From edeb1b245821c2c2aab0035031a80477df13bd22 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Mon, 17 Jun 2024 15:24:56 +0800 Subject: [PATCH] [DSIP-45] Polish the Storage SPI (#16141) --- .../api/controller/ResourcesController.java | 471 +++--- .../AbstractResourceCreateRequest.java | 34 + .../dto/resources/AbstractResourceDto.java | 29 + .../api/dto/resources/CreateDirectoryDto.java | 37 + .../dto/resources/CreateDirectoryRequest.java | 41 + .../api/dto/resources/CreateFileDto.java | 41 + .../resources/CreateFileFromContentDto.java | 39 + .../CreateFileFromContentRequest.java | 40 + .../api/dto/resources/CreateFileRequest.java | 41 + .../api/dto/resources/DeleteResourceDto.java | 36 + .../dto/resources/DeleteResourceRequest.java | 35 + .../api/dto/resources/DownloadFileDto.java | 36 + .../dto/resources/DownloadFileRequest.java | 37 + .../dto/resources/FetchFileContentDto.java | 38 + .../resources/FetchFileContentRequest.java | 37 + .../resources/PagingResourceItemRequest.java | 58 + .../api/dto/resources/QueryResourceDto.java | 34 + .../api/dto/resources/RenameDirectoryDto.java | 39 + .../dto/resources/RenameDirectoryRequest.java | 39 + .../api/dto/resources/RenameFileDto.java | 38 + .../api/dto/resources/RenameFileRequest.java | 39 + .../api/dto/resources/ResourceComponent.java | 31 +- .../api/dto/resources/UpdateFileDto.java | 41 + .../resources/UpdateFileFromContentDto.java | 39 + .../UpdateFileFromContentRequest.java | 39 + .../api/dto/resources/UpdateFileRequest.java | 41 + .../visitor/ResourceTreeVisitor.java | 6 +- .../api/exceptions/ServiceException.java | 4 + .../api/python/PythonGateway.java | 91 +- .../api/service/ResourcesService.java | 201 +-- .../service/impl/ResourcesServiceImpl.java | 1442 ++++------------- .../api/service/impl/TenantServiceImpl.java | 14 +- .../api/service/impl/UdfFuncServiceImpl.java | 14 +- .../api/service/impl/UsersServiceImpl.java | 56 +- .../dolphinscheduler/api/utils/PageInfo.java | 4 + .../api/validator/ITransformer.java | 24 + .../api/validator/IValidator.java | 24 + .../resource/AbstractResourceTransformer.java | 56 + .../resource/AbstractResourceValidator.java | 138 ++ .../resource/CreateDirectoryDtoValidator.java | 50 + .../CreateDirectoryRequestTransformer.java | 87 + .../resource/CreateFileDtoValidator.java | 47 + .../CreateFileFromContentDtoValidator.java | 46 + .../resource/DeleteResourceDtoValidator.java | 43 + .../resource/DownloadFileDtoValidator.java | 44 + .../FetchFileContentDtoValidator.java | 48 + .../FileFromContentRequestTransformer.java | 71 + .../resource/FileRequestTransformer.java | 68 + .../PagingResourceItemRequestTransformer.java | 90 + .../resource/RenameDirectoryDtoValidator.java | 50 + .../RenameDirectoryRequestTransformer.java | 56 + .../resource/RenameFileDtoValidator.java | 50 + .../RenameFileRequestTransformer.java | 47 + .../resource/UpdateFileDtoValidator.java | 57 + .../UpdateFileFromContentDtoValidator.java | 47 + ...dateFileFromContentRequestTransformer.java | 39 + .../UpdateFileRequestTransformer.java | 38 + .../api/vo/ResourceItemVO.java | 74 + .../resources/FetchFileContentResponse.java | 33 + .../api/AssertionsHelper.java | 5 + .../controller/ResourcesControllerTest.java | 143 +- .../api/python/PythonGatewayTest.java | 16 +- .../api/service/ResourcesServiceTest.java | 657 -------- .../api/service/TenantServiceTest.java | 4 +- .../api/service/UdfFuncServiceTest.java | 17 +- .../api/service/UsersServiceTest.java | 4 +- .../CreateDirectoryDtoValidatorTest.java | 136 ++ ...CreateFileFromContentDtoValidatorTest.java | 188 +++ .../FetchFileContentDtoValidatorTest.java | 197 +++ .../RenameDirectoryDtoValidatorTest.java | 188 +++ .../resource/RenameFileDtoValidatorTest.java | 202 +++ dolphinscheduler-bom/pom.xml | 7 + .../common/config/ImmutableYamlDelegate.java | 4 + .../common/constants/Constants.java | 24 - .../{ResUploadType.java => StorageType.java} | 2 +- .../common/utils/FileUtils.java | 79 +- .../common/utils/FileUtilsTest.java | 15 +- .../dao/repository/TenantDao.java | 24 + .../dao/repository/impl/TenantDaoImpl.java | 36 + .../dao/utils/TaskCacheUtils.java | 22 +- .../dao/utils/TaskCacheUtilsTest.java | 22 +- .../datasource/api/utils/CommonUtils.java | 6 +- .../security/UserGroupInformationFactory.java | 6 +- .../spi/enums/ResourceType.java | 9 + .../storage/abs/AbsStorageOperator.java | 376 +---- .../abs/AbsStorageOperatorFactory.java | 26 +- .../storage/abs/AbsStorageProperties.java | 36 + .../storage/abs/AbsStorageOperatorTest.java | 274 ---- .../storage/api/AbstractStorageOperator.java | 110 ++ .../plugin/storage/api/ResourceMetadata.java | 42 + .../storage/api/StorageConfiguration.java | 10 +- .../plugin/storage/api/StorageEntity.java | 27 +- .../plugin/storage/api/StorageOperate.java | 203 --- .../plugin/storage/api/StorageOperator.java | 157 ++ ...ctory.java => StorageOperatorFactory.java} | 4 +- .../storage/gcs/GcsStorageOperator.java | 436 ++--- .../gcs/GcsStorageOperatorFactory.java | 25 +- .../storage/gcs/GcsStorageProperties.java | 36 + .../storage/gcs/GcsStorageOperatorTest.java | 290 ---- .../dolphinscheduler-storage-hdfs/pom.xml | 6 + .../storage/hdfs/HdfsStorageOperator.java | 836 ++-------- .../hdfs/HdfsStorageOperatorFactory.java | 32 +- .../storage/hdfs/HdfsStorageProperties.java | 63 +- .../storage/hdfs/LocalStorageOperator.java | 10 - .../hdfs/LocalStorageOperatorFactory.java | 24 +- .../storage/hdfs/HdfsStorageOperatorTest.java | 197 ++- .../hdfs/LocalStorageOperatorTest.java | 322 ++++ .../hadoop-docker-compose/docker-compose.yaml | 110 ++ .../hadoop-docker-compose/hadoop.env | 45 + .../src/test/resources/storage/hello.sh | 18 + .../src/test/resources/storage/sql/demo.sql | 18 + .../storage/obs/ObsStorageOperator.java | 474 ++---- .../obs/ObsStorageOperatorFactory.java | 29 +- .../storage/obs/ObsStorageProperties.java | 40 + .../storage/obs/ObsStorageOperatorTest.java | 291 ---- .../storage/oss/OssStorageOperator.java | 470 ++---- .../oss/OssStorageOperatorFactory.java | 16 +- .../storage/oss/OssStorageOperatorTest.java | 298 ---- .../dolphinscheduler-storage-s3/pom.xml | 7 + .../plugin/storage/s3/S3StorageOperator.java | 578 +++---- .../storage/s3/S3StorageOperatorFactory.java | 25 +- .../storage/s3/S3StorageProperties.java | 38 + .../storage/s3/S3StorageOperatorTest.java | 391 ++--- .../src/test/resources/demo.sql | 17 + .../src/test/resources/student.sql | 17 + .../src/test/resources/student/student.sql | 18 + .../resource/MigrateResourceService.java | 18 +- .../components/resource/edit/use-edit.ts | 2 +- .../resource/components/resource/types.ts | 1 - .../resource/components/resource/use-file.ts | 2 +- .../runner/DefaultWorkerTaskExecutor.java | 6 +- .../DefaultWorkerTaskExecutorFactory.java | 10 +- .../worker/runner/WorkerTaskExecutor.java | 16 +- .../WorkerTaskExecutorFactoryBuilder.java | 10 +- .../utils/TaskExecutionContextUtils.java | 17 +- .../worker/utils/TaskFilesTransferUtils.java | 35 +- .../runner/DefaultWorkerTaskExecutorTest.java | 8 +- .../TaskInstanceOperationFunctionTest.java | 8 +- .../utils/TaskFilesTransferUtilsTest.java | 10 +- 139 files changed, 6355 insertions(+), 6832 deletions(-) create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/AbstractResourceCreateRequest.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/AbstractResourceDto.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateDirectoryDto.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateDirectoryRequest.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateFileDto.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateFileFromContentDto.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateFileFromContentRequest.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateFileRequest.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/DeleteResourceDto.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/DeleteResourceRequest.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/DownloadFileDto.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/DownloadFileRequest.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FetchFileContentDto.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FetchFileContentRequest.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/PagingResourceItemRequest.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/QueryResourceDto.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/RenameDirectoryDto.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/RenameDirectoryRequest.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/RenameFileDto.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/RenameFileRequest.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/UpdateFileDto.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/UpdateFileFromContentDto.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/UpdateFileFromContentRequest.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/UpdateFileRequest.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/ITransformer.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/IValidator.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/AbstractResourceTransformer.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/AbstractResourceValidator.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateDirectoryDtoValidator.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateDirectoryRequestTransformer.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateFileDtoValidator.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateFileFromContentDtoValidator.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/DeleteResourceDtoValidator.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/DownloadFileDtoValidator.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/FetchFileContentDtoValidator.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/FileFromContentRequestTransformer.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/FileRequestTransformer.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/PagingResourceItemRequestTransformer.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameDirectoryDtoValidator.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameDirectoryRequestTransformer.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameFileDtoValidator.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameFileRequestTransformer.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileDtoValidator.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileFromContentDtoValidator.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileFromContentRequestTransformer.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileRequestTransformer.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/vo/ResourceItemVO.java create mode 100644 dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/vo/resources/FetchFileContentResponse.java delete mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/CreateDirectoryDtoValidatorTest.java create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/CreateFileFromContentDtoValidatorTest.java create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/FetchFileContentDtoValidatorTest.java create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/RenameDirectoryDtoValidatorTest.java create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/RenameFileDtoValidatorTest.java rename dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/{ResUploadType.java => StorageType.java} (97%) create mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/TenantDao.java create mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/TenantDaoImpl.java create mode 100644 dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageProperties.java delete mode 100644 dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/test/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperatorTest.java create mode 100644 dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/AbstractStorageOperator.java create mode 100644 dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/ResourceMetadata.java delete mode 100644 dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperate.java create mode 100644 dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperator.java rename dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/{StorageOperateFactory.java => StorageOperatorFactory.java} (91%) create mode 100644 dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageProperties.java delete mode 100644 dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/test/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperatorTest.java create mode 100644 dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java create mode 100644 dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/resources/hadoop-docker-compose/docker-compose.yaml create mode 100644 dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/resources/hadoop-docker-compose/hadoop.env create mode 100644 dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/resources/storage/hello.sh create mode 100644 dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/resources/storage/sql/demo.sql create mode 100644 dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageProperties.java delete mode 100644 dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/test/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperatorTest.java delete mode 100644 dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/test/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperatorTest.java create mode 100644 dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageProperties.java create mode 100644 dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/test/resources/demo.sql create mode 100644 dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/test/resources/student.sql create mode 100644 dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/test/resources/student/student.sql diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java index 0338453197..620993d61b 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java @@ -28,42 +28,50 @@ import static org.apache.dolphinscheduler.api.enums.Status.QUERY_DATASOURCE_BY_T import static org.apache.dolphinscheduler.api.enums.Status.QUERY_RESOURCES_LIST_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.QUERY_RESOURCES_LIST_PAGING; import static org.apache.dolphinscheduler.api.enums.Status.QUERY_UDF_FUNCTION_LIST_PAGING_ERROR; -import static org.apache.dolphinscheduler.api.enums.Status.RESOURCE_FILE_IS_EMPTY; import static org.apache.dolphinscheduler.api.enums.Status.RESOURCE_NOT_EXIST; import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_RESOURCE_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.UPDATE_UDF_FUNCTION_ERROR; -import static org.apache.dolphinscheduler.api.enums.Status.VERIFY_RESOURCE_BY_NAME_AND_TYPE_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.VERIFY_UDF_FUNCTION_NAME_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.VIEW_RESOURCE_FILE_ON_LINE_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.VIEW_UDF_FUNCTION_ERROR; import org.apache.dolphinscheduler.api.audit.OperatorLog; import org.apache.dolphinscheduler.api.audit.enums.AuditType; -import org.apache.dolphinscheduler.api.dto.resources.DeleteDataTransferResponse; +import org.apache.dolphinscheduler.api.dto.resources.CreateDirectoryRequest; +import org.apache.dolphinscheduler.api.dto.resources.CreateFileFromContentRequest; +import org.apache.dolphinscheduler.api.dto.resources.CreateFileRequest; +import org.apache.dolphinscheduler.api.dto.resources.DeleteResourceRequest; +import org.apache.dolphinscheduler.api.dto.resources.DownloadFileRequest; +import org.apache.dolphinscheduler.api.dto.resources.FetchFileContentRequest; +import org.apache.dolphinscheduler.api.dto.resources.PagingResourceItemRequest; +import org.apache.dolphinscheduler.api.dto.resources.RenameDirectoryRequest; +import org.apache.dolphinscheduler.api.dto.resources.RenameFileRequest; +import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent; +import org.apache.dolphinscheduler.api.dto.resources.UpdateFileFromContentRequest; +import org.apache.dolphinscheduler.api.dto.resources.UpdateFileRequest; import org.apache.dolphinscheduler.api.exceptions.ApiException; import org.apache.dolphinscheduler.api.service.ResourcesService; import org.apache.dolphinscheduler.api.service.UdfFuncService; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.api.vo.ResourceItemVO; +import org.apache.dolphinscheduler.api.vo.resources.FetchFileContentResponse; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.dao.entity.User; -import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.spi.enums.ResourceType; import org.apache.commons.lang3.StringUtils; -import java.util.Map; +import java.util.List; + +import javax.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.io.Resource; -import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -77,15 +85,14 @@ import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; +import com.google.common.io.Files; + import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.Parameters; import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.tags.Tag; -/** - * resources controller - */ @Tag(name = "RESOURCES_TAG") @RestController @RequestMapping("resources") @@ -94,41 +101,34 @@ public class ResourcesController extends BaseController { @Autowired private ResourcesService resourceService; + @Autowired private UdfFuncService udfFuncService; - /** - * @param loginUser login user - * @param type type - * @param alias alias - * @param pid parent id - * @param currentDir current directory - * @return create result code - */ @Operation(summary = "createDirectory", description = "CREATE_RESOURCE_NOTES") @Parameters({ @Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)), @Parameter(name = "name", description = "RESOURCE_NAME", required = true, schema = @Schema(implementation = String.class)), - @Parameter(name = "pid", description = "RESOURCE_PID", required = true, schema = @Schema(implementation = int.class, example = "10")), @Parameter(name = "currentDir", description = "RESOURCE_CURRENT_DIR", required = true, schema = @Schema(implementation = String.class))}) @PostMapping(value = "/directory") @ApiException(CREATE_RESOURCE_ERROR) @OperatorLog(auditType = AuditType.FOLDER_CREATE) - public Result createDirectory(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam(value = "type") ResourceType type, - @RequestParam(value = "name") String alias, - @RequestParam(value = "pid") int pid, - @RequestParam(value = "currentDir") String currentDir) { - // todo verify the directory name - return resourceService.createDirectory(loginUser, alias, type, pid, currentDir); + public Result createDirectory(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestParam(value = "type") ResourceType type, + @RequestParam(value = "name") String directoryName, + @RequestParam(value = "currentDir") String parentDirectory) { + + CreateDirectoryRequest createDirectoryRequest = CreateDirectoryRequest.builder() + .loginUser(loginUser) + .directoryName(directoryName) + .type(type) + .parentAbsoluteDirectory(parentDirectory) + .build(); + resourceService.createDirectory(createDirectoryRequest); + return Result.success(null); } - /** - * create resource - * - * @return create result code - */ - @Operation(summary = "createResource", description = "CREATE_RESOURCE_NOTES") + @Operation(summary = "uploadFile", description = "CREATE_FILE") @Parameters({ @Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)), @Parameter(name = "name", description = "RESOURCE_NAME", required = true, schema = @Schema(implementation = String.class)), @@ -137,24 +137,71 @@ public class ResourcesController extends BaseController { @PostMapping() @ApiException(CREATE_RESOURCE_ERROR) @OperatorLog(auditType = AuditType.FILE_CREATE) - public Result createResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam(value = "type") ResourceType type, - @RequestParam(value = "name") String alias, - @RequestParam("file") MultipartFile file, - @RequestParam(value = "currentDir") String currentDir) { - // todo verify the file name - return resourceService.uploadResource(loginUser, alias, type, file, currentDir); + public Result createFile(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestParam(value = "type") ResourceType type, + @RequestParam(value = "name") String fileName, + @RequestParam("file") MultipartFile file, + @RequestParam(value = "currentDir") String parentDirectoryAbsolutePath) { + + CreateFileRequest uploadFileRequest = CreateFileRequest.builder() + .loginUser(loginUser) + .fileName(fileName) + .file(file) + .type(type) + .parentAbsoluteDirectory(parentDirectoryAbsolutePath) + .build(); + resourceService.createFile(uploadFileRequest); + return Result.success(); + } + + @Operation(summary = "createFileFromContent", description = "ONLINE_CREATE_RESOURCE_NOTES") + @Parameters({ + @Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)), + @Parameter(name = "fileName", description = "RESOURCE_NAME", required = true, schema = @Schema(implementation = String.class)), + @Parameter(name = "suffix", description = "SUFFIX", required = true, schema = @Schema(implementation = String.class)), + @Parameter(name = "description", description = "RESOURCE_DESC", schema = @Schema(implementation = String.class)), + @Parameter(name = "content", description = "CONTENT", required = true, schema = @Schema(implementation = String.class)), + @Parameter(name = "currentDir", description = "RESOURCE_CURRENTDIR", required = true, schema = @Schema(implementation = String.class))}) + @PostMapping(value = "/online-create") + @ApiException(CREATE_RESOURCE_FILE_ON_LINE_ERROR) + @OperatorLog(auditType = AuditType.FILE_CREATE) + public Result createFileFromContent(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestParam(value = "type") ResourceType type, + @RequestParam(value = "fileName") String fileName, + @RequestParam(value = "suffix") String fileSuffix, + @RequestParam(value = "content") String fileContent, + @RequestParam(value = "currentDir") String fileParentDirectoryAbsolutePath) { + CreateFileFromContentRequest createFileFromContentRequest = CreateFileFromContentRequest.builder() + .loginUser(loginUser) + .fileName(fileName + "." + fileSuffix) + .fileContent(fileContent) + .type(type) + .parentAbsoluteDirectory(fileParentDirectoryAbsolutePath) + .build(); + resourceService.createFileFromContent(createFileFromContentRequest); + return Result.success(); + } + + @Operation(summary = "updateFileContent", description = "UPDATE_RESOURCE_NOTES") + @Parameters({ + @Parameter(name = "content", description = "CONTENT", required = true, schema = @Schema(implementation = String.class)), + @Parameter(name = "fullName", description = "FULL_NAME", required = true, schema = @Schema(implementation = String.class)), + @Parameter(name = "tenantCode", description = "TENANT_CODE", required = true, schema = @Schema(implementation = String.class))}) + @PutMapping(value = "/update-content") + @ApiException(EDIT_RESOURCE_FILE_ON_LINE_ERROR) + @OperatorLog(auditType = AuditType.FILE_UPDATE) + public Result updateFileContent(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestParam(value = "fullName") String fileAbsolutePath, + @RequestParam(value = "content") String fileContent) { + UpdateFileFromContentRequest updateFileContentRequest = UpdateFileFromContentRequest.builder() + .loginUser(loginUser) + .fileContent(fileContent) + .fileAbsolutePath(fileAbsolutePath) + .build(); + resourceService.updateFileFromContent(updateFileContentRequest); + return Result.success(); } - /** - * update resource - * - * @param loginUser login user - * @param alias alias - * @param type resource type - * @param file resource file - * @return update result code - */ @Operation(summary = "updateResource", description = "UPDATE_RESOURCE_NOTES") @Parameters({ @Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class)), @@ -165,47 +212,39 @@ public class ResourcesController extends BaseController { @PutMapping() @ApiException(UPDATE_RESOURCE_ERROR) @OperatorLog(auditType = AuditType.FILE_UPDATE) - public Result updateResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam(value = "fullName") String fullName, - @RequestParam(value = "tenantCode", required = false) String tenantCode, - @RequestParam(value = "type") ResourceType type, - @RequestParam(value = "name") String alias, - @RequestParam(value = "file", required = false) MultipartFile file) { - return resourceService.updateResource(loginUser, fullName, tenantCode, alias, type, file); - } + public Result updateResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestParam(value = "fullName") String resourceAbsolutePath, + @RequestParam(value = "name") String resourceName, + @RequestParam(value = "file", required = false) MultipartFile file) { + if (StringUtils.isEmpty(Files.getFileExtension(resourceName))) { + RenameDirectoryRequest renameDirectoryRequest = RenameDirectoryRequest.builder() + .loginUser(loginUser) + .directoryAbsolutePath(resourceAbsolutePath) + .newDirectoryName(resourceName) + .build(); + resourceService.renameDirectory(renameDirectoryRequest); + return Result.success(); + } - /** - * query resources list - * - * @param loginUser login user - * @param type resource type - * @return resource list - */ - @Operation(summary = "queryResourceList", description = "QUERY_RESOURCE_LIST_NOTES") - @Parameters({ - @Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)), - @Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class))}) - @GetMapping(value = "/list") - @ResponseStatus(HttpStatus.OK) - @ApiException(QUERY_RESOURCES_LIST_ERROR) - public Result queryResourceList(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam(value = "type") ResourceType type, - @RequestParam(value = "fullName") String fullName) { - Map result = resourceService.queryResourceList(loginUser, type, fullName); - return returnDataList(result); + if (file == null) { + RenameFileRequest renameFileRequest = RenameFileRequest.builder() + .loginUser(loginUser) + .fileAbsolutePath(resourceAbsolutePath) + .newFileName(resourceName) + .build(); + resourceService.renameFile(renameFileRequest); + return Result.success(); + } + UpdateFileRequest updateFileRequest = UpdateFileRequest.builder() + .loginUser(loginUser) + .fileAbsolutePath(resourceAbsolutePath) + .file(file) + .build(); + resourceService.updateFile(updateFileRequest); + return Result.success(); } - /** - * query resources list paging - * - * @param loginUser login user - * @param type resource type - * @param searchVal search value - * @param pageNo page number - * @param pageSize page size - * @return resource list page - */ - @Operation(summary = "queryResourceListPaging", description = "QUERY_RESOURCE_LIST_PAGING_NOTES") + @Operation(summary = "pagingResourceItemRequest", description = "PAGING_RESOURCE_ITEM_LIST") @Parameters({ @Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)), @Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class, example = "bucket_name/tenant_name/type/ds")), @@ -215,129 +254,67 @@ public class ResourcesController extends BaseController { @GetMapping() @ResponseStatus(HttpStatus.OK) @ApiException(QUERY_RESOURCES_LIST_PAGING) - public Result> queryResourceListPaging(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam(value = "fullName") String fullName, - @RequestParam(value = "tenantCode") String tenantCode, - @RequestParam(value = "type") ResourceType type, - @RequestParam("pageNo") Integer pageNo, - @RequestParam(value = "searchVal", required = false) String searchVal, - @RequestParam("pageSize") Integer pageSize) { - checkPageParams(pageNo, pageSize); - - searchVal = ParameterUtils.handleEscapes(searchVal); - return resourceService.queryResourceListPaging(loginUser, fullName, tenantCode, type, searchVal, pageNo, - pageSize); + public Result> pagingResourceItemRequest(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestParam(value = "fullName") String resourceAbsolutePath, + @RequestParam(value = "type") ResourceType resourceType, + @RequestParam(value = "searchVal", required = false) String resourceNameKeyWord, + @RequestParam("pageNo") Integer pageNo, + @RequestParam("pageSize") Integer pageSize) { + PagingResourceItemRequest pagingResourceItemRequest = PagingResourceItemRequest.builder() + .loginUser(loginUser) + .resourceAbsolutePath(resourceAbsolutePath) + .resourceType(resourceType) + .resourceNameKeyWord(StringUtils.trim(ParameterUtils.handleEscapes(resourceNameKeyWord))) + .pageNo(pageNo) + .pageSize(pageSize) + .build(); + pagingResourceItemRequest.checkPageNoAndPageSize(); + + return Result.success(resourceService.pagingResourceItem(pagingResourceItemRequest)); } - /** - * delete resource - * - * @param loginUser login user - * @return delete result code - */ - @Operation(summary = "deleteResource", description = "DELETE_RESOURCE_BY_ID_NOTES") + // todo: this api is used for udf, we should remove it + @Operation(summary = "queryResourceList", description = "QUERY_RESOURCE_LIST_NOTES") @Parameters({ - @Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class, example = "test/"))}) - @DeleteMapping() + @Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)), + @Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class))}) + @GetMapping(value = "/list") @ResponseStatus(HttpStatus.OK) - @ApiException(DELETE_RESOURCE_ERROR) - @OperatorLog(auditType = AuditType.FILE_DELETE) - public Result deleteResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam(value = "fullName") String fullName, - @RequestParam(value = "tenantCode", required = false) String tenantCode) throws Exception { - return resourceService.delete(loginUser, fullName, tenantCode); + @ApiException(QUERY_RESOURCES_LIST_ERROR) + public Result> queryResourceList(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestParam(value = "type") ResourceType type) { + return Result.success(resourceService.queryResourceFiles(loginUser, type)); } - /** - * delete DATA_TRANSFER data - * - * @param loginUser login user - * @return delete result code - */ - @Operation(summary = "deleteDataTransferData", description = "Delete the N days ago data of DATA_TRANSFER ") + @Operation(summary = "deleteResource", description = "DELETE_RESOURCE_BY_ID_NOTES") @Parameters({ - @Parameter(name = "days", description = "N days ago", required = true, schema = @Schema(implementation = Integer.class))}) - @DeleteMapping(value = "/data-transfer") + @Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class, example = "file:////tmp/dolphinscheduler/storage/default/resources/demo.sql")) + }) + @DeleteMapping() @ResponseStatus(HttpStatus.OK) @ApiException(DELETE_RESOURCE_ERROR) - public DeleteDataTransferResponse deleteDataTransferData(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam(value = "days") Integer days) { - return resourceService.deleteDataTransferData(loginUser, days); - } - - /** - * verify resource by alias and type - * - * @param loginUser login user - * @param fullName resource full name - * @param type resource type - * @return true if the resource name not exists, otherwise return false - */ - @Operation(summary = "verifyResourceName", description = "VERIFY_RESOURCE_NAME_NOTES") - @Parameters({ - @Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)), - @Parameter(name = "fullName", description = "RESOURCE_FULL_NAME", required = true, schema = @Schema(implementation = String.class))}) - @GetMapping(value = "/verify-name") - @ResponseStatus(HttpStatus.OK) - @ApiException(VERIFY_RESOURCE_BY_NAME_AND_TYPE_ERROR) - public Result verifyResourceName(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam(value = "fullName") String fullName, - @RequestParam(value = "type") ResourceType type) { - return resourceService.verifyResourceName(fullName, type, loginUser); + @OperatorLog(auditType = AuditType.FILE_DELETE) + public Result deleteResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestParam(value = "fullName") String resourceAbsolutePath) { + DeleteResourceRequest deleteResourceRequest = DeleteResourceRequest.builder() + .loginUser(loginUser) + .resourceAbsolutePath(resourceAbsolutePath) + .build(); + resourceService.delete(deleteResourceRequest); + return Result.success(); } - /** - * query resources by type - * - * @param loginUser login user - * @param type resource type - * @return resource list - */ - @Operation(summary = "queryResourceByProgramType", description = "QUERY_RESOURCE_LIST_NOTES") + @Operation(summary = "queryResourceFileList", description = "QUERY_RESOURCE_FILE_LIST_NOTES") @Parameters({ @Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class))}) @GetMapping(value = "/query-by-type") @ResponseStatus(HttpStatus.OK) @ApiException(QUERY_RESOURCES_LIST_ERROR) - public Result queryResourceJarList(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam(value = "type") ResourceType type, - @RequestParam(value = "programType", required = false) ProgramType programType) { - return resourceService.queryResourceByProgramType(loginUser, type, programType); - } - - /** - * query resource by file name and type - * - * @param loginUser login user - * @param fileName resource full name - * @param tenantCode tenantCode of the owner of the resource - * @param type resource type - * @return true if the resource name not exists, otherwise return false - */ - @Operation(summary = "queryResourceByFileName", description = "QUERY_BY_RESOURCE_FILE_NAME") - @Parameters({ - @Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)), - @Parameter(name = "fileName", description = "RESOURCE_FILE_NAME", required = true, schema = @Schema(implementation = String.class)), - @Parameter(name = "tenantCode", description = "TENANT_CODE", required = true, schema = @Schema(implementation = String.class)),}) - @GetMapping(value = "/query-file-name") - @ResponseStatus(HttpStatus.OK) - @ApiException(RESOURCE_NOT_EXIST) - public Result queryResourceByFileName(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam(value = "fileName", required = false) String fileName, - @RequestParam(value = "tenantCode", required = false) String tenantCode, - @RequestParam(value = "type") ResourceType type) { - - return resourceService.queryResourceByFileName(loginUser, fileName, type, tenantCode); + public Result> queryResourceFileList(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestParam(value = "type") ResourceType type) { + return Result.success(resourceService.queryResourceFiles(loginUser, type)); } - /** - * view resource file online - * - * @param loginUser login user - * @param skipLineNum skip line number - * @param limit limit - * @return resource content - */ @Operation(summary = "viewResource", description = "VIEW_RESOURCE_BY_ID_NOTES") @Parameters({ @Parameter(name = "fullName", description = "RESOURCE_FULL_NAME", required = true, schema = @Schema(implementation = String.class, example = "tenant/1.png")), @@ -346,99 +323,37 @@ public class ResourcesController extends BaseController { @Parameter(name = "limit", description = "LIMIT", required = true, schema = @Schema(implementation = int.class, example = "100"))}) @GetMapping(value = "/view") @ApiException(VIEW_RESOURCE_FILE_ON_LINE_ERROR) - public Result viewResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam(value = "skipLineNum") int skipLineNum, - @RequestParam(value = "limit") int limit, - @RequestParam(value = "fullName") String fullName, - @RequestParam(value = "tenantCode") String tenantCode) { - return resourceService.readResource(loginUser, fullName, tenantCode, skipLineNum, limit); - } - - @Operation(summary = "onlineCreateResource", description = "ONLINE_CREATE_RESOURCE_NOTES") - @Parameters({ - @Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)), - @Parameter(name = "fileName", description = "RESOURCE_NAME", required = true, schema = @Schema(implementation = String.class)), - @Parameter(name = "suffix", description = "SUFFIX", required = true, schema = @Schema(implementation = String.class)), - @Parameter(name = "description", description = "RESOURCE_DESC", schema = @Schema(implementation = String.class)), - @Parameter(name = "content", description = "CONTENT", required = true, schema = @Schema(implementation = String.class)), - @Parameter(name = "currentDir", description = "RESOURCE_CURRENTDIR", required = true, schema = @Schema(implementation = String.class))}) - @PostMapping(value = "/online-create") - @ApiException(CREATE_RESOURCE_FILE_ON_LINE_ERROR) - @OperatorLog(auditType = AuditType.FILE_CREATE) - public Result createResourceFile(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam(value = "type") ResourceType type, - @RequestParam(value = "fileName") String fileName, - @RequestParam(value = "suffix") String fileSuffix, - @RequestParam(value = "content") String content, - @RequestParam(value = "currentDir") String currentDir) { - if (StringUtils.isEmpty(content)) { - log.error("resource file contents are not allowed to be empty"); - return error(RESOURCE_FILE_IS_EMPTY.getCode(), RESOURCE_FILE_IS_EMPTY.getMsg()); - } - return resourceService.createResourceFile(loginUser, type, fileName, fileSuffix, content, currentDir); - } - - /** - * edit resource file online - * - * @param loginUser login user - * @param content content - * @return update result code - */ - @Operation(summary = "updateResourceContent", description = "UPDATE_RESOURCE_NOTES") - @Parameters({ - @Parameter(name = "content", description = "CONTENT", required = true, schema = @Schema(implementation = String.class)), - @Parameter(name = "fullName", description = "FULL_NAME", required = true, schema = @Schema(implementation = String.class)), - @Parameter(name = "tenantCode", description = "TENANT_CODE", required = true, schema = @Schema(implementation = String.class))}) - @PutMapping(value = "/update-content") - @ApiException(EDIT_RESOURCE_FILE_ON_LINE_ERROR) - @OperatorLog(auditType = AuditType.FILE_UPDATE) - public Result updateResourceContent(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam(value = "fullName") String fullName, - @RequestParam(value = "tenantCode") String tenantCode, - @RequestParam(value = "content") String content) { - if (StringUtils.isEmpty(content)) { - log.error("The resource file contents are not allowed to be empty"); - return error(RESOURCE_FILE_IS_EMPTY.getCode(), RESOURCE_FILE_IS_EMPTY.getMsg()); - } - return resourceService.updateResourceContent(loginUser, fullName, tenantCode, content); + public Result viewResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestParam(value = "fullName") String resourceAbsoluteFilePath, + @RequestParam(value = "skipLineNum") int skipLineNum, + @RequestParam(value = "limit") int limit) { + FetchFileContentRequest fetchFileContentRequest = FetchFileContentRequest.builder() + .loginUser(loginUser) + .resourceFileAbsolutePath(resourceAbsoluteFilePath) + .limit(limit == -1 ? Integer.MAX_VALUE : skipLineNum) + .skipLineNum(skipLineNum) + .build(); + return Result.success(resourceService.fetchResourceFileContent(fetchFileContentRequest)); } - /** - * download resource file - * - * @param loginUser login user - * @return resource content - */ @Operation(summary = "downloadResource", description = "DOWNLOAD_RESOURCE_NOTES") @Parameters({ @Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class, example = "test/"))}) @GetMapping(value = "/download") @ResponseBody @ApiException(DOWNLOAD_RESOURCE_FILE_ERROR) - public ResponseEntity downloadResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam(value = "fullName") String fullName) throws Exception { - Resource file = resourceService.downloadResource(loginUser, fullName); - if (file == null) { - return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(RESOURCE_NOT_EXIST.getMsg()); - } - return ResponseEntity.ok() - .header(HttpHeaders.CONTENT_DISPOSITION, "attachment; filename=\"" + file.getFilename() + "\"") - .body(file); + public void downloadResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + HttpServletResponse response, + @RequestParam(value = "fullName") String fileAbsolutePath) { + + DownloadFileRequest downloadFileRequest = DownloadFileRequest.builder() + .loginUser(loginUser) + .fileAbsolutePath(fileAbsolutePath) + .build(); + + resourceService.downloadResource(response, downloadFileRequest); } - /** - * create udf function - * - * @param loginUser login user - * @param type udf type - * @param funcName function name - * @param argTypes argument types - * @param database database - * @param description description - * @param className class name - * @return create result code - */ @Operation(summary = "createUdfFunc", description = "CREATE_UDF_FUNCTION_NOTES") @Parameters({ @Parameter(name = "type", description = "UDF_TYPE", required = true, schema = @Schema(implementation = UdfType.class)), @@ -612,8 +527,8 @@ public class ResourcesController extends BaseController { @GetMapping(value = "/base-dir") @ResponseStatus(HttpStatus.OK) @ApiException(RESOURCE_NOT_EXIST) - public Result queryResourceBaseDir(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + public Result queryResourceBaseDir(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam(value = "type") ResourceType type) { - return resourceService.queryResourceBaseDir(loginUser, type); + return Result.success(resourceService.queryResourceBaseDir(loginUser, type)); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/AbstractResourceCreateRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/AbstractResourceCreateRequest.java new file mode 100644 index 0000000000..e4e0f2fa95 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/AbstractResourceCreateRequest.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.dto.resources; + +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.spi.enums.ResourceType; + +import lombok.Data; +import lombok.experimental.SuperBuilder; + +@Data +@SuperBuilder +public abstract class AbstractResourceCreateRequest { + + private User loginUser; + private String parentAbsoluteDirectory; + private ResourceType type; + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/AbstractResourceDto.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/AbstractResourceDto.java new file mode 100644 index 0000000000..6314c8542c --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/AbstractResourceDto.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.dto.resources; + +import lombok.Data; +import lombok.experimental.SuperBuilder; + +@Data +@SuperBuilder +public abstract class AbstractResourceDto { + + private String resourceAbsolutePath; + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateDirectoryDto.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateDirectoryDto.java new file mode 100644 index 0000000000..14b8eaeac2 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateDirectoryDto.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.dto.resources; + +import org.apache.dolphinscheduler.dao.entity.User; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class CreateDirectoryDto { + + private User loginUser; + + private String directoryAbsolutePath; + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateDirectoryRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateDirectoryRequest.java new file mode 100644 index 0000000000..7719fffb66 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateDirectoryRequest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.dto.resources; + +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.spi.enums.ResourceType; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class CreateDirectoryRequest { + + private User loginUser; + + private ResourceType type; + + private String parentAbsoluteDirectory; + + private String directoryName; +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateFileDto.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateFileDto.java new file mode 100644 index 0000000000..da8a37dc45 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateFileDto.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.dto.resources; + +import org.apache.dolphinscheduler.dao.entity.User; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import org.springframework.web.multipart.MultipartFile; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class CreateFileDto { + + private User loginUser; + + private String fileAbsolutePath; + + private MultipartFile file; + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateFileFromContentDto.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateFileFromContentDto.java new file mode 100644 index 0000000000..a8faf8b0aa --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateFileFromContentDto.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.dto.resources; + +import org.apache.dolphinscheduler.dao.entity.User; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class CreateFileFromContentDto { + + private User loginUser; + + private String fileAbsolutePath; + + private String fileContent; + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateFileFromContentRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateFileFromContentRequest.java new file mode 100644 index 0000000000..4b424dbd89 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateFileFromContentRequest.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.dto.resources; + +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.spi.enums.ResourceType; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class CreateFileFromContentRequest { + + private User loginUser; + private ResourceType type; + private String parentAbsoluteDirectory; + private String fileName; + private String fileContent; + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateFileRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateFileRequest.java new file mode 100644 index 0000000000..c72ffa97c4 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/CreateFileRequest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.dto.resources; + +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.spi.enums.ResourceType; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import org.springframework.web.multipart.MultipartFile; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class CreateFileRequest { + + private User loginUser; + private ResourceType type; + private String parentAbsoluteDirectory; + private String fileName; + private MultipartFile file; +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/DeleteResourceDto.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/DeleteResourceDto.java new file mode 100644 index 0000000000..c79cd0d59a --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/DeleteResourceDto.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.dto.resources; + +import org.apache.dolphinscheduler.dao.entity.User; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class DeleteResourceDto { + + private User loginUser; + private String resourceAbsolutePath; + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/DeleteResourceRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/DeleteResourceRequest.java new file mode 100644 index 0000000000..3af948d512 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/DeleteResourceRequest.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.dto.resources; + +import org.apache.dolphinscheduler.dao.entity.User; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class DeleteResourceRequest { + + private User loginUser; + private String resourceAbsolutePath; +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/DownloadFileDto.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/DownloadFileDto.java new file mode 100644 index 0000000000..fe5f0721cb --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/DownloadFileDto.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.dto.resources; + +import org.apache.dolphinscheduler.dao.entity.User; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class DownloadFileDto { + + private User loginUser; + + private String fileAbsolutePath; +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/DownloadFileRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/DownloadFileRequest.java new file mode 100644 index 0000000000..1f28a162be --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/DownloadFileRequest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.dto.resources; + +import org.apache.dolphinscheduler.dao.entity.User; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class DownloadFileRequest { + + private User loginUser; + + private String fileAbsolutePath; + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FetchFileContentDto.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FetchFileContentDto.java new file mode 100644 index 0000000000..106cbe142f --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FetchFileContentDto.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.dto.resources; + +import org.apache.dolphinscheduler.dao.entity.User; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class FetchFileContentDto { + + private User loginUser; + private String resourceFileAbsolutePath; + private int skipLineNum; + private int limit; + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FetchFileContentRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FetchFileContentRequest.java new file mode 100644 index 0000000000..8357317e85 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/FetchFileContentRequest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.dto.resources; + +import org.apache.dolphinscheduler.dao.entity.User; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class FetchFileContentRequest { + + private User loginUser; + private String resourceFileAbsolutePath; + private int skipLineNum; + private int limit; +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/PagingResourceItemRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/PagingResourceItemRequest.java new file mode 100644 index 0000000000..49d720bca3 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/PagingResourceItemRequest.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.dto.resources; + +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.exceptions.ServiceException; +import org.apache.dolphinscheduler.common.constants.Constants; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.spi.enums.ResourceType; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class PagingResourceItemRequest { + + private User loginUser; + + private String resourceAbsolutePath; + + private ResourceType resourceType; + + private String resourceNameKeyWord; + + Integer pageNo; + + Integer pageSize; + + public void checkPageNoAndPageSize() { + if (pageNo <= 0) { + throw new ServiceException(Status.REQUEST_PARAMS_NOT_VALID_ERROR, Constants.PAGE_NUMBER); + } + if (pageSize <= 0) { + throw new ServiceException(Status.REQUEST_PARAMS_NOT_VALID_ERROR, Constants.PAGE_SIZE); + } + } + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/QueryResourceDto.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/QueryResourceDto.java new file mode 100644 index 0000000000..7778ca50f8 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/QueryResourceDto.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.dto.resources; + +import java.util.List; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class QueryResourceDto { + + private List resourceAbsolutePaths; +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/RenameDirectoryDto.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/RenameDirectoryDto.java new file mode 100644 index 0000000000..2989947922 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/RenameDirectoryDto.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.dto.resources; + +import org.apache.dolphinscheduler.dao.entity.User; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class RenameDirectoryDto { + + private User loginUser; + + private String originDirectoryAbsolutePath; + + private String targetDirectoryAbsolutePath; + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/RenameDirectoryRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/RenameDirectoryRequest.java new file mode 100644 index 0000000000..640acb677d --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/RenameDirectoryRequest.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.dto.resources; + +import org.apache.dolphinscheduler.dao.entity.User; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class RenameDirectoryRequest { + + private User loginUser; + + private String directoryAbsolutePath; + + private String newDirectoryName; + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/RenameFileDto.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/RenameFileDto.java new file mode 100644 index 0000000000..091febad0e --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/RenameFileDto.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.dto.resources; + +import org.apache.dolphinscheduler.dao.entity.User; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class RenameFileDto { + + private User loginUser; + + private String originFileAbsolutePath; + + private String targetFileAbsolutePath; +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/RenameFileRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/RenameFileRequest.java new file mode 100644 index 0000000000..1b0c56be3b --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/RenameFileRequest.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.dto.resources; + +import org.apache.dolphinscheduler.dao.entity.User; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class RenameFileRequest { + + private User loginUser; + + private String fileAbsolutePath; + + private String newFileName; + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java index 4487c5a080..2aa11a6ce9 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java @@ -25,34 +25,13 @@ import java.util.List; import lombok.Data; import lombok.NoArgsConstructor; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; - /** * resource component */ @Data @NoArgsConstructor -@JsonPropertyOrder({"id", "pid", "name", "fullName", "description", "isDirctory", "children", "type"}) public abstract class ResourceComponent { - public ResourceComponent(int id, String pid, String name, String fullName, String description, boolean isDirctory) { - this.id = id; - this.pid = pid; - this.name = name; - this.fullName = fullName; - this.isDirctory = isDirctory; - int directoryFlag = isDirctory ? 1 : 0; - this.idValue = String.format("%s_%s", id, directoryFlag); - } - - /** - * id - */ - protected int id; - /** - * parent id - */ - protected String pid; /** * name */ @@ -73,10 +52,7 @@ public abstract class ResourceComponent { * is directory */ protected boolean isDirctory; - /** - * id value - */ - protected String idValue; + /** * resoruce type */ @@ -88,14 +64,11 @@ public abstract class ResourceComponent { /** * add resource component + * * @param resourceComponent resource component */ public void add(ResourceComponent resourceComponent) { children.add(resourceComponent); } - public void setIdValue(int id, boolean isDirctory) { - int directoryFlag = isDirctory ? 1 : 0; - this.idValue = String.format("%s_%s", id, directoryFlag); - } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/UpdateFileDto.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/UpdateFileDto.java new file mode 100644 index 0000000000..9213ac04d1 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/UpdateFileDto.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.dto.resources; + +import org.apache.dolphinscheduler.dao.entity.User; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import org.springframework.web.multipart.MultipartFile; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class UpdateFileDto { + + private User loginUser; + + private String fileAbsolutePath; + + private MultipartFile file; + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/UpdateFileFromContentDto.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/UpdateFileFromContentDto.java new file mode 100644 index 0000000000..0848cd8f2b --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/UpdateFileFromContentDto.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.dto.resources; + +import org.apache.dolphinscheduler.dao.entity.User; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class UpdateFileFromContentDto { + + private User loginUser; + + private String fileAbsolutePath; + + private String fileContent; + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/UpdateFileFromContentRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/UpdateFileFromContentRequest.java new file mode 100644 index 0000000000..a9564a4b25 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/UpdateFileFromContentRequest.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.dto.resources; + +import org.apache.dolphinscheduler.dao.entity.User; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class UpdateFileFromContentRequest { + + private User loginUser; + + private String fileAbsolutePath; + + private String fileContent; + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/UpdateFileRequest.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/UpdateFileRequest.java new file mode 100644 index 0000000000..10fbc65700 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/UpdateFileRequest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.dto.resources; + +import org.apache.dolphinscheduler.dao.entity.User; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import org.springframework.web.multipart.MultipartFile; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class UpdateFileRequest { + + private User loginUser; + + private String fileAbsolutePath; + + private MultipartFile file; + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java index 6c88cd84a8..1d24610af0 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java @@ -122,12 +122,8 @@ public class ResourceTreeVisitor implements Visitor { tempResourceComponent = new FileLeaf(); } - tempResourceComponent.setName(resource.getAlias()); - // tempResourceComponent.setFullName(resource.getFullName().replaceFirst("/","")); + tempResourceComponent.setName(resource.getFileName()); tempResourceComponent.setFullName(resource.getFullName()); - tempResourceComponent.setId(resource.getId()); - tempResourceComponent.setPid(resource.getPfullName()); - tempResourceComponent.setIdValue(resource.getId(), resource.isDirectory()); tempResourceComponent.setType(resource.getType()); return tempResourceComponent; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ServiceException.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ServiceException.java index 2fa3e01a1c..a0f8162fcf 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ServiceException.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/exceptions/ServiceException.java @@ -52,4 +52,8 @@ public class ServiceException extends RuntimeException { this.code = code; } + public ServiceException(String message, Exception exception) { + this(Status.INTERNAL_SERVER_ERROR_ARGS, message, exception); + } + } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java index 762ba576b0..a2ad908e47 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java @@ -31,14 +31,12 @@ import org.apache.dolphinscheduler.api.service.SchedulerService; import org.apache.dolphinscheduler.api.service.TaskDefinitionService; import org.apache.dolphinscheduler.api.service.TenantService; import org.apache.dolphinscheduler.api.service.UsersService; -import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.ComplementDependentMode; import org.apache.dolphinscheduler.common.enums.ExecutionOrder; import org.apache.dolphinscheduler.common.enums.FailureStrategy; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.ProcessExecutionTypeEnum; -import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.enums.RunMode; import org.apache.dolphinscheduler.common.enums.TaskDependType; @@ -216,22 +214,22 @@ public class PythonGateway { * If workflow do not exists in Project=`projectCode` would create a new one * If workflow already exists in Project=`projectCode` would update it * - * @param userName user name who create or update workflow - * @param projectName project name which workflow belongs to - * @param name workflow name - * @param description description - * @param globalParams global params - * @param schedule schedule for workflow, will not set schedule if null, - * and if would always fresh exists schedule if not null - * @param onlineSchedule Whether set the workflow's schedule to online state - * @param warningType warning type - * @param warningGroupId warning group id - * @param timeout timeout for workflow working, if running time longer than timeout, - * task will mark as fail - * @param workerGroup run task in which worker group - * @param taskRelationJson relation json for nodes + * @param userName user name who create or update workflow + * @param projectName project name which workflow belongs to + * @param name workflow name + * @param description description + * @param globalParams global params + * @param schedule schedule for workflow, will not set schedule if null, + * and if would always fresh exists schedule if not null + * @param onlineSchedule Whether set the workflow's schedule to online state + * @param warningType warning type + * @param warningGroupId warning group id + * @param timeout timeout for workflow working, if running time longer than timeout, + * task will mark as fail + * @param workerGroup run task in which worker group + * @param taskRelationJson relation json for nodes * @param taskDefinitionJson taskDefinitionJson - * @param otherParamsJson otherParamsJson handle other params + * @param otherParamsJson otherParamsJson handle other params * @return create result code */ public Long createOrUpdateWorkflow(String userName, @@ -300,8 +298,8 @@ public class PythonGateway { /** * get workflow * - * @param user user who create or update schedule - * @param projectCode project which workflow belongs to + * @param user user who create or update schedule + * @param projectCode project which workflow belongs to * @param workflowName workflow name */ private ProcessDefinition getWorkflow(User user, long projectCode, String workflowName) { @@ -327,13 +325,13 @@ public class PythonGateway { * It would always use latest schedule define in workflow-as-code, and set schedule online when * it's not null * - * @param user user who create or update schedule - * @param projectCode project which workflow belongs to - * @param workflowCode workflow code - * @param schedule schedule expression + * @param user user who create or update schedule + * @param projectCode project which workflow belongs to + * @param workflowCode workflow code + * @param schedule schedule expression * @param onlineSchedule Whether set the workflow's schedule to online state - * @param workerGroup work group - * @param warningType warning type + * @param workerGroup work group + * @param warningType warning type * @param warningGroupId warning group id */ private void createOrUpdateSchedule(User user, @@ -512,7 +510,7 @@ public class PythonGateway { * it will return the datasource match the type. * * @param datasourceName datasource name of datasource - * @param type datasource type + * @param type datasource type */ public DataSource getDatasource(String datasourceName, String type) { @@ -545,8 +543,8 @@ public class PythonGateway { * Get workflow object by given workflow name. It returns map contain workflow id, name, code. * Useful in Python API create subProcess task which need workflow information. * - * @param userName user who create or update schedule - * @param projectName project name which workflow belongs to + * @param userName user who create or update schedule + * @param projectName project name which workflow belongs to * @param workflowName workflow name */ public Map getWorkflowInfo(String userName, String projectName, @@ -577,9 +575,9 @@ public class PythonGateway { * Get project, workflow, task code. * Useful in Python API create dependent task which need workflow information. * - * @param projectName project name which workflow belongs to + * @param projectName project name which workflow belongs to * @param workflowName workflow name - * @param taskName task name + * @param taskName task name */ public Map getDependentInfo(String projectName, String workflowName, String taskName) { Map result = new HashMap<>(); @@ -614,25 +612,22 @@ public class PythonGateway { * Get resource by given program type and full name. It returns map contain resource id, name. * Useful in Python API create flink or spark task which need workflow information. * - * @param programType program type one of SCALA, JAVA and PYTHON - * @param fullName full name of the resource + * @param fullName full name of the resource */ - public Map getResourcesFileInfo(String programType, String fullName) { + public Map getResourcesFileInfo(String fullName) { Map result = new HashMap<>(); - Result resources = resourceService.queryResourceByProgramType(dummyAdminUser, ResourceType.FILE, - ProgramType.valueOf(programType)); - List resourcesComponent = (List) resources.getData(); - List namedResources = - resourcesComponent.stream().filter(s -> fullName.equals(s.getFullName())).collect(Collectors.toList()); + List resourceComponents = + resourceService.queryResourceFiles(dummyAdminUser, ResourceType.FILE); + List namedResources = resourceComponents.stream() + .filter(s -> fullName.equals(s.getFullName())) + .collect(Collectors.toList()); if (CollectionUtils.isEmpty(namedResources)) { - String msg = - String.format("Can not find valid resource by program type %s and name %s", programType, fullName); + String msg = String.format("Can not find valid resource by name %s", fullName); log.error(msg); throw new IllegalArgumentException(msg); } - result.put("id", namedResources.get(0).getId()); result.put("name", namedResources.get(0).getName()); return result; } @@ -671,20 +666,6 @@ public class PythonGateway { return PythonGateway.class.getPackage().getImplementationVersion(); } - /** - * create or update resource. - * If the folder is not already created, it will be - * - * @param userName user who create or update resource - * @param fullName The fullname of resource.Includes path and suffix. - * @param resourceContent content of resource - * @return StorageEntity object which contains necessary information about resource - */ - public StorageEntity createOrUpdateResource(String userName, String fullName, - String resourceContent) throws Exception { - return resourceService.createOrUpdateResource(userName, fullName, resourceContent); - } - @PostConstruct public void init() { if (apiConfig.getPythonGateway().isEnabled()) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index 54023baad9..0b501dbbc8 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -17,181 +17,99 @@ package org.apache.dolphinscheduler.api.service; -import org.apache.dolphinscheduler.api.dto.resources.DeleteDataTransferResponse; +import org.apache.dolphinscheduler.api.dto.resources.CreateDirectoryRequest; +import org.apache.dolphinscheduler.api.dto.resources.CreateFileFromContentRequest; +import org.apache.dolphinscheduler.api.dto.resources.CreateFileRequest; +import org.apache.dolphinscheduler.api.dto.resources.DeleteResourceRequest; +import org.apache.dolphinscheduler.api.dto.resources.DownloadFileRequest; +import org.apache.dolphinscheduler.api.dto.resources.FetchFileContentRequest; +import org.apache.dolphinscheduler.api.dto.resources.PagingResourceItemRequest; +import org.apache.dolphinscheduler.api.dto.resources.RenameDirectoryRequest; +import org.apache.dolphinscheduler.api.dto.resources.RenameFileRequest; +import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent; +import org.apache.dolphinscheduler.api.dto.resources.UpdateFileFromContentRequest; +import org.apache.dolphinscheduler.api.dto.resources.UpdateFileRequest; import org.apache.dolphinscheduler.api.utils.PageInfo; -import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.enums.ProgramType; +import org.apache.dolphinscheduler.api.vo.ResourceItemVO; +import org.apache.dolphinscheduler.api.vo.resources.FetchFileContentResponse; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; import org.apache.dolphinscheduler.spi.enums.ResourceType; -import java.io.IOException; -import java.util.Map; +import java.util.List; -import org.springframework.web.multipart.MultipartFile; +import javax.servlet.http.HttpServletResponse; -/** - * resources service - */ public interface ResourcesService { /** - * create directory - * - * @param loginUser login user - * @param name alias - * @param type type - * @param pid parent id - * @param currentDir current directory - * @return create directory result + * Create a new directory in the resource storage, if the directory already exists will throw exception */ - Result createDirectory(User loginUser, - String name, - ResourceType type, - int pid, - String currentDir); + void createDirectory(CreateDirectoryRequest createDirectoryRequest); /** - * create resource - * - * @param loginUser login user - * @param name alias - * @param type type - * @param file file - * @param currentDir current directory - * @return create result code + * Rename the directory in the resource storage, if the origin directory not exists or the new directory already exists will throw exception. + *

If the origin directory is empty will only update the directory name. + *

If the origin directory is not empty will move all the files and directories to the new directory. + *

After update the origin directory will be deleted. */ - Result uploadResource(User loginUser, - String name, - ResourceType type, - MultipartFile file, - String currentDir); + void renameDirectory(RenameDirectoryRequest renameDirectoryRequest); /** - * update resource - * @param loginUser login user - * @param name name - * @param type resource type - * @param file resource file - * @return update result code + * Upload a new file to the resource storage, if the file already exists will throw exception */ - Result updateResource(User loginUser, - String fullName, - String tenantCode, - String name, - ResourceType type, - MultipartFile file); + void createFile(CreateFileRequest createFileRequest); /** - * query resources list paging - * - * @param loginUser login user - * @param type resource type - * @param searchVal search value - * @param pageNo page number - * @param pageSize page size - * @return resource list page + * Update the file in the resource storage, if the origin file not exists or the new file already exists will throw exception. + *

If the new file is empty will only update the file name. + *

If the new file is not empty will update the file content and name. + *

After update the origin file will be deleted. */ - Result> queryResourceListPaging(User loginUser, String fullName, String resTenantCode, - ResourceType type, String searchVal, Integer pageNo, - Integer pageSize); + void updateFile(UpdateFileRequest updateFileRequest); /** - * query resource list - * - * @param loginUser login user - * @param type resource type - * @return resource list + * Rename the file in the resource storage, if the origin file not exists or the new file already exists will throw exception. */ - Map queryResourceList(User loginUser, ResourceType type, String fullName); + void renameFile(RenameFileRequest renameFileRequest); /** - * query resource list by program type - * - * @param loginUser login user - * @param type resource type - * @return resource list + * Create a new file in the resource storage, if the file already exists will throw exception. + * Different with {@link ResourcesService#createFile(CreateFileRequest)} this method will create a new file with the given content. */ - Result queryResourceByProgramType(User loginUser, ResourceType type, ProgramType programType); + void createFileFromContent(CreateFileFromContentRequest createFileFromContentRequest); /** - * delete resource - * - * @param loginUser login user - * @return delete result code - * @throws IOException exception + * Update the file content. */ - Result delete(User loginUser, String fullName, String tenantCode) throws IOException; + void updateFileFromContent(UpdateFileFromContentRequest updateFileContentRequest); /** - * verify resource by name and type - * @param loginUser login user - * @param fullName resource full name - * @param type resource type - * @return true if the resource name not exists, otherwise return false + * Paging query resource items. + *

If the login user is not admin will only query the resource items that under the user's tenant. + *

If the login user is admin and {@link PagingResourceItemRequest##resourceAbsolutePath} is null will return all the resource items. */ - Result verifyResourceName(String fullName, ResourceType type, User loginUser); + PageInfo pagingResourceItem(PagingResourceItemRequest pagingResourceItemRequest); /** - * verify resource by file name - * @param fileName resource file name - * @param type resource type - * @return true if the resource file name, otherwise return false + * Query the resource file items by the given resource type and program type. */ - Result queryResourceByFileName(User loginUser, String fileName, ResourceType type, String resTenantCode); + List queryResourceFiles(User loginUser, ResourceType type); /** - * view resource file online - * - * @param skipLineNum skip line number - * @param limit limit - * @param fullName fullName - * @return resource content + * Delete the resource item. + *

If the resource item is a directory will delete all the files and directories under the directory. + *

If the resource item is a file will delete the file. + *

If the resource item not exists will throw exception. */ - Result readResource(User loginUser, String fullName, String tenantCode, int skipLineNum, int limit); + void delete(DeleteResourceRequest deleteResourceRequest); /** - * create resource file online - * - * @param loginUser login user - * @param type resource type - * @param fileName file name - * @param fileSuffix file suffix - * @param content content - * @return create result code + * Fetch the file content. */ - Result createResourceFile(User loginUser, ResourceType type, String fileName, String fileSuffix, - String content, String currentDirectory); + FetchFileContentResponse fetchResourceFileContent(FetchFileContentRequest fetchFileContentRequest); - /** - * create or update resource. - * If the folder is not already created, it will be ignored and directly create the new file - * - * @param userName user who create or update resource - * @param fullName The fullname of resource.Includes path and suffix. - * @param resourceContent content of resource - */ - StorageEntity createOrUpdateResource(String userName, String fullName, String resourceContent) throws Exception; - - /** - * updateProcessInstance resource - * - * @param loginUser login user - * @param fullName full name - * @param tenantCode tenantCode - * @param content content - * @return update result cod - */ - Result updateResourceContent(User loginUser, String fullName, String tenantCode, - String content); - - /** - * download file - * - * @return resource content - * @throws IOException exception - */ - org.springframework.core.io.Resource downloadResource(User loginUser, String fullName) throws IOException; + void downloadResource(HttpServletResponse response, DownloadFileRequest downloadFileRequest); /** * Get resource by given resource type and file name. @@ -202,21 +120,6 @@ public interface ResourcesService { */ StorageEntity queryFileStatus(String userName, String fileName) throws Exception; - /** - * delete DATA_TRANSFER data in resource center - * - * @param loginUser user who query resource - * @param days number of days - */ - DeleteDataTransferResponse deleteDataTransferData(User loginUser, Integer days); - - /** - * get resource base dir - * - * @param loginUser login user - * @param type resource type - * @return - */ - Result queryResourceBaseDir(User loginUser, ResourceType type); + String queryResourceBaseDir(User loginUser, ResourceType type); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java index 6a15da17a8..0757acd816 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java @@ -17,16 +17,29 @@ package org.apache.dolphinscheduler.api.service.impl; -import static org.apache.dolphinscheduler.common.constants.Constants.ALIAS; -import static org.apache.dolphinscheduler.common.constants.Constants.CONTENT; -import static org.apache.dolphinscheduler.common.constants.Constants.EMPTY_STRING; -import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPARATOR; -import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_SS; -import static org.apache.dolphinscheduler.common.constants.Constants.JAR; -import static org.apache.dolphinscheduler.common.constants.Constants.PERIOD; - -import org.apache.dolphinscheduler.api.dto.resources.DeleteDataTransferResponse; -import org.apache.dolphinscheduler.api.dto.resources.filter.ResourceFilter; +import org.apache.dolphinscheduler.api.dto.resources.CreateDirectoryDto; +import org.apache.dolphinscheduler.api.dto.resources.CreateDirectoryRequest; +import org.apache.dolphinscheduler.api.dto.resources.CreateFileDto; +import org.apache.dolphinscheduler.api.dto.resources.CreateFileFromContentDto; +import org.apache.dolphinscheduler.api.dto.resources.CreateFileFromContentRequest; +import org.apache.dolphinscheduler.api.dto.resources.CreateFileRequest; +import org.apache.dolphinscheduler.api.dto.resources.DeleteResourceDto; +import org.apache.dolphinscheduler.api.dto.resources.DeleteResourceRequest; +import org.apache.dolphinscheduler.api.dto.resources.DownloadFileDto; +import org.apache.dolphinscheduler.api.dto.resources.DownloadFileRequest; +import org.apache.dolphinscheduler.api.dto.resources.FetchFileContentDto; +import org.apache.dolphinscheduler.api.dto.resources.FetchFileContentRequest; +import org.apache.dolphinscheduler.api.dto.resources.PagingResourceItemRequest; +import org.apache.dolphinscheduler.api.dto.resources.QueryResourceDto; +import org.apache.dolphinscheduler.api.dto.resources.RenameDirectoryDto; +import org.apache.dolphinscheduler.api.dto.resources.RenameDirectoryRequest; +import org.apache.dolphinscheduler.api.dto.resources.RenameFileDto; +import org.apache.dolphinscheduler.api.dto.resources.RenameFileRequest; +import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent; +import org.apache.dolphinscheduler.api.dto.resources.UpdateFileDto; +import org.apache.dolphinscheduler.api.dto.resources.UpdateFileFromContentDto; +import org.apache.dolphinscheduler.api.dto.resources.UpdateFileFromContentRequest; +import org.apache.dolphinscheduler.api.dto.resources.UpdateFileRequest; import org.apache.dolphinscheduler.api.dto.resources.visitor.ResourceTreeVisitor; import org.apache.dolphinscheduler.api.dto.resources.visitor.Visitor; import org.apache.dolphinscheduler.api.enums.Status; @@ -34,1269 +47,374 @@ import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.metrics.ApiServerMetrics; import org.apache.dolphinscheduler.api.service.ResourcesService; import org.apache.dolphinscheduler.api.utils.PageInfo; -import org.apache.dolphinscheduler.api.utils.RegexUtils; -import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.enums.ProgramType; -import org.apache.dolphinscheduler.common.enums.ResUploadType; +import org.apache.dolphinscheduler.api.validator.resource.CreateDirectoryDtoValidator; +import org.apache.dolphinscheduler.api.validator.resource.CreateDirectoryRequestTransformer; +import org.apache.dolphinscheduler.api.validator.resource.CreateFileDtoValidator; +import org.apache.dolphinscheduler.api.validator.resource.CreateFileFromContentDtoValidator; +import org.apache.dolphinscheduler.api.validator.resource.DeleteResourceDtoValidator; +import org.apache.dolphinscheduler.api.validator.resource.DownloadFileDtoValidator; +import org.apache.dolphinscheduler.api.validator.resource.FetchFileContentDtoValidator; +import org.apache.dolphinscheduler.api.validator.resource.FileFromContentRequestTransformer; +import org.apache.dolphinscheduler.api.validator.resource.FileRequestTransformer; +import org.apache.dolphinscheduler.api.validator.resource.PagingResourceItemRequestTransformer; +import org.apache.dolphinscheduler.api.validator.resource.RenameDirectoryDtoValidator; +import org.apache.dolphinscheduler.api.validator.resource.RenameDirectoryRequestTransformer; +import org.apache.dolphinscheduler.api.validator.resource.RenameFileDtoValidator; +import org.apache.dolphinscheduler.api.validator.resource.RenameFileRequestTransformer; +import org.apache.dolphinscheduler.api.validator.resource.UpdateFileDtoValidator; +import org.apache.dolphinscheduler.api.validator.resource.UpdateFileFromContentDtoValidator; +import org.apache.dolphinscheduler.api.validator.resource.UpdateFileFromContentRequestTransformer; +import org.apache.dolphinscheduler.api.validator.resource.UpdateFileRequestTransformer; +import org.apache.dolphinscheduler.api.vo.ResourceItemVO; +import org.apache.dolphinscheduler.api.vo.resources.FetchFileContentResponse; import org.apache.dolphinscheduler.common.utils.FileUtils; -import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.dolphinscheduler.dao.entity.Tenant; -import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.dolphinscheduler.dao.entity.User; -import org.apache.dolphinscheduler.dao.mapper.TenantMapper; -import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper; +import org.apache.dolphinscheduler.dao.repository.TenantDao; import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.dolphinscheduler.spi.enums.ResourceType; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; import java.io.File; -import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Paths; -import java.text.MessageFormat; -import java.time.LocalDateTime; -import java.time.temporal.ChronoUnit; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.UUID; import java.util.stream.Collectors; +import javax.servlet.http.HttpServletResponse; + +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; import org.springframework.web.multipart.MultipartFile; -import com.google.common.io.Files; - @Service @Slf4j public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesService { @Autowired - private UdfFuncMapper udfFunctionMapper; - - @Autowired - private TenantMapper tenantMapper; + private TenantDao tenantDao; @Autowired private UserMapper userMapper; - @Autowired(required = false) - private StorageOperate storageOperate; - - /** - * create directory - * - * @param loginUser login user - * @param name alias - * @param type type - * @param pid parent id - * @param currentDir current directory - * @return create directory result - */ - @Override - @Transactional - public Result createDirectory(User loginUser, String name, ResourceType type, int pid, String currentDir) { - Result result = new Result<>(); - if (FileUtils.directoryTraversal(name)) { - log.warn("Parameter name is invalid, name:{}.", RegexUtils.escapeNRT(name)); - putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED); - return result; - } - - User user = userMapper.selectById(loginUser.getId()); - if (user == null) { - log.error("user {} not exists", loginUser.getId()); - putMsg(result, Status.USER_NOT_EXIST, loginUser.getId()); - return result; - } - - String tenantCode = getTenantCode(user); - checkFullName(tenantCode, currentDir); - - String userResRootPath = ResourceType.UDF.equals(type) ? storageOperate.getUdfDir(tenantCode) - : storageOperate.getResDir(tenantCode); - String fullName = !currentDir.contains(userResRootPath) ? userResRootPath + name : currentDir + name; - - try { - if (checkResourceExists(fullName)) { - log.error("resource directory {} has exist, can't recreate", fullName); - putMsg(result, Status.RESOURCE_EXIST); - return result; - } - } catch (Exception e) { - log.warn("Resource exists, can not create again, fullName:{}.", fullName, e); - throw new ServiceException("resource already exists, can't recreate"); - } - - // create directory in storage - createDirectory(loginUser, fullName, type, result); - return result; - } - - /** - * create resource - * - * @param loginUser login user - * @param name alias - * @param type type - * @param file file - * @param currentDir current directory - * @return create result code - */ - @Override - @Transactional - public Result uploadResource(User loginUser, String name, ResourceType type, MultipartFile file, - String currentDir) { - Result result = new Result<>(); - - User user = userMapper.selectById(loginUser.getId()); - if (user == null) { - log.error("user {} not exists", loginUser.getId()); - putMsg(result, Status.USER_NOT_EXIST, loginUser.getId()); - return result; - } - - String tenantCode = getTenantCode(user); - checkFullName(tenantCode, currentDir); - - result = verifyFile(name, type, file); - if (!result.getCode().equals(Status.SUCCESS.getCode())) { - return result; - } - - // check resource name exists - String userResRootPath = ResourceType.UDF.equals(type) ? storageOperate.getUdfDir(tenantCode) - : storageOperate.getResDir(tenantCode); - String currDirNFileName = !currentDir.contains(userResRootPath) ? userResRootPath + name : currentDir + name; - - try { - if (checkResourceExists(currDirNFileName)) { - log.error("resource {} has exist, can't recreate", RegexUtils.escapeNRT(name)); - putMsg(result, Status.RESOURCE_EXIST); - return result; - } - } catch (Exception e) { - throw new ServiceException("resource already exists, can't recreate"); - } - if (currDirNFileName.length() > Constants.RESOURCE_FULL_NAME_MAX_LENGTH) { - log.error( - "Resource file's name is longer than max full name length, fullName:{}, " - + "fullNameSize:{}, maxFullNameSize:{}", - RegexUtils.escapeNRT(name), currDirNFileName.length(), Constants.RESOURCE_FULL_NAME_MAX_LENGTH); - putMsg(result, Status.RESOURCE_FULL_NAME_TOO_LONG_ERROR); - return result; - } - - // fail upload - if (!upload(loginUser, currDirNFileName, file, type)) { - log.error("upload resource: {} file: {} failed.", RegexUtils.escapeNRT(name), - RegexUtils.escapeNRT(file.getOriginalFilename())); - putMsg(result, Status.STORE_OPERATE_CREATE_ERROR); - throw new ServiceException( - String.format("upload resource: %s file: %s failed.", name, file.getOriginalFilename())); - } else - ApiServerMetrics.recordApiResourceUploadSize(file.getSize()); - log.info("Upload resource file complete, resourceName:{}, fileName:{}.", RegexUtils.escapeNRT(name), - RegexUtils.escapeNRT(file.getOriginalFilename())); - putMsg(result, Status.SUCCESS); - return result; - } - - /** - * check resource is exists - * - * @param fullName fullName - * @return true if resource exists - */ - private boolean checkResourceExists(String fullName) { - Boolean existResource = false; - try { - existResource = storageOperate.exists(fullName); - } catch (IOException e) { - log.error("error occurred when checking resource: " + fullName, e); - } - return Boolean.TRUE.equals(existResource); - } - - /** - * update resource - * - * @param loginUser login user - * @param resourceFullName resource full name - * @param resTenantCode tenantCode in the request field "resTenantCode" for tenant code owning the resource, - * can be different from the login user in the case of logging in as admin users. - * @param name name - * @param type resource type - * @param file resource file - * @return update result code - */ - @Override - @Transactional - public Result updateResource(User loginUser, String resourceFullName, String resTenantCode, String name, - ResourceType type, MultipartFile file) { - Result result = new Result<>(); - - User user = userMapper.selectById(loginUser.getId()); - if (user == null) { - log.error("user {} not exists", loginUser.getId()); - putMsg(result, Status.USER_NOT_EXIST, loginUser.getId()); - return result; - } - - String tenantCode = getTenantCode(user); - checkFullName(tenantCode, resourceFullName); + @Autowired + private StorageOperator storageOperator; - if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode)) { - log.error("current user does not have permission"); - putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); - return result; - } + @Autowired + private CreateDirectoryRequestTransformer createDirectoryRequestTransformer; - String defaultPath = storageOperate.getDir(type, tenantCode); + @Autowired + private CreateDirectoryDtoValidator createDirectoryDtoValidator; - StorageEntity resource; - try { - resource = storageOperate.getFileStatus(resourceFullName, defaultPath, resTenantCode, type); - } catch (Exception e) { - log.error("Get file status fail, resource path: {}", resourceFullName, e); - putMsg(result, Status.RESOURCE_NOT_EXIST); - throw new ServiceException((String.format("Get file status fail, resource path: %s", resourceFullName))); - } + @Autowired + private RenameDirectoryRequestTransformer renameDirectoryRequestTransformer; - // TODO: deal with OSS - if (resource.isDirectory() && storageOperate.returnStorageType().equals(ResUploadType.S3) - && !resource.getFileName().equals(name)) { - log.warn("Directory in S3 storage can not be renamed."); - putMsg(result, Status.S3_CANNOT_RENAME); - return result; - } + @Autowired + private RenameDirectoryDtoValidator renameDirectoryDtoValidator; - // check if updated name of the resource already exists - String originFullName = resource.getFullName(); - String originResourceName = resource.getAlias(); - - // the format of hdfs folders in the implementation has a "/" at the very end, we need to remove it. - originFullName = originFullName.endsWith("/") ? StringUtils.chop(originFullName) : originFullName; - name = name.endsWith("/") ? StringUtils.chop(name) : name; - // updated fullName - String fullName = String.format(FORMAT_SS, - originFullName.substring(0, originFullName.lastIndexOf(FOLDER_SEPARATOR) + 1), name); - if (!originResourceName.equals(name)) { - try { - if (checkResourceExists(fullName)) { - log.error("resource {} already exists, can't recreate", fullName); - putMsg(result, Status.RESOURCE_EXIST); - return result; - } - } catch (Exception e) { - throw new ServiceException(String.format("error occurs while querying resource: %s", fullName)); - } + @Autowired + private RenameFileRequestTransformer renameFileRequestTransformer; - } + @Autowired + private RenameFileDtoValidator renameFileDtoValidator; - result = verifyFile(name, type, file); - if (!result.getCode().equals(Status.SUCCESS.getCode())) { - return result; - } + @Autowired + private FileFromContentRequestTransformer createFileFromContentRequestTransformer; - Date now = new Date(); + @Autowired + private CreateFileFromContentDtoValidator createFileFromContentDtoValidator; - resource.setAlias(name); - resource.setFileName(name); - resource.setFullName(fullName); - resource.setUpdateTime(now); - if (file != null) { - resource.setSize(file.getSize()); - } + @Autowired + private FetchFileContentDtoValidator fetchFileContentDtoValidator; - // if name unchanged, return directly without moving on HDFS - if (originResourceName.equals(name) && file == null) { - return result; - } + @Autowired + private UpdateFileFromContentRequestTransformer updateFileFromContentRequestTransformer; - if (file != null) { - // fail upload - if (!upload(loginUser, fullName, file, type)) { - log.error("Storage operation error, resourceName:{}, originFileName:{}.", name, - RegexUtils.escapeNRT(file.getOriginalFilename())); - putMsg(result, Status.HDFS_OPERATION_ERROR); - throw new ServiceException( - String.format("upload resource: %s file: %s failed.", name, file.getOriginalFilename())); - } - if (!fullName.equals(originFullName)) { - try { - storageOperate.delete(originFullName, false); - } catch (IOException e) { - log.error("Resource delete error, resourceFullName:{}.", originFullName, e); - throw new ServiceException(String.format("delete resource: %s failed.", originFullName)); - } - } + @Autowired + private UpdateFileFromContentDtoValidator updateFileFromContentDtoValidator; - ApiServerMetrics.recordApiResourceUploadSize(file.getSize()); - return result; - } + @Autowired + private FileRequestTransformer createFileRequestTransformer; - // get the path of dest file in hdfs - String destHdfsFileName = fullName; - try { - log.info("start copy {} -> {}", originFullName, destHdfsFileName); - storageOperate.copy(originFullName, destHdfsFileName, true, true); - putMsg(result, Status.SUCCESS); - } catch (Exception e) { - log.error(MessageFormat.format(" copy {0} -> {1} fail", originFullName, destHdfsFileName), e); - putMsg(result, Status.HDFS_COPY_FAIL); - throw new ServiceException( - MessageFormat.format(Status.HDFS_COPY_FAIL.getMsg(), originFullName, destHdfsFileName)); - } + @Autowired + private CreateFileDtoValidator createFileDtoValidator; - return result; - } + @Autowired + private UpdateFileRequestTransformer updateFileRequestTransformer; - private Result verifyFile(String name, ResourceType type, MultipartFile file) { - Result result = new Result<>(); - putMsg(result, Status.SUCCESS); + @Autowired + private UpdateFileDtoValidator updateFileDtoValidator; - if (FileUtils.directoryTraversal(name)) { - log.warn("Parameter file alias name verify failed, fileAliasName:{}.", RegexUtils.escapeNRT(name)); - putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED); - return result; - } + @Autowired + private DeleteResourceDtoValidator deleteResourceDtoValidator; - if (file != null && FileUtils.directoryTraversal(Objects.requireNonNull(file.getOriginalFilename()))) { - log.warn("File original name verify failed, fileOriginalName:{}.", - RegexUtils.escapeNRT(file.getOriginalFilename())); - putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED); - return result; - } + @Autowired + private DownloadFileDtoValidator downloadFileDtoValidator; - if (file != null) { - // file is empty - if (file.isEmpty()) { - log.warn("Parameter file is empty, fileOriginalName:{}.", - RegexUtils.escapeNRT(file.getOriginalFilename())); - putMsg(result, Status.RESOURCE_FILE_IS_EMPTY); - return result; - } - - // file suffix - String fileSuffix = Files.getFileExtension(file.getOriginalFilename()); - String nameSuffix = Files.getFileExtension(name); - - // determine file suffix - if (!fileSuffix.equalsIgnoreCase(nameSuffix)) { - // rename file suffix and original suffix must be consistent - log.warn("Rename file suffix and original suffix must be consistent, fileOriginalName:{}.", - RegexUtils.escapeNRT(file.getOriginalFilename())); - putMsg(result, Status.RESOURCE_SUFFIX_FORBID_CHANGE); - return result; - } - - // If resource type is UDF, only jar packages are allowed to be uploaded, and the suffix must be .jar - if (Constants.UDF.equals(type.name()) && !JAR.equalsIgnoreCase(fileSuffix)) { - log.warn(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg()); - putMsg(result, Status.UDF_RESOURCE_SUFFIX_NOT_JAR); - return result; - } - if (file.getSize() > Constants.MAX_FILE_SIZE) { - log.warn( - "Resource file size is larger than max file size, fileOriginalName:{}, fileSize:{}, maxFileSize:{}.", - RegexUtils.escapeNRT(file.getOriginalFilename()), file.getSize(), Constants.MAX_FILE_SIZE); - putMsg(result, Status.RESOURCE_SIZE_EXCEED_LIMIT); - return result; - } - } - return result; - } + @Autowired + private PagingResourceItemRequestTransformer pagingResourceItemRequestTransformer; - /** - * query resources list paging - * - * @param loginUser login user - * @param fullName resource full name - * @param resTenantCode tenantCode in the request field "resTenantCode" for tenant code owning the resource, - * can be different from the login user in the case of logging in as admin users. - * @param type resource type - * @param searchVal search value - * @param pageNo page number - * @param pageSize page size - * @return resource list page - */ @Override - public Result> queryResourceListPaging(User loginUser, String fullName, - String resTenantCode, ResourceType type, - String searchVal, Integer pageNo, Integer pageSize) { - Result> result = new Result<>(); - PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); - if (storageOperate == null) { - log.warn("The resource storage is not opened."); - return Result.success(pageInfo); - } - - User user = userMapper.selectById(loginUser.getId()); - if (user == null) { - log.error("user {} not exists", loginUser.getId()); - putMsg(result, Status.USER_NOT_EXIST, loginUser.getId()); - return result; - } - - String tenantCode = getTenantCode(user); - checkFullName(tenantCode, fullName); + public void createDirectory(CreateDirectoryRequest createDirectoryRequest) { + CreateDirectoryDto createDirectoryDto = createDirectoryRequestTransformer.transform(createDirectoryRequest); + createDirectoryDtoValidator.validate(createDirectoryDto); - if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode)) { - log.error("current user does not have permission"); - putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); - return result; - } - - List resourcesList; - try { - resourcesList = queryStorageEntityList(loginUser, fullName, type, tenantCode, false); - } catch (ServiceException e) { - putMsg(result, Status.RESOURCE_NOT_EXIST); - return result; - } - - // remove leading and trailing spaces in searchVal - String trimmedSearchVal = searchVal != null ? searchVal.trim() : ""; - // filter based on trimmed searchVal - List filteredResourceList = resourcesList.stream() - .filter(x -> x.getFileName().contains(trimmedSearchVal)).collect(Collectors.toList()); - // inefficient pagination - List slicedResourcesList = filteredResourceList.stream().skip((long) (pageNo - 1) * pageSize) - .limit(pageSize).collect(Collectors.toList()); - - pageInfo.setTotal(filteredResourceList.size()); - pageInfo.setTotalList(slicedResourcesList); - result.setData(pageInfo); - putMsg(result, Status.SUCCESS); - return result; - } - - private List queryStorageEntityList(User loginUser, String fullName, ResourceType type, - String tenantCode, boolean recursive) { - String defaultPath = ""; - List resourcesList = new ArrayList<>(); - String resourceStorageType = - PropertyUtils.getString(Constants.RESOURCE_STORAGE_TYPE, ResUploadType.LOCAL.name()); - if (isAdmin(loginUser) && StringUtils.isBlank(fullName)) { - // list all tenants' resources to admin users in the root directory - List userList = userMapper.selectList(null); - Set visitedTenantEntityCode = new HashSet<>(); - for (User userEntity : userList) { - String tenantEntityCode = getTenantCode(userEntity); - if (!visitedTenantEntityCode.contains(tenantEntityCode)) { - defaultPath = storageOperate.getResDir(tenantEntityCode); - if (type.equals(ResourceType.UDF)) { - defaultPath = storageOperate.getUdfDir(tenantEntityCode); - } - try { - resourcesList.addAll(recursive - ? storageOperate.listFilesStatusRecursively(defaultPath, defaultPath, tenantEntityCode, - type) - : storageOperate.listFilesStatus(defaultPath, defaultPath, tenantEntityCode, type)); - - visitedTenantEntityCode.add(tenantEntityCode); - } catch (Exception e) { - log.error(e.getMessage() + " Resource path: {}", defaultPath, e); - throw new ServiceException( - String.format(e.getMessage() + " make sure resource path: %s exists in %s", defaultPath, - resourceStorageType)); - } - } - } - } else { - defaultPath = storageOperate.getResDir(tenantCode); - if (type.equals(ResourceType.UDF)) { - defaultPath = storageOperate.getUdfDir(tenantCode); - } - - try { - if (StringUtils.isBlank(fullName)) { - fullName = defaultPath; - } - resourcesList = - recursive ? storageOperate.listFilesStatusRecursively(fullName, defaultPath, tenantCode, type) - : storageOperate.listFilesStatus(fullName, defaultPath, tenantCode, type); - } catch (Exception e) { - log.error(e.getMessage() + " Resource path: {}", fullName, e); - throw new ServiceException(String.format(e.getMessage() + " make sure resource path: %s exists in %s", - defaultPath, resourceStorageType)); - } - } - - return resourcesList; + storageOperator.createStorageDir(createDirectoryDto.getDirectoryAbsolutePath()); + log.info("Success create directory: {}", createDirectoryRequest.getParentAbsoluteDirectory()); } - /** - * create directory - * xxx The steps to verify resources are cumbersome and can be optimized - * - * @param loginUser login user - * @param fullName full name - * @param type resource type - * @param result Result - */ - private void createDirectory(User loginUser, String fullName, ResourceType type, Result result) { - String tenantCode = tenantMapper.queryById(loginUser.getTenantId()).getTenantCode(); - // String directoryName = storageOperate.getFileName(type, tenantCode, fullName); - String resourceRootPath = storageOperate.getDir(type, tenantCode); + @Override + public void createFile(CreateFileRequest createFileRequest) { + CreateFileDto createFileDto = createFileRequestTransformer.transform(createFileRequest); + createFileDtoValidator.validate(createFileDto); + + // todo: use storage proxy + MultipartFile file = createFileDto.getFile(); + String fileAbsolutePath = createFileDto.getFileAbsolutePath(); + String srcLocalTmpFileAbsolutePath = copyFileToLocal(file); try { - if (!storageOperate.exists(resourceRootPath)) { - storageOperate.createTenantDirIfNotExists(tenantCode); - } - - if (!storageOperate.mkdir(tenantCode, fullName)) { - throw new ServiceException(String.format("Create resource directory: %s failed.", fullName)); - } - putMsg(result, Status.SUCCESS); - } catch (Exception e) { - throw new ServiceException(String.format("create resource directory: %s failed.", fullName)); + storageOperator.upload(srcLocalTmpFileAbsolutePath, fileAbsolutePath, true, false); + ApiServerMetrics.recordApiResourceUploadSize(file.getSize()); + log.info("Success upload resource file: {} complete.", fileAbsolutePath); + } catch (Exception ex) { + // If exception, clear the tmp path + FileUtils.deleteFile(srcLocalTmpFileAbsolutePath); + throw ex; } } - /** - * upload file to hdfs - * - * @param loginUser login user - * @param fullName full name - * @param file file - * @param type resource type - * @return upload success return true, otherwise false - */ - private boolean upload(User loginUser, String fullName, MultipartFile file, ResourceType type) { - // save to local - String fileSuffix = Files.getFileExtension(file.getOriginalFilename()); - String nameSuffix = Files.getFileExtension(fullName); - - // determine file suffix - if (!fileSuffix.equalsIgnoreCase(nameSuffix)) { - return false; - } - // query tenant - String tenantCode = getTenantCode(loginUser); - // random file name - String localFilename = FileUtils.getUploadFilename(tenantCode, UUID.randomUUID().toString()); - - // save file to hdfs, and delete original file - String resourcePath = storageOperate.getDir(type, tenantCode); + @Override + public void createFileFromContent(CreateFileFromContentRequest createFileFromContentRequest) { + CreateFileFromContentDto createFileFromContentDto = + createFileFromContentRequestTransformer.transform(createFileFromContentRequest); + createFileFromContentDtoValidator.validate(createFileFromContentDto); + + // todo: use storage proxy + String fileContent = createFileFromContentDto.getFileContent(); + String fileAbsolutePath = createFileFromContentDto.getFileAbsolutePath(); + String srcLocalTmpFileAbsolutePath = copyFileToLocal(fileContent); try { - // if tenant dir not exists - if (!storageOperate.exists(resourcePath)) { - storageOperate.createTenantDirIfNotExists(tenantCode); - } - org.apache.dolphinscheduler.api.utils.FileUtils.copyInputStreamToFile(file, localFilename); - storageOperate.upload(tenantCode, localFilename, fullName, true, true); - FileUtils.deleteFile(localFilename); - } catch (Exception e) { - FileUtils.deleteFile(localFilename); - log.error(e.getMessage(), e); - return false; + storageOperator.upload(srcLocalTmpFileAbsolutePath, fileAbsolutePath, true, false); + ApiServerMetrics.recordApiResourceUploadSize(fileContent.length()); + log.info("Success upload resource file: {} complete.", fileAbsolutePath); + } catch (Exception ex) { + // If exception, clear the tmp path + FileUtils.deleteFile(srcLocalTmpFileAbsolutePath); + throw ex; } - return true; } - /** - * query resource list - * - * @param loginUser login user - * @param type resource type - * @param fullName resource full name - * @return resource list - */ @Override - public Map queryResourceList(User loginUser, ResourceType type, String fullName) { - Map result = new HashMap<>(); - if (storageOperate == null) { - result.put(Constants.DATA_LIST, Collections.emptyList()); - result.put(Constants.STATUS, Status.SUCCESS); - return result; - } - - User user = userMapper.selectById(loginUser.getId()); - if (user == null) { - log.error("user {} not exists", loginUser.getId()); - putMsg(result, Status.USER_NOT_EXIST, loginUser.getId()); - return null; - } - - String tenantCode = getTenantCode(user); - checkFullName(tenantCode, fullName); - - String baseDir = storageOperate.getDir(type, tenantCode); - - List resourcesList = new ArrayList<>(); - if (StringUtils.isBlank(fullName)) { - if (isAdmin(loginUser)) { - List userList = userMapper.selectList(null); - Set visitedTenantEntityCode = new HashSet<>(); - for (User userEntity : userList) { - String tenantEntityCode = getTenantCode(userEntity); - if (!visitedTenantEntityCode.contains(tenantEntityCode)) { - baseDir = storageOperate.getDir(type, tenantEntityCode); - resourcesList.addAll(storageOperate.listFilesStatusRecursively(baseDir, baseDir, - tenantEntityCode, type)); - visitedTenantEntityCode.add(tenantEntityCode); - } - } - } else { - resourcesList = storageOperate.listFilesStatusRecursively(baseDir, baseDir, tenantCode, type); - } - } else { - resourcesList = storageOperate.listFilesStatusRecursively(fullName, baseDir, tenantCode, type); - } - - Visitor resourceTreeVisitor = new ResourceTreeVisitor(resourcesList); - result.put(Constants.DATA_LIST, resourceTreeVisitor.visit(baseDir).getChildren()); - putMsg(result, Status.SUCCESS); - - return result; + public void renameDirectory(RenameDirectoryRequest renameDirectoryRequest) { + RenameDirectoryDto renameDirectoryDto = renameDirectoryRequestTransformer.transform(renameDirectoryRequest); + renameDirectoryDtoValidator.validate(renameDirectoryDto); + + String originDirectoryAbsolutePath = renameDirectoryDto.getOriginDirectoryAbsolutePath(); + String targetDirectoryAbsolutePath = renameDirectoryDto.getTargetDirectoryAbsolutePath(); + storageOperator.copy(originDirectoryAbsolutePath, targetDirectoryAbsolutePath, true, true); + log.info("Success rename directory: {} -> {} ", originDirectoryAbsolutePath, targetDirectoryAbsolutePath); } - /** - * query resource list by program type - * - * @param loginUser login user - * @param type resource type - * @return resource list - */ @Override - public Result queryResourceByProgramType(User loginUser, ResourceType type, ProgramType programType) { - Result result = new Result<>(); - - User user = userMapper.selectById(loginUser.getId()); - if (user == null) { - log.error("user {} not exists", loginUser.getId()); - putMsg(result, Status.USER_NOT_EXIST, loginUser.getId()); - return result; - } - - Tenant tenant = tenantMapper.queryById(user.getTenantId()); - if (tenant == null) { - log.error("tenant not exists"); - putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST); - return result; - } - - String tenantCode = tenant.getTenantCode(); - - List allResourceList = queryStorageEntityList(loginUser, "", type, tenantCode, true); - - String suffix = ".jar"; - if (programType != null) { - switch (programType) { - case JAVA: - case SCALA: - break; - case PYTHON: - suffix = ".py"; - break; - default: - } - } - List resources = new ResourceFilter(suffix, new ArrayList<>(allResourceList)).filter(); - Visitor visitor = new ResourceTreeVisitor(resources); - result.setData(visitor.visit("").getChildren()); - putMsg(result, Status.SUCCESS); - return result; + public void renameFile(RenameFileRequest renameFileRequest) { + RenameFileDto renameFileDto = renameFileRequestTransformer.transform(renameFileRequest); + renameFileDtoValidator.validate(renameFileDto); + + String originFileAbsolutePath = renameFileDto.getOriginFileAbsolutePath(); + String targetFileAbsolutePath = renameFileDto.getTargetFileAbsolutePath(); + storageOperator.copy(originFileAbsolutePath, targetFileAbsolutePath, true, true); + log.info("Success rename file: {} -> {} ", originFileAbsolutePath, targetFileAbsolutePath); } - /** - * delete resource - * - * @param loginUser login user - * @param fullName resource full name - * @param resTenantCode tenantCode in the request field "resTenantCode" for tenant code owning the resource, - * can be different from the login user in the case of logging in as admin users. - * @return delete result code - * @throws IOException exception - */ @Override - @Transactional(rollbackFor = Exception.class) - public Result delete(User loginUser, String fullName, String resTenantCode) throws IOException { - Result result = new Result<>(); - - User user = userMapper.selectById(loginUser.getId()); - if (user == null) { - log.error("user {} not exists", loginUser.getId()); - putMsg(result, Status.USER_NOT_EXIST, loginUser.getId()); - return result; - } - - String tenantCode = getTenantCode(user); - checkFullName(tenantCode, fullName); - - if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode)) { - log.error("current user does not have permission"); - putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); - return result; - } - - String baseDir = storageOperate.getResDir(tenantCode); + public void updateFile(UpdateFileRequest updateFileRequest) { + UpdateFileDto updateFileDto = updateFileRequestTransformer.transform(updateFileRequest); + updateFileDtoValidator.validate(updateFileDto); - StorageEntity resource; + String srcLocalTmpFileAbsolutePath = copyFileToLocal(updateFileDto.getFile()); try { - resource = storageOperate.getFileStatus(fullName, baseDir, resTenantCode, null); - } catch (Exception e) { - log.error(e.getMessage() + " Resource path: {}", fullName, e); - putMsg(result, Status.RESOURCE_NOT_EXIST); - throw new ServiceException(String.format(e.getMessage() + " Resource path: %s", fullName)); - } - - if (resource == null) { - log.error("Resource does not exist, resource full name:{}.", fullName); - putMsg(result, Status.RESOURCE_NOT_EXIST); - return result; - } - - // recursively delete a folder - List allChildren = - storageOperate.listFilesStatusRecursively(fullName, baseDir, resTenantCode, resource.getType()) - .stream().map(storageEntity -> storageEntity.getFullName()).collect(Collectors.toList()); - - String[] allChildrenFullNameArray = allChildren.stream().toArray(String[]::new); - - // if resource type is UDF,need check whether it is bound by UDF function - if (resource.getType() == (ResourceType.UDF)) { - List udfFuncs = udfFunctionMapper.listUdfByResourceFullName(allChildrenFullNameArray); - if (CollectionUtils.isNotEmpty(udfFuncs)) { - log.warn("Resource can not be deleted because it is bound by UDF functions, udfFuncIds:{}", udfFuncs); - putMsg(result, Status.UDF_RESOURCE_IS_BOUND, udfFuncs.get(0).getFuncName()); - return result; - } + storageOperator.upload(srcLocalTmpFileAbsolutePath, updateFileDto.getFileAbsolutePath(), true, true); + ApiServerMetrics.recordApiResourceUploadSize(updateFileDto.getFile().getSize()); + log.info("Success upload resource file: {} complete.", updateFileDto.getFileAbsolutePath()); + } catch (Exception ex) { + // If exception, clear the tmp path + FileUtils.deleteFile(srcLocalTmpFileAbsolutePath); + throw ex; } - - // delete file on hdfs,S3 - storageOperate.delete(fullName, allChildren, true); - - putMsg(result, Status.SUCCESS); - - return result; } - /** - * verify resource by name and type - * - * @param loginUser login user - * @param fullName resource full name - * @param type resource type - * @return true if the resource name not exists, otherwise return false - */ @Override - public Result verifyResourceName(String fullName, ResourceType type, User loginUser) { - Result result = new Result<>(); - putMsg(result, Status.SUCCESS); - if (checkResourceExists(fullName)) { - log.error("Resource with same name exists so can not create again, resourceType:{}, resourceName:{}.", type, - RegexUtils.escapeNRT(fullName)); - putMsg(result, Status.RESOURCE_EXIST); - } - - return result; + public PageInfo pagingResourceItem(PagingResourceItemRequest pagingResourceItemRequest) { + + QueryResourceDto queryResourceDto = pagingResourceItemRequestTransformer.transform(pagingResourceItemRequest); + List resourceAbsolutePaths = queryResourceDto.getResourceAbsolutePaths(); + if (CollectionUtils.isEmpty(resourceAbsolutePaths)) { + return new PageInfo<>(pagingResourceItemRequest.getPageNo(), pagingResourceItemRequest.getPageSize()); + } + + for (String resourceAbsolutePath : resourceAbsolutePaths) { + createDirectoryDtoValidator.exceptionResourceAbsolutePathInvalidated(resourceAbsolutePath); + createDirectoryDtoValidator.exceptionUserNoResourcePermission(pagingResourceItemRequest.getLoginUser(), + resourceAbsolutePath); + } + + Integer pageNo = pagingResourceItemRequest.getPageNo(); + Integer pageSize = pagingResourceItemRequest.getPageSize(); + + List storageEntities = resourceAbsolutePaths.stream() + .flatMap(resourceAbsolutePath -> storageOperator.listStorageEntity(resourceAbsolutePath).stream()) + .collect(Collectors.toList()); + + List result = storageEntities + .stream() + .filter(storageEntity -> storageEntity.getFileName() + .contains(pagingResourceItemRequest.getResourceNameKeyWord())) + .skip((long) (pageNo - 1) * pageSize) + .limit(pageSize) + .map(ResourceItemVO::new) + .collect(Collectors.toList()); + + return PageInfo.builder() + .pageNo(pagingResourceItemRequest.getPageNo()) + .pageSize(pagingResourceItemRequest.getPageSize()) + .total(storageEntities.size()) + .totalList(result) + .build(); } - /** - * verify resource by full name or pid and type - * - * @param fileName resource file name - * @param type resource type - * @param resTenantCode tenantCode in the request field "resTenantCode" for tenant code owning the resource, - * can be different from the login user in the case of logging in as admin users. - * @return true if the resource full name or pid not exists, otherwise return false - */ @Override - public Result queryResourceByFileName(User loginUser, String fileName, ResourceType type, - String resTenantCode) { - Result result = new Result<>(); - if (StringUtils.isBlank(fileName)) { - putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR); - return result; - } - - User user = userMapper.selectById(loginUser.getId()); - if (user == null) { - log.error("user {} not exists", loginUser.getId()); - putMsg(result, Status.USER_NOT_EXIST, loginUser.getId()); - return result; - } - - String tenantCode = getTenantCode(user); - - if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode)) { - log.error("current user does not have permission"); - putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); - return result; - } - - String defaultPath = storageOperate.getDir(type, resTenantCode); - StorageEntity file; - try { - file = storageOperate.getFileStatus(defaultPath + fileName, defaultPath, resTenantCode, type); - } catch (Exception e) { - log.error(e.getMessage() + " Resource path: {}", defaultPath + fileName, e); - putMsg(result, Status.RESOURCE_NOT_EXIST); - return result; - } - - putMsg(result, Status.SUCCESS); - result.setData(file); - return result; + public List queryResourceFiles(User loginUser, ResourceType resourceType) { + Tenant tenant = tenantDao.queryOptionalById(loginUser.getTenantId()) + .orElseThrow(() -> new ServiceException(Status.TENANT_NOT_EXIST, loginUser.getTenantId())); + String storageBaseDirectory = storageOperator.getStorageBaseDirectory(tenant.getTenantCode(), resourceType); + List allResourceFiles = storageOperator.listFileStorageEntityRecursively(storageBaseDirectory); + + Visitor visitor = new ResourceTreeVisitor(allResourceFiles); + return visitor.visit("").getChildren(); } - /** - * view resource file online - * - * @param fullName resource fullName - * @param resTenantCode owner's tenant code of the resource - * @param skipLineNum skip line number - * @param limit limit - * @return resource content - */ @Override - public Result readResource(User loginUser, String fullName, String resTenantCode, int skipLineNum, - int limit) { - Result result = new Result<>(); - - User user = userMapper.selectById(loginUser.getId()); - if (user == null) { - log.error("user {} not exists", loginUser.getId()); - putMsg(result, Status.USER_NOT_EXIST, loginUser.getId()); - return result; - } - - String tenantCode = getTenantCode(user); - checkFullName(tenantCode, fullName); - - if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode)) { - log.error("current user does not have permission"); - putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); - return result; - } - - // check preview or not by file suffix - String nameSuffix = Files.getFileExtension(fullName); - String resourceViewSuffixes = FileUtils.getResourceViewSuffixes(); - if (StringUtils.isNotEmpty(resourceViewSuffixes)) { - List strList = Arrays.asList(resourceViewSuffixes.split(",")); - if (!strList.contains(nameSuffix)) { - log.error("Resource suffix does not support view,resourceFullName:{}, suffix:{}.", fullName, - nameSuffix); - putMsg(result, Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW); - return result; - } - } - - List content; - try { - if (storageOperate.exists(fullName)) { - content = storageOperate.vimFile(tenantCode, fullName, skipLineNum, limit); - long size = content.stream().mapToLong(String::length).sum(); - ApiServerMetrics.recordApiResourceDownloadSize(size); - } else { - log.error("read file {} not exist in storage", fullName); - putMsg(result, Status.RESOURCE_FILE_NOT_EXIST, fullName); - return result; - } - - } catch (Exception e) { - log.error("Resource {} read failed", fullName, e); - putMsg(result, Status.HDFS_OPERATION_ERROR); - return result; - } - - putMsg(result, Status.SUCCESS); - Map map = new HashMap<>(); - map.put(ALIAS, fullName); - map.put(CONTENT, String.join("\n", content)); - result.setData(map); - - return result; + public void delete(DeleteResourceRequest deleteResourceRequest) { + DeleteResourceDto deleteResourceDto = DeleteResourceDto.builder() + .loginUser(deleteResourceRequest.getLoginUser()) + .resourceAbsolutePath(deleteResourceRequest.getResourceAbsolutePath()) + .build(); + deleteResourceDtoValidator.validate(deleteResourceDto); + storageOperator.delete(deleteResourceDto.getResourceAbsolutePath(), true); } - /** - * create resource file online - * - * @param loginUser login user - * @param type resource type - * @param fileName file name - * @param fileSuffix file suffix - * @param content content - * @param currentDir current directory - * @return create result code - */ @Override - @Transactional - public Result createResourceFile(User loginUser, ResourceType type, String fileName, String fileSuffix, - String content, String currentDir) { - Result result = new Result<>(); - - User user = userMapper.selectById(loginUser.getId()); - if (user == null) { - log.error("user {} not exists", loginUser.getId()); - putMsg(result, Status.USER_NOT_EXIST, loginUser.getId()); - return result; - } - - String tenantCode = getTenantCode(user); - checkFullName(tenantCode, currentDir); - - if (FileUtils.directoryTraversal(fileName)) { - log.warn("File name verify failed, fileName:{}.", RegexUtils.escapeNRT(fileName)); - putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED); - return result; - } - - // check file suffix - String nameSuffix = fileSuffix.trim(); - String resourceViewSuffixes = FileUtils.getResourceViewSuffixes(); - if (StringUtils.isNotEmpty(resourceViewSuffixes)) { - List strList = Arrays.asList(resourceViewSuffixes.split(",")); - if (!strList.contains(nameSuffix)) { - log.warn("Resource suffix does not support view, suffix:{}.", nameSuffix); - putMsg(result, Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW); - return result; - } - } - - String name = fileName.trim() + "." + nameSuffix; - - String userResRootPath = storageOperate.getResDir(tenantCode); - String fullName = currentDir.contains(userResRootPath) ? currentDir + name : userResRootPath + name; - - result = verifyResourceName(fullName, type, loginUser); - if (!result.getCode().equals(Status.SUCCESS.getCode())) { - return result; - } - - result = uploadContentToStorage(fullName, tenantCode, content); - if (!result.getCode().equals(Status.SUCCESS.getCode())) { - throw new ServiceException(result.getMsg()); - } - return result; + public FetchFileContentResponse fetchResourceFileContent(FetchFileContentRequest fetchFileContentRequest) { + FetchFileContentDto fetchFileContentDto = FetchFileContentDto.builder() + .loginUser(fetchFileContentRequest.getLoginUser()) + .resourceFileAbsolutePath(fetchFileContentRequest.getResourceFileAbsolutePath()) + .skipLineNum(fetchFileContentRequest.getSkipLineNum()) + .limit(fetchFileContentRequest.getLimit()) + .build(); + fetchFileContentDtoValidator.validate(fetchFileContentDto); + + String content = storageOperator + .fetchFileContent( + fetchFileContentRequest.getResourceFileAbsolutePath(), + fetchFileContentRequest.getSkipLineNum(), + fetchFileContentRequest.getLimit()) + .stream() + .collect(Collectors.joining("\n")); + + ApiServerMetrics.recordApiResourceDownloadSize(content.length()); + + return FetchFileContentResponse.builder() + .content(content) + .build(); } @Override - @Transactional - public StorageEntity createOrUpdateResource(String userName, String filepath, - String resourceContent) throws Exception { - User user = userMapper.queryByUserNameAccurately(userName); - int suffixLabelIndex = filepath.indexOf(PERIOD); - if (suffixLabelIndex == -1) { - throw new IllegalArgumentException(String - .format("Not allow create or update resources without extension name, filepath: %s", filepath)); - } - - String defaultPath = storageOperate.getResDir(user.getTenantCode()); - String fullName = defaultPath + filepath; + public void updateFileFromContent(UpdateFileFromContentRequest updateFileContentRequest) { + UpdateFileFromContentDto updateFileFromContentDto = + updateFileFromContentRequestTransformer.transform(updateFileContentRequest); + updateFileFromContentDtoValidator.validate(updateFileFromContentDto); - Result result = uploadContentToStorage(fullName, user.getTenantCode(), resourceContent); - if (result.getCode() != Status.SUCCESS.getCode()) { - throw new ServiceException(result.getMsg()); + String srcLocalTmpFileAbsolutePath = copyFileToLocal(updateFileFromContentDto.getFileContent()); + try { + storageOperator.upload(srcLocalTmpFileAbsolutePath, updateFileFromContentDto.getFileAbsolutePath(), true, + true); + ApiServerMetrics.recordApiResourceUploadSize(updateFileFromContentDto.getFileContent().length()); + log.info("Success upload resource file: {} complete.", updateFileFromContentDto.getFileAbsolutePath()); + } catch (Exception ex) { + // If exception, clear the tmp path + FileUtils.deleteFile(srcLocalTmpFileAbsolutePath); + throw new ServiceException("Update the resource file from content: " + + updateFileFromContentDto.getFileAbsolutePath() + " failed", ex); } - return storageOperate.getFileStatus(fullName, defaultPath, user.getTenantCode(), ResourceType.FILE); } - /** - * updateProcessInstance resource - * - * @param fullName resource full name - * @param resTenantCode tenantCode in the request field "resTenantCode" for tenant code owning the resource, - * can be different from the login user in the case of logging in as admin users. - * @param content content - * @return update result cod - */ @Override - @Transactional - public Result updateResourceContent(User loginUser, String fullName, String resTenantCode, String content) { - Result result = new Result<>(); - User user = userMapper.selectById(loginUser.getId()); - if (user == null) { - log.error("user {} not exists", loginUser.getId()); - putMsg(result, Status.USER_NOT_EXIST, loginUser.getId()); - return result; - } + public void downloadResource(HttpServletResponse response, DownloadFileRequest downloadFileRequest) { + DownloadFileDto downloadFileDto = DownloadFileDto.builder() + .loginUser(downloadFileRequest.getLoginUser()) + .fileAbsolutePath(downloadFileRequest.getFileAbsolutePath()) + .build(); + downloadFileDtoValidator.validate(downloadFileDto); - String tenantCode = getTenantCode(user); - checkFullName(tenantCode, fullName); + String fileName = new File(downloadFileDto.getFileAbsolutePath()).getName(); + String localTmpFileAbsolutePath = FileUtils.getDownloadFilename(fileName); - if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode)) { - log.error("current user does not have permission"); - putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); - return result; - } - - StorageEntity resource; try { - resource = storageOperate.getFileStatus(fullName, "", resTenantCode, ResourceType.FILE); + storageOperator.download(downloadFileRequest.getFileAbsolutePath(), localTmpFileAbsolutePath, true); + int length = (int) new File(localTmpFileAbsolutePath).length(); + ApiServerMetrics.recordApiResourceDownloadSize(length); + + response.reset(); + response.setContentType("application/octet-stream"); + response.setCharacterEncoding("utf-8"); + response.setContentLength(length); + response.setHeader("Content-Disposition", "attachment;filename=" + fileName); + Files.copy(Paths.get(localTmpFileAbsolutePath), response.getOutputStream()); } catch (Exception e) { - log.error("error occurred when fetching resource information , resource full name {}", fullName); - putMsg(result, Status.RESOURCE_NOT_EXIST); - return result; - } - - if (resource == null) { - log.error("Resource does not exist, resource full name:{}.", fullName); - putMsg(result, Status.RESOURCE_NOT_EXIST); - return result; - } - - // check can edit by file suffix - String nameSuffix = Files.getFileExtension(resource.getAlias()); - String resourceViewSuffixes = FileUtils.getResourceViewSuffixes(); - if (StringUtils.isNotEmpty(resourceViewSuffixes)) { - List strList = Arrays.asList(resourceViewSuffixes.split(",")); - if (!strList.contains(nameSuffix)) { - log.warn("Resource suffix does not support view, resource full name:{}, suffix:{}.", fullName, - nameSuffix); - putMsg(result, Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW); - return result; - } - } - - result = uploadContentToStorage(resource.getFullName(), resTenantCode, content); - - if (!result.getCode().equals(Status.SUCCESS.getCode())) { - throw new ServiceException(result.getMsg()); - } else - log.info("Update resource content complete, resource full name:{}.", fullName); - return result; - } - - /** - * @param fullName resource full name - * @param tenantCode tenant code - * @param content content - * @return result - */ - private Result uploadContentToStorage(String fullName, String tenantCode, String content) { - Result result = new Result<>(); - String localFilename = ""; - try { - localFilename = FileUtils.getUploadFilename(tenantCode, UUID.randomUUID().toString()); - - if (!FileUtils.writeContent2File(content, localFilename)) { - // write file fail - log.error("Write file error, fileName:{}, content:{}.", localFilename, RegexUtils.escapeNRT(content)); - putMsg(result, Status.RESOURCE_NOT_EXIST); - return result; - } - - // get resource file path - String resourcePath = storageOperate.getResDir(tenantCode); - log.info("resource path is {}, resource dir is {}", fullName, resourcePath); - - if (!storageOperate.exists(resourcePath)) { - // create if tenant dir not exists - storageOperate.createTenantDirIfNotExists(tenantCode); - log.info("Create tenant dir because path {} does not exist, tenantCode:{}.", resourcePath, tenantCode); - } - if (storageOperate.exists(fullName)) { - storageOperate.delete(fullName, false); - } - - storageOperate.upload(tenantCode, localFilename, fullName, true, true); - } catch (Exception e) { - log.error("Upload content to storage error, tenantCode:{}, destFileName:{}.", tenantCode, localFilename, e); - result.setCode(Status.HDFS_OPERATION_ERROR.getCode()); - result.setMsg(String.format("copy %s to hdfs %s fail", localFilename, fullName)); - return result; + throw new ServiceException( + "Download the resource file: " + downloadFileRequest.getFileAbsolutePath() + " failed", e); } finally { - FileUtils.deleteFile(localFilename); - } - log.info("Upload content to storage complete, tenantCode:{}, destFileName:{}.", tenantCode, localFilename); - putMsg(result, Status.SUCCESS); - return result; - } - - /** - * download file - * - * @return resource content - */ - @Override - public org.springframework.core.io.Resource downloadResource(User loginUser, String fullName) { - if (fullName.endsWith("/")) { - log.error("resource id {} is directory,can't download it", fullName); - throw new ServiceException("can't download directory"); - } - - int userId = loginUser.getId(); - User user = userMapper.selectById(userId); - if (user == null) { - log.error("User does not exits, userId:{}.", userId); - throw new ServiceException(String.format("Resource owner id %d does not exist", userId)); - } - - String tenantCode = getTenantCode(user); - checkFullName(tenantCode, fullName); - - String[] aliasArr = fullName.split("/"); - String alias = aliasArr[aliasArr.length - 1]; - String localFileName = FileUtils.getDownloadFilename(alias); - log.info("Resource path is {}, download local filename is {}", alias, localFileName); - - try { - storageOperate.download(fullName, localFileName, true); - ApiServerMetrics.recordApiResourceDownloadSize(java.nio.file.Files.size(Paths.get(localFileName))); - return org.apache.dolphinscheduler.api.utils.FileUtils.file2Resource(localFileName); - } catch (IOException e) { - log.error("Download resource error, the path is {}, and local filename is {}, the error message is {}", - fullName, localFileName, e.getMessage()); - throw new ServiceException("Download the resource file failed ,it may be related to your storage"); + FileUtils.deleteFile(localTmpFileAbsolutePath); } } @Override - public StorageEntity queryFileStatus(String userName, String fileName) throws Exception { - // TODO: It is used in PythonGateway, should be revised - User user = userMapper.queryByUserNameAccurately(userName); - - String defaultPath = storageOperate.getResDir(user.getTenantCode()); - return storageOperate.getFileStatus(defaultPath + fileName, defaultPath, user.getTenantCode(), - ResourceType.FILE); + public StorageEntity queryFileStatus(String userName, String fileAbsolutePath) { + return storageOperator.getStorageEntity(fileAbsolutePath); } @Override - public DeleteDataTransferResponse deleteDataTransferData(User loginUser, Integer days) { - DeleteDataTransferResponse result = new DeleteDataTransferResponse(); + public String queryResourceBaseDir(User loginUser, ResourceType type) { User user = userMapper.selectById(loginUser.getId()); if (user == null) { - log.error("user {} not exists", loginUser.getId()); - putMsg(result, Status.USER_NOT_EXIST, loginUser.getId()); - return result; - } - - String tenantCode = getTenantCode(user); - - String baseFolder = storageOperate.getResourceFullName(tenantCode, "DATA_TRANSFER"); - - LocalDateTime now = LocalDateTime.now(); - now = now.minus(days, ChronoUnit.DAYS); - String deleteDate = now.toLocalDate().toString().replace("-", ""); - List storageEntities; - try { - storageEntities = new ArrayList<>( - storageOperate.listFilesStatus(baseFolder, baseFolder, tenantCode, ResourceType.FILE)); - } catch (Exception e) { - log.error("delete data transfer data error", e); - putMsg(result, Status.DELETE_RESOURCE_ERROR); - return result; - } - - List successList = new ArrayList<>(); - List failList = new ArrayList<>(); - - for (StorageEntity storageEntity : storageEntities) { - File path = new File(storageEntity.getFullName()); - String date = path.getName(); - if (date.compareTo(deleteDate) <= 0) { - try { - storageOperate.delete(storageEntity.getFullName(), true); - successList.add(storageEntity.getFullName()); - } catch (Exception ex) { - log.error("delete data transfer data {} error, please delete it manually", date, ex); - failList.add(storageEntity.getFullName()); - } - } + throw new ServiceException(Status.USER_NOT_EXIST); } - result.setSuccessList(successList); - result.setFailedList(failList); - putMsg(result, Status.SUCCESS); - return result; + Tenant tenant = tenantDao.queryOptionalById(user.getTenantId()) + .orElseThrow(() -> new ServiceException(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST)); + return storageOperator.getStorageBaseDirectory(tenant.getTenantCode(), type); } - /** - * get resource base dir - * - * @param loginUser login user - * @param type resource type - * @return - */ - @Override - public Result queryResourceBaseDir(User loginUser, ResourceType type) { - Result result = new Result<>(); - if (storageOperate == null) { - putMsg(result, Status.SUCCESS); - result.setData(EMPTY_STRING); - return result; - } - User user = userMapper.selectById(loginUser.getId()); - if (user == null) { - log.error("user {} not exists", loginUser.getId()); - putMsg(result, Status.USER_NOT_EXIST, loginUser.getId()); - return result; - } - - String tenantCode = getTenantCode(user); - - String baseDir = isAdmin(loginUser) ? storageOperate.getDir(ResourceType.ALL, tenantCode) - : storageOperate.getDir(type, tenantCode); - - putMsg(result, Status.SUCCESS); - result.setData(baseDir); - - return result; + // Copy the file to the local file system and return the local file absolute path + @SneakyThrows + private String copyFileToLocal(MultipartFile multipartFile) { + String localTmpFileAbsolutePath = FileUtils.getUploadFileLocalTmpAbsolutePath(); + FileUtils.copyInputStreamToFile(multipartFile.getInputStream(), localTmpFileAbsolutePath); + return localTmpFileAbsolutePath; } - /** - * check permission by comparing login user's tenantCode with tenantCode in the request - * - * @param isAdmin is the login user admin - * @param userTenantCode loginUser's tenantCode - * @param resTenantCode tenantCode in the request field "resTenantCode" for tenant code owning the resource, - * can be different from the login user in the case of logging in as admin users. - * @return isValid - */ - private boolean isUserTenantValid(boolean isAdmin, String userTenantCode, - String resTenantCode) throws ServiceException { - if (isAdmin) { - return true; - } - if (StringUtils.isEmpty(resTenantCode)) { - // TODO: resource tenant code will be empty when query resources list, need to be optimized - return true; - } - return resTenantCode.equals(userTenantCode); + // Copy the file to the local file system and return the local file absolute path + private String copyFileToLocal(String fileContent) { + String localTmpFileAbsolutePath = FileUtils.getUploadFileLocalTmpAbsolutePath(); + FileUtils.writeContent2File(fileContent, localTmpFileAbsolutePath); + return localTmpFileAbsolutePath; } - private String getTenantCode(User user) { - Tenant tenant = tenantMapper.queryById(user.getTenantId()); - if (tenant == null) { - throw new ServiceException(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST); - } - return tenant.getTenantCode(); - } - - private void checkFullName(String userTenantCode, String fullName) { - if (StringUtils.isEmpty(fullName)) { - return; - } - if (FOLDER_SEPARATOR.equalsIgnoreCase(fullName)) { - return; - } - // Avoid returning to the parent directory - if (fullName.contains("../")) { - throw new ServiceException(Status.ILLEGAL_RESOURCE_PATH, fullName); - } - String baseDir = storageOperate.getDir(ResourceType.ALL, userTenantCode); - if (!StringUtils.startsWith(fullName, baseDir)) { - throw new ServiceException(Status.ILLEGAL_RESOURCE_PATH, fullName); - } - } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java index e77baed1d3..7bec4d6780 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/TenantServiceImpl.java @@ -39,7 +39,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -84,7 +84,7 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService private QueueService queueService; @Autowired(required = false) - private StorageOperate storageOperate; + private StorageOperator storageOperator; /** * Check the tenant new object valid or not @@ -136,14 +136,13 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService * @param queueId queue id * @param desc description * @return create result code - * @throws Exception exception */ @Override @Transactional(rollbackFor = Exception.class) public Tenant createTenant(User loginUser, String tenantCode, int queueId, - String desc) throws Exception { + String desc) { if (!canOperatorPermissions(loginUser, null, AuthorizationType.TENANT, TENANT_CREATE)) { throw new ServiceException(Status.USER_NO_OPERATION_PERM); } @@ -154,7 +153,6 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService createTenantValid(tenant); tenantMapper.insert(tenant); - storageOperate.createTenantDirIfNotExists(tenantCode); return tenant; } @@ -209,11 +207,6 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService updateTenantValid(existsTenant, updateTenant); updateTenant.setCreateTime(existsTenant.getCreateTime()); - // updateProcessInstance tenant - // if the tenant code is modified, the original resource needs to be copied to the new tenant. - if (!Objects.equals(existsTenant.getTenantCode(), updateTenant.getTenantCode())) { - storageOperate.createTenantDirIfNotExists(tenantCode); - } int update = tenantMapper.updateById(updateTenant); if (update <= 0) { throw new ServiceException(Status.UPDATE_TENANT_ERROR); @@ -262,7 +255,6 @@ public class TenantServiceImpl extends BaseServiceImpl implements TenantService } processInstanceMapper.updateProcessInstanceByTenantCode(tenant.getTenantCode(), Constants.DEFAULT); - storageOperate.deleteTenant(tenant.getTenantCode()); } private List getProcessInstancesByTenant(Tenant tenant) { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java index 1bf7d23a6b..7cf16b3567 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java @@ -28,11 +28,10 @@ import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper; import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.commons.lang3.StringUtils; -import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.Date; @@ -62,7 +61,7 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic private UDFUserMapper udfUserMapper; @Autowired(required = false) - private StorageOperate storageOperate; + private StorageOperator storageOperator; /** * create udf function @@ -107,12 +106,7 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic return result; } - Boolean existResource = false; - try { - existResource = storageOperate.exists(fullName); - } catch (IOException e) { - log.error("Check resource error: {}", fullName, e); - } + boolean existResource = storageOperator.exists(fullName); if (!existResource) { log.error("resource full name {} is not exist", fullName); @@ -241,7 +235,7 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic Boolean doesResExist = false; try { - doesResExist = storageOperate.exists(fullName); + doesResExist = storageOperator.exists(fullName); } catch (Exception e) { log.error("udf resource :{} checking error", fullName, e); result.setCode(Status.RESOURCE_NOT_EXIST.getCode()); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java index 7b9746921c..0e91dc582d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.api.service.impl; import static org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant.USER_MANAGER; -import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.service.MetricsCleanUpService; @@ -50,7 +49,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProjectUserMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -110,7 +109,7 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService { private ProjectMapper projectMapper; @Autowired(required = false) - private StorageOperate storageOperate; + private StorageOperator storageOperator; @Autowired private K8sNamespaceUserMapper k8sNamespaceUserMapper; @@ -171,9 +170,6 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService { User user = createUser(userName, userPassword, email, tenantId, phone, queue, state); - Tenant tenant = tenantMapper.queryById(tenantId); - storageOperate.createTenantDirIfNotExists(tenant.getTenantCode()); - log.info("User is created and id is {}.", user.getId()); result.put(Constants.DATA_LIST, user); putMsg(result, Status.SUCCESS); @@ -1128,54 +1124,6 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService { return msg; } - /** - * copy resource files - * xxx unchecked - * - * @param resourceComponent resource component - * @param srcBasePath src base path - * @param dstBasePath dst base path - * @throws IOException io exception - */ - private void copyResourceFiles(String oldTenantCode, String newTenantCode, ResourceComponent resourceComponent, - String srcBasePath, String dstBasePath) { - List components = resourceComponent.getChildren(); - - try { - if (CollectionUtils.isNotEmpty(components)) { - for (ResourceComponent component : components) { - // verify whether exist - if (!storageOperate.exists( - String.format(Constants.FORMAT_S_S, srcBasePath, component.getFullName()))) { - log.error("Resource file: {} does not exist, copy error.", component.getFullName()); - throw new ServiceException(Status.RESOURCE_NOT_EXIST); - } - - if (!component.isDirctory()) { - // copy it to dst - storageOperate.copy(String.format(Constants.FORMAT_S_S, srcBasePath, component.getFullName()), - String.format(Constants.FORMAT_S_S, dstBasePath, component.getFullName()), false, true); - continue; - } - - if (CollectionUtils.isEmpty(component.getChildren())) { - // if not exist,need create it - if (!storageOperate - .exists(String.format(Constants.FORMAT_S_S, dstBasePath, component.getFullName()))) { - storageOperate.mkdir(newTenantCode, - String.format(Constants.FORMAT_S_S, dstBasePath, component.getFullName())); - } - } else { - copyResourceFiles(oldTenantCode, newTenantCode, component, srcBasePath, dstBasePath); - } - } - - } - } catch (IOException e) { - log.error("copy the resources failed,the error message is {}", e.getMessage()); - } - } - /** * registry user, default state is 0, default tenant_id is 1, no phone, no queue * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/PageInfo.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/PageInfo.java index 2cfdd8f840..85e47c96b9 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/PageInfo.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/PageInfo.java @@ -20,12 +20,16 @@ package org.apache.dolphinscheduler.api.utils; import java.util.Collections; import java.util.List; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; import lombok.Setter; import com.baomidou.mybatisplus.core.metadata.IPage; @Data +@Builder +@AllArgsConstructor public class PageInfo { /** diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/ITransformer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/ITransformer.java new file mode 100644 index 0000000000..8aedad3a90 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/ITransformer.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.validator; + +public interface ITransformer { + + R transform(T t); + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/IValidator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/IValidator.java new file mode 100644 index 0000000000..7570fa67d5 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/IValidator.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.validator; + +public interface IValidator { + + void validate(T t); + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/AbstractResourceTransformer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/AbstractResourceTransformer.java new file mode 100644 index 0000000000..4f721188dc --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/AbstractResourceTransformer.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.validator.resource; + +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.exceptions.ServiceException; +import org.apache.dolphinscheduler.api.validator.ITransformer; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.repository.TenantDao; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; +import org.apache.dolphinscheduler.spi.enums.ResourceType; + +import org.apache.commons.lang3.StringUtils; + +import lombok.AllArgsConstructor; + +@AllArgsConstructor +public abstract class AbstractResourceTransformer implements ITransformer { + + protected TenantDao tenantDao; + + protected StorageOperator storageOperator; + + protected String getParentDirectoryAbsolutePath(User loginUser, String parentAbsoluteDirectory, ResourceType type) { + String tenantCode = tenantDao.queryOptionalById(loginUser.getTenantId()) + .orElseThrow(() -> new ServiceException(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST)) + .getTenantCode(); + String userResRootPath = storageOperator.getStorageBaseDirectory(tenantCode, type); + // If the parent directory is / then will transform to userResRootPath + // This only happens when the front-end go into the resource page first + // todo: we need to change the front-end logic to avoid this + if (parentAbsoluteDirectory.equals("/")) { + return userResRootPath; + } + + if (!StringUtils.startsWith(parentAbsoluteDirectory, userResRootPath)) { + throw new ServiceException(Status.ILLEGAL_RESOURCE_PATH, parentAbsoluteDirectory); + } + return parentAbsoluteDirectory; + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/AbstractResourceValidator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/AbstractResourceValidator.java new file mode 100644 index 0000000000..35656b4d82 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/AbstractResourceValidator.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.validator.resource; + +import org.apache.dolphinscheduler.api.dto.resources.AbstractResourceDto; +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.exceptions.ServiceException; +import org.apache.dolphinscheduler.api.validator.IValidator; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.FileUtils; +import org.apache.dolphinscheduler.dao.entity.Tenant; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.repository.TenantDao; +import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; + +import org.apache.commons.lang3.StringUtils; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import org.springframework.web.multipart.MultipartFile; + +import com.google.common.io.Files; + +public abstract class AbstractResourceValidator implements IValidator { + + private static final Set FILE_SUFFIXES_WHICH_CAN_FETCH_CONTENT = new HashSet<>(Arrays.asList( + StringUtils.defaultIfBlank(FileUtils.getResourceViewSuffixes(), "").split(","))); + + protected final StorageOperator storageOperator; + + private final TenantDao tenantDao; + + public AbstractResourceValidator(StorageOperator storageOperator, TenantDao tenantDao) { + this.storageOperator = storageOperator; + this.tenantDao = tenantDao; + } + + public void exceptionResourceAbsolutePathInvalidated(String resourceAbsolutePath) { + if (StringUtils.isBlank(resourceAbsolutePath)) { + throw new ServiceException("The resource path is null"); + } + if (!resourceAbsolutePath.startsWith(storageOperator.getStorageBaseDirectory())) { + throw new ServiceException("Invalidated resource path: " + resourceAbsolutePath); + } + if (resourceAbsolutePath.contains("..")) { + throw new ServiceException("Invalidated resource path: " + resourceAbsolutePath); + } + } + + public void exceptionFileInvalidated(MultipartFile file) { + if (file == null) { + throw new ServiceException("The file is null"); + } + } + + public void exceptionFileContentInvalidated(String fileContent) { + if (StringUtils.isEmpty(fileContent)) { + throw new ServiceException("The file content is null"); + } + } + + public void exceptionFileContentCannotFetch(String fileAbsolutePath) { + String fileExtension = Files.getFileExtension(fileAbsolutePath); + if (!FILE_SUFFIXES_WHICH_CAN_FETCH_CONTENT.contains(fileExtension)) { + throw new ServiceException("The file type: " + fileExtension + " cannot be fetched"); + } + } + + public void exceptionResourceNotExists(String resourceAbsolutePath) { + if (!storageOperator.exists(resourceAbsolutePath)) { + throw new ServiceException("Thr resource is not exists: " + resourceAbsolutePath); + } + } + + public void exceptionResourceExists(String resourceAbsolutePath) { + if (storageOperator.exists(resourceAbsolutePath)) { + throw new ServiceException("The resource is already exist: " + resourceAbsolutePath); + } + } + + public void exceptionResourceIsNotDirectory(String resourceAbsolutePath) { + if (StringUtils.isNotEmpty(Files.getFileExtension(resourceAbsolutePath))) { + throw new ServiceException("The path is not a directory: " + resourceAbsolutePath); + } + } + + public void exceptionResourceIsNotFile(String fileAbsolutePath) { + if (StringUtils.isEmpty(Files.getFileExtension(fileAbsolutePath))) { + throw new ServiceException("The path is not a file: " + fileAbsolutePath); + } + } + + public void exceptionUserNoResourcePermission(User user, AbstractResourceDto resourceDto) { + exceptionUserNoResourcePermission(user, resourceDto.getResourceAbsolutePath()); + } + + public void exceptionUserNoResourcePermission(User user, String resourceAbsolutePath) { + if (user.getUserType() == UserType.ADMIN_USER) { + return; + } + // check if the user have resource tenant permission + // Parse the resource path to get the tenant code + ResourceMetadata resourceMetaData = storageOperator.getResourceMetaData(resourceAbsolutePath); + + if (!resourceAbsolutePath.startsWith(resourceMetaData.getResourceBaseDirectory())) { + throw new ServiceException("Invalidated resource path: " + resourceAbsolutePath); + } + + // todo: inject the tenant when login + Tenant tenant = tenantDao.queryOptionalById(user.getTenantId()) + .orElseThrow(() -> new ServiceException(Status.TENANT_NOT_EXIST, user.getTenantId())); + String userTenant = tenant.getTenantCode(); + if (!userTenant.equals(resourceMetaData.getTenant())) { + throw new ServiceException( + "The user's tenant is " + userTenant + " have no permission to access the resource: " + + resourceAbsolutePath); + } + } + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateDirectoryDtoValidator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateDirectoryDtoValidator.java new file mode 100644 index 0000000000..248f7fbff3 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateDirectoryDtoValidator.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.validator.resource; + +import org.apache.dolphinscheduler.api.dto.resources.CreateDirectoryDto; +import org.apache.dolphinscheduler.api.exceptions.ServiceException; +import org.apache.dolphinscheduler.dao.repository.TenantDao; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; + +import org.apache.commons.lang3.StringUtils; + +import org.springframework.stereotype.Component; + +import com.google.common.io.Files; + +@Component +public class CreateDirectoryDtoValidator extends AbstractResourceValidator { + + public CreateDirectoryDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) { + super(storageOperator, tenantDao); + } + + @Override + public void validate(CreateDirectoryDto createDirectoryDto) { + String directoryAbsolutePath = createDirectoryDto.getDirectoryAbsolutePath(); + + exceptionResourceAbsolutePathInvalidated(directoryAbsolutePath); + exceptionResourceExists(directoryAbsolutePath); + exceptionUserNoResourcePermission(createDirectoryDto.getLoginUser(), directoryAbsolutePath); + exceptionResourceIsNotDirectory(directoryAbsolutePath); + if (StringUtils.isNotEmpty(Files.getFileExtension(directoryAbsolutePath))) { + throw new ServiceException("The path is not a directory: " + directoryAbsolutePath); + } + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateDirectoryRequestTransformer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateDirectoryRequestTransformer.java new file mode 100644 index 0000000000..75985477f3 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateDirectoryRequestTransformer.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.validator.resource; + +import static com.google.common.base.Preconditions.checkNotNull; + +import org.apache.dolphinscheduler.api.dto.resources.CreateDirectoryDto; +import org.apache.dolphinscheduler.api.dto.resources.CreateDirectoryRequest; +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.exceptions.ServiceException; +import org.apache.dolphinscheduler.api.validator.ITransformer; +import org.apache.dolphinscheduler.common.utils.FileUtils; +import org.apache.dolphinscheduler.dao.repository.TenantDao; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; + +import org.apache.commons.lang3.StringUtils; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class CreateDirectoryRequestTransformer implements ITransformer { + + @Autowired + private TenantDao tenantDao; + + @Autowired + private StorageOperator storageOperator; + + @Override + public CreateDirectoryDto transform(CreateDirectoryRequest createDirectoryRequest) { + validateCreateDirectoryRequest(createDirectoryRequest); + return doTransform(createDirectoryRequest); + } + + private CreateDirectoryDto doTransform(CreateDirectoryRequest createDirectoryRequest) { + String directoryAbsolutePath = getDirectoryAbsolutePath(createDirectoryRequest); + return CreateDirectoryDto.builder() + .loginUser(createDirectoryRequest.getLoginUser()) + .directoryAbsolutePath(directoryAbsolutePath) + .build(); + } + + private void validateCreateDirectoryRequest(CreateDirectoryRequest createDirectoryRequest) { + checkNotNull(createDirectoryRequest.getLoginUser(), "loginUser is null"); + checkNotNull(createDirectoryRequest.getType(), "resource type is null"); + checkNotNull(createDirectoryRequest.getDirectoryName(), "directory name is null"); + checkNotNull(createDirectoryRequest.getParentAbsoluteDirectory(), "parent directory is null"); + + } + + private String getDirectoryAbsolutePath(CreateDirectoryRequest createDirectoryRequest) { + String tenantCode = tenantDao.queryOptionalById(createDirectoryRequest.getLoginUser().getTenantId()) + .orElseThrow(() -> new ServiceException(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST)) + .getTenantCode(); + String userResRootPath = storageOperator.getStorageBaseDirectory(tenantCode, createDirectoryRequest.getType()); + String parentDirectoryName = createDirectoryRequest.getParentAbsoluteDirectory(); + String directoryName = createDirectoryRequest.getDirectoryName(); + + // If the parent directory is / then will transform to userResRootPath + // This only happens when the front-end go into the resource page first + // todo: we need to change the front-end logic to avoid this + if (parentDirectoryName.equals("/")) { + return FileUtils.concatFilePath(userResRootPath, directoryName); + } + + if (!StringUtils.startsWith(parentDirectoryName, userResRootPath)) { + throw new ServiceException(Status.ILLEGAL_RESOURCE_PATH, parentDirectoryName); + } + return FileUtils.concatFilePath(parentDirectoryName, directoryName); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateFileDtoValidator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateFileDtoValidator.java new file mode 100644 index 0000000000..d2c91387a6 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateFileDtoValidator.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.validator.resource; + +import org.apache.dolphinscheduler.api.dto.resources.CreateFileDto; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.repository.TenantDao; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; + +import org.springframework.stereotype.Component; +import org.springframework.web.multipart.MultipartFile; + +@Component +public class CreateFileDtoValidator extends AbstractResourceValidator { + + public CreateFileDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) { + super(storageOperator, tenantDao); + } + + @Override + public void validate(CreateFileDto createFileDto) { + String fileAbsolutePath = createFileDto.getFileAbsolutePath(); + User loginUser = createFileDto.getLoginUser(); + MultipartFile file = createFileDto.getFile(); + + exceptionResourceAbsolutePathInvalidated(fileAbsolutePath); + exceptionResourceExists(fileAbsolutePath); + exceptionFileInvalidated(file); + exceptionUserNoResourcePermission(loginUser, fileAbsolutePath); + exceptionResourceIsNotFile(fileAbsolutePath); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateFileFromContentDtoValidator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateFileFromContentDtoValidator.java new file mode 100644 index 0000000000..ae61f47041 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/CreateFileFromContentDtoValidator.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.validator.resource; + +import org.apache.dolphinscheduler.api.dto.resources.CreateFileFromContentDto; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.repository.TenantDao; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; + +import org.springframework.stereotype.Component; + +@Component +public class CreateFileFromContentDtoValidator extends AbstractResourceValidator { + + public CreateFileFromContentDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) { + super(storageOperator, tenantDao); + } + + @Override + public void validate(CreateFileFromContentDto createFileFromContentDto) { + String fileAbsolutePath = createFileFromContentDto.getFileAbsolutePath(); + User loginUser = createFileFromContentDto.getLoginUser(); + String fileContent = createFileFromContentDto.getFileContent(); + + exceptionResourceAbsolutePathInvalidated(fileAbsolutePath); + exceptionResourceIsNotFile(fileAbsolutePath); + exceptionResourceExists(fileAbsolutePath); + exceptionUserNoResourcePermission(loginUser, fileAbsolutePath); + exceptionFileContentInvalidated(fileContent); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/DeleteResourceDtoValidator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/DeleteResourceDtoValidator.java new file mode 100644 index 0000000000..e337d9e07c --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/DeleteResourceDtoValidator.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.validator.resource; + +import org.apache.dolphinscheduler.api.dto.resources.DeleteResourceDto; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.repository.TenantDao; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; + +import org.springframework.stereotype.Component; + +@Component +public class DeleteResourceDtoValidator extends AbstractResourceValidator { + + public DeleteResourceDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) { + super(storageOperator, tenantDao); + } + + @Override + public void validate(DeleteResourceDto deleteResourceDto) { + String resourceAbsolutePath = deleteResourceDto.getResourceAbsolutePath(); + User loginUser = deleteResourceDto.getLoginUser(); + + exceptionResourceAbsolutePathInvalidated(resourceAbsolutePath); + exceptionResourceNotExists(resourceAbsolutePath); + exceptionUserNoResourcePermission(loginUser, resourceAbsolutePath); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/DownloadFileDtoValidator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/DownloadFileDtoValidator.java new file mode 100644 index 0000000000..ac15d279ac --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/DownloadFileDtoValidator.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.validator.resource; + +import org.apache.dolphinscheduler.api.dto.resources.DownloadFileDto; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.repository.TenantDao; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; + +import org.springframework.stereotype.Component; + +@Component +public class DownloadFileDtoValidator extends AbstractResourceValidator { + + public DownloadFileDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) { + super(storageOperator, tenantDao); + } + + @Override + public void validate(DownloadFileDto downloadFileDto) { + String fileAbsolutePath = downloadFileDto.getFileAbsolutePath(); + User loginUser = downloadFileDto.getLoginUser(); + + exceptionResourceNotExists(fileAbsolutePath); + exceptionResourceAbsolutePathInvalidated(fileAbsolutePath); + exceptionResourceIsNotFile(fileAbsolutePath); + exceptionUserNoResourcePermission(loginUser, fileAbsolutePath); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/FetchFileContentDtoValidator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/FetchFileContentDtoValidator.java new file mode 100644 index 0000000000..b69ec68a7c --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/FetchFileContentDtoValidator.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.validator.resource; + +import org.apache.dolphinscheduler.api.dto.resources.FetchFileContentDto; +import org.apache.dolphinscheduler.api.exceptions.ServiceException; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.repository.TenantDao; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; + +import org.springframework.stereotype.Component; + +@Component +public class FetchFileContentDtoValidator extends AbstractResourceValidator { + + public FetchFileContentDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) { + super(storageOperator, tenantDao); + } + + @Override + public void validate(FetchFileContentDto fetchFileContentDto) { + if (fetchFileContentDto.getSkipLineNum() < 0) { + throw new ServiceException("skipLineNum must be greater than or equal to 0"); + } + String resourceFileAbsolutePath = fetchFileContentDto.getResourceFileAbsolutePath(); + User loginUser = fetchFileContentDto.getLoginUser(); + + exceptionResourceAbsolutePathInvalidated(resourceFileAbsolutePath); + exceptionResourceIsNotFile(resourceFileAbsolutePath); + exceptionUserNoResourcePermission(loginUser, resourceFileAbsolutePath); + exceptionFileContentCannotFetch(resourceFileAbsolutePath); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/FileFromContentRequestTransformer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/FileFromContentRequestTransformer.java new file mode 100644 index 0000000000..7f29646e36 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/FileFromContentRequestTransformer.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.validator.resource; + +import static com.google.common.base.Preconditions.checkNotNull; + +import org.apache.dolphinscheduler.api.dto.resources.CreateFileFromContentDto; +import org.apache.dolphinscheduler.api.dto.resources.CreateFileFromContentRequest; +import org.apache.dolphinscheduler.common.utils.FileUtils; +import org.apache.dolphinscheduler.dao.repository.TenantDao; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; + +import org.springframework.stereotype.Component; + +@Component +public class FileFromContentRequestTransformer + extends + AbstractResourceTransformer { + + public FileFromContentRequestTransformer(TenantDao tenantDao, StorageOperator storageOperator) { + super(tenantDao, storageOperator); + } + + @Override + public CreateFileFromContentDto transform(CreateFileFromContentRequest createFileFromContentRequest) { + validateCreateFileRequest(createFileFromContentRequest); + return doTransform(createFileFromContentRequest); + } + + private void validateCreateFileRequest(CreateFileFromContentRequest createFileFromContentRequest) { + checkNotNull(createFileFromContentRequest.getLoginUser(), "loginUser is null"); + checkNotNull(createFileFromContentRequest.getType(), "resource type is null"); + checkNotNull(createFileFromContentRequest.getFileName(), "file name is null"); + checkNotNull(createFileFromContentRequest.getParentAbsoluteDirectory(), "parent directory is null"); + checkNotNull(createFileFromContentRequest.getFileContent(), "file content is null"); + } + + private CreateFileFromContentDto doTransform(CreateFileFromContentRequest createFileFromContentRequest) { + String fileAbsolutePath = getFileAbsolutePath(createFileFromContentRequest); + return CreateFileFromContentDto.builder() + .loginUser(createFileFromContentRequest.getLoginUser()) + .fileAbsolutePath(fileAbsolutePath) + .fileContent(createFileFromContentRequest.getFileContent()) + .build(); + + } + + private String getFileAbsolutePath(CreateFileFromContentRequest createFileFromContentRequest) { + String parentDirectoryAbsolutePath = getParentDirectoryAbsolutePath( + createFileFromContentRequest.getLoginUser(), + createFileFromContentRequest.getParentAbsoluteDirectory(), + createFileFromContentRequest.getType()); + return FileUtils.concatFilePath(parentDirectoryAbsolutePath, createFileFromContentRequest.getFileName()); + } + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/FileRequestTransformer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/FileRequestTransformer.java new file mode 100644 index 0000000000..c2f007dbf7 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/FileRequestTransformer.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.validator.resource; + +import static com.google.common.base.Preconditions.checkNotNull; + +import org.apache.dolphinscheduler.api.dto.resources.CreateFileDto; +import org.apache.dolphinscheduler.api.dto.resources.CreateFileRequest; +import org.apache.dolphinscheduler.common.utils.FileUtils; +import org.apache.dolphinscheduler.dao.repository.TenantDao; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; + +import org.springframework.stereotype.Component; + +@Component +public class FileRequestTransformer extends AbstractResourceTransformer { + + public FileRequestTransformer(TenantDao tenantDao, StorageOperator storageOperator) { + super(tenantDao, storageOperator); + } + + @Override + public CreateFileDto transform(CreateFileRequest createFileRequest) { + validateCreateFileRequest(createFileRequest); + return doTransform(createFileRequest); + } + + private void validateCreateFileRequest(CreateFileRequest createFileRequest) { + checkNotNull(createFileRequest.getLoginUser(), "loginUser is null"); + checkNotNull(createFileRequest.getType(), "resource type is null"); + checkNotNull(createFileRequest.getFileName(), "file name is null"); + checkNotNull(createFileRequest.getParentAbsoluteDirectory(), "parent directory is null"); + checkNotNull(createFileRequest.getFile(), "file is null"); + } + + private CreateFileDto doTransform(CreateFileRequest createFileRequest) { + String fileAbsolutePath = getFileAbsolutePath(createFileRequest); + return CreateFileDto.builder() + .loginUser(createFileRequest.getLoginUser()) + .file(createFileRequest.getFile()) + .fileAbsolutePath(fileAbsolutePath) + .build(); + + } + + private String getFileAbsolutePath(CreateFileRequest createFileRequest) { + String parentDirectoryAbsolutePath = getParentDirectoryAbsolutePath( + createFileRequest.getLoginUser(), + createFileRequest.getParentAbsoluteDirectory(), + createFileRequest.getType()); + return FileUtils.concatFilePath(parentDirectoryAbsolutePath, createFileRequest.getFileName()); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/PagingResourceItemRequestTransformer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/PagingResourceItemRequestTransformer.java new file mode 100644 index 0000000000..9be51e413e --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/PagingResourceItemRequestTransformer.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.validator.resource; + +import static com.google.common.base.Preconditions.checkNotNull; + +import org.apache.dolphinscheduler.api.dto.resources.PagingResourceItemRequest; +import org.apache.dolphinscheduler.api.dto.resources.QueryResourceDto; +import org.apache.dolphinscheduler.api.validator.ITransformer; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.dao.entity.Tenant; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.repository.TenantDao; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; +import org.apache.dolphinscheduler.spi.enums.ResourceType; + +import org.apache.commons.lang3.StringUtils; + +import java.util.List; +import java.util.stream.Collectors; + +import lombok.AllArgsConstructor; + +import org.springframework.stereotype.Component; + +import com.google.common.collect.Lists; + +@Component +@AllArgsConstructor +public class PagingResourceItemRequestTransformer implements ITransformer { + + private final StorageOperator storageOperator; + + private final TenantDao tenantDao; + + @Override + public QueryResourceDto transform(PagingResourceItemRequest pagingResourceItemRequest) { + validatePagingResourceItemRequest(pagingResourceItemRequest); + + if (StringUtils.isNotEmpty(pagingResourceItemRequest.getResourceAbsolutePath())) { + // query from the given path + return QueryResourceDto.builder() + .resourceAbsolutePaths(Lists.newArrayList(pagingResourceItemRequest.getResourceAbsolutePath())) + .build(); + } + + ResourceType resourceType = pagingResourceItemRequest.getResourceType(); + User loginUser = pagingResourceItemRequest.getLoginUser(); + if (loginUser.getUserType() == UserType.ADMIN_USER) { + // If the current user is admin + // then will query all tenant resources + List resourceAbsolutePaths = tenantDao.queryAll() + .stream() + .map(tenant -> storageOperator.getStorageBaseDirectory(tenant.getTenantCode(), resourceType)) + .collect(Collectors.toList()); + return QueryResourceDto.builder() + .resourceAbsolutePaths(resourceAbsolutePaths) + .build(); + } else { + // todo: inject the tenantCode when login + Tenant tenant = tenantDao.queryById(loginUser.getTenantId()); + String storageBaseDirectory = storageOperator.getStorageBaseDirectory(tenant.getTenantCode(), resourceType); + return QueryResourceDto.builder() + .resourceAbsolutePaths(Lists.newArrayList(storageBaseDirectory)) + .build(); + } + + } + + private void validatePagingResourceItemRequest(PagingResourceItemRequest pagingResourceItemRequest) { + checkNotNull(pagingResourceItemRequest.getLoginUser(), "loginUser is null"); + checkNotNull(pagingResourceItemRequest.getResourceType(), "resourceType is null"); + } + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameDirectoryDtoValidator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameDirectoryDtoValidator.java new file mode 100644 index 0000000000..b76ea5b6fc --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameDirectoryDtoValidator.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.validator.resource; + +import org.apache.dolphinscheduler.api.dto.resources.RenameDirectoryDto; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.repository.TenantDao; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; + +import org.springframework.stereotype.Component; + +@Component +public class RenameDirectoryDtoValidator extends AbstractResourceValidator { + + public RenameDirectoryDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) { + super(storageOperator, tenantDao); + } + + @Override + public void validate(RenameDirectoryDto renameDirectoryDto) { + String originDirectoryAbsolutePath = renameDirectoryDto.getOriginDirectoryAbsolutePath(); + User loginUser = renameDirectoryDto.getLoginUser(); + String targetDirectoryAbsolutePath = renameDirectoryDto.getTargetDirectoryAbsolutePath(); + + exceptionResourceAbsolutePathInvalidated(originDirectoryAbsolutePath); + exceptionResourceIsNotDirectory(originDirectoryAbsolutePath); + exceptionResourceNotExists(originDirectoryAbsolutePath); + exceptionUserNoResourcePermission(loginUser, originDirectoryAbsolutePath); + + exceptionResourceAbsolutePathInvalidated(targetDirectoryAbsolutePath); + exceptionResourceIsNotDirectory(targetDirectoryAbsolutePath); + exceptionResourceExists(targetDirectoryAbsolutePath); + exceptionUserNoResourcePermission(loginUser, targetDirectoryAbsolutePath); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameDirectoryRequestTransformer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameDirectoryRequestTransformer.java new file mode 100644 index 0000000000..4d6745df3d --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameDirectoryRequestTransformer.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.validator.resource; + +import org.apache.dolphinscheduler.api.dto.resources.RenameDirectoryDto; +import org.apache.dolphinscheduler.api.dto.resources.RenameDirectoryRequest; +import org.apache.dolphinscheduler.api.validator.ITransformer; + +import org.apache.commons.lang3.StringUtils; + +import java.io.File; + +import lombok.extern.slf4j.Slf4j; + +import org.springframework.stereotype.Component; + +@Slf4j +@Component +public class RenameDirectoryRequestTransformer implements ITransformer { + + @Override + public RenameDirectoryDto transform(RenameDirectoryRequest renameDirectoryRequest) { + String originDirectoryAbsolutePath = renameDirectoryRequest.getDirectoryAbsolutePath(); + String targetDirectoryName = renameDirectoryRequest.getNewDirectoryName(); + + String targetDirectoryAbsolutePath = + getTargetDirectoryAbsolutePath(originDirectoryAbsolutePath, targetDirectoryName); + + return RenameDirectoryDto.builder() + .loginUser(renameDirectoryRequest.getLoginUser()) + .originDirectoryAbsolutePath(originDirectoryAbsolutePath) + .targetDirectoryAbsolutePath(targetDirectoryAbsolutePath) + .build(); + } + + private String getTargetDirectoryAbsolutePath(String originDirectoryAbsolutePath, String targetDirectoryName) { + String originDirectoryParentAbsolutePath = StringUtils.substringBeforeLast( + originDirectoryAbsolutePath, File.separator); + return originDirectoryParentAbsolutePath + File.separator + targetDirectoryName; + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameFileDtoValidator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameFileDtoValidator.java new file mode 100644 index 0000000000..8c9e333dbb --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameFileDtoValidator.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.validator.resource; + +import org.apache.dolphinscheduler.api.dto.resources.RenameFileDto; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.repository.TenantDao; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; + +import org.springframework.stereotype.Component; + +@Component +public class RenameFileDtoValidator extends AbstractResourceValidator { + + public RenameFileDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) { + super(storageOperator, tenantDao); + } + + @Override + public void validate(RenameFileDto renameFileDto) { + String originFileAbsolutePath = renameFileDto.getOriginFileAbsolutePath(); + User loginUser = renameFileDto.getLoginUser(); + String targetFileAbsolutePath = renameFileDto.getTargetFileAbsolutePath(); + + exceptionResourceAbsolutePathInvalidated(originFileAbsolutePath); + exceptionResourceNotExists(originFileAbsolutePath); + exceptionResourceIsNotFile(originFileAbsolutePath); + exceptionUserNoResourcePermission(loginUser, originFileAbsolutePath); + + exceptionResourceAbsolutePathInvalidated(targetFileAbsolutePath); + exceptionResourceExists(targetFileAbsolutePath); + exceptionResourceIsNotFile(targetFileAbsolutePath); + exceptionUserNoResourcePermission(loginUser, targetFileAbsolutePath); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameFileRequestTransformer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameFileRequestTransformer.java new file mode 100644 index 0000000000..43cb55c461 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/RenameFileRequestTransformer.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.validator.resource; + +import org.apache.dolphinscheduler.api.dto.resources.RenameFileDto; +import org.apache.dolphinscheduler.api.dto.resources.RenameFileRequest; +import org.apache.dolphinscheduler.api.validator.ITransformer; +import org.apache.dolphinscheduler.common.utils.FileUtils; +import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class RenameFileRequestTransformer implements ITransformer { + + @Autowired + private StorageOperator storageOperator; + + @Override + public RenameFileDto transform(RenameFileRequest renameFileRequest) { + ResourceMetadata resourceMetaData = + storageOperator.getResourceMetaData(renameFileRequest.getFileAbsolutePath()); + return RenameFileDto.builder() + .loginUser(renameFileRequest.getLoginUser()) + .originFileAbsolutePath(renameFileRequest.getFileAbsolutePath()) + .targetFileAbsolutePath(FileUtils.concatFilePath(resourceMetaData.getResourceParentAbsolutePath(), + renameFileRequest.getNewFileName())) + .build(); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileDtoValidator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileDtoValidator.java new file mode 100644 index 0000000000..468c651087 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileDtoValidator.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.validator.resource; + +import org.apache.dolphinscheduler.api.dto.resources.UpdateFileDto; +import org.apache.dolphinscheduler.api.exceptions.ServiceException; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.repository.TenantDao; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; + +import java.util.Objects; + +import org.springframework.stereotype.Component; +import org.springframework.web.multipart.MultipartFile; + +import com.google.common.io.Files; + +@Component +public class UpdateFileDtoValidator extends AbstractResourceValidator { + + public UpdateFileDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) { + super(storageOperator, tenantDao); + } + + @Override + public void validate(UpdateFileDto updateFileDto) { + String fileAbsolutePath = updateFileDto.getFileAbsolutePath(); + User loginUser = updateFileDto.getLoginUser(); + MultipartFile file = updateFileDto.getFile(); + + if (!Objects.equals(Files.getFileExtension(file.getName()), + Files.getFileExtension(updateFileDto.getFileAbsolutePath()))) { + throw new ServiceException("file extension cannot not change"); + } + + exceptionResourceAbsolutePathInvalidated(fileAbsolutePath); + exceptionResourceNotExists(fileAbsolutePath); + exceptionResourceIsNotFile(fileAbsolutePath); + exceptionUserNoResourcePermission(loginUser, fileAbsolutePath); + exceptionFileInvalidated(file); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileFromContentDtoValidator.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileFromContentDtoValidator.java new file mode 100644 index 0000000000..5245759639 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileFromContentDtoValidator.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.validator.resource; + +import org.apache.dolphinscheduler.api.dto.resources.UpdateFileFromContentDto; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.repository.TenantDao; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; + +import org.springframework.stereotype.Component; + +@Component +public class UpdateFileFromContentDtoValidator extends AbstractResourceValidator { + + public UpdateFileFromContentDtoValidator(StorageOperator storageOperator, TenantDao tenantDao) { + super(storageOperator, tenantDao); + } + + @Override + public void validate(UpdateFileFromContentDto updateFileFromContentDto) { + String fileAbsolutePath = updateFileFromContentDto.getFileAbsolutePath(); + User loginUser = updateFileFromContentDto.getLoginUser(); + String fileContent = updateFileFromContentDto.getFileContent(); + + exceptionResourceAbsolutePathInvalidated(fileAbsolutePath); + exceptionResourceNotExists(fileAbsolutePath); + exceptionResourceIsNotFile(fileAbsolutePath); + exceptionUserNoResourcePermission(loginUser, fileAbsolutePath); + exceptionFileContentCannotFetch(fileAbsolutePath); + exceptionFileContentInvalidated(fileContent); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileFromContentRequestTransformer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileFromContentRequestTransformer.java new file mode 100644 index 0000000000..961d0e6049 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileFromContentRequestTransformer.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.validator.resource; + +import org.apache.dolphinscheduler.api.dto.resources.UpdateFileFromContentDto; +import org.apache.dolphinscheduler.api.dto.resources.UpdateFileFromContentRequest; +import org.apache.dolphinscheduler.api.validator.ITransformer; + +import org.springframework.stereotype.Component; + +@Component +public class UpdateFileFromContentRequestTransformer + implements + ITransformer { + + @Override + public UpdateFileFromContentDto transform(UpdateFileFromContentRequest updateFileContentRequest) { + return UpdateFileFromContentDto.builder() + .loginUser(updateFileContentRequest.getLoginUser()) + .fileAbsolutePath(updateFileContentRequest.getFileAbsolutePath()) + .fileContent(updateFileContentRequest.getFileContent()) + .build(); + } +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileRequestTransformer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileRequestTransformer.java new file mode 100644 index 0000000000..5c7646fbf9 --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/validator/resource/UpdateFileRequestTransformer.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.validator.resource; + +import org.apache.dolphinscheduler.api.dto.resources.UpdateFileDto; +import org.apache.dolphinscheduler.api.dto.resources.UpdateFileRequest; +import org.apache.dolphinscheduler.api.validator.ITransformer; + +import org.springframework.stereotype.Component; + +@Component +public class UpdateFileRequestTransformer implements ITransformer { + + @Override + public UpdateFileDto transform(UpdateFileRequest updateFileRequest) { + return UpdateFileDto.builder() + .loginUser(updateFileRequest.getLoginUser()) + .fileAbsolutePath(updateFileRequest.getFileAbsolutePath()) + .file(updateFileRequest.getFile()) + .build(); + } + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/vo/ResourceItemVO.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/vo/ResourceItemVO.java new file mode 100644 index 0000000000..9470ded4ba --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/vo/ResourceItemVO.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.vo; + +import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; +import org.apache.dolphinscheduler.spi.enums.ResourceType; + +import org.apache.commons.lang3.StringUtils; + +import java.io.File; +import java.util.Date; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class ResourceItemVO { + + // todo: remove this field, directly use fileName + private String alias; + + // todo: use tenantName instead of userName + private String userName; + + private String fileName; + + private String fullName; + + private boolean isDirectory; + + private ResourceType type; + + private long size; + + private Date createTime; + + private Date updateTime; + + public ResourceItemVO(StorageEntity storageEntity) { + this.isDirectory = storageEntity.isDirectory(); + this.alias = storageEntity.getFileName(); + this.fileName = storageEntity.getFileName(); + this.fullName = storageEntity.getFullName(); + this.type = storageEntity.getType(); + this.size = storageEntity.getSize(); + this.createTime = storageEntity.getCreateTime(); + this.updateTime = storageEntity.getUpdateTime(); + + if (isDirectory) { + alias = StringUtils.removeEndIgnoreCase(alias, File.separator); + fileName = StringUtils.removeEndIgnoreCase(fileName, File.separator); + fullName = StringUtils.removeEndIgnoreCase(fullName, File.separator); + } + } + +} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/vo/resources/FetchFileContentResponse.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/vo/resources/FetchFileContentResponse.java new file mode 100644 index 0000000000..1f228f42fb --- /dev/null +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/vo/resources/FetchFileContentResponse.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.vo.resources; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class FetchFileContentResponse { + + private String content; + +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/AssertionsHelper.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/AssertionsHelper.java index eae064bb24..21977bd50d 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/AssertionsHelper.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/AssertionsHelper.java @@ -27,6 +27,11 @@ import org.junit.jupiter.api.function.Executable; public class AssertionsHelper extends Assertions { + public static void assertThrowServiceException(String message, Executable executable) { + ServiceException exception = Assertions.assertThrows(ServiceException.class, executable); + Assertions.assertEquals(message, exception.getMessage()); + } + public static void assertThrowsServiceException(Status status, Executable executable) { ServiceException exception = Assertions.assertThrows(ServiceException.class, executable); Assertions.assertEquals(status.getCode(), exception.getCode()); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java index bfed64f9f6..01a2d47998 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.api.controller; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; @@ -28,29 +29,24 @@ import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.service.ResourcesService; import org.apache.dolphinscheduler.api.service.UdfFuncService; import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.constants.Constants; +import org.apache.dolphinscheduler.api.vo.resources.FetchFileContentResponse; import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.spi.enums.ResourceType; -import java.util.HashMap; -import java.util.Map; - import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.test.web.servlet.MvcResult; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; -/** - * resources controller test - */ +import com.fasterxml.jackson.core.type.TypeReference; + public class ResourcesControllerTest extends AbstractControllerTest { private static final Logger logger = LoggerFactory.getLogger(ResourcesControllerTest.class); @@ -61,37 +57,12 @@ public class ResourcesControllerTest extends AbstractControllerTest { @MockBean(name = "udfFuncServiceImpl") private UdfFuncService udfFuncService; - @Test - public void testQuerytResourceList() throws Exception { - Map mockResult = new HashMap<>(); - mockResult.put(Constants.STATUS, Status.SUCCESS); - Mockito.when(resourcesService.queryResourceList(Mockito.any(), Mockito.any(), Mockito.anyString())) - .thenReturn(mockResult); - - MultiValueMap paramsMap = new LinkedMultiValueMap<>(); - paramsMap.add("fullName", "dolphinscheduler/resourcePath"); - paramsMap.add("type", ResourceType.FILE.name()); - MvcResult mvcResult = mockMvc.perform(get("/resources/list") - .header(SESSION_ID, sessionId) - .params(paramsMap)) - .andExpect(status().isOk()) - .andExpect(content().contentType(MediaType.APPLICATION_JSON)) - .andReturn(); - - Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - - Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); - } - @Test public void testQueryResourceListPaging() throws Exception { Result mockResult = new Result<>(); mockResult.setCode(Status.SUCCESS.getCode()); - Mockito.when(resourcesService.queryResourceListPaging( - Mockito.any(), Mockito.anyString(), Mockito.anyString(), Mockito.any(), - Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt())) - .thenReturn(mockResult); + // Mockito.when(resourcesService.pagingResourceItem() + // .thenReturn(mockResult); MultiValueMap paramsMap = new LinkedMultiValueMap<>(); paramsMap.add("type", String.valueOf(ResourceType.FILE)); @@ -111,41 +82,17 @@ public class ResourcesControllerTest extends AbstractControllerTest { Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); - } - - @Test - public void testVerifyResourceName() throws Exception { - Result mockResult = new Result<>(); - mockResult.setCode(Status.TENANT_NOT_EXIST.getCode()); - Mockito.when(resourcesService.verifyResourceName(Mockito.anyString(), Mockito.any(), Mockito.any())) - .thenReturn(mockResult); - - MultiValueMap paramsMap = new LinkedMultiValueMap<>(); - paramsMap.add("fullName", "list_resources_1.sh"); - paramsMap.add("type", "FILE"); - - MvcResult mvcResult = mockMvc.perform(get("/resources/verify-name") - .header(SESSION_ID, sessionId) - .params(paramsMap)) - .andExpect(status().isOk()) - .andExpect(content().contentType(MediaType.APPLICATION_JSON)) - .andReturn(); - - Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - - Assertions.assertEquals(Status.TENANT_NOT_EXIST.getCode(), result.getCode().intValue()); + assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); } @Test public void testViewResource() throws Exception { - Result mockResult = new Result<>(); - mockResult.setCode(Status.HDFS_NOT_STARTUP.getCode()); - Mockito.when(resourcesService.readResource(Mockito.any(), - Mockito.anyString(), Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt())) - .thenReturn(mockResult); + FetchFileContentResponse fetchFileContentResponse = FetchFileContentResponse.builder() + .content("echo hello") + .build(); + Mockito.when(resourcesService.fetchResourceFileContent(Mockito.any())) + .thenReturn(fetchFileContentResponse); MultiValueMap paramsMap = new LinkedMultiValueMap<>(); paramsMap.add("skipLineNum", "2"); @@ -160,19 +107,17 @@ public class ResourcesControllerTest extends AbstractControllerTest { .andExpect(content().contentType(MediaType.APPLICATION_JSON)) .andReturn(); - Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), + new TypeReference>() { + }); - Assertions.assertEquals(Status.HDFS_NOT_STARTUP.getCode(), result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); + assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); + assertEquals(fetchFileContentResponse, result.getData()); } @Test public void testCreateResourceFile() throws Exception { - Result mockResult = new Result<>(); - mockResult.setCode(Status.TENANT_NOT_EXIST.getCode()); - Mockito.when(resourcesService.createResourceFile(Mockito.any(), Mockito.any(), Mockito.anyString(), - Mockito.anyString(), Mockito.anyString(), Mockito.anyString())) - .thenReturn(mockResult); + Mockito.doNothing().when(resourcesService).createFileFromContent(Mockito.any()); MultiValueMap paramsMap = new LinkedMultiValueMap<>(); paramsMap.add("type", String.valueOf(ResourceType.FILE)); @@ -190,19 +135,16 @@ public class ResourcesControllerTest extends AbstractControllerTest { .andExpect(content().contentType(MediaType.APPLICATION_JSON)) .andReturn(); - Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + Result result = + JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), new TypeReference>() { + }); - Assertions.assertEquals(Status.TENANT_NOT_EXIST.getCode(), result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); + assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); } @Test public void testUpdateResourceContent() throws Exception { - Result mockResult = new Result<>(); - mockResult.setCode(Status.TENANT_NOT_EXIST.getCode()); - Mockito.when(resourcesService.updateResourceContent(Mockito.any(), Mockito.anyString(), - Mockito.anyString(), Mockito.anyString())) - .thenReturn(mockResult); + Mockito.doNothing().when(resourcesService).updateFileFromContent(Mockito.any()); MultiValueMap paramsMap = new LinkedMultiValueMap<>(); paramsMap.add("id", "1"); @@ -217,17 +159,17 @@ public class ResourcesControllerTest extends AbstractControllerTest { .andExpect(content().contentType(MediaType.APPLICATION_JSON)) .andReturn(); - Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + Result result = + JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), new TypeReference>() { + }); - Assertions.assertEquals(Status.TENANT_NOT_EXIST.getCode(), result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); + assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); } @Test public void testDownloadResource() throws Exception { - Mockito.when(resourcesService.downloadResource(Mockito.any(), Mockito.anyString())) - .thenReturn(null); + Mockito.doNothing().when(resourcesService).downloadResource(Mockito.any(), Mockito.any()); MultiValueMap paramsMap = new LinkedMultiValueMap<>(); paramsMap.add("fullName", "dolphinscheduler/resourcePath"); @@ -235,7 +177,7 @@ public class ResourcesControllerTest extends AbstractControllerTest { MvcResult mvcResult = mockMvc.perform(get("/resources/download") .params(paramsMap) .header(SESSION_ID, sessionId)) - .andExpect(status().is(HttpStatus.BAD_REQUEST.value())) + .andExpect(status().isOk()) .andReturn(); Assertions.assertNotNull(mvcResult); @@ -269,7 +211,7 @@ public class ResourcesControllerTest extends AbstractControllerTest { Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - Assertions.assertEquals(Status.TENANT_NOT_EXIST.getCode(), result.getCode().intValue()); + assertEquals(Status.TENANT_NOT_EXIST.getCode(), result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); } @@ -289,7 +231,7 @@ public class ResourcesControllerTest extends AbstractControllerTest { Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - Assertions.assertEquals(Status.TENANT_NOT_EXIST.getCode(), result.getCode().intValue()); + assertEquals(Status.TENANT_NOT_EXIST.getCode(), result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); } @@ -323,7 +265,7 @@ public class ResourcesControllerTest extends AbstractControllerTest { Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - Assertions.assertEquals(Status.TENANT_NOT_EXIST.getCode(), result.getCode().intValue()); + assertEquals(Status.TENANT_NOT_EXIST.getCode(), result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); } @@ -348,7 +290,7 @@ public class ResourcesControllerTest extends AbstractControllerTest { Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); + assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); } @@ -370,7 +312,7 @@ public class ResourcesControllerTest extends AbstractControllerTest { Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); + assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); } @@ -392,7 +334,7 @@ public class ResourcesControllerTest extends AbstractControllerTest { Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); + assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); } @@ -410,17 +352,13 @@ public class ResourcesControllerTest extends AbstractControllerTest { Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); + assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); } @Test public void testDeleteResource() throws Exception { - Result mockResult = new Result<>(); - mockResult.setCode(Status.SUCCESS.getCode()); - Mockito.when(resourcesService.delete(Mockito.any(), Mockito.anyString(), - Mockito.anyString())) - .thenReturn(mockResult); + Mockito.doNothing().when(resourcesService).delete(Mockito.any()); MultiValueMap paramsMap = new LinkedMultiValueMap<>(); paramsMap.add("fullName", "dolphinscheduler/resourcePath"); paramsMap.add("tenantCode", "123"); @@ -431,9 +369,10 @@ public class ResourcesControllerTest extends AbstractControllerTest { .andExpect(content().contentType(MediaType.APPLICATION_JSON)) .andReturn(); - Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + Result result = + JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), new TypeReference>() { + }); - Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); + assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); } } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/python/PythonGatewayTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/python/PythonGatewayTest.java index 173ae98854..5ae26af826 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/python/PythonGatewayTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/python/PythonGatewayTest.java @@ -97,19 +97,6 @@ public class PythonGatewayTest { Assertions.assertEquals((long) result.get("taskDefinitionCode"), taskDefinition.getCode()); } - @Test - public void testCreateResource() { - User user = getTestUser(); - String resourceDir = "/dir1/dir2/"; - String resourceName = "test"; - String resourceSuffix = "py"; - String content = "content"; - String resourceFullName = resourceDir + resourceName + "." + resourceSuffix; - - Assertions.assertDoesNotThrow( - () -> pythonGateway.createOrUpdateResource(user.getUserName(), resourceFullName, content)); - } - @Test public void testQueryResourcesFileInfo() throws Exception { User user = getTestUser(); @@ -118,12 +105,11 @@ public class PythonGatewayTest { Mockito.when(resourcesService.queryFileStatus(user.getUserName(), storageEntity.getFullName())) .thenReturn(storageEntity); StorageEntity result = pythonGateway.queryResourcesFileInfo(user.getUserName(), storageEntity.getFullName()); - Assertions.assertEquals(result.getId(), storageEntity.getId()); + Assertions.assertEquals(result.getFullName(), storageEntity.getFullName()); } private StorageEntity getTestResource() { StorageEntity storageEntity = new StorageEntity(); - storageEntity.setId(1); storageEntity.setType(ResourceType.FILE); storageEntity.setFullName("/dev/test.py"); return storageEntity; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java deleted file mode 100644 index 6e94a25861..0000000000 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java +++ /dev/null @@ -1,657 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.api.service; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.when; - -import org.apache.dolphinscheduler.api.dto.resources.DeleteDataTransferResponse; -import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent; -import org.apache.dolphinscheduler.api.enums.Status; -import org.apache.dolphinscheduler.api.exceptions.ServiceException; -import org.apache.dolphinscheduler.api.permission.ResourcePermissionCheckService; -import org.apache.dolphinscheduler.api.service.impl.ResourcesServiceImpl; -import org.apache.dolphinscheduler.api.utils.PageInfo; -import org.apache.dolphinscheduler.api.utils.Result; -import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.enums.UserType; -import org.apache.dolphinscheduler.common.utils.FileUtils; -import org.apache.dolphinscheduler.common.utils.PropertyUtils; -import org.apache.dolphinscheduler.dao.entity.Tenant; -import org.apache.dolphinscheduler.dao.entity.User; -import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; -import org.apache.dolphinscheduler.dao.mapper.TenantMapper; -import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; -import org.apache.dolphinscheduler.dao.mapper.UserMapper; -import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; -import org.apache.dolphinscheduler.spi.enums.ResourceType; - -import org.apache.commons.collections4.CollectionUtils; - -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; -import org.mockito.junit.jupiter.MockitoSettings; -import org.mockito.quality.Strictness; -import org.springframework.mock.web.MockMultipartFile; - -import com.google.common.io.Files; - -/** - * resources service test - */ -@ExtendWith(MockitoExtension.class) -@MockitoSettings(strictness = Strictness.LENIENT) -public class ResourcesServiceTest { - - private static final String basePath = "/dolphinscheduler"; - private static final String tenantCode = "123"; - private static final String tenantFileResourceDir = "/dolphinscheduler/123/resources/"; - private static final String tenantUdfResourceDir = "/dolphinscheduler/123/udfs/"; - - @InjectMocks - private ResourcesServiceImpl resourcesService; - - @Mock - private TenantMapper tenantMapper; - - @Mock - private StorageOperate storageOperate; - - @Mock - private UserMapper userMapper; - - @Mock - private UdfFuncMapper udfFunctionMapper; - - @Mock - private ProcessDefinitionMapper processDefinitionMapper; - - @Mock - private ResourcePermissionCheckService resourcePermissionCheckService; - - private MockedStatic mockedStaticFileUtils; - - private MockedStatic mockedStaticFiles; - - private MockedStatic mockedStaticDolphinschedulerFileUtils; - - private MockedStatic mockedStaticPropertyUtils; - - private MockedStatic mockedStaticPaths; - - private MockedStatic filesMockedStatic; - - private Exception exception; - - @BeforeEach - public void setUp() { - mockedStaticFileUtils = Mockito.mockStatic(FileUtils.class); - mockedStaticFiles = Mockito.mockStatic(Files.class); - mockedStaticDolphinschedulerFileUtils = - Mockito.mockStatic(org.apache.dolphinscheduler.api.utils.FileUtils.class); - - mockedStaticPropertyUtils = Mockito.mockStatic(PropertyUtils.class); - mockedStaticPaths = Mockito.mockStatic(Paths.class); - filesMockedStatic = Mockito.mockStatic(java.nio.file.Files.class); - } - - @AfterEach - public void after() { - mockedStaticFileUtils.close(); - mockedStaticFiles.close(); - mockedStaticDolphinschedulerFileUtils.close(); - mockedStaticPropertyUtils.close(); - mockedStaticPaths.close(); - filesMockedStatic.close(); - } - - @Test - public void testCreateResource() { - User user = new User(); - user.setId(1); - user.setUserType(UserType.GENERAL_USER); - - // CURRENT_LOGIN_USER_TENANT_NOT_EXIST - when(userMapper.selectById(user.getId())).thenReturn(getUser()); - when(tenantMapper.queryById(1)).thenReturn(null); - ServiceException serviceException = Assertions.assertThrows(ServiceException.class, - () -> resourcesService.uploadResource(user, "ResourcesServiceTest", ResourceType.FILE, - new MockMultipartFile("test.pdf", "test.pdf", "pdf", "test".getBytes()), "/")); - assertEquals(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST.getMsg(), serviceException.getMessage()); - - // set tenant for user - user.setTenantId(1); - when(tenantMapper.queryById(1)).thenReturn(getTenant()); - when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn(basePath); - - // ILLEGAL_RESOURCE_FILE - String illegal_path = "/dolphinscheduler/123/../"; - serviceException = Assertions.assertThrows(ServiceException.class, - () -> { - MockMultipartFile mockMultipartFile = new MockMultipartFile("test.pdf", "".getBytes()); - resourcesService.uploadResource(user, "ResourcesServiceTest", ResourceType.FILE, - mockMultipartFile, illegal_path); - }); - assertEquals(new ServiceException(Status.ILLEGAL_RESOURCE_PATH, illegal_path), serviceException); - - // RESOURCE_FILE_IS_EMPTY - MockMultipartFile mockMultipartFile = new MockMultipartFile("test.pdf", "".getBytes()); - Result result = resourcesService.uploadResource(user, "ResourcesServiceTest", ResourceType.FILE, - mockMultipartFile, tenantFileResourceDir); - assertEquals(Status.RESOURCE_FILE_IS_EMPTY.getMsg(), result.getMsg()); - - // RESOURCE_SUFFIX_FORBID_CHANGE - mockMultipartFile = new MockMultipartFile("test.pdf", "test.pdf", "pdf", "test".getBytes()); - when(Files.getFileExtension("test.pdf")).thenReturn("pdf"); - when(Files.getFileExtension("ResourcesServiceTest.jar")).thenReturn("jar"); - result = resourcesService.uploadResource(user, "ResourcesServiceTest.jar", ResourceType.FILE, mockMultipartFile, - tenantFileResourceDir); - assertEquals(Status.RESOURCE_SUFFIX_FORBID_CHANGE.getMsg(), result.getMsg()); - - // UDF_RESOURCE_SUFFIX_NOT_JAR - mockMultipartFile = - new MockMultipartFile("ResourcesServiceTest.pdf", "ResourcesServiceTest.pdf", "pdf", "test".getBytes()); - when(Files.getFileExtension("ResourcesServiceTest.pdf")).thenReturn("pdf"); - result = resourcesService.uploadResource(user, "ResourcesServiceTest.pdf", ResourceType.UDF, mockMultipartFile, - tenantUdfResourceDir); - assertEquals(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg(), result.getMsg()); - - // FULL_FILE_NAME_TOO_LONG - String tooLongFileName = getRandomStringWithLength(Constants.RESOURCE_FULL_NAME_MAX_LENGTH) + ".pdf"; - mockMultipartFile = new MockMultipartFile(tooLongFileName, tooLongFileName, "pdf", "test".getBytes()); - when(Files.getFileExtension(tooLongFileName)).thenReturn("pdf"); - - // '/databasePath/tenantCode/RESOURCE/' - when(storageOperate.getResDir(tenantCode)).thenReturn(tenantFileResourceDir); - result = resourcesService.uploadResource(user, tooLongFileName, ResourceType.FILE, mockMultipartFile, - tenantFileResourceDir); - assertEquals(Status.RESOURCE_FULL_NAME_TOO_LONG_ERROR.getMsg(), result.getMsg()); - } - - @Test - public void testCreateDirecotry() throws IOException { - User user = new User(); - user.setId(1); - user.setUserType(UserType.GENERAL_USER); - - String fileName = "directoryTest"; - // RESOURCE_EXIST - user.setId(1); - user.setTenantId(1); - when(tenantMapper.queryById(1)).thenReturn(getTenant()); - when(userMapper.selectById(user.getId())).thenReturn(getUser()); - when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn(basePath); - when(storageOperate.getResDir(tenantCode)).thenReturn(tenantFileResourceDir); - when(storageOperate.exists(tenantFileResourceDir + fileName)).thenReturn(true); - Result result = resourcesService.createDirectory(user, fileName, ResourceType.FILE, -1, tenantFileResourceDir); - assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg()); - } - - @Test - public void testUpdateResource() throws Exception { - User user = new User(); - user.setId(1); - user.setUserType(UserType.GENERAL_USER); - user.setTenantId(1); - - when(userMapper.selectById(user.getId())).thenReturn(getUser()); - when(tenantMapper.queryById(1)).thenReturn(getTenant()); - when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn(basePath); - when(storageOperate.getResDir(tenantCode)).thenReturn(tenantFileResourceDir); - - // TENANT_NOT_EXIST - when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(null); - Assertions.assertThrows(ServiceException.class, () -> resourcesService.updateResource(user, - "ResourcesServiceTest1.jar", "", "ResourcesServiceTest", ResourceType.UDF, null)); - - // USER_NO_OPERATION_PERM - user.setUserType(UserType.GENERAL_USER); - // tenant who have access to resource is 123, - Tenant tenantWNoPermission = new Tenant(); - tenantWNoPermission.setTenantCode("321"); - when(tenantMapper.queryById(1)).thenReturn(tenantWNoPermission); - when(storageOperate.getDir(ResourceType.ALL, "321")).thenReturn(basePath); - - String fileName = "ResourcesServiceTest"; - Result result = resourcesService.updateResource(user, tenantFileResourceDir + fileName, - tenantCode, fileName, ResourceType.FILE, null); - assertEquals(Status.NO_CURRENT_OPERATING_PERMISSION.getMsg(), result.getMsg()); - - // SUCCESS - when(tenantMapper.queryById(1)).thenReturn(getTenant()); - when(storageOperate.exists(Mockito.any())).thenReturn(false); - - when(storageOperate.getDir(ResourceType.FILE, tenantCode)).thenReturn(tenantFileResourceDir); - when(storageOperate.getFileStatus(tenantFileResourceDir + fileName, - tenantFileResourceDir, tenantCode, ResourceType.FILE)) - .thenReturn(getStorageEntityResource(fileName)); - result = resourcesService.updateResource(user, tenantFileResourceDir + fileName, - tenantCode, fileName, ResourceType.FILE, null); - assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); - - // Tests for udf resources. - fileName = "ResourcesServiceTest.jar"; - when(storageOperate.getDir(ResourceType.UDF, tenantCode)).thenReturn(tenantUdfResourceDir); - when(storageOperate.exists(tenantUdfResourceDir + fileName)).thenReturn(true); - when(storageOperate.getFileStatus(tenantUdfResourceDir + fileName, tenantUdfResourceDir, tenantCode, - ResourceType.UDF)) - .thenReturn(getStorageEntityUdfResource(fileName)); - result = resourcesService.updateResource(user, tenantUdfResourceDir + fileName, - tenantCode, fileName, ResourceType.UDF, null); - assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); - } - - @Test - public void testQueryResourceListPaging() throws Exception { - User loginUser = new User(); - loginUser.setId(1); - loginUser.setTenantId(1); - loginUser.setTenantCode("tenant1"); - loginUser.setUserType(UserType.ADMIN_USER); - - String fileName = "ResourcesServiceTest"; - List mockResList = new ArrayList<>(); - mockResList.add(getStorageEntityResource(fileName)); - List mockUserList = new ArrayList<>(); - mockUserList.add(getUser()); - when(userMapper.selectList(null)).thenReturn(mockUserList); - when(userMapper.selectById(getUser().getId())).thenReturn(getUser()); - when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(getTenant()); - when(storageOperate.getResDir(tenantCode)).thenReturn(tenantFileResourceDir); - when(storageOperate.listFilesStatus(tenantFileResourceDir, tenantFileResourceDir, - tenantCode, ResourceType.FILE)).thenReturn(mockResList); - - Result result = resourcesService.queryResourceListPaging(loginUser, "", "", ResourceType.FILE, "Test", 1, 10); - assertEquals(Status.SUCCESS.getCode(), (int) result.getCode()); - PageInfo pageInfo = (PageInfo) result.getData(); - Assertions.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getTotalList())); - - } - - @Test - public void testQueryResourceList() { - User loginUser = getUser(); - String fileName = "ResourcesServiceTest"; - - when(userMapper.selectList(null)).thenReturn(Collections.singletonList(loginUser)); - when(userMapper.selectById(loginUser.getId())).thenReturn(loginUser); - when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(getTenant()); - when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn(basePath); - when(storageOperate.getDir(ResourceType.FILE, tenantCode)).thenReturn(tenantFileResourceDir); - when(storageOperate.getResDir(tenantCode)).thenReturn(tenantFileResourceDir); - when(storageOperate.listFilesStatusRecursively(tenantFileResourceDir, - tenantFileResourceDir, tenantCode, ResourceType.FILE)) - .thenReturn(Collections.singletonList(getStorageEntityResource(fileName))); - Map result = - resourcesService.queryResourceList(loginUser, ResourceType.FILE, tenantFileResourceDir); - assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); - List resourceList = (List) result.get(Constants.DATA_LIST); - Assertions.assertTrue(CollectionUtils.isNotEmpty(resourceList)); - - // test udf - when(storageOperate.getDir(ResourceType.UDF, tenantCode)).thenReturn(tenantUdfResourceDir); - when(storageOperate.getUdfDir(tenantCode)).thenReturn(tenantUdfResourceDir); - when(storageOperate.listFilesStatusRecursively(tenantUdfResourceDir, tenantUdfResourceDir, - tenantCode, ResourceType.UDF)).thenReturn(Arrays.asList(getStorageEntityUdfResource("test.jar"))); - loginUser.setUserType(UserType.GENERAL_USER); - result = resourcesService.queryResourceList(loginUser, ResourceType.UDF, tenantUdfResourceDir); - assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); - resourceList = (List) result.get(Constants.DATA_LIST); - Assertions.assertTrue(CollectionUtils.isNotEmpty(resourceList)); - } - - @Test - public void testDelete() throws Exception { - User loginUser = new User(); - loginUser.setId(0); - loginUser.setUserType(UserType.GENERAL_USER); - - // TENANT_NOT_EXIST - loginUser.setUserType(UserType.ADMIN_USER); - loginUser.setTenantId(2); - when(userMapper.selectById(loginUser.getId())).thenReturn(loginUser); - Assertions.assertThrows(ServiceException.class, () -> resourcesService.delete(loginUser, "", "")); - - // RESOURCE_NOT_EXIST - String fileName = "ResourcesServiceTest"; - when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(getTenant()); - when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn(basePath); - when(storageOperate.getResDir(getTenant().getTenantCode())).thenReturn(tenantFileResourceDir); - when(storageOperate.getFileStatus(tenantFileResourceDir + fileName, tenantFileResourceDir, tenantCode, null)) - .thenReturn(getStorageEntityResource(fileName)); - Result result = resourcesService.delete(loginUser, tenantFileResourceDir + "ResNotExist", tenantCode); - assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); - - // SUCCESS - loginUser.setTenantId(1); - result = resourcesService.delete(loginUser, tenantFileResourceDir + fileName, tenantCode); - assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); - } - - @Test - public void testVerifyResourceName() throws IOException { - User user = new User(); - user.setId(1); - user.setUserType(UserType.GENERAL_USER); - - String fileName = "ResourcesServiceTest"; - when(storageOperate.exists(tenantFileResourceDir + fileName)).thenReturn(true); - - Result result = resourcesService.verifyResourceName(tenantFileResourceDir + fileName, ResourceType.FILE, user); - assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg()); - - // RESOURCE_FILE_EXIST - result = resourcesService.verifyResourceName(tenantFileResourceDir + fileName, ResourceType.FILE, user); - Assertions.assertTrue(Status.RESOURCE_EXIST.getCode() == result.getCode()); - - // SUCCESS - result = resourcesService.verifyResourceName("test2", ResourceType.FILE, user); - assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); - } - - @Test - public void testReadResource() throws IOException { - // RESOURCE_NOT_EXIST - when(userMapper.selectById(getUser().getId())).thenReturn(getUser()); - when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(getTenant()); - Result result = resourcesService.readResource(getUser(), "", "", 1, 10); - assertEquals(Status.RESOURCE_FILE_NOT_EXIST.getCode(), (int) result.getCode()); - - // RESOURCE_SUFFIX_NOT_SUPPORT_VIEW - when(FileUtils.getResourceViewSuffixes()).thenReturn("class"); - result = resourcesService.readResource(getUser(), "", "", 1, 10); - assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(), result.getMsg()); - - // USER_NOT_EXIST - when(userMapper.selectById(getUser().getId())).thenReturn(null); - when(FileUtils.getResourceViewSuffixes()).thenReturn("jar"); - when(Files.getFileExtension("ResourcesServiceTest.jar")).thenReturn("jar"); - result = resourcesService.readResource(getUser(), "", "", 1, 10); - assertEquals(Status.USER_NOT_EXIST.getCode(), (int) result.getCode()); - - // TENANT_NOT_EXIST - when(userMapper.selectById(getUser().getId())).thenReturn(getUser()); - when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(null); - Assertions.assertThrows(ServiceException.class, () -> resourcesService.readResource(getUser(), "", "", 1, 10)); - - // SUCCESS - when(FileUtils.getResourceViewSuffixes()).thenReturn("jar,sh"); - when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn(basePath); - when(storageOperate.getResDir(getTenant().getTenantCode())).thenReturn(tenantFileResourceDir); - when(userMapper.selectById(getUser().getId())).thenReturn(getUser()); - when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(getTenant()); - when(storageOperate.exists(Mockito.any())).thenReturn(true); - when(storageOperate.vimFile(Mockito.any(), Mockito.any(), eq(1), eq(10))).thenReturn(getContent()); - when(Files.getFileExtension("/dolphinscheduler/123/resources/test.jar")).thenReturn("jar"); - result = resourcesService.readResource(getUser(), "/dolphinscheduler/123/resources/test.jar", tenantCode, 1, - 10); - assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); - } - - @Test - public void testCreateOrUpdateResource() throws Exception { - User user = getUser(); - when(userMapper.queryByUserNameAccurately(user.getUserName())).thenReturn(getUser()); - - // RESOURCE_SUFFIX_NOT_SUPPORT_VIEW - exception = Assertions.assertThrows(IllegalArgumentException.class, - () -> resourcesService.createOrUpdateResource(user.getUserName(), "filename", "my-content")); - Assertions.assertTrue( - exception.getMessage().contains("Not allow create or update resources without extension name")); - - // SUCCESS - String fileName = "ResourcesServiceTest"; - when(storageOperate.getResDir(user.getTenantCode())).thenReturn(tenantFileResourceDir); - when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test"); - when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true); - when(storageOperate.getFileStatus(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.any())) - .thenReturn(getStorageEntityResource(fileName)); - StorageEntity storageEntity = - resourcesService.createOrUpdateResource(user.getUserName(), "filename.txt", "my-content"); - Assertions.assertNotNull(storageEntity); - assertEquals(tenantFileResourceDir + fileName, storageEntity.getFullName()); - } - - @Test - public void testUpdateResourceContent() throws Exception { - // RESOURCE_PATH_ILLEGAL - when(userMapper.selectById(getUser().getId())).thenReturn(getUser()); - when(tenantMapper.queryById(1)).thenReturn(getTenant()); - when(storageOperate.getResDir(Mockito.anyString())).thenReturn("/tmp"); - - String fileName = "ResourcesServiceTest.jar"; - ServiceException serviceException = - Assertions.assertThrows(ServiceException.class, () -> resourcesService.updateResourceContent(getUser(), - tenantFileResourceDir + fileName, tenantCode, "content")); - assertEquals(new ServiceException(Status.ILLEGAL_RESOURCE_PATH, tenantFileResourceDir + fileName), - serviceException); - - // RESOURCE_NOT_EXIST - when(storageOperate.getDir(ResourceType.ALL, tenantCode)).thenReturn(basePath); - when(storageOperate.getResDir(Mockito.anyString())).thenReturn(tenantFileResourceDir); - when(storageOperate.getFileStatus(tenantFileResourceDir + fileName, "", tenantCode, ResourceType.FILE)) - .thenReturn(null); - Result result = resourcesService.updateResourceContent(getUser(), tenantFileResourceDir + fileName, tenantCode, - "content"); - assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); - - // RESOURCE_SUFFIX_NOT_SUPPORT_VIEW - when(FileUtils.getResourceViewSuffixes()).thenReturn("class"); - when(storageOperate.getFileStatus(tenantFileResourceDir, "", tenantCode, ResourceType.FILE)) - .thenReturn(getStorageEntityResource(fileName)); - - result = resourcesService.updateResourceContent(getUser(), tenantFileResourceDir, tenantCode, - "content"); - assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(), result.getMsg()); - - // USER_NOT_EXIST - when(userMapper.selectById(getUser().getId())).thenReturn(null); - result = resourcesService.updateResourceContent(getUser(), tenantFileResourceDir + "123.class", - tenantCode, - "content"); - Assertions.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode()); - - // TENANT_NOT_EXIST - when(userMapper.selectById(getUser().getId())).thenReturn(getUser()); - when(tenantMapper.queryById(1)).thenReturn(null); - Assertions.assertThrows(ServiceException.class, () -> resourcesService.updateResourceContent(getUser(), - tenantFileResourceDir + fileName, tenantCode, "content")); - - // SUCCESS - when(storageOperate.getFileStatus(tenantFileResourceDir + fileName, "", tenantCode, - ResourceType.FILE)).thenReturn(getStorageEntityResource(fileName)); - - when(Files.getFileExtension(Mockito.anyString())).thenReturn("jar"); - when(FileUtils.getResourceViewSuffixes()).thenReturn("jar"); - when(userMapper.selectById(getUser().getId())).thenReturn(getUser()); - when(tenantMapper.queryById(1)).thenReturn(getTenant()); - when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test"); - when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true); - result = resourcesService.updateResourceContent(getUser(), - tenantFileResourceDir + fileName, tenantCode, "content"); - assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); - } - - @Test - public void testDownloadResource() throws IOException { - when(tenantMapper.queryById(1)).thenReturn(getTenant()); - when(userMapper.selectById(1)).thenReturn(getUser()); - org.springframework.core.io.Resource resourceMock = Mockito.mock(org.springframework.core.io.Resource.class); - Path path = Mockito.mock(Path.class); - when(Paths.get(Mockito.any())).thenReturn(path); - when(java.nio.file.Files.size(Mockito.any())).thenReturn(1L); - // resource null - org.springframework.core.io.Resource resource = resourcesService.downloadResource(getUser(), ""); - Assertions.assertNull(resource); - - when(org.apache.dolphinscheduler.api.utils.FileUtils.file2Resource(Mockito.any())).thenReturn(resourceMock); - resource = resourcesService.downloadResource(getUser(), ""); - Assertions.assertNotNull(resource); - } - - @Test - public void testDeleteDataTransferData() throws Exception { - User user = getUser(); - when(userMapper.selectById(user.getId())).thenReturn(getUser()); - when(tenantMapper.queryById(user.getTenantId())).thenReturn(getTenant()); - - StorageEntity storageEntity1 = Mockito.mock(StorageEntity.class); - StorageEntity storageEntity2 = Mockito.mock(StorageEntity.class); - StorageEntity storageEntity3 = Mockito.mock(StorageEntity.class); - StorageEntity storageEntity4 = Mockito.mock(StorageEntity.class); - StorageEntity storageEntity5 = Mockito.mock(StorageEntity.class); - - when(storageEntity1.getFullName()).thenReturn("DATA_TRANSFER/20220101"); - when(storageEntity2.getFullName()).thenReturn("DATA_TRANSFER/20220102"); - when(storageEntity3.getFullName()).thenReturn("DATA_TRANSFER/20220103"); - when(storageEntity4.getFullName()).thenReturn("DATA_TRANSFER/20220104"); - when(storageEntity5.getFullName()).thenReturn("DATA_TRANSFER/20220105"); - - List storageEntityList = new ArrayList<>(); - storageEntityList.add(storageEntity1); - storageEntityList.add(storageEntity2); - storageEntityList.add(storageEntity3); - storageEntityList.add(storageEntity4); - storageEntityList.add(storageEntity5); - - when(storageOperate.listFilesStatus(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any())) - .thenReturn(storageEntityList); - - LocalDateTime localDateTime = LocalDateTime.of(2022, 1, 5, 0, 0, 0); - try (MockedStatic mockHook = Mockito.mockStatic(LocalDateTime.class)) { - mockHook.when(LocalDateTime::now).thenReturn(localDateTime); - DeleteDataTransferResponse response = resourcesService.deleteDataTransferData(user, 3); - - assertEquals(response.getSuccessList().size(), 2); - assertEquals(response.getSuccessList().get(0), "DATA_TRANSFER/20220101"); - assertEquals(response.getSuccessList().get(1), "DATA_TRANSFER/20220102"); - } - - try (MockedStatic mockHook = Mockito.mockStatic(LocalDateTime.class)) { - mockHook.when(LocalDateTime::now).thenReturn(localDateTime); - DeleteDataTransferResponse response = resourcesService.deleteDataTransferData(user, 0); - assertEquals(response.getSuccessList().size(), 5); - } - - } - - @Test - public void testCatFile() throws IOException { - // SUCCESS - List list = storageOperate.vimFile(Mockito.any(), Mockito.anyString(), eq(1), eq(10)); - Assertions.assertNotNull(list); - } - - @Test - void testQueryBaseDir() throws Exception { - User user = getUser(); - String fileName = "ResourcesServiceTest.jar"; - when(userMapper.selectById(user.getId())).thenReturn(getUser()); - when(tenantMapper.queryById(user.getTenantId())).thenReturn(getTenant()); - when(storageOperate.getDir(ResourceType.FILE, tenantCode)).thenReturn(tenantFileResourceDir); - when(storageOperate.getFileStatus(Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), - Mockito.any())).thenReturn(getStorageEntityResource(fileName)); - Result result = resourcesService.queryResourceBaseDir(user, ResourceType.FILE); - assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); - } - - private Tenant getTenant() { - Tenant tenant = new Tenant(); - tenant.setTenantCode(tenantCode); - return tenant; - } - - private User getUser() { - User user = new User(); - user.setId(1); - user.setUserType(UserType.GENERAL_USER); - user.setTenantId(1); - user.setTenantCode(tenantCode); - return user; - } - - private StorageEntity getStorageEntityResource(String fileName) { - StorageEntity entity = new StorageEntity(); - entity.setAlias(fileName); - entity.setFileName(fileName); - entity.setDirectory(false); - entity.setUserName(tenantCode); - entity.setType(ResourceType.FILE); - entity.setFullName(tenantFileResourceDir + fileName); - return entity; - } - - private StorageEntity getStorageEntityUdfResource(String fileName) { - StorageEntity entity = new StorageEntity(); - entity.setAlias(fileName); - entity.setFileName(fileName); - entity.setDirectory(false); - entity.setUserName(tenantCode); - entity.setType(ResourceType.UDF); - entity.setFullName(tenantUdfResourceDir + fileName); - - return entity; - } - - private List getContent() { - List contentList = new ArrayList<>(); - contentList.add("test"); - return contentList; - } - - private List> getResources() { - List> resources = new ArrayList<>(); - Map resource = new HashMap<>(); - resource.put("id", 1); - resource.put("resource_ids", "1"); - resources.add(resource); - return resources; - } - - private static String getRandomStringWithLength(int length) { - Random r = new Random(); - StringBuilder sb = new StringBuilder(); - while (sb.length() < length) { - char c = (char) (r.nextInt(26) + 'a'); - sb.append(c); - } - return sb.toString(); - } -} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java index 5746840e90..6925441b10 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java @@ -41,7 +41,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.commons.collections4.CollectionUtils; @@ -95,7 +95,7 @@ public class TenantServiceTest { private ResourcePermissionCheckService resourcePermissionCheckService; @Mock - private StorageOperate storageOperate; + private StorageOperator storageOperator; private static final String tenantCode = "hayden"; private static final String tenantDesc = "This is the tenant desc"; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java index f1721cbb46..5c7bba5821 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java @@ -32,11 +32,10 @@ import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper; import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.commons.collections4.CollectionUtils; -import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.Date; @@ -83,7 +82,7 @@ public class UdfFuncServiceTest { private UDFUserMapper udfUserMapper; @Mock - private StorageOperate storageOperate; + private StorageOperator storageOperator; @BeforeEach public void setUp() { @@ -111,11 +110,7 @@ public class UdfFuncServiceTest { logger.info(result.toString()); Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); // success - try { - Mockito.when(storageOperate.exists("String")).thenReturn(true); - } catch (IOException e) { - logger.error("AmazonServiceException when checking resource: String"); - } + Mockito.when(storageOperator.exists("String")).thenReturn(true); result = udfFuncService.createUdfFunction(getLoginUser(), "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", @@ -176,11 +171,7 @@ public class UdfFuncServiceTest { // success Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.UDF, 1, ApiFuncIdentificationConstant.UDF_FUNCTION_UPDATE, serviceLogger)).thenReturn(true); - try { - Mockito.when(storageOperate.exists("")).thenReturn(true); - } catch (IOException e) { - logger.error("AmazonServiceException when checking resource: "); - } + Mockito.when(storageOperator.exists("")).thenReturn(true); result = udfFuncService.updateUdfFunc(getLoginUser(), 11, "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java index 3cb71d97a0..9e2a359667 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java @@ -48,7 +48,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProjectUserMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.commons.collections4.CollectionUtils; @@ -117,7 +117,7 @@ public class UsersServiceTest { private ProjectMapper projectMapper; @Mock - private StorageOperate storageOperate; + private StorageOperator storageOperator; @Mock private ResourcePermissionCheckService resourcePermissionCheckService; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/CreateDirectoryDtoValidatorTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/CreateDirectoryDtoValidatorTest.java new file mode 100644 index 0000000000..11803cba92 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/CreateDirectoryDtoValidatorTest.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.validator.resource; + +import static org.apache.dolphinscheduler.api.AssertionsHelper.assertThrowServiceException; +import static org.mockito.Mockito.when; + +import org.apache.dolphinscheduler.api.dto.resources.CreateDirectoryDto; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.dao.entity.Tenant; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.repository.TenantDao; +import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; + +import java.util.Locale; +import java.util.Optional; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.context.i18n.LocaleContextHolder; + +@ExtendWith(MockitoExtension.class) +class CreateDirectoryDtoValidatorTest { + + @Mock + private StorageOperator storageOperator; + + @Mock + private TenantDao tenantDao; + + @InjectMocks + private CreateDirectoryDtoValidator createDirectoryDtoValidator; + + private static final String BASE_DIRECTORY = "/tmp/dolphinscheduler"; + + private User loginUser; + + @BeforeEach + public void setup() { + when(storageOperator.getStorageBaseDirectory()).thenReturn(BASE_DIRECTORY); + loginUser = new User(); + loginUser.setTenantId(1); + LocaleContextHolder.setLocale(Locale.ENGLISH); + } + + @Test + void testValidate_notUnderBaseDirectory() { + CreateDirectoryDto createDirectoryDto = CreateDirectoryDto.builder() + .loginUser(loginUser) + .directoryAbsolutePath("/tmp") + .build(); + assertThrowServiceException( + "Internal Server Error: Invalidated resource path: /tmp", + () -> createDirectoryDtoValidator.validate(createDirectoryDto)); + } + + @Test + public void testValidate_directoryPathContainsIllegalSymbolic() { + CreateDirectoryDto createDirectoryDto = CreateDirectoryDto.builder() + .loginUser(loginUser) + .directoryAbsolutePath("/tmp/dolphinscheduler/default/resources/..") + .build(); + assertThrowServiceException( + "Internal Server Error: Invalidated resource path: /tmp/dolphinscheduler/default/resources/..", + () -> createDirectoryDtoValidator.validate(createDirectoryDto)); + } + + @Test + public void testValidate_directoryExist() { + CreateDirectoryDto createDirectoryDto = CreateDirectoryDto.builder() + .loginUser(loginUser) + .directoryAbsolutePath("/tmp/dolphinscheduler/default/resources/demo") + .build(); + when(storageOperator.exists(createDirectoryDto.getDirectoryAbsolutePath())).thenReturn(true); + assertThrowServiceException( + "Internal Server Error: The resource is already exist: /tmp/dolphinscheduler/default/resources/demo", + () -> createDirectoryDtoValidator.validate(createDirectoryDto)); + } + + @Test + public void testValidate_NoPermission() { + Tenant tenant = new Tenant(); + tenant.setTenantCode("test"); + when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant)); + + CreateDirectoryDto createDirectoryDto = CreateDirectoryDto.builder() + .loginUser(loginUser) + .directoryAbsolutePath("/tmp/dolphinscheduler/default/resources/demo") + .build(); + when(storageOperator.getResourceMetaData(createDirectoryDto.getDirectoryAbsolutePath())) + .thenReturn(ResourceMetadata.builder() + .resourceAbsolutePath(createDirectoryDto.getDirectoryAbsolutePath()) + .resourceBaseDirectory(BASE_DIRECTORY) + .resourceRelativePath("demo") + .isDirectory(true) + .tenant("default") + .build()); + when(storageOperator.exists(createDirectoryDto.getDirectoryAbsolutePath())).thenReturn(false); + assertThrowServiceException( + "Internal Server Error: The user's tenant is test have no permission to access the resource: /tmp/dolphinscheduler/default/resources/demo", + () -> createDirectoryDtoValidator.validate(createDirectoryDto)); + } + + @Test + public void testValidate_pathNotDirectory() { + CreateDirectoryDto createDirectoryDto = CreateDirectoryDto.builder() + .loginUser(loginUser) + .directoryAbsolutePath("/tmp/dolphinscheduler/default/resources/demo.sql") + .build(); + loginUser.setUserType(UserType.ADMIN_USER); + assertThrowServiceException( + "Internal Server Error: The path is not a directory: /tmp/dolphinscheduler/default/resources/demo.sql", + () -> createDirectoryDtoValidator.validate(createDirectoryDto)); + } + +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/CreateFileFromContentDtoValidatorTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/CreateFileFromContentDtoValidatorTest.java new file mode 100644 index 0000000000..6312346e40 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/CreateFileFromContentDtoValidatorTest.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.validator.resource; + +import static org.apache.dolphinscheduler.api.AssertionsHelper.assertDoesNotThrow; +import static org.apache.dolphinscheduler.api.AssertionsHelper.assertThrowServiceException; +import static org.mockito.Mockito.when; + +import org.apache.dolphinscheduler.api.dto.resources.CreateFileFromContentDto; +import org.apache.dolphinscheduler.dao.entity.Tenant; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.repository.TenantDao; +import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; + +import java.util.Locale; +import java.util.Optional; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.context.i18n.LocaleContextHolder; + +@ExtendWith(MockitoExtension.class) +class CreateFileFromContentDtoValidatorTest { + + @Mock + private StorageOperator storageOperator; + + @Mock + private TenantDao tenantDao; + + @InjectMocks + private CreateFileFromContentDtoValidator createFileFromContentDtoValidator; + + private static final String BASE_DIRECTORY = "/tmp/dolphinscheduler"; + + private User loginUser; + + @BeforeEach + public void setup() { + when(storageOperator.getStorageBaseDirectory()).thenReturn(BASE_DIRECTORY); + loginUser = new User(); + loginUser.setTenantId(1); + LocaleContextHolder.setLocale(Locale.ENGLISH); + } + + @Test + void testValidate_notUnderBaseDirectory() { + CreateFileFromContentDto createFileFromContentDto = CreateFileFromContentDto.builder() + .loginUser(loginUser) + .fileAbsolutePath("/tmp") + .fileContent("select * from t") + .build(); + assertThrowServiceException( + "Internal Server Error: Invalidated resource path: /tmp", + () -> createFileFromContentDtoValidator.validate(createFileFromContentDto)); + + } + + @Test + public void testValidate_filePathContainsIllegalSymbolic() { + CreateFileFromContentDto renameDirectoryDto = CreateFileFromContentDto.builder() + .loginUser(loginUser) + .fileAbsolutePath("/tmp/dolphinscheduler/default/resources/..") + .fileContent("select * from t") + .build(); + assertThrowServiceException( + "Internal Server Error: Invalidated resource path: /tmp/dolphinscheduler/default/resources/..", + () -> createFileFromContentDtoValidator.validate(renameDirectoryDto)); + } + + @Test + public void testValidate_IsNotFile() { + CreateFileFromContentDto createFileFromContentDto = CreateFileFromContentDto.builder() + .loginUser(loginUser) + .fileAbsolutePath("/tmp/dolphinscheduler/default/resources/a") + .fileContent("select * from t") + .build(); + assertThrowServiceException( + "Internal Server Error: The path is not a file: /tmp/dolphinscheduler/default/resources/a", + () -> createFileFromContentDtoValidator.validate(createFileFromContentDto)); + } + + @Test + public void testValidate_fileAlreadyExist() { + CreateFileFromContentDto createFileFromContentDto = CreateFileFromContentDto.builder() + .loginUser(loginUser) + .fileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.sql") + .fileContent("select * from t") + .build(); + when(storageOperator.exists(createFileFromContentDto.getFileAbsolutePath())).thenReturn(true); + assertThrowServiceException( + "Internal Server Error: The resource is already exist: /tmp/dolphinscheduler/default/resources/a.sql", + () -> createFileFromContentDtoValidator.validate(createFileFromContentDto)); + } + + @Test + public void testValidate_fileNoPermission() { + Tenant tenant = new Tenant(); + tenant.setTenantCode("test"); + when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant)); + + CreateFileFromContentDto createFileFromContentDto = CreateFileFromContentDto.builder() + .loginUser(loginUser) + .fileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.sql") + .fileContent("select * from t") + .build(); + when(storageOperator.exists(createFileFromContentDto.getFileAbsolutePath())).thenReturn(false); + when(storageOperator.getResourceMetaData(createFileFromContentDto.getFileAbsolutePath())) + .thenReturn(ResourceMetadata.builder() + .resourceAbsolutePath(createFileFromContentDto.getFileAbsolutePath()) + .resourceBaseDirectory(BASE_DIRECTORY) + .resourceRelativePath("a.sql") + .isDirectory(false) + .tenant("default") + .build()); + assertThrowServiceException( + "Internal Server Error: The user's tenant is test have no permission to access the resource: /tmp/dolphinscheduler/default/resources/a.sql", + () -> createFileFromContentDtoValidator.validate(createFileFromContentDto)); + } + + @Test + public void testValidate_contentIsInvalidated() { + Tenant tenant = new Tenant(); + tenant.setTenantCode("default"); + when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant)); + + CreateFileFromContentDto createFileFromContentDto = CreateFileFromContentDto.builder() + .loginUser(loginUser) + .fileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.sql") + .fileContent("") + .build(); + when(storageOperator.exists(createFileFromContentDto.getFileAbsolutePath())).thenReturn(false); + when(storageOperator.getResourceMetaData(createFileFromContentDto.getFileAbsolutePath())) + .thenReturn(ResourceMetadata.builder() + .resourceAbsolutePath(createFileFromContentDto.getFileAbsolutePath()) + .resourceBaseDirectory(BASE_DIRECTORY) + .resourceRelativePath("a.sql") + .isDirectory(false) + .tenant("default") + .build()); + assertThrowServiceException( + "Internal Server Error: The file content is null", + () -> createFileFromContentDtoValidator.validate(createFileFromContentDto)); + } + + @Test + public void testValidate() { + Tenant tenant = new Tenant(); + tenant.setTenantCode("default"); + when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant)); + + CreateFileFromContentDto createFileFromContentDto = CreateFileFromContentDto.builder() + .loginUser(loginUser) + .fileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.sql") + .fileContent("select * from t") + .build(); + when(storageOperator.exists(createFileFromContentDto.getFileAbsolutePath())).thenReturn(false); + when(storageOperator.getResourceMetaData(createFileFromContentDto.getFileAbsolutePath())) + .thenReturn(ResourceMetadata.builder() + .resourceAbsolutePath(createFileFromContentDto.getFileAbsolutePath()) + .resourceBaseDirectory(BASE_DIRECTORY) + .resourceRelativePath("a.sql") + .isDirectory(false) + .tenant("default") + .build()); + assertDoesNotThrow(() -> createFileFromContentDtoValidator.validate(createFileFromContentDto)); + } +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/FetchFileContentDtoValidatorTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/FetchFileContentDtoValidatorTest.java new file mode 100644 index 0000000000..69c31f2a0e --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/FetchFileContentDtoValidatorTest.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.validator.resource; + +import static org.apache.dolphinscheduler.api.AssertionsHelper.assertThrowServiceException; +import static org.mockito.Mockito.when; + +import org.apache.dolphinscheduler.api.AssertionsHelper; +import org.apache.dolphinscheduler.api.dto.resources.FetchFileContentDto; +import org.apache.dolphinscheduler.dao.entity.Tenant; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.repository.TenantDao; +import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; + +import java.util.Locale; +import java.util.Optional; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; +import org.springframework.context.i18n.LocaleContextHolder; + +@MockitoSettings(strictness = Strictness.LENIENT) +@ExtendWith(MockitoExtension.class) +class FetchFileContentDtoValidatorTest { + + @Mock + private StorageOperator storageOperator; + + @Mock + private TenantDao tenantDao; + + @InjectMocks + private FetchFileContentDtoValidator fetchFileContentDtoValidator; + + private static final String BASE_DIRECTORY = "/tmp/dolphinscheduler"; + + private User loginUser; + + @BeforeEach + public void setup() { + when(storageOperator.getStorageBaseDirectory()).thenReturn(BASE_DIRECTORY); + loginUser = new User(); + loginUser.setTenantId(1); + LocaleContextHolder.setLocale(Locale.ENGLISH); + } + + @Test + void testValidate_skipLineNumInvalid() { + FetchFileContentDto fetchFileContentDto = FetchFileContentDto.builder() + .loginUser(loginUser) + .resourceFileAbsolutePath("/tmp") + .skipLineNum(-1) + .limit(-1) + .build(); + assertThrowServiceException( + "Internal Server Error: skipLineNum must be greater than or equal to 0", + () -> fetchFileContentDtoValidator.validate(fetchFileContentDto)); + + } + + @Test + void testValidate_notUnderBaseDirectory() { + FetchFileContentDto fetchFileContentDto = FetchFileContentDto.builder() + .loginUser(loginUser) + .resourceFileAbsolutePath("/tmp") + .skipLineNum(0) + .limit(-1) + .build(); + assertThrowServiceException( + "Internal Server Error: Invalidated resource path: /tmp", + () -> fetchFileContentDtoValidator.validate(fetchFileContentDto)); + + } + + @Test + public void testValidate_filePathContainsIllegalSymbolic() { + FetchFileContentDto fetchFileContentDto = FetchFileContentDto.builder() + .loginUser(loginUser) + .resourceFileAbsolutePath("/tmp/dolphinscheduler/default/resources/..") + .skipLineNum(0) + .limit(-1) + .build(); + assertThrowServiceException( + "Internal Server Error: Invalidated resource path: /tmp/dolphinscheduler/default/resources/..", + () -> fetchFileContentDtoValidator.validate(fetchFileContentDto)); + } + + @Test + public void testValidate_IsNotFile() { + FetchFileContentDto fetchFileContentDto = FetchFileContentDto.builder() + .loginUser(loginUser) + .resourceFileAbsolutePath("/tmp/dolphinscheduler/default/resources/a") + .skipLineNum(0) + .limit(-1) + .build(); + assertThrowServiceException( + "Internal Server Error: The path is not a file: /tmp/dolphinscheduler/default/resources/a", + () -> fetchFileContentDtoValidator.validate(fetchFileContentDto)); + } + + @Test + public void testValidate_fileNoPermission() { + Tenant tenant = new Tenant(); + tenant.setTenantCode("test"); + when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant)); + + FetchFileContentDto fetchFileContentDto = FetchFileContentDto.builder() + .loginUser(loginUser) + .resourceFileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.sql") + .skipLineNum(0) + .limit(-1) + .build(); + when(storageOperator.exists(fetchFileContentDto.getResourceFileAbsolutePath())).thenReturn(false); + when(storageOperator.getResourceMetaData(fetchFileContentDto.getResourceFileAbsolutePath())) + .thenReturn(ResourceMetadata.builder() + .resourceAbsolutePath(fetchFileContentDto.getResourceFileAbsolutePath()) + .resourceBaseDirectory(BASE_DIRECTORY) + .resourceRelativePath("a.sql") + .isDirectory(false) + .tenant("default") + .build()); + assertThrowServiceException( + "Internal Server Error: The user's tenant is test have no permission to access the resource: /tmp/dolphinscheduler/default/resources/a.sql", + () -> fetchFileContentDtoValidator.validate(fetchFileContentDto)); + } + + @Test + void validate_fileExtensionInvalid() { + Tenant tenant = new Tenant(); + tenant.setTenantCode("default"); + when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant)); + + FetchFileContentDto fetchFileContentDto = FetchFileContentDto.builder() + .loginUser(loginUser) + .resourceFileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.jar") + .skipLineNum(0) + .limit(-1) + .build(); + when(storageOperator.exists(fetchFileContentDto.getResourceFileAbsolutePath())).thenReturn(false); + when(storageOperator.getResourceMetaData(fetchFileContentDto.getResourceFileAbsolutePath())) + .thenReturn(ResourceMetadata.builder() + .resourceAbsolutePath(fetchFileContentDto.getResourceFileAbsolutePath()) + .resourceBaseDirectory(BASE_DIRECTORY) + .resourceRelativePath("a.jar") + .isDirectory(false) + .tenant("default") + .build()); + assertThrowServiceException( + "Internal Server Error: The file type: jar cannot be fetched", + () -> fetchFileContentDtoValidator.validate(fetchFileContentDto)); + } + + @Test + void validate() { + Tenant tenant = new Tenant(); + tenant.setTenantCode("default"); + when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant)); + + FetchFileContentDto fetchFileContentDto = FetchFileContentDto.builder() + .loginUser(loginUser) + .resourceFileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.sql") + .skipLineNum(0) + .limit(-1) + .build(); + when(storageOperator.exists(fetchFileContentDto.getResourceFileAbsolutePath())).thenReturn(false); + when(storageOperator.getResourceMetaData(fetchFileContentDto.getResourceFileAbsolutePath())) + .thenReturn(ResourceMetadata.builder() + .resourceAbsolutePath(fetchFileContentDto.getResourceFileAbsolutePath()) + .resourceBaseDirectory(BASE_DIRECTORY) + .resourceRelativePath("a.sql") + .isDirectory(false) + .tenant("default") + .build()); + AssertionsHelper.assertDoesNotThrow(() -> fetchFileContentDtoValidator.validate(fetchFileContentDto)); + } +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/RenameDirectoryDtoValidatorTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/RenameDirectoryDtoValidatorTest.java new file mode 100644 index 0000000000..43e73f3b60 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/RenameDirectoryDtoValidatorTest.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.validator.resource; + +import static org.apache.dolphinscheduler.api.AssertionsHelper.assertDoesNotThrow; +import static org.apache.dolphinscheduler.api.AssertionsHelper.assertThrowServiceException; +import static org.mockito.Mockito.when; + +import org.apache.dolphinscheduler.api.dto.resources.RenameDirectoryDto; +import org.apache.dolphinscheduler.dao.entity.Tenant; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.repository.TenantDao; +import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; + +import java.util.Locale; +import java.util.Optional; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.context.i18n.LocaleContextHolder; + +@ExtendWith(MockitoExtension.class) +class RenameDirectoryDtoValidatorTest { + + @Mock + private StorageOperator storageOperator; + + @Mock + private TenantDao tenantDao; + + @InjectMocks + private RenameDirectoryDtoValidator renameDirectoryDtoValidator; + + private static final String BASE_DIRECTORY = "/tmp/dolphinscheduler"; + + private User loginUser; + + @BeforeEach + public void setup() { + when(storageOperator.getStorageBaseDirectory()).thenReturn(BASE_DIRECTORY); + loginUser = new User(); + loginUser.setTenantId(1); + LocaleContextHolder.setLocale(Locale.ENGLISH); + } + + @Test + void testValidate_notUnderBaseDirectory() { + RenameDirectoryDto renameDirectoryDto = RenameDirectoryDto.builder() + .loginUser(loginUser) + .originDirectoryAbsolutePath("/tmp") + .targetDirectoryAbsolutePath("/tmp1") + .build(); + assertThrowServiceException( + "Internal Server Error: Invalidated resource path: /tmp", + () -> renameDirectoryDtoValidator.validate(renameDirectoryDto)); + + } + + @Test + public void testValidate_directoryPathContainsIllegalSymbolic() { + RenameDirectoryDto renameDirectoryDto = RenameDirectoryDto.builder() + .loginUser(loginUser) + .originDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/..") + .targetDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/a") + .build(); + assertThrowServiceException( + "Internal Server Error: Invalidated resource path: /tmp/dolphinscheduler/default/resources/..", + () -> renameDirectoryDtoValidator.validate(renameDirectoryDto)); + } + + @Test + public void testValidate_originDirectoryNotExist() { + RenameDirectoryDto renameDirectoryDto = RenameDirectoryDto.builder() + .loginUser(loginUser) + .originDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/a") + .targetDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/b") + .build(); + assertThrowServiceException( + "Internal Server Error: Thr resource is not exists: /tmp/dolphinscheduler/default/resources/a", + () -> renameDirectoryDtoValidator.validate(renameDirectoryDto)); + } + + @Test + public void testValidate_originDirectoryNoPermission() { + Tenant tenant = new Tenant(); + tenant.setTenantCode("test"); + when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant)); + + RenameDirectoryDto renameDirectoryDto = RenameDirectoryDto.builder() + .loginUser(loginUser) + .originDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/a") + .targetDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/b") + .build(); + when(storageOperator.exists(renameDirectoryDto.getOriginDirectoryAbsolutePath())).thenReturn(true); + when(storageOperator.getResourceMetaData(renameDirectoryDto.getOriginDirectoryAbsolutePath())) + .thenReturn(ResourceMetadata.builder() + .resourceAbsolutePath(renameDirectoryDto.getOriginDirectoryAbsolutePath()) + .resourceBaseDirectory(BASE_DIRECTORY) + .resourceRelativePath("a") + .isDirectory(true) + .tenant("default") + .build()); + assertThrowServiceException( + "Internal Server Error: The user's tenant is test have no permission to access the resource: /tmp/dolphinscheduler/default/resources/a", + () -> renameDirectoryDtoValidator.validate(renameDirectoryDto)); + } + + @Test + public void testValidate_targetDirectoryAlreadyExist() { + Tenant tenant = new Tenant(); + tenant.setTenantCode("default"); + when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant)); + + RenameDirectoryDto renameDirectoryDto = RenameDirectoryDto.builder() + .loginUser(loginUser) + .originDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/a") + .targetDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/b") + .build(); + when(storageOperator.exists(renameDirectoryDto.getOriginDirectoryAbsolutePath())).thenReturn(true); + when(storageOperator.getResourceMetaData(renameDirectoryDto.getOriginDirectoryAbsolutePath())) + .thenReturn(ResourceMetadata.builder() + .resourceAbsolutePath(renameDirectoryDto.getOriginDirectoryAbsolutePath()) + .resourceBaseDirectory(BASE_DIRECTORY) + .resourceRelativePath("a") + .isDirectory(true) + .tenant("default") + .build()); + + when(storageOperator.exists(renameDirectoryDto.getTargetDirectoryAbsolutePath())).thenReturn(true); + assertThrowServiceException( + "Internal Server Error: The resource is already exist: /tmp/dolphinscheduler/default/resources/b", + () -> renameDirectoryDtoValidator.validate(renameDirectoryDto)); + } + + @Test + public void testValidate() { + Tenant tenant = new Tenant(); + tenant.setTenantCode("default"); + when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant)); + + RenameDirectoryDto renameDirectoryDto = RenameDirectoryDto.builder() + .loginUser(loginUser) + .originDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/a") + .targetDirectoryAbsolutePath("/tmp/dolphinscheduler/default/resources/b") + .build(); + when(storageOperator.exists(renameDirectoryDto.getOriginDirectoryAbsolutePath())).thenReturn(true); + when(storageOperator.getResourceMetaData(renameDirectoryDto.getOriginDirectoryAbsolutePath())) + .thenReturn(ResourceMetadata.builder() + .resourceAbsolutePath(renameDirectoryDto.getOriginDirectoryAbsolutePath()) + .resourceBaseDirectory(BASE_DIRECTORY) + .resourceRelativePath("a") + .isDirectory(true) + .tenant("default") + .build()); + + when(storageOperator.exists(renameDirectoryDto.getTargetDirectoryAbsolutePath())).thenReturn(false); + when(storageOperator.getResourceMetaData(renameDirectoryDto.getTargetDirectoryAbsolutePath())) + .thenReturn(ResourceMetadata.builder() + .resourceAbsolutePath(renameDirectoryDto.getOriginDirectoryAbsolutePath()) + .resourceBaseDirectory(BASE_DIRECTORY) + .resourceRelativePath("b") + .isDirectory(true) + .tenant("default") + .build()); + + assertDoesNotThrow(() -> renameDirectoryDtoValidator.validate(renameDirectoryDto)); + } +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/RenameFileDtoValidatorTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/RenameFileDtoValidatorTest.java new file mode 100644 index 0000000000..79fcd8c124 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/validator/resource/RenameFileDtoValidatorTest.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.validator.resource; + +import static org.apache.dolphinscheduler.api.AssertionsHelper.assertThrowServiceException; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.mockito.Mockito.when; + +import org.apache.dolphinscheduler.api.dto.resources.RenameFileDto; +import org.apache.dolphinscheduler.dao.entity.Tenant; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.repository.TenantDao; +import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; + +import java.util.Locale; +import java.util.Optional; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.context.i18n.LocaleContextHolder; + +@ExtendWith(MockitoExtension.class) +public class RenameFileDtoValidatorTest { + + @Mock + private StorageOperator storageOperator; + + @Mock + private TenantDao tenantDao; + + @InjectMocks + private RenameFileDtoValidator renameFileDtoValidator; + + private static final String BASE_DIRECTORY = "/tmp/dolphinscheduler"; + + private User loginUser; + + @BeforeEach + public void setup() { + when(storageOperator.getStorageBaseDirectory()).thenReturn(BASE_DIRECTORY); + loginUser = new User(); + loginUser.setTenantId(1); + LocaleContextHolder.setLocale(Locale.ENGLISH); + } + + @Test + void testValidate_notUnderBaseDirectory() { + RenameFileDto renameFileDto = RenameFileDto.builder() + .loginUser(loginUser) + .originFileAbsolutePath("/tmp") + .targetFileAbsolutePath("/tmp1") + .build(); + assertThrowServiceException( + "Internal Server Error: Invalidated resource path: /tmp", + () -> renameFileDtoValidator.validate(renameFileDto)); + + } + + @Test + public void testValidate_fileAbsolutePathContainsIllegalSymbolic() { + RenameFileDto renameFileDto = RenameFileDto.builder() + .loginUser(loginUser) + .originFileAbsolutePath("/tmp/dolphinscheduler/default/resources/../a.txt") + .targetFileAbsolutePath("/tmp/dolphinscheduler/default/resources/b.txt") + .build(); + assertThrowServiceException( + "Internal Server Error: Invalidated resource path: /tmp/dolphinscheduler/default/resources/../a.txt", + () -> renameFileDtoValidator.validate(renameFileDto)); + } + + @Test + public void testValidate_originFileNotExist() { + RenameFileDto renameFileDto = RenameFileDto.builder() + .loginUser(loginUser) + .originFileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.txt") + .targetFileAbsolutePath("/tmp/dolphinscheduler/default/resources/b.txt") + .build(); + assertThrowServiceException( + "Internal Server Error: Thr resource is not exists: /tmp/dolphinscheduler/default/resources/a.txt", + () -> renameFileDtoValidator.validate(renameFileDto)); + } + + @Test + public void testValidate_originFileIsNotFile() { + RenameFileDto renameFileDto = RenameFileDto.builder() + .loginUser(loginUser) + .originFileAbsolutePath("/tmp/dolphinscheduler/default/resources/a") + .targetFileAbsolutePath("/tmp/dolphinscheduler/default/resources/b.txt") + .build(); + when(storageOperator.exists(renameFileDto.getOriginFileAbsolutePath())).thenReturn(true); + assertThrowServiceException( + "Internal Server Error: The path is not a file: /tmp/dolphinscheduler/default/resources/a", + () -> renameFileDtoValidator.validate(renameFileDto)); + } + + @Test + public void testValidate_originFileNoPermission() { + Tenant tenant = new Tenant(); + tenant.setTenantCode("test"); + when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant)); + + RenameFileDto renameFileDto = RenameFileDto.builder() + .loginUser(loginUser) + .originFileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.txt") + .targetFileAbsolutePath("/tmp/dolphinscheduler/default/resources/b.txt") + .build(); + when(storageOperator.exists(renameFileDto.getOriginFileAbsolutePath())).thenReturn(true); + when(storageOperator.getResourceMetaData(renameFileDto.getOriginFileAbsolutePath())) + .thenReturn(ResourceMetadata.builder() + .resourceAbsolutePath(renameFileDto.getOriginFileAbsolutePath()) + .resourceBaseDirectory(BASE_DIRECTORY) + .resourceRelativePath("a.txt") + .isDirectory(false) + .tenant("default") + .build()); + assertThrowServiceException( + "Internal Server Error: The user's tenant is test have no permission to access the resource: /tmp/dolphinscheduler/default/resources/a.txt", + () -> renameFileDtoValidator.validate(renameFileDto)); + } + + @Test + public void testValidate_targetFileAlreadyExist() { + Tenant tenant = new Tenant(); + tenant.setTenantCode("default"); + when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant)); + + RenameFileDto renameDirectoryDto = RenameFileDto.builder() + .loginUser(loginUser) + .originFileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.txt") + .targetFileAbsolutePath("/tmp/dolphinscheduler/default/resources/b.txt") + .build(); + when(storageOperator.exists(renameDirectoryDto.getOriginFileAbsolutePath())).thenReturn(true); + when(storageOperator.getResourceMetaData(renameDirectoryDto.getOriginFileAbsolutePath())) + .thenReturn(ResourceMetadata.builder() + .resourceAbsolutePath(renameDirectoryDto.getOriginFileAbsolutePath()) + .resourceBaseDirectory(BASE_DIRECTORY) + .resourceRelativePath("a.txt") + .isDirectory(false) + .tenant("default") + .build()); + + when(storageOperator.exists(renameDirectoryDto.getTargetFileAbsolutePath())).thenReturn(true); + assertThrowServiceException( + "Internal Server Error: The resource is already exist: /tmp/dolphinscheduler/default/resources/b.txt", + () -> renameFileDtoValidator.validate(renameDirectoryDto)); + } + + @Test + public void testValidate() { + Tenant tenant = new Tenant(); + tenant.setTenantCode("default"); + when(tenantDao.queryOptionalById(loginUser.getTenantId())).thenReturn(Optional.of(tenant)); + + RenameFileDto renameDirectoryDto = RenameFileDto.builder() + .loginUser(loginUser) + .originFileAbsolutePath("/tmp/dolphinscheduler/default/resources/a.txt") + .targetFileAbsolutePath("/tmp/dolphinscheduler/default/resources/b.txt") + .build(); + when(storageOperator.exists(renameDirectoryDto.getOriginFileAbsolutePath())).thenReturn(true); + when(storageOperator.getResourceMetaData(renameDirectoryDto.getOriginFileAbsolutePath())) + .thenReturn(ResourceMetadata.builder() + .resourceAbsolutePath(renameDirectoryDto.getOriginFileAbsolutePath()) + .resourceBaseDirectory(BASE_DIRECTORY) + .resourceRelativePath("a.txt") + .isDirectory(false) + .tenant("default") + .build()); + + when(storageOperator.exists(renameDirectoryDto.getTargetFileAbsolutePath())).thenReturn(false); + when(storageOperator.getResourceMetaData(renameDirectoryDto.getTargetFileAbsolutePath())) + .thenReturn(ResourceMetadata.builder() + .resourceAbsolutePath(renameDirectoryDto.getTargetFileAbsolutePath()) + .resourceBaseDirectory(BASE_DIRECTORY) + .resourceRelativePath("b.txt") + .isDirectory(false) + .tenant("default") + .build()); + + assertDoesNotThrow(() -> renameFileDtoValidator.validate(renameDirectoryDto)); + } + +} diff --git a/dolphinscheduler-bom/pom.xml b/dolphinscheduler-bom/pom.xml index 10c4f0e4a1..8c59d1c4ca 100644 --- a/dolphinscheduler-bom/pom.xml +++ b/dolphinscheduler-bom/pom.xml @@ -965,6 +965,13 @@ test + + org.testcontainers + minio + ${testcontainer.version} + test + + org.checkerframework checker-qual diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutableYamlDelegate.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutableYamlDelegate.java index 5806a20fd7..6f4bb34eab 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutableYamlDelegate.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/config/ImmutableYamlDelegate.java @@ -43,6 +43,10 @@ public class ImmutableYamlDelegate implements IPropertyDelegate { // read from classpath for (String fileName : yamlAbsolutePath) { try (InputStream fis = ImmutableYamlDelegate.class.getResourceAsStream(fileName)) { + if (fis == null) { + log.warn("Cannot find the file: {} under classpath", fileName); + continue; + } YamlPropertiesFactoryBean factory = new YamlPropertiesFactoryBean(); factory.setResources(new InputStreamResource(fis)); factory.afterPropertiesSet(); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java index 5ec2a2b181..d13fc42163 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java @@ -59,28 +59,6 @@ public final class Constants { */ public static final String HDFS_DEFAULT_FS = "fs.defaultFS"; - /** - * hadoop configuration - */ - public static final String HADOOP_RM_STATE_ACTIVE = "ACTIVE"; - - public static final String HADOOP_RESOURCE_MANAGER_HTTPADDRESS_PORT = "resource.manager.httpaddress.port"; - - /** - * yarn.resourcemanager.ha.rm.ids - */ - public static final String YARN_RESOURCEMANAGER_HA_RM_IDS = "yarn.resourcemanager.ha.rm.ids"; - - /** - * yarn.application.status.address - */ - public static final String YARN_APPLICATION_STATUS_ADDRESS = "yarn.application.status.address"; - - /** - * yarn.job.history.status.address - */ - public static final String YARN_JOB_HISTORY_STATUS_ADDRESS = "yarn.job.history.status.address"; - /** * hdfs configuration * resource.hdfs.root.user @@ -392,8 +370,6 @@ public final class Constants { public static final String QUEUE_NAME = "queueName"; public static final int LOG_QUERY_SKIP_LINE_NUMBER = 0; public static final int LOG_QUERY_LIMIT = 4096; - public static final String ALIAS = "alias"; - public static final String CONTENT = "content"; public static final String DEPENDENT_SPLIT = ":||"; public static final long DEPENDENT_ALL_TASK_CODE = -1; public static final long DEPENDENT_WORKFLOW_CODE = 0; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ResUploadType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StorageType.java similarity index 97% rename from dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ResUploadType.java rename to dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StorageType.java index d60a657002..ba3cad45cd 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/ResUploadType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/StorageType.java @@ -20,6 +20,6 @@ package org.apache.dolphinscheduler.common.enums; /** * data base types */ -public enum ResUploadType { +public enum StorageType { LOCAL, HDFS, S3, OSS, GCS, ABS, OBS } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java index 60629576d9..31765b19bd 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java @@ -17,14 +17,15 @@ package org.apache.dolphinscheduler.common.utils; +import static com.google.common.base.Preconditions.checkNotNull; import static org.apache.dolphinscheduler.common.constants.Constants.DATA_BASEDIR_PATH; import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPARATOR; import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S; import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_VIEW_SUFFIXES; import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_VIEW_SUFFIXES_DEFAULT_VALUE; -import static org.apache.dolphinscheduler.common.constants.DateConstants.YYYYMMDDHHMMSS; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.SystemUtils; import java.io.ByteArrayOutputStream; @@ -33,17 +34,21 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.PosixFilePermissions; +import java.util.Optional; import java.util.Set; import java.util.zip.CRC32; import java.util.zip.CheckedInputStream; import lombok.NonNull; +import lombok.SneakyThrows; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; @@ -66,32 +71,14 @@ public class FileUtils { * @return download file name */ public static String getDownloadFilename(String filename) { - String fileName = - String.format("%s/download/%s/%s", DATA_BASEDIR, DateUtils.getCurrentTime(YYYYMMDDHHMMSS), filename); - - File file = new File(fileName); - if (!file.getParentFile().exists()) { - file.getParentFile().mkdirs(); - } - - return fileName; + return Paths.get(DATA_BASEDIR, "tmp", CodeGenerateUtils.genCode() + "-" + filename).toString(); } /** - * get upload file absolute path and name - * - * @param tenantCode tenant code - * @param filename file name - * @return local file path + * Generate a local tmp absolute path of the uploaded file */ - public static String getUploadFilename(String tenantCode, String filename) { - String fileName = String.format("%s/%s/resources/%s", DATA_BASEDIR, tenantCode, filename); - File file = new File(fileName); - if (!file.getParentFile().exists()) { - file.getParentFile().mkdirs(); - } - - return fileName; + public static String getUploadFileLocalTmpAbsolutePath() { + return Paths.get(DATA_BASEDIR, "tmp", String.valueOf(CodeGenerateUtils.genCode())).toString(); } /** @@ -135,7 +122,7 @@ public class FileUtils { /** * absolute path of appInfo file * - * @param execPath directory of process execution + * @param execPath directory of process execution * @return */ public static String getAppInfoPath(String execPath) { @@ -152,7 +139,7 @@ public class FileUtils { /** * write content to file ,if parent path not exists, it will do one's utmost to mkdir * - * @param content content + * @param content content * @param filePath target file path * @return true if write success */ @@ -236,6 +223,7 @@ public class FileUtils { /** * Calculate file checksum with CRC32 algorithm + * * @param pathName * @return checksum of file/dir */ @@ -315,4 +303,45 @@ public class FileUtils { } } + public static String concatFilePath(String... paths) { + if (paths.length == 0) { + throw new IllegalArgumentException("At least one path should be provided"); + } + StringBuilder finalPath = new StringBuilder(paths[0]); + if (StringUtils.isEmpty(finalPath)) { + throw new IllegalArgumentException("The path should not be empty"); + } + String separator = File.separator; + for (int i = 1; i < paths.length; i++) { + String path = paths[i]; + if (StringUtils.isEmpty(path)) { + throw new IllegalArgumentException("The path should not be empty"); + } + if (finalPath.toString().endsWith(separator) && path.startsWith(separator)) { + finalPath.append(path.substring(separator.length())); + continue; + } + if (!finalPath.toString().endsWith(separator) && !path.startsWith(separator)) { + finalPath.append(separator).append(path); + continue; + } + finalPath.append(path); + } + return finalPath.toString(); + } + + public static String getClassPathAbsolutePath(Class clazz) { + checkNotNull(clazz, "class is null"); + return Optional.ofNullable(clazz.getResource("/")) + .map(URL::getPath) + .orElseThrow(() -> new IllegalArgumentException("class path: " + clazz + " is null")); + } + + /** + * copy input stream to file, if the file already exists, will append the content to the beginning of the file, otherwise will create a new file. + */ + @SneakyThrows + public static void copyInputStreamToFile(InputStream inputStream, String destFilename) { + org.apache.commons.io.FileUtils.copyInputStreamToFile(inputStream, new File(destFilename)); + } } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java index f06ece2a56..72ceed630f 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.common.utils; -import static org.apache.dolphinscheduler.common.constants.DateConstants.YYYYMMDDHHMMSS; - import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -31,26 +29,21 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.MockedStatic; -import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; +import com.google.common.truth.Truth; + @ExtendWith(MockitoExtension.class) public class FileUtilsTest { @Test public void testGetDownloadFilename() { - try (MockedStatic mockedDateUtils = Mockito.mockStatic(DateUtils.class)) { - mockedDateUtils.when(() -> DateUtils.getCurrentTime(YYYYMMDDHHMMSS)).thenReturn("20190101101059"); - Assertions.assertEquals("/tmp/dolphinscheduler/download/20190101101059/test", - FileUtils.getDownloadFilename("test")); - } + Truth.assertThat(FileUtils.getDownloadFilename("test")).startsWith("/tmp/dolphinscheduler/tmp/"); } @Test public void testGetUploadFilename() { - Assertions.assertEquals("/tmp/dolphinscheduler/aaa/resources/bbb", - FileUtils.getUploadFilename("aaa", "bbb")); + Truth.assertThat(FileUtils.getUploadFileLocalTmpAbsolutePath()).startsWith("/tmp/dolphinscheduler/tmp/"); } @Test diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/TenantDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/TenantDao.java new file mode 100644 index 0000000000..9f48f372da --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/TenantDao.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.repository; + +import org.apache.dolphinscheduler.dao.entity.Tenant; + +public interface TenantDao extends IDao { + +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/TenantDaoImpl.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/TenantDaoImpl.java new file mode 100644 index 0000000000..45f9a5fcba --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/repository/impl/TenantDaoImpl.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.repository.impl; + +import org.apache.dolphinscheduler.dao.entity.Tenant; +import org.apache.dolphinscheduler.dao.mapper.TenantMapper; +import org.apache.dolphinscheduler.dao.repository.BaseDao; +import org.apache.dolphinscheduler.dao.repository.TenantDao; + +import lombok.NonNull; + +import org.springframework.stereotype.Repository; + +@Repository +public class TenantDaoImpl extends BaseDao implements TenantDao { + + public TenantDaoImpl(@NonNull TenantMapper tenantMapper) { + super(tenantMapper); + } + +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskCacheUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskCacheUtils.java index 36cf6c2356..90b648386b 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskCacheUtils.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskCacheUtils.java @@ -22,7 +22,7 @@ import static org.apache.dolphinscheduler.common.constants.Constants.CRC_SUFFIX; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; @@ -65,17 +65,17 @@ public class TaskCacheUtils { * 4. input VarPool, from upstream task and workflow global parameters * @param taskInstance task instance * @param taskExecutionContext taskExecutionContext - * @param storageOperate storageOperate + * @param storageOperator storageOperate * @return cache key */ public static String generateCacheKey(TaskInstance taskInstance, TaskExecutionContext taskExecutionContext, - StorageOperate storageOperate) { + StorageOperator storageOperator) { List keyElements = new ArrayList<>(); keyElements.add(String.valueOf(taskInstance.getTaskCode())); keyElements.add(String.valueOf(taskInstance.getTaskDefinitionVersion())); keyElements.add(String.valueOf(taskInstance.getIsCache().getCode())); keyElements.add(String.valueOf(taskInstance.getEnvironmentConfig())); - keyElements.add(getTaskInputVarPoolData(taskInstance, taskExecutionContext, storageOperate)); + keyElements.add(getTaskInputVarPoolData(taskInstance, taskExecutionContext, storageOperator)); String data = StringUtils.join(keyElements, "_"); return DigestUtils.sha256Hex(data); } @@ -123,7 +123,7 @@ public class TaskCacheUtils { * taskExecutionContext taskExecutionContext */ public static String getTaskInputVarPoolData(TaskInstance taskInstance, TaskExecutionContext context, - StorageOperate storageOperate) { + StorageOperator storageOperator) { JsonNode taskParams = JSONUtils.parseObject(taskInstance.getTaskParams()); // The set of input values considered from localParams in the taskParams @@ -141,7 +141,8 @@ public class TaskCacheUtils { List fileInput = varPool.stream().filter(property -> property.getType().equals(DataType.FILE)) .collect(Collectors.toList()); fileInput.forEach( - property -> fileCheckSumMap.put(property.getProp(), getValCheckSum(property, context, storageOperate))); + property -> fileCheckSumMap.put(property.getProp(), + getValCheckSum(property, context, storageOperator))); // var pool value from workflow global parameters if (context.getPrepareParamsMap() != null) { @@ -173,17 +174,18 @@ public class TaskCacheUtils { * cache can be used if content of upstream output files are the same * @param fileProperty * @param context - * @param storageOperate + * @param storageOperator */ public static String getValCheckSum(Property fileProperty, TaskExecutionContext context, - StorageOperate storageOperate) { + StorageOperator storageOperator) { String resourceCRCPath = fileProperty.getValue() + CRC_SUFFIX; - String resourceCRCWholePath = storageOperate.getResourceFullName(context.getTenantCode(), resourceCRCPath); + String resourceCRCWholePath = + storageOperator.getStorageFileAbsolutePath(context.getTenantCode(), resourceCRCPath); String targetPath = String.format("%s/%s", context.getExecutePath(), resourceCRCPath); log.info("{} --- Remote:{} to Local:{}", "CRC file", resourceCRCWholePath, targetPath); String crcString = ""; try { - storageOperate.download(resourceCRCWholePath, targetPath, true); + storageOperator.download(resourceCRCWholePath, targetPath, true); crcString = FileUtils.readFile2Str(new FileInputStream(targetPath)); fileProperty.setValue(crcString); } catch (IOException e) { diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/TaskCacheUtilsTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/TaskCacheUtilsTest.java index 49d5a87bb6..ee88d1cd1a 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/TaskCacheUtilsTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/TaskCacheUtilsTest.java @@ -20,7 +20,7 @@ package org.apache.dolphinscheduler.dao.utils; import org.apache.dolphinscheduler.common.enums.Flag; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; @@ -46,7 +46,7 @@ class TaskCacheUtilsTest { private TaskExecutionContext taskExecutionContext; - private StorageOperate storageOperate; + private StorageOperator storageOperator; @BeforeEach void setUp() { @@ -101,7 +101,7 @@ class TaskCacheUtilsTest { prepareParamsMap.put("a", property); taskExecutionContext.setPrepareParamsMap(prepareParamsMap); - storageOperate = Mockito.mock(StorageOperate.class); + storageOperator = Mockito.mock(StorageOperator.class); } @Test @@ -128,26 +128,26 @@ class TaskCacheUtilsTest { @Test void TestGetTaskInputVarPoolData() { - TaskCacheUtils.getTaskInputVarPoolData(taskInstance, taskExecutionContext, storageOperate); + TaskCacheUtils.getTaskInputVarPoolData(taskInstance, taskExecutionContext, storageOperator); // only a=aa and c=cc will influence the result, // b=bb is a fixed value, will be considered in task version // k=kk is not in task params, will be ignored String except = "[{\"prop\":\"a\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"aa\"},{\"prop\":\"c\",\"direct\":\"IN\",\"type\":\"VARCHAR\",\"value\":\"cc\"}]"; Assertions.assertEquals(except, - TaskCacheUtils.getTaskInputVarPoolData(taskInstance, taskExecutionContext, storageOperate)); + TaskCacheUtils.getTaskInputVarPoolData(taskInstance, taskExecutionContext, storageOperator)); } @Test void TestGenerateCacheKey() { - String cacheKeyBase = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperate); + String cacheKeyBase = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperator); Property propertyI = new Property(); propertyI.setProp("i"); propertyI.setDirect(Direct.IN); propertyI.setType(DataType.VARCHAR); propertyI.setValue("ii"); taskExecutionContext.getPrepareParamsMap().put("i", propertyI); - String cacheKeyNew = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperate); + String cacheKeyNew = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperator); // i will not influence the result, because task instance not use it Assertions.assertEquals(cacheKeyBase, cacheKeyNew); @@ -157,17 +157,17 @@ class TaskCacheUtilsTest { propertyD.setType(DataType.VARCHAR); propertyD.setValue("dd"); taskExecutionContext.getPrepareParamsMap().put("i", propertyD); - String cacheKeyD = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperate); + String cacheKeyD = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperator); // d will influence the result, because task instance use it Assertions.assertNotEquals(cacheKeyBase, cacheKeyD); taskInstance.setTaskDefinitionVersion(100); - String cacheKeyE = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperate); + String cacheKeyE = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperator); // task definition version is changed, so cache key changed Assertions.assertNotEquals(cacheKeyD, cacheKeyE); taskInstance.setEnvironmentConfig("export PYTHON_LAUNCHER=/bin/python3"); - String cacheKeyF = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperate); + String cacheKeyF = TaskCacheUtils.generateCacheKey(taskInstance, taskExecutionContext, storageOperator); // EnvironmentConfig is changed, so cache key changed Assertions.assertNotEquals(cacheKeyE, cacheKeyF); } @@ -193,7 +193,7 @@ class TaskCacheUtilsTest { taskExecutionContext.setExecutePath("test"); taskExecutionContext.setTenantCode("aaa"); - String crc = TaskCacheUtils.getValCheckSum(property, taskExecutionContext, storageOperate); + String crc = TaskCacheUtils.getValCheckSum(property, taskExecutionContext, storageOperator); Assertions.assertEquals(crc, content); } diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java index 1c24785c2f..eb8f2ddef0 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-api/src/main/java/org/apache/dolphinscheduler/plugin/datasource/api/utils/CommonUtils.java @@ -29,7 +29,7 @@ import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.LOGIN_US import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.RESOURCE_UPLOAD_PATH; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.enums.ResUploadType; +import org.apache.dolphinscheduler.common.enums.StorageType; import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.apache.commons.lang3.StringUtils; @@ -72,9 +72,9 @@ public class CommonUtils { */ public static boolean getKerberosStartupState() { String resUploadStartupType = PropertyUtils.getUpperCaseString(RESOURCE_STORAGE_TYPE); - ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); + StorageType storageType = StorageType.valueOf(resUploadStartupType); Boolean kerberosStartupState = PropertyUtils.getBoolean(HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false); - return resUploadType == ResUploadType.HDFS && kerberosStartupState; + return storageType == StorageType.HDFS && kerberosStartupState; } /** diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/security/UserGroupInformationFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/security/UserGroupInformationFactory.java index 168ff3bdca..a39cdaf614 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/security/UserGroupInformationFactory.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-hive/src/main/java/org/apache/dolphinscheduler/plugin/datasource/hive/security/UserGroupInformationFactory.java @@ -20,7 +20,7 @@ package org.apache.dolphinscheduler.plugin.datasource.hive.security; import static org.apache.dolphinscheduler.common.constants.Constants.JAVA_SECURITY_KRB5_CONF; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.enums.ResUploadType; +import org.apache.dolphinscheduler.common.enums.StorageType; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils; @@ -120,10 +120,10 @@ public class UserGroupInformationFactory { public static boolean openKerberos() { String resUploadStartupType = PropertyUtils.getUpperCaseString(Constants.RESOURCE_STORAGE_TYPE); - ResUploadType resUploadType = ResUploadType.valueOf(resUploadStartupType); + StorageType storageType = StorageType.valueOf(resUploadStartupType); Boolean kerberosStartupState = PropertyUtils.getBoolean(Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false); - return resUploadType == ResUploadType.HDFS && kerberosStartupState; + return storageType == StorageType.HDFS && kerberosStartupState; } } diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/ResourceType.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/ResourceType.java index 16c49a1ecc..2dd7ca0671 100644 --- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/ResourceType.java +++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/ResourceType.java @@ -47,4 +47,13 @@ public enum ResourceType { public String getDescp() { return descp; } + + public static ResourceType getResourceType(int code) { + for (ResourceType resourceType : ResourceType.values()) { + if (resourceType.getCode() == code) { + return resourceType; + } + } + return null; + } } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperator.java index d470545bab..81b93ea207 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperator.java @@ -19,17 +19,12 @@ package org.apache.dolphinscheduler.plugin.storage.abs; import static org.apache.dolphinscheduler.common.constants.Constants.EMPTY_STRING; import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPARATOR; -import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S; -import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TYPE_FILE; -import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TYPE_UDF; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.enums.ResUploadType; import org.apache.dolphinscheduler.common.utils.FileUtils; -import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.plugin.storage.api.AbstractStorageOperator; import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; -import org.apache.dolphinscheduler.spi.enums.ResourceType; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.commons.lang3.StringUtils; @@ -37,136 +32,69 @@ import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.Closeable; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStreamReader; +import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.sql.Date; -import java.util.ArrayList; import java.util.Collections; -import java.util.LinkedList; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; -import lombok.Data; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import com.azure.core.http.rest.PagedIterable; import com.azure.storage.blob.BlobClient; import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.BlobServiceClient; import com.azure.storage.blob.BlobServiceClientBuilder; import com.azure.storage.blob.models.BlobContainerItem; -import com.azure.storage.blob.models.BlobItem; import com.azure.storage.blob.specialized.BlockBlobClient; -@Data @Slf4j -public class AbsStorageOperator implements Closeable, StorageOperate { +public class AbsStorageOperator extends AbstractStorageOperator implements Closeable, StorageOperator { - private BlobContainerClient blobContainerClient; + private final BlobContainerClient blobContainerClient; - private BlobServiceClient blobServiceClient; + private final BlobServiceClient blobServiceClient; - private String connectionString; - - private String storageAccountName; - - private String containerName; - - public AbsStorageOperator() { - - } - - public void init() { - containerName = readContainerName(); - connectionString = readConnectionString(); - storageAccountName = readAccountName(); - blobServiceClient = buildBlobServiceClient(); - blobContainerClient = buildBlobContainerClient(); - checkContainerNameExists(); - } - - protected BlobServiceClient buildBlobServiceClient() { - return new BlobServiceClientBuilder() - .endpoint("https://" + storageAccountName + ".blob.core.windows.net/") - .connectionString(connectionString) + public AbsStorageOperator(AbsStorageProperties absStorageProperties) { + super(absStorageProperties.getResourceUploadPath()); + blobServiceClient = new BlobServiceClientBuilder() + .endpoint("https://" + absStorageProperties.getStorageAccountName() + ".blob.core.windows.net/") + .connectionString(absStorageProperties.getConnectionString()) .buildClient(); - } - - protected BlobContainerClient buildBlobContainerClient() { - return blobServiceClient.getBlobContainerClient(containerName); - } - - protected String readConnectionString() { - return PropertyUtils.getString(Constants.AZURE_BLOB_STORAGE_CONNECTION_STRING); - } - - protected String readContainerName() { - return PropertyUtils.getString(Constants.AZURE_BLOB_STORAGE_CONTAINER_NAME); - } - - protected String readAccountName() { - return PropertyUtils.getString(Constants.AZURE_BLOB_STORAGE_ACCOUNT_NAME); - } - - @Override - public void createTenantDirIfNotExists(String tenantCode) throws Exception { - mkdir(tenantCode, getAbsResDir(tenantCode)); - mkdir(tenantCode, getAbsUdfDir(tenantCode)); - } - - public String getAbsResDir(String tenantCode) { - return String.format("%s/" + RESOURCE_TYPE_FILE, getAbsTenantDir(tenantCode)); - } - - public String getAbsUdfDir(String tenantCode) { - return String.format("%s/" + RESOURCE_TYPE_UDF, getAbsTenantDir(tenantCode)); - } - - public String getAbsTenantDir(String tenantCode) { - return String.format(FORMAT_S_S, getGcsDataBasePath(), tenantCode); - } - - public String getGcsDataBasePath() { - if (FOLDER_SEPARATOR.equals(RESOURCE_UPLOAD_PATH)) { - return EMPTY_STRING; - } else { - return RESOURCE_UPLOAD_PATH.replaceFirst(FOLDER_SEPARATOR, EMPTY_STRING); - } - } - - @Override - public String getResDir(String tenantCode) { - return getAbsResDir(tenantCode) + FOLDER_SEPARATOR; + blobContainerClient = blobServiceClient.getBlobContainerClient(absStorageProperties.getContainerName()); + checkContainerNameExists(absStorageProperties.getContainerName()); } @Override - public String getUdfDir(String tenantCode) { - return getAbsUdfDir(tenantCode) + FOLDER_SEPARATOR; - } - - @Override - public String getResourceFullName(String tenantCode, String fileName) { - if (fileName.startsWith(FOLDER_SEPARATOR)) { - fileName.replaceFirst(FOLDER_SEPARATOR, EMPTY_STRING); + public String getStorageBaseDirectory() { + // All directory should end with File.separator + if (getStorageBaseDirectory().startsWith("/")) { + log.warn("{} -> {} should not start with / in abs", Constants.RESOURCE_UPLOAD_PATH, + getStorageBaseDirectory()); + return getStorageBaseDirectory().substring(1); } - return String.format(FORMAT_S_S, getAbsResDir(tenantCode), fileName); + return getStorageBaseDirectory(); } + @SneakyThrows @Override - public String getFileName(ResourceType resourceType, String tenantCode, String fileName) { - if (fileName.startsWith(FOLDER_SEPARATOR)) { - fileName = fileName.replaceFirst(FOLDER_SEPARATOR, EMPTY_STRING); + public void createStorageDir(String directory) { + String objectName = directory + FOLDER_SEPARATOR; + if (isObjectExists(objectName)) { + throw new FileAlreadyExistsException("directory: " + objectName + " already exists"); } - return getDir(resourceType, tenantCode) + fileName; + BlobClient blobClient = blobContainerClient.getBlobClient(objectName); + blobClient.upload(new ByteArrayInputStream(EMPTY_STRING.getBytes()), 0); } + @SneakyThrows @Override - public void download(String srcFilePath, String dstFilePath, boolean overwrite) throws IOException { + public void download(String srcFilePath, String dstFilePath, boolean overwrite) { File dstFile = new File(dstFilePath); if (dstFile.isDirectory()) { Files.delete(dstFile.toPath()); @@ -179,7 +107,7 @@ public class AbsStorageOperator implements Closeable, StorageOperate { } @Override - public boolean exists(String fullName) throws IOException { + public boolean exists(String fullName) { return isObjectExists(fullName); } @@ -187,36 +115,14 @@ public class AbsStorageOperator implements Closeable, StorageOperate { return blobContainerClient.getBlobClient(objectName).exists(); } + @SneakyThrows @Override - public boolean delete(String filePath, boolean recursive) throws IOException { - try { - if (isObjectExists(filePath)) { - blobContainerClient.getBlobClient(filePath).delete(); - } - return true; - } catch (Exception e) { - log.error("delete the object error,the resource path is {}", filePath); - return false; - } + public void delete(String filePath, boolean recursive) { + blobContainerClient.getBlobClient(filePath).deleteIfExists(); } @Override - public boolean delete(String fullName, List childrenPathList, boolean recursive) throws IOException { - // append the resource fullName to the list for deletion. - childrenPathList.add(fullName); - - boolean result = true; - for (String filePath : childrenPathList) { - if (!delete(filePath, recursive)) { - result = false; - } - } - - return result; - } - - @Override - public boolean copy(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) throws IOException { + public void copy(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) { BlobClient srcBlobClient = blobContainerClient.getBlobClient(srcPath); BlockBlobClient dstBlobClient = blobContainerClient.getBlobClient(dstPath).getBlockBlobClient(); @@ -225,29 +131,23 @@ public class AbsStorageOperator implements Closeable, StorageOperate { if (deleteSource) { srcBlobClient.delete(); } - return true; } + @SneakyThrows @Override - public boolean upload(String tenantCode, String srcFile, String dstPath, boolean deleteSource, - boolean overwrite) throws IOException { - try { - BlobClient blobClient = blobContainerClient.getBlobClient(dstPath); - blobClient.uploadFromFile(srcFile, overwrite); - - Path srcPath = Paths.get(srcFile); - if (deleteSource) { - Files.delete(srcPath); - } - return true; - } catch (Exception e) { - log.error("upload failed,the container is {},the filePath is {}", containerName, dstPath); - return false; + public void upload(String srcFile, String dstPath, boolean deleteSource, boolean overwrite) { + BlobClient blobClient = blobContainerClient.getBlobClient(dstPath); + blobClient.uploadFromFile(srcFile, overwrite); + + Path srcPath = Paths.get(srcFile); + if (deleteSource) { + Files.delete(srcPath); } } + @SneakyThrows @Override - public List vimFile(String tenantCode, String filePath, int skipLineNums, int limit) throws IOException { + public List fetchFileContent(String filePath, int skipLineNums, int limit) { if (StringUtils.isBlank(filePath)) { log.error("file path:{} is blank", filePath); return Collections.emptyList(); @@ -263,199 +163,27 @@ public class AbsStorageOperator implements Closeable, StorageOperate { } } - @Override - public void deleteTenant(String tenantCode) throws Exception { - deleteTenantCode(tenantCode); - } - - protected void deleteTenantCode(String tenantCode) { - deleteDirectory(getResDir(tenantCode)); - deleteDirectory(getUdfDir(tenantCode)); - } - - @Override - public String getDir(ResourceType resourceType, String tenantCode) { - switch (resourceType) { - case UDF: - return getUdfDir(tenantCode); - case FILE: - return getResDir(tenantCode); - case ALL: - return getGcsDataBasePath(); - default: - return EMPTY_STRING; - } - - } - - protected void deleteDirectory(String directoryName) { - if (isObjectExists(directoryName)) { - blobContainerClient.getBlobClient(directoryName).delete(); - } - } - - @Override - public boolean mkdir(String tenantCode, String path) throws IOException { - String objectName = path + FOLDER_SEPARATOR; - if (!isObjectExists(objectName)) { - BlobClient blobClient = blobContainerClient.getBlobClient(objectName); - blobClient.upload(new ByteArrayInputStream(EMPTY_STRING.getBytes()), 0); - } - return true; - } - @Override public void close() throws IOException { } @Override - public ResUploadType returnStorageType() { - return ResUploadType.ABS; + public List listStorageEntity(String resourceAbsolutePath) { + return null; } @Override - public List listFilesStatusRecursively(String path, String defaultPath, String tenantCode, - ResourceType type) { - List storageEntityList = new ArrayList<>(); - LinkedList foldersToFetch = new LinkedList<>(); - - StorageEntity initialEntity = null; - try { - initialEntity = getFileStatus(path, defaultPath, tenantCode, type); - } catch (Exception e) { - log.error("error while listing files status recursively, path: {}", path, e); - return storageEntityList; - } - foldersToFetch.add(initialEntity); - - while (!foldersToFetch.isEmpty()) { - String pathToExplore = foldersToFetch.pop().getFullName(); - try { - List tempList = listFilesStatus(pathToExplore, defaultPath, tenantCode, type); - for (StorageEntity temp : tempList) { - if (temp.isDirectory()) { - foldersToFetch.add(temp); - } - } - storageEntityList.addAll(tempList); - } catch (Exception e) { - log.error("error while listing files stat:wus recursively, path: {}", pathToExplore, e); - } - } - - return storageEntityList; - } - - @Override - public List listFilesStatus(String path, String defaultPath, String tenantCode, - ResourceType type) throws Exception { - List storageEntityList = new ArrayList<>(); - - PagedIterable blobItems; - blobItems = blobContainerClient.listBlobsByHierarchy(path); - if (blobItems == null) { - return storageEntityList; - } - - for (BlobItem blobItem : blobItems) { - if (path.equals(blobItem.getName())) { - continue; - } - if (blobItem.isPrefix()) { - String suffix = StringUtils.difference(path, blobItem.getName()); - String fileName = StringUtils.difference(defaultPath, blobItem.getName()); - StorageEntity entity = new StorageEntity(); - entity.setAlias(suffix); - entity.setFileName(fileName); - entity.setFullName(blobItem.getName()); - entity.setDirectory(true); - entity.setUserName(tenantCode); - entity.setType(type); - entity.setSize(0); - entity.setCreateTime(null); - entity.setUpdateTime(null); - entity.setPfullName(path); - - storageEntityList.add(entity); - } else { - String[] aliasArr = blobItem.getName().split("/"); - String alias = aliasArr[aliasArr.length - 1]; - String fileName = StringUtils.difference(defaultPath, blobItem.getName()); - - StorageEntity entity = new StorageEntity(); - entity.setAlias(alias); - entity.setFileName(fileName); - entity.setFullName(blobItem.getName()); - entity.setDirectory(false); - entity.setUserName(tenantCode); - entity.setType(type); - entity.setSize(blobItem.getProperties().getContentLength()); - entity.setCreateTime(Date.from(blobItem.getProperties().getCreationTime().toInstant())); - entity.setUpdateTime(Date.from(blobItem.getProperties().getLastModified().toInstant())); - entity.setPfullName(path); - - storageEntityList.add(entity); - } - } - - return storageEntityList; + public List listFileStorageEntityRecursively(String resourceAbsolutePath) { + return null; } @Override - public StorageEntity getFileStatus(String path, String defaultPath, String tenantCode, - ResourceType type) throws Exception { - if (path.endsWith(FOLDER_SEPARATOR)) { - // the path is a directory that may or may not exist - String alias = findDirAlias(path); - String fileName = StringUtils.difference(defaultPath, path); - - StorageEntity entity = new StorageEntity(); - entity.setAlias(alias); - entity.setFileName(fileName); - entity.setFullName(path); - entity.setDirectory(true); - entity.setUserName(tenantCode); - entity.setType(type); - entity.setSize(0); - - return entity; - } else { - if (isObjectExists(path)) { - BlobClient blobClient = blobContainerClient.getBlobClient(path); - - String[] aliasArr = blobClient.getBlobName().split(FOLDER_SEPARATOR); - String alias = aliasArr[aliasArr.length - 1]; - String fileName = StringUtils.difference(defaultPath, blobClient.getBlobName()); - - StorageEntity entity = new StorageEntity(); - entity.setAlias(alias); - entity.setFileName(fileName); - entity.setFullName(blobClient.getBlobName()); - entity.setDirectory(false); - entity.setUserName(tenantCode); - entity.setType(type); - entity.setSize(blobClient.getProperties().getBlobSize()); - entity.setCreateTime(Date.from(blobClient.getProperties().getCreationTime().toInstant())); - entity.setUpdateTime(Date.from(blobClient.getProperties().getLastModified().toInstant())); - - return entity; - } else { - throw new FileNotFoundException("Object is not found in ABS container: " + containerName); - } - } - } - - private String findDirAlias(String dirPath) { - if (!dirPath.endsWith(FOLDER_SEPARATOR)) { - return dirPath; - } - - Path path = Paths.get(dirPath); - return path.getName(path.getNameCount() - 1) + FOLDER_SEPARATOR; + public StorageEntity getStorageEntity(String resourceAbsolutePath) { + return null; } - public void checkContainerNameExists() { + public void checkContainerNameExists(String containerName) { if (StringUtils.isBlank(containerName)) { throw new IllegalArgumentException(containerName + " is blank"); } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperatorFactory.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperatorFactory.java index 1909ece6f1..6428ec0980 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperatorFactory.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperatorFactory.java @@ -17,20 +17,30 @@ package org.apache.dolphinscheduler.plugin.storage.abs; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperateFactory; +import org.apache.dolphinscheduler.common.constants.Constants; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperatorFactory; import org.apache.dolphinscheduler.plugin.storage.api.StorageType; import com.google.auto.service.AutoService; -@AutoService(StorageOperateFactory.class) -public class AbsStorageOperatorFactory implements StorageOperateFactory { +@AutoService(StorageOperatorFactory.class) +public class AbsStorageOperatorFactory implements StorageOperatorFactory { @Override - public StorageOperate createStorageOperate() { - AbsStorageOperator absStorageOperator = new AbsStorageOperator(); - absStorageOperator.init(); - return absStorageOperator; + public StorageOperator createStorageOperate() { + final AbsStorageProperties absStorageProperties = getAbsStorageProperties(); + return new AbsStorageOperator(absStorageProperties); + } + + private AbsStorageProperties getAbsStorageProperties() { + return AbsStorageProperties.builder() + .containerName(PropertyUtils.getString(Constants.AZURE_BLOB_STORAGE_CONTAINER_NAME)) + .connectionString(PropertyUtils.getString(Constants.AZURE_BLOB_STORAGE_CONNECTION_STRING)) + .storageAccountName(PropertyUtils.getString(Constants.AZURE_BLOB_STORAGE_ACCOUNT_NAME)) + .resourceUploadPath(PropertyUtils.getString(Constants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler")) + .build(); } @Override diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageProperties.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageProperties.java new file mode 100644 index 0000000000..e33e165a9b --- /dev/null +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageProperties.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.storage.abs; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class AbsStorageProperties { + + private String containerName; + private String connectionString; + private String storageAccountName; + private String resourceUploadPath; + +} diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/test/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperatorTest.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/test/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperatorTest.java deleted file mode 100644 index daec0c36b2..0000000000 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/test/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperatorTest.java +++ /dev/null @@ -1,274 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.storage.abs; - -import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPARATOR; -import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; -import org.apache.dolphinscheduler.spi.enums.ResourceType; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; - -import com.azure.storage.blob.BlobClient; -import com.azure.storage.blob.BlobContainerClient; -import com.azure.storage.blob.BlobServiceClient; -import com.azure.storage.blob.specialized.BlockBlobClient; - -@ExtendWith(MockitoExtension.class) -public class AbsStorageOperatorTest { - - private static final String CONNECTION_STRING_MOCK = "CONNECTION_STRING_MOCK"; - - private static final String ACCOUNT_NAME_MOCK = "ACCOUNT_NAME_MOCK"; - - private static final String CONTAINER_NAME_MOCK = "CONTAINER_NAME_MOCK"; - - private static final String TENANT_CODE_MOCK = "TENANT_CODE_MOCK"; - - private static final String DIR_MOCK = "DIR_MOCK"; - - private static final String FILE_NAME_MOCK = "FILE_NAME_MOCK"; - - private static final String FILE_PATH_MOCK = "FILE_PATH_MOCK"; - - private static final String FULL_NAME = "/tmp/dir1/"; - - private static final String DEFAULT_PATH = "/tmp/"; - - @Mock - private BlobContainerClient blobContainerClient; - - @Mock - private BlobServiceClient blobServiceClient; - - @Mock - private BlockBlobClient blockBlobClient; - - @Mock - private BlobClient blobClient; - - private AbsStorageOperator absStorageOperator; - - @BeforeEach - public void setUp() throws Exception { - absStorageOperator = Mockito.spy(AbsStorageOperator.class); - Mockito.doReturn(CONNECTION_STRING_MOCK).when(absStorageOperator).readConnectionString(); - Mockito.doReturn(CONTAINER_NAME_MOCK).when(absStorageOperator).readContainerName(); - Mockito.doReturn(ACCOUNT_NAME_MOCK).when(absStorageOperator).readAccountName(); - Mockito.doReturn(blobContainerClient).when(absStorageOperator).buildBlobContainerClient(); - Mockito.doReturn(blobServiceClient).when(absStorageOperator).buildBlobServiceClient(); - Mockito.doNothing().when(absStorageOperator).checkContainerNameExists(); - - absStorageOperator.init(); - } - - @Test - public void testInit() throws Exception { - verify(absStorageOperator, times(1)).buildBlobServiceClient(); - verify(absStorageOperator, times(1)).buildBlobContainerClient(); - Assertions.assertEquals(CONNECTION_STRING_MOCK, absStorageOperator.getConnectionString()); - Assertions.assertEquals(CONTAINER_NAME_MOCK, absStorageOperator.getContainerName()); - Assertions.assertEquals(ACCOUNT_NAME_MOCK, absStorageOperator.getStorageAccountName()); - } - - @Test - public void createTenantResAndUdfDir() throws Exception { - doReturn(DIR_MOCK).when(absStorageOperator).getAbsResDir(TENANT_CODE_MOCK); - doReturn(DIR_MOCK).when(absStorageOperator).getAbsUdfDir(TENANT_CODE_MOCK); - doReturn(true).when(absStorageOperator).mkdir(TENANT_CODE_MOCK, DIR_MOCK); - absStorageOperator.createTenantDirIfNotExists(TENANT_CODE_MOCK); - verify(absStorageOperator, times(2)).mkdir(TENANT_CODE_MOCK, DIR_MOCK); - } - - @Test - public void getResDir() { - final String expectedResourceDir = String.format("dolphinscheduler/%s/resources/", TENANT_CODE_MOCK); - final String dir = absStorageOperator.getResDir(TENANT_CODE_MOCK); - Assertions.assertEquals(expectedResourceDir, dir); - } - - @Test - public void getUdfDir() { - final String expectedUdfDir = String.format("dolphinscheduler/%s/udfs/", TENANT_CODE_MOCK); - final String dir = absStorageOperator.getUdfDir(TENANT_CODE_MOCK); - Assertions.assertEquals(expectedUdfDir, dir); - } - - @Test - public void mkdirWhenDirExists() { - boolean isSuccess = false; - try { - final String key = DIR_MOCK + FOLDER_SEPARATOR; - Mockito.doReturn(true).when(absStorageOperator).isObjectExists(key); - isSuccess = absStorageOperator.mkdir(TENANT_CODE_MOCK, DIR_MOCK); - - } catch (IOException e) { - Assertions.fail("test failed due to unexpected IO exception"); - } - - Assertions.assertTrue(isSuccess); - } - - @Test - public void getResourceFullName() { - final String expectedResourceFileName = - String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK); - final String resourceFileName = absStorageOperator.getResourceFullName(TENANT_CODE_MOCK, FILE_NAME_MOCK); - Assertions.assertEquals(expectedResourceFileName, resourceFileName); - } - - @Test - public void getFileName() { - final String expectedFileName = - String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK); - final String fileName = absStorageOperator.getFileName(ResourceType.FILE, TENANT_CODE_MOCK, FILE_NAME_MOCK); - Assertions.assertEquals(expectedFileName, fileName); - } - - @Test - public void exists() { - boolean doesExist = false; - doReturn(true).when(absStorageOperator).isObjectExists(FILE_NAME_MOCK); - try { - doesExist = absStorageOperator.exists(FILE_NAME_MOCK); - } catch (IOException e) { - Assertions.fail("unexpected IO exception in unit test"); - } - - Assertions.assertTrue(doesExist); - } - - @Test - public void delete() { - boolean isDeleted = false; - doReturn(true).when(absStorageOperator).isObjectExists(FILE_NAME_MOCK); - Mockito.doReturn(blobClient).when(blobContainerClient).getBlobClient(Mockito.anyString()); - try { - isDeleted = absStorageOperator.delete(FILE_NAME_MOCK, true); - } catch (IOException e) { - Assertions.fail("unexpected IO exception in unit test"); - } - - Assertions.assertTrue(isDeleted); - verify(blobClient, times(1)).delete(); - } - - @Test - public void copy() { - boolean isSuccess = false; - Mockito.doReturn(blobClient).when(blobContainerClient).getBlobClient(Mockito.anyString()); - Mockito.doReturn(blockBlobClient).when(blobClient).getBlockBlobClient(); - try { - isSuccess = absStorageOperator.copy(FILE_PATH_MOCK, FILE_PATH_MOCK, false, false); - } catch (IOException e) { - Assertions.fail("unexpected IO exception in unit test"); - } - - Assertions.assertTrue(isSuccess); - } - - @Test - public void deleteTenant() { - doNothing().when(absStorageOperator).deleteTenantCode(anyString()); - try { - absStorageOperator.deleteTenant(TENANT_CODE_MOCK); - } catch (Exception e) { - Assertions.fail("unexpected exception caught in unit test"); - } - - verify(absStorageOperator, times(1)).deleteTenantCode(anyString()); - } - - @Test - public void getGcsResDir() { - final String expectedGcsResDir = String.format("dolphinscheduler/%s/resources", TENANT_CODE_MOCK); - final String gcsResDir = absStorageOperator.getAbsResDir(TENANT_CODE_MOCK); - Assertions.assertEquals(expectedGcsResDir, gcsResDir); - } - - @Test - public void getGcsUdfDir() { - final String expectedGcsUdfDir = String.format("dolphinscheduler/%s/udfs", TENANT_CODE_MOCK); - final String gcsUdfDir = absStorageOperator.getAbsUdfDir(TENANT_CODE_MOCK); - Assertions.assertEquals(expectedGcsUdfDir, gcsUdfDir); - } - - @Test - public void getGcsTenantDir() { - final String expectedGcsTenantDir = String.format(FORMAT_S_S, DIR_MOCK, TENANT_CODE_MOCK); - doReturn(DIR_MOCK).when(absStorageOperator).getGcsDataBasePath(); - final String gcsTenantDir = absStorageOperator.getAbsTenantDir(TENANT_CODE_MOCK); - Assertions.assertEquals(expectedGcsTenantDir, gcsTenantDir); - } - - @Test - public void deleteDir() { - Mockito.doReturn(blobClient).when(blobContainerClient).getBlobClient(Mockito.anyString()); - doReturn(true).when(absStorageOperator).isObjectExists(Mockito.any()); - absStorageOperator.deleteDirectory(DIR_MOCK); - verify(blobClient, times(1)).delete(); - } - - @Test - public void testGetFileStatus() throws Exception { - StorageEntity entity = - absStorageOperator.getFileStatus(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, ResourceType.FILE); - Assertions.assertEquals(FULL_NAME, entity.getFullName()); - Assertions.assertEquals("dir1/", entity.getFileName()); - } - - @Test - public void testListFilesStatus() throws Exception { - Mockito.doReturn(null).when(blobContainerClient).listBlobsByHierarchy(Mockito.any()); - List result = - absStorageOperator.listFilesStatus(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, ResourceType.FILE); - verify(blobContainerClient, times(1)).listBlobsByHierarchy(Mockito.any()); - } - - @Test - public void testListFilesStatusRecursively() throws Exception { - StorageEntity entity = new StorageEntity(); - entity.setFullName(FULL_NAME); - - doReturn(entity).when(absStorageOperator).getFileStatus(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, - ResourceType.FILE); - doReturn(Collections.EMPTY_LIST).when(absStorageOperator).listFilesStatus(anyString(), anyString(), anyString(), - Mockito.any(ResourceType.class)); - - List result = - absStorageOperator.listFilesStatusRecursively(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, - ResourceType.FILE); - Assertions.assertEquals(0, result.size()); - } -} diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/AbstractStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/AbstractStorageOperator.java new file mode 100644 index 0000000000..1ad94e47d9 --- /dev/null +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/AbstractStorageOperator.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.storage.api; + +import org.apache.dolphinscheduler.common.constants.Constants; +import org.apache.dolphinscheduler.common.utils.FileUtils; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.spi.enums.ResourceType; + +import org.apache.commons.lang3.StringUtils; + +import java.io.File; + +import com.google.common.base.Preconditions; +import com.google.common.io.Files; + +public abstract class AbstractStorageOperator implements StorageOperator { + + protected final String resourceBaseAbsolutePath; + + public AbstractStorageOperator(String resourceBaseAbsolutePath) { + Preconditions.checkNotNull(resourceBaseAbsolutePath, "Resource upload path should not be null"); + this.resourceBaseAbsolutePath = resourceBaseAbsolutePath; + } + + @Override + public ResourceMetadata getResourceMetaData(String resourceAbsolutePath) { + String storageBaseDirectory = getStorageBaseDirectory(); + String resourceSegment = StringUtils.substringAfter(resourceAbsolutePath, storageBaseDirectory); + String[] segments = StringUtils.split(resourceSegment, File.separator, 3); + if (segments.length == 0) { + throw new IllegalArgumentException("Invalid resource path: " + resourceAbsolutePath); + } + return ResourceMetadata.builder() + .resourceAbsolutePath(resourceAbsolutePath) + .resourceBaseDirectory(storageBaseDirectory) + .isDirectory(Files.getFileExtension(resourceAbsolutePath).isEmpty()) + .tenant(segments[0]) + .resourceType(segments[1].equals(FILE_FOLDER_NAME) ? ResourceType.FILE : ResourceType.UDF) + .resourceRelativePath(segments.length == 2 ? "/" : segments[2]) + .resourceParentAbsolutePath(StringUtils.substringBeforeLast(resourceAbsolutePath, File.separator)) + .build(); + } + + @Override + public String getStorageBaseDirectory() { + // All directory should end with File.separator + return PropertyUtils.getString(Constants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler"); + } + + @Override + public String getStorageBaseDirectory(String tenantCode) { + if (StringUtils.isEmpty(tenantCode)) { + throw new IllegalArgumentException("Tenant code should not be empty"); + } + // All directory should end with File.separator + return FileUtils.concatFilePath(getStorageBaseDirectory(), tenantCode); + } + + @Override + public String getStorageBaseDirectory(String tenantCode, ResourceType resourceType) { + String tenantBaseDirectory = getStorageBaseDirectory(tenantCode); + if (resourceType == null) { + throw new IllegalArgumentException("Resource type should not be null"); + } + String resourceBaseDirectory; + switch (resourceType) { + case FILE: + resourceBaseDirectory = FileUtils.concatFilePath(tenantBaseDirectory, FILE_FOLDER_NAME); + break; + case UDF: + resourceBaseDirectory = FileUtils.concatFilePath(tenantBaseDirectory, UDF_FOLDER_NAME); + break; + case ALL: + resourceBaseDirectory = tenantBaseDirectory; + break; + default: + throw new IllegalArgumentException("Resource type: " + resourceType + " not supported"); + } + // All directory should end with File.separator + return resourceBaseDirectory; + } + + @Override + public String getStorageFileAbsolutePath(String tenantCode, String fileName) { + return FileUtils.concatFilePath(getStorageBaseDirectory(tenantCode, ResourceType.FILE), fileName); + } + + protected void exceptionIfPathEmpty(String resourceAbsolutePath) { + if (StringUtils.isEmpty(resourceAbsolutePath)) { + throw new IllegalArgumentException("Resource path should not be empty"); + } + } + +} diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/ResourceMetadata.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/ResourceMetadata.java new file mode 100644 index 0000000000..6a2ee3dc63 --- /dev/null +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/ResourceMetadata.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.storage.api; + +import org.apache.dolphinscheduler.spi.enums.ResourceType; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class ResourceMetadata { + + private String resourceAbsolutePath; + + private String resourceBaseDirectory; + private String tenant; + private ResourceType resourceType; + private String resourceRelativePath; + private String resourceParentAbsolutePath; + private boolean isDirectory; + +} diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageConfiguration.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageConfiguration.java index d34e6d01de..9b30ddb450 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageConfiguration.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageConfiguration.java @@ -32,13 +32,13 @@ import org.springframework.context.annotation.Configuration; public class StorageConfiguration { @Bean - public StorageOperate storageOperate() { + public StorageOperator storageOperate() { Optional storageTypeOptional = StorageType.getStorageType(PropertyUtils.getUpperCaseString(RESOURCE_STORAGE_TYPE)); - Optional storageOperate = storageTypeOptional.map(storageType -> { - ServiceLoader storageOperateFactories = - ServiceLoader.load(StorageOperateFactory.class); - for (StorageOperateFactory storageOperateFactory : storageOperateFactories) { + Optional storageOperate = storageTypeOptional.map(storageType -> { + ServiceLoader storageOperateFactories = + ServiceLoader.load(StorageOperatorFactory.class); + for (StorageOperatorFactory storageOperateFactory : storageOperateFactories) { if (storageOperateFactory.getStorageOperate() == storageType) { return storageOperateFactory.createStorageOperate(); } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageEntity.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageEntity.java index e3639b8afe..cae9d862a0 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageEntity.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageEntity.java @@ -24,7 +24,10 @@ import org.apache.dolphinscheduler.spi.enums.ResourceType; import java.util.Date; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; // StorageEneity is an entity representing a resource in the third-part storage service. // It is only stored in t_ds_relation_resources_task if the resource is used by a task. @@ -32,32 +35,16 @@ import lombok.Data; // in table t_ds_relation_resources_task. @Data +@NoArgsConstructor +@AllArgsConstructor +@Builder public class StorageEntity { - /** - * exist only if it is stored in t_ds_relation_resources_task. - * - */ - private int id; - /** - * fullname is in a format of basepath + tenantCode + res/udf + filename - */ private String fullName; - /** - * filename is in a format of possible parent folders + alias - */ + private String fileName; - /** - * the name of the file - */ - private String alias; - /** - * parent folder time - */ private String pfullName; private boolean isDirectory; - private int userId; - private String userName; private ResourceType type; private long size; private Date createTime; diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperate.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperate.java deleted file mode 100644 index 945e361a09..0000000000 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperate.java +++ /dev/null @@ -1,203 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.storage.api; - -import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TYPE_FILE; - -import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.enums.ResUploadType; -import org.apache.dolphinscheduler.common.utils.PropertyUtils; -import org.apache.dolphinscheduler.spi.enums.ResourceType; - -import java.io.IOException; -import java.util.List; - -public interface StorageOperate { - - String RESOURCE_UPLOAD_PATH = PropertyUtils.getString(Constants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler"); - - /** - * if the resource of tenant 's exist, the resource of folder will be created - * @param tenantCode - * @throws Exception - */ - void createTenantDirIfNotExists(String tenantCode) throws Exception; - - /** - * get the resource directory of tenant - * @param tenantCode - * @return - */ - String getResDir(String tenantCode); - - /** - * return the udf directory of tenant - * @param tenantCode - * @return - */ - String getUdfDir(String tenantCode); - - /** - * create the directory that the path of tenant wanted to create - * @param tenantCode - * @param path - * @return - * @throws IOException - */ - boolean mkdir(String tenantCode, String path) throws IOException; - - /** - * get the path of the resource file (fullName) - * @param tenantCode - * @param fileName - * @return - */ - String getResourceFullName(String tenantCode, String fileName); - - /** - * get the path of the resource file excluding the base path (fileName) - */ - default String getResourceFileName(String tenantCode, String fullName) { - String resDir = getResDir(tenantCode); - String filenameReplaceResDir = fullName.replaceFirst(resDir, ""); - if (!filenameReplaceResDir.equals(fullName)) { - return filenameReplaceResDir; - } - - // Replace resource dir not effective in case of run workflow with different tenant from resource file's. - // this is backup solution to get related path, by split with RESOURCE_TYPE_FILE - return filenameReplaceResDir.contains(RESOURCE_TYPE_FILE) - ? filenameReplaceResDir.split(String.format("%s/", RESOURCE_TYPE_FILE))[1] - : filenameReplaceResDir; - } - - /** - * get the path of the file - * @param resourceType - * @param tenantCode - * @param fileName - * @return - */ - String getFileName(ResourceType resourceType, String tenantCode, String fileName); - - /** - * predicate if the resource of tenant exists - * @param fullName - * @return - * @throws IOException - */ - boolean exists(String fullName) throws IOException; - - /** - * delete the resource of filePath - * todo if the filePath is the type of directory ,the files in the filePath need to be deleted at all - * @param filePath - * @param recursive - * @return - * @throws IOException - */ - boolean delete(String filePath, boolean recursive) throws IOException; - - boolean delete(String filePath, List childrenPathArray, boolean recursive) throws IOException; - - /** - * copy the file from srcPath to dstPath - * @param srcPath - * @param dstPath - * @param deleteSource if need to delete the file of srcPath - * @param overwrite - * @return - * @throws IOException - */ - boolean copy(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) throws IOException; - - /** - * get the root path of the tenant with resourceType - * @param resourceType - * @param tenantCode - * @return - */ - String getDir(ResourceType resourceType, String tenantCode); - - /** - * upload the local srcFile to dstPath - * @param tenantCode - * @param srcFile - * @param dstPath - * @param deleteSource - * @param overwrite - * @return - * @throws IOException - */ - boolean upload(String tenantCode, String srcFile, String dstPath, boolean deleteSource, - boolean overwrite) throws IOException; - - /** - * download the srcPath to local - * - * @param srcFilePath the full path of the srcPath - * @param dstFile - * @param overwrite - * @throws IOException - */ - void download(String srcFilePath, String dstFile, boolean overwrite) throws IOException; - - /** - * vim the context of filePath - * @param tenantCode - * @param filePath - * @param skipLineNums - * @param limit - * @return - * @throws IOException - */ - List vimFile(String tenantCode, String filePath, int skipLineNums, int limit) throws IOException; - - /** - * delete the files and directory of the tenant - * - * @param tenantCode - * @throws Exception - */ - void deleteTenant(String tenantCode) throws Exception; - - /** - * return the storageType - * - * @return - */ - ResUploadType returnStorageType(); - - /** - * return files and folders in the current directory and subdirectories - * */ - List listFilesStatusRecursively(String path, String defaultPath, String tenantCode, - ResourceType type); - - /** - * return files and folders in the current directory - * */ - List listFilesStatus(String path, String defaultPath, String tenantCode, - ResourceType type) throws Exception; - - /** - * return a file status - * */ - StorageEntity getFileStatus(String path, String defaultPath, String tenantCode, - ResourceType type) throws Exception; -} diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperator.java new file mode 100644 index 0000000000..fb27bba217 --- /dev/null +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperator.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.storage.api; + +import org.apache.dolphinscheduler.spi.enums.ResourceType; + +import java.nio.file.FileAlreadyExistsException; +import java.util.List; + +public interface StorageOperator { + + String FILE_FOLDER_NAME = "resources"; + String UDF_FOLDER_NAME = "udfs"; + + ResourceMetadata getResourceMetaData(String resourceAbsolutePath); + + /** + * Get the absolute path of base directory. + * + * @return the base directory. e.g. file:///tmp/dolphinscheduler/, /tmp/dolphinscheduler/ + */ + String getStorageBaseDirectory(); + + /** + * Get the absolute path of directory which will be used by the given tenant. the tenant directory is under the base directory. + * + * @param tenantCode the tenant code, cannot be empty + * @return the tenant directory. e.g. file:///tmp/dolphinscheduler/default/ + */ + String getStorageBaseDirectory(String tenantCode); + + /** + * Get the absolute path of directory which will be used by the given tenant and resource type. the resource directory is under the tenant directory. + *

If the resource type is FILE, will be 'file:///tmp/dolphinscheduler/default/resources/'. + *

If the resource type is UDF, will be 'is file:///tmp/dolphinscheduler/default/udfs/'. + *

If the resource type is ALL, will be 'is file:///tmp/dolphinscheduler/default/'. + * + * @param tenantCode the tenant code, cannot be empty + * @param resourceType the resource type, cannot be null + * @return the resource directory. e.g. file:///tmp/dolphinscheduler/default/resources/ + */ + String getStorageBaseDirectory(String tenantCode, ResourceType resourceType); + + /** + * Get the absolute path of the file in the storage. the file will under the file resource directory. + * + * @param tenantCode the tenant code, cannot be empty + * @param fileName the file name, cannot be empty + * @return the file absolute path. e.g. file:///tmp/dolphinscheduler/default/resources/test.sh + */ + String getStorageFileAbsolutePath(String tenantCode, String fileName); + + /** + * Create a directory if the directory is already exists will throw exception(Dependent on the storage implementation). + *

If the directory is not exists, will create the directory. + *

If the parent directory is not exists, will create the parent directory. + *

If the directory is already exists, will throw {@link FileAlreadyExistsException}. + * + * @param directoryAbsolutePath the directory absolute path + */ + void createStorageDir(String directoryAbsolutePath); + + /** + * Check if the resource exists. + * + * @param resourceAbsolutePath the resource absolute path + * @return true if the resource exists, otherwise false + */ + boolean exists(String resourceAbsolutePath); + + /** + * Delete the resource, if the resourceAbsolutePath is not exists, will do nothing. + * + * @param resourceAbsolutePath the resource absolute path + * @param recursive whether to delete all the sub file/directory under the given resource + */ + void delete(String resourceAbsolutePath, boolean recursive); + + /** + * Copy the resource from the source path to the destination path. + * + * @param srcAbsolutePath the source path + * @param dstAbsolutePath the destination path + * @param deleteSource whether to delete the source path after copying + * @param overwrite whether to overwrite the destination path if it exists + */ + void copy(String srcAbsolutePath, String dstAbsolutePath, boolean deleteSource, boolean overwrite); + + /** + * Move the resource from the source path to the destination path. + * + * @param srcLocalFileAbsolutePath the source local file + * @param dstAbsolutePath the destination path + * @param deleteSource whether to delete the source path after moving + * @param overwrite whether to overwrite the destination path if it exists + */ + void upload(String srcLocalFileAbsolutePath, String dstAbsolutePath, boolean deleteSource, boolean overwrite); + + /** + * Download the resource from the source path to the destination path. + * + * @param srcFileAbsolutePath the source path + * @param dstAbsoluteFile the destination file + * @param overwrite whether to overwrite the destination file if it exists + */ + void download(String srcFileAbsolutePath, String dstAbsoluteFile, boolean overwrite); + + /** + * Fetch the content of the file. + * + * @param fileAbsolutePath the file path + * @param skipLineNums the number of lines to skip + * @param limit the number of lines to read + * @return the content of the file + */ + List fetchFileContent(String fileAbsolutePath, int skipLineNums, int limit); + + /** + * Return the {@link StorageEntity} under the given path. + *

If the path is a file, return the file status. + *

If the path is a directory, return the file/directory under the directory. + *

If the path is not exist, will return empty. + * + * @param resourceAbsolutePath the resource absolute path, cannot be empty + */ + List listStorageEntity(String resourceAbsolutePath); + + /** + * Return the {@link StorageEntity} which is file under the given path + * + * @param resourceAbsolutePath the resource absolute path, cannot be empty + */ + List listFileStorageEntityRecursively(String resourceAbsolutePath); + + /** + * Return the {@link StorageEntity} under the current directory + * + * @param resourceAbsolutePath the resource absolute path, cannot be empty + */ + StorageEntity getStorageEntity(String resourceAbsolutePath); + +} diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperateFactory.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperatorFactory.java similarity index 91% rename from dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperateFactory.java rename to dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperatorFactory.java index b3a60888c9..1e6e1f5a52 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperateFactory.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperatorFactory.java @@ -17,9 +17,9 @@ package org.apache.dolphinscheduler.plugin.storage.api; -public interface StorageOperateFactory { +public interface StorageOperatorFactory { - StorageOperate createStorageOperate(); + StorageOperator createStorageOperate(); StorageType getStorageOperate(); } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java index e4176dc58e..00aa746e73 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java @@ -18,18 +18,13 @@ package org.apache.dolphinscheduler.plugin.storage.gcs; import static org.apache.dolphinscheduler.common.constants.Constants.EMPTY_STRING; -import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPARATOR; -import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S; -import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TYPE_FILE; -import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TYPE_UDF; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.enums.ResUploadType; import org.apache.dolphinscheduler.common.utils.FileUtils; -import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.plugin.storage.api.AbstractStorageOperator; +import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata; import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; -import org.apache.dolphinscheduler.spi.enums.ResourceType; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.commons.lang3.StringUtils; @@ -37,22 +32,24 @@ import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.Closeable; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.sql.Date; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; -import lombok.Data; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import com.google.api.gax.paging.Page; @@ -64,82 +61,53 @@ import com.google.cloud.storage.Bucket; import com.google.cloud.storage.Storage; import com.google.cloud.storage.StorageOptions; -@Data @Slf4j -public class GcsStorageOperator implements Closeable, StorageOperate { +public class GcsStorageOperator extends AbstractStorageOperator implements Closeable, StorageOperator { - private Storage gcsStorage; + private final Storage gcsStorage; - private String bucketName; + private final String bucketName; - private String credential; - - public GcsStorageOperator() { - - } - - public void init() { - try { - credential = readCredentials(); - bucketName = readBucketName(); - gcsStorage = buildGcsStorage(credential); - - checkBucketNameExists(bucketName); - } catch (IOException e) { - log.error("GCS Storage operator init failed", e); - } - } - - protected Storage buildGcsStorage(String credential) throws IOException { - return StorageOptions.newBuilder() + @SneakyThrows + public GcsStorageOperator(GcsStorageProperties gcsStorageProperties) { + super(gcsStorageProperties.getResourceUploadPath()); + bucketName = gcsStorageProperties.getBucketName(); + gcsStorage = StorageOptions.newBuilder() .setCredentials(ServiceAccountCredentials.fromStream( - Files.newInputStream(Paths.get(credential)))) + Files.newInputStream(Paths.get(gcsStorageProperties.getCredential())))) .build() .getService(); - } - protected String readCredentials() { - return PropertyUtils.getString(Constants.GOOGLE_CLOUD_STORAGE_CREDENTIAL); - } - - protected String readBucketName() { - return PropertyUtils.getString(Constants.GOOGLE_CLOUD_STORAGE_BUCKET_NAME); + checkBucketNameExists(bucketName); } @Override - public void createTenantDirIfNotExists(String tenantCode) throws Exception { - mkdir(tenantCode, getGcsResDir(tenantCode)); - mkdir(tenantCode, getGcsUdfDir(tenantCode)); - } - - @Override - public String getResDir(String tenantCode) { - return getGcsResDir(tenantCode) + FOLDER_SEPARATOR; - } - - @Override - public String getUdfDir(String tenantCode) { - return getGcsUdfDir(tenantCode) + FOLDER_SEPARATOR; - } - - @Override - public String getResourceFullName(String tenantCode, String fileName) { - if (fileName.startsWith(FOLDER_SEPARATOR)) { - fileName.replaceFirst(FOLDER_SEPARATOR, EMPTY_STRING); + public String getStorageBaseDirectory() { + // All directory should end with File.separator + if (resourceBaseAbsolutePath.startsWith("/")) { + log.warn("{} -> {} should not start with / in Gcs", Constants.RESOURCE_UPLOAD_PATH, + resourceBaseAbsolutePath); + return resourceBaseAbsolutePath.substring(1); } - return String.format(FORMAT_S_S, getGcsResDir(tenantCode), fileName); + return getStorageBaseDirectory(); } + @SneakyThrows @Override - public String getFileName(ResourceType resourceType, String tenantCode, String fileName) { - if (fileName.startsWith(FOLDER_SEPARATOR)) { - fileName = fileName.replaceFirst(FOLDER_SEPARATOR, EMPTY_STRING); + public void createStorageDir(String directoryAbsolutePath) { + directoryAbsolutePath = transformAbsolutePathToGcsKey(directoryAbsolutePath); + if (exists(directoryAbsolutePath)) { + throw new FileAlreadyExistsException("directory: " + directoryAbsolutePath + " already exists"); } - return getDir(resourceType, tenantCode) + fileName; + BlobInfo blobInfo = BlobInfo.newBuilder(BlobId.of(bucketName, directoryAbsolutePath)).build(); + gcsStorage.create(blobInfo, EMPTY_STRING.getBytes(StandardCharsets.UTF_8)); } + @SneakyThrows @Override - public void download(String srcFilePath, String dstFilePath, boolean overwrite) throws IOException { + public void download(String srcFilePath, String dstFilePath, boolean overwrite) { + srcFilePath = transformAbsolutePathToGcsKey(srcFilePath); + File dstFile = new File(dstFilePath); if (dstFile.isDirectory()) { Files.delete(dstFile.toPath()); @@ -152,40 +120,26 @@ public class GcsStorageOperator implements Closeable, StorageOperate { } @Override - public boolean exists(String fullName) throws IOException { - return isObjectExists(fullName); + public boolean exists(String fullName) { + fullName = transformAbsolutePathToGcsKey(fullName); + Blob blob = gcsStorage.get(BlobId.of(bucketName, fullName)); + return blob != null && blob.exists(); } + @SneakyThrows @Override - public boolean delete(String filePath, boolean recursive) throws IOException { - try { - if (isObjectExists(filePath)) { - gcsStorage.delete(BlobId.of(bucketName, filePath)); - } - return true; - } catch (Exception e) { - log.error("delete the object error,the resource path is {}", filePath); - return false; + public void delete(String filePath, boolean recursive) { + filePath = transformAbsolutePathToGcsKey(filePath); + if (exists(filePath)) { + gcsStorage.delete(BlobId.of(bucketName, filePath)); } } @Override - public boolean delete(String fullName, List childrenPathList, boolean recursive) throws IOException { - // append the resource fullName to the list for deletion. - childrenPathList.add(fullName); - - boolean result = true; - for (String filePath : childrenPathList) { - if (!delete(filePath, recursive)) { - result = false; - } - } - - return result; - } + public void copy(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) { + srcPath = transformGcsKeyToAbsolutePath(srcPath); + dstPath = transformGcsKeyToAbsolutePath(dstPath); - @Override - public boolean copy(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) throws IOException { BlobId source = BlobId.of(bucketName, srcPath); BlobId target = BlobId.of(bucketName, dstPath); @@ -198,31 +152,30 @@ public class GcsStorageOperator implements Closeable, StorageOperate { if (deleteSource) { gcsStorage.delete(source); } - return true; } + @SneakyThrows @Override - public boolean upload(String tenantCode, String srcFile, String dstPath, boolean deleteSource, - boolean overwrite) throws IOException { - try { - BlobInfo blobInfo = BlobInfo.newBuilder( - BlobId.of(bucketName, dstPath)).build(); + public void upload(String srcFile, String dstPath, boolean deleteSource, boolean overwrite) { + dstPath = transformAbsolutePathToGcsKey(dstPath); + if (exists(dstPath) && !overwrite) { + throw new FileAlreadyExistsException("file: " + dstPath + " already exists"); + } + BlobInfo blobInfo = BlobInfo.newBuilder( + BlobId.of(bucketName, dstPath)).build(); - Path srcPath = Paths.get(srcFile); - gcsStorage.create(blobInfo, Files.readAllBytes(srcPath)); + Path srcPath = Paths.get(srcFile); + gcsStorage.create(blobInfo, Files.readAllBytes(srcPath)); - if (deleteSource) { - Files.delete(srcPath); - } - return true; - } catch (Exception e) { - log.error("upload failed,the bucketName is {},the filePath is {}", bucketName, dstPath); - return false; + if (deleteSource) { + Files.delete(srcPath); } } + @SneakyThrows @Override - public List vimFile(String tenantCode, String filePath, int skipLineNums, int limit) throws IOException { + public List fetchFileContent(String filePath, int skipLineNums, int limit) { + filePath = transformAbsolutePathToGcsKey(filePath); if (StringUtils.isBlank(filePath)) { log.error("file path:{} is blank", filePath); return Collections.emptyList(); @@ -237,232 +190,58 @@ public class GcsStorageOperator implements Closeable, StorageOperate { } } + @SneakyThrows @Override - public void deleteTenant(String tenantCode) throws Exception { - deleteTenantCode(tenantCode); - } - - protected void deleteTenantCode(String tenantCode) { - deleteDirectory(getResDir(tenantCode)); - deleteDirectory(getUdfDir(tenantCode)); - } - - @Override - public String getDir(ResourceType resourceType, String tenantCode) { - switch (resourceType) { - case UDF: - return getUdfDir(tenantCode); - case FILE: - return getResDir(tenantCode); - case ALL: - return getGcsDataBasePath(); - default: - return EMPTY_STRING; - } - - } - - protected void deleteDirectory(String directoryName) { - if (isObjectExists(directoryName)) { - gcsStorage.delete(BlobId.of(bucketName, directoryName)); - } - } - - public String getGcsResDir(String tenantCode) { - return String.format("%s/" + RESOURCE_TYPE_FILE, getGcsTenantDir(tenantCode)); - } - - public String getGcsUdfDir(String tenantCode) { - return String.format("%s/" + RESOURCE_TYPE_UDF, getGcsTenantDir(tenantCode)); - } - - public String getGcsTenantDir(String tenantCode) { - return String.format(FORMAT_S_S, getGcsDataBasePath(), tenantCode); - } - - public String getGcsDataBasePath() { - if (FOLDER_SEPARATOR.equals(RESOURCE_UPLOAD_PATH)) { - return EMPTY_STRING; - } else { - return RESOURCE_UPLOAD_PATH.replaceFirst(FOLDER_SEPARATOR, EMPTY_STRING); + public void close() throws IOException { + if (gcsStorage != null) { + gcsStorage.close(); } } @Override - public boolean mkdir(String tenantCode, String path) throws IOException { - String objectName = path + FOLDER_SEPARATOR; - if (!isObjectExists(objectName)) { - BlobInfo blobInfo = BlobInfo.newBuilder( - BlobId.of(bucketName, objectName)).build(); + public List listStorageEntity(String resourceAbsolutePath) { + resourceAbsolutePath = transformAbsolutePathToGcsKey(resourceAbsolutePath); - gcsStorage.create(blobInfo, EMPTY_STRING.getBytes(StandardCharsets.UTF_8)); - } - return true; - } - - @Override - public void close() throws IOException { - try { - if (gcsStorage != null) { - gcsStorage.close(); - } - } catch (Exception e) { - throw new IOException(e); - } + Page blobs = gcsStorage.list(bucketName, Storage.BlobListOption.prefix(resourceAbsolutePath)); + List storageEntities = new ArrayList<>(); + blobs.iterateAll().forEach(blob -> storageEntities.add(transformBlobToStorageEntity(blob))); + return storageEntities; } @Override - public ResUploadType returnStorageType() { - return ResUploadType.GCS; - } + public List listFileStorageEntityRecursively(String resourceAbsolutePath) { + resourceAbsolutePath = transformAbsolutePathToGcsKey(resourceAbsolutePath); - @Override - public List listFilesStatusRecursively(String path, String defaultPath, String tenantCode, - ResourceType type) { + Set visited = new HashSet<>(); List storageEntityList = new ArrayList<>(); - LinkedList foldersToFetch = new LinkedList<>(); - - StorageEntity initialEntity = null; - try { - initialEntity = getFileStatus(path, defaultPath, tenantCode, type); - } catch (Exception e) { - log.error("error while listing files status recursively, path: {}", path, e); - return storageEntityList; - } - foldersToFetch.add(initialEntity); + LinkedList foldersToFetch = new LinkedList<>(); + foldersToFetch.addLast(resourceAbsolutePath); while (!foldersToFetch.isEmpty()) { - String pathToExplore = foldersToFetch.pop().getFullName(); - try { - List tempList = listFilesStatus(pathToExplore, defaultPath, tenantCode, type); - for (StorageEntity temp : tempList) { - if (temp.isDirectory()) { - foldersToFetch.add(temp); + String pathToExplore = foldersToFetch.pop(); + visited.add(pathToExplore); + List tempList = listStorageEntity(pathToExplore); + for (StorageEntity temp : tempList) { + if (temp.isDirectory()) { + if (visited.contains(temp.getFullName())) { + continue; } + foldersToFetch.add(temp.getFullName()); } - storageEntityList.addAll(tempList); - } catch (Exception e) { - log.error("error while listing files stat:wus recursively, path: {}", pathToExplore, e); - } - } - - return storageEntityList; - } - - @Override - public List listFilesStatus(String path, String defaultPath, String tenantCode, - ResourceType type) throws Exception { - List storageEntityList = new ArrayList<>(); - - Page blobs; - try { - blobs = - gcsStorage.list( - bucketName, - Storage.BlobListOption.prefix(path), - Storage.BlobListOption.currentDirectory()); - } catch (Exception e) { - throw new RuntimeException("Get GCS file list exception. ", e); - } - - if (blobs == null) { - return storageEntityList; - } - - for (Blob blob : blobs.iterateAll()) { - if (path.equals(blob.getName())) { - continue; - } - if (blob.isDirectory()) { - String suffix = StringUtils.difference(path, blob.getName()); - String fileName = StringUtils.difference(defaultPath, blob.getName()); - StorageEntity entity = new StorageEntity(); - entity.setAlias(suffix); - entity.setFileName(fileName); - entity.setFullName(blob.getName()); - entity.setDirectory(true); - entity.setUserName(tenantCode); - entity.setType(type); - entity.setSize(0); - entity.setCreateTime(null); - entity.setUpdateTime(null); - entity.setPfullName(path); - - storageEntityList.add(entity); - } else { - String[] aliasArr = blob.getName().split("/"); - String alias = aliasArr[aliasArr.length - 1]; - String fileName = StringUtils.difference(defaultPath, blob.getName()); - - StorageEntity entity = new StorageEntity(); - entity.setAlias(alias); - entity.setFileName(fileName); - entity.setFullName(blob.getName()); - entity.setDirectory(false); - entity.setUserName(tenantCode); - entity.setType(type); - entity.setSize(blob.getSize()); - entity.setCreateTime(Date.from(blob.getCreateTimeOffsetDateTime().toInstant())); - entity.setUpdateTime(Date.from(blob.getUpdateTimeOffsetDateTime().toInstant())); - entity.setPfullName(path); - - storageEntityList.add(entity); } + storageEntityList.addAll(tempList); } - return storageEntityList; } @Override - public StorageEntity getFileStatus(String path, String defaultPath, String tenantCode, - ResourceType type) throws Exception { - if (path.endsWith(FOLDER_SEPARATOR)) { - // the path is a directory that may or may not exist - String alias = findDirAlias(path); - String fileName = StringUtils.difference(defaultPath, path); - - StorageEntity entity = new StorageEntity(); - entity.setAlias(alias); - entity.setFileName(fileName); - entity.setFullName(path); - entity.setDirectory(true); - entity.setUserName(tenantCode); - entity.setType(type); - entity.setSize(0); - - return entity; - } else { - if (isObjectExists(path)) { - Blob blob = gcsStorage.get(BlobId.of(bucketName, path)); - - String[] aliasArr = blob.getName().split(FOLDER_SEPARATOR); - String alias = aliasArr[aliasArr.length - 1]; - String fileName = StringUtils.difference(defaultPath, blob.getName()); - - StorageEntity entity = new StorageEntity(); - entity.setAlias(alias); - entity.setFileName(fileName); - entity.setFullName(blob.getName()); - entity.setDirectory(false); - entity.setUserName(tenantCode); - entity.setType(type); - entity.setSize(blob.getSize()); - entity.setCreateTime(Date.from(blob.getCreateTimeOffsetDateTime().toInstant())); - entity.setUpdateTime(Date.from(blob.getUpdateTimeOffsetDateTime().toInstant())); - - return entity; - } else { - throw new FileNotFoundException("Object is not found in GCS Bucket: " + bucketName); - } - } - } - - protected boolean isObjectExists(String objectName) { - Blob blob = gcsStorage.get(BlobId.of(bucketName, objectName)); - return blob != null && blob.exists(); + public StorageEntity getStorageEntity(String resourceAbsolutePath) { + resourceAbsolutePath = transformAbsolutePathToGcsKey(resourceAbsolutePath); + Blob blob = gcsStorage.get(BlobId.of(bucketName, resourceAbsolutePath)); + return transformBlobToStorageEntity(blob); } - public void checkBucketNameExists(String bucketName) { + private void checkBucketNameExists(String bucketName) { if (StringUtils.isBlank(bucketName)) { throw new IllegalArgumentException(Constants.GOOGLE_CLOUD_STORAGE_BUCKET_NAME + " is blank"); } @@ -483,12 +262,35 @@ public class GcsStorageOperator implements Closeable, StorageOperate { } } - private String findDirAlias(String dirPath) { - if (!dirPath.endsWith(FOLDER_SEPARATOR)) { - return dirPath; + private StorageEntity transformBlobToStorageEntity(Blob blob) { + String absolutePath = transformGcsKeyToAbsolutePath(blob.getName()); + + ResourceMetadata resourceMetaData = getResourceMetaData(absolutePath); + + StorageEntity entity = new StorageEntity(); + entity.setFileName(new File(absolutePath).getName()); + entity.setFullName(absolutePath); + entity.setDirectory(resourceMetaData.isDirectory()); + entity.setType(resourceMetaData.getResourceType()); + entity.setSize(blob.getSize()); + entity.setCreateTime(Date.from(blob.getCreateTimeOffsetDateTime().toInstant())); + entity.setUpdateTime(Date.from(blob.getUpdateTimeOffsetDateTime().toInstant())); + return entity; + } + + private String transformAbsolutePathToGcsKey(String absolutePath) { + ResourceMetadata resourceMetaData = getResourceMetaData(absolutePath); + if (resourceMetaData.isDirectory()) { + return FileUtils.concatFilePath(absolutePath, "/"); } + return absolutePath; + } - Path path = Paths.get(dirPath); - return path.getName(path.getNameCount() - 1) + FOLDER_SEPARATOR; + private String transformGcsKeyToAbsolutePath(String gcsKey) { + if (gcsKey.endsWith("/")) { + return gcsKey.substring(0, gcsKey.length() - 1); + } + return gcsKey; } + } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperatorFactory.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperatorFactory.java index eedd41c01f..2bc1a4bfcb 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperatorFactory.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperatorFactory.java @@ -17,20 +17,29 @@ package org.apache.dolphinscheduler.plugin.storage.gcs; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperateFactory; +import org.apache.dolphinscheduler.common.constants.Constants; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperatorFactory; import org.apache.dolphinscheduler.plugin.storage.api.StorageType; import com.google.auto.service.AutoService; -@AutoService(StorageOperateFactory.class) -public class GcsStorageOperatorFactory implements StorageOperateFactory { +@AutoService(StorageOperatorFactory.class) +public class GcsStorageOperatorFactory implements StorageOperatorFactory { @Override - public StorageOperate createStorageOperate() { - GcsStorageOperator gcsStorageOperator = new GcsStorageOperator(); - gcsStorageOperator.init(); - return gcsStorageOperator; + public StorageOperator createStorageOperate() { + final GcsStorageProperties gcsStorageProperties = getGcsStorageProperties(); + return new GcsStorageOperator(gcsStorageProperties); + } + + public GcsStorageProperties getGcsStorageProperties() { + return GcsStorageProperties.builder() + .resourceUploadPath(PropertyUtils.getString(Constants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler")) + .credential(PropertyUtils.getString(Constants.GOOGLE_CLOUD_STORAGE_CREDENTIAL)) + .bucketName(PropertyUtils.getString(Constants.GOOGLE_CLOUD_STORAGE_BUCKET_NAME)) + .build(); } @Override diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageProperties.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageProperties.java new file mode 100644 index 0000000000..5fd1b4b747 --- /dev/null +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageProperties.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.storage.gcs; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class GcsStorageProperties { + + private String bucketName; + + private String credential; + + private String resourceUploadPath; +} diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/test/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperatorTest.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/test/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperatorTest.java deleted file mode 100644 index eecde956d7..0000000000 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/test/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperatorTest.java +++ /dev/null @@ -1,290 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.storage.gcs; - -import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPARATOR; -import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; -import org.apache.dolphinscheduler.spi.enums.ResourceType; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; - -import com.google.cloud.storage.BlobId; -import com.google.cloud.storage.BlobInfo; -import com.google.cloud.storage.Storage; - -@ExtendWith(MockitoExtension.class) -public class GcsStorageOperatorTest { - - private static final String CREDENTIAL_MOCK = "CREDENTIAL_MOCK"; - - private static final String BUCKET_NAME_MOCK = "BUCKET_NAME_MOCK"; - - private static final String TENANT_CODE_MOCK = "TENANT_CODE_MOCK"; - - private static final String DIR_MOCK = "DIR_MOCK"; - - private static final String FILE_NAME_MOCK = "FILE_NAME_MOCK"; - - private static final String FILE_PATH_MOCK = "FILE_PATH_MOCK"; - - private static final String FULL_NAME = "/tmp/dir1/"; - - private static final String DEFAULT_PATH = "/tmp/"; - - @Mock - private Storage gcsStorage; - - private GcsStorageOperator gcsStorageOperator; - - @BeforeEach - public void setUp() throws Exception { - gcsStorageOperator = Mockito.spy(GcsStorageOperator.class); - Mockito.doReturn(CREDENTIAL_MOCK).when(gcsStorageOperator).readCredentials(); - Mockito.doReturn(BUCKET_NAME_MOCK).when(gcsStorageOperator).readBucketName(); - Mockito.doReturn(gcsStorage).when(gcsStorageOperator).buildGcsStorage(Mockito.anyString()); - Mockito.doNothing().when(gcsStorageOperator).checkBucketNameExists(Mockito.anyString()); - - gcsStorageOperator.init(); - } - - @Test - public void testInit() throws Exception { - verify(gcsStorageOperator, times(1)).buildGcsStorage(CREDENTIAL_MOCK); - Assertions.assertEquals(CREDENTIAL_MOCK, gcsStorageOperator.getCredential()); - Assertions.assertEquals(BUCKET_NAME_MOCK, gcsStorageOperator.getBucketName()); - } - - @Test - public void testClose() throws Exception { - doNothing().when(gcsStorage).close(); - gcsStorageOperator.close(); - verify(gcsStorage, times(1)).close(); - } - - @Test - public void createTenantResAndUdfDir() throws Exception { - doReturn(DIR_MOCK).when(gcsStorageOperator).getGcsResDir(TENANT_CODE_MOCK); - doReturn(DIR_MOCK).when(gcsStorageOperator).getGcsUdfDir(TENANT_CODE_MOCK); - doReturn(true).when(gcsStorageOperator).mkdir(TENANT_CODE_MOCK, DIR_MOCK); - gcsStorageOperator.createTenantDirIfNotExists(TENANT_CODE_MOCK); - verify(gcsStorageOperator, times(2)).mkdir(TENANT_CODE_MOCK, DIR_MOCK); - } - - @Test - public void getResDir() { - final String expectedResourceDir = String.format("dolphinscheduler/%s/resources/", TENANT_CODE_MOCK); - final String dir = gcsStorageOperator.getResDir(TENANT_CODE_MOCK); - Assertions.assertEquals(expectedResourceDir, dir); - } - - @Test - public void getUdfDir() { - final String expectedUdfDir = String.format("dolphinscheduler/%s/udfs/", TENANT_CODE_MOCK); - final String dir = gcsStorageOperator.getUdfDir(TENANT_CODE_MOCK); - Assertions.assertEquals(expectedUdfDir, dir); - } - - @Test - public void mkdirWhenDirExists() { - boolean isSuccess = false; - try { - final String key = DIR_MOCK + FOLDER_SEPARATOR; - Mockito.doReturn(true).when(gcsStorageOperator).isObjectExists(key); - isSuccess = gcsStorageOperator.mkdir(TENANT_CODE_MOCK, DIR_MOCK); - - } catch (IOException e) { - Assertions.fail("test failed due to unexpected IO exception"); - } - - Assertions.assertTrue(isSuccess); - } - - @Test - public void mkdirWhenDirNotExists() { - boolean isSuccess = true; - try { - final String key = DIR_MOCK + FOLDER_SEPARATOR; - doReturn(false).when(gcsStorageOperator).isObjectExists(key); - isSuccess = gcsStorageOperator.mkdir(TENANT_CODE_MOCK, DIR_MOCK); - verify(gcsStorage, times(1)).create(Mockito.any(BlobInfo.class), Mockito.any(byte[].class)); - } catch (IOException e) { - Assertions.fail("test failed due to unexpected IO exception"); - } - - Assertions.assertTrue(isSuccess); - } - - @Test - public void getResourceFullName() { - final String expectedResourceFullName = - String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK); - final String resourceFullName = gcsStorageOperator.getResourceFullName(TENANT_CODE_MOCK, FILE_NAME_MOCK); - Assertions.assertEquals(expectedResourceFullName, resourceFullName); - } - - @Test - public void getResourceFileName() { - final String expectedResourceFileName = FILE_NAME_MOCK; - final String resourceFullName = - String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK); - final String resourceFileName = gcsStorageOperator.getResourceFileName(TENANT_CODE_MOCK, resourceFullName); - Assertions.assertEquals(expectedResourceFileName, resourceFileName); - } - - @Test - public void getFileName() { - final String expectedFileName = - String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK); - final String fileName = gcsStorageOperator.getFileName(ResourceType.FILE, TENANT_CODE_MOCK, FILE_NAME_MOCK); - Assertions.assertEquals(expectedFileName, fileName); - } - - @Test - public void exists() { - boolean doesExist = false; - doReturn(true).when(gcsStorageOperator).isObjectExists(FILE_NAME_MOCK); - try { - doesExist = gcsStorageOperator.exists(FILE_NAME_MOCK); - } catch (IOException e) { - Assertions.fail("unexpected IO exception in unit test"); - } - - Assertions.assertTrue(doesExist); - } - - @Test - public void delete() { - boolean isDeleted = false; - doReturn(true).when(gcsStorage).delete(Mockito.any(BlobId.class)); - doReturn(true).when(gcsStorageOperator).isObjectExists(FILE_NAME_MOCK); - try { - isDeleted = gcsStorageOperator.delete(FILE_NAME_MOCK, true); - } catch (IOException e) { - Assertions.fail("unexpected IO exception in unit test"); - } - - Assertions.assertTrue(isDeleted); - verify(gcsStorage, times(1)).delete(Mockito.any(BlobId.class)); - } - - @Test - public void copy() { - boolean isSuccess = false; - doReturn(null).when(gcsStorage).copy(Mockito.any()); - try { - isSuccess = gcsStorageOperator.copy(FILE_PATH_MOCK, FILE_PATH_MOCK, false, false); - } catch (IOException e) { - Assertions.fail("unexpected IO exception in unit test"); - } - - Assertions.assertTrue(isSuccess); - verify(gcsStorage, times(1)).copy(Mockito.any()); - } - - @Test - public void deleteTenant() { - doNothing().when(gcsStorageOperator).deleteTenantCode(anyString()); - try { - gcsStorageOperator.deleteTenant(TENANT_CODE_MOCK); - } catch (Exception e) { - Assertions.fail("unexpected exception caught in unit test"); - } - - verify(gcsStorageOperator, times(1)).deleteTenantCode(anyString()); - } - - @Test - public void getGcsResDir() { - final String expectedGcsResDir = String.format("dolphinscheduler/%s/resources", TENANT_CODE_MOCK); - final String gcsResDir = gcsStorageOperator.getGcsResDir(TENANT_CODE_MOCK); - Assertions.assertEquals(expectedGcsResDir, gcsResDir); - } - - @Test - public void getGcsUdfDir() { - final String expectedGcsUdfDir = String.format("dolphinscheduler/%s/udfs", TENANT_CODE_MOCK); - final String gcsUdfDir = gcsStorageOperator.getGcsUdfDir(TENANT_CODE_MOCK); - Assertions.assertEquals(expectedGcsUdfDir, gcsUdfDir); - } - - @Test - public void getGcsTenantDir() { - final String expectedGcsTenantDir = String.format(FORMAT_S_S, DIR_MOCK, TENANT_CODE_MOCK); - doReturn(DIR_MOCK).when(gcsStorageOperator).getGcsDataBasePath(); - final String gcsTenantDir = gcsStorageOperator.getGcsTenantDir(TENANT_CODE_MOCK); - Assertions.assertEquals(expectedGcsTenantDir, gcsTenantDir); - } - - @Test - public void deleteDir() { - doReturn(true).when(gcsStorageOperator).isObjectExists(Mockito.any()); - gcsStorageOperator.deleteDirectory(DIR_MOCK); - verify(gcsStorage, times(1)).delete(Mockito.any(BlobId.class)); - } - - @Test - public void testGetFileStatus() throws Exception { - StorageEntity entity = - gcsStorageOperator.getFileStatus(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, ResourceType.FILE); - Assertions.assertEquals(FULL_NAME, entity.getFullName()); - Assertions.assertEquals("dir1/", entity.getFileName()); - } - - @Test - public void testListFilesStatus() throws Exception { - Mockito.doReturn(null).when(gcsStorage).list(Mockito.any(), Mockito.any(Storage.BlobListOption.class), - Mockito.any(Storage.BlobListOption.class)); - List result = - gcsStorageOperator.listFilesStatus(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, ResourceType.FILE); - verify(gcsStorage, times(1)).list(Mockito.any(), Mockito.any(Storage.BlobListOption.class), - Mockito.any(Storage.BlobListOption.class)); - } - - @Test - public void testListFilesStatusRecursively() throws Exception { - StorageEntity entity = new StorageEntity(); - entity.setFullName(FULL_NAME); - - doReturn(entity).when(gcsStorageOperator).getFileStatus(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, - ResourceType.FILE); - doReturn(Collections.EMPTY_LIST).when(gcsStorageOperator).listFilesStatus(anyString(), anyString(), anyString(), - Mockito.any(ResourceType.class)); - - List result = - gcsStorageOperator.listFilesStatusRecursively(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, - ResourceType.FILE); - Assertions.assertEquals(0, result.size()); - } -} diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/pom.xml b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/pom.xml index 2a07584948..04e06df244 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/pom.xml +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/pom.xml @@ -228,5 +228,11 @@ + + + org.testcontainers + testcontainers + test + diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java index 4479c93a3d..11d2549f00 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java @@ -17,340 +17,142 @@ package org.apache.dolphinscheduler.plugin.storage.hdfs; -import static org.apache.dolphinscheduler.common.constants.Constants.EMPTY_STRING; -import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPARATOR; -import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S; -import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TYPE_FILE; -import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TYPE_UDF; - import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.enums.ResUploadType; -import org.apache.dolphinscheduler.common.exception.BaseException; -import org.apache.dolphinscheduler.common.utils.HttpUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.KerberosHttpClient; -import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils; +import org.apache.dolphinscheduler.plugin.storage.api.AbstractStorageOperator; +import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata; import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; -import org.apache.dolphinscheduler.spi.enums.ResourceType; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; -import org.apache.commons.io.IOUtils; +import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.security.UserGroupInformation; import java.io.BufferedReader; import java.io.Closeable; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; +import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; import java.security.PrivilegedExceptionAction; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.LinkedList; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; -import java.util.stream.Stream; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import com.fasterxml.jackson.databind.node.ObjectNode; - @Slf4j -public class HdfsStorageOperator implements Closeable, StorageOperate { - - protected static HdfsStorageProperties hdfsProperties = new HdfsStorageProperties(); - private static final String HADOOP_UTILS_KEY = "HADOOP_UTILS_KEY"; +public class HdfsStorageOperator extends AbstractStorageOperator implements Closeable, StorageOperator { - private volatile boolean yarnEnabled = false; + private final HdfsStorageProperties hdfsProperties; private Configuration configuration; private FileSystem fs; - public HdfsStorageOperator() { - this(new HdfsStorageProperties()); - } - public HdfsStorageOperator(HdfsStorageProperties hdfsStorageProperties) { + super(hdfsStorageProperties.getResourceUploadPath()); // Overwrite config from passing hdfsStorageProperties hdfsProperties = hdfsStorageProperties; init(); initHdfsPath(); } - /** - * init dolphinscheduler root path in hdfs - */ - + @SneakyThrows private void initHdfsPath() { - Path path = new Path(RESOURCE_UPLOAD_PATH); - try { - if (!fs.exists(path)) { - fs.mkdirs(path); - } - } catch (Exception e) { - log.error(e.getMessage(), e); + Path path = new Path(resourceBaseAbsolutePath); + if (!fs.exists(path)) { + fs.mkdirs(path); + log.info("Create hdfs path: {}", path); } } - /** - * init hadoop configuration - */ - public void init() throws NullPointerException { - try { - configuration = new HdfsConfiguration(); - - String hdfsUser = hdfsProperties.getUser(); - if (CommonUtils.loadKerberosConf(configuration)) { - hdfsUser = ""; - } - - String defaultFS = getDefaultFS(); - // first get key from core-site.xml hdfs-site.xml ,if null ,then try to get from properties file - // the default is the local file system - if (StringUtils.isNotBlank(defaultFS)) { - Map fsRelatedProps = PropertyUtils.getByPrefix("fs."); - configuration.set(Constants.HDFS_DEFAULT_FS, defaultFS); - fsRelatedProps.forEach((key, value) -> configuration.set(key, value)); - } else { - log.error("property:{} can not to be empty, please set!", Constants.FS_DEFAULT_FS); - throw new NullPointerException( - String.format("property: %s can not to be empty, please set!", Constants.FS_DEFAULT_FS)); - } - - if (!defaultFS.startsWith("file")) { - log.info("get property:{} -> {}, from core-site.xml hdfs-site.xml ", Constants.FS_DEFAULT_FS, - defaultFS); - } - - if (StringUtils.isNotEmpty(hdfsUser)) { - UserGroupInformation ugi = UserGroupInformation.createRemoteUser(hdfsUser); - ugi.doAs((PrivilegedExceptionAction) () -> { - fs = FileSystem.get(configuration); - return true; - }); - } else { - log.warn("resource.hdfs.root.user is not set value!"); - fs = FileSystem.get(configuration); - } + @SneakyThrows + private void init() { + configuration = new HdfsConfiguration(); - } catch (Exception e) { - log.error(e.getMessage(), e); + if (MapUtils.isNotEmpty(hdfsProperties.getConfigurationProperties())) { + hdfsProperties.getConfigurationProperties().forEach((key, value) -> { + configuration.set(key, value); + log.info("Set HDFS prop: {} -> {}", key, value); + }); } - } - - /** - * @return Configuration - */ - public Configuration getConfiguration() { - return configuration; - } - /** - * @return DefaultFS - */ - public String getDefaultFS() { String defaultFS = hdfsProperties.getDefaultFS(); - if (StringUtils.isBlank(defaultFS)) { - defaultFS = getConfiguration().get(Constants.HDFS_DEFAULT_FS); - } - return defaultFS; - } - - /** - * get application url - * if rmHaIds contains xx, it signs not use resourcemanager - * otherwise: - * if rmHaIds is empty, single resourcemanager enabled - * if rmHaIds not empty: resourcemanager HA enabled - * - * @param applicationId application id - * @return url of application - */ - public String getApplicationUrl(String applicationId) throws BaseException { - - yarnEnabled = true; - String appUrl = StringUtils.isEmpty(hdfsProperties.getYarnResourceRmIds()) - ? hdfsProperties.getYarnAppStatusAddress() - : getAppAddress(hdfsProperties.getYarnAppStatusAddress(), hdfsProperties.getYarnResourceRmIds()); - if (StringUtils.isBlank(appUrl)) { - throw new BaseException("yarn application url generation failed"); - } - log.debug("yarn application url:{}, applicationId:{}", appUrl, applicationId); - return String.format(appUrl, hdfsProperties.getHadoopResourceManagerHttpAddressPort(), applicationId); - } - - public String getJobHistoryUrl(String applicationId) { - // eg:application_1587475402360_712719 -> job_1587475402360_712719 - String jobId = applicationId.replace("application", "job"); - return String.format(hdfsProperties.getYarnJobHistoryStatusAddress(), jobId); - } - - /** - * cat file on hdfs - * - * @param hdfsFilePath hdfs file path - * @return byte[] byte array - * @throws IOException errors - */ - public byte[] catFile(String hdfsFilePath) throws IOException { - - if (StringUtils.isBlank(hdfsFilePath)) { - log.error("hdfs file path:{} is blank", hdfsFilePath); - return new byte[0]; - } - - try (FSDataInputStream fsDataInputStream = fs.open(new Path(hdfsFilePath))) { - return IOUtils.toByteArray(fsDataInputStream); + if (StringUtils.isNotEmpty(defaultFS)) { + configuration.set(Constants.HDFS_DEFAULT_FS, hdfsProperties.getDefaultFS()); } - } - /** - * cat file on hdfs - * - * @param hdfsFilePath hdfs file path - * @param skipLineNums skip line numbers - * @param limit read how many lines - * @return content of file - * @throws IOException errors - */ - public List catFile(String hdfsFilePath, int skipLineNums, int limit) throws IOException { - - if (StringUtils.isBlank(hdfsFilePath)) { - log.error("hdfs file path:{} is blank", hdfsFilePath); - return Collections.emptyList(); + if (CommonUtils.getKerberosStartupState()) { + CommonUtils.loadKerberosConf(configuration); + fs = FileSystem.get(configuration); + log.info("Initialize HdfsStorageOperator with kerberos"); + return; } - - try (FSDataInputStream in = fs.open(new Path(hdfsFilePath))) { - BufferedReader br = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8)); - Stream stream = br.lines().skip(skipLineNums).limit(limit); - return stream.collect(Collectors.toList()); + if (StringUtils.isNotEmpty(hdfsProperties.getUser())) { + UserGroupInformation ugi = UserGroupInformation.createRemoteUser(hdfsProperties.getUser()); + ugi.doAs((PrivilegedExceptionAction) () -> { + fs = FileSystem.get(configuration); + return true; + }); + UserGroupInformation.setLoginUser(ugi); + log.info("Initialize HdfsStorageOperator with remote user: {}", hdfsProperties.getUser()); + return; } - } - - @Override - public List vimFile(String bucketName, String hdfsFilePath, int skipLineNums, - int limit) throws IOException { - return catFile(hdfsFilePath, skipLineNums, limit); - } - - @Override - public void createTenantDirIfNotExists(String tenantCode) throws IOException { - mkdir(tenantCode, getHdfsResDir(tenantCode)); - mkdir(tenantCode, getHdfsUdfDir(tenantCode)); - } - - @Override - public String getResDir(String tenantCode) { - return getHdfsResDir(tenantCode) + FOLDER_SEPARATOR; - } - - @Override - public String getUdfDir(String tenantCode) { - return getHdfsUdfDir(tenantCode) + FOLDER_SEPARATOR; - } - - /** - * make the given file and all non-existent parents into - * directories. Has the semantics of Unix 'mkdir -p'. - * Existence of the directory hierarchy is not an error. - * - * @param hdfsPath path to create - * @return mkdir result - * @throws IOException errors - */ - @Override - public boolean mkdir(String tenantCode, String hdfsPath) throws IOException { - return fs.mkdirs(new Path(addFolderSeparatorIfNotExisted(hdfsPath))); - } + fs = FileSystem.get(configuration); + log.info("Initialize HdfsStorageOperator with default user"); - @Override - public String getResourceFullName(String tenantCode, String fullName) { - return getHdfsResourceFileName(tenantCode, fullName); } @Override - public String getFileName(ResourceType resourceType, String tenantCode, String fileName) { - return getHdfsFileName(resourceType, tenantCode, fileName); + public String getStorageBaseDirectory() { + String defaultFS = hdfsProperties.getDefaultFS(); + return FileUtils.concatFilePath(defaultFS, resourceBaseAbsolutePath); } + @SneakyThrows @Override - public void download(String srcHdfsFilePath, String dstFile, boolean overwrite) throws IOException { - copyHdfsToLocal(srcHdfsFilePath, dstFile, false, overwrite); + public List fetchFileContent(String hdfsFilePath, int skipLineNums, int limit) { + try ( + FSDataInputStream in = fs.open(new Path(hdfsFilePath)); + InputStreamReader inputStreamReader = new InputStreamReader(in, StandardCharsets.UTF_8); + BufferedReader br = new BufferedReader(inputStreamReader)) { + return br.lines() + .skip(skipLineNums) + .limit(limit) + .collect(Collectors.toList()); + } } - /** - * copy files between FileSystems - * - * @param srcPath source hdfs path - * @param dstPath destination hdfs path - * @param deleteSource whether to delete the src - * @param overwrite whether to overwrite an existing file - * @return if success or not - * @throws IOException errors - */ @Override - public boolean copy(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) throws IOException { - return FileUtil.copy(fs, new Path(srcPath), fs, new Path(dstPath), deleteSource, overwrite, fs.getConf()); - } - - /** - * the src file is on the local disk. Add it to FS at - * the given dst name. - * - * @param srcFile local file - * @param dstHdfsPath destination hdfs path - * @param deleteSource whether to delete the src - * @param overwrite whether to overwrite an existing file - * @return if success or not - * @throws IOException errors - */ - public boolean copyLocalToHdfs(String srcFile, String dstHdfsPath, boolean deleteSource, - boolean overwrite) throws IOException { - Path srcPath = new Path(srcFile); - Path dstPath = new Path(dstHdfsPath); - - fs.copyFromLocalFile(deleteSource, overwrite, srcPath, dstPath); - - return true; + @SneakyThrows + public void createStorageDir(String directoryAbsolutePath) { + Path path = new Path(directoryAbsolutePath); + if (fs.exists(path)) { + throw new FileAlreadyExistsException("Directory already exists: " + directoryAbsolutePath); + } + fs.mkdirs(new Path(directoryAbsolutePath)); } + @SneakyThrows @Override - public boolean upload(String buckName, String srcFile, String dstPath, boolean deleteSource, - boolean overwrite) throws IOException { - return copyLocalToHdfs(srcFile, dstPath, deleteSource, overwrite); - } - - /** - * copy hdfs file to local - * - * @param srcHdfsFilePath source hdfs file path - * - * @param dstFile destination file - * - * @param deleteSource delete source - * - * @param overwrite overwrite - * - * @return result of copy hdfs file to local - * - * @throws IOException errors - */ - public boolean copyHdfsToLocal(String srcHdfsFilePath, String dstFile, boolean deleteSource, - boolean overwrite) throws IOException { - + public void download(String srcHdfsFilePath, String dstFile, boolean overwrite) { Path srcPath = new Path(srcHdfsFilePath); File dstPath = new File(dstFile); @@ -365,336 +167,83 @@ public class HdfsStorageOperator implements Closeable, StorageOperate { } if (!dstPath.getParentFile().exists() && !dstPath.getParentFile().mkdirs()) { - return false; + throw new IOException("Failed to create parent directory for destination file"); } - return FileUtil.copy(fs, srcPath, dstPath, deleteSource, fs.getConf()); + FileUtil.copy(fs, srcPath, dstPath, false, fs.getConf()); } - /** - * delete a file - * - * @param hdfsFilePath the path to delete. - * @param recursive if path is a directory and set to - * true, the directory is deleted else throws an exception. In - * case of a file the recursive can be set to either true or false. - * @return true if delete is successful else false. - * @throws IOException errors - */ + @SneakyThrows @Override - public boolean delete(String hdfsFilePath, boolean recursive) throws IOException { - return fs.delete(new Path(hdfsFilePath), recursive); + public void copy(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) { + FileUtil.copy(fs, new Path(srcPath), fs, new Path(dstPath), deleteSource, overwrite, fs.getConf()); } - /** - * delete a list of files - * - * @param filePath the path to delete, usually it is a directory. - * @param recursive if path is a directory and set to - * true, the directory is deleted else throws an exception. In - * case of a file the recursive can be set to either true or false. - * @return true if delete is successful else false. - * @throws IOException errors - */ - + @SneakyThrows @Override - public boolean delete(String filePath, List childrenPathArray, boolean recursive) throws IOException { - if (filePath.endsWith("/")) { - return fs.delete(new Path(filePath), true); - } - return fs.delete(new Path(filePath), recursive); + public void upload(String srcAbsoluteFilePath, + String dstAbsoluteFilePath, + boolean deleteSource, + boolean overwrite) { + Path srcPath = new Path(srcAbsoluteFilePath); + Path dstPath = new Path(dstAbsoluteFilePath); + fs.copyFromLocalFile(deleteSource, overwrite, srcPath, dstPath); } - /** - * check if exists - * - * @param hdfsFilePath source file path - * @return result of exists or not - * @throws IOException errors - */ + @SneakyThrows @Override - public boolean exists(String hdfsFilePath) throws IOException { - return fs.exists(new Path(hdfsFilePath)); + public void delete(String resourceAbsolutePath, boolean recursive) { + exceptionIfPathEmpty(resourceAbsolutePath); + fs.delete(new Path(resourceAbsolutePath), recursive); } - /** - * Gets a list of files in the directory - * - * @param path file fullName path - * @return {@link FileStatus} file status - * @throws IOException errors - */ + @SneakyThrows @Override - public List listFilesStatus(String path, String defaultPath, String tenantCode, - ResourceType type) throws IOException { - // TODO: Does listStatus truncate resultList if its size goes above certain threshold (like a 1000 in S3) - // TODO: add hdfs prefix getFile - List storageEntityList = new ArrayList<>(); - try { - Path filePath = new Path(path); - if (!fs.exists(filePath)) { - return storageEntityList; - } - FileStatus[] fileStatuses = fs.listStatus(filePath); - - // transform FileStatusArray into the StorageEntity List - for (FileStatus fileStatus : fileStatuses) { - if (fileStatus.isDirectory()) { - // the path is a directory - String fullName = fileStatus.getPath().toString(); - fullName = addFolderSeparatorIfNotExisted(fullName); - - String suffix = StringUtils.difference(path, fullName); - String fileName = StringUtils.difference(defaultPath, fullName); - - StorageEntity entity = new StorageEntity(); - entity.setAlias(suffix); - entity.setFileName(fileName); - entity.setFullName(fullName); - entity.setDirectory(true); - entity.setUserName(tenantCode); - entity.setType(type); - entity.setSize(fileStatus.getLen()); - entity.setCreateTime(new Date(fileStatus.getModificationTime())); - entity.setUpdateTime(new Date(fileStatus.getModificationTime())); - entity.setPfullName(path); - - storageEntityList.add(entity); - } else { - // the path is a file - String fullName = fileStatus.getPath().toString(); - String[] aliasArr = fullName.split("/"); - String alias = aliasArr[aliasArr.length - 1]; - - String fileName = StringUtils.difference(defaultPath, fullName); - - StorageEntity entity = new StorageEntity(); - entity.setAlias(alias); - entity.setFileName(fileName); - entity.setFullName(fullName); - entity.setDirectory(false); - entity.setUserName(tenantCode); - entity.setType(type); - entity.setSize(fileStatus.getLen()); - entity.setCreateTime(new Date(fileStatus.getModificationTime())); - entity.setUpdateTime(new Date(fileStatus.getModificationTime())); - entity.setPfullName(path); - - storageEntityList.add(entity); - } - } - } catch (FileNotFoundException e) { - throw new FileNotFoundException("The path does not exist."); - } catch (IOException e) { - throw new IOException("Get file list exception.", e); - } - - return storageEntityList; + public boolean exists(String resourceAbsolutePath) { + exceptionIfPathEmpty(resourceAbsolutePath); + return fs.exists(new Path(resourceAbsolutePath)); } + @SneakyThrows @Override - public StorageEntity getFileStatus(String path, String prefix, String tenantCode, - ResourceType type) throws IOException { - try { - FileStatus fileStatus = fs.getFileStatus(new Path(path)); - String alias = ""; - String fileName = ""; - String fullName = fileStatus.getPath().toString(); - if (fileStatus.isDirectory()) { - fullName = addFolderSeparatorIfNotExisted(fullName); - alias = findDirAlias(fullName); - fileName = StringUtils.difference(prefix, fullName); - } else { - String[] aliasArr = fileStatus.getPath().toString().split("/"); - alias = aliasArr[aliasArr.length - 1]; - fileName = StringUtils.difference(prefix, fileStatus.getPath().toString()); - } - - StorageEntity entity = new StorageEntity(); - entity.setAlias(alias); - entity.setFileName(fileName); - entity.setFullName(fullName); - entity.setDirectory(fileStatus.isDirectory()); - entity.setUserName(tenantCode); - entity.setType(type); - entity.setSize(fileStatus.getLen()); - entity.setCreateTime(new Date(fileStatus.getModificationTime())); - entity.setUpdateTime(new Date(fileStatus.getModificationTime())); - entity.setPfullName(path); - - return entity; - } catch (FileNotFoundException e) { - throw new FileNotFoundException("The path does not exist."); - } catch (IOException e) { - throw new IOException("Get file exception.", e); - } - } - - /** - * Renames Path src to Path dst. Can take place on local fs - * or remote DFS. - * - * @param src path to be renamed - * @param dst new path after rename - * @return true if rename is successful - * @throws IOException on failure - */ - public boolean rename(String src, String dst) throws IOException { - return fs.rename(new Path(src), new Path(dst)); - } - - /** - * hadoop resourcemanager enabled or not - * - * @return result - */ - public boolean isYarnEnabled() { - return yarnEnabled; - } - - /** - * get data hdfs path - * - * @return data hdfs path - */ - public static String getHdfsDataBasePath() { - String defaultFS = hdfsProperties.getDefaultFS(); - defaultFS = defaultFS.endsWith("/") ? StringUtils.chop(defaultFS) : defaultFS; - if (FOLDER_SEPARATOR.equals(RESOURCE_UPLOAD_PATH)) { - return defaultFS + ""; - } else { - return defaultFS + RESOURCE_UPLOAD_PATH; - } - } - - /** - * hdfs resource dir - * - * @param tenantCode tenant code - * @param resourceType resource type - * @return hdfs resource dir - */ - public static String getHdfsDir(ResourceType resourceType, String tenantCode) { - switch (resourceType) { - case UDF: - return getHdfsUdfDir(tenantCode); - case FILE: - return getHdfsResDir(tenantCode); - case ALL: - return getHdfsDataBasePath(); - default: - return EMPTY_STRING; + public List listStorageEntity(String resourceAbsolutePath) { + exceptionIfPathEmpty(resourceAbsolutePath); + Path path = new Path(resourceAbsolutePath); + if (!fs.exists(path)) { + return Collections.emptyList(); } + return Arrays.stream(fs.listStatus(new Path(resourceAbsolutePath))) + .map(this::transformFileStatusToResourceMetadata) + .collect(Collectors.toList()); } + @SneakyThrows @Override - public String getDir(ResourceType resourceType, String tenantCode) { - return getHdfsDir(resourceType, tenantCode); - } - - /** - * hdfs resource dir - * - * @param tenantCode tenant code - * @return hdfs resource dir - */ - public static String getHdfsResDir(String tenantCode) { - return String.format("%s/" + RESOURCE_TYPE_FILE, getHdfsTenantDir(tenantCode)); - } - - /** - * hdfs udf dir - * - * @param tenantCode tenant code - * @return get udf dir on hdfs - */ - public static String getHdfsUdfDir(String tenantCode) { - return String.format("%s/" + RESOURCE_TYPE_UDF, getHdfsTenantDir(tenantCode)); - } + public List listFileStorageEntityRecursively(String resourceAbsolutePath) { + exceptionIfPathEmpty(resourceAbsolutePath); - /** - * get hdfs file name - * - * @param resourceType resource type - * @param tenantCode tenant code - * @param fileName file name - * @return hdfs file name - */ - public static String getHdfsFileName(ResourceType resourceType, String tenantCode, String fileName) { - if (fileName.startsWith(FOLDER_SEPARATOR)) { - fileName = fileName.replaceFirst(FOLDER_SEPARATOR, ""); - } - return String.format(FORMAT_S_S, getHdfsDir(resourceType, tenantCode), fileName); - } + List result = new ArrayList<>(); - /** - * get absolute path and name for resource file on hdfs - * - * @param tenantCode tenant code - * @param fileName file name - * @return get absolute path and name for file on hdfs - */ - public static String getHdfsResourceFileName(String tenantCode, String fileName) { - if (fileName.startsWith(FOLDER_SEPARATOR)) { - fileName = fileName.replaceFirst(FOLDER_SEPARATOR, ""); - } - return String.format(FORMAT_S_S, getHdfsResDir(tenantCode), fileName); - } + LinkedList foldersToFetch = new LinkedList<>(); + foldersToFetch.addLast(resourceAbsolutePath); - /** - * get absolute path and name for udf file on hdfs - * - * @param tenantCode tenant code - * @param fileName file name - * @return get absolute path and name for udf file on hdfs - */ - public static String getHdfsUdfFileName(String tenantCode, String fileName) { - if (fileName.startsWith(FOLDER_SEPARATOR)) { - fileName = fileName.replaceFirst(FOLDER_SEPARATOR, ""); + while (!foldersToFetch.isEmpty()) { + String absolutePath = foldersToFetch.pollFirst(); + RemoteIterator remoteIterator = fs.listFiles(new Path(absolutePath), true); + while (remoteIterator.hasNext()) { + LocatedFileStatus locatedFileStatus = remoteIterator.next(); + result.add(transformFileStatusToResourceMetadata(locatedFileStatus)); + } } - return String.format(FORMAT_S_S, getHdfsUdfDir(tenantCode), fileName); - } - - /** - * @param tenantCode tenant code - * @return file directory of tenants on hdfs - */ - public static String getHdfsTenantDir(String tenantCode) { - return String.format(FORMAT_S_S, getHdfsDataBasePath(), tenantCode); + return result; } - /** - * getAppAddress - * - * @param appAddress app address - * @param rmHa resource manager ha - * @return app address - */ - public static String getAppAddress(String appAddress, String rmHa) { - - String[] split1 = appAddress.split(Constants.DOUBLE_SLASH); - - if (split1.length != 2) { - return null; - } - - String start = split1[0] + Constants.DOUBLE_SLASH; - String[] split2 = split1[1].split(Constants.COLON); - - if (split2.length != 2) { - return null; - } - - String end = Constants.COLON + split2[1]; - - // get active ResourceManager - String activeRM = YarnHAAdminUtils.getActiveRMName(start, rmHa); - - if (StringUtils.isEmpty(activeRM)) { - return null; - } - - return start + activeRM + end; + @SneakyThrows + @Override + public StorageEntity getStorageEntity(String resourceAbsolutePath) { + exceptionIfPathEmpty(resourceAbsolutePath); + FileStatus fileStatus = fs.getFileStatus(new Path(resourceAbsolutePath)); + return transformFileStatusToResourceMetadata(fileStatus); } @Override @@ -709,144 +258,21 @@ public class HdfsStorageOperator implements Closeable, StorageOperate { } } - /** - * yarn ha admin utils - */ - private static final class YarnHAAdminUtils { + private StorageEntity transformFileStatusToResourceMetadata(FileStatus fileStatus) { + Path fileStatusPath = fileStatus.getPath(); + String fileAbsolutePath = fileStatusPath.toString(); + ResourceMetadata resourceMetaData = getResourceMetaData(fileAbsolutePath); - /** - * get active resourcemanager node - * - * @param protocol http protocol - * @param rmIds yarn ha ids - * @return yarn active node - */ - public static String getActiveRMName(String protocol, String rmIds) { - - String[] rmIdArr = rmIds.split(Constants.COMMA); - - String yarnUrl = - protocol + "%s:" + hdfsProperties.getHadoopResourceManagerHttpAddressPort() + "/ws/v1/cluster/info"; - - try { - - /** - * send http get request to rm - */ - - for (String rmId : rmIdArr) { - String state = getRMState(String.format(yarnUrl, rmId)); - if (Constants.HADOOP_RM_STATE_ACTIVE.equals(state)) { - return rmId; - } - } - - } catch (Exception e) { - log.error("yarn ha application url generation failed, message:{}", e.getMessage()); - } - return null; - } - - /** - * get ResourceManager state - */ - public static String getRMState(String url) { - - String retStr = Boolean.TRUE - .equals(hdfsProperties.isHadoopSecurityAuthStartupState()) - ? KerberosHttpClient.get(url) - : HttpUtils.get(url); - - if (StringUtils.isEmpty(retStr)) { - return null; - } - // to json - ObjectNode jsonObject = JSONUtils.parseObject(retStr); - - // get ResourceManager state - if (!jsonObject.has("clusterInfo")) { - return null; - } - return jsonObject.get("clusterInfo").path("haState").asText(); - } - - } - - @Override - public void deleteTenant(String tenantCode) throws Exception { - String tenantPath = getHdfsDataBasePath() + FOLDER_SEPARATOR + tenantCode; - - if (exists(tenantPath)) { - delete(tenantPath, true); - - } - } - - @Override - public ResUploadType returnStorageType() { - return ResUploadType.HDFS; + return StorageEntity.builder() + .fileName(fileStatusPath.getName()) + .fullName(fileAbsolutePath) + .pfullName(resourceMetaData.getResourceParentAbsolutePath()) + .type(resourceMetaData.getResourceType()) + .isDirectory(fileStatus.isDirectory()) + .size(fileStatus.getLen()) + .createTime(new Date(fileStatus.getModificationTime())) + .updateTime(new Date(fileStatus.getModificationTime())) + .build(); } - @Override - public List listFilesStatusRecursively(String path, String defaultPath, String tenantCode, - ResourceType type) { - List storageEntityList = new ArrayList<>(); - - LinkedList foldersToFetch = new LinkedList<>(); - - do { - String pathToExplore = ""; - if (foldersToFetch.size() == 0) { - pathToExplore = path; - } else { - pathToExplore = foldersToFetch.pop().getFullName(); - } - - try { - List tempList = listFilesStatus(pathToExplore, defaultPath, tenantCode, type); - - for (StorageEntity temp : tempList) { - if (temp.isDirectory()) { - foldersToFetch.add(temp); - } - } - - storageEntityList.addAll(tempList); - } catch (FileNotFoundException e) { - log.error("Resource path: {}", pathToExplore, e); - // return the resources fetched before error occurs. - return storageEntityList; - } catch (IOException e) { - log.error("Resource path: {}", pathToExplore, e); - // return the resources fetched before error occurs. - return storageEntityList; - } - - } while (foldersToFetch.size() != 0); - - return storageEntityList; - - } - - /** - * find alias for directories, NOT for files - * a directory is a path ending with "/" - */ - private String findDirAlias(String myStr) { - if (!myStr.endsWith("/")) { - // Make sure system won't crush down if someone accidentally misuse the function. - return myStr; - } - int lastIndex = myStr.lastIndexOf("/"); - String subbedString = myStr.substring(0, lastIndex); - int secondLastIndex = subbedString.lastIndexOf("/"); - StringBuilder stringBuilder = new StringBuilder(); - stringBuilder.append(myStr, secondLastIndex + 1, lastIndex + 1); - - return stringBuilder.toString(); - } - - private String addFolderSeparatorIfNotExisted(String fullName) { - return fullName.endsWith(FOLDER_SEPARATOR) ? fullName : fullName + FOLDER_SEPARATOR; - } } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperatorFactory.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperatorFactory.java index d2a6ef0262..ed2aa60615 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperatorFactory.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperatorFactory.java @@ -17,20 +17,36 @@ package org.apache.dolphinscheduler.plugin.storage.hdfs; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperateFactory; +import static org.apache.dolphinscheduler.common.constants.Constants.FS_DEFAULT_FS; +import static org.apache.dolphinscheduler.common.constants.Constants.HDFS_ROOT_USER; + +import org.apache.dolphinscheduler.common.constants.Constants; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperatorFactory; import org.apache.dolphinscheduler.plugin.storage.api.StorageType; +import java.util.Map; + import com.google.auto.service.AutoService; -@AutoService(StorageOperateFactory.class) -public class HdfsStorageOperatorFactory implements StorageOperateFactory { +@AutoService(StorageOperatorFactory.class) +public class HdfsStorageOperatorFactory implements StorageOperatorFactory { @Override - public StorageOperate createStorageOperate() { - HdfsStorageOperator hdfsOperator = new HdfsStorageOperator(); - hdfsOperator.init(); - return hdfsOperator; + public StorageOperator createStorageOperate() { + final HdfsStorageProperties hdfsStorageProperties = getHdfsStorageProperties(); + return new HdfsStorageOperator(hdfsStorageProperties); + } + + private HdfsStorageProperties getHdfsStorageProperties() { + Map configurationProperties = PropertyUtils.getByPrefix("fs."); + return HdfsStorageProperties.builder() + .user(PropertyUtils.getString(HDFS_ROOT_USER)) + .defaultFS(PropertyUtils.getString(FS_DEFAULT_FS)) + .configurationProperties(configurationProperties) + .resourceUploadPath(PropertyUtils.getString(Constants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler")) + .build(); } @Override diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageProperties.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageProperties.java index a40529f584..48a2f1d179 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageProperties.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageProperties.java @@ -17,66 +17,25 @@ package org.apache.dolphinscheduler.plugin.storage.hdfs; -import static org.apache.dolphinscheduler.common.constants.Constants.FS_DEFAULT_FS; -import static org.apache.dolphinscheduler.common.constants.Constants.HADOOP_RESOURCE_MANAGER_HTTPADDRESS_PORT; -import static org.apache.dolphinscheduler.common.constants.Constants.HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE; -import static org.apache.dolphinscheduler.common.constants.Constants.HDFS_ROOT_USER; -import static org.apache.dolphinscheduler.common.constants.Constants.KERBEROS_EXPIRE_TIME; -import static org.apache.dolphinscheduler.common.constants.Constants.YARN_APPLICATION_STATUS_ADDRESS; -import static org.apache.dolphinscheduler.common.constants.Constants.YARN_JOB_HISTORY_STATUS_ADDRESS; -import static org.apache.dolphinscheduler.common.constants.Constants.YARN_RESOURCEMANAGER_HA_RM_IDS; - -import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import java.util.Map; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; - -import org.springframework.context.annotation.Configuration; +import lombok.NoArgsConstructor; @Data -@Configuration +@Builder +@NoArgsConstructor +@AllArgsConstructor public class HdfsStorageProperties { - /** - * HDFS storage user - */ - private String user = PropertyUtils.getString(HDFS_ROOT_USER); - - /** - * HDFS default fs - */ - private String defaultFS = PropertyUtils.getString(FS_DEFAULT_FS); - - /** - * YARN resource manager HA RM ids - */ - private String yarnResourceRmIds = PropertyUtils.getString(YARN_RESOURCEMANAGER_HA_RM_IDS); - - /** - * YARN application status address - */ - private String yarnAppStatusAddress = PropertyUtils.getString(YARN_APPLICATION_STATUS_ADDRESS); + private String user; - /** - * YARN job history status address - */ - private String yarnJobHistoryStatusAddress = PropertyUtils.getString(YARN_JOB_HISTORY_STATUS_ADDRESS); + private String defaultFS; - /** - * Hadoop resouece manager http address port - */ - private String hadoopResourceManagerHttpAddressPort = - PropertyUtils.getString(HADOOP_RESOURCE_MANAGER_HTTPADDRESS_PORT); + private Map configurationProperties; - /** - * Hadoop security authentication startup state - */ - private boolean hadoopSecurityAuthStartupState = - PropertyUtils.getBoolean(HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE, false); + private String resourceUploadPath; - /** - * Kerberos expire time - */ - public static int getKerberosExpireTime() { - return PropertyUtils.getInt(KERBEROS_EXPIRE_TIME, 2); - } } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperator.java index 173760b632..31c8da5074 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperator.java @@ -22,18 +22,8 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class LocalStorageOperator extends HdfsStorageOperator { - public LocalStorageOperator() { - super(new HdfsStorageProperties()); - } - public LocalStorageOperator(HdfsStorageProperties hdfsStorageProperties) { super(hdfsStorageProperties); } - @Override - public String getResourceFileName(String tenantCode, String fullName) { - // prefix schema `file:/` should be remove in local file mode - String fullNameRemoveSchema = fullName.replaceFirst(hdfsProperties.getDefaultFS(), ""); - return super.getResourceFileName(tenantCode, fullNameRemoveSchema); - } } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorFactory.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorFactory.java index 5f44eca87b..f0bb1543aa 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorFactory.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorFactory.java @@ -17,24 +17,32 @@ package org.apache.dolphinscheduler.plugin.storage.hdfs; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperateFactory; +import org.apache.dolphinscheduler.common.constants.Constants; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperatorFactory; import org.apache.dolphinscheduler.plugin.storage.api.StorageType; import com.google.auto.service.AutoService; -@AutoService(StorageOperateFactory.class) -public class LocalStorageOperatorFactory implements StorageOperateFactory { +@AutoService(StorageOperatorFactory.class) +public class LocalStorageOperatorFactory implements StorageOperatorFactory { - private static final String LOCAL_DEFAULT_FS = "file:/"; + public static final String LOCAL_DEFAULT_FS = "file:/"; @Override - public StorageOperate createStorageOperate() { - HdfsStorageProperties hdfsStorageProperties = new HdfsStorageProperties(); - hdfsStorageProperties.setDefaultFS(LOCAL_DEFAULT_FS); + public StorageOperator createStorageOperate() { + final HdfsStorageProperties hdfsStorageProperties = getHdfsStorageProperties(); return new LocalStorageOperator(hdfsStorageProperties); } + private HdfsStorageProperties getHdfsStorageProperties() { + return HdfsStorageProperties.builder() + .defaultFS(LOCAL_DEFAULT_FS) + .resourceUploadPath(PropertyUtils.getString(Constants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler")) + .build(); + } + @Override public StorageType getStorageOperate() { return StorageType.LOCAL; diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperatorTest.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperatorTest.java index 63931cba78..ee7daa2418 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperatorTest.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperatorTest.java @@ -17,62 +17,183 @@ package org.apache.dolphinscheduler.plugin.storage.hdfs; -import org.apache.dolphinscheduler.common.utils.HttpUtils; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata; +import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; import org.apache.dolphinscheduler.spi.enums.ResourceType; -import org.junit.jupiter.api.Assertions; +import java.io.File; +import java.nio.file.FileAlreadyExistsException; +import java.time.Duration; +import java.util.List; +import java.util.stream.Stream; + +import lombok.SneakyThrows; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * hadoop utils test - */ -@ExtendWith(MockitoExtension.class) -public class HdfsStorageOperatorTest { +import org.testcontainers.containers.ComposeContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.lifecycle.Startables; +import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; + +class HdfsStorageOperatorTest { + + private static HdfsStorageOperator storageOperator; + + private static ComposeContainer hdfsContainer; + + @BeforeAll + public static void setUp() throws InterruptedException { + String hdfsDockerComposeFilePath = + HdfsStorageOperatorTest.class.getResource("/hadoop-docker-compose/docker-compose.yaml").getFile(); + hdfsContainer = new ComposeContainer(new File(hdfsDockerComposeFilePath)) + .withPull(true) + .withTailChildContainers(true) + .withLocalCompose(true) + .waitingFor("namenode", Wait.forHealthcheck().withStartupTimeout(Duration.ofSeconds(60))) + .waitingFor("datanode", Wait.forHealthcheck().withStartupTimeout(Duration.ofSeconds(60))); + + Startables.deepStart(Stream.of(hdfsContainer)).join(); + + HdfsStorageProperties hdfsStorageProperties = HdfsStorageProperties.builder() + .resourceUploadPath("/tmp/dolphinscheduler") + .user("hadoop") + .defaultFS("hdfs://localhost") + // The default replication factor is 3, which is too large for the test environment. + // So we set it to 1. + .configurationProperties(ImmutableMap.of("dfs.replication", "1")) + .build(); + storageOperator = new HdfsStorageOperator(hdfsStorageProperties); + } + + @BeforeEach + public void initializeStorageFile() { + storageOperator.delete("hdfs://localhost/tmp/dolphinscheduler/test-default", true); + storageOperator.createStorageDir("hdfs://localhost/tmp/dolphinscheduler/test-default/resources/empty"); + storageOperator.createStorageDir("hdfs://localhost/tmp/dolphinscheduler/test-default/resources/sql"); + // todo: upload file and add file case + } + + @Test + public void testGetResourceMetaData() { + ResourceMetadata resourceMetaData = + storageOperator.getResourceMetaData( + "hdfs://localhost/tmp/dolphinscheduler/test-default/resources/sqlDirectory/demo.sql"); + assertEquals("hdfs://localhost/tmp/dolphinscheduler/test-default/resources/sqlDirectory/demo.sql", + resourceMetaData.getResourceAbsolutePath()); + assertEquals("hdfs://localhost/tmp/dolphinscheduler", resourceMetaData.getResourceBaseDirectory()); + assertEquals("test-default", resourceMetaData.getTenant()); + assertEquals(ResourceType.FILE, resourceMetaData.getResourceType()); + assertEquals("sqlDirectory/demo.sql", resourceMetaData.getResourceRelativePath()); + assertEquals("hdfs://localhost/tmp/dolphinscheduler/test-default/resources/sqlDirectory", + resourceMetaData.getResourceParentAbsolutePath()); + assertFalse(resourceMetaData.isDirectory()); + } + + @Test + public void testGetStorageBaseDirectory() { + assertEquals("hdfs://localhost/tmp/dolphinscheduler", storageOperator.getStorageBaseDirectory()); + } + + @Test + public void testGetStorageBaseDirectory_withTenantCode() { + assertEquals("hdfs://localhost/tmp/dolphinscheduler/default", + storageOperator.getStorageBaseDirectory("default")); + } + + @Test + public void testGetStorageBaseDirectory_withTenantCode_withFile() { + assertEquals("hdfs://localhost/tmp/dolphinscheduler/default/resources", + storageOperator.getStorageBaseDirectory("default", ResourceType.FILE)); + } - private static final Logger logger = LoggerFactory.getLogger(HdfsStorageOperatorTest.class); + @Test + public void testGetStorageBaseDirectory_withTenantCode_withAll() { + assertEquals("hdfs://localhost/tmp/dolphinscheduler/default", + storageOperator.getStorageBaseDirectory("default", ResourceType.ALL)); + } @Test - public void getHdfsTenantDir() { - HdfsStorageOperator hdfsStorageOperator = new HdfsStorageOperator(); - logger.info(hdfsStorageOperator.getHdfsTenantDir("1234")); - Assertions.assertTrue(true); + public void testGetStorageFileAbsolutePath() { + assertEquals("hdfs://localhost/tmp/dolphinscheduler/default/resources/a.sql", + storageOperator.getStorageFileAbsolutePath("default", "a.sql")); } @Test - public void getHdfsUdfFileName() { - HdfsStorageOperator hdfsStorageOperator = new HdfsStorageOperator(); - logger.info(hdfsStorageOperator.getHdfsUdfFileName("admin", "file_name")); - Assertions.assertTrue(true); + public void testCreateStorageDir_notExist() { + storageOperator.createStorageDir("hdfs://localhost/tmp/dolphinscheduler/test-default/resources/newDirectory"); + storageOperator.exists("hdfs://localhost/tmp/dolphinscheduler/test-default/resources/newDirectory"); } @Test - public void getHdfsResourceFileName() { - HdfsStorageOperator hdfsStorageOperator = new HdfsStorageOperator(); - logger.info(hdfsStorageOperator.getHdfsResourceFileName("admin", "file_name")); - Assertions.assertTrue(true); + public void testCreateStorageDir_exist() { + assertThrows(FileAlreadyExistsException.class, + () -> storageOperator + .createStorageDir("hdfs://localhost/tmp/dolphinscheduler/test-default/resources/empty")); } @Test - public void getHdfsFileName() { - HdfsStorageOperator hdfsStorageOperator = new HdfsStorageOperator(); - logger.info(hdfsStorageOperator.getHdfsFileName(ResourceType.FILE, "admin", "file_name")); - Assertions.assertTrue(true); + public void testExist_DirectoryExist() { + assertThat(storageOperator.exists("hdfs://localhost/tmp/dolphinscheduler/test-default/resources/sql")) + .isTrue(); } @Test - public void getAppAddress() { - HdfsStorageOperator hdfsStorageOperator = new HdfsStorageOperator(); - try (MockedStatic mockedHttpUtils = Mockito.mockStatic(HttpUtils.class)) { - mockedHttpUtils.when(() -> HttpUtils.get("http://ds1:8088/ws/v1/cluster/info")) - .thenReturn("{\"clusterInfo\":{\"state\":\"STARTED\",\"haState\":\"ACTIVE\"}}"); - logger.info(hdfsStorageOperator.getAppAddress("http://ds1:8088/ws/v1/cluster/apps/%s", "ds1,ds2")); - Assertions.assertTrue(true); + public void testExist_DirectoryNotExist() { + assertThat( + storageOperator.exists("hdfs://localhost/tmp/dolphinscheduler/test-default/resources/notExist")) + .isFalse(); + } + + @Test + public void testDelete_directoryExist() { + storageOperator.delete("hdfs://localhost/tmp/dolphinscheduler/test-default/resources/sql", true); + assertThat(storageOperator.exists("hdfs://localhost/tmp/dolphinscheduler/test-default/resources/sql")) + .isFalse(); + } + + @Test + public void testDelete_directoryNotExist() { + storageOperator.delete("hdfs://localhost/tmp/dolphinscheduler/test-default/resources/non", true); + assertThat(storageOperator.exists("hdfs://localhost/tmp/dolphinscheduler/test-default/resources/non")) + .isFalse(); + } + + @Test + public void testListStorageEntity_directory() { + List storageEntities = + storageOperator.listStorageEntity("hdfs://localhost/tmp/dolphinscheduler/test-default/resources/"); + assertThat(storageEntities).hasSize(2); + } + + @Test + public void testGetStorageEntity_directory() { + StorageEntity storageEntity = + storageOperator.getStorageEntity("hdfs://localhost/tmp/dolphinscheduler/test-default/resources/sql"); + assertThat(storageEntity.getFullName()) + .isEqualTo("hdfs://localhost/tmp/dolphinscheduler/test-default/resources/sql"); + assertThat(storageEntity.isDirectory()).isTrue(); + assertThat(storageEntity.getPfullName()) + .isEqualTo("hdfs://localhost/tmp/dolphinscheduler/test-default/resources"); + assertThat(storageEntity.getType()).isEqualTo(ResourceType.FILE); + assertThat(storageEntity.getFileName()).isEqualTo("sql"); + } + + @SneakyThrows + @AfterAll + public static void tearDown() { + if (storageOperator != null) { + storageOperator.close(); + } + if (hdfsContainer != null) { + hdfsContainer.stop(); } } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java new file mode 100644 index 0000000000..c90500c343 --- /dev/null +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java @@ -0,0 +1,322 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.storage.hdfs; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.apache.dolphinscheduler.common.constants.Constants; +import org.apache.dolphinscheduler.common.utils.FileUtils; +import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata; +import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; +import org.apache.dolphinscheduler.spi.enums.ResourceType; + +import org.apache.commons.lang3.StringUtils; + +import java.io.File; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; + +import lombok.SneakyThrows; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class LocalStorageOperatorTest { + + private StorageOperator storageOperator; + + private static final String resourceBaseDir = StringUtils + .substringBeforeLast(FileUtils.getClassPathAbsolutePath(LocalStorageOperatorTest.class), File.separator); + private static final String tenantCode = "default"; + private static final String baseDir = + Paths.get(resourceBaseDir, tenantCode, Constants.RESOURCE_TYPE_FILE).toString(); + + @SneakyThrows + @BeforeEach + public void setup() { + System.setProperty(Constants.RESOURCE_UPLOAD_PATH, resourceBaseDir); + + LocalStorageOperatorFactory localStorageOperatorFactory = new LocalStorageOperatorFactory(); + storageOperator = localStorageOperatorFactory.createStorageOperate(); + // create file and directory + Files.createDirectories(Paths.get(baseDir, "sqlDirectory")); + Files.createDirectories(Paths.get(baseDir, "emptyDirectory")); + Files.createFile(Paths.get(baseDir, "sqlDirectory", "demo.sql")); + Files.write(Paths.get(baseDir, "sqlDirectory", "demo.sql"), "select * from demo".getBytes()); + + } + + @Test + public void testGetResourceMetaData_directory() { + String resourceFileAbsolutePath = "file:" + baseDir; + + ResourceMetadata resourceMetaData = storageOperator.getResourceMetaData(resourceFileAbsolutePath); + assertThat(resourceMetaData.getResourceAbsolutePath()).isEqualTo("file:" + baseDir); + assertThat(resourceMetaData.getResourceBaseDirectory()).isEqualTo("file:" + resourceBaseDir); + assertThat(resourceMetaData.getTenant()).isEqualTo("default"); + assertThat(resourceMetaData.getResourceType()).isEqualTo(ResourceType.FILE); + assertThat(resourceMetaData.getResourceRelativePath()).isEqualTo("/"); + } + + @Test + public void testGetResourceMetaData_file() { + String resourceFileAbsolutePath = "file:" + Paths.get(baseDir, "sqlDirectory", "demo.sql"); + + ResourceMetadata resourceMetaData = storageOperator.getResourceMetaData(resourceFileAbsolutePath); + assertThat(resourceMetaData.getResourceAbsolutePath()).isEqualTo(resourceFileAbsolutePath); + assertThat(resourceMetaData.getResourceBaseDirectory()).isEqualTo("file:" + resourceBaseDir); + assertThat(resourceMetaData.getTenant()).isEqualTo("default"); + assertThat(resourceMetaData.getResourceType()).isEqualTo(ResourceType.FILE); + assertThat(resourceMetaData.getResourceRelativePath()).isEqualTo("sqlDirectory/demo.sql"); + } + + @Test + public void testGetResourceMetaData_invalidatedPath() { + String resourceFileAbsolutePath = Paths.get(baseDir, "sqlDirectory", "demo.sql").toString(); + + IllegalArgumentException illegalArgumentException = assertThrows(IllegalArgumentException.class, + () -> storageOperator.getResourceMetaData(resourceFileAbsolutePath)); + assertThat(illegalArgumentException.getMessage()) + .isEqualTo("Invalid resource path: " + resourceFileAbsolutePath); + } + + @Test + public void testGetStorageBaseDirectory() { + String storageBaseDirectory = storageOperator.getStorageBaseDirectory(); + assertThat(storageBaseDirectory).isEqualTo("file:" + resourceBaseDir); + } + + @Test + public void testGetStorageBaseDirectory_withTenant() { + String storageBaseDirectory = storageOperator.getStorageBaseDirectory("default"); + assertThat(storageBaseDirectory).isEqualTo("file:" + Paths.get(resourceBaseDir, tenantCode)); + } + + @Test + public void testGetStorageBaseDirectory_withTenant_withResourceTypeFile() { + String storageBaseDirectory = storageOperator.getStorageBaseDirectory("default", ResourceType.FILE); + assertThat(storageBaseDirectory) + .isEqualTo("file:" + Paths.get(resourceBaseDir, tenantCode, Constants.RESOURCE_TYPE_FILE)); + } + + @Test + public void testGetStorageBaseDirectory_withTenant_withResourceTypeUdf() { + String storageBaseDirectory = storageOperator.getStorageBaseDirectory("default", ResourceType.UDF); + assertThat(storageBaseDirectory) + .isEqualTo("file:" + Paths.get(resourceBaseDir, tenantCode, Constants.RESOURCE_TYPE_UDF)); + } + + @Test + public void testGetStorageBaseDirectory_withTenant_withResourceTypeAll() { + String storageBaseDirectory = storageOperator.getStorageBaseDirectory("default", ResourceType.ALL); + assertThat(storageBaseDirectory).isEqualTo("file:" + Paths.get(resourceBaseDir, tenantCode)); + } + + @Test + public void testGetStorageBaseDirectory_withEmptyTenant_withResourceType() { + IllegalArgumentException illegalArgumentException = assertThrows(IllegalArgumentException.class, + () -> storageOperator.getStorageBaseDirectory("", ResourceType.ALL)); + assertThat(illegalArgumentException.getMessage()).isEqualTo("Tenant code should not be empty"); + } + + @Test + public void testGetStorageBaseDirectory_withTenant_withEmptyResourceType() { + IllegalArgumentException illegalArgumentException = assertThrows(IllegalArgumentException.class, + () -> storageOperator.getStorageBaseDirectory("default", null)); + assertThat(illegalArgumentException.getMessage()).isEqualTo("Resource type should not be null"); + } + + @Test + public void testGetStorageFileAbsolutePath() { + String fileAbsolutePath = storageOperator.getStorageFileAbsolutePath("default", "test.sh"); + assertThat(fileAbsolutePath).isEqualTo( + "file:" + Paths.get(resourceBaseDir, tenantCode, Constants.RESOURCE_TYPE_FILE, "test.sh")); + } + + @SneakyThrows + @Test + public void testCreateStorageDir_notExists() { + String testDirFileAbsolutePath = + "file:" + Paths.get(resourceBaseDir, "root", Constants.RESOURCE_TYPE_FILE, "testDir"); + try { + storageOperator.createStorageDir(testDirFileAbsolutePath); + StorageEntity storageEntity = storageOperator.getStorageEntity(testDirFileAbsolutePath); + assertThat(storageEntity.getFullName()).isEqualTo(testDirFileAbsolutePath); + assertThat(storageEntity.getFileName()).isEqualTo("testDir"); + assertThat(storageEntity.getPfullName()) + .isEqualTo("file:" + Paths.get(resourceBaseDir, "root", Constants.RESOURCE_TYPE_FILE)); + assertThat(storageEntity.isDirectory()).isTrue(); + assertThat(storageEntity.getType()).isEqualTo(ResourceType.FILE); + } finally { + storageOperator.delete(testDirFileAbsolutePath, true); + } + } + + @SneakyThrows + @Test + public void testCreateStorageDir_exists() { + String testDirFileAbsolutePath = + "file:" + Paths.get(resourceBaseDir, "default", Constants.RESOURCE_TYPE_FILE, "sqlDirectory"); + assertThrows(FileAlreadyExistsException.class, () -> storageOperator.createStorageDir(testDirFileAbsolutePath)); + } + + @Test + public void testExists_fileExist() { + String resourceFileAbsolutePath = "file:" + Paths.get(baseDir, "sqlDirectory", "demo.sql"); + assertThat(storageOperator.exists(resourceFileAbsolutePath)).isTrue(); + } + + @Test + public void testExists_fileNotExist() { + String resourceFileAbsolutePath = "file:" + Paths.get(baseDir, "sqlDirectory", "demo.sh"); + assertThat(storageOperator.exists(resourceFileAbsolutePath)).isFalse(); + } + + @Test + public void testExists_directoryExist() { + String resourceFileAbsolutePath = "file:" + Paths.get(baseDir, "sqlDirectory"); + assertThat(storageOperator.exists(resourceFileAbsolutePath)).isTrue(); + } + + @Test + public void testExists_directoryNotExist() { + String resourceFileAbsolutePath = "file:" + Paths.get(baseDir, "shellDirectory"); + assertThat(storageOperator.exists(resourceFileAbsolutePath)).isFalse(); + } + + @Test + public void testDelete_directoryExist() { + String resourceFileAbsolutePath = "file:" + Paths.get(baseDir, "sqlDirectory"); + assertThat(storageOperator.exists(resourceFileAbsolutePath)).isTrue(); + + storageOperator.delete(resourceFileAbsolutePath, true); + assertThat(storageOperator.exists(resourceFileAbsolutePath)).isFalse(); + } + + @Test + public void testDelete_directoryNotExist() { + String resourceFileAbsolutePath = "file:" + Paths.get(baseDir, "shellDirectory"); + assertThat(storageOperator.exists(resourceFileAbsolutePath)).isFalse(); + + storageOperator.delete(resourceFileAbsolutePath, true); + assertThat(storageOperator.exists(resourceFileAbsolutePath)).isFalse(); + } + + @Test + public void testDelete_fileExist() { + String resourceFileAbsolutePath = "file:" + Paths.get(baseDir, "sqlDirectory", "demo.sql"); + assertThat(storageOperator.exists(resourceFileAbsolutePath)).isTrue(); + + storageOperator.delete(resourceFileAbsolutePath, true); + assertThat(storageOperator.exists(resourceFileAbsolutePath)).isFalse(); + } + + @Test + public void testDelete_fileNotExist() { + String resourceFileAbsolutePath = "file:" + Paths.get(baseDir, "sqlDirectory", "demo.sh"); + assertThat(storageOperator.exists(resourceFileAbsolutePath)).isFalse(); + + storageOperator.delete(resourceFileAbsolutePath, true); + assertThat(storageOperator.exists(resourceFileAbsolutePath)).isFalse(); + } + + @Test + public void testFetchFileContent() { + // todo: add large file test case + String resourceFileAbsolutePath = "file:" + Paths.get(baseDir, "sqlDirectory", "demo.sql"); + List content = storageOperator.fetchFileContent(resourceFileAbsolutePath, 0, 10); + assertThat(content).containsExactly("select * from demo"); + + } + + @Test + public void testListStorageEntity_directoryNotEmpty() { + String resourceFileAbsolutePath = "file:" + baseDir; + List storageEntities = storageOperator.listStorageEntity(resourceFileAbsolutePath); + assertThat(storageEntities.size()).isEqualTo(2); + + StorageEntity storageEntity1 = storageEntities.get(0); + assertThat(storageEntity1.getFullName()).isEqualTo("file:" + baseDir + "/emptyDirectory"); + assertThat(storageEntity1.getFileName()).isEqualTo("emptyDirectory"); + assertThat(storageEntity1.getPfullName()).isEqualTo("file:" + baseDir); + assertThat(storageEntity1.isDirectory()).isTrue(); + assertThat(storageEntity1.getType()).isEqualTo(ResourceType.FILE); + + StorageEntity storageEntity2 = storageEntities.get(1); + assertThat(storageEntity2.getFullName()).isEqualTo("file:" + baseDir + "/sqlDirectory"); + assertThat(storageEntity2.getFileName()).isEqualTo("sqlDirectory"); + assertThat(storageEntity2.getPfullName()).isEqualTo("file:" + baseDir); + assertThat(storageEntity2.isDirectory()).isTrue(); + assertThat(storageEntity2.getType()).isEqualTo(ResourceType.FILE); + } + + @Test + public void testListStorageEntity_directoryEmpty() { + String resourceFileAbsolutePath = "file:" + baseDir + "/emptyDirectory"; + List storageEntities = storageOperator.listStorageEntity(resourceFileAbsolutePath); + assertThat(storageEntities.size()).isEqualTo(0); + } + + @Test + public void testListStorageEntity_directoryNotExist() { + String resourceFileAbsolutePath = "file:" + baseDir + "/notExistDirectory"; + assertThat(storageOperator.listStorageEntity(resourceFileAbsolutePath)).isEmpty(); + } + + @Test + public void testListStorageEntity_file() { + String resourceFileAbsolutePath = "file:" + Paths.get(baseDir, "sqlDirectory", "demo.sql"); + List storageEntities = storageOperator.listStorageEntity(resourceFileAbsolutePath); + assertThat(storageEntities.size()).isEqualTo(1); + + StorageEntity storageEntity = storageEntities.get(0); + assertThat(storageEntity.getFullName()).isEqualTo("file:" + Paths.get(baseDir, "sqlDirectory", "demo.sql")); + assertThat(storageEntity.getFileName()).isEqualTo("demo.sql"); + assertThat(storageEntity.getPfullName()).isEqualTo("file:" + Paths.get(baseDir, "sqlDirectory")); + assertThat(storageEntity.isDirectory()).isFalse(); + assertThat(storageEntity.getType()).isEqualTo(ResourceType.FILE); + + } + + @Test + public void testListStorageEntityRecursively_directory() { + String resourceFileAbsolutePath = "file:" + baseDir; + List storageEntities = + storageOperator.listFileStorageEntityRecursively(resourceFileAbsolutePath); + assertThat(storageEntities.size()).isEqualTo(1); + + StorageEntity storageEntity2 = storageEntities.get(0); + assertThat(storageEntity2.getFullName()).isEqualTo("file:" + Paths.get(baseDir, "sqlDirectory", "demo.sql")); + assertThat(storageEntity2.getFileName()).isEqualTo("demo.sql"); + assertThat(storageEntity2.getPfullName()).isEqualTo("file:" + Paths.get(baseDir, "sqlDirectory")); + assertThat(storageEntity2.isDirectory()).isFalse(); + assertThat(storageEntity2.getType()).isEqualTo(ResourceType.FILE); + } + + @SneakyThrows + @AfterEach + public void after() { + FileUtils.deleteFile(resourceBaseDir); + } + +} diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/resources/hadoop-docker-compose/docker-compose.yaml b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/resources/hadoop-docker-compose/docker-compose.yaml new file mode 100644 index 0000000000..60ede8faf8 --- /dev/null +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/resources/hadoop-docker-compose/docker-compose.yaml @@ -0,0 +1,110 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +version: "3" + +services: + namenode: + image: sbloodys/hadoop:3.3.6 + hostname: namenode + command: [ "hdfs", "namenode" ] + ports: + - 9870:9870 + - 8020:8020 + env_file: + - ./hadoop.env + environment: + ENSURE_NAMENODE_DIR: "/tmp/hadoop-root/dfs/name" + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "1" + tty: true + stdin_open: true + restart: always + healthcheck: + test: [ "CMD", "curl", "http://namenode:9870" ] + interval: 5s + timeout: 5s + retries: 120 + datanode: + image: sbloodys/hadoop:3.3.6 + hostname: datanode + command: [ "hdfs", "datanode" ] + env_file: + - ./hadoop.env + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "1" + ports: + - 9864:9864 + tty: true + stdin_open: true + restart: always + healthcheck: + test: [ "CMD", "curl", "http://datanode:9864" ] + interval: 5s + timeout: 5s + retries: 120 + depends_on: + namenode: + condition: service_healthy + resourcemanager: + image: sbloodys/hadoop:3.3.6 + hostname: resourcemanager + command: [ "yarn", "resourcemanager" ] + ports: + - 8088:8088 + env_file: + - ./hadoop.env + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "1" + tty: true + stdin_open: true + restart: always + healthcheck: + test: [ "CMD", "curl", "http://resourcemanager:8088" ] + interval: 5s + timeout: 5s + retries: 120 + nodemanager: + image: sbloodys/hadoop:3.3.6 + hostname: nodemanager + command: [ "yarn", "nodemanager" ] + env_file: + - ./hadoop.env + logging: + driver: "json-file" + options: + max-size: "200m" + max-file: "1" + tty: true + stdin_open: true + restart: always + depends_on: + resourcemanager: + condition: service_healthy + healthcheck: + test: [ "CMD", "curl", "http://nodemanager:8042" ] + interval: 5s + timeout: 5s + retries: 120 diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/resources/hadoop-docker-compose/hadoop.env b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/resources/hadoop-docker-compose/hadoop.env new file mode 100644 index 0000000000..701cf3471f --- /dev/null +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/resources/hadoop-docker-compose/hadoop.env @@ -0,0 +1,45 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +HADOOP_HOME=/opt/hadoop +CORE-SITE.XML_fs.default.name=hdfs://namenode +CORE-SITE.XML_fs.defaultFS=hdfs://namenode +HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:8020 +HDFS-SITE.XML_dfs.replication=1 +CORE-SITE.XML_hadoop.http.staticuser.user=hadoop +MAPRED-SITE.XML_mapreduce.framework.name=yarn +MAPRED-SITE.XML_yarn.app.mapreduce.am.env=HADOOP_MAPRED_HOME=$HADOOP_HOME +MAPRED-SITE.XML_mapreduce.map.env=HADOOP_MAPRED_HOME=$HADOOP_HOME +MAPRED-SITE.XML_mapreduce.reduce.env=HADOOP_MAPRED_HOME=$HADOOP_HOME +YARN-SITE.XML_yarn.resourcemanager.hostname=resourcemanager +YARN-SITE.XML_yarn.nodemanager.pmem-check-enabled=false +YARN-SITE.XML_yarn.nodemanager.delete.debug-delay-sec=600 +YARN-SITE.XML_yarn.nodemanager.vmem-check-enabled=false +YARN-SITE.XML_yarn.nodemanager.aux-services=mapreduce_shuffle +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-applications=10000 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.maximum-am-resource-percent=0.1 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.resource-calculator=org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.queues=default +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.capacity=100 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.user-limit-factor=1 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.maximum-capacity=100 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.state=RUNNING +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_submit_applications=* +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.root.default.acl_administer_queue=* +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.node-locality-delay=40 +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings= +CAPACITY-SCHEDULER.XML_yarn.scheduler.capacity.queue-mappings-override.enable=false \ No newline at end of file diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/resources/storage/hello.sh b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/resources/storage/hello.sh new file mode 100644 index 0000000000..9ee3583189 --- /dev/null +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/resources/storage/hello.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +echo hello \ No newline at end of file diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/resources/storage/sql/demo.sql b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/resources/storage/sql/demo.sql new file mode 100644 index 0000000000..83ebbac8d0 --- /dev/null +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/resources/storage/sql/demo.sql @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +select * from t; \ No newline at end of file diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java index b644210048..6f67ccb363 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java @@ -17,19 +17,12 @@ package org.apache.dolphinscheduler.plugin.storage.obs; -import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPARATOR; -import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S; -import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TYPE_FILE; -import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TYPE_UDF; - import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.enums.ResUploadType; import org.apache.dolphinscheduler.common.utils.FileUtils; -import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.plugin.storage.api.AbstractStorageOperator; +import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata; import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; -import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; -import org.apache.dolphinscheduler.spi.enums.ResourceType; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.commons.lang3.StringUtils; @@ -37,153 +30,83 @@ import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.Closeable; import java.io.File; -import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Collections; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; -import java.util.stream.Stream; -import lombok.Data; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import com.obs.services.ObsClient; import com.obs.services.exception.ObsException; -import com.obs.services.internal.ServiceException; -import com.obs.services.model.DeleteObjectsRequest; -import com.obs.services.model.GetObjectRequest; import com.obs.services.model.ListObjectsRequest; import com.obs.services.model.ObjectListing; import com.obs.services.model.ObjectMetadata; import com.obs.services.model.ObsObject; import com.obs.services.model.PutObjectRequest; -@Data @Slf4j -public class ObsStorageOperator implements Closeable, StorageOperate { - - private String accessKeyId; - - private String accessKeySecret; - - private String bucketName; +public class ObsStorageOperator extends AbstractStorageOperator implements Closeable, StorageOperator { - private String endPoint; + private final String bucketName; - private ObsClient obsClient; + private final ObsClient obsClient; - public ObsStorageOperator() { - } - - public void init() { - this.accessKeyId = readObsAccessKeyID(); - this.accessKeySecret = readObsAccessKeySecret(); - this.endPoint = readObsEndPoint(); - this.bucketName = readObsBucketName(); - this.obsClient = buildObsClient(); + public ObsStorageOperator(ObsStorageProperties obsStorageProperties) { + super(obsStorageProperties.getResourceUploadPath()); + this.bucketName = obsStorageProperties.getBucketName(); + this.obsClient = new ObsClient( + obsStorageProperties.getAccessKeyId(), + obsStorageProperties.getAccessKeySecret(), + obsStorageProperties.getEndPoint()); ensureBucketSuccessfullyCreated(bucketName); } - protected String readObsAccessKeyID() { - return PropertyUtils.getString(TaskConstants.HUAWEI_CLOUD_ACCESS_KEY_ID); - } - - protected String readObsAccessKeySecret() { - return PropertyUtils.getString(TaskConstants.HUAWEI_CLOUD_ACCESS_KEY_SECRET); - } - - protected String readObsBucketName() { - return PropertyUtils.getString(Constants.HUAWEI_CLOUD_OBS_BUCKET_NAME); - } - - protected String readObsEndPoint() { - return PropertyUtils.getString(Constants.HUAWEI_CLOUD_OBS_END_POINT); - } - @Override public void close() throws IOException { obsClient.close(); } @Override - public void createTenantDirIfNotExists(String tenantCode) throws Exception { - mkdir(tenantCode, getObsResDir(tenantCode)); - mkdir(tenantCode, getObsUdfDir(tenantCode)); - } - - @Override - public String getResDir(String tenantCode) { - return getObsResDir(tenantCode) + FOLDER_SEPARATOR; - } - - @Override - public String getUdfDir(String tenantCode) { - return getObsUdfDir(tenantCode) + FOLDER_SEPARATOR; + public String getStorageBaseDirectory() { + // All directory should end with File.separator + if (resourceBaseAbsolutePath.startsWith("/")) { + log.warn("{} -> {} should not start with / in obs", Constants.RESOURCE_UPLOAD_PATH, + resourceBaseAbsolutePath); + return resourceBaseAbsolutePath.substring(1); + } + return resourceBaseAbsolutePath; } + @SneakyThrows @Override - public boolean mkdir(String tenantCode, String path) throws IOException { - final String key = path + FOLDER_SEPARATOR; - if (!obsClient.doesObjectExist(bucketName, key)) { - createObsPrefix(bucketName, key); + public void createStorageDir(String directoryAbsolutePath) { + directoryAbsolutePath = transformAbsolutePathToObsKey(directoryAbsolutePath); + if (obsClient.doesObjectExist(bucketName, directoryAbsolutePath)) { + throw new FileAlreadyExistsException("directory: " + directoryAbsolutePath + " already exists"); } - return true; - } - - protected void createObsPrefix(final String bucketName, final String key) { ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(0L); InputStream emptyContent = new ByteArrayInputStream(new byte[0]); - PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, key, emptyContent); + PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, directoryAbsolutePath, emptyContent); obsClient.putObject(putObjectRequest); } + @SneakyThrows @Override - public String getResourceFullName(String tenantCode, String fileName) { - if (fileName.startsWith(FOLDER_SEPARATOR)) { - fileName = fileName.replaceFirst(FOLDER_SEPARATOR, ""); - } - return String.format(FORMAT_S_S, getObsResDir(tenantCode), fileName); - } + public void download(String srcFilePath, String dstFilePath, boolean overwrite) { + srcFilePath = transformAbsolutePathToObsKey(srcFilePath); - @Override - public String getFileName(ResourceType resourceType, String tenantCode, String fileName) { - if (fileName.startsWith(FOLDER_SEPARATOR)) { - fileName = fileName.replaceFirst(FOLDER_SEPARATOR, ""); - } - return getDir(resourceType, tenantCode) + fileName; - } - - @Override - public boolean delete(String fullName, List childrenPathList, boolean recursive) throws IOException { - // append the resource fullName to the list for deletion. - childrenPathList.add(fullName); - - DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucketName); - for (String deleteKys : childrenPathList) { - deleteObjectsRequest.addKeyAndVersion(deleteKys); - } - - try { - obsClient.deleteObjects(deleteObjectsRequest); - } catch (Exception e) { - log.error("delete objects error", e); - return false; - } - - return true; - } - - @Override - public void download(String srcFilePath, String dstFilePath, boolean overwrite) throws IOException { File dstFile = new File(dstFilePath); if (dstFile.isDirectory()) { Files.delete(dstFile.toPath()); @@ -199,277 +122,115 @@ public class ObsStorageOperator implements Closeable, StorageOperate { while ((readLen = obsInputStream.read(readBuf)) > 0) { fos.write(readBuf, 0, readLen); } - } catch (ObsException e) { - throw new IOException(e); - } catch (FileNotFoundException e) { - log.error("cannot find the destination file {}", dstFilePath); - throw e; } } @Override - public boolean exists(String fileName) throws IOException { + public boolean exists(String fileName) { + fileName = transformAbsolutePathToObsKey(fileName); return obsClient.doesObjectExist(bucketName, fileName); } @Override - public boolean delete(String filePath, boolean recursive) throws IOException { - try { - obsClient.deleteObject(bucketName, filePath); - return true; - } catch (ObsException e) { - log.error("fail to delete the object, the resource path is {}", filePath, e); - return false; - } + public void delete(String filePath, boolean recursive) { + filePath = transformAbsolutePathToObsKey(filePath); + obsClient.deleteObject(bucketName, filePath); } @Override - public boolean copy(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) throws IOException { + public void copy(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) { + srcPath = transformAbsolutePathToObsKey(srcPath); + dstPath = transformAbsolutePathToObsKey(dstPath); obsClient.copyObject(bucketName, srcPath, bucketName, dstPath); if (deleteSource) { obsClient.deleteObject(bucketName, srcPath); } - return true; - } - - @Override - public String getDir(ResourceType resourceType, String tenantCode) { - switch (resourceType) { - case UDF: - return getUdfDir(tenantCode); - case FILE: - return getResDir(tenantCode); - case ALL: - return getObsDataBasePath(); - default: - return ""; - } } + @SneakyThrows @Override - public boolean upload(String tenantCode, String srcFile, String dstPath, boolean deleteSource, - boolean overwrite) throws IOException { - try { - obsClient.putObject(bucketName, dstPath, new File(srcFile)); - if (deleteSource) { - Files.delete(Paths.get(srcFile)); + public void upload(String srcFile, String dstPath, boolean deleteSource, boolean overwrite) { + dstPath = transformAbsolutePathToObsKey(dstPath); + if (obsClient.doesObjectExist(bucketName, dstPath)) { + if (!overwrite) { + throw new ObsException("file: " + dstPath + " already exists"); + } else { + obsClient.deleteObject(bucketName, dstPath); } - return true; - } catch (ObsException e) { - log.error("upload failed, the bucketName is {}, the filePath is {}", bucketName, dstPath, e); - return false; } - } - - @Override - public List vimFile(String tenantCode, String filePath, int skipLineNums, int limit) throws IOException { - if (StringUtils.isBlank(filePath)) { - log.error("file path:{} is empty", filePath); - return Collections.emptyList(); - } - ObsObject obsObject = obsClient.getObject(bucketName, filePath); - try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(obsObject.getObjectContent()))) { - Stream stream = bufferedReader.lines().skip(skipLineNums).limit(limit); - return stream.collect(Collectors.toList()); + obsClient.putObject(bucketName, dstPath, new File(srcFile)); + if (deleteSource) { + Files.delete(Paths.get(srcFile)); } - } - @Override - public ResUploadType returnStorageType() { - return ResUploadType.OBS; } + @SneakyThrows @Override - public List listFilesStatusRecursively(String path, String defaultPath, String tenantCode, - ResourceType type) { - List storageEntityList = new ArrayList<>(); - LinkedList foldersToFetch = new LinkedList<>(); - - StorageEntity initialEntity = null; - try { - initialEntity = getFileStatus(path, defaultPath, tenantCode, type); - } catch (Exception e) { - log.error("error while listing files status recursively, path: {}", path, e); - return storageEntityList; - } - foldersToFetch.add(initialEntity); - - while (!foldersToFetch.isEmpty()) { - String pathToExplore = foldersToFetch.pop().getFullName(); - try { - List tempList = listFilesStatus(pathToExplore, defaultPath, tenantCode, type); - for (StorageEntity temp : tempList) { - if (temp.isDirectory()) { - foldersToFetch.add(temp); - } - } - storageEntityList.addAll(tempList); - } catch (Exception e) { - log.error("error while listing files stat:wus recursively, path: {}", pathToExplore, e); - } + public List fetchFileContent(String filePath, int skipLineNums, int limit) { + filePath = transformAbsolutePathToObsKey(filePath); + ObsObject obsObject = obsClient.getObject(bucketName, filePath); + try ( + InputStreamReader inputStreamReader = new InputStreamReader(obsObject.getObjectContent()); + BufferedReader bufferedReader = new BufferedReader(inputStreamReader)) { + return bufferedReader + .lines() + .skip(skipLineNums) + .limit(limit) + .collect(Collectors.toList()); } - - return storageEntityList; } @Override - public List listFilesStatus(String path, String defaultPath, String tenantCode, - ResourceType type) throws Exception { - List storageEntityList = new ArrayList<>(); + public List listStorageEntity(String resourceAbsolutePath) { + resourceAbsolutePath = transformObsKeyToAbsolutePath(resourceAbsolutePath); ListObjectsRequest request = new ListObjectsRequest(); request.setBucketName(bucketName); - request.setPrefix(path); - request.setDelimiter(FOLDER_SEPARATOR); - ObjectListing result = null; - try { - result = obsClient.listObjects(request); - } catch (Exception e) { - throw new ServiceException("Get ObsClient file list exception", e); - } - - while (result != null) { - String nextMarker = result.getNextMarker(); - List objects = result.getObjects(); - - for (ObsObject object : objects) { - if (!object.getObjectKey().endsWith(FOLDER_SEPARATOR)) { - // the path is a file - String[] aliasArr = object.getObjectKey().split(FOLDER_SEPARATOR); - String alias = aliasArr[aliasArr.length - 1]; - String fileName = StringUtils.difference(defaultPath, object.getObjectKey()); - - StorageEntity entity = new StorageEntity(); - ObjectMetadata metadata = object.getMetadata(); - entity.setAlias(alias); - entity.setFileName(fileName); - entity.setFullName(object.getObjectKey()); - entity.setDirectory(false); - entity.setUserName(tenantCode); - entity.setType(type); - entity.setSize(metadata.getContentLength()); - entity.setCreateTime(metadata.getLastModified()); - entity.setUpdateTime(metadata.getLastModified()); - entity.setPfullName(path); - - storageEntityList.add(entity); - } - } + request.setPrefix(resourceAbsolutePath); + request.setDelimiter("/"); - for (String commonPrefix : result.getCommonPrefixes()) { - // the paths in commonPrefix are directories - String suffix = StringUtils.difference(path, commonPrefix); - String fileName = StringUtils.difference(defaultPath, commonPrefix); - - StorageEntity entity = new StorageEntity(); - entity.setAlias(suffix); - entity.setFileName(fileName); - entity.setFullName(commonPrefix); - entity.setDirectory(true); - entity.setUserName(tenantCode); - entity.setType(type); - entity.setSize(0); - entity.setCreateTime(null); - entity.setUpdateTime(null); - entity.setPfullName(path); - - storageEntityList.add(entity); - } - - if (!StringUtils.isNotBlank(nextMarker)) { - break; - } + ObjectListing result = obsClient.listObjects(request); - request.setMarker(nextMarker); - try { - result = obsClient.listObjects(request); - } catch (Exception e) { - throw new ServiceException("Get ObsClient file list exception", e); - } - } - return storageEntityList; + return result.getObjects() + .stream() + .map(this::transformObsObjectToStorageEntity) + .collect(Collectors.toList()); } @Override - public StorageEntity getFileStatus(String path, String defaultPath, String tenantCode, - ResourceType type) throws Exception { + public List listFileStorageEntityRecursively(String resourceAbsolutePath) { + resourceAbsolutePath = transformObsKeyToAbsolutePath(resourceAbsolutePath); - if (path.endsWith(FOLDER_SEPARATOR)) { - // the path is a directory that may or may not exist in ObsClient - String alias = findDirAlias(path); - String fileName = StringUtils.difference(defaultPath, path); - - StorageEntity entity = new StorageEntity(); - entity.setAlias(alias); - entity.setFileName(fileName); - entity.setFullName(path); - entity.setDirectory(true); - entity.setUserName(tenantCode); - entity.setType(type); - entity.setSize(0); - - return entity; + Set visited = new HashSet<>(); + List storageEntityList = new ArrayList<>(); + LinkedList foldersToFetch = new LinkedList<>(); + foldersToFetch.addLast(resourceAbsolutePath); - } else { - GetObjectRequest request = new GetObjectRequest(); - request.setBucketName(bucketName); - request.setObjectKey(path); - ObsObject object; - try { - object = obsClient.getObject(request); - } catch (Exception e) { - throw new ServiceException("Get ObsClient file list exception", e); + while (!foldersToFetch.isEmpty()) { + String pathToExplore = foldersToFetch.pop(); + visited.add(pathToExplore); + List tempList = listStorageEntity(pathToExplore); + for (StorageEntity temp : tempList) { + if (temp.isDirectory()) { + if (visited.contains(temp.getFullName())) { + continue; + } + foldersToFetch.add(temp.getFullName()); + } } - - String[] aliasArr = object.getObjectKey().split(FOLDER_SEPARATOR); - String alias = aliasArr[aliasArr.length - 1]; - String fileName = StringUtils.difference(defaultPath, object.getObjectKey()); - - StorageEntity entity = new StorageEntity(); - ObjectMetadata metadata = object.getMetadata(); - entity.setAlias(alias); - entity.setFileName(fileName); - entity.setFullName(object.getObjectKey()); - entity.setDirectory(false); - entity.setUserName(tenantCode); - entity.setType(type); - entity.setSize(metadata.getContentLength()); - entity.setCreateTime(metadata.getLastModified()); - entity.setUpdateTime(metadata.getLastModified()); - - return entity; + storageEntityList.addAll(tempList); } + return storageEntityList; } @Override - public void deleteTenant(String tenantCode) throws Exception { - deleteTenantCode(tenantCode); - } - - public String getObsResDir(String tenantCode) { - return String.format("%s/" + RESOURCE_TYPE_FILE, getObsTenantDir(tenantCode)); - } + public StorageEntity getStorageEntity(String resourceAbsolutePath) { + resourceAbsolutePath = transformObsKeyToAbsolutePath(resourceAbsolutePath); - public String getObsUdfDir(String tenantCode) { - return String.format("%s/" + RESOURCE_TYPE_UDF, getObsTenantDir(tenantCode)); - } - - public String getObsTenantDir(String tenantCode) { - return String.format(FORMAT_S_S, getObsDataBasePath(), tenantCode); - } - - public String getObsDataBasePath() { - if (FOLDER_SEPARATOR.equals(RESOURCE_UPLOAD_PATH)) { - return ""; - } else { - return RESOURCE_UPLOAD_PATH.replaceFirst(FOLDER_SEPARATOR, ""); - } - } - - protected void deleteTenantCode(String tenantCode) { - deleteDir(getResDir(tenantCode)); - deleteDir(getUdfDir(tenantCode)); + ObsObject object = obsClient.getObject(bucketName, resourceAbsolutePath); + return transformObsObjectToStorageEntity(object); } public void ensureBucketSuccessfullyCreated(String bucketName) { @@ -486,22 +247,37 @@ public class ObsStorageOperator implements Closeable, StorageOperate { log.info("bucketName: {} has been found", bucketName); } - protected void deleteDir(String directoryName) { - if (obsClient.doesObjectExist(bucketName, directoryName)) { - obsClient.deleteObject(bucketName, directoryName); + protected StorageEntity transformObsObjectToStorageEntity(ObsObject object) { + ObjectMetadata metadata = object.getMetadata(); + String fileAbsolutePath = transformObsKeyToAbsolutePath(object.getObjectKey()); + ResourceMetadata resourceMetaData = getResourceMetaData(fileAbsolutePath); + String fileExtension = com.google.common.io.Files.getFileExtension(resourceMetaData.getResourceAbsolutePath()); + + return StorageEntity.builder() + .fileName(new File(fileAbsolutePath).getName()) + .fullName(fileAbsolutePath) + .pfullName(resourceMetaData.getResourceParentAbsolutePath()) + .type(resourceMetaData.getResourceType()) + .isDirectory(StringUtils.isEmpty(fileExtension)) + .size(metadata.getContentLength()) + .createTime(metadata.getLastModified()) + .updateTime(metadata.getLastModified()) + .build(); + } + + private String transformAbsolutePathToObsKey(String absolutePath) { + ResourceMetadata resourceMetaData = getResourceMetaData(absolutePath); + if (resourceMetaData.isDirectory()) { + return FileUtils.concatFilePath(absolutePath, "/"); } + return absolutePath; } - protected ObsClient buildObsClient() { - return new ObsClient(accessKeyId, accessKeySecret, endPoint); - } - - private String findDirAlias(String dirPath) { - if (!dirPath.endsWith(FOLDER_SEPARATOR)) { - return dirPath; + private String transformObsKeyToAbsolutePath(String s3Key) { + if (s3Key.endsWith("/")) { + return s3Key.substring(0, s3Key.length() - 1); } - - Path path = Paths.get(dirPath); - return path.getName(path.getNameCount() - 1) + FOLDER_SEPARATOR; + return s3Key; } + } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperatorFactory.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperatorFactory.java index 2e67103931..66ba23ee48 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperatorFactory.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperatorFactory.java @@ -17,23 +17,32 @@ package org.apache.dolphinscheduler.plugin.storage.obs; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperateFactory; +import org.apache.dolphinscheduler.common.constants.Constants; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperatorFactory; import org.apache.dolphinscheduler.plugin.storage.api.StorageType; +import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; import com.google.auto.service.AutoService; -@AutoService({StorageOperateFactory.class}) -public class ObsStorageOperatorFactory implements StorageOperateFactory { +@AutoService({StorageOperatorFactory.class}) +public class ObsStorageOperatorFactory implements StorageOperatorFactory { - public ObsStorageOperatorFactory() { + @Override + public StorageOperator createStorageOperate() { + final ObsStorageProperties obsStorageProperties = getObsStorageProperties(); + return new ObsStorageOperator(obsStorageProperties); } - @Override - public StorageOperate createStorageOperate() { - ObsStorageOperator ossOperator = new ObsStorageOperator(); - ossOperator.init(); - return ossOperator; + private ObsStorageProperties getObsStorageProperties() { + return ObsStorageProperties.builder() + .accessKeyId(PropertyUtils.getString(TaskConstants.HUAWEI_CLOUD_ACCESS_KEY_ID)) + .accessKeySecret(PropertyUtils.getString(TaskConstants.HUAWEI_CLOUD_ACCESS_KEY_SECRET)) + .bucketName(PropertyUtils.getString(Constants.HUAWEI_CLOUD_OBS_BUCKET_NAME)) + .endPoint(PropertyUtils.getString(Constants.HUAWEI_CLOUD_OBS_END_POINT)) + .resourceUploadPath(PropertyUtils.getString(Constants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler")) + .build(); } @Override diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageProperties.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageProperties.java new file mode 100644 index 0000000000..a71e9b21b7 --- /dev/null +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageProperties.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.storage.obs; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class ObsStorageProperties { + + private String accessKeyId; + + private String accessKeySecret; + + private String bucketName; + + private String endPoint; + + private String resourceUploadPath; +} diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/test/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperatorTest.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/test/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperatorTest.java deleted file mode 100644 index 4aabdb6bd8..0000000000 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/test/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperatorTest.java +++ /dev/null @@ -1,291 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.storage.obs; - -import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPARATOR; -import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; -import org.apache.dolphinscheduler.spi.enums.ResourceType; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; - -import com.obs.services.ObsClient; - -@ExtendWith(MockitoExtension.class) -public class ObsStorageOperatorTest { - - private static final String ACCESS_KEY_ID_MOCK = "ACCESS_KEY_ID_MOCK"; - private static final String ACCESS_KEY_SECRET_MOCK = "ACCESS_KEY_SECRET_MOCK"; - private static final String END_POINT_MOCK = "END_POINT_MOCK"; - private static final String BUCKET_NAME_MOCK = "BUCKET_NAME_MOCK"; - private static final String TENANT_CODE_MOCK = "TENANT_CODE_MOCK"; - private static final String DIR_MOCK = "DIR_MOCK"; - private static final String FILE_NAME_MOCK = "FILE_NAME_MOCK"; - private static final String FILE_PATH_MOCK = "FILE_PATH_MOCK"; - private static final String FULL_NAME = "/tmp/dir1/"; - - private static final String DEFAULT_PATH = "/tmp/"; - @Mock - private ObsClient obsClientMock; - - private ObsStorageOperator obsOperator; - - @BeforeEach - public void setUp() throws Exception { - obsOperator = spy(new ObsStorageOperator()); - doReturn(ACCESS_KEY_ID_MOCK).when(obsOperator) - .readObsAccessKeyID(); - doReturn(ACCESS_KEY_SECRET_MOCK).when(obsOperator) - .readObsAccessKeySecret(); - doReturn(BUCKET_NAME_MOCK).when(obsOperator).readObsBucketName(); - doReturn(END_POINT_MOCK).when(obsOperator).readObsEndPoint(); - doReturn(obsClientMock).when(obsOperator).buildObsClient(); - doNothing().when(obsOperator).ensureBucketSuccessfullyCreated(any()); - - obsOperator.init(); - - } - - @Test - public void initObsOperator() { - verify(obsOperator, times(1)).buildObsClient(); - Assertions.assertEquals(ACCESS_KEY_ID_MOCK, obsOperator.getAccessKeyId()); - Assertions.assertEquals(ACCESS_KEY_SECRET_MOCK, obsOperator.getAccessKeySecret()); - Assertions.assertEquals(BUCKET_NAME_MOCK, obsOperator.getBucketName()); - } - - @Test - public void tearDownObsOperator() throws IOException { - doNothing().when(obsClientMock).close(); - obsOperator.close(); - verify(obsClientMock, times(1)).close(); - } - - @Test - public void createTenantResAndUdfDir() throws Exception { - doReturn(DIR_MOCK).when(obsOperator).getObsResDir(TENANT_CODE_MOCK); - doReturn(DIR_MOCK).when(obsOperator).getObsUdfDir(TENANT_CODE_MOCK); - doReturn(true).when(obsOperator).mkdir(TENANT_CODE_MOCK, DIR_MOCK); - obsOperator.createTenantDirIfNotExists(TENANT_CODE_MOCK); - verify(obsOperator, times(2)).mkdir(TENANT_CODE_MOCK, DIR_MOCK); - } - - @Test - public void getResDir() { - final String expectedResourceDir = String.format("dolphinscheduler/%s/resources/", TENANT_CODE_MOCK); - final String dir = obsOperator.getResDir(TENANT_CODE_MOCK); - Assertions.assertEquals(expectedResourceDir, dir); - } - - @Test - public void getUdfDir() { - final String expectedUdfDir = String.format("dolphinscheduler/%s/udfs/", TENANT_CODE_MOCK); - final String dir = obsOperator.getUdfDir(TENANT_CODE_MOCK); - Assertions.assertEquals(expectedUdfDir, dir); - } - - @Test - public void mkdirWhenDirExists() { - boolean isSuccess = false; - try { - final String key = DIR_MOCK + FOLDER_SEPARATOR; - doReturn(true).when(obsClientMock).doesObjectExist(BUCKET_NAME_MOCK, key); - isSuccess = obsOperator.mkdir(TENANT_CODE_MOCK, DIR_MOCK); - verify(obsClientMock, times(1)).doesObjectExist(BUCKET_NAME_MOCK, key); - - } catch (IOException e) { - Assertions.fail("test failed due to unexpected IO exception"); - } - - Assertions.assertTrue(isSuccess); - } - - @Test - public void mkdirWhenDirNotExists() { - boolean isSuccess = true; - try { - final String key = DIR_MOCK + FOLDER_SEPARATOR; - doReturn(false).when(obsClientMock).doesObjectExist(BUCKET_NAME_MOCK, key); - doNothing().when(obsOperator).createObsPrefix(BUCKET_NAME_MOCK, key); - isSuccess = obsOperator.mkdir(TENANT_CODE_MOCK, DIR_MOCK); - verify(obsClientMock, times(1)).doesObjectExist(BUCKET_NAME_MOCK, key); - verify(obsOperator, times(1)).createObsPrefix(BUCKET_NAME_MOCK, key); - - } catch (IOException e) { - Assertions.fail("test failed due to unexpected IO exception"); - } - - Assertions.assertTrue(isSuccess); - } - - @Test - public void getResourceFullName() { - final String expectedResourceFullName = - String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK); - final String resourceFullName = obsOperator.getResourceFullName(TENANT_CODE_MOCK, FILE_NAME_MOCK); - Assertions.assertEquals(expectedResourceFullName, resourceFullName); - } - - @Test - public void getResourceFileName() { - final String expectedResourceFileName = FILE_NAME_MOCK; - final String resourceFullName = - String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK); - final String resourceFileName = obsOperator.getResourceFileName(TENANT_CODE_MOCK, resourceFullName); - Assertions.assertEquals(expectedResourceFileName, resourceFileName); - } - - @Test - public void getFileName() { - final String expectedFileName = - String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK); - final String fileName = obsOperator.getFileName(ResourceType.FILE, TENANT_CODE_MOCK, FILE_NAME_MOCK); - Assertions.assertEquals(expectedFileName, fileName); - } - - @Test - public void exists() { - boolean doesExist = false; - doReturn(true).when(obsClientMock).doesObjectExist(BUCKET_NAME_MOCK, FILE_NAME_MOCK); - try { - doesExist = obsOperator.exists(FILE_NAME_MOCK); - } catch (IOException e) { - Assertions.fail("unexpected IO exception in unit test"); - } - - Assertions.assertTrue(doesExist); - verify(obsClientMock, times(1)).doesObjectExist(BUCKET_NAME_MOCK, FILE_NAME_MOCK); - } - - @Test - public void delete() { - boolean isDeleted = false; - doReturn(null).when(obsClientMock).deleteObject(anyString(), anyString()); - try { - isDeleted = obsOperator.delete(FILE_NAME_MOCK, true); - } catch (IOException e) { - Assertions.fail("unexpected IO exception in unit test"); - } - - Assertions.assertTrue(isDeleted); - verify(obsClientMock, times(1)).deleteObject(anyString(), anyString()); - } - - @Test - public void copy() { - boolean isSuccess = false; - doReturn(null).when(obsClientMock).copyObject(anyString(), anyString(), anyString(), anyString()); - try { - isSuccess = obsOperator.copy(FILE_PATH_MOCK, FILE_PATH_MOCK, false, false); - } catch (IOException e) { - Assertions.fail("unexpected IO exception in unit test"); - } - - Assertions.assertTrue(isSuccess); - verify(obsClientMock, times(1)).copyObject(anyString(), anyString(), anyString(), anyString()); - } - - @Test - public void deleteTenant() { - doNothing().when(obsOperator).deleteTenantCode(anyString()); - try { - obsOperator.deleteTenant(TENANT_CODE_MOCK); - } catch (Exception e) { - Assertions.fail("unexpected exception caught in unit test"); - } - - verify(obsOperator, times(1)).deleteTenantCode(anyString()); - } - - @Test - public void getObsResDir() { - final String expectedObsResDir = String.format("dolphinscheduler/%s/resources", TENANT_CODE_MOCK); - final String obsResDir = obsOperator.getObsResDir(TENANT_CODE_MOCK); - Assertions.assertEquals(expectedObsResDir, obsResDir); - } - - @Test - public void getObsUdfDir() { - final String expectedObsUdfDir = String.format("dolphinscheduler/%s/udfs", TENANT_CODE_MOCK); - final String obsUdfDir = obsOperator.getObsUdfDir(TENANT_CODE_MOCK); - Assertions.assertEquals(expectedObsUdfDir, obsUdfDir); - } - - @Test - public void getObsTenantDir() { - final String expectedObsTenantDir = String.format(FORMAT_S_S, DIR_MOCK, TENANT_CODE_MOCK); - doReturn(DIR_MOCK).when(obsOperator).getObsDataBasePath(); - final String obsTenantDir = obsOperator.getObsTenantDir(TENANT_CODE_MOCK); - Assertions.assertEquals(expectedObsTenantDir, obsTenantDir); - } - - @Test - public void deleteDir() { - doReturn(true).when(obsClientMock).doesObjectExist(anyString(), anyString()); - obsOperator.deleteDir(DIR_MOCK); - verify(obsClientMock, times(1)).deleteObject(anyString(), anyString()); - } - - @Test - public void testGetFileStatus() throws Exception { - StorageEntity entity = obsOperator.getFileStatus(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, ResourceType.FILE); - Assertions.assertEquals(FULL_NAME, entity.getFullName()); - Assertions.assertEquals("dir1/", entity.getFileName()); - } - - @Test - public void testListFilesStatus() throws Exception { - List result = - obsOperator.listFilesStatus("dolphinscheduler/default/resources/", - "dolphinscheduler/default/resources/", - "default", ResourceType.FILE); - Assertions.assertEquals(0, result.size()); - } - - @Test - public void testListFilesStatusRecursively() throws Exception { - StorageEntity entity = new StorageEntity(); - entity.setFullName(FULL_NAME); - - doReturn(entity).when(obsOperator).getFileStatus(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, ResourceType.FILE); - doReturn(Collections.EMPTY_LIST).when(obsOperator).listFilesStatus(anyString(), anyString(), anyString(), - Mockito.any(ResourceType.class)); - - List result = - obsOperator.listFilesStatusRecursively(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, ResourceType.FILE); - Assertions.assertEquals(0, result.size()); - } -} diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java index 61754b52a7..afdc6874c9 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java @@ -17,21 +17,16 @@ package org.apache.dolphinscheduler.plugin.storage.oss; -import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPARATOR; -import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S; -import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TYPE_FILE; -import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TYPE_UDF; - import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.enums.ResUploadType; import org.apache.dolphinscheduler.common.factory.OssClientFactory; import org.apache.dolphinscheduler.common.model.OssConnection; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.plugin.storage.api.AbstractStorageOperator; +import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata; import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.dolphinscheduler.plugin.task.api.TaskConstants; -import org.apache.dolphinscheduler.spi.enums.ResourceType; import org.apache.commons.lang3.StringUtils; @@ -44,23 +39,22 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Collections; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; -import java.util.stream.Stream; -import lombok.Data; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import com.aliyun.oss.OSS; import com.aliyun.oss.OSSException; -import com.aliyun.oss.ServiceException; -import com.aliyun.oss.model.DeleteObjectsRequest; +import com.aliyun.oss.model.GetObjectRequest; import com.aliyun.oss.model.ListObjectsV2Request; import com.aliyun.oss.model.ListObjectsV2Result; import com.aliyun.oss.model.OSSObject; @@ -68,9 +62,8 @@ import com.aliyun.oss.model.OSSObjectSummary; import com.aliyun.oss.model.ObjectMetadata; import com.aliyun.oss.model.PutObjectRequest; -@Data @Slf4j -public class OssStorageOperator implements Closeable, StorageOperate { +public class OssStorageOperator extends AbstractStorageOperator implements Closeable, StorageOperator { private String accessKeyId; @@ -86,10 +79,11 @@ public class OssStorageOperator implements Closeable, StorageOperate { private OSS ossClient; - public OssStorageOperator() { + public OssStorageOperator(String resourceBaseAbsolutePath) { + super(resourceBaseAbsolutePath); } - public void init() { + private void init() { this.accessKeyId = readOssAccessKeyID(); this.accessKeySecret = readOssAccessKeySecret(); this.endPoint = readOssEndPoint(); @@ -108,7 +102,7 @@ public class OssStorageOperator implements Closeable, StorageOperate { this.region = readOssRegion(); this.bucketName = readOssBucketName(); this.ossConnection = ossConnection; - this.ossClient = getOssClient(); + this.ossClient = buildOssClient(); ensureBucketSuccessfullyCreated(bucketName); } @@ -137,79 +131,43 @@ public class OssStorageOperator implements Closeable, StorageOperate { } @Override - public void close() throws IOException { - ossClient.shutdown(); - } - - @Override - public void createTenantDirIfNotExists(String tenantCode) throws Exception { - mkdir(tenantCode, getOssResDir(tenantCode)); - mkdir(tenantCode, getOssUdfDir(tenantCode)); + public String getStorageBaseDirectory() { + // All directory should end with File.separator + if (resourceBaseAbsolutePath.startsWith("/")) { + log.warn("{} -> {} should not start with / in Oss", Constants.RESOURCE_UPLOAD_PATH, + resourceBaseAbsolutePath); + return resourceBaseAbsolutePath.substring(1); + } + return resourceBaseAbsolutePath; } @Override - public String getResDir(String tenantCode) { - return getOssResDir(tenantCode) + FOLDER_SEPARATOR; + public void close() throws IOException { + ossClient.shutdown(); } + @SneakyThrows @Override - public String getUdfDir(String tenantCode) { - return getOssUdfDir(tenantCode) + FOLDER_SEPARATOR; - } + public void createStorageDir(String directory) { + directory = transformAbsolutePathToOssKey(directory); - @Override - public boolean mkdir(String tenantCode, String path) throws IOException { - final String key = path + FOLDER_SEPARATOR; - if (!ossClient.doesObjectExist(bucketName, key)) { - createOssPrefix(bucketName, key); + if (ossClient.doesObjectExist(bucketName, directory)) { + throw new FileAlreadyExistsException("directory: " + directory + " already exists"); } - return true; - } - - protected void createOssPrefix(final String bucketName, final String key) { ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(0); InputStream emptyContent = new ByteArrayInputStream(new byte[0]); - PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, key, emptyContent, metadata); + PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, directory, emptyContent, metadata); ossClient.putObject(putObjectRequest); } + @SneakyThrows @Override - public String getResourceFullName(String tenantCode, String fileName) { - if (fileName.startsWith(FOLDER_SEPARATOR)) { - fileName = fileName.replaceFirst(FOLDER_SEPARATOR, ""); - } - return String.format(FORMAT_S_S, getOssResDir(tenantCode), fileName); - } - - @Override - public String getFileName(ResourceType resourceType, String tenantCode, String fileName) { - if (fileName.startsWith(FOLDER_SEPARATOR)) { - fileName = fileName.replaceFirst(FOLDER_SEPARATOR, ""); - } - return getDir(resourceType, tenantCode) + fileName; - } - - @Override - public boolean delete(String fullName, List childrenPathList, boolean recursive) throws IOException { - // append the resource fullName to the list for deletion. - childrenPathList.add(fullName); - - DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucketName) - .withKeys(childrenPathList); - try { - ossClient.deleteObjects(deleteObjectsRequest); - } catch (Exception e) { - log.error("delete objects error", e); - return false; - } + public void download(String srcFilePath, + String dstFilePath, + boolean overwrite) { + srcFilePath = transformAbsolutePathToOssKey(srcFilePath); - return true; - } - - @Override - public void download(String srcFilePath, String dstFilePath, - boolean overwrite) throws IOException { File dstFile = new File(dstFilePath); if (dstFile.isDirectory()) { Files.delete(dstFile.toPath()); @@ -234,300 +192,198 @@ public class OssStorageOperator implements Closeable, StorageOperate { } @Override - public boolean exists(String fileName) throws IOException { + public boolean exists(String fileName) { + fileName = transformAbsolutePathToOssKey(fileName); return ossClient.doesObjectExist(bucketName, fileName); } @Override - public boolean delete(String filePath, boolean recursive) throws IOException { - try { - ossClient.deleteObject(bucketName, filePath); - return true; - } catch (OSSException e) { - log.error("fail to delete the object, the resource path is {}", filePath, e); - return false; - } + public void delete(String filePath, boolean recursive) { + filePath = transformAbsolutePathToOssKey(filePath); + ossClient.deleteObject(bucketName, filePath); } @Override - public boolean copy(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) throws IOException { + public void copy(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) { + srcPath = transformAbsolutePathToOssKey(srcPath); + dstPath = transformAbsolutePathToOssKey(dstPath); + ossClient.copyObject(bucketName, srcPath, bucketName, dstPath); if (deleteSource) { ossClient.deleteObject(bucketName, srcPath); } - return true; - } - - @Override - public String getDir(ResourceType resourceType, String tenantCode) { - switch (resourceType) { - case UDF: - return getUdfDir(tenantCode); - case FILE: - return getResDir(tenantCode); - case ALL: - return getOssDataBasePath(); - default: - return ""; - } } + @SneakyThrows @Override - public boolean upload(String tenantCode, String srcFile, String dstPath, boolean deleteSource, - boolean overwrite) throws IOException { - try { - ossClient.putObject(bucketName, dstPath, new File(srcFile)); - if (deleteSource) { - Files.delete(Paths.get(srcFile)); + public void upload(String srcFile, String dstPath, boolean deleteSource, boolean overwrite) { + dstPath = transformAbsolutePathToOssKey(dstPath); + if (ossClient.doesObjectExist(bucketName, dstPath)) { + if (!overwrite) { + throw new FileAlreadyExistsException("file: " + dstPath + " already exists"); + } else { + ossClient.deleteObject(bucketName, dstPath); } - return true; - } catch (OSSException e) { - log.error("upload failed, the bucketName is {}, the filePath is {}", bucketName, dstPath, e); - return false; + + } + ossClient.putObject(bucketName, dstPath, new File(srcFile)); + if (deleteSource) { + Files.delete(Paths.get(srcFile)); } } + @SneakyThrows @Override - public List vimFile(String tenantCode, String filePath, int skipLineNums, int limit) throws IOException { - if (StringUtils.isBlank(filePath)) { - log.error("file path:{} is empty", filePath); - return Collections.emptyList(); - } + public List fetchFileContent(String filePath, int skipLineNums, int limit) { + filePath = transformAbsolutePathToOssKey(filePath); OSSObject ossObject = ossClient.getObject(bucketName, filePath); - try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(ossObject.getObjectContent()))) { - Stream stream = bufferedReader.lines().skip(skipLineNums).limit(limit); - return stream.collect(Collectors.toList()); + try ( + InputStreamReader inputStreamReader = new InputStreamReader(ossObject.getObjectContent()); + BufferedReader bufferedReader = new BufferedReader(inputStreamReader)) { + return bufferedReader.lines() + .skip(skipLineNums) + .limit(limit) + .collect(Collectors.toList()); } } @Override - public ResUploadType returnStorageType() { - return ResUploadType.OSS; + public List listStorageEntity(String resourceAbsolutePath) { + final String ossResourceAbsolutePath = transformAbsolutePathToOssKey(resourceAbsolutePath); + + ListObjectsV2Request listObjectsV2Request = new ListObjectsV2Request() + .withBucketName(bucketName) + .withDelimiter("/") + .withPrefix(ossResourceAbsolutePath); + + ListObjectsV2Result listObjectsV2Result = ossClient.listObjectsV2(listObjectsV2Request); + List storageEntities = new ArrayList<>(); + storageEntities.addAll(listObjectsV2Result.getCommonPrefixes() + .stream() + .map(this::transformCommonPrefixToStorageEntity) + .collect(Collectors.toList())); + storageEntities.addAll( + listObjectsV2Result.getObjectSummaries().stream() + .filter(s3ObjectSummary -> !s3ObjectSummary.getKey().equals(resourceAbsolutePath)) + .map(this::transformOSSObjectToStorageEntity) + .collect(Collectors.toList())); + + return storageEntities; + } @Override - public List listFilesStatusRecursively(String path, String defaultPath, String tenantCode, - ResourceType type) { + public List listFileStorageEntityRecursively(String resourceAbsolutePath) { + resourceAbsolutePath = transformOssKeyToAbsolutePath(resourceAbsolutePath); + + Set visited = new HashSet<>(); List storageEntityList = new ArrayList<>(); - LinkedList foldersToFetch = new LinkedList<>(); - - StorageEntity initialEntity = null; - try { - initialEntity = getFileStatus(path, defaultPath, tenantCode, type); - } catch (Exception e) { - log.error("error while listing files status recursively, path: {}", path, e); - return storageEntityList; - } - foldersToFetch.add(initialEntity); + LinkedList foldersToFetch = new LinkedList<>(); + foldersToFetch.addLast(resourceAbsolutePath); while (!foldersToFetch.isEmpty()) { - String pathToExplore = foldersToFetch.pop().getFullName(); - try { - List tempList = listFilesStatus(pathToExplore, defaultPath, tenantCode, type); - for (StorageEntity temp : tempList) { - if (temp.isDirectory()) { - foldersToFetch.add(temp); + String pathToExplore = foldersToFetch.pop(); + visited.add(pathToExplore); + List tempList = listStorageEntity(pathToExplore); + for (StorageEntity temp : tempList) { + if (temp.isDirectory()) { + if (visited.contains(temp.getFullName())) { + continue; } + foldersToFetch.add(temp.getFullName()); } - storageEntityList.addAll(tempList); - } catch (Exception e) { - log.error("error while listing files stat:wus recursively, path: {}", pathToExplore, e); } + storageEntityList.addAll(tempList); } return storageEntityList; } @Override - public List listFilesStatus(String path, String defaultPath, String tenantCode, - ResourceType type) throws Exception { - List storageEntityList = new ArrayList<>(); - - ListObjectsV2Result result = null; - String nextContinuationToken = null; - do { - try { - ListObjectsV2Request request = new ListObjectsV2Request(); - request.setBucketName(bucketName); - request.setPrefix(path); - request.setDelimiter(FOLDER_SEPARATOR); - request.setContinuationToken(nextContinuationToken); - - result = ossClient.listObjectsV2(request); - } catch (Exception e) { - throw new ServiceException("Get OSS file list exception", e); - } - - List summaries = result.getObjectSummaries(); - - for (OSSObjectSummary summary : summaries) { - if (!summary.getKey().endsWith(FOLDER_SEPARATOR)) { - // the path is a file - String[] aliasArr = summary.getKey().split(FOLDER_SEPARATOR); - String alias = aliasArr[aliasArr.length - 1]; - String fileName = StringUtils.difference(defaultPath, summary.getKey()); - - StorageEntity entity = new StorageEntity(); - entity.setAlias(alias); - entity.setFileName(fileName); - entity.setFullName(summary.getKey()); - entity.setDirectory(false); - entity.setUserName(tenantCode); - entity.setType(type); - entity.setSize(summary.getSize()); - entity.setCreateTime(summary.getLastModified()); - entity.setUpdateTime(summary.getLastModified()); - entity.setPfullName(path); - - storageEntityList.add(entity); - } - } - - for (String commonPrefix : result.getCommonPrefixes()) { - // the paths in commonPrefix are directories - String suffix = StringUtils.difference(path, commonPrefix); - String fileName = StringUtils.difference(defaultPath, commonPrefix); - - StorageEntity entity = new StorageEntity(); - entity.setAlias(suffix); - entity.setFileName(fileName); - entity.setFullName(commonPrefix); - entity.setDirectory(true); - entity.setUserName(tenantCode); - entity.setType(type); - entity.setSize(0); - entity.setCreateTime(null); - entity.setUpdateTime(null); - entity.setPfullName(path); - - storageEntityList.add(entity); - } - - nextContinuationToken = result.getNextContinuationToken(); - } while (result.isTruncated()); - - return storageEntityList; + public StorageEntity getStorageEntity(String resourceAbsolutePath) { + resourceAbsolutePath = transformAbsolutePathToOssKey(resourceAbsolutePath); + OSSObject object = ossClient.getObject(new GetObjectRequest(bucketName, resourceAbsolutePath)); + return transformOSSObjectToStorageEntity(object); } - @Override - public StorageEntity getFileStatus(String path, String defaultPath, String tenantCode, - ResourceType type) throws Exception { - ListObjectsV2Request request = new ListObjectsV2Request(); - request.setBucketName(bucketName); - request.setPrefix(path); - request.setDelimiter(FOLDER_SEPARATOR); - - ListObjectsV2Result result; - try { - result = ossClient.listObjectsV2(request); - } catch (Exception e) { - throw new ServiceException("Get OSS file list exception", e); + public void ensureBucketSuccessfullyCreated(String bucketName) { + if (StringUtils.isBlank(bucketName)) { + throw new IllegalArgumentException("resource.alibaba.cloud.oss.bucket.name is empty"); } - List summaries = result.getObjectSummaries(); - - if (path.endsWith(FOLDER_SEPARATOR)) { - // the path is a directory that may or may not exist in OSS - String alias = findDirAlias(path); - String fileName = StringUtils.difference(defaultPath, path); - - StorageEntity entity = new StorageEntity(); - entity.setAlias(alias); - entity.setFileName(fileName); - entity.setFullName(path); - entity.setDirectory(true); - entity.setUserName(tenantCode); - entity.setType(type); - entity.setSize(0); - - return entity; - - } else { - // the path is a file - if (summaries.size() > 0) { - OSSObjectSummary summary = summaries.get(0); - String[] aliasArr = summary.getKey().split(FOLDER_SEPARATOR); - String alias = aliasArr[aliasArr.length - 1]; - String fileName = StringUtils.difference(defaultPath, summary.getKey()); - - StorageEntity entity = new StorageEntity(); - entity.setAlias(alias); - entity.setFileName(fileName); - entity.setFullName(summary.getKey()); - entity.setDirectory(false); - entity.setUserName(tenantCode); - entity.setType(type); - entity.setSize(summary.getSize()); - entity.setCreateTime(summary.getLastModified()); - entity.setUpdateTime(summary.getLastModified()); - - return entity; - } + boolean existsBucket = ossClient.doesBucketExist(bucketName); + if (!existsBucket) { + throw new IllegalArgumentException( + "bucketName: " + bucketName + " is not exists, you need to create them by yourself"); } - throw new FileNotFoundException("Object is not found in OSS Bucket: " + bucketName); + log.info("bucketName: {} has been found, the current regionName is {}", bucketName, region); } - @Override - public void deleteTenant(String tenantCode) throws Exception { - deleteTenantCode(tenantCode); + protected OSS buildOssClient() { + return OssClientFactory.buildOssClient(ossConnection); } - public String getOssResDir(String tenantCode) { - return String.format("%s/" + RESOURCE_TYPE_FILE, getOssTenantDir(tenantCode)); - } + protected StorageEntity transformOSSObjectToStorageEntity(OSSObject ossObject) { + ResourceMetadata resourceMetaData = getResourceMetaData(ossObject.getKey()); - public String getOssUdfDir(String tenantCode) { - return String.format("%s/" + RESOURCE_TYPE_UDF, getOssTenantDir(tenantCode)); + StorageEntity storageEntity = new StorageEntity(); + storageEntity.setFileName(new File(ossObject.getKey()).getName()); + storageEntity.setFullName(ossObject.getKey()); + storageEntity.setPfullName(resourceMetaData.getResourceParentAbsolutePath()); + storageEntity.setType(resourceMetaData.getResourceType()); + storageEntity.setDirectory(resourceMetaData.isDirectory()); + storageEntity.setSize(ossObject.getObjectMetadata().getContentLength()); + storageEntity.setCreateTime(ossObject.getObjectMetadata().getLastModified()); + storageEntity.setUpdateTime(ossObject.getObjectMetadata().getLastModified()); + return storageEntity; } - public String getOssTenantDir(String tenantCode) { - return String.format(FORMAT_S_S, getOssDataBasePath(), tenantCode); - } + private StorageEntity transformOSSObjectToStorageEntity(OSSObjectSummary ossObjectSummary) { + String absolutePath = transformOssKeyToAbsolutePath(ossObjectSummary.getKey()); - public String getOssDataBasePath() { - if (FOLDER_SEPARATOR.equals(RESOURCE_UPLOAD_PATH)) { - return ""; - } else { - return RESOURCE_UPLOAD_PATH.replaceFirst(FOLDER_SEPARATOR, ""); - } - } + ResourceMetadata resourceMetaData = getResourceMetaData(absolutePath); - protected void deleteTenantCode(String tenantCode) { - deleteDir(getResDir(tenantCode)); - deleteDir(getUdfDir(tenantCode)); + StorageEntity storageEntity = new StorageEntity(); + storageEntity.setFileName(new File(absolutePath).getName()); + storageEntity.setFullName(absolutePath); + storageEntity.setPfullName(resourceMetaData.getResourceParentAbsolutePath()); + storageEntity.setType(resourceMetaData.getResourceType()); + storageEntity.setDirectory(resourceMetaData.isDirectory()); + storageEntity.setSize(ossObjectSummary.getSize()); + storageEntity.setCreateTime(ossObjectSummary.getLastModified()); + storageEntity.setUpdateTime(ossObjectSummary.getLastModified()); + return storageEntity; } - public void ensureBucketSuccessfullyCreated(String bucketName) { - if (StringUtils.isBlank(bucketName)) { - throw new IllegalArgumentException("resource.alibaba.cloud.oss.bucket.name is empty"); - } + private StorageEntity transformCommonPrefixToStorageEntity(String commonPrefix) { + String absolutePath = transformOssKeyToAbsolutePath(commonPrefix); - boolean existsBucket = ossClient.doesBucketExist(bucketName); - if (!existsBucket) { - throw new IllegalArgumentException( - "bucketName: " + bucketName + " is not exists, you need to create them by yourself"); - } + ResourceMetadata resourceMetaData = getResourceMetaData(absolutePath); - log.info("bucketName: {} has been found, the current regionName is {}", bucketName, region); + StorageEntity entity = new StorageEntity(); + entity.setFileName(new File(absolutePath).getName()); + entity.setFullName(absolutePath); + entity.setDirectory(resourceMetaData.isDirectory()); + entity.setType(resourceMetaData.getResourceType()); + entity.setSize(0L); + entity.setCreateTime(null); + entity.setUpdateTime(null); + return entity; } - protected void deleteDir(String directoryName) { - if (ossClient.doesObjectExist(bucketName, directoryName)) { - ossClient.deleteObject(bucketName, directoryName); + private String transformAbsolutePathToOssKey(String absolutePath) { + ResourceMetadata resourceMetaData = getResourceMetaData(absolutePath); + if (resourceMetaData.isDirectory()) { + return FileUtils.concatFilePath(absolutePath, "/"); } + return absolutePath; } - protected OSS buildOssClient() { - return OssClientFactory.buildOssClient(ossConnection); - } - - private String findDirAlias(String dirPath) { - if (!dirPath.endsWith(FOLDER_SEPARATOR)) { - return dirPath; + private String transformOssKeyToAbsolutePath(String s3Key) { + if (s3Key.endsWith("/")) { + return s3Key.substring(0, s3Key.length() - 1); } - - Path path = Paths.get(dirPath); - return path.getName(path.getNameCount() - 1) + FOLDER_SEPARATOR; + return s3Key; } } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperatorFactory.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperatorFactory.java index 783faf59ea..966cc471ab 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperatorFactory.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperatorFactory.java @@ -17,20 +17,20 @@ package org.apache.dolphinscheduler.plugin.storage.oss; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperateFactory; +import org.apache.dolphinscheduler.common.constants.Constants; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperatorFactory; import org.apache.dolphinscheduler.plugin.storage.api.StorageType; import com.google.auto.service.AutoService; -@AutoService(StorageOperateFactory.class) -public class OssStorageOperatorFactory implements StorageOperateFactory { +@AutoService(StorageOperatorFactory.class) +public class OssStorageOperatorFactory implements StorageOperatorFactory { @Override - public StorageOperate createStorageOperate() { - OssStorageOperator ossOperator = new OssStorageOperator(); - ossOperator.init(); - return ossOperator; + public StorageOperator createStorageOperate() { + return new OssStorageOperator(PropertyUtils.getString(Constants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler")); } @Override diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/test/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperatorTest.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/test/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperatorTest.java deleted file mode 100644 index b84df0a024..0000000000 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/test/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperatorTest.java +++ /dev/null @@ -1,298 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.plugin.storage.oss; - -import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPARATOR; -import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; -import org.apache.dolphinscheduler.spi.enums.ResourceType; - -import java.io.IOException; -import java.util.Collections; -import java.util.List; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; - -import com.aliyun.oss.OSS; -import com.aliyun.oss.model.ListObjectsV2Request; -import com.aliyun.oss.model.ListObjectsV2Result; - -@ExtendWith(MockitoExtension.class) -public class OssStorageOperatorTest { - - private static final String ACCESS_KEY_ID_MOCK = "ACCESS_KEY_ID_MOCK"; - private static final String ACCESS_KEY_SECRET_MOCK = "ACCESS_KEY_SECRET_MOCK"; - private static final String REGION_MOCK = "REGION_MOCK"; - private static final String END_POINT_MOCK = "END_POINT_MOCK"; - private static final String BUCKET_NAME_MOCK = "BUCKET_NAME_MOCK"; - private static final String TENANT_CODE_MOCK = "TENANT_CODE_MOCK"; - private static final String DIR_MOCK = "DIR_MOCK"; - private static final String FILE_NAME_MOCK = "FILE_NAME_MOCK"; - private static final String FILE_PATH_MOCK = "FILE_PATH_MOCK"; - - private static final String FULL_NAME = "/tmp/dir1/"; - - private static final String DEFAULT_PATH = "/tmp/"; - - @Mock - private OSS ossClientMock; - - private OssStorageOperator ossOperator; - - @BeforeEach - public void setUp() throws Exception { - ossOperator = spy(new OssStorageOperator()); - doReturn(ACCESS_KEY_ID_MOCK).when(ossOperator) - .readOssAccessKeyID(); - doReturn(ACCESS_KEY_SECRET_MOCK).when(ossOperator) - .readOssAccessKeySecret(); - doReturn(REGION_MOCK).when(ossOperator).readOssRegion(); - doReturn(BUCKET_NAME_MOCK).when(ossOperator).readOssBucketName(); - doReturn(END_POINT_MOCK).when(ossOperator).readOssEndPoint(); - doReturn(ossClientMock).when(ossOperator).buildOssClient(); - doNothing().when(ossOperator).ensureBucketSuccessfullyCreated(any()); - - ossOperator.init(); - - } - - @Test - public void initOssOperator() { - verify(ossOperator, times(1)).buildOssClient(); - Assertions.assertEquals(ACCESS_KEY_ID_MOCK, ossOperator.getAccessKeyId()); - Assertions.assertEquals(ACCESS_KEY_SECRET_MOCK, ossOperator.getAccessKeySecret()); - Assertions.assertEquals(REGION_MOCK, ossOperator.getRegion()); - Assertions.assertEquals(BUCKET_NAME_MOCK, ossOperator.getBucketName()); - } - - @Test - public void tearDownOssOperator() throws IOException { - doNothing().when(ossClientMock).shutdown(); - ossOperator.close(); - verify(ossClientMock, times(1)).shutdown(); - } - - @Test - public void createTenantResAndUdfDir() throws Exception { - doReturn(DIR_MOCK).when(ossOperator).getOssResDir(TENANT_CODE_MOCK); - doReturn(DIR_MOCK).when(ossOperator).getOssUdfDir(TENANT_CODE_MOCK); - doReturn(true).when(ossOperator).mkdir(TENANT_CODE_MOCK, DIR_MOCK); - ossOperator.createTenantDirIfNotExists(TENANT_CODE_MOCK); - verify(ossOperator, times(2)).mkdir(TENANT_CODE_MOCK, DIR_MOCK); - } - - @Test - public void getResDir() { - final String expectedResourceDir = String.format("dolphinscheduler/%s/resources/", TENANT_CODE_MOCK); - final String dir = ossOperator.getResDir(TENANT_CODE_MOCK); - Assertions.assertEquals(expectedResourceDir, dir); - } - - @Test - public void getUdfDir() { - final String expectedUdfDir = String.format("dolphinscheduler/%s/udfs/", TENANT_CODE_MOCK); - final String dir = ossOperator.getUdfDir(TENANT_CODE_MOCK); - Assertions.assertEquals(expectedUdfDir, dir); - } - - @Test - public void mkdirWhenDirExists() { - boolean isSuccess = false; - try { - final String key = DIR_MOCK + FOLDER_SEPARATOR; - doReturn(true).when(ossClientMock).doesObjectExist(BUCKET_NAME_MOCK, key); - isSuccess = ossOperator.mkdir(TENANT_CODE_MOCK, DIR_MOCK); - verify(ossClientMock, times(1)).doesObjectExist(BUCKET_NAME_MOCK, key); - - } catch (IOException e) { - Assertions.fail("test failed due to unexpected IO exception"); - } - - Assertions.assertTrue(isSuccess); - } - - @Test - public void mkdirWhenDirNotExists() { - boolean isSuccess = true; - try { - final String key = DIR_MOCK + FOLDER_SEPARATOR; - doReturn(false).when(ossClientMock).doesObjectExist(BUCKET_NAME_MOCK, key); - doNothing().when(ossOperator).createOssPrefix(BUCKET_NAME_MOCK, key); - isSuccess = ossOperator.mkdir(TENANT_CODE_MOCK, DIR_MOCK); - verify(ossClientMock, times(1)).doesObjectExist(BUCKET_NAME_MOCK, key); - verify(ossOperator, times(1)).createOssPrefix(BUCKET_NAME_MOCK, key); - - } catch (IOException e) { - Assertions.fail("test failed due to unexpected IO exception"); - } - - Assertions.assertTrue(isSuccess); - } - - @Test - public void getResourceFullName() { - final String expectedResourceFullName = - String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK); - final String resourceFullName = ossOperator.getResourceFullName(TENANT_CODE_MOCK, FILE_NAME_MOCK); - Assertions.assertEquals(expectedResourceFullName, resourceFullName); - } - - @Test - public void getResourceFileName() { - final String expectedResourceFileName = FILE_NAME_MOCK; - final String resourceFullName = - String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK); - final String resourceFileName = ossOperator.getResourceFileName(TENANT_CODE_MOCK, resourceFullName); - Assertions.assertEquals(expectedResourceFileName, resourceFileName); - } - - @Test - public void getFileName() { - final String expectedFileName = - String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK); - final String fileName = ossOperator.getFileName(ResourceType.FILE, TENANT_CODE_MOCK, FILE_NAME_MOCK); - Assertions.assertEquals(expectedFileName, fileName); - } - - @Test - public void exists() { - boolean doesExist = false; - doReturn(true).when(ossClientMock).doesObjectExist(BUCKET_NAME_MOCK, FILE_NAME_MOCK); - try { - doesExist = ossOperator.exists(FILE_NAME_MOCK); - } catch (IOException e) { - Assertions.fail("unexpected IO exception in unit test"); - } - - Assertions.assertTrue(doesExist); - verify(ossClientMock, times(1)).doesObjectExist(BUCKET_NAME_MOCK, FILE_NAME_MOCK); - } - - @Test - public void delete() { - boolean isDeleted = false; - doReturn(null).when(ossClientMock).deleteObject(anyString(), anyString()); - try { - isDeleted = ossOperator.delete(FILE_NAME_MOCK, true); - } catch (IOException e) { - Assertions.fail("unexpected IO exception in unit test"); - } - - Assertions.assertTrue(isDeleted); - verify(ossClientMock, times(1)).deleteObject(anyString(), anyString()); - } - - @Test - public void copy() { - boolean isSuccess = false; - doReturn(null).when(ossClientMock).copyObject(anyString(), anyString(), anyString(), anyString()); - try { - isSuccess = ossOperator.copy(FILE_PATH_MOCK, FILE_PATH_MOCK, false, false); - } catch (IOException e) { - Assertions.fail("unexpected IO exception in unit test"); - } - - Assertions.assertTrue(isSuccess); - verify(ossClientMock, times(1)).copyObject(anyString(), anyString(), anyString(), anyString()); - } - - @Test - public void deleteTenant() { - doNothing().when(ossOperator).deleteTenantCode(anyString()); - try { - ossOperator.deleteTenant(TENANT_CODE_MOCK); - } catch (Exception e) { - Assertions.fail("unexpected exception caught in unit test"); - } - - verify(ossOperator, times(1)).deleteTenantCode(anyString()); - } - - @Test - public void getOssResDir() { - final String expectedOssResDir = String.format("dolphinscheduler/%s/resources", TENANT_CODE_MOCK); - final String ossResDir = ossOperator.getOssResDir(TENANT_CODE_MOCK); - Assertions.assertEquals(expectedOssResDir, ossResDir); - } - - @Test - public void getOssUdfDir() { - final String expectedOssUdfDir = String.format("dolphinscheduler/%s/udfs", TENANT_CODE_MOCK); - final String ossUdfDir = ossOperator.getOssUdfDir(TENANT_CODE_MOCK); - Assertions.assertEquals(expectedOssUdfDir, ossUdfDir); - } - - @Test - public void getOssTenantDir() { - final String expectedOssTenantDir = String.format(FORMAT_S_S, DIR_MOCK, TENANT_CODE_MOCK); - doReturn(DIR_MOCK).when(ossOperator).getOssDataBasePath(); - final String ossTenantDir = ossOperator.getOssTenantDir(TENANT_CODE_MOCK); - Assertions.assertEquals(expectedOssTenantDir, ossTenantDir); - } - - @Test - public void deleteDir() { - doReturn(true).when(ossClientMock).doesObjectExist(anyString(), anyString()); - ossOperator.deleteDir(DIR_MOCK); - verify(ossClientMock, times(1)).deleteObject(anyString(), anyString()); - } - - @Test - public void testGetFileStatus() throws Exception { - doReturn(new ListObjectsV2Result()).when(ossClientMock).listObjectsV2(Mockito.any(ListObjectsV2Request.class)); - StorageEntity entity = ossOperator.getFileStatus(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, ResourceType.FILE); - Assertions.assertEquals(FULL_NAME, entity.getFullName()); - Assertions.assertEquals("dir1/", entity.getFileName()); - } - - @Test - public void testListFilesStatus() throws Exception { - doReturn(new ListObjectsV2Result()).when(ossClientMock).listObjectsV2(Mockito.any(ListObjectsV2Request.class)); - List result = - ossOperator.listFilesStatus(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, ResourceType.FILE); - Assertions.assertEquals(0, result.size()); - } - - @Test - public void testListFilesStatusRecursively() throws Exception { - StorageEntity entity = new StorageEntity(); - entity.setFullName(FULL_NAME); - - doReturn(entity).when(ossOperator).getFileStatus(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, ResourceType.FILE); - doReturn(Collections.EMPTY_LIST).when(ossOperator).listFilesStatus(anyString(), anyString(), anyString(), - Mockito.any(ResourceType.class)); - - List result = - ossOperator.listFilesStatusRecursively(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, ResourceType.FILE); - Assertions.assertEquals(0, result.size()); - } -} diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/pom.xml b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/pom.xml index b9dfd5d1c9..2c7a1d2d99 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/pom.xml +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/pom.xml @@ -42,5 +42,12 @@ com.amazonaws aws-java-sdk-s3 + + + org.testcontainers + minio + test + + diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java index 3e4b207b50..d70a4c5990 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java @@ -17,19 +17,13 @@ package org.apache.dolphinscheduler.plugin.storage.s3; -import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPARATOR; -import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S; -import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TYPE_FILE; -import static org.apache.dolphinscheduler.common.constants.Constants.RESOURCE_TYPE_UDF; - import org.apache.dolphinscheduler.authentication.aws.AmazonS3ClientFactory; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.enums.ResUploadType; import org.apache.dolphinscheduler.common.utils.FileUtils; -import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.plugin.storage.api.AbstractStorageOperator; +import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata; import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; -import org.apache.dolphinscheduler.spi.enums.ResourceType; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.commons.lang3.StringUtils; @@ -37,28 +31,24 @@ import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.Closeable; import java.io.File; -import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Collections; +import java.util.HashSet; import java.util.LinkedList; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; -import java.util.stream.Stream; -import lombok.Data; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; -import com.amazonaws.AmazonServiceException; import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.AmazonS3Exception; -import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.ListObjectsV2Request; import com.amazonaws.services.s3.model.ListObjectsV2Result; import com.amazonaws.services.s3.model.ObjectMetadata; @@ -66,88 +56,59 @@ import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.model.S3ObjectInputStream; import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.amazonaws.services.s3.transfer.MultipleFileDownload; -import com.amazonaws.services.s3.transfer.TransferManager; -import com.amazonaws.services.s3.transfer.TransferManagerBuilder; @Slf4j -@Data -public class S3StorageOperator implements Closeable, StorageOperate { - - private String bucketName; - - private AmazonS3 s3Client; - - public S3StorageOperator() { - } - - public void init() { - bucketName = readBucketName(); - s3Client = buildS3Client(); - checkBucketNameExists(bucketName); - } - - protected AmazonS3 buildS3Client() { - return AmazonS3ClientFactory.createAmazonS3Client(PropertyUtils.getByPrefix("aws.s3.", "")); - } - - protected String readBucketName() { - return PropertyUtils.getString(Constants.AWS_S3_BUCKET_NAME); - } +public class S3StorageOperator extends AbstractStorageOperator implements Closeable, StorageOperator { - @Override - public void close() throws IOException { - s3Client.shutdown(); - } + private final String bucketName; - @Override - public void createTenantDirIfNotExists(String tenantCode) throws Exception { - mkdir(tenantCode, getS3ResDir(tenantCode)); - mkdir(tenantCode, getS3UdfDir(tenantCode)); - } + private final AmazonS3 s3Client; - @Override - public String getResDir(String tenantCode) { - return getS3ResDir(tenantCode) + FOLDER_SEPARATOR; + public S3StorageOperator(S3StorageProperties s3StorageProperties) { + super(s3StorageProperties.getResourceUploadPath()); + bucketName = s3StorageProperties.getBucketName(); + s3Client = AmazonS3ClientFactory.createAmazonS3Client(s3StorageProperties.getS3Configuration()); + exceptionWhenBucketNameNotExists(bucketName); } @Override - public String getUdfDir(String tenantCode) { - return getS3UdfDir(tenantCode) + FOLDER_SEPARATOR; - } - - @Override - public boolean mkdir(String tenantCode, String path) throws IOException { - String objectName = path + FOLDER_SEPARATOR; - if (!s3Client.doesObjectExist(bucketName, objectName)) { - ObjectMetadata metadata = new ObjectMetadata(); - metadata.setContentLength(0); - InputStream emptyContent = new ByteArrayInputStream(new byte[0]); - PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, objectName, emptyContent, metadata); - s3Client.putObject(putObjectRequest); + public String getStorageBaseDirectory() { + // All directory should end with File.separator + if (resourceBaseAbsolutePath.startsWith("/")) { + log.warn("{} -> {} should not start with / in s3", Constants.RESOURCE_UPLOAD_PATH, + resourceBaseAbsolutePath); + return resourceBaseAbsolutePath.substring(1); } - return true; + return resourceBaseAbsolutePath; } @Override - public String getResourceFullName(String tenantCode, String fileName) { - if (fileName.startsWith(FOLDER_SEPARATOR)) { - fileName = fileName.replaceFirst(FOLDER_SEPARATOR, ""); - } - return String.format(FORMAT_S_S, getS3ResDir(tenantCode), fileName); + public void close() throws IOException { + s3Client.shutdown(); } + @SneakyThrows @Override - public String getFileName(ResourceType resourceType, String tenantCode, String fileName) { - if (fileName.startsWith(FOLDER_SEPARATOR)) { - fileName = fileName.replaceFirst(FOLDER_SEPARATOR, ""); + public void createStorageDir(String directoryAbsolutePath) { + directoryAbsolutePath = transformAbsolutePathToS3Key(directoryAbsolutePath); + if (s3Client.doesObjectExist(bucketName, directoryAbsolutePath)) { + throw new FileAlreadyExistsException( + "The directory " + directoryAbsolutePath + " already exists in the bucket " + bucketName); } - return getDir(resourceType, tenantCode) + fileName; + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setContentLength(0); + InputStream emptyContent = new ByteArrayInputStream(new byte[0]); + PutObjectRequest putObjectRequest = + new PutObjectRequest(bucketName, directoryAbsolutePath, emptyContent, metadata); + s3Client.putObject(putObjectRequest); } + @SneakyThrows @Override - public void download(String srcFilePath, String dstFilePath, - boolean overwrite) throws IOException { + public void download(String srcFilePath, + String dstFilePath, + boolean overwrite) { + srcFilePath = transformAbsolutePathToS3Key(srcFilePath); File dstFile = new File(dstFilePath); if (dstFile.isDirectory()) { Files.delete(dstFile.toPath()); @@ -163,187 +124,84 @@ public class S3StorageOperator implements Closeable, StorageOperate { while ((readLen = s3is.read(readBuf)) > 0) { fos.write(readBuf, 0, readLen); } - } catch (AmazonServiceException e) { - throw new IOException(e.getMessage()); - } catch (FileNotFoundException e) { - log.error("the destination file {} not found", dstFilePath); - throw e; } } @Override - public boolean exists(String fullName) throws IOException { + public boolean exists(String fullName) { + fullName = transformAbsolutePathToS3Key(fullName); return s3Client.doesObjectExist(bucketName, fullName); } @Override - public boolean delete(String fullName, boolean recursive) throws IOException { - try { - s3Client.deleteObject(bucketName, fullName); - return true; - } catch (AmazonServiceException e) { - log.error("delete the object error,the resource path is {}", fullName); - return false; + public void delete(String absolutePath, boolean recursive) { + absolutePath = transformAbsolutePathToS3Key(absolutePath); + ResourceMetadata resourceMetaData = getResourceMetaData(absolutePath); + if (!resourceMetaData.isDirectory()) { + s3Client.deleteObject(bucketName, absolutePath); + return; } - } - - @Override - public boolean delete(String fullName, List childrenPathList, boolean recursive) throws IOException { - // append the resource fullName to the list for deletion. - childrenPathList.add(fullName); - - DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucketName) - .withKeys(childrenPathList.stream().toArray(String[]::new)); - try { - s3Client.deleteObjects(deleteObjectsRequest); - } catch (AmazonServiceException e) { - log.error("delete objects error", e); - return false; + if (recursive) { + List storageEntities = listStorageEntityRecursively(absolutePath); + for (StorageEntity storageEntity : storageEntities) { + s3Client.deleteObject(bucketName, transformAbsolutePathToS3Key(storageEntity.getFullName())); + } } - - return true; + s3Client.deleteObject(bucketName, absolutePath); } @Override - public boolean copy(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) throws IOException { + public void copy(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) { + srcPath = transformAbsolutePathToS3Key(srcPath); + dstPath = transformAbsolutePathToS3Key(dstPath); + + ResourceMetadata resourceMetaData = getResourceMetaData(srcPath); + if (resourceMetaData.isDirectory()) { + throw new UnsupportedOperationException("S3 does not support copying directories."); + } s3Client.copyObject(bucketName, srcPath, bucketName, dstPath); if (deleteSource) { s3Client.deleteObject(bucketName, srcPath); } - return true; } + @SneakyThrows @Override - public String getDir(ResourceType resourceType, String tenantCode) { - switch (resourceType) { - case UDF: - return getUdfDir(tenantCode); - case FILE: - return getResDir(tenantCode); - case ALL: - return getS3DataBasePath(); - default: - return ""; + public void upload(String srcFile, String dstPath, boolean deleteSource, boolean overwrite) { + dstPath = transformAbsolutePathToS3Key(dstPath); + + if (s3Client.doesObjectExist(bucketName, dstPath)) { + if (overwrite) { + s3Client.deleteObject(bucketName, dstPath); + } else { + throw new FileAlreadyExistsException("The file " + dstPath + " already exists in the bucket " + + bucketName + " and overwrite is not allowed."); + } } - } + s3Client.putObject(bucketName, dstPath, new File(srcFile)); - @Override - public boolean upload(String tenantCode, String srcFile, String dstPath, boolean deleteSource, - boolean overwrite) throws IOException { - try { - s3Client.putObject(bucketName, dstPath, new File(srcFile)); - - if (deleteSource) { - Files.delete(Paths.get(srcFile)); - } - return true; - } catch (AmazonServiceException e) { - log.error("upload failed,the bucketName is {},the filePath is {}", bucketName, dstPath); - return false; + if (deleteSource) { + Files.delete(Paths.get(srcFile)); } } + @SneakyThrows @Override - public List vimFile(String tenantCode, String filePath, int skipLineNums, int limit) throws IOException { - if (StringUtils.isBlank(filePath)) { - log.error("file path:{} is blank", filePath); - return Collections.emptyList(); - } + public List fetchFileContent(String filePath, int skipLineNums, int limit) { + filePath = transformAbsolutePathToS3Key(filePath); S3Object s3Object = s3Client.getObject(bucketName, filePath); - try (BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(s3Object.getObjectContent()))) { - Stream stream = bufferedReader.lines().skip(skipLineNums).limit(limit); - return stream.collect(Collectors.toList()); - } - } - - @Override - public void deleteTenant(String tenantCode) throws Exception { - deleteTenantCode(tenantCode); - } - - /** - * S3 resource dir - * - * @param tenantCode tenant code - * @return S3 resource dir - */ - public String getS3ResDir(String tenantCode) { - return String.format("%s/" + RESOURCE_TYPE_FILE, getS3TenantDir(tenantCode)); - } - - /** - * S3 udf dir - * - * @param tenantCode tenant code - * @return get udf dir on S3 - */ - public String getS3UdfDir(String tenantCode) { - return String.format("%s/" + RESOURCE_TYPE_UDF, getS3TenantDir(tenantCode)); - } - - /** - * @param tenantCode tenant code - * @return file directory of tenants on S3 - */ - public String getS3TenantDir(String tenantCode) { - return String.format(FORMAT_S_S, getS3DataBasePath(), tenantCode); - } - - /** - * get data S3 path - * - * @return data S3 path - */ - public String getS3DataBasePath() { - if (FOLDER_SEPARATOR.equals(RESOURCE_UPLOAD_PATH)) { - return ""; - } else { - return RESOURCE_UPLOAD_PATH.replaceFirst(FOLDER_SEPARATOR, ""); - } - } - - protected void deleteTenantCode(String tenantCode) { - deleteDir(getResDir(tenantCode)); - deleteDir(getUdfDir(tenantCode)); - } - - /** - * xxx untest - * upload local directory to S3 - * - * @param tenantCode - * @param keyPrefix the name of directory - * @param strPath - */ - private void uploadDirectory(String tenantCode, String keyPrefix, String strPath) { - s3Client.putObject(bucketName, tenantCode + FOLDER_SEPARATOR + keyPrefix, new File(strPath)); - } - - /** - * xxx untest - * download S3 Directory to local - * - * @param tenantCode - * @param keyPrefix the name of directory - * @param srcPath - */ - private void downloadDirectory(String tenantCode, String keyPrefix, String srcPath) { - TransferManager tm = TransferManagerBuilder.standard().withS3Client(s3Client).build(); - try { - MultipleFileDownload download = - tm.downloadDirectory(bucketName, tenantCode + FOLDER_SEPARATOR + keyPrefix, new File(srcPath)); - download.waitForCompletion(); - } catch (AmazonS3Exception | InterruptedException e) { - log.error("download the directory failed with the bucketName is {} and the keyPrefix is {}", bucketName, - tenantCode + FOLDER_SEPARATOR + keyPrefix); - Thread.currentThread().interrupt(); - } finally { - tm.shutdownNow(); + try ( + InputStreamReader inputStreamReader = new InputStreamReader(s3Object.getObjectContent()); + BufferedReader bufferedReader = new BufferedReader(inputStreamReader)) { + return bufferedReader.lines() + .skip(skipLineNums) + .limit(limit) + .collect(Collectors.toList()); } } - public void checkBucketNameExists(String bucketName) { + void exceptionWhenBucketNameNotExists(String bucketName) { if (StringUtils.isBlank(bucketName)) { throw new IllegalArgumentException("resource.aws.s3.bucket.name is blank"); } @@ -358,199 +216,135 @@ public class S3StorageOperator implements Closeable, StorageOperate { s3Client.getRegionName()); } - /** - * only delete the object of directory ,it`s better to delete the files in it -r - */ - protected void deleteDir(String directoryName) { - if (s3Client.doesObjectExist(bucketName, directoryName)) { - s3Client.deleteObject(bucketName, directoryName); - } + @Override + public List listStorageEntity(String resourceAbsolutePath) { + final String s3ResourceAbsolutePath = transformAbsolutePathToS3Key(resourceAbsolutePath); + ListObjectsV2Request listObjectsV2Request = new ListObjectsV2Request() + .withBucketName(bucketName) + .withDelimiter("/") + .withPrefix(s3ResourceAbsolutePath); + + ListObjectsV2Result listObjectsV2Result = s3Client.listObjectsV2(listObjectsV2Request); + List storageEntities = new ArrayList<>(); + storageEntities.addAll(listObjectsV2Result.getCommonPrefixes() + .stream() + .map(this::transformCommonPrefixToStorageEntity) + .collect(Collectors.toList())); + storageEntities.addAll( + listObjectsV2Result.getObjectSummaries().stream() + .filter(s3ObjectSummary -> !s3ObjectSummary.getKey().equals(s3ResourceAbsolutePath)) + .map(this::transformS3ObjectToStorageEntity) + .collect(Collectors.toList())); + + return storageEntities; } @Override - public ResUploadType returnStorageType() { - return ResUploadType.S3; + public List listFileStorageEntityRecursively(String resourceAbsolutePath) { + return listStorageEntityRecursively(resourceAbsolutePath) + .stream() + .filter(storageEntity -> !storageEntity.isDirectory()) + .collect(Collectors.toList()); } @Override - public List listFilesStatusRecursively(String path, String defaultPath, String tenantCode, - ResourceType type) { + public StorageEntity getStorageEntity(String resourceAbsolutePath) { + resourceAbsolutePath = transformAbsolutePathToS3Key(resourceAbsolutePath); + + S3Object object = s3Client.getObject(bucketName, resourceAbsolutePath); + return transformS3ObjectToStorageEntity(object); + } + + private List listStorageEntityRecursively(String resourceAbsolutePath) { + resourceAbsolutePath = transformAbsolutePathToS3Key(resourceAbsolutePath); + + Set visited = new HashSet<>(); List storageEntityList = new ArrayList<>(); - LinkedList foldersToFetch = new LinkedList<>(); - - StorageEntity initialEntity = null; - try { - initialEntity = getFileStatus(path, defaultPath, tenantCode, type); - } catch (Exception e) { - log.error("error while listing files status recursively, path: {}", path, e); - return storageEntityList; - } - foldersToFetch.add(initialEntity); + LinkedList foldersToFetch = new LinkedList<>(); + foldersToFetch.addLast(resourceAbsolutePath); while (!foldersToFetch.isEmpty()) { - String pathToExplore = foldersToFetch.pop().getFullName(); - try { - List tempList = listFilesStatus(pathToExplore, defaultPath, tenantCode, type); - for (StorageEntity temp : tempList) { - if (temp.isDirectory()) { - foldersToFetch.add(temp); + String pathToExplore = foldersToFetch.pop(); + visited.add(pathToExplore); + List tempList = listStorageEntity(pathToExplore); + for (StorageEntity temp : tempList) { + if (temp.isDirectory()) { + if (visited.contains(temp.getFullName())) { + continue; } + foldersToFetch.add(temp.getFullName()); } - storageEntityList.addAll(tempList); - } catch (Exception e) { - log.error("error while listing files status recursively, path: {}", pathToExplore, e); } + storageEntityList.addAll(tempList); } return storageEntityList; - } - @Override - public List listFilesStatus(String path, String defaultPath, String tenantCode, - ResourceType type) throws AmazonServiceException { - List storageEntityList = new ArrayList<>(); + private StorageEntity transformS3ObjectToStorageEntity(S3Object object) { - // TODO: optimize pagination - ListObjectsV2Request request = new ListObjectsV2Request(); - request.setBucketName(bucketName); - request.setPrefix(path); - request.setDelimiter("/"); - - ListObjectsV2Result v2Result; - do { - try { - v2Result = s3Client.listObjectsV2(request); - } catch (AmazonServiceException e) { - throw new AmazonServiceException("Get S3 file list exception, error type:" + e.getErrorType(), e); - } + String s3Key = object.getKey(); + String absolutePath = transformS3KeyToAbsolutePath(s3Key); - List summaries = v2Result.getObjectSummaries(); - - for (S3ObjectSummary summary : summaries) { - if (!summary.getKey().endsWith("/")) { - // the path is a file - String[] aliasArr = summary.getKey().split("/"); - String alias = aliasArr[aliasArr.length - 1]; - String fileName = StringUtils.difference(defaultPath, summary.getKey()); - - StorageEntity entity = new StorageEntity(); - entity.setAlias(alias); - entity.setFileName(fileName); - entity.setFullName(summary.getKey()); - entity.setDirectory(false); - entity.setUserName(tenantCode); - entity.setType(type); - entity.setSize(summary.getSize()); - entity.setCreateTime(summary.getLastModified()); - entity.setUpdateTime(summary.getLastModified()); - entity.setPfullName(path); - - storageEntityList.add(entity); - } - } + ResourceMetadata resourceMetaData = getResourceMetaData(absolutePath); - for (String commonPrefix : v2Result.getCommonPrefixes()) { - // the paths in commonPrefix are directories - String suffix = StringUtils.difference(path, commonPrefix); - String fileName = StringUtils.difference(defaultPath, commonPrefix); - - StorageEntity entity = new StorageEntity(); - entity.setAlias(suffix); - entity.setFileName(fileName); - entity.setFullName(commonPrefix); - entity.setDirectory(true); - entity.setUserName(tenantCode); - entity.setType(type); - entity.setSize(0); - entity.setCreateTime(null); - entity.setUpdateTime(null); - entity.setPfullName(path); - - storageEntityList.add(entity); - } + StorageEntity entity = new StorageEntity(); + entity.setFileName(new File(absolutePath).getName()); + entity.setFullName(absolutePath); + entity.setDirectory(resourceMetaData.isDirectory()); + entity.setType(resourceMetaData.getResourceType()); + entity.setSize(object.getObjectMetadata().getContentLength()); + entity.setCreateTime(object.getObjectMetadata().getLastModified()); + entity.setUpdateTime(object.getObjectMetadata().getLastModified()); + return entity; + } - request.setContinuationToken(v2Result.getContinuationToken()); + private StorageEntity transformCommonPrefixToStorageEntity(String commonPrefix) { + String absolutePath = transformS3KeyToAbsolutePath(commonPrefix); - } while (v2Result.isTruncated()); + ResourceMetadata resourceMetaData = getResourceMetaData(absolutePath); - return storageEntityList; + StorageEntity entity = new StorageEntity(); + entity.setFileName(new File(absolutePath).getName()); + entity.setFullName(absolutePath); + entity.setDirectory(resourceMetaData.isDirectory()); + entity.setType(resourceMetaData.getResourceType()); + entity.setSize(0L); + entity.setCreateTime(null); + entity.setUpdateTime(null); + return entity; } - @Override - public StorageEntity getFileStatus(String path, String defaultPath, String tenantCode, - ResourceType type) throws AmazonServiceException, FileNotFoundException { - // Notice: we do not use getObject here because intermediate directories - // may not exist in S3, which can cause getObject to throw exception. - // Since we still want to access it on frontend, this is a workaround using listObjects. - - ListObjectsV2Request request = new ListObjectsV2Request(); - request.setBucketName(bucketName); - request.setPrefix(path); - request.setDelimiter("/"); - - ListObjectsV2Result v2Result; - try { - v2Result = s3Client.listObjectsV2(request); - } catch (AmazonServiceException e) { - throw new AmazonServiceException("Get S3 file list exception, error type:" + e.getErrorType(), e); - } - - List summaries = v2Result.getObjectSummaries(); + private StorageEntity transformS3ObjectToStorageEntity(S3ObjectSummary s3ObjectSummary) { + String absolutePath = transformS3KeyToAbsolutePath(s3ObjectSummary.getKey()); - if (path.endsWith("/")) { - // the path is a directory that may or may not exist in S3 - String alias = findDirAlias(path); - String fileName = StringUtils.difference(defaultPath, path); + ResourceMetadata resourceMetaData = getResourceMetaData(absolutePath); - StorageEntity entity = new StorageEntity(); - entity.setAlias(alias); - entity.setFileName(fileName); - entity.setFullName(path); - entity.setDirectory(true); - entity.setUserName(tenantCode); - entity.setType(type); - entity.setSize(0); - - return entity; + StorageEntity entity = new StorageEntity(); + entity.setFileName(new File(absolutePath).getName()); + entity.setFullName(absolutePath); + entity.setPfullName(resourceMetaData.getResourceParentAbsolutePath()); + entity.setDirectory(resourceMetaData.isDirectory()); + entity.setType(resourceMetaData.getResourceType()); + entity.setSize(s3ObjectSummary.getSize()); + entity.setCreateTime(s3ObjectSummary.getLastModified()); + entity.setUpdateTime(s3ObjectSummary.getLastModified()); + return entity; + } - } else { - // the path is a file - if (summaries.size() > 0) { - S3ObjectSummary summary = summaries.get(0); - String[] aliasArr = summary.getKey().split("/"); - String alias = aliasArr[aliasArr.length - 1]; - String fileName = StringUtils.difference(defaultPath, summary.getKey()); - - StorageEntity entity = new StorageEntity(); - entity.setAlias(alias); - entity.setFileName(fileName); - entity.setFullName(summary.getKey()); - entity.setDirectory(false); - entity.setUserName(tenantCode); - entity.setType(type); - entity.setSize(summary.getSize()); - entity.setCreateTime(summary.getLastModified()); - entity.setUpdateTime(summary.getLastModified()); - - return entity; - } + private String transformAbsolutePathToS3Key(String absolutePath) { + ResourceMetadata resourceMetaData = getResourceMetaData(absolutePath); + if (resourceMetaData.isDirectory()) { + return FileUtils.concatFilePath(absolutePath, "/"); } - - throw new FileNotFoundException("Object is not found in S3 Bucket: " + bucketName); + return absolutePath; } - /** - * find alias for directories, NOT for files - * a directory is a path ending with "/" - */ - private String findDirAlias(String myStr) { - if (!myStr.endsWith(FOLDER_SEPARATOR)) { - // Make sure system won't crush down if someone accidentally misuse the function. - return myStr; + private String transformS3KeyToAbsolutePath(String s3Key) { + if (s3Key.endsWith("/")) { + return s3Key.substring(0, s3Key.length() - 1); } - - Path path = Paths.get(myStr); - return path.getName(path.getNameCount() - 1) + FOLDER_SEPARATOR; + return s3Key; } + } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperatorFactory.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperatorFactory.java index e1c3a41743..9b1539a4fb 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperatorFactory.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperatorFactory.java @@ -17,20 +17,29 @@ package org.apache.dolphinscheduler.plugin.storage.s3; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperateFactory; +import org.apache.dolphinscheduler.common.constants.Constants; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperatorFactory; import org.apache.dolphinscheduler.plugin.storage.api.StorageType; import com.google.auto.service.AutoService; -@AutoService(StorageOperateFactory.class) -public class S3StorageOperatorFactory implements StorageOperateFactory { +@AutoService(StorageOperatorFactory.class) +public class S3StorageOperatorFactory implements StorageOperatorFactory { @Override - public StorageOperate createStorageOperate() { - S3StorageOperator s3StorageOperator = new S3StorageOperator(); - s3StorageOperator.init(); - return s3StorageOperator; + public StorageOperator createStorageOperate() { + final S3StorageProperties s3StorageProperties = getS3StorageProperties(); + return new S3StorageOperator(s3StorageProperties); + } + + private S3StorageProperties getS3StorageProperties() { + return S3StorageProperties.builder() + .bucketName(PropertyUtils.getString(Constants.AWS_S3_BUCKET_NAME)) + .s3Configuration(PropertyUtils.getByPrefix("aws.s3.", "")) + .resourceUploadPath(PropertyUtils.getString(Constants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler")) + .build(); } @Override diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageProperties.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageProperties.java new file mode 100644 index 0000000000..b7f6f159b4 --- /dev/null +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageProperties.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.storage.s3; + +import java.util.Map; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class S3StorageProperties { + + private Map s3Configuration; + + private String bucketName; + + private String resourceUploadPath; +} diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/test/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperatorTest.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/test/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperatorTest.java index c5930d713d..dc635cceb5 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/test/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperatorTest.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/test/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperatorTest.java @@ -17,280 +17,289 @@ package org.apache.dolphinscheduler.plugin.storage.s3; -import static org.apache.dolphinscheduler.common.constants.Constants.FOLDER_SEPARATOR; -import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata; import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity; import org.apache.dolphinscheduler.spi.enums.ResourceType; -import java.io.IOException; -import java.util.Collections; +import java.nio.file.FileAlreadyExistsException; import java.util.List; +import java.util.stream.Stream; +import lombok.SneakyThrows; + +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; +import org.testcontainers.containers.MinIOContainer; +import org.testcontainers.lifecycle.Startables; +import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.ListObjectsV2Request; -import com.amazonaws.services.s3.model.ListObjectsV2Result; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; -@ExtendWith(MockitoExtension.class) public class S3StorageOperatorTest { - private static final String ACCESS_KEY_ID_MOCK = "ACCESS_KEY_ID_MOCK"; - - private static final String ACCESS_KEY_SECRET_MOCK = "ACCESS_KEY_SECRET_MOCK"; - - private static final String REGION_MOCK = "REGION_MOCK"; - - private static final String END_POINT_MOCK = "END_POINT_MOCK"; - - private static final String BUCKET_NAME_MOCK = "BUCKET_NAME_MOCK"; - - private static final String TENANT_CODE_MOCK = "TENANT_CODE_MOCK"; - - private static final String DIR_MOCK = "DIR_MOCK"; - - private static final String FILE_NAME_MOCK = "FILE_NAME_MOCK"; - - private static final String FILE_PATH_MOCK = "FILE_PATH_MOCK"; - - private static final String FULL_NAME = "/tmp/dir1/"; - - private static final String DEFAULT_PATH = "/tmp/"; - - @Mock - private AmazonS3 s3Client; - - private S3StorageOperator s3StorageOperator; + private static final String demoSql = S3StorageOperatorTest.class.getResource("/demo.sql").getFile(); + + private static MinIOContainer minIOContainer; + + private static S3StorageOperator s3StorageOperator; + + @BeforeAll + public static void setUp() throws Exception { + String bucketName = "dolphinscheduler"; + String accessKey = "accessKey123"; + String secretKey = "secretKey123"; + String region = "us-east-1"; + + minIOContainer = new MinIOContainer("minio/minio:RELEASE.2023-09-04T19-57-37Z") + .withEnv("MINIO_ACCESS_KEY", accessKey) + .withEnv("MINIO_SECRET_KEY", secretKey) + .withEnv("MINIO_REGION", region) + .withNetworkAliases(bucketName + "." + "localhost"); + + Startables.deepStart(Stream.of(minIOContainer)).join(); + + String endpoint = minIOContainer.getS3URL(); + + AmazonS3 amazonS3 = AmazonS3ClientBuilder.standard() + .withEndpointConfiguration(new AmazonS3ClientBuilder.EndpointConfiguration(endpoint, region)) + .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey))) + .withPathStyleAccessEnabled(true) + .build(); + amazonS3.createBucket(bucketName); + + S3StorageProperties s3StorageProperties = S3StorageProperties.builder() + .bucketName(bucketName) + .resourceUploadPath("tmp/dolphinscheduler") + .s3Configuration(ImmutableMap.of( + "access.key.id", accessKey, + "access.key.secret", secretKey, + "region", region, + "endpoint", endpoint)) + .build(); + s3StorageOperator = new S3StorageOperator(s3StorageProperties); + } @BeforeEach - public void setUp() throws Exception { - s3StorageOperator = Mockito.spy(new S3StorageOperator()); - - doReturn(BUCKET_NAME_MOCK).when(s3StorageOperator).readBucketName(); - Mockito.doReturn(s3Client) - .when(s3StorageOperator).buildS3Client(); - Mockito.doNothing() - .when(s3StorageOperator).checkBucketNameExists(Mockito.any()); - - s3StorageOperator.init(); + public void initializeFiles() { + s3StorageOperator.delete("tmp/dolphinscheduler/default/resources", true); + s3StorageOperator.createStorageDir("tmp/dolphinscheduler/default/resources/sqlDirectory"); + s3StorageOperator.createStorageDir("tmp/dolphinscheduler/default/resources/multipleDirectories"); + s3StorageOperator.createStorageDir("tmp/dolphinscheduler/default/resources/multipleDirectories/1"); + s3StorageOperator.createStorageDir("tmp/dolphinscheduler/default/resources/multipleDirectories/2"); + s3StorageOperator.createStorageDir("tmp/dolphinscheduler/default/resources/multipleDirectories/3"); + s3StorageOperator.upload(demoSql, "tmp/dolphinscheduler/default/resources/multipleDirectories/1/demo.sql", + false, true); + s3StorageOperator.createStorageDir("tmp/dolphinscheduler/default/resources/emptyDirectory"); + s3StorageOperator.upload(demoSql, "tmp/dolphinscheduler/default/resources/sqlDirectory/demo.sql", false, true); } @Test - public void testInit() { - verify(s3StorageOperator, times(1)).buildS3Client(); - Assertions.assertEquals(BUCKET_NAME_MOCK, s3StorageOperator.getBucketName()); + public void testGetResourceMetaData() { + ResourceMetadata resourceMetaData = + s3StorageOperator.getResourceMetaData("tmp/dolphinscheduler/default/resources/sqlDirectory/demo.sql"); + assertEquals("tmp/dolphinscheduler/default/resources/sqlDirectory/demo.sql", + resourceMetaData.getResourceAbsolutePath()); + assertEquals("tmp/dolphinscheduler", resourceMetaData.getResourceBaseDirectory()); + assertEquals("default", resourceMetaData.getTenant()); + assertEquals(ResourceType.FILE, resourceMetaData.getResourceType()); + assertEquals("sqlDirectory/demo.sql", resourceMetaData.getResourceRelativePath()); + assertEquals("tmp/dolphinscheduler/default/resources/sqlDirectory", + resourceMetaData.getResourceParentAbsolutePath()); + assertFalse(resourceMetaData.isDirectory()); } @Test - public void testTearDown() throws IOException { - doNothing().when(s3Client).shutdown(); - s3StorageOperator.close(); - verify(s3Client, times(1)).shutdown(); + public void testGetStorageBaseDirectory() { + assertEquals("tmp/dolphinscheduler", s3StorageOperator.getStorageBaseDirectory()); } @Test - public void testCreateTenantResAndUdfDir() throws Exception { - doReturn(DIR_MOCK).when(s3StorageOperator).getS3ResDir(TENANT_CODE_MOCK); - doReturn(DIR_MOCK).when(s3StorageOperator).getS3UdfDir(TENANT_CODE_MOCK); - doReturn(true).when(s3StorageOperator).mkdir(TENANT_CODE_MOCK, DIR_MOCK); - s3StorageOperator.createTenantDirIfNotExists(TENANT_CODE_MOCK); - verify(s3StorageOperator, times(2)).mkdir(TENANT_CODE_MOCK, DIR_MOCK); + public void testGetStorageBaseDirectory_withTenant() { + assertEquals("tmp/dolphinscheduler/default", s3StorageOperator.getStorageBaseDirectory("default")); } @Test - public void testGetResDir() { - final String expectedResourceDir = String.format("dolphinscheduler/%s/resources/", TENANT_CODE_MOCK); - final String dir = s3StorageOperator.getResDir(TENANT_CODE_MOCK); - Assertions.assertEquals(expectedResourceDir, dir); + public void testGetStorageBaseDirectory_withTenant_withResourceTypeFile() { + String storageBaseDirectory = s3StorageOperator.getStorageBaseDirectory("default", ResourceType.FILE); + assertThat(storageBaseDirectory).isEqualTo("tmp/dolphinscheduler/default/resources"); } @Test - public void testGetUdfDir() { - final String expectedUdfDir = String.format("dolphinscheduler/%s/udfs/", TENANT_CODE_MOCK); - final String dir = s3StorageOperator.getUdfDir(TENANT_CODE_MOCK); - Assertions.assertEquals(expectedUdfDir, dir); + public void testGetStorageBaseDirectory_withTenant_withResourceTypeAll() { + String storageBaseDirectory = s3StorageOperator.getStorageBaseDirectory("default", ResourceType.ALL); + assertThat(storageBaseDirectory).isEqualTo("tmp/dolphinscheduler/default"); } @Test - public void mkdirWhenDirExists() { - boolean isSuccess = false; - try { - final String key = DIR_MOCK + FOLDER_SEPARATOR; - doReturn(true).when(s3Client).doesObjectExist(BUCKET_NAME_MOCK, key); - isSuccess = s3StorageOperator.mkdir(TENANT_CODE_MOCK, DIR_MOCK); - verify(s3Client, times(1)).doesObjectExist(BUCKET_NAME_MOCK, key); - - } catch (IOException e) { - Assertions.fail("test failed due to unexpected IO exception"); - } - - Assertions.assertTrue(isSuccess); + public void testGetStorageFileAbsolutePath() { + assertThat(s3StorageOperator.getStorageFileAbsolutePath("default", "demo.sql")) + .isEqualTo("tmp/dolphinscheduler/default/resources/demo.sql"); } @Test - public void mkdirWhenDirNotExists() { - boolean isSuccess = true; - try { - final String key = DIR_MOCK + FOLDER_SEPARATOR; - doReturn(false).when(s3Client).doesObjectExist(BUCKET_NAME_MOCK, key); - isSuccess = s3StorageOperator.mkdir(TENANT_CODE_MOCK, DIR_MOCK); - verify(s3Client, times(1)).doesObjectExist(BUCKET_NAME_MOCK, key); - - } catch (IOException e) { - Assertions.fail("test failed due to unexpected IO exception"); - } + public void testCreateStorageDir_notExist() { + String dirName = "tmp/dolphinscheduler/default/resources/testDirectory"; + s3StorageOperator.createStorageDir(dirName); + assertTrue(s3StorageOperator.exists(dirName)); - Assertions.assertTrue(isSuccess); } @Test - public void getResourceFullName() { - final String expectedResourceFullName = - String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK); - final String resourceFullName = s3StorageOperator.getResourceFullName(TENANT_CODE_MOCK, FILE_NAME_MOCK); - Assertions.assertEquals(expectedResourceFullName, resourceFullName); + public void testCreateStorageDir_exist() { + final String dirName = "tmp/dolphinscheduler/default/resources/emptyDirectory"; + Assertions.assertThrows(FileAlreadyExistsException.class, () -> s3StorageOperator.createStorageDir(dirName)); } @Test - public void getResourceFileName() { - final String expectedResourceFileName = FILE_NAME_MOCK; - final String resourceFullName = - String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK); - final String resourceFileName = s3StorageOperator.getResourceFileName(TENANT_CODE_MOCK, resourceFullName); - Assertions.assertEquals(expectedResourceFileName, resourceFileName); + public void testExists_fileExist() { + assertTrue(s3StorageOperator.exists("tmp/dolphinscheduler/default/resources/sqlDirectory/demo.sql")); } @Test - public void getFileName() { - final String expectedFileName = - String.format("dolphinscheduler/%s/resources/%s", TENANT_CODE_MOCK, FILE_NAME_MOCK); - final String fileName = s3StorageOperator.getFileName(ResourceType.FILE, TENANT_CODE_MOCK, FILE_NAME_MOCK); - Assertions.assertEquals(expectedFileName, fileName); + public void testExists_fileNotExist() { + assertFalse(s3StorageOperator.exists("tmp/dolphinscheduler/default/resources/sqlDirectory/notExist.sql")); } @Test - public void exists() { - boolean doesExist = false; - doReturn(true).when(s3Client).doesObjectExist(BUCKET_NAME_MOCK, FILE_NAME_MOCK); - try { - doesExist = s3StorageOperator.exists(FILE_NAME_MOCK); - } catch (IOException e) { - Assertions.fail("unexpected IO exception in unit test"); - } + public void testExists_directoryExist() { + assertTrue(s3StorageOperator.exists("tmp/dolphinscheduler/default/resources/sqlDirectory")); + } - Assertions.assertTrue(doesExist); - verify(s3Client, times(1)).doesObjectExist(BUCKET_NAME_MOCK, FILE_NAME_MOCK); + @Test + public void testExists_directoryNotExist() { + assertFalse(s3StorageOperator.exists("tmp/dolphinscheduler/default/resources/notExistDirectory")); } @Test - public void delete() { - doNothing().when(s3Client).deleteObject(anyString(), anyString()); - try { - s3StorageOperator.delete(FILE_NAME_MOCK, true); - } catch (IOException e) { - Assertions.fail("unexpected IO exception in unit test"); - } + public void delete_fileExist() { + s3StorageOperator.delete("tmp/dolphinscheduler/default/resources/sqlDirectory/demo.sql", true); + assertFalse(s3StorageOperator.exists("tmp/dolphinscheduler/default/resources/sqlDirectory/demo.sql")); + } - verify(s3Client, times(1)).deleteObject(anyString(), anyString()); + @Test + public void delete_fileNotExist() { + s3StorageOperator.delete("tmp/dolphinscheduler/default/resources/sqlDirectory/notExist.sql", true); + assertFalse(s3StorageOperator.exists("tmp/dolphinscheduler/default/resources/sqlDirectory/notExist.sql")); } @Test - public void copy() { - boolean isSuccess = false; - doReturn(null).when(s3Client).copyObject(anyString(), anyString(), anyString(), anyString()); - try { - isSuccess = s3StorageOperator.copy(FILE_PATH_MOCK, FILE_PATH_MOCK, false, false); - } catch (IOException e) { - Assertions.fail("unexpected IO exception in unit test"); - } + public void delete_directoryExist() { + s3StorageOperator.delete("tmp/dolphinscheduler/default/resources/sqlDirectory", true); + assertFalse(s3StorageOperator.exists("/tmp/dolphinscheduler/default/resources/sqlDirectory")); + } - Assertions.assertTrue(isSuccess); - verify(s3Client, times(1)).copyObject(anyString(), anyString(), anyString(), anyString()); + @Test + public void delete_directoryNotExist() { + s3StorageOperator.delete("tmp/dolphinscheduler/default/resources/notExist", true); + assertFalse(s3StorageOperator.exists("tmp/dolphinscheduler/default/resources/notExist")); } @Test - public void deleteTenant() { - doNothing().when(s3StorageOperator).deleteTenantCode(anyString()); - try { - s3StorageOperator.deleteTenant(TENANT_CODE_MOCK); - } catch (Exception e) { - Assertions.fail("unexpected exception caught in unit test"); - } + public void copy_file() { + s3StorageOperator.copy("tmp/dolphinscheduler/default/resources/sqlDirectory/demo.sql", + "tmp/dolphinscheduler/default/resources/sqlDirectory/demo_copy.sql", true, true); + assertTrue(s3StorageOperator.exists("tmp/dolphinscheduler/default/resources/sqlDirectory/demo_copy.sql")); + assertFalse(s3StorageOperator.exists("tmp/dolphinscheduler/default/resources/sqlDirectory/demo.sql")); + } - verify(s3StorageOperator, times(1)).deleteTenantCode(anyString()); + @Test + public void copy_directory() { + assertThrows(UnsupportedOperationException.class, + () -> s3StorageOperator.copy("tmp/dolphinscheduler/default/resources/sqlDirectory", + "tmp/dolphinscheduler/default/resources/sqlDirectory_copy", true, true)); } @Test - public void testGetS3ResDir() { - final String expectedS3ResDir = String.format("dolphinscheduler/%s/resources", TENANT_CODE_MOCK); - final String s3ResDir = s3StorageOperator.getS3ResDir(TENANT_CODE_MOCK); - Assertions.assertEquals(expectedS3ResDir, s3ResDir); + public void testUpload_file() { + String file = S3StorageOperatorTest.class.getResource("/student.sql").getFile(); + s3StorageOperator.upload(file, "tmp/dolphinscheduler/default/resources/sqlDirectory/student.sql", false, true); + assertTrue(s3StorageOperator.exists("tmp/dolphinscheduler/default/resources/sqlDirectory/student.sql")); } @Test - public void testGetS3UdfDir() { - final String expectedS3UdfDir = String.format("dolphinscheduler/%s/udfs", TENANT_CODE_MOCK); - final String s3UdfDir = s3StorageOperator.getS3UdfDir(TENANT_CODE_MOCK); - Assertions.assertEquals(expectedS3UdfDir, s3UdfDir); + public void testFetchFileContent() { + List strings = s3StorageOperator + .fetchFileContent("tmp/dolphinscheduler/default/resources/sqlDirectory/demo.sql", 0, 2); + assertThat(strings).hasSize(2); } @Test - public void testGetS3TenantDir() { - final String expectedS3TenantDir = String.format(FORMAT_S_S, DIR_MOCK, TENANT_CODE_MOCK); - doReturn(DIR_MOCK).when(s3StorageOperator).getS3DataBasePath(); - final String s3TenantDir = s3StorageOperator.getS3TenantDir(TENANT_CODE_MOCK); - Assertions.assertEquals(expectedS3TenantDir, s3TenantDir); + public void testListStorageEntity_file() { + List storageEntities = + s3StorageOperator.listStorageEntity("tmp/dolphinscheduler/default/resources/sqlDirectory"); + assertThat(storageEntities).hasSize(1); + + StorageEntity storageEntity = storageEntities.get(0); + assertThat(storageEntity.getFullName()) + .isEqualTo("tmp/dolphinscheduler/default/resources/sqlDirectory/demo.sql"); + assertThat(storageEntity.getFileName()) + .isEqualTo("demo.sql"); + assertThat(storageEntity.isDirectory()).isFalse(); + assertThat(storageEntity.getPfullName()).isEqualTo("tmp/dolphinscheduler/default/resources/sqlDirectory"); + assertThat(storageEntity.getType()).isEqualTo(ResourceType.FILE); } @Test - public void deleteDir() { - doReturn(true).when(s3Client).doesObjectExist(anyString(), anyString()); - s3StorageOperator.deleteDir(DIR_MOCK); - verify(s3Client, times(1)).deleteObject(anyString(), anyString()); + public void testListStorageEntity_directory() { + List storageEntities = + s3StorageOperator.listStorageEntity("tmp/dolphinscheduler/default/resources"); + assertThat(storageEntities).hasSize(3); + } @Test - public void testGetFileStatus() throws Exception { - doReturn(new ListObjectsV2Result()).when(s3Client).listObjectsV2(Mockito.any(ListObjectsV2Request.class)); - StorageEntity entity = - s3StorageOperator.getFileStatus(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, ResourceType.FILE); - Assertions.assertEquals(FULL_NAME, entity.getFullName()); - Assertions.assertEquals("dir1/", entity.getFileName()); + public void testListStorageEntity_directoryNotExist() { + List storageEntities = + s3StorageOperator.listStorageEntity("tmp/dolphinscheduler/notExist/resources"); + assertThat(storageEntities).isEmpty(); + } @Test - public void testListFilesStatus() throws Exception { - doReturn(new ListObjectsV2Result()).when(s3Client).listObjectsV2(Mockito.any(ListObjectsV2Request.class)); - List result = - s3StorageOperator.listFilesStatus(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, ResourceType.FILE); - Assertions.assertEquals(0, result.size()); + public void testListStorageEntityRecursively() { + List storageEntities = + s3StorageOperator + .listFileStorageEntityRecursively("tmp/dolphinscheduler/default/resources/multipleDirectories"); + assertThat(storageEntities).hasSize(1); + + StorageEntity storageEntity = storageEntities.get(0); + assertThat(storageEntity.getFullName()) + .isEqualTo("tmp/dolphinscheduler/default/resources/multipleDirectories/1/demo.sql"); + assertThat(storageEntity.getFileName()) + .isEqualTo("demo.sql"); + assertThat(storageEntity.isDirectory()).isFalse(); + assertThat(storageEntity.getPfullName()) + .isEqualTo("tmp/dolphinscheduler/default/resources/multipleDirectories/1"); + assertThat(storageEntity.getType()).isEqualTo(ResourceType.FILE); + } @Test - public void testListFilesStatusRecursively() throws Exception { - StorageEntity entity = new StorageEntity(); - entity.setFullName(FULL_NAME); - - doReturn(entity).when(s3StorageOperator).getFileStatus(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, - ResourceType.FILE); - doReturn(Collections.EMPTY_LIST).when(s3StorageOperator).listFilesStatus(anyString(), anyString(), anyString(), - Mockito.any(ResourceType.class)); - - List result = - s3StorageOperator.listFilesStatusRecursively(FULL_NAME, DEFAULT_PATH, TENANT_CODE_MOCK, - ResourceType.FILE); - Assertions.assertEquals(0, result.size()); + public void testExceptionWhenBucketNameNotExists() { + Assertions.assertDoesNotThrow(() -> s3StorageOperator.exceptionWhenBucketNameNotExists("dolphinscheduler")); } + + @SneakyThrows + @AfterAll + public static void tearDown() { + if (s3StorageOperator != null) { + s3StorageOperator.close(); + } + if (minIOContainer != null) { + minIOContainer.stop(); + } + } + } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/test/resources/demo.sql b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/test/resources/demo.sql new file mode 100644 index 0000000000..2c3a4cd616 --- /dev/null +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/test/resources/demo.sql @@ -0,0 +1,17 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +select * from t_ds_version; \ No newline at end of file diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/test/resources/student.sql b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/test/resources/student.sql new file mode 100644 index 0000000000..d91aaf0803 --- /dev/null +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/test/resources/student.sql @@ -0,0 +1,17 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +select * from t_ds_student; \ No newline at end of file diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/test/resources/student/student.sql b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/test/resources/student/student.sql new file mode 100644 index 0000000000..6eab26f880 --- /dev/null +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/test/resources/student/student.sql @@ -0,0 +1,18 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +select * from t_ds_student; \ No newline at end of file diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/resource/MigrateResourceService.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/resource/MigrateResourceService.java index 8961af1223..26e06ec6fe 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/resource/MigrateResourceService.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/resource/MigrateResourceService.java @@ -22,12 +22,11 @@ import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S; import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.dolphinscheduler.spi.enums.ResourceType; import org.apache.commons.lang3.StringUtils; -import java.io.IOException; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -50,7 +49,7 @@ public class MigrateResourceService { private static final Logger logger = LoggerFactory.getLogger(MigrateResourceService.class); @Autowired - private StorageOperate storageOperate; + private StorageOperator storageOperator; @Autowired private TenantMapper tenantMapper; @@ -83,11 +82,11 @@ public class MigrateResourceService { try { oriFullName = oriFullName.startsWith("/") ? oriFullName.substring(1) : oriFullName; if (ResourceType.FILE.getCode() == type) { - storageOperate.copy(oriFullName, + storageOperator.copy(oriFullName, String.format(FORMAT_S_S, resMigrateBasePath, oriFullName), true, true); } else if (ResourceType.UDF.getCode() == type) { String fullName = String.format(FORMAT_S_S, udfMigrateBasePath, oriFullName); - storageOperate.copy(oriFullName, fullName, true, true); + storageOperator.copy(oriFullName, fullName, true, true); // change relative udfs resourceName List udfs = udfFuncMapper.listUdfByResourceId(new Integer[]{id}); @@ -96,7 +95,7 @@ public class MigrateResourceService { udfFuncMapper.updateById(udf); }); } - } catch (IOException e) { + } catch (Exception e) { logger.error("Migrate resource: {} failed: {}", item, e); } } @@ -121,12 +120,11 @@ public class MigrateResourceService { } public String createMigrateDirByType(String targetTenantCode, ResourceType type) { - String migrateBasePath = type.equals(ResourceType.FILE) ? storageOperate.getResDir(targetTenantCode) - : storageOperate.getUdfDir(targetTenantCode); + String migrateBasePath = storageOperator.getStorageBaseDirectory(targetTenantCode, type); migrateBasePath += MIGRATE_BASE_DIR; try { - storageOperate.mkdir(targetTenantCode, migrateBasePath); - } catch (IOException e) { + storageOperator.createStorageDir(migrateBasePath); + } catch (Exception e) { logger.error("create migrate base directory {} failed", migrateBasePath); return ""; } diff --git a/dolphinscheduler-ui/src/views/resource/components/resource/edit/use-edit.ts b/dolphinscheduler-ui/src/views/resource/components/resource/edit/use-edit.ts index ab6c05e463..2ec97926b3 100644 --- a/dolphinscheduler-ui/src/views/resource/components/resource/edit/use-edit.ts +++ b/dolphinscheduler-ui/src/views/resource/components/resource/edit/use-edit.ts @@ -32,7 +32,7 @@ export function useEdit(state: any) { const getResourceView = (fullName: string, tenantCode: string) => { const params = { skipLineNum: 0, - limit: 3000, + limit: -1, fullName: fullName, tenantCode: tenantCode } diff --git a/dolphinscheduler-ui/src/views/resource/components/resource/types.ts b/dolphinscheduler-ui/src/views/resource/components/resource/types.ts index e33af74686..f208380178 100644 --- a/dolphinscheduler-ui/src/views/resource/components/resource/types.ts +++ b/dolphinscheduler-ui/src/views/resource/components/resource/types.ts @@ -133,7 +133,6 @@ export interface IUploadDefaultValue { name: string file: string type: ResourceType - pid: number currentDir: string } } diff --git a/dolphinscheduler-ui/src/views/resource/components/resource/use-file.ts b/dolphinscheduler-ui/src/views/resource/components/resource/use-file.ts index 5c9207c21a..bcac6e76a9 100644 --- a/dolphinscheduler-ui/src/views/resource/components/resource/use-file.ts +++ b/dolphinscheduler-ui/src/views/resource/components/resource/use-file.ts @@ -72,7 +72,7 @@ export function useFileState( const getResourceView = (fullName: string, tenantCode: string) => { const params = { skipLineNum: 0, - limit: 3000, + limit: -1, fullName: fullName, tenantCode: tenantCode } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutor.java index ea44f1790f..f5abf9e577 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutor.java @@ -17,7 +17,7 @@ package org.apache.dolphinscheduler.server.worker.runner; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; @@ -34,12 +34,12 @@ public class DefaultWorkerTaskExecutor extends WorkerTaskExecutor { public DefaultWorkerTaskExecutor(@NonNull TaskExecutionContext taskExecutionContext, @NonNull WorkerConfig workerConfig, @NonNull WorkerMessageSender workerMessageSender, - @Nullable StorageOperate storageOperate, + @Nullable StorageOperator storageOperator, @NonNull WorkerRegistryClient workerRegistryClient) { super(taskExecutionContext, workerConfig, workerMessageSender, - storageOperate, + storageOperator, workerRegistryClient); } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutorFactory.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutorFactory.java index 20fa6a5e2a..085deafb09 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutorFactory.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutorFactory.java @@ -17,7 +17,7 @@ package org.apache.dolphinscheduler.server.worker.runner; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient; @@ -34,18 +34,18 @@ public class DefaultWorkerTaskExecutorFactory private final @NonNull TaskExecutionContext taskExecutionContext; private final @NonNull WorkerConfig workerConfig; private final @NonNull WorkerMessageSender workerMessageSender; - private final @Nullable StorageOperate storageOperate; + private final @Nullable StorageOperator storageOperator; private final @NonNull WorkerRegistryClient workerRegistryClient; public DefaultWorkerTaskExecutorFactory(@NonNull TaskExecutionContext taskExecutionContext, @NonNull WorkerConfig workerConfig, @NonNull WorkerMessageSender workerMessageSender, - @Nullable StorageOperate storageOperate, + @Nullable StorageOperator storageOperator, @NonNull WorkerRegistryClient workerRegistryClient) { this.taskExecutionContext = taskExecutionContext; this.workerConfig = workerConfig; this.workerMessageSender = workerMessageSender; - this.storageOperate = storageOperate; + this.storageOperator = storageOperator; this.workerRegistryClient = workerRegistryClient; } @@ -55,7 +55,7 @@ public class DefaultWorkerTaskExecutorFactory taskExecutionContext, workerConfig, workerMessageSender, - storageOperate, + storageOperator, workerRegistryClient); } } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutor.java index 604b0b5743..c9382f170d 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutor.java @@ -33,7 +33,7 @@ import org.apache.dolphinscheduler.extract.base.client.SingletonJdkDynamicRpcCli import org.apache.dolphinscheduler.extract.base.utils.Host; import org.apache.dolphinscheduler.extract.master.transportor.ITaskInstanceExecutionEvent; import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; @@ -76,7 +76,7 @@ public abstract class WorkerTaskExecutor implements Runnable { protected final TaskExecutionContext taskExecutionContext; protected final WorkerConfig workerConfig; protected final WorkerMessageSender workerMessageSender; - protected final @Nullable StorageOperate storageOperate; + protected final @Nullable StorageOperator storageOperator; protected final WorkerRegistryClient workerRegistryClient; protected @Nullable AbstractTask task; @@ -85,12 +85,12 @@ public abstract class WorkerTaskExecutor implements Runnable { @NonNull TaskExecutionContext taskExecutionContext, @NonNull WorkerConfig workerConfig, @NonNull WorkerMessageSender workerMessageSender, - @Nullable StorageOperate storageOperate, + @Nullable StorageOperator storageOperator, @NonNull WorkerRegistryClient workerRegistryClient) { this.taskExecutionContext = taskExecutionContext; this.workerConfig = workerConfig; this.workerMessageSender = workerMessageSender; - this.storageOperate = storageOperate; + this.storageOperator = storageOperator; this.workerRegistryClient = workerRegistryClient; SensitiveDataConverter.addMaskPattern(K8S_CONFIG_REGEX); } @@ -223,12 +223,12 @@ public abstract class WorkerTaskExecutor implements Runnable { log.info("Create TaskChannel: {} successfully", taskChannel.getClass().getName()); - ResourceContext resourceContext = TaskExecutionContextUtils.downloadResourcesIfNeeded(originTenant, taskChannel, - storageOperate, taskExecutionContext); + ResourceContext resourceContext = TaskExecutionContextUtils.downloadResourcesIfNeeded(taskChannel, + storageOperator, taskExecutionContext); taskExecutionContext.setResourceContext(resourceContext); log.info("Download resources successfully: \n{}", taskExecutionContext.getResourceContext()); - TaskFilesTransferUtils.downloadUpstreamFiles(taskExecutionContext, storageOperate); + TaskFilesTransferUtils.downloadUpstreamFiles(taskExecutionContext, storageOperator); log.info("Download upstream files: {} successfully", TaskFilesTransferUtils.getFileLocalParams(taskExecutionContext, Direct.IN)); @@ -282,7 +282,7 @@ public abstract class WorkerTaskExecutor implements Runnable { taskExecutionContext.setEndTime(System.currentTimeMillis()); // upload out files and modify the "OUT FILE" property in VarPool - TaskFilesTransferUtils.uploadOutputFiles(taskExecutionContext, storageOperate); + TaskFilesTransferUtils.uploadOutputFiles(taskExecutionContext, storageOperator); log.info("Upload output files: {} successfully", TaskFilesTransferUtils.getFileLocalParams(taskExecutionContext, Direct.OUT)); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorFactoryBuilder.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorFactoryBuilder.java index 56f3207884..4cb1739abc 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorFactoryBuilder.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutorFactoryBuilder.java @@ -17,7 +17,7 @@ package org.apache.dolphinscheduler.server.worker.runner; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient; @@ -36,7 +36,7 @@ public class WorkerTaskExecutorFactoryBuilder { private WorkerMessageSender workerMessageSender; @Autowired(required = false) - private StorageOperate storageOperate; + private StorageOperator storageOperator; @Autowired private WorkerRegistryClient workerRegistryClient; @@ -45,11 +45,11 @@ public class WorkerTaskExecutorFactoryBuilder { WorkerConfig workerConfig, WorkerMessageSender workerMessageSender, WorkerTaskExecutorThreadPool workerManager, - StorageOperate storageOperate, + StorageOperator storageOperator, WorkerRegistryClient workerRegistryClient) { this.workerConfig = workerConfig; this.workerMessageSender = workerMessageSender; - this.storageOperate = storageOperate; + this.storageOperator = storageOperator; this.workerRegistryClient = workerRegistryClient; } @@ -57,7 +57,7 @@ public class WorkerTaskExecutorFactoryBuilder { return new DefaultWorkerTaskExecutorFactory(taskExecutionContext, workerConfig, workerMessageSender, - storageOperate, + storageOperator, workerRegistryClient); } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java index 1b486f265c..70db04a6c8 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java @@ -18,7 +18,8 @@ package org.apache.dolphinscheduler.server.worker.utils; import org.apache.dolphinscheduler.common.utils.FileUtils; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; +import org.apache.dolphinscheduler.plugin.storage.api.ResourceMetadata; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; @@ -65,9 +66,8 @@ public class TaskExecutionContextUtils { } } - public static ResourceContext downloadResourcesIfNeeded(String tenant, - TaskChannel taskChannel, - StorageOperate storageOperate, + public static ResourceContext downloadResourcesIfNeeded(TaskChannel taskChannel, + StorageOperator storageOperator, TaskExecutionContext taskExecutionContext) { AbstractParameters abstractParameters = taskChannel.parseParameters( ParametersNode.builder() @@ -86,14 +86,15 @@ public class TaskExecutionContextUtils { for (ResourceInfo resourceInfo : resourceFilesList) { String resourceAbsolutePathInStorage = resourceInfo.getResourceName(); - String resourceRelativePath = storageOperate.getResourceFileName(tenant, resourceAbsolutePathInStorage); - String resourceAbsolutePathInLocal = Paths.get(taskWorkingDirectory, resourceRelativePath).toString(); + ResourceMetadata resourceMetaData = storageOperator.getResourceMetaData(resourceAbsolutePathInStorage); + String resourceAbsolutePathInLocal = + Paths.get(taskWorkingDirectory, resourceMetaData.getResourceRelativePath()).toString(); File file = new File(resourceAbsolutePathInLocal); if (!file.exists()) { try { long resourceDownloadStartTime = System.currentTimeMillis(); - storageOperate.download(resourceAbsolutePathInStorage, resourceAbsolutePathInLocal, true); - log.debug("Download resource file {} under: {} successfully", resourceAbsolutePathInStorage, + storageOperator.download(resourceAbsolutePathInStorage, resourceAbsolutePathInLocal, true); + log.info("Download resource file {} -> {} successfully", resourceAbsolutePathInStorage, resourceAbsolutePathInLocal); FileUtils.setFileTo755(file); WorkerServerMetrics diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskFilesTransferUtils.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskFilesTransferUtils.java index f060c0a17d..47cd9bdaf8 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskFilesTransferUtils.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskFilesTransferUtils.java @@ -22,7 +22,7 @@ import static org.apache.dolphinscheduler.common.constants.Constants.CRC_SUFFIX; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; @@ -65,11 +65,11 @@ public class TaskFilesTransferUtils { * upload output files to resource storage * * @param taskExecutionContext is the context of task - * @param storageOperate is the storage operate + * @param storageOperator is the storage operate * @throws TaskException TaskException */ public static void uploadOutputFiles(TaskExecutionContext taskExecutionContext, - StorageOperate storageOperate) throws TaskException { + StorageOperator storageOperator) throws TaskException { // get OUTPUT FILE parameters List localParamsProperty = getFileLocalParams(taskExecutionContext, Direct.OUT); if (localParamsProperty.isEmpty()) { @@ -102,15 +102,15 @@ public class TaskFilesTransferUtils { try { // upload file to storage String resourceWholePath = - storageOperate.getResourceFullName(taskExecutionContext.getTenantCode(), resourcePath); + storageOperator.getStorageFileAbsolutePath(taskExecutionContext.getTenantCode(), resourcePath); String resourceCRCWholePath = - storageOperate.getResourceFullName(taskExecutionContext.getTenantCode(), resourceCRCPath); + storageOperator.getStorageFileAbsolutePath(taskExecutionContext.getTenantCode(), + resourceCRCPath); log.info("{} --- Local:{} to Remote:{}", property, srcPath, resourceWholePath); - storageOperate.upload(taskExecutionContext.getTenantCode(), srcPath, resourceWholePath, false, true); + storageOperator.upload(srcPath, resourceWholePath, false, true); log.info("{} --- Local:{} to Remote:{}", "CRC file", srcCRCPath, resourceCRCWholePath); - storageOperate.upload(taskExecutionContext.getTenantCode(), srcCRCPath, resourceCRCWholePath, false, - true); - } catch (IOException ex) { + storageOperator.upload(srcCRCPath, resourceCRCWholePath, false, true); + } catch (Exception ex) { throw new TaskException("Upload file to storage error", ex); } @@ -134,10 +134,11 @@ public class TaskFilesTransferUtils { * only download files which are defined in the task parameters * * @param taskExecutionContext is the context of task - * @param storageOperate is the storage operate + * @param storageOperator is the storage operate * @throws TaskException task exception */ - public static void downloadUpstreamFiles(TaskExecutionContext taskExecutionContext, StorageOperate storageOperate) { + public static void downloadUpstreamFiles(TaskExecutionContext taskExecutionContext, + StorageOperator storageOperator) { // get "IN FILE" parameters List localParamsProperty = getFileLocalParams(taskExecutionContext, Direct.IN); @@ -178,14 +179,10 @@ public class TaskFilesTransferUtils { downloadPath = targetPath; } - try { - String resourceWholePath = - storageOperate.getResourceFullName(taskExecutionContext.getTenantCode(), resourcePath); - log.info("{} --- Remote:{} to Local:{}", property, resourceWholePath, downloadPath); - storageOperate.download(resourceWholePath, downloadPath, true); - } catch (IOException ex) { - throw new TaskException("Download file from storage error", ex); - } + String resourceWholePath = + storageOperator.getStorageFileAbsolutePath(taskExecutionContext.getTenantCode(), resourcePath); + log.info("{} --- Remote:{} to Local:{}", property, resourceWholePath, downloadPath); + storageOperator.download(resourceWholePath, downloadPath, true); // unpack if the data is packaged if (isPack) { diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutorTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutorTest.java index e211fcdd1f..f451c09012 100644 --- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutorTest.java +++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/DefaultWorkerTaskExecutorTest.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.server.worker.runner; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; @@ -39,7 +39,7 @@ public class DefaultWorkerTaskExecutorTest { private WorkerMessageSender workerMessageSender = Mockito.mock(WorkerMessageSender.class); - private StorageOperate storageOperate = Mockito.mock(StorageOperate.class); + private StorageOperator storageOperator = Mockito.mock(StorageOperator.class); private WorkerRegistryClient workerRegistryClient = Mockito.mock(WorkerRegistryClient.class); @@ -55,7 +55,7 @@ public class DefaultWorkerTaskExecutorTest { taskExecutionContext, workerConfig, workerMessageSender, - storageOperate, + storageOperator, workerRegistryClient); Assertions.assertAll(workerTaskExecutor::run); @@ -78,7 +78,7 @@ public class DefaultWorkerTaskExecutorTest { taskExecutionContext, workerConfig, workerMessageSender, - storageOperate, + storageOperator, workerRegistryClient); Assertions.assertAll(workerTaskExecutor::run); diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstanceOperationFunctionTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstanceOperationFunctionTest.java index f761dd61d6..cc17d74e02 100644 --- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstanceOperationFunctionTest.java +++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/runner/operator/TaskInstanceOperationFunctionTest.java @@ -31,7 +31,7 @@ import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstancePauseR import org.apache.dolphinscheduler.extract.worker.transportor.TaskInstancePauseResponse; import org.apache.dolphinscheduler.extract.worker.transportor.UpdateWorkflowHostRequest; import org.apache.dolphinscheduler.extract.worker.transportor.UpdateWorkflowHostResponse; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; @@ -71,7 +71,7 @@ public class TaskInstanceOperationFunctionTest { private WorkerTaskExecutorThreadPool workerManager = Mockito.mock(WorkerTaskExecutorThreadPool.class); - private StorageOperate storageOperate = Mockito.mock(StorageOperate.class); + private StorageOperator storageOperator = Mockito.mock(StorageOperator.class); private WorkerRegistryClient workerRegistryClient = Mockito.mock(WorkerRegistryClient.class); @@ -92,7 +92,7 @@ public class TaskInstanceOperationFunctionTest { workerConfig, workerMessageSender, workerManager, - storageOperate, + storageOperator, workerRegistryClient); TaskInstanceDispatchOperationFunction taskInstanceDispatchOperationFunction = @@ -186,7 +186,7 @@ public class TaskInstanceOperationFunctionTest { workerConfig, workerMessageSender, workerManager, - storageOperate, + storageOperator, workerRegistryClient); TaskInstanceDispatchOperationFunction taskInstanceDispatchOperationFunction = diff --git a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/utils/TaskFilesTransferUtilsTest.java b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/utils/TaskFilesTransferUtilsTest.java index 9b8d5a9129..a5a425f9a9 100644 --- a/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/utils/TaskFilesTransferUtilsTest.java +++ b/dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/utils/TaskFilesTransferUtilsTest.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.server.worker.utils; import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; +import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.enums.DataType; import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; @@ -99,8 +99,8 @@ public class TaskFilesTransferUtilsTest { List oriProperties = TaskFilesTransferUtils.getVarPools(taskExecutionContext); - StorageOperate storageOperate = Mockito.mock(StorageOperate.class); - TaskFilesTransferUtils.uploadOutputFiles(taskExecutionContext, storageOperate); + StorageOperator storageOperator = Mockito.mock(StorageOperator.class); + TaskFilesTransferUtils.uploadOutputFiles(taskExecutionContext, storageOperator); System.out.println(taskExecutionContext.getVarPool()); String exceptFolder = @@ -154,10 +154,10 @@ public class TaskFilesTransferUtilsTest { .endTime(endTime) .build(); - StorageOperate storageOperate = Mockito.mock(StorageOperate.class); + StorageOperator storageOperator = Mockito.mock(StorageOperator.class); Mockito.mockStatic(ZipUtil.class); Assertions.assertDoesNotThrow( - () -> TaskFilesTransferUtils.downloadUpstreamFiles(taskExecutionContext, storageOperate)); + () -> TaskFilesTransferUtils.downloadUpstreamFiles(taskExecutionContext, storageOperator)); } @Test