diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java index e2f3c6f33e..ff65bfd94b 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java @@ -55,6 +55,7 @@ import org.apache.dolphinscheduler.spi.enums.ResourceType; import org.apache.commons.lang3.StringUtils; +import java.io.IOException; import java.util.Map; import org.slf4j.Logger; @@ -139,7 +140,6 @@ public class ResourcesController extends BaseController { @Parameter(name = "name", description = "RESOURCE_NAME", required = true, schema = @Schema(implementation = String.class)), @Parameter(name = "description", description = "RESOURCE_DESC", schema = @Schema(implementation = String.class)), @Parameter(name = "file", description = "RESOURCE_FILE", required = true, schema = @Schema(implementation = MultipartFile.class)), - @Parameter(name = "pid", description = "RESOURCE_PID", required = true, schema = @Schema(implementation = int.class, example = "10")), @Parameter(name = "currentDir", description = "RESOURCE_CURRENT_DIR", required = true, schema = @Schema(implementation = String.class)) }) @PostMapping() @@ -150,10 +150,9 @@ public class ResourcesController extends BaseController { @RequestParam(value = "name") String alias, @RequestParam(value = "description", required = false) String description, @RequestParam("file") MultipartFile file, - @RequestParam(value = "pid") int pid, @RequestParam(value = "currentDir") String currentDir) { // todo verify the file name - return resourceService.createResource(loginUser, alias, description, type, file, pid, currentDir); + return resourceService.createResource(loginUser, alias, description, type, file, currentDir); } /** @@ -161,7 +160,6 @@ public class ResourcesController extends BaseController { * * @param loginUser login user * @param alias alias - * @param resourceId resource id * @param type resource type * @param description description * @param file resource file @@ -169,23 +167,24 @@ public class ResourcesController extends BaseController { */ @Operation(summary = "updateResource", description = "UPDATE_RESOURCE_NOTES") @Parameters({ - @Parameter(name = "id", description = "RESOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "100")), + @Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class)), + @Parameter(name = "tenantCode", description = "TENANT_CODE", required = true, schema = @Schema(implementation = String.class)), @Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)), @Parameter(name = "name", description = "RESOURCE_NAME", required = true, schema = @Schema(implementation = String.class)), @Parameter(name = "description", description = "RESOURCE_DESC", schema = @Schema(implementation = String.class)), @Parameter(name = "file", description = "RESOURCE_FILE", required = true, schema = @Schema(implementation = MultipartFile.class)) }) - @PutMapping(value = "/{id}") + @PutMapping() @ApiException(UPDATE_RESOURCE_ERROR) @AccessLogAnnotation(ignoreRequestArgs = "loginUser") public Result updateResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @PathVariable(value = "id") int resourceId, + @RequestParam(value = "fullName") String fullName, + @RequestParam(value = "tenantCode", required = false) String tenantCode, @RequestParam(value = "type") ResourceType type, @RequestParam(value = "name") String alias, @RequestParam(value = "description", required = false) String description, @RequestParam(value = "file", required = false) MultipartFile file) { - // todo verify the resource name - return resourceService.updateResource(loginUser, resourceId, alias, description, type, file); + return resourceService.updateResource(loginUser, fullName, tenantCode, alias, description, type, file); } /** @@ -197,15 +196,17 @@ public class ResourcesController extends BaseController { */ @Operation(summary = "queryResourceList", description = "QUERY_RESOURCE_LIST_NOTES") @Parameters({ - @Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)) + @Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)), + @Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class)) }) @GetMapping(value = "/list") @ResponseStatus(HttpStatus.OK) @ApiException(QUERY_RESOURCES_LIST_ERROR) @AccessLogAnnotation(ignoreRequestArgs = "loginUser") public Result queryResourceList(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam(value = "type") ResourceType type) { - Map result = resourceService.queryResourceList(loginUser, type); + @RequestParam(value = "type") ResourceType type, + @RequestParam(value = "fullName") String fullName) { + Map result = resourceService.queryResourceList(loginUser, type, fullName); return returnDataList(result); } @@ -222,7 +223,7 @@ public class ResourcesController extends BaseController { @Operation(summary = "queryResourceListPaging", description = "QUERY_RESOURCE_LIST_PAGING_NOTES") @Parameters({ @Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)), - @Parameter(name = "id", description = "RESOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "10")), + @Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class, example = "bucket_name/tenant_name/type/ds")), @Parameter(name = "searchVal", description = "SEARCH_VAL", schema = @Schema(implementation = String.class)), @Parameter(name = "pageNo", description = "PAGE_NO", required = true, schema = @Schema(implementation = int.class, example = "1")), @Parameter(name = "pageSize", description = "PAGE_SIZE", required = true, schema = @Schema(implementation = int.class, example = "20")) @@ -232,8 +233,9 @@ public class ResourcesController extends BaseController { @ApiException(QUERY_RESOURCES_LIST_PAGING) @AccessLogAnnotation(ignoreRequestArgs = "loginUser") public Result queryResourceListPaging(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestParam(value = "fullName") String fullName, + @RequestParam(value = "tenantCode") String tenantCode, @RequestParam(value = "type") ResourceType type, - @RequestParam(value = "id") int id, @RequestParam("pageNo") Integer pageNo, @RequestParam(value = "searchVal", required = false) String searchVal, @RequestParam("pageSize") Integer pageSize) { @@ -243,7 +245,8 @@ public class ResourcesController extends BaseController { } searchVal = ParameterUtils.handleEscapes(searchVal); - result = resourceService.queryResourceListPaging(loginUser, id, type, searchVal, pageNo, pageSize); + result = resourceService.queryResourceListPaging(loginUser, fullName, tenantCode, type, searchVal, pageNo, + pageSize); return result; } @@ -251,20 +254,20 @@ public class ResourcesController extends BaseController { * delete resource * * @param loginUser login user - * @param resourceId resource id * @return delete result code */ @Operation(summary = "deleteResource", description = "DELETE_RESOURCE_BY_ID_NOTES") @Parameters({ - @Parameter(name = "id", description = "RESOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "100")) + @Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class, example = "test/")) }) - @DeleteMapping(value = "/{id}") + @DeleteMapping() @ResponseStatus(HttpStatus.OK) @ApiException(DELETE_RESOURCE_ERROR) @AccessLogAnnotation(ignoreRequestArgs = "loginUser") public Result deleteResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @PathVariable(value = "id") int resourceId) throws Exception { - return resourceService.delete(loginUser, resourceId); + @RequestParam(value = "fullName") String fullName, + @RequestParam(value = "tenantCode", required = false) String tenantCode) throws Exception { + return resourceService.delete(loginUser, fullName, tenantCode); } /** @@ -312,55 +315,56 @@ public class ResourcesController extends BaseController { } /** - * query resource by full name and type + * query resource by file name and type * * @param loginUser login user - * @param fullName resource full name + * @param fileName resource full name + * @param tenantCode tenantcode of the owner of the resource * @param type resource type - * @param id resource id * @return true if the resource name not exists, otherwise return false */ - @Operation(summary = "queryResource", description = "QUERY_BY_RESOURCE_NAME") + @Operation(summary = "queryResourceByFileName", description = "QUERY_BY_RESOURCE_FILE_NAME") @Parameters({ @Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)), - @Parameter(name = "fullName", description = "RESOURCE_FULL_NAME", required = true, schema = @Schema(implementation = String.class)), - @Parameter(name = "id", description = "RESOURCE_ID", required = false, schema = @Schema(implementation = int.class, example = "10")) + @Parameter(name = "fileName", description = "RESOURCE_FILE_NAME", required = true, schema = @Schema(implementation = String.class)), + @Parameter(name = "tenantCode", description = "TENANT_CODE", required = true, schema = @Schema(implementation = String.class)), }) - @GetMapping(value = "/{id}") + @GetMapping(value = "/query-file-name") @ResponseStatus(HttpStatus.OK) @ApiException(RESOURCE_NOT_EXIST) @AccessLogAnnotation(ignoreRequestArgs = "loginUser") - public Result queryResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam(value = "fullName", required = false) String fullName, - @PathVariable(value = "id", required = false) Integer id, - @RequestParam(value = "type") ResourceType type) { + public Result queryResourceByFileName(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestParam(value = "fileName", required = false) String fileName, + @RequestParam(value = "tenantCode", required = false) String tenantCode, + @RequestParam(value = "type") ResourceType type) { - return resourceService.queryResource(loginUser, fullName, id, type); + return resourceService.queryResourceByFileName(loginUser, fileName, type, tenantCode); } /** * view resource file online * * @param loginUser login user - * @param resourceId resource id * @param skipLineNum skip line number * @param limit limit * @return resource content */ @Operation(summary = "viewResource", description = "VIEW_RESOURCE_BY_ID_NOTES") @Parameters({ - @Parameter(name = "id", description = "RESOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "100")), + @Parameter(name = "fullName", description = "RESOURCE_FULL_NAME", required = true, schema = @Schema(implementation = String.class, example = "tenant/1.png")), + @Parameter(name = "tenantCode", description = "TENANT_CODE", required = true, schema = @Schema(implementation = String.class)), @Parameter(name = "skipLineNum", description = "SKIP_LINE_NUM", required = true, schema = @Schema(implementation = int.class, example = "100")), @Parameter(name = "limit", description = "LIMIT", required = true, schema = @Schema(implementation = int.class, example = "100")) }) - @GetMapping(value = "/{id}/view") + @GetMapping(value = "/view") @ApiException(VIEW_RESOURCE_FILE_ON_LINE_ERROR) @AccessLogAnnotation(ignoreRequestArgs = "loginUser") public Result viewResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @PathVariable(value = "id") int resourceId, @RequestParam(value = "skipLineNum") int skipLineNum, - @RequestParam(value = "limit") int limit) { - return resourceService.readResource(loginUser, resourceId, skipLineNum, limit); + @RequestParam(value = "limit") int limit, + @RequestParam(value = "fullName") String fullName, + @RequestParam(value = "tenantCode") String tenantCode) { + return resourceService.readResource(loginUser, fullName, tenantCode, skipLineNum, limit); } /** @@ -375,7 +379,6 @@ public class ResourcesController extends BaseController { @Parameter(name = "suffix", description = "SUFFIX", required = true, schema = @Schema(implementation = String.class)), @Parameter(name = "description", description = "RESOURCE_DESC", schema = @Schema(implementation = String.class)), @Parameter(name = "content", description = "CONTENT", required = true, schema = @Schema(implementation = String.class)), - @Parameter(name = "pid", description = "RESOURCE_PID", required = true, schema = @Schema(implementation = int.class, example = "10")), @Parameter(name = "currentDir", description = "RESOURCE_CURRENTDIR", required = true, schema = @Schema(implementation = String.class)) }) @PostMapping(value = "/online-create") @@ -387,13 +390,12 @@ public class ResourcesController extends BaseController { @RequestParam(value = "suffix") String fileSuffix, @RequestParam(value = "description", required = false) String description, @RequestParam(value = "content") String content, - @RequestParam(value = "pid") int pid, @RequestParam(value = "currentDir") String currentDir) { if (StringUtils.isEmpty(content)) { logger.error("resource file contents are not allowed to be empty"); return error(RESOURCE_FILE_IS_EMPTY.getCode(), RESOURCE_FILE_IS_EMPTY.getMsg()); } - return resourceService.onlineCreateResource(loginUser, type, fileName, fileSuffix, description, content, pid, + return resourceService.onlineCreateResource(loginUser, type, fileName, fileSuffix, description, content, currentDir); } @@ -401,46 +403,46 @@ public class ResourcesController extends BaseController { * edit resource file online * * @param loginUser login user - * @param resourceId resource id * @param content content * @return update result code */ @Operation(summary = "updateResourceContent", description = "UPDATE_RESOURCE_NOTES") @Parameters({ - @Parameter(name = "id", description = "RESOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "100")), - @Parameter(name = "content", description = "CONTENT", required = true, schema = @Schema(implementation = String.class)) + @Parameter(name = "content", description = "CONTENT", required = true, schema = @Schema(implementation = String.class)), + @Parameter(name = "fullName", description = "FULL_NAME", required = true, schema = @Schema(implementation = String.class)), + @Parameter(name = "tenantCode", description = "TENANT_CODE", required = true, schema = @Schema(implementation = String.class)) }) - @PutMapping(value = "/{id}/update-content") + @PutMapping(value = "/update-content") @ApiException(EDIT_RESOURCE_FILE_ON_LINE_ERROR) @AccessLogAnnotation(ignoreRequestArgs = "loginUser") public Result updateResourceContent(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @PathVariable(value = "id") int resourceId, + @RequestParam(value = "fullName") String fullName, + @RequestParam(value = "tenantCode") String tenantCode, @RequestParam(value = "content") String content) { if (StringUtils.isEmpty(content)) { logger.error("The resource file contents are not allowed to be empty"); return error(RESOURCE_FILE_IS_EMPTY.getCode(), RESOURCE_FILE_IS_EMPTY.getMsg()); } - return resourceService.updateResourceContent(loginUser, resourceId, content); + return resourceService.updateResourceContent(loginUser, fullName, tenantCode, content); } /** * download resource file * * @param loginUser login user - * @param resourceId resource id * @return resource content */ @Operation(summary = "downloadResource", description = "DOWNLOAD_RESOURCE_NOTES") @Parameters({ - @Parameter(name = "id", description = "RESOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "100")) + @Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class, example = "test/")) }) - @GetMapping(value = "/{id}/download") + @GetMapping(value = "/download") @ResponseBody @ApiException(DOWNLOAD_RESOURCE_FILE_ERROR) @AccessLogAnnotation(ignoreRequestArgs = "loginUser") public ResponseEntity downloadResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @PathVariable(value = "id") int resourceId) throws Exception { - Resource file = resourceService.downloadResource(loginUser, resourceId); + @RequestParam(value = "fullName") String fullName) throws Exception { + Resource file = resourceService.downloadResource(loginUser, fullName); if (file == null) { return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(RESOURCE_NOT_EXIST.getMsg()); } @@ -460,7 +462,6 @@ public class ResourcesController extends BaseController { * @param database database * @param description description * @param className class name - * @param resourceId resource id * @return create result code */ @Operation(summary = "createUdfFunc", description = "CREATE_UDF_FUNCTION_NOTES") @@ -474,7 +475,7 @@ public class ResourcesController extends BaseController { @Parameter(name = "resourceId", description = "RESOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "100")) }) - @PostMapping(value = "/{resourceId}/udf-func") + @PostMapping(value = "/udf-func") @ResponseStatus(HttpStatus.CREATED) @ApiException(CREATE_UDF_FUNCTION_ERROR) @AccessLogAnnotation(ignoreRequestArgs = "loginUser") @@ -482,20 +483,20 @@ public class ResourcesController extends BaseController { @RequestParam(value = "type") UdfType type, @RequestParam(value = "funcName") String funcName, @RequestParam(value = "className") String className, + @RequestParam(value = "fullName") String fullName, @RequestParam(value = "argTypes", required = false) String argTypes, @RequestParam(value = "database", required = false) String database, - @RequestParam(value = "description", required = false) String description, - @PathVariable(value = "resourceId") int resourceId) { + @RequestParam(value = "description", required = false) String description) { // todo verify the sourceName - return udfFuncService.createUdfFunction(loginUser, funcName, className, argTypes, database, description, type, - resourceId); + return udfFuncService.createUdfFunction(loginUser, funcName, className, fullName, + argTypes, database, description, type); } /** * view udf function * * @param loginUser login user - * @param id resource id + * @param id udf function id * @return udf function detail */ @Operation(summary = "viewUIUdfFunction", description = "VIEW_UDF_FUNCTION_NOTES") @@ -521,7 +522,6 @@ public class ResourcesController extends BaseController { * @param argTypes argument types * @param database data base * @param description description - * @param resourceId resource id * @param className class name * @param udfFuncId udf function id * @return update result code @@ -534,11 +534,9 @@ public class ResourcesController extends BaseController { @Parameter(name = "className", description = "CLASS_NAME", required = true, schema = @Schema(implementation = String.class)), @Parameter(name = "argTypes", description = "ARG_TYPES", schema = @Schema(implementation = String.class)), @Parameter(name = "database", description = "DATABASE_NAME", schema = @Schema(implementation = String.class)), - @Parameter(name = "description", description = "UDF_DESC", schema = @Schema(implementation = String.class)), - @Parameter(name = "resourceId", description = "RESOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "100")) - + @Parameter(name = "description", description = "UDF_DESC", schema = @Schema(implementation = String.class)) }) - @PutMapping(value = "/{resourceId}/udf-func/{id}") + @PutMapping(value = "/udf-func/{id}") @ApiException(UPDATE_UDF_FUNCTION_ERROR) @AccessLogAnnotation(ignoreRequestArgs = "loginUser") public Result updateUdfFunc(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @@ -549,9 +547,9 @@ public class ResourcesController extends BaseController { @RequestParam(value = "argTypes", required = false) String argTypes, @RequestParam(value = "database", required = false) String database, @RequestParam(value = "description", required = false) String description, - @PathVariable(value = "resourceId") int resourceId) { - return udfFuncService.updateUdfFunc(loginUser, udfFuncId, funcName, className, argTypes, database, description, - type, resourceId); + @RequestParam(value = "fullName") String fullName) { + return udfFuncService.updateUdfFunc(loginUser, udfFuncId, funcName, className, + argTypes, database, description, type, fullName); } /** @@ -731,23 +729,26 @@ public class ResourcesController extends BaseController { } /** - * query resource by resource id + * query a resource by resource full name * * @param loginUser login user - * @param id resource id + * @param fullName resource full name * @return resource */ - @Operation(summary = "queryResourceById", description = "QUERY_BY_RESOURCE_NAME") + @Operation(summary = "queryResourceByFullName", description = "QUERY_BY_RESOURCE_FULL_NAME") @Parameters({ - @Parameter(name = "id", description = "RESOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "10")) + @Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)), + @Parameter(name = "fullName", description = "RESOURCE_FULL_NAME", required = true, schema = @Schema(implementation = String.class)), }) - @GetMapping(value = "/{id}/query") + @GetMapping(value = "/query-full-name") @ResponseStatus(HttpStatus.OK) @ApiException(RESOURCE_NOT_EXIST) @AccessLogAnnotation(ignoreRequestArgs = "loginUser") - public Result queryResourceById(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @PathVariable(value = "id", required = true) Integer id) { + public Result queryResourceByFullName(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + @RequestParam(value = "type") ResourceType type, + @RequestParam(value = "fullName") String fullName, + @RequestParam(value = "tenantCode") String tenantCode) throws IOException { - return resourceService.queryResourceById(loginUser, id); + return resourceService.queryResourceByFullName(loginUser, fullName, tenantCode, type); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java index e520212adf..708f9efc0a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java @@ -35,7 +35,7 @@ import com.fasterxml.jackson.annotation.JsonPropertyOrder; @JsonPropertyOrder({"id", "pid", "name", "fullName", "description", "isDirctory", "children", "type"}) public abstract class ResourceComponent { - public ResourceComponent(int id, int pid, String name, String fullName, String description, boolean isDirctory) { + public ResourceComponent(int id, String pid, String name, String fullName, String description, boolean isDirctory) { this.id = id; this.pid = pid; this.name = name; @@ -53,7 +53,7 @@ public abstract class ResourceComponent { /** * parent id */ - protected int pid; + protected String pid; /** * name */ diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java index 5b84fde1a5..1f404c31e1 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java @@ -19,10 +19,11 @@ package org.apache.dolphinscheduler.api.dto.resources.visitor; import org.apache.dolphinscheduler.api.dto.resources.Directory; import org.apache.dolphinscheduler.api.dto.resources.FileLeaf; import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent; -import org.apache.dolphinscheduler.dao.entity.Resource; +import org.apache.dolphinscheduler.service.storage.StorageEntity; import java.util.ArrayList; import java.util.List; +import java.util.Objects; /** * resource tree visitor @@ -32,7 +33,7 @@ public class ResourceTreeVisitor implements Visitor { /** * resource list */ - private List resourceList; + private List resourceList; public ResourceTreeVisitor() { } @@ -41,7 +42,7 @@ public class ResourceTreeVisitor implements Visitor { * constructor * @param resourceList resource list */ - public ResourceTreeVisitor(List resourceList) { + public ResourceTreeVisitor(List resourceList) { this.resourceList = resourceList; } @@ -50,14 +51,15 @@ public class ResourceTreeVisitor implements Visitor { * @return resoruce component */ @Override - public ResourceComponent visit() { + public ResourceComponent visit(String rootPath) { ResourceComponent rootDirectory = new Directory(); - for (Resource resource : resourceList) { + for (StorageEntity resource : resourceList) { // judge whether is root node - if (rootNode(resource)) { + if (rootNode(resource, rootPath)) { + // if it is a root node. ResourceComponent tempResourceComponent = getResourceComponent(resource); rootDirectory.add(tempResourceComponent); - tempResourceComponent.setChildren(setChildren(tempResourceComponent.getId(), resourceList)); + tempResourceComponent.setChildren(setChildren(tempResourceComponent.getFullName(), resourceList)); } } return rootDirectory; @@ -65,20 +67,21 @@ public class ResourceTreeVisitor implements Visitor { /** * set children - * @param id id + * @param fullName unique path * @param list resource list * @return resource component list */ - public static List setChildren(int id, List list) { + public static List setChildren(String fullName, List list) { + // id is the unique value, List childList = new ArrayList<>(); - for (Resource resource : list) { - if (id == resource.getPid()) { + for (StorageEntity resource : list) { + if (Objects.equals(fullName, resource.getPfullName())) { ResourceComponent tempResourceComponent = getResourceComponent(resource); childList.add(tempResourceComponent); } } for (ResourceComponent resourceComponent : childList) { - resourceComponent.setChildren(setChildren(resourceComponent.getId(), list)); + resourceComponent.setChildren(setChildren(resourceComponent.getFullName(), list)); } if (childList.size() == 0) { return new ArrayList<>(); @@ -91,17 +94,18 @@ public class ResourceTreeVisitor implements Visitor { * @param resource resource * @return true if it is the root node */ - public boolean rootNode(Resource resource) { + public boolean rootNode(StorageEntity resource, String rootPath) { boolean isRootNode = true; - if (resource.getPid() != -1) { - for (Resource parent : resourceList) { - if (resource.getPid() == parent.getId()) { + if (!Objects.equals(resource.getPfullName(), rootPath)) { + for (StorageEntity parent : resourceList) { + if (Objects.equals(resource.getPfullName(), parent.getFullName())) { isRootNode = false; break; } } } + return isRootNode; } @@ -110,7 +114,7 @@ public class ResourceTreeVisitor implements Visitor { * @param resource resource * @return resource component */ - private static ResourceComponent getResourceComponent(Resource resource) { + private static ResourceComponent getResourceComponent(StorageEntity resource) { ResourceComponent tempResourceComponent; if (resource.isDirectory()) { tempResourceComponent = new Directory(); @@ -119,9 +123,10 @@ public class ResourceTreeVisitor implements Visitor { } tempResourceComponent.setName(resource.getAlias()); - tempResourceComponent.setFullName(resource.getFullName().replaceFirst("/", "")); + // tempResourceComponent.setFullName(resource.getFullName().replaceFirst("/","")); + tempResourceComponent.setFullName(resource.getFullName()); tempResourceComponent.setId(resource.getId()); - tempResourceComponent.setPid(resource.getPid()); + tempResourceComponent.setPid(resource.getPfullName()); tempResourceComponent.setIdValue(resource.getId(), resource.isDirectory()); tempResourceComponent.setDescription(resource.getDescription()); tempResourceComponent.setType(resource.getType()); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/Visitor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/Visitor.java index 382bcc5e16..1792b2769c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/Visitor.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/Visitor.java @@ -28,5 +28,5 @@ public interface Visitor { * visit * @return resource component */ - ResourceComponent visit(); + ResourceComponent visit(String rootPath); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckService.java index b8016c5926..9f5fde0501 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckService.java @@ -38,7 +38,6 @@ public interface ResourcePermissionCheckService { * @param authorizationType * @param userId * @param logger - * @param * @return */ Set userOwnedResourceIdsAcquisition(Object authorizationType, Integer userId, Logger logger); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index 4de830c074..04f477e36b 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -57,9 +57,8 @@ public interface ResourcesService { * @param loginUser login user * @param name alias * @param desc description - * @param file file * @param type type - * @param pid parent id + * @param file file * @param currentDir current directory * @return create result code */ @@ -68,13 +67,11 @@ public interface ResourcesService { String desc, ResourceType type, MultipartFile file, - int pid, String currentDir); /** * update resource * @param loginUser login user - * @param resourceId resource id * @param name name * @param desc description * @param type resource type @@ -82,7 +79,8 @@ public interface ResourcesService { * @return update result code */ Result updateResource(User loginUser, - int resourceId, + String fullName, + String tenantCode, String name, String desc, ResourceType type, @@ -98,8 +96,8 @@ public interface ResourcesService { * @param pageSize page size * @return resource list page */ - Result queryResourceListPaging(User loginUser, int directoryId, ResourceType type, String searchVal, Integer pageNo, - Integer pageSize); + Result queryResourceListPaging(User loginUser, String fullName, String resTenantCode, + ResourceType type, String searchVal, Integer pageNo, Integer pageSize); /** * query resource list @@ -108,7 +106,7 @@ public interface ResourcesService { * @param type resource type * @return resource list */ - Map queryResourceList(User loginUser, ResourceType type); + Map queryResourceList(User loginUser, ResourceType type, String fullName); /** * query resource list by program type @@ -123,11 +121,10 @@ public interface ResourcesService { * delete resource * * @param loginUser login user - * @param resourceId resource id * @return delete result code * @throws IOException exception */ - Result delete(User loginUser, int resourceId) throws IOException; + Result delete(User loginUser, String fullName, String tenantCode) throws IOException; /** * verify resource by name and type @@ -139,23 +136,22 @@ public interface ResourcesService { Result verifyResourceName(String fullName, ResourceType type, User loginUser); /** - * verify resource by full name or pid and type - * @param fullName resource full name - * @param id resource id + * verify resource by file name + * @param fileName resource file name * @param type resource type - * @return true if the resource full name or pid not exists, otherwise return false + * @return true if the resource file name, otherwise return false */ - Result queryResource(User loginUser, String fullName, Integer id, ResourceType type); + Result queryResourceByFileName(User loginUser, String fileName, ResourceType type, String resTenantCode); /** * view resource file online * - * @param resourceId resource id * @param skipLineNum skip line number * @param limit limit + * @param fullName fullName * @return resource content */ - Result readResource(User loginUser, int resourceId, int skipLineNum, int limit); + Result readResource(User loginUser, String fullName, String tenantCode, int skipLineNum, int limit); /** * create resource file online @@ -169,7 +165,7 @@ public interface ResourcesService { * @return create result code */ Result onlineCreateResource(User loginUser, ResourceType type, String fileName, String fileSuffix, - String desc, String content, int pid, String currentDirectory); + String desc, String content, String currentDirectory); /** * create or update resource. @@ -203,16 +199,16 @@ public interface ResourcesService { * @param content content * @return update result cod */ - Result updateResourceContent(User loginUser, int resourceId, String content); + Result updateResourceContent(User loginUser, String fullName, String tenantCode, + String content); /** * download file * - * @param resourceId resource id * @return resource content * @throws IOException exception */ - org.springframework.core.io.Resource downloadResource(User loginUser, int resourceId) throws IOException; + org.springframework.core.io.Resource downloadResource(User loginUser, String fullName) throws IOException; /** * list all file @@ -270,9 +266,11 @@ public interface ResourcesService { /** * get resource by id - * @param resourceId resource id + * @param fullName resource full name + * @param tenantCode owner's tenant code of resource * @return resource */ - Result queryResourceById(User loginUser, Integer resourceId); + Result queryResourceByFullName(User loginUser, String fullName, String tenantCode, + ResourceType type) throws IOException; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java index 0cac41663f..35be4066d8 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java @@ -35,18 +35,17 @@ public interface UdfFuncService { * @param argTypes argument types * @param database database * @param desc description - * @param resourceId resource id * @param className class name * @return create result code */ Result createUdfFunction(User loginUser, String funcName, String className, + String fullName, String argTypes, String database, String desc, - UdfType type, - int resourceId); + UdfType type); /** * query udf function @@ -66,6 +65,7 @@ public interface UdfFuncService { * @param database data base * @param desc description * @param resourceId resource id + * @param fullName resource full name * @param className class name * @return update result code */ @@ -77,7 +77,7 @@ public interface UdfFuncService { String database, String desc, UdfType type, - int resourceId); + String fullName); /** * query udf function list paging diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java index 85ac4734b8..d82a0e4b7d 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java @@ -26,13 +26,13 @@ import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S; import static org.apache.dolphinscheduler.common.constants.Constants.JAR; import static org.apache.dolphinscheduler.common.constants.Constants.PERIOD; -import org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant; import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent; import org.apache.dolphinscheduler.api.dto.resources.filter.ResourceFilter; import org.apache.dolphinscheduler.api.dto.resources.visitor.ResourceTreeVisitor; import org.apache.dolphinscheduler.api.dto.resources.visitor.Visitor; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ServiceException; +import org.apache.dolphinscheduler.api.service.ProcessDefinitionService; import org.apache.dolphinscheduler.api.service.ResourcesService; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.RegexUtils; @@ -44,18 +44,26 @@ import org.apache.dolphinscheduler.common.enums.ResUploadType; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog; +import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation; import org.apache.dolphinscheduler.dao.entity.Resource; -import org.apache.dolphinscheduler.dao.entity.ResourcesUser; +import org.apache.dolphinscheduler.dao.entity.ResourcesTask; +import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.dolphinscheduler.dao.entity.User; -import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionLogMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationMapper; import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; +import org.apache.dolphinscheduler.dao.mapper.ResourceTaskMapper; import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper; +import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper; -import org.apache.dolphinscheduler.dao.utils.ResourceProcessDefinitionUtils; +import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; +import org.apache.dolphinscheduler.service.process.ProcessService; +import org.apache.dolphinscheduler.service.storage.StorageEntity; import org.apache.dolphinscheduler.service.storage.StorageOperate; import org.apache.dolphinscheduler.spi.enums.ResourceType; @@ -77,19 +85,16 @@ import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.UUID; -import java.util.regex.Matcher; import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.dao.DuplicateKeyException; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.web.multipart.MultipartFile; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.SerializationFeature; import com.google.common.base.Joiner; import com.google.common.io.Files; @@ -105,9 +110,21 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe @Autowired private ResourceMapper resourcesMapper; + @Autowired + private ResourceTaskMapper resourceTaskMapper; + + @Autowired + private TaskDefinitionMapper taskDefinitionMapper; + @Autowired private UdfFuncMapper udfFunctionMapper; + @Autowired + private ProcessService processService; + + @Autowired + private ProcessDefinitionService processDefinitionService; + @Autowired private TenantMapper tenantMapper; @@ -118,7 +135,10 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe private ResourceUserMapper resourceUserMapper; @Autowired - private ProcessDefinitionMapper processDefinitionMapper; + private ProcessDefinitionLogMapper processDefinitionLogMapper; + + @Autowired + private ProcessTaskRelationMapper processTaskRelationMapper; @Autowired(required = false) private StorageOperate storageOperate; @@ -143,14 +163,6 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe int pid, String currentDir) { Result result = new Result<>(); - String funcPermissionKey = type.equals(ResourceType.FILE) ? ApiFuncIdentificationConstant.FOLDER_ONLINE_CREATE - : ApiFuncIdentificationConstant.UDF_FOLDER_ONLINE_CREATE; - boolean canOperatorPermissions = - canOperatorPermissions(loginUser, null, AuthorizationType.RESOURCE_FILE_ID, funcPermissionKey); - if (!canOperatorPermissions) { - putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); - return result; - } result = checkResourceUploadStartupState(); if (!result.getCode().equals(Status.SUCCESS.getCode())) { @@ -162,48 +174,49 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe return result; } - if (checkDescriptionLength(description)) { - logger.warn("Parameter description is too long."); - putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR); + User user = userMapper.selectById(loginUser.getId()); + if (user == null) { + logger.error("user {} not exists", loginUser.getId()); + putMsg(result, Status.USER_NOT_EXIST, loginUser.getId()); return result; } - String fullName = getFullName(currentDir, name); - result = verifyResource(loginUser, type, fullName, pid); - if (!result.getCode().equals(Status.SUCCESS.getCode())) { + Tenant tenant = tenantMapper.queryById(user.getTenantId()); + if (tenant == null) { + logger.error("tenant not exists"); + putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST); return result; } - if (checkResourceExists(fullName, type.ordinal())) { - logger.warn("Resource directory exists, can not create again, fullName:{}.", fullName); - putMsg(result, Status.RESOURCE_EXIST); + String tenantCode = tenant.getTenantCode(); + + if (!isUserTenantValid(isAdmin(loginUser), tenantCode, "")) { + logger.error("current user does not have permission"); + putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); return result; } - Date now = new Date(); + if (checkDescriptionLength(description)) { + logger.warn("Parameter description is too long."); + putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR); + return result; + } - Resource resource = - new Resource(pid, name, fullName, true, description, name, loginUser.getId(), type, 0, now, now); + String userResRootPath = ResourceType.UDF.equals(type) ? storageOperate.getUdfDir(tenantCode) + : storageOperate.getResDir(tenantCode); + String fullName = !currentDir.contains(userResRootPath) ? userResRootPath + name : currentDir + name; try { - resourcesMapper.insert(resource); - putMsg(result, Status.SUCCESS); - permissionPostHandle(resource.getType(), loginUser, resource.getId()); - Map resultMap = new HashMap<>(); - for (Map.Entry entry : new BeanMap(resource).entrySet()) { - if (!"class".equalsIgnoreCase(entry.getKey().toString())) { - resultMap.put(entry.getKey().toString(), entry.getValue()); - } + if (checkResourceExists(fullName)) { + logger.error("resource directory {} has exist, can't recreate", fullName); + putMsg(result, Status.RESOURCE_EXIST); + return result; } - result.setData(resultMap); - } catch (DuplicateKeyException e) { - logger.warn("Resource directory exists, can not create again, fullName:{}.", fullName); - putMsg(result, Status.RESOURCE_EXIST); - return result; } catch (Exception e) { logger.warn("Resource exists, can not create again, fullName:{}.", fullName, e); throw new ServiceException("resource already exists, can't recreate"); } + // create directory in storage createDirectory(loginUser, fullName, type, result); return result; @@ -220,9 +233,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe * @param loginUser login user * @param name alias * @param desc description - * @param file file * @param type type - * @param pid parent id + * @param file file * @param currentDir current directory * @return create result code */ @@ -233,36 +245,33 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe String desc, ResourceType type, MultipartFile file, - int pid, String currentDir) { Result result = new Result<>(); - String funcPermissionKey = type.equals(ResourceType.FILE) ? ApiFuncIdentificationConstant.FILE_UPLOAD - : ApiFuncIdentificationConstant.UDF_UPLOAD; - boolean canOperatorPermissions = - canOperatorPermissions(loginUser, null, AuthorizationType.RESOURCE_FILE_ID, funcPermissionKey); - if (!canOperatorPermissions) { - putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); - return result; - } + result = checkResourceUploadStartupState(); if (!result.getCode().equals(Status.SUCCESS.getCode())) { return result; } - result = verifyPid(loginUser, pid); - if (!result.getCode().equals(Status.SUCCESS.getCode())) { + User user = userMapper.selectById(loginUser.getId()); + if (user == null) { + logger.error("user {} not exists", loginUser.getId()); + putMsg(result, Status.USER_NOT_EXIST, loginUser.getId()); return result; } - if (checkDescriptionLength(desc)) { - logger.warn("Parameter description is too long."); - putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR); + + Tenant tenant = tenantMapper.queryById(user.getTenantId()); + if (tenant == null) { + logger.error("tenant not exists"); + putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST); return result; } - // make sure login user has tenant - String tenantCode = getTenantCode(loginUser.getId(), result); - if (StringUtils.isEmpty(tenantCode)) { - logger.error("Tenant of current login user does not specified, loginUserName:{}.", loginUser.getUserName()); + String tenantCode = tenant.getTenantCode(); + + if (!isUserTenantValid(isAdmin(loginUser), tenantCode, "")) { + logger.error("current user does not have permission"); + putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); return result; } @@ -272,45 +281,32 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe } // check resource name exists - String fullName = getFullName(currentDir, name); - if (checkResourceExists(fullName, type.ordinal())) { - logger.warn("Resource exists, can not create again, fullName:{}.", RegexUtils.escapeNRT(name)); - putMsg(result, Status.RESOURCE_EXIST); - return result; - } - if (fullName.length() > Constants.RESOURCE_FULL_NAME_MAX_LENGTH) { - logger.warn( - "Resource file's name is longer than max full name length, fullName:{}, fullNameSize:{}, maxFullNameSize:{}", - RegexUtils.escapeNRT(name), fullName.length(), Constants.RESOURCE_FULL_NAME_MAX_LENGTH); - putMsg(result, Status.RESOURCE_FULL_NAME_TOO_LONG_ERROR); - return result; - } - - Date now = new Date(); - Resource resource = new Resource(pid, name, fullName, false, desc, file.getOriginalFilename(), - loginUser.getId(), type, file.getSize(), now, now); + String userResRootPath = ResourceType.UDF.equals(type) ? storageOperate.getUdfDir(tenantCode) + : storageOperate.getResDir(tenantCode); + String currDirNFileName = !currentDir.contains(userResRootPath) ? userResRootPath + name : currentDir + name; try { - resourcesMapper.insert(resource); - updateParentResourceSize(resource, resource.getSize()); - putMsg(result, Status.SUCCESS); - permissionPostHandle(resource.getType(), loginUser, resource.getId()); - Map resultMap = new HashMap<>(); - for (Map.Entry entry : new BeanMap(resource).entrySet()) { - if (!"class".equalsIgnoreCase(entry.getKey().toString())) { - resultMap.put(entry.getKey().toString(), entry.getValue()); - } + if (checkResourceExists(currDirNFileName)) { + logger.error("resource {} has exist, can't recreate", RegexUtils.escapeNRT(name)); + putMsg(result, Status.RESOURCE_EXIST); + return result; } - result.setData(resultMap); } catch (Exception e) { - logger.warn("Resource exists, can not create again, fullName:{}.", fullName, e); throw new ServiceException("resource already exists, can't recreate"); } + if (currDirNFileName.length() > Constants.RESOURCE_FULL_NAME_MAX_LENGTH) { + logger.error( + "Resource file's name is longer than max full name length, fullName:{}, " + + "fullNameSize:{}, maxFullNameSize:{}", + RegexUtils.escapeNRT(name), currDirNFileName.length(), Constants.RESOURCE_FULL_NAME_MAX_LENGTH); + putMsg(result, Status.RESOURCE_FULL_NAME_TOO_LONG_ERROR); + return result; + } // fail upload - if (!upload(loginUser, fullName, file, type)) { - logger.error("Upload resource file failed, resourceName:{}, fileName:{}.", - RegexUtils.escapeNRT(name), RegexUtils.escapeNRT(file.getOriginalFilename())); + if (!upload(loginUser, currDirNFileName, file, type)) { + logger.error("upload resource: {} file: {} failed.", RegexUtils.escapeNRT(name), + RegexUtils.escapeNRT(file.getOriginalFilename())); putMsg(result, Status.STORE_OPERATE_CREATE_ERROR); throw new ServiceException( String.format("upload resource: %s file: %s failed.", name, file.getOriginalFilename())); @@ -354,11 +350,15 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe * check resource is exists * * @param fullName fullName - * @param type type * @return true if resource exists */ - private boolean checkResourceExists(String fullName, int type) { - Boolean existResource = resourcesMapper.existResource(fullName, type); + private boolean checkResourceExists(String fullName) { + Boolean existResource = false; + try { + existResource = storageOperate.exists(fullName); + } catch (IOException e) { + logger.error("error occurred when checking resource: " + fullName, e); + } return Boolean.TRUE.equals(existResource); } @@ -366,7 +366,9 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe * update resource * * @param loginUser login user - * @param resourceId resource id + * @param resourceFullName resource full name + * @param resTenantCode tenantCode in the request field "resTenantCode" for tenant code owning the resource, + * can be different from the login user in the case of logging in as admin users. * @param name name * @param desc description * @param type resource type @@ -376,37 +378,52 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe @Override @Transactional public Result updateResource(User loginUser, - int resourceId, + String resourceFullName, + String resTenantCode, String name, String desc, ResourceType type, MultipartFile file) { Result result = new Result<>(); - String funcPermissionKey = type.equals(ResourceType.FILE) ? ApiFuncIdentificationConstant.FILE_UPDATE - : ApiFuncIdentificationConstant.UDF_UPDATE; - boolean canOperatorPermissions = - canOperatorPermissions(loginUser, new Object[]{resourceId}, checkResourceType(type), funcPermissionKey); - if (!canOperatorPermissions) { - putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); - return result; - } + result = checkResourceUploadStartupState(); if (!result.getCode().equals(Status.SUCCESS.getCode())) { return result; } - Resource resource = resourcesMapper.selectById(resourceId); - if (resource == null) { - logger.error("Resource does not exist, resourceId:{}.", resourceId); - putMsg(result, Status.RESOURCE_NOT_EXIST); + User user = userMapper.selectById(loginUser.getId()); + if (user == null) { + logger.error("user {} not exists", loginUser.getId()); + putMsg(result, Status.USER_NOT_EXIST, loginUser.getId()); return result; } - if (checkDescriptionLength(desc)) { - logger.warn("Parameter description is too long."); - putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR); + + Tenant tenant = tenantMapper.queryById(user.getTenantId()); + if (tenant == null) { + logger.error("tenant not exists"); + putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST); return result; } + String tenantCode = tenant.getTenantCode(); + + if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode)) { + logger.error("current user does not have permission"); + putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); + return result; + } + + String defaultPath = storageOperate.getResDir(tenantCode); + + StorageEntity resource; + try { + resource = storageOperate.getFileStatus(resourceFullName, defaultPath, resTenantCode, type); + } catch (Exception e) { + logger.error("Get file status fail, resource path: {}", resourceFullName, e); + putMsg(result, Status.RESOURCE_NOT_EXIST); + throw new ServiceException((String.format("Get file status fail, resource path: %s", resourceFullName))); + } + if (!PropertyUtils.getResUploadStartupState()) { logger.error("Storage does not start up, resource upload startup state: {}.", PropertyUtils.getResUploadStartupState()); @@ -423,21 +440,33 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe } if (file == null && name.equals(resource.getAlias()) && desc.equals(resource.getDescription())) { - logger.info("Resource does not need to be updated due to no change, resourceId:{}.", resourceId); + logger.info("Resource does not need to be updated due to no change, resource full name:{}.", + resourceFullName); putMsg(result, Status.SUCCESS); return result; } - // check resource already exists + // check if updated name of the resource already exists String originFullName = resource.getFullName(); String originResourceName = resource.getAlias(); + // the format of hdfs folders in the implementation has a "/" at the very end, we need to remove it. + originFullName = originFullName.endsWith("/") ? StringUtils.chop(originFullName) : originFullName; + name = name.endsWith("/") ? StringUtils.chop(name) : name; + // updated fullName String fullName = String.format(FORMAT_SS, originFullName.substring(0, originFullName.lastIndexOf(FOLDER_SEPARATOR) + 1), name); - if (!originResourceName.equals(name) && checkResourceExists(fullName, type.ordinal())) { - logger.warn("Resource exists, can not create again, fullName:{}.", RegexUtils.escapeNRT(name)); - putMsg(result, Status.RESOURCE_EXIST); - return result; + if (!originResourceName.equals(name)) { + try { + if (checkResourceExists(fullName)) { + logger.error("resource {} already exists, can't recreate", fullName); + putMsg(result, Status.RESOURCE_EXIST); + return result; + } + } catch (Exception e) { + throw new ServiceException(String.format("error occurs while querying resource: %s", fullName)); + } + } result = verifyFile(name, type, file); @@ -445,60 +474,6 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe return result; } - // query tenant by user id - String tenantCode = getTenantCode(resource.getUserId(), result); - if (StringUtils.isEmpty(tenantCode)) { - logger.error("Tenant of current login user does not specified, loginUserName:{}.", loginUser.getUserName()); - return result; - } - // verify whether the resource exists in storage - // get the path of origin file in storage - String originFileName = storageOperate.getFileName(resource.getType(), tenantCode, originFullName); - try { - if (!storageOperate.exists(tenantCode, originFileName)) { - logger.error( - "Resource file does not exist in {} storage, tenantCode:{}, resourceId:{}, originFileName:{}.", - resource.getType(), tenantCode, resourceId, originFileName); - putMsg(result, Status.RESOURCE_NOT_EXIST); - return result; - } - } catch (IOException e) { - logger.error("Storage operation error, storageType:{}, tenantCode:{}, resourceId:{}, originFileName:{}.", - resource.getType(), tenantCode, resourceId, originFileName, e); - throw new ServiceException(Status.HDFS_OPERATION_ERROR); - } - - if (!resource.isDirectory()) { - // get the origin file suffix - String originSuffix = Files.getFileExtension(originFullName); - String suffix = Files.getFileExtension(fullName); - boolean suffixIsChanged = false; - if (StringUtils.isBlank(suffix) && StringUtils.isNotBlank(originSuffix)) { - suffixIsChanged = true; - } - if (StringUtils.isNotBlank(suffix) && !suffix.equals(originSuffix)) { - suffixIsChanged = true; - } - // verify whether suffix is changed - if (suffixIsChanged) { - // need verify whether this resource is authorized to other users - Map columnMap = new HashMap<>(); - columnMap.put("resources_id", resourceId); - - List resourcesUsers = resourceUserMapper.selectByMap(columnMap); - if (CollectionUtils.isNotEmpty(resourcesUsers)) { - List userIds = - resourcesUsers.stream().map(ResourcesUser::getUserId).collect(Collectors.toList()); - List users = userMapper.selectBatchIds(userIds); - String userNames = users.stream().map(User::getUserName).collect(Collectors.toList()).toString(); - logger.error("resource is authorized to user {},suffix not allowed to be modified", userNames); - putMsg(result, Status.RESOURCE_IS_AUTHORIZED, userNames); - return result; - } - } - } - - // updateResource data Date now = new Date(); long originFileSize = resource.getSize(); @@ -511,97 +486,123 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe resource.setSize(file.getSize()); } - try { - resourcesMapper.updateById(resource); - if (resource.isDirectory()) { - List childrenResource = listAllChildren(resource, false); - if (CollectionUtils.isNotEmpty(childrenResource)) { - String matcherFullName = Matcher.quoteReplacement(fullName); - List childResourceList; - Integer[] childResIdArray = childrenResource.toArray(new Integer[childrenResource.size()]); - List resourceList = resourcesMapper.listResourceByIds(childResIdArray); - childResourceList = resourceList.stream().map(t -> { - t.setFullName(t.getFullName().replaceFirst(originFullName, matcherFullName)); - t.setUpdateTime(now); - return t; - }).collect(Collectors.toList()); - resourcesMapper.batchUpdateResource(childResourceList); - - if (ResourceType.UDF.equals(resource.getType())) { - List udfFuncs = udfFunctionMapper.listUdfByResourceId(childResIdArray); - if (CollectionUtils.isNotEmpty(udfFuncs)) { - udfFuncs = udfFuncs.stream().map(t -> { - t.setResourceName(t.getResourceName().replaceFirst(originFullName, matcherFullName)); - t.setUpdateTime(now); - return t; - }).collect(Collectors.toList()); - udfFunctionMapper.batchUpdateUdfFunc(udfFuncs); - } - } - } - } else if (ResourceType.UDF.equals(resource.getType())) { - List udfFuncs = udfFunctionMapper.listUdfByResourceId(new Integer[]{resourceId}); - if (CollectionUtils.isNotEmpty(udfFuncs)) { - udfFuncs = udfFuncs.stream().map(t -> { - t.setResourceName(fullName); - t.setUpdateTime(now); - return t; - }).collect(Collectors.toList()); - udfFunctionMapper.batchUpdateUdfFunc(udfFuncs); - } + // if name unchanged, return directly without moving on HDFS + if (originResourceName.equals(name) && file == null) { + return result; + } + List existResourcesList; + if (resource.isDirectory()) { + existResourcesList = resourceTaskMapper.selectSubfoldersFullNames(originFullName + FOLDER_SEPARATOR); + } else { + existResourcesList = resourceTaskMapper.selectByMap( + Collections.singletonMap("full_name", originFullName)); + } + + if (existResourcesList.size() > 0 && !fullName.equals(originFullName)) { + // check if any related task is online. If it is, it can not be updated. + for (ResourcesTask existResource : existResourcesList) { + int taskId = existResource.getTaskId(); + if (processService.isTaskOnline(taskDefinitionMapper.selectById(taskId).getCode())) { + logger.error("can't be updated,because it is used of process definition that's online"); + logger.error("resource task relation id:{} is used of task code {}", existResource.getId(), + taskDefinitionMapper.selectById(taskId).getCode()); + putMsg(result, Status.RESOURCE_IS_USED); + return result; + } } - putMsg(result, Status.SUCCESS); - Map resultMap = new HashMap<>(); - for (Map.Entry entry : new BeanMap(resource).entrySet()) { - if (!Constants.CLASS.equalsIgnoreCase(entry.getKey().toString())) { - resultMap.put(entry.getKey().toString(), entry.getValue()); + for (ResourcesTask existResource : existResourcesList) { + int taskId = existResource.getTaskId(); + long taskCode = taskDefinitionMapper.selectById(taskId).getCode(); + + List processTaskRelation = processTaskRelationMapper.selectByMap( + Collections.singletonMap("post_task_code", taskCode)); + if (processTaskRelation.size() > 0) { + long processDefinitionCode = processTaskRelation.get(0).getProcessDefinitionCode(); + int processDefinitionVersion = processTaskRelation.get(0).getProcessDefinitionVersion(); + List taskRelationList = processTaskRelationMapper.queryByProcessCode( + processTaskRelation.get(0).getProjectCode(), + processDefinitionCode); + + List taskDefinitionLogList = new ArrayList<>(); + + if (taskRelationList.size() > 0) { + ProcessDefinitionLog processDefinition = + processDefinitionLogMapper.queryByDefinitionCodeAndVersion( + processDefinitionCode, processDefinitionVersion); + for (ProcessTaskRelation taskRelation : taskRelationList) { + long taskCodeInProcess = taskRelation.getPostTaskCode(); + TaskDefinition taskDefinition = taskDefinitionMapper.queryByCode(taskCodeInProcess); + if (taskCodeInProcess == taskCode) { + // originFullName is a prefix if isDirectory is true + taskDefinition.setTaskParams(RemoveResourceFromResourceList(originFullName, + taskDefinition.getTaskParams(), + resource.isDirectory())); + // if isDirectory is true, fullName is the new prefix. we replace old prefix + // of resource fullname with the new prefix. + // if isDirectory is false, fullName is the new path. + taskDefinition.setTaskParams(AddResourceToResourceList(originFullName, + fullName, + existResource.getFullName(), + taskDefinition.getTaskParams(), + resource.isDirectory())); + } + taskDefinitionLogList.add(taskDefinition); + } + + // update workflow & task definition associated to the resource + if (processDefinition != null) { + processDefinitionService.updateProcessDefinition(loginUser, + processDefinition.getProjectCode(), + processDefinition.getName(), + processDefinition.getCode(), + processDefinition.getDescription(), + processDefinition.getGlobalParams(), + processDefinition.getLocations(), + processDefinition.getTimeout(), + tenantCode, + JSONUtils.toJsonString(taskRelationList.toArray()), + JSONUtils.toJsonString(taskDefinitionLogList.toArray()), + "", + processDefinition.getExecutionType()); + } + } } } - result.setData(resultMap); - } catch (Exception e) { - logger.error(Status.UPDATE_RESOURCE_ERROR.getMsg(), e); - throw new ServiceException(Status.UPDATE_RESOURCE_ERROR); - } - - // if name unchanged, return directly without moving on HDFS - if (originResourceName.equals(name) && file == null) { - return result; } if (file != null) { // fail upload if (!upload(loginUser, fullName, file, type)) { - logger.error("Storage operation error, resourceId:{}, resourceName:{}, originFileName:{}.", - resourceId, name, RegexUtils.escapeNRT(file.getOriginalFilename())); + logger.error("Storage operation error, resourceName:{}, originFileName:{}.", + name, RegexUtils.escapeNRT(file.getOriginalFilename())); putMsg(result, Status.HDFS_OPERATION_ERROR); throw new ServiceException( String.format("upload resource: %s file: %s failed.", name, file.getOriginalFilename())); } if (!fullName.equals(originFullName)) { try { - storageOperate.delete(tenantCode, originFileName, false); + storageOperate.delete(originFullName, false); } catch (IOException e) { logger.error("Resource delete error, resourceFullName:{}.", originFullName, e); throw new ServiceException(String.format("delete resource: %s failed.", originFullName)); } } - updateParentResourceSize(resource, resource.getSize() - originFileSize); return result; } // get the path of dest file in hdfs - String destHdfsFileName = storageOperate.getFileName(resource.getType(), tenantCode, fullName); - + String destHdfsFileName = fullName; try { - logger.info("Start copy file {} -> {}.", originFileName, destHdfsFileName); - storageOperate.copy(originFileName, destHdfsFileName, true, true); + logger.info("start copy {} -> {}", originFullName, destHdfsFileName); + storageOperate.copy(originFullName, destHdfsFileName, true, true); } catch (Exception e) { - logger.error(MessageFormat.format("Copy file {0} -> {1} fail.", originFileName, destHdfsFileName), e); + logger.error(MessageFormat.format(" copy {0} -> {1} fail", originFullName, destHdfsFileName), e); putMsg(result, Status.HDFS_COPY_FAIL); - throw new ServiceException(Status.HDFS_COPY_FAIL); + throw new ServiceException(MessageFormat.format( + Status.HDFS_COPY_FAIL.getMsg(), originFullName, destHdfsFileName)); } return result; @@ -667,6 +668,9 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe * query resources list paging * * @param loginUser login user + * @param fullName resource full name + * @param resTenantCode tenantCode in the request field "resTenantCode" for tenant code owning the resource, + * can be different from the login user in the case of logging in as admin users. * @param type resource type * @param searchVal search value * @param pageNo page number @@ -674,30 +678,91 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe * @return resource list page */ @Override - public Result queryResourceListPaging(User loginUser, int directoryId, ResourceType type, String searchVal, - Integer pageNo, Integer pageSize) { + public Result queryResourceListPaging(User loginUser, String fullName, String resTenantCode, + ResourceType type, String searchVal, Integer pageNo, Integer pageSize) { Result result = new Result<>(); - Page page = new Page<>(pageNo, pageSize); - if (directoryId != -1) { - Resource directory = resourcesMapper.selectById(directoryId); - if (directory == null) { - logger.error("Resource does not exist, resourceId:{}.", directoryId); - putMsg(result, Status.RESOURCE_NOT_EXIST); - return result; - } + PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); + + User user = userMapper.selectById(loginUser.getId()); + if (user == null) { + logger.error("user {} not exists", loginUser.getId()); + putMsg(result, Status.USER_NOT_EXIST, loginUser.getId()); + return result; } - PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); - Set resourcesIds = resourcePermissionCheckService - .userOwnedResourceIdsAcquisition(checkResourceType(type), loginUser.getId(), logger); - if (resourcesIds.isEmpty()) { - result.setData(pageInfo); - putMsg(result, Status.SUCCESS); + + Tenant tenant = tenantMapper.queryById(user.getTenantId()); + if (tenant == null) { + logger.error("tenant not exists"); + putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST); + return result; + } + + String tenantCode = tenant.getTenantCode(); + + if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode)) { + logger.error("current user does not have permission"); + putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); return result; } - IPage resourceIPage = resourcesMapper.queryResourcePaging(page, directoryId, type.ordinal(), - searchVal, new ArrayList<>(resourcesIds)); - pageInfo.setTotal((int) resourceIPage.getTotal()); - pageInfo.setTotalList(resourceIPage.getRecords()); + + String defaultPath = ""; + List resourcesList = new ArrayList<>(); + + if (isAdmin(loginUser) && StringUtils.isBlank(fullName)) { + // list all tenants' resources to admin users in the root directory + List userList = userMapper.selectList(null); + Set visitedTenantEntityCode = new HashSet<>(); + for (User userEntity : userList) { + String tenantEntityCode = tenantMapper.queryById(userEntity.getTenantId()).getTenantCode(); + if (!visitedTenantEntityCode.contains(tenantEntityCode)) { + defaultPath = storageOperate.getResDir(tenantEntityCode); + if (type.equals(ResourceType.UDF)) { + defaultPath = storageOperate.getUdfDir(tenantEntityCode); + } + try { + resourcesList.addAll(storageOperate.listFilesStatus(defaultPath, defaultPath, + tenantEntityCode, type)); + + visitedTenantEntityCode.add(tenantEntityCode); + } catch (Exception e) { + logger.error(e.getMessage() + " Resource path: {}", defaultPath, e); + putMsg(result, Status.RESOURCE_NOT_EXIST); + throw new ServiceException(String.format(e.getMessage() + + " make sure resource path: %s exists in hdfs", defaultPath)); + } + } + } + } else { + defaultPath = storageOperate.getResDir(tenantCode); + if (type.equals(ResourceType.UDF)) { + defaultPath = storageOperate.getUdfDir(tenantCode); + } + + try { + if (StringUtils.isBlank(fullName)) { + resourcesList = storageOperate.listFilesStatus(defaultPath, defaultPath, tenantCode, type); + } else { + resourcesList = storageOperate.listFilesStatus(fullName, defaultPath, tenantCode, type); + } + } catch (Exception e) { + logger.error(e.getMessage() + " Resource path: {}", fullName, e); + putMsg(result, Status.RESOURCE_NOT_EXIST); + throw new ServiceException(String.format(e.getMessage() + + " make sure resource path: %s exists in hdfs", defaultPath)); + } + } + + // remove leading and trailing spaces in searchVal + String trimmedSearchVal = searchVal != null ? searchVal.trim() : ""; + // filter based on trimmed searchVal + List filteredResourceList = resourcesList.stream() + .filter(x -> x.getFileName().matches("(.*)" + trimmedSearchVal + "(.*)")).collect(Collectors.toList()); + // inefficient pagination + List slicedResourcesList = filteredResourceList.stream().skip((long) (pageNo - 1) * pageSize) + .limit(pageSize).collect(Collectors.toList()); + + pageInfo.setTotal(resourcesList.size()); + pageInfo.setTotalList(slicedResourcesList); result.setData(pageInfo); putMsg(result, Status.SUCCESS); return result; @@ -714,22 +779,22 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe */ private void createDirectory(User loginUser, String fullName, ResourceType type, Result result) { String tenantCode = tenantMapper.queryById(loginUser.getTenantId()).getTenantCode(); - String directoryName = storageOperate.getFileName(type, tenantCode, fullName); + // String directoryName = storageOperate.getFileName(type, tenantCode, fullName); String resourceRootPath = storageOperate.getDir(type, tenantCode); try { - if (!storageOperate.exists(tenantCode, resourceRootPath)) { + if (!storageOperate.exists(resourceRootPath)) { storageOperate.createTenantDirIfNotExists(tenantCode); } - if (!storageOperate.mkdir(tenantCode, directoryName)) { - logger.error("create resource directory {} failed", directoryName); + if (!storageOperate.mkdir(tenantCode, fullName)) { + logger.error("create resource directory {} failed", fullName); putMsg(result, Status.STORE_OPERATE_CREATE_ERROR); - throw new ServiceException(String.format("create resource directory: %s failed.", directoryName)); + // throw new ServiceException(String.format("create resource directory: %s failed.", fullName)); } } catch (Exception e) { - logger.error("create resource directory {} failed", directoryName); + logger.error("create resource directory {} failed", fullName); putMsg(result, Status.STORE_OPERATE_CREATE_ERROR); - throw new ServiceException(String.format("create resource directory: %s failed.", directoryName)); + throw new ServiceException(String.format("create resource directory: %s failed.", fullName)); } } @@ -755,15 +820,14 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe String localFilename = FileUtils.getUploadFilename(tenantCode, UUID.randomUUID().toString()); // save file to hdfs, and delete original file - String fileName = storageOperate.getFileName(type, tenantCode, fullName); String resourcePath = storageOperate.getDir(type, tenantCode); try { // if tenant dir not exists - if (!storageOperate.exists(tenantCode, resourcePath)) { + if (!storageOperate.exists(resourcePath)) { storageOperate.createTenantDirIfNotExists(tenantCode); } org.apache.dolphinscheduler.api.utils.FileUtils.copyInputStreamToFile(file, localFilename); - storageOperate.upload(tenantCode, localFilename, fileName, true, true); + storageOperate.upload(tenantCode, localFilename, fullName, true, true); } catch (Exception e) { FileUtils.deleteFile(localFilename); logger.error(e.getMessage(), e); @@ -777,15 +841,67 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe * * @param loginUser login user * @param type resource type + * @param fullName resource full name * @return resource list */ @Override - public Map queryResourceList(User loginUser, ResourceType type) { + public Map queryResourceList(User loginUser, ResourceType type, String fullName) { Map result = new HashMap<>(); - List allResourceList = queryAuthoredResourceList(loginUser, type); - Visitor resourceTreeVisitor = new ResourceTreeVisitor(allResourceList); - result.put(Constants.DATA_LIST, resourceTreeVisitor.visit().getChildren()); + User user = userMapper.selectById(loginUser.getId()); + if (user == null) { + logger.error("user {} not exists", loginUser.getId()); + putMsg(result, Status.USER_NOT_EXIST, loginUser.getId()); + return null; + } + + Tenant tenant = tenantMapper.queryById(user.getTenantId()); + if (tenant == null) { + logger.error("tenant not exists"); + putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST); + return null; + } + String tenantCode = tenant.getTenantCode(); + + String defaultPath = ""; + List resourcesList = new ArrayList<>(); + + if (StringUtils.isBlank(fullName)) { + if (isAdmin(loginUser)) { + List userList = userMapper.selectList(null); + Set visitedTenantEntityCode = new HashSet<>(); + for (User userEntity : userList) { + Tenant tt = tenantMapper.queryById(userEntity.getTenantId()); + String tenantEntityCode = tenantMapper.queryById(userEntity.getTenantId()).getTenantCode(); + if (!visitedTenantEntityCode.contains(tenantEntityCode)) { + defaultPath = storageOperate.getResDir(tenantEntityCode); + if (type.equals(ResourceType.UDF)) { + defaultPath = storageOperate.getUdfDir(tenantEntityCode); + } + resourcesList.addAll(storageOperate.listFilesStatusRecursively(defaultPath, defaultPath, + tenantEntityCode, type)); + visitedTenantEntityCode.add(tenantEntityCode); + } + } + } else { + defaultPath = storageOperate.getResDir(tenantCode); + if (type.equals(ResourceType.UDF)) { + defaultPath = storageOperate.getUdfDir(tenantCode); + } + + resourcesList = storageOperate.listFilesStatusRecursively(defaultPath, defaultPath, tenantCode, type); + } + } else { + defaultPath = storageOperate.getResDir(tenantCode); + if (type.equals(ResourceType.UDF)) { + defaultPath = storageOperate.getUdfDir(tenantCode); + } + + resourcesList = storageOperate.listFilesStatusRecursively(fullName, defaultPath, tenantCode, type); + } + + Visitor resourceTreeVisitor = new ResourceTreeVisitor(resourcesList); + result.put(Constants.DATA_LIST, resourceTreeVisitor.visit(defaultPath).getChildren()); putMsg(result, Status.SUCCESS); return result; @@ -824,121 +940,264 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe } } List resources = new ResourceFilter(suffix, new ArrayList<>(allResourceList)).filter(); - Visitor resourceTreeVisitor = new ResourceTreeVisitor(resources); - result.setData(resourceTreeVisitor.visit().getChildren()); + // Transform into StorageEntity for compatibility + List transformedResourceList = resources.stream() + .map(this::createStorageEntityBasedOnResource) + .collect(Collectors.toList()); + Visitor visitor = new ResourceTreeVisitor(transformedResourceList); + result.setData(visitor.visit("").getChildren()); putMsg(result, Status.SUCCESS); return result; } + /** + * transform resource object into StorageEntity object + * + * @param resource a resource object + * @return a storageEntity object + */ + private StorageEntity createStorageEntityBasedOnResource(Resource resource) { + StorageEntity entity = new StorageEntity(); + entity.setFullName(resource.getFullName()); + entity.setPfullName(resource.getPid() == -1 ? "" + : resourcesMapper.selectById(resource.getPid()).getFullName()); + entity.setDirectory(resource.isDirectory()); + entity.setAlias(resource.getAlias()); + entity.setId(resource.getId()); + entity.setType(resource.getType()); + entity.setDescription(resource.getDescription()); + + return entity; + } + /** * delete resource * * @param loginUser login user - * @param resourceId resource id + * @param fullName resource full name + * @param resTenantCode tenantCode in the request field "resTenantCode" for tenant code owning the resource, + * can be different from the login user in the case of logging in as admin users. * @return delete result code * @throws IOException exception */ @Override - @Transactional - public Result delete(User loginUser, int resourceId) throws IOException { - // get resource by id - Result resultCheck = new Result<>(); - Resource resource = resourcesMapper.selectById(resourceId); - if (resource == null) { - logger.error("Resource does not exist, resourceId:{}.", resourceId); - putMsg(resultCheck, Status.RESOURCE_NOT_EXIST); - return resultCheck; - } - String funcPermissionKey = - resource.getType().equals(ResourceType.FILE) ? ApiFuncIdentificationConstant.FILE_DELETE - : ApiFuncIdentificationConstant.UDF_DELETE; - boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{resourceId}, - checkResourceType(resource.getType()), funcPermissionKey); - if (!canOperatorPermissions) { - putMsg(resultCheck, Status.NO_CURRENT_OPERATING_PERMISSION); - return resultCheck; - } + @Transactional(rollbackFor = Exception.class) + public Result delete(User loginUser, String fullName, + String resTenantCode) throws IOException { + Result result = new Result<>(); - Result result = checkResourceUploadStartupState(); + result = checkResourceUploadStartupState(); if (!result.getCode().equals(Status.SUCCESS.getCode())) { return result; } - if (!canOperator(loginUser, resource.getUserId())) { - putMsg(result, Status.USER_NO_OPERATION_PERM); + + User user = userMapper.selectById(loginUser.getId()); + if (user == null) { + logger.error("user {} not exists", loginUser.getId()); + putMsg(result, Status.USER_NOT_EXIST, loginUser.getId()); + return result; + } + + Tenant tenant = tenantMapper.queryById(user.getTenantId()); + if (tenant == null) { + logger.error("tenant not exists"); + putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST); return result; } - String tenantCode = getTenantCode(resource.getUserId(), result); - if (StringUtils.isEmpty(tenantCode)) { - logger.error("Tenant of current login user does not specified, loginUserName:{}.", loginUser.getUserName()); + String tenantCode = tenant.getTenantCode(); + + if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode)) { + logger.error("current user does not have permission"); + putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); return result; } - // get all resource id of process definitions those are released - List> list = processDefinitionMapper.listResources(); - Map> resourceProcessMap = - ResourceProcessDefinitionUtils.getResourceProcessDefinitionMap(list); - Set resourceIdSet = resourceProcessMap.keySet(); - // get all children of the resource - List allChildren = listAllChildren(resource, true); + String defaultPath = storageOperate.getResDir(tenantCode); + StorageEntity resource; + try { + resource = storageOperate.getFileStatus(fullName, defaultPath, resTenantCode, null); + } catch (Exception e) { + logger.error(e.getMessage() + " Resource path: {}", fullName, e); + putMsg(result, Status.RESOURCE_NOT_EXIST); + throw new ServiceException(String.format(e.getMessage() + " Resource path: %s", fullName)); + } - Integer[] needDeleteResourceIdArray = allChildren.toArray(new Integer[allChildren.size()]); - if (needDeleteResourceIdArray.length >= 2) { - logger.warn("Resource can not be deleted because there are files or folders {} in the current directory.", - resource.getFileName()); - putMsg(result, Status.RESOURCE_HAS_FOLDER, resource.getFileName()); + if (resource == null) { + logger.error("Resource does not exist, resource full name:{}.", fullName); + putMsg(result, Status.RESOURCE_NOT_EXIST); return result; } + // recursively delete a folder + List allChildren = storageOperate.listFilesStatusRecursively(fullName, defaultPath, + resTenantCode, resource.getType()).stream().map(storageEntity -> storageEntity.getFullName()) + .collect(Collectors.toList()); + + Set resourcesNeedToDeleteSet = new HashSet<>(); + String[] allChildrenFullNameArray = allChildren.stream().toArray(String[]::new); + + // check before using allChildrenFullNameArray to query full names. + if (allChildrenFullNameArray.length != 0) { + resourcesNeedToDeleteSet.addAll( + resourceTaskMapper.selectBatchFullNames(allChildrenFullNameArray)); + } + // if resource type is UDF,need check whether it is bound by UDF function if (resource.getType() == (ResourceType.UDF)) { - List udfFuncs = udfFunctionMapper.listUdfByResourceId(needDeleteResourceIdArray); - List udfFuncIds = udfFuncs.stream().map(UdfFunc::getId).collect(Collectors.toList()); + List udfFuncs = udfFunctionMapper.listUdfByResourceFullName(allChildrenFullNameArray); if (CollectionUtils.isNotEmpty(udfFuncs)) { logger.warn("Resource can not be deleted because it is bound by UDF functions, udfFuncIds:{}", - udfFuncIds); + udfFuncs); putMsg(result, Status.UDF_RESOURCE_IS_BOUND, udfFuncs.get(0).getFuncName()); return result; } } - if (resourceIdSet.contains(resource.getPid())) { - logger.warn( - "Resource can not be deleted because it is used by process definition, resourceId:{}, processDefinitionCode:{}.", - resource.getId(), resource.getPid()); - putMsg(result, Status.RESOURCE_IS_USED); - return result; - } - resourceIdSet.retainAll(allChildren); - if (CollectionUtils.isNotEmpty(resourceIdSet)) { - for (Integer resId : resourceIdSet) { - logger.warn( - "Resource can not be deleted because it is used by process definition, resourceId:{}, processDefinitionCode:{}.", - resId, resourceProcessMap.get(resId)); + // delete data in database + if (resourcesNeedToDeleteSet.size() > 0) { + for (ResourcesTask resourcesTask : resourcesNeedToDeleteSet) { + int taskId = resourcesTask.getTaskId(); + if (processService.isTaskOnline(taskDefinitionMapper.selectById(taskId).getCode())) { + logger.error("can't be deleted,because it is used of process definition that's online"); + logger.error("resource task relation id:{} is used of task code {}", resourcesTask.getId(), + taskDefinitionMapper.selectById(taskId).getCode()); + putMsg(result, Status.RESOURCE_IS_USED); + return result; + } } - putMsg(result, Status.RESOURCE_IS_USED); - return result; - } - // get hdfs file by type - String storageFilename = storageOperate.getFileName(resource.getType(), tenantCode, resource.getFullName()); - // delete data in database - resourcesMapper.selectBatchIds(Arrays.asList(needDeleteResourceIdArray)).forEach(item -> { - updateParentResourceSize(item, item.getSize() * -1); - }); - resourcesMapper.deleteIds(needDeleteResourceIdArray); - resourceUserMapper.deleteResourceUserArray(0, needDeleteResourceIdArray); + for (ResourcesTask existResource : resourcesNeedToDeleteSet) { + int taskId = existResource.getTaskId(); + long taskCode = taskDefinitionMapper.selectById(taskId).getCode(); + + // use taskCode to get processDefinitionCode, then get a list of processDefinitionLog. + List processTaskRelation = processTaskRelationMapper.selectByMap( + Collections.singletonMap("post_task_code", taskCode)); + if (processTaskRelation.size() > 0) { + long processDefinitionCode = processTaskRelation.get(0).getProcessDefinitionCode(); + int processDefinitionVersion = processTaskRelation.get(0).getProcessDefinitionVersion(); + List taskRelationList = processTaskRelationMapper.queryByProcessCode( + processTaskRelation.get(0).getProjectCode(), + processDefinitionCode); + + List taskDefinitionLogList = new ArrayList<>(); + + if (taskRelationList.size() > 0) { + ProcessDefinitionLog processDefinition = + processDefinitionLogMapper.queryByDefinitionCodeAndVersion( + processDefinitionCode, processDefinitionVersion); + for (ProcessTaskRelation taskRelation : taskRelationList) { + long taskCodeInProcess = taskRelation.getPostTaskCode(); + TaskDefinition taskDefinition = taskDefinitionMapper.queryByCode(taskCodeInProcess); + if (taskCodeInProcess == taskCode) { + taskDefinition.setTaskParams(RemoveResourceFromResourceList(existResource.getFullName(), + taskDefinition.getTaskParams(), false)); + } + taskDefinitionLogList.add(taskDefinition); + } + + // update workflow & task definition associated to the resource + if (processDefinition != null) { + processDefinitionService.updateProcessDefinition(loginUser, + processDefinition.getProjectCode(), + processDefinition.getName(), + processDefinition.getCode(), + processDefinition.getDescription(), + processDefinition.getGlobalParams(), + processDefinition.getLocations(), + processDefinition.getTimeout(), + tenantCode, + JSONUtils.toJsonString(taskRelationList.toArray()), + JSONUtils.toJsonString(taskDefinitionLogList.toArray()), + "", + processDefinition.getExecutionType()); + } + } + } + } + } - // delete file on hdfs + // delete file on hdfs,S3 + storageOperate.delete(fullName, allChildren, true); - // delete file on storage - storageOperate.delete(tenantCode, storageFilename, true); - logger.info("Resource delete complete, tenantCode:{}, fileName:{}.", tenantCode, storageFilename); putMsg(result, Status.SUCCESS); return result; } + private String RemoveResourceFromResourceList(String stringToDelete, String taskParameter, boolean isDir) { + Map taskParameters = JSONUtils.parseObject( + taskParameter, + new TypeReference>() { + }); + if (taskParameters.containsKey("resourceList")) { + String resourceListStr = JSONUtils.toJsonString(taskParameters.get("resourceList")); + List resourceInfoList = JSONUtils.toList(resourceListStr, ResourceInfo.class); + List updatedResourceInfoList; + if (isDir) { + String stringToDeleteWSeparator = stringToDelete + FOLDER_SEPARATOR; + // use start with to identify any prefix matching folder path + updatedResourceInfoList = resourceInfoList.stream() + .filter(Objects::nonNull) + .filter(resourceInfo -> !resourceInfo.getResourceName().startsWith(stringToDeleteWSeparator)) + .collect(Collectors.toList()); + } else { + updatedResourceInfoList = resourceInfoList.stream() + .filter(Objects::nonNull) + .filter(resourceInfo -> !resourceInfo.getResourceName().equals(stringToDelete)) + .collect(Collectors.toList()); + } + taskParameters.put("resourceList", updatedResourceInfoList); + return JSONUtils.toJsonString(taskParameters); + } + return taskParameter; + } + + private String AddResourceToResourceList(String oldPrefix, String newPrefix, String resFullName, + String taskParameter, boolean isDir) { + Map taskParameters = JSONUtils.parseObject( + taskParameter, + new TypeReference>() { + }); + + if (taskParameters.containsKey("resourceList")) { + String resourceListStr = JSONUtils.toJsonString(taskParameters.get("resourceList")); + List resourceInfos = JSONUtils.toList(resourceListStr, ResourceInfo.class); + + // add updated resource to replace the original resource. + ResourceInfo newResource = new ResourceInfo(); + if (isDir) { + // we add spearator here because we dont want rare cases like + // oldFullName: .../folderToDelete and a resource path: .../folderToDeleteAnotherFolder + // Therefore, we make sure the oldFullName has a format of .../folderToDelete/ when + // modifying resourceFullNames in taskDefinition. + String oldFullNameWSeparator = oldPrefix + FOLDER_SEPARATOR; + String newFullNameWSpearator = newPrefix + FOLDER_SEPARATOR; + + newResource.setResourceName(resFullName.replace(oldFullNameWSeparator, newFullNameWSpearator)); + } else { + newResource.setResourceName(newPrefix); + } + resourceInfos.add(newResource); + + taskParameters.put("resourceList", resourceInfos); + + return JSONUtils.toJsonString(taskParameters); + } + return taskParameter; + } + + private String RemoveResourceFromIdsNew(int idToDelete, String idNews) { + + String[] resourceIds = idNews.split(","); + Set resourceIdSet = Arrays.stream(resourceIds) + .map(Integer::parseInt) + .filter(integerId -> !integerId.equals(idToDelete)) + .collect(Collectors.toSet()); + return Joiner.on(",").join(resourceIdSet); + } + /** * verify resource by name and type * @@ -950,43 +1209,11 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe @Override public Result verifyResourceName(String fullName, ResourceType type, User loginUser) { Result result = new Result<>(); - String funcPermissionKey = type.equals(ResourceType.FILE) ? ApiFuncIdentificationConstant.FILE_RENAME - : ApiFuncIdentificationConstant.UDF_FILE_VIEW; - boolean canOperatorPermissions = - canOperatorPermissions(loginUser, null, AuthorizationType.RESOURCE_FILE_ID, funcPermissionKey); - if (!canOperatorPermissions) { - putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); - return result; - } putMsg(result, Status.SUCCESS); - if (checkResourceExists(fullName, type.ordinal())) { - logger.warn("Resource with same name exists so can not create again, resourceType:{}, resourceName:{}.", + if (checkResourceExists(fullName)) { + logger.error("Resource with same name exists so can not create again, resourceType:{}, resourceName:{}.", type, RegexUtils.escapeNRT(fullName)); putMsg(result, Status.RESOURCE_EXIST); - } else { - // query tenant - Tenant tenant = tenantMapper.queryById(loginUser.getTenantId()); - if (tenant != null) { - String tenantCode = tenant.getTenantCode(); - try { - String filename = storageOperate.getFileName(type, tenantCode, fullName); - if (storageOperate.exists(tenantCode, filename)) { - logger.warn( - "Resource file with same name exists so can not create again, tenantCode:{}, resourceName:{}.", - tenantCode, RegexUtils.escapeNRT(filename)); - putMsg(result, Status.RESOURCE_FILE_EXIST, filename); - } - - } catch (Exception e) { - logger.error("Verify resource name failed, resourceName:{}.", RegexUtils.escapeNRT(fullName), e); - putMsg(result, Status.STORE_OPERATE_CREATE_ERROR); - } - } else { - logger.error( - "Tenant does not exist or tenant of current login user does not specified, loginUserName:{}.", - loginUser.getUserName()); - putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST); - } } return result; @@ -995,158 +1222,189 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe /** * verify resource by full name or pid and type * - * @param fullName resource full name - * @param id resource id + * @param fileName resource file name * @param type resource type + * @param resTenantCode tenantCode in the request field "resTenantCode" for tenant code owning the resource, + * can be different from the login user in the case of logging in as admin users. * @return true if the resource full name or pid not exists, otherwise return false */ @Override - public Result queryResource(User loginUser, String fullName, Integer id, ResourceType type) { + public Result queryResourceByFileName(User loginUser, String fileName, ResourceType type, + String resTenantCode) { Result result = new Result<>(); - if (StringUtils.isBlank(fullName) && id == null) { - logger.warn("Parameter fullName and id is invalid."); + if (StringUtils.isBlank(fileName)) { putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR); return result; } - Resource resource; - if (StringUtils.isNotBlank(fullName)) { - List resourceList = resourcesMapper.queryResource(fullName, type.ordinal()); - if (CollectionUtils.isEmpty(resourceList)) { - logger.error("Resources do not exist, fullName:{}.", - RegexUtils.escapeNRT(fullName)); - putMsg(result, Status.RESOURCE_NOT_EXIST); - return result; - } - resource = resourceList.get(0); - } else { - resource = resourcesMapper.selectById(id); - if (resource == null) { - logger.error("Resource does not exist, resourceId:{}.", id); - putMsg(result, Status.RESOURCE_NOT_EXIST); - return result; - } - int pid = resource.getPid(); - resource = resourcesMapper.selectById(pid); - if (resource == null) { - logger.error("Resource does not exist, resourceId:{}.", pid); - putMsg(result, Status.RESOURCE_NOT_EXIST); - return result; - } + + User user = userMapper.selectById(loginUser.getId()); + if (user == null) { + logger.error("user {} not exists", loginUser.getId()); + putMsg(result, Status.USER_NOT_EXIST, loginUser.getId()); + return result; + } + + Tenant tenant = tenantMapper.queryById(user.getTenantId()); + if (tenant == null) { + logger.error("tenant not exists"); + putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST); + return result; } - String funcPermissionKey = type.equals(ResourceType.FILE) ? ApiFuncIdentificationConstant.FILE_VIEW - : ApiFuncIdentificationConstant.UDF_FILE_VIEW; - boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{resource.getId()}, - checkResourceType(type), funcPermissionKey); - if (!canOperatorPermissions) { + + String tenantCode = tenant.getTenantCode(); + + if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode)) { + logger.error("current user does not have permission"); putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); return result; } + + String defaultPath = storageOperate.getResDir(resTenantCode); + if (type.equals(ResourceType.UDF)) { + defaultPath = storageOperate.getUdfDir(resTenantCode); + } + + StorageEntity file; + try { + file = storageOperate.getFileStatus(defaultPath + fileName, defaultPath, resTenantCode, type); + } catch (Exception e) { + logger.error(e.getMessage() + " Resource path: {}", defaultPath + fileName, e); + putMsg(result, Status.RESOURCE_NOT_EXIST); + return result; + } + putMsg(result, Status.SUCCESS); - result.setData(resource); + result.setData(file); return result; } /** * get resource by id - * @param id resource id + * @param fullName resource full name + * @param resTenantCode tenantCode in the request field "resTenantCode" for tenant code owning the resource, + * can be different from the login user in the case of logging in as admin users. * @return resource */ @Override - public Result queryResourceById(User loginUser, Integer id) { + public Result queryResourceByFullName(User loginUser, String fullName, String resTenantCode, + ResourceType type) throws IOException { Result result = new Result<>(); - Resource resource = resourcesMapper.selectById(id); - if (resource == null) { - logger.error("Resource does not exist, resourceId:{}.", id); - putMsg(result, Status.RESOURCE_NOT_EXIST); + + User user = userMapper.selectById(loginUser.getId()); + if (user == null) { + logger.error("user {} not exists", loginUser.getId()); + putMsg(result, Status.USER_NOT_EXIST, loginUser.getId()); return result; } - String funcPermissionKey = - resource.getType().equals(ResourceType.FILE) ? ApiFuncIdentificationConstant.FILE_VIEW - : ApiFuncIdentificationConstant.UDF_FILE_VIEW; - boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{id}, - checkResourceType(resource.getType()), funcPermissionKey); - if (!canOperatorPermissions) { + + Tenant tenant = tenantMapper.queryById(user.getTenantId()); + if (tenant == null) { + logger.error("tenant not exists"); + putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST); + return result; + } + + String tenantCode = tenant.getTenantCode(); + + if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode)) { + logger.error("current user does not have permission"); putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); return result; } + + String defaultPath = storageOperate.getResDir(resTenantCode); + if (type.equals(ResourceType.UDF)) { + defaultPath = storageOperate.getUdfDir(resTenantCode); + } + + StorageEntity file; + try { + file = storageOperate.getFileStatus(fullName, defaultPath, resTenantCode, type); + } catch (Exception e) { + logger.error(e.getMessage() + " Resource path: {}", fullName, e); + putMsg(result, Status.RESOURCE_NOT_EXIST); + throw new ServiceException(String.format(e.getMessage() + " Resource path: %s", fullName)); + } + putMsg(result, Status.SUCCESS); - result.setData(resource); + result.setData(file); return result; } /** * view resource file online * - * @param resourceId resource id + * @param fullName resource fullName + * @param resTenantCode owner's tenant code of the resource * @param skipLineNum skip line number * @param limit limit * @return resource content */ @Override - public Result readResource(User loginUser, int resourceId, int skipLineNum, int limit) { + public Result readResource(User loginUser, String fullName, String resTenantCode, + int skipLineNum, int limit) { Result result = checkResourceUploadStartupState(); if (!result.getCode().equals(Status.SUCCESS.getCode())) { return result; } - // get resource by id - Resource resource = resourcesMapper.selectById(resourceId); - if (resource == null) { - logger.error("Resource does not exist, resourceId:{}.", resourceId); - putMsg(result, Status.RESOURCE_NOT_EXIST); + + User user = userMapper.selectById(loginUser.getId()); + if (user == null) { + logger.error("user {} not exists", loginUser.getId()); + putMsg(result, Status.USER_NOT_EXIST, loginUser.getId()); + return result; + } + + Tenant tenant = tenantMapper.queryById(user.getTenantId()); + if (tenant == null) { + logger.error("tenant not exists"); + putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST); return result; } - String funcPermissionKey = - resource.getType().equals(ResourceType.FILE) ? ApiFuncIdentificationConstant.FILE_VIEW - : ApiFuncIdentificationConstant.UDF_FILE_VIEW; - boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{resourceId}, - checkResourceType(resource.getType()), funcPermissionKey); - if (!canOperatorPermissions) { + + String tenantCode = tenant.getTenantCode(); + + if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode)) { + logger.error("current user does not have permission"); putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); return result; } + // check preview or not by file suffix - String nameSuffix = Files.getFileExtension(resource.getAlias()); + String nameSuffix = Files.getFileExtension(fullName); String resourceViewSuffixes = FileUtils.getResourceViewSuffixes(); if (StringUtils.isNotEmpty(resourceViewSuffixes)) { List strList = Arrays.asList(resourceViewSuffixes.split(",")); if (!strList.contains(nameSuffix)) { - logger.warn("Resource suffix does not support view, resourceId:{}, suffix:{}.", resourceId, nameSuffix); + logger.error("Resource suffix does not support view,resourceFullName:{}, suffix:{}.", fullName, + nameSuffix); putMsg(result, Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW); return result; } } - String tenantCode = getTenantCode(resource.getUserId(), result); - if (StringUtils.isEmpty(tenantCode)) { - logger.error("Tenant of resource user does not specified, resourceUserIde:{}.", resource.getUserId()); - return result; - } - - // source path - String resourceFileName = storageOperate.getResourceFileName(tenantCode, resource.getFullName()); - logger.info("Resource file path is {}.", resourceFileName); + List content = new ArrayList<>(); try { - if (storageOperate.exists(tenantCode, resourceFileName)) { - List content = storageOperate.vimFile(tenantCode, resourceFileName, skipLineNum, limit); - - logger.info( - "Vim file content in path {} success, tenantCode:{}, fileName:{}, skipLineNum:{}, limit:{}.", - resourceFileName, tenantCode, resourceFileName, skipLineNum, limit); - putMsg(result, Status.SUCCESS); - Map map = new HashMap<>(); - map.put(ALIAS, resource.getAlias()); - map.put(CONTENT, String.join("\n", content)); - result.setData(map); + if (storageOperate.exists(fullName)) { + content = storageOperate.vimFile(tenantCode, fullName, skipLineNum, limit); } else { - logger.error("File does not exist in storage, fileName:{}.", resourceFileName); - putMsg(result, Status.RESOURCE_FILE_NOT_EXIST, resourceFileName); + logger.error("read file {} not exist in storage", fullName); + putMsg(result, Status.RESOURCE_FILE_NOT_EXIST, fullName); + return result; } } catch (Exception e) { - logger.error("Resource file read error, fileName:{}.", resourceFileName, e); + logger.error("Resource {} read failed", fullName, e); putMsg(result, Status.HDFS_OPERATION_ERROR); + return result; } + putMsg(result, Status.SUCCESS); + Map map = new HashMap<>(); + map.put(ALIAS, fullName); + map.put(CONTENT, String.join("\n", content)); + result.setData(map); + return result; } @@ -1159,36 +1417,47 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe * @param fileSuffix file suffix * @param desc description * @param content content - * @param pid pid * @param currentDir current directory * @return create result code */ @Override @Transactional public Result onlineCreateResource(User loginUser, ResourceType type, String fileName, String fileSuffix, - String desc, String content, int pid, String currentDir) { + String desc, String content, String currentDir) { Result result = new Result<>(); - boolean canOperatorPermissions = canOperatorPermissions(loginUser, null, AuthorizationType.RESOURCE_FILE_ID, - ApiFuncIdentificationConstant.FILE_ONLINE_CREATE); - if (!canOperatorPermissions) { - putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); - return result; - } result = checkResourceUploadStartupState(); if (!result.getCode().equals(Status.SUCCESS.getCode())) { return result; } + + User user = userMapper.selectById(loginUser.getId()); + if (user == null) { + logger.error("user {} not exists", loginUser.getId()); + putMsg(result, Status.USER_NOT_EXIST, loginUser.getId()); + return result; + } + + Tenant tenant = tenantMapper.queryById(user.getTenantId()); + if (tenant == null) { + logger.error("tenant not exists"); + putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST); + return result; + } + + String tenantCode = tenant.getTenantCode(); + + if (!isUserTenantValid(isAdmin(loginUser), tenantCode, "")) { + logger.error("current user does not have permission"); + putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); + return result; + } + if (FileUtils.directoryTraversal(fileName)) { logger.warn("File name verify failed, fileName:{}.", RegexUtils.escapeNRT(fileName)); putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED); return result; } - if (checkDescriptionLength(desc)) { - logger.warn("Parameter description is too long."); - putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR); - return result; - } // check file suffix String nameSuffix = fileSuffix.trim(); @@ -1203,31 +1472,19 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe } String name = fileName.trim() + "." + nameSuffix; - String fullName = getFullName(currentDir, name); - result = verifyResource(loginUser, type, fullName, pid); - if (!result.getCode().equals(Status.SUCCESS.getCode())) { - return result; - } - - // save data - Date now = new Date(); - Resource resource = new Resource(pid, name, fullName, false, desc, name, loginUser.getId(), type, - content.getBytes().length, now, now); - resourcesMapper.insert(resource); - updateParentResourceSize(resource, resource.getSize()); - - putMsg(result, Status.SUCCESS); - permissionPostHandle(resource.getType(), loginUser, resource.getId()); - Map resultMap = new HashMap<>(); - for (Map.Entry entry : new BeanMap(resource).entrySet()) { - if (!Constants.CLASS.equalsIgnoreCase(entry.getKey().toString())) { - resultMap.put(entry.getKey().toString(), entry.getValue()); - } + String fullName = ""; + String userResRootPath = storageOperate.getResDir(tenantCode); + if (!currentDir.contains(userResRootPath)) { + fullName = userResRootPath + name; + } else { + fullName = currentDir + name; } - result.setData(resultMap); - String tenantCode = tenantMapper.queryById(loginUser.getTenantId()).getTenantCode(); + result = verifyResourceName(fullName, type, loginUser); + if (!result.getCode().equals(Status.SUCCESS.getCode())) { + return result; + } result = uploadContentToStorage(loginUser, fullName, tenantCode, content); if (!result.getCode().equals(Status.SUCCESS.getCode())) { @@ -1250,9 +1507,11 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe @Transactional public Result onlineCreateOrUpdateResourceWithDir(User loginUser, String fileFullName, String desc, String content) { - if (checkResourceExists(fileFullName, ResourceType.FILE.ordinal())) { + // TODO: need update to third party service + if (checkResourceExists(fileFullName)) { Resource resource = resourcesMapper.queryResource(fileFullName, ResourceType.FILE.ordinal()).get(0); - Result result = this.updateResourceContent(loginUser, resource.getId(), content); + Result result = this.updateResourceContent(loginUser, fileFullName, + resource.getUserName(), content); if (result.getCode() == Status.SUCCESS.getCode()) { resource.setDescription(desc); Map resultMap = new HashMap<>(); @@ -1279,7 +1538,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe } } return this.onlineCreateResource( - loginUser, ResourceType.FILE, resourceName, resourceSuffix, desc, content, pid, + loginUser, ResourceType.FILE, resourceName, resourceSuffix, desc, content, currDirPath.toString()); } } @@ -1311,7 +1570,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe private int queryOrCreateDirId(User user, int pid, String currentDir, String dirName) { String dirFullName = currentDir + FOLDER_SEPARATOR + dirName; - if (checkResourceExists(dirFullName, ResourceType.FILE.ordinal())) { + if (checkResourceExists(dirFullName)) { List resourceList = resourcesMapper.queryResource(dirFullName, ResourceType.FILE.ordinal()); return resourceList.get(0).getId(); } else { @@ -1319,8 +1578,12 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe Result createDirResult = this.createDirectory( user, dirName, EMPTY_STRING, ResourceType.FILE, pid, currentDir); if (createDirResult.getCode() == Status.SUCCESS.getCode()) { - Map resultMap = (Map) createDirResult.getData(); - return resultMap.get("id") == null ? -1 : (Integer) resultMap.get("id"); + // Map resultMap = (Map) createDirResult.getData(); + // return resultMap.get("id") == null ? -1 : (Integer) resultMap.get("id"); + + // Since resource is kept in third party services, its id will always be -1. + return -1; + } else { String msg = String.format("Create dir error, dirFullName:%s.", dirFullName); logger.error(msg); @@ -1379,76 +1642,89 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe /** * updateProcessInstance resource * - * @param resourceId resource id + * @param fullName resource full name + * @param resTenantCode tenantCode in the request field "resTenantCode" for tenant code owning the resource, + * can be different from the login user in the case of logging in as admin users. * @param content content * @return update result cod */ @Override @Transactional - public Result updateResourceContent(User loginUser, int resourceId, String content) { + public Result updateResourceContent(User loginUser, String fullName, String resTenantCode, + String content) { Result result = checkResourceUploadStartupState(); if (!result.getCode().equals(Status.SUCCESS.getCode())) { return result; } - Resource resource = resourcesMapper.selectById(resourceId); - if (resource == null) { - logger.error("Resource does not exist, resourceId:{}.", resourceId); - putMsg(result, Status.RESOURCE_NOT_EXIST); + User user = userMapper.selectById(loginUser.getId()); + if (user == null) { + logger.error("user {} not exists", loginUser.getId()); + putMsg(result, Status.USER_NOT_EXIST, loginUser.getId()); return result; } - String funcPermissionKey = - resource.getType().equals(ResourceType.FILE) ? ApiFuncIdentificationConstant.FILE_UPDATE - : ApiFuncIdentificationConstant.UDF_UPDATE; - boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{resourceId}, - checkResourceType(resource.getType()), funcPermissionKey); - if (!canOperatorPermissions) { + + Tenant tenant = tenantMapper.queryById(user.getTenantId()); + if (tenant == null) { + logger.error("tenant not exists"); + putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST); + return result; + } + + String tenantCode = tenant.getTenantCode(); + + if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode)) { + logger.error("current user does not have permission"); putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION); return result; } + + StorageEntity resource; + try { + resource = storageOperate.getFileStatus(fullName, "", resTenantCode, ResourceType.FILE); + } catch (Exception e) { + logger.error("error occurred when fetching resource information , resource full name {}", fullName); + putMsg(result, Status.RESOURCE_NOT_EXIST); + return result; + } + + if (resource == null) { + logger.error("Resource does not exist, resource full name:{}.", fullName); + putMsg(result, Status.RESOURCE_NOT_EXIST); + return result; + } + // check can edit by file suffix String nameSuffix = Files.getFileExtension(resource.getAlias()); String resourceViewSuffixes = FileUtils.getResourceViewSuffixes(); if (StringUtils.isNotEmpty(resourceViewSuffixes)) { List strList = Arrays.asList(resourceViewSuffixes.split(",")); if (!strList.contains(nameSuffix)) { - logger.warn("Resource suffix does not support view, resourceId:{}, suffix:{}.", resourceId, nameSuffix); + logger.warn("Resource suffix does not support view, resource full name:{}, suffix:{}.", + fullName, nameSuffix); putMsg(result, Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW); return result; } } - String tenantCode = getTenantCode(resource.getUserId(), result); - if (StringUtils.isEmpty(tenantCode)) { - logger.error("Tenant of resource user does not specified, resourceUserIde:{}.", resource.getUserId()); - return result; - } - long originFileSize = resource.getSize(); - resource.setSize(content.getBytes().length); - resource.setUpdateTime(new Date()); - resourcesMapper.updateById(resource); - - result = uploadContentToStorage(loginUser, resource.getFullName(), tenantCode, content); - updateParentResourceSize(resource, resource.getSize() - originFileSize); + result = uploadContentToStorage(loginUser, resource.getFullName(), resTenantCode, content); if (!result.getCode().equals(Status.SUCCESS.getCode())) { throw new ServiceException(result.getMsg()); } else - logger.info("Update resource content complete, resourceId:{}.", resourceId); + logger.info("Update resource content complete, resource full name:{}.", fullName); return result; } /** - * @param resourceName resource name + * @param fullName resource full name * @param tenantCode tenant code * @param content content * @return result */ - private Result uploadContentToStorage(User loginUser, String resourceName, String tenantCode, - String content) { + private Result uploadContentToStorage(User loginUser, String fullName, String tenantCode, String content) { Result result = new Result<>(); String localFilename = ""; - String storageFileName = ""; try { localFilename = FileUtils.getUploadFilename(tenantCode, UUID.randomUUID().toString()); @@ -1461,42 +1737,40 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe } // get resource file path - storageFileName = storageOperate.getResourceFileName(tenantCode, resourceName); String resourcePath = storageOperate.getResDir(tenantCode); - logger.info("Resource file name is {}, resource dir is {}.", storageFileName, resourcePath); + logger.info("resource path is {}, resource dir is {}", fullName, resourcePath); - if (!storageOperate.exists(tenantCode, resourcePath)) { + if (!storageOperate.exists(resourcePath)) { // create if tenant dir not exists storageOperate.createTenantDirIfNotExists(tenantCode); logger.info("Create tenant dir because path {} does not exist, tenantCode:{}.", resourcePath, tenantCode); } - if (storageOperate.exists(tenantCode, storageFileName)) { - storageOperate.delete(tenantCode, storageFileName, false); + if (storageOperate.exists(fullName)) { + storageOperate.delete(fullName, false); } - storageOperate.upload(tenantCode, localFilename, storageFileName, true, true); + storageOperate.upload(tenantCode, localFilename, fullName, true, true); } catch (Exception e) { - logger.error("Upload content to storage error, tenantCode:{}, destFileName:{}.", tenantCode, - storageFileName, e); + logger.error("Upload content to storage error, tenantCode:{}, destFileName:{}.", tenantCode, localFilename, + e); result.setCode(Status.HDFS_OPERATION_ERROR.getCode()); - result.setMsg(String.format("copy %s to hdfs %s fail", localFilename, storageFileName)); + result.setMsg(String.format("copy %s to hdfs %s fail", localFilename, fullName)); return result; } - logger.info("Upload content to storage complete, tenantCode:{}, destFileName:{}.", tenantCode, storageFileName); + logger.info("Upload content to storage complete, tenantCode:{}, destFileName:{}.", tenantCode, localFilename); putMsg(result, Status.SUCCESS); return result; } /** * download file - * - * @param resourceId resource id * @return resource content * @throws IOException exception */ @Override - public org.springframework.core.io.Resource downloadResource(User loginUser, int resourceId) throws IOException { + public org.springframework.core.io.Resource downloadResource(User loginUser, + String fullName) throws IOException { // if resource upload startup if (!PropertyUtils.getResUploadStartupState()) { logger.warn("Storage does not start up, resource upload startup state: {}.", @@ -1504,59 +1778,42 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe throw new ServiceException("hdfs not startup"); } - Resource resource = resourcesMapper.selectById(resourceId); - if (resource == null) { - logger.error("Resource does not exist, resourceId:{}.", resourceId); - return null; - } - - String funcPermissionKey = - resource.getType().equals(ResourceType.FILE) ? ApiFuncIdentificationConstant.FILE_DOWNLOAD - : ApiFuncIdentificationConstant.UDF_DOWNLOAD; - boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{resourceId}, - checkResourceType(resource.getType()), funcPermissionKey); - if (!canOperatorPermissions) { - logger.error("{}: {}", Status.NO_CURRENT_OPERATING_PERMISSION.getMsg(), - PropertyUtils.getResUploadStartupState()); - throw new ServiceException(Status.NO_CURRENT_OPERATING_PERMISSION.getMsg()); - } - if (resource.isDirectory()) { - logger.warn("Resource is a directory so can not download it, resourceId:{}.", resourceId); + if (fullName.endsWith("/")) { + logger.error("resource id {} is directory,can't download it", fullName); throw new ServiceException("can't download directory"); } - int userId = resource.getUserId(); + int userId = loginUser.getId(); User user = userMapper.selectById(userId); if (user == null) { logger.error("User does not exits, userId:{}.", userId); - throw new ServiceException(String.format("resource owner id %d does not exist", userId)); - } - - Tenant tenant = tenantMapper.queryById(user.getTenantId()); - if (tenant == null) { - logger.error("Tenant does not exists, tenantId:{}.", user.getTenantId()); - throw new ServiceException( - String.format("The tenant id %d of resource owner does not exist", user.getTenantId())); + throw new ServiceException(String.format("Resource owner id %d does not exist", userId)); } - String tenantCode = tenant.getTenantCode(); + String tenantCode = ""; - String fileName = storageOperate.getFileName(resource.getType(), tenantCode, resource.getFullName()); + if (user.getTenantId() != 0) { + Tenant tenant = tenantMapper.queryById(user.getTenantId()); + if (tenant == null) { + logger.error("Tenant id {} not exists", user.getTenantId()); + throw new ServiceException( + String.format("The tenant id %d of resource owner not exist", user.getTenantId())); + } + tenantCode = tenant.getTenantCode(); + } - String localFileName = FileUtils.getDownloadFilename(resource.getAlias()); - logger.info("Resource path is {}, download local filename is {}.", fileName, localFileName); + String[] aliasArr = fullName.split("/"); + String alias = aliasArr[aliasArr.length - 1]; + String localFileName = FileUtils.getDownloadFilename(alias); + logger.info("Resource path is {}, download local filename is {}", alias, localFileName); try { - storageOperate.download(tenantCode, fileName, localFileName, false, true); - org.springframework.core.io.Resource file2Resource = - org.apache.dolphinscheduler.api.utils.FileUtils.file2Resource(localFileName); - if (file2Resource != null) { - logger.info("Download resource complete, path:{}, localFileName:{}.", fileName, localFileName); - } - return file2Resource; + storageOperate.download(tenantCode, fullName, localFileName, false, true); + return org.apache.dolphinscheduler.api.utils.FileUtils.file2Resource(localFileName); } catch (IOException e) { - logger.error("Download resource error, path:{}, localFileName:{}.", fileName, localFileName, e); - throw new ServerException("download the resource file failed ,it may be related to your storage"); + logger.error("Download resource error, the path is {}, and local filename is {}, the error message is {}", + fullName, localFileName, e.getMessage()); + throw new ServerException("Download the resource file failed ,it may be related to your storage"); } } @@ -1585,8 +1842,12 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe } List list; if (CollectionUtils.isNotEmpty(resourceList)) { - Visitor visitor = new ResourceTreeVisitor(resourceList); - list = visitor.visit().getChildren(); + // Transform into StorageEntity for compatibility + List transformedResourceList = resourceList.stream() + .map(this::createStorageEntityBasedOnResource) + .collect(Collectors.toList()); + Visitor visitor = new ResourceTreeVisitor(transformedResourceList); + list = visitor.visit("").getChildren(); } else { list = new ArrayList<>(0); } @@ -1597,12 +1858,13 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe } @Override - public Resource queryResourcesFileInfo(String userName, String fullName) { + public Resource queryResourcesFileInfo(String userName, String fileName) { + // TODO: It is used in PythonGateway, should be revised User user = userMapper.queryByUserNameAccurately(userName); - Result resourceResponse = this.queryResource(user, fullName, null, ResourceType.FILE); + Result resourceResponse = this.queryResourceByFileName(user, fileName, ResourceType.FILE, ""); if (resourceResponse.getCode() != Status.SUCCESS.getCode()) { String msg = - String.format("Query resource by fullName failed, userName:%s, fullName:%s", userName, fullName); + String.format("Query resource by fullName failed, userName:%s, fullName:%s", userName, fileName); logger.error(msg); throw new IllegalArgumentException(msg); } @@ -1637,8 +1899,12 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe } else { list = new ArrayList<>(0); } - Visitor visitor = new ResourceTreeVisitor(list); - result.put(Constants.DATA_LIST, visitor.visit().getChildren()); + // Transform into StorageEntity for compatibility + List transformedResourceList = list.stream() + .map(this::createStorageEntityBasedOnResource) + .collect(Collectors.toList()); + Visitor visitor = new ResourceTreeVisitor(transformedResourceList); + result.put(Constants.DATA_LIST, visitor.visit("").getChildren()); putMsg(result, Status.SUCCESS); return result; } @@ -1717,13 +1983,17 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe } List authedResources = queryResourceList(userId, Constants.AUTHORIZE_WRITABLE_PERM); - Visitor visitor = new ResourceTreeVisitor(authedResources); - String visit = JSONUtils.toJsonString(visitor.visit(), SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS); + // Transform into StorageEntity for compatibility + List transformedResourceList = authedResources.stream() + .map(this::createStorageEntityBasedOnResource) + .collect(Collectors.toList()); + Visitor visitor = new ResourceTreeVisitor(transformedResourceList); + String visit = JSONUtils.toJsonString(visitor.visit(""), SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS); logger.info(visit); String jsonTreeStr = - JSONUtils.toJsonString(visitor.visit().getChildren(), SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS); + JSONUtils.toJsonString(visitor.visit("").getChildren(), SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS); logger.info(jsonTreeStr); - result.put(Constants.DATA_LIST, visitor.visit().getChildren()); + result.put(Constants.DATA_LIST, visitor.visit("").getChildren()); putMsg(result, Status.SUCCESS); return result; } @@ -1742,30 +2012,6 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe } } - /** - * get tenantCode by UserId - * - * @param userId user id - * @param result return result - * @return tenant code - */ - private String getTenantCode(int userId, Result result) { - User user = userMapper.selectById(userId); - if (user == null) { - logger.error("user {} not exists", userId); - putMsg(result, Status.USER_NOT_EXIST, userId); - return null; - } - - Tenant tenant = tenantMapper.queryById(user.getTenantId()); - if (tenant == null) { - logger.error("tenant not exists"); - putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST); - return null; - } - return tenant.getTenantCode(); - } - /** * list all children id * @@ -1832,4 +2078,27 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe private AuthorizationType checkResourceType(ResourceType type) { return type.equals(ResourceType.FILE) ? AuthorizationType.RESOURCE_FILE_ID : AuthorizationType.UDF_FILE; } + + /** + * check permission by comparing login user's tenantCode with tenantCode in the request + * + * @param isAdmin is the login user admin + * @param userTenantCode loginUser's tenantCode + * @param resTenantCode tenantCode in the request field "resTenantCode" for tenant code owning the resource, + * can be different from the login user in the case of logging in as admin users. + * @return isValid + */ + private boolean isUserTenantValid(boolean isAdmin, String userTenantCode, + String resTenantCode) throws ServiceException { + if (!isAdmin) { + resTenantCode = resTenantCode == null ? "" : resTenantCode; + if (!StringUtils.isBlank(resTenantCode) && !resTenantCode.equals(userTenantCode)) { + // if an ordinary user directly send a query API with a different tenantCode and fullName "", + // still he/she does not have read permission. + return false; + } + } + + return true; + } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java index c5e5da007c..a7973eff94 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java @@ -25,15 +25,16 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.common.utils.PropertyUtils; -import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper; import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; +import org.apache.dolphinscheduler.service.storage.StorageOperate; import org.apache.commons.lang3.StringUtils; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.Date; @@ -66,6 +67,9 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic @Autowired private UDFUserMapper udfUserMapper; + @Autowired(required = false) + private StorageOperate storageOperate; + /** * create udf function * @@ -75,7 +79,6 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic * @param argTypes argument types * @param database database * @param desc description - * @param resourceId resource id * @param className class name * @return create result code */ @@ -84,11 +87,11 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic public Result createUdfFunction(User loginUser, String funcName, String className, + String fullName, String argTypes, String database, String desc, - UdfType type, - int resourceId) { + UdfType type) { Result result = new Result<>(); boolean canOperatorPermissions = canOperatorPermissions(loginUser, null, AuthorizationType.UDF, @@ -117,9 +120,15 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic return result; } - Resource resource = resourceMapper.selectById(resourceId); - if (resource == null) { - logger.error("Resource does not exist, resourceId:{}.", resourceId); + Boolean existResource = false; + try { + existResource = storageOperate.exists(fullName); + } catch (IOException e) { + logger.error("Check resource error: {}", fullName, e); + } + + if (!existResource) { + logger.error("resource full name {} is not exist", fullName); putMsg(result, Status.RESOURCE_NOT_EXIST); return result; } @@ -137,8 +146,9 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic udf.setDatabase(database); } udf.setDescription(desc); - udf.setResourceId(resourceId); - udf.setResourceName(resource.getFullName()); + // set resourceId to -1 because we do not store resource to db anymore, instead we use fullName + udf.setResourceId(-1); + udf.setResourceName(fullName); udf.setType(type); udf.setCreateTime(now); @@ -178,7 +188,7 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic } UdfFunc udfFunc = udfFuncMapper.selectById(id); if (udfFunc == null) { - logger.error("Resource does not exist, resourceId:{}.", id); + logger.error("Resource does not exist, udf func id:{}.", id); putMsg(result, Status.RESOURCE_NOT_EXIST); return result; } @@ -196,7 +206,7 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic * @param argTypes argument types * @param database data base * @param desc description - * @param resourceId resource id + * @param fullName resource full name * @param className class name * @return update result code */ @@ -209,7 +219,7 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic String database, String desc, UdfType type, - int resourceId) { + String fullName) { Result result = new Result<>(); boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{udfFuncId}, @@ -251,13 +261,23 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic } } - Resource resource = resourceMapper.selectById(resourceId); - if (resource == null) { - logger.error("Resource does not exist, resourceId:{}.", resourceId); + Boolean doesResExist = false; + try { + doesResExist = storageOperate.exists(fullName); + } catch (Exception e) { + logger.error("udf resource checking error", fullName); result.setCode(Status.RESOURCE_NOT_EXIST.getCode()); result.setMsg(Status.RESOURCE_NOT_EXIST.getMsg()); return result; } + + if (!doesResExist) { + logger.error("resource full name {} is not exist", fullName); + result.setCode(Status.RESOURCE_NOT_EXIST.getCode()); + result.setMsg(Status.RESOURCE_NOT_EXIST.getMsg()); + return result; + } + Date now = new Date(); udf.setFuncName(funcName); udf.setClassName(className); @@ -266,8 +286,9 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic udf.setDatabase(database); } udf.setDescription(desc); - udf.setResourceId(resourceId); - udf.setResourceName(resource.getFullName()); + // set resourceId to -1 because we do not store resource to db anymore, instead we use fullName + udf.setResourceId(-1); + udf.setResourceName(fullName); udf.setType(type); udf.setUpdateTime(now); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java index 611f37bf30..3b65634979 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java @@ -1173,7 +1173,7 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService { if (CollectionUtils.isNotEmpty(components)) { for (ResourceComponent component : components) { // verify whether exist - if (!storageOperate.exists(oldTenantCode, + if (!storageOperate.exists( String.format(Constants.FORMAT_S_S, srcBasePath, component.getFullName()))) { logger.error("Resource file: {} does not exist, copy error.", component.getFullName()); throw new ServiceException(Status.RESOURCE_NOT_EXIST); @@ -1188,8 +1188,8 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService { if (CollectionUtils.isEmpty(component.getChildren())) { // if not exist,need create it - if (!storageOperate.exists(oldTenantCode, - String.format(Constants.FORMAT_S_S, dstBasePath, component.getFullName()))) { + if (!storageOperate + .exists(String.format(Constants.FORMAT_S_S, dstBasePath, component.getFullName()))) { storageOperate.mkdir(newTenantCode, String.format(Constants.FORMAT_S_S, dstBasePath, component.getFullName())); } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java index 8c43deb997..6bfcddcb29 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java @@ -65,11 +65,15 @@ public class ResourcesControllerTest extends AbstractControllerTest { public void testQuerytResourceList() throws Exception { Map mockResult = new HashMap<>(); mockResult.put(Constants.STATUS, Status.SUCCESS); - Mockito.when(resourcesService.queryResourceList(Mockito.any(), Mockito.any())).thenReturn(mockResult); + Mockito.when(resourcesService.queryResourceList(Mockito.any(), Mockito.any(), Mockito.anyString())) + .thenReturn(mockResult); + MultiValueMap paramsMap = new LinkedMultiValueMap<>(); + paramsMap.add("fullName", "dolphinscheduler/resourcePath"); + paramsMap.add("type", ResourceType.FILE.name()); MvcResult mvcResult = mockMvc.perform(get("/resources/list") .header(SESSION_ID, sessionId) - .param("type", ResourceType.FILE.name())) + .params(paramsMap)) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON)) .andReturn(); @@ -85,8 +89,8 @@ public class ResourcesControllerTest extends AbstractControllerTest { Result mockResult = new Result<>(); mockResult.setCode(Status.SUCCESS.getCode()); Mockito.when(resourcesService.queryResourceListPaging( - Mockito.any(), Mockito.anyInt(), Mockito.any(), Mockito.anyString(), Mockito.anyInt(), - Mockito.anyInt())) + Mockito.any(), Mockito.anyString(), Mockito.anyString(), Mockito.any(), + Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt())) .thenReturn(mockResult); MultiValueMap paramsMap = new LinkedMultiValueMap<>(); @@ -95,6 +99,8 @@ public class ResourcesControllerTest extends AbstractControllerTest { paramsMap.add("pageNo", "1"); paramsMap.add("searchVal", "test"); paramsMap.add("pageSize", "1"); + paramsMap.add("fullName", "dolphinscheduler/resourcePath"); + paramsMap.add("tenantCode", "123"); MvcResult mvcResult = mockMvc.perform(get("/resources") .header(SESSION_ID, sessionId) @@ -137,14 +143,17 @@ public class ResourcesControllerTest extends AbstractControllerTest { public void testViewResource() throws Exception { Result mockResult = new Result<>(); mockResult.setCode(Status.HDFS_NOT_STARTUP.getCode()); - Mockito.when(resourcesService.readResource(Mockito.any(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt())) + Mockito.when(resourcesService.readResource(Mockito.any(), + Mockito.anyString(), Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt())) .thenReturn(mockResult); MultiValueMap paramsMap = new LinkedMultiValueMap<>(); paramsMap.add("skipLineNum", "2"); paramsMap.add("limit", "100"); + paramsMap.add("fullName", "dolphinscheduler/resourcePath"); + paramsMap.add("tenantCode", "123"); - MvcResult mvcResult = mockMvc.perform(get("/resources/{id}/view", "5") + MvcResult mvcResult = mockMvc.perform(get("/resources/view") .header(SESSION_ID, sessionId) .params(paramsMap)) .andExpect(status().isOk()) @@ -163,7 +172,7 @@ public class ResourcesControllerTest extends AbstractControllerTest { mockResult.setCode(Status.TENANT_NOT_EXIST.getCode()); Mockito.when(resourcesService .onlineCreateResource(Mockito.any(), Mockito.any(), Mockito.anyString(), Mockito.anyString(), - Mockito.anyString(), Mockito.anyString(), Mockito.anyInt(), Mockito.anyString())) + Mockito.anyString(), Mockito.anyString(), Mockito.anyString())) .thenReturn(mockResult); MultiValueMap paramsMap = new LinkedMultiValueMap<>(); @@ -192,14 +201,17 @@ public class ResourcesControllerTest extends AbstractControllerTest { public void testUpdateResourceContent() throws Exception { Result mockResult = new Result<>(); mockResult.setCode(Status.TENANT_NOT_EXIST.getCode()); - Mockito.when(resourcesService.updateResourceContent(Mockito.any(), Mockito.anyInt(), Mockito.anyString())) + Mockito.when(resourcesService.updateResourceContent(Mockito.any(), Mockito.anyString(), + Mockito.anyString(), Mockito.anyString())) .thenReturn(mockResult); MultiValueMap paramsMap = new LinkedMultiValueMap<>(); paramsMap.add("id", "1"); paramsMap.add("content", "echo test_1111"); + paramsMap.add("fullName", "dolphinscheduler/resourcePath"); + paramsMap.add("tenantCode", "123"); - MvcResult mvcResult = mockMvc.perform(put("/resources/1/update-content") + MvcResult mvcResult = mockMvc.perform(put("/resources/update-content") .header(SESSION_ID, sessionId) .params(paramsMap)) .andExpect(status().isOk()) @@ -215,9 +227,14 @@ public class ResourcesControllerTest extends AbstractControllerTest { @Test public void testDownloadResource() throws Exception { - Mockito.when(resourcesService.downloadResource(Mockito.any(), Mockito.anyInt())).thenReturn(null); + Mockito.when(resourcesService.downloadResource(Mockito.any(), Mockito.anyString())) + .thenReturn(null); - MvcResult mvcResult = mockMvc.perform(get("/resources/{id}/download", 5) + MultiValueMap paramsMap = new LinkedMultiValueMap<>(); + paramsMap.add("fullName", "dolphinscheduler/resourcePath"); + + MvcResult mvcResult = mockMvc.perform(get("/resources/download") + .params(paramsMap) .header(SESSION_ID, sessionId)) .andExpect(status().is(HttpStatus.BAD_REQUEST.value())) .andReturn(); @@ -231,7 +248,7 @@ public class ResourcesControllerTest extends AbstractControllerTest { mockResult.setCode(Status.TENANT_NOT_EXIST.getCode()); Mockito.when(udfFuncService .createUdfFunction(Mockito.any(), Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), - Mockito.anyString(), Mockito.anyString(), Mockito.any(), Mockito.anyInt())) + Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.any())) .thenReturn(mockResult); MultiValueMap paramsMap = new LinkedMultiValueMap<>(); @@ -242,8 +259,9 @@ public class ResourcesControllerTest extends AbstractControllerTest { paramsMap.add("database", "database"); paramsMap.add("description", "description"); paramsMap.add("resourceId", "1"); + paramsMap.add("fullName", "dolphinscheduler/resourcePath"); - MvcResult mvcResult = mockMvc.perform(post("/resources/{resourceId}/udf-func", "123") + MvcResult mvcResult = mockMvc.perform(post("/resources/udf-func") .header(SESSION_ID, sessionId) .params(paramsMap)) .andExpect(status().isCreated()) @@ -282,7 +300,8 @@ public class ResourcesControllerTest extends AbstractControllerTest { mockResult.setCode(Status.TENANT_NOT_EXIST.getCode()); Mockito.when(udfFuncService .updateUdfFunc(Mockito.any(), Mockito.anyInt(), Mockito.anyString(), Mockito.anyString(), - Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.any(), Mockito.anyInt())) + Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.any(), + Mockito.anyString())) .thenReturn(mockResult); MultiValueMap paramsMap = new LinkedMultiValueMap<>(); @@ -294,8 +313,9 @@ public class ResourcesControllerTest extends AbstractControllerTest { paramsMap.add("database", "database"); paramsMap.add("description", "description"); paramsMap.add("resourceId", "1"); + paramsMap.add("fullName", "dolphinscheduler/resourcePath"); - MvcResult mvcResult = mockMvc.perform(put("/resources/{resourceId}/udf-func/{id}", "123", "456") + MvcResult mvcResult = mockMvc.perform(put("/resources/udf-func/{id}", "456") .header(SESSION_ID, sessionId) .params(paramsMap)) .andExpect(status().isOk()) @@ -465,10 +485,15 @@ public class ResourcesControllerTest extends AbstractControllerTest { public void testDeleteResource() throws Exception { Result mockResult = new Result<>(); mockResult.setCode(Status.SUCCESS.getCode()); - Mockito.when(resourcesService.delete(Mockito.any(), Mockito.anyInt())).thenReturn(mockResult); - - MvcResult mvcResult = mockMvc.perform(delete("/resources/{id}", "123") - .header(SESSION_ID, sessionId)) + Mockito.when(resourcesService.delete(Mockito.any(), Mockito.anyString(), + Mockito.anyString())) + .thenReturn(mockResult); + MultiValueMap paramsMap = new LinkedMultiValueMap<>(); + paramsMap.add("fullName", "dolphinscheduler/resourcePath"); + paramsMap.add("tenantCode", "123"); + MvcResult mvcResult = mockMvc.perform(delete("/resources") + .header(SESSION_ID, sessionId) + .params(paramsMap)) .andExpect(status().isOk()) .andExpect(content().contentType(MediaType.APPLICATION_JSON)) .andReturn(); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java index 7178cbfce3..9d239c911b 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java @@ -27,9 +27,12 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.utils.JSONUtils; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.MediaType; @@ -41,17 +44,21 @@ public class TenantControllerTest extends AbstractControllerTest { private static final Logger logger = LoggerFactory.getLogger(TenantControllerTest.class); + private MockedStatic mockedStaticPropertyUtils; + @Test public void testCreateTenant() throws Exception { + mockedStaticPropertyUtils = Mockito.mockStatic(PropertyUtils.class); + MultiValueMap paramsMap = new LinkedMultiValueMap<>(); paramsMap.add("tenantCode", "hayden"); paramsMap.add("queueId", "1"); paramsMap.add("description", "tenant description"); + Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); - MvcResult mvcResult = mockMvc.perform(post("/tenants/") + MvcResult mvcResult = mockMvc.perform(post("/tenants") .header(SESSION_ID, sessionId) .params(paramsMap)) - .andExpect(status().isCreated()) .andExpect(content().contentType(MediaType.APPLICATION_JSON)) .andReturn(); @@ -59,6 +66,7 @@ public class TenantControllerTest extends AbstractControllerTest { Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); + mockedStaticPropertyUtils.close(); } @Test diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitorTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitorTest.java index f5ca3c559d..c51145b3a9 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitorTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitorTest.java @@ -17,7 +17,7 @@ package org.apache.dolphinscheduler.api.dto.resources.visitor; import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent; -import org.apache.dolphinscheduler.dao.entity.Resource; +import org.apache.dolphinscheduler.service.storage.StorageEntity; import java.util.ArrayList; import java.util.List; @@ -31,53 +31,43 @@ import org.junit.jupiter.api.Test; public class ResourceTreeVisitorTest { @Test - public void visit() throws Exception { - List resourceList = new ArrayList<>(); + public void visit() { + List resourceList = new ArrayList<>(); + + StorageEntity resource1 = new StorageEntity(); + resource1.setFullName("/default/a"); + resource1.setPfullName("/default"); + + StorageEntity resource2 = new StorageEntity(); + resource1.setFullName("/default/a/a1.txt"); + resource1.setPfullName("/default/a"); - Resource resource1 = new Resource(3, -1, "b", "/b", true); - Resource resource2 = new Resource(4, 2, "a1.txt", "/a/a1.txt", false); - Resource resource3 = new Resource(5, 3, "b1.txt", "/b/b1.txt", false); - Resource resource4 = new Resource(6, 3, "b2.jar", "/b/b2.jar", false); - Resource resource5 = new Resource(7, -1, "b2", "/b2", true); - Resource resource6 = new Resource(8, -1, "b2", "/b/b2", true); - Resource resource7 = new Resource(9, 8, "c2.jar", "/b/b2/c2.jar", false); resourceList.add(resource1); resourceList.add(resource2); - resourceList.add(resource3); - resourceList.add(resource4); - resourceList.add(resource5); - resourceList.add(resource6); - resourceList.add(resource7); ResourceTreeVisitor resourceTreeVisitor = new ResourceTreeVisitor(resourceList); - ResourceComponent resourceComponent = resourceTreeVisitor.visit(); + ResourceComponent resourceComponent = resourceTreeVisitor.visit("/default"); Assertions.assertNotNull(resourceComponent.getChildren()); } @Test - public void rootNode() throws Exception { - List resourceList = new ArrayList<>(); + public void rootNode() { + List resourceList = new ArrayList<>(); + + StorageEntity resource1 = new StorageEntity(); + resource1.setFullName("/default/a"); + resource1.setPfullName("/default"); + + StorageEntity resource2 = new StorageEntity(); + resource1.setFullName("/default/a/a1.txt"); + resource1.setPfullName("/default/a"); - Resource resource1 = new Resource(3, -1, "b", "/b", true); - Resource resource2 = new Resource(4, 2, "a1.txt", "/a/a1.txt", false); - Resource resource3 = new Resource(5, 3, "b1.txt", "/b/b1.txt", false); - Resource resource4 = new Resource(6, 3, "b2.jar", "/b/b2.jar", false); - Resource resource5 = new Resource(7, -1, "b2", "/b2", true); - Resource resource6 = new Resource(8, -1, "b2", "/b/b2", true); - Resource resource7 = new Resource(9, 8, "c2.jar", "/b/b2/c2.jar", false); resourceList.add(resource1); resourceList.add(resource2); - resourceList.add(resource3); - resourceList.add(resource4); - resourceList.add(resource5); - resourceList.add(resource6); - resourceList.add(resource7); ResourceTreeVisitor resourceTreeVisitor = new ResourceTreeVisitor(resourceList); - Assertions.assertTrue(resourceTreeVisitor.rootNode(resource1)); - Assertions.assertTrue(resourceTreeVisitor.rootNode(resource2)); - Assertions.assertFalse(resourceTreeVisitor.rootNode(resource3)); - + Assertions.assertTrue(resourceTreeVisitor.rootNode(resource1, "/default")); + Assertions.assertFalse(resourceTreeVisitor.rootNode(resource2, "/default")); } } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java index a4b0d79111..ecfbc72548 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.api.service; import static org.mockito.ArgumentMatchers.eq; -import org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.permission.ResourcePermissionCheckService; import org.apache.dolphinscheduler.api.service.impl.BaseServiceImpl; @@ -27,7 +26,6 @@ import org.apache.dolphinscheduler.api.service.impl.ResourcesServiceImpl; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils; @@ -41,6 +39,7 @@ import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper; +import org.apache.dolphinscheduler.service.storage.StorageEntity; import org.apache.dolphinscheduler.service.storage.StorageOperate; import org.apache.dolphinscheduler.spi.enums.ResourceType; @@ -145,11 +144,6 @@ public class ResourcesServiceTest { @Test public void testCreateResource() { - Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, - null, 1, ApiFuncIdentificationConstant.FILE_UPLOAD, serviceLogger)).thenReturn(true); - Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID, - null, 1, serviceLogger)).thenReturn(true); - Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); User user = new User(); user.setId(1); @@ -158,10 +152,10 @@ public class ResourcesServiceTest { // CURRENT_LOGIN_USER_TENANT_NOT_EXIST MockMultipartFile mockMultipartFile = new MockMultipartFile("test.pdf", "test.pdf", "pdf", "test".getBytes()); Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); - Mockito.when(userMapper.selectById(1)).thenReturn(getUser()); + Mockito.when(userMapper.selectById(user.getId())).thenReturn(getUser()); Mockito.when(tenantMapper.queryById(1)).thenReturn(null); Result result = resourcesService.createResource(user, "ResourcesServiceTest", "ResourcesServiceTest", - ResourceType.FILE, mockMultipartFile, -1, "/"); + ResourceType.FILE, mockMultipartFile, "/"); logger.info(result.toString()); Assertions.assertEquals(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST.getMsg(), result.getMsg()); // set tenant for user @@ -171,7 +165,7 @@ public class ResourcesServiceTest { // HDFS_NOT_STARTUP Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); result = resourcesService.createResource(user, "ResourcesServiceTest", "ResourcesServiceTest", - ResourceType.FILE, null, -1, "/"); + ResourceType.FILE, null, "/"); logger.info(result.toString()); Assertions.assertEquals(Status.STORAGE_NOT_STARTUP.getMsg(), result.getMsg()); @@ -179,7 +173,7 @@ public class ResourcesServiceTest { mockMultipartFile = new MockMultipartFile("test.pdf", "".getBytes()); Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); result = resourcesService.createResource(user, "ResourcesServiceTest", "ResourcesServiceTest", - ResourceType.FILE, mockMultipartFile, -1, "/"); + ResourceType.FILE, mockMultipartFile, "/"); logger.info(result.toString()); Assertions.assertEquals(Status.RESOURCE_FILE_IS_EMPTY.getMsg(), result.getMsg()); @@ -188,47 +182,33 @@ public class ResourcesServiceTest { Mockito.when(Files.getFileExtension("test.pdf")).thenReturn("pdf"); Mockito.when(Files.getFileExtension("ResourcesServiceTest.jar")).thenReturn("jar"); result = resourcesService.createResource(user, "ResourcesServiceTest.jar", "ResourcesServiceTest", - ResourceType.FILE, mockMultipartFile, -1, "/"); + ResourceType.FILE, mockMultipartFile, "/"); logger.info(result.toString()); Assertions.assertEquals(Status.RESOURCE_SUFFIX_FORBID_CHANGE.getMsg(), result.getMsg()); // UDF_RESOURCE_SUFFIX_NOT_JAR - Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, - null, 1, ApiFuncIdentificationConstant.UDF_UPLOAD, serviceLogger)).thenReturn(true); - Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID, - null, 1, serviceLogger)).thenReturn(true); - mockMultipartFile = new MockMultipartFile("ResourcesServiceTest.pdf", "ResourcesServiceTest.pdf", "pdf", "test".getBytes()); Mockito.when(Files.getFileExtension("ResourcesServiceTest.pdf")).thenReturn("pdf"); result = resourcesService.createResource(user, "ResourcesServiceTest.pdf", "ResourcesServiceTest", - ResourceType.UDF, mockMultipartFile, -1, "/"); + ResourceType.UDF, mockMultipartFile, "/"); logger.info(result.toString()); Assertions.assertEquals(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg(), result.getMsg()); // FULL_FILE_NAME_TOO_LONG - Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, - null, 1, ApiFuncIdentificationConstant.FILE_UPLOAD, serviceLogger)).thenReturn(true); - Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID, - null, 1, serviceLogger)).thenReturn(true); - String tooLongFileName = getRandomStringWithLength(Constants.RESOURCE_FULL_NAME_MAX_LENGTH) + ".pdf"; mockMultipartFile = new MockMultipartFile(tooLongFileName, tooLongFileName, "pdf", "test".getBytes()); - Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); Mockito.when(Files.getFileExtension(tooLongFileName)).thenReturn("pdf"); + // '/databasePath/tenantCode/RESOURCE/' + Mockito.when(storageOperate.getResDir("123")).thenReturn("/dolphinscheduler/123/resources/"); result = resourcesService.createResource(user, tooLongFileName, tooLongFileName, ResourceType.FILE, - mockMultipartFile, -1, "/"); + mockMultipartFile, "/"); logger.info(result.toString()); Assertions.assertEquals(Status.RESOURCE_FULL_NAME_TOO_LONG_ERROR.getMsg(), result.getMsg()); } @Test public void testCreateDirecotry() { - Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, - null, 1, ApiFuncIdentificationConstant.FOLDER_ONLINE_CREATE, serviceLogger)).thenReturn(true); - Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID, - null, 1, serviceLogger)).thenReturn(true); - Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); User user = new User(); user.setId(1); @@ -239,23 +219,18 @@ public class ResourcesServiceTest { logger.info(result.toString()); Assertions.assertEquals(Status.STORAGE_NOT_STARTUP.getMsg(), result.getMsg()); - // PARENT_RESOURCE_NOT_EXIST + // RESOURCE_EXIST user.setId(1); user.setTenantId(1); Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); + Mockito.when(userMapper.selectById(user.getId())).thenReturn(getUser()); Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); - Mockito.when(resourcesMapper.selectById(Mockito.anyInt())).thenReturn(null); - Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, - null, 1, ApiFuncIdentificationConstant.FOLDER_ONLINE_CREATE, serviceLogger)).thenReturn(true); - Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID, - null, 1, serviceLogger)).thenReturn(true); - Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, - null, 1, ApiFuncIdentificationConstant.FILE_RENAME, serviceLogger)).thenReturn(true); - - result = resourcesService.createDirectory(user, "directoryTest", "directory test", ResourceType.FILE, 1, "/"); - logger.info(result.toString()); - Assertions.assertEquals(Status.PARENT_RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); - // RESOURCE_EXIST + Mockito.when(storageOperate.getResDir("123")).thenReturn("/dolphinscheduler/123/resources/"); + try { + Mockito.when(storageOperate.exists("/dolphinscheduler/123/resources/directoryTest")).thenReturn(true); + } catch (IOException e) { + logger.error(e.getMessage(), e); + } Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); Mockito.when(resourcesMapper.existResource("/directoryTest", 0)).thenReturn(true); result = resourcesService.createDirectory(user, "directoryTest", "directory test", ResourceType.FILE, -1, "/"); @@ -272,131 +247,126 @@ public class ResourcesServiceTest { @Test public void testUpdateResource() { - Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, - null, 1, ApiFuncIdentificationConstant.FILE_UPDATE, serviceLogger)).thenReturn(true); - Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID, - new Object[]{1}, 1, serviceLogger)).thenReturn(true); - - Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); User user = new User(); user.setId(1); user.setUserType(UserType.GENERAL_USER); + user.setTenantId(1); + + Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); + Mockito.when(userMapper.selectById(user.getId())).thenReturn(getUser()); + Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); + Mockito.when(storageOperate.getResDir("123")).thenReturn("/dolphinscheduler/123/resources/"); + // HDFS_NOT_STARTUP - Result result = resourcesService.updateResource(user, 1, "ResourcesServiceTest", "ResourcesServiceTest", - ResourceType.FILE, null); + Result result = resourcesService.updateResource(user, "ResourcesServiceTest", + "123", "ResourcesServiceTest", "", ResourceType.FILE, null); logger.info(result.toString()); Assertions.assertEquals(Status.STORAGE_NOT_STARTUP.getMsg(), result.getMsg()); - // RESOURCE_NOT_EXIST - Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, - null, 1, ApiFuncIdentificationConstant.FILE_UPDATE, serviceLogger)).thenReturn(true); - Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID, - new Object[]{0}, 1, serviceLogger)).thenReturn(true); - Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource()); - Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); - result = resourcesService.updateResource(user, 0, "ResourcesServiceTest", "ResourcesServiceTest", - ResourceType.FILE, null); - logger.info(result.toString()); - Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); - // USER_NO_OPERATION_PERM - user.setId(2); + Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); user.setUserType(UserType.GENERAL_USER); - Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, - null, 2, ApiFuncIdentificationConstant.FILE_UPDATE, serviceLogger)).thenReturn(true); - Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID, - new Object[]{1}, 2, serviceLogger)).thenReturn(false); - result = resourcesService.updateResource(user, 1, "ResourcesServiceTest", "ResourcesServiceTest", - ResourceType.FILE, null); + // tenant who have access to resource is 123, + Tenant tenantWNoPermission = new Tenant(); + tenantWNoPermission.setTenantCode("321"); + Mockito.when(tenantMapper.queryById(1)).thenReturn(tenantWNoPermission); + result = resourcesService.updateResource(user, + "/dolphinscheduler/123/resources/ResourcesServiceTest", + "123", + "ResourcesServiceTest", "", ResourceType.FILE, null); logger.info(result.toString()); Assertions.assertEquals(Status.NO_CURRENT_OPERATING_PERMISSION.getMsg(), result.getMsg()); - // RESOURCE_NOT_EXIST - user.setId(1); - Mockito.when(userMapper.selectById(1)).thenReturn(getUser()); + // SUCCESS Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); - Mockito.when(storageOperate.getFileName(Mockito.any(), Mockito.any(), Mockito.anyString())).thenReturn("test1"); - - Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.UDF_FILE, null, - 1, ApiFuncIdentificationConstant.UDF_UPDATE, serviceLogger)).thenReturn(true); - Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.UDF_FILE, new Object[]{1}, - 1, serviceLogger)).thenReturn(true); try { - Mockito.when(storageOperate.exists(Mockito.any(), Mockito.any())).thenReturn(false); + Mockito.when(storageOperate.exists(Mockito.any())).thenReturn(false); } catch (IOException e) { logger.error(e.getMessage(), e); } - result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest", - ResourceType.UDF, null); - Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); - // SUCCESS - user.setId(1); - Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); try { - Mockito.when(storageOperate.exists(Mockito.any(), Mockito.any())).thenReturn(true); - } catch (IOException e) { - logger.error(e.getMessage(), e); + Mockito.when(storageOperate.getFileStatus("/dolphinscheduler/123/resources/ResourcesServiceTest", + "/dolphinscheduler/123/resources/", + "123", ResourceType.FILE)).thenReturn(getStorageEntityResource()); + result = resourcesService.updateResource(user, "/dolphinscheduler/123/resources/ResourcesServiceTest", + "123", + "ResourcesServiceTest", "", ResourceType.FILE, null); + logger.info(result.toString()); + Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); + } catch (Exception e) { + logger.error(e.getMessage() + " Resource path: {}", "/dolphinscheduler/123/resources/ResourcesServiceTest", + e); } - Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null, - 1, ApiFuncIdentificationConstant.FILE_UPDATE, serviceLogger)).thenReturn(true); - Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID, - new Object[]{1}, - 1, serviceLogger)).thenReturn(true); - result = resourcesService.updateResource(user, 1, "ResourcesServiceTest.jar", "ResourcesServiceTest", - ResourceType.FILE, null); - logger.info(result.toString()); - Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); - + // Tests for udf resources. // RESOURCE_EXIST - Mockito.when(resourcesMapper.existResource("/ResourcesServiceTest1.jar", 0)).thenReturn(true); - result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest", - ResourceType.FILE, null); - logger.info(result.toString()); - Assertions.assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg()); - // USER_NOT_EXIST + try { + Mockito.when(storageOperate.exists("/dolphinscheduler/123/resources/ResourcesServiceTest2.jar")) + .thenReturn(true); + } catch (IOException e) { + logger.error("error occurred when checking resource: " + + "/dolphinscheduler/123/resources/ResourcesServiceTest2.jar"); + } - Mockito.when(userMapper.selectById(Mockito.anyInt())).thenReturn(null); - result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest", - ResourceType.UDF, null); + try { + Mockito.when(storageOperate.getFileStatus("/dolphinscheduler/123/resources/ResourcesServiceTest1.jar", + "/dolphinscheduler/123/resources/", + "123", ResourceType.UDF)).thenReturn(getStorageEntityUdfResource()); + } catch (Exception e) { + logger.error(e.getMessage() + " Resource path: {}", + "/dolphinscheduler/123/resources/ResourcesServiceTest1.jar", e); + } + result = resourcesService.updateResource(user, "/dolphinscheduler/123/resources/ResourcesServiceTest1.jar", + "123", "ResourcesServiceTest2.jar", "", ResourceType.UDF, null); logger.info(result.toString()); - Assertions.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode()); + Assertions.assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg()); // TENANT_NOT_EXIST - Mockito.when(userMapper.selectById(1)).thenReturn(getUser()); Mockito.when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(null); - result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest", - ResourceType.UDF, null); + result = resourcesService.updateResource(user, "ResourcesServiceTest1.jar", + "", "ResourcesServiceTest", "", ResourceType.UDF, null); logger.info(result.toString()); Assertions.assertEquals(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST.getMsg(), result.getMsg()); // SUCCESS Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); - result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest1.jar", - ResourceType.UDF, null); + result = resourcesService.updateResource(user, "/dolphinscheduler/123/resources/ResourcesServiceTest1.jar", + "123", "ResourcesServiceTest1.jar", "", ResourceType.UDF, null); logger.info(result.toString()); Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); - } @Test public void testQueryResourceListPaging() { User loginUser = new User(); loginUser.setId(1); + loginUser.setTenantId(1); + loginUser.setTenantCode("tenant1"); loginUser.setUserType(UserType.ADMIN_USER); IPage resourcePage = new Page<>(1, 10); resourcePage.setTotal(1); resourcePage.setRecords(getResourceList()); - Mockito.when(resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.RESOURCE_FILE_ID, - 1, resourceLogger)).thenReturn(getSetIds()); + List mockResList = new ArrayList(); + mockResList.add(getStorageEntityResource()); + List mockUserList = new ArrayList(); + mockUserList.add(getUser()); + Mockito.when(userMapper.selectList(null)).thenReturn(mockUserList); + Mockito.when(userMapper.selectById(getUser().getId())).thenReturn(getUser()); + Mockito.when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(getTenant()); + Mockito.when(storageOperate.getResDir("123")).thenReturn("/dolphinscheduler/123/resources/"); - Mockito.when( - resourcesMapper.queryResourcePaging(Mockito.any(Page.class), eq(-1), eq(0), eq("test"), Mockito.any())) - .thenReturn(resourcePage); - Result result = resourcesService.queryResourceListPaging(loginUser, -1, ResourceType.FILE, "test", 1, 10); + try { + Mockito.when(storageOperate.listFilesStatus("/dolphinscheduler/123/resources/", + "/dolphinscheduler/123/resources/", + "123", ResourceType.FILE)).thenReturn(mockResList); + } catch (Exception e) { + logger.error("QueryResourceListPaging Error"); + } + Result result = resourcesService.queryResourceListPaging(loginUser, "", "", + ResourceType.FILE, "Test", 1, 10); logger.info(result.toString()); Assertions.assertEquals(Status.SUCCESS.getCode(), (int) result.getCode()); PageInfo pageInfo = (PageInfo) result.getData(); @@ -410,30 +380,33 @@ public class ResourcesServiceTest { loginUser.setId(0); loginUser.setUserType(UserType.ADMIN_USER); - Mockito.when(resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.RESOURCE_FILE_ID, - 0, resourceLogger)).thenReturn(getSetIds()); - Mockito.when(resourcesMapper.selectBatchIds(Mockito.anySet())).thenReturn(getResourceList()); - - Map result = resourcesService.queryResourceList(loginUser, ResourceType.FILE); + Mockito.when(userMapper.selectList(null)).thenReturn(Arrays.asList(loginUser)); + Mockito.when(userMapper.selectById(loginUser.getId())).thenReturn(loginUser); + Mockito.when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(getTenant()); + Mockito.when(storageOperate.getResDir("123")).thenReturn("/dolphinscheduler/123/resources/"); + Mockito.when(storageOperate.listFilesStatusRecursively("/dolphinscheduler/123/resources/", + "/dolphinscheduler/123/resources/", + "123", + ResourceType.FILE)).thenReturn(Arrays.asList(getStorageEntityResource())); + Map result = resourcesService.queryResourceList(loginUser, ResourceType.FILE, ""); logger.info(result.toString()); Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); List resourceList = (List) result.get(Constants.DATA_LIST); Assertions.assertTrue(CollectionUtils.isNotEmpty(resourceList)); // test udf - Mockito.when(resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.UDF_FILE, 0, - resourceLogger)).thenReturn(getSetIds()); - Mockito.when(resourcesMapper.selectBatchIds(Mockito.anySet())) - .thenReturn(Arrays.asList(getResource(11, ResourceType.UDF), - getResource(10, ResourceType.UDF), getResource(9, ResourceType.UDF), - getResource(8, ResourceType.UDF))); - + Mockito.when(storageOperate.getUdfDir("123")).thenReturn("/dolphinscheduler/123/udfs/"); + Mockito.when(storageOperate.listFilesStatusRecursively("/dolphinscheduler/123/udfs/", + "/dolphinscheduler/123/udfs/", + "123", + ResourceType.UDF)) + .thenReturn(Arrays.asList(getStorageEntityUdfResource())); loginUser.setUserType(UserType.GENERAL_USER); - result = resourcesService.queryResourceList(loginUser, ResourceType.UDF); + result = resourcesService.queryResourceList(loginUser, ResourceType.UDF, ""); logger.info(result.toString()); Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); resourceList = (List) result.get(Constants.DATA_LIST); - Assertions.assertTrue(resourceList.size() == 4); + Assertions.assertTrue(CollectionUtils.isNotEmpty(resourceList)); } @Test @@ -443,59 +416,34 @@ public class ResourcesServiceTest { loginUser.setId(0); loginUser.setUserType(UserType.GENERAL_USER); Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); - Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource()); - Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); - - Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null, - 0, ApiFuncIdentificationConstant.FILE_DELETE, serviceLogger)).thenReturn(true); - Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID, - new Object[]{1}, - 0, serviceLogger)).thenReturn(true); try { // HDFS_NOT_STARTUP - Result result = resourcesService.delete(loginUser, 1); + Result result = resourcesService.delete(loginUser, "", ""); logger.info(result.toString()); Assertions.assertEquals(Status.STORAGE_NOT_STARTUP.getMsg(), result.getMsg()); - // RESOURCE_NOT_EXIST - Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); - Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource()); - - Mockito.when( - resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null, - 0, ApiFuncIdentificationConstant.FILE_DELETE, serviceLogger)) - .thenReturn(true); - - result = resourcesService.delete(loginUser, 2); - logger.info(result.toString()); - Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); - - // USER_NO_OPERATION_PERM - result = resourcesService.delete(loginUser, 2); - logger.info(result.toString()); - Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); - // TENANT_NOT_EXIST + Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); loginUser.setUserType(UserType.ADMIN_USER); loginUser.setTenantId(2); - Mockito.when(userMapper.selectById(Mockito.anyInt())).thenReturn(loginUser); - Mockito.when( - resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null, - 0, ApiFuncIdentificationConstant.FILE_DELETE, serviceLogger)) - .thenReturn(true); - Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID, - new Object[]{1}, - 0, serviceLogger)).thenReturn(true); - result = resourcesService.delete(loginUser, 1); + Mockito.when(userMapper.selectById(loginUser.getId())).thenReturn(loginUser); + result = resourcesService.delete(loginUser, "", ""); logger.info(result.toString()); Assertions.assertEquals(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST.getMsg(), result.getMsg()); + // RESOURCE_NOT_EXIST + Mockito.when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(getTenant()); + Mockito.when(storageOperate.getFileStatus("/dolphinscheduler/123/resources/ResourcesServiceTest", + null, "123", null)) + .thenReturn(getStorageEntityResource()); + result = resourcesService.delete(loginUser, "/dolphinscheduler/123/resources/ResNotExist", "123"); + logger.info(result.toString()); + Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); + // SUCCESS loginUser.setTenantId(1); - Mockito.when(processDefinitionMapper.listResources()).thenReturn(getResources()); - Mockito.when(resourcesMapper.deleteIds(Mockito.any())).thenReturn(1); - Mockito.when(resourceUserMapper.deleteResourceUserArray(Mockito.anyInt(), Mockito.any())).thenReturn(1); - result = resourcesService.delete(loginUser, 1); + result = resourcesService.delete(loginUser, "/dolphinscheduler/123/resources/ResourcesServiceTest", + "123"); logger.info(result.toString()); Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); @@ -508,29 +456,19 @@ public class ResourcesServiceTest { @Test public void testVerifyResourceName() { - Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null, - 1, ApiFuncIdentificationConstant.FILE_RENAME, serviceLogger)).thenReturn(true); - Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null, - 1, serviceLogger)).thenReturn(true); User user = new User(); user.setId(1); user.setUserType(UserType.GENERAL_USER); - Mockito.when(resourcesMapper.existResource("/ResourcesServiceTest.jar", 0)).thenReturn(true); + try { + Mockito.when(storageOperate.exists("/ResourcesServiceTest.jar")).thenReturn(true); + } catch (IOException e) { + logger.error("error occurred when checking resource: /ResourcesServiceTest.jar\""); + } Result result = resourcesService.verifyResourceName("/ResourcesServiceTest.jar", ResourceType.FILE, user); logger.info(result.toString()); Assertions.assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg()); - // TENANT_NOT_EXIST - Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); - String unExistFullName = "/test.jar"; - - result = resourcesService.verifyResourceName("/test.jar", ResourceType.FILE, user); - logger.info(result.toString()); - Assertions.assertEquals(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST.getMsg(), result.getMsg()); - // RESOURCE_FILE_EXIST - user.setTenantId(1); - result = resourcesService.verifyResourceName("/ResourcesServiceTest.jar", ResourceType.FILE, user); logger.info(result.toString()); Assertions.assertTrue(Status.RESOURCE_EXIST.getCode() == result.getCode()); @@ -544,123 +482,89 @@ public class ResourcesServiceTest { @Test public void testReadResource() { - - Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null, - 1, ApiFuncIdentificationConstant.FILE_VIEW, serviceLogger)).thenReturn(true); - Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID, - new Object[]{1}, - 1, serviceLogger)).thenReturn(true); Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); // HDFS_NOT_STARTUP - Result result = resourcesService.readResource(getUser(), 1, 1, 10); + Result result = resourcesService.readResource(getUser(), "", "", 1, 10); logger.info(result.toString()); Assertions.assertEquals(Status.STORAGE_NOT_STARTUP.getMsg(), result.getMsg()); // RESOURCE_NOT_EXIST - Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource()); Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); - Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null, - 1, ApiFuncIdentificationConstant.FILE_VIEW, serviceLogger)).thenReturn(true); - - result = resourcesService.readResource(getUser(), 2, 1, 10); + Mockito.when(userMapper.selectById(getUser().getId())).thenReturn(getUser()); + Mockito.when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(getTenant()); + result = resourcesService.readResource(getUser(), "", "", 1, 10); logger.info(result.toString()); - Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); + Assertions.assertEquals(Status.RESOURCE_FILE_NOT_EXIST.getCode(), (int) result.getCode()); // RESOURCE_SUFFIX_NOT_SUPPORT_VIEW - Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null, - 1, ApiFuncIdentificationConstant.FILE_VIEW, serviceLogger)).thenReturn(true); - Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID, - new Object[]{1}, - 1, serviceLogger)).thenReturn(true); Mockito.when(FileUtils.getResourceViewSuffixes()).thenReturn("class"); Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); - result = resourcesService.readResource(getUser(), 1, 1, 10); + result = resourcesService.readResource(getUser(), "", "", 1, 10); logger.info(result.toString()); Assertions.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(), result.getMsg()); // USER_NOT_EXIST + Mockito.when(userMapper.selectById(getUser().getId())).thenReturn(null); Mockito.when(FileUtils.getResourceViewSuffixes()).thenReturn("jar"); Mockito.when(Files.getFileExtension("ResourcesServiceTest.jar")).thenReturn("jar"); - result = resourcesService.readResource(getUser(), 1, 1, 10); + result = resourcesService.readResource(getUser(), "", "", 1, 10); logger.info(result.toString()); Assertions.assertEquals(Status.USER_NOT_EXIST.getCode(), (int) result.getCode()); // TENANT_NOT_EXIST - Mockito.when(userMapper.selectById(1)).thenReturn(getUser()); - result = resourcesService.readResource(getUser(), 1, 1, 10); + Mockito.when(userMapper.selectById(getUser().getId())).thenReturn(getUser()); + Mockito.when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(null); + result = resourcesService.readResource(getUser(), "", "", 1, 10); logger.info(result.toString()); Assertions.assertEquals(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST.getMsg(), result.getMsg()); - // RESOURCE_FILE_NOT_EXIST - Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); - - result = resourcesService.readResource(getUser(), 1, 1, 10); - logger.info(result.toString()); - Assertions.assertEquals(Status.RESOURCE_FILE_NOT_EXIST.getCode(), (int) result.getCode()); - // SUCCESS + Mockito.when(userMapper.selectById(getUser().getId())).thenReturn(getUser()); + Mockito.when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(getTenant()); try { - Mockito.when(storageOperate.exists(Mockito.any(), Mockito.any())).thenReturn(true); + Mockito.when(storageOperate.exists(Mockito.any())).thenReturn(true); Mockito.when(storageOperate.vimFile(Mockito.any(), Mockito.any(), eq(1), eq(10))).thenReturn(getContent()); } catch (IOException e) { logger.error("storage error", e); } - result = resourcesService.readResource(getUser(), 1, 1, 10); + Mockito.when(Files.getFileExtension("test.jar")).thenReturn("jar"); + result = resourcesService.readResource(getUser(), "test.jar", "", 1, 10); logger.info(result.toString()); Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); - } @Test public void testOnlineCreateResource() { - - Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null, - 1, ApiFuncIdentificationConstant.FILE_ONLINE_CREATE, serviceLogger)).thenReturn(true); - Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null, - 1, serviceLogger)).thenReturn(true); - Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); User user = getUser(); user.setId(1); + Mockito.when(userMapper.selectById(user.getId())).thenReturn(getUser()); + Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); + // HDFS_NOT_STARTUP Result result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content", - -1, "/"); + "/"); logger.info(result.toString()); Assertions.assertEquals(Status.STORAGE_NOT_STARTUP.getMsg(), result.getMsg()); // RESOURCE_SUFFIX_NOT_SUPPORT_VIEW Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); Mockito.when(FileUtils.getResourceViewSuffixes()).thenReturn("class"); - result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content", -1, + result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content", "/"); logger.info(result.toString()); Assertions.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(), result.getMsg()); - // RuntimeException - try { - Mockito.when(FileUtils.getResourceViewSuffixes()).thenReturn("jar"); - Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); - result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content", - -1, "/"); - } catch (RuntimeException ex) { - logger.info(result.toString()); - Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), ex.getMessage()); - } - // SUCCESS - Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null, - 1, ApiFuncIdentificationConstant.FILE_RENAME, serviceLogger)).thenReturn(true); - Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null, - 1, serviceLogger)).thenReturn(true); - + Mockito.when(FileUtils.getResourceViewSuffixes()).thenReturn("jar"); + Mockito.when(storageOperate.getResDir("123")).thenReturn("/dolphinscheduler/123/resources/"); Mockito.when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test"); Mockito.when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true); - result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content", -1, + result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content", "/"); logger.info(result.toString()); Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); - } @Test @@ -684,141 +588,138 @@ public class ResourcesServiceTest { Resource dir2 = new Resource(); dir2.setFullName(resourceDir); dir2.setUserId(user.getId()); - Mockito.when(resourcesMapper.queryResource(dir1.getFullName(), ResourceType.FILE.ordinal())) - .thenReturn(Collections.singletonList(dir1)); - Mockito.when(resourcesMapper.queryResource(resourceDir, ResourceType.FILE.ordinal())).thenReturn(null); - - Tenant tenant = getTenant(); - Mockito.when(resourcePermissionCheckService.operationPermissionCheck( - AuthorizationType.RESOURCE_FILE_ID, null, 1, ApiFuncIdentificationConstant.FOLDER_ONLINE_CREATE, - serviceLogger)).thenReturn(true); - Mockito.when(resourcePermissionCheckService.resourcePermissionCheck( - AuthorizationType.RESOURCE_FILE_ID, null, 1, serviceLogger)).thenReturn(true); + + Mockito.when(storageOperate.getDir(ResourceType.FILE, "123")).thenReturn("/dolphinscheduler/123/resources/"); + Mockito.when(storageOperate.getResDir("123")).thenReturn("/dolphinscheduler/123/resources/"); + Mockito.when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test"); + Mockito.when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true); try { - Mockito.when(storageOperate.mkdir(tenant.getTenantCode(), null)).thenReturn(true); + Mockito.when(storageOperate.mkdir("123", "/dolphinscheduler/123/resources" + dir1Path)).thenReturn(true); + Mockito.when(storageOperate.mkdir("123", "/dolphinscheduler/123/resources" + dir2Path)).thenReturn(true); } catch (IOException e) { - logger.error("storage error", e); + logger.error("create resource directory {} failed", fullName); } - - Mockito.when(resourcePermissionCheckService.operationPermissionCheck( - AuthorizationType.RESOURCE_FILE_ID, null, 1, ApiFuncIdentificationConstant.FILE_ONLINE_CREATE, - serviceLogger)).thenReturn(true); - Mockito.when(resourcePermissionCheckService.resourcePermissionCheck( - AuthorizationType.RESOURCE_FILE_ID, null, 1, serviceLogger)).thenReturn(true); Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); - Mockito.when(resourcePermissionCheckService.operationPermissionCheck( - AuthorizationType.RESOURCE_FILE_ID, null, 1, ApiFuncIdentificationConstant.FILE_RENAME, serviceLogger)) - .thenReturn(true); - Mockito.when(resourcePermissionCheckService.resourcePermissionCheck( - AuthorizationType.RESOURCE_FILE_ID, null, 1, serviceLogger)).thenReturn(true); - Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); - Mockito.when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test"); - Mockito.when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true); - + Mockito.when(userMapper.selectById(user.getId())).thenReturn(getUser()); + Mockito.when(tenantMapper.queryById(user.getTenantId())).thenReturn(getTenant()); Result result = resourcesService.onlineCreateOrUpdateResourceWithDir(user, fullName, desc, content); Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); } - @Test - public void testQueryResourcesFileInfo() { - User user = getUser(); - String userName = "test-user"; - Mockito.when(userMapper.queryByUserNameAccurately(userName)).thenReturn(user); - Resource file = new Resource(); - file.setFullName("/dir/file1.py"); - file.setId(1); - Mockito.when(resourcesMapper.queryResource(file.getFullName(), ResourceType.FILE.ordinal())) - .thenReturn(Collections.singletonList(file)); - Mockito.when(resourcePermissionCheckService.operationPermissionCheck( - AuthorizationType.RESOURCE_FILE_ID, null, user.getId(), ApiFuncIdentificationConstant.FILE_VIEW, - serviceLogger)).thenReturn(true); - Mockito.when(resourcePermissionCheckService.resourcePermissionCheck( - AuthorizationType.RESOURCE_FILE_ID, new Object[]{file.getId()}, user.getId(), serviceLogger)) - .thenReturn(true); - Resource result = resourcesService.queryResourcesFileInfo(userName, file.getFullName()); - Assertions.assertEquals(file.getFullName(), result.getFullName()); - } + // TODO: revise this testcase after modifying PythonGateway.java + // @Test + // public void testQueryResourcesFileInfo() { + // User user = getUser(); + // String userName = "test-user"; + // Mockito.when(userMapper.queryByUserNameAccurately(userName)).thenReturn(user); + // Resource file = new Resource(); + // file.setFullName("/dir/file1.py"); + // file.setId(1); + // Mockito.when(resourcesMapper.queryResource(file.getFullName(), ResourceType.FILE.ordinal())) + // .thenReturn(Collections.singletonList(file)); + // Mockito.when(resourcePermissionCheckService.operationPermissionCheck( + // AuthorizationType.RESOURCE_FILE_ID, null, user.getId(), ApiFuncIdentificationConstant.FILE_VIEW, + // serviceLogger)).thenReturn(true); + // Mockito.when(resourcePermissionCheckService.resourcePermissionCheck( + // AuthorizationType.RESOURCE_FILE_ID, new Object[]{file.getId()}, user.getId(), serviceLogger)) + // .thenReturn(true); + // Mockito.when(userMapper.selectById(1)).thenReturn(getUser()); + // Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); + // Resource result = resourcesService.queryResourcesFileInfo(userName, file.getFullName()); + // Assertions.assertEquals(file.getFullName(), result.getFullName()); + // } @Test public void testUpdateResourceContent() { Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); // HDFS_NOT_STARTUP - Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null, - 1, ApiFuncIdentificationConstant.FILE_UPDATE, serviceLogger)).thenReturn(true); - Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID, - new Object[]{1}, - 1, serviceLogger)).thenReturn(true); - - Result result = resourcesService.updateResourceContent(getUser(), 1, "content"); + Result result = resourcesService.updateResourceContent(getUser(), "", "", "content"); logger.info(result.toString()); Assertions.assertEquals(Status.STORAGE_NOT_STARTUP.getMsg(), result.getMsg()); // RESOURCE_NOT_EXIST - Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null, - 1, ApiFuncIdentificationConstant.FILE_UPDATE, serviceLogger)).thenReturn(true); Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); - Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource()); - result = resourcesService.updateResourceContent(getUser(), 2, "content"); + Mockito.when(userMapper.selectById(getUser().getId())).thenReturn(getUser()); + Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); + + try { + Mockito.when(storageOperate.getFileStatus("/dolphinscheduler/123/resources/ResourcesServiceTest.jar", + "", + "123", ResourceType.FILE)).thenReturn(null); + } catch (Exception e) { + logger.error(e.getMessage() + " Resource path: {}", "", e); + } + + result = resourcesService.updateResourceContent(getUser(), + "/dolphinscheduler/123/resources/ResourcesServiceTest.jar", + "123", "content"); logger.info(result.toString()); Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); // RESOURCE_SUFFIX_NOT_SUPPORT_VIEW - Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null, - 1, ApiFuncIdentificationConstant.FILE_UPDATE, serviceLogger)).thenReturn(true); - Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID, - new Object[]{1}, - 1, serviceLogger)).thenReturn(true); Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); Mockito.when(FileUtils.getResourceViewSuffixes()).thenReturn("class"); - result = resourcesService.updateResourceContent(getUser(), 1, "content"); + try { + Mockito.when(storageOperate.getFileStatus("", "", "123", ResourceType.FILE)) + .thenReturn(getStorageEntityResource()); + } catch (Exception e) { + logger.error(e.getMessage() + " Resource path: {}", "", e); + } + + result = resourcesService.updateResourceContent(getUser(), "", "123", "content"); logger.info(result.toString()); Assertions.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(), result.getMsg()); // USER_NOT_EXIST - Mockito.when(FileUtils.getResourceViewSuffixes()).thenReturn("jar"); - Mockito.when(Files.getFileExtension("ResourcesServiceTest.jar")).thenReturn("jar"); - result = resourcesService.updateResourceContent(getUser(), 1, "content"); + Mockito.when(userMapper.selectById(getUser().getId())).thenReturn(null); + result = resourcesService.updateResourceContent(getUser(), "", "123", "content"); logger.info(result.toString()); Assertions.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode()); // TENANT_NOT_EXIST - Mockito.when(userMapper.selectById(1)).thenReturn(getUser()); - result = resourcesService.updateResourceContent(getUser(), 1, "content"); + Mockito.when(userMapper.selectById(getUser().getId())).thenReturn(getUser()); + Mockito.when(tenantMapper.queryById(1)).thenReturn(null); + result = resourcesService.updateResourceContent(getUser(), "", "123", "content"); logger.info(result.toString()); Assertions.assertTrue(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST.getCode() == result.getCode()); // SUCCESS + try { + Mockito.when(storageOperate.getFileStatus("/dolphinscheduler/123/resources/ResourcesServiceTest.jar", + "", + "123", ResourceType.FILE)).thenReturn(getStorageEntityResource()); + } catch (Exception e) { + logger.error(e.getMessage() + " Resource path: {}", "", e); + } + + Mockito.when(Files.getFileExtension(Mockito.anyString())).thenReturn("jar"); + Mockito.when(FileUtils.getResourceViewSuffixes()).thenReturn("jar"); + Mockito.when(userMapper.selectById(getUser().getId())).thenReturn(getUser()); Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); Mockito.when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test"); Mockito.when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true); - result = resourcesService.updateResourceContent(getUser(), 1, "content"); + result = resourcesService.updateResourceContent(getUser(), + "/dolphinscheduler/123/resources/ResourcesServiceTest.jar", + "123", "content"); logger.info(result.toString()); Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); } @Test public void testDownloadResource() { - - Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null, - 1, ApiFuncIdentificationConstant.FILE_DOWNLOAD, serviceLogger)).thenReturn(true); - Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID, - new Object[]{1}, - 1, serviceLogger)).thenReturn(true); - Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); Mockito.when(userMapper.selectById(1)).thenReturn(getUser()); org.springframework.core.io.Resource resourceMock = Mockito.mock(org.springframework.core.io.Resource.class); try { // resource null - org.springframework.core.io.Resource resource = resourcesService.downloadResource(getUser(), 1); + org.springframework.core.io.Resource resource = resourcesService.downloadResource(getUser(), ""); Assertions.assertNull(resource); - Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource()); Mockito.when(org.apache.dolphinscheduler.api.utils.FileUtils.file2Resource(Mockito.any())) .thenReturn(resourceMock); - resource = resourcesService.downloadResource(getUser(), 1); + resource = resourcesService.downloadResource(getUser(), ""); Assertions.assertNotNull(resource); } catch (Exception e) { logger.error("DownloadResource error", e); @@ -1016,7 +917,6 @@ public class ResourcesServiceTest { } private Resource getResource() { - Resource resource = new Resource(); resource.setPid(-1); resource.setUserId(1); @@ -1027,6 +927,19 @@ public class ResourcesServiceTest { return resource; } + private StorageEntity getStorageEntityResource() { + StorageEntity entity = new StorageEntity(); + entity.setAlias("ResourcesServiceTest"); + entity.setFileName("ResourcesServiceTest"); + entity.setDirectory(false); + entity.setDescription(""); + entity.setUserName("123"); + entity.setType(ResourceType.FILE); + entity.setFullName("/dolphinscheduler/123/resources/ResourcesServiceTest"); + + return entity; + } + private Resource getResource(int resourceId) { Resource resource = new Resource(); @@ -1064,6 +977,19 @@ public class ResourcesServiceTest { return resource; } + private StorageEntity getStorageEntityUdfResource() { + StorageEntity entity = new StorageEntity(); + entity.setAlias("ResourcesServiceTest1.jar"); + entity.setFileName("ResourcesServiceTest1.jar"); + entity.setDirectory(false); + entity.setDescription(""); + entity.setUserName("123"); + entity.setType(ResourceType.UDF); + entity.setFullName("/dolphinscheduler/123/resources/ResourcesServiceTest1.jar"); + + return entity; + } + private UdfFunc getUdfFunc() { UdfFunc udfFunc = new UdfFunc(); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java index 0607d95c9c..32f24a69f9 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java @@ -40,6 +40,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper; +import org.apache.dolphinscheduler.service.storage.StorageOperate; import org.apache.commons.collections.CollectionUtils; @@ -96,6 +97,9 @@ public class TenantServiceTest { @Mock private ResourcePermissionCheckService resourcePermissionCheckService; + @Mock + private StorageOperate storageOperate; + private static final String tenantCode = "hayden"; private static final String tenantDesc = "This is the tenant desc"; private static final String queue = "queue"; diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java index 29ce54afad..9f12cc23c2 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java @@ -34,9 +34,11 @@ import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper; import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; +import org.apache.dolphinscheduler.service.storage.StorageOperate; import org.apache.commons.collections.CollectionUtils; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.Date; @@ -85,6 +87,9 @@ public class UdfFuncServiceTest { @Mock private UDFUserMapper udfUserMapper; + @Mock + private StorageOperate storageOperate; + @BeforeEach public void setUp() { mockedStaticPropertyUtils = Mockito.mockStatic(PropertyUtils.class); @@ -108,21 +113,27 @@ public class UdfFuncServiceTest { // hdfs not start Result result = udfFuncService.createUdfFunction(getLoginUser(), "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", - "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, Integer.MAX_VALUE); + "UdfFuncServiceTest", "UdfFuncServiceTest", "", UdfType.HIVE); logger.info(result.toString()); Assertions.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(), result.getMsg()); // resource not exist Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); + result = udfFuncService.createUdfFunction(getLoginUser(), "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", - "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, Integer.MAX_VALUE); + "UdfFuncServiceTest", "UdfFuncServiceTest", "", UdfType.HIVE); logger.info(result.toString()); Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg()); // success - Mockito.when(resourceMapper.selectById(1)).thenReturn(getResource()); + try { + Mockito.when(storageOperate.exists("String")).thenReturn(true); + } catch (IOException e) { + logger.error("AmazonServiceException when checking resource: String"); + } + result = udfFuncService.createUdfFunction(getLoginUser(), "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", - "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 1); + "UdfFuncServiceTest", "UdfFuncServiceTest", "", UdfType.HIVE); logger.info(result.toString()); Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); } @@ -154,7 +165,6 @@ public class UdfFuncServiceTest { Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); Mockito.when(udfFuncMapper.selectUdfById(1)).thenReturn(getUdfFunc()); - Mockito.when(resourceMapper.selectById(1)).thenReturn(getResource()); // UDF_FUNCTION_NOT_EXIST Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.UDF, null, 1, @@ -163,7 +173,7 @@ public class UdfFuncServiceTest { serviceLogger)).thenReturn(true); Result result = udfFuncService.updateUdfFunc(getLoginUser(), 12, "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", - "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 1); + "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, ""); logger.info(result.toString()); Assertions.assertTrue(Status.UDF_FUNCTION_NOT_EXIST.getCode() == result.getCode()); @@ -172,7 +182,7 @@ public class UdfFuncServiceTest { serviceLogger)).thenReturn(true); result = udfFuncService.updateUdfFunc(getLoginUser(), 1, "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", - "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 1); + "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, ""); logger.info(result.toString()); Assertions.assertTrue(Status.HDFS_NOT_STARTUP.getCode() == result.getCode()); @@ -185,16 +195,22 @@ public class UdfFuncServiceTest { Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); result = udfFuncService.updateUdfFunc(getLoginUser(), 11, "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", - "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 12); + "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, ""); logger.info(result.toString()); Assertions.assertTrue(Status.RESOURCE_NOT_EXIST.getCode() == result.getCode()); // success Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.UDF, null, 1, ApiFuncIdentificationConstant.UDF_FUNCTION_UPDATE, serviceLogger)).thenReturn(true); + try { + Mockito.when(storageOperate.exists("")).thenReturn(true); + } catch (IOException e) { + logger.error("AmazonServiceException when checking resource: "); + } + result = udfFuncService.updateUdfFunc(getLoginUser(), 11, "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", - "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 1); + "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, ""); logger.info(result.toString()); Assertions.assertTrue(Status.SUCCESS.getCode() == result.getCode()); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResourcesTask.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResourcesTask.java new file mode 100644 index 0000000000..799aa3e901 --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResourcesTask.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.dolphinscheduler.dao.entity; + +import org.apache.dolphinscheduler.spi.enums.ResourceType; + +import lombok.Data; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; + +@Data +@TableName("t_ds_relation_resources_task") +public class ResourcesTask { + + @TableId(value = "id", type = IdType.AUTO) + private Integer id; + + private String fullName; + + private int taskId; + + private ResourceType type; + + public ResourcesTask(int id, String fullName, int taskId, ResourceType type) { + this.id = id; + this.fullName = fullName; + this.taskId = taskId; + this.type = type; + } + + public ResourcesTask(int taskId, String fullName, ResourceType type) { + this.taskId = taskId; + this.fullName = fullName; + this.type = type; + } +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java index 45c70cb723..c90d4ea068 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java @@ -54,7 +54,6 @@ public interface ResourceMapper extends BaseMapper { /** * resource page * @param page page - * @param userId userId * @param id id * @param type type * @param searchVal searchVal diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceTaskMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceTaskMapper.java new file mode 100644 index 0000000000..bdd1542c49 --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceTaskMapper.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ + +package org.apache.dolphinscheduler.dao.mapper; + +import org.apache.dolphinscheduler.dao.entity.ResourcesTask; + +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; + +/** + * resource task relation mapper interface + */ +public interface ResourceTaskMapper extends BaseMapper { + + Integer existResourceByTaskIdNFullName(@Param("taskId") int task_id, @Param("fullName") String fullName); + + int deleteIds(@Param("resIds") Integer[] resIds); + + int updateResource(@Param("id") int id, @Param("fullName") String fullName); + + List selectBatchFullNames(@Param("fullNameArr") String[] fullNameArr); + + List selectSubfoldersFullNames(@Param("folderPath") String folderPath); +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapper.java index eb3d1f7e40..629cd9bf7b 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapper.java @@ -75,7 +75,7 @@ public interface TaskDefinitionMapper extends BaseMapper { List countDefinitionGroupByUser(@Param("projectCodes") Long[] projectCodes); /** - * list all resource ids + * list all resource ids and task_params containing resourceList * * @return task ids list */ diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java index 5573aedd8d..6bc8049c7d 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java @@ -49,7 +49,7 @@ public interface UdfFuncMapper extends BaseMapper { /** * udf function page * @param page page - * @param userId userId + * @param ids userId * @param searchVal searchVal * @return udf function IPage */ @@ -59,7 +59,7 @@ public interface UdfFuncMapper extends BaseMapper { /** * query udf function by type - * @param userId userId + * @param ids userId * @param type type * @return udf function list */ @@ -95,6 +95,13 @@ public interface UdfFuncMapper extends BaseMapper { */ List listUdfByResourceId(@Param("resourceIds") Integer[] resourceIds); + /** + * list UDF by resource fullName + * @param resourceFullNames resource fullName array + * @return UDF function list + */ + List listUdfByResourceFullName(@Param("resourceFullNames") String[] resourceFullNames); + /** * list authorized UDF by resource id * @param resourceIds resource id array diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtils.java index 69d9788b32..91484a476f 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtils.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtils.java @@ -66,4 +66,34 @@ public class ResourceProcessDefinitionUtils { return resourceResult; } + + public static Map> getResourceObjectMap(List> resourceList, + String objectName, Class clazz) { + // resourceId -> task ids or code depends on the objectName + Map> resourceResult = new HashMap<>(); + + if (CollectionUtils.isNotEmpty(resourceList)) { + for (Map resourceMap : resourceList) { + // get resName and resource_id_news, td_id + T taskId = (T) resourceMap.get(objectName); + String[] resourceIds = ((String) resourceMap.get("resource_ids")) + .split(","); + + Set resourceIdSet = + Arrays.stream(resourceIds).map(Integer::parseInt).collect(Collectors.toSet()); + for (Integer resourceId : resourceIdSet) { + Set codeSet; + if (resourceResult.containsKey(resourceId)) { + codeSet = resourceResult.get(resourceId); + } else { + codeSet = new HashSet<>(); + } + codeSet.add(taskId); + resourceResult.put(resourceId, codeSet); + } + } + } + + return resourceResult; + } } diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml index a69150b3f7..f654b0bd77 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml @@ -168,7 +168,7 @@ + select + id + from t_ds_relation_resources_task + where full_name = #{fullName} and task_id = #{taskId} + + + + + + UPDATE t_ds_relation_resources_task SET full_name=#{fullName} WHERE id=#{id} + + + + delete from t_ds_relation_resources_task where id in + + #{i} + + + + + \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapper.xml index 9c7d6a7b59..2c6a5cbe8a 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapper.xml @@ -96,8 +96,8 @@ #{taskDefinition.projectCode},#{taskDefinition.userId},#{taskDefinition.taskType},#{taskDefinition.taskParams},#{taskDefinition.flag}, #{taskDefinition.taskPriority},#{taskDefinition.workerGroup},#{taskDefinition.environmentCode},#{taskDefinition.failRetryTimes}, #{taskDefinition.failRetryInterval},#{taskDefinition.timeoutFlag},#{taskDefinition.timeoutNotifyStrategy},#{taskDefinition.timeout}, - #{taskDefinition.delayTime},#{taskDefinition.resourceIds},#{taskDefinition.createTime},#{taskDefinition.updateTime}, #{taskDefinition.taskGroupId}, - #{taskDefinition.taskExecuteType}) + #{taskDefinition.delayTime},#{taskDefinition.resourceIds},#{taskDefinition.createTime},#{taskDefinition.updateTime}, + #{taskDefinition.taskGroupId}, #{taskDefinition.taskExecuteType}) +