From 5b6b0ceb31aab5ddb79ae4b2551d104acc6d9f00 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Sat, 23 Dec 2023 22:47:59 +0800 Subject: [PATCH] Fix resource file usage(Delete Resource/ResourceUser which is deprecated) (#15355) --- .../api/controller/ResourcesController.java | 42 -- .../api/controller/UsersController.java | 24 - .../ResourcePermissionCheckServiceImpl.java | 45 -- .../api/service/ResourcesService.java | 27 -- .../api/service/UsersService.java | 10 - .../service/impl/ResourcesServiceImpl.java | 249 +--------- .../api/service/impl/UdfFuncServiceImpl.java | 4 - .../api/service/impl/UsersServiceImpl.java | 122 ----- .../api/controller/ProjectControllerTest.java | 3 +- .../controller/ResourcesControllerTest.java | 22 - .../controller/SchedulerControllerTest.java | 4 +- .../api/controller/UsersControllerTest.java | 18 - .../v2/ProjectV2ControllerTest.java | 3 +- .../permission/FilePermissionCheckTest.java | 106 ----- .../api/service/ResourcesServiceTest.java | 175 +------ .../api/service/UdfFuncServiceTest.java | 16 - .../api/service/UsersServiceTest.java | 46 -- .../common/enums/AuthorizationType.java | 3 + .../common/utils/JSONUtils.java | 4 +- .../dolphinscheduler/dao/entity/Resource.java | 166 ------- .../dao/entity/ResourcesUser.java | 61 --- .../dao/entity/TaskDefinition.java | 3 +- .../dao/entity/TaskInstance.java | 3 - .../dao/mapper/ResourceMapper.java | 157 ------- .../dao/mapper/ResourceUserMapper.java | 60 --- .../dao/utils/TaskCacheUtils.java | 2 +- .../dao/utils/TaskInstanceUtils.java | 1 - .../dao/mapper/ResourceMapper.xml | 198 -------- .../resources/sql/dolphinscheduler_h2.sql | 2 + .../resources/sql/dolphinscheduler_mysql.sql | 2 + .../sql/dolphinscheduler_postgresql.sql | 4 +- .../dao/mapper/ResourceMapperTest.java | 440 ------------------ .../dao/mapper/ResourceUserMapperTest.java | 111 ----- .../builder/TaskExecutionContextBuilder.java | 7 - .../runner/StreamTaskExecuteRunnable.java | 45 -- .../runner/TaskExecutionContextFactory.java | 23 - .../service/process/ProcessService.java | 8 - .../service/process/ProcessServiceImpl.java | 112 +---- .../service/process/ProcessServiceTest.java | 5 - .../storage/abs/AbsStorageOperator.java | 9 +- .../plugin/storage/api/StorageOperate.java | 12 +- .../storage/gcs/GcsStorageOperator.java | 9 +- .../storage/hdfs/HdfsStorageOperator.java | 9 +- .../storage/obs/ObsStorageOperator.java | 9 +- .../storage/oss/OssStorageOperator.java | 8 +- .../plugin/storage/s3/S3StorageOperator.java | 8 +- .../task/api/AbstractCommandExecutor.java | 7 +- .../plugin/task/api/TaskExecutionContext.java | 7 +- .../plugin/task/api/model/ResourceInfo.java | 4 +- .../resource/UdfFuncParameters.java | 4 - .../task/api/resource/ResourceContext.java | 67 +++ .../BaseLinuxShellInterceptorBuilder.java | 7 +- .../task/datax/DataxParametersTest.java | 3 +- .../plugin/task/flink/FlinkArgsUtilsTest.java | 15 +- .../plugin/task/flink/FlinkArgsUtils.java | 4 +- .../plugin/task/flink/FlinkArgsUtilsTest.java | 23 +- .../plugin/task/hivecli/HiveCliTask.java | 4 +- .../plugin/task/hivecli/HiveCliTaskTest.java | 13 +- .../plugin/task/java/JavaTask.java | 16 +- .../plugin/task/java/JavaTaskTest.java | 47 +- .../plugin/task/mr/MapReduceArgsUtils.java | 4 +- .../plugin/task/spark/SparkTask.java | 8 +- .../plugin/task/spark/SparkTaskTest.java | 14 +- .../plugin/task/sql/SqlTask.java | 9 +- .../tools/resource/MigrateResource.java | 4 +- .../resource/MigrateResourceService.java | 54 ++- .../worker/runner/WorkerTaskExecutor.java | 30 +- ...ls.java => TaskExecutionContextUtils.java} | 88 ++-- .../worker/utils/TaskFilesTransferUtils.java | 2 +- 69 files changed, 334 insertions(+), 2497 deletions(-) delete mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/permission/FilePermissionCheckTest.java delete mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Resource.java delete mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResourcesUser.java delete mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java delete mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.java delete mode 100644 dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml delete mode 100644 dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java delete mode 100644 dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapperTest.java create mode 100644 dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/resource/ResourceContext.java rename dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/{TaskExecutionCheckerUtils.java => TaskExecutionContextUtils.java} (59%) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java index 6fff27acaf..2cdead61ca 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java @@ -17,9 +17,7 @@ package org.apache.dolphinscheduler.api.controller; -import static org.apache.dolphinscheduler.api.enums.Status.AUTHORIZED_FILE_RESOURCE_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.AUTHORIZED_UDF_FUNCTION_ERROR; -import static org.apache.dolphinscheduler.api.enums.Status.AUTHORIZE_RESOURCE_TREE; import static org.apache.dolphinscheduler.api.enums.Status.CREATE_RESOURCE_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.CREATE_RESOURCE_FILE_ON_LINE_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.CREATE_UDF_FUNCTION_ERROR; @@ -625,46 +623,6 @@ public class ResourcesController extends BaseController { return udfFuncService.delete(loginUser, udfFuncId); } - /** - * authorized file resource list - * - * @param loginUser login user - * @param userId user id - * @return authorized result - */ - @Operation(summary = "authorizedFile", description = "AUTHORIZED_FILE_NOTES") - @Parameters({ - @Parameter(name = "userId", description = "USER_ID", required = true, schema = @Schema(implementation = int.class, example = "100")) - }) - @GetMapping(value = "/authed-file") - @ResponseStatus(HttpStatus.CREATED) - @ApiException(AUTHORIZED_FILE_RESOURCE_ERROR) - public Result authorizedFile(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam("userId") Integer userId) { - Map result = resourceService.authorizedFile(loginUser, userId); - return returnDataList(result); - } - - /** - * unauthorized file resource list - * - * @param loginUser login user - * @param userId user id - * @return unauthorized result code - */ - @Operation(summary = "authorizeResourceTree", description = "AUTHORIZE_RESOURCE_TREE_NOTES") - @Parameters({ - @Parameter(name = "userId", description = "USER_ID", required = true, schema = @Schema(implementation = int.class, example = "100")) - }) - @GetMapping(value = "/authed-resource-tree") - @ResponseStatus(HttpStatus.CREATED) - @ApiException(AUTHORIZE_RESOURCE_TREE) - public Result authorizeResourceTree(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam("userId") Integer userId) { - Map result = resourceService.authorizeResourceTree(loginUser, userId); - return returnDataList(result); - } - /** * unauthorized udf function * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java index a5a4486e6c..cc40c6a98c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/UsersController.java @@ -24,7 +24,6 @@ import static org.apache.dolphinscheduler.api.enums.Status.GET_USER_INFO_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.GRANT_DATASOURCE_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.GRANT_K8S_NAMESPACE_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.GRANT_PROJECT_ERROR; -import static org.apache.dolphinscheduler.api.enums.Status.GRANT_RESOURCE_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.GRANT_UDF_FUNCTION_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.QUERY_USER_LIST_PAGING_ERROR; import static org.apache.dolphinscheduler.api.enums.Status.REVOKE_PROJECT_ERROR; @@ -318,29 +317,6 @@ public class UsersController extends BaseController { return returnDataList(result); } - /** - * grant resource - * - * @param loginUser login user - * @param userId user id - * @param resourceIds resource id array - * @return grant result code - */ - @Operation(summary = "grantResource", description = "GRANT_RESOURCE_NOTES") - @Parameters({ - @Parameter(name = "userId", description = "USER_ID", required = true, schema = @Schema(implementation = int.class, example = "100")), - @Parameter(name = "resourceIds", description = "RESOURCE_IDS", required = true, schema = @Schema(implementation = String.class)) - }) - @PostMapping(value = "/grant-file") - @ResponseStatus(HttpStatus.OK) - @ApiException(GRANT_RESOURCE_ERROR) - public Result grantResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam(value = "userId") int userId, - @RequestParam(value = "resourceIds") String resourceIds) { - Map result = usersService.grantResources(loginUser, userId, resourceIds); - return returnDataList(result); - } - /** * grant udf function * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckServiceImpl.java index b35a89cfdf..355c2257b9 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckServiceImpl.java @@ -44,7 +44,6 @@ import org.apache.dolphinscheduler.dao.entity.Environment; import org.apache.dolphinscheduler.dao.entity.K8sNamespace; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.Queue; -import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.TaskGroup; import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.UdfFunc; @@ -58,17 +57,12 @@ import org.apache.dolphinscheduler.dao.mapper.EnvironmentMapper; import org.apache.dolphinscheduler.dao.mapper.K8sNamespaceMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.QueueMapper; -import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; -import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper; import org.apache.dolphinscheduler.dao.mapper.TaskGroupMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper; import org.apache.dolphinscheduler.service.process.ProcessService; -import org.apache.commons.collections4.CollectionUtils; - -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashSet; @@ -207,45 +201,6 @@ public class ResourcePermissionCheckServiceImpl } } - @Component - public static class FilePermissionCheck implements ResourceAcquisitionAndPermissionCheck { - - private final ResourceMapper resourceMapper; - - private final ResourceUserMapper resourceUserMapper; - - public FilePermissionCheck(ResourceMapper resourceMapper, ResourceUserMapper resourceUserMapper) { - this.resourceMapper = resourceMapper; - this.resourceUserMapper = resourceUserMapper; - } - - @Override - public List authorizationTypes() { - return Arrays.asList(AuthorizationType.RESOURCE_FILE_ID, AuthorizationType.UDF_FILE); - } - - @Override - public Set listAuthorizedResourceIds(int userId, Logger logger) { - List relationResources; - if (userId == 0) { - relationResources = new ArrayList<>(); - } else { - // query resource relation - List resIds = resourceUserMapper.queryResourcesIdListByUserIdAndPerm(userId, 0); - relationResources = CollectionUtils.isEmpty(resIds) ? new ArrayList<>() - : resourceMapper.queryResourceListById(resIds); - } - List ownResourceList = resourceMapper.queryResourceListAuthored(userId, -1); - relationResources.addAll(ownResourceList); - return relationResources.stream().map(Resource::getId).collect(toSet()); - } - - @Override - public boolean permissionCheck(int userId, String permissionKey, Logger logger) { - return true; - } - } - @Component public static class UdfFuncPermissionCheck implements ResourceAcquisitionAndPermissionCheck { diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index ac078e7ff2..7ed99c50c9 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -193,15 +193,6 @@ public interface ResourcesService { */ org.springframework.core.io.Resource downloadResource(User loginUser, String fullName) throws IOException; - /** - * list all file - * - * @param loginUser login user - * @param userId user id - * @return unauthorized result code - */ - Map authorizeResourceTree(User loginUser, Integer userId); - /** * Get resource by given resource type and full name. * Useful in Python API create task which need processDefinition information. @@ -219,15 +210,6 @@ public interface ResourcesService { */ DeleteDataTransferResponse deleteDataTransferData(User loginUser, Integer days); - /** - * unauthorized file - * - * @param loginUser login user - * @param userId user id - * @return unauthorized result code - */ - Map unauthorizedFile(User loginUser, Integer userId); - /** * unauthorized udf function * @@ -246,15 +228,6 @@ public interface ResourcesService { */ Map authorizedUDFFunction(User loginUser, Integer userId); - /** - * authorized file - * - * @param loginUser login user - * @param userId user id - * @return authorized result - */ - Map authorizedFile(User loginUser, Integer userId); - /** * get resource by id * @param fullName resource full name diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java index e0f984eb13..45e5964825 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java @@ -183,16 +183,6 @@ public interface UsersService { */ Map revokeProject(User loginUser, int userId, long projectCode); - /** - * grant resource - * - * @param loginUser login user - * @param userId user id - * @param resourceIds resource id array - * @return grant result code - */ - Map grantResources(User loginUser, int userId, String resourceIds); - /** * grant udf function * diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java index 0a8e21071f..89f281fa7a 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java @@ -27,7 +27,6 @@ import static org.apache.dolphinscheduler.common.constants.Constants.JAR; import static org.apache.dolphinscheduler.common.constants.Constants.PERIOD; import org.apache.dolphinscheduler.api.dto.resources.DeleteDataTransferResponse; -import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent; import org.apache.dolphinscheduler.api.dto.resources.filter.ResourceFilter; import org.apache.dolphinscheduler.api.dto.resources.visitor.ResourceTreeVisitor; import org.apache.dolphinscheduler.api.dto.resources.visitor.Visitor; @@ -45,14 +44,9 @@ import org.apache.dolphinscheduler.common.enums.ResUploadType; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils; -import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.dolphinscheduler.dao.entity.User; -import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionLogMapper; -import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationMapper; -import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; -import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper; @@ -91,8 +85,6 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.web.multipart.MultipartFile; import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.google.common.base.Joiner; import com.google.common.io.Files; /** @@ -102,9 +94,6 @@ import com.google.common.io.Files; @Slf4j public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesService { - @Autowired - private ResourceMapper resourcesMapper; - @Autowired private UdfFuncMapper udfFunctionMapper; @@ -114,15 +103,6 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe @Autowired private UserMapper userMapper; - @Autowired - private ResourceUserMapper resourceUserMapper; - - @Autowired - private ProcessDefinitionLogMapper processDefinitionLogMapper; - - @Autowired - private ProcessTaskRelationMapper processTaskRelationMapper; - @Autowired(required = false) private StorageOperate storageOperate; @@ -267,36 +247,6 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe return result; } - /** - * update the folder's size of the resource - * - * @param resource the current resource - * @param size size - */ - private void updateParentResourceSize(Resource resource, long size) { - if (resource.getSize() > 0) { - String[] splits = resource.getFullName().split("/"); - for (int i = 1; i < splits.length; i++) { - String parentFullName = Joiner.on("/").join(Arrays.copyOfRange(splits, 0, i)); - if (StringUtils.isNotBlank(parentFullName)) { - List resources = - resourcesMapper.queryResource(parentFullName, resource.getType().ordinal()); - if (CollectionUtils.isNotEmpty(resources)) { - Resource parentResource = resources.get(0); - if (parentResource.getSize() + size >= 0) { - parentResource.setSize(parentResource.getSize() + size); - } else { - parentResource.setSize(0L); - } - resourcesMapper.updateById(parentResource); - log.info("Resource size update complete, resourceFullName:{}, newSize:{}.", - parentResource.getFullName(), parentResource.getSize()); - } - } - } - } - } - /** * check resource is exists * @@ -647,14 +597,10 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe } if (!storageOperate.mkdir(tenantCode, fullName)) { - log.error("create resource directory {} failed", fullName); - putMsg(result, Status.STORE_OPERATE_CREATE_ERROR); - // throw new ServiceException(String.format("create resource directory: %s failed.", fullName)); + throw new ServiceException(String.format("Create resource directory: %s failed.", fullName)); } putMsg(result, Status.SUCCESS); } catch (Exception e) { - log.error("create resource directory {} failed", fullName); - putMsg(result, Status.STORE_OPERATE_CREATE_ERROR); throw new ServiceException(String.format("create resource directory: %s failed.", fullName)); } } @@ -817,25 +763,6 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe return result; } - /** - * transform resource object into StorageEntity object - * - * @param resource a resource object - * @return a storageEntity object - */ - private StorageEntity createStorageEntityBasedOnResource(Resource resource) { - StorageEntity entity = new StorageEntity(); - entity.setFullName(resource.getFullName()); - entity.setPfullName(resource.getPid() == -1 ? "" - : resourcesMapper.selectById(resource.getPid()).getFullName()); - entity.setDirectory(resource.isDirectory()); - entity.setAlias(resource.getAlias()); - entity.setId(resource.getId()); - entity.setType(resource.getType()); - - return entity; - } - /** * delete resource * @@ -1358,7 +1285,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe log.info("Resource path is {}, download local filename is {}", alias, localFileName); try { - storageOperate.download(tenantCode, fullName, localFileName, true); + storageOperate.download(fullName, localFileName, true); ApiServerMetrics.recordApiResourceDownloadSize(java.nio.file.Files.size(Paths.get(localFileName))); return org.apache.dolphinscheduler.api.utils.FileUtils.file2Resource(localFileName); } catch (IOException e) { @@ -1368,46 +1295,6 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe } } - /** - * list all file - * - * @param loginUser login user - * @param userId user id - * @return unauthorized result code - */ - @Override - public Map authorizeResourceTree(User loginUser, Integer userId) { - Map result = new HashMap<>(); - if (resourcePermissionCheckService.functionDisabled()) { - putMsg(result, Status.FUNCTION_DISABLED); - return result; - } - - List resourceList; - if (isAdmin(loginUser)) { - // admin gets all resources except userId - resourceList = resourcesMapper.queryResourceExceptUserId(userId); - } else { - // non-admins users get their own resources - resourceList = resourcesMapper.queryResourceListAuthored(loginUser.getId(), -1); - } - List list; - if (CollectionUtils.isNotEmpty(resourceList)) { - // Transform into StorageEntity for compatibility - List transformedResourceList = resourceList.stream() - .map(this::createStorageEntityBasedOnResource) - .collect(Collectors.toList()); - Visitor visitor = new ResourceTreeVisitor(transformedResourceList); - list = visitor.visit("").getChildren(); - } else { - list = new ArrayList<>(0); - } - - result.put(Constants.DATA_LIST, list); - putMsg(result, Status.SUCCESS); - return result; - } - @Override public StorageEntity queryFileStatus(String userName, String fileName) throws Exception { // TODO: It is used in PythonGateway, should be revised @@ -1469,44 +1356,6 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe return result; } - /** - * unauthorized file - * - * @param loginUser login user - * @param userId user id - * @return unauthorized result code - */ - @Override - public Map unauthorizedFile(User loginUser, Integer userId) { - Map result = new HashMap<>(); - - List resourceList; - if (isAdmin(loginUser)) { - // admin gets all resources except userId - resourceList = resourcesMapper.queryResourceExceptUserId(userId); - } else { - // non-admins users get their own resources - resourceList = resourcesMapper.queryResourceListAuthored(loginUser.getId(), -1); - } - List list; - if (resourceList != null && !resourceList.isEmpty()) { - Set resourceSet = new HashSet<>(resourceList); - List authedResourceList = queryResourceList(userId, Constants.AUTHORIZE_WRITABLE_PERM); - getAuthorizedResourceList(resourceSet, authedResourceList); - list = new ArrayList<>(resourceSet); - } else { - list = new ArrayList<>(0); - } - // Transform into StorageEntity for compatibility - List transformedResourceList = list.stream() - .map(this::createStorageEntityBasedOnResource) - .collect(Collectors.toList()); - Visitor visitor = new ResourceTreeVisitor(transformedResourceList); - result.put(Constants.DATA_LIST, visitor.visit("").getChildren()); - putMsg(result, Status.SUCCESS); - return result; - } - /** * unauthorized udf function * @@ -1565,37 +1414,6 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe return result; } - /** - * authorized file - * - * @param loginUser login user - * @param userId user id - * @return authorized result - */ - @Override - public Map authorizedFile(User loginUser, Integer userId) { - Map result = new HashMap<>(); - if (resourcePermissionCheckService.functionDisabled()) { - putMsg(result, Status.FUNCTION_DISABLED); - return result; - } - - List authedResources = queryResourceList(userId, Constants.AUTHORIZE_WRITABLE_PERM); - // Transform into StorageEntity for compatibility - List transformedResourceList = authedResources.stream() - .map(this::createStorageEntityBasedOnResource) - .collect(Collectors.toList()); - Visitor visitor = new ResourceTreeVisitor(transformedResourceList); - String visit = JSONUtils.toJsonString(visitor.visit(""), SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS); - log.info(visit); - String jsonTreeStr = - JSONUtils.toJsonString(visitor.visit("").getChildren(), SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS); - log.info(jsonTreeStr); - result.put(Constants.DATA_LIST, visitor.visit("").getChildren()); - putMsg(result, Status.SUCCESS); - return result; - } - /** * get resource base dir * @@ -1649,69 +1467,6 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe } } - /** - * list all children id - * - * @param resource resource - * @param containSelf whether add self to children list - * @return all children id - */ - List listAllChildren(Resource resource, boolean containSelf) { - List childList = new ArrayList<>(); - if (resource.getId() != null && containSelf) { - childList.add(resource.getId()); - } - - if (resource.isDirectory()) { - listAllChildren(resource.getId(), childList); - } - return childList; - } - - /** - * list all children id - * - * @param resourceId resource id - * @param childList child list - */ - void listAllChildren(int resourceId, List childList) { - List children = resourcesMapper.listChildren(resourceId); - for (int childId : children) { - childList.add(childId); - listAllChildren(childId, childList); - } - } - - /** - * query authored resource list (own and authorized) - * - * @param loginUser login user - * @param type ResourceType - * @return all authored resource list - */ - private List queryAuthoredResourceList(User loginUser, ResourceType type) { - Set resourceIds = resourcePermissionCheckService - .userOwnedResourceIdsAcquisition(checkResourceType(type), loginUser.getId(), log); - if (resourceIds.isEmpty()) { - return Collections.emptyList(); - } - List resources = resourcesMapper.selectBatchIds(resourceIds); - resources = resources.stream().filter(rs -> rs.getType() == type).collect(Collectors.toList()); - return resources; - } - - /** - * query resource list by userId and perm - * - * @param userId userId - * @param perm perm - * @return resource list - */ - private List queryResourceList(Integer userId, int perm) { - List resIds = resourceUserMapper.queryResourcesIdListByUserIdAndPerm(userId, perm); - return CollectionUtils.isEmpty(resIds) ? new ArrayList<>() : resourcesMapper.queryResourceListById(resIds); - } - private AuthorizationType checkResourceType(ResourceType type) { return type.equals(ResourceType.FILE) ? AuthorizationType.RESOURCE_FILE_ID : AuthorizationType.UDF_FILE; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java index 05f8f6a20e..1bf7d23a6b 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java @@ -26,7 +26,6 @@ import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.dolphinscheduler.dao.entity.User; -import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper; import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; @@ -56,9 +55,6 @@ import com.baomidou.mybatisplus.extension.plugins.pagination.Page; @Slf4j public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncService { - @Autowired - private ResourceMapper resourceMapper; - @Autowired private UdfFuncMapper udfFuncMapper; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java index 3b54df1725..0c331a4c0c 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java @@ -38,8 +38,6 @@ import org.apache.dolphinscheduler.dao.entity.DatasourceUser; import org.apache.dolphinscheduler.dao.entity.K8sNamespaceUser; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.ProjectUser; -import org.apache.dolphinscheduler.dao.entity.Resource; -import org.apache.dolphinscheduler.dao.entity.ResourcesUser; import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.UDFUser; import org.apache.dolphinscheduler.dao.entity.User; @@ -47,15 +45,11 @@ import org.apache.dolphinscheduler.dao.mapper.AccessTokenMapper; import org.apache.dolphinscheduler.dao.mapper.AlertGroupMapper; import org.apache.dolphinscheduler.dao.mapper.DataSourceUserMapper; import org.apache.dolphinscheduler.dao.mapper.K8sNamespaceUserMapper; -import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectUserMapper; -import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; -import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper; -import org.apache.dolphinscheduler.dao.utils.ResourceProcessDefinitionUtils; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; import org.apache.commons.collections4.CollectionUtils; @@ -103,12 +97,6 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService { @Autowired private ProjectUserMapper projectUserMapper; - @Autowired - private ResourceUserMapper resourceUserMapper; - - @Autowired - private ResourceMapper resourceMapper; - @Autowired private DataSourceUserMapper datasourceUserMapper; @@ -118,9 +106,6 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService { @Autowired private AlertGroupMapper alertGroupMapper; - @Autowired - private ProcessDefinitionMapper processDefinitionMapper; - @Autowired private ProjectMapper projectMapper; @@ -754,113 +739,6 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService { return result; } - /** - * grant resource - * - * @param loginUser login user - * @param userId user id - * @param resourceIds resource id array - * @return grant result code - */ - @Override - @Transactional - public Map grantResources(User loginUser, int userId, String resourceIds) { - Map result = new HashMap<>(); - - if (resourcePermissionCheckService.functionDisabled()) { - putMsg(result, Status.FUNCTION_DISABLED); - return result; - } - User user = userMapper.selectById(userId); - if (user == null) { - log.error("User does not exist, userId:{}.", userId); - putMsg(result, Status.USER_NOT_EXIST, userId); - return result; - } - - Set needAuthorizeResIds = new HashSet<>(); - if (StringUtils.isNotBlank(resourceIds)) { - String[] resourceFullIdArr = resourceIds.split(","); - // need authorize resource id set - for (String resourceFullId : resourceFullIdArr) { - String[] resourceIdArr = resourceFullId.split("-"); - for (int i = 0; i <= resourceIdArr.length - 1; i++) { - int resourceIdValue = Integer.parseInt(resourceIdArr[i]); - needAuthorizeResIds.add(resourceIdValue); - } - } - } - - // get the authorized resource id list by user id - List resIds = - resourceUserMapper.queryResourcesIdListByUserIdAndPerm(userId, Constants.AUTHORIZE_WRITABLE_PERM); - List oldAuthorizedRes = - CollectionUtils.isEmpty(resIds) ? new ArrayList<>() : resourceMapper.queryResourceListById(resIds); - // if resource type is UDF,need check whether it is bound by UDF function - Set oldAuthorizedResIds = oldAuthorizedRes.stream().map(Resource::getId).collect(Collectors.toSet()); - - // get the unauthorized resource id list - oldAuthorizedResIds.removeAll(needAuthorizeResIds); - - if (CollectionUtils.isNotEmpty(oldAuthorizedResIds)) { - - // get all resource id of process definitions those are released - List> list = processDefinitionMapper.listResourcesByUser(userId); - Map> resourceProcessMap = - ResourceProcessDefinitionUtils.getResourceProcessDefinitionMap(list); - Set resourceIdSet = resourceProcessMap.keySet(); - - resourceIdSet.retainAll(oldAuthorizedResIds); - if (CollectionUtils.isNotEmpty(resourceIdSet)) { - for (Integer resId : resourceIdSet) { - log.error("Resource id:{} is used by process definition {}", resId, - resourceProcessMap.get(resId)); - } - putMsg(result, Status.RESOURCE_IS_USED); - return result; - } - - } - - resourceUserMapper.deleteResourceUser(userId, 0); - - if (check(result, StringUtils.isEmpty(resourceIds), Status.SUCCESS)) { - log.warn("Parameter resourceIds is empty."); - return result; - } - - for (int resourceIdValue : needAuthorizeResIds) { - Resource resource = resourceMapper.selectById(resourceIdValue); - if (resource == null) { - log.error("Resource does not exist, resourceId:{}.", resourceIdValue); - putMsg(result, Status.RESOURCE_NOT_EXIST); - return result; - } - - Date now = new Date(); - ResourcesUser resourcesUser = new ResourcesUser(); - resourcesUser.setUserId(userId); - resourcesUser.setResourcesId(resourceIdValue); - if (resource.isDirectory()) { - resourcesUser.setPerm(Constants.AUTHORIZE_READABLE_PERM); - } else { - resourcesUser.setPerm(Constants.AUTHORIZE_WRITABLE_PERM); - } - - resourcesUser.setCreateTime(now); - resourcesUser.setUpdateTime(now); - resourceUserMapper.insert(resourcesUser); - - } - - log.info("User is granted permission for resources, userId:{}, resourceIds:{}.", user.getId(), - needAuthorizeResIds); - - putMsg(result, Status.SUCCESS); - - return result; - } - /** * grant udf function * diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java index 1caf333ab1..e14cd5ac38 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ProjectControllerTest.java @@ -23,7 +23,6 @@ import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.dao.entity.Project; -import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; @@ -97,7 +96,7 @@ public class ProjectControllerTest { int pageSize = 10; String searchVal = ""; - Result result = Result.success(new PageInfo(1, 10)); + Result result = Result.success(new PageInfo(1, 10)); Mockito.when(projectService.queryProjectListPaging(user, pageSize, pageNo, searchVal)).thenReturn(result); Result response = projectController.queryProjectListPaging(user, searchVal, pageSize, pageNo); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java index 8a41591e28..1cf4272903 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java @@ -397,28 +397,6 @@ public class ResourcesControllerTest extends AbstractControllerTest { logger.info(mvcResult.getResponse().getContentAsString()); } - @Test - public void testAuthorizedFile() throws Exception { - Map mockResult = new HashMap<>(); - mockResult.put(Constants.STATUS, Status.SUCCESS); - Mockito.when(resourcesService.authorizedFile(Mockito.any(), Mockito.anyInt())).thenReturn(mockResult); - - MultiValueMap paramsMap = new LinkedMultiValueMap<>(); - paramsMap.add("userId", "2"); - - MvcResult mvcResult = mockMvc.perform(get("/resources/authed-file") - .header(SESSION_ID, sessionId) - .params(paramsMap)) - .andExpect(status().isCreated()) - .andExpect(content().contentType(MediaType.APPLICATION_JSON)) - .andReturn(); - - Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - - Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); - } - @Test public void testAuthorizedUDFFunction() throws Exception { Map mockResult = new HashMap<>(); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/SchedulerControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/SchedulerControllerTest.java index 39dd645247..5644065a33 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/SchedulerControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/SchedulerControllerTest.java @@ -34,7 +34,7 @@ import org.apache.dolphinscheduler.common.enums.FailureStrategy; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.dao.entity.Resource; +import org.apache.dolphinscheduler.dao.entity.Schedule; import org.apache.dolphinscheduler.dao.entity.User; import org.junit.jupiter.api.Assertions; @@ -163,7 +163,7 @@ public class SchedulerControllerTest extends AbstractControllerTest { paramsMap.add("pageNo", "1"); paramsMap.add("pageSize", "30"); - PageInfo pageInfo = new PageInfo<>(1, 10); + PageInfo pageInfo = new PageInfo<>(1, 10); Result mockResult = Result.success(pageInfo); Mockito.when(schedulerService.querySchedule(isA(User.class), isA(Long.class), isA(Long.class), diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/UsersControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/UsersControllerTest.java index 77b5d85442..1f6eca0572 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/UsersControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/UsersControllerTest.java @@ -145,24 +145,6 @@ public class UsersControllerTest extends AbstractControllerTest { logger.info(mvcResult.getResponse().getContentAsString()); } - @Test - public void testGrantResource() throws Exception { - MultiValueMap paramsMap = new LinkedMultiValueMap<>(); - paramsMap.add("userId", "32"); - paramsMap.add("resourceIds", "5"); - - MvcResult mvcResult = mockMvc.perform(post("/users/grant-file") - .header(SESSION_ID, sessionId) - .params(paramsMap)) - .andExpect(status().isOk()) - .andExpect(content().contentType(MediaType.APPLICATION_JSON)) - .andReturn(); - - Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); - Assertions.assertEquals(Status.USER_NOT_EXIST.getCode(), result.getCode().intValue()); - logger.info(mvcResult.getResponse().getContentAsString()); - } - @Test public void testGrantUDFFunc() throws Exception { MultiValueMap paramsMap = new LinkedMultiValueMap<>(); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/v2/ProjectV2ControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/v2/ProjectV2ControllerTest.java index f11f8fa41b..eb47639af6 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/v2/ProjectV2ControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/v2/ProjectV2ControllerTest.java @@ -27,7 +27,6 @@ import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.dao.entity.Project; -import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; @@ -101,7 +100,7 @@ public class ProjectV2ControllerTest { projectQueryReq.setPageNo(1); projectQueryReq.setPageSize(10); - Result result = Result.success(new PageInfo(1, 10)); + Result result = Result.success(new PageInfo(1, 10)); Mockito.when(projectService.queryProjectListPaging(user, projectQueryReq.getPageSize(), projectQueryReq.getPageNo(), projectQueryReq.getSearchVal())).thenReturn(result); Result response = projectV2Controller.queryProjectListPaging(user, projectQueryReq); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/permission/FilePermissionCheckTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/permission/FilePermissionCheckTest.java deleted file mode 100644 index ed060f5398..0000000000 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/permission/FilePermissionCheckTest.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.api.permission; - -import org.apache.dolphinscheduler.common.enums.AuthorizationType; -import org.apache.dolphinscheduler.common.enums.UserType; -import org.apache.dolphinscheduler.dao.entity.Resource; -import org.apache.dolphinscheduler.dao.entity.User; -import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; -import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper; - -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@ExtendWith(MockitoExtension.class) -public class FilePermissionCheckTest { - - private static final Logger logger = LoggerFactory.getLogger(FilePermissionCheckTest.class); - @InjectMocks - private ResourcePermissionCheckServiceImpl.FilePermissionCheck filePermissionCheck; - - @Mock - private ResourceMapper resourceMapper; - - @Mock - private ResourceUserMapper resourceUserMapper; - - @Test - public void testPermissionCheck() { - User user = getLoginUser(); - Assertions.assertTrue(filePermissionCheck.permissionCheck(user.getId(), null, logger)); - } - - @Test - public void testAuthorizationTypes() { - List authorizationTypes = filePermissionCheck.authorizationTypes(); - Assertions.assertEquals(Arrays.asList(AuthorizationType.RESOURCE_FILE_ID, AuthorizationType.UDF_FILE), - authorizationTypes); - } - - @Test - public void testListAuthorizedResourceIds() { - // ADMIN_USER - User user = getAdminUser(); - Resource resource = new Resource(); - Set ids = new HashSet(); - ids.add(resource.getId()); - List resources = Arrays.asList(resource); - - Mockito.when(resourceMapper.queryResourceListAuthored(user.getId(), -1)).thenReturn(resources); - Assertions.assertEquals(ids, filePermissionCheck.listAuthorizedResourceIds(user.getId(), logger)); - - // GENERAL_USER - user = getLoginUser(); - Resource resource1 = new Resource(); - ids.add(resource1.getId()); - Mockito.when(resourceMapper.queryResourceListAuthored(user.getId(), -1)).thenReturn(resources); - Mockito.when(resourceUserMapper.queryResourcesIdListByUserIdAndPerm(user.getId(), 0)) - .thenReturn(Arrays.asList(resource1.getId())); - - Assertions.assertEquals(ids, filePermissionCheck.listAuthorizedResourceIds(user.getId(), logger)); - } - - private User getLoginUser() { - User loginUser = new User(); - loginUser.setUserType(UserType.GENERAL_USER); - loginUser.setUserName("test"); - loginUser.setId(1); - return loginUser; - } - - private User getAdminUser() { - User loginUser = new User(); - loginUser.setUserType(UserType.ADMIN_USER); - loginUser.setUserName("test"); - loginUser.setId(0); - return loginUser; - } -} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java index 1295a8f3f3..77bdb29c1b 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.api.service; import static org.mockito.ArgumentMatchers.eq; import org.apache.dolphinscheduler.api.dto.resources.DeleteDataTransferResponse; +import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.permission.ResourcePermissionCheckService; @@ -31,13 +32,10 @@ import org.apache.dolphinscheduler.common.constants.Constants; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.PropertyUtils; -import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; -import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; -import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper; @@ -77,8 +75,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.mock.web.MockMultipartFile; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import com.google.common.io.Files; /** @@ -93,9 +89,6 @@ public class ResourcesServiceTest { @InjectMocks private ResourcesServiceImpl resourcesService; - @Mock - private ResourceMapper resourcesMapper; - @Mock private TenantMapper tenantMapper; @@ -111,9 +104,6 @@ public class ResourcesServiceTest { @Mock private ProcessDefinitionMapper processDefinitionMapper; - @Mock - private ResourceUserMapper resourceUserMapper; - @Mock private ResourcePermissionCheckService resourcePermissionCheckService; @@ -228,7 +218,6 @@ public class ResourcesServiceTest { } catch (IOException e) { logger.error(e.getMessage(), e); } - Mockito.when(resourcesMapper.existResource("/directoryTest", 0)).thenReturn(true); Result result = resourcesService.createDirectory(user, "directoryTest", ResourceType.FILE, -1, "/"); logger.info(result.toString()); Assertions.assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg()); @@ -325,10 +314,6 @@ public class ResourcesServiceTest { loginUser.setTenantId(1); loginUser.setTenantCode("tenant1"); loginUser.setUserType(UserType.ADMIN_USER); - IPage resourcePage = new Page<>(1, 10); - resourcePage.setTotal(1); - resourcePage.setRecords(getResourceList()); - List mockResList = new ArrayList(); mockResList.add(getStorageEntityResource()); List mockUserList = new ArrayList(); @@ -371,7 +356,7 @@ public class ResourcesServiceTest { Map result = resourcesService.queryResourceList(loginUser, ResourceType.FILE, ""); logger.info(result.toString()); Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); - List resourceList = (List) result.get(Constants.DATA_LIST); + List resourceList = (List) result.get(Constants.DATA_LIST); Assertions.assertTrue(CollectionUtils.isNotEmpty(resourceList)); // test udf @@ -385,7 +370,7 @@ public class ResourcesServiceTest { result = resourcesService.queryResourceList(loginUser, ResourceType.UDF, ""); logger.info(result.toString()); Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); - resourceList = (List) result.get(Constants.DATA_LIST); + resourceList = (List) result.get(Constants.DATA_LIST); Assertions.assertTrue(CollectionUtils.isNotEmpty(resourceList)); } @@ -630,66 +615,6 @@ public class ResourcesServiceTest { } - @Test - public void testAuthorizeResourceTree() { - User user = getUser(); - user.setId(1); - user.setUserType(UserType.ADMIN_USER); - int userId = 3; - - // test admin user - List resIds = new ArrayList<>(); - resIds.add(1); - Mockito.when(resourcePermissionCheckService.functionDisabled()).thenReturn(false); - Mockito.when(resourcesMapper.queryResourceExceptUserId(userId)).thenReturn(getResourceList()); - Map result = resourcesService.authorizeResourceTree(user, userId); - logger.info(result.toString()); - Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); - List resources = (List) result.get(Constants.DATA_LIST); - Assertions.assertTrue(CollectionUtils.isNotEmpty(resources)); - - // test non-admin user - user.setId(2); - user.setUserType(UserType.GENERAL_USER); - Mockito.when(resourcesMapper.queryResourceListAuthored(user.getId(), -1)).thenReturn(getResourceList()); - result = resourcesService.authorizeResourceTree(user, userId); - logger.info(result.toString()); - Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); - resources = (List) result.get(Constants.DATA_LIST); - Assertions.assertTrue(CollectionUtils.isNotEmpty(resources)); - } - - @Test - public void testUnauthorizedFile() { - User user = getUser(); - user.setId(1); - user.setUserType(UserType.ADMIN_USER); - int userId = 3; - - // test admin user - List resIds = new ArrayList<>(); - resIds.add(1); - Mockito.when(resourcesMapper.queryResourceExceptUserId(userId)).thenReturn(getResourceList()); - Mockito.when(resourceUserMapper.queryResourcesIdListByUserIdAndPerm(Mockito.anyInt(), Mockito.anyInt())) - .thenReturn(resIds); - Mockito.when(resourcesMapper.queryResourceListById(Mockito.any())).thenReturn(getSingleResourceList()); - Map result = resourcesService.unauthorizedFile(user, userId); - logger.info(result.toString()); - Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); - List resources = (List) result.get(Constants.DATA_LIST); - Assertions.assertTrue(CollectionUtils.isNotEmpty(resources)); - - // test non-admin user - user.setId(2); - user.setUserType(UserType.GENERAL_USER); - Mockito.when(resourcesMapper.queryResourceListAuthored(user.getId(), -1)).thenReturn(getResourceList()); - result = resourcesService.unauthorizedFile(user, userId); - logger.info(result.toString()); - Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); - resources = (List) result.get(Constants.DATA_LIST); - Assertions.assertTrue(CollectionUtils.isNotEmpty(resources)); - } - @Test public void testUnauthorizedUDFFunction() { User user = getUser(); @@ -745,39 +670,6 @@ public class ResourcesServiceTest { Assertions.assertTrue(CollectionUtils.isNotEmpty(udfFuncs)); } - @Test - public void testAuthorizedFile() { - User user = getUser(); - user.setId(1); - user.setUserType(UserType.ADMIN_USER); - int userId = 3; - - // test admin user - List resIds = new ArrayList<>(); - resIds.add(1); - Mockito.when(resourcePermissionCheckService.functionDisabled()).thenReturn(false); - Mockito.when(resourceUserMapper.queryResourcesIdListByUserIdAndPerm(Mockito.anyInt(), Mockito.anyInt())) - .thenReturn(resIds); - Mockito.when(resourcesMapper.queryResourceListById(Mockito.any())).thenReturn(getResourceList()); - Map result = resourcesService.authorizedFile(user, userId); - logger.info(result.toString()); - Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); - List resources = (List) result.get(Constants.DATA_LIST); - Assertions.assertTrue(CollectionUtils.isNotEmpty(resources)); - - // test non-admin user - user.setId(2); - user.setUserType(UserType.GENERAL_USER); - Mockito.when(resourceUserMapper.queryResourcesIdListByUserIdAndPerm(Mockito.anyInt(), Mockito.anyInt())) - .thenReturn(resIds); - Mockito.when(resourcesMapper.queryResourceListById(Mockito.any())).thenReturn(getResourceList()); - result = resourcesService.authorizedFile(user, userId); - logger.info(result.toString()); - Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); - resources = (List) result.get(Constants.DATA_LIST); - Assertions.assertTrue(CollectionUtils.isNotEmpty(resources)); - } - @Test public void testDeleteDataTransferData() throws Exception { User user = getUser(); @@ -853,15 +745,6 @@ public class ResourcesServiceTest { Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); } - private List getResourceList() { - - List resources = new ArrayList<>(); - resources.add(getResource(1)); - resources.add(getResource(2)); - resources.add(getResource(3)); - return resources; - } - private Set getSetIds() { Set resources = new HashSet<>(); @@ -869,27 +752,12 @@ public class ResourcesServiceTest { return resources; } - private List getSingleResourceList() { - return Collections.singletonList(getResource(1)); - } - private Tenant getTenant() { Tenant tenant = new Tenant(); tenant.setTenantCode("123"); return tenant; } - private Resource getResource() { - Resource resource = new Resource(); - resource.setPid(-1); - resource.setUserId(1); - resource.setDescription("ResourcesServiceTest.jar"); - resource.setAlias("ResourcesServiceTest.jar"); - resource.setFullName("/ResourcesServiceTest.jar"); - resource.setType(ResourceType.FILE); - return resource; - } - private StorageEntity getStorageEntityResource() { StorageEntity entity = new StorageEntity(); entity.setAlias("ResourcesServiceTest"); @@ -902,43 +770,6 @@ public class ResourcesServiceTest { return entity; } - private Resource getResource(int resourceId) { - - Resource resource = new Resource(); - resource.setId(resourceId); - resource.setPid(-1); - resource.setUserId(1); - resource.setDescription("ResourcesServiceTest.jar"); - resource.setAlias("ResourcesServiceTest.jar"); - resource.setFullName("/ResourcesServiceTest.jar"); - resource.setType(ResourceType.FILE); - return resource; - } - - private Resource getResource(int resourceId, ResourceType type) { - - Resource resource = new Resource(); - resource.setId(resourceId); - resource.setPid(-1); - resource.setUserId(1); - resource.setDescription("ResourcesServiceTest.jar"); - resource.setAlias("ResourcesServiceTest.jar"); - resource.setFullName("/ResourcesServiceTest.jar"); - resource.setType(type); - return resource; - } - - private Resource getUdfResource() { - - Resource resource = new Resource(); - resource.setUserId(1); - resource.setDescription("udfTest"); - resource.setAlias("udfTest.jar"); - resource.setFullName("/udfTest.jar"); - resource.setType(ResourceType.UDF); - return resource; - } - private StorageEntity getStorageEntityUdfResource() { StorageEntity entity = new StorageEntity(); entity.setAlias("ResourcesServiceTest1.jar"); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java index e32d68a0e0..f1721cbb46 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java @@ -28,10 +28,8 @@ import org.apache.dolphinscheduler.common.enums.AuthorizationType; import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.utils.PropertyUtils; -import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.dolphinscheduler.dao.entity.User; -import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper; import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; @@ -78,9 +76,6 @@ public class UdfFuncServiceTest { @InjectMocks private UdfFuncServiceImpl udfFuncService; - @Mock - private ResourceMapper resourceMapper; - @Mock private UdfFuncMapper udfFuncMapper; @@ -291,17 +286,6 @@ public class UdfFuncServiceTest { return loginUser; } - /** - * get resourceId - */ - private Resource getResource() { - - Resource resource = new Resource(); - resource.setId(1); - resource.setAlias("test"); - return resource; - } - private List getList() { List udfFuncList = new ArrayList<>(); udfFuncList.add(getUdfFunc()); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java index 27ee150b50..36e3abfa3e 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java @@ -37,7 +37,6 @@ import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.utils.EncryptionUtils; import org.apache.dolphinscheduler.dao.entity.AlertGroup; import org.apache.dolphinscheduler.dao.entity.Project; -import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.AccessTokenMapper; @@ -46,13 +45,10 @@ import org.apache.dolphinscheduler.dao.mapper.DataSourceUserMapper; import org.apache.dolphinscheduler.dao.mapper.K8sNamespaceUserMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectUserMapper; -import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; -import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper; import org.apache.dolphinscheduler.dao.mapper.UserMapper; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; -import org.apache.dolphinscheduler.spi.enums.ResourceType; import org.apache.commons.collections4.CollectionUtils; @@ -99,9 +95,6 @@ public class UsersServiceTest { @Mock private TenantMapper tenantMapper; - @Mock - private ResourceMapper resourceMapper; - @Mock private AlertGroupMapper alertGroupMapper; @@ -111,9 +104,6 @@ public class UsersServiceTest { @Mock private ProjectUserMapper projectUserMapper; - @Mock - private ResourceUserMapper resourceUserMapper; - @Mock private MetricsCleanUpService metricsCleanUpService; @@ -526,26 +516,6 @@ public class UsersServiceTest { Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); } - @Test - public void testGrantResources() { - String resourceIds = "100000,120000"; - when(userMapper.selectById(1)).thenReturn(getUser()); - User loginUser = new User(); - - // user not exist - loginUser.setUserType(UserType.ADMIN_USER); - Map result = usersService.grantResources(loginUser, 2, resourceIds); - logger.info(result.toString()); - Assertions.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS)); - // success - when(resourceMapper.selectById(Mockito.anyInt())).thenReturn(getResource()); - when(resourceUserMapper.deleteResourceUser(1, 0)).thenReturn(1); - result = usersService.grantResources(loginUser, 1, resourceIds); - logger.info(result.toString()); - Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); - - } - @Test public void testGrantUDFFunction() { String udfIds = "100000,120000"; @@ -931,22 +901,6 @@ public class UsersServiceTest { return tenant; } - /** - * get resource - * - * @return resource - */ - private Resource getResource() { - Resource resource = new Resource(); - resource.setPid(-1); - resource.setUserId(1); - resource.setDescription("ResourcesServiceTest.jar"); - resource.setAlias("ResourcesServiceTest.jar"); - resource.setFullName("/ResourcesServiceTest.jar"); - resource.setType(ResourceType.FILE); - return resource; - } - private List getAlertGroups() { List alertGroups = new ArrayList<>(); AlertGroup alertGroup = new AlertGroup(); diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuthorizationType.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuthorizationType.java index 49905df3eb..79ef986cd3 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuthorizationType.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/enums/AuthorizationType.java @@ -44,8 +44,11 @@ public enum AuthorizationType { * 16 USER; * 17 Data_Quality; */ + @Deprecated RESOURCE_FILE_ID(0, "resource file id"), + @Deprecated RESOURCE_FILE_NAME(1, "resource file name"), + @Deprecated UDF_FILE(2, "udf file"), DATASOURCE(3, "data source"), UDF(4, "udf function"), diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java index 5e216ff27e..6750b364f9 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/JSONUtils.java @@ -21,6 +21,7 @@ import static com.fasterxml.jackson.databind.DeserializationFeature.ACCEPT_EMPTY import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES; import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL; import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS; +import static com.fasterxml.jackson.databind.SerializationFeature.FAIL_ON_EMPTY_BEANS; import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.dolphinscheduler.common.constants.DateConstants.YYYY_MM_DD_HH_MM_SS; @@ -65,7 +66,7 @@ import com.google.common.base.Strings; * json utils */ @Slf4j -public class JSONUtils { +public final class JSONUtils { static { log.info("init timezone: {}", TimeZone.getDefault()); @@ -76,6 +77,7 @@ public class JSONUtils { .configure(ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true) .configure(READ_UNKNOWN_ENUM_VALUES_AS_NULL, true) .configure(REQUIRE_SETTERS_FOR_GETTERS, true) + .configure(FAIL_ON_EMPTY_BEANS, false) .addModule(new SimpleModule() .addSerializer(LocalDateTime.class, new LocalDateTimeSerializer()) .addDeserializer(LocalDateTime.class, new LocalDateTimeDeserializer())) diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Resource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Resource.java deleted file mode 100644 index 9ecc047e6d..0000000000 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Resource.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.dao.entity; - -import org.apache.dolphinscheduler.spi.enums.ResourceType; - -import java.util.Date; -import java.util.Objects; - -import lombok.Data; -import lombok.NoArgsConstructor; - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableField; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; - -@Data -@NoArgsConstructor -@TableName("t_ds_resources") -public class Resource { - - /** - * id - */ - @TableId(value = "id", type = IdType.AUTO) - private Integer id; - - /** - * parent id - */ - private int pid; - - /** - * resource alias - */ - private String alias; - - /** - * full name - */ - private String fullName; - - /** - * is directory - */ - private boolean isDirectory = false; - - /** - * description - */ - private String description; - - /** - * file alias - */ - private String fileName; - - /** - * user id - */ - private int userId; - - /** - * resource type - */ - private ResourceType type; - - /** - * resource size - */ - private long size; - - /** - * create time - */ - private Date createTime; - - /** - * update time - */ - private Date updateTime; - - /** - * user name - */ - @TableField(exist = false) - private String userName; - - public Resource(int id, String alias, String fileName, String description, int userId, - ResourceType type, long size, - Date createTime, Date updateTime) { - this.id = id; - this.alias = alias; - this.fileName = fileName; - this.description = description; - this.userId = userId; - this.type = type; - this.size = size; - this.createTime = createTime; - this.updateTime = updateTime; - } - - public Resource(int id, int pid, String alias, String fullName, boolean isDirectory) { - this.id = id; - this.pid = pid; - this.alias = alias; - this.fullName = fullName; - this.isDirectory = isDirectory; - } - - public Resource(int pid, String alias, String fullName, boolean isDirectory, String description, String fileName, - int userId, ResourceType type, long size, Date createTime, Date updateTime) { - this.pid = pid; - this.alias = alias; - this.fullName = fullName; - this.isDirectory = isDirectory; - this.description = description; - this.fileName = fileName; - this.userId = userId; - this.type = type; - this.size = size; - this.createTime = createTime; - this.updateTime = updateTime; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - Resource resource = (Resource) o; - - if (!Objects.equals(id, resource.id)) { - return false; - } - return alias.equals(resource.alias); - - } - - @Override - public int hashCode() { - int result = id; - result = 31 * result + alias.hashCode(); - return result; - } -} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResourcesUser.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResourcesUser.java deleted file mode 100644 index 3126f2604d..0000000000 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResourcesUser.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.dao.entity; - -import java.util.Date; - -import lombok.Data; - -import com.baomidou.mybatisplus.annotation.IdType; -import com.baomidou.mybatisplus.annotation.TableId; -import com.baomidou.mybatisplus.annotation.TableName; - -@Data -@TableName("t_ds_relation_resources_user") -public class ResourcesUser { - - /** - * id - */ - @TableId(value = "id", type = IdType.AUTO) - private Integer id; - - /** - * user id - */ - private int userId; - - /** - * resource id - */ - private int resourcesId; - - /** - * permission - */ - private int perm; - - /** - * create time - */ - private Date createTime; - - /** - * update time - */ - private Date updateTime; -} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskDefinition.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskDefinition.java index 18c3a405a7..2fd2b02987 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskDefinition.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskDefinition.java @@ -179,8 +179,9 @@ public class TaskDefinition { private int delayTime; /** - * resource ids + * resource ids we do */ + @Deprecated private String resourceIds; /** diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java index b205aa8ac7..b292641008 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java @@ -258,9 +258,6 @@ public class TaskInstance implements Serializable { private String executorName; - @TableField(exist = false) - private Map resources; - /** * delay execution time. */ diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java deleted file mode 100644 index c90d4ea068..0000000000 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.dao.mapper; - -import org.apache.dolphinscheduler.dao.entity.Resource; - -import org.apache.ibatis.annotations.Param; - -import java.util.List; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.baomidou.mybatisplus.core.metadata.IPage; - -/** - * resource mapper interface - */ -public interface ResourceMapper extends BaseMapper { - - /** - * query resource list - * @param fullName full name - * @param userId userId - * @param type type - * @return resource list - */ - List queryResourceList(@Param("fullName") String fullName, - @Param("userId") int userId, - @Param("type") int type); - - /** - * query resource list - * @param userId userId - * @param type type - * @return resource list - */ - List queryResourceListAuthored(@Param("userId") int userId, - @Param("type") int type); - - /** - * resource page - * @param page page - * @param id id - * @param type type - * @param searchVal searchVal - * @param resIds resIds - * @return resource page - */ - IPage queryResourcePaging(IPage page, - @Param("id") int id, - @Param("type") int type, - @Param("searchVal") String searchVal, - @Param("resIds") List resIds); - - /** - * query resource except userId - * @param userId userId - * @return resource list - */ - List queryResourceExceptUserId(@Param("userId") int userId); - - /** - * list authorized resource - * @param userId userId - * @param resNames resNames - * @param T - * @return resource list - */ - List listAuthorizedResource(@Param("userId") int userId, @Param("resNames") T[] resNames); - - /** - * list resources by id - * @param resIds resIds - * @return resource list - */ - List queryResourceListById(@Param("resIds") List resIds); - - /** - * list authorized resource - * @param userId userId - * @param resIds resIds - * @param T - * @return resource list - */ - List listAuthorizedResourceById(@Param("userId") int userId, @Param("resIds") T[] resIds); - - /** - * delete resource by id array - * @param resIds resource id array - * @return delete num - */ - int deleteIds(@Param("resIds") Integer[] resIds); - - /** - * list children - * @param direcotyId directory id - * @return resource id array - */ - List listChildren(@Param("direcotyId") int direcotyId); - - /** - * query resource by full name or pid - * @param fullName full name - * @param type resource type - * @return resource - */ - List queryResource(@Param("fullName") String fullName, @Param("type") int type); - - /** - * list resource by id array - * @param resIds resource id array - * @return resource list - */ - List listResourceByIds(@Param("resIds") Integer[] resIds); - - /** - * update resource - * @param resourceList resource list - * @return update num - */ - int batchUpdateResource(@Param("resourceList") List resourceList); - - /** - * check resource exist - * @param fullName full name - * @param userId userId - * @param type type - * @return true if exist else return null - */ - Boolean existResourceByUser(@Param("fullName") String fullName, - @Param("userId") int userId, - @Param("type") int type); - - /** - * check resource exist - * @param fullName full name - * @param type type - * @return true if exist else return null - */ - Boolean existResource(@Param("fullName") String fullName, - @Param("type") int type); - -} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.java deleted file mode 100644 index 6fc36ea425..0000000000 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.dao.mapper; - -import org.apache.dolphinscheduler.dao.entity.ResourcesUser; - -import org.apache.ibatis.annotations.Param; - -import java.util.List; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; - -/** - * resource user relation mapper interface - */ -public interface ResourceUserMapper extends BaseMapper { - - /** - * query resourcesId list by userId and perm - * @param userId userId - * @param perm perm - * @return resourcesId list result - */ - List queryResourcesIdListByUserIdAndPerm(@Param("userId") int userId, - @Param("perm") int perm); - - /** - * delete resource user relation - * @param userId userId - * @param resourceId resourceId - * @return delete result - */ - int deleteResourceUser(@Param("userId") int userId, - @Param("resourceId") int resourceId); - - /** - * delete resource user relation - * @param userId userId - * @param resIds resource Ids - * @return delete result - */ - int deleteResourceUserArray(@Param("userId") int userId, - @Param("resIds") Integer[] resIds); - -} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskCacheUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskCacheUtils.java index 838a77808f..36cf6c2356 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskCacheUtils.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskCacheUtils.java @@ -183,7 +183,7 @@ public class TaskCacheUtils { log.info("{} --- Remote:{} to Local:{}", "CRC file", resourceCRCWholePath, targetPath); String crcString = ""; try { - storageOperate.download(context.getTenantCode(), resourceCRCWholePath, targetPath, true); + storageOperate.download(resourceCRCWholePath, targetPath, true); crcString = FileUtils.readFile2Str(new FileInputStream(targetPath)); fileProperty.setValue(crcString); } catch (IOException e) { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java index 2fc9fbdf5c..f75a2f5e11 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/TaskInstanceUtils.java @@ -71,7 +71,6 @@ public class TaskInstanceUtils { target.setExecutorId(source.getExecutorId()); target.setVarPool(source.getVarPool()); target.setExecutorName(source.getExecutorName()); - target.setResources(source.getResources()); target.setDelayTime(source.getDelayTime()); target.setDryRun(source.getDryRun()); target.setTaskGroupId(source.getTaskGroupId()); diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml deleted file mode 100644 index a87aac4b7b..0000000000 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml +++ /dev/null @@ -1,198 +0,0 @@ - - - - - - - ${alias}.id, ${alias}.alias, ${alias}.file_name, ${alias}.description, ${alias}.user_id, ${alias}.type, ${alias}.size, ${alias}.create_time, ${alias}.update_time, - ${alias}.pid, ${alias}.full_name, ${alias}.is_directory - - - - - - - - - - - - - delete from t_ds_resources - - where id in - - #{i} - - - - - - - - - - - update t_ds_resources - - full_name=#{resource.fullName}, - update_time=#{resource.updateTime} - - - id=#{resource.id} - - - - - - - - - - diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql index 2e18052aa6..4d470f9dfc 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql @@ -784,6 +784,7 @@ CREATE TABLE t_ds_relation_project_user -- ---------------------------- -- Table structure for t_ds_relation_resources_user -- ---------------------------- +-- Deprecated DROP TABLE IF EXISTS t_ds_relation_resources_user CASCADE; CREATE TABLE t_ds_relation_resources_user ( @@ -818,6 +819,7 @@ CREATE TABLE t_ds_relation_udfs_user -- ---------------------------- -- Table structure for t_ds_resources -- ---------------------------- +-- Deprecated DROP TABLE IF EXISTS t_ds_resources CASCADE; CREATE TABLE t_ds_resources ( diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql index a264a868e5..f9ffde83f3 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql @@ -783,6 +783,7 @@ CREATE TABLE `t_ds_relation_project_user` ( -- ---------------------------- -- Table structure for t_ds_relation_resources_user -- ---------------------------- +-- Deprecated DROP TABLE IF EXISTS `t_ds_relation_resources_user`; CREATE TABLE `t_ds_relation_resources_user` ( `id` int(11) NOT NULL AUTO_INCREMENT, @@ -815,6 +816,7 @@ CREATE TABLE `t_ds_relation_udfs_user` ( -- ---------------------------- -- Table structure for t_ds_resources -- ---------------------------- +-- Deprecated DROP TABLE IF EXISTS `t_ds_resources`; CREATE TABLE `t_ds_resources` ( `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'key', diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql index f013ee9373..d8214c7a1f 100644 --- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql +++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql @@ -695,7 +695,7 @@ create index relation_project_user_id_index on t_ds_relation_project_user (user_ -- -- Table structure for table t_ds_relation_resources_user -- - +-- Deprecated DROP TABLE IF EXISTS t_ds_relation_resources_user; CREATE TABLE t_ds_relation_resources_user ( id int NOT NULL , @@ -726,7 +726,7 @@ CREATE TABLE t_ds_relation_udfs_user ( -- -- Table structure for table t_ds_resources -- - +-- Deprecated DROP TABLE IF EXISTS t_ds_resources; CREATE TABLE t_ds_resources ( id int NOT NULL , diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java deleted file mode 100644 index 6c6da06238..0000000000 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java +++ /dev/null @@ -1,440 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.dao.mapper; - -import static java.util.stream.Collectors.toList; - -import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.common.enums.UserType; -import org.apache.dolphinscheduler.dao.BaseDaoTest; -import org.apache.dolphinscheduler.dao.entity.Resource; -import org.apache.dolphinscheduler.dao.entity.ResourcesUser; -import org.apache.dolphinscheduler.dao.entity.Tenant; -import org.apache.dolphinscheduler.dao.entity.User; -import org.apache.dolphinscheduler.spi.enums.ResourceType; - -import org.apache.commons.collections4.CollectionUtils; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.List; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; - -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; - -public class ResourceMapperTest extends BaseDaoTest { - - @Autowired - private ResourceMapper resourceMapper; - - @Autowired - private ResourceUserMapper resourceUserMapper; - - @Autowired - private TenantMapper tenantMapper; - - @Autowired - private UserMapper userMapper; - - /** - * insert - * - * @return Resource - */ - private Resource insertOne() { - // insertOne - Resource resource = new Resource(); - resource.setAlias("ut-resource"); - resource.setFullName("/ut-resource"); - resource.setPid(-1); - resource.setDirectory(false); - resource.setType(ResourceType.FILE); - resource.setUserId(111); - int status = resourceMapper.insert(resource); - if (status != 1) { - Assertions.fail("insert data error"); - } - return resource; - } - - /** - * create resource by user - * - * @param user user - * @return Resource - */ - private Resource createResource(User user, boolean isDirectory, ResourceType resourceType, int pid, String alias, - String fullName) { - // insertOne - Resource resource = new Resource(); - resource.setDirectory(isDirectory); - resource.setType(resourceType); - resource.setAlias(alias); - resource.setFullName(fullName); - resource.setUserId(user.getId()); - int status = resourceMapper.insert(resource); - if (status != 1) { - Assertions.fail("insert data error"); - } - return resource; - } - - /** - * create resource by user - * - * @param user user - * @return Resource - */ - private Resource createResource(User user) { - // insertOne - String alias = String.format("ut-resource-%s", user.getUserName()); - String fullName = String.format("/%s", alias); - - Resource resource = createResource(user, false, ResourceType.FILE, -1, alias, fullName); - return resource; - } - - /** - * create user - * - * @return User - */ - private User createGeneralUser(String userName) { - User user = new User(); - user.setUserName(userName); - user.setUserPassword("1"); - user.setEmail("xx@123.com"); - user.setUserType(UserType.GENERAL_USER); - user.setCreateTime(new Date()); - user.setTenantId(1); - user.setUpdateTime(new Date()); - int status = userMapper.insert(user); - - if (status != 1) { - Assertions.fail("insert data error"); - } - return user; - } - - /** - * create resource user - * - * @return ResourcesUser - */ - private ResourcesUser createResourcesUser(Resource resource, User user) { - // insertOne - ResourcesUser resourcesUser = new ResourcesUser(); - resourcesUser.setCreateTime(new Date()); - resourcesUser.setUpdateTime(new Date()); - resourcesUser.setUserId(user.getId()); - resourcesUser.setResourcesId(resource.getId()); - resourcesUser.setPerm(7); - resourceUserMapper.insert(resourcesUser); - return resourcesUser; - } - - @Test - public void testInsert() { - Resource resource = insertOne(); - Assertions.assertNotNull(resource.getId()); - Assertions.assertTrue(resource.getId() > 0); - } - - /** - * test update - */ - @Test - public void testUpdate() { - // insertOne - Resource resource = insertOne(); - resource.setCreateTime(new Date()); - // update - int update = resourceMapper.updateById(resource); - Assertions.assertEquals(1, update); - } - - /** - * test delete - */ - @Test - public void testDelete() { - Resource resourceMap = insertOne(); - int delete = resourceMapper.deleteById(resourceMap.getId()); - Assertions.assertEquals(1, delete); - } - - /** - * test query - */ - @Test - public void testQuery() { - Resource resource = insertOne(); - // query - List resources = resourceMapper.selectList(null); - Assertions.assertNotEquals(resources.size(), 0); - } - - /** - * test query resource list - */ - @Test - public void testQueryResourceList() { - - Resource resource = insertOne(); - - String alias = ""; - int userId = resource.getUserId(); - int type = resource.getType().ordinal(); - List resources = resourceMapper.queryResourceList( - alias, - userId, - type); - - Assertions.assertNotEquals(resources.size(), 0); - } - - /** - * test page - */ - @Test - public void testQueryResourcePaging() { - User user = new User(); - user.setUserName("11"); - user.setUserPassword("1"); - user.setEmail("xx@123.com"); - user.setUserType(UserType.GENERAL_USER); - user.setCreateTime(new Date()); - user.setTenantId(1); - user.setUpdateTime(new Date()); - userMapper.insert(user); - Resource resource = new Resource(); - resource.setAlias("ut-resource"); - resource.setFullName("/ut-resource"); - resource.setPid(-1); - resource.setDirectory(false); - resource.setType(ResourceType.FILE); - resource.setUserId(user.getId()); - resourceMapper.insert(resource); - - Page page = new Page(1, 3); - - IPage resourceIPage = resourceMapper.queryResourcePaging( - page, - -1, - resource.getType().ordinal(), - "", - new ArrayList<>(resource.getId())); - IPage resourceIPage1 = resourceMapper.queryResourcePaging( - page, - -1, - resource.getType().ordinal(), - "", - null); - Assertions.assertEquals(resourceIPage.getTotal(), 1); - Assertions.assertEquals(resourceIPage1.getTotal(), 1); - - } - - /** - * test authed resource list - */ - @Test - public void testQueryResourceListAuthored() { - Resource resource = insertOne(); - - List resIds = resourceUserMapper.queryResourcesIdListByUserIdAndPerm(resource.getUserId(), - Constants.AUTHORIZE_WRITABLE_PERM); - List resources = - CollectionUtils.isEmpty(resIds) ? new ArrayList<>() : resourceMapper.queryResourceListById(resIds); - - ResourcesUser resourcesUser = new ResourcesUser(); - - resourcesUser.setResourcesId(resource.getId()); - resourcesUser.setUserId(1110); - resourcesUser.setPerm(Constants.AUTHORIZE_WRITABLE_PERM); - resourceUserMapper.insert(resourcesUser); - - List resIds1 = - resourceUserMapper.queryResourcesIdListByUserIdAndPerm(1110, Constants.AUTHORIZE_WRITABLE_PERM); - List resources1 = - CollectionUtils.isEmpty(resIds1) ? new ArrayList<>() : resourceMapper.queryResourceListById(resIds1); - - Assertions.assertEquals(0, resources.size()); - Assertions.assertNotEquals(0, resources1.size()); - - } - - /** - * test authed resource list - */ - @Test - public void testQueryAuthorizedResourceList() { - Resource resource = insertOne(); - - List resIds = resourceUserMapper.queryResourcesIdListByUserIdAndPerm(resource.getUserId(), - Constants.AUTHORIZE_WRITABLE_PERM); - List resources = - CollectionUtils.isEmpty(resIds) ? new ArrayList<>() : resourceMapper.queryResourceListById(resIds); - - resourceMapper.deleteById(resource.getId()); - Assertions.assertEquals(0, resources.size()); - } - - /** - * test query resource expect userId - */ - @Test - public void testQueryResourceExceptUserId() { - Resource resource = insertOne(); - List resources = resourceMapper.queryResourceExceptUserId( - 11111); - Assertions.assertNotEquals(resources.size(), 0); - } - - /** - * test query tenant code by resource name - */ - @Test - public void testQueryTenantCodeByResourceName() { - - Tenant tenant = new Tenant(); - tenant.setTenantCode("ut tenant code for resource"); - int tenantInsertStatus = tenantMapper.insert(tenant); - - if (tenantInsertStatus != 1) { - Assertions.fail("insert tenant data error"); - } - - User user = new User(); - user.setTenantId(tenant.getId()); - user.setUserName("ut user"); - int userInsertStatus = userMapper.insert(user); - - if (userInsertStatus != 1) { - Assertions.fail("insert user data error"); - } - - Resource resource = insertOne(); - resource.setUserId(user.getId()); - int userUpdateStatus = resourceMapper.updateById(resource); - if (userUpdateStatus != 1) { - Assertions.fail("update user data error"); - } - - List resourceList = resourceMapper.queryResource(resource.getFullName(), ResourceType.FILE.ordinal()); - - int resourceUserId = resourceList.get(0).getUserId(); - User resourceUser = userMapper.selectById(resourceUserId); - Tenant resourceTenant = tenantMapper.selectById(resourceUser.getTenantId()); - - Assertions.assertEquals("ut tenant code for resource", resourceTenant.getTenantCode()); - - } - - @Test - public void testListAuthorizedResource() { - // create a general user - User generalUser1 = createGeneralUser("user1"); - User generalUser2 = createGeneralUser("user2"); - // create one resource - Resource resource = createResource(generalUser2); - Resource unauthorizedResource = createResource(generalUser1); - - // need download resources - String[] resNames = new String[]{resource.getFullName(), unauthorizedResource.getFullName()}; - - List resources = resourceMapper.listAuthorizedResource(generalUser2.getId(), resNames); - - Assertions.assertEquals(generalUser2.getId().intValue(), resource.getUserId()); - Assertions.assertFalse( - resources.stream().map(t -> t.getFullName()).collect(toList()).containsAll(Arrays.asList(resNames))); - - // authorize object unauthorizedResource to generalUser - createResourcesUser(unauthorizedResource, generalUser2); - List authorizedResources = resourceMapper.listAuthorizedResource(generalUser2.getId(), resNames); - Assertions.assertTrue(authorizedResources.stream().map(t -> t.getFullName()).collect(toList()) - .containsAll(Arrays.asList(resource.getFullName()))); - - } - - @Test - public void deleteIdsTest() { - // create a general user - User generalUser1 = createGeneralUser("user1"); - User generalUser = createGeneralUser("user"); - - Resource resource = createResource(generalUser); - Resource resource1 = createResource(generalUser1); - - List resourceList = new ArrayList<>(); - resourceList.add(resource.getId()); - resourceList.add(resource1.getId()); - int result = resourceMapper.deleteIds(resourceList.toArray(new Integer[resourceList.size()])); - Assertions.assertEquals(result, 2); - } - - @Test - public void queryResourceListAuthoredTest() { - // create a general user - User generalUser1 = createGeneralUser("user1"); - User generalUser2 = createGeneralUser("user2"); - // create resource - Resource resource = createResource(generalUser1); - createResourcesUser(resource, generalUser2); - - List resourceList = - resourceMapper.queryResourceListAuthored(generalUser2.getId(), ResourceType.FILE.ordinal()); - Assertions.assertNotNull(resourceList); - - resourceList = resourceMapper.queryResourceListAuthored(generalUser2.getId(), ResourceType.FILE.ordinal()); - Assertions.assertFalse(resourceList.contains(resource)); - } - - @Test - public void batchUpdateResourceTest() { - // create a general user - User generalUser1 = createGeneralUser("user1"); - // create resource - Resource resource = createResource(generalUser1); - resource.setFullName(String.format("%s-update", resource.getFullName())); - resource.setUpdateTime(new Date()); - List resourceList = new ArrayList<>(); - resourceList.add(resource); - int result = resourceMapper.batchUpdateResource(resourceList); - if (result != resourceList.size()) { - Assertions.fail("batch update resource data error"); - } - } - - @Test - public void existResourceTest() { - String fullName = "/ut-resource"; - int userId = 111; - int type = ResourceType.FILE.getCode(); - Assertions.assertNull(resourceMapper.existResourceByUser(fullName, userId, type)); - Assertions.assertNull(resourceMapper.existResource(fullName, type)); - insertOne(); - Assertions.assertTrue(resourceMapper.existResourceByUser(fullName, userId, type)); - Assertions.assertTrue(resourceMapper.existResource(fullName, type)); - } -} diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapperTest.java deleted file mode 100644 index a25610008f..0000000000 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapperTest.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.dao.mapper; - -import org.apache.dolphinscheduler.common.constants.Constants; -import org.apache.dolphinscheduler.dao.BaseDaoTest; -import org.apache.dolphinscheduler.dao.entity.ResourcesUser; - -import java.util.Date; -import java.util.List; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; - -public class ResourceUserMapperTest extends BaseDaoTest { - - @Autowired - private ResourceUserMapper resourceUserMapper; - - /** - * insert - * @return ResourcesUser - */ - private ResourcesUser insertOne() { - // insertOne - ResourcesUser resourcesUser = new ResourcesUser(); - resourcesUser.setCreateTime(new Date()); - resourcesUser.setUpdateTime(new Date()); - resourcesUser.setUserId(11111); - resourcesUser.setResourcesId(1110); - resourcesUser.setPerm(Constants.AUTHORIZE_WRITABLE_PERM); - resourceUserMapper.insert(resourcesUser); - return resourcesUser; - } - - /** - * test update - */ - @Test - public void testUpdate() { - // insertOne - ResourcesUser queue = insertOne(); - queue.setCreateTime(new Date()); - // update - int update = resourceUserMapper.updateById(queue); - Assertions.assertEquals(1, update); - } - - /** - * test delete - */ - @Test - public void testDelete() { - ResourcesUser queue = insertOne(); - int delete = resourceUserMapper.deleteById(queue.getId()); - Assertions.assertEquals(1, delete); - } - - /** - * test query - */ - @Test - public void testQuery() { - ResourcesUser queue = insertOne(); - // query - List queues = resourceUserMapper.selectList(null); - Assertions.assertNotEquals(0, queues.size()); - } - - /** - * test delete - */ - @Test - public void testDeleteResourceUser() { - - ResourcesUser queue = insertOne(); - int delete = resourceUserMapper.deleteResourceUser( - queue.getUserId(), - queue.getResourcesId()); - Assertions.assertNotEquals(0, delete); - } - - /** - * test delete - */ - @Test - public void testDeleteResourceUserArray() { - - ResourcesUser resourcesUser = insertOne(); - Integer[] resourceIdArray = new Integer[]{resourcesUser.getResourcesId()}; - int delete = resourceUserMapper.deleteResourceUserArray( - resourcesUser.getUserId(), - resourceIdArray); - Assertions.assertNotEquals(0, delete); - } -} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/builder/TaskExecutionContextBuilder.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/builder/TaskExecutionContextBuilder.java index e4e69043e6..c0a5abc369 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/builder/TaskExecutionContextBuilder.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/builder/TaskExecutionContextBuilder.java @@ -26,7 +26,6 @@ import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.TaskInstance; -import org.apache.dolphinscheduler.plugin.task.api.DataQualityTaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.K8sTaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; @@ -67,7 +66,6 @@ public class TaskExecutionContextBuilder { taskExecutionContext.setWorkerGroup(taskInstance.getWorkerGroup()); taskExecutionContext.setEnvironmentConfig(taskInstance.getEnvironmentConfig()); taskExecutionContext.setHost(taskInstance.getHost()); - taskExecutionContext.setResources(taskInstance.getResources()); taskExecutionContext.setDelayTime(taskInstance.getDelayTime()); taskExecutionContext.setVarPool(taskInstance.getVarPool()); taskExecutionContext.setDryRun(taskInstance.getDryRun()); @@ -122,11 +120,6 @@ public class TaskExecutionContextBuilder { return this; } - public TaskExecutionContextBuilder buildDataQualityTaskExecutionContext(DataQualityTaskExecutionContext dataQualityTaskExecutionContext) { - taskExecutionContext.setDataQualityTaskExecutionContext(dataQualityTaskExecutionContext); - return this; - } - public TaskExecutionContextBuilder buildResourceParametersInfo(ResourceParametersHelper parametersHelper) { taskExecutionContext.setResourceParametersHelper(parametersHelper); return this; diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteRunnable.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteRunnable.java index ab6fc555a6..2f61507e72 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteRunnable.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/StreamTaskExecuteRunnable.java @@ -27,7 +27,6 @@ import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.dao.entity.Environment; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation; -import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationMapper; @@ -43,7 +42,6 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.model.Property; -import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; @@ -60,9 +58,7 @@ import org.apache.dolphinscheduler.server.master.runner.dispatcher.WorkerTaskDis import org.apache.dolphinscheduler.server.master.runner.execute.DefaultTaskExecuteRunnableFactory; import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.apache.dolphinscheduler.service.process.ProcessService; -import org.apache.dolphinscheduler.spi.enums.ResourceType; -import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import java.util.Date; @@ -70,10 +66,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Set; import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.stream.Collectors; -import java.util.stream.Stream; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -319,8 +312,6 @@ public class StreamTaskExecuteRunnable implements Runnable { return null; } - taskInstance.setResources(getResourceFullNames(taskInstance)); - TaskChannel taskChannel = taskPluginManager.getTaskChannel(taskInstance.getTaskType()); ResourceParametersHelper resources = taskChannel.getResources(taskInstance.getTaskParams()); @@ -350,42 +341,6 @@ public class StreamTaskExecuteRunnable implements Runnable { return taskExecutionContext; } - /** - * get resource map key is full name and value is tenantCode - */ - protected Map getResourceFullNames(TaskInstance taskInstance) { - Map resourcesMap = new HashMap<>(); - AbstractParameters baseParam = taskPluginManager.getParameters(ParametersNode.builder() - .taskType(taskInstance.getTaskType()).taskParams(taskInstance.getTaskParams()).build()); - if (baseParam != null) { - List projectResourceFiles = baseParam.getResourceFilesList(); - if (CollectionUtils.isNotEmpty(projectResourceFiles)) { - - // filter the resources that the resource id equals 0 - Set oldVersionResources = - projectResourceFiles.stream().filter(t -> t.getId() == null).collect(Collectors.toSet()); - if (CollectionUtils.isNotEmpty(oldVersionResources)) { - oldVersionResources.forEach(t -> resourcesMap.put(t.getRes(), - processService.queryTenantCodeByResName(t.getRes(), ResourceType.FILE))); - } - - // get the resource id in order to get the resource names in batch - Stream resourceIdStream = projectResourceFiles.stream().map(ResourceInfo::getId); - Set resourceIdsSet = resourceIdStream.collect(Collectors.toSet()); - - if (CollectionUtils.isNotEmpty(resourceIdsSet)) { - Integer[] resourceIds = resourceIdsSet.toArray(new Integer[resourceIdsSet.size()]); - - List resources = processService.listResourceByIds(resourceIds); - resources.forEach(t -> resourcesMap.put(t.getFullName(), - processService.queryTenantCodeByResName(t.getFullName(), ResourceType.FILE))); - } - } - } - - return resourcesMap; - } - protected boolean handleTaskEvent(TaskEvent taskEvent) throws StateEventHandleException, StateEventHandleError { measureTaskState(taskEvent); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java index 3fb2a9c850..d436004719 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/TaskExecutionContextFactory.java @@ -54,7 +54,6 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.dp.ConnectorType; import org.apache.dolphinscheduler.plugin.task.api.enums.dp.ExecuteSqlType; import org.apache.dolphinscheduler.plugin.task.api.model.JdbcInfo; import org.apache.dolphinscheduler.plugin.task.api.model.Property; -import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.K8sTaskParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; @@ -72,13 +71,11 @@ import org.apache.dolphinscheduler.server.master.exception.TaskExecutionContextC import org.apache.dolphinscheduler.service.expand.CuringParamsService; import org.apache.dolphinscheduler.service.process.ProcessService; import org.apache.dolphinscheduler.spi.enums.DbType; -import org.apache.dolphinscheduler.spi.enums.ResourceType; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -113,7 +110,6 @@ public class TaskExecutionContextFactory { public TaskExecutionContext createTaskExecutionContext(TaskInstance taskInstance) throws TaskExecutionContextCreateException { ProcessInstance workflowInstance = taskInstance.getProcessInstance(); - taskInstance.setResources(getResourceFullNames(taskInstance)); ResourceParametersHelper resources = Optional.ofNullable(taskPluginManager.getTaskChannel(taskInstance.getTaskType())) @@ -159,22 +155,6 @@ public class TaskExecutionContextFactory { taskExecutionContext.setK8sTaskExecutionContext(k8sTaskExecutionContext); } - private Map getResourceFullNames(TaskInstance taskInstance) { - Map resourcesMap = new HashMap<>(); - AbstractParameters baseParam = taskPluginManager.getParameters(ParametersNode.builder() - .taskType(taskInstance.getTaskType()).taskParams(taskInstance.getTaskParams()).build()); - if (baseParam != null) { - List projectResourceFiles = baseParam.getResourceFilesList(); - if (CollectionUtils.isNotEmpty(projectResourceFiles)) { - // TODO: Modify this part to accomodate(migrate) oldversionresources in the future. - projectResourceFiles.forEach(file -> resourcesMap.put(file.getResourceName(), - processService.queryTenantCodeByResName(file.getResourceName(), ResourceType.FILE))); - } - } - - return resourcesMap; - } - private void setTaskResourceInfo(ResourceParametersHelper resourceParametersHelper) { if (Objects.isNull(resourceParametersHelper)) { return; @@ -219,9 +199,6 @@ public class TaskExecutionContextFactory { udfFuncList.forEach(udfFunc -> { UdfFuncParameters udfFuncParameters = JSONUtils.parseObject(JSONUtils.toJsonString(udfFunc), UdfFuncParameters.class); - udfFuncParameters.setDefaultFS(PropertyUtils.getString(Constants.FS_DEFAULT_FS)); - String tenantCode = processService.queryTenantCodeByResName(udfFunc.getResourceName(), ResourceType.UDF); - udfFuncParameters.setTenantCode(tenantCode); map.put(udfFunc.getId(), udfFuncParameters); }); } diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index ad9b82974a..70d2593a32 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -39,7 +39,6 @@ import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation; import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.ProjectUser; -import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.Schedule; import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog; @@ -50,7 +49,6 @@ import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.service.exceptions.CronParseException; import org.apache.dolphinscheduler.service.model.TaskNode; -import org.apache.dolphinscheduler.spi.enums.ResourceType; import java.util.List; import java.util.Map; @@ -135,8 +133,6 @@ public interface ProcessService { List queryUdfFunListByIds(Integer[] ids); - String queryTenantCodeByResName(String resName, ResourceType resourceType); - List selectAllByProcessDefineCode(long[] codes); String queryUserQueueByProcessInstance(ProcessInstance processInstance); @@ -149,10 +145,6 @@ public interface ProcessService { User getUserById(int userId); - Resource getResourceById(int resourceId); - - List listResourceByIds(Integer[] resIds); - String formatTaskAppId(TaskInstance taskInstance); int switchVersion(ProcessDefinition processDefinition, ProcessDefinitionLog processDefinitionLog); diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java index d065d62f53..44945d03fb 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java @@ -69,7 +69,6 @@ import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation; import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog; import org.apache.dolphinscheduler.dao.entity.Project; import org.apache.dolphinscheduler.dao.entity.ProjectUser; -import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.Schedule; import org.apache.dolphinscheduler.dao.entity.TaskDefinition; import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog; @@ -89,7 +88,6 @@ import org.apache.dolphinscheduler.dao.mapper.DqRuleInputEntryMapper; import org.apache.dolphinscheduler.dao.mapper.DqRuleMapper; import org.apache.dolphinscheduler.dao.mapper.DqTaskStatisticsValueMapper; import org.apache.dolphinscheduler.dao.mapper.EnvironmentMapper; -import org.apache.dolphinscheduler.dao.mapper.ErrorCommandMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionLogMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapMapper; @@ -97,8 +95,6 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationLogMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; -import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; -import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper; import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; @@ -125,8 +121,6 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.enums.dp.DqTaskState; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; -import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; -import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; import org.apache.dolphinscheduler.plugin.task.api.parameters.SubProcessParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.TaskTimeoutParameter; import org.apache.dolphinscheduler.service.command.CommandService; @@ -137,7 +131,6 @@ import org.apache.dolphinscheduler.service.expand.CuringParamsService; import org.apache.dolphinscheduler.service.model.TaskNode; import org.apache.dolphinscheduler.service.utils.ClusterConfUtils; import org.apache.dolphinscheduler.service.utils.DagHelper; -import org.apache.dolphinscheduler.spi.enums.ResourceType; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; @@ -169,7 +162,6 @@ import org.springframework.transaction.annotation.Transactional; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.base.Joiner; import com.google.common.base.Strings; import com.google.common.collect.Lists; @@ -226,15 +218,6 @@ public class ProcessServiceImpl implements ProcessService { @Autowired private UdfFuncMapper udfFuncMapper; - @Autowired - private ResourceMapper resourceMapper; - - @Autowired - private ResourceUserMapper resourceUserMapper; - - @Autowired - private ErrorCommandMapper errorCommandMapper; - @Autowired private TenantMapper tenantMapper; @@ -1382,7 +1365,6 @@ public class ProcessServiceImpl implements ProcessService { return new ResourceInfo(); } resourceInfo = new ResourceInfo(); - resourceInfo.setId(-1); resourceInfo.setResourceName(resourceFullName); log.info("updated resource info {}", JSONUtils.toJsonString(resourceInfo)); @@ -1617,34 +1599,6 @@ public class ProcessServiceImpl implements ProcessService { return udfFuncMapper.queryUdfByIdStr(ids, null); } - /** - * find tenant code by resource name - * - * @param resName resource name - * @param resourceType resource type - * @return tenant code - */ - @Override - public String queryTenantCodeByResName(String resName, ResourceType resourceType) { - // in order to query tenant code successful although the version is older - String fullName = resName.startsWith("/") ? resName : String.format("/%s", resName); - - List resourceList = resourceMapper.queryResource(fullName, resourceType.ordinal()); - if (CollectionUtils.isEmpty(resourceList)) { - return ""; - } - int userId = resourceList.get(0).getUserId(); - User user = userMapper.selectById(userId); - if (Objects.isNull(user)) { - return ""; - } - Tenant tenant = tenantMapper.queryById(user.getTenantId()); - if (Objects.isNull(tenant)) { - return ""; - } - return tenant.getTenantCode(); - } - /** * find schedule list by process define codes. * @@ -1723,20 +1677,6 @@ public class ProcessServiceImpl implements ProcessService { Set originResSet = new HashSet<>(Arrays.asList(needChecks)); switch (authorizationType) { - case RESOURCE_FILE_ID: - case UDF_FILE: - List ownUdfResources = resourceMapper.listAuthorizedResourceById(userId, needChecks); - addAuthorizedResources(ownUdfResources, userId); - Set authorizedResourceFiles = - ownUdfResources.stream().map(Resource::getId).collect(toSet()); - originResSet.removeAll(authorizedResourceFiles); - break; - case RESOURCE_FILE_NAME: - List ownResources = resourceMapper.listAuthorizedResource(userId, needChecks); - addAuthorizedResources(ownResources, userId); - Set authorizedResources = ownResources.stream().map(Resource::getFullName).collect(toSet()); - originResSet.removeAll(authorizedResources); - break; case DATASOURCE: Set authorizedDatasources = dataSourceMapper.listAuthorizedDataSource(userId, needChecks) .stream().map(DataSource::getId).collect(toSet()); @@ -1768,28 +1708,6 @@ public class ProcessServiceImpl implements ProcessService { return userMapper.selectById(userId); } - /** - * get resource by resource id - * - * @param resourceId resource id - * @return Resource - */ - @Override - public Resource getResourceById(int resourceId) { - return resourceMapper.selectById(resourceId); - } - - /** - * list resources by ids - * - * @param resIds resIds - * @return resource list - */ - @Override - public List listResourceByIds(Integer[] resIds) { - return resourceMapper.listResourceByIds(resIds); - } - /** * format task app id in task instance */ @@ -1880,22 +1798,10 @@ public class ProcessServiceImpl implements ProcessService { * @param taskDefinition taskDefinition * @return resource ids */ + @Deprecated @Override public String getResourceIds(TaskDefinition taskDefinition) { - Set resourceIds = null; - AbstractParameters params = taskPluginManager.getParameters(ParametersNode.builder() - .taskType(taskDefinition.getTaskType()).taskParams(taskDefinition.getTaskParams()).build()); - - if (params != null && CollectionUtils.isNotEmpty(params.getResourceFilesList())) { - resourceIds = params.getResourceFilesList().stream() - .map(ResourceInfo::getId) - .filter(Objects::nonNull) - .collect(toSet()); - } - if (CollectionUtils.isEmpty(resourceIds)) { - return ""; - } - return Joiner.on(",").join(resourceIds); + return ""; } @Override @@ -2136,20 +2042,6 @@ public class ProcessServiceImpl implements ProcessService { return processTaskRelationLogList.stream().map(r -> (ProcessTaskRelation) r).collect(Collectors.toList()); } - /** - * add authorized resources - * - * @param ownResources own resources - * @param userId userId - */ - private void addAuthorizedResources(List ownResources, int userId) { - List relationResourceIds = resourceUserMapper.queryResourcesIdListByUserIdAndPerm(userId, 7); - List relationResources = CollectionUtils.isNotEmpty(relationResourceIds) - ? resourceMapper.queryResourceListById(relationResourceIds) - : new ArrayList<>(); - ownResources.addAll(relationResources); - } - /** * Use temporarily before refactoring taskNode */ diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java index 9944b1fdce..70157f6a05 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java @@ -59,7 +59,6 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationLogMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationMapper; -import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper; import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper; import org.apache.dolphinscheduler.dao.mapper.TaskGroupMapper; @@ -151,8 +150,6 @@ public class ProcessServiceTest { @Mock private ProcessDefinitionLogMapper processDefineLogMapper; @Mock - private ResourceMapper resourceMapper; - @Mock private TaskGroupMapper taskGroupMapper; @Mock private DataSourceMapper dataSourceMapper; @@ -736,12 +733,10 @@ public class ProcessServiceTest { // test normal situation ResourceInfo resourceInfoNormal = new ResourceInfo(); - resourceInfoNormal.setId(1); resourceInfoNormal.setResourceName("/test.txt"); ResourceInfo updatedResourceInfo3 = processService.updateResourceInfo(0, resourceInfoNormal); - Assertions.assertEquals(-1, updatedResourceInfo3.getId().intValue()); Assertions.assertEquals("/test.txt", updatedResourceInfo3.getResourceName()); } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperator.java index f7c4231b9f..bb899030c5 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-abs/src/main/java/org/apache/dolphinscheduler/plugin/storage/abs/AbsStorageOperator.java @@ -148,12 +148,6 @@ public class AbsStorageOperator implements Closeable, StorageOperate { return getAbsUdfDir(tenantCode) + FOLDER_SEPARATOR; } - @Override - public String getResourceFileName(String tenantCode, String fullName) { - String resDir = getResDir(tenantCode); - return fullName.replaceFirst(resDir, ""); - } - @Override public String getResourceFullName(String tenantCode, String fileName) { if (fileName.startsWith(FOLDER_SEPARATOR)) { @@ -171,8 +165,7 @@ public class AbsStorageOperator implements Closeable, StorageOperate { } @Override - public void download(String tenantCode, String srcFilePath, String dstFilePath, - boolean overwrite) throws IOException { + public void download(String srcFilePath, String dstFilePath, boolean overwrite) throws IOException { File dstFile = new File(dstFilePath); if (dstFile.isDirectory()) { Files.delete(dstFile.toPath()); diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperate.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperate.java index 5934323afc..ab096808b0 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperate.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-api/src/main/java/org/apache/dolphinscheduler/plugin/storage/api/StorageOperate.java @@ -69,10 +69,11 @@ public interface StorageOperate { /** * get the path of the resource file excluding the base path (fileName) - * @param fullName - * @return */ - String getResourceFileName(String tenantCode, String fullName); + default String getResourceFileName(String tenantCode, String fullName) { + String resDir = getResDir(tenantCode); + return fullName.replaceFirst(resDir, ""); + } /** * get the path of the file @@ -137,14 +138,13 @@ public interface StorageOperate { /** * download the srcPath to local - * @param tenantCode + * * @param srcFilePath the full path of the srcPath * @param dstFile * @param overwrite * @throws IOException */ - void download(String tenantCode, String srcFilePath, String dstFile, - boolean overwrite) throws IOException; + void download(String srcFilePath, String dstFile, boolean overwrite) throws IOException; /** * vim the context of filePath diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java index 94c015dfe4..9d09c2c9bb 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-gcs/src/main/java/org/apache/dolphinscheduler/plugin/storage/gcs/GcsStorageOperator.java @@ -129,12 +129,6 @@ public class GcsStorageOperator implements Closeable, StorageOperate { return String.format(FORMAT_S_S, getGcsResDir(tenantCode), fileName); } - @Override - public String getResourceFileName(String tenantCode, String fullName) { - String resDir = getResDir(tenantCode); - return fullName.replaceFirst(resDir, ""); - } - @Override public String getFileName(ResourceType resourceType, String tenantCode, String fileName) { if (fileName.startsWith(FOLDER_SEPARATOR)) { @@ -144,8 +138,7 @@ public class GcsStorageOperator implements Closeable, StorageOperate { } @Override - public void download(String tenantCode, String srcFilePath, String dstFilePath, - boolean overwrite) throws IOException { + public void download(String srcFilePath, String dstFilePath, boolean overwrite) throws IOException { File dstFile = new File(dstFilePath); if (dstFile.isDirectory()) { Files.delete(dstFile.toPath()); diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java index 04449fd0c3..f5891a5b92 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/main/java/org/apache/dolphinscheduler/plugin/storage/hdfs/HdfsStorageOperator.java @@ -281,20 +281,13 @@ public class HdfsStorageOperator implements Closeable, StorageOperate { return getHdfsResourceFileName(tenantCode, fullName); } - @Override - public String getResourceFileName(String tenantCode, String fullName) { - String resDir = getResDir(tenantCode); - return fullName.replaceFirst(resDir, ""); - } - @Override public String getFileName(ResourceType resourceType, String tenantCode, String fileName) { return getHdfsFileName(resourceType, tenantCode, fileName); } @Override - public void download(String tenantCode, String srcHdfsFilePath, String dstFile, - boolean overwrite) throws IOException { + public void download(String srcHdfsFilePath, String dstFile, boolean overwrite) throws IOException { copyHdfsToLocal(srcHdfsFilePath, dstFile, false, overwrite); } diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java index 6dc5318519..b660c59ab4 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java @@ -153,12 +153,6 @@ public class ObsStorageOperator implements Closeable, StorageOperate { return String.format(FORMAT_S_S, getObsResDir(tenantCode), fileName); } - @Override - public String getResourceFileName(String tenantCode, String fullName) { - String resDir = getResDir(tenantCode); - return fullName.replaceFirst(resDir, ""); - } - @Override public String getFileName(ResourceType resourceType, String tenantCode, String fileName) { if (fileName.startsWith(FOLDER_SEPARATOR)) { @@ -188,8 +182,7 @@ public class ObsStorageOperator implements Closeable, StorageOperate { } @Override - public void download(String tenantCode, String srcFilePath, String dstFilePath, - boolean overwrite) throws IOException { + public void download(String srcFilePath, String dstFilePath, boolean overwrite) throws IOException { File dstFile = new File(dstFilePath); if (dstFile.isDirectory()) { Files.delete(dstFile.toPath()); diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java index 80e8175cdd..2b2d18fd27 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-oss/src/main/java/org/apache/dolphinscheduler/plugin/storage/oss/OssStorageOperator.java @@ -181,12 +181,6 @@ public class OssStorageOperator implements Closeable, StorageOperate { return String.format(FORMAT_S_S, getOssResDir(tenantCode), fileName); } - @Override - public String getResourceFileName(String tenantCode, String fullName) { - String resDir = getResDir(tenantCode); - return fullName.replaceFirst(resDir, ""); - } - @Override public String getFileName(ResourceType resourceType, String tenantCode, String fileName) { if (fileName.startsWith(FOLDER_SEPARATOR)) { @@ -213,7 +207,7 @@ public class OssStorageOperator implements Closeable, StorageOperate { } @Override - public void download(String tenantCode, String srcFilePath, String dstFilePath, + public void download(String srcFilePath, String dstFilePath, boolean overwrite) throws IOException { File dstFile = new File(dstFilePath); if (dstFile.isDirectory()) { diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java index 86d3216aa8..884c672d53 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-s3/src/main/java/org/apache/dolphinscheduler/plugin/storage/s3/S3StorageOperator.java @@ -185,12 +185,6 @@ public class S3StorageOperator implements Closeable, StorageOperate { return String.format(FORMAT_S_S, getS3ResDir(tenantCode), fileName); } - @Override - public String getResourceFileName(String tenantCode, String fullName) { - String resDir = getResDir(tenantCode); - return fullName.replaceFirst(resDir, ""); - } - @Override public String getFileName(ResourceType resourceType, String tenantCode, String fileName) { if (fileName.startsWith(FOLDER_SEPARATOR)) { @@ -200,7 +194,7 @@ public class S3StorageOperator implements Closeable, StorageOperate { } @Override - public void download(String tenantCode, String srcFilePath, String dstFilePath, + public void download(String srcFilePath, String dstFilePath, boolean overwrite) throws IOException { File dstFile = new File(dstFilePath); if (dstFile.isDirectory()) { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java index e54df42944..8aa4c36af2 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/AbstractCommandExecutor.java @@ -22,7 +22,6 @@ import static org.apache.dolphinscheduler.common.constants.Constants.SLEEP_TIME_ import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE; import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_KILL; -import org.apache.dolphinscheduler.common.constants.TenantConstants; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; @@ -131,11 +130,7 @@ public abstract class AbstractCommandExecutor { // Set sudo (This is only work in Linux) iShellInterceptorBuilder.sudoMode(OSUtils.isSudoEnable()); // Set tenant (This is only work in Linux) - if (TenantConstants.DEFAULT_TENANT_CODE.equals(taskRequest.getTenantCode())) { - iShellInterceptorBuilder.runUser(TenantConstants.BOOTSTRAPT_SYSTEM_USER); - } else { - iShellInterceptorBuilder.runUser(taskRequest.getTenantCode()); - } + iShellInterceptorBuilder.runUser(taskRequest.getTenantCode()); // Set CPU Quota (This is only work in Linux) if (taskRequest.getCpuQuota() != null) { iShellInterceptorBuilder.cpuQuota(taskRequest.getCpuQuota()); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskExecutionContext.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskExecutionContext.java index 61f61fbf9c..4304ad2c13 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskExecutionContext.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskExecutionContext.java @@ -21,6 +21,7 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskTimeoutStrategy; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper; +import org.apache.dolphinscheduler.plugin.task.api.resource.ResourceContext; import java.io.Serializable; import java.util.Map; @@ -227,10 +228,8 @@ public class TaskExecutionContext implements Serializable { * k8s TaskExecutionContext */ private K8sTaskExecutionContext k8sTaskExecutionContext; - /** - * resources full name and tenant code - */ - private Map resources; + + private ResourceContext resourceContext; /** * taskInstance varPool diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/ResourceInfo.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/ResourceInfo.java index b4f4b5c861..47d726d4ce 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/ResourceInfo.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/model/ResourceInfo.java @@ -25,12 +25,14 @@ public class ResourceInfo { /** * res the name of the resource that was uploaded */ + @Deprecated private Integer id; /** - * full name of the resource that was uploaded + * ResourceAbsolutePathInStorage */ private String resourceName; + @Deprecated private String res; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/UdfFuncParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/UdfFuncParameters.java index 3830b5299d..a36c8d84b5 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/UdfFuncParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/resource/UdfFuncParameters.java @@ -90,10 +90,6 @@ public class UdfFuncParameters extends AbstractResourceParameters { */ private UdfType type; - private String tenantCode; - - private String defaultFS; - /** * create time */ diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/resource/ResourceContext.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/resource/ResourceContext.java new file mode 100644 index 0000000000..687d1aeb95 --- /dev/null +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/resource/ResourceContext.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.task.api.resource; + +import static com.google.common.base.Preconditions.checkNotNull; + +import java.util.HashMap; +import java.util.Map; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.ToString; + +@ToString +public class ResourceContext { + + /** + * ResourceAbsolutePathInStorage -> ResourceItem + */ + private final Map resourceItemMap; + + public ResourceContext() { + this.resourceItemMap = new HashMap<>(); + } + + public void addResourceItem(ResourceItem resourceItem) { + checkNotNull(resourceItem); + resourceItemMap.put(resourceItem.getResourceAbsolutePathInStorage(), resourceItem); + } + + public ResourceItem getResourceItem(String resourceAbsolutePathInStorage) { + ResourceItem resourceItem = resourceItemMap.get(resourceAbsolutePathInStorage); + if (resourceItem == null) { + throw new IllegalArgumentException("Cannot find the resourceItem: " + resourceAbsolutePathInStorage); + } + return resourceItem; + } + + @Data + @Builder + @AllArgsConstructor + @NoArgsConstructor + public static class ResourceItem { + + private String resourceAbsolutePathInStorage; + private String resourceRelativePath; + private String resourceAbsolutePathInLocal; + } + +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/shell/BaseLinuxShellInterceptorBuilder.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/shell/BaseLinuxShellInterceptorBuilder.java index d5a858a437..c391ca87a7 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/shell/BaseLinuxShellInterceptorBuilder.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/shell/BaseLinuxShellInterceptorBuilder.java @@ -63,7 +63,12 @@ public abstract class BaseLinuxShellInterceptorBuilder generateBootstrapCommand() throws FileOperateException { diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParametersTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParametersTest.java index 79792f491c..821668ae8f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParametersTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-datax/src/test/java/org/apache/dolphinscheduler/plugin/task/datax/DataxParametersTest.java @@ -58,7 +58,6 @@ public class DataxParametersTest { DataxParameters dataxParameters = new DataxParameters(); List resourceInfoList = new ArrayList<>(); ResourceInfo resourceInfo = new ResourceInfo(); - resourceInfo.setId(2); resourceInfo.setResourceName("/hdfs.keytab"); resourceInfoList.add(resourceInfo); @@ -90,7 +89,7 @@ public class DataxParametersTest { + "jobSpeedRecord=1, " + "xms=0, " + "xmx=-100, " - + "resourceList=[{\"id\":2,\"resourceName\":\"/hdfs.keytab\",\"res\":null}]" + + "resourceList=[{\"id\":null,\"resourceName\":\"/hdfs.keytab\",\"res\":null}]" + "}"; Assertions.assertEquals(expected, dataxParameters.toString()); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/test/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkArgsUtilsTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/test/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkArgsUtilsTest.java index c274ee2594..6c2c4c6100 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/test/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkArgsUtilsTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink-stream/src/test/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkArgsUtilsTest.java @@ -19,8 +19,8 @@ package org.apache.dolphinscheduler.plugin.task.flink; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; +import org.apache.dolphinscheduler.plugin.task.api.resource.ResourceContext; -import java.util.HashMap; import java.util.List; import org.junit.jupiter.api.Assertions; @@ -38,9 +38,7 @@ public class FlinkArgsUtilsTest { flinkParameters.setDeployMode(flinkDeployMode); flinkParameters.setParallelism(4); ResourceInfo resourceInfo = new ResourceInfo(); - resourceInfo.setId(1); resourceInfo.setResourceName("/opt/job.jar"); - resourceInfo.setRes("/opt/job.jar"); flinkParameters.setMainJar(resourceInfo); flinkParameters.setMainClass("org.example.Main"); flinkParameters.setSlot(4); @@ -54,9 +52,14 @@ public class FlinkArgsUtilsTest { TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); taskExecutionContext.setTaskAppId("app-id"); taskExecutionContext.setExecutePath("/tmp/execution"); - HashMap map = new HashMap<>(); - map.put("/opt/job.jar", "/opt/job.jar"); - taskExecutionContext.setResources(map); + + ResourceContext.ResourceItem resourceItem = new ResourceContext.ResourceItem(); + resourceItem.setResourceAbsolutePathInLocal("/opt/job.jar"); + resourceItem.setResourceAbsolutePathInStorage("/opt/job.jar"); + + ResourceContext resourceContext = new ResourceContext(); + resourceContext.addResourceItem(resourceItem); + taskExecutionContext.setResourceContext(resourceContext); return taskExecutionContext; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkArgsUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkArgsUtils.java index 6d28ffcc6d..fe374c9d8f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkArgsUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/main/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkArgsUtils.java @@ -20,6 +20,7 @@ package org.apache.dolphinscheduler.plugin.task.flink; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; +import org.apache.dolphinscheduler.plugin.task.api.resource.ResourceContext; import org.apache.dolphinscheduler.plugin.task.api.utils.ArgsUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; @@ -288,7 +289,8 @@ public class FlinkArgsUtils { if (ProgramType.PYTHON == programType) { args.add(FlinkConstants.FLINK_PYTHON); } - args.add(taskExecutionContext.getResources().get(mainJar.getResourceName())); + ResourceContext resourceContext = taskExecutionContext.getResourceContext(); + args.add(resourceContext.getResourceItem(mainJar.getResourceName()).getResourceAbsolutePathInLocal()); } String mainArgs = flinkParameters.getMainArgs(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/test/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkArgsUtilsTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/test/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkArgsUtilsTest.java index 64d9931f22..4ef1ad7091 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/test/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkArgsUtilsTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-flink/src/test/java/org/apache/dolphinscheduler/plugin/task/flink/FlinkArgsUtilsTest.java @@ -19,8 +19,8 @@ package org.apache.dolphinscheduler.plugin.task.flink; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; +import org.apache.dolphinscheduler.plugin.task.api.resource.ResourceContext; -import java.util.HashMap; import java.util.List; import org.junit.jupiter.api.Assertions; @@ -38,9 +38,7 @@ public class FlinkArgsUtilsTest { flinkParameters.setDeployMode(flinkDeployMode); flinkParameters.setParallelism(4); ResourceInfo resourceInfo = new ResourceInfo(); - resourceInfo.setId(1); resourceInfo.setResourceName("/opt/job.jar"); - resourceInfo.setRes("/opt/job.jar"); flinkParameters.setMainJar(resourceInfo); flinkParameters.setMainClass("org.example.Main"); flinkParameters.setSlot(4); @@ -54,9 +52,14 @@ public class FlinkArgsUtilsTest { TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); taskExecutionContext.setTaskAppId("app-id"); taskExecutionContext.setExecutePath("/tmp/execution"); - HashMap map = new HashMap<>(); - map.put("/opt/job.jar", "/opt/job.jar"); - taskExecutionContext.setResources(map); + + ResourceContext.ResourceItem resourceItem = new ResourceContext.ResourceItem(); + resourceItem.setResourceAbsolutePathInLocal("/opt/job.jar"); + resourceItem.setResourceAbsolutePathInStorage("/opt/job.jar"); + + ResourceContext resourceContext = new ResourceContext(); + resourceContext.addResourceItem(resourceItem); + taskExecutionContext.setResourceContext(resourceContext); return taskExecutionContext; } @@ -71,7 +74,7 @@ public class FlinkArgsUtilsTest { } @Test - public void testRunJarInClusterMode() throws Exception { + public void testRunJarInClusterMode() { FlinkParameters flinkParameters = buildTestFlinkParametersWithDeployMode(FlinkDeployMode.CLUSTER); flinkParameters.setFlinkVersion("1.11"); List commandLine1 = @@ -99,7 +102,7 @@ public class FlinkArgsUtilsTest { } @Test - public void testRunJarInLocalMode() throws Exception { + public void testRunJarInLocalMode() { FlinkParameters flinkParameters = buildTestFlinkParametersWithDeployMode(FlinkDeployMode.LOCAL); List commandLine = FlinkArgsUtils.buildRunCommandLine(buildTestTaskExecutionContext(), flinkParameters); @@ -109,7 +112,7 @@ public class FlinkArgsUtilsTest { } @Test - public void testRunSql() throws Exception { + public void testRunSql() { FlinkParameters flinkParameters = buildTestFlinkParametersWithDeployMode(FlinkDeployMode.CLUSTER); flinkParameters.setProgramType(ProgramType.SQL); List commandLine = FlinkArgsUtils.buildRunCommandLine(buildTestTaskExecutionContext(), flinkParameters); @@ -119,7 +122,7 @@ public class FlinkArgsUtilsTest { } @Test - public void testInitOptionsInLocalMode() throws Exception { + public void testInitOptionsInLocalMode() { List initOptions = FlinkArgsUtils.buildInitOptionsForSql(buildTestFlinkParametersWithDeployMode(FlinkDeployMode.LOCAL)); Assertions.assertEquals(2, initOptions.size()); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTask.java index 0ca6a2c39a..6630bfb8b7 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/main/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTask.java @@ -31,6 +31,7 @@ import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; +import org.apache.dolphinscheduler.plugin.task.api.resource.ResourceContext; import org.apache.dolphinscheduler.plugin.task.api.shell.IShellInterceptorBuilder; import org.apache.dolphinscheduler.plugin.task.api.shell.ShellInterceptorBuilderFactory; import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; @@ -137,8 +138,9 @@ public class HiveCliTask extends AbstractRemoteTask { try { resourceFileName = resourceInfos.get(0).getResourceName(); + ResourceContext resourceContext = taskExecutionContext.getResourceContext(); sqlContent = FileUtils.readFileToString( - new File(String.format("%s/%s", taskExecutionContext.getExecutePath(), resourceFileName)), + new File(resourceContext.getResourceItem(resourceFileName).getResourceAbsolutePathInLocal()), StandardCharsets.UTF_8); } catch (IOException e) { log.error("read hive sql content from file {} error ", resourceFileName, e); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/test/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/test/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskTest.java index 506651208b..824ad49f89 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/test/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-hivecli/src/test/java/org/apache/dolphinscheduler/plugin/task/hivecli/HiveCliTaskTest.java @@ -24,6 +24,7 @@ import static org.mockito.Mockito.when; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; +import org.apache.dolphinscheduler.plugin.task.api.resource.ResourceContext; import org.apache.commons.io.FileUtils; @@ -72,9 +73,17 @@ public class HiveCliTaskTest { } @Test - public void hiveCliTaskExecuteSqlFromFile() throws Exception { + public void hiveCliTaskExecuteSqlFromFile() { String hiveCliTaskParameters = buildHiveCliTaskExecuteSqlFromFileParameters(); - HiveCliTask hiveCliTask = prepareHiveCliTaskForTest(hiveCliTaskParameters); + TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); + taskExecutionContext.setTaskParams(hiveCliTaskParameters); + ResourceContext resourceContext = new ResourceContext(); + resourceContext.addResourceItem(new ResourceContext.ResourceItem("/sql_tasks/hive_task.sql", "123_node.sql", + "/sql_tasks/hive_task.sql")); + taskExecutionContext.setResourceContext(resourceContext); + + HiveCliTask hiveCliTask = spy(new HiveCliTask(taskExecutionContext)); + doReturn("123_node.sql").when(hiveCliTask).generateSqlScriptFile(Mockito.any()); hiveCliTask.init(); Assertions.assertEquals(hiveCliTask.buildCommand(), EXPECTED_HIVE_CLI_TASK_EXECUTE_FROM_FILE_COMMAND); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java index 5b4811bf8b..43dd5de586 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/main/java/org/apache/dolphinscheduler/plugin/task/java/JavaTask.java @@ -32,6 +32,7 @@ import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.model.TaskResponse; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; +import org.apache.dolphinscheduler.plugin.task.api.resource.ResourceContext; import org.apache.dolphinscheduler.plugin.task.api.shell.IShellInterceptorBuilder; import org.apache.dolphinscheduler.plugin.task.api.shell.ShellInterceptorBuilderFactory; import org.apache.dolphinscheduler.plugin.task.api.utils.MapUtils; @@ -176,7 +177,9 @@ public class JavaTask extends AbstractTask { * @return String **/ protected String buildJarCommand() { - String mainJarName = taskRequest.getResources().get(javaParameters.getMainJar().getResourceName()); + ResourceContext resourceContext = taskRequest.getResourceContext(); + String mainJarName = resourceContext.getResourceItem(javaParameters.getMainJar().getResourceName()) + .getResourceAbsolutePathInLocal(); StringBuilder builder = new StringBuilder(); builder.append(getJavaCommandPath()) .append("java").append(" ") @@ -279,14 +282,17 @@ public class JavaTask extends AbstractTask { } else { builder.append("--class-path"); } - builder.append(" ").append(JavaConstants.CLASSPATH_CURRENT_DIR) + builder.append(" ") + .append(JavaConstants.CLASSPATH_CURRENT_DIR) .append(JavaConstants.PATH_SEPARATOR) .append(taskRequest.getExecutePath()); - Map resourceMap = taskRequest.getResources(); + ResourceContext resourceContext = taskRequest.getResourceContext(); for (ResourceInfo info : javaParameters.getResourceFilesList()) { builder.append(JavaConstants.PATH_SEPARATOR); - builder.append(taskRequest.getExecutePath()).append(FOLDER_SEPARATOR) - .append(resourceMap.get(info.getResourceName())); + builder + .append(taskRequest.getExecutePath()) + .append(FOLDER_SEPARATOR) + .append(resourceContext.getResourceItem(info.getResourceName()).getResourceAbsolutePathInLocal()); } return builder.toString(); } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java index 4e6bc30e1f..39e1dac18d 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-java/src/test/java/org/apache/dolphinscheduler/plugin/task/java/JavaTaskTest.java @@ -28,6 +28,7 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.ApplicationInfo; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; +import org.apache.dolphinscheduler.plugin.task.api.resource.ResourceContext; import org.apache.dolphinscheduler.plugin.task.java.exception.JavaSourceFileExistException; import org.apache.dolphinscheduler.plugin.task.java.exception.PublicClassNotFoundException; import org.apache.dolphinscheduler.plugin.task.java.exception.RunTypeNotFoundException; @@ -39,7 +40,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.HashMap; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -212,9 +212,7 @@ public class JavaTaskTest { javaParameters.setJvmArgs("-xms:50m"); javaParameters.setMainArgs("-host 127.0.0.1 -port 8080"); ResourceInfo resourceJar = new ResourceInfo(); - resourceJar.setId(2); resourceJar.setResourceName("/opt/share/jar/resource2.jar"); - resourceJar.setRes("I'm resource2.jar"); ArrayList resourceInfoArrayList = new ArrayList<>(); resourceInfoArrayList.add(resourceJar); javaParameters.setResourceList(resourceInfoArrayList); @@ -236,9 +234,7 @@ public class JavaTaskTest { property.setType(VARCHAR); javaParameters.setLocalParams(localParams); ResourceInfo mainJar = new ResourceInfo(); - mainJar.setId(1); mainJar.setResourceName("/opt/share/jar/main.jar"); - mainJar.setRes("I'm main.jar"); javaParameters.setMainJar(mainJar); return javaParameters; } @@ -253,11 +249,23 @@ public class JavaTaskTest { taskExecutionContext.setTaskParams(JSONUtils.toJsonString(createJavaParametersObject(RUN_TYPE_JAVA))); taskExecutionContext.setExecutePath("/tmp/dolphinscheduler/test/executepath"); taskExecutionContext.setTaskAppId("runJavaType"); - HashMap map = new HashMap<>(); - map.put("/opt/share/jar/resource2.jar", "opt/share/jar/resource2.jar"); - map.put("/opt/share/jar/main.jar", "opt/share/jar/main.jar"); - map.put("/JavaTaskTest.java", "JavaTaskTest.java"); - taskExecutionContext.setResources(map); + ResourceContext.ResourceItem resourceItem1 = new ResourceContext.ResourceItem(); + resourceItem1.setResourceAbsolutePathInStorage("/opt/share/jar/resource2.jar"); + resourceItem1.setResourceAbsolutePathInLocal("opt/share/jar/resource2.jar"); + + ResourceContext.ResourceItem resourceItem2 = new ResourceContext.ResourceItem(); + resourceItem2.setResourceAbsolutePathInStorage("/opt/share/jar/main.jar"); + resourceItem2.setResourceAbsolutePathInLocal("opt/share/jar/main.jar"); + + ResourceContext.ResourceItem resourceItem3 = new ResourceContext.ResourceItem(); + resourceItem2.setResourceAbsolutePathInStorage("/JavaTaskTest.java"); + resourceItem2.setResourceAbsolutePathInLocal("JavaTaskTest.java"); + + ResourceContext resourceContext = new ResourceContext(); + resourceContext.addResourceItem(resourceItem1); + resourceContext.addResourceItem(resourceItem2); + resourceContext.addResourceItem(resourceItem3); + taskExecutionContext.setResourceContext(resourceContext); JavaTask javaTask = new JavaTask(taskExecutionContext); javaTask.init(); return javaTask; @@ -268,15 +276,24 @@ public class JavaTaskTest { * * @return JavaTask **/ - public JavaTask runJarType() { + private JavaTask runJarType() { TaskExecutionContext taskExecutionContext = new TaskExecutionContext(); taskExecutionContext.setTaskParams(JSONUtils.toJsonString(createJavaParametersObject(RUN_TYPE_JAR))); taskExecutionContext.setExecutePath("/tmp/dolphinscheduler/test/executepath"); taskExecutionContext.setTaskAppId("runJavaType"); - HashMap map = new HashMap<>(); - map.put("/opt/share/jar/resource2.jar", "opt/share/jar/resource2.jar"); - map.put("/opt/share/jar/main.jar", "opt/share/jar/main.jar"); - taskExecutionContext.setResources(map); + ResourceContext.ResourceItem resourceItem1 = new ResourceContext.ResourceItem(); + resourceItem1.setResourceAbsolutePathInStorage("/opt/share/jar/resource2.jar"); + resourceItem1.setResourceAbsolutePathInLocal("opt/share/jar/resource2.jar"); + + ResourceContext.ResourceItem resourceItem2 = new ResourceContext.ResourceItem(); + resourceItem2.setResourceAbsolutePathInStorage("/opt/share/jar/main.jar"); + resourceItem2.setResourceAbsolutePathInLocal("opt/share/jar/main.jar"); + + ResourceContext resourceContext = new ResourceContext(); + resourceContext.addResourceItem(resourceItem1); + resourceContext.addResourceItem(resourceItem2); + taskExecutionContext.setResourceContext(resourceContext); + JavaTask javaTask = new JavaTask(taskExecutionContext); javaTask.init(); return javaTask; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceArgsUtils.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceArgsUtils.java index 97e3bcc661..dfa08db1e5 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceArgsUtils.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-mr/src/main/java/org/apache/dolphinscheduler/plugin/task/mr/MapReduceArgsUtils.java @@ -24,6 +24,7 @@ import static org.apache.dolphinscheduler.plugin.task.mr.MapReduceTaskConstants. import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; +import org.apache.dolphinscheduler.plugin.task.api.resource.ResourceContext; import org.apache.dolphinscheduler.plugin.task.api.utils.ArgsUtils; import org.apache.commons.lang3.StringUtils; @@ -52,7 +53,8 @@ public class MapReduceArgsUtils { ResourceInfo mainJar = param.getMainJar(); if (mainJar != null) { args.add(JAR); - args.add(taskExecutionContext.getResources().get(mainJar.getResourceName())); + ResourceContext resourceContext = taskExecutionContext.getResourceContext(); + args.add(resourceContext.getResourceItem(mainJar.getResourceName()).getResourceAbsolutePathInLocal()); } ProgramType programType = param.getProgramType(); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java index 99a9d9e61b..a0d1f3fc77 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/main/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTask.java @@ -31,6 +31,7 @@ import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.Property; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; +import org.apache.dolphinscheduler.plugin.task.api.resource.ResourceContext; import org.apache.dolphinscheduler.plugin.task.api.utils.ArgsUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; @@ -179,7 +180,8 @@ public class SparkTask extends AbstractYarnTask { ResourceInfo mainJar = sparkParameters.getMainJar(); if (programType != ProgramType.SQL) { - args.add(taskExecutionContext.getResources().get(mainJar.getResourceName())); + ResourceContext resourceContext = taskExecutionContext.getResourceContext(); + args.add(resourceContext.getResourceItem(mainJar.getResourceName()).getResourceAbsolutePathInLocal()); } String mainArgs = sparkParameters.getMainArgs(); @@ -200,8 +202,10 @@ public class SparkTask extends AbstractYarnTask { try { resourceFileName = resourceInfos.get(0).getResourceName(); + ResourceContext resourceContext = taskExecutionContext.getResourceContext(); sqlContent = FileUtils.readFileToString( - new File(String.format("%s/%s", taskExecutionContext.getExecutePath(), resourceFileName)), + new File( + resourceContext.getResourceItem(resourceFileName).getResourceAbsolutePathInLocal()), StandardCharsets.UTF_8); } catch (IOException e) { log.error("read sql content from file {} error ", resourceFileName, e); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskTest.java index ba031bb304..78d8968e59 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-spark/src/test/java/org/apache/dolphinscheduler/plugin/task/spark/SparkTaskTest.java @@ -20,9 +20,9 @@ package org.apache.dolphinscheduler.plugin.task.spark; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; +import org.apache.dolphinscheduler.plugin.task.api.resource.ResourceContext; import java.util.Collections; -import java.util.HashMap; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -61,10 +61,14 @@ public class SparkTaskTest { public void testBuildCommandWithSparkSubmit() { String parameters = buildSparkParametersWithSparkSubmit(); TaskExecutionContext taskExecutionContext = Mockito.mock(TaskExecutionContext.class); - HashMap map = new HashMap<>(); - map.put("/lib/dolphinscheduler-task-spark.jar", "/lib/dolphinscheduler-task-spark.jar"); - Mockito.when(taskExecutionContext.getResources()).thenReturn(map); + ResourceContext.ResourceItem resourceItem = new ResourceContext.ResourceItem(); + resourceItem.setResourceAbsolutePathInStorage("/lib/dolphinscheduler-task-spark.jar"); + resourceItem.setResourceAbsolutePathInLocal("/lib/dolphinscheduler-task-spark.jar"); + ResourceContext resourceContext = new ResourceContext(); + resourceContext.addResourceItem(resourceItem); + Mockito.when(taskExecutionContext.getTaskParams()).thenReturn(parameters); + Mockito.when(taskExecutionContext.getResourceContext()).thenReturn(resourceContext); SparkTask sparkTask = Mockito.spy(new SparkTask(taskExecutionContext)); sparkTask.init(); Assertions.assertEquals( @@ -113,8 +117,6 @@ public class SparkTaskTest { sparkParameters.setExecutorMemory("1G"); sparkParameters.setExecutorCores(2); ResourceInfo resourceInfo = new ResourceInfo(); - resourceInfo.setId(1); - resourceInfo.setRes("dolphinscheduler-task-spark.jar"); resourceInfo.setResourceName("/lib/dolphinscheduler-task-spark.jar"); sparkParameters.setMainJar(resourceInfo); return JSONUtils.toJsonString(sparkParameters); diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java index acadd98975..260ea99fea 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java @@ -20,7 +20,6 @@ package org.apache.dolphinscheduler.plugin.task.sql; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceClientProvider; -import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils; import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils; import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.SQLTaskExecutionContext; @@ -36,6 +35,7 @@ import org.apache.dolphinscheduler.plugin.task.api.model.TaskAlertInfo; import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.SqlParameters; import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.UdfFuncParameters; +import org.apache.dolphinscheduler.plugin.task.api.resource.ResourceContext; import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils; import org.apache.dolphinscheduler.plugin.task.sql.utils.SqlSplitUtils; import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam; @@ -518,11 +518,10 @@ public class SqlTask extends AbstractTask { */ private List buildJarSql(List udfFuncParameters) { return udfFuncParameters.stream().map(value -> { - String defaultFS = value.getDefaultFS(); - String prefixPath = defaultFS.startsWith("file://") ? "file://" : defaultFS; - String uploadPath = CommonUtils.getHdfsUdfDir(value.getTenantCode()); String resourceFullName = value.getResourceName(); - return String.format("add jar %s", resourceFullName); + ResourceContext resourceContext = taskExecutionContext.getResourceContext(); + return String.format("add jar %s", + resourceContext.getResourceItem(resourceFullName).getResourceAbsolutePathInLocal()); }).collect(Collectors.toList()); } diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/resource/MigrateResource.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/resource/MigrateResource.java index e519dc05d7..3247003eac 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/resource/MigrateResource.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/resource/MigrateResource.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.tools.resource; +import java.sql.SQLException; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -45,7 +47,7 @@ public class MigrateResource { private MigrateResourceService migrateResourceService; @Override - public void run(String... args) { + public void run(String... args) throws SQLException { String targetTenantCode = args[0]; logger.info("Moving all unmanaged resources to tenant: {}", targetTenantCode); migrateResourceService.migrateResourceOnce(targetTenantCode); diff --git a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/resource/MigrateResourceService.java b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/resource/MigrateResourceService.java index 942aa1ede7..8961af1223 100644 --- a/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/resource/MigrateResourceService.java +++ b/dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/resource/MigrateResourceService.java @@ -19,9 +19,7 @@ package org.apache.dolphinscheduler.tools.resource; import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S; -import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.UdfFunc; -import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper; import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; @@ -30,7 +28,16 @@ import org.apache.dolphinscheduler.spi.enums.ResourceType; import org.apache.commons.lang3.StringUtils; import java.io.IOException; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; + +import javax.sql.DataSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,14 +56,14 @@ public class MigrateResourceService { private TenantMapper tenantMapper; @Autowired - private ResourceMapper resourceMapper; + private UdfFuncMapper udfFuncMapper; @Autowired - private UdfFuncMapper udfFuncMapper; + private DataSource dataSource; private static final String MIGRATE_BASE_DIR = ".migrate"; - public void migrateResourceOnce(String targetTenantCode) { + public void migrateResourceOnce(String targetTenantCode) throws SQLException { if (true != tenantMapper.existTenant(targetTenantCode)) { logger.error("Tenant not exists!"); return; @@ -67,31 +74,50 @@ public class MigrateResourceService { if (StringUtils.isEmpty(resMigrateBasePath) || StringUtils.isEmpty(udfMigrateBasePath)) { return; } - // migrate all unmanaged resources and udfs once - List resources = resourceMapper.queryResourceExceptUserId(-1); - resources.forEach(resource -> { + List> resources = getAllResources(); + for (Map item : resources) { + String oriFullName = (String) item.get("full_name"); + int type = (int) item.get("type"); + int id = (int) item.get("id"); try { - String oriFullName = resource.getFullName(); oriFullName = oriFullName.startsWith("/") ? oriFullName.substring(1) : oriFullName; - if (resource.getType().equals(ResourceType.FILE)) { + if (ResourceType.FILE.getCode() == type) { storageOperate.copy(oriFullName, String.format(FORMAT_S_S, resMigrateBasePath, oriFullName), true, true); - } else if (resource.getType().equals(ResourceType.UDF)) { + } else if (ResourceType.UDF.getCode() == type) { String fullName = String.format(FORMAT_S_S, udfMigrateBasePath, oriFullName); storageOperate.copy(oriFullName, fullName, true, true); // change relative udfs resourceName - List udfs = udfFuncMapper.listUdfByResourceId(new Integer[]{resource.getId()}); + List udfs = udfFuncMapper.listUdfByResourceId(new Integer[]{id}); udfs.forEach(udf -> { udf.setResourceName(fullName); udfFuncMapper.updateById(udf); }); } } catch (IOException e) { - logger.error("Migrate resource failed: {}", e.getMessage()); + logger.error("Migrate resource: {} failed: {}", item, e); } - }); + } + } + + private List> getAllResources() throws SQLException { + try ( + Connection connection = dataSource.getConnection(); + PreparedStatement preparedStatement = + connection.prepareStatement("select * from t_ds_resources where user_id != -1"); + ResultSet resultSet = preparedStatement.executeQuery()) { + List> result = new ArrayList<>(); + while (resultSet.next()) { + Map item = new HashMap<>(); + item.put("id", resultSet.getInt("id")); + item.put("full_name", resultSet.getString("full_name")); + item.put("type", resultSet.getInt("type")); + result.add(item); + } + return result; + } } public String createMigrateDirByType(String targetTenantCode, ResourceType type) { diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutor.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutor.java index c0c7c35b7d..c335f34994 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutor.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/runner/WorkerTaskExecutor.java @@ -36,6 +36,7 @@ import org.apache.dolphinscheduler.plugin.datasource.api.utils.CommonUtils; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; import org.apache.dolphinscheduler.plugin.task.api.AbstractTask; import org.apache.dolphinscheduler.plugin.task.api.TaskCallBack; +import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContextCacheManager; @@ -45,12 +46,13 @@ import org.apache.dolphinscheduler.plugin.task.api.enums.Direct; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.log.TaskInstanceLogHeader; import org.apache.dolphinscheduler.plugin.task.api.model.TaskAlertInfo; +import org.apache.dolphinscheduler.plugin.task.api.resource.ResourceContext; import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils; import org.apache.dolphinscheduler.plugin.task.api.utils.ProcessUtils; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.registry.WorkerRegistryClient; import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender; -import org.apache.dolphinscheduler.server.worker.utils.TaskExecutionCheckerUtils; +import org.apache.dolphinscheduler.server.worker.utils.TaskExecutionContextUtils; import org.apache.dolphinscheduler.server.worker.utils.TaskFilesTransferUtils; import java.io.File; @@ -208,23 +210,33 @@ public abstract class WorkerTaskExecutor implements Runnable { log.info("Send task status {} master: {}", TaskExecutionStatus.RUNNING_EXECUTION.name(), taskExecutionContext.getHost()); - TaskExecutionCheckerUtils.checkTenantExist(workerConfig, taskExecutionContext); + // In most of case the origin tenant is the same as the current tenant + // Except `default` tenant. The originTenant is used to download the resources + String originTenant = taskExecutionContext.getTenantCode(); + String tenant = TaskExecutionContextUtils.getOrCreateTenant(workerConfig, taskExecutionContext); + taskExecutionContext.setTenantCode(tenant); log.info("TenantCode: {} check successfully", taskExecutionContext.getTenantCode()); - TaskExecutionCheckerUtils.createProcessLocalPathIfAbsent(taskExecutionContext); + TaskExecutionContextUtils.createProcessLocalPathIfAbsent(taskExecutionContext); log.info("WorkflowInstanceExecDir: {} check successfully", taskExecutionContext.getExecutePath()); - TaskExecutionCheckerUtils.downloadResourcesIfNeeded(storageOperate, taskExecutionContext); - log.info("Download resources: {} successfully", taskExecutionContext.getResources()); + TaskChannel taskChannel = + Optional.ofNullable(taskPluginManager.getTaskChannelMap().get(taskExecutionContext.getTaskType())) + .orElseThrow(() -> new TaskPluginException(taskExecutionContext.getTaskType() + + " task plugin not found, please check the task type is correct.")); + + log.info("Create TaskChannel: {} successfully", taskChannel.getClass().getName()); + + ResourceContext resourceContext = TaskExecutionContextUtils.downloadResourcesIfNeeded(originTenant, taskChannel, + storageOperate, taskExecutionContext); + taskExecutionContext.setResourceContext(resourceContext); + log.info("Download resources successfully: \n{}", taskExecutionContext.getResourceContext()); TaskFilesTransferUtils.downloadUpstreamFiles(taskExecutionContext, storageOperate); log.info("Download upstream files: {} successfully", TaskFilesTransferUtils.getFileLocalParams(taskExecutionContext, Direct.IN)); - task = Optional.ofNullable(taskPluginManager.getTaskChannelMap().get(taskExecutionContext.getTaskType())) - .map(taskChannel -> taskChannel.createTask(taskExecutionContext)) - .orElseThrow(() -> new TaskPluginException(taskExecutionContext.getTaskType() - + " task plugin not found, please check the task type is correct.")); + task = taskChannel.createTask(taskExecutionContext); log.info("Task plugin instance: {} create successfully", taskExecutionContext.getTaskType()); // todo: remove the init method, this should initialize in constructor method diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionCheckerUtils.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java similarity index 59% rename from dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionCheckerUtils.java rename to dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java index 195e4d3a0a..bbe3b1ab4b 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionCheckerUtils.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionContextUtils.java @@ -21,36 +21,37 @@ import org.apache.dolphinscheduler.common.constants.TenantConstants; import org.apache.dolphinscheduler.common.utils.FileUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.plugin.storage.api.StorageOperate; +import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; import org.apache.dolphinscheduler.plugin.task.api.TaskException; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; +import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo; +import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters; +import org.apache.dolphinscheduler.plugin.task.api.parameters.ParametersNode; +import org.apache.dolphinscheduler.plugin.task.api.resource.ResourceContext; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.worker.metrics.WorkerServerMetrics; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.SystemUtils; -import org.apache.commons.lang3.tuple.Pair; import java.io.File; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; import java.util.List; -import java.util.Map; import lombok.extern.slf4j.Slf4j; @Slf4j -public class TaskExecutionCheckerUtils { +public class TaskExecutionContextUtils { - public static void checkTenantExist(WorkerConfig workerConfig, TaskExecutionContext taskExecutionContext) { + public static String getOrCreateTenant(WorkerConfig workerConfig, TaskExecutionContext taskExecutionContext) { try { String tenantCode = taskExecutionContext.getTenantCode(); if (TenantConstants.DEFAULT_TENANT_CODE.equals(tenantCode)) { - log.warn("Current tenant is default tenant, will use {} to execute the task", + log.info("Current tenant is default tenant, will use {} to execute the task", TenantConstants.BOOTSTRAPT_SYSTEM_USER); - return; + return TenantConstants.BOOTSTRAPT_SYSTEM_USER; } boolean osUserExistFlag; // if Using distributed is true and Currently supported systems are linux,Should not let it @@ -70,6 +71,7 @@ public class TaskExecutionCheckerUtils { throw new TaskException( String.format("TenantCode: %s doesn't exist", tenantCode)); } + return tenantCode; } catch (TaskException ex) { throw ex; } catch (Exception ex) { @@ -97,51 +99,55 @@ public class TaskExecutionCheckerUtils { } } - public static void downloadResourcesIfNeeded(StorageOperate storageOperate, - TaskExecutionContext taskExecutionContext) { - String execLocalPath = taskExecutionContext.getExecutePath(); - String tenant = taskExecutionContext.getTenantCode(); - String actualTenant = - TenantConstants.DEFAULT_TENANT_CODE.equals(tenant) ? TenantConstants.BOOTSTRAPT_SYSTEM_USER : tenant; + public static ResourceContext downloadResourcesIfNeeded(String tenant, + TaskChannel taskChannel, + StorageOperate storageOperate, + TaskExecutionContext taskExecutionContext) { + AbstractParameters abstractParameters = taskChannel.parseParameters( + ParametersNode.builder() + .taskType(taskExecutionContext.getTaskType()) + .taskParams(taskExecutionContext.getTaskParams()) + .build()); - Map projectRes = taskExecutionContext.getResources(); - if (MapUtils.isEmpty(projectRes)) { - return; + List resourceFilesList = abstractParameters.getResourceFilesList(); + if (CollectionUtils.isEmpty(resourceFilesList)) { + log.debug("There is no resource file need to download"); + return new ResourceContext(); } - List> downloadFiles = new ArrayList<>(); - projectRes.keySet().forEach(fullName -> { - String fileName = storageOperate.getResourceFileName(actualTenant, fullName); - projectRes.put(fullName, fileName); - File resFile = new File(execLocalPath, fileName); - boolean notExist = !resFile.exists(); - if (notExist) { - downloadFiles.add(Pair.of(fullName, fileName)); - } else { - log.warn("Resource file : {} already exists will not download again ", resFile.getName()); - } - }); - if (CollectionUtils.isNotEmpty(downloadFiles)) { - for (Pair fileDownload : downloadFiles) { - try { - String fullName = fileDownload.getLeft(); - String fileName = fileDownload.getRight(); + ResourceContext resourceContext = new ResourceContext(); + String taskWorkingDirectory = taskExecutionContext.getExecutePath(); + for (ResourceInfo resourceInfo : resourceFilesList) { + String resourceAbsolutePathInStorage = resourceInfo.getResourceName(); + String resourceRelativePath = storageOperate.getResourceFileName(tenant, resourceAbsolutePathInStorage); + String resourceAbsolutePathInLocal = Paths.get(taskWorkingDirectory, resourceRelativePath).toString(); + File file = new File(resourceAbsolutePathInLocal); + if (!file.exists()) { + try { long resourceDownloadStartTime = System.currentTimeMillis(); - - Path localFileAbsolutePath = Paths.get(execLocalPath, fileName); - storageOperate.download(actualTenant, fullName, localFileAbsolutePath.toString(), true); - log.info("Download resource file {} under: {} successfully", fileName, localFileAbsolutePath); + storageOperate.download(resourceAbsolutePathInStorage, resourceAbsolutePathInLocal, true); + log.debug("Download resource file {} under: {} successfully", resourceAbsolutePathInStorage, + resourceAbsolutePathInLocal); WorkerServerMetrics .recordWorkerResourceDownloadTime(System.currentTimeMillis() - resourceDownloadStartTime); - WorkerServerMetrics.recordWorkerResourceDownloadSize(Files.size(localFileAbsolutePath)); + WorkerServerMetrics + .recordWorkerResourceDownloadSize(Files.size(Paths.get(resourceAbsolutePathInLocal))); WorkerServerMetrics.incWorkerResourceDownloadSuccessCount(); - } catch (Exception e) { + } catch (Exception ex) { WorkerServerMetrics.incWorkerResourceDownloadFailureCount(); - throw new TaskException(String.format("Download resource file: %s error", fileDownload), e); + throw new TaskException( + String.format("Download resource file: %s error", resourceAbsolutePathInStorage), ex); } } + ResourceContext.ResourceItem resourceItem = ResourceContext.ResourceItem.builder() + .resourceAbsolutePathInStorage(resourceAbsolutePathInStorage) + .resourceRelativePath(resourceRelativePath) + .resourceAbsolutePathInLocal(resourceAbsolutePathInLocal) + .build(); + resourceContext.addResourceItem(resourceItem); } + return resourceContext; } } diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskFilesTransferUtils.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskFilesTransferUtils.java index e591b3610e..87438c614b 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskFilesTransferUtils.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskFilesTransferUtils.java @@ -178,7 +178,7 @@ public class TaskFilesTransferUtils { String resourceWholePath = storageOperate.getResourceFullName(taskExecutionContext.getTenantCode(), resourcePath); log.info("{} --- Remote:{} to Local:{}", property, resourceWholePath, downloadPath); - storageOperate.download(taskExecutionContext.getTenantCode(), resourceWholePath, downloadPath, true); + storageOperate.download(resourceWholePath, downloadPath, true); } catch (IOException ex) { throw new TaskException("Download file from storage error", ex); }