From 733acdfd76c10cd1948da7a96e222bbbfd62b777 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Sat, 11 Apr 2020 17:27:10 +0800 Subject: [PATCH] It is necessary to check whether the resource is valid (#2399) * init full_name in dml of version 1.2.2 * redefine request parameter resourceIds * redefine request parameter resourceIds * Actually grant resource file if choose the directory * To cancel authorized resource need check whether it is used by the process definition which is online * If release the process definition online,It is necessary to check whether resource is valid * update ResourceServiceTest and ResourceMapperTest * add batchUpdateResourceTest * add getHdfsFileNameTest and getHdfsResourceFileNameTest * update ResourceServiceTest and ResourceMapperTest * extract getResourceProcessMap to ResourceProcessDefinitonUtils Co-authored-by: dailidong --- .../dolphinscheduler/api/enums/Status.java | 1 + .../api/service/ProcessDefinitionService.java | 17 ++++ .../api/service/ResourcesService.java | 99 +++++++++---------- .../api/service/UsersService.java | 71 ++++++++++--- .../api/service/ResourcesServiceTest.java | 2 +- .../api/service/UsersServiceTest.java | 30 +++++- .../dolphinscheduler/common/Constants.java | 8 ++ .../common/utils/HadoopUtilsTest.java | 16 +++ .../dao/mapper/ResourceMapper.java | 4 +- .../dao/mapper/ResourceUserMapper.java | 9 ++ .../dao/mapper/UdfFuncMapper.java | 7 +- .../utils/ResourceProcessDefinitionUtils.java | 61 ++++++++++++ .../dao/mapper/ResourceMapper.xml | 14 ++- .../dao/mapper/ResourceUserMapper.xml | 13 +++ .../mapper/ProcessDefinitionMapperTest.java | 11 +++ .../dao/mapper/ResourceMapperTest.java | 33 +++++++ .../dao/mapper/ResourceUserMapperTest.java | 30 ++++-- .../ResourceProcessDefinitionUtilsTest.java | 41 ++++++++ 18 files changed, 386 insertions(+), 81 deletions(-) create mode 100644 dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtils.java create mode 100644 dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtilsTest.java diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java index 416dc0ef54..2cbf22e199 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java @@ -190,6 +190,7 @@ public enum Status { UDF_RESOURCE_IS_BOUND(20013, "udf resource file is bound by UDF functions:{0}","udf函数绑定了资源文件[{0}]"), RESOURCE_IS_USED(20014, "resource file is used by process definition","资源文件被上线的流程定义使用了"), PARENT_RESOURCE_NOT_EXIST(20015, "parent resource not exist","父资源文件不存在"), + RESOURCE_NOT_EXIST_OR_NO_PERMISSION(20016, "resource not exist or no permission,please view the task node and remove error resource","请检查任务节点并移除无权限或者已删除的资源"), USER_NO_OPERATION_PERM(30001, "user has no operation privilege", "当前用户没有操作权限"), diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java index 655e70a95d..eed9c78e74 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java @@ -44,6 +44,7 @@ import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.entity.*; import org.apache.dolphinscheduler.dao.mapper.*; import org.apache.dolphinscheduler.dao.utils.DagHelper; +import org.apache.dolphinscheduler.service.permission.PermissionCheck; import org.apache.dolphinscheduler.service.process.ProcessService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -143,6 +144,7 @@ public class ProcessDefinitionService extends BaseDAGService { processDefine.setTimeout(processData.getTimeout()); processDefine.setTenantId(processData.getTenantId()); processDefine.setModifyBy(loginUser.getUserName()); + processDefine.setResourceIds(getResourceIds(processData)); //custom global params List globalParamsList = processData.getGlobalParams(); @@ -333,6 +335,7 @@ public class ProcessDefinitionService extends BaseDAGService { processDefine.setTimeout(processData.getTimeout()); processDefine.setTenantId(processData.getTenantId()); processDefine.setModifyBy(loginUser.getUserName()); + processDefine.setResourceIds(getResourceIds(processData)); //custom global params List globalParamsList = new ArrayList<>(); @@ -476,6 +479,20 @@ public class ProcessDefinitionService extends BaseDAGService { switch (state) { case ONLINE: + // To check resources whether they are already cancel authorized or deleted + String resourceIds = processDefinition.getResourceIds(); + if (StringUtils.isNotBlank(resourceIds)) { + Integer[] resourceIdArray = Arrays.stream(resourceIds.split(",")).map(Integer::parseInt).toArray(Integer[]::new); + PermissionCheck permissionCheck = new PermissionCheck(AuthorizationType.RESOURCE_FILE_ID,processService,resourceIdArray,loginUser.getId(),logger); + try { + permissionCheck.checkPermission(); + } catch (Exception e) { + logger.error(e.getMessage(),e); + putMsg(result, Status.RESOURCE_NOT_EXIST_OR_NO_PERMISSION, "releaseState"); + return result; + } + } + processDefinition.setReleaseState(state); processDefineMapper.updateById(processDefinition); break; diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index ff87aadbc7..c98b7c31b9 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -36,6 +36,7 @@ import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.*; +import org.apache.dolphinscheduler.dao.utils.ResourceProcessDefinitionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -176,6 +177,21 @@ public class ResourcesService extends BaseService { putMsg(result, Status.HDFS_NOT_STARTUP); return result; } + + if (pid != -1) { + Resource parentResource = resourcesMapper.selectById(pid); + + if (parentResource == null) { + putMsg(result, Status.PARENT_RESOURCE_NOT_EXIST); + return result; + } + + if (!hasPerm(loginUser, parentResource.getUserId())) { + putMsg(result, Status.USER_NO_OPERATION_PERM); + return result; + } + } + // file is empty if (file.isEmpty()) { logger.error("file is empty: {}", file.getOriginalFilename()); @@ -416,6 +432,14 @@ public class ResourcesService extends BaseService { if (isAdmin(loginUser)) { userId= 0; } + if (direcotryId != -1) { + Resource directory = resourcesMapper.selectById(direcotryId); + if (directory == null) { + putMsg(result, Status.RESOURCE_NOT_EXIST); + return result; + } + } + IPage resourceIPage = resourcesMapper.queryResourcePaging(page, userId,direcotryId, type.ordinal(), searchVal); PageInfo pageInfo = new PageInfo(pageNo, pageSize); @@ -505,8 +529,12 @@ public class ResourcesService extends BaseService { Map result = new HashMap<>(5); - Set allResourceList = getAllResources(loginUser, type); - Visitor resourceTreeVisitor = new ResourceTreeVisitor(new ArrayList<>(allResourceList)); + int userId = loginUser.getId(); + if(isAdmin(loginUser)){ + userId = 0; + } + List allResourceList = resourcesMapper.queryResourceListAuthored(userId, type.ordinal(),0); + Visitor resourceTreeVisitor = new ResourceTreeVisitor(allResourceList); //JSONArray jsonArray = JSON.parseArray(JSON.toJSONString(resourceTreeVisitor.visit().getChildren(), SerializerFeature.SortField)); result.put(Constants.DATA_LIST, resourceTreeVisitor.visit().getChildren()); putMsg(result,Status.SUCCESS); @@ -519,7 +547,7 @@ public class ResourcesService extends BaseService { * @param loginUser login user * @return all resource set */ - private Set getAllResources(User loginUser, ResourceType type) { + /*private Set getAllResources(User loginUser, ResourceType type) { int userId = loginUser.getId(); boolean listChildren = true; if(isAdmin(loginUser)){ @@ -540,7 +568,7 @@ public class ResourcesService extends BaseService { } } return allResourceList; - } + }*/ /** * query resource list @@ -553,7 +581,7 @@ public class ResourcesService extends BaseService { Map result = new HashMap<>(5); - Set allResourceList = getAllResources(loginUser, type); + List allResourceList = resourcesMapper.queryResourceListAuthored(loginUser.getId(), type.ordinal(),0); List resources = new ResourceFilter(".jar",new ArrayList<>(allResourceList)).filter(); Visitor resourceTreeVisitor = new ResourceTreeVisitor(resources); result.put(Constants.DATA_LIST, resourceTreeVisitor.visit().getChildren()); @@ -592,15 +620,6 @@ public class ResourcesService extends BaseService { putMsg(result, Status.USER_NO_OPERATION_PERM); return result; } - //if resource type is UDF,need check whether it is bound by UDF functon - if (resource.getType() == (ResourceType.UDF)) { - List udfFuncs = udfFunctionMapper.listUdfByResourceId(new int[]{resourceId}); - if (CollectionUtils.isNotEmpty(udfFuncs)) { - logger.error("can't be deleted,because it is bound by UDF functions:{}",udfFuncs.toString()); - putMsg(result,Status.UDF_RESOURCE_IS_BOUND,udfFuncs.get(0).getFuncName()); - return result; - } - } String tenantCode = getTenantCode(resource.getUserId(),result); if (StringUtils.isEmpty(tenantCode)){ @@ -608,10 +627,22 @@ public class ResourcesService extends BaseService { } // get all resource id of process definitions those is released - Map> resourceProcessMap = getResourceProcessMap(); + List> list = processDefinitionMapper.listResources(); + Map> resourceProcessMap = ResourceProcessDefinitionUtils.getResourceProcessDefinitionMap(list); Set resourceIdSet = resourceProcessMap.keySet(); // get all children of the resource List allChildren = listAllChildren(resource); + Integer[] needDeleteResourceIdArray = allChildren.toArray(new Integer[allChildren.size()]); + + //if resource type is UDF,need check whether it is bound by UDF functon + if (resource.getType() == (ResourceType.UDF)) { + List udfFuncs = udfFunctionMapper.listUdfByResourceId(needDeleteResourceIdArray); + if (CollectionUtils.isNotEmpty(udfFuncs)) { + logger.error("can't be deleted,because it is bound by UDF functions:{}",udfFuncs.toString()); + putMsg(result,Status.UDF_RESOURCE_IS_BOUND,udfFuncs.get(0).getFuncName()); + return result; + } + } if (resourceIdSet.contains(resource.getPid())) { logger.error("can't be deleted,because it is used of process definition"); @@ -632,8 +663,8 @@ public class ResourcesService extends BaseService { String hdfsFilename = HadoopUtils.getHdfsFileName(resource.getType(), tenantCode, resource.getFullName()); //delete data in database - resourcesMapper.deleteIds(allChildren.toArray(new Integer[allChildren.size()])); - resourceUserMapper.deleteResourceUser(0, resourceId); + resourcesMapper.deleteIds(needDeleteResourceIdArray); + resourceUserMapper.deleteResourceUserArray(0, needDeleteResourceIdArray); //delete file on hdfs HadoopUtils.getInstance().delete(hdfsFilename, true); @@ -1191,38 +1222,4 @@ public class ResourcesService extends BaseService { } } - /** - * get resource process map key is resource id,value is the set of process definition - * @return resource process definition map - */ - private Map> getResourceProcessMap(){ - Map map = new HashMap<>(); - Map> result = new HashMap<>(); - List> list = processDefinitionMapper.listResources(); - if (CollectionUtils.isNotEmpty(list)) { - for (Map tempMap : list) { - - map.put((Integer) tempMap.get("id"), (String)tempMap.get("resource_ids")); - } - } - - for (Map.Entry entry : map.entrySet()) { - Integer mapKey = entry.getKey(); - String[] arr = entry.getValue().split(","); - Set mapValues = Arrays.stream(arr).map(Integer::parseInt).collect(Collectors.toSet()); - for (Integer value : mapValues) { - if (result.containsKey(value)) { - Set set = result.get(value); - set.add(mapKey); - result.put(value, set); - } else { - Set set = new HashSet<>(); - set.add(mapKey); - result.put(value, set); - } - } - } - return result; - } - } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java index 1e5ec9e369..4671188d28 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UsersService.java @@ -16,6 +16,8 @@ */ package org.apache.dolphinscheduler.api.service; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.CheckUtils; import org.apache.dolphinscheduler.api.utils.PageInfo; @@ -23,15 +25,10 @@ import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.UserType; -import org.apache.dolphinscheduler.common.utils.CollectionUtils; -import org.apache.dolphinscheduler.common.utils.EncryptionUtils; -import org.apache.dolphinscheduler.common.utils.HadoopUtils; -import org.apache.dolphinscheduler.common.utils.PropertyUtils; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.entity.*; import org.apache.dolphinscheduler.dao.mapper.*; +import org.apache.dolphinscheduler.dao.utils.ResourceProcessDefinitionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -39,6 +36,7 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import java.util.*; +import java.util.stream.Collectors; /** * user service @@ -72,6 +70,9 @@ public class UsersService extends BaseService { @Autowired private AlertGroupMapper alertGroupMapper; + @Autowired + private ProcessDefinitionMapper processDefinitionMapper; + /** * create user, only system admin have permission @@ -483,23 +484,71 @@ public class UsersService extends BaseService { return result; } + String[] resourceFullIdArr = resourceIds.split(","); + // need authorize resource id set + Set needAuthorizeResIds = new HashSet(); + for (String resourceFullId : resourceFullIdArr) { + String[] resourceIdArr = resourceFullId.split("-"); + for (int i=0;i<=resourceIdArr.length-1;i++) { + int resourceIdValue = Integer.parseInt(resourceIdArr[i]); + needAuthorizeResIds.add(resourceIdValue); + } + } + + //get the authorized resource id list by user id + List oldAuthorizedRes = resourceMapper.queryAuthorizedResourceList(userId); + //if resource type is UDF,need check whether it is bound by UDF functon + Set oldAuthorizedResIds = oldAuthorizedRes.stream().map(t -> t.getId()).collect(Collectors.toSet()); + + //get the unauthorized resource id list + oldAuthorizedResIds.removeAll(needAuthorizeResIds); + + if (CollectionUtils.isNotEmpty(oldAuthorizedResIds)) { + + // get all resource id of process definitions those is released + List> list = processDefinitionMapper.listResources(); + Map> resourceProcessMap = ResourceProcessDefinitionUtils.getResourceProcessDefinitionMap(list); + Set resourceIdSet = resourceProcessMap.keySet(); + + resourceIdSet.retainAll(oldAuthorizedResIds); + if (CollectionUtils.isNotEmpty(resourceIdSet)) { + logger.error("can't be deleted,because it is used of process definition"); + for (Integer resId : resourceIdSet) { + logger.error("resource id:{} is used of process definition {}",resId,resourceProcessMap.get(resId)); + } + putMsg(result, Status.RESOURCE_IS_USED); + return result; + } + + } + resourcesUserMapper.deleteResourceUser(userId, 0); if (check(result, StringUtils.isEmpty(resourceIds), Status.SUCCESS)) { return result; } - String[] resourcesIdArr = resourceIds.split(","); + for (int resourceIdValue : needAuthorizeResIds) { + Resource resource = resourceMapper.selectById(resourceIdValue); + if (resource == null) { + putMsg(result, Status.RESOURCE_NOT_EXIST); + return result; + } - for (String resourceId : resourcesIdArr) { Date now = new Date(); ResourcesUser resourcesUser = new ResourcesUser(); resourcesUser.setUserId(userId); - resourcesUser.setResourcesId(Integer.parseInt(resourceId)); - resourcesUser.setPerm(7); + resourcesUser.setResourcesId(resourceIdValue); + if (resource.isDirectory()) { + resourcesUser.setPerm(Constants.AUTHORIZE_READABLE_PERM); + }else{ + resourcesUser.setPerm(Constants.AUTHORIZE_WRITABLE_PERM); + } + resourcesUser.setCreateTime(now); resourcesUser.setUpdateTime(now); resourcesUserMapper.insert(resourcesUser); + } putMsg(result, Status.SUCCESS); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java index d73eba8bdc..4f9176d699 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java @@ -242,7 +242,7 @@ public class ResourcesServiceTest { User loginUser = new User(); loginUser.setId(0); loginUser.setUserType(UserType.ADMIN_USER); - Mockito.when(resourcesMapper.queryResourceListAuthored(0, 0)).thenReturn(getResourceList()); + Mockito.when(resourcesMapper.queryResourceListAuthored(0, 0,0)).thenReturn(getResourceList()); Map result = resourcesService.queryResourceList(loginUser, ResourceType.FILE); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java index efe9022ad7..58ee6fdf6c 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java @@ -18,13 +18,16 @@ package org.apache.dolphinscheduler.api.service; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.apache.avro.generic.GenericData; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.common.utils.EncryptionUtils; +import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.User; import org.apache.dolphinscheduler.dao.mapper.*; @@ -68,6 +71,8 @@ public class UsersServiceTest { private DataSourceUserMapper datasourceUserMapper; @Mock private AlertGroupMapper alertGroupMapper; + @Mock + private ResourceMapper resourceMapper; private String queueName ="UsersServiceTestQueue"; @@ -301,9 +306,13 @@ public class UsersServiceTest { logger.info(result.toString()); Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS)); //success + when(resourceMapper.queryAuthorizedResourceList(1)).thenReturn(new ArrayList()); + + when(resourceMapper.selectById(Mockito.anyInt())).thenReturn(getResource()); result = usersService.grantResources(loginUser, 1, resourceIds); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + } @@ -476,11 +485,30 @@ public class UsersServiceTest { return user; } - + /** + * get tenant + * @return tenant + */ private Tenant getTenant(){ Tenant tenant = new Tenant(); tenant.setId(1); return tenant; } + /** + * get resource + * @return resource + */ + private Resource getResource(){ + + Resource resource = new Resource(); + resource.setPid(-1); + resource.setUserId(1); + resource.setDescription("ResourcesServiceTest.jar"); + resource.setAlias("ResourcesServiceTest.jar"); + resource.setFullName("/ResourcesServiceTest.jar"); + resource.setType(ResourceType.FILE); + return resource; + } + } \ No newline at end of file diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 94349b7b5d..853ab95d1c 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -946,4 +946,12 @@ public final class Constants { * schedule time */ public static final String PARAMETER_SHECDULE_TIME = "schedule.time"; + /** + * authorize writable perm + */ + public static final int AUTHORIZE_WRITABLE_PERM=7; + /** + * authorize readable perm + */ + public static final int AUTHORIZE_READABLE_PERM=4; } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java index 8948e69f74..b7bf2209d6 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java @@ -16,6 +16,7 @@ */ package org.apache.dolphinscheduler.common.utils; +import org.apache.dolphinscheduler.common.enums.ResourceType; import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; @@ -85,4 +86,19 @@ public class HadoopUtilsTest { List stringList = HadoopUtils.getInstance().catFile("/dolphinscheduler/hdfs/resources/WCSparkPython.py", 0, 1000); logger.info(String.join(",",stringList)); } + + @Test + public void getHdfsFileNameTest(){ + logger.info(HadoopUtils.getHdfsFileName(ResourceType.FILE,"test","/test")); + } + + @Test + public void getHdfsResourceFileNameTest(){ + logger.info(HadoopUtils.getHdfsResourceFileName("test","/test")); + } + + @Test + public void getHdfsUdfFileNameTest(){ + logger.info(HadoopUtils.getHdfsUdfFileName("test","/test.jar")); + } } \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java index 6d3ff60dc8..f58cc7d496 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java @@ -43,11 +43,13 @@ public interface ResourceMapper extends BaseMapper { * query resource list * @param userId userId * @param type type + * @param perm perm * @return resource list */ List queryResourceListAuthored( @Param("userId") int userId, - @Param("type") int type); + @Param("type") int type, + @Param("perm") int perm); /** diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.java index 6e973d7cc0..176f7d8eb4 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.java @@ -34,4 +34,13 @@ public interface ResourceUserMapper extends BaseMapper { int deleteResourceUser(@Param("userId") int userId, @Param("resourceId") int resourceId); + /** + * delete resource user relation + * @param userId userId + * @param resIds resource Ids + * @return delete result + */ + int deleteResourceUserArray(@Param("userId") int userId, + @Param("resIds") Integer[] resIds); + } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java index 9bc47d7a54..a2ce6b29b8 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java @@ -81,9 +81,8 @@ public interface UdfFuncMapper extends BaseMapper { /** * list authorized UDF function * @param userId userId - * @param udfIds udfIds - * @param T - * @return Udf function list + * @param udfIds UDF function id array + * @return UDF function list */ List listAuthorizedUdfFunc (@Param("userId") int userId,@Param("udfIds")T[] udfIds); @@ -92,7 +91,7 @@ public interface UdfFuncMapper extends BaseMapper { * @param resourceIds resource id array * @return UDF function list */ - List listUdfByResourceId(@Param("resourceIds") int[] resourceIds); + List listUdfByResourceId(@Param("resourceIds") Integer[] resourceIds); /** * list authorized UDF by resource id diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtils.java new file mode 100644 index 0000000000..b334603a1a --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtils.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.dao.utils; + +import org.apache.dolphinscheduler.common.utils.CollectionUtils; + +import java.util.*; +import java.util.stream.Collectors; + +/** + * resource process definition utils + */ +public class ResourceProcessDefinitionUtils { + /** + * get resource process map key is resource id,value is the set of process definition + * @param list the map key is process definition id and value is resource_ids + * @return resource process definition map + */ + public static Map> getResourceProcessDefinitionMap(List> list) { + Map map = new HashMap<>(); + Map> result = new HashMap<>(); + if (CollectionUtils.isNotEmpty(list)) { + for (Map tempMap : list) { + + map.put((Integer) tempMap.get("id"), (String)tempMap.get("resource_ids")); + } + } + + for (Map.Entry entry : map.entrySet()) { + Integer mapKey = entry.getKey(); + String[] arr = entry.getValue().split(","); + Set mapValues = Arrays.stream(arr).map(Integer::parseInt).collect(Collectors.toSet()); + for (Integer value : mapValues) { + if (result.containsKey(value)) { + Set set = result.get(value); + set.add(mapKey); + result.put(value, set); + } else { + Set set = new HashSet<>(); + set.add(mapKey); + result.put(value, set); + } + } + } + return result; + } +} diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml index c1fe50fd47..6b1c9b7e34 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml @@ -39,7 +39,11 @@ and type=#{type} - + + and id in (select resources_id from t_ds_relation_resources_user where user_id=#{userId} and perm=#{perm} + union select id as resources_id from t_ds_resources where user_id=#{userId}) + + and id in (select resources_id from t_ds_relation_resources_user where user_id=#{userId} union select id as resources_id from t_ds_resources where user_id=#{userId}) @@ -48,7 +52,7 @@ select * from t_ds_resources where type=#{type} and pid=#{id} - + and id in (select resources_id from t_ds_relation_resources_user where user_id=#{userId} union select id as resources_id from t_ds_resources where user_id=#{userId}) @@ -60,7 +64,7 @@ select * from t_ds_resources - where id in (select resources_id from t_ds_relation_resources_user where user_id=#{userId} + where id in (select resources_id from t_ds_relation_resources_user where user_id=#{userId} and perm=7 union select id as resources_id from t_ds_resources where user_id=#{userId}) and id in diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.xml index 6a89e47c2f..7fdd09fecc 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.xml @@ -29,4 +29,17 @@ and resources_id = #{resourceId} + + + delete + from t_ds_relation_resources_user + where 1 = 1 + + and user_id = #{userId} + + and resources_id in + + #{i} + + \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java index d467979ed2..9dafbe138c 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapperTest.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.dao.mapper; +import org.apache.dolphinscheduler.common.enums.ReleaseState; import org.apache.dolphinscheduler.common.enums.UserType; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; @@ -32,6 +33,7 @@ import org.springframework.transaction.annotation.Transactional; import java.util.Date; import java.util.List; +import java.util.Map; @RunWith(SpringRunner.class) @SpringBootTest @@ -215,4 +217,13 @@ public class ProcessDefinitionMapperTest { ); Assert.assertNotEquals(processDefinitions.size(), 0); } + + @Test + public void listResourcesTest(){ + ProcessDefinition processDefinition = insertOne(); + processDefinition.setResourceIds("3,5"); + processDefinition.setReleaseState(ReleaseState.ONLINE); + List> maps = processDefinitionMapper.listResources(); + Assert.assertNotNull(maps); + } } \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java index 82380e4b3a..818f88fb49 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.dao.mapper; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.dao.entity.Resource; @@ -138,6 +139,7 @@ public class ResourceMapperTest { resourcesUser.setUpdateTime(new Date()); resourcesUser.setUserId(user.getId()); resourcesUser.setResourcesId(resource.getId()); + resourcesUser.setPerm(7); resourceUserMapper.insert(resourcesUser); return resourcesUser; } @@ -247,6 +249,7 @@ public class ResourceMapperTest { resourcesUser.setResourcesId(resource.getId()); resourcesUser.setUserId(1110); + resourcesUser.setPerm(Constants.AUTHORIZE_WRITABLE_PERM); resourceUserMapper.insert(resourcesUser); List resources1 = resourceMapper.queryAuthorizedResourceList(1110); @@ -351,4 +354,34 @@ public class ResourceMapperTest { int result = resourceMapper.deleteIds(resourceList.toArray(new Integer[resourceList.size()])); Assert.assertEquals(result,2); } + + @Test + public void queryResourceListAuthoredTest(){ + // create a general user + User generalUser1 = createGeneralUser("user1"); + User generalUser2 = createGeneralUser("user2"); + // create resource + Resource resource = createResource(generalUser1); + createResourcesUser(resource, generalUser2); + + List resourceList = resourceMapper.queryResourceListAuthored(generalUser2.getId(), ResourceType.FILE.ordinal(), 0); + Assert.assertNotNull(resourceList); + + resourceList = resourceMapper.queryResourceListAuthored(generalUser2.getId(), ResourceType.FILE.ordinal(), 4); + Assert.assertFalse(resourceList.contains(resource)); + } + + @Test + public void batchUpdateResourceTest(){ + // create a general user + User generalUser1 = createGeneralUser("user1"); + // create resource + Resource resource = createResource(generalUser1); + resource.setFullName(String.format("%s-update",resource.getFullName())); + resource.setUpdateTime(new Date()); + List resourceList = new ArrayList<>(); + resourceList.add(resource); + int result = resourceMapper.batchUpdateResource(resourceList); + Assert.assertTrue(result>0); + } } \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapperTest.java index 9244c9e13d..26ae55800a 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapperTest.java @@ -17,6 +17,7 @@ package org.apache.dolphinscheduler.dao.mapper; +import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.dao.entity.ResourcesUser; import org.junit.Assert; import org.junit.Test; @@ -47,13 +48,14 @@ public class ResourceUserMapperTest { */ private ResourcesUser insertOne(){ //insertOne - ResourcesUser queue = new ResourcesUser(); - queue.setCreateTime(new Date()); - queue.setUpdateTime(new Date()); - queue.setUserId(11111); - queue.setResourcesId(1110); - resourceUserMapper.insert(queue); - return queue; + ResourcesUser resourcesUser = new ResourcesUser(); + resourcesUser.setCreateTime(new Date()); + resourcesUser.setUpdateTime(new Date()); + resourcesUser.setUserId(11111); + resourcesUser.setResourcesId(1110); + resourcesUser.setPerm(Constants.AUTHORIZE_WRITABLE_PERM); + resourceUserMapper.insert(resourcesUser); + return resourcesUser; } /** @@ -102,4 +104,18 @@ public class ResourceUserMapperTest { queue.getResourcesId()); Assert.assertNotEquals(delete, 0); } + + /** + * test delete + */ + @Test + public void testDeleteResourceUserArray() { + + ResourcesUser resourcesUser = insertOne(); + Integer[] resourceIdArray = new Integer[]{resourcesUser.getResourcesId()}; + int delete = resourceUserMapper.deleteResourceUserArray( + resourcesUser.getUserId(), + resourceIdArray); + Assert.assertNotEquals(delete, 0); + } } \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtilsTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtilsTest.java new file mode 100644 index 0000000000..914a5010ca --- /dev/null +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtilsTest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.dao.utils; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * resource process definition utils test + */ +public class ResourceProcessDefinitionUtilsTest { + @Test + public void getResourceProcessDefinitionMapTest(){ + List> mapList = new ArrayList<>(); + Map map = new HashMap(); + map.put("id",1); + map.put("resource_ids","1,2,3"); + mapList.add(map); + Assert.assertNotNull(ResourceProcessDefinitionUtils.getResourceProcessDefinitionMap(mapList)); + } + +} \ No newline at end of file