From ba039dc252a2ddfe316297a2955f0ea9f966432a Mon Sep 17 00:00:00 2001 From: zhuangchong <37063904+zhuangchong@users.noreply.github.com> Date: Sat, 20 Mar 2021 12:50:26 +0800 Subject: [PATCH] [Improvement][dao] Resource features SQL slow query optimization. (#4947) * Resourse features SQL slow query optimization. * update resource mapper queryResourcePaging method. * split resource and resource_uesr sql statement. * determine the size of the collection. * update process service class code style. * update ResourceUserMapper java class code style. * add resourceUserMapper in the test config file. * solve code smell * split paging query join statement. * update ResourceUserMapper class. * remove sql join statement. * update resource test class code style. * update ResourcesServiceTest test class. * rerun ut. * update user service test class. * solve query resource list method code smell. --- .../service/impl/ResourcesServiceImpl.java | 66 +++++++--- .../api/service/impl/UsersServiceImpl.java | 3 +- .../api/service/ResourcesServiceTest.java | 11 +- .../api/service/UsersServiceTest.java | 1 - .../dao/mapper/ResourceMapper.java | 43 +++--- .../dao/mapper/ResourceUserMapper.java | 16 ++- .../dao/mapper/ResourceMapper.xml | 123 ++++++++++-------- .../dao/mapper/ResourceUserMapper.xml | 14 ++ .../dao/mapper/ResourceMapperTest.java | 60 +++++---- .../server/registry/DependencyConfig.java | 6 + .../TaskCallbackServiceTestConfig.java | 6 + .../service/process/ProcessService.java | 39 +++++- 12 files changed, 252 insertions(+), 136 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java index f4da1dc0d1..4b7cc592ac 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java @@ -529,8 +529,10 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe } } - IPage resourceIPage = resourcesMapper.queryResourcePaging(page, - userId,directoryId, type.ordinal(), searchVal); + List resourcesIds = resourceUserMapper.queryResourcesIdListByUserIdAndPerm(userId, 0); + + IPage resourceIPage = resourcesMapper.queryResourcePaging(page, userId, directoryId, type.ordinal(), searchVal,resourcesIds); + PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); pageInfo.setTotalCount((int)resourceIPage.getTotal()); pageInfo.setLists(resourceIPage.getRecords()); @@ -615,15 +617,10 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe @Override public Map queryResourceList(User loginUser, ResourceType type) { Map result = new HashMap<>(); - - int userId = loginUser.getId(); - if (isAdmin(loginUser)) { - userId = 0; - } - List allResourceList = resourcesMapper.queryResourceListAuthored(userId, type.ordinal(),0); + List allResourceList = queryAuthoredResourceList(loginUser, type); Visitor resourceTreeVisitor = new ResourceTreeVisitor(allResourceList); result.put(Constants.DATA_LIST, resourceTreeVisitor.visit().getChildren()); - putMsg(result,Status.SUCCESS); + putMsg(result, Status.SUCCESS); return result; } @@ -638,11 +635,10 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe @Override public Map queryResourceByProgramType(User loginUser, ResourceType type, ProgramType programType) { Map result = new HashMap<>(); + + List allResourceList = queryAuthoredResourceList(loginUser, type); + String suffix = ".jar"; - int userId = loginUser.getId(); - if (isAdmin(loginUser)) { - userId = 0; - } if (programType != null) { switch (programType) { case JAVA: @@ -654,11 +650,10 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe default: } } - List allResourceList = resourcesMapper.queryResourceListAuthored(userId, type.ordinal(),0); - List resources = new ResourceFilter(suffix,new ArrayList<>(allResourceList)).filter(); + List resources = new ResourceFilter(suffix, new ArrayList<>(allResourceList)).filter(); Visitor resourceTreeVisitor = new ResourceTreeVisitor(resources); result.put(Constants.DATA_LIST, resourceTreeVisitor.visit().getChildren()); - putMsg(result,Status.SUCCESS); + putMsg(result, Status.SUCCESS); return result; } @@ -1173,8 +1168,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe List list; if (resourceList != null && !resourceList.isEmpty()) { Set resourceSet = new HashSet<>(resourceList); - List authedResourceList = resourcesMapper.queryAuthorizedResourceList(userId); - + List authedResourceList = queryResourceList(userId, Constants.AUTHORIZE_WRITABLE_PERM); getAuthorizedResourceList(resourceSet, authedResourceList); list = new ArrayList<>(resourceSet); } else { @@ -1249,7 +1243,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe if (isNotAdmin(loginUser, result)) { return result; } - List authedResources = resourcesMapper.queryAuthorizedResourceList(userId); + List authedResources = queryResourceList(userId, Constants.AUTHORIZE_WRITABLE_PERM); Visitor visitor = new ResourceTreeVisitor(authedResources); String visit = JSONUtils.toJsonString(visitor.visit(), SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS); logger.info(visit); @@ -1329,4 +1323,38 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe } } + /** + * query authored resource list (own and authorized) + * @param loginUser login user + * @param type ResourceType + * @return all authored resource list + */ + private List queryAuthoredResourceList(User loginUser, ResourceType type) { + List relationResources; + int userId = loginUser.getId(); + if (isAdmin(loginUser)) { + userId = 0; + relationResources = new ArrayList<>(); + } else { + // query resource relation + relationResources = queryResourceList(userId, 0); + } + + List ownResourceList = resourcesMapper.queryResourceListAuthored(userId, type.ordinal()); + ownResourceList.addAll(relationResources); + + return ownResourceList; + } + + /** + * query resource list by userId and perm + * @param userId userId + * @param perm perm + * @return resource list + */ + private List queryResourceList(Integer userId, int perm) { + List resIds = resourceUserMapper.queryResourcesIdListByUserIdAndPerm(userId, perm); + return CollectionUtils.isEmpty(resIds) ? new ArrayList<>() : resourcesMapper.queryResourceListById(resIds); + } + } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java index 865f4c329a..62561aea66 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java @@ -586,7 +586,8 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService { } //get the authorized resource id list by user id - List oldAuthorizedRes = resourceMapper.queryAuthorizedResourceList(userId); + List resIds = resourceUserMapper.queryResourcesIdListByUserIdAndPerm(userId, Constants.AUTHORIZE_WRITABLE_PERM); + List oldAuthorizedRes = CollectionUtils.isEmpty(resIds) ? new ArrayList<>() : resourceMapper.queryResourceListById(resIds); //if resource type is UDF,need check whether it is bound by UDF function Set oldAuthorizedResIds = oldAuthorizedRes.stream().map(Resource::getId).collect(Collectors.toSet()); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java index 89bd3df243..931a41933e 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java @@ -266,8 +266,9 @@ public class ResourcesServiceTest { IPage resourcePage = new Page<>(1, 10); resourcePage.setTotal(1); resourcePage.setRecords(getResourceList()); + Mockito.when(resourcesMapper.queryResourcePaging(Mockito.any(Page.class), - Mockito.eq(0), Mockito.eq(-1), Mockito.eq(0), Mockito.eq("test"))).thenReturn(resourcePage); + Mockito.eq(0), Mockito.eq(-1), Mockito.eq(0), Mockito.eq("test"), Mockito.any())).thenReturn(resourcePage); Map result = resourcesService.queryResourceListPaging(loginUser, -1, ResourceType.FILE, "test", 1, 10); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); @@ -281,7 +282,7 @@ public class ResourcesServiceTest { User loginUser = new User(); loginUser.setId(0); loginUser.setUserType(UserType.ADMIN_USER); - Mockito.when(resourcesMapper.queryResourceListAuthored(0, 0, 0)).thenReturn(getResourceList()); + Mockito.when(resourcesMapper.queryResourceListAuthored(0, 0)).thenReturn(getResourceList()); Map result = resourcesService.queryResourceList(loginUser, ResourceType.FILE); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); @@ -615,7 +616,11 @@ public class ResourcesServiceTest { Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); //SUCCESS user.setUserType(UserType.ADMIN_USER); - Mockito.when(resourcesMapper.queryAuthorizedResourceList(1)).thenReturn(getResourceList()); + + List resIds = new ArrayList<>(); + resIds.add(1); + Mockito.when(resourceUserMapper.queryResourcesIdListByUserIdAndPerm(Mockito.anyInt(), Mockito.anyInt())).thenReturn(resIds); + Mockito.when(resourcesMapper.queryResourceListById(Mockito.any())).thenReturn(getResourceList()); result = resourcesService.authorizedFile(user, 1); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java index 39b8cda3d4..e8707fc01e 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java @@ -325,7 +325,6 @@ public class UsersServiceTest { logger.info(result.toString()); Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS)); //success - when(resourceMapper.queryAuthorizedResourceList(1)).thenReturn(new ArrayList()); when(resourceMapper.selectById(Mockito.anyInt())).thenReturn(getResource()); when(resourceUserMapper.deleteResourceUser(1, 0)).thenReturn(1); result = usersService.grantResources(loginUser, 1, resourceIds); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java index 96b203963b..8fbd9a34b9 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java @@ -14,15 +14,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.dao.mapper; import org.apache.dolphinscheduler.dao.entity.Resource; -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.baomidou.mybatisplus.core.metadata.IPage; + import org.apache.ibatis.annotations.Param; import java.util.List; +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import com.baomidou.mybatisplus.core.metadata.IPage; + /** * resource mapper interface */ @@ -43,14 +46,10 @@ public interface ResourceMapper extends BaseMapper { * query resource list * @param userId userId * @param type type - * @param perm perm * @return resource list */ - List queryResourceListAuthored( - @Param("userId") int userId, - @Param("type") int type, - @Param("perm") int perm); - + List queryResourceListAuthored(@Param("userId") int userId, + @Param("type") int type); /** * resource page @@ -59,20 +58,15 @@ public interface ResourceMapper extends BaseMapper { * @param id id * @param type type * @param searchVal searchVal + * @param resIds resIds * @return resource page */ IPage queryResourcePaging(IPage page, @Param("userId") int userId, @Param("id") int id, @Param("type") int type, - @Param("searchVal") String searchVal); - - /** - * query Authed resource list - * @param userId userId - * @return resource list - */ - List queryAuthorizedResourceList(@Param("userId") int userId); + @Param("searchVal") String searchVal, + @Param("resIds") List resIds); /** * query resource except userId @@ -81,14 +75,6 @@ public interface ResourceMapper extends BaseMapper { */ List queryResourceExceptUserId(@Param("userId") int userId); - /** - * query tenant code by name - * @param resName resource name - * @param resType resource type - * @return tenant code - */ - String queryTenantCodeByResourceName(@Param("resName") String resName,@Param("resType") int resType); - /** * list authorized resource * @param userId userId @@ -96,9 +82,14 @@ public interface ResourceMapper extends BaseMapper { * @param T * @return resource list */ - List listAuthorizedResource(@Param("userId") int userId,@Param("resNames")T[] resNames); - + List listAuthorizedResource(@Param("userId") int userId, @Param("resNames") T[] resNames); + /** + * list resources by id + * @param resIds resIds + * @return resource list + */ + List queryResourceListById(@Param("resIds") List resIds); /** * list authorized resource diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.java index 176f7d8eb4..dbaca256cb 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.java @@ -14,17 +14,31 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.dao.mapper; import org.apache.dolphinscheduler.dao.entity.ResourcesUser; -import com.baomidou.mybatisplus.core.mapper.BaseMapper; + import org.apache.ibatis.annotations.Param; +import java.util.List; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; + /** * resource user relation mapper interface */ public interface ResourceUserMapper extends BaseMapper { + /** + * query resourcesId list by userId and perm + * @param userId userId + * @param perm perm + * @return resourcesId list result + */ + List queryResourcesIdListByUserIdAndPerm(@Param("userId") int userId, + @Param("perm") int perm); + /** * delete resource user relation * @param userId userId diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml index c67fd7291c..7a8b043c09 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml @@ -18,90 +18,82 @@ - - id, alias, file_name, description, user_id, type, size, create_time, update_time, - pid, full_name, is_directory - ${alias}.id, ${alias}.alias, ${alias}.file_name, ${alias}.description, ${alias}.user_id, ${alias}.type, ${alias}.size, ${alias}.create_time, ${alias}.update_time, ${alias}.pid, ${alias}.full_name, ${alias}.is_directory - select - from t_ds_resources r,t_ds_relation_resources_user rel - where r.id = rel.resources_id AND rel.user_id = #{userId} and perm=7 - - - + + select - - from t_ds_resources - where type = #{type} - and full_name = #{fullName} + + + + from t_ds_resources r + where r.type = #{type} + and r.full_name = #{fullName} @@ -159,9 +168,11 @@ + select + resources_id + from t_ds_relation_resources_user rel + where 1 = 1 + + and rel.user_id=#{userId} and rel.perm=#{perm} + + + and rel.user_id=#{userId} + + + delete from t_ds_relation_resources_user diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java index f06e4fcd9b..011d69c672 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java @@ -14,18 +14,29 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.dolphinscheduler.dao.mapper; +import static java.util.stream.Collectors.toList; + +import static org.hamcrest.Matchers.greaterThan; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; -import com.baomidou.mybatisplus.core.metadata.IPage; -import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; import org.apache.dolphinscheduler.dao.entity.Resource; import org.apache.dolphinscheduler.dao.entity.ResourcesUser; import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.User; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; + import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -35,15 +46,8 @@ import org.springframework.test.annotation.Rollback; import org.springframework.test.context.junit4.SpringRunner; import org.springframework.transaction.annotation.Transactional; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Date; -import java.util.List; - -import static java.util.stream.Collectors.toList; -import static org.hamcrest.Matchers.greaterThan; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; @RunWith(SpringRunner.class) @SpringBootTest @@ -238,14 +242,16 @@ public class ResourceMapperTest { 0, -1, resource.getType().ordinal(), - "" + "", + new ArrayList<>() ); IPage resourceIPage1 = resourceMapper.queryResourcePaging( page, 1110, -1, resource.getType().ordinal(), - "" + "", + null ); Assert.assertNotEquals(resourceIPage.getTotal(), 0); Assert.assertNotEquals(resourceIPage1.getTotal(), 0); @@ -259,7 +265,8 @@ public class ResourceMapperTest { public void testQueryResourceListAuthored() { Resource resource = insertOne(); - List resources = resourceMapper.queryAuthorizedResourceList(resource.getUserId()); + List resIds = resourceUserMapper.queryResourcesIdListByUserIdAndPerm(resource.getUserId(), Constants.AUTHORIZE_WRITABLE_PERM); + List resources = CollectionUtils.isEmpty(resIds) ? new ArrayList<>() : resourceMapper.queryResourceListById(resIds); ResourcesUser resourcesUser = new ResourcesUser(); @@ -268,7 +275,8 @@ public class ResourceMapperTest { resourcesUser.setPerm(Constants.AUTHORIZE_WRITABLE_PERM); resourceUserMapper.insert(resourcesUser); - List resources1 = resourceMapper.queryAuthorizedResourceList(1110); + List resIds1 = resourceUserMapper.queryResourcesIdListByUserIdAndPerm(1110, Constants.AUTHORIZE_WRITABLE_PERM); + List resources1 = CollectionUtils.isEmpty(resIds1) ? new ArrayList<>() : resourceMapper.queryResourceListById(resIds1); Assert.assertEquals(0, resources.size()); Assert.assertNotEquals(0, resources1.size()); @@ -282,7 +290,8 @@ public class ResourceMapperTest { public void testQueryAuthorizedResourceList() { Resource resource = insertOne(); - List resources = resourceMapper.queryAuthorizedResourceList(resource.getUserId()); + List resIds = resourceUserMapper.queryResourcesIdListByUserIdAndPerm(resource.getUserId(), Constants.AUTHORIZE_WRITABLE_PERM); + List resources = CollectionUtils.isEmpty(resIds) ? new ArrayList<>() : resourceMapper.queryResourceListById(resIds); resourceMapper.deleteById(resource.getId()); Assert.assertEquals(0, resources.size()); @@ -306,7 +315,6 @@ public class ResourceMapperTest { @Test public void testQueryTenantCodeByResourceName() { - Tenant tenant = new Tenant(); tenant.setTenantCode("ut tenant code for resource"); int tenantInsertStatus = tenantMapper.insert(tenant); @@ -324,7 +332,6 @@ public class ResourceMapperTest { Assert.fail("insert user data error"); } - Resource resource = insertOne(); resource.setUserId(user.getId()); int userUpdateStatus = resourceMapper.updateById(resource); @@ -332,12 +339,13 @@ public class ResourceMapperTest { Assert.fail("update user data error"); } - String resource1 = resourceMapper.queryTenantCodeByResourceName( - resource.getFullName(), ResourceType.FILE.ordinal() - ); + List resourceList = resourceMapper.queryResource(resource.getFullName(), ResourceType.FILE.ordinal()); + int resourceUserId = resourceList.get(0).getUserId(); + User resourceUser = userMapper.selectById(resourceUserId); + Tenant resourceTenant = tenantMapper.selectById(resourceUser.getTenantId()); - Assert.assertEquals("ut tenant code for resource", resource1); + Assert.assertEquals("ut tenant code for resource", resourceTenant.getTenantCode()); } @@ -358,7 +366,6 @@ public class ResourceMapperTest { Assert.assertEquals(generalUser2.getId(), resource.getUserId()); Assert.assertFalse(resources.stream().map(t -> t.getFullName()).collect(toList()).containsAll(Arrays.asList(resNames))); - // authorize object unauthorizedResource to generalUser createResourcesUser(unauthorizedResource, generalUser2); List authorizedResources = resourceMapper.listAuthorizedResource(generalUser2.getId(), resNames); @@ -390,10 +397,10 @@ public class ResourceMapperTest { Resource resource = createResource(generalUser1); createResourcesUser(resource, generalUser2); - List resourceList = resourceMapper.queryResourceListAuthored(generalUser2.getId(), ResourceType.FILE.ordinal(), 0); + List resourceList = resourceMapper.queryResourceListAuthored(generalUser2.getId(), ResourceType.FILE.ordinal()); Assert.assertNotNull(resourceList); - resourceList = resourceMapper.queryResourceListAuthored(generalUser2.getId(), ResourceType.FILE.ordinal(), 4); + resourceList = resourceMapper.queryResourceListAuthored(generalUser2.getId(), ResourceType.FILE.ordinal()); Assert.assertFalse(resourceList.contains(resource)); } @@ -422,4 +429,5 @@ public class ResourceMapperTest { insertOne(); Assert.assertTrue(resourceMapper.existResource(fullName, userId, type)); } -} \ No newline at end of file +} + diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/DependencyConfig.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/DependencyConfig.java index d51d64d68d..f237bca112 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/DependencyConfig.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/DependencyConfig.java @@ -30,6 +30,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; +import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper; import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper; @@ -130,6 +131,11 @@ public class DependencyConfig { return Mockito.mock(ResourceMapper.class); } + @Bean + public ResourceUserMapper resourceUserMapper() { + return Mockito.mock(ResourceUserMapper.class); + } + @Bean public ErrorCommandMapper errorCommandMapper() { return Mockito.mock(ErrorCommandMapper.class); diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTestConfig.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTestConfig.java index 224764c882..f4876a697f 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTestConfig.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTestConfig.java @@ -30,6 +30,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; +import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper; import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper; @@ -118,6 +119,11 @@ public class TaskCallbackServiceTestConfig { return Mockito.mock(ResourceMapper.class); } + @Bean + public ResourceUserMapper resourceUserMapper() { + return Mockito.mock(ResourceUserMapper.class); + } + @Bean public ErrorCommandMapper errorCommandMapper() { return Mockito.mock(ErrorCommandMapper.class); diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java index fe73eaedfb..55b1d0f9c1 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java @@ -76,6 +76,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapMapper; import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; +import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper; import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper; import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper; import org.apache.dolphinscheduler.dao.mapper.TenantMapper; @@ -155,6 +156,9 @@ public class ProcessService { @Autowired private ResourceMapper resourceMapper; + @Autowired + private ResourceUserMapper resourceUserMapper; + @Autowired private ErrorCommandMapper errorCommandMapper; @@ -1794,7 +1798,21 @@ public class ProcessService { public String queryTenantCodeByResName(String resName, ResourceType resourceType) { // in order to query tenant code successful although the version is older String fullName = resName.startsWith("/") ? resName : String.format("/%s", resName); - return resourceMapper.queryTenantCodeByResourceName(fullName, resourceType.ordinal()); + + List resourceList = resourceMapper.queryResource(fullName, resourceType.ordinal()); + if (CollectionUtils.isEmpty(resourceList)) { + return StringUtils.EMPTY; + } + int userId = resourceList.get(0).getUserId(); + User user = userMapper.selectById(userId); + if (Objects.isNull(user)) { + return StringUtils.EMPTY; + } + Tenant tenant = tenantMapper.selectById(user.getTenantId()); + if (Objects.isNull(tenant)) { + return StringUtils.EMPTY; + } + return tenant.getTenantCode(); } /** @@ -2036,11 +2054,15 @@ public class ProcessService { switch (authorizationType) { case RESOURCE_FILE_ID: case UDF_FILE: - Set authorizedResourceFiles = resourceMapper.listAuthorizedResourceById(userId, needChecks).stream().map(Resource::getId).collect(toSet()); + List ownUdfResources = resourceMapper.listAuthorizedResourceById(userId, needChecks); + addAuthorizedResources(ownUdfResources, userId); + Set authorizedResourceFiles = ownUdfResources.stream().map(Resource::getId).collect(toSet()); originResSet.removeAll(authorizedResourceFiles); break; case RESOURCE_FILE_NAME: - Set authorizedResources = resourceMapper.listAuthorizedResource(userId, needChecks).stream().map(Resource::getFullName).collect(toSet()); + List ownResources = resourceMapper.listAuthorizedResource(userId, needChecks); + addAuthorizedResources(ownResources, userId); + Set authorizedResources = ownResources.stream().map(Resource::getFullName).collect(toSet()); originResSet.removeAll(authorizedResources); break; case DATASOURCE: @@ -2163,4 +2185,15 @@ public class ProcessService { } return JSONUtils.toJsonString(processData); } + + /** + * add authorized resources + * @param ownResources own resources + * @param userId userId + */ + private void addAuthorizedResources(List ownResources, int userId) { + List relationResourceIds = resourceUserMapper.queryResourcesIdListByUserIdAndPerm(userId, 7); + List relationResources = CollectionUtils.isNotEmpty(relationResourceIds) ? resourceMapper.queryResourceListById(relationResourceIds) : new ArrayList<>(); + ownResources.addAll(relationResources); + } }