Browse Source

[Improvement][dao] Resource features SQL slow query optimization. (#4947)

* Resourse features SQL slow query optimization.

* update resource mapper queryResourcePaging method.

* split resource and resource_uesr sql statement.

* determine the size of the collection.

* update process service class code style.

* update ResourceUserMapper java class code style.

* add resourceUserMapper in the test config file.

* solve code smell

* split paging query join statement.

* update ResourceUserMapper class.

* remove sql join statement.

* update resource test class code style.

* update ResourcesServiceTest test class.

* rerun ut.

* update user service test class.

* solve query resource list method code smell.
pull/3/MERGE
zhuangchong 4 years ago committed by GitHub
parent
commit
ba039dc252
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 66
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
  2. 3
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java
  3. 11
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java
  4. 1
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java
  5. 43
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java
  6. 16
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.java
  7. 123
      dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml
  8. 14
      dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.xml
  9. 58
      dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java
  10. 6
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/DependencyConfig.java
  11. 6
      dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTestConfig.java
  12. 39
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java

66
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java

@ -529,8 +529,10 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
}
}
IPage<Resource> resourceIPage = resourcesMapper.queryResourcePaging(page,
userId,directoryId, type.ordinal(), searchVal);
List<Integer> resourcesIds = resourceUserMapper.queryResourcesIdListByUserIdAndPerm(userId, 0);
IPage<Resource> resourceIPage = resourcesMapper.queryResourcePaging(page, userId, directoryId, type.ordinal(), searchVal,resourcesIds);
PageInfo<Resource> pageInfo = new PageInfo<>(pageNo, pageSize);
pageInfo.setTotalCount((int)resourceIPage.getTotal());
pageInfo.setLists(resourceIPage.getRecords());
@ -615,15 +617,10 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
@Override
public Map<String, Object> queryResourceList(User loginUser, ResourceType type) {
Map<String, Object> result = new HashMap<>();
int userId = loginUser.getId();
if (isAdmin(loginUser)) {
userId = 0;
}
List<Resource> allResourceList = resourcesMapper.queryResourceListAuthored(userId, type.ordinal(),0);
List<Resource> allResourceList = queryAuthoredResourceList(loginUser, type);
Visitor resourceTreeVisitor = new ResourceTreeVisitor(allResourceList);
result.put(Constants.DATA_LIST, resourceTreeVisitor.visit().getChildren());
putMsg(result,Status.SUCCESS);
putMsg(result, Status.SUCCESS);
return result;
}
@ -638,11 +635,10 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
@Override
public Map<String, Object> queryResourceByProgramType(User loginUser, ResourceType type, ProgramType programType) {
Map<String, Object> result = new HashMap<>();
List<Resource> allResourceList = queryAuthoredResourceList(loginUser, type);
String suffix = ".jar";
int userId = loginUser.getId();
if (isAdmin(loginUser)) {
userId = 0;
}
if (programType != null) {
switch (programType) {
case JAVA:
@ -654,11 +650,10 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
default:
}
}
List<Resource> allResourceList = resourcesMapper.queryResourceListAuthored(userId, type.ordinal(),0);
List<Resource> resources = new ResourceFilter(suffix,new ArrayList<>(allResourceList)).filter();
List<Resource> resources = new ResourceFilter(suffix, new ArrayList<>(allResourceList)).filter();
Visitor resourceTreeVisitor = new ResourceTreeVisitor(resources);
result.put(Constants.DATA_LIST, resourceTreeVisitor.visit().getChildren());
putMsg(result,Status.SUCCESS);
putMsg(result, Status.SUCCESS);
return result;
}
@ -1173,8 +1168,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
List<Resource> list;
if (resourceList != null && !resourceList.isEmpty()) {
Set<Resource> resourceSet = new HashSet<>(resourceList);
List<Resource> authedResourceList = resourcesMapper.queryAuthorizedResourceList(userId);
List<Resource> authedResourceList = queryResourceList(userId, Constants.AUTHORIZE_WRITABLE_PERM);
getAuthorizedResourceList(resourceSet, authedResourceList);
list = new ArrayList<>(resourceSet);
} else {
@ -1249,7 +1243,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
if (isNotAdmin(loginUser, result)) {
return result;
}
List<Resource> authedResources = resourcesMapper.queryAuthorizedResourceList(userId);
List<Resource> authedResources = queryResourceList(userId, Constants.AUTHORIZE_WRITABLE_PERM);
Visitor visitor = new ResourceTreeVisitor(authedResources);
String visit = JSONUtils.toJsonString(visitor.visit(), SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS);
logger.info(visit);
@ -1329,4 +1323,38 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
}
}
/**
* query authored resource list (own and authorized)
* @param loginUser login user
* @param type ResourceType
* @return all authored resource list
*/
private List<Resource> queryAuthoredResourceList(User loginUser, ResourceType type) {
List<Resource> relationResources;
int userId = loginUser.getId();
if (isAdmin(loginUser)) {
userId = 0;
relationResources = new ArrayList<>();
} else {
// query resource relation
relationResources = queryResourceList(userId, 0);
}
List<Resource> ownResourceList = resourcesMapper.queryResourceListAuthored(userId, type.ordinal());
ownResourceList.addAll(relationResources);
return ownResourceList;
}
/**
* query resource list by userId and perm
* @param userId userId
* @param perm perm
* @return resource list
*/
private List<Resource> queryResourceList(Integer userId, int perm) {
List<Integer> resIds = resourceUserMapper.queryResourcesIdListByUserIdAndPerm(userId, perm);
return CollectionUtils.isEmpty(resIds) ? new ArrayList<>() : resourcesMapper.queryResourceListById(resIds);
}
}

3
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java

@ -586,7 +586,8 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService {
}
//get the authorized resource id list by user id
List<Resource> oldAuthorizedRes = resourceMapper.queryAuthorizedResourceList(userId);
List<Integer> resIds = resourceUserMapper.queryResourcesIdListByUserIdAndPerm(userId, Constants.AUTHORIZE_WRITABLE_PERM);
List<Resource> oldAuthorizedRes = CollectionUtils.isEmpty(resIds) ? new ArrayList<>() : resourceMapper.queryResourceListById(resIds);
//if resource type is UDF,need check whether it is bound by UDF function
Set<Integer> oldAuthorizedResIds = oldAuthorizedRes.stream().map(Resource::getId).collect(Collectors.toSet());

11
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java

@ -266,8 +266,9 @@ public class ResourcesServiceTest {
IPage<Resource> resourcePage = new Page<>(1, 10);
resourcePage.setTotal(1);
resourcePage.setRecords(getResourceList());
Mockito.when(resourcesMapper.queryResourcePaging(Mockito.any(Page.class),
Mockito.eq(0), Mockito.eq(-1), Mockito.eq(0), Mockito.eq("test"))).thenReturn(resourcePage);
Mockito.eq(0), Mockito.eq(-1), Mockito.eq(0), Mockito.eq("test"), Mockito.any())).thenReturn(resourcePage);
Map<String, Object> result = resourcesService.queryResourceListPaging(loginUser, -1, ResourceType.FILE, "test", 1, 10);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
@ -281,7 +282,7 @@ public class ResourcesServiceTest {
User loginUser = new User();
loginUser.setId(0);
loginUser.setUserType(UserType.ADMIN_USER);
Mockito.when(resourcesMapper.queryResourceListAuthored(0, 0, 0)).thenReturn(getResourceList());
Mockito.when(resourcesMapper.queryResourceListAuthored(0, 0)).thenReturn(getResourceList());
Map<String, Object> result = resourcesService.queryResourceList(loginUser, ResourceType.FILE);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
@ -615,7 +616,11 @@ public class ResourcesServiceTest {
Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS));
//SUCCESS
user.setUserType(UserType.ADMIN_USER);
Mockito.when(resourcesMapper.queryAuthorizedResourceList(1)).thenReturn(getResourceList());
List<Integer> resIds = new ArrayList<>();
resIds.add(1);
Mockito.when(resourceUserMapper.queryResourcesIdListByUserIdAndPerm(Mockito.anyInt(), Mockito.anyInt())).thenReturn(resIds);
Mockito.when(resourcesMapper.queryResourceListById(Mockito.any())).thenReturn(getResourceList());
result = resourcesService.authorizedFile(user, 1);
logger.info(result.toString());
Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));

1
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java

@ -325,7 +325,6 @@ public class UsersServiceTest {
logger.info(result.toString());
Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS));
//success
when(resourceMapper.queryAuthorizedResourceList(1)).thenReturn(new ArrayList<Resource>());
when(resourceMapper.selectById(Mockito.anyInt())).thenReturn(getResource());
when(resourceUserMapper.deleteResourceUser(1, 0)).thenReturn(1);
result = usersService.grantResources(loginUser, 1, resourceIds);

43
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java

@ -14,15 +14,18 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.mapper;
import org.apache.dolphinscheduler.dao.entity.Resource;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
import org.apache.ibatis.annotations.Param;
import java.util.List;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import com.baomidou.mybatisplus.core.metadata.IPage;
/**
* resource mapper interface
*/
@ -43,14 +46,10 @@ public interface ResourceMapper extends BaseMapper<Resource> {
* query resource list
* @param userId userId
* @param type type
* @param perm perm
* @return resource list
*/
List<Resource> queryResourceListAuthored(
@Param("userId") int userId,
@Param("type") int type,
@Param("perm") int perm);
List<Resource> queryResourceListAuthored(@Param("userId") int userId,
@Param("type") int type);
/**
* resource page
@ -59,20 +58,15 @@ public interface ResourceMapper extends BaseMapper<Resource> {
* @param id id
* @param type type
* @param searchVal searchVal
* @param resIds resIds
* @return resource page
*/
IPage<Resource> queryResourcePaging(IPage<Resource> page,
@Param("userId") int userId,
@Param("id") int id,
@Param("type") int type,
@Param("searchVal") String searchVal);
/**
* query Authed resource list
* @param userId userId
* @return resource list
*/
List<Resource> queryAuthorizedResourceList(@Param("userId") int userId);
@Param("searchVal") String searchVal,
@Param("resIds") List<Integer> resIds);
/**
* query resource except userId
@ -81,14 +75,6 @@ public interface ResourceMapper extends BaseMapper<Resource> {
*/
List<Resource> queryResourceExceptUserId(@Param("userId") int userId);
/**
* query tenant code by name
* @param resName resource name
* @param resType resource type
* @return tenant code
*/
String queryTenantCodeByResourceName(@Param("resName") String resName,@Param("resType") int resType);
/**
* list authorized resource
* @param userId userId
@ -96,9 +82,14 @@ public interface ResourceMapper extends BaseMapper<Resource> {
* @param <T> T
* @return resource list
*/
<T> List<Resource> listAuthorizedResource(@Param("userId") int userId,@Param("resNames")T[] resNames);
<T> List<Resource> listAuthorizedResource(@Param("userId") int userId, @Param("resNames") T[] resNames);
/**
* list resources by id
* @param resIds resIds
* @return resource list
*/
List<Resource> queryResourceListById(@Param("resIds") List<Integer> resIds);
/**
* list authorized resource

16
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.java

@ -14,17 +14,31 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.mapper;
import org.apache.dolphinscheduler.dao.entity.ResourcesUser;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
import org.apache.ibatis.annotations.Param;
import java.util.List;
import com.baomidou.mybatisplus.core.mapper.BaseMapper;
/**
* resource user relation mapper interface
*/
public interface ResourceUserMapper extends BaseMapper<ResourcesUser> {
/**
* query resourcesId list by userId and perm
* @param userId userId
* @param perm perm
* @return resourcesId list result
*/
List<Integer> queryResourcesIdListByUserIdAndPerm(@Param("userId") int userId,
@Param("perm") int perm);
/**
* delete resource user relation
* @param userId userId

123
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.xml

@ -18,90 +18,82 @@
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="org.apache.dolphinscheduler.dao.mapper.ResourceMapper">
<sql id="baseSql">
id, alias, file_name, description, user_id, type, size, create_time, update_time,
pid, full_name, is_directory
</sql>
<sql id="baseSqlV2">
${alias}.id, ${alias}.alias, ${alias}.file_name, ${alias}.description, ${alias}.user_id, ${alias}.type, ${alias}.size, ${alias}.create_time, ${alias}.update_time,
${alias}.pid, ${alias}.full_name, ${alias}.is_directory
</sql>
<select id="queryResourceList" resultType="org.apache.dolphinscheduler.dao.entity.Resource">
select
<include refid="baseSql"/>
from t_ds_resources
<include refid="baseSqlV2">
<property name="alias" value="r"/>
</include>
from t_ds_resources r
where 1= 1
<if test="fullName != null and fullName != ''">
and full_name = #{fullName}
and r.full_name = #{fullName}
</if>
<if test="type != -1">
and type = #{type}
and r.type = #{type}
</if>
<if test="userId != 0">
and user_id = #{userId}
and r.user_id = #{userId}
</if>
</select>
<select id="queryResourceListAuthored" resultType="org.apache.dolphinscheduler.dao.entity.Resource">
select
<include refid="baseSql"/>
from t_ds_resources
<include refid="baseSqlV2">
<property name="alias" value="r"/>
</include>
from t_ds_resources r
where 1 = 1
<if test="type != -1">
and type=#{type}
and r.type=#{type}
</if>
<if test="userId != 0 and perm != 0">
and id in (select resources_id from t_ds_relation_resources_user where user_id=#{userId} and perm=#{perm}
union select id as resources_id from t_ds_resources where user_id=#{userId})
</if>
<if test="userId != 0 and perm == 0">
and id in (select resources_id from t_ds_relation_resources_user where user_id=#{userId}
union select id as resources_id from t_ds_resources where user_id=#{userId})
<if test="userId != 0">
and r.user_id=#{userId}
</if>
</select>
<select id="queryResourcePaging" resultType="org.apache.dolphinscheduler.dao.entity.Resource">
select
d.id, d.alias, d.file_name, d.description, d.user_id, d.type, d.size, d.create_time, d.update_time,
d.pid, d.full_name, d.is_directory,
u.user_name as user_name
<include refid="baseSqlV2">
<property name="alias" value="d"/>
</include>
from t_ds_resources d
join t_ds_user u on d.user_id = u.id
where d.type=#{type} and d.pid=#{id}
<if test="userId != 0">
and d.id in (select resources_id from t_ds_relation_resources_user where user_id=#{userId}
union select id as resources_id from t_ds_resources where user_id=#{userId})
and (
<if test="resIds != null and resIds.size() > 0">
and d.id in
<foreach collection="resIds" item="i" open="(" close=") or" separator=",">
#{i}
</foreach>
</if>
d.user_id=#{userId} )
</if>
<if test="searchVal != null and searchVal != ''">
and d.alias like concat('%', #{searchVal}, '%')
</if>
order by d.update_time desc
</select>
<select id="queryAuthorizedResourceList" resultType="org.apache.dolphinscheduler.dao.entity.Resource">
<select id="queryResourceExceptUserId" resultType="org.apache.dolphinscheduler.dao.entity.Resource">
select
<include refid="baseSqlV2">
<property name="alias" value="r"/>
</include>
from t_ds_resources r,t_ds_relation_resources_user rel
where r.id = rel.resources_id AND rel.user_id = #{userId} and perm=7
</select>
<select id="queryResourceExceptUserId" resultType="org.apache.dolphinscheduler.dao.entity.Resource">
select
<include refid="baseSql"/>
from t_ds_resources
where user_id <![CDATA[ <> ]]> #{userId}
</select>
<select id="queryTenantCodeByResourceName" resultType="java.lang.String">
select tenant_code
from t_ds_tenant t, t_ds_user u, t_ds_resources res
where t.id = u.tenant_id and u.id = res.user_id and res.type=#{resType}
and res.full_name= #{resName}
from t_ds_resources r
where r.user_id <![CDATA[ <> ]]> #{userId}
</select>
<select id="listAuthorizedResource" resultType="org.apache.dolphinscheduler.dao.entity.Resource">
select
<include refid="baseSql"/>
from t_ds_resources
where type=0
and id in (select resources_id from t_ds_relation_resources_user where user_id=#{userId} and perm=7
union select id as resources_id from t_ds_resources where user_id=#{userId})
<include refid="baseSqlV2">
<property name="alias" value="r"/>
</include>
from t_ds_resources r
where r.type = 0
and r.user_id=#{userId}
<if test="resNames != null and resNames.length > 0">
and full_name in
<foreach collection="resNames" item="i" open="(" close=")" separator=",">
@ -109,12 +101,27 @@
</foreach>
</if>
</select>
<select id="queryResourceListById" resultType="org.apache.dolphinscheduler.dao.entity.Resource">
select
<include refid="baseSqlV2">
<property name="alias" value="r"/>
</include>
from t_ds_resources r
where 1 = 1
<if test="resIds != null and resIds.size() > 0">
and r.id in
<foreach collection="resIds" item="i" open="(" close=")" separator=",">
#{i}
</foreach>
</if>
</select>
<select id="listAuthorizedResourceById" resultType="org.apache.dolphinscheduler.dao.entity.Resource">
select
<include refid="baseSql"/>
from t_ds_resources
where id in (select resources_id from t_ds_relation_resources_user where user_id=#{userId} and perm=7
union select id as resources_id from t_ds_resources where user_id=#{userId})
<include refid="baseSqlV2">
<property name="alias" value="r"/>
</include>
from t_ds_resources r
where r.user_id=#{userId}
<if test="resIds != null and resIds.length > 0">
and id in
<foreach collection="resIds" item="i" open="(" close=")" separator=",">
@ -138,10 +145,12 @@
<select id="queryResource" resultType="org.apache.dolphinscheduler.dao.entity.Resource">
select
<include refid="baseSql"/>
from t_ds_resources
where type = #{type}
and full_name = #{fullName}
<include refid="baseSqlV2">
<property name="alias" value="r"/>
</include>
from t_ds_resources r
where r.type = #{type}
and r.full_name = #{fullName}
</select>
<update id="batchUpdateResource" parameterType="java.util.List">
@ -159,9 +168,11 @@
<select id="listResourceByIds" resultType="org.apache.dolphinscheduler.dao.entity.Resource">
select
<include refid="baseSql"/>
from t_ds_resources
where id in
<include refid="baseSqlV2">
<property name="alias" value="r"/>
</include>
from t_ds_resources r
where r.id in
<foreach collection="resIds" item="i" open="(" close=")" separator=",">
#{i}
</foreach>

14
dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceUserMapper.xml

@ -18,6 +18,20 @@
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper">
<select id="queryResourcesIdListByUserIdAndPerm" resultType="java.lang.Integer">
select
resources_id
from t_ds_relation_resources_user rel
where 1 = 1
<if test="userId != 0 and perm != 0">
and rel.user_id=#{userId} and rel.perm=#{perm}
</if>
<if test="userId != 0 and perm == 0">
and rel.user_id=#{userId}
</if>
</select>
<delete id="deleteResourceUser">
delete
from t_ds_relation_resources_user

58
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapperTest.java

@ -14,18 +14,29 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.mapper;
import static java.util.stream.Collectors.toList;
import static org.hamcrest.Matchers.greaterThan;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ResourceType;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.dao.entity.Resource;
import org.apache.dolphinscheduler.dao.entity.ResourcesUser;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.entity.User;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -35,15 +46,8 @@ import org.springframework.test.annotation.Rollback;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.transaction.annotation.Transactional;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import static java.util.stream.Collectors.toList;
import static org.hamcrest.Matchers.greaterThan;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
@RunWith(SpringRunner.class)
@SpringBootTest
@ -238,14 +242,16 @@ public class ResourceMapperTest {
0,
-1,
resource.getType().ordinal(),
""
"",
new ArrayList<>()
);
IPage<Resource> resourceIPage1 = resourceMapper.queryResourcePaging(
page,
1110,
-1,
resource.getType().ordinal(),
""
"",
null
);
Assert.assertNotEquals(resourceIPage.getTotal(), 0);
Assert.assertNotEquals(resourceIPage1.getTotal(), 0);
@ -259,7 +265,8 @@ public class ResourceMapperTest {
public void testQueryResourceListAuthored() {
Resource resource = insertOne();
List<Resource> resources = resourceMapper.queryAuthorizedResourceList(resource.getUserId());
List<Integer> resIds = resourceUserMapper.queryResourcesIdListByUserIdAndPerm(resource.getUserId(), Constants.AUTHORIZE_WRITABLE_PERM);
List<Resource> resources = CollectionUtils.isEmpty(resIds) ? new ArrayList<>() : resourceMapper.queryResourceListById(resIds);
ResourcesUser resourcesUser = new ResourcesUser();
@ -268,7 +275,8 @@ public class ResourceMapperTest {
resourcesUser.setPerm(Constants.AUTHORIZE_WRITABLE_PERM);
resourceUserMapper.insert(resourcesUser);
List<Resource> resources1 = resourceMapper.queryAuthorizedResourceList(1110);
List<Integer> resIds1 = resourceUserMapper.queryResourcesIdListByUserIdAndPerm(1110, Constants.AUTHORIZE_WRITABLE_PERM);
List<Resource> resources1 = CollectionUtils.isEmpty(resIds1) ? new ArrayList<>() : resourceMapper.queryResourceListById(resIds1);
Assert.assertEquals(0, resources.size());
Assert.assertNotEquals(0, resources1.size());
@ -282,7 +290,8 @@ public class ResourceMapperTest {
public void testQueryAuthorizedResourceList() {
Resource resource = insertOne();
List<Resource> resources = resourceMapper.queryAuthorizedResourceList(resource.getUserId());
List<Integer> resIds = resourceUserMapper.queryResourcesIdListByUserIdAndPerm(resource.getUserId(), Constants.AUTHORIZE_WRITABLE_PERM);
List<Resource> resources = CollectionUtils.isEmpty(resIds) ? new ArrayList<>() : resourceMapper.queryResourceListById(resIds);
resourceMapper.deleteById(resource.getId());
Assert.assertEquals(0, resources.size());
@ -306,7 +315,6 @@ public class ResourceMapperTest {
@Test
public void testQueryTenantCodeByResourceName() {
Tenant tenant = new Tenant();
tenant.setTenantCode("ut tenant code for resource");
int tenantInsertStatus = tenantMapper.insert(tenant);
@ -324,7 +332,6 @@ public class ResourceMapperTest {
Assert.fail("insert user data error");
}
Resource resource = insertOne();
resource.setUserId(user.getId());
int userUpdateStatus = resourceMapper.updateById(resource);
@ -332,12 +339,13 @@ public class ResourceMapperTest {
Assert.fail("update user data error");
}
String resource1 = resourceMapper.queryTenantCodeByResourceName(
resource.getFullName(), ResourceType.FILE.ordinal()
);
List<Resource> resourceList = resourceMapper.queryResource(resource.getFullName(), ResourceType.FILE.ordinal());
int resourceUserId = resourceList.get(0).getUserId();
User resourceUser = userMapper.selectById(resourceUserId);
Tenant resourceTenant = tenantMapper.selectById(resourceUser.getTenantId());
Assert.assertEquals("ut tenant code for resource", resource1);
Assert.assertEquals("ut tenant code for resource", resourceTenant.getTenantCode());
}
@ -358,7 +366,6 @@ public class ResourceMapperTest {
Assert.assertEquals(generalUser2.getId(), resource.getUserId());
Assert.assertFalse(resources.stream().map(t -> t.getFullName()).collect(toList()).containsAll(Arrays.asList(resNames)));
// authorize object unauthorizedResource to generalUser
createResourcesUser(unauthorizedResource, generalUser2);
List<Resource> authorizedResources = resourceMapper.listAuthorizedResource(generalUser2.getId(), resNames);
@ -390,10 +397,10 @@ public class ResourceMapperTest {
Resource resource = createResource(generalUser1);
createResourcesUser(resource, generalUser2);
List<Resource> resourceList = resourceMapper.queryResourceListAuthored(generalUser2.getId(), ResourceType.FILE.ordinal(), 0);
List<Resource> resourceList = resourceMapper.queryResourceListAuthored(generalUser2.getId(), ResourceType.FILE.ordinal());
Assert.assertNotNull(resourceList);
resourceList = resourceMapper.queryResourceListAuthored(generalUser2.getId(), ResourceType.FILE.ordinal(), 4);
resourceList = resourceMapper.queryResourceListAuthored(generalUser2.getId(), ResourceType.FILE.ordinal());
Assert.assertFalse(resourceList.contains(resource));
}
@ -423,3 +430,4 @@ public class ResourceMapperTest {
Assert.assertTrue(resourceMapper.existResource(fullName, userId, type));
}
}

6
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/DependencyConfig.java

@ -30,6 +30,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.dao.mapper.ResourceMapper;
import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper;
import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
@ -130,6 +131,11 @@ public class DependencyConfig {
return Mockito.mock(ResourceMapper.class);
}
@Bean
public ResourceUserMapper resourceUserMapper() {
return Mockito.mock(ResourceUserMapper.class);
}
@Bean
public ErrorCommandMapper errorCommandMapper() {
return Mockito.mock(ErrorCommandMapper.class);

6
dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskCallbackServiceTestConfig.java

@ -30,6 +30,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.dao.mapper.ResourceMapper;
import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper;
import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
@ -118,6 +119,11 @@ public class TaskCallbackServiceTestConfig {
return Mockito.mock(ResourceMapper.class);
}
@Bean
public ResourceUserMapper resourceUserMapper() {
return Mockito.mock(ResourceUserMapper.class);
}
@Bean
public ErrorCommandMapper errorCommandMapper() {
return Mockito.mock(ErrorCommandMapper.class);

39
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessService.java

@ -76,6 +76,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.dao.mapper.ResourceMapper;
import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper;
import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
@ -155,6 +156,9 @@ public class ProcessService {
@Autowired
private ResourceMapper resourceMapper;
@Autowired
private ResourceUserMapper resourceUserMapper;
@Autowired
private ErrorCommandMapper errorCommandMapper;
@ -1794,7 +1798,21 @@ public class ProcessService {
public String queryTenantCodeByResName(String resName, ResourceType resourceType) {
// in order to query tenant code successful although the version is older
String fullName = resName.startsWith("/") ? resName : String.format("/%s", resName);
return resourceMapper.queryTenantCodeByResourceName(fullName, resourceType.ordinal());
List<Resource> resourceList = resourceMapper.queryResource(fullName, resourceType.ordinal());
if (CollectionUtils.isEmpty(resourceList)) {
return StringUtils.EMPTY;
}
int userId = resourceList.get(0).getUserId();
User user = userMapper.selectById(userId);
if (Objects.isNull(user)) {
return StringUtils.EMPTY;
}
Tenant tenant = tenantMapper.selectById(user.getTenantId());
if (Objects.isNull(tenant)) {
return StringUtils.EMPTY;
}
return tenant.getTenantCode();
}
/**
@ -2036,11 +2054,15 @@ public class ProcessService {
switch (authorizationType) {
case RESOURCE_FILE_ID:
case UDF_FILE:
Set<Integer> authorizedResourceFiles = resourceMapper.listAuthorizedResourceById(userId, needChecks).stream().map(Resource::getId).collect(toSet());
List<Resource> ownUdfResources = resourceMapper.listAuthorizedResourceById(userId, needChecks);
addAuthorizedResources(ownUdfResources, userId);
Set<Integer> authorizedResourceFiles = ownUdfResources.stream().map(Resource::getId).collect(toSet());
originResSet.removeAll(authorizedResourceFiles);
break;
case RESOURCE_FILE_NAME:
Set<String> authorizedResources = resourceMapper.listAuthorizedResource(userId, needChecks).stream().map(Resource::getFullName).collect(toSet());
List<Resource> ownResources = resourceMapper.listAuthorizedResource(userId, needChecks);
addAuthorizedResources(ownResources, userId);
Set<String> authorizedResources = ownResources.stream().map(Resource::getFullName).collect(toSet());
originResSet.removeAll(authorizedResources);
break;
case DATASOURCE:
@ -2163,4 +2185,15 @@ public class ProcessService {
}
return JSONUtils.toJsonString(processData);
}
/**
* add authorized resources
* @param ownResources own resources
* @param userId userId
*/
private void addAuthorizedResources(List<Resource> ownResources, int userId) {
List<Integer> relationResourceIds = resourceUserMapper.queryResourcesIdListByUserIdAndPerm(userId, 7);
List<Resource> relationResources = CollectionUtils.isNotEmpty(relationResourceIds) ? resourceMapper.queryResourceListById(relationResourceIds) : new ArrayList<>();
ownResources.addAll(relationResources);
}
}

Loading…
Cancel
Save