From 5f5c08402fdcecca8c35f4dc3021cc089949ef13 Mon Sep 17 00:00:00 2001 From: lgcareer <18610854716@163.com> Date: Fri, 14 Aug 2020 16:47:01 +0800 Subject: [PATCH] [Fix-3469][api]Should filter the resource by the different program type (#3498) * [Fix-3463][api]Fixed that run the sql task will be failure after rename the udf resource * [Fix-3469][api]Should list python file and jar file * [Fix-3469][api]Should filter the resource by the different program type * [Fix-3469][api]fix the code smell --- .../api/controller/ResourcesController.java | 13 +++++++--- .../dto/resources/filter/ResourceFilter.java | 25 +++++++++++++------ .../api/service/ResourcesService.java | 19 +++++++++++--- .../controller/ResourcesControllerTest.java | 25 +++++++++++++++++++ .../resources/filter/ResourceFilterTest.java | 8 ++++-- .../dao/upgrade/UpgradeDao.java | 8 ++++++ 6 files changed, 82 insertions(+), 16 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java index 2f2cb9ca6f..697d364dfd 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java @@ -27,6 +27,7 @@ import org.apache.dolphinscheduler.api.service.ResourcesService; import org.apache.dolphinscheduler.api.service.UdfFuncService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.common.utils.ParameterUtils; @@ -275,7 +276,7 @@ public class ResourcesController extends BaseController { * @param type resource type * @return resource list */ - @ApiOperation(value = "queryResourceJarList", notes = "QUERY_RESOURCE_LIST_NOTES") + @ApiOperation(value = "queryResourceByProgramType", notes = "QUERY_RESOURCE_LIST_NOTES") @ApiImplicitParams({ @ApiImplicitParam(name = "type", value = "RESOURCE_TYPE", required = true, dataType = "ResourceType") }) @@ -283,10 +284,14 @@ public class ResourcesController extends BaseController { @ResponseStatus(HttpStatus.OK) @ApiException(QUERY_RESOURCES_LIST_ERROR) public Result queryResourceJarList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam(value = "type") ResourceType type + @RequestParam(value = "type") ResourceType type, + @RequestParam(value = "programType",required = false) ProgramType programType ) { - logger.info("query resource list, login user:{}, resource type:{}", loginUser.getUserName(), type.toString()); - Map result = resourceService.queryResourceJarList(loginUser, type); + String programTypeName = programType == null ? "" : programType.name(); + String userName = loginUser.getUserName(); + userName = userName.replaceAll("[\n|\r|\t]", "_"); + logger.info("query resource list, login user:{}, resource type:{}, program type:{}", userName,programTypeName); + Map result = resourceService.queryResourceByProgramType(loginUser, type,programType); return returnDataList(result); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java index c918a160af..9c3a84cf43 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java @@ -31,24 +31,29 @@ public class ResourceFilter implements IFilter { /** * resource suffix */ - private String suffix; + private Set suffixs = new HashSet<>(); /** * resource list */ private List resourceList; /** - * parent list + * constructor + * @param suffix resource suffix + * @param resourceList resource list */ - //Set parentList = new HashSet<>(); + public ResourceFilter(String suffix, List resourceList) { + this.suffixs.add(suffix); + this.resourceList = resourceList; + } /** * constructor - * @param suffix resource suffix + * @param suffixs resource suffixs * @param resourceList resource list */ - public ResourceFilter(String suffix, List resourceList) { - this.suffix = suffix; + public ResourceFilter(Set suffixs, List resourceList) { + this.suffixs = suffixs; this.resourceList = resourceList; } @@ -59,7 +64,13 @@ public class ResourceFilter implements IFilter { public Set fileFilter(){ Set resources = resourceList.stream().filter(t -> { String alias = t.getAlias(); - return alias.endsWith(suffix); + boolean result = false; + for (String suffix : suffixs) { + if (alias.endsWith(suffix)) { + result = true; + } + } + return result; }).collect(Collectors.toSet()); return resources; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index e1a9c85aa1..f66e035eae 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -30,6 +30,7 @@ import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.entity.*; @@ -654,21 +655,33 @@ public class ResourcesService extends BaseService { } /** - * query resource list + * query resource list by program type * * @param loginUser login user * @param type resource type * @return resource list */ - public Map queryResourceJarList(User loginUser, ResourceType type) { + public Map queryResourceByProgramType(User loginUser, ResourceType type, ProgramType programType) { Map result = new HashMap<>(5); + String suffix = ".jar"; int userId = loginUser.getId(); if(isAdmin(loginUser)){ userId = 0; } + if (programType != null) { + switch (programType) { + case JAVA: + break; + case SCALA: + break; + case PYTHON: + suffix = ".py"; + break; + } + } List allResourceList = resourcesMapper.queryResourceListAuthored(userId, type.ordinal(),0); - List resources = new ResourceFilter(".jar",new ArrayList<>(allResourceList)).filter(); + List resources = new ResourceFilter(suffix,new ArrayList<>(allResourceList)).filter(); Visitor resourceTreeVisitor = new ResourceTreeVisitor(resources); result.put(Constants.DATA_LIST, resourceTreeVisitor.visit().getChildren()); putMsg(result,Status.SUCCESS); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java index a56e3f83ef..a2ebd270bb 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.api.controller; import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.common.utils.JSONUtils; @@ -452,4 +453,28 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); } + + @Test + public void testqueryResourceJarList() throws Exception { + + MultiValueMap paramsMap = new LinkedMultiValueMap<>(); + paramsMap.add("type", ResourceType.FILE.name()); + //paramsMap.add("programType", ProgramType.PYTHON.name()); + paramsMap.add("programType", "JAVA"); + + + MvcResult mvcResult = mockMvc.perform(get("/resources/list/jar") + .header(SESSION_ID, sessionId) + .params(paramsMap)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) + .andReturn(); + + Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + result.getCode().equals(Status.SUCCESS.getCode()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); + + Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); + logger.info(mvcResult.getResponse().getContentAsString()); + } } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java index 8a4a16c4f0..8e3be6a2c9 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java @@ -23,7 +23,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; /** * resource filter test @@ -48,9 +50,11 @@ public class ResourceFilterTest { allList.add(resource5); allList.add(resource6); allList.add(resource7); + Set suffixSet = new HashSet<>(); + suffixSet.add(".jar"); + suffixSet.add(".txt"); - - ResourceFilter resourceFilter = new ResourceFilter(".jar",allList); + ResourceFilter resourceFilter = new ResourceFilter(suffixSet,allList); List resourceList = resourceFilter.filter(); Assert.assertNotNull(resourceList); resourceList.stream().forEach(t-> logger.info(t.toString())); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java index 3d35b68aac..d111a8c4f8 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java @@ -343,6 +343,14 @@ public abstract class UpgradeDao extends AbstractBaseDao { if (param != null) { List resourceList = JSONUtils.toList(param.getString("resourceList"), ResourceInfo.class); + ResourceInfo mainJar = JSONUtils.parseObject(param.getString("mainJar"), ResourceInfo.class); + if (mainJar != null && mainJar.getId() == 0) { + String fullName = mainJar.getRes().startsWith("/") ? mainJar.getRes() : String.format("/%s",mainJar.getRes()); + if (resourcesMap.containsKey(fullName)) { + mainJar.setId(resourcesMap.get(fullName)); + param.put("mainJar",JSONUtils.parseObject(JSONObject.toJSONString(mainJar))); + } + } if (CollectionUtils.isNotEmpty(resourceList)) { List newResourceList = resourceList.stream().map(resInfo -> {