Browse Source

[Fix-3469][api]Should filter the resource by the different program type (#3498)

* [Fix-3463][api]Fixed that run the sql task will be failure after rename the udf resource

* [Fix-3469][api]Should list python file and jar file

* [Fix-3469][api]Should filter the resource by the different program type

* [Fix-3469][api]fix the code smell
pull/3/MERGE
lgcareer 4 years ago committed by GitHub
parent
commit
5f5c08402f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 13
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java
  2. 25
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java
  3. 19
      dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
  4. 25
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java
  5. 8
      dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java
  6. 8
      dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java

13
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java

@ -27,6 +27,7 @@ import org.apache.dolphinscheduler.api.service.ResourcesService;
import org.apache.dolphinscheduler.api.service.UdfFuncService; import org.apache.dolphinscheduler.api.service.UdfFuncService;
import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.ResourceType;
import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.common.enums.UdfType;
import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils;
@ -275,7 +276,7 @@ public class ResourcesController extends BaseController {
* @param type resource type * @param type resource type
* @return resource list * @return resource list
*/ */
@ApiOperation(value = "queryResourceJarList", notes = "QUERY_RESOURCE_LIST_NOTES") @ApiOperation(value = "queryResourceByProgramType", notes = "QUERY_RESOURCE_LIST_NOTES")
@ApiImplicitParams({ @ApiImplicitParams({
@ApiImplicitParam(name = "type", value = "RESOURCE_TYPE", required = true, dataType = "ResourceType") @ApiImplicitParam(name = "type", value = "RESOURCE_TYPE", required = true, dataType = "ResourceType")
}) })
@ -283,10 +284,14 @@ public class ResourcesController extends BaseController {
@ResponseStatus(HttpStatus.OK) @ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_RESOURCES_LIST_ERROR) @ApiException(QUERY_RESOURCES_LIST_ERROR)
public Result queryResourceJarList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, public Result queryResourceJarList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@RequestParam(value = "type") ResourceType type @RequestParam(value = "type") ResourceType type,
@RequestParam(value = "programType",required = false) ProgramType programType
) { ) {
logger.info("query resource list, login user:{}, resource type:{}", loginUser.getUserName(), type.toString()); String programTypeName = programType == null ? "" : programType.name();
Map<String, Object> result = resourceService.queryResourceJarList(loginUser, type); String userName = loginUser.getUserName();
userName = userName.replaceAll("[\n|\r|\t]", "_");
logger.info("query resource list, login user:{}, resource type:{}, program type:{}", userName,programTypeName);
Map<String, Object> result = resourceService.queryResourceByProgramType(loginUser, type,programType);
return returnDataList(result); return returnDataList(result);
} }

25
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java

@ -31,24 +31,29 @@ public class ResourceFilter implements IFilter {
/** /**
* resource suffix * resource suffix
*/ */
private String suffix; private Set<String> suffixs = new HashSet<>();
/** /**
* resource list * resource list
*/ */
private List<Resource> resourceList; private List<Resource> resourceList;
/** /**
* parent list * constructor
* @param suffix resource suffix
* @param resourceList resource list
*/ */
//Set<Resource> parentList = new HashSet<>(); public ResourceFilter(String suffix, List<Resource> resourceList) {
this.suffixs.add(suffix);
this.resourceList = resourceList;
}
/** /**
* constructor * constructor
* @param suffix resource suffix * @param suffixs resource suffixs
* @param resourceList resource list * @param resourceList resource list
*/ */
public ResourceFilter(String suffix, List<Resource> resourceList) { public ResourceFilter(Set<String> suffixs, List<Resource> resourceList) {
this.suffix = suffix; this.suffixs = suffixs;
this.resourceList = resourceList; this.resourceList = resourceList;
} }
@ -59,7 +64,13 @@ public class ResourceFilter implements IFilter {
public Set<Resource> fileFilter(){ public Set<Resource> fileFilter(){
Set<Resource> resources = resourceList.stream().filter(t -> { Set<Resource> resources = resourceList.stream().filter(t -> {
String alias = t.getAlias(); String alias = t.getAlias();
return alias.endsWith(suffix); boolean result = false;
for (String suffix : suffixs) {
if (alias.endsWith(suffix)) {
result = true;
}
}
return result;
}).collect(Collectors.toSet()); }).collect(Collectors.toSet());
return resources; return resources;
} }

19
dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java

@ -30,6 +30,7 @@ import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.ResourceType;
import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.*;
import org.apache.dolphinscheduler.dao.entity.*; import org.apache.dolphinscheduler.dao.entity.*;
@ -654,21 +655,33 @@ public class ResourcesService extends BaseService {
} }
/** /**
* query resource list * query resource list by program type
* *
* @param loginUser login user * @param loginUser login user
* @param type resource type * @param type resource type
* @return resource list * @return resource list
*/ */
public Map<String, Object> queryResourceJarList(User loginUser, ResourceType type) { public Map<String, Object> queryResourceByProgramType(User loginUser, ResourceType type, ProgramType programType) {
Map<String, Object> result = new HashMap<>(5); Map<String, Object> result = new HashMap<>(5);
String suffix = ".jar";
int userId = loginUser.getId(); int userId = loginUser.getId();
if(isAdmin(loginUser)){ if(isAdmin(loginUser)){
userId = 0; userId = 0;
} }
if (programType != null) {
switch (programType) {
case JAVA:
break;
case SCALA:
break;
case PYTHON:
suffix = ".py";
break;
}
}
List<Resource> allResourceList = resourcesMapper.queryResourceListAuthored(userId, type.ordinal(),0); List<Resource> allResourceList = resourcesMapper.queryResourceListAuthored(userId, type.ordinal(),0);
List<Resource> resources = new ResourceFilter(".jar",new ArrayList<>(allResourceList)).filter(); List<Resource> resources = new ResourceFilter(suffix,new ArrayList<>(allResourceList)).filter();
Visitor resourceTreeVisitor = new ResourceTreeVisitor(resources); Visitor resourceTreeVisitor = new ResourceTreeVisitor(resources);
result.put(Constants.DATA_LIST, resourceTreeVisitor.visit().getChildren()); result.put(Constants.DATA_LIST, resourceTreeVisitor.visit().getChildren());
putMsg(result,Status.SUCCESS); putMsg(result,Status.SUCCESS);

25
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java

@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.api.controller;
import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSON;
import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.ResourceType;
import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.common.enums.UdfType;
import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.JSONUtils;
@ -452,4 +453,28 @@ public class ResourcesControllerTest extends AbstractControllerTest{
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString()); logger.info(mvcResult.getResponse().getContentAsString());
} }
@Test
public void testqueryResourceJarList() throws Exception {
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("type", ResourceType.FILE.name());
//paramsMap.add("programType", ProgramType.PYTHON.name());
paramsMap.add("programType", "JAVA");
MvcResult mvcResult = mockMvc.perform(get("/resources/list/jar")
.header(SESSION_ID, sessionId)
.params(paramsMap))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
.andReturn();
Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
result.getCode().equals(Status.SUCCESS.getCode());
JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString());
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
} }

8
dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java

@ -23,7 +23,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Set;
/** /**
* resource filter test * resource filter test
@ -48,9 +50,11 @@ public class ResourceFilterTest {
allList.add(resource5); allList.add(resource5);
allList.add(resource6); allList.add(resource6);
allList.add(resource7); allList.add(resource7);
Set<String> suffixSet = new HashSet<>();
suffixSet.add(".jar");
suffixSet.add(".txt");
ResourceFilter resourceFilter = new ResourceFilter(suffixSet,allList);
ResourceFilter resourceFilter = new ResourceFilter(".jar",allList);
List<Resource> resourceList = resourceFilter.filter(); List<Resource> resourceList = resourceFilter.filter();
Assert.assertNotNull(resourceList); Assert.assertNotNull(resourceList);
resourceList.stream().forEach(t-> logger.info(t.toString())); resourceList.stream().forEach(t-> logger.info(t.toString()));

8
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java

@ -343,6 +343,14 @@ public abstract class UpgradeDao extends AbstractBaseDao {
if (param != null) { if (param != null) {
List<ResourceInfo> resourceList = JSONUtils.toList(param.getString("resourceList"), ResourceInfo.class); List<ResourceInfo> resourceList = JSONUtils.toList(param.getString("resourceList"), ResourceInfo.class);
ResourceInfo mainJar = JSONUtils.parseObject(param.getString("mainJar"), ResourceInfo.class);
if (mainJar != null && mainJar.getId() == 0) {
String fullName = mainJar.getRes().startsWith("/") ? mainJar.getRes() : String.format("/%s",mainJar.getRes());
if (resourcesMap.containsKey(fullName)) {
mainJar.setId(resourcesMap.get(fullName));
param.put("mainJar",JSONUtils.parseObject(JSONObject.toJSONString(mainJar)));
}
}
if (CollectionUtils.isNotEmpty(resourceList)) { if (CollectionUtils.isNotEmpty(resourceList)) {
List<ResourceInfo> newResourceList = resourceList.stream().map(resInfo -> { List<ResourceInfo> newResourceList = resourceList.stream().map(resInfo -> {

Loading…
Cancel
Save