diff --git a/.github/workflows/ci_e2e.yml b/.github/workflows/ci_e2e.yml index 82c81ef4e1..eaffc04e62 100644 --- a/.github/workflows/ci_e2e.yml +++ b/.github/workflows/ci_e2e.yml @@ -66,9 +66,9 @@ jobs: run: cd ./e2e && mvn -B clean test - name: Collect logs if: failure() - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v2 with: name: dslogs - path: /var/lib/docker/volumes/docker-swarm_dolphinscheduler-logs/_data + path: ${{ github.workspace }}/docker/docker-swarm/dolphinscheduler-logs diff --git a/dolphinscheduler-alert/pom.xml b/dolphinscheduler-alert/pom.xml index 5958bfbb8f..a416f2f08e 100644 --- a/dolphinscheduler-alert/pom.xml +++ b/dolphinscheduler-alert/pom.xml @@ -21,7 +21,7 @@ org.apache.dolphinscheduler dolphinscheduler - 1.3.2-SNAPSHOT + 1.3.3-SNAPSHOT dolphinscheduler-alert ${project.artifactId} diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index 5c822ee362..c49e186198 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -21,7 +21,7 @@ org.apache.dolphinscheduler dolphinscheduler - 1.3.2-SNAPSHOT + 1.3.3-SNAPSHOT dolphinscheduler-api ${project.artifactId} diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java index 2f2cb9ca6f..697d364dfd 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java @@ -27,6 +27,7 @@ import org.apache.dolphinscheduler.api.service.ResourcesService; import org.apache.dolphinscheduler.api.service.UdfFuncService; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.common.utils.ParameterUtils; @@ -275,7 +276,7 @@ public class ResourcesController extends BaseController { * @param type resource type * @return resource list */ - @ApiOperation(value = "queryResourceJarList", notes = "QUERY_RESOURCE_LIST_NOTES") + @ApiOperation(value = "queryResourceByProgramType", notes = "QUERY_RESOURCE_LIST_NOTES") @ApiImplicitParams({ @ApiImplicitParam(name = "type", value = "RESOURCE_TYPE", required = true, dataType = "ResourceType") }) @@ -283,10 +284,14 @@ public class ResourcesController extends BaseController { @ResponseStatus(HttpStatus.OK) @ApiException(QUERY_RESOURCES_LIST_ERROR) public Result queryResourceJarList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, - @RequestParam(value = "type") ResourceType type + @RequestParam(value = "type") ResourceType type, + @RequestParam(value = "programType",required = false) ProgramType programType ) { - logger.info("query resource list, login user:{}, resource type:{}", loginUser.getUserName(), type.toString()); - Map result = resourceService.queryResourceJarList(loginUser, type); + String programTypeName = programType == null ? "" : programType.name(); + String userName = loginUser.getUserName(); + userName = userName.replaceAll("[\n|\r|\t]", "_"); + logger.info("query resource list, login user:{}, resource type:{}, program type:{}", userName,programTypeName); + Map result = resourceService.queryResourceByProgramType(loginUser, type,programType); return returnDataList(result); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java index c918a160af..9c3a84cf43 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java @@ -31,24 +31,29 @@ public class ResourceFilter implements IFilter { /** * resource suffix */ - private String suffix; + private Set suffixs = new HashSet<>(); /** * resource list */ private List resourceList; /** - * parent list + * constructor + * @param suffix resource suffix + * @param resourceList resource list */ - //Set parentList = new HashSet<>(); + public ResourceFilter(String suffix, List resourceList) { + this.suffixs.add(suffix); + this.resourceList = resourceList; + } /** * constructor - * @param suffix resource suffix + * @param suffixs resource suffixs * @param resourceList resource list */ - public ResourceFilter(String suffix, List resourceList) { - this.suffix = suffix; + public ResourceFilter(Set suffixs, List resourceList) { + this.suffixs = suffixs; this.resourceList = resourceList; } @@ -59,7 +64,13 @@ public class ResourceFilter implements IFilter { public Set fileFilter(){ Set resources = resourceList.stream().filter(t -> { String alias = t.getAlias(); - return alias.endsWith(suffix); + boolean result = false; + for (String suffix : suffixs) { + if (alias.endsWith(suffix)) { + result = true; + } + } + return result; }).collect(Collectors.toSet()); return resources; } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index e1a9c85aa1..f66e035eae 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -30,6 +30,7 @@ import org.apache.dolphinscheduler.api.exceptions.ServiceException; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.entity.*; @@ -654,21 +655,33 @@ public class ResourcesService extends BaseService { } /** - * query resource list + * query resource list by program type * * @param loginUser login user * @param type resource type * @return resource list */ - public Map queryResourceJarList(User loginUser, ResourceType type) { + public Map queryResourceByProgramType(User loginUser, ResourceType type, ProgramType programType) { Map result = new HashMap<>(5); + String suffix = ".jar"; int userId = loginUser.getId(); if(isAdmin(loginUser)){ userId = 0; } + if (programType != null) { + switch (programType) { + case JAVA: + break; + case SCALA: + break; + case PYTHON: + suffix = ".py"; + break; + } + } List allResourceList = resourcesMapper.queryResourceListAuthored(userId, type.ordinal(),0); - List resources = new ResourceFilter(".jar",new ArrayList<>(allResourceList)).filter(); + List resources = new ResourceFilter(suffix,new ArrayList<>(allResourceList)).filter(); Visitor resourceTreeVisitor = new ResourceTreeVisitor(resources); result.put(Constants.DATA_LIST, resourceTreeVisitor.visit().getChildren()); putMsg(result,Status.SUCCESS); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java index a56e3f83ef..a2ebd270bb 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.api.controller; import com.alibaba.fastjson.JSON; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.common.enums.ProgramType; import org.apache.dolphinscheduler.common.enums.ResourceType; import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.common.utils.JSONUtils; @@ -452,4 +453,28 @@ public class ResourcesControllerTest extends AbstractControllerTest{ Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); logger.info(mvcResult.getResponse().getContentAsString()); } + + @Test + public void testqueryResourceJarList() throws Exception { + + MultiValueMap paramsMap = new LinkedMultiValueMap<>(); + paramsMap.add("type", ResourceType.FILE.name()); + //paramsMap.add("programType", ProgramType.PYTHON.name()); + paramsMap.add("programType", "JAVA"); + + + MvcResult mvcResult = mockMvc.perform(get("/resources/list/jar") + .header(SESSION_ID, sessionId) + .params(paramsMap)) + .andExpect(status().isOk()) + .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8)) + .andReturn(); + + Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class); + result.getCode().equals(Status.SUCCESS.getCode()); + JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString()); + + Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue()); + logger.info(mvcResult.getResponse().getContentAsString()); + } } diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java index 8a4a16c4f0..8e3be6a2c9 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java @@ -23,7 +23,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; /** * resource filter test @@ -48,9 +50,11 @@ public class ResourceFilterTest { allList.add(resource5); allList.add(resource6); allList.add(resource7); + Set suffixSet = new HashSet<>(); + suffixSet.add(".jar"); + suffixSet.add(".txt"); - - ResourceFilter resourceFilter = new ResourceFilter(".jar",allList); + ResourceFilter resourceFilter = new ResourceFilter(suffixSet,allList); List resourceList = resourceFilter.filter(); Assert.assertNotNull(resourceList); resourceList.stream().forEach(t-> logger.info(t.toString())); diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index 290fa91b54..86e9de6cf8 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -21,7 +21,7 @@ org.apache.dolphinscheduler dolphinscheduler - 1.3.2-SNAPSHOT + 1.3.3-SNAPSHOT dolphinscheduler-common dolphinscheduler-common diff --git a/dolphinscheduler-dao/pom.xml b/dolphinscheduler-dao/pom.xml index c97b52d201..8ef43b51b6 100644 --- a/dolphinscheduler-dao/pom.xml +++ b/dolphinscheduler-dao/pom.xml @@ -21,7 +21,7 @@ org.apache.dolphinscheduler dolphinscheduler - 1.3.2-SNAPSHOT + 1.3.3-SNAPSHOT dolphinscheduler-dao ${project.artifactId} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java index 3d35b68aac..d111a8c4f8 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java @@ -343,6 +343,14 @@ public abstract class UpgradeDao extends AbstractBaseDao { if (param != null) { List resourceList = JSONUtils.toList(param.getString("resourceList"), ResourceInfo.class); + ResourceInfo mainJar = JSONUtils.parseObject(param.getString("mainJar"), ResourceInfo.class); + if (mainJar != null && mainJar.getId() == 0) { + String fullName = mainJar.getRes().startsWith("/") ? mainJar.getRes() : String.format("/%s",mainJar.getRes()); + if (resourcesMap.containsKey(fullName)) { + mainJar.setId(resourcesMap.get(fullName)); + param.put("mainJar",JSONUtils.parseObject(JSONObject.toJSONString(mainJar))); + } + } if (CollectionUtils.isNotEmpty(resourceList)) { List newResourceList = resourceList.stream().map(resInfo -> { diff --git a/dolphinscheduler-dist/pom.xml b/dolphinscheduler-dist/pom.xml index ca3834e90b..8accc18f4c 100644 --- a/dolphinscheduler-dist/pom.xml +++ b/dolphinscheduler-dist/pom.xml @@ -20,7 +20,7 @@ dolphinscheduler org.apache.dolphinscheduler - 1.3.2-SNAPSHOT + 1.3.3-SNAPSHOT 4.0.0 diff --git a/dolphinscheduler-plugin-api/pom.xml b/dolphinscheduler-plugin-api/pom.xml index 2bffba811d..b5963a0ea3 100644 --- a/dolphinscheduler-plugin-api/pom.xml +++ b/dolphinscheduler-plugin-api/pom.xml @@ -21,7 +21,7 @@ org.apache.dolphinscheduler dolphinscheduler - 1.3.2-SNAPSHOT + 1.3.3-SNAPSHOT dolphinscheduler-plugin-api ${project.artifactId} diff --git a/dolphinscheduler-remote/pom.xml b/dolphinscheduler-remote/pom.xml index dcc57633bb..7a6aef783e 100644 --- a/dolphinscheduler-remote/pom.xml +++ b/dolphinscheduler-remote/pom.xml @@ -19,7 +19,7 @@ dolphinscheduler org.apache.dolphinscheduler - 1.3.2-SNAPSHOT + 1.3.3-SNAPSHOT 4.0.0 diff --git a/dolphinscheduler-server/pom.xml b/dolphinscheduler-server/pom.xml index 4cbce0ab47..f32207bc77 100644 --- a/dolphinscheduler-server/pom.xml +++ b/dolphinscheduler-server/pom.xml @@ -21,7 +21,7 @@ org.apache.dolphinscheduler dolphinscheduler - 1.3.2-SNAPSHOT + 1.3.3-SNAPSHOT dolphinscheduler-server dolphinscheduler-server diff --git a/dolphinscheduler-service/pom.xml b/dolphinscheduler-service/pom.xml index 3c4ea53ebb..4e0bdbb983 100644 --- a/dolphinscheduler-service/pom.xml +++ b/dolphinscheduler-service/pom.xml @@ -19,7 +19,7 @@ dolphinscheduler org.apache.dolphinscheduler - 1.3.2-SNAPSHOT + 1.3.3-SNAPSHOT 4.0.0 diff --git a/dolphinscheduler-ui/pom.xml b/dolphinscheduler-ui/pom.xml index 13644bad91..53b872929b 100644 --- a/dolphinscheduler-ui/pom.xml +++ b/dolphinscheduler-ui/pom.xml @@ -20,7 +20,7 @@ dolphinscheduler org.apache.dolphinscheduler - 1.3.2-SNAPSHOT + 1.3.3-SNAPSHOT 4.0.0 diff --git a/pom.xml b/pom.xml index 80dceb71f6..f13cf9a05d 100644 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,7 @@ 4.0.0 org.apache.dolphinscheduler dolphinscheduler - 1.3.2-SNAPSHOT + 1.3.3-SNAPSHOT pom ${project.artifactId} http://dolphinscheduler.apache.org