diff --git a/.github/workflows/ci_e2e.yml b/.github/workflows/ci_e2e.yml
index 82c81ef4e1..eaffc04e62 100644
--- a/.github/workflows/ci_e2e.yml
+++ b/.github/workflows/ci_e2e.yml
@@ -66,9 +66,9 @@ jobs:
run: cd ./e2e && mvn -B clean test
- name: Collect logs
if: failure()
- uses: actions/upload-artifact@v1
+ uses: actions/upload-artifact@v2
with:
name: dslogs
- path: /var/lib/docker/volumes/docker-swarm_dolphinscheduler-logs/_data
+ path: ${{ github.workspace }}/docker/docker-swarm/dolphinscheduler-logs
diff --git a/dolphinscheduler-alert/pom.xml b/dolphinscheduler-alert/pom.xml
index 5958bfbb8f..a416f2f08e 100644
--- a/dolphinscheduler-alert/pom.xml
+++ b/dolphinscheduler-alert/pom.xml
@@ -21,7 +21,7 @@
org.apache.dolphinscheduler
dolphinscheduler
- 1.3.2-SNAPSHOT
+ 1.3.3-SNAPSHOT
dolphinscheduler-alert
${project.artifactId}
diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml
index 5c822ee362..c49e186198 100644
--- a/dolphinscheduler-api/pom.xml
+++ b/dolphinscheduler-api/pom.xml
@@ -21,7 +21,7 @@
org.apache.dolphinscheduler
dolphinscheduler
- 1.3.2-SNAPSHOT
+ 1.3.3-SNAPSHOT
dolphinscheduler-api
${project.artifactId}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java
index 2f2cb9ca6f..697d364dfd 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java
@@ -27,6 +27,7 @@ import org.apache.dolphinscheduler.api.service.ResourcesService;
import org.apache.dolphinscheduler.api.service.UdfFuncService;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.enums.ResourceType;
import org.apache.dolphinscheduler.common.enums.UdfType;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
@@ -275,7 +276,7 @@ public class ResourcesController extends BaseController {
* @param type resource type
* @return resource list
*/
- @ApiOperation(value = "queryResourceJarList", notes = "QUERY_RESOURCE_LIST_NOTES")
+ @ApiOperation(value = "queryResourceByProgramType", notes = "QUERY_RESOURCE_LIST_NOTES")
@ApiImplicitParams({
@ApiImplicitParam(name = "type", value = "RESOURCE_TYPE", required = true, dataType = "ResourceType")
})
@@ -283,10 +284,14 @@ public class ResourcesController extends BaseController {
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_RESOURCES_LIST_ERROR)
public Result queryResourceJarList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
- @RequestParam(value = "type") ResourceType type
+ @RequestParam(value = "type") ResourceType type,
+ @RequestParam(value = "programType",required = false) ProgramType programType
) {
- logger.info("query resource list, login user:{}, resource type:{}", loginUser.getUserName(), type.toString());
- Map result = resourceService.queryResourceJarList(loginUser, type);
+ String programTypeName = programType == null ? "" : programType.name();
+ String userName = loginUser.getUserName();
+ userName = userName.replaceAll("[\n|\r|\t]", "_");
+ logger.info("query resource list, login user:{}, resource type:{}, program type:{}", userName,programTypeName);
+ Map result = resourceService.queryResourceByProgramType(loginUser, type,programType);
return returnDataList(result);
}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java
index c918a160af..9c3a84cf43 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java
@@ -31,24 +31,29 @@ public class ResourceFilter implements IFilter {
/**
* resource suffix
*/
- private String suffix;
+ private Set suffixs = new HashSet<>();
/**
* resource list
*/
private List resourceList;
/**
- * parent list
+ * constructor
+ * @param suffix resource suffix
+ * @param resourceList resource list
*/
- //Set parentList = new HashSet<>();
+ public ResourceFilter(String suffix, List resourceList) {
+ this.suffixs.add(suffix);
+ this.resourceList = resourceList;
+ }
/**
* constructor
- * @param suffix resource suffix
+ * @param suffixs resource suffixs
* @param resourceList resource list
*/
- public ResourceFilter(String suffix, List resourceList) {
- this.suffix = suffix;
+ public ResourceFilter(Set suffixs, List resourceList) {
+ this.suffixs = suffixs;
this.resourceList = resourceList;
}
@@ -59,7 +64,13 @@ public class ResourceFilter implements IFilter {
public Set fileFilter(){
Set resources = resourceList.stream().filter(t -> {
String alias = t.getAlias();
- return alias.endsWith(suffix);
+ boolean result = false;
+ for (String suffix : suffixs) {
+ if (alias.endsWith(suffix)) {
+ result = true;
+ }
+ }
+ return result;
}).collect(Collectors.toSet());
return resources;
}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
index e1a9c85aa1..f66e035eae 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
@@ -30,6 +30,7 @@ import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.enums.ResourceType;
import org.apache.dolphinscheduler.common.utils.*;
import org.apache.dolphinscheduler.dao.entity.*;
@@ -654,21 +655,33 @@ public class ResourcesService extends BaseService {
}
/**
- * query resource list
+ * query resource list by program type
*
* @param loginUser login user
* @param type resource type
* @return resource list
*/
- public Map queryResourceJarList(User loginUser, ResourceType type) {
+ public Map queryResourceByProgramType(User loginUser, ResourceType type, ProgramType programType) {
Map result = new HashMap<>(5);
+ String suffix = ".jar";
int userId = loginUser.getId();
if(isAdmin(loginUser)){
userId = 0;
}
+ if (programType != null) {
+ switch (programType) {
+ case JAVA:
+ break;
+ case SCALA:
+ break;
+ case PYTHON:
+ suffix = ".py";
+ break;
+ }
+ }
List allResourceList = resourcesMapper.queryResourceListAuthored(userId, type.ordinal(),0);
- List resources = new ResourceFilter(".jar",new ArrayList<>(allResourceList)).filter();
+ List resources = new ResourceFilter(suffix,new ArrayList<>(allResourceList)).filter();
Visitor resourceTreeVisitor = new ResourceTreeVisitor(resources);
result.put(Constants.DATA_LIST, resourceTreeVisitor.visit().getChildren());
putMsg(result,Status.SUCCESS);
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java
index a56e3f83ef..a2ebd270bb 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java
@@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.api.controller;
import com.alibaba.fastjson.JSON;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.Result;
+import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.enums.ResourceType;
import org.apache.dolphinscheduler.common.enums.UdfType;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
@@ -452,4 +453,28 @@ public class ResourcesControllerTest extends AbstractControllerTest{
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
+
+ @Test
+ public void testqueryResourceJarList() throws Exception {
+
+ MultiValueMap paramsMap = new LinkedMultiValueMap<>();
+ paramsMap.add("type", ResourceType.FILE.name());
+ //paramsMap.add("programType", ProgramType.PYTHON.name());
+ paramsMap.add("programType", "JAVA");
+
+
+ MvcResult mvcResult = mockMvc.perform(get("/resources/list/jar")
+ .header(SESSION_ID, sessionId)
+ .params(paramsMap))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
+ .andReturn();
+
+ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
+ result.getCode().equals(Status.SUCCESS.getCode());
+ JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString());
+
+ Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
+ logger.info(mvcResult.getResponse().getContentAsString());
+ }
}
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java
index 8a4a16c4f0..8e3be6a2c9 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java
@@ -23,7 +23,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
/**
* resource filter test
@@ -48,9 +50,11 @@ public class ResourceFilterTest {
allList.add(resource5);
allList.add(resource6);
allList.add(resource7);
+ Set suffixSet = new HashSet<>();
+ suffixSet.add(".jar");
+ suffixSet.add(".txt");
-
- ResourceFilter resourceFilter = new ResourceFilter(".jar",allList);
+ ResourceFilter resourceFilter = new ResourceFilter(suffixSet,allList);
List resourceList = resourceFilter.filter();
Assert.assertNotNull(resourceList);
resourceList.stream().forEach(t-> logger.info(t.toString()));
diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml
index 290fa91b54..86e9de6cf8 100644
--- a/dolphinscheduler-common/pom.xml
+++ b/dolphinscheduler-common/pom.xml
@@ -21,7 +21,7 @@
org.apache.dolphinscheduler
dolphinscheduler
- 1.3.2-SNAPSHOT
+ 1.3.3-SNAPSHOT
dolphinscheduler-common
dolphinscheduler-common
diff --git a/dolphinscheduler-dao/pom.xml b/dolphinscheduler-dao/pom.xml
index c97b52d201..8ef43b51b6 100644
--- a/dolphinscheduler-dao/pom.xml
+++ b/dolphinscheduler-dao/pom.xml
@@ -21,7 +21,7 @@
org.apache.dolphinscheduler
dolphinscheduler
- 1.3.2-SNAPSHOT
+ 1.3.3-SNAPSHOT
dolphinscheduler-dao
${project.artifactId}
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
index 3d35b68aac..d111a8c4f8 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
@@ -343,6 +343,14 @@ public abstract class UpgradeDao extends AbstractBaseDao {
if (param != null) {
List resourceList = JSONUtils.toList(param.getString("resourceList"), ResourceInfo.class);
+ ResourceInfo mainJar = JSONUtils.parseObject(param.getString("mainJar"), ResourceInfo.class);
+ if (mainJar != null && mainJar.getId() == 0) {
+ String fullName = mainJar.getRes().startsWith("/") ? mainJar.getRes() : String.format("/%s",mainJar.getRes());
+ if (resourcesMap.containsKey(fullName)) {
+ mainJar.setId(resourcesMap.get(fullName));
+ param.put("mainJar",JSONUtils.parseObject(JSONObject.toJSONString(mainJar)));
+ }
+ }
if (CollectionUtils.isNotEmpty(resourceList)) {
List newResourceList = resourceList.stream().map(resInfo -> {
diff --git a/dolphinscheduler-dist/pom.xml b/dolphinscheduler-dist/pom.xml
index ca3834e90b..8accc18f4c 100644
--- a/dolphinscheduler-dist/pom.xml
+++ b/dolphinscheduler-dist/pom.xml
@@ -20,7 +20,7 @@
dolphinscheduler
org.apache.dolphinscheduler
- 1.3.2-SNAPSHOT
+ 1.3.3-SNAPSHOT
4.0.0
diff --git a/dolphinscheduler-plugin-api/pom.xml b/dolphinscheduler-plugin-api/pom.xml
index 2bffba811d..b5963a0ea3 100644
--- a/dolphinscheduler-plugin-api/pom.xml
+++ b/dolphinscheduler-plugin-api/pom.xml
@@ -21,7 +21,7 @@
org.apache.dolphinscheduler
dolphinscheduler
- 1.3.2-SNAPSHOT
+ 1.3.3-SNAPSHOT
dolphinscheduler-plugin-api
${project.artifactId}
diff --git a/dolphinscheduler-remote/pom.xml b/dolphinscheduler-remote/pom.xml
index dcc57633bb..7a6aef783e 100644
--- a/dolphinscheduler-remote/pom.xml
+++ b/dolphinscheduler-remote/pom.xml
@@ -19,7 +19,7 @@
dolphinscheduler
org.apache.dolphinscheduler
- 1.3.2-SNAPSHOT
+ 1.3.3-SNAPSHOT
4.0.0
diff --git a/dolphinscheduler-server/pom.xml b/dolphinscheduler-server/pom.xml
index 4cbce0ab47..f32207bc77 100644
--- a/dolphinscheduler-server/pom.xml
+++ b/dolphinscheduler-server/pom.xml
@@ -21,7 +21,7 @@
org.apache.dolphinscheduler
dolphinscheduler
- 1.3.2-SNAPSHOT
+ 1.3.3-SNAPSHOT
dolphinscheduler-server
dolphinscheduler-server
diff --git a/dolphinscheduler-service/pom.xml b/dolphinscheduler-service/pom.xml
index 3c4ea53ebb..4e0bdbb983 100644
--- a/dolphinscheduler-service/pom.xml
+++ b/dolphinscheduler-service/pom.xml
@@ -19,7 +19,7 @@
dolphinscheduler
org.apache.dolphinscheduler
- 1.3.2-SNAPSHOT
+ 1.3.3-SNAPSHOT
4.0.0
diff --git a/dolphinscheduler-ui/pom.xml b/dolphinscheduler-ui/pom.xml
index 13644bad91..53b872929b 100644
--- a/dolphinscheduler-ui/pom.xml
+++ b/dolphinscheduler-ui/pom.xml
@@ -20,7 +20,7 @@
dolphinscheduler
org.apache.dolphinscheduler
- 1.3.2-SNAPSHOT
+ 1.3.3-SNAPSHOT
4.0.0
diff --git a/pom.xml b/pom.xml
index 80dceb71f6..f13cf9a05d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -19,7 +19,7 @@
4.0.0
org.apache.dolphinscheduler
dolphinscheduler
- 1.3.2-SNAPSHOT
+ 1.3.3-SNAPSHOT
pom
${project.artifactId}
http://dolphinscheduler.apache.org