Merge remote-tracking branch 'remotes/upstream/1.3.2-release' into 1.3.2-release

.github/workflows/ci_e2e.yml

@@ -66,9 +66,9 @@ jobs:
         run: cd ./e2e && mvn -B clean test
       - name: Collect logs
         if: failure()
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v2
         with:
           name: dslogs
-          path: /var/lib/docker/volumes/docker-swarm_dolphinscheduler-logs/_data
+          path: ${{ github.workspace }}/docker/docker-swarm/dolphinscheduler-logs
 
 

dolphinscheduler-alert/pom.xml

@@ -21,7 +21,7 @@
     <parent>
         <groupId>org.apache.dolphinscheduler</groupId>
         <artifactId>dolphinscheduler</artifactId>
-        <version>1.3.2-SNAPSHOT</version>
+        <version>1.3.3-SNAPSHOT</version>
     </parent>
     <artifactId>dolphinscheduler-alert</artifactId>
     <name>${project.artifactId}</name>

dolphinscheduler-api/pom.xml

@@ -21,7 +21,7 @@
   <parent>
     <groupId>org.apache.dolphinscheduler</groupId>
     <artifactId>dolphinscheduler</artifactId>
-    <version>1.3.2-SNAPSHOT</version>
+    <version>1.3.3-SNAPSHOT</version>
   </parent>
   <artifactId>dolphinscheduler-api</artifactId>
   <name>${project.artifactId}</name>

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java

@@ -27,6 +27,7 @@ import org.apache.dolphinscheduler.api.service.ResourcesService;
 import org.apache.dolphinscheduler.api.service.UdfFuncService;
 import org.apache.dolphinscheduler.api.utils.Result;
 import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.enums.ProgramType;
 import org.apache.dolphinscheduler.common.enums.ResourceType;
 import org.apache.dolphinscheduler.common.enums.UdfType;
 import org.apache.dolphinscheduler.common.utils.ParameterUtils;
@@ -275,7 +276,7 @@ public class ResourcesController extends BaseController {
      * @param type      resource type
      * @return resource list
      */
-    @ApiOperation(value = "queryResourceJarList", notes = "QUERY_RESOURCE_LIST_NOTES")
+    @ApiOperation(value = "queryResourceByProgramType", notes = "QUERY_RESOURCE_LIST_NOTES")
     @ApiImplicitParams({
             @ApiImplicitParam(name = "type", value = "RESOURCE_TYPE", required = true, dataType = "ResourceType")
     })
@@ -283,10 +284,14 @@ public class ResourcesController extends BaseController {
     @ResponseStatus(HttpStatus.OK)
     @ApiException(QUERY_RESOURCES_LIST_ERROR)
     public Result queryResourceJarList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
-                                       @RequestParam(value = "type") ResourceType type
+                                       @RequestParam(value = "type") ResourceType type,
+                                       @RequestParam(value = "programType",required = false) ProgramType programType
     ) {
-        logger.info("query resource list, login user:{}, resource type:{}", loginUser.getUserName(), type.toString());
-        Map<String, Object> result = resourceService.queryResourceJarList(loginUser, type);
+        String programTypeName = programType == null ? "" : programType.name();
+        String userName = loginUser.getUserName();
+        userName = userName.replaceAll("[\n|\r|\t]", "_");
+        logger.info("query resource list, login user:{}, resource type:{}, program type:{}", userName,programTypeName);
+        Map<String, Object> result = resourceService.queryResourceByProgramType(loginUser, type,programType);
         return returnDataList(result);
     }
 

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java

@@ -31,24 +31,29 @@ public class ResourceFilter implements IFilter {
     /**
      * resource suffix
      */
-    private String suffix;
+    private Set<String> suffixs = new HashSet<>();
     /**
      * resource list
      */
     private List<Resource> resourceList;
 
     /**
-     * parent list
+     * constructor
+     * @param suffix        resource suffix
+     * @param resourceList  resource list
      */
-    //Set<Resource> parentList =  new HashSet<>();
+    public ResourceFilter(String suffix, List<Resource> resourceList) {
+        this.suffixs.add(suffix);
+        this.resourceList = resourceList;
+    }
 
     /**
      * constructor
-     * @param suffix        resource suffix
+     * @param suffixs        resource suffixs
      * @param resourceList  resource list
      */
-    public ResourceFilter(String suffix, List<Resource> resourceList) {
-        this.suffix = suffix;
+    public ResourceFilter(Set<String> suffixs, List<Resource> resourceList) {
+        this.suffixs = suffixs;
         this.resourceList = resourceList;
     }
 
@@ -59,7 +64,13 @@ public class ResourceFilter implements IFilter {
     public Set<Resource> fileFilter(){
         Set<Resource> resources = resourceList.stream().filter(t -> {
             String alias = t.getAlias();
-            return alias.endsWith(suffix);
+            boolean result = false;
+            for (String suffix : suffixs) {
+                if (alias.endsWith(suffix)) {
+                    result = true;
+                }
+            }
+            return result;
         }).collect(Collectors.toSet());
         return resources;
     }

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java

@@ -30,6 +30,7 @@ import org.apache.dolphinscheduler.api.exceptions.ServiceException;
 import org.apache.dolphinscheduler.api.utils.PageInfo;
 import org.apache.dolphinscheduler.api.utils.Result;
 import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.enums.ProgramType;
 import org.apache.dolphinscheduler.common.enums.ResourceType;
 import org.apache.dolphinscheduler.common.utils.*;
 import org.apache.dolphinscheduler.dao.entity.*;
@@ -654,21 +655,33 @@ public class ResourcesService extends BaseService {
     }
 
     /**
-     * query resource list
+     * query resource list by program type
      *
      * @param loginUser login user
      * @param type resource type
      * @return resource list
      */
-    public Map<String, Object> queryResourceJarList(User loginUser, ResourceType type) {
+    public Map<String, Object> queryResourceByProgramType(User loginUser, ResourceType type, ProgramType programType) {
 
         Map<String, Object> result = new HashMap<>(5);
+        String suffix = ".jar";
         int userId = loginUser.getId();
         if(isAdmin(loginUser)){
             userId = 0;
         }
+        if (programType != null) {
+            switch (programType) {
+                case JAVA:
+                    break;
+                case SCALA:
+                    break;
+                case PYTHON:
+                    suffix = ".py";
+                    break;
+            }
+        }
         List<Resource> allResourceList = resourcesMapper.queryResourceListAuthored(userId, type.ordinal(),0);
-        List<Resource> resources = new ResourceFilter(".jar",new ArrayList<>(allResourceList)).filter();
+        List<Resource> resources = new ResourceFilter(suffix,new ArrayList<>(allResourceList)).filter();
         Visitor resourceTreeVisitor = new ResourceTreeVisitor(resources);
         result.put(Constants.DATA_LIST, resourceTreeVisitor.visit().getChildren());
         putMsg(result,Status.SUCCESS);

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java

@@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.api.controller;
 import com.alibaba.fastjson.JSON;
 import org.apache.dolphinscheduler.api.enums.Status;
 import org.apache.dolphinscheduler.api.utils.Result;
+import org.apache.dolphinscheduler.common.enums.ProgramType;
 import org.apache.dolphinscheduler.common.enums.ResourceType;
 import org.apache.dolphinscheduler.common.enums.UdfType;
 import org.apache.dolphinscheduler.common.utils.JSONUtils;
@@ -452,4 +453,28 @@ public class ResourcesControllerTest extends AbstractControllerTest{
         Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
         logger.info(mvcResult.getResponse().getContentAsString());
     }
+
+    @Test
+    public void testqueryResourceJarList() throws Exception {
+
+        MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
+        paramsMap.add("type", ResourceType.FILE.name());
+        //paramsMap.add("programType", ProgramType.PYTHON.name());
+        paramsMap.add("programType", "JAVA");
+
+
+        MvcResult mvcResult = mockMvc.perform(get("/resources/list/jar")
+                .header(SESSION_ID, sessionId)
+                .params(paramsMap))
+                .andExpect(status().isOk())
+                .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
+                .andReturn();
+
+        Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
+        result.getCode().equals(Status.SUCCESS.getCode());
+        JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString());
+
+        Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
+        logger.info(mvcResult.getResponse().getContentAsString());
+    }
 }

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java

@@ -23,7 +23,9 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 /**
  * resource filter test
@@ -48,9 +50,11 @@ public class ResourceFilterTest {
         allList.add(resource5);
         allList.add(resource6);
         allList.add(resource7);
+        Set<String> suffixSet = new HashSet<>();
+        suffixSet.add(".jar");
+        suffixSet.add(".txt");
 
-
-        ResourceFilter resourceFilter = new ResourceFilter(".jar",allList);
+        ResourceFilter resourceFilter = new ResourceFilter(suffixSet,allList);
         List<Resource> resourceList = resourceFilter.filter();
         Assert.assertNotNull(resourceList);
         resourceList.stream().forEach(t-> logger.info(t.toString()));

dolphinscheduler-common/pom.xml

@@ -21,7 +21,7 @@
 	<parent>
 		<groupId>org.apache.dolphinscheduler</groupId>
 		<artifactId>dolphinscheduler</artifactId>
-		<version>1.3.2-SNAPSHOT</version>
+		<version>1.3.3-SNAPSHOT</version>
 	</parent>
 	<artifactId>dolphinscheduler-common</artifactId>
 	<name>dolphinscheduler-common</name>

dolphinscheduler-dao/pom.xml

@@ -21,7 +21,7 @@
 	<parent>
 		<groupId>org.apache.dolphinscheduler</groupId>
 		<artifactId>dolphinscheduler</artifactId>
-		<version>1.3.2-SNAPSHOT</version>
+		<version>1.3.3-SNAPSHOT</version>
 	</parent>
 	<artifactId>dolphinscheduler-dao</artifactId>
 	<name>${project.artifactId}</name>

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java

@@ -343,6 +343,14 @@ public abstract class UpgradeDao extends AbstractBaseDao {
                     if (param != null) {
 
                         List<ResourceInfo> resourceList = JSONUtils.toList(param.getString("resourceList"), ResourceInfo.class);
+                        ResourceInfo mainJar = JSONUtils.parseObject(param.getString("mainJar"), ResourceInfo.class);
+                        if (mainJar != null && mainJar.getId() == 0) {
+                            String fullName = mainJar.getRes().startsWith("/") ? mainJar.getRes() : String.format("/%s",mainJar.getRes());
+                            if (resourcesMap.containsKey(fullName)) {
+                                mainJar.setId(resourcesMap.get(fullName));
+                                param.put("mainJar",JSONUtils.parseObject(JSONObject.toJSONString(mainJar)));
+                            }
+                        }
 
                         if (CollectionUtils.isNotEmpty(resourceList)) {
                             List<ResourceInfo> newResourceList = resourceList.stream().map(resInfo -> {

dolphinscheduler-dist/pom.xml

@@ -20,7 +20,7 @@
     <parent>
         <artifactId>dolphinscheduler</artifactId>
         <groupId>org.apache.dolphinscheduler</groupId>
-        <version>1.3.2-SNAPSHOT</version>
+        <version>1.3.3-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

dolphinscheduler-plugin-api/pom.xml

@@ -21,7 +21,7 @@
     <parent>
         <groupId>org.apache.dolphinscheduler</groupId>
         <artifactId>dolphinscheduler</artifactId>
-        <version>1.3.2-SNAPSHOT</version>
+        <version>1.3.3-SNAPSHOT</version>
     </parent>
     <artifactId>dolphinscheduler-plugin-api</artifactId>
     <name>${project.artifactId}</name>

dolphinscheduler-remote/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <artifactId>dolphinscheduler</artifactId>
         <groupId>org.apache.dolphinscheduler</groupId>
-        <version>1.3.2-SNAPSHOT</version>
+        <version>1.3.3-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

dolphinscheduler-server/pom.xml

@@ -21,7 +21,7 @@
 	<parent>
 		<groupId>org.apache.dolphinscheduler</groupId>
 		<artifactId>dolphinscheduler</artifactId>
-		<version>1.3.2-SNAPSHOT</version>
+		<version>1.3.3-SNAPSHOT</version>
 	</parent>
 	<artifactId>dolphinscheduler-server</artifactId>
 	<name>dolphinscheduler-server</name>

dolphinscheduler-service/pom.xml

@@ -19,7 +19,7 @@
     <parent>
         <artifactId>dolphinscheduler</artifactId>
         <groupId>org.apache.dolphinscheduler</groupId>
-        <version>1.3.2-SNAPSHOT</version>
+        <version>1.3.3-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

dolphinscheduler-ui/pom.xml

@@ -20,7 +20,7 @@
   <parent>
     <artifactId>dolphinscheduler</artifactId>
     <groupId>org.apache.dolphinscheduler</groupId>
-    <version>1.3.2-SNAPSHOT</version>
+    <version>1.3.3-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
 

pom.xml

@@ -19,7 +19,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>org.apache.dolphinscheduler</groupId>
     <artifactId>dolphinscheduler</artifactId>
-    <version>1.3.2-SNAPSHOT</version>
+    <version>1.3.3-SNAPSHOT</version>
     <packaging>pom</packaging>
     <name>${project.artifactId}</name>
     <url>http://dolphinscheduler.apache.org</url>