Merge remote-tracking branch 'remotes/upstream/1.3.2-release' into 1.3.2-release

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java

@@ -564,7 +564,7 @@ public class ResourcesController extends BaseController {
     @GetMapping(value = "/udf-func/list-paging")
     @ResponseStatus(HttpStatus.OK)
     @ApiException(QUERY_UDF_FUNCTION_LIST_PAGING_ERROR)
-    public Result queryUdfFuncList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
+    public Result<Object> queryUdfFuncListPaging(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
                                    @RequestParam("pageNo") Integer pageNo,
                                    @RequestParam(value = "searchVal", required = false) String searchVal,
                                    @RequestParam("pageSize") Integer pageSize
@@ -581,23 +581,25 @@ public class ResourcesController extends BaseController {
     }
 
     /**
-     * query resource list by type
+     * query udf func list by type
      *
      * @param loginUser login user
      * @param type      resource type
      * @return resource list
      */
-    @ApiOperation(value = "queryResourceList", notes = "QUERY_RESOURCE_LIST_NOTES")
+    @ApiOperation(value = "queryUdfFuncList", notes = "QUERY_UDF_FUNC_LIST_NOTES")
     @ApiImplicitParams({
             @ApiImplicitParam(name = "type", value = "UDF_TYPE", required = true, dataType = "UdfType")
     })
     @GetMapping(value = "/udf-func/list")
     @ResponseStatus(HttpStatus.OK)
     @ApiException(QUERY_DATASOURCE_BY_TYPE_ERROR)
-    public Result queryResourceList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
+    public Result<Object> queryUdfFuncList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
                                     @RequestParam("type") UdfType type) {
-        logger.info("query datasource list, user:{}, type:{}", loginUser.getUserName(), type);
+        String userName = loginUser.getUserName();
-        Map<String, Object> result = udfFuncService.queryResourceList(loginUser, type.ordinal());
+        userName = userName.replaceAll("[\n|\r|\t]", "_");
+        logger.info("query udf func list, user:{}, type:{}", userName, type);
+        Map<String, Object> result = udfFuncService.queryUdfFuncList(loginUser, type.ordinal());
         return returnDataList(result);
     }
 

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java

@@ -38,6 +38,7 @@ import org.apache.dolphinscheduler.common.model.TaskNode;
 import org.apache.dolphinscheduler.common.model.TaskNodeRelation;
 import org.apache.dolphinscheduler.common.process.ProcessDag;
 import org.apache.dolphinscheduler.common.process.Property;
+import org.apache.dolphinscheduler.common.process.ResourceInfo;
 import org.apache.dolphinscheduler.common.task.AbstractParameters;
 import org.apache.dolphinscheduler.common.thread.Stopper;
 import org.apache.dolphinscheduler.common.utils.*;
@@ -176,7 +177,7 @@ public class ProcessDefinitionService extends BaseDAGService {
             String taskParameter = taskNode.getParams();
             AbstractParameters params = TaskParametersUtils.getParameters(taskNode.getType(),taskParameter);
             if (CollectionUtils.isNotEmpty(params.getResourceFilesList())) {
-                Set<Integer> tempSet = params.getResourceFilesList().stream().map(t->t.getId()).collect(Collectors.toSet());
+                Set<Integer> tempSet = params.getResourceFilesList().stream().filter(t->t.getId()!=0).map(ResourceInfo::getId).collect(Collectors.toSet());
                 resourceIds.addAll(tempSet);
             }
         }

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java

@@ -436,14 +436,38 @@ public class ResourcesService extends BaseService {
                 if (CollectionUtils.isNotEmpty(childrenResource)) {
                     String matcherFullName = Matcher.quoteReplacement(fullName);
                     List<Resource> childResourceList = new ArrayList<>();
-                    List<Resource> resourceList = resourcesMapper.listResourceByIds(childrenResource.toArray(new Integer[childrenResource.size()]));
+                    Integer[] childResIdArray = childrenResource.toArray(new Integer[childrenResource.size()]);
+                    List<Resource> resourceList = resourcesMapper.listResourceByIds(childResIdArray);
                     childResourceList = resourceList.stream().map(t -> {
                         t.setFullName(t.getFullName().replaceFirst(originFullName, matcherFullName));
                         t.setUpdateTime(now);
                         return t;
                     }).collect(Collectors.toList());
                     resourcesMapper.batchUpdateResource(childResourceList);
+
+                    if (ResourceType.UDF.equals(resource.getType())) {
+                        List<UdfFunc> udfFuncs = udfFunctionMapper.listUdfByResourceId(childResIdArray);
+                        if (CollectionUtils.isNotEmpty(udfFuncs)) {
+                            udfFuncs = udfFuncs.stream().map(t -> {
+                                t.setResourceName(t.getResourceName().replaceFirst(originFullName, matcherFullName));
+                                t.setUpdateTime(now);
+                                return t;
+                            }).collect(Collectors.toList());
+                            udfFunctionMapper.batchUpdateUdfFunc(udfFuncs);
+                        }
+                    }
                 }
+            } else if (ResourceType.UDF.equals(resource.getType())) {
+                List<UdfFunc> udfFuncs = udfFunctionMapper.listUdfByResourceId(new Integer[]{resourceId});
+                if (CollectionUtils.isNotEmpty(udfFuncs)) {
+                    udfFuncs = udfFuncs.stream().map(t -> {
+                        t.setResourceName(fullName);
+                        t.setUpdateTime(now);
+                        return t;
+                    }).collect(Collectors.toList());
+                    udfFunctionMapper.batchUpdateUdfFunc(udfFuncs);
+                }
+
             }
 
             putMsg(result, Status.SUCCESS);

dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java

@@ -279,15 +279,19 @@ public class UdfFuncService extends BaseService{
     }
 
     /**
-     * query data resource by type
+     * query udf list
      *
      * @param loginUser login user
-     * @param type  resource type
+     * @param type  udf type
-     * @return resource list
+     * @return udf func list
      */
-    public Map<String, Object> queryResourceList(User loginUser, Integer type) {
+    public Map<String, Object> queryUdfFuncList(User loginUser, Integer type) {
         Map<String, Object> result = new HashMap<>(5);
-        List<UdfFunc> udfFuncList = udfFuncMapper.getUdfFuncByType(loginUser.getId(), type);
+        int userId = loginUser.getId();
+        if (isAdmin(loginUser)) {
+            userId = 0;
+        }
+        List<UdfFunc> udfFuncList = udfFuncMapper.getUdfFuncByType(userId, type);
 
         result.put(Constants.DATA_LIST, udfFuncList);
         putMsg(result, Status.SUCCESS);

dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java

@@ -149,9 +149,11 @@ public class UdfFuncServiceTest {
     }
 
     @Test
-    public  void testQueryResourceList(){
+    public  void testQueryUdfFuncList(){
-        Mockito.when(udfFuncMapper.getUdfFuncByType(1, 1)).thenReturn(getList());
+        User user = getLoginUser();
-        Map<String, Object> result = udfFuncService.queryResourceList(getLoginUser(),1);
+        user.setUserType(UserType.GENERAL_USER);
+        Mockito.when(udfFuncMapper.getUdfFuncByType(user.getId(), UdfType.HIVE.ordinal())).thenReturn(getList());
+        Map<String, Object> result = udfFuncService.queryUdfFuncList(user,UdfType.HIVE.ordinal());
         logger.info(result.toString());
         Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS));
         List<UdfFunc> udfFuncList = (List<UdfFunc>) result.get(Constants.DATA_LIST);

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java

@@ -16,9 +16,9 @@
  */
 package org.apache.dolphinscheduler.dao.mapper;
 
-import org.apache.dolphinscheduler.dao.entity.UdfFunc;
 import com.baomidou.mybatisplus.core.mapper.BaseMapper;
 import com.baomidou.mybatisplus.core.metadata.IPage;
+import org.apache.dolphinscheduler.dao.entity.UdfFunc;
 import org.apache.ibatis.annotations.Param;
 
 import java.util.List;
@@ -100,5 +100,12 @@ public interface UdfFuncMapper extends BaseMapper<UdfFunc> {
      */
     List<UdfFunc> listAuthorizedUdfByResourceId(@Param("userId") int userId,@Param("resourceIds") int[] resourceIds);
 
+    /**
+     * batch update udf func
+     * @param udfFuncList  udf list
+     * @return update num
+     */
+    int batchUpdateUdfFunc(@Param("udfFuncList") List<UdfFunc> udfFuncList);
+
 
 }

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java

@@ -117,6 +117,8 @@ public class DolphinSchedulerManager {
                     upgradeDao.upgradeDolphinScheduler(schemaDir);
                     if ("1.3.0".equals(schemaVersion)) {
                         upgradeDao.upgradeDolphinSchedulerWorkerGroup();
+                    } else if ("1.3.2".equals(schemaVersion)) {
+                        upgradeDao.upgradeDolphinSchedulerResourceList();
                     }
                     version = schemaVersion;
                 }

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ResourceDao.java

@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.dao.upgrade;
+
+import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * resource dao
+ */
+public class ResourceDao {
+    public static final Logger logger = LoggerFactory.getLogger(ProcessDefinitionDao.class);
+
+    /**
+     * list all resources
+     * @param conn connection
+     * @return map that key is full_name and value is id
+     */
+    Map<String,Integer> listAllResources(Connection conn){
+        Map<String,Integer> resourceMap = new HashMap<>();
+
+        String sql = String.format("SELECT id,full_name FROM t_ds_resources");
+        ResultSet rs = null;
+        PreparedStatement pstmt = null;
+        try {
+            pstmt = conn.prepareStatement(sql);
+            rs = pstmt.executeQuery();
+
+            while (rs.next()){
+                Integer id = rs.getInt(1);
+                String fullName = rs.getString(2);
+                resourceMap.put(fullName,id);
+            }
+
+        } catch (Exception e) {
+            logger.error(e.getMessage(),e);
+            throw new RuntimeException("sql: " + sql, e);
+        } finally {
+            ConnectionUtils.releaseResource(rs, pstmt, conn);
+        }
+
+        return resourceMap;
+    }
+
+}

dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java

@@ -19,10 +19,8 @@ package org.apache.dolphinscheduler.dao.upgrade;
 import com.alibaba.fastjson.JSONArray;
 import com.alibaba.fastjson.JSONObject;
 import org.apache.dolphinscheduler.common.enums.DbType;
-import org.apache.dolphinscheduler.common.utils.ConnectionUtils;
+import org.apache.dolphinscheduler.common.process.ResourceInfo;
-import org.apache.dolphinscheduler.common.utils.SchemaUtils;
+import org.apache.dolphinscheduler.common.utils.*;
-import org.apache.dolphinscheduler.common.utils.ScriptRunner;
-import org.apache.dolphinscheduler.common.utils.StringUtils;
 import org.apache.dolphinscheduler.dao.AbstractBaseDao;
 import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory;
 import org.slf4j.Logger;
@@ -36,7 +34,9 @@ import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.text.MessageFormat;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 public abstract class UpgradeDao extends AbstractBaseDao {
 
@@ -270,6 +270,15 @@ public abstract class UpgradeDao extends AbstractBaseDao {
     public void upgradeDolphinSchedulerWorkerGroup() {
         updateProcessDefinitionJsonWorkerGroup();
     }
+
+    /**
+     * upgrade DolphinScheduler resource list
+     * ds-1.3.2 modify the resource list for process definition json
+     */
+    public void upgradeDolphinSchedulerResourceList() {
+        updateProcessDefinitionJsonResourceList();
+    }
+
     /**
      * updateProcessDefinitionJsonWorkerGroup
      */
@@ -288,6 +297,7 @@ public abstract class UpgradeDao extends AbstractBaseDao {
                 for (int i = 0 ;i < tasks.size() ; i++){
                     JSONObject task = tasks.getJSONObject(i);
                     Integer workerGroupId = task.getInteger("workerGroupId");
+                    if (workerGroupId != null) {
                         if (workerGroupId == -1) {
                             task.put("workerGroup", "default");
                         } else {
@@ -295,6 +305,8 @@ public abstract class UpgradeDao extends AbstractBaseDao {
                         }
                     }
 
+                }
+
                 jsonObject.remove(jsonObject.getString("tasks"));
 
                 jsonObject.put("tasks",tasks);
@@ -310,6 +322,58 @@ public abstract class UpgradeDao extends AbstractBaseDao {
 
     }
 
+    /**
+     * updateProcessDefinitionJsonResourceList
+     */
+    protected void updateProcessDefinitionJsonResourceList(){
+        ResourceDao resourceDao = new ResourceDao();
+        ProcessDefinitionDao processDefinitionDao = new ProcessDefinitionDao();
+        Map<Integer,String> replaceProcessDefinitionMap = new HashMap<>();
+        try {
+            Map<String,Integer> resourcesMap = resourceDao.listAllResources(dataSource.getConnection());
+            Map<Integer,String> processDefinitionJsonMap = processDefinitionDao.queryAllProcessDefinition(dataSource.getConnection());
+
+            for (Map.Entry<Integer,String> entry : processDefinitionJsonMap.entrySet()){
+                JSONObject jsonObject = JSONObject.parseObject(entry.getValue());
+                JSONArray tasks = JSONArray.parseArray(jsonObject.getString("tasks"));
+
+                for (int i = 0 ;i < tasks.size() ; i++){
+                    JSONObject task = tasks.getJSONObject(i);
+                    JSONObject param = (JSONObject) task.get("params");
+                    if (param != null) {
+
+                        List<ResourceInfo> resourceList = JSONUtils.toList(param.getString("resourceList"), ResourceInfo.class);
+
+                        if (CollectionUtils.isNotEmpty(resourceList)) {
+                            List<ResourceInfo> newResourceList = resourceList.stream().map(resInfo -> {
+                                String fullName = resInfo.getRes().startsWith("/") ? resInfo.getRes() : String.format("/%s",resInfo.getRes());
+                                if (resInfo.getId() == 0 && resourcesMap.containsKey(fullName)) {
+                                    resInfo.setId(resourcesMap.get(fullName));
+                                }
+                                return resInfo;
+                            }).collect(Collectors.toList());
+                            param.put("resourceList",JSONArray.parse(JSONObject.toJSONString(newResourceList)));
+                        }
+                    }
+                    task.put("params",param);
+
+                }
+
+                jsonObject.remove(jsonObject.getString("tasks"));
+
+                jsonObject.put("tasks",tasks);
+
+                replaceProcessDefinitionMap.put(entry.getKey(),jsonObject.toJSONString());
+            }
+            if (replaceProcessDefinitionMap.size() > 0){
+                processDefinitionDao.updateProcessDefinitionJson(dataSource.getConnection(),replaceProcessDefinitionMap);
+            }
+        }catch (Exception e){
+            logger.error("update process definition json resource list error",e);
+        }
+
+    }
+
     /**
      * upgradeDolphinScheduler DML
      * @param schemaDir schemaDir

dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml

@@ -111,4 +111,17 @@
             </foreach>
         </if>
     </select>
+
+    <update id="batchUpdateUdfFunc" parameterType="java.util.List">
+        <foreach collection="udfFuncList" item="udf" index="index" open="" close="" separator =";">
+            update t_ds_udfs
+            <set>
+                resource_name=#{udf.resourceName},
+                update_time=#{udf.updateTime}
+            </set>
+            <where>
+                id=#{udf.id}
+            </where>
+        </foreach>
+    </update>
 </mapper>

dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapperTest.java

@@ -33,6 +33,7 @@ import org.springframework.test.annotation.Rollback;
 import org.springframework.test.context.junit4.SpringRunner;
 import org.springframework.transaction.annotation.Transactional;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
 import java.util.List;
@@ -303,4 +304,16 @@ public class UdfFuncMapperTest {
         authorizedUdfFunc = udfFuncMapper.listAuthorizedUdfFunc(generalUser1.getId(), udfFuncIds);
         Assert.assertTrue(authorizedUdfFunc.stream().map(t -> t.getId()).collect(toList()).containsAll(Arrays.asList(udfFuncIds)));
     }
+
+    @Test
+    public void batchUpdateUdfFuncTest(){
+        //create general user
+        User generalUser1 = createGeneralUser("user1");
+        UdfFunc udfFunc = insertOne(generalUser1);
+        udfFunc.setResourceName("/updateTest");
+        List<UdfFunc> udfFuncList = new ArrayList<>();
+        udfFuncList.add(udfFunc);
+        Assert.assertTrue(udfFuncMapper.batchUpdateUdfFunc(udfFuncList)>0);
+
+    }
 }

dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java

@@ -667,12 +667,10 @@ public class MasterExecThread implements Runnable {
         List<String> depNameList = taskNode.getDepList();
         for(String depsNode : depNameList ){
             if(!dag.containsNode(depsNode)
+                    || skipTaskNodeList.containsKey(depsNode)
                     || forbiddenTaskList.containsKey(depsNode)){
                 continue;
             }
-            if(skipTaskNodeList.containsKey(depsNode)){
-                return DependResult.FAILED;
-            }
             // all the dependencies must be completed
             if(!completeTaskList.containsKey(depsNode)){
                 return DependResult.WAITING;

dolphinscheduler-ui/build/webpack.config.prod.js

@@ -51,12 +51,7 @@ const config = merge.smart(baseConfig, {
     minimizer: [
       new TerserPlugin({
         terserOptions: {
-          compress: {
+          compress: {}
-            warnings: false,
-            drop_console: true,
-            drop_debugger: true,
-            pure_funcs: ['console.log']
-          }
         },
         cache: true,
         parallel: true,

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js

@@ -118,8 +118,8 @@ Dag.prototype.toolbarEvent = function ({ item, code, is }) {
 Dag.prototype.backfill = function (arg) {
   if (arg) {
     let locationsValue = store.state.dag.locations
-    const locationsValue1 = store.state.dag.locations
+    const locationsValue1 = _.cloneDeep(store.state.dag.locations)
-    const locationsValue2 = store.state.dag.locations
+    const locationsValue2 = _.cloneDeep(store.state.dag.locations)
     const arr = []
     for (const i in locationsValue1) {
       const objs = {}

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue

@@ -522,9 +522,9 @@
         } else {
           rearList = []
         }
-        let targetarr = $(`#${id}`).attr('data-targetarr')
+        let target = $(`#${id}`).attr('data-targetarr')
-        if (targetarr) {
+        if (target) {
-          let nodearr = targetarr.split(',')
+          let nodearr = target.split(',')
           nodearr.forEach(v => {
             let nodeobj = {}
             nodeobj.value = $(`#${v}`).find('.name-p').text()
@@ -602,9 +602,7 @@
     },
     watch: {
       'tasks': {
-        deep: true,
         handler (o) {
-
           // Edit state does not allow deletion of node a...
           this.setIsEditDag(true)
         }

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue

@@ -375,10 +375,8 @@
           resourceIdArr = isResourceId.map(item=>{
             return item.id
           })
-          Array.prototype.diff = function(a) {
+          let diffSet
-            return this.filter(function(i) {return a.indexOf(i) < 0;});
+          diffSet = _.xorWith(this.resourceList, resourceIdArr, _.isEqual)
-          };
-          let diffSet = this.resourceList.diff(resourceIdArr);
           let optionsCmp = []
           if(diffSet.length>0) {
             diffSet.forEach(item=>{
@@ -477,7 +475,7 @@
         // Non-null objects represent backfill
         if (!_.isEmpty(o)) {
           this.mainClass = o.params.mainClass || ''
-          if(o.params.mainJar.res) {
+          if(!o.params.mainJar.id) {
             this.marjarId(o.params.mainJar.res)
           } else if(o.params.mainJar.res=='') {
             this.mainJar = ''

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue

@@ -216,10 +216,8 @@
           resourceIdArr = isResourceId.map(item=>{
             return item.id
           })
-          Array.prototype.diff = function(a) {
+          let diffSet
-            return this.filter(function(i) {return a.indexOf(i) < 0;});
+          diffSet = _.xorWith(this.resourceList, resourceIdArr, _.isEqual)
-          };
-          let diffSet = this.resourceList.diff(resourceIdArr);
           let optionsCmp = []
           if(diffSet.length>0) {
             diffSet.forEach(item=>{
@@ -356,7 +354,7 @@
         // Non-null objects represent backfill
         if (!_.isEmpty(o)) {
           this.mainClass = o.params.mainClass || ''
-          if(o.params.mainJar.res) {
+          if(!o.params.mainJar.id) {
             this.marjarId(o.params.mainJar.res)
           } else if(o.params.mainJar.res=='') {
             this.mainJar = ''

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue

@@ -208,10 +208,8 @@
           resourceIdArr = isResourceId.map(item=>{
             return item.id
           })
-          Array.prototype.diff = function(a) {
+          let diffSet
-            return this.filter(function(i) {return a.indexOf(i) < 0;});
+          diffSet = _.xorWith(this.resourceList, resourceIdArr, _.isEqual)
-          };
-          let diffSet = this.resourceList.diff(resourceIdArr);
           let optionsCmp = []
           if(diffSet.length>0) {
             diffSet.forEach(item=>{

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue

@@ -20,11 +20,7 @@
       <div slot="text">{{$t('Script')}}</div>
       <div slot="content">
         <div class="from-mirror">
-          <textarea
+          <textarea id="code-shell-mirror" name="code-shell-mirror" style="opacity: 0"></textarea>
-            id="code-shell-mirror"
-            name="code-shell-mirror"
-            style="opacity: 0">
-          </textarea>
           <a class="ans-modal-box-max">
             <em class="ans-icon-max" @click="setEditorVal"></em>
           </a>
@@ -250,10 +246,8 @@
           resourceIdArr = isResourceId.map(item=>{
             return item.id
           })
-          Array.prototype.diff = function(a) {
+          let diffSet
-            return this.filter(function(i) {return a.indexOf(i) < 0;});
+          diffSet = _.xorWith(this.resourceList, resourceIdArr, _.isEqual)
-          };
-          let diffSet = this.resourceList.diff(resourceIdArr);
           let optionsCmp = []
           if(diffSet.length>0) {
             diffSet.forEach(item=>{
@@ -364,9 +358,12 @@
       }
     },
     mounted () {
+      // Added delay loading in script input box
+      this.$nextTick(() => {
         setTimeout(() => {
           this._handlerEditor()
-      }, 200)
+        }, 350)
+      })
     },
     destroyed () {
       if (editor) {

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue

@@ -335,10 +335,8 @@
           resourceIdArr = isResourceId.map(item=>{
             return item.id
           })
-          Array.prototype.diff = function(a) {
+          let diffSet
-            return this.filter(function(i) {return a.indexOf(i) < 0;});
+          diffSet = _.xorWith(this.resourceList, resourceIdArr, _.isEqual)
-          };
-          let diffSet = this.resourceList.diff(resourceIdArr);
           let optionsCmp = []
           if(diffSet.length>0) {
             diffSet.forEach(item=>{
@@ -525,7 +523,7 @@
         // Non-null objects represent backfill
         if (!_.isEmpty(o)) {
           this.mainClass = o.params.mainClass || ''
-          if(o.params.mainJar.res) {
+          if(!o.params.mainJar.id) {
             this.marjarId(o.params.mainJar.res)
           } else if(o.params.mainJar.res=='') {
             this.mainJar = ''

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js

@@ -36,7 +36,7 @@ import {
 } from './util'
 import mStart from '@/conf/home/pages/projects/pages/definition/pages/list/_source/start'
 
-const JSP = function () {
+let JSP = function () {
   this.dag = {}
   this.selectedElement = {}
 
@@ -687,37 +687,55 @@ JSP.prototype.saveStore = function () {
 
 JSP.prototype.handleEvent = function () {
   this.JspInstance.bind('beforeDrop', function (info) {
-    const sourceId = info.sourceId// 出
+    console.log(info)
-    const targetId = info.targetId// 入
+    const rtTargetArr = (id) => {
+      let ids = $(`#${id}`).attr('data-targetarr')
+      return ids ? ids.split(',') : []
+    }
+    let sourceId = info['sourceId']// 出
+    let targetId = info['targetId']// 入
+    console.log(sourceId,targetId)
+    let rtTargetArrs = rtTargetArr(targetId)
+    let rtSouceArrs = rtTargetArr(sourceId)
+    /**
+    * When connecting, connection is prohibited when the sourceId and target nodes are empty
+    */
+    if(!sourceId && !targetId) {
+      Vue.$message.warning(`${i18n.$t('This canvas is abnormal and the node connection cannot be made. Please save or exit the current workflow')}`)
+      return false
+    }
     /**
      * Recursive search for nodes
      */
     let recursiveVal
     const recursiveTargetarr = (arr, targetId) => {
-      for (const i in arr) {
+        for (let i in arr) {
           if (arr[i] === targetId) {
             recursiveVal = targetId
           } else {
-          recursiveTargetarr(rtTargetarrArr(arr[i]), targetId)
+            let targetArr = rtTargetArr(arr[i])
+            recursiveTargetarr(targetArr, targetId)
           }
         }
       return recursiveVal
     }
     
     // Connection to connected nodes is not allowed
-    if (_.findIndex(rtTargetarrArr(targetId), v => v === sourceId) !== -1) {
+    if (_.findIndex(rtTargetArrs, v => v === sourceId) !== -1) {
+      console.log(rtTargetArrs,'not allowed')
       return false
     }
     
     // Recursive form to find if the target Targetarr has a sourceId
-    if (recursiveTargetarr(rtTargetarrArr(sourceId), targetId)) {
+    if (recursiveTargetarr(rtSouceArrs, targetId)) {
+      console.log('has a sourceId')
       return false
     }
-
+    if ($(`#${sourceId}`).attr('data-tasks-type') === 'CONDITIONS' && parseInt($(`#${sourceId}`).attr('data-nodenumber')) === 2) {
-    if ($(`#${sourceId}`).attr('data-tasks-type') === 'CONDITIONS' && $(`#${sourceId}`).attr('data-nodenumber') === 2) {
       return false
     } else {
-      $(`#${sourceId}`).attr('data-nodenumber', Number($(`#${sourceId}`).attr('data-nodenumber')) + 1)
+      console.log('data-nodenumber')
+      $(`#${sourceId}`).attr('data-nodenumber', parseInt($(`#${sourceId}`).attr('data-nodenumber')) + 1)
     }
 
     // Storage node dependency information

dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/util.js

@@ -32,8 +32,9 @@ const rtTargetarrArr = (id) => {
  */
 const saveTargetarr = (valId, domId) => {
   let $target = $(`#${domId}`)
-  let targetStr = $target.attr('data-targetarr') ? $target.attr('data-targetarr') + `,${valId}` : `${valId}`
+  let targetStr = $target.attr('data-targetarr') ? $target.attr('data-targetarr') + `,${valId}` : `${valId}`;
   $target.attr('data-targetarr', targetStr)
+  console.log('Store node id to targetarr',targetStr)
 }
 
 const rtBantpl = () => {

dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/subUdfDirectory/_source/list.vue

@@ -43,7 +43,7 @@
           <th scope="col" width="140">
             <span>{{$t('Update Time')}}</span>
           </th>
-          <th scope="col" width="110">
+          <th scope="col" width="130">
             <span>{{$t('Operation')}}</span>
           </th>
         </tr>
@@ -76,6 +76,16 @@
             <span v-else>-</span>
           </td>
           <td>
+            <x-button
+                    type="info"
+                    shape="circle"
+                    size="xsmall"
+                    data-toggle="tooltip"
+                    :title="$t('ReUpload File')"
+                    :disabled="item.directory? true: false"
+                    @click="_reUpload(item)"
+                    icon="ans-icon-upload">
+            </x-button>
             <x-button
                     type="info"
                     shape="circle"
@@ -127,6 +137,7 @@
   import mRename from './rename'
   import { downloadFile } from '@/module/download'
   import { bytesToSize } from '@/module/util/util'
+  import { findComponentDownward } from '@/module/util'
   import localStore from '@/module/util/localStorage'
 
   export default {
@@ -155,6 +166,12 @@
           this.$router.push({ path: `/resource/udf/subUdfDirectory/${item.id}` })
         }
       },
+      /**
+       * File Upload
+       */
+      _reUpload (item) {
+        findComponentDownward(this.$root, 'roof-nav')._fileChildReUpload('UDF', item, this.$route.params.id)
+      },
       _rtSize (val) {
         return bytesToSize(parseInt(val))
       },

dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js

@@ -19,6 +19,25 @@ import _ from 'lodash'
 import io from '@/module/io'
 import { tasksState } from '@/conf/home/pages/dag/_source/config'
 
+// delete 'definitionList' from tasks
+const deleteDefinitionList = (tasks) => {
+  const newTasks = [];
+  tasks.forEach(item => {
+    const newItem = Object.assign({}, item);
+    if(newItem.dependence && newItem.dependence.dependTaskList) {
+      newItem.dependence.dependTaskList.forEach(dependTaskItem => {
+        if (dependTaskItem.dependItemList) {
+          dependTaskItem.dependItemList.forEach(dependItem => {
+            Reflect.deleteProperty(dependItem, 'definitionList');
+          })
+        }
+      })
+    }
+    newTasks.push(newItem);
+  });
+  return newTasks;
+}
+
 export default {
   /**
    *  Task status acquisition
@@ -193,7 +212,7 @@ export default {
     return new Promise((resolve, reject) => {
       const data = {
         globalParams: state.globalParams,
-        tasks: state.tasks,
+        tasks: deleteDefinitionList(state.tasks),
         tenantId: state.tenantId,
         timeout: state.timeout
       }
@@ -217,7 +236,7 @@ export default {
     return new Promise((resolve, reject) => {
       const data = {
         globalParams: state.globalParams,
-        tasks: state.tasks,
+        tasks: deleteDefinitionList(state.tasks),
         tenantId: state.tenantId,
         timeout: state.timeout
       }

dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js

@@ -600,5 +600,6 @@ export default {
   'Directory detail': 'Directory detail',
   'Connection name': 'Connection name',
   'Current connection settings': 'Current connection settings',
-  'Please save the DAG before formatting': 'Please save the DAG before formatting'
+  'Please save the DAG before formatting': 'Please save the DAG before formatting',
+  'This canvas is abnormal and the node connection cannot be made. Please save or exit the current workflow': 'This canvas is abnormal and the node connection cannot be made. Please save or exit the current workflow'
 }

dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js

@@ -606,5 +606,6 @@ export default {
   'Directory detail': '查看目录详情',
   'Connection name': '连线名',
   'Current connection settings': '当前连线设置',
-  'Please save the DAG before formatting': '格式化前请先保存DAG'
+  'Please save the DAG before formatting': '格式化前请先保存DAG',
+  'This canvas is abnormal and the node connection cannot be made. Please save or exit the current workflow': '此画布异常，无法进行节点连线，请保存或退出当前工作流'
 }

sql/upgrade/1.2.0_schema/mysql/dolphinscheduler_dml.sql

@@ -23,3 +23,4 @@ UPDATE QRTZ_JOB_DETAILS SET SCHED_NAME='DolphinScheduler' WHERE SCHED_NAME='Easy
 UPDATE QRTZ_JOB_DETAILS SET JOB_CLASS_NAME='org.apache.dolphinscheduler.dao.quartz.ProcessScheduleJob' WHERE JOB_CLASS_NAME='cn.escheduler.server.quartz.ProcessScheduleJob';
 UPDATE QRTZ_LOCKS SET SCHED_NAME='DolphinScheduler' WHERE SCHED_NAME='EasyScheduler';
 UPDATE QRTZ_SCHEDULER_STATE SET SCHED_NAME='DolphinScheduler' WHERE SCHED_NAME='EasyScheduler';
+UPDATE t_ds_user SET phone = '' WHERE phone = 'xx';

sql/upgrade/1.3.0_schema/mysql/dolphinscheduler_dml.sql

@@ -24,3 +24,4 @@ UPDATE t_ds_task_instance instance SET `worker_group`=IFNULL((SELECT name from t
 UPDATE t_ds_schedules schedule SET `worker_group`=IFNULL((SELECT name from t_ds_worker_group WHERE schedule.worker_group=CONCAT(id,'')),'default');
 UPDATE t_ds_command command SET `worker_group`=IFNULL((SELECT name from t_ds_worker_group WHERE command.worker_group=CONCAT(id,'')),'default');
 UPDATE t_ds_error_command command SET `worker_group`=IFNULL((SELECT name from t_ds_worker_group WHERE command.worker_group=CONCAT(id,'')),'default');
+UPDATE t_ds_user SET phone = '' WHERE phone = 'xx';

sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_dml.sql

@@ -22,3 +22,4 @@ UPDATE t_ds_task_instance instance SET worker_group=COALESCE((SELECT name from t
 UPDATE t_ds_schedules schedule SET worker_group=COALESCE((SELECT name from t_ds_worker_group WHERE schedule.worker_group=CONCAT(id,'')),'default');
 UPDATE t_ds_command command SET worker_group=COALESCE((SELECT name from t_ds_worker_group WHERE command.worker_group=CONCAT(id,'')),'default');
 UPDATE t_ds_error_command command SET worker_group=COALESCE((SELECT name from t_ds_worker_group WHERE command.worker_group=CONCAT(id,'')),'default');
+UPDATE t_ds_user SET phone = '' WHERE phone = 'xx';

sql/upgrade/1.3.2_schema/mysql/dolphinscheduler_ddl.sql

@@ -0,0 +1,16 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/

sql/upgrade/1.3.2_schema/mysql/dolphinscheduler_dml.sql

@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+SET sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));
+SET FOREIGN_KEY_CHECKS=0;
+UPDATE t_ds_user SET phone = '' WHERE phone = 'xx';

sql/upgrade/1.3.2_schema/postgresql/dolphinscheduler_ddl.sql

@@ -0,0 +1,16 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/

sql/upgrade/1.3.2_schema/postgresql/dolphinscheduler_dml.sql

@@ -0,0 +1,17 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+UPDATE t_ds_user SET phone = '' WHERE phone = 'xx';