diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java index 2af8ccd767..2f2cb9ca6f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java @@ -564,7 +564,7 @@ public class ResourcesController extends BaseController { @GetMapping(value = "/udf-func/list-paging") @ResponseStatus(HttpStatus.OK) @ApiException(QUERY_UDF_FUNCTION_LIST_PAGING_ERROR) - public Result queryUdfFuncList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + public Result queryUdfFuncListPaging(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam("pageNo") Integer pageNo, @RequestParam(value = "searchVal", required = false) String searchVal, @RequestParam("pageSize") Integer pageSize @@ -581,23 +581,25 @@ public class ResourcesController extends BaseController { } /** - * query resource list by type + * query udf func list by type * * @param loginUser login user * @param type resource type * @return resource list */ - @ApiOperation(value = "queryResourceList", notes = "QUERY_RESOURCE_LIST_NOTES") + @ApiOperation(value = "queryUdfFuncList", notes = "QUERY_UDF_FUNC_LIST_NOTES") @ApiImplicitParams({ @ApiImplicitParam(name = "type", value = "UDF_TYPE", required = true, dataType = "UdfType") }) @GetMapping(value = "/udf-func/list") @ResponseStatus(HttpStatus.OK) @ApiException(QUERY_DATASOURCE_BY_TYPE_ERROR) - public Result queryResourceList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, + public Result queryUdfFuncList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser, @RequestParam("type") UdfType type) { - logger.info("query datasource list, user:{}, type:{}", loginUser.getUserName(), type); - Map result = udfFuncService.queryResourceList(loginUser, type.ordinal()); + String userName = loginUser.getUserName(); + userName = userName.replaceAll("[\n|\r|\t]", "_"); + logger.info("query udf func list, user:{}, type:{}", userName, type); + Map result = udfFuncService.queryUdfFuncList(loginUser, type.ordinal()); return returnDataList(result); } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java index 881e2fed1a..19b38ea46f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ProcessDefinitionService.java @@ -38,6 +38,7 @@ import org.apache.dolphinscheduler.common.model.TaskNode; import org.apache.dolphinscheduler.common.model.TaskNodeRelation; import org.apache.dolphinscheduler.common.process.ProcessDag; import org.apache.dolphinscheduler.common.process.Property; +import org.apache.dolphinscheduler.common.process.ResourceInfo; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.utils.*; @@ -176,7 +177,7 @@ public class ProcessDefinitionService extends BaseDAGService { String taskParameter = taskNode.getParams(); AbstractParameters params = TaskParametersUtils.getParameters(taskNode.getType(),taskParameter); if (CollectionUtils.isNotEmpty(params.getResourceFilesList())) { - Set tempSet = params.getResourceFilesList().stream().map(t->t.getId()).collect(Collectors.toSet()); + Set tempSet = params.getResourceFilesList().stream().filter(t->t.getId()!=0).map(ResourceInfo::getId).collect(Collectors.toSet()); resourceIds.addAll(tempSet); } } diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java index e0ee7117c4..e1a9c85aa1 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java @@ -436,14 +436,38 @@ public class ResourcesService extends BaseService { if (CollectionUtils.isNotEmpty(childrenResource)) { String matcherFullName = Matcher.quoteReplacement(fullName); List childResourceList = new ArrayList<>(); - List resourceList = resourcesMapper.listResourceByIds(childrenResource.toArray(new Integer[childrenResource.size()])); + Integer[] childResIdArray = childrenResource.toArray(new Integer[childrenResource.size()]); + List resourceList = resourcesMapper.listResourceByIds(childResIdArray); childResourceList = resourceList.stream().map(t -> { t.setFullName(t.getFullName().replaceFirst(originFullName, matcherFullName)); t.setUpdateTime(now); return t; }).collect(Collectors.toList()); resourcesMapper.batchUpdateResource(childResourceList); + + if (ResourceType.UDF.equals(resource.getType())) { + List udfFuncs = udfFunctionMapper.listUdfByResourceId(childResIdArray); + if (CollectionUtils.isNotEmpty(udfFuncs)) { + udfFuncs = udfFuncs.stream().map(t -> { + t.setResourceName(t.getResourceName().replaceFirst(originFullName, matcherFullName)); + t.setUpdateTime(now); + return t; + }).collect(Collectors.toList()); + udfFunctionMapper.batchUpdateUdfFunc(udfFuncs); + } + } } + } else if (ResourceType.UDF.equals(resource.getType())) { + List udfFuncs = udfFunctionMapper.listUdfByResourceId(new Integer[]{resourceId}); + if (CollectionUtils.isNotEmpty(udfFuncs)) { + udfFuncs = udfFuncs.stream().map(t -> { + t.setResourceName(fullName); + t.setUpdateTime(now); + return t; + }).collect(Collectors.toList()); + udfFunctionMapper.batchUpdateUdfFunc(udfFuncs); + } + } putMsg(result, Status.SUCCESS); diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java index 8a0bf748bb..4d4aec738f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java @@ -279,15 +279,19 @@ public class UdfFuncService extends BaseService{ } /** - * query data resource by type + * query udf list * * @param loginUser login user - * @param type resource type - * @return resource list + * @param type udf type + * @return udf func list */ - public Map queryResourceList(User loginUser, Integer type) { + public Map queryUdfFuncList(User loginUser, Integer type) { Map result = new HashMap<>(5); - List udfFuncList = udfFuncMapper.getUdfFuncByType(loginUser.getId(), type); + int userId = loginUser.getId(); + if (isAdmin(loginUser)) { + userId = 0; + } + List udfFuncList = udfFuncMapper.getUdfFuncByType(userId, type); result.put(Constants.DATA_LIST, udfFuncList); putMsg(result, Status.SUCCESS); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java index 9ec24bbb50..884e9b6b36 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java @@ -149,9 +149,11 @@ public class UdfFuncServiceTest { } @Test - public void testQueryResourceList(){ - Mockito.when(udfFuncMapper.getUdfFuncByType(1, 1)).thenReturn(getList()); - Map result = udfFuncService.queryResourceList(getLoginUser(),1); + public void testQueryUdfFuncList(){ + User user = getLoginUser(); + user.setUserType(UserType.GENERAL_USER); + Mockito.when(udfFuncMapper.getUdfFuncByType(user.getId(), UdfType.HIVE.ordinal())).thenReturn(getList()); + Map result = udfFuncService.queryUdfFuncList(user,UdfType.HIVE.ordinal()); logger.info(result.toString()); Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); List udfFuncList = (List) result.get(Constants.DATA_LIST); diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java index a2ce6b29b8..db5e6a454d 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java @@ -16,9 +16,9 @@ */ package org.apache.dolphinscheduler.dao.mapper; -import org.apache.dolphinscheduler.dao.entity.UdfFunc; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import com.baomidou.mybatisplus.core.metadata.IPage; +import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.ibatis.annotations.Param; import java.util.List; @@ -100,5 +100,12 @@ public interface UdfFuncMapper extends BaseMapper { */ List listAuthorizedUdfByResourceId(@Param("userId") int userId,@Param("resourceIds") int[] resourceIds); + /** + * batch update udf func + * @param udfFuncList udf list + * @return update num + */ + int batchUpdateUdfFunc(@Param("udfFuncList") List udfFuncList); + } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java index 8d1d862640..b2daae28cb 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/DolphinSchedulerManager.java @@ -117,6 +117,8 @@ public class DolphinSchedulerManager { upgradeDao.upgradeDolphinScheduler(schemaDir); if ("1.3.0".equals(schemaVersion)) { upgradeDao.upgradeDolphinSchedulerWorkerGroup(); + } else if ("1.3.2".equals(schemaVersion)) { + upgradeDao.upgradeDolphinSchedulerResourceList(); } version = schemaVersion; } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ResourceDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ResourceDao.java new file mode 100644 index 0000000000..b47971e1f0 --- /dev/null +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ResourceDao.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.dao.upgrade; + +import org.apache.dolphinscheduler.common.utils.ConnectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.HashMap; +import java.util.Map; + +/** + * resource dao + */ +public class ResourceDao { + public static final Logger logger = LoggerFactory.getLogger(ProcessDefinitionDao.class); + + /** + * list all resources + * @param conn connection + * @return map that key is full_name and value is id + */ + Map listAllResources(Connection conn){ + Map resourceMap = new HashMap<>(); + + String sql = String.format("SELECT id,full_name FROM t_ds_resources"); + ResultSet rs = null; + PreparedStatement pstmt = null; + try { + pstmt = conn.prepareStatement(sql); + rs = pstmt.executeQuery(); + + while (rs.next()){ + Integer id = rs.getInt(1); + String fullName = rs.getString(2); + resourceMap.put(fullName,id); + } + + } catch (Exception e) { + logger.error(e.getMessage(),e); + throw new RuntimeException("sql: " + sql, e); + } finally { + ConnectionUtils.releaseResource(rs, pstmt, conn); + } + + return resourceMap; + } + +} diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java index f9458a82bf..3d35b68aac 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java @@ -19,10 +19,8 @@ package org.apache.dolphinscheduler.dao.upgrade; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; import org.apache.dolphinscheduler.common.enums.DbType; -import org.apache.dolphinscheduler.common.utils.ConnectionUtils; -import org.apache.dolphinscheduler.common.utils.SchemaUtils; -import org.apache.dolphinscheduler.common.utils.ScriptRunner; -import org.apache.dolphinscheduler.common.utils.StringUtils; +import org.apache.dolphinscheduler.common.process.ResourceInfo; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.dao.AbstractBaseDao; import org.apache.dolphinscheduler.dao.datasource.ConnectionFactory; import org.slf4j.Logger; @@ -36,7 +34,9 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.text.MessageFormat; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; public abstract class UpgradeDao extends AbstractBaseDao { @@ -270,6 +270,15 @@ public abstract class UpgradeDao extends AbstractBaseDao { public void upgradeDolphinSchedulerWorkerGroup() { updateProcessDefinitionJsonWorkerGroup(); } + + /** + * upgrade DolphinScheduler resource list + * ds-1.3.2 modify the resource list for process definition json + */ + public void upgradeDolphinSchedulerResourceList() { + updateProcessDefinitionJsonResourceList(); + } + /** * updateProcessDefinitionJsonWorkerGroup */ @@ -288,11 +297,14 @@ public abstract class UpgradeDao extends AbstractBaseDao { for (int i = 0 ;i < tasks.size() ; i++){ JSONObject task = tasks.getJSONObject(i); Integer workerGroupId = task.getInteger("workerGroupId"); - if (workerGroupId == -1) { - task.put("workerGroup", "default"); - }else { - task.put("workerGroup", oldWorkerGroupMap.get(workerGroupId)); + if (workerGroupId != null) { + if (workerGroupId == -1) { + task.put("workerGroup", "default"); + } else { + task.put("workerGroup", oldWorkerGroupMap.get(workerGroupId)); + } } + } jsonObject.remove(jsonObject.getString("tasks")); @@ -310,6 +322,58 @@ public abstract class UpgradeDao extends AbstractBaseDao { } + /** + * updateProcessDefinitionJsonResourceList + */ + protected void updateProcessDefinitionJsonResourceList(){ + ResourceDao resourceDao = new ResourceDao(); + ProcessDefinitionDao processDefinitionDao = new ProcessDefinitionDao(); + Map replaceProcessDefinitionMap = new HashMap<>(); + try { + Map resourcesMap = resourceDao.listAllResources(dataSource.getConnection()); + Map processDefinitionJsonMap = processDefinitionDao.queryAllProcessDefinition(dataSource.getConnection()); + + for (Map.Entry entry : processDefinitionJsonMap.entrySet()){ + JSONObject jsonObject = JSONObject.parseObject(entry.getValue()); + JSONArray tasks = JSONArray.parseArray(jsonObject.getString("tasks")); + + for (int i = 0 ;i < tasks.size() ; i++){ + JSONObject task = tasks.getJSONObject(i); + JSONObject param = (JSONObject) task.get("params"); + if (param != null) { + + List resourceList = JSONUtils.toList(param.getString("resourceList"), ResourceInfo.class); + + if (CollectionUtils.isNotEmpty(resourceList)) { + List newResourceList = resourceList.stream().map(resInfo -> { + String fullName = resInfo.getRes().startsWith("/") ? resInfo.getRes() : String.format("/%s",resInfo.getRes()); + if (resInfo.getId() == 0 && resourcesMap.containsKey(fullName)) { + resInfo.setId(resourcesMap.get(fullName)); + } + return resInfo; + }).collect(Collectors.toList()); + param.put("resourceList",JSONArray.parse(JSONObject.toJSONString(newResourceList))); + } + } + task.put("params",param); + + } + + jsonObject.remove(jsonObject.getString("tasks")); + + jsonObject.put("tasks",tasks); + + replaceProcessDefinitionMap.put(entry.getKey(),jsonObject.toJSONString()); + } + if (replaceProcessDefinitionMap.size() > 0){ + processDefinitionDao.updateProcessDefinitionJson(dataSource.getConnection(),replaceProcessDefinitionMap); + } + }catch (Exception e){ + logger.error("update process definition json resource list error",e); + } + + } + /** * upgradeDolphinScheduler DML * @param schemaDir schemaDir diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml index e38d1637d6..18de6db620 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml @@ -111,4 +111,17 @@ + + + + update t_ds_udfs + + resource_name=#{udf.resourceName}, + update_time=#{udf.updateTime} + + + id=#{udf.id} + + + \ No newline at end of file diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapperTest.java index 47d8d89b40..9399c51f3e 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapperTest.java @@ -33,6 +33,7 @@ import org.springframework.test.annotation.Rollback; import org.springframework.test.context.junit4.SpringRunner; import org.springframework.transaction.annotation.Transactional; +import java.util.ArrayList; import java.util.Arrays; import java.util.Date; import java.util.List; @@ -303,4 +304,16 @@ public class UdfFuncMapperTest { authorizedUdfFunc = udfFuncMapper.listAuthorizedUdfFunc(generalUser1.getId(), udfFuncIds); Assert.assertTrue(authorizedUdfFunc.stream().map(t -> t.getId()).collect(toList()).containsAll(Arrays.asList(udfFuncIds))); } + + @Test + public void batchUpdateUdfFuncTest(){ + //create general user + User generalUser1 = createGeneralUser("user1"); + UdfFunc udfFunc = insertOne(generalUser1); + udfFunc.setResourceName("/updateTest"); + List udfFuncList = new ArrayList<>(); + udfFuncList.add(udfFunc); + Assert.assertTrue(udfFuncMapper.batchUpdateUdfFunc(udfFuncList)>0); + + } } \ No newline at end of file diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java index 6692598ce6..15cc814b26 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterExecThread.java @@ -667,12 +667,10 @@ public class MasterExecThread implements Runnable { List depNameList = taskNode.getDepList(); for(String depsNode : depNameList ){ if(!dag.containsNode(depsNode) + || skipTaskNodeList.containsKey(depsNode) || forbiddenTaskList.containsKey(depsNode)){ continue; } - if(skipTaskNodeList.containsKey(depsNode)){ - return DependResult.FAILED; - } // all the dependencies must be completed if(!completeTaskList.containsKey(depsNode)){ return DependResult.WAITING; diff --git a/dolphinscheduler-ui/build/webpack.config.prod.js b/dolphinscheduler-ui/build/webpack.config.prod.js index 4bb90d54d1..1024ac6724 100644 --- a/dolphinscheduler-ui/build/webpack.config.prod.js +++ b/dolphinscheduler-ui/build/webpack.config.prod.js @@ -51,12 +51,7 @@ const config = merge.smart(baseConfig, { minimizer: [ new TerserPlugin({ terserOptions: { - compress: { - warnings: false, - drop_console: true, - drop_debugger: true, - pure_funcs: ['console.log'] - } + compress: {} }, cache: true, parallel: true, diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js index 5a41091029..fef884c387 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.js @@ -118,8 +118,8 @@ Dag.prototype.toolbarEvent = function ({ item, code, is }) { Dag.prototype.backfill = function (arg) { if (arg) { let locationsValue = store.state.dag.locations - const locationsValue1 = store.state.dag.locations - const locationsValue2 = store.state.dag.locations + const locationsValue1 = _.cloneDeep(store.state.dag.locations) + const locationsValue2 = _.cloneDeep(store.state.dag.locations) const arr = [] for (const i in locationsValue1) { const objs = {} diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue index 25516bf265..98aaaa5ff4 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/dag.vue @@ -522,9 +522,9 @@ } else { rearList = [] } - let targetarr = $(`#${id}`).attr('data-targetarr') - if (targetarr) { - let nodearr = targetarr.split(',') + let target = $(`#${id}`).attr('data-targetarr') + if (target) { + let nodearr = target.split(',') nodearr.forEach(v => { let nodeobj = {} nodeobj.value = $(`#${v}`).find('.name-p').text() @@ -602,9 +602,7 @@ }, watch: { 'tasks': { - deep: true, handler (o) { - // Edit state does not allow deletion of node a... this.setIsEditDag(true) } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue index 3f85f36992..0fe5d0cbdb 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/flink.vue @@ -375,10 +375,8 @@ resourceIdArr = isResourceId.map(item=>{ return item.id }) - Array.prototype.diff = function(a) { - return this.filter(function(i) {return a.indexOf(i) < 0;}); - }; - let diffSet = this.resourceList.diff(resourceIdArr); + let diffSet + diffSet = _.xorWith(this.resourceList, resourceIdArr, _.isEqual) let optionsCmp = [] if(diffSet.length>0) { diffSet.forEach(item=>{ @@ -477,7 +475,7 @@ // Non-null objects represent backfill if (!_.isEmpty(o)) { this.mainClass = o.params.mainClass || '' - if(o.params.mainJar.res) { + if(!o.params.mainJar.id) { this.marjarId(o.params.mainJar.res) } else if(o.params.mainJar.res=='') { this.mainJar = '' diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue index d05c6983d6..26d74bd990 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/mr.vue @@ -216,10 +216,8 @@ resourceIdArr = isResourceId.map(item=>{ return item.id }) - Array.prototype.diff = function(a) { - return this.filter(function(i) {return a.indexOf(i) < 0;}); - }; - let diffSet = this.resourceList.diff(resourceIdArr); + let diffSet + diffSet = _.xorWith(this.resourceList, resourceIdArr, _.isEqual) let optionsCmp = [] if(diffSet.length>0) { diffSet.forEach(item=>{ @@ -356,7 +354,7 @@ // Non-null objects represent backfill if (!_.isEmpty(o)) { this.mainClass = o.params.mainClass || '' - if(o.params.mainJar.res) { + if(!o.params.mainJar.id) { this.marjarId(o.params.mainJar.res) } else if(o.params.mainJar.res=='') { this.mainJar = '' diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue index 851f8bee8a..40b8c6a0b7 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/python.vue @@ -208,10 +208,8 @@ resourceIdArr = isResourceId.map(item=>{ return item.id }) - Array.prototype.diff = function(a) { - return this.filter(function(i) {return a.indexOf(i) < 0;}); - }; - let diffSet = this.resourceList.diff(resourceIdArr); + let diffSet + diffSet = _.xorWith(this.resourceList, resourceIdArr, _.isEqual) let optionsCmp = [] if(diffSet.length>0) { diffSet.forEach(item=>{ diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue index 7a462a1f27..48c5322b16 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/shell.vue @@ -20,11 +20,7 @@
{{$t('Script')}}
- + @@ -250,10 +246,8 @@ resourceIdArr = isResourceId.map(item=>{ return item.id }) - Array.prototype.diff = function(a) { - return this.filter(function(i) {return a.indexOf(i) < 0;}); - }; - let diffSet = this.resourceList.diff(resourceIdArr); + let diffSet + diffSet = _.xorWith(this.resourceList, resourceIdArr, _.isEqual) let optionsCmp = [] if(diffSet.length>0) { diffSet.forEach(item=>{ @@ -364,9 +358,12 @@ } }, mounted () { - setTimeout(() => { - this._handlerEditor() - }, 200) + // Added delay loading in script input box + this.$nextTick(() => { + setTimeout(() => { + this._handlerEditor() + }, 350) + }) }, destroyed () { if (editor) { diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue index 4cceb3b27b..2c59707c69 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/formModel/tasks/spark.vue @@ -335,10 +335,8 @@ resourceIdArr = isResourceId.map(item=>{ return item.id }) - Array.prototype.diff = function(a) { - return this.filter(function(i) {return a.indexOf(i) < 0;}); - }; - let diffSet = this.resourceList.diff(resourceIdArr); + let diffSet + diffSet = _.xorWith(this.resourceList, resourceIdArr, _.isEqual) let optionsCmp = [] if(diffSet.length>0) { diffSet.forEach(item=>{ @@ -525,7 +523,7 @@ // Non-null objects represent backfill if (!_.isEmpty(o)) { this.mainClass = o.params.mainClass || '' - if(o.params.mainJar.res) { + if(!o.params.mainJar.id) { this.marjarId(o.params.mainJar.res) } else if(o.params.mainJar.res=='') { this.mainJar = '' diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js index 3e2d7d944e..34f78b03b8 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/jsPlumbHandle.js @@ -36,7 +36,7 @@ import { } from './util' import mStart from '@/conf/home/pages/projects/pages/definition/pages/list/_source/start' -const JSP = function () { +let JSP = function () { this.dag = {} this.selectedElement = {} @@ -687,37 +687,55 @@ JSP.prototype.saveStore = function () { JSP.prototype.handleEvent = function () { this.JspInstance.bind('beforeDrop', function (info) { - const sourceId = info.sourceId// 出 - const targetId = info.targetId// 入 + console.log(info) + const rtTargetArr = (id) => { + let ids = $(`#${id}`).attr('data-targetarr') + return ids ? ids.split(',') : [] + } + let sourceId = info['sourceId']// 出 + let targetId = info['targetId']// 入 + console.log(sourceId,targetId) + let rtTargetArrs = rtTargetArr(targetId) + let rtSouceArrs = rtTargetArr(sourceId) + /** + * When connecting, connection is prohibited when the sourceId and target nodes are empty + */ + if(!sourceId && !targetId) { + Vue.$message.warning(`${i18n.$t('This canvas is abnormal and the node connection cannot be made. Please save or exit the current workflow')}`) + return false + } /** * Recursive search for nodes */ let recursiveVal const recursiveTargetarr = (arr, targetId) => { - for (const i in arr) { - if (arr[i] === targetId) { - recursiveVal = targetId - } else { - recursiveTargetarr(rtTargetarrArr(arr[i]), targetId) + for (let i in arr) { + if (arr[i] === targetId) { + recursiveVal = targetId + } else { + let targetArr = rtTargetArr(arr[i]) + recursiveTargetarr(targetArr, targetId) + } } - } return recursiveVal } - + // Connection to connected nodes is not allowed - if (_.findIndex(rtTargetarrArr(targetId), v => v === sourceId) !== -1) { + if (_.findIndex(rtTargetArrs, v => v === sourceId) !== -1) { + console.log(rtTargetArrs,'not allowed') return false } - + // Recursive form to find if the target Targetarr has a sourceId - if (recursiveTargetarr(rtTargetarrArr(sourceId), targetId)) { + if (recursiveTargetarr(rtSouceArrs, targetId)) { + console.log('has a sourceId') return false } - - if ($(`#${sourceId}`).attr('data-tasks-type') === 'CONDITIONS' && $(`#${sourceId}`).attr('data-nodenumber') === 2) { + if ($(`#${sourceId}`).attr('data-tasks-type') === 'CONDITIONS' && parseInt($(`#${sourceId}`).attr('data-nodenumber')) === 2) { return false } else { - $(`#${sourceId}`).attr('data-nodenumber', Number($(`#${sourceId}`).attr('data-nodenumber')) + 1) + console.log('data-nodenumber') + $(`#${sourceId}`).attr('data-nodenumber', parseInt($(`#${sourceId}`).attr('data-nodenumber')) + 1) } // Storage node dependency information diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/util.js b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/util.js index e4c78d29fb..c0037dca7f 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/util.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/dag/_source/plugIn/util.js @@ -32,8 +32,9 @@ const rtTargetarrArr = (id) => { */ const saveTargetarr = (valId, domId) => { let $target = $(`#${domId}`) - let targetStr = $target.attr('data-targetarr') ? $target.attr('data-targetarr') + `,${valId}` : `${valId}` + let targetStr = $target.attr('data-targetarr') ? $target.attr('data-targetarr') + `,${valId}` : `${valId}`; $target.attr('data-targetarr', targetStr) + console.log('Store node id to targetarr',targetStr) } const rtBantpl = () => { diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/subUdfDirectory/_source/list.vue b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/subUdfDirectory/_source/list.vue index ddb097ee0f..610eb1dbd8 100755 --- a/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/subUdfDirectory/_source/list.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/resource/pages/udf/pages/subUdfDirectory/_source/list.vue @@ -43,7 +43,7 @@ {{$t('Update Time')}} - + {{$t('Operation')}} @@ -76,6 +76,16 @@ - + + { + const newTasks = []; + tasks.forEach(item => { + const newItem = Object.assign({}, item); + if(newItem.dependence && newItem.dependence.dependTaskList) { + newItem.dependence.dependTaskList.forEach(dependTaskItem => { + if (dependTaskItem.dependItemList) { + dependTaskItem.dependItemList.forEach(dependItem => { + Reflect.deleteProperty(dependItem, 'definitionList'); + }) + } + }) + } + newTasks.push(newItem); + }); + return newTasks; +} + export default { /** * Task status acquisition @@ -193,7 +212,7 @@ export default { return new Promise((resolve, reject) => { const data = { globalParams: state.globalParams, - tasks: state.tasks, + tasks: deleteDefinitionList(state.tasks), tenantId: state.tenantId, timeout: state.timeout } @@ -217,7 +236,7 @@ export default { return new Promise((resolve, reject) => { const data = { globalParams: state.globalParams, - tasks: state.tasks, + tasks: deleteDefinitionList(state.tasks), tenantId: state.tenantId, timeout: state.timeout } diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js index 170ebbc38d..5f7a707f87 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/en_US.js @@ -600,5 +600,6 @@ export default { 'Directory detail': 'Directory detail', 'Connection name': 'Connection name', 'Current connection settings': 'Current connection settings', - 'Please save the DAG before formatting': 'Please save the DAG before formatting' + 'Please save the DAG before formatting': 'Please save the DAG before formatting', + 'This canvas is abnormal and the node connection cannot be made. Please save or exit the current workflow': 'This canvas is abnormal and the node connection cannot be made. Please save or exit the current workflow' } diff --git a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js index 704b447b12..5dec5096b3 100755 --- a/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js +++ b/dolphinscheduler-ui/src/js/module/i18n/locale/zh_CN.js @@ -606,5 +606,6 @@ export default { 'Directory detail': '查看目录详情', 'Connection name': '连线名', 'Current connection settings': '当前连线设置', - 'Please save the DAG before formatting': '格式化前请先保存DAG' + 'Please save the DAG before formatting': '格式化前请先保存DAG', + 'This canvas is abnormal and the node connection cannot be made. Please save or exit the current workflow': '此画布异常，无法进行节点连线，请保存或退出当前工作流' } diff --git a/sql/upgrade/1.2.0_schema/mysql/dolphinscheduler_dml.sql b/sql/upgrade/1.2.0_schema/mysql/dolphinscheduler_dml.sql index 6aeafcd700..e2a08756a6 100644 --- a/sql/upgrade/1.2.0_schema/mysql/dolphinscheduler_dml.sql +++ b/sql/upgrade/1.2.0_schema/mysql/dolphinscheduler_dml.sql @@ -22,4 +22,5 @@ UPDATE QRTZ_FIRED_TRIGGERS SET SCHED_NAME='DolphinScheduler' WHERE SCHED_NAME='E UPDATE QRTZ_JOB_DETAILS SET SCHED_NAME='DolphinScheduler' WHERE SCHED_NAME='EasyScheduler'; UPDATE QRTZ_JOB_DETAILS SET JOB_CLASS_NAME='org.apache.dolphinscheduler.dao.quartz.ProcessScheduleJob' WHERE JOB_CLASS_NAME='cn.escheduler.server.quartz.ProcessScheduleJob'; UPDATE QRTZ_LOCKS SET SCHED_NAME='DolphinScheduler' WHERE SCHED_NAME='EasyScheduler'; -UPDATE QRTZ_SCHEDULER_STATE SET SCHED_NAME='DolphinScheduler' WHERE SCHED_NAME='EasyScheduler'; \ No newline at end of file +UPDATE QRTZ_SCHEDULER_STATE SET SCHED_NAME='DolphinScheduler' WHERE SCHED_NAME='EasyScheduler'; +UPDATE t_ds_user SET phone = '' WHERE phone = 'xx'; \ No newline at end of file diff --git a/sql/upgrade/1.3.0_schema/mysql/dolphinscheduler_dml.sql b/sql/upgrade/1.3.0_schema/mysql/dolphinscheduler_dml.sql index 6f0e145176..661ed9e827 100644 --- a/sql/upgrade/1.3.0_schema/mysql/dolphinscheduler_dml.sql +++ b/sql/upgrade/1.3.0_schema/mysql/dolphinscheduler_dml.sql @@ -23,4 +23,5 @@ UPDATE t_ds_process_instance instance SET `worker_group`=IFNULL((SELECT name fro UPDATE t_ds_task_instance instance SET `worker_group`=IFNULL((SELECT name from t_ds_worker_group WHERE instance.worker_group=CONCAT(id,'')),'default'); UPDATE t_ds_schedules schedule SET `worker_group`=IFNULL((SELECT name from t_ds_worker_group WHERE schedule.worker_group=CONCAT(id,'')),'default'); UPDATE t_ds_command command SET `worker_group`=IFNULL((SELECT name from t_ds_worker_group WHERE command.worker_group=CONCAT(id,'')),'default'); -UPDATE t_ds_error_command command SET `worker_group`=IFNULL((SELECT name from t_ds_worker_group WHERE command.worker_group=CONCAT(id,'')),'default'); \ No newline at end of file +UPDATE t_ds_error_command command SET `worker_group`=IFNULL((SELECT name from t_ds_worker_group WHERE command.worker_group=CONCAT(id,'')),'default'); +UPDATE t_ds_user SET phone = '' WHERE phone = 'xx'; \ No newline at end of file diff --git a/sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_dml.sql b/sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_dml.sql index fba03152ee..a748eae6cf 100644 --- a/sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_dml.sql +++ b/sql/upgrade/1.3.0_schema/postgresql/dolphinscheduler_dml.sql @@ -21,4 +21,5 @@ UPDATE t_ds_process_instance instance SET worker_group=COALESCE((SELECT name fro UPDATE t_ds_task_instance instance SET worker_group=COALESCE((SELECT name from t_ds_worker_group WHERE instance.worker_group=CONCAT(id,'')),'default'); UPDATE t_ds_schedules schedule SET worker_group=COALESCE((SELECT name from t_ds_worker_group WHERE schedule.worker_group=CONCAT(id,'')),'default'); UPDATE t_ds_command command SET worker_group=COALESCE((SELECT name from t_ds_worker_group WHERE command.worker_group=CONCAT(id,'')),'default'); -UPDATE t_ds_error_command command SET worker_group=COALESCE((SELECT name from t_ds_worker_group WHERE command.worker_group=CONCAT(id,'')),'default'); \ No newline at end of file +UPDATE t_ds_error_command command SET worker_group=COALESCE((SELECT name from t_ds_worker_group WHERE command.worker_group=CONCAT(id,'')),'default'); +UPDATE t_ds_user SET phone = '' WHERE phone = 'xx'; \ No newline at end of file diff --git a/sql/upgrade/1.3.2_schema/mysql/dolphinscheduler_ddl.sql b/sql/upgrade/1.3.2_schema/mysql/dolphinscheduler_ddl.sql new file mode 100644 index 0000000000..38964cc551 --- /dev/null +++ b/sql/upgrade/1.3.2_schema/mysql/dolphinscheduler_ddl.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ \ No newline at end of file diff --git a/sql/upgrade/1.3.2_schema/mysql/dolphinscheduler_dml.sql b/sql/upgrade/1.3.2_schema/mysql/dolphinscheduler_dml.sql new file mode 100644 index 0000000000..383d8a4407 --- /dev/null +++ b/sql/upgrade/1.3.2_schema/mysql/dolphinscheduler_dml.sql @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +SET sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY','')); +SET FOREIGN_KEY_CHECKS=0; +UPDATE t_ds_user SET phone = '' WHERE phone = 'xx'; \ No newline at end of file diff --git a/sql/upgrade/1.3.2_schema/postgresql/dolphinscheduler_ddl.sql b/sql/upgrade/1.3.2_schema/postgresql/dolphinscheduler_ddl.sql new file mode 100644 index 0000000000..38964cc551 --- /dev/null +++ b/sql/upgrade/1.3.2_schema/postgresql/dolphinscheduler_ddl.sql @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ \ No newline at end of file diff --git a/sql/upgrade/1.3.2_schema/postgresql/dolphinscheduler_dml.sql b/sql/upgrade/1.3.2_schema/postgresql/dolphinscheduler_dml.sql new file mode 100644 index 0000000000..bf043ad469 --- /dev/null +++ b/sql/upgrade/1.3.2_schema/postgresql/dolphinscheduler_dml.sql @@ -0,0 +1,17 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +UPDATE t_ds_user SET phone = '' WHERE phone = 'xx'; \ No newline at end of file