From 8fa8cb559aec0fb1228a9e622db99a665060dae7 Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Wed, 18 Dec 2019 23:03:49 +0800 Subject: [PATCH 1/4] Solve jertty packing garbled (#1518) * Password verification and v-for add key * DAG automatic layout * combo.js replacement * combo.js replacement * webpack upgrade to 4 * packjson has a file name that has not changed * Solve jertty packing garbled --- dolphinscheduler-ui/src/view/home/index.html | 1 + dolphinscheduler-ui/src/view/login/index.html | 1 + 2 files changed, 2 insertions(+) diff --git a/dolphinscheduler-ui/src/view/home/index.html b/dolphinscheduler-ui/src/view/home/index.html index ff7ee7b66a..a89ab8b01c 100644 --- a/dolphinscheduler-ui/src/view/home/index.html +++ b/dolphinscheduler-ui/src/view/home/index.html @@ -19,6 +19,7 @@ + diff --git a/dolphinscheduler-ui/src/view/login/index.html b/dolphinscheduler-ui/src/view/login/index.html index 595492b7e3..30eb792335 100644 --- a/dolphinscheduler-ui/src/view/login/index.html +++ b/dolphinscheduler-ui/src/view/login/index.html @@ -19,6 +19,7 @@ + From 66217be9903e757fb2375b67d741cbec13656b40 Mon Sep 17 00:00:00 2001 From: Yelli <51317527+Yeleights@users.noreply.github.com> Date: Thu, 19 Dec 2019 11:29:52 +0800 Subject: [PATCH 2/4] Modify Queue already add to User not change bug -- fix #1514 (#1517) * add Spark Version in Spark Component add Spark Version in Spark Component * add license for SparkVersion.class add license * 1 add spark task UT 2 add spark version param check * add assert check for sparkTaskTest * fix bug:user queue field does not change with queue table modify * modify check queue in using method name and updateUserByQueue method param name modify check queue in using method name and updateUserByQueue method param name --- .../api/service/QueueService.java | 58 ++++++++++++++----- .../dao/mapper/UserMapper.java | 14 +++++ .../dao/mapper/UserMapper.xml | 10 ++++ 3 files changed, 66 insertions(+), 16 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java index 5ac3ace25c..f860517d0f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/QueueService.java @@ -26,6 +26,7 @@ import org.apache.dolphinscheduler.dao.mapper.QueueMapper; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.commons.lang.StringUtils; +import org.apache.dolphinscheduler.dao.mapper.UserMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -47,6 +48,9 @@ public class QueueService extends BaseService { @Autowired private QueueMapper queueMapper; + @Autowired + private UserMapper userMapper; + /** * query queue list * @@ -70,9 +74,9 @@ public class QueueService extends BaseService { * query queue list paging * * @param loginUser login user - * @param pageNo page number + * @param pageNo page number * @param searchVal search value - * @param pageSize page size + * @param pageSize page size * @return queue list */ public Map queryList(User loginUser, String searchVal, Integer pageNo, Integer pageSize) { @@ -86,7 +90,7 @@ public class QueueService extends BaseService { IPage queueList = queueMapper.queryQueuePaging(page, searchVal); - Integer count = (int)queueList.getTotal(); + Integer count = (int) queueList.getTotal(); PageInfo pageInfo = new PageInfo<>(pageNo, pageSize); pageInfo.setTotalCount(count); pageInfo.setLists(queueList.getRecords()); @@ -100,7 +104,7 @@ public class QueueService extends BaseService { * create queue * * @param loginUser login user - * @param queue queue + * @param queue queue * @param queueName queue name * @return create result */ @@ -110,12 +114,12 @@ public class QueueService extends BaseService { return result; } - if(StringUtils.isEmpty(queue)){ + if (StringUtils.isEmpty(queue)) { putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, queue); return result; } - if(StringUtils.isEmpty(queueName)){ + if (StringUtils.isEmpty(queueName)) { putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, queueName); return result; } @@ -148,8 +152,8 @@ public class QueueService extends BaseService { * update queue * * @param loginUser login user - * @param queue queue - * @param id queue id + * @param queue queue + * @param id queue id * @param queueName queue name * @return update result code */ @@ -173,7 +177,7 @@ public class QueueService extends BaseService { // check queue name is exist if (!queueName.equals(queueObj.getQueueName())) { - if(checkQueueNameExist(queueName)){ + if (checkQueueNameExist(queueName)) { putMsg(result, Status.QUEUE_NAME_EXIST, queueName); return result; } @@ -181,12 +185,19 @@ public class QueueService extends BaseService { // check queue value is exist if (!queue.equals(queueObj.getQueue())) { - if(checkQueueExist(queue)){ + if (checkQueueExist(queue)) { putMsg(result, Status.QUEUE_VALUE_EXIST, queue); return result; } } + // check old queue using by any user + if (checkIfQueueIsInUsing(queueObj.getQueueName(), queueName)) { + //update user related old queue + Integer relatedUserNums = userMapper.updateUserQueue(queueObj.getQueueName(), queueName); + logger.info("old queue have related {} user, exec update user success.", relatedUserNums); + } + // update queue Date now = new Date(); queueObj.setQueue(queue); @@ -194,6 +205,7 @@ public class QueueService extends BaseService { queueObj.setUpdateTime(now); queueMapper.updateById(queueObj); + putMsg(result, Status.SUCCESS); return result; @@ -202,12 +214,12 @@ public class QueueService extends BaseService { /** * verify queue and queueName * - * @param queue queue + * @param queue queue * @param queueName queue name * @return true if the queue name not exists, otherwise return false */ public Result verifyQueue(String queue, String queueName) { - Result result=new Result(); + Result result = new Result(); if (StringUtils.isEmpty(queue)) { putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, queue); @@ -220,13 +232,13 @@ public class QueueService extends BaseService { } - if(checkQueueNameExist(queueName)){ + if (checkQueueNameExist(queueName)) { logger.error("queue name {} has exist, can't create again.", queueName); putMsg(result, Status.QUEUE_NAME_EXIST, queueName); return result; } - if(checkQueueExist(queue)){ + if (checkQueueExist(queue)) { logger.error("queue value {} has exist, can't create again.", queue); putMsg(result, Status.QUEUE_VALUE_EXIST, queue); return result; @@ -240,21 +252,35 @@ public class QueueService extends BaseService { * check queue exist * if exists return true，not exists return false * check queue exist + * * @param queue queue * @return true if the queue not exists, otherwise return false */ private boolean checkQueueExist(String queue) { - return queueMapper.queryAllQueueList(queue, null).size()>0 ? true : false; + return queueMapper.queryAllQueueList(queue, null).size() > 0; } /** * check queue name exist * if exists return true，not exists return false + * * @param queueName queue name * @return true if the queue name not exists, otherwise return false */ private boolean checkQueueNameExist(String queueName) { - return queueMapper.queryAllQueueList(null ,queueName).size() > 0 ? true : false; + return queueMapper.queryAllQueueList(null, queueName).size() > 0; + } + + /** + * check old queue name using by any user + * if need to update user + * + * @param oldQueue old queue name + * @param newQueue new queue name + * @return true if need to update user + */ + private boolean checkIfQueueIsInUsing (String oldQueue, String newQueue) { + return !oldQueue.equals(newQueue) && userMapper.queryUserListByQueue(oldQueue).size() > 0; } } diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UserMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UserMapper.java index 2c194b4afa..1468f10b90 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UserMapper.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UserMapper.java @@ -95,4 +95,18 @@ public interface UserMapper extends BaseMapper { */ User queryUserByToken(@Param("token") String token); + /** + * query user by queue name + * @param queueName queue name + * @return user list + */ + List queryUserListByQueue(@Param("queueName") String queueName); + + /** + * update user with old queue + * @param oldQueue old queue name + * @param newQueue new queue name + * @return update rows + */ + Integer updateUserQueue(@Param("oldQueue") String oldQueue, @Param("newQueue") String newQueue); } diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UserMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UserMapper.xml index 6046ad22eb..fcf8a137e6 100644 --- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UserMapper.xml +++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UserMapper.xml @@ -69,4 +69,14 @@ from t_ds_user u ,t_ds_access_token t where u.id = t.user_id and token=#{token} and t.expire_time > NOW() + + + update t_ds_user + set queue = #{newQueue} + where queue = #{oldQueue} + \ No newline at end of file From 67cde18cc314004f8f9fdba6a271850d1be84ca6 Mon Sep 17 00:00:00 2001 From: Tboy Date: Thu, 19 Dec 2019 11:31:51 +0800 Subject: [PATCH 3/4] fix #1515 (#1516) --- .../dolphinscheduler/server/worker/runner/FetchTaskThread.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java index ae4ee792c5..68f27d76d8 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/runner/FetchTaskThread.java @@ -147,9 +147,8 @@ public class FetchTaskThread implements Runnable{ //check memory and cpu usage and threads boolean runCheckFlag = OSUtils.checkResource(workerConfig.getWorkerMaxCpuloadAvg(), workerConfig.getWorkerReservedMemory()) && checkThreadCount(poolExecutor); - Thread.sleep(Constants.SLEEP_TIME_MILLIS); - if(!runCheckFlag) { + Thread.sleep(Constants.SLEEP_TIME_MILLIS); continue; } From 0567b5829c9ce13d64a13480403ccf6d46cc783a Mon Sep 17 00:00:00 2001 From: zhukai Date: Thu, 19 Dec 2019 11:33:08 +0800 Subject: [PATCH 4/4] Add the UTs of FuncUtils and JSONUtilsTest. (#1507) 1.Add FuncUtilsTest.java and JSONUtilsTest.java 2.Add null check in FuncUtils.mkString 3.Add pom config --- .../alert/utils/FuncUtils.java | 7 ++ .../alert/utils/FuncUtilsTest.java | 60 ++++++++++ .../alert/utils/JSONUtilsTest.java | 112 ++++++++++++++++++ pom.xml | 7 +- 4 files changed, 183 insertions(+), 3 deletions(-) create mode 100644 dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/FuncUtilsTest.java create mode 100644 dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java diff --git a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java index e682fde2e7..186a983e05 100644 --- a/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java +++ b/dolphinscheduler-alert/src/main/java/org/apache/dolphinscheduler/alert/utils/FuncUtils.java @@ -16,9 +16,16 @@ */ package org.apache.dolphinscheduler.alert.utils; +import org.apache.commons.lang.StringUtils; + public class FuncUtils { static public String mkString(Iterable list, String split) { + + if (null == list || StringUtils.isEmpty(split)){ + return null; + } + StringBuilder sb = new StringBuilder(); boolean first = true; for (String item : list) { diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/FuncUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/FuncUtilsTest.java new file mode 100644 index 0000000000..e2b25d961d --- /dev/null +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/FuncUtilsTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.utils; + +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class FuncUtilsTest { + + private static final Logger logger = LoggerFactory.getLogger(FuncUtilsTest.class); + + /** + * Test mkString + */ + @Test + public void testMKString() { + + //Define users list + Iterable users = Arrays.asList("user1", "user2", "user3"); + //Define split + String split = "|"; + + //Invoke mkString with correctParams + String result = FuncUtils.mkString(users, split); + logger.info(result); + + //Expected result string + assertEquals(result, "user1|user2|user3"); + + //Null list expected return null + result = FuncUtils.mkString(null, split); + assertNull(result); + + //Null split expected return null + result = FuncUtils.mkString(users, null); + assertNull(result); + + } +} \ No newline at end of file diff --git a/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java new file mode 100644 index 0000000000..cb63a22d79 --- /dev/null +++ b/dolphinscheduler-alert/src/test/java/org/apache/dolphinscheduler/alert/utils/JSONUtilsTest.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.alert.utils; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class JSONUtilsTest { + + private static final Logger logger = LoggerFactory.getLogger(JSONUtilsTest.class); + + public List> list = new ArrayList<>(); + + public String expected = null; + + @Before + public void setUp() throws Exception { + + //Define expected json string + expected = "[{\"mysql service name\":\"mysql200\",\"mysql address\":\"192.168.xx.xx\",\"port\":\"3306\",\"no index of number\":\"80\",\"database client connections\":\"190\"}]"; + + //Initial map + LinkedHashMap map = new LinkedHashMap<>(); + map.put("mysql service name","mysql200"); + map.put("mysql address","192.168.xx.xx"); + map.put("port","3306"); + map.put("no index of number","80"); + map.put("database client connections","190"); + + //Add map into list + list.add(map); + } + + + /** + * Test toJsonString + */ + @Test + public void testToJsonString() { + + //Invoke toJsonString + String result = JSONUtils.toJsonString(list); + logger.info(result); + + //Equal result with expected string + assertEquals(result,expected); + + //If param is null, then return null string + result = JSONUtils.toJsonString(null); + logger.info(result); + + assertEquals(result,"null"); + + } + + /** + * Test toList + */ + @Test + public void testToList() { + + //Invoke toList + List result = JSONUtils.toList(expected ,LinkedHashMap.class); + //Equal list size=1 + assertEquals(result.size(),1); + + //Transform entity to LinkedHashMap + LinkedHashMap entity = result.get(0); + + //Equal expected values + assertEquals(entity.get("mysql service name"),"mysql200"); + assertEquals(entity.get("mysql address"),"192.168.xx.xx"); + assertEquals(entity.get("port"),"3306"); + assertEquals(entity.get("no index of number"),"80"); + assertEquals(entity.get("database client connections"),"190"); + + //If param is null, then return null + result = JSONUtils.toList(null ,LinkedHashMap.class); + assertNull(result); + + //If param is incorrect, then return null and log error message + result = JSONUtils.toList("}{" ,LinkedHashMap.class); + assertNull(result); + + } + +} diff --git a/pom.xml b/pom.xml index a56fe6e892..4b10ba9c42 100644 --- a/pom.xml +++ b/pom.xml @@ -613,14 +613,15 @@ **/common/utils/*.java - **/api/utils/CheckUtilsTest.java - **/api/utils/FileUtilsTest.java **/common/graph/*.java **/common/queue/*.java **/api/utils/CheckUtilsTest.java **/api/utils/FileUtilsTest.java - **/alert/utils/ExcelUtilsTest.java + + **/alert/utils/ExcelUtilsTest.java + **/alert/utils/FuncUtilsTest.java + **/alert/utils/JSONUtilsTest.java