From 7eeeb9f5206b0b03abfafa67ecd3f3663a658070 Mon Sep 17 00:00:00 2001 From: lilin Date: Sun, 22 Dec 2019 14:54:32 +0800 Subject: [PATCH 01/31] add MonitorServiceTest --- .../api/service/MonitorServiceTest.java | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java new file mode 100644 index 0000000000..ef4b684f57 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service; + +import org.apache.dolphinscheduler.api.ApiApplicationServer; +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.model.Server; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +import java.util.List; +import java.util.Map; + +@RunWith(SpringRunner.class) +@SpringBootTest(classes = ApiApplicationServer.class) +public class MonitorServiceTest { + + private static final Logger logger = LoggerFactory.getLogger(MonitorServiceTest.class); + + @Autowired + private MonitorService monitorService; + + @Test + public void testQueryDatabaseState(){ + + Map result = monitorService.queryDatabaseState(null); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } + @Test + public void testQueryMaster(){ + + Map result = monitorService.queryMaster(null); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } + @Test + public void testQueryZookeeperState(){ + Map result = monitorService.queryZookeeperState(null); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } + + @Test + public void testGetServerListFromZK(){ + List serverList = monitorService.getServerListFromZK(true); + logger.info(serverList.toString()); + } + +} From 84df6ed8dea87f64206b87918b2d2941a6633941 Mon Sep 17 00:00:00 2001 From: loushang Date: Wed, 25 Dec 2019 17:06:58 +0800 Subject: [PATCH 02/31] refactor zk tree cache --- .../common/zk/ZookeeperCachedOperator.java | 48 +++++----- .../common/zk/ZookeeperOperator.java | 8 +- .../server/zk/ZKMasterClient.java | 94 ++++++++++--------- .../server/zk/ZKWorkerClient.java | 54 ++++++----- 4 files changed, 109 insertions(+), 95 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperCachedOperator.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperCachedOperator.java index cf4980147e..daec765315 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperCachedOperator.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperCachedOperator.java @@ -16,8 +16,10 @@ */ package org.apache.dolphinscheduler.common.zk; +import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.ChildData; import org.apache.curator.framework.recipes.cache.TreeCache; +import org.apache.curator.framework.recipes.cache.TreeCacheEvent; import org.apache.curator.framework.recipes.cache.TreeCacheListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,30 +36,37 @@ public class ZookeeperCachedOperator extends ZookeeperOperator { private final Logger logger = LoggerFactory.getLogger(ZookeeperCachedOperator.class); - //kay is zk path, value is TreeCache - private ConcurrentHashMap allCaches = new ConcurrentHashMap<>(); + TreeCache treeCache; /** - * @param cachePath zk path - * @param listener operator + * register a unified listener of /${dsRoot}, */ - public void registerListener(final String cachePath, final TreeCacheListener listener) { - TreeCache newCache = new TreeCache(zkClient, cachePath); - logger.info("add listener to zk path: {}", cachePath); + @Override + protected void registerListener() { + treeCache = new TreeCache(zkClient, getZookeeperConfig().getDsRoot()); + logger.info("add listener to zk path: {}", getZookeeperConfig().getDsRoot()); try { - newCache.start(); + treeCache.start(); } catch (Exception e) { - logger.error("add listener to zk path: {} failed", cachePath); + logger.error("add listener to zk path: {} failed", getZookeeperConfig().getDsRoot()); throw new RuntimeException(e); } - newCache.getListenable().addListener(listener); + treeCache.getListenable().addListener((client, event) -> { + String path = null == event.getData() ? "" : event.getData().getPath(); + if (path.isEmpty()) { + return; + } + dataChanged(client, event, path); + }); - allCaches.put(cachePath, newCache); } + //for sub class + protected void dataChanged(final CuratorFramework client, final TreeCacheEvent event, final String path){} + public String getFromCache(final String cachePath, final String key) { - ChildData resultInCache = allCaches.get(checkNotNull(cachePath)).getCurrentData(key); + ChildData resultInCache = treeCache.getCurrentData(key); if (null != resultInCache) { return null == resultInCache.getData() ? null : new String(resultInCache.getData(), StandardCharsets.UTF_8); } @@ -65,18 +74,15 @@ public class ZookeeperCachedOperator extends ZookeeperOperator { } public TreeCache getTreeCache(final String cachePath) { - return allCaches.get(checkNotNull(cachePath)); + return treeCache; } public void close() { - - allCaches.forEach((path, cache) -> { - cache.close(); - try { - Thread.sleep(500); - } catch (InterruptedException ignore) { - } - }); + treeCache.close(); + try { + Thread.sleep(500); + } catch (InterruptedException ignore) { + } super.close(); } } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java index f4d72f436e..2203f2ef3c 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperOperator.java @@ -57,11 +57,13 @@ public class ZookeeperOperator implements InitializingBean { public void afterPropertiesSet() throws Exception { this.zkClient = buildClient(); initStateLister(); - //init(); + registerListener(); } - //for subclass - //protected void init(){} + /** + * this method is for sub class, + */ + protected void registerListener(){} public void initStateLister() { checkNotNull(zkClient); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java index 2aec6ecaf6..168db366c6 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java @@ -101,12 +101,6 @@ public class ZKMasterClient extends AbstractZKClient { // init system znode this.initSystemZNode(); - // monitor master - this.listenerMaster(); - - // monitor worker - this.listenerWorker(); - // register master this.registerMaster(); @@ -158,31 +152,24 @@ public class ZKMasterClient extends AbstractZKClient { } } - /** - * monitor master + * handle path events that this class cares about + * @param client zkClient + * @param event path event + * @param path zk path */ - public void listenerMaster(){ - registerListener(getZNodeParentPath(ZKNodeType.MASTER), new AbstractListener() { - @Override - protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String path) { - switch (event.getType()) { - case NODE_ADDED: - logger.info("master node added : {}", path); - break; - case NODE_REMOVED: - String serverHost = getHostByEventDataPath(path); - if (checkServerSelfDead(serverHost, ZKNodeType.MASTER)) { - return; - } - removeZKNodePath(path, ZKNodeType.MASTER, true); - break; - default: - break; - } - } - }); -} + @Override + protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String path) { + if(path.equals(getZNodeParentPath(ZKNodeType.MASTER))){ //monitor master + logger.info("master path event touch down"); + handleMasterEvent(event,path); + + }else if(path.equals(getZNodeParentPath(ZKNodeType.WORKER))){ //monitor worker + logger.info("worker path event touch down"); + handleWorkerEvent(event,path); + } + //other path event, ignore + } /** * remove zookeeper node path @@ -273,25 +260,40 @@ public class ZKMasterClient extends AbstractZKClient { } /** - * monitor worker + * monitor master */ - public void listenerWorker(){ - registerListener(getZNodeParentPath(ZKNodeType.WORKER), new AbstractListener() { - @Override - protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String path) { - switch (event.getType()) { - case NODE_ADDED: - logger.info("worker node added : {}", path); - break; - case NODE_REMOVED: - logger.info("worker node deleted : {}", path); - removeZKNodePath(path, ZKNodeType.WORKER, true); - break; - default: - break; + public void handleMasterEvent(TreeCacheEvent event, String path){ + switch (event.getType()) { + case NODE_ADDED: + logger.info("master node added : {}", path); + break; + case NODE_REMOVED: + String serverHost = getHostByEventDataPath(path); + if (checkServerSelfDead(serverHost, ZKNodeType.MASTER)) { + return; } - } - }); + removeZKNodePath(path, ZKNodeType.MASTER, true); + break; + default: + break; + } + } + + /** + * monitor worker + */ + public void handleWorkerEvent(TreeCacheEvent event, String path){ + switch (event.getType()) { + case NODE_ADDED: + logger.info("worker node added : {}", path); + break; + case NODE_REMOVED: + logger.info("worker node deleted : {}", path); + removeZKNodePath(path, ZKNodeType.WORKER, true); + break; + default: + break; + } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java index 0dd1cf15be..192d9e0e58 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java @@ -61,9 +61,6 @@ public class ZKWorkerClient extends AbstractZKClient { // init system znode this.initSystemZNode(); - // monitor worker - this.listenerWorker(); - // register worker this.registWorker(); } @@ -83,31 +80,38 @@ public class ZKWorkerClient extends AbstractZKClient { System.exit(-1); } } - + /** - * monitor worker + * handle path events that this class cares about + * @param client zkClient + * @param event path event + * @param path zk path */ - private void listenerWorker(){ - registerListener(getZNodeParentPath(ZKNodeType.WORKER), new AbstractListener() { - @Override - protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String path) { - switch (event.getType()) { - case NODE_ADDED: - logger.info("worker node added : {}", path); - break; - case NODE_REMOVED: - //find myself dead - String serverHost = getHostByEventDataPath(path); - if(checkServerSelfDead(serverHost, ZKNodeType.WORKER)){ - return; - } - break; - default: - break; - } - } - }); + @Override + protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String path) { + if(path.equals(getZNodeParentPath(ZKNodeType.WORKER))){ + handleWorkerEvent(event,path); + } + } + /** + * monitor worker + */ + public void handleWorkerEvent(TreeCacheEvent event, String path){ + switch (event.getType()) { + case NODE_ADDED: + logger.info("worker node added : {}", path); + break; + case NODE_REMOVED: + //find myself dead + String serverHost = getHostByEventDataPath(path); + if(checkServerSelfDead(serverHost, ZKNodeType.WORKER)){ + return; + } + break; + default: + break; + } } /** From cc1a5ff97e9067bcca1488619807a9510620a058 Mon Sep 17 00:00:00 2001 From: loushang Date: Wed, 25 Dec 2019 19:25:39 +0800 Subject: [PATCH 03/31] refactor zk tree cache --- .../common/zk/AbstractListener.java | 35 ------------------- .../server/zk/ZKMasterClient.java | 10 ++---- .../server/zk/ZKWorkerClient.java | 9 +---- 3 files changed, 3 insertions(+), 51 deletions(-) delete mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractListener.java diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractListener.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractListener.java deleted file mode 100644 index d84b9f7e11..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractListener.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.common.zk; - -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.recipes.cache.TreeCacheEvent; -import org.apache.curator.framework.recipes.cache.TreeCacheListener; - -public abstract class AbstractListener implements TreeCacheListener { - - @Override - public final void childEvent(final CuratorFramework client, final TreeCacheEvent event) throws Exception { - String path = null == event.getData() ? "" : event.getData().getPath(); - if (path.isEmpty()) { - return; - } - dataChanged(client, event, path); - } - - protected abstract void dataChanged(final CuratorFramework client, final TreeCacheEvent event, final String path); -} diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java index 168db366c6..a26a217665 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java @@ -21,7 +21,6 @@ import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ZKNodeType; import org.apache.dolphinscheduler.common.model.Server; -import org.apache.dolphinscheduler.common.zk.AbstractListener; import org.apache.dolphinscheduler.common.zk.AbstractZKClient; import org.apache.dolphinscheduler.dao.AlertDao; import org.apache.dolphinscheduler.dao.DaoFactory; @@ -31,9 +30,6 @@ import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.utils.ProcessUtils; import org.apache.commons.lang.StringUtils; import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.recipes.cache.PathChildrenCache; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; import org.apache.curator.framework.recipes.locks.InterProcessMutex; import org.apache.curator.utils.ThreadUtils; import org.slf4j.Logger; @@ -160,12 +156,10 @@ public class ZKMasterClient extends AbstractZKClient { */ @Override protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String path) { - if(path.equals(getZNodeParentPath(ZKNodeType.MASTER))){ //monitor master - logger.info("master path event touch down"); + if(path.startsWith(getZNodeParentPath(ZKNodeType.MASTER)+Constants.SINGLE_SLASH)){ //monitor master handleMasterEvent(event,path); - }else if(path.equals(getZNodeParentPath(ZKNodeType.WORKER))){ //monitor worker - logger.info("worker path event touch down"); + }else if(path.startsWith(getZNodeParentPath(ZKNodeType.WORKER)+Constants.SINGLE_SLASH)){ //monitor worker handleWorkerEvent(event,path); } //other path event, ignore diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java index 192d9e0e58..2e063d50d5 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKWorkerClient.java @@ -19,20 +19,13 @@ package org.apache.dolphinscheduler.server.zk; import org.apache.curator.framework.recipes.cache.TreeCacheEvent; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ZKNodeType; -import org.apache.dolphinscheduler.common.zk.AbstractListener; import org.apache.dolphinscheduler.common.zk.AbstractZKClient; import org.apache.commons.lang.StringUtils; import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.recipes.cache.PathChildrenCache; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; -import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; -import org.apache.curator.utils.ThreadUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; -import java.util.concurrent.ThreadFactory; - /** * zookeeper worker client @@ -89,7 +82,7 @@ public class ZKWorkerClient extends AbstractZKClient { */ @Override protected void dataChanged(CuratorFramework client, TreeCacheEvent event, String path) { - if(path.equals(getZNodeParentPath(ZKNodeType.WORKER))){ + if(path.startsWith(getZNodeParentPath(ZKNodeType.WORKER)+Constants.SINGLE_SLASH)){ handleWorkerEvent(event,path); } } From 6bec773792e7828a2fb6416818a479d343e82bfa Mon Sep 17 00:00:00 2001 From: lilin Date: Fri, 27 Dec 2019 14:18:44 +0800 Subject: [PATCH 04/31] mobile phone need 11 number --- .../java/org/apache/dolphinscheduler/api/utils/CheckUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java index 6c9f714721..a7867f1ba2 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/utils/CheckUtils.java @@ -104,7 +104,7 @@ public class CheckUtils { * @return true if phone regex valid, otherwise return false */ public static boolean checkPhone(String phone) { - return StringUtils.isEmpty(phone) || phone.length() <= 11; + return StringUtils.isEmpty(phone) || phone.length() == 11; } From 9c6f9fce20ea2d84c0a191309625f2a61897cfec Mon Sep 17 00:00:00 2001 From: lilin Date: Fri, 27 Dec 2019 14:21:48 +0800 Subject: [PATCH 05/31] delete file --- .../api/service/MonitorServiceTest.java | 71 ------------------- 1 file changed, 71 deletions(-) delete mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java deleted file mode 100644 index ef4b684f57..0000000000 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.api.service; - -import org.apache.dolphinscheduler.api.ApiApplicationServer; -import org.apache.dolphinscheduler.api.enums.Status; -import org.apache.dolphinscheduler.common.Constants; -import org.apache.dolphinscheduler.common.model.Server; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; - -import java.util.List; -import java.util.Map; - -@RunWith(SpringRunner.class) -@SpringBootTest(classes = ApiApplicationServer.class) -public class MonitorServiceTest { - - private static final Logger logger = LoggerFactory.getLogger(MonitorServiceTest.class); - - @Autowired - private MonitorService monitorService; - - @Test - public void testQueryDatabaseState(){ - - Map result = monitorService.queryDatabaseState(null); - logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); - } - @Test - public void testQueryMaster(){ - - Map result = monitorService.queryMaster(null); - logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); - } - @Test - public void testQueryZookeeperState(){ - Map result = monitorService.queryZookeeperState(null); - logger.info(result.toString()); - Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); - } - - @Test - public void testGetServerListFromZK(){ - List serverList = monitorService.getServerListFromZK(true); - logger.info(serverList.toString()); - } - -} From 05a5a413cf9a1dc770848b9f023134665875b4f6 Mon Sep 17 00:00:00 2001 From: lilin Date: Fri, 27 Dec 2019 14:57:55 +0800 Subject: [PATCH 06/31] delAlertgroupById add entity exist check --- .../dolphinscheduler/api/service/AlertGroupService.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java index 40fc65b4dc..63f50f936f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/AlertGroupService.java @@ -193,6 +193,12 @@ public class AlertGroupService extends BaseService{ if (checkAdmin(loginUser, result)){ return result; } + //check exist + AlertGroup alertGroup = alertGroupMapper.selectById(id); + if (alertGroup == null) { + putMsg(result, Status.ALERT_GROUP_NOT_EXIST); + return result; + } userAlertGroupMapper.deleteByAlertgroupId(id); alertGroupMapper.deleteById(id); From ce76f3eab763e944a8a8149a5babbc90a085b215 Mon Sep 17 00:00:00 2001 From: lilin Date: Fri, 27 Dec 2019 18:11:48 +0800 Subject: [PATCH 07/31] 1 Modify judgment order 2 before delete check entity exist --- .../api/service/UdfFuncService.java | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java index 8694f4e7b3..7b4ce63e24 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java @@ -187,6 +187,12 @@ public class UdfFuncService extends BaseService{ // verify udfFunc is exist UdfFunc udf = udfFuncMapper.selectUdfById(udfFuncId); + if (udf == null) { + result.put(Constants.STATUS, Status.UDF_FUNCTION_NOT_EXIST); + result.put(Constants.MSG, Status.UDF_FUNCTION_NOT_EXIST.getMsg()); + return result; + } + // if resource upload startup if (!PropertyUtils.getResUploadStartupState()){ logger.error("resource upload startup state: {}", PropertyUtils.getResUploadStartupState()); @@ -194,12 +200,6 @@ public class UdfFuncService extends BaseService{ return result; } - if (udf == null) { - result.put(Constants.STATUS, Status.UDF_FUNCTION_NOT_EXIST); - result.put(Constants.MSG, Status.UDF_FUNCTION_NOT_EXIST.getMsg()); - return result; - } - // verify udfFuncName is exist if (!funcName.equals(udf.getFuncName())) { if (checkUdfFuncNameExists(funcName)) { @@ -303,7 +303,12 @@ public class UdfFuncService extends BaseService{ @Transactional(rollbackFor = Exception.class) public Result delete(int id) { Result result = new Result(); - + //check exist + UdfFunc udfFunc = udfFuncMapper.selectById(id); + if (udfFunc == null) { + putMsg(result, Status.RESOURCE_NOT_EXIST); + return result; + } udfFuncMapper.deleteById(id); udfUserMapper.deleteByUdfFuncId(id); putMsg(result, Status.SUCCESS); From 62f54eed7f25ee207bee1b4731b08263ab68aadc Mon Sep 17 00:00:00 2001 From: lilin Date: Fri, 27 Dec 2019 18:21:21 +0800 Subject: [PATCH 08/31] =?UTF-8?q?1=20Modify=20judgment=20order(=E8=B0=83?= =?UTF-8?q?=E6=95=B4=E9=A1=BA=E5=BA=8F=EF=BC=8C=E5=85=88=E5=88=A4=E6=96=AD?= =?UTF-8?q?=E6=98=AF=E5=90=A6=E5=AD=98=E5=9C=A8)=202=20before=20delete=20c?= =?UTF-8?q?heck=20entity=20exist?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../org/apache/dolphinscheduler/api/service/UdfFuncService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java index 7b4ce63e24..df439ce38f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java @@ -306,7 +306,7 @@ public class UdfFuncService extends BaseService{ //check exist UdfFunc udfFunc = udfFuncMapper.selectById(id); if (udfFunc == null) { - putMsg(result, Status.RESOURCE_NOT_EXIST); + putMsg(result, Status.UDF_FUNCTION_NOT_EXIST); return result; } udfFuncMapper.deleteById(id); From cfd3266ca24c4c287629495dff02f88f9f1323c7 Mon Sep 17 00:00:00 2001 From: Technoboy- Date: Sat, 28 Dec 2019 21:35:10 +0800 Subject: [PATCH 09/31] we should insert alert DB once , and trigger this type of alert 3 times --- .../apache/dolphinscheduler/server/master/MasterServer.java | 6 ++---- .../apache/dolphinscheduler/server/worker/WorkerServer.java | 4 +--- .../apache/dolphinscheduler/server/zk/ZKMasterClient.java | 6 ++---- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index e8c8b6779e..1f5b16a25a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -152,10 +152,8 @@ public class MasterServer implements IStoppable { @Override public void run() { if (zkMasterClient.getActiveMasterNum() <= 1) { - for (int i = 0; i < Constants.DOLPHINSCHEDULER_WARN_TIMES_FAILOVER; i++) { - zkMasterClient.getAlertDao().sendServerStopedAlert( - 1, OSUtils.getHost(), "Master-Server"); - } + zkMasterClient.getAlertDao().sendServerStopedAlert( + 1, OSUtils.getHost(), "Master-Server"); } stop("shutdownhook"); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index fe9c4991ac..ea8330e18a 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -177,9 +177,7 @@ public class WorkerServer implements IStoppable { public void run() { // worker server exit alert if (zkWorkerClient.getActiveMasterNum() <= 1) { - for (int i = 0; i < Constants.DOLPHINSCHEDULER_WARN_TIMES_FAILOVER; i++) { - alertDao.sendServerStopedAlert(1, OSUtils.getHost(), "Worker-Server"); - } + alertDao.sendServerStopedAlert(1, OSUtils.getHost(), "Worker-Server"); } stop("shutdownhook"); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java index a26a217665..1c7b737f5b 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java @@ -247,10 +247,8 @@ public class ZKMasterClient extends AbstractZKClient { */ private void alertServerDown(String serverHost, ZKNodeType zkNodeType) { - String serverType = zkNodeType.toString(); - for (int i = 0; i < Constants.DOLPHINSCHEDULER_WARN_TIMES_FAILOVER; i++) { - alertDao.sendServerStopedAlert(1, serverHost, serverType); - } + String serverType = zkNodeType.toString(); + alertDao.sendServerStopedAlert(1, serverHost, serverType); } /** From b053322466761354b56bb760b3a32d4d127a60a9 Mon Sep 17 00:00:00 2001 From: Technoboy- Date: Sun, 29 Dec 2019 16:54:40 +0800 Subject: [PATCH 10/31] refactor AbstractZKClient --- .../common/zk/AbstractZKClient.java | 96 +++++++------------ 1 file changed, 36 insertions(+), 60 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java index 0e95dddb36..c3ba718270 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java @@ -16,6 +16,22 @@ */ package org.apache.dolphinscheduler.common.zk; +import static org.apache.dolphinscheduler.common.Constants.ADD_ZK_OP; +import static org.apache.dolphinscheduler.common.Constants.DELETE_ZK_OP; +import static org.apache.dolphinscheduler.common.Constants.MASTER_PREFIX; +import static org.apache.dolphinscheduler.common.Constants.SINGLE_SLASH; +import static org.apache.dolphinscheduler.common.Constants.UNDERLINE; +import static org.apache.dolphinscheduler.common.Constants.WORKER_PREFIX; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.imps.CuratorFrameworkState; +import org.apache.curator.framework.recipes.locks.InterProcessMutex; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.IStoppable; import org.apache.dolphinscheduler.common.enums.ZKNodeType; @@ -23,26 +39,9 @@ import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.ResInfo; -import org.apache.commons.configuration.Configuration; -import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.configuration.PropertiesConfiguration; -import org.apache.commons.lang3.StringUtils; -import org.apache.curator.RetryPolicy; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.framework.imps.CuratorFrameworkState; -import org.apache.curator.framework.recipes.locks.InterProcessMutex; -import org.apache.curator.framework.state.ConnectionState; -import org.apache.curator.framework.state.ConnectionStateListener; -import org.apache.curator.retry.ExponentialBackoffRetry; -import org.apache.zookeeper.CreateMode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; - -import static org.apache.dolphinscheduler.common.Constants.*; - /** * abstract zookeeper client @@ -70,8 +69,7 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ return; } - byte[] bytes = zkClient.getData().forPath(znode); - String resInfoStr = new String(bytes); + String resInfoStr = super.get(znode); String[] splits = resInfoStr.split(Constants.COMMA); if (splits.length != Constants.HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH){ return; @@ -107,8 +105,7 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ String type = serverType.equals(MASTER_PREFIX) ? MASTER_PREFIX : WORKER_PREFIX; String deadServerPath = getDeadZNodeParentPath() + SINGLE_SLASH + type + UNDERLINE + ipSeqNo; - if(zkClient.checkExists().forPath(zNode) == null || - zkClient.checkExists().forPath(deadServerPath) != null ){ + if(!isExisted(zNode) || isExisted(deadServerPath)){ return true; } @@ -118,14 +115,12 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ public void removeDeadServerByHost(String host, String serverType) throws Exception { - List deadServers = zkClient.getChildren().forPath(getDeadZNodeParentPath()); + List deadServers = super.getChildrenKeys(getDeadZNodeParentPath()); for(String serverPath : deadServers){ if(serverPath.startsWith(serverType+UNDERLINE+host)){ - String server = getDeadZNodeParentPath() + SINGLE_SLASH + serverPath; - if(zkClient.checkExists().forPath(server) != null){ - zkClient.delete().forPath(server); - logger.info("{} server {} deleted from zk dead server path success" , serverType , host); - } + String server = getDeadZNodeParentPath() + SINGLE_SLASH + serverPath; + super.remove(server); + logger.info("{} server {} deleted from zk dead server path success" , serverType , host); } } } @@ -143,8 +138,8 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ // create temporary sequence nodes for master znode String parentPath = getZNodeParentPath(zkNodeType); String serverPathPrefix = parentPath + "/" + OSUtils.getHost(); - String registerPath = zkClient.create().withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath( - serverPathPrefix + UNDERLINE, heartbeatZKInfo.getBytes()); + String registerPath = serverPathPrefix + UNDERLINE; + super.persistEphemeral(registerPath, heartbeatZKInfo); logger.info("register {} node {} success" , zkNodeType.toString(), registerPath); return registerPath; } @@ -165,7 +160,7 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ } registerPath = createZNodePath(zkNodeType); - // handle dead server + // handle dead server handleDeadServer(registerPath, zkNodeType, Constants.DELETE_ZK_OP); return registerPath; @@ -196,10 +191,10 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ }else if(opType.equals(ADD_ZK_OP)){ String deadServerPath = getDeadZNodeParentPath() + SINGLE_SLASH + type + UNDERLINE + ipSeqNo; - if(zkClient.checkExists().forPath(deadServerPath) == null){ + if(!super.isExisted(deadServerPath)){ //add dead server info to zk dead server path : /dead-servers/ - zkClient.create().forPath(deadServerPath,(type + UNDERLINE + ipSeqNo).getBytes()); + super.persist(deadServerPath,(type + UNDERLINE + ipSeqNo)); logger.info("{} server dead , and {} added to zk dead server path success" , zkNodeType.toString(), zNode); @@ -226,19 +221,13 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ List childrenList = new ArrayList<>(); try { // read master node parent path from conf - if(zkClient.checkExists().forPath(getZNodeParentPath(ZKNodeType.MASTER)) != null){ - childrenList = zkClient.getChildren().forPath(getZNodeParentPath(ZKNodeType.MASTER)); + if(super.isExisted(getZNodeParentPath(ZKNodeType.MASTER))){ + childrenList = super.getChildrenKeys(getZNodeParentPath(ZKNodeType.MASTER)); } } catch (Exception e) { - if(e.getMessage().contains("java.lang.IllegalStateException: instance must be started")){ - logger.error("zookeeper service not started",e); - }else{ - logger.error(e.getMessage(),e); - } - - }finally { - return childrenList.size(); + logger.error("getActiveMasterNum error",e); } + return childrenList.size(); } /** @@ -280,10 +269,9 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ Map masterMap = new HashMap<>(); try { String path = getZNodeParentPath(zkNodeType); - List serverList = getZkClient().getChildren().forPath(path); + List serverList = super.getChildrenKeys(path); for(String server : serverList){ - byte[] bytes = getZkClient().getData().forPath(path + "/" + server); - masterMap.putIfAbsent(server, new String(bytes)); + masterMap.putIfAbsent(server, super.get(path + "/" + server)); } } catch (Exception e) { logger.error("get server list failed : " + e.getMessage(), e); @@ -430,27 +418,15 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ */ protected void initSystemZNode(){ try { - createNodePath(getMasterZNodeParentPath()); - createNodePath(getWorkerZNodeParentPath()); - createNodePath(getDeadZNodeParentPath()); + persist(getMasterZNodeParentPath(), ""); + persist(getWorkerZNodeParentPath(), ""); + persist(getDeadZNodeParentPath(), ""); } catch (Exception e) { logger.error("init system znode failed : " + e.getMessage(),e); } } - /** - * create zookeeper node path if not exists - * @param zNodeParentPath zookeeper parent path - * @throws Exception errors - */ - private void createNodePath(String zNodeParentPath) throws Exception { - if(null == zkClient.checkExists().forPath(zNodeParentPath)){ - zkClient.create().creatingParentContainersIfNeeded() - .withMode(CreateMode.PERSISTENT).forPath(zNodeParentPath); - } - } - /** * server self dead, stop all threads * @param serverHost server host From 503be5f4007cc05bdcab268eb6f63c5cd72344fb Mon Sep 17 00:00:00 2001 From: samz406 Date: Mon, 30 Dec 2019 17:24:34 +0800 Subject: [PATCH 11/31] add service UT (#1637) * User update not check params * user phone update when noteEmpty * modify saveWorkerGroup may NPE * add some service UT * add service ut include --- .../api/service/AccessTokenServiceTest.java | 181 ++++++++ .../api/service/AlertGroupServiceTest.java | 219 +++++++++ .../api/service/MonitorServiceTest.java | 101 ++++ .../api/service/ProjectServiceTest.java | 356 ++++++++++++++ .../api/service/QueueServiceTest.java | 208 +++++++++ .../api/service/SessionServiceTest.java | 121 ++++- .../api/service/TenantServiceTest.java | 225 ++++++++- .../api/service/UsersServiceTest.java | 434 +++++++++++++++++- .../api/service/WorkerGroupServiceTest.java | 173 +++++++ pom.xml | 9 + 10 files changed, 1990 insertions(+), 37 deletions(-) create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/QueueServiceTest.java create mode 100644 dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java new file mode 100644 index 0000000000..a9a5f67b0b --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AccessTokenServiceTest.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service; + +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.apache.commons.lang3.time.DateUtils; +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.dao.entity.AccessToken; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.AccessTokenMapper; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class AccessTokenServiceTest { + + + private static final Logger logger = LoggerFactory.getLogger(AccessTokenServiceTest.class); + + + @InjectMocks + private AccessTokenService accessTokenService ; + + @Mock + private AccessTokenMapper accessTokenMapper; + + @Before + public void setUp() { + + } + + + @After + public void after(){ + + } + + + + @Test + public void testQueryAccessTokenList(){ + + IPage tokenPage = new Page<>(); + tokenPage.setRecords(getList()); + tokenPage.setTotal(1L); + when(accessTokenMapper.selectAccessTokenPage(any(Page.class),eq("zhangsan"),eq(0))).thenReturn(tokenPage); + + User user =new User(); + Map result = accessTokenService.queryAccessTokenList(user,"zhangsan",1,10); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + PageInfo pageInfo = (PageInfo) result.get(Constants.DATA_LIST); + Assert.assertTrue(pageInfo.getTotalCount()>0); + } + + @Test + public void testCreateToken(){ + + + when(accessTokenMapper.insert(any(AccessToken.class))).thenReturn(2); + Map result = accessTokenService.createToken(1,getDate(),"AccessTokenServiceTest"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } + + @Test + public void testGenerateToken(){ + + Map result = accessTokenService.generateToken(Integer.MAX_VALUE,getDate()); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + String token = (String) result.get(Constants.DATA_LIST); + Assert.assertNotNull(token); + } + + @Test + public void testDelAccessTokenById(){ + + when(accessTokenMapper.selectById(1)).thenReturn(getEntity()); + User userLogin = new User(); + // not exist + Map result = accessTokenService.delAccessTokenById(userLogin,0); + logger.info(result.toString()); + Assert.assertEquals(Status.ACCESS_TOKEN_NOT_EXIST,result.get(Constants.STATUS)); + // no operate + result = accessTokenService.delAccessTokenById(userLogin,1); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + //success + userLogin.setId(1); + userLogin.setUserType(UserType.ADMIN_USER); + result = accessTokenService.delAccessTokenById(userLogin,1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } + + @Test + public void testUpdateToken(){ + + when(accessTokenMapper.selectById(1)).thenReturn(getEntity()); + Map result = accessTokenService.updateToken(1,Integer.MAX_VALUE,getDate(),"token"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + // not exist + result = accessTokenService.updateToken(2,Integer.MAX_VALUE,getDate(),"token"); + logger.info(result.toString()); + Assert.assertEquals(Status.ACCESS_TOKEN_NOT_EXIST,result.get(Constants.STATUS)); + + } + + /** + * create entity + * @return + */ + private AccessToken getEntity(){ + AccessToken accessToken = new AccessToken(); + accessToken.setId(1); + accessToken.setUserId(1); + accessToken.setToken("AccessTokenServiceTest"); + Date date = DateUtils.addDays(new Date(),30); + accessToken.setExpireTime(date); + return accessToken; + } + + /** + * entity list + * @return + */ + private List getList(){ + + List list = new ArrayList<>(); + list.add(getEntity()); + return list; + } + + + + /** + * get dateStr + * @return + */ + private String getDate(){ + Date date = DateUtils.addDays(new Date(),30); + return org.apache.dolphinscheduler.common.utils.DateUtils.dateToString(date); + } +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java new file mode 100644 index 0000000000..4a31902af9 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/AlertGroupServiceTest.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service; + +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.AlertType; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.dao.entity.AlertGroup; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.AlertGroupMapper; +import org.apache.dolphinscheduler.dao.mapper.UserAlertGroupMapper; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; + +@RunWith(MockitoJUnitRunner.class) +public class AlertGroupServiceTest { + + private static final Logger logger = LoggerFactory.getLogger(AlertGroupServiceTest.class); + + @InjectMocks + private AlertGroupService alertGroupService; + @Mock + private AlertGroupMapper alertGroupMapper; + @Mock + private UserAlertGroupMapper userAlertGroupMapper; + + private String groupName = "AlertGroupServiceTest"; + + @Before + public void setUp() { + } + + + @After + public void after(){ + + } + + + + @Test + public void testQueryAlertgroup(){ + + Mockito.when(alertGroupMapper.queryAllGroupList()).thenReturn(getList()); + HashMap result= alertGroupService.queryAlertgroup(); + logger.info(result.toString()); + List alertGroups = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(alertGroups)); + } + @Test + public void testListPaging(){ + IPage page = new Page<>(1,10); + page.setTotal(1L); + page.setRecords(getList()); + Mockito.when(alertGroupMapper.queryAlertGroupPage(any(Page.class),eq(groupName))).thenReturn(page); + User user = new User(); + // no operate + Map result = alertGroupService.listPaging(user,groupName,1,10); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + //success + user.setUserType(UserType.ADMIN_USER); + result = alertGroupService.listPaging(user,groupName,1,10); + logger.info(result.toString()); + PageInfo pageInfo = (PageInfo) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists())); + + } + @Test + public void testCreateAlertgroup(){ + + + Mockito.when(alertGroupMapper.insert(any(AlertGroup.class))).thenReturn(2); + User user = new User(); + //no operate + Map result = alertGroupService.createAlertgroup(user,groupName, AlertType.EMAIL,groupName); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + user.setUserType(UserType.ADMIN_USER); + //success + result = alertGroupService.createAlertgroup(user,groupName, AlertType.EMAIL,groupName); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } + @Test + public void testUpdateAlertgroup(){ + + User user = new User(); + // no operate + Map result = alertGroupService.updateAlertgroup(user,1,groupName, AlertType.SMS,groupName); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + user.setUserType(UserType.ADMIN_USER); + // not exist + result = alertGroupService.updateAlertgroup(user,1,groupName, AlertType.SMS,groupName); + logger.info(result.toString()); + Assert.assertEquals(Status.ALERT_GROUP_NOT_EXIST,result.get(Constants.STATUS)); + //success + Mockito.when(alertGroupMapper.selectById(2)).thenReturn(getEntity()); + result = alertGroupService.updateAlertgroup(user,2,groupName, AlertType.SMS,groupName); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + + } + @Test + public void testDelAlertgroupById(){ + + User user = new User(); + // no operate + Map result = alertGroupService.delAlertgroupById(user,1); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + user.setUserType(UserType.ADMIN_USER); + // not exist + result = alertGroupService.delAlertgroupById(user,2); + logger.info(result.toString()); + Assert.assertEquals(Status.ALERT_GROUP_NOT_EXIST,result.get(Constants.STATUS)); + //success + Mockito.when(alertGroupMapper.selectById(2)).thenReturn(getEntity()); + result = alertGroupService.delAlertgroupById(user,2); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + + + } + @Test + public void testGrantUser(){ + + Map result = alertGroupService.grantUser(getLoginUser(),1,"123,321"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } + @Test + public void testVerifyGroupName(){ + //group name not exist + Result result = alertGroupService.verifyGroupName(getLoginUser(), groupName); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + Mockito.when(alertGroupMapper.queryByGroupName(groupName)).thenReturn(getList()); + + //group name exist + result = alertGroupService.verifyGroupName(getLoginUser(), groupName); + logger.info(result.toString()); + Assert.assertEquals(Status.ALERT_GROUP_EXIST.getMsg(),result.getMsg()); + } + + + /** + * create admin user + * @return + */ + private User getLoginUser(){ + + User loginUser = new User(); + loginUser.setUserType(UserType.ADMIN_USER); + loginUser.setId(99999999); + return loginUser; + } + + /** + * get list + * @return + */ + private List getList(){ + List alertGroups = new ArrayList<>(); + alertGroups.add(getEntity()); + return alertGroups; + } + + /** + * get entity + * @return + */ + private AlertGroup getEntity(){ + AlertGroup alertGroup = new AlertGroup(); + alertGroup.setId(1); + alertGroup.setGroupName(groupName); + alertGroup.setGroupType(AlertType.EMAIL); + return alertGroup; + } + +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java new file mode 100644 index 0000000000..b155d5959a --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/MonitorServiceTest.java @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service; + +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.model.Server; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.dao.MonitorDBDao; +import org.apache.dolphinscheduler.dao.entity.MonitorRecord; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@RunWith(MockitoJUnitRunner.class) +public class MonitorServiceTest { + + private static final Logger logger = LoggerFactory.getLogger(MonitorServiceTest.class); + + @InjectMocks + private MonitorService monitorService; + @Mock + private MonitorDBDao monitorDBDao; + + + @Test + public void testQueryDatabaseState(){ + + Mockito.when(monitorDBDao.queryDatabaseState()).thenReturn(getList()); + Map result = monitorService.queryDatabaseState(null); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + List monitorRecordList = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(monitorRecordList)); + } + @Test + public void testQueryMaster(){ + //TODO need zk +// Map result = monitorService.queryMaster(null); +// logger.info(result.toString()); +// Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } + @Test + public void testQueryZookeeperState(){ + //TODO need zk +// Map result = monitorService.queryZookeeperState(null); +// logger.info(result.toString()); +// Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } + + @Test + public void testGetServerListFromZK(){ + //TODO need zk +// List serverList = monitorService.getServerListFromZK(true); +// logger.info(serverList.toString()); + } + + private List getList(){ + List monitorRecordList = new ArrayList<>(); + monitorRecordList.add(getEntity()); + return monitorRecordList; + } + + private MonitorRecord getEntity(){ + MonitorRecord monitorRecord = new MonitorRecord(); + monitorRecord.setDbType(DbType.MYSQL); + return monitorRecord; + } + + private List getServerList(){ + List servers = new ArrayList<>(); + servers.add(new Server()); + return servers; + } + +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java new file mode 100644 index 0000000000..51f9e148d1 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ProjectServiceTest.java @@ -0,0 +1,356 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service; + +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.entity.Project; +import org.apache.dolphinscheduler.dao.entity.ProjectUser; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; +import org.apache.dolphinscheduler.dao.mapper.ProjectMapper; +import org.apache.dolphinscheduler.dao.mapper.ProjectUserMapper; +import org.apache.dolphinscheduler.dao.mapper.UserMapper; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@RunWith(MockitoJUnitRunner.class) +public class ProjectServiceTest { + + + private static final Logger logger = LoggerFactory.getLogger(ProjectServiceTest.class); + + @InjectMocks + private ProjectService projectService; + @Mock + private ProjectMapper projectMapper; + @Mock + private UserMapper userMapper; + @Mock + private ProjectUserMapper projectUserMapper; + @Mock + private ProcessDefinitionMapper processDefinitionMapper; + + + + private String projectName = "ProjectServiceTest"; + + private String userName = "ProjectServiceTest"; + + @Before + public void setUp() { + + } + + + @After + public void after(){ + + } + + @Test + public void testCreateProject(){ + + User loginUser = getLoginUser(); + loginUser.setId(1); + Map result = projectService.createProject(loginUser, projectName, getDesc()); + logger.info(result.toString()); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR,result.get(Constants.STATUS)); + + //project name exist + Mockito.when(projectMapper.queryByName(projectName)).thenReturn(getProject()); + result = projectService.createProject(loginUser, projectName, projectName); + logger.info(result.toString()); + Assert.assertEquals(Status.PROJECT_ALREADY_EXISTS,result.get(Constants.STATUS)); + + //success + Mockito.when(projectMapper.insert(Mockito.any(Project.class))).thenReturn(1); + result = projectService.createProject(loginUser, "test", "test"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + + + } + @Test + public void testQueryById(){ + + //not exist + Map result = projectService.queryById(Integer.MAX_VALUE); + Assert.assertEquals(Status.PROJECT_NOT_FOUNT,result.get(Constants.STATUS)); + logger.info(result.toString()); + + //success + Mockito.when(projectMapper.selectById(1)).thenReturn(getProject()); + result = projectService.queryById(1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + + } + @Test + public void testCheckProjectAndAuth(){ + + Mockito.when(projectUserMapper.queryProjectRelation(1, 1)).thenReturn(getProjectUser()); + User loginUser = getLoginUser(); + + Map result = projectService.checkProjectAndAuth(loginUser,null,projectName); + logger.info(result.toString()); + Status status = (Status)result.get(Constants.STATUS); + Assert.assertEquals(Status.PROJECT_NOT_FOUNT,result.get(Constants.STATUS)); + + Project project = getProject(); + //USER_NO_OPERATION_PROJECT_PERM + project.setUserId(2); + result = projectService.checkProjectAndAuth(loginUser,project,projectName); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PROJECT_PERM,result.get(Constants.STATUS)); + + //success + project.setUserId(1); + result = projectService.checkProjectAndAuth(loginUser,project,projectName); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + + } + + @Test + public void testHasProjectAndPerm(){ + + // Mockito.when(projectUserMapper.queryProjectRelation(1, 1)).thenReturn(getProjectUser()); + User loginUser = getLoginUser(); + Project project = getProject(); + Map result = new HashMap<>(); + // not exist user + User tempUser = new User(); + tempUser.setId(Integer.MAX_VALUE); + boolean checkResult = projectService.hasProjectAndPerm(tempUser,project,result); + logger.info(result.toString()); + Assert.assertFalse(checkResult); + + //success + result = new HashMap<>(); + project.setUserId(1); + checkResult = projectService.hasProjectAndPerm(loginUser,project,result); + logger.info(result.toString()); + Assert.assertTrue(checkResult); + } + @Test + public void testQueryProjectListPaging(){ + + IPage page = new Page<>(1,10); + page.setRecords(getList()); + page.setTotal(1L); + Mockito.when(projectMapper.queryProjectListPaging(Mockito.any(Page.class), Mockito.eq(1), Mockito.eq(projectName))).thenReturn(page); + User loginUser = getLoginUser(); + + // project owner + Map result = projectService.queryProjectListPaging(loginUser,10,1,projectName); + logger.info(result.toString()); + PageInfo pageInfo = (PageInfo) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists())); + + //admin + Mockito.when(projectMapper.queryProjectListPaging(Mockito.any(Page.class), Mockito.eq(0), Mockito.eq(projectName))).thenReturn(page); + loginUser.setUserType(UserType.ADMIN_USER); + result = projectService.queryProjectListPaging(loginUser,10,1,projectName); + logger.info(result.toString()); + pageInfo = (PageInfo) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists())); + } + @Test + public void testDeleteProject(){ + + Mockito.when(projectMapper.selectById(1)).thenReturn(getProject()); + User loginUser = getLoginUser(); + //PROJECT_NOT_FOUNT + Map result= projectService.deleteProject(loginUser,12); + logger.info(result.toString()); + Assert.assertEquals(Status.PROJECT_NOT_FOUNT,result.get(Constants.STATUS)); + loginUser.setId(2); + //USER_NO_OPERATION_PROJECT_PERM + result= projectService.deleteProject(loginUser,1); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PROJECT_PERM,result.get(Constants.STATUS)); + + //DELETE_PROJECT_ERROR_DEFINES_NOT_NULL + Mockito.when(processDefinitionMapper.queryAllDefinitionList(1)).thenReturn(getProcessDefinitions()); + loginUser.setUserType(UserType.ADMIN_USER); + result= projectService.deleteProject(loginUser,1); + logger.info(result.toString()); + Assert.assertEquals(Status.DELETE_PROJECT_ERROR_DEFINES_NOT_NULL,result.get(Constants.STATUS)); + + //success + Mockito.when(projectMapper.deleteById(1)).thenReturn(1); + Mockito.when(processDefinitionMapper.queryAllDefinitionList(1)).thenReturn(new ArrayList<>()); + result= projectService.deleteProject(loginUser,1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + + + } + + @Test + public void testUpdate(){ + + User loginUser = getLoginUser(); + Project project = getProject(); + project.setId(2); + Mockito.when(projectMapper.queryByName(projectName)).thenReturn(project); + Mockito.when( projectMapper.selectById(1)).thenReturn(getProject()); + // PROJECT_NOT_FOUNT + Map result = projectService.update(loginUser,12,projectName,"desc"); + logger.info(result.toString()); + Assert.assertEquals(Status.PROJECT_NOT_FOUNT,result.get(Constants.STATUS)); + + //PROJECT_ALREADY_EXISTS + result = projectService.update(loginUser,1,projectName,"desc"); + logger.info(result.toString()); + Assert.assertEquals(Status.PROJECT_ALREADY_EXISTS,result.get(Constants.STATUS)); + + //success + project.setUserId(1); + Mockito.when(projectMapper.updateById(Mockito.any(Project.class))).thenReturn(1); + result = projectService.update(loginUser,1,"test","desc"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + + } + @Test + public void testQueryAuthorizedProject(){ + + User loginUser = getLoginUser(); + + Mockito.when(projectMapper.queryAuthedProjectListByUserId(1)).thenReturn(getList()); + //USER_NO_OPERATION_PERM + Map result = projectService.queryAuthorizedProject(loginUser,3); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM,result.get(Constants.STATUS)); + + //success + loginUser.setUserType(UserType.ADMIN_USER); + result = projectService.queryAuthorizedProject(loginUser,1); + logger.info(result.toString()); + List projects = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(projects)); + + } + @Test + public void testQueryAllProjectList(){ + + Mockito.when(projectMapper.selectList(null)).thenReturn(getList()); + Mockito.when(processDefinitionMapper.selectList(null)).thenReturn(getProcessDefinitions()); + + Map result = projectService.queryAllProjectList(); + logger.info(result.toString()); + List projects = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(projects)); + + } + @Test + public void testQueryUnauthorizedProject(){ + // Mockito.when(projectMapper.queryAuthedProjectListByUserId(1)).thenReturn(getList()); + Mockito.when(projectMapper.queryProjectExceptUserId(2)).thenReturn(getList()); + + User loginUser = new User(); + loginUser.setUserType(UserType.ADMIN_USER); + + Map result = projectService.queryUnauthorizedProject(loginUser,2); + logger.info(result.toString()); + List projects = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(projects)); + } + + + private Project getProject(){ + Project project = new Project(); + project.setId(1); + project.setName(projectName); + project.setUserId(1); + return project; + } + + private List getList(){ + List list = new ArrayList<>(); + list.add(getProject()); + return list; + } + + + /** + * create admin user + * @return + */ + private User getLoginUser(){ + + User loginUser = new User(); + loginUser.setUserType(UserType.GENERAL_USER); + loginUser.setUserName(userName); + loginUser.setId(1); + return loginUser; + + } + + /** + * get project user + + */ + private ProjectUser getProjectUser(){ + ProjectUser projectUser = new ProjectUser(); + projectUser.setProjectId(1); + projectUser.setUserId(1); + return projectUser; + } + + private List getProcessDefinitions(){ + List list = new ArrayList<>(); + ProcessDefinition processDefinition = new ProcessDefinition(); + processDefinition.setProjectId(1); + list.add(processDefinition); + return list; + } + + + + + private String getDesc(){ + return "projectUserMapper.deleteProjectRelation(projectId,userId)projectUserMappe" + + ".deleteProjectRelation(projectId,userId)projectUserMappe" + + "r.deleteProjectRelation(projectId,userId)projectUserMapper" + + ".deleteProjectRelation(projectId,userId)projectUserMapper.deleteProjectRelation(projectId,userId)"; + } + + +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/QueueServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/QueueServiceTest.java new file mode 100644 index 0000000000..dbae95b181 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/QueueServiceTest.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service; + +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.api.utils.Result; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.dao.entity.Queue; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.QueueMapper; +import org.apache.dolphinscheduler.dao.mapper.UserMapper; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@RunWith(MockitoJUnitRunner.class) +public class QueueServiceTest { + + private static final Logger logger = LoggerFactory.getLogger(QueueServiceTest.class); + + @InjectMocks + private QueueService queueService; + @Mock + private QueueMapper queueMapper; + @Mock + private UserMapper userMapper; + private String queueName = "QueueServiceTest"; + + @Before + public void setUp() { + } + + + @After + public void after(){ + } + + @Test + public void testQueryList(){ + + Mockito.when(queueMapper.selectList(null)).thenReturn(getQueueList()); + Map result = queueService.queryList(getLoginUser()); + logger.info(result.toString()); + List queueList = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(queueList)); + + } + @Test + public void testQueryListPage(){ + + IPage page = new Page<>(1,10); + page.setTotal(1L); + page.setRecords(getQueueList()); + Mockito.when(queueMapper.queryQueuePaging(Mockito.any(Page.class), Mockito.eq(queueName))).thenReturn(page); + Map result = queueService.queryList(getLoginUser(),queueName,1,10); + logger.info(result.toString()); + PageInfo pageInfo = (PageInfo) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists())); + } + @Test + public void testCreateQueue(){ + + // queue is null + Map result = queueService.createQueue(getLoginUser(),null,queueName); + logger.info(result.toString()); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR,result.get(Constants.STATUS)); + // queueName is null + result = queueService.createQueue(getLoginUser(),queueName,null); + logger.info(result.toString()); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR,result.get(Constants.STATUS)); + // correct + result = queueService.createQueue(getLoginUser(),queueName,queueName); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + + } + @Test + public void testUpdateQueue(){ + + Mockito.when(queueMapper.selectById(1)).thenReturn(getQueue()); + Mockito.when(queueMapper.queryAllQueueList("test", null)).thenReturn(getQueueList()); + Mockito.when(queueMapper.queryAllQueueList(null, "test")).thenReturn(getQueueList()); + Mockito.when(userMapper.queryUserListByQueue(queueName)).thenReturn(getUserList()); + + // not exist + Map result = queueService.updateQueue(getLoginUser(),0,"queue",queueName); + logger.info(result.toString()); + Assert.assertEquals(Status.QUEUE_NOT_EXIST.getCode(),((Status)result.get(Constants.STATUS)).getCode()); + //no need update + result = queueService.updateQueue(getLoginUser(),1,queueName,queueName); + logger.info(result.toString()); + Assert.assertEquals(Status.NEED_NOT_UPDATE_QUEUE.getCode(),((Status)result.get(Constants.STATUS)).getCode()); + //queue exist + result = queueService.updateQueue(getLoginUser(),1,"test",queueName); + logger.info(result.toString()); + Assert.assertEquals(Status.QUEUE_VALUE_EXIST.getCode(),((Status)result.get(Constants.STATUS)).getCode()); + // queueName exist + result = queueService.updateQueue(getLoginUser(),1,"test1","test"); + logger.info(result.toString()); + Assert.assertEquals(Status.QUEUE_NAME_EXIST.getCode(),((Status)result.get(Constants.STATUS)).getCode()); + //success + result = queueService.updateQueue(getLoginUser(),1,"test1","test1"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getCode(),((Status)result.get(Constants.STATUS)).getCode()); + + } + @Test + public void testVerifyQueue(){ + + Mockito.when(queueMapper.queryAllQueueList(queueName, null)).thenReturn(getQueueList()); + Mockito.when(queueMapper.queryAllQueueList(null, queueName)).thenReturn(getQueueList()); + + //queue null + Result result = queueService.verifyQueue(null,queueName); + logger.info(result.toString()); + Assert.assertEquals(result.getCode().intValue(), Status.REQUEST_PARAMS_NOT_VALID_ERROR.getCode()); + + //queueName null + result = queueService.verifyQueue(queueName,null); + logger.info(result.toString()); + Assert.assertEquals(result.getCode().intValue(), Status.REQUEST_PARAMS_NOT_VALID_ERROR.getCode()); + + //exist queueName + result = queueService.verifyQueue(queueName,queueName); + logger.info(result.toString()); + Assert.assertEquals(result.getCode().intValue(), Status.QUEUE_NAME_EXIST.getCode()); + + //exist queue + result = queueService.verifyQueue(queueName,"test"); + logger.info(result.toString()); + Assert.assertEquals(result.getCode().intValue(), Status.QUEUE_VALUE_EXIST.getCode()); + + // success + result = queueService.verifyQueue("test","test"); + logger.info(result.toString()); + Assert.assertEquals(result.getCode().intValue(), Status.SUCCESS.getCode()); + + + } + /** + * create admin user + * @return + */ + private User getLoginUser(){ + + User loginUser = new User(); + loginUser.setUserType(UserType.ADMIN_USER); + loginUser.setId(99999999); + return loginUser; + } + + private List getUserList(){ + List list = new ArrayList<>(); + list.add(getLoginUser()); + return list; + } + + + /** + * get queue + * @return + */ + private Queue getQueue(){ + Queue queue = new Queue(); + queue.setId(1); + queue.setQueue(queueName); + queue.setQueueName(queueName); + return queue; + } + + private List getQueueList(){ + List queueList = new ArrayList<>(); + queueList.add(getQueue()); + return queueList; + } + +} diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/SessionServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/SessionServiceTest.java index 1920586706..72545a347c 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/SessionServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/SessionServiceTest.java @@ -16,37 +16,130 @@ */ package org.apache.dolphinscheduler.api.service; -import org.apache.dolphinscheduler.api.ApiApplicationServer; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DateUtils; +import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.dao.entity.Session; import org.apache.dolphinscheduler.dao.entity.User; -import org.apache.commons.lang3.StringUtils; +import org.apache.dolphinscheduler.dao.mapper.SessionMapper; +import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.mock.web.MockCookie; +import org.springframework.mock.web.MockHttpServletRequest; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + -@RunWith(SpringRunner.class) -@SpringBootTest(classes = ApiApplicationServer.class) +@RunWith(MockitoJUnitRunner.class) public class SessionServiceTest { private static final Logger logger = LoggerFactory.getLogger(SessionServiceTest.class); - @Autowired + @InjectMocks private SessionService sessionService; + @Mock + private SessionMapper sessionMapper; + + private String sessionId ="aaaaaaaaaaaaaaaaaa"; + + @Before + public void setUp() { + } + + + @After + public void after(){ + } + + /** + * create session + */ + @Test + public void testGetSession(){ + + + Mockito.when(sessionMapper.selectById(sessionId)).thenReturn(getSession()); + // get sessionId from header + MockHttpServletRequest mockHttpServletRequest = new MockHttpServletRequest(); + mockHttpServletRequest.addHeader(Constants.SESSION_ID,sessionId); + mockHttpServletRequest.addHeader("HTTP_X_FORWARDED_FOR","127.0.0.1"); + //query + Session session = sessionService.getSession(mockHttpServletRequest); + Assert.assertNotNull(session); + logger.info("session ip {}",session.getIp()); + + // get sessionId from cookie + mockHttpServletRequest = new MockHttpServletRequest(); + mockHttpServletRequest.addHeader("HTTP_X_FORWARDED_FOR","127.0.0.1"); + MockCookie mockCookie = new MockCookie(Constants.SESSION_ID,sessionId); + mockHttpServletRequest.setCookies(mockCookie); + //query + session = sessionService.getSession(mockHttpServletRequest); + Assert.assertNotNull(session); + logger.info("session ip {}",session.getIp()); + Assert.assertEquals(session.getIp(),"127.0.0.1"); + + + } + + /** + * create session + */ + @Test + public void testCreateSession(){ + + String ip = "127.0.0.1"; + User user = new User(); + user.setUserType(UserType.GENERAL_USER); + user.setId(1); + Mockito.when(sessionMapper.queryByUserId(1)).thenReturn(getSessions()); + String sessionId = sessionService.createSession(user, ip); + logger.info("createSessionId is "+sessionId); + Assert.assertTrue(StringUtils.isNotEmpty(sessionId)); + } + /** + * sign out + * remove ip restrictions + */ @Test - public void createSession(){ + public void testSignOut(){ + + int userId = 88888888; + String ip = "127.0.0.1"; + User user = new User(); + user.setId(userId); + sessionService.signOut(ip ,user); - User loginUser = new User(); - loginUser.setId(1); - loginUser.setUserType(UserType.GENERAL_USER); + } - String session = sessionService.createSession(loginUser, "127.0.0.1"); - Assert.assertTrue(StringUtils.isNotEmpty(session)); + private Session getSession(){ + Session session = new Session(); + session.setId(sessionId); + session.setIp("127.0.0.1"); + session.setLastLoginTime(DateUtils.addDays(new Date(),40)); + session.setUserId(1); + return session; } + + private List getSessions(){ + List sessionList = new ArrayList<>(); + sessionList.add(getSession()); + return sessionList; + } + + } \ No newline at end of file diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java index ddd604ea88..31c8c0222d 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java @@ -16,38 +16,241 @@ */ package org.apache.dolphinscheduler.api.service; -import org.apache.dolphinscheduler.api.ApiApplicationServer; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper; +import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; +import org.apache.dolphinscheduler.dao.mapper.TenantMapper; +import org.apache.dolphinscheduler.dao.mapper.UserMapper; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; +import java.util.ArrayList; +import java.util.List; import java.util.Map; -@RunWith(SpringRunner.class) -@SpringBootTest(classes = ApiApplicationServer.class) +@RunWith(MockitoJUnitRunner.class) public class TenantServiceTest { private static final Logger logger = LoggerFactory.getLogger(TenantServiceTest.class); - @Autowired + @InjectMocks private TenantService tenantService; + @Mock + private TenantMapper tenantMapper; + @Mock + private ProcessDefinitionMapper processDefinitionMapper; + @Mock + private ProcessInstanceMapper processInstanceMapper; + @Mock + private UserMapper userMapper; + + private String tenantCode ="TenantServiceTest"; + private String tenantName ="TenantServiceTest"; + + + @Test + public void testCreateTenant(){ + + User loginUser = getLoginUser(); + Mockito.when(tenantMapper.queryByTenantCode(tenantCode)).thenReturn(getList()); + try { + //check tenantCode + Map result = tenantService.createTenant(getLoginUser(), "%!1111", tenantName, 1, "TenantServiceTest"); + logger.info(result.toString()); + Assert.assertEquals(Status.VERIFY_TENANT_CODE_ERROR,result.get(Constants.STATUS)); + + //check exist + result = tenantService.createTenant(loginUser, tenantCode, tenantName, 1, "TenantServiceTest"); + logger.info(result.toString()); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR,result.get(Constants.STATUS)); + + // success + result = tenantService.createTenant(loginUser, "test", "test", 1, "TenantServiceTest"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } catch (Exception e) { + logger.error("create tenant error",e); + Assert.assertTrue(false); + } + } + + @Test + public void testQueryTenantListPage(){ + + IPage page = new Page<>(1,10); + page.setRecords(getList()); + page.setTotal(1L); + Mockito.when(tenantMapper.queryTenantPaging(Mockito.any(Page.class), Mockito.eq("TenantServiceTest"))).thenReturn(page); + Map result = tenantService.queryTenantList(getLoginUser(), "TenantServiceTest", 1, 10); + logger.info(result.toString()); + PageInfo pageInfo = (PageInfo) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists())); + + } + + @Test + public void testUpdateTenant(){ + + Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); + try { + // id not exist + Map result = tenantService.updateTenant(getLoginUser(), 912222, tenantCode, tenantName, 1, "desc"); + logger.info(result.toString()); + // success + Assert.assertEquals(Status.TENANT_NOT_EXIST,result.get(Constants.STATUS)); + result = tenantService.updateTenant(getLoginUser(), 1, tenantCode, "TenantServiceTest001", 1, "desc"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } catch (Exception e) { + logger.error("update tenant error",e); + Assert.assertTrue(false); + } + + } @Test - public void queryTenantList(){ + public void testDeleteTenantById(){ + + Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); + Mockito.when(processInstanceMapper.queryByTenantIdAndStatus(1, Constants.NOT_TERMINATED_STATES)).thenReturn(getInstanceList()); + Mockito.when(processDefinitionMapper.queryDefinitionListByTenant(2)).thenReturn(getDefinitionsList()); + Mockito.when( userMapper.queryUserListByTenant(3)).thenReturn(getUserList()); + + try { + //TENANT_NOT_EXIST + Map result = tenantService.deleteTenantById(getLoginUser(),12); + logger.info(result.toString()); + Assert.assertEquals(Status.TENANT_NOT_EXIST,result.get(Constants.STATUS)); + + //DELETE_TENANT_BY_ID_FAIL + result = tenantService.deleteTenantById(getLoginUser(),1); + logger.info(result.toString()); + Assert.assertEquals(Status.DELETE_TENANT_BY_ID_FAIL,result.get(Constants.STATUS)); + + //DELETE_TENANT_BY_ID_FAIL_DEFINES + Mockito.when(tenantMapper.queryById(2)).thenReturn(getTenant(2)); + result = tenantService.deleteTenantById(getLoginUser(),2); + logger.info(result.toString()); + Assert.assertEquals(Status.DELETE_TENANT_BY_ID_FAIL_DEFINES,result.get(Constants.STATUS)); + + //DELETE_TENANT_BY_ID_FAIL_USERS + Mockito.when(tenantMapper.queryById(3)).thenReturn(getTenant(3)); + result = tenantService.deleteTenantById(getLoginUser(),3); + logger.info(result.toString()); + Assert.assertEquals(Status.DELETE_TENANT_BY_ID_FAIL_USERS,result.get(Constants.STATUS)); + + // success + Mockito.when(tenantMapper.queryById(4)).thenReturn(getTenant(4)); + result = tenantService.deleteTenantById(getLoginUser(),4); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } catch (Exception e) { + logger.error("delete tenant error",e); + Assert.assertTrue(false); + } + } + + @Test + public void testQueryTenantList(){ + + Mockito.when( tenantMapper.selectList(null)).thenReturn(getList()); + Map result = tenantService.queryTenantList(getLoginUser()); + logger.info(result.toString()); + List tenantList = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(tenantList)); + } + + @Test + public void testVerifyTenantCode(){ + + Mockito.when(tenantMapper.queryByTenantCode(tenantCode)).thenReturn(getList()); + // tenantCode not exist + Result result = tenantService.verifyTenantCode("s00000000000l887888885554444sfjdskfjslakslkdf"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + // tenantCode exist + result = tenantService.verifyTenantCode(getTenant().getTenantCode()); + logger.info(result.toString()); + Assert.assertEquals(Status.TENANT_NAME_EXIST.getMsg(),result.getMsg()); + } + + + /** + * get user + * @return + */ + private User getLoginUser(){ User loginUser = new User(); loginUser.setUserType(UserType.ADMIN_USER); - Map map = tenantService.queryTenantList(loginUser); - Assert.assertEquals(Status.SUCCESS, map.get(Constants.STATUS)); - logger.info(map.toString()); + return loginUser; + } + /** + * get list + * @return + */ + private List getList(){ + List tenantList = new ArrayList<>(); + tenantList.add(getTenant()); + return tenantList; } + + /** + * get tenant + * @return + */ + private Tenant getTenant(){ + return getTenant(1); + } + /** + * get tenant + * @return + */ + private Tenant getTenant(int id){ + Tenant tenant = new Tenant(); + tenant.setId(id); + tenant.setTenantCode(tenantCode); + tenant.setTenantName(tenantName); + return tenant; + } + + private List getUserList(){ + List userList = new ArrayList<>(); + userList.add(getLoginUser()); + return userList; + } + + private List getInstanceList(){ + List processInstances = new ArrayList<>(); + ProcessInstance processInstance = new ProcessInstance(); + processInstances.add(processInstance); + return processInstances; + } + + private List getDefinitionsList(){ + List processDefinitions = new ArrayList<>(); + ProcessDefinition processDefinition = new ProcessDefinition(); + processDefinitions.add(processDefinition); + return processDefinitions; + } + + } \ No newline at end of file diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java index 21f0f7f3b3..30aabe93f2 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java @@ -16,39 +16,449 @@ */ package org.apache.dolphinscheduler.api.service; -import org.apache.dolphinscheduler.api.ApiApplicationServer; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.EncryptionUtils; +import org.apache.dolphinscheduler.dao.entity.Tenant; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.*; +import org.junit.After; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; +import java.util.ArrayList; +import java.util.List; import java.util.Map; -@RunWith(SpringRunner.class) -@SpringBootTest(classes = ApiApplicationServer.class) +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) public class UsersServiceTest { private static final Logger logger = LoggerFactory.getLogger(UsersServiceTest.class); - @Autowired + @InjectMocks private UsersService usersService; + @Mock + private UserMapper userMapper; + @Mock + private TenantMapper tenantMapper; + @Mock + private ProjectUserMapper projectUserMapper; + @Mock + private ResourceUserMapper resourcesUserMapper; + @Mock + private UDFUserMapper udfUserMapper; + @Mock + private DataSourceUserMapper datasourceUserMapper; + @Mock + private AlertGroupMapper alertGroupMapper; + + private String queueName ="UsersServiceTestQueue"; + + + @Before + public void before(){ + + + } + @After + public void after(){ + + } + + + @Test + public void testCreateUser(){ + + User user = new User(); + user.setUserType(UserType.ADMIN_USER); + String userName = "userTest0001~"; + String userPassword = "userTest"; + String email = "123@qq.com"; + int tenantId = Integer.MAX_VALUE; + String phone= "13456432345"; + try { + //userName error + Map result = usersService.createUser(user, userName, userPassword, email, tenantId, phone, queueName); + logger.info(result.toString()); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS)); + + userName = "userTest0001"; + userPassword = "userTest000111111111111111"; + //password error + result = usersService.createUser(user, userName, userPassword, email, tenantId, phone, queueName); + logger.info(result.toString()); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS)); + + userPassword = "userTest0001"; + email = "1q.com"; + //email error + result = usersService.createUser(user, userName, userPassword, email, tenantId, phone, queueName); + logger.info(result.toString()); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS)); + + email = "122222@qq.com"; + phone ="2233"; + //phone error + result = usersService.createUser(user, userName, userPassword, email, tenantId, phone, queueName); + logger.info(result.toString()); + Assert.assertEquals(Status.REQUEST_PARAMS_NOT_VALID_ERROR, result.get(Constants.STATUS)); + + phone = "13456432345"; + //tenantId not exists + result = usersService.createUser(user, userName, userPassword, email, tenantId, phone, queueName); + logger.info(result.toString()); + Assert.assertEquals(Status.TENANT_NOT_EXIST, result.get(Constants.STATUS)); + //success + Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant()); + result = usersService.createUser(user, userName, userPassword, email, 1, phone, queueName); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + + } catch (Exception e) { + logger.error(Status.CREATE_USER_ERROR.getMsg(),e); + Assert.assertTrue(false); + } + } + + @Test + public void testQueryUser(){ + + String userName = "userTest0001"; + String userPassword = "userTest0001"; + when(userMapper.queryUserByNamePassword(userName,EncryptionUtils.getMd5(userPassword))).thenReturn(getGeneralUser()); + User queryUser = usersService.queryUser(userName, userPassword); + logger.info(queryUser.toString()); + Assert.assertTrue(queryUser!=null); + } + + + + @Test + public void testQueryUserList(){ + + + User user = new User(); + + //no operate + Map result = usersService.queryUserList(user); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); + + //success + user.setUserType(UserType.ADMIN_USER); + when(userMapper.selectList(null )).thenReturn(getUserList()); + result = usersService.queryUserList(user); + List userList = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(userList.size()>0); + } + + @Test + public void testQueryUserListPage(){ + + + User user = new User(); + IPage page = new Page<>(1,10); + page.setRecords(getUserList()); + when(userMapper.queryUserPaging(any(Page.class), eq("userTest"))).thenReturn(page); + + //no operate + Map result = usersService.queryUserList(user,"userTest",1,10); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); + + //success + user.setUserType(UserType.ADMIN_USER); + result = usersService.queryUserList(user,"userTest",1,10); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + PageInfo pageInfo = (PageInfo) result.get(Constants.DATA_LIST); + Assert.assertTrue(pageInfo.getLists().size()>0); + } + + @Test + public void testUpdateUser(){ + + String userName = "userTest0001"; + String userPassword = "userTest0001"; + try { + //user not exist + Map result = usersService.updateUser(0,userName,userPassword,"3443@qq.com",1,"13457864543","queue"); + Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS)); + logger.info(result.toString()); + + //success + when(userMapper.selectById(1)).thenReturn(getUser()); + result = usersService.updateUser(1,userName,userPassword,"32222s@qq.com",1,"13457864543","queue"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + } catch (Exception e) { + logger.error("update user error",e); + Assert.assertTrue(false); + } + } + + @Test + public void testDeleteUserById(){ + + User loginUser = new User(); + try { + when(userMapper.queryTenantCodeByUserId(1)).thenReturn(getUser()); + when(userMapper.selectById(1)).thenReturn(getUser()); + + //no operate + Map result = usersService.deleteUserById(loginUser,3); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); + + // user not exist + loginUser.setUserType(UserType.ADMIN_USER); + result = usersService.deleteUserById(loginUser,3); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS)); + + //success + result = usersService.deleteUserById(loginUser,1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + } catch (Exception e) { + logger.error("delete user error",e); + Assert.assertTrue(false); + } + + + } + + @Test + public void testGrantProject(){ + + when(userMapper.selectById(1)).thenReturn(getUser()); + User loginUser = new User(); + String projectIds= "100000,120000"; + Map result = usersService.grantProject(loginUser, 1, projectIds); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); + //user not exist + loginUser.setUserType(UserType.ADMIN_USER); + result = usersService.grantProject(loginUser, 2, projectIds); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS)); + //success + result = usersService.grantProject(loginUser, 1, projectIds); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + } + + @Test + public void testGrantResources(){ + + String resourceIds = "100000,120000"; + when(userMapper.selectById(1)).thenReturn(getUser()); + User loginUser = new User(); + Map result = usersService.grantResources(loginUser, 1, resourceIds); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); + //user not exist + loginUser.setUserType(UserType.ADMIN_USER); + result = usersService.grantResources(loginUser, 2, resourceIds); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS)); + //success + result = usersService.grantResources(loginUser, 1, resourceIds); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + } + + + @Test + public void testGrantUDFFunction(){ + + String udfIds = "100000,120000"; + when(userMapper.selectById(1)).thenReturn(getUser()); + User loginUser = new User(); + Map result = usersService.grantUDFFunction(loginUser, 1, udfIds); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); + //user not exist + loginUser.setUserType(UserType.ADMIN_USER); + result = usersService.grantUDFFunction(loginUser, 2, udfIds); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS)); + //success + result = usersService.grantUDFFunction(loginUser, 1, udfIds); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + } + + @Test + public void testGrantDataSource(){ + + String datasourceIds = "100000,120000"; + when(userMapper.selectById(1)).thenReturn(getUser()); + User loginUser = new User(); + Map result = usersService.grantDataSource(loginUser, 1, datasourceIds); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); + //user not exist + loginUser.setUserType(UserType.ADMIN_USER); + result = usersService.grantDataSource(loginUser, 2, datasourceIds); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NOT_EXIST, result.get(Constants.STATUS)); + //success + result = usersService.grantDataSource(loginUser, 1, datasourceIds); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + + } + @Test public void getUserInfo(){ User loginUser = new User(); - loginUser.setId(19); - loginUser.setUserType(UserType.GENERAL_USER); - Map map = usersService.getUserInfo(loginUser); - Assert.assertEquals(Status.SUCCESS, map.get(Constants.STATUS)); - logger.info(map.toString()); + loginUser.setUserName("admin"); + loginUser.setUserType(UserType.ADMIN_USER); + // get admin user + Map result = usersService.getUserInfo(loginUser); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + User tempUser = (User) result.get(Constants.DATA_LIST); + //check userName + Assert.assertEquals("admin",tempUser.getUserName()); + //get general user + loginUser.setUserType(null); + loginUser.setId(1); + when(userMapper.queryDetailsById(1)).thenReturn(getGeneralUser()); + result = usersService.getUserInfo(loginUser); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + tempUser = (User) result.get(Constants.DATA_LIST); + //check userName + Assert.assertEquals("userTest0001",tempUser.getUserName()); } + + + @Test + public void testQueryAllGeneralUsers(){ + + User loginUser = new User(); + //no operate + Map result = usersService.queryAllGeneralUsers(loginUser); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); + //success + loginUser.setUserType(UserType.ADMIN_USER); + when(userMapper.queryAllGeneralUser()).thenReturn(getUserList()); + result = usersService.queryAllGeneralUsers(loginUser); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + List userList = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(userList)); + } + + @Test + public void testVerifyUserName(){ + + //not exist user + Result result = usersService.verifyUserName("admin89899"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(), result.getMsg()); + //exist user + when(userMapper.queryByUserNameAccurately("userTest0001")).thenReturn(getUser()); + result = usersService.verifyUserName("userTest0001"); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NAME_EXIST.getMsg(), result.getMsg()); + } + + @Test + public void testUnauthorizedUser(){ + + User loginUser = new User(); + when(userMapper.selectList(null )).thenReturn(getUserList()); + when( userMapper.queryUserListByAlertGroupId(2)).thenReturn(getUserList()); + //no operate + Map result = usersService.unauthorizedUser(loginUser, 2); + logger.info(result.toString()); + loginUser.setUserType(UserType.ADMIN_USER); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); + //success + result = usersService.unauthorizedUser(loginUser, 2); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + } + + + @Test + public void testAuthorizedUser(){ + + User loginUser = new User(); + when(userMapper.queryUserListByAlertGroupId(2)).thenReturn(getUserList()); + //no operate + Map result = usersService.authorizedUser(loginUser, 2); + logger.info(result.toString()); + Assert.assertEquals(Status.USER_NO_OPERATION_PERM, result.get(Constants.STATUS)); + //success + loginUser.setUserType(UserType.ADMIN_USER); + result = usersService.authorizedUser(loginUser, 2); + Assert.assertEquals(Status.SUCCESS, result.get(Constants.STATUS)); + List userList = (List) result.get(Constants.DATA_LIST); + logger.info(result.toString()); + Assert.assertTrue(CollectionUtils.isNotEmpty(userList)); + } + + /** + * get user + * @return + */ + private User getGeneralUser(){ + + User user = new User(); + user.setUserType(UserType.GENERAL_USER); + user.setUserName("userTest0001"); + user.setUserPassword("userTest0001"); + return user; + } + + + private List getUserList(){ + List userList = new ArrayList<>(); + userList.add(getGeneralUser()); + return userList; + } + + /** + * get user + */ + private User getUser(){ + + User user = new User(); + user.setUserType(UserType.ADMIN_USER); + user.setUserName("userTest0001"); + user.setUserPassword("userTest0001"); + return user; + } + + + private Tenant getTenant(){ + Tenant tenant = new Tenant(); + tenant.setId(1); + return tenant; + } + } \ No newline at end of file diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java new file mode 100644 index 0000000000..2c535054a7 --- /dev/null +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/WorkerGroupServiceTest.java @@ -0,0 +1,173 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.api.service; + +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import org.apache.dolphinscheduler.api.enums.Status; +import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.dao.entity.ProcessInstance; +import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.entity.WorkerGroup; +import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper; +import org.apache.dolphinscheduler.dao.mapper.WorkerGroupMapper; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@RunWith(MockitoJUnitRunner.class) +public class WorkerGroupServiceTest { + + private static final Logger logger = LoggerFactory.getLogger(WorkerGroupServiceTest.class); + + @InjectMocks + private WorkerGroupService workerGroupService; + @Mock + private WorkerGroupMapper workerGroupMapper; + @Mock + private ProcessInstanceMapper processInstanceMapper; + + + private String groupName="groupName000001"; + + + /** + * create or update a worker group + */ + @Test + public void testSaveWorkerGroup(){ + + User user = new User(); + // general user add + user.setUserType(UserType.GENERAL_USER); + Map result = workerGroupService.saveWorkerGroup(user, 0, groupName, "127.0.0.1"); + logger.info(result.toString()); + Assert.assertEquals( Status.USER_NO_OPERATION_PERM.getMsg(),(String) result.get(Constants.MSG)); + + //success + user.setUserType(UserType.ADMIN_USER); + result = workerGroupService.saveWorkerGroup(user, 0, groupName, "127.0.0.1"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),(String)result.get(Constants.MSG)); + // group name exist + Mockito.when(workerGroupMapper.selectById(2)).thenReturn(getWorkerGroup(2)); + Mockito.when(workerGroupMapper.queryWorkerGroupByName(groupName)).thenReturn(getList()); + result = workerGroupService.saveWorkerGroup(user, 2, groupName, "127.0.0.1"); + logger.info(result.toString()); + Assert.assertEquals(Status.NAME_EXIST,result.get(Constants.STATUS)); + + } + + /** + * query worker group paging + */ + @Test + public void testQueryAllGroupPaging(){ + + User user = new User(); + // general user add + user.setUserType(UserType.GENERAL_USER); + Map result = workerGroupService.queryAllGroupPaging(user, 1, 10, groupName); + logger.info(result.toString()); + Assert.assertEquals((String) result.get(Constants.MSG), Status.USER_NO_OPERATION_PERM.getMsg()); + //success + user.setUserType(UserType.ADMIN_USER); + Page page = new Page<>(1,10); + page.setRecords(getList()); + page.setSize(1L); + Mockito.when(workerGroupMapper.queryListPaging(Mockito.any(Page.class), Mockito.eq(groupName))).thenReturn(page); + result = workerGroupService.queryAllGroupPaging(user, 1, 10, groupName); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),(String)result.get(Constants.MSG)); + PageInfo pageInfo = (PageInfo) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists())); + } + + /** + * delete group by id + */ + @Test + public void testDeleteWorkerGroupById(){ + + //DELETE_WORKER_GROUP_BY_ID_FAIL + Mockito.when(processInstanceMapper.queryByWorkerGroupIdAndStatus(1, Constants.NOT_TERMINATED_STATES)).thenReturn(getProcessInstanceList()); + Map result = workerGroupService.deleteWorkerGroupById(1); + logger.info(result.toString()); + Assert.assertEquals(Status.DELETE_WORKER_GROUP_BY_ID_FAIL.getCode(),((Status) result.get(Constants.STATUS)).getCode()); + + //correct + result = workerGroupService.deleteWorkerGroupById(2); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),(String)result.get(Constants.MSG)); + + } + + @Test + public void testQueryAllGroup(){ + Mockito.when(workerGroupMapper.queryAllWorkerGroup()).thenReturn(getList()); + Map result = workerGroupService.queryAllGroup(); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),(String)result.get(Constants.MSG)); + List workerGroupList = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(workerGroupList.size()>0); + } + + + /** + * get processInstances + * @return + */ + private List getProcessInstanceList(){ + + List processInstances = new ArrayList<>(); + processInstances.add(new ProcessInstance()); + return processInstances; + } + /** + * get Group + * @return + */ + private WorkerGroup getWorkerGroup(int id){ + WorkerGroup workerGroup = new WorkerGroup(); + workerGroup.setName(groupName); + workerGroup.setId(id); + return workerGroup; + } + private WorkerGroup getWorkerGroup(){ + + return getWorkerGroup(1); + } + + private List getList(){ + List list = new ArrayList<>(); + list.add(getWorkerGroup()); + return list; + } + +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 7936bf7dc8..fafa7be6ef 100644 --- a/pom.xml +++ b/pom.xml @@ -666,6 +666,15 @@ **/api/utils/CheckUtilsTest.java **/api/utils/FileUtilsTest.java **/api/enums/*.java + **/api/service/AccessTokenServiceTest.java + **/api/service/QueueServiceTest.java + **/api/service/MonitorServiceTest.java + **/api/service/SessionServiceTest.java + **/api/service/UsersServiceTest.java + **/api/service/TenantServiceTest.java + **/api/service/WorkerGroupServiceTest.java + **/api/service/AlertGroupServiceTest.java + **/api/service/ProjectServiceTest.java **/alert/utils/ExcelUtilsTest.java **/alert/utils/FuncUtilsTest.java **/alert/utils/JSONUtilsTest.java From 5e935d579eec25701a6ce1d29fea95b161173ba3 Mon Sep 17 00:00:00 2001 From: Tboy Date: Mon, 30 Dec 2019 17:59:06 +0800 Subject: [PATCH 12/31] refactor AbstractZKClient (#1627) * we should insert alert DB once , and trigger this type of alert 3 times * refactor AbstractZKClient --- .../common/zk/AbstractZKClient.java | 96 +++++++------------ 1 file changed, 36 insertions(+), 60 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java index 0e95dddb36..c3ba718270 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java @@ -16,6 +16,22 @@ */ package org.apache.dolphinscheduler.common.zk; +import static org.apache.dolphinscheduler.common.Constants.ADD_ZK_OP; +import static org.apache.dolphinscheduler.common.Constants.DELETE_ZK_OP; +import static org.apache.dolphinscheduler.common.Constants.MASTER_PREFIX; +import static org.apache.dolphinscheduler.common.Constants.SINGLE_SLASH; +import static org.apache.dolphinscheduler.common.Constants.UNDERLINE; +import static org.apache.dolphinscheduler.common.Constants.WORKER_PREFIX; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.imps.CuratorFrameworkState; +import org.apache.curator.framework.recipes.locks.InterProcessMutex; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.IStoppable; import org.apache.dolphinscheduler.common.enums.ZKNodeType; @@ -23,26 +39,9 @@ import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.ResInfo; -import org.apache.commons.configuration.Configuration; -import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.configuration.PropertiesConfiguration; -import org.apache.commons.lang3.StringUtils; -import org.apache.curator.RetryPolicy; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.framework.imps.CuratorFrameworkState; -import org.apache.curator.framework.recipes.locks.InterProcessMutex; -import org.apache.curator.framework.state.ConnectionState; -import org.apache.curator.framework.state.ConnectionStateListener; -import org.apache.curator.retry.ExponentialBackoffRetry; -import org.apache.zookeeper.CreateMode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; - -import static org.apache.dolphinscheduler.common.Constants.*; - /** * abstract zookeeper client @@ -70,8 +69,7 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ return; } - byte[] bytes = zkClient.getData().forPath(znode); - String resInfoStr = new String(bytes); + String resInfoStr = super.get(znode); String[] splits = resInfoStr.split(Constants.COMMA); if (splits.length != Constants.HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH){ return; @@ -107,8 +105,7 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ String type = serverType.equals(MASTER_PREFIX) ? MASTER_PREFIX : WORKER_PREFIX; String deadServerPath = getDeadZNodeParentPath() + SINGLE_SLASH + type + UNDERLINE + ipSeqNo; - if(zkClient.checkExists().forPath(zNode) == null || - zkClient.checkExists().forPath(deadServerPath) != null ){ + if(!isExisted(zNode) || isExisted(deadServerPath)){ return true; } @@ -118,14 +115,12 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ public void removeDeadServerByHost(String host, String serverType) throws Exception { - List deadServers = zkClient.getChildren().forPath(getDeadZNodeParentPath()); + List deadServers = super.getChildrenKeys(getDeadZNodeParentPath()); for(String serverPath : deadServers){ if(serverPath.startsWith(serverType+UNDERLINE+host)){ - String server = getDeadZNodeParentPath() + SINGLE_SLASH + serverPath; - if(zkClient.checkExists().forPath(server) != null){ - zkClient.delete().forPath(server); - logger.info("{} server {} deleted from zk dead server path success" , serverType , host); - } + String server = getDeadZNodeParentPath() + SINGLE_SLASH + serverPath; + super.remove(server); + logger.info("{} server {} deleted from zk dead server path success" , serverType , host); } } } @@ -143,8 +138,8 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ // create temporary sequence nodes for master znode String parentPath = getZNodeParentPath(zkNodeType); String serverPathPrefix = parentPath + "/" + OSUtils.getHost(); - String registerPath = zkClient.create().withMode(CreateMode.EPHEMERAL_SEQUENTIAL).forPath( - serverPathPrefix + UNDERLINE, heartbeatZKInfo.getBytes()); + String registerPath = serverPathPrefix + UNDERLINE; + super.persistEphemeral(registerPath, heartbeatZKInfo); logger.info("register {} node {} success" , zkNodeType.toString(), registerPath); return registerPath; } @@ -165,7 +160,7 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ } registerPath = createZNodePath(zkNodeType); - // handle dead server + // handle dead server handleDeadServer(registerPath, zkNodeType, Constants.DELETE_ZK_OP); return registerPath; @@ -196,10 +191,10 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ }else if(opType.equals(ADD_ZK_OP)){ String deadServerPath = getDeadZNodeParentPath() + SINGLE_SLASH + type + UNDERLINE + ipSeqNo; - if(zkClient.checkExists().forPath(deadServerPath) == null){ + if(!super.isExisted(deadServerPath)){ //add dead server info to zk dead server path : /dead-servers/ - zkClient.create().forPath(deadServerPath,(type + UNDERLINE + ipSeqNo).getBytes()); + super.persist(deadServerPath,(type + UNDERLINE + ipSeqNo)); logger.info("{} server dead , and {} added to zk dead server path success" , zkNodeType.toString(), zNode); @@ -226,19 +221,13 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ List childrenList = new ArrayList<>(); try { // read master node parent path from conf - if(zkClient.checkExists().forPath(getZNodeParentPath(ZKNodeType.MASTER)) != null){ - childrenList = zkClient.getChildren().forPath(getZNodeParentPath(ZKNodeType.MASTER)); + if(super.isExisted(getZNodeParentPath(ZKNodeType.MASTER))){ + childrenList = super.getChildrenKeys(getZNodeParentPath(ZKNodeType.MASTER)); } } catch (Exception e) { - if(e.getMessage().contains("java.lang.IllegalStateException: instance must be started")){ - logger.error("zookeeper service not started",e); - }else{ - logger.error(e.getMessage(),e); - } - - }finally { - return childrenList.size(); + logger.error("getActiveMasterNum error",e); } + return childrenList.size(); } /** @@ -280,10 +269,9 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ Map masterMap = new HashMap<>(); try { String path = getZNodeParentPath(zkNodeType); - List serverList = getZkClient().getChildren().forPath(path); + List serverList = super.getChildrenKeys(path); for(String server : serverList){ - byte[] bytes = getZkClient().getData().forPath(path + "/" + server); - masterMap.putIfAbsent(server, new String(bytes)); + masterMap.putIfAbsent(server, super.get(path + "/" + server)); } } catch (Exception e) { logger.error("get server list failed : " + e.getMessage(), e); @@ -430,27 +418,15 @@ public abstract class AbstractZKClient extends ZookeeperCachedOperator{ */ protected void initSystemZNode(){ try { - createNodePath(getMasterZNodeParentPath()); - createNodePath(getWorkerZNodeParentPath()); - createNodePath(getDeadZNodeParentPath()); + persist(getMasterZNodeParentPath(), ""); + persist(getWorkerZNodeParentPath(), ""); + persist(getDeadZNodeParentPath(), ""); } catch (Exception e) { logger.error("init system znode failed : " + e.getMessage(),e); } } - /** - * create zookeeper node path if not exists - * @param zNodeParentPath zookeeper parent path - * @throws Exception errors - */ - private void createNodePath(String zNodeParentPath) throws Exception { - if(null == zkClient.checkExists().forPath(zNodeParentPath)){ - zkClient.create().creatingParentContainersIfNeeded() - .withMode(CreateMode.PERSISTENT).forPath(zNodeParentPath); - } - } - /** * server self dead, stop all threads * @param serverHost server host From 426c027ddf738f3a6671195f2afd6a655ebc166a Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Mon, 30 Dec 2019 18:11:14 +0800 Subject: [PATCH 13/31] Click the run button to add the workflow name echo #1339 Point 11 (#1638) * Increase script text box to zoom in and fix log loading * Timing operation adds default value for start and end dates * Click the run button to add the workflow name echo --- .../pages/definition/pages/list/_source/start.vue | 8 ++++++++ .../pages/projects/pages/index/_source/chartConfig.js | 2 +- .../projects/pages/index/_source/processStateCount.vue | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue index dfc7f49c20..2049b6b16e 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/definition/pages/list/_source/start.vue @@ -19,6 +19,12 @@
{{$t('Please set the parameters before starting')}}
+
+
+ {{$t('Process Name')}} +
+
{{workflowName}}
+
{{$t('Failure Strategy')}} @@ -174,6 +180,7 @@ processDefinitionId: 0, failureStrategy: 'CONTINUE', warningTypeList: warningTypeList, + workflowName: '', warningType: '', notifyGroupList: [], warningGroupId: '', @@ -276,6 +283,7 @@ this.warningGroupId = '' }) }) + this.workflowName = this.item.name }, computed: {}, components: { mEmail, mPriority, mWorkerGroups } diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/chartConfig.js b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/chartConfig.js index f693ed578e..b8601a69c9 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/chartConfig.js +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/chartConfig.js @@ -27,7 +27,7 @@ let pie = { avoidLabelOverlap: true, // Whether to prevent the label overlap policy hoverAnimation: true, // Whether to enable hover to enlarge the animation on the sector. radius: ['30%', '60%'], - center: ['50%', '50%'], + center: ['53%', '60%'], label: { align: 'left', normal: { diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/processStateCount.vue b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/processStateCount.vue index c21579a434..d121034932 100644 --- a/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/processStateCount.vue +++ b/dolphinscheduler-ui/src/js/conf/home/pages/projects/pages/index/_source/processStateCount.vue @@ -29,7 +29,7 @@ {{$t('Number')}} {{$t('State')}} - + {{$index+1}} {{item.value}} {{item.key}} From d16500e8bee48f63b80235ce97a576af1b5a150d Mon Sep 17 00:00:00 2001 From: bao liang <29528966+lenboo@users.noreply.github.com> Date: Mon, 30 Dec 2019 18:14:31 +0800 Subject: [PATCH 14/31] [Feature] merge some configurations #1635 (#1636) * merge hadoop.properties into common.properties * merge hadoop,zookeeper.properties into common.properties remove combined.properties/master.properties/worker.properties * change db user/pwd to test/test --- .../resources/application-combined.properties | 40 ------- .../dolphinscheduler/common/Constants.java | 21 +--- .../common/utils/PropertyUtils.java | 3 +- .../common/zk/ZookeeperConfig.java | 2 +- .../resources/{common => }/common.properties | 51 ++++++-- .../resources/common/hadoop/hadoop.properties | 35 ------ .../src/main/resources/quartz.properties | 18 +-- .../src/main/resources/zookeeper.properties | 43 ------- .../dao/utils/PropertyUtils.java | 2 +- ...-dao.properties => application.properties} | 0 .../resources/application-master.properties | 18 --- .../resources/application-worker.properties | 18 --- install.sh | 109 ++++++------------ script/dolphinscheduler-daemon.sh | 8 +- 14 files changed, 100 insertions(+), 268 deletions(-) delete mode 100644 dolphinscheduler-api/src/main/resources/application-combined.properties rename dolphinscheduler-common/src/main/resources/{common => }/common.properties (67%) delete mode 100644 dolphinscheduler-common/src/main/resources/common/hadoop/hadoop.properties delete mode 100644 dolphinscheduler-common/src/main/resources/zookeeper.properties rename dolphinscheduler-dao/src/main/resources/{application-dao.properties => application.properties} (100%) delete mode 100644 dolphinscheduler-server/src/main/resources/application-master.properties delete mode 100644 dolphinscheduler-server/src/main/resources/application-worker.properties diff --git a/dolphinscheduler-api/src/main/resources/application-combined.properties b/dolphinscheduler-api/src/main/resources/application-combined.properties deleted file mode 100644 index 0264c4f4b6..0000000000 --- a/dolphinscheduler-api/src/main/resources/application-combined.properties +++ /dev/null @@ -1,40 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -logging.config=classpath:combined_logback.xml - -# server port -server.port=12345 - -# session config -server.servlet.session.timeout=7200 - -server.servlet.context-path=/dolphinscheduler/ - -# file size limit for upload -spring.servlet.multipart.max-file-size=1024MB -spring.servlet.multipart.max-request-size=1024MB - -#post content -server.jetty.max-http-post-size=5000000 - -spring.messages.encoding=UTF-8 - -#i18n classpath folder , file prefix messages， if have many files, use "," seperator -spring.messages.basename=i18n/messages - -server.is-combined-server=true diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java index 6073f218a4..1222faa1a7 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java @@ -26,25 +26,10 @@ import java.util.regex.Pattern; */ public final class Constants { - /** - * zookeeper properties path - */ - public static final String ZOOKEEPER_PROPERTIES_PATH = "zookeeper.properties"; - - /** - * hadoop properties path - */ - public static final String HADOOP_PROPERTIES_PATH = "/common/hadoop/hadoop.properties"; - /** * common properties path */ - public static final String COMMON_PROPERTIES_PATH = "/common/common.properties"; - - /** - * dao properties path - */ - public static final String DAO_PROPERTIES_PATH = "application.properties"; + public static final String COMMON_PROPERTIES_PATH = "/common.properties"; /** * fs.defaultFS @@ -227,6 +212,8 @@ public final class Constants { public static final String ZOOKEEPER_CONNECTION_TIMEOUT = "zookeeper.connection.timeout"; public static final String ZOOKEEPER_RETRY_SLEEP = "zookeeper.retry.sleep"; + public static final String ZOOKEEPER_RETRY_BASE_SLEEP = "zookeeper.retry.base.sleep"; + public static final String ZOOKEEPER_RETRY_MAX_SLEEP = "zookeeper.retry.max.sleep"; public static final String ZOOKEEPER_RETRY_MAXTIME = "zookeeper.retry.maxtime"; @@ -469,7 +456,7 @@ public final class Constants { /** * task record configuration path */ - public static final String APPLICATION_PROPERTIES = "application-dao.properties"; + public static final String APPLICATION_PROPERTIES = "application.properties"; public static final String TASK_RECORD_URL = "task.record.datasource.url"; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java index ea01d025a7..c3e8197079 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/PropertyUtils.java @@ -29,7 +29,6 @@ import java.util.Map; import java.util.Properties; import static org.apache.dolphinscheduler.common.Constants.COMMON_PROPERTIES_PATH; -import static org.apache.dolphinscheduler.common.Constants.HADOOP_PROPERTIES_PATH; /** * property utils @@ -51,7 +50,7 @@ public class PropertyUtils { } private void init(){ - String[] propertyFiles = new String[]{HADOOP_PROPERTIES_PATH,COMMON_PROPERTIES_PATH}; + String[] propertyFiles = new String[]{COMMON_PROPERTIES_PATH}; for (String fileName : propertyFiles) { InputStream fis = null; try { diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperConfig.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperConfig.java index a90a147425..75a9f6c5f4 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperConfig.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/ZookeeperConfig.java @@ -24,7 +24,7 @@ import org.springframework.stereotype.Component; * zookeeper conf */ @Component -@PropertySource("classpath:zookeeper.properties") +@PropertySource("classpath:common.properties") public class ZookeeperConfig { //zk connect config diff --git a/dolphinscheduler-common/src/main/resources/common/common.properties b/dolphinscheduler-common/src/main/resources/common.properties similarity index 67% rename from dolphinscheduler-common/src/main/resources/common/common.properties rename to dolphinscheduler-common/src/main/resources/common.properties index 01e321167d..9826c56593 100644 --- a/dolphinscheduler-common/src/main/resources/common/common.properties +++ b/dolphinscheduler-common/src/main/resources/common.properties @@ -18,14 +18,21 @@ #task queue implementation, default "zookeeper" dolphinscheduler.queue.impl=zookeeper -# user data directory path, self configuration, please make sure the directory exists and have read write permissions -data.basedir.path=/tmp/dolphinscheduler +#zookeeper cluster. multiple are separated by commas. eg. 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181 +zookeeper.quorum=localhost:2181 -# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions -data.download.basedir.path=/tmp/dolphinscheduler/download +#dolphinscheduler root directory +zookeeper.dolphinscheduler.root=/dolphinscheduler -# process execute directory. self configuration, please make sure the directory exists and have read write permissions -process.exec.basepath=/tmp/dolphinscheduler/exec +#dolphinscheduler failover directory +zookeeper.session.timeout=300 +zookeeper.connection.timeout=300 +zookeeper.retry.base.sleep=100 +zookeeper.retry.max.sleep=30000 +zookeeper.retry.maxtime=5 + +# resource upload startup type : HDFS,S3,NONE +res.upload.startup.type=NONE # Users who have permission to create directories under the HDFS root path hdfs.root.user=hdfs @@ -33,8 +40,15 @@ hdfs.root.user=hdfs # data base dir, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions。"/dolphinscheduler" is recommended data.store2hdfs.basepath=/dolphinscheduler -# resource upload startup type : HDFS,S3,NONE -res.upload.startup.type=NONE +# user data directory path, self configuration, please make sure the directory exists and have read write permissions +data.basedir.path=/tmp/dolphinscheduler + +# directory path for user data download. self configuration, please make sure the directory exists and have read write permissions +data.download.basedir.path=/tmp/dolphinscheduler/download + +# process execute directory. self configuration, please make sure the directory exists and have read write permissions +process.exec.basepath=/tmp/dolphinscheduler/exec + # whether kerberos starts hadoop.security.authentication.startup.state=false @@ -57,3 +71,24 @@ resource.view.suffixs=txt,log,sh,conf,cfg,py,java,sql,hql,xml,properties # is development state? default "false" development.state=true + +# ha or single namenode,If namenode ha needs to copy core-site.xml and hdfs-site.xml +# to the conf directory，support s3，for example : s3a://dolphinscheduler +fs.defaultFS=hdfs://mycluster:8020 + +# s3 need，s3 endpoint +fs.s3a.endpoint=http://192.168.199.91:9010 + +# s3 need，s3 access key +fs.s3a.access.key=A3DXS30FO22544RE + +# s3 need，s3 secret key +fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK + +#resourcemanager ha note this need ips , this empty if single +yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx + +# If it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine +yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s + + diff --git a/dolphinscheduler-common/src/main/resources/common/hadoop/hadoop.properties b/dolphinscheduler-common/src/main/resources/common/hadoop/hadoop.properties deleted file mode 100644 index 2c19b4a52e..0000000000 --- a/dolphinscheduler-common/src/main/resources/common/hadoop/hadoop.properties +++ /dev/null @@ -1,35 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# ha or single namenode,If namenode ha needs to copy core-site.xml and hdfs-site.xml -# to the conf directory，support s3，for example : s3a://dolphinscheduler -fs.defaultFS=hdfs://mycluster:8020 - -# s3 need，s3 endpoint -fs.s3a.endpoint=http://192.168.199.91:9010 - -# s3 need，s3 access key -fs.s3a.access.key=A3DXS30FO22544RE - -# s3 need，s3 secret key -fs.s3a.secret.key=OloCLq3n+8+sdPHUhJ21XrSxTC+JK - -#resourcemanager ha note this need ips , this empty if single -yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx - -# If it is a single resourcemanager, you only need to configure one host name. If it is resourcemanager HA, the default configuration is fine -yarn.application.status.address=http://ark1:8088/ws/v1/cluster/apps/%s \ No newline at end of file diff --git a/dolphinscheduler-common/src/main/resources/quartz.properties b/dolphinscheduler-common/src/main/resources/quartz.properties index d388691c3f..684aabcd66 100644 --- a/dolphinscheduler-common/src/main/resources/quartz.properties +++ b/dolphinscheduler-common/src/main/resources/quartz.properties @@ -18,6 +18,15 @@ #============================================================================ # Configure Main Scheduler Properties #============================================================================ +#org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.StdJDBCDelegate +org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.PostgreSQLDelegate +#org.quartz.dataSource.myDs.driver = com.mysql.jdbc.Driver +org.quartz.dataSource.myDs.driver = org.postgresql.Driver +#org.quartz.dataSource.myDs.URL = jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?characterEncoding=utf8 +org.quartz.dataSource.myDs.URL = jdbc:postgresql://localhost:5432/dolphinscheduler?characterEncoding=utf8 +org.quartz.dataSource.myDs.user = test +org.quartz.dataSource.myDs.password = test + org.quartz.scheduler.instanceName = DolphinScheduler org.quartz.scheduler.instanceId = AUTO org.quartz.scheduler.makeSchedulerThreadDaemon = true @@ -37,8 +46,7 @@ org.quartz.threadPool.threadPriority = 5 #============================================================================ org.quartz.jobStore.class = org.quartz.impl.jdbcjobstore.JobStoreTX -#org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.StdJDBCDelegate -org.quartz.jobStore.driverDelegateClass = org.quartz.impl.jdbcjobstore.PostgreSQLDelegate + org.quartz.jobStore.tablePrefix = QRTZ_ org.quartz.jobStore.isClustered = true org.quartz.jobStore.misfireThreshold = 60000 @@ -49,11 +57,5 @@ org.quartz.jobStore.dataSource = myDs # Configure Datasources #============================================================================ org.quartz.dataSource.myDs.connectionProvider.class = org.apache.dolphinscheduler.server.quartz.DruidConnectionProvider -#org.quartz.dataSource.myDs.driver = com.mysql.jdbc.Driver -org.quartz.dataSource.myDs.driver = org.postgresql.Driver -#org.quartz.dataSource.myDs.URL = jdbc:mysql://192.168.xx.xx:3306/dolphinscheduler?characterEncoding=utf8 -org.quartz.dataSource.myDs.URL = jdbc:postgresql://192.168.xx.xx:5432/dolphinscheduler?characterEncoding=utf8 -org.quartz.dataSource.myDs.user = xx -org.quartz.dataSource.myDs.password = xx org.quartz.dataSource.myDs.maxConnections = 10 org.quartz.dataSource.myDs.validationQuery = select 1 \ No newline at end of file diff --git a/dolphinscheduler-common/src/main/resources/zookeeper.properties b/dolphinscheduler-common/src/main/resources/zookeeper.properties deleted file mode 100644 index a560de4f23..0000000000 --- a/dolphinscheduler-common/src/main/resources/zookeeper.properties +++ /dev/null @@ -1,43 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -#zookeeper cluster. multiple are separated by commas. eg. 192.168.xx.xx:2181,192.168.xx.xx:2181,192.168.xx.xx:2181 -zookeeper.quorum=localhost:2181 - -#dolphinscheduler root directory -zookeeper.dolphinscheduler.root=/dolphinscheduler - -#zookeeper server dirctory -#zookeeper.dolphinscheduler.dead.servers=/dolphinscheduler/dead-servers -#zookeeper.dolphinscheduler.masters=/dolphinscheduler/masters -#zookeeper.dolphinscheduler.workers=/dolphinscheduler/workers - -#zookeeper lock dirctory -#zookeeper.dolphinscheduler.lock.masters=/dolphinscheduler/lock/masters -#zookeeper.dolphinscheduler.lock.workers=/dolphinscheduler/lock/workers - -#dolphinscheduler failover directory -#zookeeper.dolphinscheduler.lock.failover.masters=/dolphinscheduler/lock/failover/masters -#zookeeper.dolphinscheduler.lock.failover.workers=/dolphinscheduler/lock/failover/workers -#zookeeper.dolphinscheduler.lock.failover.startup.masters=/dolphinscheduler/lock/failover/startup-masters - -#dolphinscheduler failover directory -zookeeper.session.timeout=300 -zookeeper.connection.timeout=300 -zookeeper.retry.base.sleep=100 -zookeeper.retry.max.sleep=30000 -zookeeper.retry.maxtime=5 \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PropertyUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PropertyUtils.java index e92491e096..cdd481a5d7 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PropertyUtils.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/PropertyUtils.java @@ -49,7 +49,7 @@ public class PropertyUtils { * init */ private void init(){ - String[] propertyFiles = new String[]{Constants.DAO_PROPERTIES_PATH}; + String[] propertyFiles = new String[]{Constants.APPLICATION_PROPERTIES}; for (String fileName : propertyFiles) { InputStream fis = null; try { diff --git a/dolphinscheduler-dao/src/main/resources/application-dao.properties b/dolphinscheduler-dao/src/main/resources/application.properties similarity index 100% rename from dolphinscheduler-dao/src/main/resources/application-dao.properties rename to dolphinscheduler-dao/src/main/resources/application.properties diff --git a/dolphinscheduler-server/src/main/resources/application-master.properties b/dolphinscheduler-server/src/main/resources/application-master.properties deleted file mode 100644 index 49d28e810f..0000000000 --- a/dolphinscheduler-server/src/main/resources/application-master.properties +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -logging.config=classpath:master_logback.xml \ No newline at end of file diff --git a/dolphinscheduler-server/src/main/resources/application-worker.properties b/dolphinscheduler-server/src/main/resources/application-worker.properties deleted file mode 100644 index be4319d204..0000000000 --- a/dolphinscheduler-server/src/main/resources/application-worker.properties +++ /dev/null @@ -1,18 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -logging.config=classpath:worker_logback.xml diff --git a/install.sh b/install.sh index b73ca09079..85e470051f 100644 --- a/install.sh +++ b/install.sh @@ -218,30 +218,6 @@ keytabPath="$installPath/conf/hdfs.headless.keytab" # zk root directory zkRoot="/dolphinscheduler" -# used to record the zk directory of the hanging machine -zkDeadServers="$zkRoot/dead-servers" - -# masters directory -zkMasters="$zkRoot/masters" - -# workers directory -zkWorkers="$zkRoot/workers" - -# zk master distributed lock -mastersLock="$zkRoot/lock/masters" - -# zk worker distributed lock -workersLock="$zkRoot/lock/workers" - -# zk master fault-tolerant distributed lock -mastersFailover="$zkRoot/lock/failover/masters" - -# zk worker fault-tolerant distributed lock -workersFailover="$zkRoot/lock/failover/workers" - -# zk master start fault tolerant distributed lock -mastersStartupFailover="$zkRoot/lock/failover/startup-masters" - # zk session timeout zkSessionTimeout="300" @@ -321,11 +297,10 @@ apiMaxHttpPostSize="5000000" # 1,replace file echo "1,replace file" if [ $dbtype == "mysql" ];then - sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:mysql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application-dao.properties - sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application-dao.properties - sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application-dao.properties - sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=com.mysql.jdbc.Driver#g" conf/application-dao.properties - + sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:mysql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application.properties + sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application.properties + sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application.properties + sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=com.mysql.jdbc.Driver#g" conf/application.properties sed -i ${txt} "s#org.quartz.dataSource.myDs.URL.*#org.quartz.dataSource.myDs.URL=jdbc:mysql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/quartz.properties sed -i ${txt} "s#org.quartz.dataSource.myDs.user.*#org.quartz.dataSource.myDs.user=${username}#g" conf/quartz.properties @@ -335,10 +310,10 @@ if [ $dbtype == "mysql" ];then fi if [ $dbtype == "postgresql" ];then - sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:postgresql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application-dao.properties - sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application-dao.properties - sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application-dao.properties - sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=org.postgresql.Driver#g" conf/application-dao.properties + sed -i ${txt} "s#spring.datasource.url.*#spring.datasource.url=jdbc:postgresql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/application.properties + sed -i ${txt} "s#spring.datasource.username.*#spring.datasource.username=${username}#g" conf/application.properties + sed -i ${txt} "s#spring.datasource.password.*#spring.datasource.password=${passowrd}#g" conf/application.properties + sed -i ${txt} "s#spring.datasource.driver-class-name.*#spring.datasource.driver-class-name=org.postgresql.Driver#g" conf/application.properties sed -i ${txt} "s#org.quartz.dataSource.myDs.URL.*#org.quartz.dataSource.myDs.URL=jdbc:postgresql://${dbhost}/${dbname}?characterEncoding=UTF-8#g" conf/quartz.properties sed -i ${txt} "s#org.quartz.dataSource.myDs.user.*#org.quartz.dataSource.myDs.user=${username}#g" conf/quartz.properties @@ -349,46 +324,34 @@ fi -sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${defaultFS}#g" conf/common/hadoop/hadoop.properties -sed -i ${txt} "s#fs.s3a.endpoint.*#fs.s3a.endpoint=${s3Endpoint}#g" conf/common/hadoop/hadoop.properties -sed -i ${txt} "s#fs.s3a.access.key.*#fs.s3a.access.key=${s3AccessKey}#g" conf/common/hadoop/hadoop.properties -sed -i ${txt} "s#fs.s3a.secret.key.*#fs.s3a.secret.key=${s3SecretKey}#g" conf/common/hadoop/hadoop.properties -sed -i ${txt} "s#yarn.resourcemanager.ha.rm.ids.*#yarn.resourcemanager.ha.rm.ids=${yarnHaIps}#g" conf/common/hadoop/hadoop.properties -sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:8088/ws/v1/cluster/apps/%s#g" conf/common/hadoop/hadoop.properties - - -sed -i ${txt} "s#data.basedir.path.*#data.basedir.path=${programPath}#g" conf/common/common.properties -sed -i ${txt} "s#data.download.basedir.path.*#data.download.basedir.path=${downloadPath}#g" conf/common/common.properties -sed -i ${txt} "s#process.exec.basepath.*#process.exec.basepath=${execPath}#g" conf/common/common.properties -sed -i ${txt} "s#hdfs.root.user.*#hdfs.root.user=${hdfsRootUser}#g" conf/common/common.properties -sed -i ${txt} "s#data.store2hdfs.basepath.*#data.store2hdfs.basepath=${hdfsPath}#g" conf/common/common.properties -sed -i ${txt} "s#res.upload.startup.type.*#res.upload.startup.type=${resUploadStartupType}#g" conf/common/common.properties -sed -i ${txt} "s#dolphinscheduler.env.path.*#dolphinscheduler.env.path=${shellEnvPath}#g" conf/common/common.properties -sed -i ${txt} "s#resource.view.suffixs.*#resource.view.suffixs=${resSuffixs}#g" conf/common/common.properties -sed -i ${txt} "s#development.state.*#development.state=${devState}#g" conf/common/common.properties -sed -i ${txt} "s#hadoop.security.authentication.startup.state.*#hadoop.security.authentication.startup.state=${kerberosStartUp}#g" conf/common/common.properties -sed -i ${txt} "s#java.security.krb5.conf.path.*#java.security.krb5.conf.path=${krb5ConfPath}#g" conf/common/common.properties -sed -i ${txt} "s#login.user.keytab.username.*#login.user.keytab.username=${keytabUserName}#g" conf/common/common.properties -sed -i ${txt} "s#login.user.keytab.path.*#login.user.keytab.path=${keytabPath}#g" conf/common/common.properties - -sed -i ${txt} "s#zookeeper.quorum.*#zookeeper.quorum=${zkQuorum}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.root.*#zookeeper.dolphinscheduler.root=${zkRoot}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.dead.servers.*#zookeeper.dolphinscheduler.dead.servers=${zkDeadServers}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.masters.*#zookeeper.dolphinscheduler.masters=${zkMasters}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.workers.*#zookeeper.dolphinscheduler.workers=${zkWorkers}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.masters.*#zookeeper.dolphinscheduler.lock.masters=${mastersLock}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.workers.*#zookeeper.dolphinscheduler.lock.workers=${workersLock}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.failover.masters.*#zookeeper.dolphinscheduler.lock.failover.masters=${mastersFailover}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.failover.workers.*#zookeeper.dolphinscheduler.lock.failover.workers=${workersFailover}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.dolphinscheduler.lock.failover.startup.masters.*#zookeeper.dolphinscheduler.lock.failover.startup.masters=${mastersStartupFailover}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.session.timeout.*#zookeeper.session.timeout=${zkSessionTimeout}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.connection.timeout.*#zookeeper.connection.timeout=${zkConnectionTimeout}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.retry.sleep.*#zookeeper.retry.sleep=${zkRetrySleep}#g" conf/zookeeper.properties -sed -i ${txt} "s#zookeeper.retry.maxtime.*#zookeeper.retry.maxtime=${zkRetryMaxtime}#g" conf/zookeeper.properties - -sed -i ${txt} "s#server.port.*#server.port=${masterPort}#g" conf/application-master.properties -sed -i ${txt} "s#server.port.*#server.port=${workerPort}#g" conf/application-worker.properties - +sed -i ${txt} "s#fs.defaultFS.*#fs.defaultFS=${defaultFS}#g" conf/common.properties +sed -i ${txt} "s#fs.s3a.endpoint.*#fs.s3a.endpoint=${s3Endpoint}#g" conf/common.properties +sed -i ${txt} "s#fs.s3a.access.key.*#fs.s3a.access.key=${s3AccessKey}#g" conf/common.properties +sed -i ${txt} "s#fs.s3a.secret.key.*#fs.s3a.secret.key=${s3SecretKey}#g" conf/common.properties +sed -i ${txt} "s#yarn.resourcemanager.ha.rm.ids.*#yarn.resourcemanager.ha.rm.ids=${yarnHaIps}#g" conf/common.properties +sed -i ${txt} "s#yarn.application.status.address.*#yarn.application.status.address=http://${singleYarnIp}:8088/ws/v1/cluster/apps/%s#g" conf/common.properties + + +sed -i ${txt} "s#data.basedir.path.*#data.basedir.path=${programPath}#g" conf/common.properties +sed -i ${txt} "s#data.download.basedir.path.*#data.download.basedir.path=${downloadPath}#g" conf/common.properties +sed -i ${txt} "s#process.exec.basepath.*#process.exec.basepath=${execPath}#g" conf/common.properties +sed -i ${txt} "s#hdfs.root.user.*#hdfs.root.user=${hdfsRootUser}#g" conf/common.properties +sed -i ${txt} "s#data.store2hdfs.basepath.*#data.store2hdfs.basepath=${hdfsPath}#g" conf/common.properties +sed -i ${txt} "s#res.upload.startup.type.*#res.upload.startup.type=${resUploadStartupType}#g" conf/common.properties +sed -i ${txt} "s#dolphinscheduler.env.path.*#dolphinscheduler.env.path=${shellEnvPath}#g" conf/common.properties +sed -i ${txt} "s#resource.view.suffixs.*#resource.view.suffixs=${resSuffixs}#g" conf/common.properties +sed -i ${txt} "s#development.state.*#development.state=${devState}#g" conf/common.properties +sed -i ${txt} "s#hadoop.security.authentication.startup.state.*#hadoop.security.authentication.startup.state=${kerberosStartUp}#g" conf/common.properties +sed -i ${txt} "s#java.security.krb5.conf.path.*#java.security.krb5.conf.path=${krb5ConfPath}#g" conf/common.properties +sed -i ${txt} "s#login.user.keytab.username.*#login.user.keytab.username=${keytabUserName}#g" conf/common.properties +sed -i ${txt} "s#login.user.keytab.path.*#login.user.keytab.path=${keytabPath}#g" conf/common.properties + +sed -i ${txt} "s#zookeeper.quorum.*#zookeeper.quorum=${zkQuorum}#g" conf/common.properties +sed -i ${txt} "s#zookeeper.dolphinscheduler.root.*#zookeeper.dolphinscheduler.root=${zkRoot}#g" conf/common.properties +sed -i ${txt} "s#zookeeper.session.timeout.*#zookeeper.session.timeout=${zkSessionTimeout}#g" conf/common.properties +sed -i ${txt} "s#zookeeper.connection.timeout.*#zookeeper.connection.timeout=${zkConnectionTimeout}#g" conf/common.properties +sed -i ${txt} "s#zookeeper.retry.sleep.*#zookeeper.retry.sleep=${zkRetrySleep}#g" conf/common.properties +sed -i ${txt} "s#zookeeper.retry.maxtime.*#zookeeper.retry.maxtime=${zkRetryMaxtime}#g" conf/common.properties sed -i ${txt} "s#server.port.*#server.port=${apiServerPort}#g" conf/application-api.properties sed -i ${txt} "s#server.servlet.session.timeout.*#server.servlet.session.timeout=${apiServerSessionTimeout}#g" conf/application-api.properties diff --git a/script/dolphinscheduler-daemon.sh b/script/dolphinscheduler-daemon.sh index 292f2eca39..b3310c384f 100644 --- a/script/dolphinscheduler-daemon.sh +++ b/script/dolphinscheduler-daemon.sh @@ -57,13 +57,13 @@ pid=$DOLPHINSCHEDULER_LOG_DIR/dolphinscheduler-$command.pid cd $DOLPHINSCHEDULER_HOME if [ "$command" = "api-server" ]; then - LOG_FILE="-Dspring.profiles.active=api" + LOG_FILE="-Dlogging.config=classpath:apiserver_logback.xml -Dspring.profiles.active=api" CLASS=org.apache.dolphinscheduler.api.ApiApplicationServer elif [ "$command" = "master-server" ]; then - LOG_FILE="-Dspring.profiles.active=master -Ddruid.mysql.usePingMethod=false" + LOG_FILE="-Dlogging.config=classpath:master_logback.xml -Ddruid.mysql.usePingMethod=false" CLASS=org.apache.dolphinscheduler.server.master.MasterServer elif [ "$command" = "worker-server" ]; then - LOG_FILE="-Dspring.profiles.active=worker -Ddruid.mysql.usePingMethod=false" + LOG_FILE="-Dlogging.config=classpath:worker_logback.xml -Ddruid.mysql.usePingMethod=false" CLASS=org.apache.dolphinscheduler.server.worker.WorkerServer elif [ "$command" = "alert-server" ]; then LOG_FILE="-Dlogback.configurationFile=conf/alert_logback.xml" @@ -71,7 +71,7 @@ elif [ "$command" = "alert-server" ]; then elif [ "$command" = "logger-server" ]; then CLASS=org.apache.dolphinscheduler.server.rpc.LoggerServer elif [ "$command" = "combined-server" ]; then - LOG_FILE="-Dspring.profiles.active=combined" + LOG_FILE="-Dlogging.config=classpath:combined_logback.xml -Dspring.profiles.active=api -Dserver.is-combined-server=true" CLASS=org.apache.dolphinscheduler.api.CombinedApplicationServer else echo "Error: No command named \`$command' was found." From cc786e6d76b5eb208bef7a23872cca6c2fb60920 Mon Sep 17 00:00:00 2001 From: "dk.technoboy" Date: Mon, 30 Dec 2019 18:18:18 +0800 Subject: [PATCH 15/31] remove swagger-annotations --- dolphinscheduler-dao/pom.xml | 6 ------ .../java/org/apache/dolphinscheduler/dao/entity/User.java | 5 ----- 2 files changed, 11 deletions(-) diff --git a/dolphinscheduler-dao/pom.xml b/dolphinscheduler-dao/pom.xml index 85467e12d4..af1cdb8b78 100644 --- a/dolphinscheduler-dao/pom.xml +++ b/dolphinscheduler-dao/pom.xml @@ -162,12 +162,6 @@ spring-test test - - io.swagger - swagger-annotations - 1.5.20 - compile - org.yaml snakeyaml diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java index c4df0679ef..b6def18922 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java @@ -22,8 +22,6 @@ import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; import lombok.Data; import java.util.Date; @@ -33,7 +31,6 @@ import java.util.Date; */ @Data @TableName("t_ds_user") -@ApiModel(description = "UserModelDesc") public class User { /** @@ -45,13 +42,11 @@ public class User { /** * user name */ - @ApiModelProperty(name = "userName", notes = "USER_NAME",dataType = "String",required = true) private String userName; /** * user password */ - @ApiModelProperty(name = "userPassword", notes = "USER_PASSWORD",dataType = "String",required = true) private String userPassword; /** From 96d49a09501874b9ed8de28638ef48231f8fdd29 Mon Sep 17 00:00:00 2001 From: lilin Date: Mon, 30 Dec 2019 18:26:10 +0800 Subject: [PATCH 16/31] remove delete check exist --- .../dolphinscheduler/api/service/UdfFuncService.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java index df439ce38f..20324928fb 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java @@ -303,12 +303,7 @@ public class UdfFuncService extends BaseService{ @Transactional(rollbackFor = Exception.class) public Result delete(int id) { Result result = new Result(); - //check exist - UdfFunc udfFunc = udfFuncMapper.selectById(id); - if (udfFunc == null) { - putMsg(result, Status.UDF_FUNCTION_NOT_EXIST); - return result; - } + udfFuncMapper.deleteById(id); udfUserMapper.deleteByUdfFuncId(id); putMsg(result, Status.SUCCESS); From dc4e46db410ef7fcd2a1d5efd89ffdd509b76245 Mon Sep 17 00:00:00 2001 From: loushang Date: Mon, 30 Dec 2019 18:58:38 +0800 Subject: [PATCH 17/31] fix Monitor bug --- .../api/service/MonitorService.java | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java index 91e58ef6cc..e9877216f1 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java @@ -33,6 +33,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import static org.apache.dolphinscheduler.common.utils.Preconditions.*; + /** * monitor service */ @@ -116,20 +118,9 @@ public class MonitorService extends BaseService{ } public List getServerListFromZK(boolean isMaster){ - List servers = new ArrayList<>(); - ZookeeperMonitor zookeeperMonitor = null; - try{ - zookeeperMonitor = new ZookeeperMonitor(); - ZKNodeType zkNodeType = isMaster ? ZKNodeType.MASTER : ZKNodeType.WORKER; - servers = zookeeperMonitor.getServersList(zkNodeType); - }catch (Exception e){ - throw e; - }finally { - if(zookeeperMonitor != null){ - zookeeperMonitor.close(); - } - } - return servers; + + ZKNodeType zkNodeType = isMaster ? ZKNodeType.MASTER : ZKNodeType.WORKER; + return checkNotNull(zookeeperMonitor).getServersList(zkNodeType); } } From b73979360e91dccdd7e9761b6d3caa53ffce765a Mon Sep 17 00:00:00 2001 From: qiaozhanwei Date: Mon, 30 Dec 2019 19:01:08 +0800 Subject: [PATCH 18/31] DataSourceMapperTest UT modify #1465 (#1642) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * remove LogViewServiceGrpc.java file and pom modify * remove kazoo * remove kazoo * remove kazoo * remove common monitor package * add license * remove kazoo modify * remove kazoo modify * remove kazoo modify * remove kazoo modify * remove kazoo modify * remove kazoo modify * install.sh remove python kazoo * add system param whether repeat running * remove kazoo modify * BusinessTimeUtils remove whther repeat running inner param * add AccessTokenMapperTest UT * CI UT yml modify，start postgresql and zookeeper by default * add AlertGroupMapperTest UT in github action * Conflicts reslove * AlertMappert UT modify * AlertMappert UT modify * AlertMappert UT modify * CommandMapperTest UT modify * DataSourceMapperTest UT modify --- .../dao/mapper/DataSourceMapperTest.java | 299 +++++++++++++----- 1 file changed, 226 insertions(+), 73 deletions(-) diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapperTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapperTest.java index 79b25dd5fa..c826236239 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapperTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/mapper/DataSourceMapperTest.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.dao.mapper; import org.apache.dolphinscheduler.common.enums.DbType; +import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.entity.DataSource; import org.apache.dolphinscheduler.dao.entity.DatasourceUser; import com.baomidou.mybatisplus.core.metadata.IPage; @@ -27,88 +28,116 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.annotation.Rollback; import org.springframework.test.context.junit4.SpringRunner; +import org.springframework.transaction.annotation.Transactional; -import java.util.Date; -import java.util.List; +import java.util.*; +import static org.hamcrest.Matchers.*; +import static org.junit.Assert.*; + +/** + * datasource mapper test + */ @RunWith(SpringRunner.class) @SpringBootTest +@Transactional +@Rollback(true) public class DataSourceMapperTest { + /** + * datasource mapper + */ @Autowired DataSourceMapper dataSourceMapper; + /** + * datasource user relation mapper + */ @Autowired DataSourceUserMapper dataSourceUserMapper; /** - * insert - * @return DataSource + * test insert */ - private DataSource insertOne(){ - //insertOne - DataSource dataSource = new DataSource(); - dataSource.setUserId(4); - dataSource.setName("data source test"); - dataSource.setType(DbType.MYSQL); - dataSource.setNote("mysql test"); - dataSource.setConnectionParams("hello mysql"); - dataSource.setUpdateTime(new Date()); - dataSource.setCreateTime(new Date()); - dataSourceMapper.insert(dataSource); - return dataSource; + @Test + public void testInsert(){ + DataSource dataSource = createDataSource(); + assertNotNull(dataSource.getId()); + assertThat(dataSource.getId(), greaterThan(0)); } /** - * test update + * test query */ @Test - public void testUpdate(){ - //insertOne - DataSource dataSource = insertOne(); - //update - dataSource.setUpdateTime(new Date()); - int update = dataSourceMapper.updateById(dataSource); - Assert.assertEquals(update, 1); - dataSourceMapper.deleteById(dataSource.getId()); + public void testSelectById() { + DataSource expectedDataSource = createDataSource(); + DataSource actualDataSource = dataSourceMapper.selectById(expectedDataSource.getId()); + assertEquals(expectedDataSource, actualDataSource); } + /** - * test delete + * test query */ @Test - public void testDelete(){ + public void testUpdate() { + DataSource expectedDataSource = createDataSource(); + + expectedDataSource.setName("modify " + expectedDataSource.getName()); + expectedDataSource.setNote("modifiy " + expectedDataSource.getNote()); + expectedDataSource.setUserId(2); + expectedDataSource.setType(DbType.HIVE); + expectedDataSource.setConnectionParams("modify " + expectedDataSource.getConnectionParams()); + expectedDataSource.setUpdateTime(DateUtils.getCurrentDate()); - DataSource dataSource = insertOne(); - int delete = dataSourceMapper.deleteById(dataSource.getId()); - Assert.assertEquals(delete, 1); + dataSourceMapper.updateById(expectedDataSource); + + DataSource actualDataSource = dataSourceMapper.selectById(expectedDataSource.getId()); + + assertEquals(expectedDataSource, actualDataSource); } + /** - * test query + * test delete */ @Test - public void testQuery() { - DataSource dataSource = insertOne(); - //query - List dataSources = dataSourceMapper.selectList(null); - Assert.assertNotEquals(dataSources.size(), 0); - dataSourceMapper.deleteById(dataSource.getId()); + public void testDelete(){ + DataSource expectedDataSource = createDataSource(); + + dataSourceMapper.deleteById(expectedDataSource.getId()); + + DataSource actualDataSource = dataSourceMapper.selectById(expectedDataSource.getId()); + + assertNull(actualDataSource); } + + /** * test query datasource by type */ @Test public void testQueryDataSourceByType() { - DataSource dataSource = insertOne(); - //query - List dataSources = dataSourceMapper.queryDataSourceByType( - 0, DbType.MYSQL.ordinal() - ); - Assert.assertNotEquals(dataSources.size(), 0); - dataSourceMapper.deleteById(dataSource.getId()); + Integer userId = 1; + + Map datasourceMap = createDataSourceMap(userId, "test"); + + List actualDataSources = dataSourceMapper.queryDataSourceByType( + 0, DbType.MYSQL.ordinal()); + + assertThat(actualDataSources.size(), greaterThanOrEqualTo(2)); + + for (DataSource actualDataSource : actualDataSources){ + DataSource expectedDataSource = datasourceMap.get(actualDataSource.getId()); + if (expectedDataSource != null){ + assertEquals(expectedDataSource,actualDataSource); + } + } + } /** @@ -116,12 +145,23 @@ public class DataSourceMapperTest { */ @Test public void testSelectPaging() { - DataSource dataSource = insertOne(); - Page page = new Page(1, 3); - IPage dataSourceIPage = dataSourceMapper.selectPaging(page, - 4, null); - Assert.assertNotEquals(dataSourceIPage.getTotal(), 0); - dataSourceMapper.deleteById(dataSource.getId()); + String name = "test"; + Integer userId = 1; + + Map expectedDataSourceMap = createDataSourceMap(userId, name); + + Page page = new Page(0, 4); + + IPage dataSourceIPage = dataSourceMapper.selectPaging(page, userId, name); + List actualDataSources = dataSourceIPage.getRecords(); + + for (DataSource actualDataSource : actualDataSources){ + DataSource expectedDataSource = expectedDataSourceMap.get(actualDataSource.getId()); + if (expectedDataSource != null){ + assertEquals(expectedDataSource,actualDataSource); + } + } + } /** @@ -129,10 +169,17 @@ public class DataSourceMapperTest { */ @Test public void testQueryDataSourceByName() { - DataSource dataSource = insertOne(); - List dataSources = dataSourceMapper.queryDataSourceByName("data source test"); - Assert.assertNotEquals(dataSources.size(), 0); - dataSourceMapper.deleteById(dataSource.getId()); + String name = "test"; + DataSource expectedDataSource = createDataSource(name); + + List actualDataSources = dataSourceMapper.queryDataSourceByName(name); + + for (DataSource actualDataSource : actualDataSources){ + if (expectedDataSource.getId() == actualDataSource.getId()){ + assertEquals(expectedDataSource,actualDataSource); + } + } + } /** @@ -140,17 +187,20 @@ public class DataSourceMapperTest { */ @Test public void testQueryAuthedDatasource() { + String name = "test"; + Integer userId = 1; - DataSource dataSource = insertOne(); - DatasourceUser datasourceUser = new DatasourceUser(); - datasourceUser.setUserId(3); - datasourceUser.setDatasourceId(dataSource.getId()); - dataSourceUserMapper.insert(datasourceUser); + Map expectedDataSourceMap = createDataSourceMap(userId, name); + + List actualDataSources = dataSourceMapper.queryAuthedDatasource(userId); + + for (DataSource actualDataSource : actualDataSources){ + DataSource expectedDataSource = expectedDataSourceMap.get(actualDataSource.getId()); + if (expectedDataSource != null){ + assertEquals(expectedDataSource,actualDataSource); + } + } - List dataSources = dataSourceMapper.queryAuthedDatasource(3); - Assert.assertNotEquals(dataSources.size(), 0); - dataSourceMapper.deleteById(dataSource.getId()); - dataSourceUserMapper.deleteById(datasourceUser.getId()); } /** @@ -158,10 +208,19 @@ public class DataSourceMapperTest { */ @Test public void testQueryDatasourceExceptUserId() { - DataSource dataSource = insertOne(); - List dataSources = dataSourceMapper.queryDatasourceExceptUserId(3); - Assert.assertNotEquals(dataSources.size(), 0); - dataSourceMapper.deleteById(dataSource.getId()); + String name = "test"; + Integer userId = 1; + + Map expectedDataSourceMap = createDataSourceMap(userId, name); + + List actualDataSources = dataSourceMapper.queryDatasourceExceptUserId(userId); + + for (DataSource actualDataSource : actualDataSources){ + DataSource expectedDataSource = expectedDataSourceMap.get(actualDataSource.getId()); + if (expectedDataSource != null){ + assertEquals(expectedDataSource,actualDataSource); + } + } } /** @@ -169,13 +228,107 @@ public class DataSourceMapperTest { */ @Test public void testListAllDataSourceByType() { + Integer count = 10; + + Map expectedDataSourceMap = createDataSourceMap(count); + + List actualDataSources = dataSourceMapper.listAllDataSourceByType(DbType.MYSQL.ordinal()); + + assertThat(actualDataSources.size(), greaterThanOrEqualTo(count)); + + for (DataSource actualDataSource : actualDataSources){ + DataSource expectedDataSource = expectedDataSourceMap.get(actualDataSource.getId()); + if (expectedDataSource != null){ + assertEquals(expectedDataSource,actualDataSource); + } + } + } + + /** + * create datasource relation + * @param userId + */ + private Map createDataSourceMap(Integer userId,String name){ + + Map dataSourceMap = new HashMap<>(); + + DataSource dataSource = createDataSource(userId, name); + + dataSourceMap.put(dataSource.getId(),dataSource); + + DataSource otherDataSource = createDataSource(userId + 1,name); + + DatasourceUser datasourceUser = new DatasourceUser(); + + datasourceUser.setDatasourceId(otherDataSource.getId()); + datasourceUser.setUserId(userId); + datasourceUser.setPerm(7); + datasourceUser.setCreateTime(DateUtils.getCurrentDate()); + datasourceUser.setUpdateTime(DateUtils.getCurrentDate()); + + dataSourceUserMapper.insert(datasourceUser); + + dataSourceMap.put(otherDataSource.getId(), otherDataSource); + + return dataSourceMap; + } + + /** + * create datasource map + * @param count datasource count + * @return datasource map + */ + private Map createDataSourceMap(Integer count){ + Map dataSourceMap = new HashMap<>(); - DataSource dataSource = insertOne(); + for (int i = 0 ; i < count ;i++){ + DataSource dataSource = createDataSource("test"); + dataSourceMap.put(dataSource.getId(),dataSource); + } - List dataSources = dataSourceMapper.queryDataSourceByType(4, DbType.MYSQL.ordinal()); - Assert.assertNotEquals(dataSources.size(), 0); - List dataSources2 = dataSourceMapper.queryDataSourceByType(10091, DbType.MYSQL.ordinal()); - Assert.assertEquals(dataSources2.size(), 0); - dataSourceMapper.deleteById(dataSource.getId()); + return dataSourceMap; } + + + /** + * create datasource + * @return datasource + */ + private DataSource createDataSource(){ + return createDataSource(1,"test"); + } + + + /** + * create datasource + * @param name name + * @return datasource + */ + private DataSource createDataSource(String name){ + return createDataSource(1,name); + } + + /** + * create datasource + * @param userId userId + * @param name name + * @return datasource + */ + private DataSource createDataSource(Integer userId,String name){ + Random random = new Random(); + DataSource dataSource = new DataSource(); + dataSource.setUserId(userId); + dataSource.setName(name); + dataSource.setType(DbType.MYSQL); + dataSource.setNote("mysql test"); + dataSource.setConnectionParams("hello mysql"); + dataSource.setUpdateTime(DateUtils.getCurrentDate()); + dataSource.setCreateTime(DateUtils.getCurrentDate()); + + dataSourceMapper.insert(dataSource); + + return dataSource; + } + + } \ No newline at end of file From e1d39e1e8e5e283c9e45f80a609013d654579abd Mon Sep 17 00:00:00 2001 From: AmoryWang <2421902248@qq.com> Date: Mon, 30 Dec 2019 19:42:48 +0800 Subject: [PATCH 19/31] add UT for common thread package (#1628) * add common thread ut * Update ThreadUtilsTest.java add license header Co-authored-by: dailidong --- .../common/threadutils/ThreadUtilsTest.java | 190 ++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java new file mode 100644 index 0000000000..52c8031e75 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/threadutils/ThreadUtilsTest.java @@ -0,0 +1,190 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.threadutils; + +import org.apache.dolphinscheduler.common.thread.Stopper; +import org.apache.dolphinscheduler.common.thread.ThreadPoolExecutors; +import org.apache.dolphinscheduler.common.thread.ThreadUtils; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Calendar; +import java.util.concurrent.*; + +import static org.junit.Assert.*; + + +public class ThreadUtilsTest { + private static final Logger logger = LoggerFactory.getLogger(ThreadUtilsTest.class); + /** + * create a naming thread + */ + @Test + public void testNewDaemonFixedThreadExecutor() { + // create core size and max size are all 3 + ExecutorService testExec = ThreadUtils.newDaemonFixedThreadExecutor("test-exec-thread",10); + + for (int i = 0; i < 19; i++) { + final int index = i; + testExec.submit(() -> { + System.out.println("do some work index " + index); + }); + } + assertFalse(testExec.isShutdown()); + testExec.shutdownNow(); + assertTrue(testExec.isShutdown()); + + } + + /** + * test schedulerThreadExecutor as for print time in scheduler + * default check thread is 1 + */ + @Test + public void testNewDaemonScheduleThreadExecutor() { + + ScheduledExecutorService scheduleService = ThreadUtils.newDaemonThreadScheduledExecutor("scheduler-thread", 1); + Calendar start = Calendar.getInstance(); + Calendar globalTimer = Calendar.getInstance(); + globalTimer.set(2019, Calendar.DECEMBER, 1, 0, 0, 0); + // current + Calendar end = Calendar.getInstance(); + end.set(2019, Calendar.DECEMBER, 1, 0, 0, 3); + Runnable schedulerTask = new Runnable() { + @Override + public void run() { + start.set(2019, Calendar.DECEMBER, 1, 0, 0, 0); + int index = 0; + // send heart beat work + while (start.getTime().getTime() <= end.getTime().getTime()) { + System.out.println("worker here"); + System.out.println(index ++); + start.add(Calendar.SECOND, 1); + globalTimer.add(Calendar.SECOND, 1); + } + System.out.println("time is " + System.currentTimeMillis()); + } + }; + scheduleService.scheduleAtFixedRate(schedulerTask, 2, 10, TimeUnit.SECONDS); + assertFalse(scheduleService.isShutdown()); + try { + Thread.sleep(60000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + scheduleService.shutdownNow(); + assertTrue(scheduleService.isShutdown()); + } + + /** + * test stopper is working normal + */ + @Test + public void testStopper() { + assertTrue(Stopper.isRunning()); + Stopper.stop(); + assertTrue(Stopper.isStoped()); + } + + /** + * test threadPoolExecutors with 3 workers and current each 5 tasks + * @throws InterruptedException + */ + @Test + public void testThreadInfo() throws InterruptedException { + ThreadPoolExecutors workers = ThreadPoolExecutors.getInstance("worker", 3); + for (int i = 0; i < 5; ++i ) { + int index = i; + workers.execute(() -> { + for (int j = 0; j < 10; ++j) { + try { + Thread.sleep(1000); + System.out.printf("worker %d is doing the task", index); + System.out.println(); + workers.printStatus(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + }); + workers.submit(() -> { + for (int j = 0; j < 10; ++j) { + try { + Thread.sleep(1000); + System.out.printf("worker_2 %d is doing the task", index); + System.out.println(); + workers.printStatus(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + }); + } + Thread.sleep(50001); + workers.shutdown(); + } + + /** + * test a single daemon thread pool + */ + @Test + public void testNewDaemonSingleThreadExecutor() { + ExecutorService threadTest = ThreadUtils.newDaemonSingleThreadExecutor("thread_test"); + threadTest.execute(() -> { + for (int i = 0; i < 100; ++i) { + System.out.println("daemon working "); + } + + }); + assertFalse(threadTest.isShutdown()); + threadTest.shutdownNow(); + assertTrue(threadTest.isShutdown()); + } + + @Test + public void testNewDaemonCachedThreadPool() { + + ThreadPoolExecutor threadPoolExecutor = ThreadUtils.newDaemonCachedThreadPool("threadTest-"); + Thread thread1 = threadPoolExecutor.getThreadFactory().newThread(() -> { + for (int i = 0; i < 10; ++i) { + System.out.println("this task is with index " + i ); + } + }); + assertTrue(thread1.getName().startsWith("threadTest-")); + assertFalse(threadPoolExecutor.isShutdown()); + threadPoolExecutor.shutdown(); + assertTrue(threadPoolExecutor.isShutdown()); + } + + @Test + public void testNewDaemonCachedThreadPoolWithThreadNumber() { + ThreadPoolExecutor threadPoolExecutor = ThreadUtils.newDaemonCachedThreadPool("threadTest--", 3, 10); + for (int i = 0; i < 10; ++ i) { + threadPoolExecutor.getThreadFactory().newThread(() -> { + assertEquals(3, threadPoolExecutor.getActiveCount()); + System.out.println("this task is first work to do"); + }); + } + assertFalse(threadPoolExecutor.isShutdown()); + threadPoolExecutor.shutdown(); + assertTrue(threadPoolExecutor.isShutdown()); + } + + + +} From a51242d445cfcc00086235f8e6b202c4c8577ebe Mon Sep 17 00:00:00 2001 From: Jave-Chen Date: Mon, 30 Dec 2019 21:21:39 +0800 Subject: [PATCH 20/31] Merge ut sonarcloud into one file (#1647) * merge ut and sonarcloud to one action turn on sonarcloud ut * fix error * test in github --- .github/workflows/SonarCloud.yml | 51 -------------------------------- .github/workflows/ci_ut.yml | 13 ++++++++ 2 files changed, 13 insertions(+), 51 deletions(-) delete mode 100644 .github/workflows/SonarCloud.yml diff --git a/.github/workflows/SonarCloud.yml b/.github/workflows/SonarCloud.yml deleted file mode 100644 index fd5b0585db..0000000000 --- a/.github/workflows/SonarCloud.yml +++ /dev/null @@ -1,51 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -name: SonarCloud -on: [push, pull_request] -jobs: - sonarCloudTrigger: - name: SonarCloud Trigger - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - uses: actions/setup-java@v1 - with: - java-version: 8 - - uses: actions/cache@v1 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-sonarqube - restore-keys: | - ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-sonarqube - ${{ runner.os }}-maven- - - name: Maven clean - run: mvn clean - - name: Run SonarCloud analyse - run: > - mvn clean --batch-mode - org.jacoco:jacoco-maven-plugin:prepare-agent - verify - org.sonarsource.scanner.maven:sonar-maven-plugin:sonar - -Dmaven.test.skip=true - -Dsonar.host.url=https://sonarcloud.io - -Dsonar.organization=apache - -Dsonar.projectKey=apache-dolphinscheduler - -Dsonar.login=e4058004bc6be89decf558ac819aa1ecbee57682 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index c415d35619..f76c403a8c 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -49,6 +49,19 @@ jobs: export MAVEN_OPTS='-Dmaven.repo.local=.m2/repository -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -XX:-UseGCOverheadLimit -Xmx3g' mvn test -Dmaven.test.skip=false cobertura:cobertura CODECOV_TOKEN="09c2663f-b091-4258-8a47-c981827eb29a" bash <(curl -s https://codecov.io/bash) + - name: Run SonarCloud analysis + run: > + mvn clean --batch-mode + org.jacoco:jacoco-maven-plugin:prepare-agent + verify + org.sonarsource.scanner.maven:sonar-maven-plugin:sonar + -Dsonar.host.url=https://sonarcloud.io + -Dsonar.organization=apache + -Dsonar.projectKey=apache-dolphinscheduler + -Dsonar.login=e4058004bc6be89decf558ac819aa1ecbee57682 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - name: Collect logs run: | mkdir -p ${LOG_DIR} From b549ee3901c3fc823512f86d1a795af7a9efb08a Mon Sep 17 00:00:00 2001 From: loushang Date: Tue, 31 Dec 2019 10:30:20 +0800 Subject: [PATCH 21/31] fix zk monitor bug --- .../apache/dolphinscheduler/api/service/MonitorService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java index e9877216f1..118c5ce936 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/MonitorService.java @@ -119,8 +119,9 @@ public class MonitorService extends BaseService{ public List getServerListFromZK(boolean isMaster){ + checkNotNull(zookeeperMonitor); ZKNodeType zkNodeType = isMaster ? ZKNodeType.MASTER : ZKNodeType.WORKER; - return checkNotNull(zookeeperMonitor).getServersList(zkNodeType); + return zookeeperMonitor.getServersList(zkNodeType); } } From 677e9cfd3344b11aeaa9f9d3f8ab2695f6d6aef4 Mon Sep 17 00:00:00 2001 From: Tboy Date: Tue, 31 Dec 2019 16:03:08 +0800 Subject: [PATCH 22/31] remove commons-lang3 dependancy (#1655) --- dolphinscheduler-common/pom.xml | 5 - .../common/job/db/ClickHouseDataSource.java | 2 +- .../common/job/db/DB2ServerDataSource.java | 2 +- .../common/job/db/HiveDataSource.java | 7 +- .../common/job/db/MySQLDataSource.java | 2 +- .../common/job/db/OracleDataSource.java | 2 +- .../common/job/db/PostgreDataSource.java | 2 +- .../common/job/db/SQLServerDataSource.java | 2 +- .../common/job/db/SparkDataSource.java | 2 +- .../common/model/TaskNode.java | 2 +- .../common/utils/CommonUtils.java | 1 - .../common/utils/EncryptionUtils.java | 1 - .../common/utils/EnumUtils.java | 33 +++++ .../common/utils/HadoopUtils.java | 1 - .../common/utils/OSUtils.java | 6 +- .../common/utils/SchemaUtils.java | 1 - .../common/utils/StringUtils.java | 125 ++++++++++++++++++ .../common/utils/TaskParametersUtils.java | 1 - .../common/zk/AbstractZKClient.java | 2 +- .../common/utils/EncryptionUtilsTest.java | 2 - 20 files changed, 170 insertions(+), 31 deletions(-) create mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EnumUtils.java create mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index 1bfac571ba..10cede7988 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -330,11 +330,6 @@ - - org.apache.commons - commons-lang3 - - org.postgresql postgresql diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/ClickHouseDataSource.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/ClickHouseDataSource.java index c2e1f86d30..dfcda99141 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/ClickHouseDataSource.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/ClickHouseDataSource.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.common.job.db; -import org.apache.commons.lang3.StringUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/DB2ServerDataSource.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/DB2ServerDataSource.java index 8783c8ebbe..e6d7b94b51 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/DB2ServerDataSource.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/DB2ServerDataSource.java @@ -16,8 +16,8 @@ */ package org.apache.dolphinscheduler.common.job.db; -import org.apache.commons.lang3.StringUtils; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/HiveDataSource.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/HiveDataSource.java index fc1be356f5..2970354e6b 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/HiveDataSource.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/HiveDataSource.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.common.job.db; -import org.apache.commons.lang3.StringUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -29,9 +29,6 @@ public class HiveDataSource extends BaseDataSource { private static final Logger logger = LoggerFactory.getLogger(HiveDataSource.class); - - - /** * gets the JDBC url for the data source connection * @return @@ -49,8 +46,6 @@ public class HiveDataSource extends BaseDataSource { jdbcUrl += ";principal=" + getPrincipal(); } - - if (StringUtils.isNotEmpty(getOther())) { jdbcUrl += ";" + getOther(); } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/MySQLDataSource.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/MySQLDataSource.java index 0e850ee2de..f50de0dee3 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/MySQLDataSource.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/MySQLDataSource.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.common.job.db; -import org.apache.commons.lang3.StringUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/OracleDataSource.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/OracleDataSource.java index 67c035d5be..ddc30d939a 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/OracleDataSource.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/OracleDataSource.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.common.job.db; -import org.apache.commons.lang3.StringUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/PostgreDataSource.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/PostgreDataSource.java index 332a9cb936..ecc29c73e8 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/PostgreDataSource.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/PostgreDataSource.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.common.job.db; -import org.apache.commons.lang3.StringUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SQLServerDataSource.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SQLServerDataSource.java index 084b10d425..7a51ff7a45 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SQLServerDataSource.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SQLServerDataSource.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.common.job.db; -import org.apache.commons.lang3.StringUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SparkDataSource.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SparkDataSource.java index a449e398c6..638664c47d 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SparkDataSource.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/job/db/SparkDataSource.java @@ -16,7 +16,7 @@ */ package org.apache.dolphinscheduler.common.job.db; -import org.apache.commons.lang3.StringUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java index 076966ef72..40efd0a24f 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/model/TaskNode.java @@ -26,7 +26,7 @@ import com.alibaba.fastjson.JSONObject; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import org.apache.commons.lang3.StringUtils; +import org.apache.dolphinscheduler.common.utils.StringUtils; import java.io.IOException; import java.util.List; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java index 1203cbb26b..842c74edc0 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/CommonUtils.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.common.utils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ResUploadType; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EncryptionUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EncryptionUtils.java index dd4d176b3a..c153ec817a 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EncryptionUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EncryptionUtils.java @@ -17,7 +17,6 @@ package org.apache.dolphinscheduler.common.utils; import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.lang3.StringUtils; /** * encryption utils diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EnumUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EnumUtils.java new file mode 100644 index 0000000000..dc2e3a0750 --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/EnumUtils.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils; + + + +public class EnumUtils { + + public static > E getEnum(final Class enumClass, final String enumName) { + if (enumName == null) { + return null; + } + try { + return Enum.valueOf(enumClass, enumName); + } catch (final IllegalArgumentException ex) { + return null; + } + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java index 6a10a87b51..072266fb7f 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/HadoopUtils.java @@ -23,7 +23,6 @@ import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONException; import com.alibaba.fastjson.JSONObject; import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; import org.apache.hadoop.fs.FileSystem; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java index 20336afd67..0c061fc0ba 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java @@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.common.utils; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.shell.ShellExecutor; import org.apache.commons.configuration.Configuration; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import oshi.SystemInfo; @@ -180,7 +179,7 @@ public class OSUtils { private static List getUserListFromMac() throws IOException { String result = exeCmd("dscl . list /users"); if (StringUtils.isNotEmpty(result)) { - return Arrays.asList(StringUtils.split(result, "\n")); + return Arrays.asList(result.split( "\n")); } return Collections.emptyList(); @@ -251,9 +250,8 @@ public class OSUtils { */ public static String getGroup() throws IOException { String result = exeCmd("groups"); - if (StringUtils.isNotEmpty(result)) { - String[] groupInfo = StringUtils.split(result); + String[] groupInfo = result.split(" "); return groupInfo[0]; } diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java index cd3a2dfc21..a4e16ba8b6 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java @@ -16,7 +16,6 @@ */ package org.apache.dolphinscheduler.common.utils; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java new file mode 100644 index 0000000000..64dd4f3dcd --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/StringUtils.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils; + + +import java.nio.charset.StandardCharsets; +import java.util.regex.Pattern; + + +public class StringUtils { + + public static final int INDEX_NOT_FOUND = -1; + + public static final String EMPTY = ""; + + public static boolean isEmpty(final CharSequence cs) { + return cs == null || cs.length() == 0; + } + + public static boolean isNotEmpty(final CharSequence cs) { + return !isEmpty(cs); + } + + public static boolean isBlank(CharSequence cs){ + int strLen; + if (cs == null || (strLen = cs.length()) == 0) { + return true; + } + for (int i = 0; i < strLen; i++) { + if (Character.isWhitespace(cs.charAt(i)) == false) { + return false; + } + } + return true; + } + + public static boolean isNotBlank(CharSequence str){ + return !isBlank(str); + } + + public static String substringBefore(final String str, final String separator) { + if (isBlank(str) || separator == null) { + return str; + } + if (separator.isEmpty()) { + return EMPTY; + } + final int pos = str.indexOf(separator); + if (pos == INDEX_NOT_FOUND) { + return str; + } + return str.substring(0, pos); + } + + public static String substringAfter(final String str, final String separator) { + if (isBlank(str)) { + return str; + } + if (separator == null) { + return EMPTY; + } + final int pos = str.indexOf(separator); + if (pos == INDEX_NOT_FOUND) { + return EMPTY; + } + return str.substring(pos + separator.length()); + } + + public static String substringAfterLast(final String str, final String separator) { + if (isEmpty(str)) { + return str; + } + if (isEmpty(separator)) { + return EMPTY; + } + final int pos = str.lastIndexOf(separator); + if (pos == INDEX_NOT_FOUND || pos == str.length() - separator.length()) { + return EMPTY; + } + return str.substring(pos + separator.length()); + } + + public static String getUtf8String(byte[] bytes){ + return new String(bytes, StandardCharsets.UTF_8); + } + + public static byte[] getUtf8Bytes(String str){ + return str.getBytes(StandardCharsets.UTF_8); + } + + public static boolean hasChinese(String str) { + if (str == null) { + return false; + } + Pattern pattern = Pattern.compile("[\\u4E00-\\u9FBF]+"); + return pattern.matcher(str).find(); + } + + public static boolean hasSpace(String str) { + if (str == null) { + return false; + } + int len = str.length(); + for (int i = 0; i < len; i++) { + if (str.charAt(i) == ' ') { + return true; + } + } + return false; + } +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java index 757d205dea..28e2593359 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtils.java @@ -28,7 +28,6 @@ import org.apache.dolphinscheduler.common.task.shell.ShellParameters; import org.apache.dolphinscheduler.common.task.spark.SparkParameters; import org.apache.dolphinscheduler.common.task.sql.SqlParameters; import org.apache.dolphinscheduler.common.task.subprocess.SubProcessParameters; -import org.apache.commons.lang3.EnumUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java index c3ba718270..b03b88ae65 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/zk/AbstractZKClient.java @@ -28,7 +28,6 @@ import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.commons.lang3.StringUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.imps.CuratorFrameworkState; import org.apache.curator.framework.recipes.locks.InterProcessMutex; @@ -39,6 +38,7 @@ import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.common.utils.ResInfo; +import org.apache.dolphinscheduler.common.utils.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/EncryptionUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/EncryptionUtilsTest.java index b72e7b7d64..fc7f65930c 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/EncryptionUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/EncryptionUtilsTest.java @@ -16,8 +16,6 @@ */ package org.apache.dolphinscheduler.common.utils; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.lang3.StringUtils; import org.junit.Assert; import org.junit.Test; From bc8e06bb08be1eb7361e88f6c4f2c799aa170677 Mon Sep 17 00:00:00 2001 From: khadgarmage Date: Tue, 31 Dec 2019 22:51:56 +0800 Subject: [PATCH 23/31] SchemaUtils.java TaskParametersUtils.java unit test (#1660) * schemautils test and TaskParametersUtils test --- .../common/utils/SchemaUtils.java | 24 ++-- .../common/utils/SchemaUtilsTest.java | 119 ++++++++++++++++++ .../common/utils/TaskParametersUtilsTest.java | 48 +++++++ 3 files changed, 179 insertions(+), 12 deletions(-) create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtilsTest.java diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java index a4e16ba8b6..7d341f3b20 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/SchemaUtils.java @@ -35,7 +35,7 @@ import java.util.regex.Pattern; * */ public class SchemaUtils { - + private static final Logger logger = LoggerFactory.getLogger(SchemaUtils.class); private static Pattern p = Pattern.compile("\\s*|\t|\r|\n"); @@ -50,11 +50,11 @@ public class SchemaUtils { if(schemaDirArr == null || schemaDirArr.length == 0) { return null; } - + for(File file : schemaDirArr) { schemaDirList.add(file.getName()); } - + Collections.sort(schemaDirList , new Comparator() { @Override public int compare(Object o1 , Object o2){ @@ -66,23 +66,23 @@ public class SchemaUtils { if(version1.equals(version2)) { return 0; } - + if(SchemaUtils.isAGreatVersion(version1, version2)) { return 1; } - + return -1; - + } catch (Exception e) { logger.error(e.getMessage(),e); throw new RuntimeException(e); } } }); - + return schemaDirList; } - + /** * Determine whether schemaVersion is higher than version * @param schemaVersion schema version @@ -93,7 +93,7 @@ public class SchemaUtils { if(StringUtils.isEmpty(schemaVersion) || StringUtils.isEmpty(version)) { throw new RuntimeException("schemaVersion or version is empty"); } - + String[] schemaVersionArr = schemaVersion.split("\\."); String[] versionArr = version.split("\\."); int arrLength = schemaVersionArr.length < versionArr.length ? schemaVersionArr.length : versionArr.length; @@ -104,11 +104,11 @@ public class SchemaUtils { return false; } } - + // If the version and schema version is the same from 0 up to the arrlength-1 element,whoever has a larger arrLength has a larger version number return schemaVersionArr.length > versionArr.length; } - + /** * Gets the current software version number of the system * @return current software version @@ -127,7 +127,7 @@ public class SchemaUtils { } return soft_version; } - + /** * Strips the string of space carriage returns and tabs * @param str string diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java new file mode 100644 index 0000000000..907a09e458 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/SchemaUtilsTest.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileNotFoundException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +@RunWith(PowerMockRunner.class) +@PrepareForTest({ LoggerFactory.class, FileUtils.class }) +public class SchemaUtilsTest { + + @Test + public void testReplaceBlank() { + Assert.assertEquals("abc", SchemaUtils.replaceBlank(" abc")); + Assert.assertEquals("abc", SchemaUtils.replaceBlank("abc ")); + Assert.assertEquals("abc", SchemaUtils.replaceBlank("a b c")); + Assert.assertEquals("abc", SchemaUtils.replaceBlank("a b c")); + Assert.assertEquals("", SchemaUtils.replaceBlank(" ")); + Assert.assertEquals("", SchemaUtils.replaceBlank(null)); + Assert.assertEquals("我怕的你", SchemaUtils.replaceBlank("我怕的 你")); + } + + @Test + public void testGetSoftVersion() { + // file not found + try { + SchemaUtils.getSoftVersion(); + } catch (RuntimeException e) { + Assert.assertEquals("Failed to get the product version description file. The file could not be found", + e.getMessage()); + } + + // file exists, fmt is invalid + FileUtils.writeContent2File("32432423", "sql/soft_version"); + Assert.assertEquals("32432423", SchemaUtils.getSoftVersion()); + } + + @Test + public void testIsAGreatVersion() { + // param is null + try { + SchemaUtils.isAGreatVersion(null, null); + } catch (RuntimeException e) { + Assert.assertEquals("schemaVersion or version is empty", e.getMessage()); + } + + // param is "" + try { + SchemaUtils.isAGreatVersion("", ""); + } catch (RuntimeException e) { + Assert.assertEquals("schemaVersion or version is empty", e.getMessage()); + } + Assert.assertFalse(SchemaUtils.isAGreatVersion("1", "1")); + Assert.assertTrue(SchemaUtils.isAGreatVersion("2", "1")); + Assert.assertTrue(SchemaUtils.isAGreatVersion("1.1", "1")); + Assert.assertTrue(SchemaUtils.isAGreatVersion("1.1", "1.0.1")); + Assert.assertFalse(SchemaUtils.isAGreatVersion("1.1", "1.2")); + Assert.assertTrue(SchemaUtils.isAGreatVersion("1.1.1", "1.1")); + Assert.assertTrue(SchemaUtils.isAGreatVersion("10.1.1", "1.01.100")); + try { + SchemaUtils.isAGreatVersion("10.1.1", ".1"); + } catch (Exception e) { + Assert.assertNotNull(e); + } + try { + SchemaUtils.isAGreatVersion("a.1.1", "b.1"); + } catch (Exception e) { + Assert.assertNotNull(e); + } + } + + @Test + public void testGetAllSchemaList() { + //normal + PowerMockito.mockStatic(FileUtils.class); + File[] files = new File[4]; + files[0] = new File("sql/upgrade/1.2.0_schema"); + files[1] = new File("sql/upgrade/1.0.1_schema"); + files[2] = new File("sql/upgrade/1.0.2_schema"); + files[3] = new File("sql/upgrade/1.1.0_schema"); + PowerMockito.when(FileUtils.getAllDir("sql/upgrade")).thenReturn(files); + List real = SchemaUtils.getAllSchemaList(); + List expect = Arrays.asList("1.0.1_schema", "1.0.2_schema", + "1.1.0_schema", "1.2.0_schema"); + Assert.assertTrue(CollectionUtils.isEqualCollection(real, expect)); + + //normal + files = new File[0]; + PowerMockito.when(FileUtils.getAllDir("sql/upgrade")).thenReturn(files); + real = SchemaUtils.getAllSchemaList(); + Assert.assertNull(real); + } +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtilsTest.java new file mode 100644 index 0000000000..db4a86bc26 --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/TaskParametersUtilsTest.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.LoggerFactory; + +@RunWith(PowerMockRunner.class) +@PrepareForTest(LoggerFactory.class) +public class TaskParametersUtilsTest { + + @Test + public void testGetParameters() { + Assert.assertNull(TaskParametersUtils.getParameters("xx", "ttt")); + Assert.assertNull(TaskParametersUtils.getParameters("SHELL", "ttt")); + Assert.assertNotNull(TaskParametersUtils.getParameters("SHELL", "{}")); + Assert.assertNotNull(TaskParametersUtils.getParameters("SQL", "{}")); + Assert.assertNotNull(TaskParametersUtils.getParameters("SUB_PROCESS", "{}")); + Assert.assertNotNull(TaskParametersUtils.getParameters("PROCEDURE", "{}")); + Assert.assertNotNull(TaskParametersUtils.getParameters("MR", "{}")); + Assert.assertNotNull(TaskParametersUtils.getParameters("SPARK", "{}")); + Assert.assertNotNull(TaskParametersUtils.getParameters("PYTHON", "{}")); + Assert.assertNotNull(TaskParametersUtils.getParameters("DEPENDENT", "{}")); + Assert.assertNotNull(TaskParametersUtils.getParameters("FLINK", "{}")); + Assert.assertNotNull(TaskParametersUtils.getParameters("HTTP", "{}")); + } +} From dc6c18902a864cb5a00f57d6fc9a9d91fbd3bbe8 Mon Sep 17 00:00:00 2001 From: samz406 Date: Tue, 31 Dec 2019 23:21:09 +0800 Subject: [PATCH 24/31] modify UdfFuncServiceTest UT (#1644) --- dolphinscheduler-api/pom.xml | 18 ++ .../api/service/UdfFuncServiceTest.java | 196 ++++++++++++++++-- pom.xml | 1 + 3 files changed, 201 insertions(+), 14 deletions(-) diff --git a/dolphinscheduler-api/pom.xml b/dolphinscheduler-api/pom.xml index 06a444aa6d..014799b0b6 100644 --- a/dolphinscheduler-api/pom.xml +++ b/dolphinscheduler-api/pom.xml @@ -250,5 +250,23 @@ ${servlet-api.version} + + org.powermock + powermock-module-junit4 + test + + + + org.powermock + powermock-api-mockito2 + test + + + org.mockito + mockito-core + + + + \ No newline at end of file diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java index 814a9ee7cd..9ec24bbb50 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java @@ -16,43 +16,211 @@ */ package org.apache.dolphinscheduler.api.service; -import org.apache.dolphinscheduler.api.ApiApplicationServer; +import com.baomidou.mybatisplus.core.metadata.IPage; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.PageInfo; +import org.apache.dolphinscheduler.api.utils.Result; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.UdfType; import org.apache.dolphinscheduler.common.enums.UserType; +import org.apache.dolphinscheduler.common.utils.CollectionUtils; +import org.apache.dolphinscheduler.common.utils.PropertyUtils; +import org.apache.dolphinscheduler.dao.entity.Resource; +import org.apache.dolphinscheduler.dao.entity.UdfFunc; import org.apache.dolphinscheduler.dao.entity.User; +import org.apache.dolphinscheduler.dao.mapper.ResourceMapper; +import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper; +import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.powermock.api.mockito.PowerMockito; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.junit4.SpringRunner; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; import java.util.Map; -@RunWith(SpringRunner.class) -@SpringBootTest(classes = ApiApplicationServer.class) +@RunWith(PowerMockRunner.class) +@PrepareForTest(PropertyUtils.class) public class UdfFuncServiceTest { private static final Logger logger = LoggerFactory.getLogger(UdfFuncServiceTest.class); - @Autowired + @InjectMocks private UdfFuncService udfFuncService; + @Mock + private ResourceMapper resourceMapper; + @Mock + private UdfFuncMapper udfFuncMapper; + @Mock + private UDFUserMapper udfUserMapper; + + + @Before + public void setUp() { + PowerMockito.mockStatic(PropertyUtils.class); + } + + @Test + public void testCreateUdfFunction(){ + + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); + //hdfs not start + Result result = udfFuncService.createUdfFunction(getLoginUser(), "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, Integer.MAX_VALUE); + logger.info(result.toString()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(),result.getMsg()); + //resource not exist + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); + result = udfFuncService.createUdfFunction(getLoginUser(), "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, Integer.MAX_VALUE); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(),result.getMsg()); + // success + PowerMockito.when(resourceMapper.selectById(1)).thenReturn(getResource()); + result = udfFuncService.createUdfFunction(getLoginUser(), "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + } @Test - public void queryUdfFuncListPaging(){ + public void testQueryUdfFuncDetail(){ + + PowerMockito.when(udfFuncMapper.selectById(1)).thenReturn(getUdfFunc()); + //resource not exist + Map result = udfFuncService.queryUdfFuncDetail(2); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST,result.get(Constants.STATUS)); + // success + result = udfFuncService.queryUdfFuncDetail(1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + } + + @Test + public void testUpdateUdfFunc(){ + + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false); + PowerMockito.when(udfFuncMapper.selectUdfById(1)).thenReturn(getUdfFunc()); + PowerMockito.when(resourceMapper.selectById(1)).thenReturn(getResource()); + + //UDF_FUNCTION_NOT_EXIST + Map result = udfFuncService.updateUdfFunc(12, "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 1); + logger.info(result.toString()); + Assert.assertEquals(Status.UDF_FUNCTION_NOT_EXIST,result.get(Constants.STATUS)); + + //HDFS_NOT_STARTUP + result = udfFuncService.updateUdfFunc(1, "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 1); + logger.info(result.toString()); + Assert.assertEquals(Status.HDFS_NOT_STARTUP,result.get(Constants.STATUS)); + + //RESOURCE_NOT_EXIST + PowerMockito.when(udfFuncMapper.selectUdfById(11)).thenReturn(getUdfFunc()); + PowerMockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true); + result = udfFuncService.updateUdfFunc(11, "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 12); + logger.info(result.toString()); + Assert.assertEquals(Status.RESOURCE_NOT_EXIST,result.get(Constants.STATUS)); + + //success + result = udfFuncService.updateUdfFunc(11, "UdfFuncServiceTest", "org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String", "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + + } + + @Test + public void testQueryUdfFuncListPaging(){ + + IPage page = new Page<>(1,10); + page.setTotal(1L); + page.setRecords(getList()); + Mockito.when(udfFuncMapper.queryUdfFuncPaging(Mockito.any(Page.class), Mockito.eq(0),Mockito.eq("test"))).thenReturn(page); + Map result = udfFuncService.queryUdfFuncListPaging(getLoginUser(),"test",1,10); + logger.info(result.toString()); + PageInfo pageInfo = (PageInfo) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(pageInfo.getLists())); + } + + @Test + public void testQueryResourceList(){ + Mockito.when(udfFuncMapper.getUdfFuncByType(1, 1)).thenReturn(getList()); + Map result = udfFuncService.queryResourceList(getLoginUser(),1); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS,result.get(Constants.STATUS)); + List udfFuncList = (List) result.get(Constants.DATA_LIST); + Assert.assertTrue(CollectionUtils.isNotEmpty(udfFuncList)); + } + + @Test + public void testDelete(){ + Result result= udfFuncService.delete(122); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + } + + @Test + public void testVerifyUdfFuncByName(){ + + //success + Mockito.when(udfFuncMapper.queryUdfByIdStr(null, "UdfFuncServiceTest")).thenReturn(getList()); + Result result = udfFuncService.verifyUdfFuncByName("test"); + logger.info(result.toString()); + Assert.assertEquals(Status.SUCCESS.getMsg(),result.getMsg()); + //exist + result = udfFuncService.verifyUdfFuncByName("UdfFuncServiceTest"); + logger.info(result.toString()); + Assert.assertEquals(Status.UDF_FUNCTION_EXISTS.getMsg(),result.getMsg()); + } + + /** + * create admin user + * @return + */ + private User getLoginUser(){ User loginUser = new User(); - loginUser.setId(-1); - loginUser.setUserType(UserType.GENERAL_USER); + loginUser.setUserType(UserType.ADMIN_USER); + loginUser.setId(1); + return loginUser; + } + + /** + * get resourceId + */ + private Resource getResource(){ - Map map = udfFuncService.queryUdfFuncListPaging(loginUser, "", 1, 10); - Assert.assertEquals(Status.SUCCESS, map.get(Constants.STATUS)); + Resource resource = new Resource(); + resource.setId(1); + resource.setAlias("test"); + return resource; + } - PageInfo pageInfo = (PageInfo) map.get("data"); - logger.info(pageInfo.getLists().toString()); + private List getList(){ + List udfFuncList = new ArrayList<>(); + udfFuncList.add(getUdfFunc()); + return udfFuncList; + } + /** + * get UdfFunc id + */ + private UdfFunc getUdfFunc(){ + UdfFunc udfFunc = new UdfFunc(); + udfFunc.setFuncName("UdfFuncServiceTest"); + udfFunc.setClassName("org.apache.dolphinscheduler.api.service.UdfFuncServiceTest"); + udfFunc.setResourceId(0); + udfFunc.setResourceName("UdfFuncServiceTest"); + udfFunc.setCreateTime(new Date()); + udfFunc.setDatabase("database"); + udfFunc.setUpdateTime(new Date()); + udfFunc.setType(UdfType.HIVE); + return udfFunc; } } \ No newline at end of file diff --git a/pom.xml b/pom.xml index fafa7be6ef..5e94b16768 100644 --- a/pom.xml +++ b/pom.xml @@ -675,6 +675,7 @@ **/api/service/WorkerGroupServiceTest.java **/api/service/AlertGroupServiceTest.java **/api/service/ProjectServiceTest.java + **/api/service/UdfFuncServiceTest.java **/alert/utils/ExcelUtilsTest.java **/alert/utils/FuncUtilsTest.java **/alert/utils/JSONUtilsTest.java From b4711194438ff34b3aa4d6e3430e0ec920dace06 Mon Sep 17 00:00:00 2001 From: zhukai Date: Tue, 31 Dec 2019 23:33:27 +0800 Subject: [PATCH 25/31] Add ParamUtilsTest which is the UT of ParamUtils (#1634) 1. Add ParamUtilsTest. 2. Add a null check in the method convert. 3. Add the UT path to the root pom. --- .../server/utils/ParamUtils.java | 4 + .../server/utils/ParamUtilsTest.java | 129 ++++++++++++++++++ pom.xml | 1 + 3 files changed, 134 insertions(+) create mode 100644 dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ParamUtils.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ParamUtils.java index 3965b0e8f4..1d7a80daf0 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ParamUtils.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/utils/ParamUtils.java @@ -93,6 +93,10 @@ public class ParamUtils { * @return Map of converted */ public static Map convert(Map paramsMap){ + if(paramsMap == null){ + return null; + } + Map map = new HashMap<>(); Iterator> iter = paramsMap.entrySet().iterator(); while (iter.hasNext()){ diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java new file mode 100644 index 0000000000..b91bff7659 --- /dev/null +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.server.utils; + +import com.alibaba.fastjson.JSON; +import org.apache.dolphinscheduler.common.enums.CommandType; +import org.apache.dolphinscheduler.common.enums.DataType; +import org.apache.dolphinscheduler.common.enums.Direct; +import org.apache.dolphinscheduler.common.process.Property; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +/** + * Test ParamUtils + */ +public class ParamUtilsTest { + + private static final Logger logger = LoggerFactory.getLogger(ParamUtilsTest.class); + + //Define global variables + public Map globalParams = new HashMap<>(); + + public Map globalParamsMap = new HashMap<>(); + + public Map localParams = new HashMap<>(); + + /** + * Init params + * @throws Exception + */ + @Before + public void setUp() throws Exception { + + Property property = new Property(); + property.setProp("global_param"); + property.setDirect(Direct.IN); + property.setType(DataType.VARCHAR); + property.setValue("${system.biz.date}"); + globalParams.put("global_param", property); + + globalParamsMap.put("global_param", "${system.biz.date}"); + + Property localProperty = new Property(); + localProperty.setProp("local_param"); + localProperty.setDirect(Direct.IN); + localProperty.setType(DataType.VARCHAR); + localProperty.setValue("${global_param}"); + localParams.put("local_param", localProperty); + } + + /** + * Test convert + */ + @Test + public void testConvert() { + + //The expected value + String expected = "{\"global_param\":{\"direct\":\"IN\",\"prop\":\"global_param\",\"type\":\"VARCHAR\",\"value\":\"20191229\"},\"local_param\":{\"direct\":\"IN\",\"prop\":\"local_param\",\"type\":\"VARCHAR\",\"value\":\"20191229\"}}"; + + //The expected value when globalParams is null but localParams is not null + String expected1 = "{\"local_param\":{\"direct\":\"IN\",\"prop\":\"local_param\",\"type\":\"VARCHAR\",\"value\":\"20191229\"}}"; + + //Invoke convert + Map paramsMap = ParamUtils.convert(globalParams, globalParamsMap, localParams, CommandType.START_PROCESS, new Date()); + String result = JSON.toJSONString(paramsMap); + assertEquals(expected, result); + + for (Map.Entry entry : paramsMap.entrySet()) { + + String key = entry.getKey(); + Property prop = entry.getValue(); + logger.info(key + " : " + prop.getValue()); + } + + //Invoke convert with null globalParams + Map paramsMap1 = ParamUtils.convert(null, globalParamsMap, localParams, CommandType.START_PROCESS, new Date()); + String result1 = JSON.toJSONString(paramsMap1); + assertEquals(expected1, result1); + + //Null check, invoke convert with null globalParams and null localParams + Map paramsMap2 = ParamUtils.convert(null, globalParamsMap, null, CommandType.START_PROCESS, new Date()); + assertNull(paramsMap2); + } + + /** + * Test the overload method of convert + */ + @Test + public void testConvert1() { + + //The expected value + String expected = "{\"global_param\":\"${system.biz.date}\"}"; + + //Invoke convert + Map paramsMap = ParamUtils.convert(globalParams); + String result = JSON.toJSONString(paramsMap); + assertEquals(expected, result); + + logger.info(result); + + //Null check + Map paramsMap1 = ParamUtils.convert(null); + assertNull(paramsMap1); + } +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 5e94b16768..6892b387f1 100644 --- a/pom.xml +++ b/pom.xml @@ -682,6 +682,7 @@ **/alert/utils/PropertyUtilsTest.java **/server/utils/SparkArgsUtilsTest.java **/server/utils/FlinkArgsUtilsTest.java + **/server/utils/ParamUtilsTest.java **/dao/mapper/AccessTokenMapperTest.java **/dao/mapper/AlertGroupMapperTest.java **/dao/mapper/AlertMapperTest.java From f311c966ab2f7cfcbea8cb8983689ddbdacab7b7 Mon Sep 17 00:00:00 2001 From: zhukai Date: Wed, 1 Jan 2020 21:09:19 +0800 Subject: [PATCH 26/31] Fix bug in #1634 ,replace the current datetime with a fixed time (#1665) --- .../server/utils/ParamUtilsTest.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java index b91bff7659..60a5259f44 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ParamUtilsTest.java @@ -27,6 +27,7 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Calendar; import java.util.Date; import java.util.HashMap; import java.util.Map; @@ -84,8 +85,13 @@ public class ParamUtilsTest { //The expected value when globalParams is null but localParams is not null String expected1 = "{\"local_param\":{\"direct\":\"IN\",\"prop\":\"local_param\",\"type\":\"VARCHAR\",\"value\":\"20191229\"}}"; + //Define expected date , the month is 0-base + Calendar calendar = Calendar.getInstance(); + calendar.set(2019,11,30); + Date date = calendar.getTime(); + //Invoke convert - Map paramsMap = ParamUtils.convert(globalParams, globalParamsMap, localParams, CommandType.START_PROCESS, new Date()); + Map paramsMap = ParamUtils.convert(globalParams, globalParamsMap, localParams, CommandType.START_PROCESS, date); String result = JSON.toJSONString(paramsMap); assertEquals(expected, result); @@ -97,12 +103,12 @@ public class ParamUtilsTest { } //Invoke convert with null globalParams - Map paramsMap1 = ParamUtils.convert(null, globalParamsMap, localParams, CommandType.START_PROCESS, new Date()); + Map paramsMap1 = ParamUtils.convert(null, globalParamsMap, localParams, CommandType.START_PROCESS, date); String result1 = JSON.toJSONString(paramsMap1); assertEquals(expected1, result1); //Null check, invoke convert with null globalParams and null localParams - Map paramsMap2 = ParamUtils.convert(null, globalParamsMap, null, CommandType.START_PROCESS, new Date()); + Map paramsMap2 = ParamUtils.convert(null, globalParamsMap, null, CommandType.START_PROCESS, date); assertNull(paramsMap2); } From 7e53ceb3c423a2a49343c2e9e96f36211f82a9c4 Mon Sep 17 00:00:00 2001 From: Jave-Chen Date: Wed, 1 Jan 2020 21:28:39 +0800 Subject: [PATCH 27/31] fix 0% UT coverage on new code . issue #1611 (#1662) --- .github/workflows/ci_ut.yml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml index f76c403a8c..152d7ee8c9 100644 --- a/.github/workflows/ci_ut.yml +++ b/.github/workflows/ci_ut.yml @@ -51,17 +51,16 @@ jobs: CODECOV_TOKEN="09c2663f-b091-4258-8a47-c981827eb29a" bash <(curl -s https://codecov.io/bash) - name: Run SonarCloud analysis run: > - mvn clean --batch-mode - org.jacoco:jacoco-maven-plugin:prepare-agent - verify - org.sonarsource.scanner.maven:sonar-maven-plugin:sonar - -Dsonar.host.url=https://sonarcloud.io - -Dsonar.organization=apache - -Dsonar.projectKey=apache-dolphinscheduler - -Dsonar.login=e4058004bc6be89decf558ac819aa1ecbee57682 + mvn clean --batch-mode + verify + org.sonarsource.scanner.maven:sonar-maven-plugin:3.6.1.1688:sonar + -Dsonar.host.url=https://sonarcloud.io + -Dsonar.organization=apache + -Dsonar.projectKey=apache-dolphinscheduler + -Dsonar.login=e4058004bc6be89decf558ac819aa1ecbee57682 env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - name: Collect logs run: | mkdir -p ${LOG_DIR} From bb414b079451005ab1ab200830bed896eb482d4c Mon Sep 17 00:00:00 2001 From: khadgarmage Date: Thu, 2 Jan 2020 10:15:37 +0800 Subject: [PATCH 28/31] delete bytes file (#1669) --- .../dolphinscheduler/common/utils/Bytes.java | 699 ------------------ .../server/worker/task/http/HttpTask.java | 6 +- 2 files changed, 3 insertions(+), 702 deletions(-) delete mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/Bytes.java diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/Bytes.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/Bytes.java deleted file mode 100644 index 39050d9952..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/Bytes.java +++ /dev/null @@ -1,699 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.common.utils; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.nio.ByteBuffer; -import java.nio.charset.StandardCharsets; - -/** - * Utility class that handles Bytes - */ -public class Bytes { - - private static final Logger logger = LoggerFactory.getLogger(Bytes.class); - public static final String UTF8_ENCODING = "UTF-8"; - //An empty instance. - public static final byte [] EMPTY_BYTE_ARRAY = new byte [0]; - - /** - * Size of int in bytes - */ - public static final int SIZEOF_INT = Integer.SIZE / Byte.SIZE; - - /** - * Size of long in bytes - */ - public static final int SIZEOF_LONG = Long.SIZE / Byte.SIZE; - - /** - * Size of short in bytes - */ - public static final int SIZEOF_SHORT = Short.SIZE / Byte.SIZE; - - - - /** - * Put bytes at the specified byte array position. - * @param tgtBytes the byte array - * @param tgtOffset position in the array - * @param srcBytes array to write out - * @param srcOffset source offset - * @param srcLength source length - * @return incremented offset - */ - public static int putBytes(byte[] tgtBytes, int tgtOffset, byte[] srcBytes, - int srcOffset, int srcLength) { - System.arraycopy(srcBytes, srcOffset, tgtBytes, tgtOffset, srcLength); - return tgtOffset + srcLength; - } - - /** - * Write a single byte out to the specified byte array position. - * @param bytes the byte array - * @param offset position in the array - * @param b byte to write out - * @return incremented offset - */ - public static int putByte(byte[] bytes, int offset, byte b) { - bytes[offset] = b; - return offset + 1; - } - - /** - * Returns a new byte array, copied from the passed ByteBuffer. - * @param bb A ByteBuffer - * @return the byte array - */ - public static byte[] toBytes(ByteBuffer bb) { - int length = bb.limit(); - byte [] result = new byte[length]; - System.arraycopy(bb.array(), bb.arrayOffset(), result, 0, length); - return result; - } - - /** - * @param b Presumed UTF-8 encoded byte array. - * @return String made from b - */ - public static String toString(final byte [] b) { - if (b == null) { - return null; - } - return toString(b, 0, b.length); - } - - /** - * Joins two byte arrays together using a separator. - * @param b1 The first byte array. - * @param sep The separator to use. - * @param b2 The second byte array. - * @return two byte arrays together using a separator. - */ - public static String toString(final byte [] b1, - String sep, - final byte [] b2) { - return toString(b1, 0, b1.length) + sep + toString(b2, 0, b2.length); - } - - /** - * This method will convert utf8 encoded bytes into a string. If - * an UnsupportedEncodingException occurs, this method will eat it - * and return null instead. - * - * @param b Presumed UTF-8 encoded byte array. - * @param off offset into array - * @param len length of utf-8 sequence - * @return String made from b or null - */ - public static String toString(final byte [] b, int off, int len) { - if (b == null) { - return null; - } - if (len == 0) { - return ""; - } - return new String(b, off, len, StandardCharsets.UTF_8); - } - - - /** - * Converts a string to a UTF-8 byte array. - * @param s string - * @return the byte array - */ - public static byte[] toBytes(String s) { - return s.getBytes(StandardCharsets.UTF_8); - } - - /** - * Convert a boolean to a byte array. True becomes -1 - * and false becomes 0. - * - * @param b value - * @return b encoded in a byte array. - */ - public static byte [] toBytes(final boolean b) { - return new byte[] { b ? (byte) -1 : (byte) 0 }; - } - - /** - * Reverses {@link #toBytes(boolean)} - * @param b array - * @return True or false. - */ - public static boolean toBoolean(final byte [] b) { - if (b.length != 1) { - throw new IllegalArgumentException("Array has wrong size: " + b.length); - } - return b[0] != (byte) 0; - } - - /** - * Convert a long value to a byte array using big-endian. - * - * @param val value to convert - * @return the byte array - */ - public static byte[] toBytes(long val) { - byte [] b = new byte[8]; - for (int i = 7; i > 0; i--) { - b[i] = (byte) val; - val >>>= 8; - } - b[0] = (byte) val; - return b; - } - - /** - * Converts a byte array to a long value. Reverses - * {@link #toBytes(long)} - * @param bytes array - * @return the long value - */ - public static long toLong(byte[] bytes) { - return toLong(bytes, 0, SIZEOF_LONG); - } - - /** - * Converts a byte array to a long value. Assumes there will be - * {@link #SIZEOF_LONG} bytes available. - * - * @param bytes bytes - * @param offset offset - * @return the long value - */ - public static long toLong(byte[] bytes, int offset) { - return toLong(bytes, offset, SIZEOF_LONG); - } - - /** - * Converts a byte array to a long value. - * - * @param bytes array of bytes - * @param offset offset into array - * @param length length of data (must be {@link #SIZEOF_LONG}) - * @return the long value - * @throws IllegalArgumentException if length is not {@link #SIZEOF_LONG} or - * if there's not enough room in the array at the offset indicated. - */ - public static long toLong(byte[] bytes, int offset, final int length) { - if (length != SIZEOF_LONG || offset + length > bytes.length) { - throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_LONG); - } - long l = 0; - for(int i = offset; i < offset + length; i++) { - l <<= 8; - l ^= bytes[i] & 0xFF; - } - return l; - } - - private static IllegalArgumentException - explainWrongLengthOrOffset(final byte[] bytes, - final int offset, - final int length, - final int expectedLength) { - String reason; - if (length != expectedLength) { - reason = "Wrong length: " + length + ", expected " + expectedLength; - } else { - reason = "offset (" + offset + ") + length (" + length + ") exceed the" - + " capacity of the array: " + bytes.length; - } - return new IllegalArgumentException(reason); - } - - /** - * Put a long value out to the specified byte array position. - * @param bytes the byte array - * @param offset position in the array - * @param val long to write out - * @return incremented offset - * @throws IllegalArgumentException if the byte array given doesn't have - * enough room at the offset specified. - */ - public static int putLong(byte[] bytes, int offset, long val) { - if (bytes.length - offset < SIZEOF_LONG) { - throw new IllegalArgumentException("Not enough room to put a long at" - + " offset " + offset + " in a " + bytes.length + " byte array"); - } - for(int i = offset + 7; i > offset; i--) { - bytes[i] = (byte) val; - val >>>= 8; - } - bytes[offset] = (byte) val; - return offset + SIZEOF_LONG; - } - - /** - * Presumes float encoded as IEEE 754 floating-point "single format" - * @param bytes byte array - * @return Float made from passed byte array. - */ - public static float toFloat(byte [] bytes) { - return toFloat(bytes, 0); - } - - /** - * Presumes float encoded as IEEE 754 floating-point "single format" - * @param bytes array to convert - * @param offset offset into array - * @return Float made from passed byte array. - */ - public static float toFloat(byte [] bytes, int offset) { - return Float.intBitsToFloat(toInt(bytes, offset, SIZEOF_INT)); - } - - /** - * @param bytes byte array - * @param offset offset to write to - * @param f float value - * @return New offset in bytes - */ - public static int putFloat(byte [] bytes, int offset, float f) { - return putInt(bytes, offset, Float.floatToRawIntBits(f)); - } - - /** - * @param f float value - * @return the float represented as byte [] - */ - public static byte [] toBytes(final float f) { - // Encode it as int - return Bytes.toBytes(Float.floatToRawIntBits(f)); - } - - /** - * @param bytes byte array - * @return Return double made from passed bytes. - */ - public static double toDouble(final byte [] bytes) { - return toDouble(bytes, 0); - } - - /** - * @param bytes byte array - * @param offset offset where double is - * @return Return double made from passed bytes. - */ - public static double toDouble(final byte [] bytes, final int offset) { - return Double.longBitsToDouble(toLong(bytes, offset, SIZEOF_LONG)); - } - - /** - * @param bytes byte array - * @param offset offset to write to - * @param d value - * @return New offset into array bytes - */ - public static int putDouble(byte [] bytes, int offset, double d) { - return putLong(bytes, offset, Double.doubleToLongBits(d)); - } - - /** - * Serialize a double as the IEEE 754 double format output. The resultant - * array will be 8 bytes long. - * - * @param d value - * @return the double represented as byte [] - */ - public static byte [] toBytes(final double d) { - // Encode it as a long - return Bytes.toBytes(Double.doubleToRawLongBits(d)); - } - - /** - * Convert an int value to a byte array - * @param val value - * @return the byte array - */ - public static byte[] toBytes(int val) { - byte [] b = new byte[4]; - for(int i = 3; i > 0; i--) { - b[i] = (byte) val; - val >>>= 8; - } - b[0] = (byte) val; - return b; - } - - /** - * Converts a byte array to an int value - * @param bytes byte array - * @return the int value - */ - public static int toInt(byte[] bytes) { - return toInt(bytes, 0, SIZEOF_INT); - } - - /** - * Converts a byte array to an int value - * @param bytes byte array - * @param offset offset into array - * @return the int value - */ - public static int toInt(byte[] bytes, int offset) { - return toInt(bytes, offset, SIZEOF_INT); - } - - /** - * Converts a byte array to an int value - * @param bytes byte array - * @param offset offset into array - * @param length length of int (has to be {@link #SIZEOF_INT}) - * @return the int value - * @throws IllegalArgumentException if length is not {@link #SIZEOF_INT} or - * if there's not enough room in the array at the offset indicated. - */ - public static int toInt(byte[] bytes, int offset, final int length) { - if (length != SIZEOF_INT || offset + length > bytes.length) { - throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_INT); - } - int n = 0; - for(int i = offset; i < (offset + length); i++) { - n <<= 8; - n ^= bytes[i] & 0xFF; - } - return n; - } - - /** - * Put an int value out to the specified byte array position. - * @param bytes the byte array - * @param offset position in the array - * @param val int to write out - * @return incremented offset - * @throws IllegalArgumentException if the byte array given doesn't have - * enough room at the offset specified. - */ - public static int putInt(byte[] bytes, int offset, int val) { - if (bytes.length - offset < SIZEOF_INT) { - throw new IllegalArgumentException("Not enough room to put an int at" - + " offset " + offset + " in a " + bytes.length + " byte array"); - } - for(int i= offset + 3; i > offset; i--) { - bytes[i] = (byte) val; - val >>>= 8; - } - bytes[offset] = (byte) val; - return offset + SIZEOF_INT; - } - - /** - * Convert a short value to a byte array of {@link #SIZEOF_SHORT} bytes long. - * @param val value - * @return the byte array - */ - public static byte[] toBytes(short val) { - byte[] b = new byte[SIZEOF_SHORT]; - b[1] = (byte) val; - val >>= 8; - b[0] = (byte) val; - return b; - } - - /** - * Converts a byte array to a short value - * @param bytes byte array - * @return the short value - */ - public static short toShort(byte[] bytes) { - return toShort(bytes, 0, SIZEOF_SHORT); - } - - /** - * Converts a byte array to a short value - * @param bytes byte array - * @param offset offset into array - * @return the short value - */ - public static short toShort(byte[] bytes, int offset) { - return toShort(bytes, offset, SIZEOF_SHORT); - } - - /** - * Converts a byte array to a short value - * @param bytes byte array - * @param offset offset into array - * @param length length, has to be {@link #SIZEOF_SHORT} - * @return the short value - * @throws IllegalArgumentException if length is not {@link #SIZEOF_SHORT} - * or if there's not enough room in the array at the offset indicated. - */ - public static short toShort(byte[] bytes, int offset, final int length) { - if (length != SIZEOF_SHORT || offset + length > bytes.length) { - throw explainWrongLengthOrOffset(bytes, offset, length, SIZEOF_SHORT); - } - short n = 0; - n ^= bytes[offset] & 0xFF; - n <<= 8; - n ^= bytes[offset+1] & 0xFF; - return n; - } - - /** - * This method will get a sequence of bytes from pos to limit, - * but will restore pos after. - * @param buf buffer - * @return byte array - */ - public static byte[] getBytes(ByteBuffer buf) { - int savedPos = buf.position(); - byte [] newBytes = new byte[buf.remaining()]; - buf.get(newBytes); - buf.position(savedPos); - return newBytes; - } - - /** - * Put a short value out to the specified byte array position. - * @param bytes the byte array - * @param offset position in the array - * @param val short to write out - * @return incremented offset - * @throws IllegalArgumentException if the byte array given doesn't have - * enough room at the offset specified. - */ - public static int putShort(byte[] bytes, int offset, short val) { - if (bytes.length - offset < SIZEOF_SHORT) { - throw new IllegalArgumentException("Not enough room to put a short at" - + " offset " + offset + " in a " + bytes.length + " byte array"); - } - bytes[offset+1] = (byte) val; - val >>= 8; - bytes[offset] = (byte) val; - return offset + SIZEOF_SHORT; - } - - /** - * Convert a BigDecimal value to a byte array - * - * @param val value - * @return the byte array - */ - public static byte[] toBytes(BigDecimal val) { - byte[] valueBytes = val.unscaledValue().toByteArray(); - byte[] result = new byte[valueBytes.length + SIZEOF_INT]; - int offset = putInt(result, 0, val.scale()); - putBytes(result, offset, valueBytes, 0, valueBytes.length); - return result; - } - - - /** - * Converts a byte array to a BigDecimal - * - * @param bytes bytes array - * @return the char value - */ - public static BigDecimal toBigDecimal(byte[] bytes) { - return toBigDecimal(bytes, 0, bytes.length); - } - - /** - * Converts a byte array to a BigDecimal value - * - * @param bytes bytes array - * @param offset offset - * @param length length - * @return the char value - */ - public static BigDecimal toBigDecimal(byte[] bytes, int offset, final int length) { - if (bytes == null || length < SIZEOF_INT + 1 || - (offset + length > bytes.length)) { - return null; - } - - int scale = toInt(bytes, offset); - byte[] tcBytes = new byte[length - SIZEOF_INT]; - System.arraycopy(bytes, offset + SIZEOF_INT, tcBytes, 0, length - SIZEOF_INT); - return new BigDecimal(new BigInteger(tcBytes), scale); - } - - /** - * Put a BigDecimal value out to the specified byte array position. - * - * @param bytes the byte array - * @param offset position in the array - * @param val BigDecimal to write out - * @return incremented offset - */ - public static int putBigDecimal(byte[] bytes, int offset, BigDecimal val) { - if (bytes == null) { - return offset; - } - - byte[] valueBytes = val.unscaledValue().toByteArray(); - byte[] result = new byte[valueBytes.length + SIZEOF_INT]; - offset = putInt(result, offset, val.scale()); - return putBytes(result, offset, valueBytes, 0, valueBytes.length); - } - - /** - * @param a lower half - * @param b upper half - * @return New array that has a in lower half and b in upper half. - */ - public static byte [] add(final byte [] a, final byte [] b) { - return add(a, b, EMPTY_BYTE_ARRAY); - } - - /** - * @param a first third - * @param b second third - * @param c third third - * @return New array made from a, b and c - */ - public static byte [] add(final byte [] a, final byte [] b, final byte [] c) { - byte [] result = new byte[a.length + b.length + c.length]; - System.arraycopy(a, 0, result, 0, a.length); - System.arraycopy(b, 0, result, a.length, b.length); - System.arraycopy(c, 0, result, a.length + b.length, c.length); - return result; - } - - /** - * @param a array - * @param length amount of bytes to grab - * @return First length bytes from a - */ - public static byte [] head(final byte [] a, final int length) { - if (a.length < length) { - return null; - } - byte [] result = new byte[length]; - System.arraycopy(a, 0, result, 0, length); - return result; - } - - /** - * @param a array - * @param length amount of bytes to snarf - * @return Last length bytes from a - */ - public static byte [] tail(final byte [] a, final int length) { - if (a.length < length) { - return null; - } - byte [] result = new byte[length]; - System.arraycopy(a, a.length - length, result, 0, length); - return result; - } - - /** - * @param a array - * @param length new array size - * @return Value in a plus length prepended 0 bytes - */ - public static byte [] padHead(final byte [] a, final int length) { - byte[] padding = getPadding(length); - return add(padding,a); - } - - private static byte[] getPadding(int length) { - byte[] padding = new byte[length]; - for (int i = 0; i < length; i++) { - padding[i] = 0; - } - return padding; - } - - /** - * @param a array - * @param length new array size - * @return Value in a plus length appended 0 bytes - */ - public static byte [] padTail(final byte [] a, final int length) { - byte[] padding = getPadding(length); - return add(a,padding); - } - - - - /** - * @param bytes array to hash - * @param offset offset to start from - * @param length length to hash - * @return hash code - * */ - public static int hashCode(byte[] bytes, int offset, int length) { - int hash = 1; - for (int i = offset; i < offset + length; i++) { - hash = (31 * hash) + (int) bytes[i]; - } - return hash; - } - - /** - * @param t operands - * @return Array of byte arrays made from passed array of Text - */ - public static byte [][] toByteArrays(final String [] t) { - byte [][] result = new byte[t.length][]; - for (int i = 0; i < t.length; i++) { - result[i] = Bytes.toBytes(t[i]); - } - return result; - } - - /** - * @param column operand - * @return A byte array of a byte array where first and only entry is - * column - */ - public static byte [][] toByteArrays(final String column) { - return toByteArrays(toBytes(column)); - } - - /** - * @param column operand - * @return A byte array of a byte array where first and only entry is - * column - */ - public static byte [][] toByteArrays(final byte [] column) { - byte [][] result = new byte[1][]; - result[0] = column; - return result; - } - - -} diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java index 993310f6ec..39bcd922c8 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/http/HttpTask.java @@ -27,7 +27,6 @@ import org.apache.dolphinscheduler.common.process.HttpProperty; import org.apache.dolphinscheduler.common.process.Property; import org.apache.dolphinscheduler.common.task.AbstractParameters; import org.apache.dolphinscheduler.common.task.http.HttpParameters; -import org.apache.dolphinscheduler.common.utils.Bytes; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.common.utils.ParameterUtils; import org.apache.dolphinscheduler.common.utils.SpringApplicationContext; @@ -50,6 +49,7 @@ import org.apache.http.util.EntityUtils; import org.slf4j.Logger; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -176,7 +176,7 @@ public class HttpTask extends AbstractTask { if (entity == null) { return null; } - String webPage = EntityUtils.toString(entity, Bytes.UTF8_ENCODING); + String webPage = EntityUtils.toString(entity, StandardCharsets.UTF_8.name()); return webPage; } @@ -264,7 +264,7 @@ public class HttpTask extends AbstractTask { } } StringEntity postingString = new StringEntity(jsonParam.toString(), Charsets.UTF_8); - postingString.setContentEncoding(Bytes.UTF8_ENCODING); + postingString.setContentEncoding(StandardCharsets.UTF_8.name()); postingString.setContentType(APPLICATION_JSON); builder.setEntity(postingString); } From 22a2d43d7c491d5a1acc5176c4ca26b842d1c99b Mon Sep 17 00:00:00 2001 From: break60 <790061044@qq.com> Date: Thu, 2 Jan 2020 10:25:09 +0800 Subject: [PATCH 29/31] Fix api url (#1672) --- dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js | 2 +- .../src/js/module/components/fileUpdate/definitionUpdate.vue | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js b/dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js index 7ef6193d65..b854dc8ade 100644 --- a/dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js +++ b/dolphinscheduler-ui/src/js/conf/home/store/dag/actions.js @@ -270,7 +270,7 @@ export default { resolve() return } - io.get(`projects/queryAllProjectList`, payload, res => { + io.get(`projects/query-project-list`, payload, res => { state.projectListS = res.data resolve(res.data) }).catch(res => { diff --git a/dolphinscheduler-ui/src/js/module/components/fileUpdate/definitionUpdate.vue b/dolphinscheduler-ui/src/js/module/components/fileUpdate/definitionUpdate.vue index f3124b900e..e8d8ee3087 100644 --- a/dolphinscheduler-ui/src/js/module/components/fileUpdate/definitionUpdate.vue +++ b/dolphinscheduler-ui/src/js/module/components/fileUpdate/definitionUpdate.vue @@ -141,7 +141,7 @@ let self = this let formData = new FormData() formData.append('file', this.file) - io.post(`projects/importProcessDefinition`, res => { + io.post(`projects/import-definition`, res => { this.$message.success(res.msg) resolve() self.$emit('onUpdate') From 0bfb0e4b164c8a9cf26ee84fb3a4a9bf6a4bb3a3 Mon Sep 17 00:00:00 2001 From: "DK.Pino" Date: Thu, 2 Jan 2020 17:14:34 +0800 Subject: [PATCH 30/31] remove lombok depend in common dodule (#1677) --- dolphinscheduler-common/pom.xml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/dolphinscheduler-common/pom.xml b/dolphinscheduler-common/pom.xml index 10cede7988..955e956251 100644 --- a/dolphinscheduler-common/pom.xml +++ b/dolphinscheduler-common/pom.xml @@ -600,13 +600,6 @@ compile - - org.projectlombok - lombok - ${lombok.version} - compile - - org.springframework spring-context From 30dd867acbc12a46dbe181d507b4510d819e4079 Mon Sep 17 00:00:00 2001 From: "DK.Pino" Date: Thu, 2 Jan 2020 17:26:02 +0800 Subject: [PATCH 31/31] remove lombok depend in dao dodule (#1683) --- .../apache/dolphinscheduler/dao/entity/ProcessDefinition.java | 2 -- .../java/org/apache/dolphinscheduler/dao/entity/Schedule.java | 2 -- .../org/apache/dolphinscheduler/dao/entity/TaskInstance.java | 2 -- .../main/java/org/apache/dolphinscheduler/dao/entity/User.java | 2 -- .../org/apache/dolphinscheduler/dao/entity/UserAlertGroup.java | 2 -- .../org/apache/dolphinscheduler/dao/entity/WorkerGroup.java | 2 -- 6 files changed, 12 deletions(-) diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java index d3de4123b0..cd0494ecc6 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ProcessDefinition.java @@ -25,7 +25,6 @@ import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; import com.baomidou.mybatisplus.core.toolkit.StringUtils; -import lombok.Data; import java.util.Date; import java.util.List; @@ -36,7 +35,6 @@ import java.util.stream.Collectors; /** * process definition */ -@Data @TableName("t_ds_process_definition") public class ProcessDefinition { /** diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Schedule.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Schedule.java index 65ae4d8e12..cfda49df6e 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Schedule.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Schedule.java @@ -20,7 +20,6 @@ import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; -import lombok.Data; import org.apache.dolphinscheduler.common.enums.FailureStrategy; import org.apache.dolphinscheduler.common.enums.Priority; import org.apache.dolphinscheduler.common.enums.ReleaseState; @@ -32,7 +31,6 @@ import java.util.Date; * schedule * */ -@Data @TableName("t_ds_schedules") public class Schedule { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java index 3492fe5b17..2db1eda8f4 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/TaskInstance.java @@ -26,14 +26,12 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; -import lombok.Data; import java.util.Date; /** * task instance */ -@Data @TableName("t_ds_task_instance") public class TaskInstance { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java index b6def18922..8849be0b83 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/User.java @@ -22,14 +22,12 @@ import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; -import lombok.Data; import java.util.Date; /** * user */ -@Data @TableName("t_ds_user") public class User { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UserAlertGroup.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UserAlertGroup.java index 84742468b6..e86b32e075 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UserAlertGroup.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/UserAlertGroup.java @@ -20,14 +20,12 @@ import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableField; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; -import lombok.Data; import java.util.Date; /** * user alert group */ -@Data @TableName("t_ds_relation_user_alertgroup") public class UserAlertGroup { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkerGroup.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkerGroup.java index 5c4595b1cf..a732dbbe6e 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkerGroup.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/WorkerGroup.java @@ -19,14 +19,12 @@ package org.apache.dolphinscheduler.dao.entity; import com.baomidou.mybatisplus.annotation.IdType; import com.baomidou.mybatisplus.annotation.TableId; import com.baomidou.mybatisplus.annotation.TableName; -import lombok.Data; import java.util.Date; /** * worker group for task running */ -@Data @TableName("t_ds_worker_group") public class WorkerGroup {