From 9fd728908fd41b8aecc9565687060f96f413936a Mon Sep 17 00:00:00 2001 From: Kevin Date: Sun, 14 Jun 2020 10:27:13 +0800 Subject: [PATCH 1/4] remove unused import #2715 (#2718) Co-authored-by: dailidong Co-authored-by: gabry.wu <8545796+gabrywu@users.noreply.github.com> --- .../apache/dolphinscheduler/api/service/UsersServiceTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java index 4c857cbcb1..7df3091c70 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UsersServiceTest.java @@ -18,7 +18,6 @@ package org.apache.dolphinscheduler.api.service; import com.baomidou.mybatisplus.core.metadata.IPage; import com.baomidou.mybatisplus.extension.plugins.pagination.Page; -import org.apache.avro.generic.GenericData; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.api.utils.PageInfo; import org.apache.dolphinscheduler.api.utils.Result; From 0e7403e6a8ad75416937122740397eb66742bf0f Mon Sep 17 00:00:00 2001 From: Yichao Yang <1048262223@qq.com> Date: Sun, 14 Jun 2020 11:17:12 +0800 Subject: [PATCH 2/4] [Feature-2927][common] Fix NEF when get info from OSUtils (#2927) (#2964) * [Feature-2927][common] Fix NEF when get info from OSUtils (#2927) * Update OSUtils.java Change java doc Co-authored-by: gabry.wu <8545796+gabrywu@users.noreply.github.com> --- .../dolphinscheduler/common/utils/OSUtils.java | 15 ++++++++++++--- .../common/utils/OSUtilsTest.java | 8 ++++---- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java index 652b981e1e..e3b2cc2720 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java @@ -53,6 +53,12 @@ public class OSUtils { private static final SystemInfo SI = new SystemInfo(); public static final String TWO_DECIMAL = "0.00"; + /** + * return -1 when the function can not get hardware env info + * e.g {@link OSUtils#loadAverage()} {@link OSUtils#cpuUsage()} + */ + public static final double NEGATIVE_ONE = -1; + private static HardwareAbstractionLayer hal = SI.getHardware(); private OSUtils() {} @@ -118,9 +124,11 @@ public class OSUtils { */ public static double loadAverage() { double loadAverage = hal.getProcessor().getSystemLoadAverage(); + if (Double.isNaN(loadAverage)) { + return NEGATIVE_ONE; + } DecimalFormat df = new DecimalFormat(TWO_DECIMAL); - df.setRoundingMode(RoundingMode.HALF_UP); return Double.parseDouble(df.format(loadAverage)); } @@ -133,10 +141,12 @@ public class OSUtils { public static double cpuUsage() { CentralProcessor processor = hal.getProcessor(); double cpuUsage = processor.getSystemCpuLoad(); + if (Double.isNaN(cpuUsage)) { + return NEGATIVE_ONE; + } DecimalFormat df = new DecimalFormat(TWO_DECIMAL); df.setRoundingMode(RoundingMode.HALF_UP); - return Double.parseDouble(df.format(cpuUsage)); } @@ -393,7 +403,6 @@ public class OSUtils { return null; } - /** * whether is macOS * @return true if mac diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java index b955787c69..e1fa0c563b 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java @@ -41,15 +41,15 @@ public class OSUtilsTest { public void testOSMetric(){ if (!OSUtils.isWindows()) { double availablePhysicalMemorySize = OSUtils.availablePhysicalMemorySize(); - Assert.assertTrue(availablePhysicalMemorySize > 0.0f); + Assert.assertTrue(availablePhysicalMemorySize >= 0.0d); double totalMemorySize = OSUtils.totalMemorySize(); - Assert.assertTrue(totalMemorySize > 0.0f); + Assert.assertTrue(totalMemorySize >= 0.0d); double loadAverage = OSUtils.loadAverage(); logger.info("loadAverage {}", loadAverage); double memoryUsage = OSUtils.memoryUsage(); - Assert.assertTrue(memoryUsage > 0.0f); + Assert.assertTrue(memoryUsage >= 0.0d); double cpuUsage = OSUtils.cpuUsage(); - Assert.assertTrue(cpuUsage > 0.0f); + Assert.assertTrue(cpuUsage >= 0.0d || cpuUsage == -1.0d); } else { // TODO window ut } From 3a7fd725c7b67950540c3959fda359e6ab66f229 Mon Sep 17 00:00:00 2001 From: superScala <66515261+superScala@users.noreply.github.com> Date: Sun, 14 Jun 2020 13:49:20 +0800 Subject: [PATCH 3/4] first show: remove unnecessary code (#2968) Co-authored-by: gabry --- .../apache/dolphinscheduler/remote/NettyRemotingServer.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java index dbeb318f2d..080c586d00 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/NettyRemotingServer.java @@ -208,9 +208,7 @@ public class NettyRemotingServer { if(workGroup != null){ this.workGroup.shutdownGracefully(); } - if(defaultExecutor != null){ - defaultExecutor.shutdown(); - } + defaultExecutor.shutdown(); } catch (Exception ex) { logger.error("netty server close exception", ex); } From a58d1b3b453e73eb9065f281ce66127e8d69da54 Mon Sep 17 00:00:00 2001 From: 743294668 <41327198+743294668@users.noreply.github.com> Date: Mon, 15 Jun 2020 11:12:22 +0800 Subject: [PATCH 4/4] Modify OracleDB connection string format. (#2970) Co-authored-by: taoyuxin --- .../api/service/DataSourceService.java | 1 + .../api/service/DataSourceServiceTest.java | 9 +++ .../dao/datasource/BaseDataSource.java | 2 +- .../dao/datasource/OracleDataSource.java | 16 ++-- .../dao/datasource/OracleDataSourceTest.java | 74 +++++++++++++++++++ pom.xml | 1 + 6 files changed, 93 insertions(+), 10 deletions(-) create mode 100644 dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/OracleDataSourceTest.java diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java index 1bfacfb93d..c25cba67d2 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/DataSourceService.java @@ -514,6 +514,7 @@ public class DataSourceService extends BaseService{ } Map parameterMap = new LinkedHashMap(6); + parameterMap.put(TYPE, connectType); parameterMap.put(Constants.ADDRESS, address); parameterMap.put(Constants.DATABASE, database); parameterMap.put(Constants.JDBC_URL, jdbcUrl); diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java index e47103f9b6..d280b204f4 100644 --- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java +++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/DataSourceServiceTest.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.api.service; import org.apache.dolphinscheduler.api.ApiApplicationServer; import org.apache.dolphinscheduler.api.enums.Status; import org.apache.dolphinscheduler.common.Constants; +import org.apache.dolphinscheduler.common.enums.DbConnectType; import org.apache.dolphinscheduler.common.enums.DbType; import org.apache.dolphinscheduler.common.enums.UserType; import org.apache.dolphinscheduler.dao.entity.User; @@ -50,4 +51,12 @@ public class DataSourceServiceTest { Map map = dataSourceService.queryDataSourceList(loginUser, DbType.MYSQL.ordinal()); Assert.assertEquals(Status.SUCCESS, map.get(Constants.STATUS)); } + + @Test + public void buildParameter(){ + String param = dataSourceService.buildParameter("","", DbType.ORACLE, "192.168.9.1","1521","im" + ,"","test","test", DbConnectType.ORACLE_SERVICE_NAME,""); + String expected = "{\"type\":\"ORACLE_SERVICE_NAME\",\"address\":\"jdbc:oracle:thin:@//192.168.9.1:1521\",\"database\":\"im\",\"jdbcUrl\":\"jdbc:oracle:thin:@//192.168.9.1:1521/im\",\"user\":\"test\",\"password\":\"test\"}"; + Assert.assertEquals(expected, param); + } } \ No newline at end of file diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java index 1132147faf..5d2a41c8ed 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java @@ -97,7 +97,7 @@ public abstract class BaseDataSource { * append database * @param jdbcUrl jdbc url */ - private void appendDatabase(StringBuilder jdbcUrl) { + protected void appendDatabase(StringBuilder jdbcUrl) { if (dbTypeSelector() == DbType.SQLSERVER) { jdbcUrl.append(";databaseName=").append(getDatabase()); } else { diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/OracleDataSource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/OracleDataSource.java index c457583075..8067663ddd 100644 --- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/OracleDataSource.java +++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/OracleDataSource.java @@ -19,8 +19,6 @@ package org.apache.dolphinscheduler.dao.datasource; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.DbConnectType; import org.apache.dolphinscheduler.common.enums.DbType; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * data source of Oracle @@ -46,16 +44,16 @@ public class OracleDataSource extends BaseDataSource { } /** - * gets the JDBC url for the data source connection - * @return jdbc url + * append service name or SID */ @Override - public String getJdbcUrl() { - String jdbcUrl = getAddress(); - if (jdbcUrl.lastIndexOf("/") != (jdbcUrl.length() - 1)) { - jdbcUrl += "/"; + protected void appendDatabase(StringBuilder jdbcUrl) { + if (getType() == DbConnectType.ORACLE_SID) { + jdbcUrl.append(":"); + } else { + jdbcUrl.append("/"); } - return jdbcUrl; + jdbcUrl.append(getDatabase()); } /** diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/OracleDataSourceTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/OracleDataSourceTest.java new file mode 100644 index 0000000000..5d3d63c168 --- /dev/null +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/OracleDataSourceTest.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.dao.datasource; + +import org.apache.dolphinscheduler.common.enums.DbConnectType; +import org.junit.Assert; +import org.junit.Test; + +public class OracleDataSourceTest { + + @Test + public void testGetOracleJdbcUrl() { + OracleDataSource oracleDataSource = new OracleDataSource(); + oracleDataSource.setType(DbConnectType.ORACLE_SERVICE_NAME); + oracleDataSource.setAddress("jdbc:oracle:thin:@//127.0.0.1:1521"); + oracleDataSource.setDatabase("test"); + oracleDataSource.setPassword("123456"); + oracleDataSource.setUser("test"); + Assert.assertEquals("jdbc:oracle:thin:@//127.0.0.1:1521/test", oracleDataSource.getJdbcUrl()); + //set fake principal + oracleDataSource.setPrincipal("fake principal"); + Assert.assertEquals("jdbc:oracle:thin:@//127.0.0.1:1521/test", oracleDataSource.getJdbcUrl()); + //set fake other + oracleDataSource.setOther("charset=UTF-8"); + Assert.assertEquals("jdbc:oracle:thin:@//127.0.0.1:1521/test?charset=UTF-8", oracleDataSource.getJdbcUrl()); + + OracleDataSource oracleDataSource2 = new OracleDataSource(); + oracleDataSource2.setAddress("jdbc:oracle:thin:@127.0.0.1:1521"); + oracleDataSource2.setDatabase("orcl"); + oracleDataSource2.setPassword("123456"); + oracleDataSource2.setUser("test"); + oracleDataSource2.setType(DbConnectType.ORACLE_SID); + Assert.assertEquals("jdbc:oracle:thin:@127.0.0.1:1521:orcl", oracleDataSource2.getJdbcUrl()); + //set fake principal + oracleDataSource2.setPrincipal("fake principal"); + Assert.assertEquals("jdbc:oracle:thin:@127.0.0.1:1521:orcl", oracleDataSource2.getJdbcUrl()); + //set fake other + oracleDataSource2.setOther("charset=UTF-8"); + Assert.assertEquals("jdbc:oracle:thin:@127.0.0.1:1521:orcl?charset=UTF-8", oracleDataSource2.getJdbcUrl()); + } + + @Test + public void testAppendDatabase() { + OracleDataSource oracleDataSource = new OracleDataSource(); + oracleDataSource.setAddress("jdbc:oracle:thin:@//127.0.0.1:1521"); + oracleDataSource.setDatabase("test"); + oracleDataSource.setType(DbConnectType.ORACLE_SERVICE_NAME); + StringBuilder jdbcUrl = new StringBuilder(oracleDataSource.getAddress()); + oracleDataSource.appendDatabase(jdbcUrl); + Assert.assertEquals("jdbc:oracle:thin:@//127.0.0.1:1521/test", jdbcUrl.toString()); + + OracleDataSource oracleDataSource2 = new OracleDataSource(); + oracleDataSource2.setAddress("jdbc:oracle:thin:@127.0.0.1:1521"); + oracleDataSource2.setDatabase("orcl"); + oracleDataSource2.setType(DbConnectType.ORACLE_SID); + StringBuilder jdbcUrl2 = new StringBuilder(oracleDataSource2.getAddress()); + oracleDataSource2.appendDatabase(jdbcUrl2); + Assert.assertEquals("jdbc:oracle:thin:@127.0.0.1:1521:orcl", jdbcUrl2.toString()); + } +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index bdac1f16aa..57269748cf 100644 --- a/pom.xml +++ b/pom.xml @@ -845,6 +845,7 @@ **/dao/mapper/UserMapperTest.java **/dao/utils/DagHelperTest.java **/dao/AlertDaoTest.java + **/dao/datasource/OracleDataSourceTest.java **/plugin/model/AlertDataTest.java **/plugin/model/AlertInfoTest.java **/plugin/utils/PropertyUtilsTest.java