From f3dfaacca28b3b71c993bb543dd3b9fffcc62613 Mon Sep 17 00:00:00 2001 From: wenjun <861923274@qq.com> Date: Thu, 15 Apr 2021 20:25:01 +0800 Subject: [PATCH] [Improvement][Service] Optimize query log (#5205) * [Improvement][Service] Optimize query log * remove IPUtils --- .../common/utils/IpUtils.java | 63 -------- .../common/utils/IpUtilsTest.java | 55 ------- .../remote/utils/Constants.java | 4 +- .../remote/utils/IPUtils.java | 145 ------------------ .../service/log/LogClientService.java | 4 +- .../service/log/LogClientServiceTest.java | 14 +- pom.xml | 1 - 7 files changed, 12 insertions(+), 274 deletions(-) delete mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java delete mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/IpUtilsTest.java delete mode 100644 dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/IPUtils.java diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java deleted file mode 100644 index 63d43e7b69..0000000000 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/IpUtils.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.common.utils; - -/** - * http utils - */ -public class IpUtils { - - private IpUtils() { - throw new UnsupportedOperationException("Construct IpUtils"); - } - - public static final String DOT = "."; - - /** - * ip str to long

- * - * @param ipStr ip string - * @return ip to long - */ - public static Long ipToLong(String ipStr) { - String[] ipSet = ipStr.split("\\" + DOT); - - return Long.parseLong(ipSet[0]) << 24 | Long.parseLong(ipSet[1]) << 16 | Long.parseLong(ipSet[2]) << 8 | Long.parseLong(ipSet[3]); - } - - /** - * long to ip - * - * @param ipLong the long number converted from IP - * @return String - */ - public static String longToIp(long ipLong) { - long[] ipNumbers = new long[4]; - long tmp = 0xFF; - ipNumbers[0] = ipLong >> 24 & tmp; - ipNumbers[1] = ipLong >> 16 & tmp; - ipNumbers[2] = ipLong >> 8 & tmp; - ipNumbers[3] = ipLong & tmp; - - return ipNumbers[0] + DOT - + ipNumbers[1] + DOT - + ipNumbers[2] + DOT - + ipNumbers[3]; - } - -} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/IpUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/IpUtilsTest.java deleted file mode 100644 index ec6ffa35a7..0000000000 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/IpUtilsTest.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.common.utils; - -import org.junit.Assert; -import org.junit.Test; - -public class IpUtilsTest { - - @Test - public void ipToLong() { - - String ip = "192.168.110.1"; - String ip2 = "0.0.0.0"; - long longNumber = IpUtils.ipToLong(ip); - long longNumber2 = IpUtils.ipToLong(ip2); - System.out.println(longNumber); - Assert.assertEquals(3232263681L, longNumber); - Assert.assertEquals(0L, longNumber2); - - String ip3 = "255.255.255.255"; - long longNumber3 = IpUtils.ipToLong(ip3); - System.out.println(longNumber3); - Assert.assertEquals(4294967295L, longNumber3); - - } - - @Test - public void longToIp() { - - String ip = "192.168.110.1"; - String ip2 = "0.0.0.0"; - long longNum = 3232263681L; - String i1 = IpUtils.longToIp(longNum); - - String i2 = IpUtils.longToIp(0); - - Assert.assertEquals(ip, i1); - Assert.assertEquals(ip2, i2); - } -} \ No newline at end of file diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java index e708032493..9ac1a9a72f 100644 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java +++ b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/Constants.java @@ -17,6 +17,8 @@ package org.apache.dolphinscheduler.remote.utils; +import org.apache.dolphinscheduler.common.utils.NetUtils; + import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; @@ -48,7 +50,7 @@ public class Constants { public static final int CPUS = Runtime.getRuntime().availableProcessors(); - public static final String LOCAL_ADDRESS = IPUtils.getFirstNoLoopbackIP4Address(); + public static final String LOCAL_ADDRESS = NetUtils.getHost(); /** * netty epoll enable switch diff --git a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/IPUtils.java b/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/IPUtils.java deleted file mode 100644 index 91f61f852d..0000000000 --- a/dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/utils/IPUtils.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.dolphinscheduler.remote.utils; - -import org.apache.dolphinscheduler.remote.exceptions.RemoteException; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.net.*; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Enumeration; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -public class IPUtils { - - private IPUtils() { - throw new IllegalStateException(IPUtils.class.getName()); - } - - private static final Logger logger = LoggerFactory.getLogger(IPUtils.class); - - private static String localHost = "unknown"; - - static { - String host = System.getenv("HOSTNAME"); - if (isNotEmpty(host)) { - localHost = host; - } else { - - try { - String hostName = InetAddress.getLocalHost().getHostName(); - if (isNotEmpty(hostName)) { - localHost = hostName; - } - } catch (UnknownHostException e) { - logger.error("get hostName error!", e); - } - } - } - - public static String getLocalHost() { - return localHost; - } - - - public static String getFirstNoLoopbackIP4Address() { - Collection allNoLoopbackIP4Addresses = getNoLoopbackIP4Addresses(); - if (allNoLoopbackIP4Addresses.isEmpty()) { - return null; - } - return allNoLoopbackIP4Addresses.iterator().next(); - } - - public static Collection getNoLoopbackIP4Addresses() { - Collection noLoopbackIP4Addresses = new ArrayList<>(); - Collection allInetAddresses = getAllHostAddress(); - - for (InetAddress address : allInetAddresses) { - if (!address.isLoopbackAddress() && !address.isSiteLocalAddress() - && !Inet6Address.class.isInstance(address)) { - noLoopbackIP4Addresses.add(address.getHostAddress()); - } - } - if (noLoopbackIP4Addresses.isEmpty()) { - for (InetAddress address : allInetAddresses) { - if (!address.isLoopbackAddress() && !Inet6Address.class.isInstance(address)) { - noLoopbackIP4Addresses.add(address.getHostAddress()); - } - } - } - return noLoopbackIP4Addresses; - } - - public static Collection getAllHostAddress() { - try { - Enumeration networkInterfaces = NetworkInterface.getNetworkInterfaces(); - Collection addresses = new ArrayList<>(); - - while (networkInterfaces.hasMoreElements()) { - NetworkInterface networkInterface = networkInterfaces.nextElement(); - Enumeration inetAddresses = networkInterface.getInetAddresses(); - while (inetAddresses.hasMoreElements()) { - InetAddress inetAddress = inetAddresses.nextElement(); - addresses.add(inetAddress); - } - } - - return addresses; - } catch (SocketException e) { - throw new RemoteException(e.getMessage(), e); - } - } - - public static String getIpByHostName(String host) { - InetAddress address = null; - try { - address = InetAddress.getByName(host); - } catch (UnknownHostException e) { - logger.error("get IP error", e); - } - if (address == null) { - return ""; - } - return address.getHostAddress(); - - } - - private static boolean isEmpty(final CharSequence cs) { - return cs == null || cs.length() == 0; - } - - private static boolean isNotEmpty(final CharSequence cs) { - return !isEmpty(cs); - } - - public static boolean isIp(String addr) { - if (addr.length() < 7 || addr.length() > 15 || "".equals(addr)) { - return false; - } - - String ipRegex = "([1-9]|[1-9]\\d|1\\d{2}|2[0-4]\\d|25[0-5])(\\.(\\d|[1-9]\\d|1\\d{2}|2[0-4]\\d|25[0-5])){3}"; - Pattern pat = Pattern.compile(ipRegex); - - Matcher mat = pat.matcher(addr); - - return mat.find(); - } -} diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java index 5f0f5453be..22092e51ce 100644 --- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java +++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClientService.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.service.log; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.LoggerUtils; +import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.remote.NettyRemotingClient; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.log.GetLogBytesRequestCommand; @@ -31,7 +32,6 @@ import org.apache.dolphinscheduler.remote.command.log.ViewLogRequestCommand; import org.apache.dolphinscheduler.remote.command.log.ViewLogResponseCommand; import org.apache.dolphinscheduler.remote.config.NettyClientConfig; import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.remote.utils.IPUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -118,7 +118,7 @@ public class LogClientService { String result = ""; final Host address = new Host(host, port); try { - if (IPUtils.getLocalHost().equals(host)) { + if (NetUtils.getHost().equals(host)) { result = LoggerUtils.readWholeFileContent(request.getPath()); } else { Command command = request.convert2Command(); diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LogClientServiceTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LogClientServiceTest.java index 58d7c1e19c..6a6fb4c3bd 100644 --- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LogClientServiceTest.java +++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LogClientServiceTest.java @@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.service.log; import org.apache.dolphinscheduler.common.utils.JSONUtils; import org.apache.dolphinscheduler.common.utils.LoggerUtils; +import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.remote.NettyRemotingClient; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.log.GetLogBytesResponseCommand; @@ -26,7 +27,6 @@ import org.apache.dolphinscheduler.remote.command.log.RemoveTaskLogResponseComma import org.apache.dolphinscheduler.remote.command.log.RollViewLogResponseCommand; import org.apache.dolphinscheduler.remote.command.log.ViewLogResponseCommand; import org.apache.dolphinscheduler.remote.utils.Host; -import org.apache.dolphinscheduler.remote.utils.IPUtils; import java.nio.charset.StandardCharsets; @@ -40,7 +40,7 @@ import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; @RunWith(PowerMockRunner.class) -@PrepareForTest({LogClientService.class, IPUtils.class, LoggerUtils.class, NettyRemotingClient.class}) +@PrepareForTest({LogClientService.class, NetUtils.class, LoggerUtils.class, NettyRemotingClient.class}) public class LogClientServiceTest { @Test @@ -49,8 +49,8 @@ public class LogClientServiceTest { int port = 1234; String path = "/tmp/log"; - PowerMockito.mockStatic(IPUtils.class); - PowerMockito.when(IPUtils.getLocalHost()).thenReturn(localMachine); + PowerMockito.mockStatic(NetUtils.class); + PowerMockito.when(NetUtils.getHost()).thenReturn(localMachine); PowerMockito.mockStatic(LoggerUtils.class); PowerMockito.when(LoggerUtils.readWholeFileContent(Mockito.anyString())).thenReturn("application_xx_11"); @@ -61,12 +61,12 @@ public class LogClientServiceTest { @Test public void testViewLogFromRemote() throws Exception { - String localMachine = "LOCAL_MACHINE"; + String localMachine = "127.0.0.1"; int port = 1234; String path = "/tmp/log"; - PowerMockito.mockStatic(IPUtils.class); - PowerMockito.when(IPUtils.getLocalHost()).thenReturn(localMachine + "1"); + PowerMockito.mockStatic(NetUtils.class); + PowerMockito.when(NetUtils.getHost()).thenReturn(localMachine + "1"); NettyRemotingClient remotingClient = PowerMockito.mock(NettyRemotingClient.class); PowerMockito.whenNew(NettyRemotingClient.class).withAnyArguments().thenReturn(remotingClient); diff --git a/pom.xml b/pom.xml index 90051ca53c..ee15f71f3a 100644 --- a/pom.xml +++ b/pom.xml @@ -864,7 +864,6 @@ **/common/utils/DependentUtilsTest.java **/common/utils/EncryptionUtilsTest.java **/common/utils/FileUtilsTest.java - **/common/utils/IpUtilsTest.java **/common/utils/JSONUtilsTest.java **/common/utils/LoggerUtilsTest.java **/common/utils/NetUtilsTest.java