From d1282fdd96147ca345f2a60d4e5c3554820f6fc9 Mon Sep 17 00:00:00 2001 From: CalvinKirs Date: Thu, 2 Jul 2020 19:16:37 +0800 Subject: [PATCH] In the case of multiple network cards, a valid host is returned (#2924) * In the case of multiple network cards, a valid host is returned * In the case of multiple network cards, a valid host is returned * In the case of multiple network cards, a valid host is returned * In the case of multiple network cards, a valid host is returned * fix LOCAL_ADDRESS not assigned value * fix LOCAL_ADDRESS not assigned value * fix LOCAL_ADDRESS not assigned value * fix lock error * fix lock error * end * end * end * end * code smell * code smell * code smell * code smell * code smell * code smell * code smell * lowering * messy * small change * compile error Co-authored-by: dailidong Co-authored-by: gabry.wu --- .../common/utils/NetUtils.java | 209 ++++++++++++++++++ .../common/utils/OSUtils.java | 15 -- .../common/os/OSUtilsTest.java | 6 - .../common/utils/NetUtilsTest.java | 57 +++++ .../common/utils/OSUtilsTest.java | 2 +- .../master/registry/MasterRegistry.java | 8 +- .../runner/ConditionsTaskExecThread.java | 4 +- .../runner/DependentTaskExecThread.java | 4 +- .../master/runner/MasterSchedulerService.java | 3 +- .../processor/TaskExecuteProcessor.java | 30 +-- .../worker/registry/WorkerRegistry.java | 6 +- .../server/zk/ZKMasterClient.java | 8 +- .../executor/NettyExecutorManagerTest.java | 6 +- .../host/RoundRobinHostManagerTest.java | 4 +- .../registry/ZookeeperNodeManagerTest.java | 6 +- .../utils/ExecutionContextTestUtils.java | 4 +- .../worker/registry/WorkerRegistryTest.java | 5 +- 17 files changed, 309 insertions(+), 68 deletions(-) create mode 100644 dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/NetUtilsTest.java diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java new file mode 100644 index 0000000000..3bc80c7bfc --- /dev/null +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.*; +import java.util.Enumeration; +import java.util.LinkedList; +import java.util.List; +import java.util.Optional; +import java.util.regex.Pattern; + +import static java.util.Collections.emptyList; + +/** + * NetUtils + */ +public class NetUtils { + + + private NetUtils() { + throw new IllegalStateException("Utility class"); + } + + private static Logger logger = LoggerFactory.getLogger(NetUtils.class); + + private static final Pattern IP_PATTERN = Pattern.compile("\\d{1,3}(\\.\\d{1,3}){3,5}$"); + + private static String ANY_HOST_VALUE = "0.0.0.0"; + + private static String LOCAL_HOST_VALUE = "127.0.0.1"; + + private static InetAddress LOCAL_ADDRESS = null; + + private static volatile String HOST_ADDRESS; + + public static String getHost() { + if (HOST_ADDRESS != null) { + return HOST_ADDRESS; + } + + InetAddress address = getLocalAddress(); + if (address != null) { + HOST_ADDRESS = address.getHostAddress(); + return HOST_ADDRESS; + } + return LOCAL_HOST_VALUE; + } + + private static InetAddress getLocalAddress() { + if (null != LOCAL_ADDRESS) { + return LOCAL_ADDRESS; + } + return getLocalAddress0(); + } + + /** + * Find first valid IP from local network card + * + * @return first valid local IP + */ + private static synchronized InetAddress getLocalAddress0() { + if (null != LOCAL_ADDRESS) { + return LOCAL_ADDRESS; + } + InetAddress localAddress = null; + NetworkInterface networkInterface = findNetworkInterface(); + Enumeration addresses = networkInterface.getInetAddresses(); + while (addresses.hasMoreElements()) { + Optional addressOp = toValidAddress(addresses.nextElement()); + if (addressOp.isPresent()) { + try { + if (addressOp.get().isReachable(100)) { + LOCAL_ADDRESS = addressOp.get(); + return LOCAL_ADDRESS; + } + } catch (IOException e) { + logger.warn("test address id reachable io exception", e); + } + } + } + + try { + localAddress = InetAddress.getLocalHost(); + } catch (UnknownHostException e) { + logger.warn("InetAddress get LocalHost exception", e); + } + Optional addressOp = toValidAddress(localAddress); + if (addressOp.isPresent()) { + LOCAL_ADDRESS = addressOp.get(); + } + return LOCAL_ADDRESS; + } + + private static Optional toValidAddress(InetAddress address) { + if (address instanceof Inet6Address) { + Inet6Address v6Address = (Inet6Address) address; + if (isPreferIPV6Address()) { + return Optional.ofNullable(normalizeV6Address(v6Address)); + } + } + if (isValidV4Address(address)) { + return Optional.of(address); + } + return Optional.empty(); + } + + private static InetAddress normalizeV6Address(Inet6Address address) { + String addr = address.getHostAddress(); + int i = addr.lastIndexOf('%'); + if (i > 0) { + try { + return InetAddress.getByName(addr.substring(0, i) + '%' + address.getScopeId()); + } catch (UnknownHostException e) { + logger.debug("Unknown IPV6 address: ", e); + } + } + return address; + } + + public static boolean isValidV4Address(InetAddress address) { + + if (address == null || address.isLoopbackAddress()) { + return false; + } + String name = address.getHostAddress(); + return (name != null + && IP_PATTERN.matcher(name).matches() + && !ANY_HOST_VALUE.equals(name) + && !LOCAL_HOST_VALUE.equals(name)); + } + + /** + * Check if an ipv6 address + * + * @return true if it is reachable + */ + private static boolean isPreferIPV6Address() { + return Boolean.getBoolean("java.net.preferIPv6Addresses"); + } + + /** + * Get the suitable {@link NetworkInterface} + * + * @return If no {@link NetworkInterface} is available , return null + */ + private static NetworkInterface findNetworkInterface() { + + List validNetworkInterfaces = emptyList(); + + try { + validNetworkInterfaces = getValidNetworkInterfaces(); + } catch (SocketException e) { + logger.warn("ValidNetworkInterfaces exception", e); + } + + + return validNetworkInterfaces.get(0); + + } + + /** + * Get the valid {@link NetworkInterface network interfaces} + * + * @throws SocketException SocketException if an I/O error occurs. + */ + private static List getValidNetworkInterfaces() throws SocketException { + List validNetworkInterfaces = new LinkedList<>(); + Enumeration interfaces = NetworkInterface.getNetworkInterfaces(); + while (interfaces.hasMoreElements()) { + NetworkInterface networkInterface = interfaces.nextElement(); + if (ignoreNetworkInterface(networkInterface)) { // ignore + continue; + } + validNetworkInterfaces.add(networkInterface); + } + return validNetworkInterfaces; + } + + /** + * @param networkInterface {@link NetworkInterface} + * @return if the specified {@link NetworkInterface} should be ignored, return true + * @throws SocketException SocketException if an I/O error occurs. + */ + public static boolean ignoreNetworkInterface(NetworkInterface networkInterface) throws SocketException { + return networkInterface == null + || networkInterface.isLoopback() + || networkInterface.isVirtual() + || !networkInterface.isUp(); + } + +} diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java index 171a017b4e..c3fcb50602 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java @@ -23,8 +23,6 @@ import java.io.InputStreamReader; import java.lang.management.ManagementFactory; import java.lang.management.RuntimeMXBean; import java.math.RoundingMode; -import java.net.InetAddress; -import java.net.UnknownHostException; import java.text.DecimalFormat; import java.util.ArrayList; import java.util.Arrays; @@ -409,19 +407,6 @@ public class OSUtils { return Integer.parseInt(runtimeMXBean.getName().split("@")[0]); } - /** - * get local host - * @return host - */ - public static String getHost(){ - try { - return InetAddress.getLocalHost().getHostAddress(); - } catch (UnknownHostException e) { - logger.error(e.getMessage(),e); - } - return null; - } - /** * whether is macOS * @return true if mac diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java index 1815e48f84..b2fd887d3d 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/os/OSUtilsTest.java @@ -35,12 +35,6 @@ public class OSUtilsTest { private static Logger logger = LoggerFactory.getLogger(OSUtilsTest.class); - @Test - public void getHost(){ - logger.info(OSUtils.getHost()); - } - - @Test public void memoryUsage() { logger.info("memoryUsage : {}", OSUtils.memoryUsage());// 0.3361799418926239 diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/NetUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/NetUtilsTest.java new file mode 100644 index 0000000000..59276295ae --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/NetUtilsTest.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.dolphinscheduler.common.utils; + +import org.junit.Test; + +import java.net.InetAddress; + +import static org.junit.Assert.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * NetUtilsTest + */ +public class NetUtilsTest { + + @Test + public void testGetLocalHost() { + assertNotNull(NetUtils.getHost()); + } + + @Test + public void testIsValidAddress() { + assertFalse(NetUtils.isValidV4Address(null)); + InetAddress address = mock(InetAddress.class); + when(address.isLoopbackAddress()).thenReturn(true); + assertFalse(NetUtils.isValidV4Address(address)); + address = mock(InetAddress.class); + when(address.getHostAddress()).thenReturn("localhost"); + assertFalse(NetUtils.isValidV4Address(address)); + address = mock(InetAddress.class); + when(address.getHostAddress()).thenReturn("0.0.0.0"); + assertFalse(NetUtils.isValidV4Address(address)); + address = mock(InetAddress.class); + when(address.getHostAddress()).thenReturn("127.0.0.1"); + assertFalse(NetUtils.isValidV4Address(address)); + address = mock(InetAddress.class); + when(address.getHostAddress()).thenReturn("1.2.3.4"); + assertTrue(NetUtils.isValidV4Address(address)); + } + +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java index 44c88f8b20..8f40bccdc8 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java @@ -93,7 +93,7 @@ public class OSUtilsTest { } @Test public void getHost(){ - String host = OSUtils.getHost(); + String host = NetUtils.getHost(); Assert.assertNotNull(host); Assert.assertNotEquals("", host); } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistry.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistry.java index 3ddb9e73d2..1668f8ae92 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistry.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/registry/MasterRegistry.java @@ -21,7 +21,7 @@ import org.apache.curator.framework.state.ConnectionState; import org.apache.curator.framework.state.ConnectionStateListener; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.registry.HeartBeatTask; @@ -80,7 +80,7 @@ public class MasterRegistry { * registry */ public void registry() { - String address = OSUtils.getHost(); + String address = NetUtils.getHost(); String localNodePath = getMasterPath(); zookeeperRegistryCenter.getZookeeperCachedOperator().persistEphemeral(localNodePath, ""); zookeeperRegistryCenter.getZookeeperCachedOperator().getZkClient().getConnectionStateListenable().addListener(new ConnectionStateListener() { @@ -132,7 +132,9 @@ public class MasterRegistry { * @return */ private String getLocalAddress(){ - return OSUtils.getHost() + Constants.COLON + masterConfig.getListenPort(); + + return NetUtils.getHost() + ":" + masterConfig.getListenPort(); + } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java index 61fc27f49a..826a7c8fb9 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/ConditionsTaskExecThread.java @@ -25,7 +25,7 @@ import org.apache.dolphinscheduler.common.task.dependent.DependentParameters; import org.apache.dolphinscheduler.common.utils.DependentUtils; import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.LoggerUtils; -import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.slf4j.LoggerFactory; @@ -123,7 +123,7 @@ public class ConditionsTaskExecThread extends MasterBaseTaskExecThread { private void initTaskParameters() { this.taskInstance.setLogPath(getTaskLogPath(taskInstance)); - this.taskInstance.setHost(OSUtils.getHost() + Constants.COLON + masterConfig.getListenPort()); + this.taskInstance.setHost(NetUtils.getHost() + Constants.COLON + masterConfig.getListenPort()); taskInstance.setState(ExecutionStatus.RUNNING_EXEUTION); taskInstance.setStartTime(new Date()); this.processService.saveTaskInstance(taskInstance); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java index 882f588ab6..a67e24bcf2 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/DependentTaskExecThread.java @@ -26,7 +26,7 @@ import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.utils.DependentUtils; import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.common.utils.LoggerUtils; -import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.utils.DependentExecute; import org.slf4j.LoggerFactory; @@ -172,7 +172,7 @@ public class DependentTaskExecThread extends MasterBaseTaskExecThread { private void initTaskParameters() { taskInstance.setLogPath(getTaskLogPath(taskInstance)); - taskInstance.setHost(OSUtils.getHost() + Constants.COLON + masterConfig.getListenPort()); + taskInstance.setHost(NetUtils.getHost() + Constants.COLON + masterConfig.getListenPort()); taskInstance.setState(ExecutionStatus.RUNNING_EXEUTION); taskInstance.setStartTime(new Date()); processService.updateTaskInstance(taskInstance); diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java index 87e16596b4..ea6dbcc060 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/master/runner/MasterSchedulerService.java @@ -21,6 +21,7 @@ import org.apache.curator.framework.recipes.locks.InterProcessMutex; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.thread.Stopper; import org.apache.dolphinscheduler.common.thread.ThreadUtils; +import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.common.utils.OSUtils; import org.apache.dolphinscheduler.dao.entity.Command; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; @@ -158,6 +159,6 @@ public class MasterSchedulerService extends Thread { } private String getLocalAddress(){ - return OSUtils.getHost() + ":" + masterConfig.getListenPort(); + return NetUtils.getHost() + ":" + masterConfig.getListenPort(); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java index 4a2767f138..788a1d7119 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/processor/TaskExecuteProcessor.java @@ -18,21 +18,15 @@ package org.apache.dolphinscheduler.server.worker.processor; -import java.util.Date; -import java.util.Optional; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; - +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.sift.SiftingAppender; +import com.github.rholder.retry.RetryException; +import io.netty.channel.Channel; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.TaskType; import org.apache.dolphinscheduler.common.thread.ThreadUtils; -import org.apache.dolphinscheduler.common.utils.FileUtils; -import org.apache.dolphinscheduler.common.utils.JSONUtils; -import org.apache.dolphinscheduler.common.utils.LoggerUtils; -import org.apache.dolphinscheduler.common.utils.OSUtils; -import org.apache.dolphinscheduler.common.utils.Preconditions; -import org.apache.dolphinscheduler.common.utils.RetryerUtils; +import org.apache.dolphinscheduler.common.utils.*; import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.TaskExecuteAckCommand; @@ -47,11 +41,10 @@ import org.apache.dolphinscheduler.service.bean.SpringApplicationContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.github.rholder.retry.RetryException; - -import ch.qos.logback.classic.LoggerContext; -import ch.qos.logback.classic.sift.SiftingAppender; -import io.netty.channel.Channel; +import java.util.Date; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; /** * worker request processor @@ -98,12 +91,13 @@ public class TaskExecuteProcessor implements NettyRequestProcessor { String contextJson = taskRequestCommand.getTaskExecutionContext(); TaskExecutionContext taskExecutionContext = JSONUtils.parseObject(contextJson, TaskExecutionContext.class); + if(taskExecutionContext == null){ logger.error("task execution context is null"); return; } - taskExecutionContext.setHost(OSUtils.getHost() + ":" + workerConfig.getListenPort()); + taskExecutionContext.setHost(NetUtils.getHost() + ":" + workerConfig.getListenPort()); // custom logger Logger taskLogger = LoggerFactory.getLogger(LoggerUtils.buildTaskId(LoggerUtils.TASK_LOGGER_INFO_PREFIX, @@ -197,4 +191,4 @@ public class TaskExecuteProcessor implements NettyRequestProcessor { taskExecutionContext.getProcessInstanceId(), taskExecutionContext.getTaskInstanceId()); } -} +} \ No newline at end of file diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistry.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistry.java index 715db3966e..e1349ea9fe 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistry.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistry.java @@ -21,7 +21,7 @@ import org.apache.curator.framework.state.ConnectionState; import org.apache.curator.framework.state.ConnectionStateListener; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.utils.DateUtils; -import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.remote.utils.NamedThreadFactory; import org.apache.dolphinscheduler.server.registry.HeartBeatTask; @@ -87,7 +87,7 @@ public class WorkerRegistry { * registry */ public void registry() { - String address = OSUtils.getHost(); + String address = NetUtils.getHost(); String localNodePath = getWorkerPath(); zookeeperRegistryCenter.getZookeeperCachedOperator().persistEphemeral(localNodePath, ""); zookeeperRegistryCenter.getZookeeperCachedOperator().getZkClient().getConnectionStateListenable().addListener(new ConnectionStateListener() { @@ -149,7 +149,7 @@ public class WorkerRegistry { * @return */ private String getLocalAddress(){ - return OSUtils.getHost() + Constants.COLON + workerConfig.getListenPort(); + return NetUtils.getHost() + ":" + workerConfig.getListenPort(); } } diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java index 686d73d8ac..6abb381583 100644 --- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java +++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/zk/ZKMasterClient.java @@ -25,7 +25,7 @@ import org.apache.dolphinscheduler.common.enums.ExecutionStatus; import org.apache.dolphinscheduler.common.enums.ZKNodeType; import org.apache.dolphinscheduler.common.model.Server; import org.apache.dolphinscheduler.common.thread.ThreadUtils; -import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; import org.apache.dolphinscheduler.server.builder.TaskExecutionContextBuilder; @@ -41,7 +41,7 @@ import org.springframework.stereotype.Component; import java.util.Date; import java.util.List; -import static org.apache.dolphinscheduler.common.Constants.*; +import static org.apache.dolphinscheduler.common.Constants.SLEEP_TIME_MILLIS; /** @@ -75,7 +75,7 @@ public class ZKMasterClient extends AbstractZKClient { // init system znode this.initSystemZNode(); - while (!checkZKNodeExists(OSUtils.getHost(), ZKNodeType.MASTER)){ + while (!checkZKNodeExists(NetUtils.getHost(), ZKNodeType.MASTER)){ ThreadUtils.sleep(SLEEP_TIME_MILLIS); } @@ -155,7 +155,7 @@ public class ZKMasterClient extends AbstractZKClient { * @throws Exception exception */ private void failoverServerWhenDown(String serverHost, ZKNodeType zkNodeType) throws Exception { - if(StringUtils.isEmpty(serverHost) || serverHost.startsWith(OSUtils.getHost())){ + if(StringUtils.isEmpty(serverHost) || serverHost.startsWith(NetUtils.getHost())){ return ; } switch (zkNodeType){ diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java index 5955f46056..a1c6b71437 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/executor/NettyExecutorManagerTest.java @@ -17,7 +17,7 @@ package org.apache.dolphinscheduler.server.master.dispatch.executor; import org.apache.dolphinscheduler.common.enums.CommandType; -import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; @@ -78,7 +78,7 @@ public class NettyExecutorManagerTest { .buildProcessDefinitionRelatedInfo(processDefinition) .create(); ExecutionContext executionContext = new ExecutionContext(context.toCommand(), ExecutorType.WORKER); - executionContext.setHost(Host.of(OSUtils.getHost() + ":" + serverConfig.getListenPort())); + executionContext.setHost(Host.of(NetUtils.getHost() + ":" + serverConfig.getListenPort())); Boolean execute = nettyExecutorManager.execute(executionContext); Assert.assertTrue(execute); nettyRemotingServer.close(); @@ -97,7 +97,7 @@ public class NettyExecutorManagerTest { .buildProcessDefinitionRelatedInfo(processDefinition) .create(); ExecutionContext executionContext = new ExecutionContext(context.toCommand(), ExecutorType.WORKER); - executionContext.setHost(Host.of(OSUtils.getHost() + ":4444")); + executionContext.setHost(Host.of(NetUtils.getHost() + ":4444")); nettyExecutorManager.execute(executionContext); } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java index e223a762dd..b70cf6d3e8 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/master/dispatch/host/RoundRobinHostManagerTest.java @@ -17,7 +17,7 @@ package org.apache.dolphinscheduler.server.master.dispatch.host; -import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.common.utils.StringUtils; import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.server.master.dispatch.context.ExecutionContext; @@ -73,6 +73,6 @@ public class RoundRobinHostManagerTest { ExecutionContext context = ExecutionContextTestUtils.getExecutionContext(10000); Host host = roundRobinHostManager.select(context); Assert.assertTrue(StringUtils.isNotEmpty(host.getAddress())); - Assert.assertTrue(host.getAddress().equalsIgnoreCase(OSUtils.getHost() + ":" + workerConfig.getListenPort())); + Assert.assertTrue(host.getAddress().equalsIgnoreCase(NetUtils.getHost() + ":" + workerConfig.getListenPort())); } } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/ZookeeperNodeManagerTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/ZookeeperNodeManagerTest.java index c99dfc1c9f..0e780a59ce 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/ZookeeperNodeManagerTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/registry/ZookeeperNodeManagerTest.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.server.registry; import org.apache.dolphinscheduler.common.utils.CollectionUtils; -import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.server.master.config.MasterConfig; import org.apache.dolphinscheduler.server.master.registry.MasterRegistry; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; @@ -74,7 +74,7 @@ public class ZookeeperNodeManagerTest { Set masterNodes = zookeeperNodeManager.getMasterNodes(); Assert.assertTrue(CollectionUtils.isNotEmpty(masterNodes)); Assert.assertEquals(1, masterNodes.size()); - Assert.assertEquals(OSUtils.getHost() + ":" + masterConfig.getListenPort(), masterNodes.iterator().next()); + Assert.assertEquals(NetUtils.getHost() + ":" + masterConfig.getListenPort(), masterNodes.iterator().next()); } @Test @@ -102,6 +102,6 @@ public class ZookeeperNodeManagerTest { Set workerNodes = zookeeperNodeManager.getWorkerGroupNodes("default"); Assert.assertTrue(CollectionUtils.isNotEmpty(workerNodes)); Assert.assertEquals(1, workerNodes.size()); - Assert.assertEquals(OSUtils.getHost() + ":" + workerConfig.getListenPort(), workerNodes.iterator().next()); + Assert.assertEquals(NetUtils.getHost() + ":" + workerConfig.getListenPort(), workerNodes.iterator().next()); } } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ExecutionContextTestUtils.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ExecutionContextTestUtils.java index 26d904f798..3d570b24b9 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ExecutionContextTestUtils.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/utils/ExecutionContextTestUtils.java @@ -18,7 +18,7 @@ package org.apache.dolphinscheduler.server.utils; import org.apache.dolphinscheduler.common.enums.CommandType; -import org.apache.dolphinscheduler.common.utils.OSUtils; +import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.dao.entity.ProcessDefinition; import org.apache.dolphinscheduler.dao.entity.ProcessInstance; import org.apache.dolphinscheduler.dao.entity.TaskInstance; @@ -47,7 +47,7 @@ public class ExecutionContextTestUtils { .buildProcessDefinitionRelatedInfo(processDefinition) .create(); ExecutionContext executionContext = new ExecutionContext(context.toCommand(), ExecutorType.WORKER); - executionContext.setHost(Host.of(OSUtils.getHost() + ":" + port)); + executionContext.setHost(Host.of(NetUtils.getHost() + ":" + port)); return executionContext; } diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryTest.java index 6ecff51f70..b34ba8bee9 100644 --- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryTest.java +++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/worker/registry/WorkerRegistryTest.java @@ -17,8 +17,7 @@ package org.apache.dolphinscheduler.server.worker.registry; -import org.apache.dolphinscheduler.common.utils.OSUtils; -import org.apache.dolphinscheduler.remote.utils.Constants; +import org.apache.dolphinscheduler.common.utils.NetUtils; import org.apache.dolphinscheduler.server.registry.ZookeeperRegistryCenter; import org.apache.dolphinscheduler.server.worker.config.WorkerConfig; import org.apache.dolphinscheduler.server.zk.SpringZKServer; @@ -60,7 +59,7 @@ public class WorkerRegistryTest { workerRegistry.registry(); String workerPath = zookeeperRegistryCenter.getWorkerPath(); Assert.assertEquals(DEFAULT_WORKER_GROUP, workerConfig.getWorkerGroup().trim()); - String instancePath = workerPath + "/" + workerConfig.getWorkerGroup().trim() + "/" + (OSUtils.getHost() + ":" + workerConfig.getListenPort()); + String instancePath = workerPath + "/" + workerConfig.getWorkerGroup().trim() + "/" + (NetUtils.getHost() + ":" + workerConfig.getListenPort()); TimeUnit.SECONDS.sleep(workerConfig.getWorkerHeartbeatInterval() + 2); //wait heartbeat info write into zk node String heartbeat = zookeeperRegistryCenter.getZookeeperCachedOperator().get(instancePath); Assert.assertEquals(HEARTBEAT_FOR_ZOOKEEPER_INFO_LENGTH, heartbeat.split(",").length);