From 255fda6df91d76b82b461022f3bb2fa7c3d0c01f Mon Sep 17 00:00:00 2001 From: Shiwen Cheng Date: Sat, 6 Mar 2021 22:57:44 +0800 Subject: [PATCH] [1.3.6-prepare][Fix-4905][K8s] Fix incorrect host problem in minikube #4906 (#4970) --- .../common/utils/OSUtils.java | 41 +++++++++++++---- .../dolphinscheduler/common/CommonTest.java | 46 +++++++++++++++++++ .../common/utils/OSUtilsTest.java | 29 ++++++++++-- 3 files changed, 101 insertions(+), 15 deletions(-) create mode 100644 dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/CommonTest.java diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java index 7b255c2e63..d39d2bdcda 100644 --- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java +++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OSUtils.java @@ -16,30 +16,38 @@ */ package org.apache.dolphinscheduler.common.utils; -import org.apache.commons.configuration.Configuration; import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.shell.ShellExecutor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import oshi.SystemInfo; -import oshi.hardware.CentralProcessor; -import oshi.hardware.GlobalMemory; -import oshi.hardware.HardwareAbstractionLayer; -import java.lang.management.OperatingSystemMXBean; +import org.apache.commons.configuration.Configuration; + import java.io.BufferedReader; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.lang.management.ManagementFactory; +import java.lang.management.OperatingSystemMXBean; import java.lang.management.RuntimeMXBean; import java.math.RoundingMode; import java.net.InetAddress; import java.net.UnknownHostException; import java.text.DecimalFormat; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.StringTokenizer; import java.util.regex.Pattern; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import oshi.SystemInfo; +import oshi.hardware.CentralProcessor; +import oshi.hardware.GlobalMemory; +import oshi.hardware.HardwareAbstractionLayer; + /** * os utils * @@ -50,6 +58,9 @@ public class OSUtils { public static final ThreadLocal taskLoggerThreadLocal = new ThreadLocal<>(); + private static final Pattern STS_PATTERN = Pattern.compile("-\\d+$"); // StatefulSet pattern + private static final Pattern IP_PATTERN = Pattern.compile("\\d{1,3}(\\.\\d{1,3}){3,5}$"); + private static final SystemInfo SI = new SystemInfo(); public static final String TWO_DECIMAL = "0.00"; @@ -439,7 +450,17 @@ public class OSUtils { */ public static String getHost(InetAddress inetAddress){ if (inetAddress != null) { - return Constants.KUBERNETES_MODE ? inetAddress.getHostName() : inetAddress.getHostAddress(); + if (Constants.KUBERNETES_MODE) { + String canonicalHost = inetAddress.getCanonicalHostName(); + if (!canonicalHost.contains(".") || IP_PATTERN.matcher(canonicalHost).matches()) { + String host = inetAddress.getHostName(); + if (STS_PATTERN.matcher(host).find()) { + return String.format("%s.%s", host, host.replaceFirst("\\d+$", "headless")); + } + } + return canonicalHost; + } + return inetAddress.getHostAddress(); } return null; } diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/CommonTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/CommonTest.java new file mode 100644 index 0000000000..3752fe670c --- /dev/null +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/CommonTest.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.common; + +import static org.junit.Assert.assertTrue; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; + +import org.junit.Test; + +/** + * CommonTest + */ +public class CommonTest { + + public static void setFinalStatic(Field field, Object newValue) throws NoSuchFieldException, IllegalAccessException { + field.setAccessible(true); + Field modifiersField = Field.class.getDeclaredField("modifiers"); + modifiersField.setAccessible(true); + modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); + field.set(null, newValue); + } + + @Test + public void testSetFinalStatic() throws Exception { + setFinalStatic(Constants.class.getDeclaredField("KUBERNETES_MODE"), true); + assertTrue(Constants.KUBERNETES_MODE); + } + +} diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java index cdda544b96..39c9bf8601 100644 --- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java +++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/OSUtilsTest.java @@ -16,17 +16,24 @@ */ package org.apache.dolphinscheduler.common.utils; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.apache.dolphinscheduler.common.CommonTest; +import org.apache.dolphinscheduler.common.Constants; + import org.apache.commons.configuration.Configuration; import org.apache.commons.configuration.PropertiesConfiguration; -import org.apache.dolphinscheduler.common.Constants; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.List; + import org.junit.Assert; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.List; - public class OSUtilsTest { private static final Logger logger = LoggerFactory.getLogger(OSUtilsTest.class); @@ -94,10 +101,22 @@ public class OSUtilsTest { Assert.assertEquals("localhost:1234", OSUtils.getAddr("localhost", 1234)); } @Test - public void getHost(){ + public void getHost() throws Exception { String host = OSUtils.getHost(); Assert.assertNotNull(host); Assert.assertNotEquals("", host); + InetAddress address = mock(InetAddress.class); + when(address.getCanonicalHostName()).thenReturn("dolphinscheduler-worker-0.dolphinscheduler-worker-headless.default.svc.cluster.local"); + when(address.getHostName()).thenReturn("dolphinscheduler-worker-0"); + when(address.getHostAddress()).thenReturn("172.17.0.15"); + Assert.assertEquals("172.17.0.15", OSUtils.getHost(address)); + CommonTest.setFinalStatic(Constants.class.getDeclaredField("KUBERNETES_MODE"), true); + Assert.assertEquals("dolphinscheduler-worker-0.dolphinscheduler-worker-headless.default.svc.cluster.local", OSUtils.getHost(address)); + address = mock(InetAddress.class); + when(address.getCanonicalHostName()).thenReturn("dolphinscheduler-worker-0"); + when(address.getHostName()).thenReturn("dolphinscheduler-worker-0"); + CommonTest.setFinalStatic(Constants.class.getDeclaredField("KUBERNETES_MODE"), true); + Assert.assertEquals("dolphinscheduler-worker-0.dolphinscheduler-worker-headless", OSUtils.getHost(address)); } @Test public void checkResource(){