Browse Source

[Improvement][Test] Fully remove the usage of powermock from the whole project (#12244)

* Fully remove the usage of powermock from the whole project

* Upgrade org.reflections to 0.10.12
3.2.0-release
Eric Gao 2 years ago committed by GitHub
parent
commit
9ab79e064c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 12
      dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java
  2. 47
      dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/AlertServerTest.java
  3. 8
      dolphinscheduler-bom/pom.xml
  4. 7
      dolphinscheduler-common/pom.xml
  5. 4
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
  6. 32
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/KubernetesUtils.java
  7. 15
      dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java
  8. 11
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java
  9. 24
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java
  10. 22
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java
  11. 64
      dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/NetUtilsTest.java
  12. 2
      dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/factory/NettyRemotingClientFactory.java
  13. 33
      dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/factory/NettyRemotingServerFactory.java
  14. 2
      dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/config/ServiceBean.java
  15. 2
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/alert/AlertClientService.java
  16. 2
      dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClient.java
  17. 2
      dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/alert/AlertClientServiceTest.java
  18. 2
      dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LogClientTest.java
  19. 8
      dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/DateUtils.java
  20. 21
      dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessorTest.java
  21. 19
      pom.xml
  22. 4
      tools/dependencies/known-dependencies.txt

12
dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java

@ -23,7 +23,7 @@ import org.apache.dolphinscheduler.common.thread.ThreadUtils;
import org.apache.dolphinscheduler.dao.PluginDao; import org.apache.dolphinscheduler.dao.PluginDao;
import org.apache.dolphinscheduler.remote.NettyRemotingServer; import org.apache.dolphinscheduler.remote.NettyRemotingServer;
import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.CommandType;
import org.apache.dolphinscheduler.remote.config.NettyServerConfig; import org.apache.dolphinscheduler.remote.factory.NettyRemotingServerFactory;
import java.io.Closeable; import java.io.Closeable;
@ -108,20 +108,16 @@ public class AlertServer implements Closeable {
} }
} }
private void checkTable() { protected void checkTable() {
if (!pluginDao.checkPluginDefineTableExist()) { if (!pluginDao.checkPluginDefineTableExist()) {
logger.error("Plugin Define Table t_ds_plugin_define Not Exist . Please Create it First !"); logger.error("Plugin Define Table t_ds_plugin_define Not Exist . Please Create it First !");
System.exit(1); System.exit(1);
} }
} }
private void startServer() { protected void startServer() {
NettyServerConfig serverConfig = new NettyServerConfig(); nettyRemotingServer = NettyRemotingServerFactory.buildNettyRemotingServer(alertConfig.getPort());
serverConfig.setListenPort(alertConfig.getPort());
nettyRemotingServer = new NettyRemotingServer(serverConfig);
nettyRemotingServer.registerProcessor(CommandType.ALERT_SEND_REQUEST, alertRequestProcessor); nettyRemotingServer.registerProcessor(CommandType.ALERT_SEND_REQUEST, alertRequestProcessor);
nettyRemotingServer.start(); nettyRemotingServer.start();
} }
} }

47
dolphinscheduler-alert/dolphinscheduler-alert-server/src/test/java/org/apache/dolphinscheduler/alert/AlertServerTest.java

@ -17,18 +17,26 @@
package org.apache.dolphinscheduler.alert; package org.apache.dolphinscheduler.alert;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import org.apache.dolphinscheduler.dao.PluginDao; import org.apache.dolphinscheduler.dao.PluginDao;
import org.apache.dolphinscheduler.remote.NettyRemotingServer; import org.apache.dolphinscheduler.remote.NettyRemotingServer;
import org.apache.dolphinscheduler.remote.config.NettyServerConfig; import org.apache.dolphinscheduler.remote.factory.NettyRemotingServerFactory;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks; import org.mockito.InjectMocks;
import org.mockito.Mock; import org.mockito.Mock;
import org.mockito.MockedStatic;
import org.mockito.Mockito; import org.mockito.Mockito;
import org.mockito.Spy;
import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoExtension;
import org.powermock.reflect.Whitebox;
@ExtendWith(MockitoExtension.class) @ExtendWith(MockitoExtension.class)
public class AlertServerTest { public class AlertServerTest {
@ -42,7 +50,11 @@ public class AlertServerTest {
@Mock @Mock
private AlertSenderService alertSenderService; private AlertSenderService alertSenderService;
@Mock
private NettyRemotingServer nettyRemotingServer;
@InjectMocks @InjectMocks
@Spy
private AlertServer alertServer; private AlertServer alertServer;
@BeforeEach @BeforeEach
@ -50,20 +62,31 @@ public class AlertServerTest {
Mockito.lenient().when(pluginDao.checkPluginDefineTableExist()).thenReturn(true); Mockito.lenient().when(pluginDao.checkPluginDefineTableExist()).thenReturn(true);
Mockito.lenient().when(alertConfig.getPort()).thenReturn(50052); Mockito.lenient().when(alertConfig.getPort()).thenReturn(50052);
Mockito.doNothing().when(alertSenderService).start();
} }
@Test @Test
public void alertServerStartSuccessfully() { public void alertServerRunSuccessfully() {
doNothing().when(alertServer).checkTable();
doNothing().when(alertServer).startServer();
alertServer.run(null); alertServer.run(null);
NettyRemotingServer nettyRemotingServer = Whitebox.getInternalState(alertServer, "nettyRemotingServer"); Mockito.verify(alertServer, times(1)).checkTable();
Mockito.verify(alertServer, times(1)).startServer();
NettyServerConfig nettyServerConfig = Whitebox.getInternalState(nettyRemotingServer, "serverConfig"); Mockito.verify(alertSenderService, times(1)).start();
}
Assertions.assertEquals(50052, nettyServerConfig.getListenPort());
@Test
public void alertServerServerStartWithExpectedListeningPort() {
try (
MockedStatic<NettyRemotingServerFactory> mockedNettyRemotingServerFactory =
mockStatic(NettyRemotingServerFactory.class)) {
mockedNettyRemotingServerFactory.when(() -> NettyRemotingServerFactory.buildNettyRemotingServer(anyInt()))
.thenReturn(nettyRemotingServer);
alertServer.startServer();
mockedNettyRemotingServerFactory.verify(() -> NettyRemotingServerFactory.buildNettyRemotingServer(50052));
verify(nettyRemotingServer, times(1)).registerProcessor(any(), any());
verify(nettyRemotingServer, times(1)).start();
}
} }
} }

8
dolphinscheduler-bom/pom.xml

@ -77,7 +77,7 @@
<servlet-api.version>2.5</servlet-api.version> <servlet-api.version>2.5</servlet-api.version>
<springfox.version>3.0.0</springfox.version> <springfox.version>3.0.0</springfox.version>
<guava-retry.version>2.0.0</guava-retry.version> <guava-retry.version>2.0.0</guava-retry.version>
<reflections.version>0.9.12</reflections.version> <reflections.version>0.10.2</reflections.version>
<py4j.version>0.10.9</py4j.version> <py4j.version>0.10.9</py4j.version>
<jsr305.version>3.0.0</jsr305.version> <jsr305.version>3.0.0</jsr305.version>
<commons-compress.version>1.21</commons-compress.version> <commons-compress.version>1.21</commons-compress.version>
@ -162,6 +162,12 @@
<groupId>com.cronutils</groupId> <groupId>com.cronutils</groupId>
<artifactId>cron-utils</artifactId> <artifactId>cron-utils</artifactId>
<version>${cron-utils.version}</version> <version>${cron-utils.version}</version>
<exclusions>
<exclusion>
<groupId>org.javassist</groupId>
<artifactId>javassist</artifactId>
</exclusion>
</exclusions>
</dependency> </dependency>
<dependency> <dependency>

7
dolphinscheduler-common/pom.xml

@ -65,6 +65,13 @@
<artifactId>mockito-core</artifactId> <artifactId>mockito-core</artifactId>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-inline</artifactId>
<version>3.12.4</version>
<!-- TODO: move this dependency to root pom after removing powermock in the whole project -->
<scope>test</scope>
</dependency>
<dependency> <dependency>
<groupId>org.springframework</groupId> <groupId>org.springframework</groupId>

4
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java

@ -20,7 +20,6 @@ package org.apache.dolphinscheduler.common;
import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus; import org.apache.dolphinscheduler.common.enums.WorkflowExecutionStatus;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus; import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.SystemUtils; import org.apache.commons.lang3.SystemUtils;
import java.time.Duration; import java.time.Duration;
@ -797,9 +796,6 @@ public final class Constants {
*/ */
public static final String PSTREE = "pstree"; public static final String PSTREE = "pstree";
public static final boolean KUBERNETES_MODE = !StringUtils.isEmpty(System.getenv("KUBERNETES_SERVICE_HOST"))
&& !StringUtils.isEmpty(System.getenv("KUBERNETES_SERVICE_PORT"));
/** /**
* dry run flag * dry run flag
*/ */

32
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/KubernetesUtils.java

@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.common.utils;
import org.apache.commons.lang3.StringUtils;
import lombok.experimental.UtilityClass;
@UtilityClass
public class KubernetesUtils {
public boolean isKubernetesMode() {
return !StringUtils.isEmpty(System.getenv("KUBERNETES_SERVICE_HOST"))
&& !StringUtils.isEmpty(System.getenv("KUBERNETES_SERVICE_PORT"));
}
}

15
dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/NetUtils.java

@ -21,6 +21,8 @@ import static java.util.Collections.emptyList;
import org.apache.dolphinscheduler.common.Constants; import org.apache.dolphinscheduler.common.Constants;
import org.apache.http.conn.util.InetAddressUtils;
import java.io.IOException; import java.io.IOException;
import java.net.Inet6Address; import java.net.Inet6Address;
import java.net.InetAddress; import java.net.InetAddress;
@ -33,7 +35,6 @@ import java.util.List;
import java.util.Objects; import java.util.Objects;
import java.util.Optional; import java.util.Optional;
import org.apache.http.conn.util.InetAddressUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -75,7 +76,7 @@ public class NetUtils {
*/ */
public static String getHost(InetAddress inetAddress) { public static String getHost(InetAddress inetAddress) {
if (inetAddress != null) { if (inetAddress != null) {
if (Constants.KUBERNETES_MODE) { if (KubernetesUtils.isKubernetesMode()) {
String canonicalHost = inetAddress.getCanonicalHostName(); String canonicalHost = inetAddress.getCanonicalHostName();
String[] items = canonicalHost.split("\\."); String[] items = canonicalHost.split("\\.");
if (items.length == 6 && "svc".equals(items[3])) { if (items.length == 6 && "svc".equals(items[3])) {
@ -98,7 +99,7 @@ public class NetUtils {
HOST_ADDRESS = getHost(address); HOST_ADDRESS = getHost(address);
return HOST_ADDRESS; return HOST_ADDRESS;
} }
return Constants.KUBERNETES_MODE ? "localhost" : "127.0.0.1"; return KubernetesUtils.isKubernetesMode() ? "localhost" : "127.0.0.1";
} }
private static InetAddress getLocalAddress() { private static InetAddress getLocalAddress() {
@ -258,8 +259,9 @@ public class NetUtils {
} }
private static boolean isSpecifyNetworkInterface(NetworkInterface networkInterface) { private static boolean isSpecifyNetworkInterface(NetworkInterface networkInterface) {
String preferredNetworkInterface = PropertyUtils.getString(Constants.DOLPHIN_SCHEDULER_NETWORK_INTERFACE_PREFERRED, String preferredNetworkInterface =
System.getProperty(Constants.DOLPHIN_SCHEDULER_NETWORK_INTERFACE_PREFERRED)); PropertyUtils.getString(Constants.DOLPHIN_SCHEDULER_NETWORK_INTERFACE_PREFERRED,
System.getProperty(Constants.DOLPHIN_SCHEDULER_NETWORK_INTERFACE_PREFERRED));
return Objects.equals(networkInterface.getDisplayName(), preferredNetworkInterface); return Objects.equals(networkInterface.getDisplayName(), preferredNetworkInterface);
} }
@ -267,7 +269,8 @@ public class NetUtils {
if (validNetworkInterfaces.isEmpty()) { if (validNetworkInterfaces.isEmpty()) {
return null; return null;
} }
String networkPriority = PropertyUtils.getString(Constants.DOLPHIN_SCHEDULER_NETWORK_PRIORITY_STRATEGY, NETWORK_PRIORITY_DEFAULT); String networkPriority = PropertyUtils.getString(Constants.DOLPHIN_SCHEDULER_NETWORK_PRIORITY_STRATEGY,
NETWORK_PRIORITY_DEFAULT);
if (NETWORK_PRIORITY_DEFAULT.equalsIgnoreCase(networkPriority)) { if (NETWORK_PRIORITY_DEFAULT.equalsIgnoreCase(networkPriority)) {
return findAddressByDefaultPolicy(validNetworkInterfaces); return findAddressByDefaultPolicy(validNetworkInterfaces);
} else if (NETWORK_PRIORITY_INNER.equalsIgnoreCase(networkPriority)) { } else if (NETWORK_PRIORITY_INNER.equalsIgnoreCase(networkPriority)) {

11
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/CommonUtilsTest.java

@ -17,27 +17,22 @@
package org.apache.dolphinscheduler.common.utils; package org.apache.dolphinscheduler.common.utils;
import org.apache.dolphinscheduler.spi.utils.PropertyUtils;
import org.apache.hadoop.security.UserGroupInformation;
import java.net.InetAddress; import java.net.InetAddress;
import java.net.UnknownHostException; import java.net.UnknownHostException;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.powermock.core.classloader.annotations.PrepareForTest; import org.mockito.junit.MockitoJUnitRunner;
import org.powermock.modules.junit4.PowerMockRunner;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
/** /**
* configuration test * configuration test
*/ */
@RunWith(PowerMockRunner.class) @RunWith(MockitoJUnitRunner.class)
@PrepareForTest(value = {PropertyUtils.class, UserGroupInformation.class})
public class CommonUtilsTest { public class CommonUtilsTest {
private static final Logger logger = LoggerFactory.getLogger(CommonUtilsTest.class); private static final Logger logger = LoggerFactory.getLogger(CommonUtilsTest.class);
@Test @Test

24
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java

@ -27,26 +27,26 @@ import java.io.FileNotFoundException;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito; import org.mockito.MockedStatic;
import org.powermock.core.classloader.annotations.PrepareForTest; import org.mockito.Mockito;
import org.powermock.modules.junit4.PowerMockRunner; import org.mockito.junit.MockitoJUnitRunner;
@RunWith(PowerMockRunner.class) @RunWith(MockitoJUnitRunner.class)
@PrepareForTest(DateUtils.class)
public class FileUtilsTest { public class FileUtilsTest {
@Test @Test
public void testGetDownloadFilename() { public void testGetDownloadFilename() {
PowerMockito.mockStatic(DateUtils.class); try (MockedStatic<DateUtils> mockedDateUtils = Mockito.mockStatic(DateUtils.class)) {
PowerMockito.when(DateUtils.getCurrentTime(YYYYMMDDHHMMSS)).thenReturn("20190101101059"); mockedDateUtils.when(() -> DateUtils.getCurrentTime(YYYYMMDDHHMMSS)).thenReturn("20190101101059");
Assert.assertEquals("/tmp/dolphinscheduler/download/20190101101059/test", Assert.assertEquals("/tmp/dolphinscheduler/download/20190101101059/test",
FileUtils.getDownloadFilename("test")); FileUtils.getDownloadFilename("test"));
}
} }
@Test @Test
public void testGetUploadFilename() { public void testGetUploadFilename() {
Assert.assertEquals("/tmp/dolphinscheduler/aaa/resources/bbb", Assert.assertEquals("/tmp/dolphinscheduler/aaa/resources/bbb",
FileUtils.getUploadFilename("aaa","bbb")); FileUtils.getUploadFilename("aaa", "bbb"));
} }
@Test @Test
@ -68,9 +68,9 @@ public class FileUtilsTest {
@Test @Test
public void testSetValue() { public void testSetValue() {
try { try {
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"true"); PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "true");
Assert.assertTrue(PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE)); Assert.assertTrue(PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE));
PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE,"false"); PropertyUtils.setValue(Constants.DATASOURCE_ENCRYPTION_ENABLE, "false");
Assert.assertFalse(PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE)); Assert.assertFalse(PropertyUtils.getBoolean(Constants.DATASOURCE_ENCRYPTION_ENABLE));
} catch (Exception e) { } catch (Exception e) {
Assert.assertTrue(false); Assert.assertTrue(false);

22
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/HadoopUtilsTest.java

@ -22,20 +22,18 @@ import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.powermock.api.mockito.PowerMockito; import org.mockito.MockedStatic;
import org.powermock.core.classloader.annotations.PrepareForTest; import org.mockito.Mockito;
import org.powermock.core.classloader.annotations.SuppressStaticInitializationFor; import org.mockito.junit.MockitoJUnitRunner;
import org.powermock.modules.junit4.PowerMockRunner;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
/** /**
* hadoop utils test * hadoop utils test
*/ */
@RunWith(PowerMockRunner.class) @RunWith(MockitoJUnitRunner.class)
@PrepareForTest(value = {HadoopUtils.class})
@SuppressStaticInitializationFor("org.apache.dolphinscheduler.common.utils.HttpUtils")
public class HadoopUtilsTest { public class HadoopUtilsTest {
private static final Logger logger = LoggerFactory.getLogger(HadoopUtilsTest.class); private static final Logger logger = LoggerFactory.getLogger(HadoopUtilsTest.class);
@Test @Test
@ -64,10 +62,12 @@ public class HadoopUtilsTest {
@Test @Test
public void getAppAddress() { public void getAppAddress() {
PowerMockito.mockStatic(HttpUtils.class); try (MockedStatic<HttpUtils> mockedHttpUtils = Mockito.mockStatic(HttpUtils.class)) {
PowerMockito.when(HttpUtils.get("http://ds1:8088/ws/v1/cluster/info")).thenReturn("{\"clusterInfo\":{\"state\":\"STARTED\",\"haState\":\"ACTIVE\"}}"); mockedHttpUtils.when(() -> HttpUtils.get("http://ds1:8088/ws/v1/cluster/info"))
logger.info(HadoopUtils.getAppAddress("http://ds1:8088/ws/v1/cluster/apps/%s", "ds1,ds2")); .thenReturn("{\"clusterInfo\":{\"state\":\"STARTED\",\"haState\":\"ACTIVE\"}}");
Assert.assertTrue(true); logger.info(HadoopUtils.getAppAddress("http://ds1:8088/ws/v1/cluster/apps/%s", "ds1,ds2"));
Assert.assertTrue(true);
}
} }
} }

64
dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/NetUtilsTest.java

@ -22,21 +22,15 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
import org.apache.dolphinscheduler.common.Constants;
import java.net.InetAddress; import java.net.InetAddress;
import org.junit.After;
import org.junit.Test; import org.junit.Test;
import org.powermock.reflect.Whitebox; import org.mockito.MockedStatic;
public class NetUtilsTest { public class NetUtilsTest {
@After
public void reset() {
Whitebox.setInternalState(Constants.class, "KUBERNETES_MODE", false);
}
@Test @Test
public void testGetAddr() { public void testGetAddr() {
@ -46,32 +40,42 @@ public class NetUtilsTest {
} }
@Test @Test
public void testGetHost() { public void testGetHostInKubernetesMode() {
InetAddress address = mock(InetAddress.class); try (MockedStatic<KubernetesUtils> mockedKubernetesUtils = mockStatic(KubernetesUtils.class)) {
when(address.getCanonicalHostName()).thenReturn("dolphinscheduler-worker-0.dolphinscheduler-worker-headless.default.svc.cluster.local"); mockedKubernetesUtils.when(() -> KubernetesUtils.isKubernetesMode()).thenReturn(true);
when(address.getHostName()).thenReturn("dolphinscheduler-worker-0");
when(address.getHostAddress()).thenReturn("172.17.0.15");
assertEquals("172.17.0.15", NetUtils.getHost(address));
Whitebox.setInternalState(Constants.class, "KUBERNETES_MODE", true);
assertEquals("dolphinscheduler-worker-0.dolphinscheduler-worker-headless", NetUtils.getHost(address));
address = mock(InetAddress.class); InetAddress address = mock(InetAddress.class);
when(address.getCanonicalHostName()).thenReturn("busybox-1.default-subdomain.my-namespace.svc.cluster-domain.example"); when(address.getCanonicalHostName())
when(address.getHostName()).thenReturn("busybox-1"); .thenReturn("dolphinscheduler-worker-0.dolphinscheduler-worker-headless.default.svc.cluster.local");
Whitebox.setInternalState(Constants.class, "KUBERNETES_MODE", true); when(address.getHostName()).thenReturn("dolphinscheduler-worker-0");
assertEquals("busybox-1.default-subdomain", NetUtils.getHost(address)); assertEquals("dolphinscheduler-worker-0.dolphinscheduler-worker-headless", NetUtils.getHost(address));
address = mock(InetAddress.class); address = mock(InetAddress.class);
when(address.getCanonicalHostName()).thenReturn("dolphinscheduler.cluster-domain.example"); when(address.getCanonicalHostName())
when(address.getHostName()).thenReturn("dolphinscheduler"); .thenReturn("busybox-1.default-subdomain.my-namespace.svc.cluster-domain.example");
Whitebox.setInternalState(Constants.class, "KUBERNETES_MODE", true); when(address.getHostName()).thenReturn("busybox-1");
assertEquals("dolphinscheduler.cluster-domain.example", NetUtils.getHost(address)); assertEquals("busybox-1.default-subdomain", NetUtils.getHost(address));
address = mock(InetAddress.class); address = mock(InetAddress.class);
when(address.getCanonicalHostName()).thenReturn("dolphinscheduler-worker-0"); when(address.getCanonicalHostName()).thenReturn("dolphinscheduler.cluster-domain.example");
when(address.getHostName()).thenReturn("dolphinscheduler");
assertEquals("dolphinscheduler.cluster-domain.example", NetUtils.getHost(address));
address = mock(InetAddress.class);
when(address.getCanonicalHostName()).thenReturn("dolphinscheduler-worker-0");
when(address.getHostName()).thenReturn("dolphinscheduler-worker-0");
assertEquals("dolphinscheduler-worker-0", NetUtils.getHost(address));
}
}
@Test
public void testGetHostInNonKubernetesMode() {
InetAddress address = mock(InetAddress.class);
when(address.getCanonicalHostName())
.thenReturn("dolphinscheduler-worker-0.dolphinscheduler-worker-headless.default.svc.cluster.local");
when(address.getHostName()).thenReturn("dolphinscheduler-worker-0"); when(address.getHostName()).thenReturn("dolphinscheduler-worker-0");
Whitebox.setInternalState(Constants.class, "KUBERNETES_MODE", true); when(address.getHostAddress()).thenReturn("172.17.0.15");
assertEquals("dolphinscheduler-worker-0", NetUtils.getHost(address)); assertEquals("172.17.0.15", NetUtils.getHost(address));
} }
@Test @Test

2
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/factory/NettyRemotingClientFactory.java → dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/factory/NettyRemotingClientFactory.java

@ -15,7 +15,7 @@
* limitations under the License. * limitations under the License.
*/ */
package org.apache.dolphinscheduler.service.factory; package org.apache.dolphinscheduler.remote.factory;
import org.apache.dolphinscheduler.remote.NettyRemotingClient; import org.apache.dolphinscheduler.remote.NettyRemotingClient;
import org.apache.dolphinscheduler.remote.config.NettyClientConfig; import org.apache.dolphinscheduler.remote.config.NettyClientConfig;

33
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/remote/factory/NettyRemotingServerFactory.java

@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.remote.factory;
import org.apache.dolphinscheduler.remote.NettyRemotingServer;
import org.apache.dolphinscheduler.remote.config.NettyServerConfig;
import lombok.experimental.UtilityClass;
@UtilityClass
public class NettyRemotingServerFactory {
public NettyRemotingServer buildNettyRemotingServer(int listenPort) {
NettyServerConfig serverConfig = new NettyServerConfig();
serverConfig.setListenPort(listenPort);
return new NettyRemotingServer(serverConfig);
}
}

2
dolphinscheduler-remote/src/main/java/org/apache/dolphinscheduler/rpc/config/ServiceBean.java

@ -49,7 +49,7 @@ public class ServiceBean {
if (initialized.get()) { if (initialized.get()) {
return; return;
} }
Reflections f = new Reflections("org/apache/dolphinscheduler/"); Reflections f = new Reflections("org.apache.dolphinscheduler.");
List<Class<?>> list = new ArrayList<>(f.getTypesAnnotatedWith(RpcService.class)); List<Class<?>> list = new ArrayList<>(f.getTypesAnnotatedWith(RpcService.class));
list.forEach(rpcClass -> { list.forEach(rpcClass -> {
RpcService rpcService = rpcClass.getAnnotation(RpcService.class); RpcService rpcService = rpcClass.getAnnotation(RpcService.class);

2
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/alert/AlertClientService.java

@ -21,9 +21,9 @@ import org.apache.dolphinscheduler.remote.NettyRemotingClient;
import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.Command;
import org.apache.dolphinscheduler.remote.command.alert.AlertSendRequestCommand; import org.apache.dolphinscheduler.remote.command.alert.AlertSendRequestCommand;
import org.apache.dolphinscheduler.remote.command.alert.AlertSendResponseCommand; import org.apache.dolphinscheduler.remote.command.alert.AlertSendResponseCommand;
import org.apache.dolphinscheduler.remote.factory.NettyRemotingClientFactory;
import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.remote.utils.JsonSerializer; import org.apache.dolphinscheduler.remote.utils.JsonSerializer;
import org.apache.dolphinscheduler.service.factory.NettyRemotingClientFactory;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;

2
dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/LogClient.java

@ -34,8 +34,8 @@ import org.apache.dolphinscheduler.remote.command.log.RollViewLogResponseCommand
import org.apache.dolphinscheduler.remote.command.log.ViewLogRequestCommand; import org.apache.dolphinscheduler.remote.command.log.ViewLogRequestCommand;
import org.apache.dolphinscheduler.remote.command.log.ViewLogResponseCommand; import org.apache.dolphinscheduler.remote.command.log.ViewLogResponseCommand;
import org.apache.dolphinscheduler.remote.exceptions.RemotingException; import org.apache.dolphinscheduler.remote.exceptions.RemotingException;
import org.apache.dolphinscheduler.remote.factory.NettyRemotingClientFactory;
import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.service.factory.NettyRemotingClientFactory;
import java.util.List; import java.util.List;

2
dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/alert/AlertClientServiceTest.java

@ -23,7 +23,7 @@ import org.apache.dolphinscheduler.remote.command.Command;
import org.apache.dolphinscheduler.remote.command.alert.AlertSendRequestCommand; import org.apache.dolphinscheduler.remote.command.alert.AlertSendRequestCommand;
import org.apache.dolphinscheduler.remote.command.alert.AlertSendResponseCommand; import org.apache.dolphinscheduler.remote.command.alert.AlertSendResponseCommand;
import org.apache.dolphinscheduler.remote.command.alert.AlertSendResponseResult; import org.apache.dolphinscheduler.remote.command.alert.AlertSendResponseResult;
import org.apache.dolphinscheduler.service.factory.NettyRemotingClientFactory; import org.apache.dolphinscheduler.remote.factory.NettyRemotingClientFactory;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;

2
dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/LogClientTest.java

@ -26,8 +26,8 @@ import org.apache.dolphinscheduler.remote.command.log.GetLogBytesResponseCommand
import org.apache.dolphinscheduler.remote.command.log.RemoveTaskLogResponseCommand; import org.apache.dolphinscheduler.remote.command.log.RemoveTaskLogResponseCommand;
import org.apache.dolphinscheduler.remote.command.log.RollViewLogResponseCommand; import org.apache.dolphinscheduler.remote.command.log.RollViewLogResponseCommand;
import org.apache.dolphinscheduler.remote.command.log.ViewLogResponseCommand; import org.apache.dolphinscheduler.remote.command.log.ViewLogResponseCommand;
import org.apache.dolphinscheduler.remote.factory.NettyRemotingClientFactory;
import org.apache.dolphinscheduler.remote.utils.Host; import org.apache.dolphinscheduler.remote.utils.Host;
import org.apache.dolphinscheduler.service.factory.NettyRemotingClientFactory;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;

8
dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/utils/DateUtils.java

@ -45,7 +45,8 @@ public class DateUtils {
/** /**
* a default datetime formatter for the timestamp * a default datetime formatter for the timestamp
*/ */
private static final DateTimeFormatter DEFAULT_DATETIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); private static final DateTimeFormatter DEFAULT_DATETIME_FORMATTER =
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
private static final Logger logger = LoggerFactory.getLogger(DateUtils.class); private static final Logger logger = LoggerFactory.getLogger(DateUtils.class);
@ -193,7 +194,6 @@ public class DateUtils {
return calendar.get(Calendar.HOUR_OF_DAY); return calendar.get(Calendar.HOUR_OF_DAY);
} }
/** /**
* compare two dates * compare two dates
* *
@ -431,10 +431,6 @@ public class DateUtils {
return TimeZone.getTimeZone(timezoneId); return TimeZone.getTimeZone(timezoneId);
} }
/**
* get timestamp in String
* PowerMock 2.0.9 fails to mock System.currentTimeMillis(), this method helps in UT
*/
public static String getTimestampString() { public static String getTimestampString() {
return String.valueOf(System.currentTimeMillis()); return String.valueOf(System.currentTimeMillis());
} }

21
dolphinscheduler-worker/src/test/java/org/apache/dolphinscheduler/server/worker/processor/TaskDispatchProcessorTest.java

@ -17,9 +17,7 @@
package org.apache.dolphinscheduler.server.worker.processor; package org.apache.dolphinscheduler.server.worker.processor;
import io.netty.channel.Channel;
import org.apache.dolphinscheduler.common.storage.StorageOperate; import org.apache.dolphinscheduler.common.storage.StorageOperate;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannel;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.remote.command.Command; import org.apache.dolphinscheduler.remote.command.Command;
import org.apache.dolphinscheduler.remote.command.CommandType; import org.apache.dolphinscheduler.remote.command.CommandType;
@ -29,19 +27,22 @@ import org.apache.dolphinscheduler.server.worker.rpc.WorkerMessageSender;
import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread; import org.apache.dolphinscheduler.server.worker.runner.WorkerManagerThread;
import org.apache.dolphinscheduler.service.alert.AlertClientService; import org.apache.dolphinscheduler.service.alert.AlertClientService;
import org.apache.dolphinscheduler.service.task.TaskPluginManager; import org.apache.dolphinscheduler.service.task.TaskPluginManager;
import java.util.Date;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import org.mockito.InjectMocks; import org.mockito.InjectMocks;
import org.mockito.Mock; import org.mockito.Mock;
import org.mockito.Mockito; import org.mockito.Mockito;
import org.powermock.modules.junit4.PowerMockRunner; import org.mockito.junit.MockitoJUnitRunner;
import java.util.Date; import io.netty.channel.Channel;
/** /**
* test task execute processor * test task execute processor
*/ */
@RunWith(PowerMockRunner.class) @RunWith(MockitoJUnitRunner.class)
public class TaskDispatchProcessorTest { public class TaskDispatchProcessorTest {
@InjectMocks @InjectMocks
@ -68,25 +69,21 @@ public class TaskDispatchProcessorTest {
@Test @Test
public void process() { public void process() {
Channel channel = Mockito.mock(Channel.class); Channel channel = Mockito.mock(Channel.class);
TaskChannel taskChannel = Mockito.mock(TaskChannel.class);
Mockito.when(taskPluginManager.getTaskChannel(Mockito.anyString())).thenReturn(taskChannel);
TaskExecutionContext taskExecutionContext = getTaskExecutionContext(); TaskExecutionContext taskExecutionContext = getTaskExecutionContext();
Command dispatchCommand = createDispatchCommand(taskExecutionContext); Command dispatchCommand = createDispatchCommand(taskExecutionContext);
taskDispatchProcessor.process(channel, dispatchCommand); taskDispatchProcessor.process(channel, dispatchCommand);
Mockito.verify(workerManagerThread, Mockito.atMostOnce()).offer(Mockito.any()); Mockito.verify(workerManagerThread, Mockito.atMostOnce()).offer(Mockito.any());
Mockito.verify(workerMessageSender, Mockito.never()).sendMessageWithRetry(taskExecutionContext, "localhost:5678", CommandType.TASK_REJECT); Mockito.verify(workerMessageSender, Mockito.never()).sendMessageWithRetry(taskExecutionContext,
"localhost:5678", CommandType.TASK_REJECT);
} }
public Command createDispatchCommand(TaskExecutionContext taskExecutionContext) { public Command createDispatchCommand(TaskExecutionContext taskExecutionContext) {
return new TaskDispatchCommand( return new TaskDispatchCommand(
taskExecutionContext, taskExecutionContext,
"localhost:5678", "localhost:5678",
"localhost:1234", "localhost:1234",
System.currentTimeMillis() System.currentTimeMillis()).convert2Command();
).convert2Command();
} }
public TaskExecutionContext getTaskExecutionContext() { public TaskExecutionContext getTaskExecutionContext() {

19
pom.xml

@ -81,7 +81,6 @@
<auto-service.version>1.0.1</auto-service.version> <auto-service.version>1.0.1</auto-service.version>
<jacoco.skip>false</jacoco.skip> <jacoco.skip>false</jacoco.skip>
<maven-jar-plugin.version>3.2.0</maven-jar-plugin.version> <maven-jar-plugin.version>3.2.0</maven-jar-plugin.version>
<powermock.version>2.0.9</powermock.version>
<exec-maven-plugin.version>3.0.0</exec-maven-plugin.version> <exec-maven-plugin.version>3.0.0</exec-maven-plugin.version>
<owasp-dependency-check-maven.version>7.1.2</owasp-dependency-check-maven.version> <owasp-dependency-check-maven.version>7.1.2</owasp-dependency-check-maven.version>
<lombok.version>1.18.20</lombok.version> <lombok.version>1.18.20</lombok.version>
@ -363,24 +362,6 @@
<version>${lombok.version}</version> <version>${lombok.version}</version>
<scope>provided</scope> <scope>provided</scope>
</dependency> </dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-api-mockito2</artifactId>
<version>${powermock.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-module-junit4</artifactId>
<version>${powermock.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-core</artifactId>
<version>${powermock.version}</version>
<scope>test</scope>
</dependency>
</dependencies> </dependencies>
<build> <build>

4
tools/dependencies/known-dependencies.txt

@ -124,7 +124,7 @@ jakarta.websocket-api-1.1.2.jar
jakarta.xml.bind-api-2.3.3.jar jakarta.xml.bind-api-2.3.3.jar
jamon-runtime-2.3.1.jar jamon-runtime-2.3.1.jar
janino-3.0.16.jar janino-3.0.16.jar
javassist-3.27.0-GA.jar javassist-3.28.0-GA.jar
javax.activation-api-1.2.0.jar javax.activation-api-1.2.0.jar
javax.annotation-api-1.3.2.jar javax.annotation-api-1.3.2.jar
javax.el-3.0.0.jar javax.el-3.0.0.jar
@ -266,7 +266,7 @@ py4j-0.10.9.jar
quartz-2.3.2.jar quartz-2.3.2.jar
re2j-1.1.jar re2j-1.1.jar
reactive-streams-1.0.4.jar reactive-streams-1.0.4.jar
reflections-0.9.12.jar reflections-0.10.2.jar
regions-2.17.282.jar regions-2.17.282.jar
reload4j-1.2.18.3.jar reload4j-1.2.18.3.jar
sdk-core-2.17.282.jar sdk-core-2.17.282.jar

Loading…
Cancel
Save