[Chore] Use testcontainer origin exposed port rather than generate random port (#16195)

dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImplTest.java

@@ -29,7 +29,6 @@ import org.apache.dolphinscheduler.common.enums.WarningType;
 import org.apache.dolphinscheduler.common.utils.DateUtils;
 import org.apache.dolphinscheduler.dao.BaseDaoTest;
 import org.apache.dolphinscheduler.dao.entity.Command;
-import org.apache.dolphinscheduler.dao.mapper.CommandMapper;
 import org.apache.dolphinscheduler.dao.repository.CommandDao;
 
 import org.apache.commons.lang3.RandomUtils;
@@ -39,34 +38,28 @@ import java.util.stream.Collectors;
 
 import org.junit.jupiter.api.RepeatedTest;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.test.annotation.DirtiesContext;
 
-import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
+@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
-
 class CommandDaoImplTest extends BaseDaoTest {
 
     @Autowired
     private CommandDao commandDao;
 
-    @Autowired
+    @RepeatedTest(value = 10)
-    private CommandMapper commandMapper;
-
-    @RepeatedTest(value = 100)
     void fetchCommandByIdSlot() {
-        // clear all commands
-        commandMapper.delete(new QueryWrapper<Command>().ge("id", -1));
-
         int totalSlot = RandomUtils.nextInt(1, 10);
         int currentSlotIndex = RandomUtils.nextInt(0, totalSlot);
         int fetchSize = RandomUtils.nextInt(10, 100);
         int idStep = RandomUtils.nextInt(1, 5);
         int commandSize = RandomUtils.nextInt(currentSlotIndex, 1000);
         // Generate commandSize commands
-        int id = 0;
+        int id = 1;
         for (int j = 0; j < commandSize; j++) {
-            id += idStep;
             Command command = generateCommand(CommandType.START_PROCESS, 0);
             command.setId(id);
             commandDao.insert(command);
+            id += idStep;
         }
 
         List<Command> commands = commandDao.queryCommandByIdSlot(currentSlotIndex, totalSlot, idStep, fetchSize);

dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryTestCase.java

@@ -51,6 +51,7 @@ public class EtcdRegistryTestCase extends RegistryTestCase<EtcdRegistry> {
                 .build()
                 .cluster();
         etcdCluster.start();
+        System.clearProperty("registry.endpoints");
         System.setProperty("registry.endpoints",
                 etcdCluster.clientEndpoints().stream().map(URI::toString).collect(Collectors.joining(",")));
     }

dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/test/java/org/apache/dolphinscheduler/plugin/registry/jdbc/MysqlJdbcRegistryTestCase.java

@@ -17,8 +17,6 @@
 
 package org.apache.dolphinscheduler.plugin.registry.jdbc;
 
-import org.apache.commons.lang3.RandomUtils;
-
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.Statement;
@@ -37,8 +35,6 @@ import org.testcontainers.containers.wait.strategy.Wait;
 import org.testcontainers.lifecycle.Startables;
 import org.testcontainers.utility.DockerImageName;
 
-import com.google.common.collect.Lists;
-
 @ActiveProfiles("mysql")
 class MysqlJdbcRegistryTestCase extends JdbcRegistryTestCase {
 
@@ -55,11 +51,11 @@ class MysqlJdbcRegistryTestCase extends JdbcRegistryTestCase {
                 .withExposedPorts(3306)
                 .waitingFor(Wait.forHealthcheck().withStartupTimeout(Duration.ofSeconds(300)));
 
-        int exposedPort = RandomUtils.nextInt(10000, 65535);
-        mysqlContainer.setPortBindings(Lists.newArrayList(exposedPort + ":3306"));
         Startables.deepStart(Stream.of(mysqlContainer)).join();
 
-        String jdbcUrl = "jdbc:mysql://localhost:" + exposedPort + "/dolphinscheduler?useSSL=false&serverTimezone=UTC";
+        String jdbcUrl = "jdbc:mysql://localhost:" + mysqlContainer.getMappedPort(3306)
+                + "/dolphinscheduler?useSSL=false&serverTimezone=UTC";
+        System.clearProperty("spring.datasource.url");
         System.setProperty("spring.datasource.url", jdbcUrl);
 
         try (

dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/test/java/org/apache/dolphinscheduler/plugin/registry/jdbc/PostgresqlJdbcRegistryTestCase.java

@@ -17,8 +17,6 @@
 
 package org.apache.dolphinscheduler.plugin.registry.jdbc;
 
-import org.apache.commons.lang3.RandomUtils;
-
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.Statement;
@@ -37,8 +35,6 @@ import org.testcontainers.containers.PostgreSQLContainer;
 import org.testcontainers.lifecycle.Startables;
 import org.testcontainers.utility.DockerImageName;
 
-import com.google.common.collect.Lists;
-
 @ActiveProfiles("postgresql")
 @SpringBootTest(classes = {JdbcRegistryProperties.class})
 @SpringBootApplication(scanBasePackageClasses = JdbcRegistryProperties.class)
@@ -55,12 +51,11 @@ public class PostgresqlJdbcRegistryTestCase extends JdbcRegistryTestCase {
                 .withDatabaseName("dolphinscheduler")
                 .withNetwork(Network.newNetwork())
                 .withExposedPorts(5432);
-        int exposedPort = RandomUtils.nextInt(10000, 65535);
 
-        postgresqlContainer.setPortBindings(Lists.newArrayList(exposedPort + ":5432"));
         Startables.deepStart(Stream.of(postgresqlContainer)).join();
 
-        String jdbcUrl = "jdbc:postgresql://localhost:" + exposedPort + "/dolphinscheduler";
+        String jdbcUrl = "jdbc:postgresql://localhost:" + postgresqlContainer.getMappedPort(5432) + "/dolphinscheduler";
+        System.clearProperty("spring.datasource.url");
         System.setProperty("spring.datasource.url", jdbcUrl);
         try (
                 Connection connection = DriverManager.getConnection(jdbcUrl, "root", "root");

dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryTestCase.java

@@ -19,8 +19,6 @@ package org.apache.dolphinscheduler.plugin.registry.zookeeper;
 
 import org.apache.dolphinscheduler.plugin.registry.RegistryTestCase;
 
-import org.apache.commons.lang3.RandomUtils;
-
 import java.util.stream.Stream;
 
 import lombok.SneakyThrows;
@@ -35,8 +33,6 @@ import org.testcontainers.containers.Network;
 import org.testcontainers.lifecycle.Startables;
 import org.testcontainers.utility.DockerImageName;
 
-import com.google.common.collect.Lists;
-
 @SpringBootTest(classes = ZookeeperRegistryProperties.class)
 @SpringBootApplication(scanBasePackageClasses = ZookeeperRegistryProperties.class)
 class ZookeeperRegistryTestCase extends RegistryTestCase<ZookeeperRegistry> {
@@ -52,11 +48,11 @@ class ZookeeperRegistryTestCase extends RegistryTestCase<ZookeeperRegistry> {
     @BeforeAll
     public static void setUpTestingServer() {
         zookeeperContainer = new GenericContainer<>(DockerImageName.parse("zookeeper:3.8"))
-                .withNetwork(NETWORK);
+                .withNetwork(NETWORK)
-        int randomPort = RandomUtils.nextInt(10000, 65535);
+                .withExposedPorts(2181);
-        zookeeperContainer.setPortBindings(Lists.newArrayList(randomPort + ":2181"));
         Startables.deepStart(Stream.of(zookeeperContainer)).join();
-        System.setProperty("registry.zookeeper.connect-string", "localhost:" + randomPort);
+        System.clearProperty("registry.zookeeper.connect-string");
+        System.setProperty("registry.zookeeper.connect-string", "localhost:" + zookeeperContainer.getMappedPort(2181));
     }
 
     @SneakyThrows

dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java

@@ -52,6 +52,7 @@ class LocalStorageOperatorTest {
     @BeforeEach
     public void setup() {
         Files.createDirectories(Paths.get(resourceBaseDir));
+        System.clearProperty(Constants.RESOURCE_UPLOAD_PATH);
         System.setProperty(Constants.RESOURCE_UPLOAD_PATH, resourceBaseDir);
 
         LocalStorageOperatorFactory localStorageOperatorFactory = new LocalStorageOperatorFactory();

dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/jupiter/DolphinSchedulerDatabaseContainerExtension.java

@@ -20,7 +20,6 @@ package org.apache.dolphinscheduler.tools.datasource.jupiter;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.ServiceLoader;
-import java.util.stream.Stream;
 
 import lombok.extern.slf4j.Slf4j;
 
@@ -28,7 +27,6 @@ import org.junit.jupiter.api.extension.AfterAllCallback;
 import org.junit.jupiter.api.extension.BeforeAllCallback;
 import org.junit.jupiter.api.extension.ExtensionContext;
 import org.testcontainers.containers.GenericContainer;
-import org.testcontainers.lifecycle.Startables;
 import org.testcontainers.utility.DockerImageName;
 
 @Slf4j
@@ -39,12 +37,6 @@ public class DolphinSchedulerDatabaseContainerExtension implements BeforeAllCall
     @Override
     public void beforeAll(ExtensionContext context) {
         databaseContainer = getDataSourceContainer(context);
-        log.info("Create {} successfully.", databaseContainer.getDockerImageName());
-        databaseContainer.start();
-
-        log.info("Starting {}...", databaseContainer.getDockerImageName());
-        Startables.deepStart(Stream.of(databaseContainer)).join();
-        log.info("{} started", databaseContainer.getDockerImageName());
 
     }
 

dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/mysql/DolphinSchedulerMysqlProfile.java

@@ -27,6 +27,7 @@ import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
 
 import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.annotation.DirtiesContext;
 import org.springframework.test.context.ActiveProfiles;
 
 @Inherited
@@ -34,6 +35,7 @@ import org.springframework.test.context.ActiveProfiles;
 @Retention(RetentionPolicy.RUNTIME)
 @ActiveProfiles("mysql")
 @SpringBootTest(classes = {UpgradeDolphinScheduler.class, DaoConfiguration.class})
+@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
 public @interface DolphinSchedulerMysqlProfile {
 
 }

dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/mysql/MysqlDatabaseContainerProvider.java

@@ -20,15 +20,20 @@ package org.apache.dolphinscheduler.tools.datasource.mysql;
 import org.apache.dolphinscheduler.tools.datasource.jupiter.DatabaseContainerProvider;
 import org.apache.dolphinscheduler.tools.datasource.jupiter.DolphinSchedulerDatabaseContainer;
 
+import java.util.stream.Stream;
+
+import lombok.extern.slf4j.Slf4j;
+
 import org.testcontainers.containers.GenericContainer;
 import org.testcontainers.containers.MySQLContainer;
 import org.testcontainers.containers.Network;
 import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.lifecycle.Startables;
 import org.testcontainers.utility.DockerImageName;
 
 import com.google.auto.service.AutoService;
-import com.google.common.collect.Lists;
 
+@Slf4j
 @AutoService(DatabaseContainerProvider.class)
 public class MysqlDatabaseContainerProvider implements DatabaseContainerProvider {
 
@@ -43,7 +48,19 @@ public class MysqlDatabaseContainerProvider implements DatabaseContainerProvider
                 .withNetwork(NETWORK)
                 .withExposedPorts(3306)
                 .waitingFor(Wait.forHealthcheck());
-        mysqlContainer.setPortBindings(Lists.newArrayList("3306:3306"));
+
+        log.info("Create {} successfully.", mysqlContainer.getDockerImageName());
+        mysqlContainer.start();
+
+        log.info("Starting {}...", mysqlContainer.getDockerImageName());
+        Startables.deepStart(Stream.of(mysqlContainer)).join();
+        log.info("{} started", mysqlContainer.getDockerImageName());
+
+        String jdbcUrl = "jdbc:mysql://localhost:" + mysqlContainer.getMappedPort(3306)
+                + "/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8";
+        System.clearProperty("spring.datasource.url");
+        System.setProperty("spring.datasource.url", jdbcUrl);
+
         return mysqlContainer;
     }
 

dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/postgresql/DolphinSchedulerPostgresqlProfile.java

@@ -27,12 +27,14 @@ import java.lang.annotation.RetentionPolicy;
 import java.lang.annotation.Target;
 
 import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.annotation.DirtiesContext;
 import org.springframework.test.context.ActiveProfiles;
 @Inherited
 @Target(ElementType.TYPE)
 @Retention(RetentionPolicy.RUNTIME)
 @ActiveProfiles("postgresql")
 @SpringBootTest(classes = {UpgradeDolphinScheduler.class, DaoConfiguration.class})
+@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
 public @interface DolphinSchedulerPostgresqlProfile {
 
 }

dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/postgresql/PostgresqlDatabaseContainerProvider.java

@@ -20,15 +20,17 @@ package org.apache.dolphinscheduler.tools.datasource.postgresql;
 import org.apache.dolphinscheduler.tools.datasource.jupiter.DatabaseContainerProvider;
 import org.apache.dolphinscheduler.tools.datasource.jupiter.DolphinSchedulerDatabaseContainer;
 
+import java.util.stream.Stream;
+
 import lombok.extern.slf4j.Slf4j;
 
 import org.testcontainers.containers.GenericContainer;
 import org.testcontainers.containers.Network;
 import org.testcontainers.containers.PostgreSQLContainer;
+import org.testcontainers.lifecycle.Startables;
 import org.testcontainers.utility.DockerImageName;
 
 import com.google.auto.service.AutoService;
-import com.google.common.collect.Lists;
 
 @Slf4j
 @AutoService(DatabaseContainerProvider.class)
@@ -47,8 +49,18 @@ public class PostgresqlDatabaseContainerProvider implements DatabaseContainerPro
                         .withDatabaseName("dolphinscheduler")
                         .withNetwork(NETWORK)
                         .withExposedPorts(5432);
-        postgresqlContainer.setPortBindings(Lists.newArrayList("5432:5432"));
 
+        log.info("Create {} successfully.", postgresqlContainer.getDockerImageName());
+        postgresqlContainer.start();
+
+        log.info("Starting {}...", postgresqlContainer.getDockerImageName());
+        Startables.deepStart(Stream.of(postgresqlContainer)).join();
+        log.info("{} started", postgresqlContainer.getDockerImageName());
+
+        String jdbcUrl = "jdbc:mysql://localhost:" + postgresqlContainer.getMappedPort(5432)
+                + "/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8";
+        System.clearProperty("spring.datasource.url");
+        System.setProperty("spring.datasource.url", jdbcUrl);
         return postgresqlContainer;
     }