From f0e0395b36bba5977135a3b7b4472b91ec388e50 Mon Sep 17 00:00:00 2001 From: Wenjun Ruan Date: Fri, 21 Jun 2024 22:19:54 +0800 Subject: [PATCH] [Chore] Use testcontainer origin exposed port rather than generate random port (#16195) --- .../repository/impl/CommandDaoImplTest.java | 17 +++++---------- .../registry/etcd/EtcdRegistryTestCase.java | 1 + .../jdbc/MysqlJdbcRegistryTestCase.java | 10 +++------ .../jdbc/PostgresqlJdbcRegistryTestCase.java | 9 ++------ .../zookeeper/ZookeeperRegistryTestCase.java | 12 ++++------- .../hdfs/LocalStorageOperatorTest.java | 1 + ...inSchedulerDatabaseContainerExtension.java | 8 ------- .../mysql/DolphinSchedulerMysqlProfile.java | 2 ++ .../mysql/MysqlDatabaseContainerProvider.java | 21 +++++++++++++++++-- .../DolphinSchedulerPostgresqlProfile.java | 2 ++ .../PostgresqlDatabaseContainerProvider.java | 16 ++++++++++++-- 11 files changed, 53 insertions(+), 46 deletions(-) diff --git a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImplTest.java b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImplTest.java index 1897ad7fa2..ec12bbb2f2 100644 --- a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImplTest.java +++ b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImplTest.java @@ -29,7 +29,6 @@ import org.apache.dolphinscheduler.common.enums.WarningType; import org.apache.dolphinscheduler.common.utils.DateUtils; import org.apache.dolphinscheduler.dao.BaseDaoTest; import org.apache.dolphinscheduler.dao.entity.Command; -import org.apache.dolphinscheduler.dao.mapper.CommandMapper; import org.apache.dolphinscheduler.dao.repository.CommandDao; import org.apache.commons.lang3.RandomUtils; @@ -39,34 +38,28 @@ import java.util.stream.Collectors; import org.junit.jupiter.api.RepeatedTest; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.annotation.DirtiesContext; -import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; - +@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD) class CommandDaoImplTest extends BaseDaoTest { @Autowired private CommandDao commandDao; - @Autowired - private CommandMapper commandMapper; - - @RepeatedTest(value = 100) + @RepeatedTest(value = 10) void fetchCommandByIdSlot() { - // clear all commands - commandMapper.delete(new QueryWrapper().ge("id", -1)); - int totalSlot = RandomUtils.nextInt(1, 10); int currentSlotIndex = RandomUtils.nextInt(0, totalSlot); int fetchSize = RandomUtils.nextInt(10, 100); int idStep = RandomUtils.nextInt(1, 5); int commandSize = RandomUtils.nextInt(currentSlotIndex, 1000); // Generate commandSize commands - int id = 0; + int id = 1; for (int j = 0; j < commandSize; j++) { - id += idStep; Command command = generateCommand(CommandType.START_PROCESS, 0); command.setId(id); commandDao.insert(command); + id += idStep; } List commands = commandDao.queryCommandByIdSlot(currentSlotIndex, totalSlot, idStep, fetchSize); diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryTestCase.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryTestCase.java index 1e751c1862..39bfea8cc7 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryTestCase.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryTestCase.java @@ -51,6 +51,7 @@ public class EtcdRegistryTestCase extends RegistryTestCase { .build() .cluster(); etcdCluster.start(); + System.clearProperty("registry.endpoints"); System.setProperty("registry.endpoints", etcdCluster.clientEndpoints().stream().map(URI::toString).collect(Collectors.joining(","))); } diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/test/java/org/apache/dolphinscheduler/plugin/registry/jdbc/MysqlJdbcRegistryTestCase.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/test/java/org/apache/dolphinscheduler/plugin/registry/jdbc/MysqlJdbcRegistryTestCase.java index 0db601693b..6dca1aeb2a 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/test/java/org/apache/dolphinscheduler/plugin/registry/jdbc/MysqlJdbcRegistryTestCase.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/test/java/org/apache/dolphinscheduler/plugin/registry/jdbc/MysqlJdbcRegistryTestCase.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.plugin.registry.jdbc; -import org.apache.commons.lang3.RandomUtils; - import java.sql.Connection; import java.sql.DriverManager; import java.sql.Statement; @@ -37,8 +35,6 @@ import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.lifecycle.Startables; import org.testcontainers.utility.DockerImageName; -import com.google.common.collect.Lists; - @ActiveProfiles("mysql") class MysqlJdbcRegistryTestCase extends JdbcRegistryTestCase { @@ -55,11 +51,11 @@ class MysqlJdbcRegistryTestCase extends JdbcRegistryTestCase { .withExposedPorts(3306) .waitingFor(Wait.forHealthcheck().withStartupTimeout(Duration.ofSeconds(300))); - int exposedPort = RandomUtils.nextInt(10000, 65535); - mysqlContainer.setPortBindings(Lists.newArrayList(exposedPort + ":3306")); Startables.deepStart(Stream.of(mysqlContainer)).join(); - String jdbcUrl = "jdbc:mysql://localhost:" + exposedPort + "/dolphinscheduler?useSSL=false&serverTimezone=UTC"; + String jdbcUrl = "jdbc:mysql://localhost:" + mysqlContainer.getMappedPort(3306) + + "/dolphinscheduler?useSSL=false&serverTimezone=UTC"; + System.clearProperty("spring.datasource.url"); System.setProperty("spring.datasource.url", jdbcUrl); try ( diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/test/java/org/apache/dolphinscheduler/plugin/registry/jdbc/PostgresqlJdbcRegistryTestCase.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/test/java/org/apache/dolphinscheduler/plugin/registry/jdbc/PostgresqlJdbcRegistryTestCase.java index a86533dcaf..f34015e5a2 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/test/java/org/apache/dolphinscheduler/plugin/registry/jdbc/PostgresqlJdbcRegistryTestCase.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/test/java/org/apache/dolphinscheduler/plugin/registry/jdbc/PostgresqlJdbcRegistryTestCase.java @@ -17,8 +17,6 @@ package org.apache.dolphinscheduler.plugin.registry.jdbc; -import org.apache.commons.lang3.RandomUtils; - import java.sql.Connection; import java.sql.DriverManager; import java.sql.Statement; @@ -37,8 +35,6 @@ import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.lifecycle.Startables; import org.testcontainers.utility.DockerImageName; -import com.google.common.collect.Lists; - @ActiveProfiles("postgresql") @SpringBootTest(classes = {JdbcRegistryProperties.class}) @SpringBootApplication(scanBasePackageClasses = JdbcRegistryProperties.class) @@ -55,12 +51,11 @@ public class PostgresqlJdbcRegistryTestCase extends JdbcRegistryTestCase { .withDatabaseName("dolphinscheduler") .withNetwork(Network.newNetwork()) .withExposedPorts(5432); - int exposedPort = RandomUtils.nextInt(10000, 65535); - postgresqlContainer.setPortBindings(Lists.newArrayList(exposedPort + ":5432")); Startables.deepStart(Stream.of(postgresqlContainer)).join(); - String jdbcUrl = "jdbc:postgresql://localhost:" + exposedPort + "/dolphinscheduler"; + String jdbcUrl = "jdbc:postgresql://localhost:" + postgresqlContainer.getMappedPort(5432) + "/dolphinscheduler"; + System.clearProperty("spring.datasource.url"); System.setProperty("spring.datasource.url", jdbcUrl); try ( Connection connection = DriverManager.getConnection(jdbcUrl, "root", "root"); diff --git a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryTestCase.java b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryTestCase.java index 60d8520b81..73c784b2df 100644 --- a/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryTestCase.java +++ b/dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryTestCase.java @@ -19,8 +19,6 @@ package org.apache.dolphinscheduler.plugin.registry.zookeeper; import org.apache.dolphinscheduler.plugin.registry.RegistryTestCase; -import org.apache.commons.lang3.RandomUtils; - import java.util.stream.Stream; import lombok.SneakyThrows; @@ -35,8 +33,6 @@ import org.testcontainers.containers.Network; import org.testcontainers.lifecycle.Startables; import org.testcontainers.utility.DockerImageName; -import com.google.common.collect.Lists; - @SpringBootTest(classes = ZookeeperRegistryProperties.class) @SpringBootApplication(scanBasePackageClasses = ZookeeperRegistryProperties.class) class ZookeeperRegistryTestCase extends RegistryTestCase { @@ -52,11 +48,11 @@ class ZookeeperRegistryTestCase extends RegistryTestCase { @BeforeAll public static void setUpTestingServer() { zookeeperContainer = new GenericContainer<>(DockerImageName.parse("zookeeper:3.8")) - .withNetwork(NETWORK); - int randomPort = RandomUtils.nextInt(10000, 65535); - zookeeperContainer.setPortBindings(Lists.newArrayList(randomPort + ":2181")); + .withNetwork(NETWORK) + .withExposedPorts(2181); Startables.deepStart(Stream.of(zookeeperContainer)).join(); - System.setProperty("registry.zookeeper.connect-string", "localhost:" + randomPort); + System.clearProperty("registry.zookeeper.connect-string"); + System.setProperty("registry.zookeeper.connect-string", "localhost:" + zookeeperContainer.getMappedPort(2181)); } @SneakyThrows diff --git a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java index 7a277afbce..f6ed76e265 100644 --- a/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java +++ b/dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java @@ -52,6 +52,7 @@ class LocalStorageOperatorTest { @BeforeEach public void setup() { Files.createDirectories(Paths.get(resourceBaseDir)); + System.clearProperty(Constants.RESOURCE_UPLOAD_PATH); System.setProperty(Constants.RESOURCE_UPLOAD_PATH, resourceBaseDir); LocalStorageOperatorFactory localStorageOperatorFactory = new LocalStorageOperatorFactory(); diff --git a/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/jupiter/DolphinSchedulerDatabaseContainerExtension.java b/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/jupiter/DolphinSchedulerDatabaseContainerExtension.java index 4b463e1ef9..a264ca09e9 100644 --- a/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/jupiter/DolphinSchedulerDatabaseContainerExtension.java +++ b/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/jupiter/DolphinSchedulerDatabaseContainerExtension.java @@ -20,7 +20,6 @@ package org.apache.dolphinscheduler.tools.datasource.jupiter; import java.util.HashMap; import java.util.Map; import java.util.ServiceLoader; -import java.util.stream.Stream; import lombok.extern.slf4j.Slf4j; @@ -28,7 +27,6 @@ import org.junit.jupiter.api.extension.AfterAllCallback; import org.junit.jupiter.api.extension.BeforeAllCallback; import org.junit.jupiter.api.extension.ExtensionContext; import org.testcontainers.containers.GenericContainer; -import org.testcontainers.lifecycle.Startables; import org.testcontainers.utility.DockerImageName; @Slf4j @@ -39,12 +37,6 @@ public class DolphinSchedulerDatabaseContainerExtension implements BeforeAllCall @Override public void beforeAll(ExtensionContext context) { databaseContainer = getDataSourceContainer(context); - log.info("Create {} successfully.", databaseContainer.getDockerImageName()); - databaseContainer.start(); - - log.info("Starting {}...", databaseContainer.getDockerImageName()); - Startables.deepStart(Stream.of(databaseContainer)).join(); - log.info("{} started", databaseContainer.getDockerImageName()); } diff --git a/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/mysql/DolphinSchedulerMysqlProfile.java b/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/mysql/DolphinSchedulerMysqlProfile.java index f32f4d80d4..a8670d5934 100644 --- a/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/mysql/DolphinSchedulerMysqlProfile.java +++ b/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/mysql/DolphinSchedulerMysqlProfile.java @@ -27,6 +27,7 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ActiveProfiles; @Inherited @@ -34,6 +35,7 @@ import org.springframework.test.context.ActiveProfiles; @Retention(RetentionPolicy.RUNTIME) @ActiveProfiles("mysql") @SpringBootTest(classes = {UpgradeDolphinScheduler.class, DaoConfiguration.class}) +@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD) public @interface DolphinSchedulerMysqlProfile { } diff --git a/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/mysql/MysqlDatabaseContainerProvider.java b/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/mysql/MysqlDatabaseContainerProvider.java index 80d0b8978d..d27a897920 100644 --- a/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/mysql/MysqlDatabaseContainerProvider.java +++ b/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/mysql/MysqlDatabaseContainerProvider.java @@ -20,15 +20,20 @@ package org.apache.dolphinscheduler.tools.datasource.mysql; import org.apache.dolphinscheduler.tools.datasource.jupiter.DatabaseContainerProvider; import org.apache.dolphinscheduler.tools.datasource.jupiter.DolphinSchedulerDatabaseContainer; +import java.util.stream.Stream; + +import lombok.extern.slf4j.Slf4j; + import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.MySQLContainer; import org.testcontainers.containers.Network; import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.lifecycle.Startables; import org.testcontainers.utility.DockerImageName; import com.google.auto.service.AutoService; -import com.google.common.collect.Lists; +@Slf4j @AutoService(DatabaseContainerProvider.class) public class MysqlDatabaseContainerProvider implements DatabaseContainerProvider { @@ -43,7 +48,19 @@ public class MysqlDatabaseContainerProvider implements DatabaseContainerProvider .withNetwork(NETWORK) .withExposedPorts(3306) .waitingFor(Wait.forHealthcheck()); - mysqlContainer.setPortBindings(Lists.newArrayList("3306:3306")); + + log.info("Create {} successfully.", mysqlContainer.getDockerImageName()); + mysqlContainer.start(); + + log.info("Starting {}...", mysqlContainer.getDockerImageName()); + Startables.deepStart(Stream.of(mysqlContainer)).join(); + log.info("{} started", mysqlContainer.getDockerImageName()); + + String jdbcUrl = "jdbc:mysql://localhost:" + mysqlContainer.getMappedPort(3306) + + "/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8"; + System.clearProperty("spring.datasource.url"); + System.setProperty("spring.datasource.url", jdbcUrl); + return mysqlContainer; } diff --git a/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/postgresql/DolphinSchedulerPostgresqlProfile.java b/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/postgresql/DolphinSchedulerPostgresqlProfile.java index 1a5a07ad93..73b02f8a76 100644 --- a/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/postgresql/DolphinSchedulerPostgresqlProfile.java +++ b/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/postgresql/DolphinSchedulerPostgresqlProfile.java @@ -27,12 +27,14 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ActiveProfiles; @Inherited @Target(ElementType.TYPE) @Retention(RetentionPolicy.RUNTIME) @ActiveProfiles("postgresql") @SpringBootTest(classes = {UpgradeDolphinScheduler.class, DaoConfiguration.class}) +@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD) public @interface DolphinSchedulerPostgresqlProfile { } diff --git a/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/postgresql/PostgresqlDatabaseContainerProvider.java b/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/postgresql/PostgresqlDatabaseContainerProvider.java index c09b7fae01..d8157d7029 100644 --- a/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/postgresql/PostgresqlDatabaseContainerProvider.java +++ b/dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/postgresql/PostgresqlDatabaseContainerProvider.java @@ -20,15 +20,17 @@ package org.apache.dolphinscheduler.tools.datasource.postgresql; import org.apache.dolphinscheduler.tools.datasource.jupiter.DatabaseContainerProvider; import org.apache.dolphinscheduler.tools.datasource.jupiter.DolphinSchedulerDatabaseContainer; +import java.util.stream.Stream; + import lombok.extern.slf4j.Slf4j; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.Network; import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.lifecycle.Startables; import org.testcontainers.utility.DockerImageName; import com.google.auto.service.AutoService; -import com.google.common.collect.Lists; @Slf4j @AutoService(DatabaseContainerProvider.class) @@ -47,8 +49,18 @@ public class PostgresqlDatabaseContainerProvider implements DatabaseContainerPro .withDatabaseName("dolphinscheduler") .withNetwork(NETWORK) .withExposedPorts(5432); - postgresqlContainer.setPortBindings(Lists.newArrayList("5432:5432")); + log.info("Create {} successfully.", postgresqlContainer.getDockerImageName()); + postgresqlContainer.start(); + + log.info("Starting {}...", postgresqlContainer.getDockerImageName()); + Startables.deepStart(Stream.of(postgresqlContainer)).join(); + log.info("{} started", postgresqlContainer.getDockerImageName()); + + String jdbcUrl = "jdbc:mysql://localhost:" + postgresqlContainer.getMappedPort(5432) + + "/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8"; + System.clearProperty("spring.datasource.url"); + System.setProperty("spring.datasource.url", jdbcUrl); return postgresqlContainer; }