[CI][E2E] Try to fix e2e timeout error (#14690)

* try to fix e2e timeout error

* remove use file

(cherry picked from commit dec5228d62)

dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/basic/docker-compose.yaml

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-version: "2.1"
+version: "3.8"
 
 services:
   dolphinscheduler:
@@ -23,14 +23,14 @@ services:
     environment:
       MASTER_MAX_CPU_LOAD_AVG: 100
       WORKER_TENANT_AUTO_CREATE: 'true'
-    expose:
-      - 12345
+    ports:
+      - "12345:12345"
     networks:
       - e2e
     healthcheck:
       test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
       interval: 5s
-      timeout: 60s
+      timeout: 5s
       retries: 120
 
 networks:

dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-clickhouse/docker-compose.yaml

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-version: "2.1"
+version: "3.8"
 
 services:
   dolphinscheduler:
@@ -23,14 +23,14 @@ services:
     environment:
       MASTER_MAX_CPU_LOAD_AVG: 100
       WORKER_TENANT_AUTO_CREATE: 'true'
-    expose:
-      - 12345
+    ports:
+      - "12345:12345"
     networks:
       - e2e
     healthcheck:
       test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
       interval: 5s
-      timeout: 300s
+      timeout: 5s
       retries: 120
     depends_on:
       clickhouse:
@@ -45,8 +45,8 @@ services:
       nofile:
         soft: 262144
         hard: 262144
-    expose:
-      - 8123
+    ports:
+      - "8123:8123"
     mem_limit: 3G
     mem_reservation: 1G
     networks:
@@ -54,7 +54,7 @@ services:
     healthcheck:
       test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"]
       interval: 5s
-      timeout: 300s
+      timeout: 5s
       retries: 120
 
 networks:

dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-hive/docker-compose.yaml

@@ -23,15 +23,16 @@ services:
     environment:
       MASTER_MAX_CPU_LOAD_AVG: 100
       WORKER_TENANT_AUTO_CREATE: 'true'
-    expose:
-      - 12345
+    ports:
+      - "12345:12345"
     networks:
       - e2e
     healthcheck:
       test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
       interval: 5s
-      timeout: 300s
+      timeout: 5s
       retries: 120
+      start_period: 60s
     depends_on:
       hive-server:
         condition: service_healthy
@@ -43,12 +44,12 @@ services:
       - ./hadoop-hive.env
     networks:
       - e2e
-    expose:
-      - "50070"
+    ports:
+      - "50070:50070"
     healthcheck:
       test: [ "CMD", "curl", "http://localhost:50070/" ]
       interval: 5s
-      timeout: 300s
+      timeout: 5s
       retries: 120
   datanode:
     image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8
@@ -58,12 +59,12 @@ services:
       SERVICE_PRECONDITION: "namenode:50070"
     networks:
       - e2e
-    expose:
-      - "50075"
+    ports:
+      - "50075:50075"
     healthcheck:
       test: [ "CMD", "curl", "http://localhost:50075" ]
       interval: 5s
-      timeout: 300s
+      timeout: 5s
       retries: 120
   hive-server:
     image: bde2020/hive:2.3.2-postgresql-metastore
@@ -74,8 +75,8 @@ services:
     environment:
       HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://hive-metastore/metastore"
       SERVICE_PRECONDITION: "hive-metastore:9083"
-    expose:
-      - "10000"
+    ports:
+      - "10000:10000"
     depends_on:
       datanode:
         condition: service_healthy
@@ -84,7 +85,7 @@ services:
     healthcheck:
       test: beeline -u "jdbc:hive2://127.0.0.1:10000/default" -n health_check -e "show databases;"
       interval: 5s
-      timeout: 300s
+      timeout: 5s
       retries: 120
   hive-metastore:
     image: bde2020/hive:2.3.2-postgresql-metastore
@@ -95,8 +96,8 @@ services:
       - e2e
     environment:
       SERVICE_PRECONDITION: "namenode:50070 datanode:50075 hive-metastore-postgresql:5432"
-    expose:
-      - "9083"
+    ports:
+      - "9083:9083"
     depends_on:
       hive-metastore-postgresql:
         condition: service_healthy
@@ -104,12 +105,12 @@ services:
     image: bde2020/hive-metastore-postgresql:2.3.0
     networks:
       - e2e
-    expose:
-      - "5432"
+    ports:
+      - "5432:5432"
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
       interval: 5s
-      timeout: 300s
+      timeout: 5s
       retries: 120
 
 

dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-version: "2.1"
+version: "3.8"
 
 services:
   dolphinscheduler:
@@ -23,8 +23,8 @@ services:
     environment:
       MASTER_MAX_CPU_LOAD_AVG: 100
       WORKER_TENANT_AUTO_CREATE: 'true'
-    expose:
-      - 12345
+    ports:
+      - "12345:12345"
     networks:
       - e2e
     volumes:
@@ -33,7 +33,7 @@ services:
     healthcheck:
       test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
       interval: 5s
-      timeout: 300s
+      timeout: 5s
       retries: 120
     depends_on:
       mysql:
@@ -46,12 +46,12 @@ services:
       MYSQL_ROOT_PASSWORD: 123456
     networks:
       - e2e
-    expose:
-      - 3306
+    ports:
+      - "3306:3306"
     healthcheck:
       test: mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD
       interval: 5s
-      timeout: 300s
+      timeout: 5s
       retries: 120
 
 networks:

dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-postgresql/docker-compose.yaml

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-version: "2.1"
+version: "3.8"
 
 services:
   dolphinscheduler:
@@ -23,8 +23,8 @@ services:
     environment:
       MASTER_MAX_CPU_LOAD_AVG: 100
       WORKER_TENANT_AUTO_CREATE: 'true'
-    expose:
-      - 12345
+    ports:
+      - "12345:12345"
     networks:
       - e2e
     healthcheck:
@@ -42,8 +42,8 @@ services:
       POSTGRES_PASSWORD: postgres
     networks:
       - e2e
-    expose:
-      - 5432
+    ports:
+      - "5432:5432"
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
       interval: 5s

dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-sqlserver/docker-compose.yaml

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-version: "2.1"
+version: "3.8"
 
 services:
   dolphinscheduler:
@@ -23,28 +23,28 @@ services:
     environment:
       MASTER_MAX_CPU_LOAD_AVG: 100
       WORKER_TENANT_AUTO_CREATE: 'true'
-    expose:
-      - 12345
+    ports:
+      - "12345:12345"
     networks:
       - e2e
     healthcheck:
       test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
       interval: 5s
-      timeout: 300s
+      timeout: 5s
       retries: 120
     depends_on:
       sqlserver:
         condition: service_healthy
   sqlserver:
     image: alexk002/sqlserver2019_demo:1
-    expose:
-      - 1433
+    ports:
+      - "1433:1433"
     networks:
       - e2e
     healthcheck:
       test: /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P OcP2020123 -Q "SELECT 1"
       interval: 5s
-      timeout: 300s
+      timeout: 5s
       retries: 120
 
 networks:

dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml

@@ -23,14 +23,14 @@ services:
     environment:
       MASTER_MAX_CPU_LOAD_AVG: 100
       WORKER_TENANT_AUTO_CREATE: 'true'
-    expose:
-      - 12345
+    ports:
+      - "12345:12345"
     networks:
       - e2e
     healthcheck:
       test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
       interval: 5s
-      timeout: 300s
+      timeout: 5s
       retries: 120
     volumes:
       - ./common.properties:/opt/dolphinscheduler/conf/common.properties
@@ -46,7 +46,7 @@ services:
     stdin_open: true
     command: server /data --console-address ":9001"
     ports:
-      - 9000:9000
+      - "9000:9000"
     networks:
       - e2e
     environment:
@@ -55,7 +55,7 @@ services:
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
       interval: 5s
-      timeout: 300s
+      timeout: 5s
       retries: 120
   mc:
     image: minio/mc:RELEASE.2022-01-07T06-01-38Z

dolphinscheduler-e2e/dolphinscheduler-e2e-core/src/main/java/org/apache/dolphinscheduler/e2e/core/DolphinSchedulerExtension.java

@@ -64,6 +64,8 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
 
     private final int LOCAL_PORT = 5173;
 
+    private final int DOCKER_PORT = 12345;
+
     private RemoteWebDriver driver;
     private DockerComposeContainer<?> compose;
     private BrowserWebDriverContainer<?> browser;
@@ -72,6 +74,8 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
 
     private Path record;
 
+    private final String serviceName = "dolphinscheduler_1";
+
     @Override
     @SuppressWarnings("UnstableApiUsage")
     public void beforeAll(ExtensionContext context) throws IOException {
@@ -89,7 +93,7 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
         setBrowserContainerByOsName();
 
         if (compose != null) {
-            Testcontainers.exposeHostPorts(compose.getServicePort("dolphinscheduler_1", 12345));
+            Testcontainers.exposeHostPorts(compose.getServicePort(serviceName, DOCKER_PORT));
             browser.withAccessToHost(true);
         }
         browser.start();
@@ -123,7 +127,7 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
         compose = createDockerCompose(context);
         compose.start();
 
-        address = HostAndPort.fromParts("host.testcontainers.internal", compose.getServicePort("dolphinscheduler_1", 12345));
+        address = HostAndPort.fromParts("host.testcontainers.internal", compose.getServicePort(serviceName, DOCKER_PORT));
         rootPath = "/dolphinscheduler/ui/";
     }
 
@@ -202,9 +206,10 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
         compose = new DockerComposeContainer<>(files)
             .withPull(true)
             .withTailChildContainers(true)
-            .withExposedService("dolphinscheduler_1", 12345)
-            .withLogConsumer("dolphinscheduler_1", outputFrame -> LOGGER.info(outputFrame.getUtf8String()))
-            .waitingFor("dolphinscheduler_1", Wait.forHealthcheck().withStartupTimeout(Duration.ofSeconds(300)));
+            .withLocalCompose(true)
+            .withExposedService(serviceName, DOCKER_PORT, Wait.forListeningPort().withStartupTimeout(Duration.ofSeconds(300)))
+            .withLogConsumer(serviceName, outputFrame -> LOGGER.info(outputFrame.getUtf8String()))
+            .waitingFor(serviceName, Wait.forHealthcheck().withStartupTimeout(Duration.ofSeconds(300)));
 
         return compose;
     }