Browse Source

[CI][E2E] Try to fix e2e timeout error (#14690)

* try to fix e2e timeout error

* remove use file
3.2.1-prepare
xiangzihao 1 year ago committed by GitHub
parent
commit
dec5228d62
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 8
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/basic/docker-compose.yaml
  2. 14
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-clickhouse/docker-compose.yaml
  3. 35
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-hive/docker-compose.yaml
  4. 14
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml
  5. 10
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-postgresql/docker-compose.yaml
  6. 14
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-sqlserver/docker-compose.yaml
  7. 10
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml
  8. 15
      dolphinscheduler-e2e/dolphinscheduler-e2e-core/src/main/java/org/apache/dolphinscheduler/e2e/core/DolphinSchedulerExtension.java

8
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/basic/docker-compose.yaml

@ -15,7 +15,7 @@
# limitations under the License. # limitations under the License.
# #
version: "2.1" version: "3.8"
services: services:
dolphinscheduler: dolphinscheduler:
@ -23,14 +23,14 @@ services:
environment: environment:
MASTER_MAX_CPU_LOAD_AVG: 100 MASTER_MAX_CPU_LOAD_AVG: 100
WORKER_TENANT_AUTO_CREATE: 'true' WORKER_TENANT_AUTO_CREATE: 'true'
expose: ports:
- 12345 - "12345:12345"
networks: networks:
- e2e - e2e
healthcheck: healthcheck:
test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ] test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
interval: 5s interval: 5s
timeout: 60s timeout: 5s
retries: 120 retries: 120
networks: networks:

14
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-clickhouse/docker-compose.yaml

@ -15,7 +15,7 @@
# limitations under the License. # limitations under the License.
# #
version: "2.1" version: "3.8"
services: services:
dolphinscheduler: dolphinscheduler:
@ -23,14 +23,14 @@ services:
environment: environment:
MASTER_MAX_CPU_LOAD_AVG: 100 MASTER_MAX_CPU_LOAD_AVG: 100
WORKER_TENANT_AUTO_CREATE: 'true' WORKER_TENANT_AUTO_CREATE: 'true'
expose: ports:
- 12345 - "12345:12345"
networks: networks:
- e2e - e2e
healthcheck: healthcheck:
test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ] test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
interval: 5s interval: 5s
timeout: 300s timeout: 5s
retries: 120 retries: 120
depends_on: depends_on:
clickhouse: clickhouse:
@ -45,8 +45,8 @@ services:
nofile: nofile:
soft: 262144 soft: 262144
hard: 262144 hard: 262144
expose: ports:
- 8123 - "8123:8123"
mem_limit: 3G mem_limit: 3G
mem_reservation: 1G mem_reservation: 1G
networks: networks:
@ -54,7 +54,7 @@ services:
healthcheck: healthcheck:
test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"] test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"]
interval: 5s interval: 5s
timeout: 300s timeout: 5s
retries: 120 retries: 120
networks: networks:

35
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-hive/docker-compose.yaml

@ -23,15 +23,16 @@ services:
environment: environment:
MASTER_MAX_CPU_LOAD_AVG: 100 MASTER_MAX_CPU_LOAD_AVG: 100
WORKER_TENANT_AUTO_CREATE: 'true' WORKER_TENANT_AUTO_CREATE: 'true'
expose: ports:
- 12345 - "12345:12345"
networks: networks:
- e2e - e2e
healthcheck: healthcheck:
test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ] test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
interval: 5s interval: 5s
timeout: 300s timeout: 5s
retries: 120 retries: 120
start_period: 60s
depends_on: depends_on:
hive-server: hive-server:
condition: service_healthy condition: service_healthy
@ -43,12 +44,12 @@ services:
- ./hadoop-hive.env - ./hadoop-hive.env
networks: networks:
- e2e - e2e
expose: ports:
- "50070" - "50070:50070"
healthcheck: healthcheck:
test: [ "CMD", "curl", "http://localhost:50070/" ] test: [ "CMD", "curl", "http://localhost:50070/" ]
interval: 5s interval: 5s
timeout: 300s timeout: 5s
retries: 120 retries: 120
datanode: datanode:
image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8 image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8
@ -58,12 +59,12 @@ services:
SERVICE_PRECONDITION: "namenode:50070" SERVICE_PRECONDITION: "namenode:50070"
networks: networks:
- e2e - e2e
expose: ports:
- "50075" - "50075:50075"
healthcheck: healthcheck:
test: [ "CMD", "curl", "http://localhost:50075" ] test: [ "CMD", "curl", "http://localhost:50075" ]
interval: 5s interval: 5s
timeout: 300s timeout: 5s
retries: 120 retries: 120
hive-server: hive-server:
image: bde2020/hive:2.3.2-postgresql-metastore image: bde2020/hive:2.3.2-postgresql-metastore
@ -74,8 +75,8 @@ services:
environment: environment:
HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://hive-metastore/metastore" HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://hive-metastore/metastore"
SERVICE_PRECONDITION: "hive-metastore:9083" SERVICE_PRECONDITION: "hive-metastore:9083"
expose: ports:
- "10000" - "10000:10000"
depends_on: depends_on:
datanode: datanode:
condition: service_healthy condition: service_healthy
@ -84,7 +85,7 @@ services:
healthcheck: healthcheck:
test: beeline -u "jdbc:hive2://127.0.0.1:10000/default" -n health_check -e "show databases;" test: beeline -u "jdbc:hive2://127.0.0.1:10000/default" -n health_check -e "show databases;"
interval: 5s interval: 5s
timeout: 300s timeout: 5s
retries: 120 retries: 120
hive-metastore: hive-metastore:
image: bde2020/hive:2.3.2-postgresql-metastore image: bde2020/hive:2.3.2-postgresql-metastore
@ -95,8 +96,8 @@ services:
- e2e - e2e
environment: environment:
SERVICE_PRECONDITION: "namenode:50070 datanode:50075 hive-metastore-postgresql:5432" SERVICE_PRECONDITION: "namenode:50070 datanode:50075 hive-metastore-postgresql:5432"
expose: ports:
- "9083" - "9083:9083"
depends_on: depends_on:
hive-metastore-postgresql: hive-metastore-postgresql:
condition: service_healthy condition: service_healthy
@ -104,12 +105,12 @@ services:
image: bde2020/hive-metastore-postgresql:2.3.0 image: bde2020/hive-metastore-postgresql:2.3.0
networks: networks:
- e2e - e2e
expose: ports:
- "5432" - "5432:5432"
healthcheck: healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres"] test: ["CMD-SHELL", "pg_isready -U postgres"]
interval: 5s interval: 5s
timeout: 300s timeout: 5s
retries: 120 retries: 120

14
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml

@ -15,7 +15,7 @@
# limitations under the License. # limitations under the License.
# #
version: "2.1" version: "3.8"
services: services:
dolphinscheduler: dolphinscheduler:
@ -23,8 +23,8 @@ services:
environment: environment:
MASTER_MAX_CPU_LOAD_AVG: 100 MASTER_MAX_CPU_LOAD_AVG: 100
WORKER_TENANT_AUTO_CREATE: 'true' WORKER_TENANT_AUTO_CREATE: 'true'
expose: ports:
- 12345 - "12345:12345"
networks: networks:
- e2e - e2e
volumes: volumes:
@ -33,7 +33,7 @@ services:
healthcheck: healthcheck:
test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ] test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
interval: 5s interval: 5s
timeout: 300s timeout: 5s
retries: 120 retries: 120
depends_on: depends_on:
mysql: mysql:
@ -46,12 +46,12 @@ services:
MYSQL_ROOT_PASSWORD: 123456 MYSQL_ROOT_PASSWORD: 123456
networks: networks:
- e2e - e2e
expose: ports:
- 3306 - "3306:3306"
healthcheck: healthcheck:
test: mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD test: mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD
interval: 5s interval: 5s
timeout: 300s timeout: 5s
retries: 120 retries: 120
networks: networks:

10
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-postgresql/docker-compose.yaml

@ -15,7 +15,7 @@
# limitations under the License. # limitations under the License.
# #
version: "2.1" version: "3.8"
services: services:
dolphinscheduler: dolphinscheduler:
@ -23,8 +23,8 @@ services:
environment: environment:
MASTER_MAX_CPU_LOAD_AVG: 100 MASTER_MAX_CPU_LOAD_AVG: 100
WORKER_TENANT_AUTO_CREATE: 'true' WORKER_TENANT_AUTO_CREATE: 'true'
expose: ports:
- 12345 - "12345:12345"
networks: networks:
- e2e - e2e
healthcheck: healthcheck:
@ -42,8 +42,8 @@ services:
POSTGRES_PASSWORD: postgres POSTGRES_PASSWORD: postgres
networks: networks:
- e2e - e2e
expose: ports:
- 5432 - "5432:5432"
healthcheck: healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres"] test: ["CMD-SHELL", "pg_isready -U postgres"]
interval: 5s interval: 5s

14
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-sqlserver/docker-compose.yaml

@ -15,7 +15,7 @@
# limitations under the License. # limitations under the License.
# #
version: "2.1" version: "3.8"
services: services:
dolphinscheduler: dolphinscheduler:
@ -23,28 +23,28 @@ services:
environment: environment:
MASTER_MAX_CPU_LOAD_AVG: 100 MASTER_MAX_CPU_LOAD_AVG: 100
WORKER_TENANT_AUTO_CREATE: 'true' WORKER_TENANT_AUTO_CREATE: 'true'
expose: ports:
- 12345 - "12345:12345"
networks: networks:
- e2e - e2e
healthcheck: healthcheck:
test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ] test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
interval: 5s interval: 5s
timeout: 300s timeout: 5s
retries: 120 retries: 120
depends_on: depends_on:
sqlserver: sqlserver:
condition: service_healthy condition: service_healthy
sqlserver: sqlserver:
image: alexk002/sqlserver2019_demo:1 image: alexk002/sqlserver2019_demo:1
expose: ports:
- 1433 - "1433:1433"
networks: networks:
- e2e - e2e
healthcheck: healthcheck:
test: /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P OcP2020123 -Q "SELECT 1" test: /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P OcP2020123 -Q "SELECT 1"
interval: 5s interval: 5s
timeout: 300s timeout: 5s
retries: 120 retries: 120
networks: networks:

10
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml

@ -23,14 +23,14 @@ services:
environment: environment:
MASTER_MAX_CPU_LOAD_AVG: 100 MASTER_MAX_CPU_LOAD_AVG: 100
WORKER_TENANT_AUTO_CREATE: 'true' WORKER_TENANT_AUTO_CREATE: 'true'
expose: ports:
- 12345 - "12345:12345"
networks: networks:
- e2e - e2e
healthcheck: healthcheck:
test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ] test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
interval: 5s interval: 5s
timeout: 300s timeout: 5s
retries: 120 retries: 120
volumes: volumes:
- ./common.properties:/opt/dolphinscheduler/conf/common.properties - ./common.properties:/opt/dolphinscheduler/conf/common.properties
@ -46,7 +46,7 @@ services:
stdin_open: true stdin_open: true
command: server /data --console-address ":9001" command: server /data --console-address ":9001"
ports: ports:
- 9000:9000 - "9000:9000"
networks: networks:
- e2e - e2e
environment: environment:
@ -55,7 +55,7 @@ services:
healthcheck: healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
interval: 5s interval: 5s
timeout: 300s timeout: 5s
retries: 120 retries: 120
mc: mc:
image: minio/mc:RELEASE.2022-01-07T06-01-38Z image: minio/mc:RELEASE.2022-01-07T06-01-38Z

15
dolphinscheduler-e2e/dolphinscheduler-e2e-core/src/main/java/org/apache/dolphinscheduler/e2e/core/DolphinSchedulerExtension.java

@ -64,6 +64,8 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
private final int LOCAL_PORT = 5173; private final int LOCAL_PORT = 5173;
private final int DOCKER_PORT = 12345;
private RemoteWebDriver driver; private RemoteWebDriver driver;
private DockerComposeContainer<?> compose; private DockerComposeContainer<?> compose;
private BrowserWebDriverContainer<?> browser; private BrowserWebDriverContainer<?> browser;
@ -72,6 +74,8 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
private Path record; private Path record;
private final String serviceName = "dolphinscheduler_1";
@Override @Override
@SuppressWarnings("UnstableApiUsage") @SuppressWarnings("UnstableApiUsage")
public void beforeAll(ExtensionContext context) throws IOException { public void beforeAll(ExtensionContext context) throws IOException {
@ -89,7 +93,7 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
setBrowserContainerByOsName(); setBrowserContainerByOsName();
if (compose != null) { if (compose != null) {
Testcontainers.exposeHostPorts(compose.getServicePort("dolphinscheduler_1", 12345)); Testcontainers.exposeHostPorts(compose.getServicePort(serviceName, DOCKER_PORT));
browser.withAccessToHost(true); browser.withAccessToHost(true);
} }
browser.start(); browser.start();
@ -123,7 +127,7 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
compose = createDockerCompose(context); compose = createDockerCompose(context);
compose.start(); compose.start();
address = HostAndPort.fromParts("host.testcontainers.internal", compose.getServicePort("dolphinscheduler_1", 12345)); address = HostAndPort.fromParts("host.testcontainers.internal", compose.getServicePort(serviceName, DOCKER_PORT));
rootPath = "/dolphinscheduler/ui/"; rootPath = "/dolphinscheduler/ui/";
} }
@ -202,9 +206,10 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
compose = new DockerComposeContainer<>(files) compose = new DockerComposeContainer<>(files)
.withPull(true) .withPull(true)
.withTailChildContainers(true) .withTailChildContainers(true)
.withExposedService("dolphinscheduler_1", 12345) .withLocalCompose(true)
.withLogConsumer("dolphinscheduler_1", outputFrame -> LOGGER.info(outputFrame.getUtf8String())) .withExposedService(serviceName, DOCKER_PORT, Wait.forListeningPort().withStartupTimeout(Duration.ofSeconds(300)))
.waitingFor("dolphinscheduler_1", Wait.forHealthcheck().withStartupTimeout(Duration.ofSeconds(300))); .withLogConsumer(serviceName, outputFrame -> LOGGER.info(outputFrame.getUtf8String()))
.waitingFor(serviceName, Wait.forHealthcheck().withStartupTimeout(Duration.ofSeconds(300)));
return compose; return compose;
} }

Loading…
Cancel
Save