github
/
DolphinScheduler
mirror of https://gitee.com/dolphinscheduler/DolphinScheduler.git
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

[CI][E2E] Try to fix e2e timeout error (#14690) 

* try to fix e2e timeout error

* remove use file
3.2.1-prepare
xiangzihao 1 year ago committed by GitHub
parent
5ec9085113
commit
dec5228d62
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 63 additions and 57 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 8
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/basic/docker-compose.yaml
  2. 14
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-clickhouse/docker-compose.yaml
  3. 35
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-hive/docker-compose.yaml
  4. 14
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml
  5. 10
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-postgresql/docker-compose.yaml
  6. 14
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-sqlserver/docker-compose.yaml
  7. 10
      dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml
  8. 15
      dolphinscheduler-e2e/dolphinscheduler-e2e-core/src/main/java/org/apache/dolphinscheduler/e2e/core/DolphinSchedulerExtension.java

8
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/basic/docker-compose.yaml
Unescape View File

@ -15,7 +15,7 @@
# limitations under the License. # limitations under the License.
# #
version: "2.1" version: "3.8"
services: services:
dolphinscheduler: dolphinscheduler:
@ -23,14 +23,14 @@ services:
environment: environment:
MASTER_MAX_CPU_LOAD_AVG: 100 MASTER_MAX_CPU_LOAD_AVG: 100
WORKER_TENANT_AUTO_CREATE: 'true' WORKER_TENANT_AUTO_CREATE: 'true'
expose: ports:
- 12345 - "12345:12345"
networks: networks:
- e2e - e2e
healthcheck: healthcheck:
test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ] test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
interval: 5s interval: 5s
timeout: 60s timeout: 5s
retries: 120 retries: 120
networks: networks:

14
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-clickhouse/docker-compose.yaml
Unescape View File

@ -15,7 +15,7 @@
# limitations under the License. # limitations under the License.
# #
version: "2.1" version: "3.8"
services: services:
dolphinscheduler: dolphinscheduler:
@ -23,14 +23,14 @@ services:
environment: environment:
MASTER_MAX_CPU_LOAD_AVG: 100 MASTER_MAX_CPU_LOAD_AVG: 100
WORKER_TENANT_AUTO_CREATE: 'true' WORKER_TENANT_AUTO_CREATE: 'true'
expose: ports:
- 12345 - "12345:12345"
networks: networks:
- e2e - e2e
healthcheck: healthcheck:
test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ] test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
interval: 5s interval: 5s
timeout: 300s timeout: 5s
retries: 120 retries: 120
depends_on: depends_on:
clickhouse: clickhouse:
@ -45,8 +45,8 @@ services:
nofile: nofile:
soft: 262144 soft: 262144
hard: 262144 hard: 262144
expose: ports:
- 8123 - "8123:8123"
mem_limit: 3G mem_limit: 3G
mem_reservation: 1G mem_reservation: 1G
networks: networks:
@ -54,7 +54,7 @@ services:
healthcheck: healthcheck:
test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"] test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"]
interval: 5s interval: 5s
timeout: 300s timeout: 5s
retries: 120 retries: 120
networks: networks:

35
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-hive/docker-compose.yaml
Unescape View File

@ -23,15 +23,16 @@ services:
environment: environment:
MASTER_MAX_CPU_LOAD_AVG: 100 MASTER_MAX_CPU_LOAD_AVG: 100
WORKER_TENANT_AUTO_CREATE: 'true' WORKER_TENANT_AUTO_CREATE: 'true'
expose: ports:
- 12345 - "12345:12345"
networks: networks:
- e2e - e2e
healthcheck: healthcheck:
test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ] test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
interval: 5s interval: 5s
timeout: 300s timeout: 5s
retries: 120 retries: 120
start_period: 60s
depends_on: depends_on:
hive-server: hive-server:
condition: service_healthy condition: service_healthy
@ -43,12 +44,12 @@ services:
- ./hadoop-hive.env - ./hadoop-hive.env
networks: networks:
- e2e - e2e
expose: ports:
- "50070" - "50070:50070"
healthcheck: healthcheck:
test: [ "CMD", "curl", "http://localhost:50070/" ] test: [ "CMD", "curl", "http://localhost:50070/" ]
interval: 5s interval: 5s
timeout: 300s timeout: 5s
retries: 120 retries: 120
datanode: datanode:
image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8 image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8
@ -58,12 +59,12 @@ services:
SERVICE_PRECONDITION: "namenode:50070" SERVICE_PRECONDITION: "namenode:50070"
networks: networks:
- e2e - e2e
expose: ports:
- "50075" - "50075:50075"
healthcheck: healthcheck:
test: [ "CMD", "curl", "http://localhost:50075" ] test: [ "CMD", "curl", "http://localhost:50075" ]
interval: 5s interval: 5s
timeout: 300s timeout: 5s
retries: 120 retries: 120
hive-server: hive-server:
image: bde2020/hive:2.3.2-postgresql-metastore image: bde2020/hive:2.3.2-postgresql-metastore
@ -74,8 +75,8 @@ services:
environment: environment:
HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://hive-metastore/metastore" HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://hive-metastore/metastore"
SERVICE_PRECONDITION: "hive-metastore:9083" SERVICE_PRECONDITION: "hive-metastore:9083"
expose: ports:
- "10000" - "10000:10000"
depends_on: depends_on:
datanode: datanode:
condition: service_healthy condition: service_healthy
@ -84,7 +85,7 @@ services:
healthcheck: healthcheck:
test: beeline -u "jdbc:hive2://127.0.0.1:10000/default" -n health_check -e "show databases;" test: beeline -u "jdbc:hive2://127.0.0.1:10000/default" -n health_check -e "show databases;"
interval: 5s interval: 5s
timeout: 300s timeout: 5s
retries: 120 retries: 120
hive-metastore: hive-metastore:
image: bde2020/hive:2.3.2-postgresql-metastore image: bde2020/hive:2.3.2-postgresql-metastore
@ -95,8 +96,8 @@ services:
- e2e - e2e
environment: environment:
SERVICE_PRECONDITION: "namenode:50070 datanode:50075 hive-metastore-postgresql:5432" SERVICE_PRECONDITION: "namenode:50070 datanode:50075 hive-metastore-postgresql:5432"
expose: ports:
- "9083" - "9083:9083"
depends_on: depends_on:
hive-metastore-postgresql: hive-metastore-postgresql:
condition: service_healthy condition: service_healthy
@ -104,12 +105,12 @@ services:
image: bde2020/hive-metastore-postgresql:2.3.0 image: bde2020/hive-metastore-postgresql:2.3.0
networks: networks:
- e2e - e2e
expose: ports:
- "5432" - "5432:5432"
healthcheck: healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres"] test: ["CMD-SHELL", "pg_isready -U postgres"]
interval: 5s interval: 5s
timeout: 300s timeout: 5s
retries: 120 retries: 120

14
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml
Unescape View File

@ -15,7 +15,7 @@
# limitations under the License. # limitations under the License.
# #
version: "2.1" version: "3.8"
services: services:
dolphinscheduler: dolphinscheduler:
@ -23,8 +23,8 @@ services:
environment: environment:
MASTER_MAX_CPU_LOAD_AVG: 100 MASTER_MAX_CPU_LOAD_AVG: 100
WORKER_TENANT_AUTO_CREATE: 'true' WORKER_TENANT_AUTO_CREATE: 'true'
expose: ports:
- 12345 - "12345:12345"
networks: networks:
- e2e - e2e
volumes: volumes:
@ -33,7 +33,7 @@ services:
healthcheck: healthcheck:
test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ] test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
interval: 5s interval: 5s
timeout: 300s timeout: 5s
retries: 120 retries: 120
depends_on: depends_on:
mysql: mysql:
@ -46,12 +46,12 @@ services:
MYSQL_ROOT_PASSWORD: 123456 MYSQL_ROOT_PASSWORD: 123456
networks: networks:
- e2e - e2e
expose: ports:
- 3306 - "3306:3306"
healthcheck: healthcheck:
test: mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD test: mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD
interval: 5s interval: 5s
timeout: 300s timeout: 5s
retries: 120 retries: 120
networks: networks:

10
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-postgresql/docker-compose.yaml
Unescape View File

@ -15,7 +15,7 @@
# limitations under the License. # limitations under the License.
# #
version: "2.1" version: "3.8"
services: services:
dolphinscheduler: dolphinscheduler:
@ -23,8 +23,8 @@ services:
environment: environment:
MASTER_MAX_CPU_LOAD_AVG: 100 MASTER_MAX_CPU_LOAD_AVG: 100
WORKER_TENANT_AUTO_CREATE: 'true' WORKER_TENANT_AUTO_CREATE: 'true'
expose: ports:
- 12345 - "12345:12345"
networks: networks:
- e2e - e2e
healthcheck: healthcheck:
@ -42,8 +42,8 @@ services:
POSTGRES_PASSWORD: postgres POSTGRES_PASSWORD: postgres
networks: networks:
- e2e - e2e
expose: ports:
- 5432 - "5432:5432"
healthcheck: healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres"] test: ["CMD-SHELL", "pg_isready -U postgres"]
interval: 5s interval: 5s

14
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-sqlserver/docker-compose.yaml
Unescape View File

@ -15,7 +15,7 @@
# limitations under the License. # limitations under the License.
# #
version: "2.1" version: "3.8"
services: services:
dolphinscheduler: dolphinscheduler:
@ -23,28 +23,28 @@ services:
environment: environment:
MASTER_MAX_CPU_LOAD_AVG: 100 MASTER_MAX_CPU_LOAD_AVG: 100
WORKER_TENANT_AUTO_CREATE: 'true' WORKER_TENANT_AUTO_CREATE: 'true'
expose: ports:
- 12345 - "12345:12345"
networks: networks:
- e2e - e2e
healthcheck: healthcheck:
test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ] test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
interval: 5s interval: 5s
timeout: 300s timeout: 5s
retries: 120 retries: 120
depends_on: depends_on:
sqlserver: sqlserver:
condition: service_healthy condition: service_healthy
sqlserver: sqlserver:
image: alexk002/sqlserver2019_demo:1 image: alexk002/sqlserver2019_demo:1
expose: ports:
- 1433 - "1433:1433"
networks: networks:
- e2e - e2e
healthcheck: healthcheck:
test: /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P OcP2020123 -Q "SELECT 1" test: /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P OcP2020123 -Q "SELECT 1"
interval: 5s interval: 5s
timeout: 300s timeout: 5s
retries: 120 retries: 120
networks: networks:

10
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml
Unescape View File

@ -23,14 +23,14 @@ services:
environment: environment:
MASTER_MAX_CPU_LOAD_AVG: 100 MASTER_MAX_CPU_LOAD_AVG: 100
WORKER_TENANT_AUTO_CREATE: 'true' WORKER_TENANT_AUTO_CREATE: 'true'
expose: ports:
- 12345 - "12345:12345"
networks: networks:
- e2e - e2e
healthcheck: healthcheck:
test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ] test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
interval: 5s interval: 5s
timeout: 300s timeout: 5s
retries: 120 retries: 120
volumes: volumes:
- ./common.properties:/opt/dolphinscheduler/conf/common.properties - ./common.properties:/opt/dolphinscheduler/conf/common.properties
@ -46,7 +46,7 @@ services:
stdin_open: true stdin_open: true
command: server /data --console-address ":9001" command: server /data --console-address ":9001"
ports: ports:
- 9000:9000 - "9000:9000"
networks: networks:
- e2e - e2e
environment: environment:
@ -55,7 +55,7 @@ services:
healthcheck: healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
interval: 5s interval: 5s
timeout: 300s timeout: 5s
retries: 120 retries: 120
mc: mc:
image: minio/mc:RELEASE.2022-01-07T06-01-38Z image: minio/mc:RELEASE.2022-01-07T06-01-38Z

15
dolphinscheduler-e2e/dolphinscheduler-e2e-core/src/main/java/org/apache/dolphinscheduler/e2e/core/DolphinSchedulerExtension.java
Unescape View File

@ -64,6 +64,8 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
private final int LOCAL_PORT = 5173; private final int LOCAL_PORT = 5173;
private final int DOCKER_PORT = 12345;
private RemoteWebDriver driver; private RemoteWebDriver driver;
private DockerComposeContainer<?> compose; private DockerComposeContainer<?> compose;
private BrowserWebDriverContainer<?> browser; private BrowserWebDriverContainer<?> browser;
@ -72,6 +74,8 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
private Path record; private Path record;
private final String serviceName = "dolphinscheduler_1";
@Override @Override
@SuppressWarnings("UnstableApiUsage") @SuppressWarnings("UnstableApiUsage")
public void beforeAll(ExtensionContext context) throws IOException { public void beforeAll(ExtensionContext context) throws IOException {
@ -89,7 +93,7 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
setBrowserContainerByOsName(); setBrowserContainerByOsName();
if (compose != null) { if (compose != null) {
Testcontainers.exposeHostPorts(compose.getServicePort("dolphinscheduler_1", 12345)); Testcontainers.exposeHostPorts(compose.getServicePort(serviceName, DOCKER_PORT));
browser.withAccessToHost(true); browser.withAccessToHost(true);
} }
browser.start(); browser.start();
@ -123,7 +127,7 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
compose = createDockerCompose(context); compose = createDockerCompose(context);
compose.start(); compose.start();
address = HostAndPort.fromParts("host.testcontainers.internal", compose.getServicePort("dolphinscheduler_1", 12345)); address = HostAndPort.fromParts("host.testcontainers.internal", compose.getServicePort(serviceName, DOCKER_PORT));
rootPath = "/dolphinscheduler/ui/"; rootPath = "/dolphinscheduler/ui/";
} }
@ -202,9 +206,10 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
compose = new DockerComposeContainer<>(files) compose = new DockerComposeContainer<>(files)
.withPull(true) .withPull(true)
.withTailChildContainers(true) .withTailChildContainers(true)
.withExposedService("dolphinscheduler_1", 12345) .withLocalCompose(true)
.withLogConsumer("dolphinscheduler_1", outputFrame -> LOGGER.info(outputFrame.getUtf8String())) .withExposedService(serviceName, DOCKER_PORT, Wait.forListeningPort().withStartupTimeout(Duration.ofSeconds(300)))
.waitingFor("dolphinscheduler_1", Wait.forHealthcheck().withStartupTimeout(Duration.ofSeconds(300))); .withLogConsumer(serviceName, outputFrame -> LOGGER.info(outputFrame.getUtf8String()))
.waitingFor(serviceName, Wait.forHealthcheck().withStartupTimeout(Duration.ofSeconds(300)));
return compose; return compose;
} }

Write Preview
Loading…
Cancel
Save