From e50739465dde8e424e15282f1c5ffaf4b910b759 Mon Sep 17 00:00:00 2001 From: xiangzihao <460888207@qq.com> Date: Wed, 15 Jun 2022 16:29:36 +0800 Subject: [PATCH] [Feature-10411] Add tenant api test (#10442) * add feature_10411 * add feature_10411 * update README.md * fix README.md deadlink * fix error log output * fix comment --- .github/workflows/api-test.yml | 139 +++++++ dolphinscheduler-api-test/README.md | 44 ++ .../dolphinscheduler-api-test-case/pom.xml | 40 ++ .../api.test/cases/TenantAPITest.java | 113 ++++++ .../api.test/entity/HttpResponse.java | 34 ++ .../api.test/entity/HttpResponseBody.java | 40 ++ .../api.test/entity/LoginResponseData.java | 32 ++ .../entity/TenantListPagingResponseData.java | 43 ++ .../TenantListPagingResponseTotalList.java | 47 +++ .../api.test/pages/LoginPage.java | 40 ++ .../api.test/pages/security/TenantPage.java | 67 +++ .../api.test/utils/JSONUtils.java | 384 ++++++++++++++++++ .../api.test/utils/RequestClient.java | 171 ++++++++ .../docker/basic/docker-compose.yaml | 37 ++ .../datasource-clickhouse/docker-compose.yaml | 61 +++ .../datasource-hive/docker-compose.yaml | 117 ++++++ .../docker/datasource-hive/hadoop-hive.env | 50 +++ .../datasource-mysql/docker-compose.yaml | 56 +++ .../docker/datasource-mysql/download-mysql.sh | 27 ++ .../datasource-postgresql/docker-compose.yaml | 54 +++ .../datasource-sqlserver/docker-compose.yaml | 51 +++ .../docker/file-manage/common.properties | 90 ++++ .../docker/file-manage/docker-compose.yaml | 78 ++++ .../dolphinscheduler-api-test-core/pom.xml | 32 ++ .../api/test/core/Constants.java | 58 +++ .../api/test/core/DolphinScheduler.java | 41 ++ .../test/core/DolphinSchedulerExtension.java | 80 ++++ .../src/main/resources/log4j2.xml | 31 ++ dolphinscheduler-api-test/lombok.config | 21 + dolphinscheduler-api-test/pom.xml | 142 +++++++ 30 files changed, 2220 insertions(+) create mode 100644 .github/workflows/api-test.yml create mode 100644 dolphinscheduler-api-test/README.md create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-case/pom.xml create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/cases/TenantAPITest.java create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/entity/HttpResponse.java create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/entity/HttpResponseBody.java create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/entity/LoginResponseData.java create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/entity/TenantListPagingResponseData.java create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/entity/TenantListPagingResponseTotalList.java create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/pages/LoginPage.java create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/pages/security/TenantPage.java create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/utils/JSONUtils.java create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/utils/RequestClient.java create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/basic/docker-compose.yaml create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-clickhouse/docker-compose.yaml create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-hive/docker-compose.yaml create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-hive/hadoop-hive.env create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-mysql/download-mysql.sh create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-postgresql/docker-compose.yaml create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-sqlserver/docker-compose.yaml create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/common.properties create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/docker-compose.yaml create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-core/pom.xml create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-core/src/main/java/org/apache/dolphinscheduler/api/test/core/Constants.java create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-core/src/main/java/org/apache/dolphinscheduler/api/test/core/DolphinScheduler.java create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-core/src/main/java/org/apache/dolphinscheduler/api/test/core/DolphinSchedulerExtension.java create mode 100644 dolphinscheduler-api-test/dolphinscheduler-api-test-core/src/main/resources/log4j2.xml create mode 100644 dolphinscheduler-api-test/lombok.config create mode 100644 dolphinscheduler-api-test/pom.xml diff --git a/.github/workflows/api-test.yml b/.github/workflows/api-test.yml new file mode 100644 index 0000000000..3a60e68bb1 --- /dev/null +++ b/.github/workflows/api-test.yml @@ -0,0 +1,139 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +on: + pull_request: + push: + branches: + - dev + +name: API-Test + +concurrency: + group: api-test-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + + +jobs: + paths-filter: + name: API-Test-Path-Filter + runs-on: ubuntu-latest + outputs: + not-ignore: ${{ steps.filter.outputs.not-ignore }} + steps: + - uses: actions/checkout@v2 + - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721 + id: filter + with: + filters: | + not-ignore: + - '!(docs/**)' + build: + name: API-Test-Build + needs: paths-filter + if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Sanity Check + uses: ./.github/actions/sanity-check + - name: Cache local Maven repository + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-maven- + - name: Build Image + run: | + ./mvnw -B clean install \ + -Dmaven.test.skip \ + -Dmaven.javadoc.skip \ + -Dmaven.checkstyle.skip \ + -Pdocker,release -Ddocker.tag=ci \ + -pl dolphinscheduler-standalone-server -am + - name: Export Docker Images + run: | + docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \ + && du -sh /tmp/standalone-image.tar + - uses: actions/upload-artifact@v2 + name: Upload Docker Images + with: + name: standalone-image + path: /tmp/standalone-image.tar + retention-days: 1 + api-test: + name: ${{ matrix.case.name }} + needs: build + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + matrix: + case: + - name: Tenant + class: org.apache.dolphinscheduler.api.test.cases.TenantAPITest + env: + RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Cache local Maven repository + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-maven- + - uses: actions/download-artifact@v2 + name: Download Docker Images + with: + name: standalone-image + path: /tmp + - name: Load Docker Images + run: | + docker load -i /tmp/standalone-image.tar + - name: Run Test + run: | + ./mvnw -B -f dolphinscheduler-api-test/pom.xml -am \ + -DfailIfNoTests=false \ + -Dtest=${{ matrix.case.class }} test + - uses: actions/upload-artifact@v2 + if: always() + name: Upload Recording + with: + name: recording-${{ matrix.case.name }} + path: ${{ env.RECORDING_PATH }} + retention-days: 1 + result: + name: API-Test-Result + runs-on: ubuntu-latest + timeout-minutes: 30 + needs: [ api-test, paths-filter ] + if: always() + steps: + - name: Status + run: | + if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then + echo "Skip API Test!" + exit 0 + fi + if [[ ${{ needs.api-test.result }} != 'success' ]]; then + echo "API test Failed!" + exit -1 + fi diff --git a/dolphinscheduler-api-test/README.md b/dolphinscheduler-api-test/README.md new file mode 100644 index 0000000000..6036566c3b --- /dev/null +++ b/dolphinscheduler-api-test/README.md @@ -0,0 +1,44 @@ +# DolphinScheduler Backend API Test + +## Page Object Model + +DolphinScheduler API test respects +the [Page Object Model (POM)](https://www.selenium.dev/documentation/guidelines/page_object_models/) design pattern. +Every page of DolphinScheduler's api is abstracted into a class for better maintainability. + +### Example + +The login page's api is abstracted +as [`LoginPage`](dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/pages/LoginPage.java), with the +following fields, + +```java +public HttpResponse login(String username, String password) { + Map params = new HashMap<>(); + + params.put("userName", username); + params.put("userPassword", password); + + RequestClient requestClient = new RequestClient(); + + return requestClient.post("/login", null, params); +} +``` + +where `userName`, `userPassword` are the main elements on UI that we are interested in. + +## Test Environment Setup + +DolphinScheduler API test uses [testcontainers](https://www.testcontainers.org) to set up the testing +environment, with docker compose. + +Typically, every test case needs one or more `docker-compose.yaml` files to set up all needed components, and expose the +DolphinScheduler UI port for testing. You can use `@DolphinScheduler(composeFiles = "")` and pass +the `docker-compose.yaml` files to automatically set up the environment in the test class. + +```java + +@DolphinScheduler(composeFiles = "docker/tenant/docker-compose.yaml") +class TenantAPITest { +} +``` diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/pom.xml b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/pom.xml new file mode 100644 index 0000000000..cf10add5ab --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/pom.xml @@ -0,0 +1,40 @@ + + + + + + dolphinscheduler-api-test + org.apache.dolphinscheduler + 1.0-SNAPSHOT + + 4.0.0 + + dolphinscheduler-api-test-case + + + + org.apache.dolphinscheduler + dolphinscheduler-api-test-core + ${project.version} + + + diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/cases/TenantAPITest.java b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/cases/TenantAPITest.java new file mode 100644 index 0000000000..177c800336 --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/cases/TenantAPITest.java @@ -0,0 +1,113 @@ +/* + * Licensed to Apache Software Foundation (ASF) under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Apache Software Foundation (ASF) licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.dolphinscheduler.api.test.cases; + +import lombok.extern.slf4j.Slf4j; +import org.apache.dolphinscheduler.api.test.core.DolphinScheduler; +import org.apache.dolphinscheduler.api.test.entity.HttpResponse; +import org.apache.dolphinscheduler.api.test.entity.LoginResponseData; +import org.apache.dolphinscheduler.api.test.entity.TenantListPagingResponseData; +import org.apache.dolphinscheduler.api.test.entity.TenantListPagingResponseTotalList; +import org.apache.dolphinscheduler.api.test.pages.LoginPage; +import org.apache.dolphinscheduler.api.test.pages.security.TenantPage; +import org.apache.dolphinscheduler.api.test.utils.JSONUtils; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; + + +@DolphinScheduler(composeFiles = "docker/basic/docker-compose.yaml") +@Slf4j +public class TenantAPITest { + private static final String tenant = System.getProperty("user.name"); + + private static final String user = "admin"; + + private static final String password = "dolphinscheduler123"; + + private static String sessionId = null; + + private static Integer existTenantId = null; + + @BeforeAll + public static void setup() { + LoginPage loginPage = new LoginPage(); + HttpResponse loginHttpResponse = loginPage.login(user, password); + + sessionId = JSONUtils.convertValue(loginHttpResponse.body().data(), LoginResponseData.class).sessionId(); + } + + @AfterAll + public static void cleanup() { + LOGGER.info("success cleanup"); + } + + @Test + @Order(1) + public void testCreateTenant() { + TenantPage tenantPage = new TenantPage(); + + HttpResponse createTenantHttpResponse = tenantPage.createTenant(sessionId, tenant, 1, ""); + + Assertions.assertTrue(createTenantHttpResponse.body().success()); + } + + @Test + @Order(2) + public void testDuplicateCreateTenant() { + TenantPage tenantPage = new TenantPage(); + + HttpResponse createTenantHttpResponse = tenantPage.createTenant(sessionId, tenant, 1, ""); + + Assertions.assertFalse(createTenantHttpResponse.body().success()); + } + + @Test + @Order(5) + public void testGetTenantListPaging() { + TenantPage tenantPage = new TenantPage(); + + HttpResponse createTenantHttpResponse = tenantPage.getTenantListPaging(sessionId, 1, 10, ""); + boolean result = false; + + for (TenantListPagingResponseTotalList tenantListPagingResponseTotalList : JSONUtils.convertValue(createTenantHttpResponse.body().data(), TenantListPagingResponseData.class).totalList()) { + if (tenantListPagingResponseTotalList.tenantCode().equals(tenant)) { + result = true; + existTenantId = tenantListPagingResponseTotalList.id(); + break; + } + } + + Assertions.assertTrue(createTenantHttpResponse.body().success()); + Assertions.assertTrue(result); + } + + @Test + @Order(10) + public void testDeleteTenant() { + TenantPage tenantPage = new TenantPage(); + + HttpResponse deleteTenantHttpResponse = tenantPage.deleteTenant(sessionId, existTenantId); + + Assertions.assertTrue(deleteTenantHttpResponse.body().success()); + } +} diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/entity/HttpResponse.java b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/entity/HttpResponse.java new file mode 100644 index 0000000000..5cd87e99ad --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/entity/HttpResponse.java @@ -0,0 +1,34 @@ +/* + * Licensed to Apache Software Foundation (ASF) under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Apache Software Foundation (ASF) licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.dolphinscheduler.api.test.entity; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@AllArgsConstructor +@NoArgsConstructor +@Data +public class HttpResponse { + + private int statusCode; + + private HttpResponseBody body; +} diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/entity/HttpResponseBody.java b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/entity/HttpResponseBody.java new file mode 100644 index 0000000000..a0f2eccb94 --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/entity/HttpResponseBody.java @@ -0,0 +1,40 @@ +/* + * Licensed to Apache Software Foundation (ASF) under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Apache Software Foundation (ASF) licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.dolphinscheduler.api.test.entity; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@AllArgsConstructor +@NoArgsConstructor +@Data +public class HttpResponseBody { + + private Integer code; + + private String msg; + + private Object data; + + private Boolean failed; + + private Boolean success; +} diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/entity/LoginResponseData.java b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/entity/LoginResponseData.java new file mode 100644 index 0000000000..38a03eaa2c --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/entity/LoginResponseData.java @@ -0,0 +1,32 @@ +/* + * Licensed to Apache Software Foundation (ASF) under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Apache Software Foundation (ASF) licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.dolphinscheduler.api.test.entity; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@AllArgsConstructor +@NoArgsConstructor +@Data +public class LoginResponseData { + + private String sessionId; +} diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/entity/TenantListPagingResponseData.java b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/entity/TenantListPagingResponseData.java new file mode 100644 index 0000000000..be9131cce2 --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/entity/TenantListPagingResponseData.java @@ -0,0 +1,43 @@ +/* + * Licensed to Apache Software Foundation (ASF) under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Apache Software Foundation (ASF) licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.dolphinscheduler.api.test.entity; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@AllArgsConstructor +@NoArgsConstructor +@Data +public class TenantListPagingResponseData { + private Integer currentPage; + + private Integer pageSize; + + private Integer start; + + private Integer total; + + private List totalList; + + private Integer totalPage; +} diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/entity/TenantListPagingResponseTotalList.java b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/entity/TenantListPagingResponseTotalList.java new file mode 100644 index 0000000000..4667ce6921 --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/entity/TenantListPagingResponseTotalList.java @@ -0,0 +1,47 @@ +/* + * Licensed to Apache Software Foundation (ASF) under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Apache Software Foundation (ASF) licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.dolphinscheduler.api.test.entity; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.Date; + +@AllArgsConstructor +@NoArgsConstructor +@Data +public class TenantListPagingResponseTotalList { + private Date createTime; + + private Date updateTime; + + private String description; + + private Integer id; + + private String queue; + + private Integer queueId; + + private String queueName; + + private String tenantCode; +} diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/pages/LoginPage.java b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/pages/LoginPage.java new file mode 100644 index 0000000000..25790b39ef --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/pages/LoginPage.java @@ -0,0 +1,40 @@ +/* + * Licensed to Apache Software Foundation (ASF) under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Apache Software Foundation (ASF) licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.dolphinscheduler.api.test.pages; + + +import org.apache.dolphinscheduler.api.test.entity.HttpResponse; +import org.apache.dolphinscheduler.api.test.utils.RequestClient; + +import java.util.HashMap; +import java.util.Map; + +public final class LoginPage { + public HttpResponse login(String username, String password) { + Map params = new HashMap<>(); + + params.put("userName", username); + params.put("userPassword", password); + + RequestClient requestClient = new RequestClient(); + + return requestClient.post("/login", null, params); + } +} diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/pages/security/TenantPage.java b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/pages/security/TenantPage.java new file mode 100644 index 0000000000..dc16d61424 --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/pages/security/TenantPage.java @@ -0,0 +1,67 @@ +/* + * Licensed to Apache Software Foundation (ASF) under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Apache Software Foundation (ASF) licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.dolphinscheduler.api.test.pages.security; + + +import org.apache.dolphinscheduler.api.test.core.Constants; +import org.apache.dolphinscheduler.api.test.entity.HttpResponse; +import org.apache.dolphinscheduler.api.test.utils.RequestClient; + +import java.util.HashMap; +import java.util.Map; + +public final class TenantPage { + public HttpResponse createTenant(String sessionId, String tenant, Integer queueId, String description) { + Map params = new HashMap<>(); + params.put("tenantCode", tenant); + params.put("queueId", queueId); + params.put("description", description); + + Map headers = new HashMap<>(); + headers.put(Constants.SESSION_ID_KEY, sessionId); + + RequestClient requestClient = new RequestClient(); + + return requestClient.post("/tenants", headers, params); + } + + public HttpResponse getTenantListPaging(String sessionId, Integer pageNo, Integer pageSize, String searchVal) { + Map headers = new HashMap<>(); + headers.put(Constants.SESSION_ID_KEY, sessionId); + + Map params = new HashMap<>(); + params.put("pageSize", pageSize); + params.put("pageNo", pageNo); + params.put("searchVal", searchVal); + + RequestClient requestClient = new RequestClient(); + + return requestClient.get("/tenants/", headers, params); + } + + public HttpResponse deleteTenant(String sessionId, Integer tenantId) { + Map headers = new HashMap<>(); + headers.put(Constants.SESSION_ID_KEY, sessionId); + + RequestClient requestClient = new RequestClient(); + + return requestClient.delete(String.format("/tenants/%s", tenantId), headers, null); + } +} diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/utils/JSONUtils.java b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/utils/JSONUtils.java new file mode 100644 index 0000000000..e49341c632 --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/utils/JSONUtils.java @@ -0,0 +1,384 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.test.utils; +import static java.nio.charset.StandardCharsets.UTF_8; + +import static com.fasterxml.jackson.databind.DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT; +import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES; +import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL; +import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS; + +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.TimeZone; + +import org.apache.dolphinscheduler.api.test.core.Constants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import com.fasterxml.jackson.databind.type.CollectionType; +import org.testcontainers.shaded.org.apache.commons.lang.StringUtils; + +/** + * json utils + */ +public class JSONUtils { + + private static final Logger logger = LoggerFactory.getLogger(JSONUtils.class); + + static { + logger.info("init timezone: {}",TimeZone.getDefault()); + } + + /** + * can use static singleton, inject: just make sure to reuse! + */ + private static final ObjectMapper objectMapper = new ObjectMapper() + .configure(FAIL_ON_UNKNOWN_PROPERTIES, false) + .configure(ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT, true) + .configure(READ_UNKNOWN_ENUM_VALUES_AS_NULL, true) + .configure(REQUIRE_SETTERS_FOR_GETTERS, true) + .setTimeZone(TimeZone.getDefault()) + .setDateFormat(new SimpleDateFormat(Constants.YYYY_MM_DD_HH_MM_SS)); + + private JSONUtils() { + throw new UnsupportedOperationException("Construct JSONUtils"); + } + + public static ArrayNode createArrayNode() { + return objectMapper.createArrayNode(); + } + + public static ObjectNode createObjectNode() { + return objectMapper.createObjectNode(); + } + + public static JsonNode toJsonNode(Object obj) { + return objectMapper.valueToTree(obj); + } + + /** + * json representation of object + * + * @param object object + * @param feature feature + * @return object to json string + */ + public static String toJsonString(Object object, SerializationFeature feature) { + try { + ObjectWriter writer = objectMapper.writer(feature); + return writer.writeValueAsString(object); + } catch (Exception e) { + logger.error("object to json exception!", e); + } + + return null; + } + + /** + * This method deserializes the specified Json into an object of the specified class. It is not + * suitable to use if the specified class is a generic type since it will not have the generic + * type information because of the Type Erasure feature of Java. Therefore, this method should not + * be used if the desired type is a generic type. Note that this method works fine if the any of + * the fields of the specified object are generics, just the object itself should not be a + * generic type. + * + * @param json the string from which the object is to be deserialized + * @param clazz the class of T + * @param T + * @return an object of type T from the string + * classOfT + */ + public static T parseObject(String json, Class clazz) { + if (StringUtils.isEmpty(json)) { + return null; + } + + try { + return objectMapper.readValue(json, clazz); + } catch (Exception e) { + logger.error("parse object exception!", e); + } + return null; + } + + /** + * deserialize + * + * @param src byte array + * @param clazz class + * @param deserialize type + * @return deserialize type + */ + public static T parseObject(byte[] src, Class clazz) { + if (src == null) { + return null; + } + String json = new String(src, UTF_8); + return parseObject(json, clazz); + } + + /** + * json to list + * + * @param json json string + * @param clazz class + * @param T + * @return list + */ + public static List toList(String json, Class clazz) { + if (StringUtils.isEmpty(json)) { + return Collections.emptyList(); + } + + try { + CollectionType listType = objectMapper.getTypeFactory().constructCollectionType(ArrayList.class, clazz); + return objectMapper.readValue(json, listType); + } catch (Exception e) { + logger.error("parse list exception!", e); + } + + return Collections.emptyList(); + } + + /** + * check json object valid + * + * @param json json + * @return true if valid + */ + public static boolean checkJsonValid(String json) { + + if (StringUtils.isEmpty(json)) { + return false; + } + + try { + objectMapper.readTree(json); + return true; + } catch (IOException e) { + logger.error("check json object valid exception!", e); + } + + return false; + } + + /** + * Method for finding a JSON Object field with specified name in this + * node or its child nodes, and returning value it has. + * If no matching field is found in this node or its descendants, returns null. + * + * @param jsonNode json node + * @param fieldName Name of field to look for + * @return Value of first matching node found, if any; null if none + */ + public static String findValue(JsonNode jsonNode, String fieldName) { + JsonNode node = jsonNode.findValue(fieldName); + + if (node == null) { + return null; + } + + return node.asText(); + } + + /** + * json to map + * {@link #toMap(String, Class, Class)} + * + * @param json json + * @return json to map + */ + public static Map toMap(String json) { + return parseObject(json, new TypeReference>() {}); + } + + /** + * json to map + * + * @param json json + * @param classK classK + * @param classV classV + * @param K + * @param V + * @return to map + */ + public static Map toMap(String json, Class classK, Class classV) { + if (StringUtils.isEmpty(json)) { + return Collections.emptyMap(); + } + + try { + return objectMapper.readValue(json, new TypeReference>() { + }); + } catch (Exception e) { + logger.error("json to map exception!", e); + } + + return Collections.emptyMap(); + } + + /** + * from the key-value generated json to get the str value no matter the real type of value + * @param json the json str + * @param nodeName key + * @return the str value of key + */ + public static String getNodeString(String json, String nodeName) { + try { + JsonNode rootNode = objectMapper.readTree(json); + JsonNode jsonNode = rootNode.findValue(nodeName); + if (Objects.isNull(jsonNode)) { + return ""; + } + return jsonNode.isTextual() ? jsonNode.asText() : jsonNode.toString(); + } catch (JsonProcessingException e) { + return ""; + } + } + + /** + * json to object + * + * @param json json string + * @param type type reference + * @param + * @return return parse object + */ + public static T parseObject(String json, TypeReference type) { + if (StringUtils.isEmpty(json)) { + return null; + } + + try { + return objectMapper.readValue(json, type); + } catch (Exception e) { + logger.error("json to map exception!", e); + } + + return null; + } + + /** + * object to json string + * + * @param object object + * @return json string + */ + public static String toJsonString(Object object) { + try { + return objectMapper.writeValueAsString(object); + } catch (Exception e) { + throw new RuntimeException("Object json deserialization exception.", e); + } + } + + /** + * serialize to json byte + * + * @param obj object + * @param object type + * @return byte array + */ + public static byte[] toJsonByteArray(T obj) { + if (obj == null) { + return null; + } + String json = ""; + try { + json = toJsonString(obj); + } catch (Exception e) { + logger.error("json serialize exception.", e); + } + + return json.getBytes(UTF_8); + } + + public static ObjectNode parseObject(String text) { + try { + if (text.isEmpty()) { + return parseObject(text, ObjectNode.class); + } else { + return (ObjectNode) objectMapper.readTree(text); + } + } catch (Exception e) { + throw new RuntimeException("String json deserialization exception.", e); + } + } + + public static ArrayNode parseArray(String text) { + try { + return (ArrayNode) objectMapper.readTree(text); + } catch (Exception e) { + throw new RuntimeException("Json deserialization exception.", e); + } + } + + /** + * json serializer + */ + public static class JsonDataSerializer extends JsonSerializer { + + @Override + public void serialize(String value, JsonGenerator gen, SerializerProvider provider) throws IOException { + gen.writeRawValue(value); + } + + } + + /** + * json data deserializer + */ + public static class JsonDataDeserializer extends JsonDeserializer { + + @Override + public String deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { + JsonNode node = p.getCodec().readTree(p); + if (node instanceof TextNode) { + return node.asText(); + } else { + return node.toString(); + } + } + + } + + public static T convertValue(Object value, Class targetType) { + return objectMapper.convertValue(value, targetType); + } +} diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/utils/RequestClient.java b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/utils/RequestClient.java new file mode 100644 index 0000000000..e1c1cda8e5 --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/utils/RequestClient.java @@ -0,0 +1,171 @@ +/* + * Licensed to Apache Software Foundation (ASF) under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Apache Software Foundation (ASF) licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.dolphinscheduler.api.test.utils; + +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.dolphinscheduler.api.test.core.Constants; +import org.apache.dolphinscheduler.api.test.entity.HttpResponse; +import org.apache.dolphinscheduler.api.test.entity.HttpResponseBody; +import org.testcontainers.shaded.okhttp3.FormBody; +import org.testcontainers.shaded.okhttp3.Headers; +import org.testcontainers.shaded.okhttp3.MediaType; +import org.testcontainers.shaded.okhttp3.OkHttpClient; +import org.testcontainers.shaded.okhttp3.Request; +import org.testcontainers.shaded.okhttp3.RequestBody; +import org.testcontainers.shaded.okhttp3.Response; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +@Slf4j +public class RequestClient { + + private OkHttpClient httpClient = null; + + public RequestClient() { + this.httpClient = new OkHttpClient(); + } + + @SneakyThrows + public HttpResponse get(String url, Map headers, Map params) { + String requestUrl = String.format("%s%s%s", Constants.DOLPHINSCHEDULER_API_URL, url, getParams(params)); + + Headers headersBuilder = new Headers.Builder().build(); + if (headers != null) { + headersBuilder = Headers.of(headers); + } + + LOGGER.info("GET request to {}, Headers: {}", requestUrl, headersBuilder); + Request request = new Request.Builder() + .url(requestUrl) + .headers(headersBuilder) + .get() + .build(); + + Response response = this.httpClient.newCall(request).execute(); + + HttpResponseBody responseData = null; + int responseCode = response.code(); + if (response.body() != null) { + responseData = JSONUtils.parseObject(response.body().string(), HttpResponseBody.class); + } + response.close(); + + HttpResponse httpResponse = new HttpResponse(responseCode, responseData); + + LOGGER.info("GET response: {}", httpResponse); + + return httpResponse; + } + + public static String getParams(Map params) { + StringBuilder sb = new StringBuilder(Constants.QUESTION_MARK); + if (params.size() > 0) { + for (Map.Entry item : params.entrySet()) { + Object value = item.getValue(); + if (Objects.nonNull(value)) { + sb.append(Constants.AND_MARK); + sb.append(item.getKey()); + sb.append(Constants.EQUAL_MARK); + sb.append(value); + } + } + return sb.toString(); + } else { + return ""; + } + } + + @SneakyThrows + public HttpResponse post(String url, Map headers, Map params) { + if (headers == null) { + headers = new HashMap<>(); + } + + String requestUrl = String.format("%s%s", Constants.DOLPHINSCHEDULER_API_URL, url); + + headers.put("Content-Type", Constants.REQUEST_CONTENT_TYPE); + + Headers headersBuilder = Headers.of(headers); + + RequestBody requestBody = FormBody.create(MediaType.parse(Constants.REQUEST_CONTENT_TYPE), getParams(params)); + + LOGGER.info("POST request to {}, Headers: {}, Params: {}", requestUrl, headersBuilder, params); + Request request = new Request.Builder() + .headers(headersBuilder) + .url(requestUrl) + .post(requestBody) + .build(); + + Response response = this.httpClient.newCall(request).execute(); + + + int responseCode = response.code(); + HttpResponseBody responseData = null; + if (response.body() != null) { + responseData = JSONUtils.parseObject(response.body().string(), HttpResponseBody.class); + } + response.close(); + + HttpResponse httpResponse = new HttpResponse(responseCode, responseData); + + LOGGER.info("POST response: {}", httpResponse); + + return httpResponse; + } + + @SneakyThrows + public HttpResponse delete(String url, Map headers, Map params) { + if (headers == null) { + headers = new HashMap<>(); + } + + String requestUrl = String.format("%s%s", Constants.DOLPHINSCHEDULER_API_URL, url); + + headers.put("Content-Type", Constants.REQUEST_CONTENT_TYPE); + + Headers headersBuilder = Headers.of(headers); + + LOGGER.info("DELETE request to {}, Headers: {}, Params: {}", requestUrl, headersBuilder, params); + Request request = new Request.Builder() + .headers(headersBuilder) + .url(requestUrl) + .delete() + .build(); + + Response response = this.httpClient.newCall(request).execute(); + + + int responseCode = response.code(); + HttpResponseBody responseData = null; + if (response.body() != null) { + responseData = JSONUtils.parseObject(response.body().string(), HttpResponseBody.class); + } + response.close(); + + HttpResponse httpResponse = new HttpResponse(responseCode, responseData); + + LOGGER.info("DELETE response: {}", httpResponse); + + return httpResponse; + } +} diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/basic/docker-compose.yaml b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/basic/docker-compose.yaml new file mode 100644 index 0000000000..aeb18a9417 --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/basic/docker-compose.yaml @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3.8" + +services: + dolphinscheduler: + image: apache/dolphinscheduler-standalone-server:ci + environment: + MASTER_MAX_CPU_LOAD_AVG: 100 + WORKER_TENANT_AUTO_CREATE: 'true' + ports: + - "12345:12345" + networks: + - api-test + healthcheck: + test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ] + interval: 5s + timeout: 60s + retries: 120 + +networks: + api-test: diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-clickhouse/docker-compose.yaml b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-clickhouse/docker-compose.yaml new file mode 100644 index 0000000000..fea9fb869c --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-clickhouse/docker-compose.yaml @@ -0,0 +1,61 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3.8" + +services: + dolphinscheduler: + image: apache/dolphinscheduler-standalone-server:ci + environment: + MASTER_MAX_CPU_LOAD_AVG: 100 + WORKER_TENANT_AUTO_CREATE: 'true' + ports: + - "12345:12345" + networks: + - api-test + healthcheck: + test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ] + interval: 5s + timeout: 60s + retries: 120 + depends_on: + clickhouse: + condition: service_healthy + clickhouse: + image: yandex/clickhouse-server:21.12.3.32 + environment: + - CLICKHOUSE_DB=ch_test + - CLICKHOUSE_USER=ch_test + - CLICKHOUSE_PASSWORD=ch_test + ulimits: + nofile: + soft: 262144 + hard: 262144 + expose: + - 8123 + mem_limit: 3G + mem_reservation: 1G + networks: + - e2e + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"] + interval: 5s + timeout: 60s + retries: 120 + +networks: + api-test: diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-hive/docker-compose.yaml b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-hive/docker-compose.yaml new file mode 100644 index 0000000000..1d15805a86 --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-hive/docker-compose.yaml @@ -0,0 +1,117 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3.8" + +services: + dolphinscheduler: + image: apache/dolphinscheduler-standalone-server:ci + environment: + MASTER_MAX_CPU_LOAD_AVG: 100 + WORKER_TENANT_AUTO_CREATE: 'true' + ports: + - "12345:12345" + networks: + - api-test + healthcheck: + test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ] + interval: 5s + timeout: 60s + retries: 120 + depends_on: + hive-server: + condition: service_healthy + namenode: + image: bde2020/hadoop-namenode:2.0.0-hadoop2.7.4-java8 + environment: + - CLUSTER_NAME=test + env_file: + - ./hadoop-hive.env + networks: + - e2e + expose: + - "50070" + healthcheck: + test: [ "CMD", "curl", "http://localhost:50070/" ] + interval: 5s + timeout: 60s + retries: 120 + datanode: + image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8 + env_file: + - ./hadoop-hive.env + environment: + SERVICE_PRECONDITION: "namenode:50070" + networks: + - e2e + expose: + - "50075" + healthcheck: + test: [ "CMD", "curl", "http://localhost:50075" ] + interval: 5s + timeout: 60s + retries: 120 + hive-server: + image: bde2020/hive:2.3.2-postgresql-metastore + env_file: + - ./hadoop-hive.env + networks: + - e2e + environment: + HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://hive-metastore/metastore" + SERVICE_PRECONDITION: "hive-metastore:9083" + expose: + - "10000" + depends_on: + datanode: + condition: service_healthy + namenode: + condition: service_healthy + healthcheck: + test: beeline -u "jdbc:hive2://127.0.0.1:10000/default" -n health_check -e "show databases;" + interval: 5s + timeout: 60s + retries: 120 + hive-metastore: + image: bde2020/hive:2.3.2-postgresql-metastore + env_file: + - ./hadoop-hive.env + command: /opt/hive/bin/hive --service metastore + networks: + - e2e + environment: + SERVICE_PRECONDITION: "namenode:50070 datanode:50075 hive-metastore-postgresql:5432" + expose: + - "9083" + depends_on: + hive-metastore-postgresql: + condition: service_healthy + hive-metastore-postgresql: + image: bde2020/hive-metastore-postgresql:2.3.0 + networks: + - e2e + expose: + - "5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 5s + timeout: 60s + retries: 120 + + +networks: + api-test: diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-hive/hadoop-hive.env b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-hive/hadoop-hive.env new file mode 100644 index 0000000000..9acb760b1c --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-hive/hadoop-hive.env @@ -0,0 +1,50 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +HIVE_SITE_CONF_javax_jdo_option_ConnectionURL=jdbc:postgresql://hive-metastore-postgresql/metastore +HIVE_SITE_CONF_javax_jdo_option_ConnectionDriverName=org.postgresql.Driver +HIVE_SITE_CONF_javax_jdo_option_ConnectionUserName=hive +HIVE_SITE_CONF_javax_jdo_option_ConnectionPassword=hive +HIVE_SITE_CONF_datanucleus_autoCreateSchema=false +HIVE_SITE_CONF_hive_metastore_uris=thrift://hive-metastore:9083 +HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check=false +HIVE_SITE_CONF_hive_server2_thrift_bind_host=0.0.0.0 +HIVE_SITE_CONF_hive_server2_thrift_port=10000 + +CORE_CONF_fs_defaultFS=hdfs://namenode:8020 +CORE_CONF_hadoop_http_staticuser_user=root +CORE_CONF_hadoop_proxyuser_hue_hosts=* +CORE_CONF_hadoop_proxyuser_hue_groups=* +CORE_CONF_hadoop_proxyuser_hive_hosts=* + +HDFS_CONF_dfs_webhdfs_enabled=true +HDFS_CONF_dfs_permissions_enabled=false + +YARN_CONF_yarn_log___aggregation___enable=true +YARN_CONF_yarn_resourcemanager_recovery_enabled=true +YARN_CONF_yarn_resourcemanager_store_class=org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore +YARN_CONF_yarn_resourcemanager_fs_state___store_uri=/rmstate +YARN_CONF_yarn_nodemanager_remote___app___log___dir=/app-logs +YARN_CONF_yarn_log_server_url=http://historyserver:8188/applicationhistory/logs/ +YARN_CONF_yarn_timeline___service_enabled=true +YARN_CONF_yarn_timeline___service_generic___application___history_enabled=true +YARN_CONF_yarn_resourcemanager_system___metrics___publisher_enabled=true +YARN_CONF_yarn_resourcemanager_hostname=resourcemanager +YARN_CONF_yarn_timeline___service_hostname=historyserver +YARN_CONF_yarn_resourcemanager_address=resourcemanager:8032 +YARN_CONF_yarn_resourcemanager_scheduler_address=resourcemanager:8030 +YARN_CONF_yarn_resourcemanager_resource__tracker_address=resourcemanager:8031 \ No newline at end of file diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml new file mode 100644 index 0000000000..d4134ef3f9 --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml @@ -0,0 +1,56 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3.8" + +services: + dolphinscheduler: + image: apache/dolphinscheduler-standalone-server:ci + environment: + MASTER_MAX_CPU_LOAD_AVG: 100 + WORKER_TENANT_AUTO_CREATE: 'true' + ports: + - "12345:12345" + networks: + - api-test + volumes: + - ./download-mysql.sh:/tmp/download-mysql.sh + entrypoint: [ 'bash', '-c', '/bin/bash /tmp/download-mysql.sh && /opt/dolphinscheduler/bin/start.sh && tail -f /dev/null' ] + healthcheck: + test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ] + interval: 5s + timeout: 60s + retries: 120 + depends_on: + mysql: + condition: service_healthy + mysql: + image: mysql:5.7.36 + command: --default-authentication-plugin=mysql_native_password + restart: always + environment: + MYSQL_ROOT_PASSWORD: 123456 + expose: + - 3306 + healthcheck: + test: mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD + interval: 5s + timeout: 60s + retries: 120 + +networks: + api-test: \ No newline at end of file diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-mysql/download-mysql.sh b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-mysql/download-mysql.sh new file mode 100644 index 0000000000..ef0ea7309c --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-mysql/download-mysql.sh @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -ex + +DS_HOME=/opt/dolphinscheduler/libs/standalone-server +MYSQL_URL="https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.16/mysql-connector-java-8.0.16.jar" +MYSQL_DRIVER="mysql-connector-java-8.0.16.jar" + +if ! curl -Lo "${DS_HOME}/${MYSQL_DRIVER}" ${MYSQL_URL}; then + echo "Fail to download ${MYSQL_DRIVER}." + exit 1 +fi diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-postgresql/docker-compose.yaml b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-postgresql/docker-compose.yaml new file mode 100644 index 0000000000..a1696ea724 --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-postgresql/docker-compose.yaml @@ -0,0 +1,54 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3.8" + +services: + dolphinscheduler: + image: apache/dolphinscheduler-standalone-server:ci + environment: + MASTER_MAX_CPU_LOAD_AVG: 100 + WORKER_TENANT_AUTO_CREATE: 'true' + ports: + - "12345:12345" + networks: + - api-test + healthcheck: + test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ] + interval: 5s + timeout: 60s + retries: 120 + depends_on: + postgres: + condition: service_healthy + postgres: + image: postgres:14.1 + restart: always + environment: + POSTGRES_PASSWORD: postgres + networks: + - e2e + expose: + - 5432 + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 5s + timeout: 60s + retries: 120 + +networks: + api-test: diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-sqlserver/docker-compose.yaml b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-sqlserver/docker-compose.yaml new file mode 100644 index 0000000000..e8dde399b9 --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/datasource-sqlserver/docker-compose.yaml @@ -0,0 +1,51 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3.8" + +services: + dolphinscheduler: + image: apache/dolphinscheduler-standalone-server:ci + environment: + MASTER_MAX_CPU_LOAD_AVG: 100 + WORKER_TENANT_AUTO_CREATE: 'true' + ports: + - "12345:12345" + networks: + - api-test + healthcheck: + test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ] + interval: 5s + timeout: 60s + retries: 120 + depends_on: + sqlserver: + condition: service_healthy + sqlserver: + image: alexk002/sqlserver2019_demo:1 + expose: + - 1433 + networks: + - e2e + healthcheck: + test: /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P OcP2020123 -Q "SELECT 1" + interval: 5s + timeout: 60s + retries: 120 + +networks: + api-test: diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/common.properties b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/common.properties new file mode 100644 index 0000000000..dcf839ff76 --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/common.properties @@ -0,0 +1,90 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# user data local directory path, please make sure the directory exists and have read write permissions +data.basedir.path=/tmp/dolphinscheduler + +# resource storage type: HDFS, S3, NONE +resource.storage.type=S3 + +# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended +resource.upload.path=/dolphinscheduler + +# whether to startup kerberos +hadoop.security.authentication.startup.state=false + +# java.security.krb5.conf path +java.security.krb5.conf.path=/opt/krb5.conf + +# login user from keytab username +login.user.keytab.username=hdfs-mycluster@ESZ.COM + +# login user from keytab path +login.user.keytab.path=/opt/hdfs.headless.keytab + +# kerberos expire time, the unit is hour +kerberos.expire.time=2 + +# resource view suffixs +#resource.view.suffixs=txt,log,sh,bat,conf,cfg,py,java,sql,xml,hql,properties,json,yml,yaml,ini,js + +# if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path +hdfs.root.user=hdfs + +# if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir +fs.defaultFS=s3a://dolphinscheduler + + +# resourcemanager port, the default value is 8088 if not specified +resource.manager.httpaddress.port=8088 + +# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty +yarn.resourcemanager.ha.rm.ids=192.168.xx.xx,192.168.xx.xx + +# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname +yarn.application.status.address=http://ds1:%s/ws/v1/cluster/apps/%s + +# job history status url when application number threshold is reached(default 10000, maybe it was set to 1000) +yarn.job.history.status.address=http://ds1:19888/ws/v1/history/mapreduce/jobs/%s + +# datasource encryption enable +datasource.encryption.enable=false + +# datasource encryption salt +datasource.encryption.salt=!@#$%^&* + +# use sudo or not, if set true, executing user is tenant user and deploy user needs sudo permissions; if set false, executing user is the deploy user and doesn't need sudo permissions +sudo.enable=true + +# network interface preferred like eth0, default: empty +#dolphin.scheduler.network.interface.preferred= + +# network IP gets priority, default: inner outer +#dolphin.scheduler.network.priority.strategy=default +# system env path +#dolphinscheduler.env.path=dolphinscheduler_env.sh +# development state +development.state=false +# rpc port +alert.rpc.port=50052 +aws.access.key.id=accessKey123 +aws.secret.access.key=secretKey123 +aws.region=us-east-1 +aws.endpoint=http://s3:9000 + +# Task resource limit state +task.resource.limit.state=false \ No newline at end of file diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/docker-compose.yaml b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/docker-compose.yaml new file mode 100644 index 0000000000..c672c0cb0e --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/docker-compose.yaml @@ -0,0 +1,78 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3.8" + +services: + dolphinscheduler: + image: apache/dolphinscheduler-standalone-server:ci + environment: + MASTER_MAX_CPU_LOAD_AVG: 100 + WORKER_TENANT_AUTO_CREATE: 'true' + ports: + - "12345:12345" + networks: + - api-test + healthcheck: + test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ] + interval: 5s + timeout: 60s + retries: 120 + volumes: + - ./common.properties:/opt/dolphinscheduler/conf/common.properties + depends_on: + s3: + condition: service_healthy + mc: + condition: service_completed_successfully + s3: + image: minio/minio:RELEASE.2022-01-25T19-56-04Z + hostname: s3 + tty: true + stdin_open: true + command: server /data --console-address ":9001" + ports: + - 9000:9000 + networks: + - e2e + environment: + MINIO_ROOT_USER: accessKey123 + MINIO_ROOT_PASSWORD: secretKey123 + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 5s + timeout: 120s + retries: 120 + mc: + image: minio/mc:RELEASE.2022-01-07T06-01-38Z + entrypoint: bash + networks: + - e2e + command: + - -c + - mc alias set s3 http://s3:9000 accessKey123 secretKey123 && mc mb s3/dolphinscheduler + depends_on: + s3: + condition: service_healthy + +networks: + api-test: + driver: bridge + ipam: + config: + - subnet: 10.1.0.0/24 + gateway: 10.1.0.1 diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-core/pom.xml b/dolphinscheduler-api-test/dolphinscheduler-api-test-core/pom.xml new file mode 100644 index 0000000000..69ceaa5183 --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-core/pom.xml @@ -0,0 +1,32 @@ + + + + + + dolphinscheduler-api-test + org.apache.dolphinscheduler + 1.0-SNAPSHOT + + 4.0.0 + + dolphinscheduler-api-test-core + diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-core/src/main/java/org/apache/dolphinscheduler/api/test/core/Constants.java b/dolphinscheduler-api-test/dolphinscheduler-api-test-core/src/main/java/org/apache/dolphinscheduler/api/test/core/Constants.java new file mode 100644 index 0000000000..4db7f1d0e0 --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-core/src/main/java/org/apache/dolphinscheduler/api/test/core/Constants.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.test.core; + +import lombok.experimental.UtilityClass; + +import java.nio.file.Path; +import java.nio.file.Paths; + +@UtilityClass +public final class Constants { + + /** + * backend api url + */ + public static final String DOLPHINSCHEDULER_API_URL = "http://0.0.0.0:12345/dolphinscheduler"; + + /** + * backend api request header's content type + */ + public static final String REQUEST_CONTENT_TYPE = "application/x-www-form-urlencoded"; + + /** + * header's session id's key + */ + public static final String SESSION_ID_KEY = "sessionId"; + + /** + * simple date format + */ + public static final String YYYY_MM_DD_HH_MM_SS = "yyyy-MM-dd HH:mm:ss"; + + /** + * docker compose default healthy timeout + */ + public static final Integer DOCKER_COMPOSE_DEFAULT_TIMEOUT = 180; + + public static final String QUESTION_MARK = "?"; + + public static final String EQUAL_MARK = "="; + + public static final String AND_MARK = "&"; +} diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-core/src/main/java/org/apache/dolphinscheduler/api/test/core/DolphinScheduler.java b/dolphinscheduler-api-test/dolphinscheduler-api-test-core/src/main/java/org/apache/dolphinscheduler/api/test/core/DolphinScheduler.java new file mode 100644 index 0000000000..5c3b28485e --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-core/src/main/java/org/apache/dolphinscheduler/api/test/core/DolphinScheduler.java @@ -0,0 +1,41 @@ +/* + * Licensed to Apache Software Foundation (ASF) under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Apache Software Foundation (ASF) licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.dolphinscheduler.api.test.core; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +import org.junit.jupiter.api.MethodOrderer.OrderAnnotation; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.extension.ExtendWith; +import org.testcontainers.junit.jupiter.Testcontainers; + +@Inherited +@Testcontainers +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@TestMethodOrder(OrderAnnotation.class) +@ExtendWith(DolphinSchedulerExtension.class) +public @interface DolphinScheduler { + String[] composeFiles(); +} diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-core/src/main/java/org/apache/dolphinscheduler/api/test/core/DolphinSchedulerExtension.java b/dolphinscheduler-api-test/dolphinscheduler-api-test-core/src/main/java/org/apache/dolphinscheduler/api/test/core/DolphinSchedulerExtension.java new file mode 100644 index 0000000000..c0164a63ca --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-core/src/main/java/org/apache/dolphinscheduler/api/test/core/DolphinSchedulerExtension.java @@ -0,0 +1,80 @@ +/* + * Licensed to Apache Software Foundation (ASF) under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Apache Software Foundation (ASF) licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.dolphinscheduler.api.test.core; + +import java.io.File; +import java.net.URL; +import java.time.Duration; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import lombok.SneakyThrows; +import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.testcontainers.containers.DockerComposeContainer; +import org.testcontainers.containers.wait.strategy.Wait; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCallback { + private final boolean LOCAL_MODE = Objects.equals(System.getProperty("local"), "true"); + + private final String SERVICE_NAME = "dolphinscheduler_1"; + + private DockerComposeContainer compose; + + @Override + public void beforeAll(ExtensionContext context) { + if (!LOCAL_MODE) { + compose = createDockerCompose(context); + compose.start(); + } + } + + @Override + public void afterAll(ExtensionContext context) { + if (compose != null) { + compose.stop(); + } + } + + private DockerComposeContainer createDockerCompose(ExtensionContext context) { + final Class clazz = context.getRequiredTestClass(); + final DolphinScheduler annotation = clazz.getAnnotation(DolphinScheduler.class); + final List files = Stream.of(annotation.composeFiles()) + .map(it -> DolphinScheduler.class.getClassLoader().getResource(it)) + .filter(Objects::nonNull) + .map(URL::getPath) + .map(File::new) + .collect(Collectors.toList()); + + compose = new DockerComposeContainer<>(files) + .withPull(true) + .withTailChildContainers(true) + .withLogConsumer(SERVICE_NAME, outputFrame -> LOGGER.info(outputFrame.getUtf8String())) + .waitingFor(SERVICE_NAME, Wait.forHealthcheck().withStartupTimeout(Duration.ofSeconds(Constants.DOCKER_COMPOSE_DEFAULT_TIMEOUT))); + + return compose; + } +} diff --git a/dolphinscheduler-api-test/dolphinscheduler-api-test-core/src/main/resources/log4j2.xml b/dolphinscheduler-api-test/dolphinscheduler-api-test-core/src/main/resources/log4j2.xml new file mode 100644 index 0000000000..167e6e6d97 --- /dev/null +++ b/dolphinscheduler-api-test/dolphinscheduler-api-test-core/src/main/resources/log4j2.xml @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + diff --git a/dolphinscheduler-api-test/lombok.config b/dolphinscheduler-api-test/lombok.config new file mode 100644 index 0000000000..15f8d6f2ae --- /dev/null +++ b/dolphinscheduler-api-test/lombok.config @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +lombok.accessors.fluent=true +lombok.log.fieldname=LOGGER +lombok.accessors.fluent=true +lombok.anyConstructor.addConstructorProperties=true \ No newline at end of file diff --git a/dolphinscheduler-api-test/pom.xml b/dolphinscheduler-api-test/pom.xml new file mode 100644 index 0000000000..31d29c2b77 --- /dev/null +++ b/dolphinscheduler-api-test/pom.xml @@ -0,0 +1,142 @@ + + + + 4.0.0 + + org.apache.dolphinscheduler + dolphinscheduler-api-test + pom + 1.0-SNAPSHOT + + + dolphinscheduler-api-test-core + dolphinscheduler-api-test-case + + + + 8 + 8 + UTF-8 + + 5.7.2 + 3.141.59 + 1.18.24 + 3.23.1 + 4.1.0 + 1.5.30 + 1.7.36 + 2.17.2 + 31.0.1-jre + 2.13.2 + + + + + org.slf4j + slf4j-api + ${slf4j-api.version} + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j-slf4j-impl.version} + + + + org.junit.jupiter + junit-jupiter + + + + org.testcontainers + testcontainers + + + org.testcontainers + junit-jupiter + + + + org.assertj + assertj-core + ${assertj-core.version} + test + + + + org.projectlombok + lombok + ${lombok.version} + provided + + + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + + + + + + + com.google.guava + guava + ${guava.version} + + + org.junit + junit-bom + ${junit.version} + import + pom + + + org.testcontainers + testcontainers-bom + 1.16.3 + import + pom + + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.22.2 + + + +