From 30746d9762b350df530187a628c58c547c7f7ba1 Mon Sep 17 00:00:00 2001 From: Jiajie Zhong Date: Fri, 18 Mar 2022 16:24:40 +0800 Subject: [PATCH] [python] Add integrated test to python gateway server (#8966) * [python] Add integrated test to python gateway server * Build java code and create standalone server image in GA * Add component start docker in python * Run example to make sure it work to it close: #8035 * Fix build docker image working directory * Fix working directory --- .github/workflows/py-ci.yml | 66 ++++++++++++- .../pydolphinscheduler/DEVELOP.md | 32 +++++- .../pydolphinscheduler/pytest.ini | 2 + .../pydolphinscheduler/setup.py | 1 + .../pydolphinscheduler/core/configuration.py | 4 +- .../tests/core/test_configuration.py | 2 +- .../tests/integration/__init__.py | 18 ++++ .../tests/integration/test_submit_examples.py | 58 +++++++++++ .../tests/testing/constants.py | 9 ++ .../tests/testing/docker_wrapper.py | 98 +++++++++++++++++++ .../pydolphinscheduler/tox.ini | 7 +- .../src/main/docker/Dockerfile | 2 +- 12 files changed, 288 insertions(+), 11 deletions(-) create mode 100644 dolphinscheduler-python/pydolphinscheduler/tests/integration/__init__.py create mode 100644 dolphinscheduler-python/pydolphinscheduler/tests/integration/test_submit_examples.py create mode 100644 dolphinscheduler-python/pydolphinscheduler/tests/testing/docker_wrapper.py diff --git a/.github/workflows/py-ci.yml b/.github/workflows/py-ci.yml index 49599f1c38..bec1d74f3f 100644 --- a/.github/workflows/py-ci.yml +++ b/.github/workflows/py-ci.yml @@ -93,7 +93,7 @@ jobs: - name: Run Tests Build Docs run: | python -m tox -vv -e doc-build-test - verify-local-ci: + local-ci: name: Local CI timeout-minutes: 15 needs: @@ -112,3 +112,67 @@ jobs: - name: Run Tests Build Docs run: | python -m tox -vv -e local-ci + build-image: + name: Build Image + runs-on: ubuntu-latest + # Switch to project root directory to run mvnw command + defaults: + run: + working-directory: ./ + timeout-minutes: 20 + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - name: Sanity Check + uses: ./.github/actions/sanity-check + - name: Cache local Maven repository + uses: actions/cache@v2 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-maven- + - name: Build Image + run: | + ./mvnw -B clean install \ + -Dmaven.test.skip \ + -Dmaven.javadoc.skip \ + -Dmaven.checkstyle.skip \ + -Pdocker,release -Ddocker.tag=ci \ + -pl dolphinscheduler-standalone-server -am + - name: Export Docker Images + run: | + docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \ + && du -sh /tmp/standalone-image.tar + - uses: actions/upload-artifact@v2 + name: Upload Docker Images + with: + name: standalone-image + path: /tmp/standalone-image.tar + retention-days: 1 + integrate-test: + name: Integrate Test + timeout-minutes: 20 + needs: + - build-image + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/download-artifact@v2 + name: Download Docker Images + with: + name: standalone-image + path: /tmp + - name: Load Docker Images + run: | + docker load -i /tmp/standalone-image.tar + - name: Set up Python 3.7 + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Install Dependences + run: | + python -m pip install --upgrade ${{ env.DEPENDENCES }} + - name: Run Tests Build Docs + run: | + python -m tox -vv -e integrate-test diff --git a/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md b/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md index 063f00454e..a0dbb410f9 100644 --- a/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md +++ b/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md @@ -67,8 +67,8 @@ integration service run by GitHub Action to test whether the patch is good or no section [With GitHub Action](#with-github-action) see more detail. And to make more convenience to local tests, we also have the way to run your [test automated with tox](#automated-testing-with-tox) -locally. It is helpful when your try to find out the detail when continuous integration in GitHub Action failed, -or you have a great patch and want to test local first. +locally(*run all tests except integrate test with need docker environment*). It is helpful when your try to find out the +detail when continuous integration in GitHub Action failed, or you have a great patch and want to test local first. Besides [automated testing with tox](#automated-testing-with-tox) locally, we also have a [manual way](#manually) run tests. And it is scattered commands to reproduce each step of the integration test we told about. @@ -76,14 +76,14 @@ run tests. And it is scattered commands to reproduce each step of the integratio * Remote * [With GitHub Action](#with-github-action) * Local - * [Automated Testing With tox](#automated-testing-with-tox) - * [Manually](#manually) + * [Automated Testing With tox](#automated-testing-with-tox)(including all but integrate test) + * [Manually](#manually)(with integrate test) ### With GitHub Action GitHub Action test in various environment for pydolphinscheduler, including different python version in `3.6|3.7|3.8|3.9` and operating system `linux|macOS|windows`. It will trigger and run automatically when you -submit pull requests to `apache/dolphinscheduler`. +submit pull requests to `apache/dolphinscheduler`. ### Automated Testing With tox @@ -165,6 +165,28 @@ It would not only run unit test but also show each file coverage which cover rat line show you total coverage of you code. If your CI failed with coverage you could go and find some reason by this command output. +#### Integrate Test + +Integrate Test can not run when you execute command `tox -e local-ci` because it needs external environment +including [Docker](https://docs.docker.com/get-docker/) and specific image build by [maven](https://maven.apache.org/install.html). +Here we would show you the step to run integrate test in directory `dolphinscheduler-python/pydolphinscheduler/tests/integration`. + +```shell +# Go to project root directory and build Docker image +cd ../../ + +# Build Docker image +./mvnw -B clean install \ + -Dmaven.test.skip \ + -Dmaven.javadoc.skip \ + -Dmaven.checkstyle.skip \ + -Pdocker,release -Ddocker.tag=ci \ + -pl dolphinscheduler-standalone-server -am + +# Go to pydolphinscheduler root directory and run integrate tests +tox -e integrate-test +``` + ## Add LICENSE When New Dependencies Adding When you add a new package in pydolphinscheduler, you should also add the package's LICENSE to directory diff --git a/dolphinscheduler-python/pydolphinscheduler/pytest.ini b/dolphinscheduler-python/pydolphinscheduler/pytest.ini index f2c7ae6489..9ed8ccf2b6 100644 --- a/dolphinscheduler-python/pydolphinscheduler/pytest.ini +++ b/dolphinscheduler-python/pydolphinscheduler/pytest.ini @@ -20,3 +20,5 @@ addopts = --ignore=tests/test_java_gateway.py # add path here to skip pytest scan it norecursedirs = tests/testing + # Integration test run seperated which do not calculate coverage + tests/integration diff --git a/dolphinscheduler-python/pydolphinscheduler/setup.py b/dolphinscheduler-python/pydolphinscheduler/setup.py index 23c0b9b392..9632bb1fd9 100644 --- a/dolphinscheduler-python/pydolphinscheduler/setup.py +++ b/dolphinscheduler-python/pydolphinscheduler/setup.py @@ -58,6 +58,7 @@ test = [ "freezegun>=1.1", "coverage>=6.1", "pytest-cov>=3.0", + "docker>=5.0.3", ] style = [ diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/configuration.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/configuration.py index 9c5a5d7a8e..8b5665d304 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/configuration.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/core/configuration.py @@ -134,8 +134,8 @@ JAVA_GATEWAY_AUTO_CONVERT = configs.get_bool("java_gateway.auto_convert") USER_NAME = configs.get("default.user.name") USER_PASSWORD = configs.get("default.user.password") USER_EMAIL = configs.get("default.user.email") -USER_PHONE = configs.get("default.user.phone") -USER_STATE = configs.get("default.user.state") +USER_PHONE = str(configs.get("default.user.phone")) +USER_STATE = configs.get_int("default.user.state") # Workflow Settings WORKFLOW_PROJECT = configs.get("default.workflow.project") diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_configuration.py b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_configuration.py index a0704c9738..45d4477bcd 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/core/test_configuration.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/core/test_configuration.py @@ -163,7 +163,7 @@ def test_single_config_get_set_not_exists_key(): ("USER_NAME", "userPythonGateway"), ("USER_PASSWORD", "userPythonGateway"), ("USER_EMAIL", "userPythonGateway@dolphinscheduler.com"), - ("USER_PHONE", 11111111111), + ("USER_PHONE", "11111111111"), ("USER_STATE", 1), ("WORKFLOW_PROJECT", "project-pydolphin"), ("WORKFLOW_TENANT", "tenant_pydolphin"), diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/integration/__init__.py b/dolphinscheduler-python/pydolphinscheduler/tests/integration/__init__.py new file mode 100644 index 0000000000..8079bfba1f --- /dev/null +++ b/dolphinscheduler-python/pydolphinscheduler/tests/integration/__init__.py @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Test integration between Python API and PythonGatewayServer.""" diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/integration/test_submit_examples.py b/dolphinscheduler-python/pydolphinscheduler/tests/integration/test_submit_examples.py new file mode 100644 index 0000000000..423c6c3a6f --- /dev/null +++ b/dolphinscheduler-python/pydolphinscheduler/tests/integration/test_submit_examples.py @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Test whether success submit examples DAG to PythonGatewayServer.""" + +from pathlib import Path + +import pytest + +from tests.testing.constants import ignore_exec_examples +from tests.testing.docker_wrapper import DockerWrapper +from tests.testing.path import path_example + + +@pytest.fixture(scope="module") +def setup_docker(): + """Set up and teardown docker env for fixture.""" + docker_wrapper = DockerWrapper( + image="apache/dolphinscheduler-standalone-server:ci", + container_name="ci-dolphinscheduler-standalone-server", + ) + ports = {"25333/tcp": 25333} + container = docker_wrapper.run_until_log( + log="Started StandaloneServer in", tty=True, ports=ports + ) + assert container is not None + yield + docker_wrapper.remove_container() + + +@pytest.mark.parametrize( + "example_path", + [ + path + for path in path_example.iterdir() + if path.is_file() and path.stem not in ignore_exec_examples + ], +) +def test_exec_white_list_example(setup_docker, example_path: Path): + """Test execute examples and submit DAG to PythonGatewayServer.""" + try: + exec(example_path.read_text()) + except Exception: + raise Exception("Run example %s failed.", example_path.stem) diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/testing/constants.py b/dolphinscheduler-python/pydolphinscheduler/tests/testing/constants.py index 7e214ff892..63f0fdd822 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tests/testing/constants.py +++ b/dolphinscheduler-python/pydolphinscheduler/tests/testing/constants.py @@ -29,6 +29,15 @@ task_without_example = { "procedure", } +# The examples ignore test to run it. Those examples could not be run directly cause it need other +# support like resource files, data source and etc. But we should try to run them later for more coverage +ignore_exec_examples = { + "task_datax_example", + "task_flink_example", + "task_map_reduce_example", + "task_spark_example", +} + # pydolphinscheduler environment home ENV_PYDS_HOME = "PYDOLPHINSCHEDULER_HOME" diff --git a/dolphinscheduler-python/pydolphinscheduler/tests/testing/docker_wrapper.py b/dolphinscheduler-python/pydolphinscheduler/tests/testing/docker_wrapper.py new file mode 100644 index 0000000000..a3d0b6ea7a --- /dev/null +++ b/dolphinscheduler-python/pydolphinscheduler/tests/testing/docker_wrapper.py @@ -0,0 +1,98 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Wrap docker commands for easier create docker container.""" + +import time +from typing import Optional + +import docker +from docker.errors import ImageNotFound +from docker.models.containers import Container + + +class DockerWrapper: + """Wrap docker commands for easier create docker container. + + :param image: The image to create docker container. + """ + + def __init__(self, image: str, container_name: str): + self._client = docker.from_env() + self.image = image + self.container_name = container_name + + def run(self, *args, **kwargs) -> Container: + """Create and run a new container. + + This method would return immediately after the container started, if you wish it return container + object when specific service start, you could see :func:`run_until_log` which return container + object when specific output log appear in docker. + """ + if not self.images_exists: + raise ValueError("Docker image named %s do not exists.", self.image) + return self._client.containers.run( + image=self.image, name=self.container_name, detach=True, *args, **kwargs + ) + + def run_until_log( + self, log: str, remove_exists: Optional[bool] = True, *args, **kwargs + ) -> Container: + """Create and run a new container, return when specific log appear. + + It will call :func:`run` inside this method. And after container started, it would not + return it immediately but run command `docker logs` to see whether specific log appear. + It will raise `RuntimeError` when 10 minutes after but specific log do not appear. + """ + if remove_exists: + self.remove_container() + + log_byte = str.encode(log) + container = self.run(*args, **kwargs) + + timeout_threshold = 10 * 60 + start_time = time.time() + while time.time() <= start_time + timeout_threshold: + if log_byte in container.logs(tail=1000): + break + time.sleep(2) + # Stop container and raise error when reach timeout threshold but do not appear specific log output + else: + container.remove(force=True) + raise RuntimeError( + "Can not capture specific log `%s` in %d seconds, remove container.", + (log, timeout_threshold), + ) + return container + + def remove_container(self): + """Remove container which already running.""" + containers = self._client.containers.list( + all=True, filters={"name": self.container_name} + ) + if containers: + for container in containers: + container.remove(force=True) + + @property + def images_exists(self) -> bool: + """Check whether the image exists in local docker repository or not.""" + try: + self._client.images.get(self.image) + return True + except ImageNotFound: + return False diff --git a/dolphinscheduler-python/pydolphinscheduler/tox.ini b/dolphinscheduler-python/pydolphinscheduler/tox.ini index e1cfea18cf..8e9280f057 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tox.ini +++ b/dolphinscheduler-python/pydolphinscheduler/tox.ini @@ -16,7 +16,7 @@ # under the License. [tox] -envlist = local-ci, auto-lint, lint, doc-build-test, code-test, py{36,37,38,39} +envlist = local-ci, auto-lint, lint, doc-build-test, code-test, integrate-test, py{36,37,38,39} [testenv] whitelist_externals = make @@ -48,6 +48,11 @@ commands = make -C {toxinidir}/docs clean make -C {toxinidir}/docs html +[testenv:integrate-test] +extras = test +commands = + python -m pytest tests/integration/ + [testenv:local-ci] extras = dev commands = diff --git a/dolphinscheduler-standalone-server/src/main/docker/Dockerfile b/dolphinscheduler-standalone-server/src/main/docker/Dockerfile index 1822047132..d1ae2b3c98 100644 --- a/dolphinscheduler-standalone-server/src/main/docker/Dockerfile +++ b/dolphinscheduler-standalone-server/src/main/docker/Dockerfile @@ -29,6 +29,6 @@ WORKDIR $DOLPHINSCHEDULER_HOME ADD ./target/standalone-server $DOLPHINSCHEDULER_HOME -EXPOSE 12345 +EXPOSE 12345 25333 CMD [ "/bin/bash", "./bin/start.sh" ]