From 90f9aed11dafe1617b5b80418573b645f3c79072 Mon Sep 17 00:00:00 2001 From: Jiajie Zhong Date: Sat, 12 Mar 2022 21:27:36 +0800 Subject: [PATCH] [python] Add tox for out of box test (#8837) * [python] Add tox for out of box test * Add `tox` for easy and complete test * Correct package data attribute * Add more dirs in flake8 ignore * Change GA from raw to tox * Fix env setter * Temp skip sanity * Rm not exists job * Add code test * fix * rm all need * fix * fix * fix * fix * fix * fix * Migrate pip upgrede to tox setting * Change install_commands to commands * fix config of install command * Add GA env to do that * Fix env error * Finial commit * remove duplicate * Change doc * Change dependence of ci * Change dependence of ci --- .github/workflows/py-ci.yml | 79 +++++++++---------- .gitignore | 1 + .../pydolphinscheduler/.coveragerc | 4 +- .../pydolphinscheduler/.flake8 | 4 +- .../pydolphinscheduler/DEVELOP.md | 73 +++++++++++++---- .../pydolphinscheduler/setup.py | 3 +- .../pydolphinscheduler/tox.ini | 48 +++++++++++ 7 files changed, 153 insertions(+), 59 deletions(-) create mode 100644 dolphinscheduler-python/pydolphinscheduler/tox.ini diff --git a/.github/workflows/py-ci.yml b/.github/workflows/py-ci.yml index db430caea9..d10d2559e8 100644 --- a/.github/workflows/py-ci.yml +++ b/.github/workflows/py-ci.yml @@ -25,22 +25,21 @@ on: paths: - 'dolphinscheduler-python/**' +concurrency: + group: py-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + defaults: run: working-directory: dolphinscheduler-python/pydolphinscheduler +# We have to update setuptools wheel to package with package_data, LICENSE, NOTICE +env: + DEPENDENCES: pip setuptools wheel tox + jobs: - sanity: - name: Sanity - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - name: Sanity Check - uses: ./.github/actions/sanity-check lint: - name: Code Style + name: Lint runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -48,19 +47,15 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.7 - - name: Install Development Dependences - run: pip install -e .[style] - - name: Run Isort Checking - run: isort --check . - - name: Run Black Checking - run: black --check . - - name: Run Flake8 Checking - run: flake8 + - name: Install Dependences + run: | + python -m pip install --upgrade ${{ env.DEPENDENCES }} + - name: Run All Lint Check + run: | + python -m tox -vv -e lint pytest: name: Pytest - needs: - - lint - - sanity + needs: lint runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -73,16 +68,15 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies & pydolphinscheduler + - name: Install Dependences run: | - pip install -e .[test] - - name: Run tests + python -m pip install --upgrade ${{ env.DEPENDENCES }} + - name: Run All Tests run: | - pytest - coverage: - name: Tests coverage - needs: - - pytest + python -m tox -vv -e code-test + doc-build: + name: Docs Build Test + needs: lint runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -90,14 +84,17 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.7 - - name: Install Development Dependences + - name: Install Dependences run: | - pip install -e .[test] - - name: Run Tests && Check coverage - run: coverage run && coverage report - doc-build: - name: Document Build Test - needs: pytest + python -m pip install --upgrade ${{ env.DEPENDENCES }} + - name: Run Tests Build Docs + run: | + python -m tox -vv -e doc-build-test + verify-local-ci: + name: Local CI + needs: + - pytest + - doc-build runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -105,9 +102,9 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.7 - - name: Install Development Dependences + - name: Install Dependences + run: | + python -m pip install --upgrade ${{ env.DEPENDENCES }} + - name: Run Tests Build Docs run: | - pip install -e .[doc] - - name: Test Build Document - working-directory: dolphinscheduler-python/pydolphinscheduler/docs - run: make clean && make html + python -m tox -vv -e local-ci diff --git a/.gitignore b/.gitignore index 0a0aaaa6fd..765259a5b0 100644 --- a/.gitignore +++ b/.gitignore @@ -55,6 +55,7 @@ dolphinscheduler-worker/logs # ------------------ # Cache __pycache__/ +.tox/ # Build build/ diff --git a/dolphinscheduler-python/pydolphinscheduler/.coveragerc b/dolphinscheduler-python/pydolphinscheduler/.coveragerc index 52286d6cab..16205094c2 100644 --- a/dolphinscheduler-python/pydolphinscheduler/.coveragerc +++ b/dolphinscheduler-python/pydolphinscheduler/.coveragerc @@ -20,8 +20,10 @@ command_line = -m pytest omit = # Ignore all test cases in tests/ tests/* + # Ignore examples directory + */pydolphinscheduler/examples/* # TODO. Temporary ignore java_gateway file, because we could not find good way to test it. - src/pydolphinscheduler/java_gateway.py + */pydolphinscheduler/java_gateway.py [report] # Don’t report files that are 100% covered diff --git a/dolphinscheduler-python/pydolphinscheduler/.flake8 b/dolphinscheduler-python/pydolphinscheduler/.flake8 index 7f659a21b3..9fbada8c9d 100644 --- a/dolphinscheduler-python/pydolphinscheduler/.flake8 +++ b/dolphinscheduler-python/pydolphinscheduler/.flake8 @@ -26,7 +26,9 @@ exclude = old, build, dist, - htmlcov + htmlcov, + .tox, + dist, ignore = # It's clear and not need to add docstring D107, # D107: Don't require docstrings on __init__ diff --git a/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md b/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md index ef7dfa4060..4158b5b66a 100644 --- a/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md +++ b/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md @@ -60,7 +60,53 @@ pydolphinscheduler tasks object, we use tasks to define exact job we want Dolphi we only support `shell` task to execute shell task. [This link][all-task] list all tasks support in DolphinScheduler and would be implemented in the further. -## Code Style +## Test Your Code + +Linting and tests is very important for open source project, so we pay more attention to it. We have continuous +integration service run by GitHub Action to test whether the patch is good or not, which you could jump to +section [With GitHub Action](#with-github-action) see more detail. + +And to make more convenience to local tests, we also have the way to run your [test automated with tox](#automated-testing-with-tox) +locally. It is helpful when your try to find out the detail when continuous integration in GitHub Action failed, +or you have a great patch and want to test local first. + +Besides [automated testing with tox](#automated-testing-with-tox) locally, we also have a [manual way](#manually) +run tests. And it is scattered commands to reproduce each step of the integration test we told about. + +* Remote + * [With GitHub Action](#with-github-action) +* Local + * [Automated Testing With tox](#automated-testing-with-tox) + * [Manually](#manually) + +### With GitHub Action + +GitHub Action test in various environment for pydolphinscheduler, including different python version in +`3.6|3.7|3.8|3.9` and operating system `linux|macOS|windows`. It will trigger and run automatically when you +submit pull requests to `apache/dolphinscheduler`. + +### Automated Testing With tox + +[tox](https://tox.wiki) is a package aims to automate and standardize testing in Python, both our continuous +integration and local test use it to run actual task. To use it, you should install it first + +```shell +python -m pip install --upgrade tox +``` + +After installation, you could run a single command to run all the tests, it is almost like test in GitHub Action +but not so much different environment. + +```shell +tox -e local-ci +``` + +It will take a while when you run it the first time, because it has to install dependencies and make some prepare, +and the next time you run it will be faster. + +### Manually + +#### Code Style We use [isort][isort] to automatically keep Python imports alphabetically, and use [Black][black] for code formatter and [Flake8][flake8] for pep8 checker. If you use [pycharm][pycharm]or [IntelliJ IDEA][idea], @@ -74,33 +120,30 @@ GitHub, you could also run static check locally. # but Flake8 just hint when code style not match pep8 # Run Isort -isort . +python -m isort . # Run Black -black . +python -m black . # Run Flake8 -flake8 +python -m flake8 ``` -## Testing +#### Testing -pydolphinscheduler using [pytest][pytest] to test our codebase. GitHub Action will run our test when you create -pull request or commit to dev branch, with python version `3.6|3.7|3.8|3.9` and operating system `linux|macOS|windows`. - -To test locally, you could directly run pytest after set `PYTHONPATH` +pydolphinscheduler using [pytest][pytest] to run all tests in directory `tests`. You could run tests by the commands ```shell -PYTHONPATH=src/ pytest +python -m pytest --cov=pydolphinscheduler --cov-config=.coveragerc tests/ ``` -We try to keep pydolphinscheduler usable through unit test coverage. 90% test coverage is our target, but for -now, we require test coverage up to 85%, and each pull request leas than 85% would fail our CI step -`Tests coverage`. We use [coverage][coverage] to check our test coverage, and you could check it locally by -run command. +Besides run tests, it will also check the unit test [coverage][coverage] threshold, for now when test cover less than 90% +will fail the coverage, as well as our GitHub Action. + +The command above will check test coverage automatically, and you could also test the coverage by command. ```shell -coverage run && coverage report +python -m coverage run && python -m coverage report ``` It would not only run unit test but also show each file coverage which cover rate less than 100%, and `TOTAL` diff --git a/dolphinscheduler-python/pydolphinscheduler/setup.py b/dolphinscheduler-python/pydolphinscheduler/setup.py index 7b5cda86db..ffb08880df 100644 --- a/dolphinscheduler-python/pydolphinscheduler/setup.py +++ b/dolphinscheduler-python/pydolphinscheduler/setup.py @@ -57,6 +57,7 @@ test = [ "pytest>=6.2", "freezegun>=1.1", "coverage>=6.1", + "pytest-cov>=3.0", ] style = [ @@ -144,7 +145,7 @@ setup( package_dir={"": "src"}, include_package_data=True, package_data={ - "examples": ["examples.tutorial.py"], + "pydolphinscheduler": ["core/default_config.yaml"], }, platforms=["any"], classifiers=[ diff --git a/dolphinscheduler-python/pydolphinscheduler/tox.ini b/dolphinscheduler-python/pydolphinscheduler/tox.ini new file mode 100644 index 0000000000..865c862ec8 --- /dev/null +++ b/dolphinscheduler-python/pydolphinscheduler/tox.ini @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[tox] +envlist = local-ci, lint, doc-build-test, code-test, py{36,37,38,39} + +[testenv] +whitelist_externals = make + +[testenv:lint] +extras = style +commands = + python -m isort --check . + python -m black --check . + python -m flake8 + +[testenv:code-test] +extras = test +# Run both tests and coverage +commands = + python -m pytest --cov=pydolphinscheduler --cov-config={toxinidir}/.coveragerc tests/ + +[testenv:doc-build-test] +extras = doc +commands = + make -C {toxinidir}/docs clean + make -C {toxinidir}/docs html + +[testenv:local-ci] +extras = dev +commands = + {[testenv:lint]commands} + {[testenv:code-test]commands} + {[testenv:doc-build-test]commands}