[python] Add tox for out of box test (#8837)

* [python] Add tox for out of box test * Add `tox` for easy and complete test * Correct package data attribute * Add more dirs in flake8 ignore * Change GA from raw to tox * Fix env setter * Temp skip sanity * Rm not exists job * Add code test * fix * rm all need * fix * fix * fix * fix * fix * fix * Migrate pip upgrede to tox setting * Change install_commands to commands * fix config of install command * Add GA env to do that * Fix env error * Finial commit * remove duplicate * Change doc * Change dependence of ci * Change dependence of ci
2 years ago · 90f9aed11d
7 changed files with 153 additions and 59 deletions
--- a/.github/workflows/py-ci.yml
+++ b/.github/workflows/py-ci.yml
@ -25,22 +25,21 @@ on:
    paths:
      - 'dolphinscheduler-python/**'

+concurrency:
+  group: py-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 defaults:
  run:
    working-directory: dolphinscheduler-python/pydolphinscheduler

+# We have to update setuptools wheel to package with package_data, LICENSE, NOTICE
+env:
+  DEPENDENCES: pip setuptools wheel tox
+
 jobs:
-  sanity:
-    name: Sanity
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: true
-      - name: Sanity Check
-        uses: ./.github/actions/sanity-check
  lint:
-    name: Code Style
+    name: Lint
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
@ -48,19 +47,15 @@ jobs:
        uses: actions/setup-python@v2
        with:
          python-version: 3.7
-      - name: Install Development Dependences
-        run: pip install -e .[style]
-      - name: Run Isort Checking
-        run: isort --check .
-      - name: Run Black Checking
-        run: black --check .
-      - name: Run Flake8 Checking
-        run: flake8
+      - name: Install Dependences
+        run: |
+          python -m pip install --upgrade ${{ env.DEPENDENCES }}
+      - name: Run All Lint Check
+        run: |
+          python -m tox -vv -e lint
  pytest:
    name: Pytest
-    needs:
-      - lint
-      - sanity
+    needs: lint
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
@ -73,16 +68,15 @@ jobs:
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
-      - name: Install dependencies & pydolphinscheduler
+      - name: Install Dependences
        run: |
-          pip install -e .[test]
-      - name: Run tests
+          python -m pip install --upgrade ${{ env.DEPENDENCES }}
+      - name: Run All Tests
        run: |
-          pytest
-  coverage:
-    name: Tests coverage
-    needs:
-      - pytest
+          python -m tox -vv -e code-test
+  doc-build:
+    name: Docs Build Test
+    needs: lint
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
@ -90,14 +84,17 @@ jobs:
        uses: actions/setup-python@v2
        with:
          python-version: 3.7
-      - name: Install Development Dependences
+      - name: Install Dependences
        run: |
-          pip install -e .[test]
-      - name: Run Tests && Check coverage
-        run: coverage run && coverage report
-  doc-build:
-    name: Document Build Test
-    needs: pytest
+          python -m pip install --upgrade ${{ env.DEPENDENCES }}
+      - name: Run Tests Build Docs
+        run: |
+          python -m tox -vv -e doc-build-test
+  verify-local-ci:
+    name: Local CI
+    needs:
+      - pytest
+      - doc-build
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
@ -105,9 +102,9 @@ jobs:
        uses: actions/setup-python@v2
        with:
          python-version: 3.7
-      - name: Install Development Dependences
+      - name: Install Dependences
+        run: |
+          python -m pip install --upgrade ${{ env.DEPENDENCES }}
+      - name: Run Tests Build Docs
        run: |
-          pip install -e .[doc]
-      - name: Test Build Document
-        working-directory: dolphinscheduler-python/pydolphinscheduler/docs
-        run: make clean && make html
+          python -m tox -vv -e local-ci
--- a/.gitignore
+++ b/.gitignore
@ -55,6 +55,7 @@ dolphinscheduler-worker/logs
 # ------------------
 # Cache
 __pycache__/
+.tox/

 # Build
 build/
--- a/dolphinscheduler-python/pydolphinscheduler/.coveragerc
+++ b/dolphinscheduler-python/pydolphinscheduler/.coveragerc
@ -20,8 +20,10 @@ command_line = -m pytest
 omit = 
    # Ignore all test cases in tests/
    tests/*
+    # Ignore examples directory
+    */pydolphinscheduler/examples/*
    # TODO. Temporary ignore java_gateway file, because we could not find good way to test it.
-    src/pydolphinscheduler/java_gateway.py
+    */pydolphinscheduler/java_gateway.py

 [report]
 # Don’t report files that are 100% covered
--- a/dolphinscheduler-python/pydolphinscheduler/.flake8
+++ b/dolphinscheduler-python/pydolphinscheduler/.flake8
@ -26,7 +26,9 @@ exclude =
    old,
    build,
    dist,
-    htmlcov
+    htmlcov,
+    .tox,
+    dist,
 ignore = 
    # It's clear and not need to add docstring
    D107,  # D107: Don't require docstrings on __init__
--- a/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md
+++ b/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md
@ -60,7 +60,53 @@ pydolphinscheduler tasks object, we use tasks to define exact job we want Dolphi
 we only support `shell` task to execute shell task. [This link][all-task] list all tasks support in DolphinScheduler
 and would be implemented in the further.

-## Code Style
+## Test Your Code
+
+Linting and tests is very important for open source project, so we pay more attention to it. We have continuous
+integration service run by GitHub Action to test whether the patch is good or not, which you could jump to
+section [With GitHub Action](#with-github-action) see more detail.
+
+And to make more convenience to local tests, we also have the way to run your [test automated with tox](#automated-testing-with-tox)
+locally. It is helpful when your try to find out the detail when continuous integration in GitHub Action failed,
+or you have a great patch and want to test local first.
+
+Besides [automated testing with tox](#automated-testing-with-tox) locally, we also have a [manual way](#manually)
+run tests. And it is scattered commands to reproduce each step of the integration test we told about.
+
+* Remote
+  * [With GitHub Action](#with-github-action)
+* Local
+  * [Automated Testing With tox](#automated-testing-with-tox)
+  * [Manually](#manually)
+
+### With GitHub Action
+
+GitHub Action test in various environment for pydolphinscheduler, including different python version in
+`3.6|3.7|3.8|3.9` and operating system `linux|macOS|windows`. It will trigger and run automatically when you
+submit pull requests to `apache/dolphinscheduler`. 
+
+### Automated Testing With tox
+
+[tox](https://tox.wiki) is a package aims to automate and standardize testing in Python, both our continuous
+integration and local test use it to run actual task. To use it, you should install it first
+
+```shell
+python -m pip install --upgrade tox
+```
+
+After installation, you could run a single command to run all the tests, it is almost like test in GitHub Action
+but not so much different environment.
+
+```shell
+tox -e local-ci
+```
+
+It will take a while when you run it the first time, because it has to install dependencies and make some prepare,
+and the next time you run it will be faster.
+
+### Manually
+
+#### Code Style

 We use [isort][isort] to automatically keep Python imports alphabetically, and use [Black][black] for code
 formatter and [Flake8][flake8] for pep8 checker. If you use [pycharm][pycharm]or [IntelliJ IDEA][idea],
@ -74,33 +120,30 @@ GitHub, you could also run static check locally.
 # but Flake8 just hint when code style not match pep8

 # Run Isort
-isort .
+python -m isort .

 # Run Black
-black .
+python -m black .

 # Run Flake8
-flake8
+python -m flake8
 ```

-## Testing
+#### Testing

-pydolphinscheduler using [pytest][pytest] to test our codebase. GitHub Action will run our test when you create
-pull request or commit to dev branch, with python version `3.6|3.7|3.8|3.9` and operating system `linux|macOS|windows`.
-
-To test locally, you could directly run pytest after set `PYTHONPATH` 
+pydolphinscheduler using [pytest][pytest] to run all tests in directory `tests`. You could run tests by the commands

 ```shell
-PYTHONPATH=src/ pytest
+python -m pytest --cov=pydolphinscheduler --cov-config=.coveragerc tests/
 ```

-We try to keep pydolphinscheduler usable through unit test coverage. 90% test coverage is our target, but for
-now, we require test coverage up to 85%, and each pull request leas than 85% would fail our CI step
-`Tests coverage`. We use [coverage][coverage] to check our test coverage, and you could check it locally by
-run command.
+Besides run tests, it will also check the unit test [coverage][coverage] threshold, for now when test cover less than 90%
+will fail the coverage, as well as our GitHub Action.
+
+The command above will check test coverage automatically, and you could also test the coverage by command.

 ```shell
-coverage run && coverage report
+python -m coverage run && python -m  coverage report
 ```

 It would not only run unit test but also show each file coverage which cover rate less than 100%, and `TOTAL`
--- a/dolphinscheduler-python/pydolphinscheduler/setup.py
+++ b/dolphinscheduler-python/pydolphinscheduler/setup.py
@ -57,6 +57,7 @@ test = [
    "pytest>=6.2",
    "freezegun>=1.1",
    "coverage>=6.1",
+    "pytest-cov>=3.0",
 ]

 style = [
@ -144,7 +145,7 @@ setup(
    package_dir={"": "src"},
    include_package_data=True,
    package_data={
-        "examples": ["examples.tutorial.py"],
+        "pydolphinscheduler": ["core/default_config.yaml"],
    },
    platforms=["any"],
    classifiers=[
--- a/dolphinscheduler-python/pydolphinscheduler/tox.ini
+++ b/dolphinscheduler-python/pydolphinscheduler/tox.ini
@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[tox]
+envlist = local-ci, lint, doc-build-test, code-test, py{36,37,38,39}
+
+[testenv]
+whitelist_externals = make
+
+[testenv:lint]
+extras = style
+commands =
+    python -m isort --check .
+    python -m black --check .
+    python -m flake8
+    
+[testenv:code-test]
+extras = test
+# Run both tests and coverage
+commands =
+    python -m pytest --cov=pydolphinscheduler --cov-config={toxinidir}/.coveragerc tests/
+
+[testenv:doc-build-test]
+extras = doc
+commands =
+    make -C {toxinidir}/docs clean
+    make -C {toxinidir}/docs html
+
+[testenv:local-ci]
+extras = dev
+commands =
+    {[testenv:lint]commands}
+    {[testenv:code-test]commands}
+    {[testenv:doc-build-test]commands}