diff --git a/.github/workflows/py-ci.yml b/.github/workflows/py-ci.yml index fdc55bac10..99b79c7d11 100644 --- a/.github/workflows/py-ci.yml +++ b/.github/workflows/py-ci.yml @@ -97,6 +97,10 @@ jobs: timeout-minutes: 15 needs: lint runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + env-list: [doc-build, doc-build-multi] steps: - uses: actions/checkout@v2 - name: Set up Python 3.7 @@ -106,9 +110,9 @@ jobs: - name: Install Dependences run: | python -m pip install --upgrade ${{ env.DEPENDENCES }} - - name: Run Tests Build Docs + - name: Run Build Docs Tests ${{ matrix.env-list }} run: | - python -m tox -vv -e doc-build-test + python -m tox -vv -e ${{ matrix.env-list }} local-ci: name: Local CI timeout-minutes: 15 diff --git a/docs/configs/docsdev.js b/docs/configs/docsdev.js index 65de9eb5fa..d8d03806d1 100644 --- a/docs/configs/docsdev.js +++ b/docs/configs/docsdev.js @@ -426,7 +426,7 @@ export default { }, { title: 'PyDolphinScheduler', - link: '/python/index.html', + link: '/python/dev/index.html', }, { title: 'Health Check', @@ -1005,7 +1005,7 @@ export default { }, { title: 'PyDolphinScheduler', - link: '/python/index.html', + link: '/python/dev/index.html', }, { title: '健康检查', diff --git a/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md b/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md index c893418487..c27249311c 100644 --- a/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md +++ b/dolphinscheduler-python/pydolphinscheduler/DEVELOP.md @@ -146,10 +146,24 @@ python -m flake8 #### Testing -## Build Docs +## Build Document We use [sphinx][sphinx] to build docs. Dolphinscheduler Python API CI would automatically build docs when you submit pull request in -GitHub. You may locally ensure docs could be built suceessfully in case the failure blocks CI. +GitHub. You may locally ensure docs could be built successfully in case the failure blocks CI, you can build by tox or manual. + +### Build Document Automatically with tox + +We integrated document build process into tox, you can build the latest document and all document(including history documents) via +single command + +```shell +# Build the latest document in dev branch +tox -e doc-build +# Build all documents, which including the latest and all history documents +tox -e doc-build-multi +``` + +### Build Document Manually To build docs locally, install sphinx and related python modules first via: @@ -157,13 +171,16 @@ To build docs locally, install sphinx and related python modules first via: python -m pip install '.[doc]' ``` -Then +Then go to document directory and execute the build command ```shell cd pydolphinscheduler/docs/ make clean && make html ``` +> NOTE: We support build multiple versions of documents with [sphinx-multiversion](https://holzhaus.github.io/sphinx-multiversion/master/index.html), +> you can build with command `git fetch --tags && make clean && make multiversion` + ## Testing pydolphinscheduler using [pytest][pytest] to test our codebase. GitHub Action will run our test when you create diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/Makefile b/dolphinscheduler-python/pydolphinscheduler/docs/Makefile index 985198a779..ff2c4ebb44 100644 --- a/dolphinscheduler-python/pydolphinscheduler/docs/Makefile +++ b/dolphinscheduler-python/pydolphinscheduler/docs/Makefile @@ -24,6 +24,7 @@ # Add opts `turn warnings into errors` strict sphinx-build behavior SPHINXOPTS ?= -W SPHINXBUILD ?= sphinx-build +SPHINXMULTIVERSION ?= sphinx-multiversion SOURCEDIR = source BUILDDIR = build @@ -37,3 +38,7 @@ help: # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +# Create multiple version of docs +multiversion: + @$(SPHINXMULTIVERSION) "$(SOURCEDIR)" "$(BUILDDIR)/html" diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/_templates/versioning.html b/dolphinscheduler-python/pydolphinscheduler/docs/source/_templates/versioning.html new file mode 100644 index 0000000000..47136c45cf --- /dev/null +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/_templates/versioning.html @@ -0,0 +1,27 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +#} + +{% if versions %} +

{{ _('Versions') }}

+ +{% endif %} diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/_templates/versions.html b/dolphinscheduler-python/pydolphinscheduler/docs/source/_templates/versions.html new file mode 100644 index 0000000000..51b7271e9c --- /dev/null +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/_templates/versions.html @@ -0,0 +1,46 @@ +{# + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +#} + +{%- if current_version %} +
+ + Other Versions + v: {{ current_version.name }} + + +
+ {%- if versions.tags %} +
+
Tags
+ {%- for item in versions.tags %} +
{{ item.name }}
+ {%- endfor %} +
+ {%- endif %} + {%- if versions.branches %} +
+
Branches
+ {%- for item in versions.branches %} +
{{ item.name }}
+ {%- endfor %} +
+ {%- endif %} +
+
+{%- endif %} diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/api.rst b/dolphinscheduler-python/pydolphinscheduler/docs/source/api.rst index 570e06d6e9..b170b6f870 100644 --- a/dolphinscheduler-python/pydolphinscheduler/docs/source/api.rst +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/api.rst @@ -24,8 +24,8 @@ Core .. automodule:: pydolphinscheduler.core :inherited-members: -Sides ------ +Models +------ .. automodule:: pydolphinscheduler.models :inherited-members: diff --git a/dolphinscheduler-python/pydolphinscheduler/docs/source/conf.py b/dolphinscheduler-python/pydolphinscheduler/docs/source/conf.py index b162e0cc50..23fc117fb7 100644 --- a/dolphinscheduler-python/pydolphinscheduler/docs/source/conf.py +++ b/dolphinscheduler-python/pydolphinscheduler/docs/source/conf.py @@ -26,17 +26,31 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) + +import os +import sys +from pathlib import Path + +# For sphinx-multiversion, we need to build API docs of the corresponding package version, related issue: +# https://github.com/Holzhaus/sphinx-multiversion/issues/42 +pkg_src_dir = ( + Path(os.environ.get("SPHINX_MULTIVERSION_SOURCEDIR", default=".")) + .joinpath("../../src") + .resolve() +) +sys.path.insert(0, str(pkg_src_dir)) +# Debug to uncomment this to see the source path +# print("=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=") +# print(pkg_src_dir) +# [print(p) for p in sys.path] +# print("=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=") # -- Project information ----------------------------------------------------- project = "pydolphinscheduler" copyright = "2022, apache" -author = "apache" +author = "apache dolphinscheduler contributors" # The full version, including alpha/beta/rc tags release = "0.0.1" @@ -60,11 +74,25 @@ extensions = [ # Add inline tabbed content "sphinx_inline_tabs", "sphinx_copybutton", + "sphinx_multiversion", ] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] +# sphinx_multiversion configuration +html_sidebars = { + "**": [ + "versioning.html", + ], +} +# Match all exists tag for pydolphinscheduler expect version 2.0.4(not release apache dolphinscheduler) +smv_tag_whitelist = r"^(?!2.0.4)\d+\.\d+\.\d+$" +smv_branch_whitelist = "dev" +smv_remote_whitelist = r"^(origin|upstream)$" +smv_released_pattern = "^refs/tags/.*$" +smv_outputdir_format = "versions/{ref.name}" + # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path. diff --git a/dolphinscheduler-python/pydolphinscheduler/setup.py b/dolphinscheduler-python/pydolphinscheduler/setup.py index 5a181223e4..0edd24bd7b 100644 --- a/dolphinscheduler-python/pydolphinscheduler/setup.py +++ b/dolphinscheduler-python/pydolphinscheduler/setup.py @@ -53,6 +53,10 @@ doc = [ "sphinx-click>=3.0", "sphinx-inline-tabs", "sphinx-copybutton>=0.4.0", + # Unreleased package have a feature we want(use correct version package for API ref), so we install from + # GitHub directly, see also: + # https://github.com/Holzhaus/sphinx-multiversion/issues/42#issuecomment-1210539786 + "sphinx-multiversion @ git+https://github.com/Holzhaus/sphinx-multiversion#egg=sphinx-multiversion", ] test = [ diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/__init__.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/__init__.py index 42594002a3..b289954caa 100644 --- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/__init__.py +++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/__init__.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -"""Init Side package, Side package keep object related to DolphinScheduler but not in the Core part.""" +"""Init Models package, keeping object related to DolphinScheduler covert from Java Gateway Service.""" from pydolphinscheduler.models.base import Base from pydolphinscheduler.models.base_side import BaseSide diff --git a/dolphinscheduler-python/pydolphinscheduler/tox.ini b/dolphinscheduler-python/pydolphinscheduler/tox.ini index 4e21bcb513..067412acee 100644 --- a/dolphinscheduler-python/pydolphinscheduler/tox.ini +++ b/dolphinscheduler-python/pydolphinscheduler/tox.ini @@ -16,10 +16,12 @@ # under the License. [tox] -envlist = local-ci, auto-lint, lint, doc-build-test, code-test, integrate-test, local-integrate-test, py{36,37,38,39} +envlist = local-ci, auto-lint, lint, doc-build, doc-build-multi, code-test, integrate-test, local-integrate-test, py{36,37,38,39} [testenv] -whitelist_externals = make +allowlist_externals = + make + git [testenv:auto-lint] extras = style @@ -42,12 +44,20 @@ extras = test commands = python -m pytest --cov=pydolphinscheduler --cov-config={toxinidir}/.coveragerc tests/ -[testenv:doc-build-test] +[testenv:doc-build] extras = doc commands = make -C {toxinidir}/docs clean make -C {toxinidir}/docs html +[testenv:doc-build-multi] +extras = doc +commands = + # Get all tags for `multiversion` subcommand + git fetch --tags + make -C {toxinidir}/docs clean + make -C {toxinidir}/docs multiversion + [testenv:integrate-test] extras = test commands = @@ -65,4 +75,4 @@ extras = dev commands = {[testenv:lint]commands} {[testenv:code-test]commands} - {[testenv:doc-build-test]commands} + {[testenv:doc-build]commands}