Browse Source

[python] Add multiple version of document (#11391)

* Using sphinx-multiversion to support multiple version
of python api document
* change python api sidebar link
* fix whitelist_externals deprecated and add git

close: #8227
3.1.0-release
Jiajie Zhong 2 years ago committed by GitHub
parent
commit
26afd655cb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 8
      .github/workflows/py-ci.yml
  2. 4
      docs/configs/docsdev.js
  3. 23
      dolphinscheduler-python/pydolphinscheduler/DEVELOP.md
  4. 5
      dolphinscheduler-python/pydolphinscheduler/docs/Makefile
  5. 27
      dolphinscheduler-python/pydolphinscheduler/docs/source/_templates/versioning.html
  6. 46
      dolphinscheduler-python/pydolphinscheduler/docs/source/_templates/versions.html
  7. 4
      dolphinscheduler-python/pydolphinscheduler/docs/source/api.rst
  8. 38
      dolphinscheduler-python/pydolphinscheduler/docs/source/conf.py
  9. 4
      dolphinscheduler-python/pydolphinscheduler/setup.py
  10. 2
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/__init__.py
  11. 18
      dolphinscheduler-python/pydolphinscheduler/tox.ini

8
.github/workflows/py-ci.yml

@ -97,6 +97,10 @@ jobs:
timeout-minutes: 15
needs: lint
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
env-list: [doc-build, doc-build-multi]
steps:
- uses: actions/checkout@v2
- name: Set up Python 3.7
@ -106,9 +110,9 @@ jobs:
- name: Install Dependences
run: |
python -m pip install --upgrade ${{ env.DEPENDENCES }}
- name: Run Tests Build Docs
- name: Run Build Docs Tests ${{ matrix.env-list }}
run: |
python -m tox -vv -e doc-build-test
python -m tox -vv -e ${{ matrix.env-list }}
local-ci:
name: Local CI
timeout-minutes: 15

4
docs/configs/docsdev.js

@ -426,7 +426,7 @@ export default {
},
{
title: 'PyDolphinScheduler',
link: '/python/index.html',
link: '/python/dev/index.html',
},
{
title: 'Health Check',
@ -1005,7 +1005,7 @@ export default {
},
{
title: 'PyDolphinScheduler',
link: '/python/index.html',
link: '/python/dev/index.html',
},
{
title: '健康检查',

23
dolphinscheduler-python/pydolphinscheduler/DEVELOP.md

@ -146,10 +146,24 @@ python -m flake8
#### Testing
## Build Docs
## Build Document
We use [sphinx][sphinx] to build docs. Dolphinscheduler Python API CI would automatically build docs when you submit pull request in
GitHub. You may locally ensure docs could be built suceessfully in case the failure blocks CI.
GitHub. You may locally ensure docs could be built successfully in case the failure blocks CI, you can build by tox or manual.
### Build Document Automatically with tox
We integrated document build process into tox, you can build the latest document and all document(including history documents) via
single command
```shell
# Build the latest document in dev branch
tox -e doc-build
# Build all documents, which including the latest and all history documents
tox -e doc-build-multi
```
### Build Document Manually
To build docs locally, install sphinx and related python modules first via:
@ -157,13 +171,16 @@ To build docs locally, install sphinx and related python modules first via:
python -m pip install '.[doc]'
```
Then
Then go to document directory and execute the build command
```shell
cd pydolphinscheduler/docs/
make clean && make html
```
> NOTE: We support build multiple versions of documents with [sphinx-multiversion](https://holzhaus.github.io/sphinx-multiversion/master/index.html),
> you can build with command `git fetch --tags && make clean && make multiversion`
## Testing
pydolphinscheduler using [pytest][pytest] to test our codebase. GitHub Action will run our test when you create

5
dolphinscheduler-python/pydolphinscheduler/docs/Makefile

@ -24,6 +24,7 @@
# Add opts `turn warnings into errors` strict sphinx-build behavior
SPHINXOPTS ?= -W
SPHINXBUILD ?= sphinx-build
SPHINXMULTIVERSION ?= sphinx-multiversion
SOURCEDIR = source
BUILDDIR = build
@ -37,3 +38,7 @@ help:
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
%: Makefile
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
# Create multiple version of docs
multiversion:
@$(SPHINXMULTIVERSION) "$(SOURCEDIR)" "$(BUILDDIR)/html"

27
dolphinscheduler-python/pydolphinscheduler/docs/source/_templates/versioning.html

@ -0,0 +1,27 @@
{#
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
#}
{% if versions %}
<h3>{{ _('Versions') }}</h3>
<ul>
{%- for item in versions %}
<li><a href="{{ item.url }}">{{ item.name }}</a></li>
{%- endfor %}
</ul>
{% endif %}

46
dolphinscheduler-python/pydolphinscheduler/docs/source/_templates/versions.html

@ -0,0 +1,46 @@
{#
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
#}
{%- if current_version %}
<div class="rst-versions" data-toggle="rst-versions" role="note" aria-label="versions">
<span class="rst-current-version" data-toggle="rst-current-version">
<span class="fa fa-book"> Other Versions</span>
v: {{ current_version.name }}
<span class="fa fa-caret-down"></span>
</span>
<div class="rst-other-versions">
{%- if versions.tags %}
<dl>
<dt>Tags</dt>
{%- for item in versions.tags %}
<dd><a href="{{ item.url }}">{{ item.name }}</a></dd>
{%- endfor %}
</dl>
{%- endif %}
{%- if versions.branches %}
<dl>
<dt>Branches</dt>
{%- for item in versions.branches %}
<dd><a href="{{ item.url }}">{{ item.name }}</a></dd>
{%- endfor %}
</dl>
{%- endif %}
</div>
</div>
{%- endif %}

4
dolphinscheduler-python/pydolphinscheduler/docs/source/api.rst

@ -24,8 +24,8 @@ Core
.. automodule:: pydolphinscheduler.core
:inherited-members:
Sides
-----
Models
------
.. automodule:: pydolphinscheduler.models
:inherited-members:

38
dolphinscheduler-python/pydolphinscheduler/docs/source/conf.py

@ -26,17 +26,31 @@
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
# import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))
import os
import sys
from pathlib import Path
# For sphinx-multiversion, we need to build API docs of the corresponding package version, related issue:
# https://github.com/Holzhaus/sphinx-multiversion/issues/42
pkg_src_dir = (
Path(os.environ.get("SPHINX_MULTIVERSION_SOURCEDIR", default="."))
.joinpath("../../src")
.resolve()
)
sys.path.insert(0, str(pkg_src_dir))
# Debug to uncomment this to see the source path
# print("=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=")
# print(pkg_src_dir)
# [print(p) for p in sys.path]
# print("=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=")
# -- Project information -----------------------------------------------------
project = "pydolphinscheduler"
copyright = "2022, apache"
author = "apache"
author = "apache dolphinscheduler contributors"
# The full version, including alpha/beta/rc tags
release = "0.0.1"
@ -60,11 +74,25 @@ extensions = [
# Add inline tabbed content
"sphinx_inline_tabs",
"sphinx_copybutton",
"sphinx_multiversion",
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ["_templates"]
# sphinx_multiversion configuration
html_sidebars = {
"**": [
"versioning.html",
],
}
# Match all exists tag for pydolphinscheduler expect version 2.0.4(not release apache dolphinscheduler)
smv_tag_whitelist = r"^(?!2.0.4)\d+\.\d+\.\d+$"
smv_branch_whitelist = "dev"
smv_remote_whitelist = r"^(origin|upstream)$"
smv_released_pattern = "^refs/tags/.*$"
smv_outputdir_format = "versions/{ref.name}"
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.

4
dolphinscheduler-python/pydolphinscheduler/setup.py

@ -53,6 +53,10 @@ doc = [
"sphinx-click>=3.0",
"sphinx-inline-tabs",
"sphinx-copybutton>=0.4.0",
# Unreleased package have a feature we want(use correct version package for API ref), so we install from
# GitHub directly, see also:
# https://github.com/Holzhaus/sphinx-multiversion/issues/42#issuecomment-1210539786
"sphinx-multiversion @ git+https://github.com/Holzhaus/sphinx-multiversion#egg=sphinx-multiversion",
]
test = [

2
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/models/__init__.py

@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
"""Init Side package, Side package keep object related to DolphinScheduler but not in the Core part."""
"""Init Models package, keeping object related to DolphinScheduler covert from Java Gateway Service."""
from pydolphinscheduler.models.base import Base
from pydolphinscheduler.models.base_side import BaseSide

18
dolphinscheduler-python/pydolphinscheduler/tox.ini

@ -16,10 +16,12 @@
# under the License.
[tox]
envlist = local-ci, auto-lint, lint, doc-build-test, code-test, integrate-test, local-integrate-test, py{36,37,38,39}
envlist = local-ci, auto-lint, lint, doc-build, doc-build-multi, code-test, integrate-test, local-integrate-test, py{36,37,38,39}
[testenv]
whitelist_externals = make
allowlist_externals =
make
git
[testenv:auto-lint]
extras = style
@ -42,12 +44,20 @@ extras = test
commands =
python -m pytest --cov=pydolphinscheduler --cov-config={toxinidir}/.coveragerc tests/
[testenv:doc-build-test]
[testenv:doc-build]
extras = doc
commands =
make -C {toxinidir}/docs clean
make -C {toxinidir}/docs html
[testenv:doc-build-multi]
extras = doc
commands =
# Get all tags for `multiversion` subcommand
git fetch --tags
make -C {toxinidir}/docs clean
make -C {toxinidir}/docs multiversion
[testenv:integrate-test]
extras = test
commands =
@ -65,4 +75,4 @@ extras = dev
commands =
{[testenv:lint]commands}
{[testenv:code-test]commands}
{[testenv:doc-build-test]commands}
{[testenv:doc-build]commands}

Loading…
Cancel
Save