Jay Chung
2 years ago
committed by
zhuangchong
190 changed files with 40 additions and 14887 deletions
@ -1,205 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
name: Python API |
||||
|
||||
on: |
||||
push: |
||||
branches: |
||||
- dev |
||||
paths: |
||||
- 'dolphinscheduler-python/**' |
||||
pull_request: |
||||
|
||||
concurrency: |
||||
group: py-${{ github.event.pull_request.number || github.ref }} |
||||
cancel-in-progress: true |
||||
|
||||
defaults: |
||||
run: |
||||
working-directory: dolphinscheduler-python/pydolphinscheduler |
||||
|
||||
# We have to update setuptools wheel to package with package_data, LICENSE, NOTICE |
||||
env: |
||||
DEPENDENCES: pip setuptools wheel tox |
||||
|
||||
jobs: |
||||
paths-filter: |
||||
name: Python-Path-Filter |
||||
runs-on: ubuntu-latest |
||||
outputs: |
||||
not-docs: ${{ steps.filter.outputs.not-docs }} |
||||
py-change: ${{ steps.filter.outputs.py-change }} |
||||
steps: |
||||
- uses: actions/checkout@v2 |
||||
- uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721 |
||||
id: filter |
||||
with: |
||||
filters: | |
||||
not-docs: |
||||
- '!(docs/**)' |
||||
py-change: |
||||
- 'dolphinscheduler-python/pydolphinscheduler/**' |
||||
lint: |
||||
name: Lint |
||||
if: ${{ (needs.paths-filter.outputs.py-change == 'true') || (github.event_name == 'push') }} |
||||
timeout-minutes: 15 |
||||
needs: paths-filter |
||||
runs-on: ubuntu-latest |
||||
steps: |
||||
- uses: actions/checkout@v2 |
||||
- name: Set up Python 3.7 |
||||
uses: actions/setup-python@v4 |
||||
with: |
||||
python-version: 3.7 |
||||
- name: Install Dependences |
||||
run: | |
||||
python -m pip install --upgrade ${{ env.DEPENDENCES }} |
||||
- name: Run All Lint Check |
||||
run: | |
||||
python -m tox -vv -e lint |
||||
pytest: |
||||
name: Pytest |
||||
timeout-minutes: 15 |
||||
needs: lint |
||||
runs-on: ${{ matrix.os }} |
||||
strategy: |
||||
fail-fast: false |
||||
matrix: |
||||
# YAML parse `3.10` to `3.1`, so we have to add quotes for `'3.10'`, see also: |
||||
# https://github.com/actions/setup-python/issues/160#issuecomment-724485470 |
||||
python-version: [3.6, 3.7, 3.8, 3.9, '3.10', 3.11-dev] |
||||
os: [ubuntu-latest, macOS-latest, windows-latest] |
||||
# Skip because dependence [py4j](https://pypi.org/project/py4j/) not work on those environments |
||||
exclude: |
||||
- os: windows-latest |
||||
python-version: '3.10' |
||||
- os: windows-latest |
||||
python-version: 3.11-dev |
||||
steps: |
||||
- uses: actions/checkout@v2 |
||||
- name: Set up Python ${{ matrix.python-version }} |
||||
uses: actions/setup-python@v4 |
||||
with: |
||||
python-version: ${{ matrix.python-version }} |
||||
- name: Install Dependences |
||||
run: | |
||||
python -m pip install --upgrade ${{ env.DEPENDENCES }} |
||||
- name: Run All Tests |
||||
run: | |
||||
python -m tox -vv -e code-test |
||||
doc-build: |
||||
name: Docs Build Test |
||||
timeout-minutes: 15 |
||||
needs: lint |
||||
runs-on: ubuntu-latest |
||||
strategy: |
||||
fail-fast: false |
||||
matrix: |
||||
env-list: [doc-build, doc-build-multi] |
||||
steps: |
||||
- uses: actions/checkout@v2 |
||||
- name: Set up Python 3.7 |
||||
uses: actions/setup-python@v4 |
||||
with: |
||||
python-version: 3.7 |
||||
- name: Install Dependences |
||||
run: | |
||||
python -m pip install --upgrade ${{ env.DEPENDENCES }} |
||||
- name: Run Build Docs Tests ${{ matrix.env-list }} |
||||
run: | |
||||
python -m tox -vv -e ${{ matrix.env-list }} |
||||
local-ci: |
||||
name: Local CI |
||||
timeout-minutes: 15 |
||||
needs: |
||||
- pytest |
||||
- doc-build |
||||
runs-on: ubuntu-latest |
||||
steps: |
||||
- uses: actions/checkout@v2 |
||||
- name: Set up Python 3.7 |
||||
uses: actions/setup-python@v4 |
||||
with: |
||||
python-version: 3.7 |
||||
- name: Install Dependences |
||||
run: | |
||||
python -m pip install --upgrade ${{ env.DEPENDENCES }} |
||||
- name: Run Tests Build Docs |
||||
run: | |
||||
python -m tox -vv -e local-ci |
||||
integrate-test: |
||||
name: Integrate Test |
||||
if: ${{ (needs.paths-filter.outputs.not-docs == 'true') || (github.event_name == 'push') }} |
||||
runs-on: ubuntu-latest |
||||
needs: paths-filter |
||||
timeout-minutes: 30 |
||||
steps: |
||||
- uses: actions/checkout@v2 |
||||
with: |
||||
submodules: true |
||||
- name: Sanity Check |
||||
uses: ./.github/actions/sanity-check |
||||
with: |
||||
token: ${{ secrets.GITHUB_TOKEN }} |
||||
- name: Cache local Maven repository |
||||
uses: actions/cache@v3 |
||||
with: |
||||
path: ~/.m2/repository |
||||
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} |
||||
restore-keys: ${{ runner.os }}-maven- |
||||
# Switch to project root directory to run mvnw command |
||||
- name: Build Image |
||||
working-directory: ./ |
||||
run: | |
||||
./mvnw -B clean install \ |
||||
-Dmaven.test.skip \ |
||||
-Dmaven.javadoc.skip \ |
||||
-Dcheckstyle.skip=true \ |
||||
-Pdocker,release -Ddocker.tag=ci \ |
||||
-pl dolphinscheduler-standalone-server -am |
||||
- name: Set up Python 3.7 |
||||
uses: actions/setup-python@v4 |
||||
with: |
||||
python-version: 3.7 |
||||
- name: Install Dependences |
||||
run: | |
||||
python -m pip install --upgrade ${{ env.DEPENDENCES }} |
||||
- name: Run Integrate Tests |
||||
run: | |
||||
python -m tox -vv -e integrate-test |
||||
result: |
||||
name: Python |
||||
runs-on: ubuntu-latest |
||||
timeout-minutes: 30 |
||||
needs: [ paths-filter, local-ci, integrate-test ] |
||||
if: always() |
||||
steps: |
||||
- name: Status |
||||
# We need change CWD to current directory to avoid global default working directory not exists |
||||
working-directory: ./ |
||||
run: | |
||||
if [[ ${{ needs.paths-filter.outputs.not-docs }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then |
||||
echo "Only document change, skip both python unit and integrate test!" |
||||
exit 0 |
||||
fi |
||||
if [[ ${{ needs.paths-filter.outputs.py-change }} == 'false' && ${{ needs.integrate-test.result }} == 'success' && ${{ github.event_name }} == 'pull_request' ]]; then |
||||
echo "No python code change, and integrate test pass!" |
||||
exit 0 |
||||
fi |
||||
if [[ ${{ needs.integrate-test.result }} != 'success' || ${{ needs.local-ci.result }} != 'success' ]]; then |
||||
echo "py-ci Failed!" |
||||
exit -1 |
||||
fi |
||||
@ -1,34 +0,0 @@
|
||||
<!-- |
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more |
||||
~ contributor license agreements. See the NOTICE file distributed with |
||||
~ this work for additional information regarding copyright ownership. |
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
~ (the "License"); you may not use this file except in compliance with |
||||
~ the License. You may obtain a copy of the License at |
||||
~ |
||||
~ http://www.apache.org/licenses/LICENSE-2.0 |
||||
~ |
||||
~ Unless required by applicable law or agreed to in writing, software |
||||
~ distributed under the License is distributed on an "AS IS" BASIS, |
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
~ See the License for the specific language governing permissions and |
||||
~ limitations under the License. |
||||
--> |
||||
|
||||
<assembly |
||||
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0" |
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" |
||||
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd"> |
||||
<id>python-api</id> |
||||
<formats> |
||||
<format>dir</format> |
||||
</formats> |
||||
<includeBaseDirectory>false</includeBaseDirectory> |
||||
|
||||
<fileSets> |
||||
<fileSet> |
||||
<directory>${basedir}/../dolphinscheduler-python/pydolphinscheduler/dist</directory> |
||||
<outputDirectory>.</outputDirectory> |
||||
</fileSet> |
||||
</fileSets> |
||||
</assembly> |
||||
@ -1,34 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
[run] |
||||
command_line = -m pytest |
||||
omit = |
||||
# Ignore all test cases in tests/ |
||||
tests/* |
||||
# Ignore examples directory |
||||
*/pydolphinscheduler/examples/* |
||||
# TODO. Temporary ignore java_gateway file, because we could not find good way to test it. |
||||
*/pydolphinscheduler/java_gateway.py |
||||
|
||||
[report] |
||||
# Don’t report files that are 100% covered |
||||
skip_covered = True |
||||
show_missing = True |
||||
precision = 2 |
||||
# Report will fail when coverage under 90.00% |
||||
fail_under = 90 |
||||
@ -1,19 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
[settings] |
||||
profile=black |
||||
@ -1,56 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# See https://pre-commit.com for more information |
||||
# See https://pre-commit.com/hooks.html for more hooks |
||||
|
||||
default_stages: [commit, push] |
||||
default_language_version: |
||||
# force all python hooks to run python3 |
||||
python: python3 |
||||
repos: |
||||
- repo: https://github.com/pycqa/isort |
||||
rev: 5.10.1 |
||||
hooks: |
||||
- id: isort |
||||
name: isort (python) |
||||
- repo: https://github.com/psf/black |
||||
rev: 22.1.0 |
||||
hooks: |
||||
- id: black |
||||
- repo: https://github.com/pycqa/flake8 |
||||
rev: 4.0.1 |
||||
hooks: |
||||
- id: flake8 |
||||
additional_dependencies: [ |
||||
'flake8-docstrings>=1.6', |
||||
'flake8-black>=0.2', |
||||
] |
||||
# pre-commit run in the root, so we have to point out the full path of configuration |
||||
args: [ |
||||
--config, |
||||
dolphinscheduler-python/pydolphinscheduler/.flake8 |
||||
] |
||||
- repo: https://github.com/pycqa/autoflake |
||||
rev: v1.4 |
||||
hooks: |
||||
- id: autoflake |
||||
args: [ |
||||
--remove-all-unused-imports, |
||||
--ignore-init-module-imports, |
||||
--in-place |
||||
] |
||||
@ -1,228 +0,0 @@
|
||||
Apache License |
||||
Version 2.0, January 2004 |
||||
http://www.apache.org/licenses/ |
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION |
||||
|
||||
1. Definitions. |
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction, |
||||
and distribution as defined by Sections 1 through 9 of this document. |
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by |
||||
the copyright owner that is granting the License. |
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all |
||||
other entities that control, are controlled by, or are under common |
||||
control with that entity. For the purposes of this definition, |
||||
"control" means (i) the power, direct or indirect, to cause the |
||||
direction or management of such entity, whether by contract or |
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the |
||||
outstanding shares, or (iii) beneficial ownership of such entity. |
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity |
||||
exercising permissions granted by this License. |
||||
|
||||
"Source" form shall mean the preferred form for making modifications, |
||||
including but not limited to software source code, documentation |
||||
source, and configuration files. |
||||
|
||||
"Object" form shall mean any form resulting from mechanical |
||||
transformation or translation of a Source form, including but |
||||
not limited to compiled object code, generated documentation, |
||||
and conversions to other media types. |
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or |
||||
Object form, made available under the License, as indicated by a |
||||
copyright notice that is included in or attached to the work |
||||
(an example is provided in the Appendix below). |
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object |
||||
form, that is based on (or derived from) the Work and for which the |
||||
editorial revisions, annotations, elaborations, or other modifications |
||||
represent, as a whole, an original work of authorship. For the purposes |
||||
of this License, Derivative Works shall not include works that remain |
||||
separable from, or merely link (or bind by name) to the interfaces of, |
||||
the Work and Derivative Works thereof. |
||||
|
||||
"Contribution" shall mean any work of authorship, including |
||||
the original version of the Work and any modifications or additions |
||||
to that Work or Derivative Works thereof, that is intentionally |
||||
submitted to Licensor for inclusion in the Work by the copyright owner |
||||
or by an individual or Legal Entity authorized to submit on behalf of |
||||
the copyright owner. For the purposes of this definition, "submitted" |
||||
means any form of electronic, verbal, or written communication sent |
||||
to the Licensor or its representatives, including but not limited to |
||||
communication on electronic mailing lists, source code control systems, |
||||
and issue tracking systems that are managed by, or on behalf of, the |
||||
Licensor for the purpose of discussing and improving the Work, but |
||||
excluding communication that is conspicuously marked or otherwise |
||||
designated in writing by the copyright owner as "Not a Contribution." |
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity |
||||
on behalf of whom a Contribution has been received by Licensor and |
||||
subsequently incorporated within the Work. |
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of |
||||
this License, each Contributor hereby grants to You a perpetual, |
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable |
||||
copyright license to reproduce, prepare Derivative Works of, |
||||
publicly display, publicly perform, sublicense, and distribute the |
||||
Work and such Derivative Works in Source or Object form. |
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of |
||||
this License, each Contributor hereby grants to You a perpetual, |
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable |
||||
(except as stated in this section) patent license to make, have made, |
||||
use, offer to sell, sell, import, and otherwise transfer the Work, |
||||
where such license applies only to those patent claims licensable |
||||
by such Contributor that are necessarily infringed by their |
||||
Contribution(s) alone or by combination of their Contribution(s) |
||||
with the Work to which such Contribution(s) was submitted. If You |
||||
institute patent litigation against any entity (including a |
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work |
||||
or a Contribution incorporated within the Work constitutes direct |
||||
or contributory patent infringement, then any patent licenses |
||||
granted to You under this License for that Work shall terminate |
||||
as of the date such litigation is filed. |
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the |
||||
Work or Derivative Works thereof in any medium, with or without |
||||
modifications, and in Source or Object form, provided that You |
||||
meet the following conditions: |
||||
|
||||
(a) You must give any other recipients of the Work or |
||||
Derivative Works a copy of this License; and |
||||
|
||||
(b) You must cause any modified files to carry prominent notices |
||||
stating that You changed the files; and |
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works |
||||
that You distribute, all copyright, patent, trademark, and |
||||
attribution notices from the Source form of the Work, |
||||
excluding those notices that do not pertain to any part of |
||||
the Derivative Works; and |
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its |
||||
distribution, then any Derivative Works that You distribute must |
||||
include a readable copy of the attribution notices contained |
||||
within such NOTICE file, excluding those notices that do not |
||||
pertain to any part of the Derivative Works, in at least one |
||||
of the following places: within a NOTICE text file distributed |
||||
as part of the Derivative Works; within the Source form or |
||||
documentation, if provided along with the Derivative Works; or, |
||||
within a display generated by the Derivative Works, if and |
||||
wherever such third-party notices normally appear. The contents |
||||
of the NOTICE file are for informational purposes only and |
||||
do not modify the License. You may add Your own attribution |
||||
notices within Derivative Works that You distribute, alongside |
||||
or as an addendum to the NOTICE text from the Work, provided |
||||
that such additional attribution notices cannot be construed |
||||
as modifying the License. |
||||
|
||||
You may add Your own copyright statement to Your modifications and |
||||
may provide additional or different license terms and conditions |
||||
for use, reproduction, or distribution of Your modifications, or |
||||
for any such Derivative Works as a whole, provided Your use, |
||||
reproduction, and distribution of the Work otherwise complies with |
||||
the conditions stated in this License. |
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise, |
||||
any Contribution intentionally submitted for inclusion in the Work |
||||
by You to the Licensor shall be under the terms and conditions of |
||||
this License, without any additional terms or conditions. |
||||
Notwithstanding the above, nothing herein shall supersede or modify |
||||
the terms of any separate license agreement you may have executed |
||||
with Licensor regarding such Contributions. |
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade |
||||
names, trademarks, service marks, or product names of the Licensor, |
||||
except as required for reasonable and customary use in describing the |
||||
origin of the Work and reproducing the content of the NOTICE file. |
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or |
||||
agreed to in writing, Licensor provides the Work (and each |
||||
Contributor provides its Contributions) on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or |
||||
implied, including, without limitation, any warranties or conditions |
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A |
||||
PARTICULAR PURPOSE. You are solely responsible for determining the |
||||
appropriateness of using or redistributing the Work and assume any |
||||
risks associated with Your exercise of permissions under this License. |
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory, |
||||
whether in tort (including negligence), contract, or otherwise, |
||||
unless required by applicable law (such as deliberate and grossly |
||||
negligent acts) or agreed to in writing, shall any Contributor be |
||||
liable to You for damages, including any direct, indirect, special, |
||||
incidental, or consequential damages of any character arising as a |
||||
result of this License or out of the use or inability to use the |
||||
Work (including but not limited to damages for loss of goodwill, |
||||
work stoppage, computer failure or malfunction, or any and all |
||||
other commercial damages or losses), even if such Contributor |
||||
has been advised of the possibility of such damages. |
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing |
||||
the Work or Derivative Works thereof, You may choose to offer, |
||||
and charge a fee for, acceptance of support, warranty, indemnity, |
||||
or other liability obligations and/or rights consistent with this |
||||
License. However, in accepting such obligations, You may act only |
||||
on Your own behalf and on Your sole responsibility, not on behalf |
||||
of any other Contributor, and only if You agree to indemnify, |
||||
defend, and hold each Contributor harmless for any liability |
||||
incurred by, or claims asserted against, such Contributor by reason |
||||
of your accepting any such warranty or additional liability. |
||||
|
||||
END OF TERMS AND CONDITIONS |
||||
|
||||
APPENDIX: How to apply the Apache License to your work. |
||||
|
||||
To apply the Apache License to your work, attach the following |
||||
boilerplate notice, with the fields enclosed by brackets "{}" |
||||
replaced with your own identifying information. (Don't include |
||||
the brackets!) The text should be enclosed in the appropriate |
||||
comment syntax for the file format. We also recommend that a |
||||
file or class name and description of purpose be included on the |
||||
same "printed page" as the copyright notice for easier |
||||
identification within third-party archives. |
||||
|
||||
Copyright {yyyy} {name of copyright owner} |
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); |
||||
you may not use this file except in compliance with the License. |
||||
You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
Unless required by applicable law or agreed to in writing, software |
||||
distributed under the License is distributed on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
See the License for the specific language governing permissions and |
||||
limitations under the License. |
||||
|
||||
============================================================================ |
||||
Apache DolphinScheduler Python API SUBCOMPONENTS: |
||||
|
||||
The Apache DolphinScheduler Python API project contains subcomponents |
||||
with separate copyright notices and license terms. Your use of the source |
||||
code for the these subcomponents is subject to the terms and conditions |
||||
of the following licenses. |
||||
|
||||
======================================================================== |
||||
BSD licenses |
||||
======================================================================== |
||||
|
||||
The following components are provided under a BSD license. See project link for details. |
||||
The text of each license is also included at licenses/LICENSE-[project].txt. |
||||
|
||||
py4j v0.10 (https://github.com/py4j/py4j) |
||||
click v8.0 (https://github.com/pallets/click) |
||||
|
||||
======================================================================== |
||||
MIT licenses |
||||
======================================================================== |
||||
|
||||
The following components are provided under the MIT License. See project link for details. |
||||
The text of each license is also included at licenses/LICENSE-[project].txt. |
||||
|
||||
ruamel.yaml v0.17 (https://sourceforge.net/projects/ruamel-yaml/) |
||||
@ -1,5 +0,0 @@
|
||||
Apache DolphinScheduler |
||||
Copyright 2017-2022 The Apache Software Foundation |
||||
|
||||
This product includes software developed at |
||||
The Apache Software Foundation (http://www.apache.org/). |
||||
@ -1,35 +0,0 @@
|
||||
<!-- |
||||
Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
--> |
||||
|
||||
# Release |
||||
|
||||
**PyDolphinScheduler** office release is in [ASF Distribution Directory](https://downloads.apache.org/dolphinscheduler/), |
||||
and it should be released together with [apache-dolphinscheduler](https://github.com/apache/dolphinscheduler). |
||||
|
||||
## To ASF Distribution Directory |
||||
|
||||
You could release to [ASF Distribution Directory](https://downloads.apache.org/dolphinscheduler/) according to |
||||
[release guide](../../docs/docs/en/contribute/release/release-prepare.md) in DolphinScheduler |
||||
website. |
||||
|
||||
## To PyPi |
||||
|
||||
[PyPI](https://pypi.org), Python Package Index, is a repository of software for the Python programming language. |
||||
User could install Python package from it. Release to PyPi make user easier to install and try PyDolphinScheduler, |
||||
There is an official way to package project from [PyPA](https://packaging.python.org/en/latest/tutorials/packaging-projects) |
||||
@ -1,44 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# Minimal makefile for Sphinx documentation
|
||||
#
|
||||
|
||||
# You can set these variables from the command line, and also
|
||||
# from the environment for the first two.
|
||||
|
||||
# Add opts `turn warnings into errors` strict sphinx-build behavior
|
||||
SPHINXOPTS ?= -W
|
||||
SPHINXBUILD ?= sphinx-build
|
||||
SPHINXMULTIVERSION ?= sphinx-multiversion
|
||||
SOURCEDIR = source
|
||||
BUILDDIR = build
|
||||
|
||||
# Put it first so that "make" without argument is like "make help".
|
||||
help: |
||||
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
||||
|
||||
.PHONY: help Makefile |
||||
|
||||
# Catch-all target: route all unknown targets to Sphinx using the new
|
||||
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
|
||||
%: Makefile |
||||
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
|
||||
|
||||
# Create multiple version of docs
|
||||
multiversion: |
||||
@$(SPHINXMULTIVERSION) "$(SOURCEDIR)" "$(BUILDDIR)/html"
|
||||
@ -1,54 +0,0 @@
|
||||
REM Licensed to the Apache Software Foundation (ASF) under one |
||||
REM or more contributor license agreements. See the NOTICE file |
||||
REM distributed with this work for additional information |
||||
REM regarding copyright ownership. The ASF licenses this file |
||||
REM to you under the Apache License, Version 2.0 (the |
||||
REM "License"); you may not use this file except in compliance |
||||
REM with the License. You may obtain a copy of the License at |
||||
REM |
||||
REM http://www.apache.org/licenses/LICENSE-2.0 |
||||
REM |
||||
REM Unless required by applicable law or agreed to in writing, |
||||
REM software distributed under the License is distributed on an |
||||
REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
REM KIND, either express or implied. See the License for the |
||||
REM specific language governing permissions and limitations |
||||
REM under the License. |
||||
|
||||
@ECHO OFF |
||||
|
||||
pushd %~dp0 |
||||
|
||||
REM Command file for Sphinx documentation |
||||
|
||||
if "%SPHINXBUILD%" == "" ( |
||||
set SPHINXBUILD=sphinx-build |
||||
) |
||||
set SOURCEDIR=source |
||||
set BUILDDIR=build |
||||
REM Add opts `turn warnings into errors` strict sphinx-build behavior |
||||
set SPHINXOPTS=-W |
||||
|
||||
if "%1" == "" goto help |
||||
|
||||
%SPHINXBUILD% >NUL 2>NUL |
||||
if errorlevel 9009 ( |
||||
echo. |
||||
echo.The 'sphinx-build' command was not found. Make sure you have Sphinx |
||||
echo.installed, then set the SPHINXBUILD environment variable to point |
||||
echo.to the full path of the 'sphinx-build' executable. Alternatively you |
||||
echo.may add the Sphinx directory to PATH. |
||||
echo. |
||||
echo.If you don't have Sphinx installed, grab it from |
||||
echo.https://www.sphinx-doc.org/ |
||||
exit /b 1 |
||||
) |
||||
|
||||
%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% |
||||
goto end |
||||
|
||||
:help |
||||
%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% |
||||
|
||||
:end |
||||
popd |
||||
@ -1,27 +0,0 @@
|
||||
{# |
||||
Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
#} |
||||
|
||||
{% if versions %} |
||||
<h3>{{ _('Versions') }}</h3> |
||||
<ul> |
||||
{%- for item in versions %} |
||||
<li><a href="{{ item.url }}">{{ item.name }}</a></li> |
||||
{%- endfor %} |
||||
</ul> |
||||
{% endif %} |
||||
@ -1,46 +0,0 @@
|
||||
{# |
||||
Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
#} |
||||
|
||||
{%- if current_version %} |
||||
<div class="rst-versions" data-toggle="rst-versions" role="note" aria-label="versions"> |
||||
<span class="rst-current-version" data-toggle="rst-current-version"> |
||||
<span class="fa fa-book"> Other Versions</span> |
||||
v: {{ current_version.name }} |
||||
<span class="fa fa-caret-down"></span> |
||||
</span> |
||||
<div class="rst-other-versions"> |
||||
{%- if versions.tags %} |
||||
<dl> |
||||
<dt>Tags</dt> |
||||
{%- for item in versions.tags %} |
||||
<dd><a href="{{ item.url }}">{{ item.name }}</a></dd> |
||||
{%- endfor %} |
||||
</dl> |
||||
{%- endif %} |
||||
{%- if versions.branches %} |
||||
<dl> |
||||
<dt>Branches</dt> |
||||
{%- for item in versions.branches %} |
||||
<dd><a href="{{ item.url }}">{{ item.name }}</a></dd> |
||||
{%- endfor %} |
||||
</dl> |
||||
{%- endif %} |
||||
</div> |
||||
</div> |
||||
{%- endif %} |
||||
@ -1,47 +0,0 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
API |
||||
=== |
||||
|
||||
Core |
||||
---- |
||||
|
||||
.. automodule:: pydolphinscheduler.core |
||||
:inherited-members: |
||||
|
||||
Models |
||||
------ |
||||
|
||||
.. automodule:: pydolphinscheduler.models |
||||
:inherited-members: |
||||
|
||||
Tasks |
||||
----- |
||||
|
||||
.. automodule:: pydolphinscheduler.tasks |
||||
:inherited-members: |
||||
|
||||
Constants |
||||
--------- |
||||
|
||||
.. automodule:: pydolphinscheduler.constants |
||||
|
||||
Exceptions |
||||
---------- |
||||
|
||||
.. automodule:: pydolphinscheduler.exceptions |
||||
@ -1,36 +0,0 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
Command Line Interface |
||||
====================== |
||||
|
||||
*PyDolphinScheduler* have mechanism call CLI(command line interface) to help user control it in Shell. |
||||
|
||||
Prepare |
||||
------- |
||||
|
||||
You have to :ref:`install PyDolphinScheduler <start:installing pydolphinscheduler>` first before you using |
||||
its CLI |
||||
|
||||
Usage |
||||
----- |
||||
|
||||
Here is basic usage about the command line of *PyDolphinScheduler* |
||||
|
||||
.. click:: pydolphinscheduler.cli.commands:cli |
||||
:prog: pydolphinscheduler |
||||
:nested: full |
||||
@ -1,151 +0,0 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
Concepts |
||||
======== |
||||
|
||||
In this section, you would know the core concepts of *PyDolphinScheduler*. |
||||
|
||||
Process Definition |
||||
------------------ |
||||
|
||||
Process definition describe the whole things except `tasks`_ and `tasks dependence`_, which including |
||||
name, schedule interval, schedule start time and end time. You would know scheduler |
||||
|
||||
Process definition could be initialized in normal assign statement or in context manger. |
||||
|
||||
.. code-block:: python |
||||
|
||||
# Initialization with assign statement |
||||
pd = ProcessDefinition(name="my first process definition") |
||||
|
||||
# Or context manger |
||||
with ProcessDefinition(name="my first process definition") as pd: |
||||
pd.submit() |
||||
|
||||
Process definition is the main object communicate between *PyDolphinScheduler* and DolphinScheduler daemon. |
||||
After process definition and task is be declared, you could use `submit` and `run` notify server your definition. |
||||
|
||||
If you just want to submit your definition and create workflow, without run it, you should use attribute `submit`. |
||||
But if you want to run the workflow after you submit it, you could use attribute `run`. |
||||
|
||||
.. code-block:: python |
||||
|
||||
# Just submit definition, without run it |
||||
pd.submit() |
||||
|
||||
# Both submit and run definition |
||||
pd.run() |
||||
|
||||
Schedule |
||||
~~~~~~~~ |
||||
|
||||
We use parameter `schedule` determine the schedule interval of workflow, *PyDolphinScheduler* support seven |
||||
asterisks expression, and each of the meaning of position as below |
||||
|
||||
.. code-block:: text |
||||
|
||||
* * * * * * * |
||||
┬ ┬ ┬ ┬ ┬ ┬ ┬ |
||||
│ │ │ │ │ │ │ |
||||
│ │ │ │ │ │ └─── year |
||||
│ │ │ │ │ └───── day of week (0 - 7) (0 to 6 are Sunday to Saturday, or use names; 7 is Sunday, the same as 0) |
||||
│ │ │ │ └─────── month (1 - 12) |
||||
│ │ │ └───────── day of month (1 - 31) |
||||
│ │ └─────────── hour (0 - 23) |
||||
│ └───────────── min (0 - 59) |
||||
└─────────────── second (0 - 59) |
||||
|
||||
Here we add some example crontab: |
||||
|
||||
- `0 0 0 * * ? *`: Workflow execute every day at 00:00:00. |
||||
- `10 2 * * * ? *`: Workflow execute hourly day at ten pass two. |
||||
- `10,11 20 0 1,2 * ? *`: Workflow execute first and second day of month at 00:20:10 and 00:20:11. |
||||
|
||||
Tenant |
||||
~~~~~~ |
||||
|
||||
Tenant is the user who run task command in machine or in virtual machine. it could be assign by simple string. |
||||
|
||||
.. code-block:: python |
||||
|
||||
# |
||||
pd = ProcessDefinition(name="process definition tenant", tenant="tenant_exists") |
||||
|
||||
.. note:: |
||||
|
||||
Make should tenant exists in target machine, otherwise it will raise an error when you try to run command |
||||
|
||||
Tasks |
||||
----- |
||||
|
||||
Task is the minimum unit running actual job, and it is nodes of DAG, aka directed acyclic graph. You could define |
||||
what you want to in the task. It have some required parameter to make uniqueness and definition. |
||||
|
||||
Here we use :py:meth:`pydolphinscheduler.tasks.Shell` as example, parameter `name` and `command` is required and must be provider. Parameter |
||||
`name` set name to the task, and parameter `command` declare the command you wish to run in this task. |
||||
|
||||
.. code-block:: python |
||||
|
||||
# We named this task as "shell", and just run command `echo shell task` |
||||
shell_task = Shell(name="shell", command="echo shell task") |
||||
|
||||
If you want to see all type of tasks, you could see :doc:`tasks/index`. |
||||
|
||||
Tasks Dependence |
||||
~~~~~~~~~~~~~~~~ |
||||
|
||||
You could define many tasks in on single `Process Definition`_. If all those task is in parallel processing, |
||||
then you could leave them alone without adding any additional information. But if there have some tasks should |
||||
not be run unless pre task in workflow have be done, we should set task dependence to them. Set tasks dependence |
||||
have two mainly way and both of them is easy. You could use bitwise operator `>>` and `<<`, or task attribute |
||||
`set_downstream` and `set_upstream` to do it. |
||||
|
||||
.. code-block:: python |
||||
|
||||
# Set task1 as task2 upstream |
||||
task1 >> task2 |
||||
# You could use attribute `set_downstream` too, is same as `task1 >> task2` |
||||
task1.set_downstream(task2) |
||||
|
||||
# Set task1 as task2 downstream |
||||
task1 << task2 |
||||
# It is same as attribute `set_upstream` |
||||
task1.set_upstream(task2) |
||||
|
||||
# Beside, we could set dependence between task and sequence of tasks, |
||||
# we set `task1` is upstream to both `task2` and `task3`. It is useful |
||||
# for some tasks have same dependence. |
||||
task1 >> [task2, task3] |
||||
|
||||
Task With Process Definition |
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
||||
|
||||
In most of data orchestration cases, you should assigned attribute `process_definition` to task instance to |
||||
decide workflow of task. You could set `process_definition` in both normal assign or in context manger mode |
||||
|
||||
.. code-block:: python |
||||
|
||||
# Normal assign, have to explicit declaration and pass `ProcessDefinition` instance to task |
||||
pd = ProcessDefinition(name="my first process definition") |
||||
shell_task = Shell(name="shell", command="echo shell task", process_definition=pd) |
||||
|
||||
# Context manger, `ProcessDefinition` instance pd would implicit declaration to task |
||||
with ProcessDefinition(name="my first process definition") as pd: |
||||
shell_task = Shell(name="shell", command="echo shell task", |
||||
|
||||
With both `Process Definition`_, `Tasks`_ and `Tasks Dependence`_, we could build a workflow with multiple tasks. |
||||
@ -1,121 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Configuration file for the Sphinx documentation builder. |
||||
# |
||||
# This file only contains a selection of the most common options. For a full |
||||
# list see the documentation: |
||||
# https://www.sphinx-doc.org/en/master/usage/configuration.html |
||||
|
||||
# -- Path setup -------------------------------------------------------------- |
||||
|
||||
# If extensions (or modules to document with autodoc) are in another directory, |
||||
# add these directories to sys.path here. If the directory is relative to the |
||||
# documentation root, use os.path.abspath to make it absolute, like shown here. |
||||
|
||||
import os |
||||
import sys |
||||
from pathlib import Path |
||||
|
||||
# For sphinx-multiversion, we need to build API docs of the corresponding package version, related issue: |
||||
# https://github.com/Holzhaus/sphinx-multiversion/issues/42 |
||||
pkg_src_dir = ( |
||||
Path(os.environ.get("SPHINX_MULTIVERSION_SOURCEDIR", default=".")) |
||||
.joinpath("../../src") |
||||
.resolve() |
||||
) |
||||
sys.path.insert(0, str(pkg_src_dir)) |
||||
# Debug to uncomment this to see the source path |
||||
# print("=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=") |
||||
# print(pkg_src_dir) |
||||
# [print(p) for p in sys.path] |
||||
# print("=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=") |
||||
|
||||
|
||||
# -- Project information ----------------------------------------------------- |
||||
|
||||
project = "pydolphinscheduler" |
||||
copyright = "2022, apache" |
||||
author = "apache dolphinscheduler contributors" |
||||
|
||||
# The full version, including alpha/beta/rc tags |
||||
release = "0.0.1" |
||||
|
||||
|
||||
# -- General configuration --------------------------------------------------- |
||||
|
||||
# Add any Sphinx extension module names here, as strings. They can be |
||||
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom |
||||
# ones. |
||||
extensions = [ |
||||
# Measures durations of Sphinx processing |
||||
"sphinx.ext.duration", |
||||
# Semi-automatic make docstrings to document |
||||
"sphinx.ext.autodoc", |
||||
"sphinx.ext.viewcode", |
||||
"sphinx.ext.autosectionlabel", |
||||
"sphinx_rtd_theme", |
||||
# Documenting command line interface |
||||
"sphinx_click.ext", |
||||
# Add inline tabbed content |
||||
"sphinx_inline_tabs", |
||||
"sphinx_copybutton", |
||||
"sphinx_multiversion", |
||||
] |
||||
|
||||
# Add any paths that contain templates here, relative to this directory. |
||||
templates_path = ["_templates"] |
||||
|
||||
# sphinx_multiversion configuration |
||||
html_sidebars = { |
||||
"**": [ |
||||
"versioning.html", |
||||
], |
||||
} |
||||
# Match all exists tag for pydolphinscheduler expect version 2.0.4(not release apache dolphinscheduler) |
||||
smv_tag_whitelist = r"^(?!2.0.4)\d+\.\d+\.\d+$" |
||||
smv_branch_whitelist = "dev" |
||||
smv_remote_whitelist = r"^(origin|upstream)$" |
||||
smv_released_pattern = "^refs/tags/.*$" |
||||
smv_outputdir_format = "versions/{ref.name}" |
||||
|
||||
# List of patterns, relative to source directory, that match files and |
||||
# directories to ignore when looking for source files. |
||||
# This pattern also affects html_static_path and html_extra_path. |
||||
exclude_patterns = [] |
||||
|
||||
autodoc_default_options = { |
||||
"members": True, |
||||
"show-inheritance": True, |
||||
"private-members": True, |
||||
"undoc-members": True, |
||||
"member-order": "groupwise", |
||||
} |
||||
|
||||
autosectionlabel_prefix_document = True |
||||
|
||||
# -- Options for HTML output ------------------------------------------------- |
||||
|
||||
# The theme to use for HTML and HTML Help pages. See the documentation for |
||||
# a list of builtin themes. |
||||
# |
||||
html_theme = "sphinx_rtd_theme" |
||||
|
||||
# Add any paths that contain custom static files (such as style sheets) here, |
||||
# relative to this directory. They are copied after the builtin static files, |
||||
# so a file named "default.css" will overwrite the builtin "default.css". |
||||
html_static_path = ["_static"] |
||||
@ -1,218 +0,0 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
Configuration |
||||
============= |
||||
|
||||
pydolphinscheduler has a built-in module setting necessary configuration to start and run your workflow code. |
||||
You could directly use them if you only want to run a quick start or for a simple job like POC. But if you |
||||
want to deep use pydolphinscheduler and even use it in production. You should probably need to modify and |
||||
change the built-in configuration. |
||||
|
||||
We have two ways to modify the configuration: |
||||
|
||||
- `Using Environment Variables`_: The more lightweight way to modify the configuration. it is useful in |
||||
containerization scenarios, like docker and k8s, or when you like to temporarily override configs in the |
||||
configuration file. |
||||
- `Using Configuration File`_: The more general way to modify the configuration. It is useful when you want |
||||
to persist and manage configuration files in one single file. |
||||
|
||||
Using Environment Variables |
||||
--------------------------- |
||||
|
||||
You could change the configuration by adding or modifying the operating system's environment variables. No |
||||
matter what way you used, as long as you can successfully modify the environment variables. We use two common |
||||
ways, `Bash <by bash>`_ and `Python OS Module <by python os module>`_, as examples: |
||||
|
||||
By Bash |
||||
^^^^^^^ |
||||
|
||||
Setting environment variables via `Bash` is the most straightforward and easiest way. We give some examples about |
||||
how to change them by Bash. |
||||
|
||||
.. code-block:: bash |
||||
|
||||
# Modify Java Gateway Address |
||||
export PYDS_JAVA_GATEWAY_ADDRESS="192.168.1.1" |
||||
|
||||
# Modify Workflow Default User |
||||
export PYDS_WORKFLOW_USER="custom-user" |
||||
|
||||
After executing the commands above, both ``PYDS_JAVA_GATEWAY_ADDRESS`` and ``PYDS_WORKFLOW_USER`` will be changed. |
||||
The next time you execute and submit your workflow, it will submit to host `192.168.1.1`, and with workflow's user |
||||
named `custom-user`. |
||||
|
||||
By Python OS Module |
||||
^^^^^^^^^^^^^^^^^^^ |
||||
|
||||
pydolphinscheduler is a Python API for Apache DolphinScheduler, and you could modify or add system environment |
||||
variables via Python ``os`` module. In this example, we change variables as the same value as we change in |
||||
`Bash <by bash>`_. It will take effect the next time you run your workflow, and call workflow ``run`` or ``submit`` |
||||
method next to ``os.environ`` statement. |
||||
|
||||
.. code-block:: python |
||||
|
||||
import os |
||||
# Modify Java Gateway Address |
||||
os.environ["PYDS_JAVA_GATEWAY_ADDRESS"] = "192.168.1.1" |
||||
|
||||
# Modify Workflow Default User |
||||
os.environ["PYDS_WORKFLOW_USER"] = "custom-user" |
||||
|
||||
All Configurations in Environment Variables |
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ |
||||
|
||||
All environment variables as below, and you could modify their value via `Bash <by bash>`_ or `Python OS Module <by python os module>`_ |
||||
|
||||
+------------------+------------------------------------+--------------------------------------------------------------------------------------------------------------------+ |
||||
| Variable Section | Variable Name | description | |
||||
+==================+====================================+====================================================================================================================+ |
||||
| | ``PYDS_JAVA_GATEWAY_ADDRESS`` | Default Java gateway address, will use its value when it is set. | |
||||
+ +------------------------------------+--------------------------------------------------------------------------------------------------------------------+ |
||||
| Java Gateway | ``PYDS_JAVA_GATEWAY_PORT`` | Default Java gateway port, will use its value when it is set. | |
||||
+ +------------------------------------+--------------------------------------------------------------------------------------------------------------------+ |
||||
| | ``PYDS_JAVA_GATEWAY_AUTO_CONVERT`` | Default boolean Java gateway auto convert, will use its value when it is set. | |
||||
+------------------+------------------------------------+--------------------------------------------------------------------------------------------------------------------+ |
||||
| | ``PYDS_USER_NAME`` | Default user name, will use when user's ``name`` when does not specify. | |
||||
+ +------------------------------------+--------------------------------------------------------------------------------------------------------------------+ |
||||
| | ``PYDS_USER_PASSWORD`` | Default user password, will use when user's ``password`` when does not specify. | |
||||
+ +------------------------------------+--------------------------------------------------------------------------------------------------------------------+ |
||||
| Default User | ``PYDS_USER_EMAIL`` | Default user email, will use when user's ``email`` when does not specify. | |
||||
+ +------------------------------------+--------------------------------------------------------------------------------------------------------------------+ |
||||
| | ``PYDS_USER_PHONE`` | Default user phone, will use when user's ``phone`` when does not specify. | |
||||
+ +------------------------------------+--------------------------------------------------------------------------------------------------------------------+ |
||||
| | ``PYDS_USER_STATE`` | Default user state, will use when user's ``state`` when does not specify. | |
||||
+------------------+------------------------------------+--------------------------------------------------------------------------------------------------------------------+ |
||||
| | ``PYDS_WORKFLOW_PROJECT`` | Default workflow project name, will use its value when workflow does not specify the attribute ``project``. | |
||||
+ +------------------------------------+--------------------------------------------------------------------------------------------------------------------+ |
||||
| | ``PYDS_WORKFLOW_TENANT`` | Default workflow tenant, will use its value when workflow does not specify the attribute ``tenant``. | |
||||
+ +------------------------------------+--------------------------------------------------------------------------------------------------------------------+ |
||||
| Default Workflow | ``PYDS_WORKFLOW_USER`` | Default workflow user, will use its value when workflow does not specify the attribute ``user``. | |
||||
+ +------------------------------------+--------------------------------------------------------------------------------------------------------------------+ |
||||
| | ``PYDS_WORKFLOW_QUEUE`` | Default workflow queue, will use its value when workflow does not specify the attribute ``queue``. | |
||||
+ +------------------------------------+--------------------------------------------------------------------------------------------------------------------+ |
||||
| | ``PYDS_WORKFLOW_WORKER_GROUP`` | Default workflow worker group, will use its value when workflow does not specify the attribute ``worker_group``. | |
||||
+ +------------------------------------+--------------------------------------------------------------------------------------------------------------------+ |
||||
| | ``PYDS_WORKFLOW_RELEASE_STATE`` | Default workflow release state, will use its value when workflow does not specify the attribute ``release_state``. | |
||||
+ +------------------------------------+--------------------------------------------------------------------------------------------------------------------+ |
||||
| | ``PYDS_WORKFLOW_TIME_ZONE`` | Default workflow worker group, will use its value when workflow does not specify the attribute ``timezone``. | |
||||
+ +------------------------------------+--------------------------------------------------------------------------------------------------------------------+ |
||||
| | ``PYDS_WORKFLOW_WARNING_TYPE`` | Default workflow warning type, will use its value when workflow does not specify the attribute ``warning_type``. | |
||||
+------------------+------------------------------------+--------------------------------------------------------------------------------------------------------------------+ |
||||
|
||||
.. note:: |
||||
|
||||
The scope of setting configuration via environment variable is in the workflow, and it will not change the |
||||
value of the configuration file. The :doc:`CLI <cli>` command ``config --get`` and ``config --set`` operate |
||||
the value of the configuration file, so the command ``config --get`` may return a different value from what |
||||
you set in the environment variable, and command ``config --get`` will never change your environment variable. |
||||
|
||||
Using Configuration File |
||||
------------------------ |
||||
|
||||
If you want to persist and manage configuration in a file instead of environment variables, or maybe you want |
||||
want to save your configuration file to a version control system, like Git or SVN, and the way to change |
||||
configuration by file is the best choice. |
||||
|
||||
Export Configuration File |
||||
^^^^^^^^^^^^^^^^^^^^^^^^^ |
||||
|
||||
pydolphinscheduler allows you to change the built-in configurations via CLI or editor you like. pydolphinscheduler |
||||
integrated built-in configurations in its package, but you could also export it locally by CLI |
||||
|
||||
.. code-block:: bash |
||||
|
||||
pydolphinscheduler config --init |
||||
|
||||
And it will create a new YAML file in the path `~/pydolphinscheduler/config.yaml` by default. If you want to export |
||||
it to another path, you should set `PYDS_HOME` before you run command :code:`pydolphinscheduler config --init`. |
||||
|
||||
.. code-block:: bash |
||||
|
||||
export PYDS_HOME=<CUSTOM_PATH> |
||||
pydolphinscheduler config --init |
||||
|
||||
After that, your configuration file will export into `<CUSTOM_PATH>/config.yaml` instead of the default path. |
||||
|
||||
Change Configuration |
||||
^^^^^^^^^^^^^^^^^^^^ |
||||
|
||||
In section `export configuration file`_ you export the configuration file locally, and as a local file, you could |
||||
edit it with any editor you like. After you save your change in your editor, the latest configuration will work |
||||
when you run your workflow code. |
||||
|
||||
You could also query or change the configuration via CLI :code:`config --get <config>` or :code:`config --get <config> <val>`. |
||||
Both `--get` and `--set` could be called one or more times in single command, and you could only set the leaf |
||||
node of the configuration but could get the parent configuration, there are simple examples below: |
||||
|
||||
.. code-block:: bash |
||||
|
||||
# Get single configuration in the leaf node, |
||||
# The output look like below: |
||||
# java_gateway.address = 127.0.0.1 |
||||
pydolphinscheduler config --get java_gateway.address |
||||
|
||||
# Get multiple configuration in the leaf node, |
||||
# The output look like below: |
||||
# java_gateway.address = 127.0.0.1 |
||||
# java_gateway.port = 25333 |
||||
pydolphinscheduler config --get java_gateway.address --get java_gateway.port |
||||
|
||||
|
||||
# Get parent configuration which contain multiple leaf nodes, |
||||
# The output look like below: |
||||
# java_gateway = ordereddict([('address', '127.0.0.1'), ('port', 25333), ('auto_convert', True)]) |
||||
pydolphinscheduler config --get java_gateway |
||||
|
||||
# Set single configuration, |
||||
# The output look like below: |
||||
# Set configuration done. |
||||
pydolphinscheduler config --set java_gateway.address 192.168.1.1 |
||||
|
||||
# Set multiple configuration |
||||
# The output look like below: |
||||
# Set configuration done. |
||||
pydolphinscheduler config --set java_gateway.address 192.168.1.1 --set java_gateway.port 25334 |
||||
|
||||
# Set configuration not in leaf node will fail |
||||
# The output look like below: |
||||
# Raise error. |
||||
pydolphinscheduler config --set java_gateway 192.168.1.1,25334,True |
||||
|
||||
For more information about our CLI, you could see document :doc:`cli`. |
||||
|
||||
All Configurations in File |
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^ |
||||
|
||||
Here are all our configurations for pydolphinscheduler. |
||||
|
||||
.. literalinclude:: ../../src/pydolphinscheduler/default_config.yaml |
||||
:language: yaml |
||||
:lines: 18- |
||||
|
||||
Priority |
||||
-------- |
||||
|
||||
We have two ways to modify the configuration and there is a built-in config in pydolphinscheduler too. It is |
||||
very important to understand the priority of the configuration when you use them. The overview of configuration |
||||
priority is. |
||||
|
||||
``Environment Variables > Configurations File > Built-in Configurations`` |
||||
|
||||
This means that your setting in environment variables or configurations file will overwrite the built-in one. |
||||
And you could temporarily modify configurations by setting environment variables without modifying the global |
||||
config in the configuration file. |
||||
@ -1,30 +0,0 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
HOWTOs |
||||
====== |
||||
|
||||
pydolphinscheduler HOWTOs are documents that cover a single, specific topic, and attempt to cover it fairly |
||||
completely. This collection is an effort to foster documentation that is more detailed than the :doc:`../concept` |
||||
and :doc:`../tutorial`. |
||||
|
||||
Currently, the HOWTOs are: |
||||
|
||||
.. toctree:: |
||||
:maxdepth: 2 |
||||
|
||||
remote-submit |
||||
@ -1,51 +0,0 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
Submit Your Code from Different machine |
||||
======================================= |
||||
|
||||
Generally, we use pydolphinscheduler as a client to DolphinScheduler, and consider we may change our workflow |
||||
code frequently, the best practice is running :ref:`python gateway service <start:start python gateway service>` |
||||
in your server machine and submit the workflow code from your development machine, like a laptop or PC. This behavior |
||||
is supported by pydolphinscheduler out of box with one or two single command lines. |
||||
|
||||
Export Configuration File |
||||
------------------------- |
||||
|
||||
.. code-block:: bash |
||||
|
||||
pydolphinscheduler config --init |
||||
|
||||
your could find more detail in :ref:`configuration exporting <config:export configuration file>` |
||||
|
||||
Run API Server in Other Host |
||||
---------------------------- |
||||
|
||||
.. code-block:: bash |
||||
|
||||
pydolphinscheduler config --set java_gateway.address <your-api-server-ip-or-hostname> |
||||
|
||||
your could find more detail in :ref:`configuration setting <config:change configuration>` |
||||
|
||||
Run API Server in Other Port |
||||
---------------------------- |
||||
|
||||
.. code-block:: bash |
||||
|
||||
pydolphinscheduler config --set java_gateway.port <your-python-gateway-service-port> |
||||
|
||||
your could find more detail in :ref:`configuration setting <config:change configuration>` |
||||
@ -1,171 +0,0 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
Getting Started |
||||
=============== |
||||
|
||||
To get started with *PyDolphinScheduler* you must ensure python and pip |
||||
installed on your machine, if you're already set up, you can skip straight |
||||
to `Installing PyDolphinScheduler`_, otherwise please continue with |
||||
`Installing Python`_. |
||||
|
||||
Installing Python |
||||
----------------- |
||||
|
||||
How to install `python` and `pip` depends on what operating system |
||||
you're using. The python wiki provides up to date |
||||
`instructions for all platforms here`_. When you entering the website |
||||
and choice your operating system, you would be offered the choice and |
||||
select python version. *PyDolphinScheduler* recommend use version above |
||||
Python 3.6 and we highly recommend you install *Stable Releases* instead |
||||
of *Pre-releases*. |
||||
|
||||
After you have download and installed Python, you should open your terminal, |
||||
typing and running :code:`python --version` to check whether the installation |
||||
is correct or not. If all thing good, you could see the version in console |
||||
without error(here is a example after Python 3.8.7 installed) |
||||
|
||||
.. code-block:: bash |
||||
|
||||
python --version |
||||
|
||||
Will see detail of Python version, such as *Python 3.8.7* |
||||
|
||||
Installing PyDolphinScheduler |
||||
----------------------------- |
||||
|
||||
After Python is already installed on your machine following section |
||||
`installing Python`_, it easy to *PyDolphinScheduler* by pip. |
||||
|
||||
.. code-block:: bash |
||||
|
||||
python -m pip install apache-dolphinscheduler |
||||
|
||||
The latest version of *PyDolphinScheduler* would be installed after you run above |
||||
command in your terminal. You could go and `start Python Gateway Service`_ to finish |
||||
the prepare, and then go to :doc:`tutorial` to make your hand dirty. But if you |
||||
want to install the unreleased version of *PyDolphinScheduler*, you could go and see |
||||
section `installing PyDolphinScheduler in dev branch`_ for more detail. |
||||
|
||||
.. note:: |
||||
|
||||
Currently, we released multiple pre-release package in PyPI, you can see all released package |
||||
including pre-release in `release history <https://pypi.org/project/apache-dolphinscheduler/#history>`_. |
||||
You can fix the the package version if you want to install pre-release package, for example if |
||||
you want to install version `3.0.0-beta-2` package, you can run command |
||||
:code:`python -m pip install apache-dolphinscheduler==3.0.0b2`. |
||||
|
||||
Installing PyDolphinScheduler In DEV Branch |
||||
------------------------------------------- |
||||
|
||||
Because the project is developing and some of the features still not release. |
||||
If you want to try some thing unreleased you could install from the source code |
||||
which we hold in GitHub |
||||
|
||||
.. code-block:: bash |
||||
|
||||
# Clone Apache DolphinScheduler repository |
||||
git clone git@github.com:apache/dolphinscheduler.git |
||||
# Install PyDolphinScheduler in develop mode |
||||
cd dolphinscheduler-python/pydolphinscheduler && python -m pip install -e . |
||||
|
||||
After you installed *PyDolphinScheduler*, please remember `start Python Gateway Service`_ |
||||
which waiting for *PyDolphinScheduler*'s workflow definition require. |
||||
|
||||
Above command will clone whole dolphinscheduler source code to local, maybe you want to install latest pydolphinscheduler |
||||
package directly and do not care about other code(including Python gateway service code), you can execute command |
||||
|
||||
.. code-block:: bash |
||||
|
||||
# Must escape the '&' character by adding '\' |
||||
pip install -e "git+https://github.com/apache/dolphinscheduler.git#egg=apache-dolphinscheduler&subdirectory=dolphinscheduler-python/pydolphinscheduler" |
||||
|
||||
Start Python Gateway Service |
||||
---------------------------- |
||||
|
||||
Since **PyDolphinScheduler** is Python API for `Apache DolphinScheduler`_, it |
||||
could define workflow and tasks structure, but could not run it unless you |
||||
`install Apache DolphinScheduler`_ and start its API server which including |
||||
Python gateway service in it. We only and some key steps here and you could |
||||
go `install Apache DolphinScheduler`_ for more detail |
||||
|
||||
.. code-block:: bash |
||||
|
||||
# Start DolphinScheduler api-server which including python gateway service |
||||
./bin/dolphinscheduler-daemon.sh start api-server |
||||
|
||||
To check whether the server is alive or not, you could run :code:`jps`. And |
||||
the server is health if keyword `ApiApplicationServer` in the console. |
||||
|
||||
.. code-block:: bash |
||||
|
||||
jps |
||||
# .... |
||||
# 201472 ApiApplicationServer |
||||
# .... |
||||
|
||||
.. note:: |
||||
|
||||
Please make sure you already enabled started Python gateway service along with `api-server`. The configuration is in |
||||
yaml config path `python-gateway.enabled : true` in api-server's configuration path in `api-server/conf/application.yaml`. |
||||
The default value is true and Python gateway service start when api server is been started. |
||||
|
||||
Run an Example |
||||
-------------- |
||||
|
||||
Before run an example for pydolphinscheduler, you should get the example code from it source code. You could run |
||||
single bash command to get it |
||||
|
||||
.. code-block:: bash |
||||
|
||||
wget https://raw.githubusercontent.com/apache/dolphinscheduler/dev/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial.py |
||||
|
||||
or you could copy-paste the content from `tutorial source code`_. And then you could run the example in your |
||||
terminal |
||||
|
||||
.. code-block:: bash |
||||
|
||||
python tutorial.py |
||||
|
||||
If you want to submit your workflow to a remote API server, which means that your workflow script is different |
||||
from the API server, you should first change pydolphinscheduler configuration and then submit the workflow script |
||||
|
||||
.. code-block:: bash |
||||
|
||||
pydolphinscheduler config --init |
||||
pydolphinscheduler config --set java_gateway.address <YOUR-API-SERVER-IP-OR-HOSTNAME> |
||||
python tutorial.py |
||||
|
||||
.. note:: |
||||
|
||||
You could see more information in :doc:`config` about all the configurations pydolphinscheduler supported. |
||||
|
||||
After that, you could go and see your DolphinScheduler web UI to find out a new workflow created by pydolphinscheduler, |
||||
and the path of web UI is `Project -> Workflow -> Workflow Definition`. |
||||
|
||||
|
||||
What's More |
||||
----------- |
||||
|
||||
If you do not familiar with *PyDolphinScheduler*, you could go to :doc:`tutorial` and see how it works. But |
||||
if you already know the basic usage or concept of *PyDolphinScheduler*, you could go and play with all |
||||
:doc:`tasks/index` *PyDolphinScheduler* supports, or see our :doc:`howto/index` about useful cases. |
||||
|
||||
.. _`instructions for all platforms here`: https://wiki.python.org/moin/BeginnersGuide/Download |
||||
.. _`Apache DolphinScheduler`: https://dolphinscheduler.apache.org |
||||
.. _`install Apache DolphinScheduler`: https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/guide/installation/standalone.html |
||||
.. _`tutorial source code`: https://raw.githubusercontent.com/apache/dolphinscheduler/dev/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial.py |
||||
@ -1,40 +0,0 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
Condition |
||||
========= |
||||
|
||||
A condition task type's example and dive into information of **PyDolphinScheduler**. |
||||
|
||||
Example |
||||
------- |
||||
|
||||
.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_condition_example.py |
||||
:start-after: [start workflow_declare] |
||||
:end-before: [end workflow_declare] |
||||
|
||||
Dive Into |
||||
--------- |
||||
|
||||
.. automodule:: pydolphinscheduler.tasks.condition |
||||
|
||||
YAML file example |
||||
----------------- |
||||
|
||||
.. literalinclude:: ../../../examples/yaml_define/Condition.yaml |
||||
:start-after: # under the License. |
||||
:language: yaml |
||||
@ -1,46 +0,0 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
Datax |
||||
===== |
||||
|
||||
A DataX task type's example and dive into information of **PyDolphinScheduler**. |
||||
|
||||
Example |
||||
------- |
||||
|
||||
.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_datax_example.py |
||||
:start-after: [start workflow_declare] |
||||
:end-before: [end workflow_declare] |
||||
|
||||
Dive Into |
||||
--------- |
||||
|
||||
.. automodule:: pydolphinscheduler.tasks.datax |
||||
|
||||
YAML file example |
||||
----------------- |
||||
|
||||
.. literalinclude:: ../../../examples/yaml_define/DataX.yaml |
||||
:start-after: # under the License. |
||||
:language: yaml |
||||
|
||||
|
||||
example_datax.json: |
||||
|
||||
.. literalinclude:: ../../../examples/yaml_define/example_datax.json |
||||
:language: json |
||||
@ -1,47 +0,0 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
Dependent |
||||
========= |
||||
|
||||
A dependent task type's example and dive into information of **PyDolphinScheduler**. |
||||
|
||||
Example |
||||
------- |
||||
|
||||
.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_dependent_example.py |
||||
:start-after: [start workflow_declare] |
||||
:end-before: [end workflow_declare] |
||||
|
||||
Dive Into |
||||
--------- |
||||
|
||||
.. automodule:: pydolphinscheduler.tasks.dependent |
||||
|
||||
|
||||
YAML file example |
||||
----------------- |
||||
|
||||
.. literalinclude:: ../../../examples/yaml_define/Dependent.yaml |
||||
:start-after: # under the License. |
||||
:language: yaml |
||||
|
||||
Dependent_External.yaml: |
||||
|
||||
.. literalinclude:: ../../../examples/yaml_define/Dependent_External.yaml |
||||
:start-after: # under the License. |
||||
:language: yaml |
||||
@ -1,41 +0,0 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
DVC |
||||
=== |
||||
|
||||
A DVC task type's example and dive into information of **PyDolphinScheduler**. |
||||
|
||||
Example |
||||
------- |
||||
|
||||
.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_dvc_example.py |
||||
:start-after: [start workflow_declare] |
||||
:end-before: [end workflow_declare] |
||||
|
||||
Dive Into |
||||
--------- |
||||
|
||||
.. automodule:: pydolphinscheduler.tasks.dvc |
||||
|
||||
|
||||
YAML file example |
||||
----------------- |
||||
|
||||
.. literalinclude:: ../../../examples/yaml_define/Dvc.yaml |
||||
:start-after: # under the License. |
||||
:language: yaml |
||||
@ -1,40 +0,0 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
Flink |
||||
===== |
||||
|
||||
A flink task type's example and dive into information of **PyDolphinScheduler**. |
||||
|
||||
Example |
||||
------- |
||||
|
||||
.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_flink_example.py |
||||
:start-after: [start workflow_declare] |
||||
:end-before: [end workflow_declare] |
||||
|
||||
Dive Into |
||||
--------- |
||||
|
||||
.. automodule:: pydolphinscheduler.tasks.flink |
||||
|
||||
YAML file example |
||||
----------------- |
||||
|
||||
.. literalinclude:: ../../../examples/yaml_define/Flink.yaml |
||||
:start-after: # under the License. |
||||
:language: yaml |
||||
@ -1,33 +0,0 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
Python Function Wrapper |
||||
======================= |
||||
|
||||
A decorator covert Python function into pydolphinscheduler's task. |
||||
|
||||
Example |
||||
------- |
||||
|
||||
.. literalinclude:: ../../../src/pydolphinscheduler/examples/tutorial_decorator.py |
||||
:start-after: [start tutorial] |
||||
:end-before: [end tutorial] |
||||
|
||||
Dive Into |
||||
--------- |
||||
|
||||
.. automodule:: pydolphinscheduler.tasks.func_wrap |
||||
@ -1,29 +0,0 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
HTTP |
||||
==== |
||||
|
||||
.. automodule:: pydolphinscheduler.tasks.http |
||||
|
||||
|
||||
YAML file example |
||||
----------------- |
||||
|
||||
.. literalinclude:: ../../../examples/yaml_define/Http.yaml |
||||
:start-after: # under the License. |
||||
:language: yaml |
||||
@ -1,48 +0,0 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
Tasks |
||||
===== |
||||
|
||||
In this section |
||||
|
||||
.. toctree:: |
||||
:maxdepth: 1 |
||||
|
||||
func_wrap |
||||
shell |
||||
sql |
||||
python |
||||
http |
||||
|
||||
switch |
||||
condition |
||||
dependent |
||||
|
||||
spark |
||||
flink |
||||
map_reduce |
||||
procedure |
||||
|
||||
datax |
||||
sub_process |
||||
|
||||
sagemaker |
||||
mlflow |
||||
openmldb |
||||
pytorch |
||||
dvc |
||||
@ -1,42 +0,0 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
Map Reduce |
||||
========== |
||||
|
||||
|
||||
A Map Reduce task type's example and dive into information of **PyDolphinScheduler**. |
||||
|
||||
Example |
||||
------- |
||||
|
||||
.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_map_reduce_example.py |
||||
:start-after: [start workflow_declare] |
||||
:end-before: [end workflow_declare] |
||||
|
||||
Dive Into |
||||
--------- |
||||
|
||||
.. automodule:: pydolphinscheduler.tasks.map_reduce |
||||
|
||||
|
||||
YAML file example |
||||
----------------- |
||||
|
||||
.. literalinclude:: ../../../examples/yaml_define/MapReduce.yaml |
||||
:start-after: # under the License. |
||||
:language: yaml |
||||
@ -1,42 +0,0 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
MLflow |
||||
========= |
||||
|
||||
|
||||
A MLflow task type's example and dive into information of **PyDolphinScheduler**. |
||||
|
||||
Example |
||||
------- |
||||
|
||||
.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_mlflow_example.py |
||||
:start-after: [start workflow_declare] |
||||
:end-before: [end workflow_declare] |
||||
|
||||
Dive Into |
||||
--------- |
||||
|
||||
.. automodule:: pydolphinscheduler.tasks.mlflow |
||||
|
||||
|
||||
YAML file example |
||||
----------------- |
||||
|
||||
.. literalinclude:: ../../../examples/yaml_define/mlflow.yaml |
||||
:start-after: # under the License. |
||||
:language: yaml |
||||
@ -1,42 +0,0 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
OpenMLDB |
||||
========= |
||||
|
||||
|
||||
A OpenMLDB task type's example and dive into information of **PyDolphinScheduler**. |
||||
|
||||
Example |
||||
------- |
||||
|
||||
.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_openmldb_example.py |
||||
:start-after: [start workflow_declare] |
||||
:end-before: [end workflow_declare] |
||||
|
||||
Dive Into |
||||
--------- |
||||
|
||||
.. automodule:: pydolphinscheduler.tasks.openmldb |
||||
|
||||
|
||||
YAML file example |
||||
----------------- |
||||
|
||||
.. literalinclude:: ../../../examples/yaml_define/OpenMLDB.yaml |
||||
:start-after: # under the License. |
||||
:language: yaml |
||||
@ -1,29 +0,0 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
Procedure |
||||
========= |
||||
|
||||
.. automodule:: pydolphinscheduler.tasks.procedure |
||||
|
||||
|
||||
YAML file example |
||||
----------------- |
||||
|
||||
.. literalinclude:: ../../../examples/yaml_define/Procedure.yaml |
||||
:start-after: # under the License. |
||||
:language: yaml |
||||
@ -1,29 +0,0 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
Python |
||||
====== |
||||
|
||||
.. automodule:: pydolphinscheduler.tasks.python |
||||
|
||||
|
||||
YAML file example |
||||
----------------- |
||||
|
||||
.. literalinclude:: ../../../examples/yaml_define/Python.yaml |
||||
:start-after: # under the License. |
||||
:language: yaml |
||||
@ -1,42 +0,0 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
Pytorch |
||||
======= |
||||
|
||||
|
||||
A Pytorch task type's example and dive into information of **PyDolphinScheduler**. |
||||
|
||||
Example |
||||
------- |
||||
|
||||
.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_pytorch_example.py |
||||
:start-after: [start workflow_declare] |
||||
:end-before: [end workflow_declare] |
||||
|
||||
Dive Into |
||||
--------- |
||||
|
||||
.. automodule:: pydolphinscheduler.tasks.pytorch |
||||
|
||||
|
||||
YAML file example |
||||
----------------- |
||||
|
||||
.. literalinclude:: ../../../examples/yaml_define/Pytorch.yaml |
||||
:start-after: # under the License. |
||||
:language: yaml |
||||
@ -1,46 +0,0 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
SageMaker |
||||
========= |
||||
|
||||
|
||||
A SageMaker task type's example and dive into information of **PyDolphinScheduler**. |
||||
|
||||
Example |
||||
------- |
||||
|
||||
.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_sagemaker_example.py |
||||
:start-after: [start workflow_declare] |
||||
:end-before: [end workflow_declare] |
||||
|
||||
Dive Into |
||||
--------- |
||||
|
||||
.. automodule:: pydolphinscheduler.tasks.sagemaker |
||||
|
||||
YAML file example |
||||
----------------- |
||||
|
||||
.. literalinclude:: ../../../examples/yaml_define/Sagemaker.yaml |
||||
:start-after: # under the License. |
||||
:language: yaml |
||||
|
||||
example_sagemaker_params.json: |
||||
|
||||
.. literalinclude:: ../../../examples/yaml_define/example_sagemaker_params.json |
||||
:language: json |
||||
@ -1,41 +0,0 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
Shell |
||||
===== |
||||
|
||||
A shell task type's example and dive into information of **PyDolphinScheduler**. |
||||
|
||||
Example |
||||
------- |
||||
|
||||
.. literalinclude:: ../../../src/pydolphinscheduler/examples/tutorial.py |
||||
:start-after: [start workflow_declare] |
||||
:end-before: [end task_relation_declare] |
||||
|
||||
Dive Into |
||||
--------- |
||||
|
||||
.. automodule:: pydolphinscheduler.tasks.shell |
||||
|
||||
|
||||
YAML file example |
||||
----------------- |
||||
|
||||
.. literalinclude:: ../../../examples/yaml_define/Shell.yaml |
||||
:start-after: # under the License. |
||||
:language: yaml |
||||
@ -1,41 +0,0 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
Spark |
||||
===== |
||||
|
||||
A spark task type's example and dive into information of **PyDolphinScheduler**. |
||||
|
||||
Example |
||||
------- |
||||
|
||||
.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_spark_example.py |
||||
:start-after: [start workflow_declare] |
||||
:end-before: [end workflow_declare] |
||||
|
||||
Dive Into |
||||
--------- |
||||
|
||||
.. automodule:: pydolphinscheduler.tasks.spark |
||||
|
||||
|
||||
YAML file example |
||||
----------------- |
||||
|
||||
.. literalinclude:: ../../../examples/yaml_define/Spark.yaml |
||||
:start-after: # under the License. |
||||
:language: yaml |
||||
@ -1,35 +0,0 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
SQL |
||||
=== |
||||
|
||||
.. automodule:: pydolphinscheduler.tasks.sql |
||||
|
||||
|
||||
YAML file example |
||||
----------------- |
||||
|
||||
.. literalinclude:: ../../../examples/yaml_define/Sql.yaml |
||||
:start-after: # under the License. |
||||
:language: yaml |
||||
|
||||
example_sql.sql: |
||||
|
||||
.. literalinclude:: ../../../examples/yaml_define/example_sql.sql |
||||
:start-after: */ |
||||
:language: sql |
||||
@ -1,38 +0,0 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
Sub Process |
||||
=========== |
||||
|
||||
.. automodule:: pydolphinscheduler.tasks.sub_process |
||||
|
||||
|
||||
YAML file example |
||||
----------------- |
||||
|
||||
.. literalinclude:: ../../../examples/yaml_define/SubProcess.yaml |
||||
:start-after: # under the License. |
||||
:language: yaml |
||||
|
||||
|
||||
|
||||
example_subprocess.yaml: |
||||
|
||||
.. literalinclude:: ../../../examples/yaml_define/example_sub_workflow.yaml |
||||
:start-after: # under the License. |
||||
:language: yaml |
||||
|
||||
@ -1,42 +0,0 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
Switch |
||||
====== |
||||
|
||||
A switch task type's example and dive into information of **PyDolphinScheduler**. |
||||
|
||||
Example |
||||
------- |
||||
|
||||
.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_switch_example.py |
||||
:start-after: [start workflow_declare] |
||||
:end-before: [end workflow_declare] |
||||
|
||||
Dive Into |
||||
--------- |
||||
|
||||
.. automodule:: pydolphinscheduler.tasks.switch |
||||
|
||||
|
||||
YAML file example |
||||
----------------- |
||||
|
||||
.. literalinclude:: ../../../examples/yaml_define/Switch.yaml |
||||
:start-after: # under the License. |
||||
:language: yaml |
||||
|
||||
@ -1,319 +0,0 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
Tutorial |
||||
======== |
||||
|
||||
This tutorial shows you the basic concept of *PyDolphinScheduler* and tells all |
||||
things you should know before you submit or run your first workflow. If you |
||||
still have not installed *PyDolphinScheduler* and start DolphinScheduler, you |
||||
could go and see :ref:`how to getting start PyDolphinScheduler <start:getting started>` firstly. |
||||
|
||||
Overview of Tutorial |
||||
-------------------- |
||||
|
||||
Here have an overview of our tutorial, and it looks a little complex but does not |
||||
worry about that because we explain this example below as detail as possible. |
||||
|
||||
There are two types of tutorials: traditional and task decorator. |
||||
|
||||
- **Traditional Way**: More general, support many :doc:`built-in task types <tasks/index>`, it is convenient |
||||
when you build your workflow at the beginning. |
||||
- **Task Decorator**: A Python decorator allow you to wrap your function into pydolphinscheduler's task. Less |
||||
versatility to the traditional way because it only supported Python functions and without build-in tasks |
||||
supported. But it is helpful if your workflow is all built with Python or if you already have some Python |
||||
workflow code and want to migrate them to pydolphinscheduler. |
||||
- **YAML File**: We can use pydolphinscheduler CLI to create process using YAML file: :code:`pydolphinscheduler yaml -f tutorial.yaml`. |
||||
We can find more YAML file examples in `examples/yaml_define <https://github.com/apache/dolphinscheduler/tree/dev/dolphinscheduler-python/pydolphinscheduler/examples/yaml_define>`_ |
||||
|
||||
.. tab:: Tradition |
||||
|
||||
.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py |
||||
:dedent: 0 |
||||
:start-after: [start tutorial] |
||||
:end-before: [end tutorial] |
||||
|
||||
.. tab:: Task Decorator |
||||
|
||||
.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial_decorator.py |
||||
:dedent: 0 |
||||
:start-after: [start tutorial] |
||||
:end-before: [end tutorial] |
||||
|
||||
.. tab:: YAML File |
||||
|
||||
.. literalinclude:: ../../examples/yaml_define/tutorial.yaml |
||||
:start-after: # under the License. |
||||
:language: yaml |
||||
|
||||
Import Necessary Module |
||||
----------------------- |
||||
|
||||
First of all, we should import the necessary module which we would use later just like other Python packages. |
||||
|
||||
.. tab:: Tradition |
||||
|
||||
.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py |
||||
:dedent: 0 |
||||
:start-after: [start package_import] |
||||
:end-before: [end package_import] |
||||
|
||||
In tradition tutorial we import :class:`pydolphinscheduler.core.process_definition.ProcessDefinition` and |
||||
:class:`pydolphinscheduler.tasks.shell.Shell`. |
||||
|
||||
If you want to use other task type you could click and :doc:`see all tasks we support <tasks/index>` |
||||
|
||||
.. tab:: Task Decorator |
||||
|
||||
.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial_decorator.py |
||||
:dedent: 0 |
||||
:start-after: [start package_import] |
||||
:end-before: [end package_import] |
||||
|
||||
In task decorator tutorial we import :class:`pydolphinscheduler.core.process_definition.ProcessDefinition` and |
||||
:func:`pydolphinscheduler.tasks.func_wrap.task`. |
||||
|
||||
Process Definition Declaration |
||||
------------------------------ |
||||
|
||||
We should instantiate :class:`pydolphinscheduler.core.process_definition.ProcessDefinition` object after we |
||||
import them from `import necessary module`_. Here we declare basic arguments for process definition(aka, workflow). |
||||
We define the name of :code:`ProcessDefinition`, using `Python context manager`_ and it **the only required argument** |
||||
for `ProcessDefinition`. Besides, we also declare three arguments named :code:`schedule` and :code:`start_time` |
||||
which setting workflow schedule interval and schedule start_time, and argument :code:`tenant` defines which tenant |
||||
will be running this task in the DolphinScheduler worker. See :ref:`section tenant <concept:tenant>` in |
||||
*PyDolphinScheduler* :doc:`concept` for more information. |
||||
|
||||
.. tab:: Tradition |
||||
|
||||
.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py |
||||
:dedent: 0 |
||||
:start-after: [start workflow_declare] |
||||
:end-before: [end workflow_declare] |
||||
|
||||
.. tab:: Task Decorator |
||||
|
||||
.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial_decorator.py |
||||
:dedent: 0 |
||||
:start-after: [start workflow_declare] |
||||
:end-before: [end workflow_declare] |
||||
|
||||
.. tab:: YAML File |
||||
|
||||
.. literalinclude:: ../../examples/yaml_define/tutorial.yaml |
||||
:start-after: # under the License. |
||||
:end-before: # Define the tasks under the workflow |
||||
:language: yaml |
||||
|
||||
We could find more detail about :code:`ProcessDefinition` in :ref:`concept about process definition <concept:process definition>` |
||||
if you are interested in it. For all arguments of object process definition, you could find in the |
||||
:class:`pydolphinscheduler.core.process_definition` API documentation. |
||||
|
||||
Task Declaration |
||||
---------------- |
||||
|
||||
.. tab:: Tradition |
||||
|
||||
We declare four tasks to show how to create tasks, and both of them are simple tasks of |
||||
:class:`pydolphinscheduler.tasks.shell` which runs `echo` command in the terminal. Besides the argument |
||||
`command` with :code:`echo` command, we also need to set the argument `name` for each task |
||||
*(not only shell task, `name` is required for each type of task)*. |
||||
|
||||
.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py |
||||
:dedent: 0 |
||||
:start-after: [start task_declare] |
||||
:end-before: [end task_declare] |
||||
|
||||
Besides shell task, *PyDolphinScheduler* supports multiple tasks and you could find in :doc:`tasks/index`. |
||||
|
||||
.. tab:: Task Decorator |
||||
|
||||
We declare four tasks to show how to create tasks, and both of them are created by the task decorator which |
||||
using :func:`pydolphinscheduler.tasks.func_wrap.task`. All we have to do is add a decorator named |
||||
:code:`@task` to existing Python function, and then use them inside :class:`pydolphinscheduler.core.process_definition` |
||||
|
||||
.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial_decorator.py |
||||
:dedent: 0 |
||||
:start-after: [start task_declare] |
||||
:end-before: [end task_declare] |
||||
|
||||
It makes our workflow more Pythonic, but be careful that when we use task decorator mode mean we only use |
||||
Python function as a task and could not use the :doc:`built-in tasks <tasks/index>` most of the cases. |
||||
|
||||
.. tab:: YAML File |
||||
|
||||
.. literalinclude:: ../../examples/yaml_define/tutorial.yaml |
||||
:start-after: # Define the tasks under the workflow |
||||
:language: yaml |
||||
|
||||
Setting Task Dependence |
||||
----------------------- |
||||
|
||||
After we declare both process definition and task, we have four tasks that are independent and will be running |
||||
in parallel. If you want to start one task until some task is finished, you have to set dependence on those |
||||
tasks. |
||||
|
||||
Set task dependence is quite easy by task's attribute :code:`set_downstream` and :code:`set_upstream` or by |
||||
bitwise operators :code:`>>` and :code:`<<` |
||||
|
||||
In this tutorial, task `task_parent` is the leading task of the whole workflow, then task `task_child_one` and |
||||
task `task_child_two` are its downstream tasks. Task `task_union` will not run unless both task `task_child_one` |
||||
and task `task_child_two` was done, because both two task is `task_union`'s upstream. |
||||
|
||||
.. tab:: Tradition |
||||
|
||||
.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py |
||||
:dedent: 0 |
||||
:start-after: [start task_relation_declare] |
||||
:end-before: [end task_relation_declare] |
||||
|
||||
.. tab:: Task Decorator |
||||
|
||||
.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial_decorator.py |
||||
:dedent: 0 |
||||
:start-after: [start task_relation_declare] |
||||
:end-before: [end task_relation_declare] |
||||
|
||||
.. tab:: YAML File |
||||
|
||||
We can use :code:`deps:[]` to set task dependence |
||||
|
||||
.. literalinclude:: ../../examples/yaml_define/tutorial.yaml |
||||
:start-after: # Define the tasks under the workflow |
||||
:language: yaml |
||||
|
||||
.. note:: |
||||
|
||||
We could set task dependence in batch mode if they have the same downstream or upstream by declaring those |
||||
tasks as task groups. In tutorial, We declare task `task_child_one` and `task_child_two` as task group named |
||||
`task_group`, then set `task_group` as downstream of task `task_parent`. You could see more detail in |
||||
:ref:`concept:Tasks Dependence` for more detail about how to set task dependence. |
||||
|
||||
Submit Or Run Workflow |
||||
---------------------- |
||||
|
||||
After that, we finish our workflow definition, with four tasks and task dependence, but all these things are |
||||
local, we should let the DolphinScheduler daemon know how the definition of workflow. So the last thing we |
||||
have to do is submit the workflow to the DolphinScheduler daemon. |
||||
|
||||
Fortunately, we have a convenient method to submit workflow via `ProcessDefinition` attribute :code:`run` which |
||||
will create workflow definition as well as workflow schedule. |
||||
|
||||
.. tab:: Tradition |
||||
|
||||
.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py |
||||
:dedent: 0 |
||||
:start-after: [start submit_or_run] |
||||
:end-before: [end submit_or_run] |
||||
|
||||
.. tab:: Task Decorator |
||||
|
||||
.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial_decorator.py |
||||
:dedent: 0 |
||||
:start-after: [start submit_or_run] |
||||
:end-before: [end submit_or_run] |
||||
|
||||
.. tab:: YAML File |
||||
|
||||
pydolphinscheduler YAML CLI always submit workflow. We can run the workflow if we set :code:`run: true` |
||||
|
||||
.. code-block:: yaml |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "tutorial" |
||||
run: true |
||||
|
||||
At last, we could execute this workflow code in your terminal like other Python scripts, running |
||||
:code:`python tutorial.py` to trigger and execute it. |
||||
|
||||
.. note:: |
||||
|
||||
If you do not start your DolphinScheduler API server, you could find how to start it in |
||||
:ref:`start:start Python gateway service` for more detail. Besides attribute :code:`run`, we have attribute |
||||
:code:`submit` for object `ProcessDefinition` which just submits workflow to the daemon but does not set |
||||
the workflow schedule information. For more detail, you could see :ref:`concept:process definition`. |
||||
|
||||
DAG Graph After Tutorial Run |
||||
---------------------------- |
||||
|
||||
After we run the tutorial code, you could log in DolphinScheduler web UI, go and see the |
||||
`DolphinScheduler project page`_. They is a new process definition be created by *PyDolphinScheduler* and it |
||||
named "tutorial" or "tutorial_decorator". The task graph of workflow like below: |
||||
|
||||
.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py |
||||
:language: text |
||||
:lines: 24-28 |
||||
|
||||
Create Process Using YAML File |
||||
------------------------------ |
||||
|
||||
We can use pydolphinscheduler CLI to create process using YAML file |
||||
|
||||
.. code-block:: bash |
||||
|
||||
pydolphinscheduler yaml -f Shell.yaml |
||||
|
||||
We can use the following four special grammars to define workflows more flexibly. |
||||
|
||||
- :code:`$FILE{"file_name"}`: Read the file (:code:`file_name`) contents and replace them to that location. |
||||
- :code:`$WORKFLOW{"other_workflow.yaml"}`: Refer to another process defined using YAML file (:code:`other_workflow.yaml`) and replace the process name in this location. |
||||
- :code:`$ENV{env_name}`: Read the environment variable (:code:`env_name`) and replace it to that location. |
||||
- :code:`${CONFIG.key_name}`: Read the configuration value of key (:code:`key_name`) and it them to that location. |
||||
|
||||
|
||||
In addition, when loading the file path use :code:`$FILE{"file_name"}` or :code:`$WORKFLOW{"other_workflow.yaml"}`, pydolphinscheduler will search in the path of the YAMl file if the file does not exist. |
||||
|
||||
For exmaples, our file directory structure is as follows: |
||||
|
||||
.. code-block:: bash |
||||
|
||||
. |
||||
└── yaml_define |
||||
├── Condition.yaml |
||||
├── DataX.yaml |
||||
├── Dependent_External.yaml |
||||
├── Dependent.yaml |
||||
├── example_datax.json |
||||
├── example_sql.sql |
||||
├── example_subprocess.yaml |
||||
├── Flink.yaml |
||||
├── Http.yaml |
||||
├── MapReduce.yaml |
||||
├── MoreConfiguration.yaml |
||||
├── Procedure.yaml |
||||
├── Python.yaml |
||||
├── Shell.yaml |
||||
├── Spark.yaml |
||||
├── Sql.yaml |
||||
├── SubProcess.yaml |
||||
└── Switch.yaml |
||||
|
||||
After we run |
||||
|
||||
.. code-block:: bash |
||||
|
||||
pydolphinscheduler yaml -file yaml_define/SubProcess.yaml |
||||
|
||||
|
||||
the :code:`$WORKFLOW{"example_sub_workflow.yaml"}` will be set to :code:`$WORKFLOW{"yaml_define/example_sub_workflow.yaml"}`, because :code:`./example_subprocess.yaml` does not exist and :code:`yaml_define/example_sub_workflow.yaml` does. |
||||
|
||||
Furthermore, this feature supports recursion all the way down. |
||||
|
||||
|
||||
.. _`DolphinScheduler project page`: https://dolphinscheduler.apache.org/en-us/docs/latest/user_doc/guide/project.html |
||||
.. _`Python context manager`: https://docs.python.org/3/library/stdtypes.html#context-manager-types |
||||
@ -1,43 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "Condition" |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- { "task_type": "Shell", "name": "pre_task_1", "command": "echo pre_task_1" } |
||||
- { "task_type": "Shell", "name": "pre_task_2", "command": "echo pre_task_2" } |
||||
- { "task_type": "Shell", "name": "pre_task_3", "command": "echo pre_task_3" } |
||||
- { "task_type": "Shell", "name": "success_branch", "command": "echo success_branch" } |
||||
- { "task_type": "Shell", "name": "fail_branch", "command": "echo fail_branch" } |
||||
|
||||
- name: condition |
||||
task_type: Condition |
||||
success_task: success_branch |
||||
failed_task: fail_branch |
||||
op: AND |
||||
groups: |
||||
- op: AND |
||||
groups: |
||||
- task: pre_task_1 |
||||
flag: true |
||||
- task: pre_task_2 |
||||
flag: true |
||||
- task: pre_task_3 |
||||
flag: false |
||||
@ -1,33 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "DataX" |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- name: task |
||||
task_type: DataX |
||||
datasource_name: db |
||||
datatarget_name: db |
||||
sql: show tables; |
||||
target_table: table_test |
||||
|
||||
- name: task_custon_config |
||||
task_type: CustomDataX |
||||
json: $FILE{"example_datax.json"} |
||||
@ -1,76 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
workflow: |
||||
name: "Dependent" |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- name: dependent |
||||
task_type: Dependent |
||||
denpendence: |
||||
op: and |
||||
groups: |
||||
- op: or |
||||
groups: |
||||
- project_name: pydolphin |
||||
process_definition_name: task_dependent_external |
||||
dependent_task_name: task_1 |
||||
|
||||
- project_name: pydolphin |
||||
process_definition_name: task_dependent_external |
||||
dependent_task_name: task_2 |
||||
|
||||
- op: and |
||||
groups: |
||||
- project_name: pydolphin |
||||
process_definition_name: task_dependent_external |
||||
dependent_task_name: task_1 |
||||
dependent_date: LAST_WEDNESDAY |
||||
|
||||
- project_name: pydolphin |
||||
process_definition_name: task_dependent_external |
||||
dependent_task_name: task_2 |
||||
dependent_date: last24Hours |
||||
|
||||
- name: dependent_var |
||||
task_type: Dependent |
||||
denpendence: |
||||
op: and |
||||
groups: |
||||
- op: or |
||||
# we can use ${CONFIG.WORKFLOW_PROJECT} to set the value to configuration.WORKFLOW_PROJECT |
||||
# we can use $WORKFLOW{"Dependent_External.yaml"} to create or update a workflow from dependent_external.yaml and set the value to that workflow name |
||||
groups: |
||||
- project_name: ${CONFIG.WORKFLOW_PROJECT} |
||||
process_definition_name: $WORKFLOW{"Dependent_External.yaml"} |
||||
dependent_task_name: task_1 |
||||
|
||||
- project_name: ${CONFIG.WORKFLOW_PROJECT} |
||||
process_definition_name: $WORKFLOW{"Dependent_External.yaml"} |
||||
dependent_task_name: task_2 |
||||
- op: and |
||||
groups: |
||||
- project_name: ${CONFIG.WORKFLOW_PROJECT} |
||||
process_definition_name: $WORKFLOW{"Dependent_External.yaml"} |
||||
dependent_task_name: task_1 |
||||
dependent_date: LAST_WEDNESDAY |
||||
|
||||
- project_name: ${CONFIG.WORKFLOW_PROJECT} |
||||
process_definition_name: $WORKFLOW{"Dependent_External.yaml"} |
||||
dependent_task_name: task_2 |
||||
dependent_date: last24Hours |
||||
@ -1,26 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "task_dependent_external" |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- { "task_type": "Shell", "name": "task_1", "command": "echo task 1" } |
||||
- { "task_type": "Shell", "name": "task_2", "command": "echo task 2" } |
||||
- { "task_type": "Shell", "name": "task_3", "command": "echo task 3" } |
||||
@ -1,46 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define variable `repository` |
||||
repository: &repository "git@github.com:<YOUR-NAME-OR-ORG>/dvc-data-repository-example.git" |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "DVC" |
||||
release_state: "offline" |
||||
|
||||
# Define the tasks under the process |
||||
tasks: |
||||
- name: init_dvc |
||||
task_type: DVCInit |
||||
repository: *repository |
||||
store_url: ~/dvc_data |
||||
|
||||
- name: upload_data |
||||
task_type: DVCUpload |
||||
repository: *repository |
||||
data_path_in_dvc_repository: "iris" |
||||
data_path_in_worker: ~/source/iris |
||||
version: v1 |
||||
message: upload iris data v1 |
||||
|
||||
- name: download_data |
||||
task_type: DVCDownload |
||||
repository: *repository |
||||
data_path_in_dvc_repository: "iris" |
||||
data_path_in_worker: ~/target/iris |
||||
version: v1 |
||||
@ -1,29 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "Flink" |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- name: task |
||||
task_type: Flink |
||||
main_class: org.apache.flink.streaming.examples.wordcount.WordCount |
||||
main_package: test_java.jar |
||||
program_type: JAVA |
||||
deploy_mode: local |
||||
@ -1,37 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "Http" |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- name: task |
||||
task_type: Http |
||||
url: "https://httpbin.org/get" |
||||
http_method: "GET" |
||||
http_params: |
||||
- { "prop": "a", "httpParametersType": "PARAMETER", "value": "1" } |
||||
- { "prop": "b", "httpParametersType": "PARAMETER", "value": "2" } |
||||
- { |
||||
"prop": "Content-Type", |
||||
"httpParametersType": "header", |
||||
"value": "test", |
||||
} |
||||
http_check_condition: "STATUS_CODE_CUSTOM" |
||||
condition: "404" |
||||
@ -1,29 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "MapReduce" |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- name: task |
||||
task_type: MR |
||||
main_class: wordcount |
||||
main_package: test_java.jar |
||||
program_type: SCALA |
||||
main_args: /dolphinscheduler/tenant_exists/resources/file.txt /output/ds |
||||
@ -1,40 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "MoreConfiguration" |
||||
param: |
||||
n: 1 |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- name: shell_0 |
||||
task_type: Shell |
||||
description: "yaml define task" |
||||
flag: "YES" |
||||
command: | |
||||
echo "$ENV{HOME}" |
||||
echo "${n}" |
||||
task_priority: "HIGH" |
||||
delay_time: 20 |
||||
fail_retry_times: 30 |
||||
fail_retry_interval: 5 |
||||
timeout_flag: "CLOSE" |
||||
timeout: 60 |
||||
local_params: |
||||
- { "prop": "n", "direct": "IN", "type": "VARCHAR", "value": "${n}" } |
||||
@ -1,33 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "OpenMLDB" |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- name: OpenMLDB |
||||
task_type: OpenMLDB |
||||
zookeeper: "127.0.0.1:2181" |
||||
zookeeper_path: "/openmldb" |
||||
execute_mode: "online" |
||||
sql: | |
||||
USE demo_db; |
||||
set @@job_timeout=200000; |
||||
LOAD DATA INFILE 'file:///tmp/train_sample.csv' |
||||
INTO TABLE talkingdata OPTIONS(mode='overwrite'); |
||||
@ -1,27 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "Procedure" |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- name: task |
||||
task_type: Procedure |
||||
datasource_name: db |
||||
method: show tables; |
||||
@ -1,30 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "Python" |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- name: python |
||||
task_type: Python |
||||
definition: | |
||||
import os |
||||
print(os) |
||||
print("1") |
||||
print("2") |
||||
@ -1,53 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "Pytorch" |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
|
||||
# run project with existing environment |
||||
- name: task_existing_env |
||||
task_type: pytorch |
||||
script: main.py |
||||
script_params: --dry-run --no-cuda |
||||
project_path: https://github.com/pytorch/examples#mnist |
||||
python_command: /home/anaconda3/envs/pytorch/bin/python3 |
||||
|
||||
|
||||
# run project with creating conda environment |
||||
- name: task_conda_env |
||||
task_type: pytorch |
||||
script: main.py |
||||
script_params: --dry-run --no-cuda |
||||
project_path: https://github.com/pytorch/examples#mnist |
||||
is_create_environment: True |
||||
python_env_tool: conda |
||||
requirements: requirements.txt |
||||
conda_python_version: 3.7 |
||||
|
||||
# run project with creating virtualenv environment |
||||
- name: task_virtualenv_env |
||||
task_type: pytorch |
||||
script: main.py |
||||
script_params: --dry-run --no-cuda |
||||
project_path: https://github.com/pytorch/examples#mnist |
||||
is_create_environment: True |
||||
python_env_tool: virtualenv |
||||
requirements: requirements.txt |
||||
@ -1,28 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "Sagemaker" |
||||
release_state: "offline" |
||||
|
||||
# Define the tasks under the process |
||||
tasks: |
||||
- name: sagemaker |
||||
task_type: Sagemaker |
||||
sagemaker_request_json: $FILE{"example_sagemaker_params.json"} |
||||
|
||||
@ -1,40 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "Shell" |
||||
release_state: "offline" |
||||
run: true |
||||
|
||||
# Define the tasks under the process |
||||
tasks: |
||||
- name: task_parent |
||||
task_type: Shell |
||||
command: | |
||||
echo hello pydolphinscheduler |
||||
echo run task parent |
||||
|
||||
- name: task_child_one |
||||
task_type: Shell |
||||
deps: [task_parent] |
||||
command: echo "child one" |
||||
|
||||
- name: task_child_two |
||||
task_type: Shell |
||||
deps: [task_parent] |
||||
command: echo "child two" |
||||
@ -1,45 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "Sql" |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- name: task_base |
||||
task_type: Sql |
||||
datasource_name: "db" |
||||
sql: show tables; |
||||
|
||||
- name: task_multi_line |
||||
task_type: Sql |
||||
datasource_name: "db" |
||||
sql: | |
||||
show tables; |
||||
select id from version where id=1; |
||||
|
||||
- name: task_file |
||||
task_type: Sql |
||||
datasource_name: "db" |
||||
sql: $FILE{"example_sql.sql"} |
||||
|
||||
# Or you can define task "task_union" it with one line |
||||
- { "task_type": "Sql", "name": "task_base_one_line", "datasource_name": "db", "sql": "select id from version where id=1;"} |
||||
|
||||
# Or you can define task "task_union" it with one line |
||||
- { "task_type": "Sql", "name": "task_file_one_line", "datasource_name": "db", "sql": '$FILE{"example_sql.sql"}'} |
||||
@ -1,27 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "SubWorkflow" |
||||
|
||||
tasks: |
||||
- name: example_workflow |
||||
task_type: SubProcess |
||||
process_definition_name: $WORKFLOW{"example_sub_workflow.yaml"} |
||||
|
||||
- { "task_type": "Shell", "deps": [example_workflow], "name": "task_3", "command": "echo task 3" } |
||||
@ -1,39 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "Switch" |
||||
param: |
||||
var: 1 |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- name: switch_child_1 |
||||
task_type: Shell |
||||
command: echo switch_child_1 |
||||
|
||||
- name: switch_child_2 |
||||
task_type: Shell |
||||
command: echo switch_child_2 |
||||
|
||||
- name: switch |
||||
task_type: Switch |
||||
condition: |
||||
- task: switch_child_1 |
||||
condition: "${var} > 1" |
||||
- task: switch_child_2 |
||||
@ -1,62 +0,0 @@
|
||||
{ |
||||
"job": { |
||||
"content": [ |
||||
{ |
||||
"reader": { |
||||
"name": "mysqlreader", |
||||
"parameter": { |
||||
"username": "usr", |
||||
"password": "pwd", |
||||
"column": [ |
||||
"id", |
||||
"name", |
||||
"code", |
||||
"description" |
||||
], |
||||
"splitPk": "id", |
||||
"connection": [ |
||||
{ |
||||
"table": [ |
||||
"source_table" |
||||
], |
||||
"jdbcUrl": [ |
||||
"jdbc:mysql://127.0.0.1:3306/source_db" |
||||
] |
||||
} |
||||
] |
||||
} |
||||
}, |
||||
"writer": { |
||||
"name": "mysqlwriter", |
||||
"parameter": { |
||||
"writeMode": "insert", |
||||
"username": "usr", |
||||
"password": "pwd", |
||||
"column": [ |
||||
"id", |
||||
"name" |
||||
], |
||||
"connection": [ |
||||
{ |
||||
"jdbcUrl": "jdbc:mysql://127.0.0.1:3306/target_db", |
||||
"table": [ |
||||
"target_table" |
||||
] |
||||
} |
||||
] |
||||
} |
||||
} |
||||
} |
||||
], |
||||
"setting": { |
||||
"errorLimit": { |
||||
"percentage": 0, |
||||
"record": 0 |
||||
}, |
||||
"speed": { |
||||
"channel": 1, |
||||
"record": 1000 |
||||
} |
||||
} |
||||
} |
||||
} |
||||
@ -1,18 +0,0 @@
|
||||
{ |
||||
"ParallelismConfiguration":{ |
||||
"MaxParallelExecutionSteps":1 |
||||
}, |
||||
"PipelineExecutionDescription":"run pipeline using ds", |
||||
"PipelineExecutionDisplayName":"ds-sagemaker-pipeline", |
||||
"PipelineName":"DsSagemakerPipeline", |
||||
"PipelineParameters":[ |
||||
{ |
||||
"Name":"InputData", |
||||
"Value": "s3://sagemaker/dataset/dataset.csv" |
||||
}, |
||||
{ |
||||
"Name":"InferenceData", |
||||
"Value": "s3://sagemaker/dataset/inference.csv" |
||||
} |
||||
] |
||||
} |
||||
@ -1,22 +0,0 @@
|
||||
/* |
||||
* Licensed to the Apache Software Foundation (ASF) under one or more |
||||
* contributor license agreements. See the NOTICE file distributed with |
||||
* this work for additional information regarding copyright ownership. |
||||
* The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
* (the "License"); you may not use this file except in compliance with |
||||
* the License. You may obtain a copy of the License at |
||||
* |
||||
* http://www.apache.org/licenses/LICENSE-2.0 |
||||
* |
||||
* Unless required by applicable law or agreed to in writing, software |
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
* See the License for the specific language governing permissions and |
||||
* limitations under the License. |
||||
*/ |
||||
|
||||
select id from version where id=1; |
||||
select id from version where id=2; |
||||
select id from version where id=3; |
||||
select id from version where id=4; |
||||
select id from version where id=5; |
||||
@ -1,26 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "example_workflow_for_sub_workflow" |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- { "task_type": "Shell", "name": "task_1", "command": "echo task 1" } |
||||
- { "task_type": "Shell", "deps": [task_1], "name": "task_2", "command": "echo task 2" } |
||||
- { "task_type": "Shell", "deps": [task_2], "name": "task_3", "command": "echo task 3" } |
||||
@ -1,69 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
|
||||
# Define variable `mlflow_tracking_uri` |
||||
mlflow_tracking_uri: &mlflow_tracking_uri "http://127.0.0.1:5000" |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "MLflow" |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- name: train_xgboost_native |
||||
task_type: MLFlowProjectsCustom |
||||
repository: https://github.com/mlflow/mlflow#examples/xgboost/xgboost_native |
||||
mlflow_tracking_uri: *mlflow_tracking_uri |
||||
parameters: -P learning_rate=0.2 -P colsample_bytree=0.8 -P subsample=0.9 |
||||
experiment_name: xgboost |
||||
|
||||
- name: train_automl |
||||
task_type: MLFlowProjectsAutoML |
||||
mlflow_tracking_uri: *mlflow_tracking_uri |
||||
parameters: time_budget=30;estimator_list=['lgbm'] |
||||
experiment_name: automl_iris |
||||
model_name: iris_A |
||||
automl_tool: flaml |
||||
data_path: /data/examples/iris |
||||
|
||||
- name: deploy_docker |
||||
task_type: MLflowModels |
||||
deps: [train_automl] |
||||
model_uri: models:/iris_A/Production |
||||
mlflow_tracking_uri: *mlflow_tracking_uri |
||||
deploy_mode: DOCKER |
||||
port: 7002 |
||||
|
||||
- name: train_basic_algorithm |
||||
task_type: MLFlowProjectsBasicAlgorithm |
||||
mlflow_tracking_uri: *mlflow_tracking_uri |
||||
parameters: n_estimators=200;learning_rate=0.2 |
||||
experiment_name: basic_algorithm_iris |
||||
model_name: iris_B |
||||
algorithm: lightgbm |
||||
data_path: /data/examples/iris |
||||
search_params: max_depth=[5, 10];n_estimators=[100, 200] |
||||
|
||||
- name: deploy_mlflow |
||||
deps: [train_basic_algorithm] |
||||
task_type: MLflowModels |
||||
model_uri: models:/iris_B/Production |
||||
mlflow_tracking_uri: *mlflow_tracking_uri |
||||
deploy_mode: MLFLOW |
||||
port: 7001 |
||||
|
||||
@ -1,46 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Define the workflow |
||||
workflow: |
||||
name: "tutorial" |
||||
schedule: "0 0 0 * * ? *" |
||||
start_time: "2021-01-01" |
||||
tenant: "tenant_exists" |
||||
release_state: "offline" |
||||
run: true |
||||
|
||||
# Define the tasks under the workflow |
||||
tasks: |
||||
- name: task_parent |
||||
task_type: Shell |
||||
command: echo hello pydolphinscheduler |
||||
|
||||
- name: task_child_one |
||||
task_type: Shell |
||||
deps: [task_parent] |
||||
command: echo "child one" |
||||
|
||||
- name: task_child_two |
||||
task_type: Shell |
||||
deps: [task_parent] |
||||
command: echo "child two" |
||||
|
||||
- name: task_union |
||||
task_type: Shell |
||||
deps: [task_child_one, task_child_two] |
||||
command: echo "union" |
||||
@ -1,21 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more |
||||
# contributor license agreements. See the NOTICE file distributed with |
||||
# this work for additional information regarding copyright ownership. |
||||
# The ASF licenses this file to You under the Apache License, Version 2.0 |
||||
# (the "License"); you may not use this file except in compliance with |
||||
# the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
[pytest] |
||||
# add path here to skip pytest scan it |
||||
norecursedirs = |
||||
tests/testing |
||||
# Integration test run seperated which do not calculate coverage, it will run in `tox -e integrate-test` |
||||
tests/integration |
||||
@ -1,16 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
@ -1,22 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Init root of pydolphinscheduler.""" |
||||
|
||||
from pkg_resources import get_distribution |
||||
|
||||
__version__ = get_distribution("apache-dolphinscheduler").version |
||||
@ -1,18 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Commands line interface of pydolphinscheduler.""" |
||||
@ -1,106 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Commands line interface's command of pydolphinscheduler.""" |
||||
|
||||
import click |
||||
from click import echo |
||||
|
||||
import pydolphinscheduler |
||||
from pydolphinscheduler.configuration import ( |
||||
get_single_config, |
||||
init_config_file, |
||||
set_single_config, |
||||
) |
||||
from pydolphinscheduler.core.yaml_process_define import create_process_definition |
||||
|
||||
version_option_val = ["major", "minor", "micro"] |
||||
|
||||
|
||||
@click.group() |
||||
def cli(): |
||||
"""Apache DolphinScheduler Python API's command line interface.""" |
||||
|
||||
|
||||
@cli.command() |
||||
@click.option( |
||||
"--part", |
||||
"-p", |
||||
required=False, |
||||
type=click.Choice(version_option_val, case_sensitive=False), |
||||
multiple=False, |
||||
help="The part of version your want to get.", |
||||
) |
||||
def version(part: str) -> None: |
||||
"""Show current version of pydolphinscheduler.""" |
||||
if part: |
||||
idx = version_option_val.index(part) |
||||
echo(f"{pydolphinscheduler.__version__.split('.')[idx]}") |
||||
else: |
||||
echo(f"{pydolphinscheduler.__version__}") |
||||
|
||||
|
||||
@cli.command() |
||||
@click.option( |
||||
"--init", |
||||
"-i", |
||||
is_flag=True, |
||||
help="Initialize and create configuration file to `PYDS_HOME`.", |
||||
) |
||||
@click.option( |
||||
"--set", |
||||
"-s", |
||||
"setter", |
||||
multiple=True, |
||||
type=click.Tuple([str, str]), |
||||
help="Set specific setting to config file." |
||||
"Use multiple ``--set <KEY> <VAL>`` options to set multiple configs", |
||||
) |
||||
@click.option( |
||||
"--get", |
||||
"-g", |
||||
"getter", |
||||
multiple=True, |
||||
type=str, |
||||
help="Get specific setting from config file." |
||||
"Use multiple ``--get <KEY>`` options to get multiple configs", |
||||
) |
||||
def config(getter, setter, init) -> None: |
||||
"""Manage the configuration for pydolphinscheduler.""" |
||||
if init: |
||||
init_config_file() |
||||
elif getter: |
||||
click.echo("The configuration query as below:\n") |
||||
configs_kv = [f"{key} = {get_single_config(key)}" for key in getter] |
||||
click.echo("\n".join(configs_kv)) |
||||
elif setter: |
||||
for key, val in setter: |
||||
set_single_config(key, val) |
||||
click.echo("Set configuration done.") |
||||
|
||||
|
||||
@cli.command() |
||||
@click.option( |
||||
"--yaml_file", |
||||
"-f", |
||||
required=True, |
||||
help="YAML file path", |
||||
type=click.Path(exists=True), |
||||
) |
||||
def yaml(yaml_file) -> None: |
||||
"""Create process definition using YAML file.""" |
||||
create_process_definition(yaml_file) |
||||
@ -1,193 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Configuration module for pydolphinscheduler.""" |
||||
import os |
||||
from pathlib import Path |
||||
from typing import Any |
||||
|
||||
from pydolphinscheduler.exceptions import PyDSConfException |
||||
from pydolphinscheduler.utils import file |
||||
from pydolphinscheduler.utils.yaml_parser import YamlParser |
||||
|
||||
BUILD_IN_CONFIG_PATH = Path(__file__).resolve().parent.joinpath("default_config.yaml") |
||||
|
||||
|
||||
def config_path() -> Path: |
||||
"""Get the path of pydolphinscheduler configuration file.""" |
||||
pyds_home = os.environ.get("PYDS_HOME", "~/pydolphinscheduler") |
||||
config_file_path = Path(pyds_home).joinpath("config.yaml").expanduser() |
||||
return config_file_path |
||||
|
||||
|
||||
def get_configs() -> YamlParser: |
||||
"""Get all configuration settings from configuration file. |
||||
|
||||
Will use custom configuration file first if it exists, otherwise default configuration file in |
||||
default path. |
||||
""" |
||||
path = str(config_path()) if config_path().exists() else BUILD_IN_CONFIG_PATH |
||||
with open(path, mode="r") as f: |
||||
return YamlParser(f.read()) |
||||
|
||||
|
||||
def init_config_file() -> None: |
||||
"""Initialize configuration file by default configs.""" |
||||
if config_path().exists(): |
||||
raise PyDSConfException( |
||||
"Initialize configuration false to avoid overwrite configure by accident, file already exists " |
||||
"in %s, if you wan to overwrite the exists configure please remove the exists file manually.", |
||||
str(config_path()), |
||||
) |
||||
file.write(content=str(get_configs()), to_path=str(config_path())) |
||||
|
||||
|
||||
def get_single_config(key: str) -> Any: |
||||
"""Get single config to configuration file. |
||||
|
||||
Support get from nested keys by delimiter ``.``. |
||||
|
||||
For example, yaml config as below: |
||||
|
||||
.. code-block:: yaml |
||||
|
||||
one: |
||||
two1: |
||||
three: value1 |
||||
two2: value2 |
||||
|
||||
you could get ``value1`` and ``value2`` by nested path |
||||
|
||||
.. code-block:: python |
||||
|
||||
value1 = get_single_config("one.two1.three") |
||||
value2 = get_single_config("one.two2") |
||||
|
||||
:param key: The config key want to get it value. |
||||
""" |
||||
config = get_configs() |
||||
if key not in config: |
||||
raise PyDSConfException( |
||||
"Configuration path %s do not exists. Can not get configuration.", key |
||||
) |
||||
return config[key] |
||||
|
||||
|
||||
def set_single_config(key: str, value: Any) -> None: |
||||
"""Change single config to configuration file. |
||||
|
||||
For example, yaml config as below: |
||||
|
||||
.. code-block:: yaml |
||||
|
||||
one: |
||||
two1: |
||||
three: value1 |
||||
two2: value2 |
||||
|
||||
you could change ``value1`` to ``value3``, also change ``value2`` to ``value4`` by nested path assigned |
||||
|
||||
.. code-block:: python |
||||
|
||||
set_single_config["one.two1.three"] = "value3" |
||||
set_single_config["one.two2"] = "value4" |
||||
|
||||
:param key: The config key want change. |
||||
:param value: The new value want to set. |
||||
""" |
||||
config = get_configs() |
||||
if key not in config: |
||||
raise PyDSConfException( |
||||
"Configuration path %s do not exists. Can not set configuration.", key |
||||
) |
||||
config[key] = value |
||||
file.write(content=str(config), to_path=str(config_path()), overwrite=True) |
||||
|
||||
|
||||
def get_int(val: Any) -> int: |
||||
"""Covert value to int.""" |
||||
return int(val) |
||||
|
||||
|
||||
def get_bool(val: Any) -> bool: |
||||
"""Covert value to boolean.""" |
||||
if isinstance(val, str): |
||||
return val.lower() in {"true", "t"} |
||||
elif isinstance(val, int): |
||||
return val == 1 |
||||
else: |
||||
return bool(val) |
||||
|
||||
|
||||
# Start Common Configuration Settings |
||||
|
||||
# Add configs as module variables to avoid read configuration multiple times when |
||||
# Get common configuration setting |
||||
# Set or get multiple configs in single time |
||||
configs: YamlParser = get_configs() |
||||
|
||||
# Java Gateway Settings |
||||
JAVA_GATEWAY_ADDRESS = os.environ.get( |
||||
"PYDS_JAVA_GATEWAY_ADDRESS", configs.get("java_gateway.address") |
||||
) |
||||
JAVA_GATEWAY_PORT = get_int( |
||||
os.environ.get("PYDS_JAVA_GATEWAY_PORT", configs.get("java_gateway.port")) |
||||
) |
||||
JAVA_GATEWAY_AUTO_CONVERT = get_bool( |
||||
os.environ.get( |
||||
"PYDS_JAVA_GATEWAY_AUTO_CONVERT", configs.get("java_gateway.auto_convert") |
||||
) |
||||
) |
||||
|
||||
# User Settings |
||||
USER_NAME = os.environ.get("PYDS_USER_NAME", configs.get("default.user.name")) |
||||
USER_PASSWORD = os.environ.get( |
||||
"PYDS_USER_PASSWORD", configs.get("default.user.password") |
||||
) |
||||
USER_EMAIL = os.environ.get("PYDS_USER_EMAIL", configs.get("default.user.email")) |
||||
USER_PHONE = str(os.environ.get("PYDS_USER_PHONE", configs.get("default.user.phone"))) |
||||
USER_STATE = get_int( |
||||
os.environ.get("PYDS_USER_STATE", configs.get("default.user.state")) |
||||
) |
||||
|
||||
# Workflow Settings |
||||
WORKFLOW_PROJECT = os.environ.get( |
||||
"PYDS_WORKFLOW_PROJECT", configs.get("default.workflow.project") |
||||
) |
||||
WORKFLOW_TENANT = os.environ.get( |
||||
"PYDS_WORKFLOW_TENANT", configs.get("default.workflow.tenant") |
||||
) |
||||
WORKFLOW_USER = os.environ.get( |
||||
"PYDS_WORKFLOW_USER", configs.get("default.workflow.user") |
||||
) |
||||
WORKFLOW_QUEUE = os.environ.get( |
||||
"PYDS_WORKFLOW_QUEUE", configs.get("default.workflow.queue") |
||||
) |
||||
WORKFLOW_RELEASE_STATE = os.environ.get( |
||||
"PYDS_WORKFLOW_RELEASE_STATE", configs.get("default.workflow.release_state") |
||||
) |
||||
WORKFLOW_WORKER_GROUP = os.environ.get( |
||||
"PYDS_WORKFLOW_WORKER_GROUP", configs.get("default.workflow.worker_group") |
||||
) |
||||
WORKFLOW_TIME_ZONE = os.environ.get( |
||||
"PYDS_WORKFLOW_TIME_ZONE", configs.get("default.workflow.time_zone") |
||||
) |
||||
WORKFLOW_WARNING_TYPE = os.environ.get( |
||||
"PYDS_WORKFLOW_WARNING_TYPE", configs.get("default.workflow.warning_type") |
||||
) |
||||
|
||||
# End Common Configuration Setting |
||||
@ -1,30 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Init pydolphinscheduler.core package.""" |
||||
|
||||
from pydolphinscheduler.core.database import Database |
||||
from pydolphinscheduler.core.engine import Engine |
||||
from pydolphinscheduler.core.process_definition import ProcessDefinition |
||||
from pydolphinscheduler.core.task import Task |
||||
|
||||
__all__ = [ |
||||
"Database", |
||||
"Engine", |
||||
"ProcessDefinition", |
||||
"Task", |
||||
] |
||||
@ -1,62 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Module database.""" |
||||
|
||||
from typing import Dict |
||||
|
||||
from py4j.protocol import Py4JJavaError |
||||
|
||||
from pydolphinscheduler.exceptions import PyDSParamException |
||||
from pydolphinscheduler.java_gateway import JavaGate |
||||
|
||||
|
||||
class Database(dict): |
||||
"""database object, get information about database. |
||||
|
||||
You provider database_name contain connection information, it decisions which |
||||
database type and database instance would run task. |
||||
""" |
||||
|
||||
def __init__(self, database_name: str, type_key, database_key, *args, **kwargs): |
||||
super().__init__(*args, **kwargs) |
||||
self._database = {} |
||||
self.database_name = database_name |
||||
self[type_key] = self.database_type |
||||
self[database_key] = self.database_id |
||||
|
||||
@property |
||||
def database_type(self) -> str: |
||||
"""Get database type from java gateway, a wrapper for :func:`get_database_info`.""" |
||||
return self.get_database_info(self.database_name).get("type") |
||||
|
||||
@property |
||||
def database_id(self) -> str: |
||||
"""Get database id from java gateway, a wrapper for :func:`get_database_info`.""" |
||||
return self.get_database_info(self.database_name).get("id") |
||||
|
||||
def get_database_info(self, name) -> Dict: |
||||
"""Get database info from java gateway, contains database id, type, name.""" |
||||
if self._database: |
||||
return self._database |
||||
else: |
||||
try: |
||||
self._database = JavaGate().get_datasource_info(name) |
||||
# Handler database source do not exists error, for now we just terminate the process. |
||||
except Py4JJavaError as ex: |
||||
raise PyDSParamException(str(ex.java_exception)) |
||||
return self._database |
||||
@ -1,94 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Module engine.""" |
||||
|
||||
from typing import Dict, Optional |
||||
|
||||
from py4j.protocol import Py4JJavaError |
||||
|
||||
from pydolphinscheduler.core.task import Task |
||||
from pydolphinscheduler.exceptions import PyDSParamException |
||||
from pydolphinscheduler.java_gateway import JavaGate |
||||
|
||||
|
||||
class ProgramType(str): |
||||
"""Type of program engine runs, for now it just contain `JAVA`, `SCALA` and `PYTHON`.""" |
||||
|
||||
JAVA = "JAVA" |
||||
SCALA = "SCALA" |
||||
PYTHON = "PYTHON" |
||||
|
||||
|
||||
class Engine(Task): |
||||
"""Task engine object, declare behavior for engine task to dolphinscheduler. |
||||
|
||||
This is the parent class of spark, flink and mr tasks, |
||||
and is used to provide the programType, mainClass and mainJar task parameters for reuse. |
||||
""" |
||||
|
||||
def __init__( |
||||
self, |
||||
name: str, |
||||
task_type: str, |
||||
main_class: str, |
||||
main_package: str, |
||||
program_type: Optional[ProgramType] = ProgramType.SCALA, |
||||
*args, |
||||
**kwargs |
||||
): |
||||
super().__init__(name, task_type, *args, **kwargs) |
||||
self.main_class = main_class |
||||
self.main_package = main_package |
||||
self.program_type = program_type |
||||
self._resource = {} |
||||
|
||||
def get_resource_info(self, program_type, main_package): |
||||
"""Get resource info from java gateway, contains resource id, name.""" |
||||
if self._resource: |
||||
return self._resource |
||||
else: |
||||
try: |
||||
self._resource = JavaGate().get_resources_file_info( |
||||
program_type, main_package |
||||
) |
||||
# Handler source do not exists error, for now we just terminate the process. |
||||
except Py4JJavaError as ex: |
||||
raise PyDSParamException(str(ex.java_exception)) |
||||
return self._resource |
||||
|
||||
def get_jar_id(self) -> int: |
||||
"""Get jar id from java gateway, a wrapper for :func:`get_resource_info`.""" |
||||
return self.get_resource_info(self.program_type, self.main_package).get("id") |
||||
|
||||
@property |
||||
def task_params(self, camel_attr: bool = True, custom_attr: set = None) -> Dict: |
||||
"""Override Task.task_params for engine children task. |
||||
|
||||
children task have some specials attribute for task_params, and is odd if we |
||||
directly set as python property, so we Override Task.task_params here. |
||||
""" |
||||
params = super().task_params |
||||
custom_params = { |
||||
"programType": self.program_type, |
||||
"mainClass": self.main_class, |
||||
"mainJar": { |
||||
"id": self.get_jar_id(), |
||||
}, |
||||
} |
||||
params.update(custom_params) |
||||
return params |
||||
@ -1,73 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Module resource.""" |
||||
|
||||
from typing import Optional |
||||
|
||||
from pydolphinscheduler.exceptions import PyDSParamException |
||||
from pydolphinscheduler.java_gateway import JavaGate |
||||
from pydolphinscheduler.models import Base |
||||
|
||||
|
||||
class Resource(Base): |
||||
"""resource object, will define the resources that you want to create or update. |
||||
|
||||
:param name: The fullname of resource.Includes path and suffix. |
||||
:param content: The description of resource. |
||||
:param description: The description of resource. |
||||
:param user_name: The user name of resource. |
||||
""" |
||||
|
||||
_DEFINE_ATTR = {"name", "content", "description", "user_name"} |
||||
|
||||
def __init__( |
||||
self, |
||||
name: str, |
||||
content: Optional[str] = None, |
||||
description: Optional[str] = None, |
||||
user_name: Optional[str] = None, |
||||
): |
||||
super().__init__(name, description) |
||||
self.content = content |
||||
self.user_name = user_name |
||||
self._resource_code = None |
||||
|
||||
def get_info_from_database(self): |
||||
"""Get resource info from java gateway, contains resource id, name.""" |
||||
if not self.user_name: |
||||
raise PyDSParamException( |
||||
"`user_name` is required when querying resources from python gate." |
||||
) |
||||
return JavaGate().query_resources_file_info(self.user_name, self.name) |
||||
|
||||
def get_id_from_database(self): |
||||
"""Get resource id from java gateway.""" |
||||
return self.get_info_from_database().getId() |
||||
|
||||
def create_or_update_resource(self): |
||||
"""Create or update resource via java gateway.""" |
||||
if not self.content or not self.user_name: |
||||
raise PyDSParamException( |
||||
"`user_name` and `content` are required when create or update resource from python gate." |
||||
) |
||||
JavaGate().create_or_update_resource( |
||||
self.user_name, |
||||
self.name, |
||||
self.content, |
||||
self.description, |
||||
) |
||||
@ -1,466 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Parse YAML file to create process.""" |
||||
|
||||
import logging |
||||
import os |
||||
import re |
||||
from pathlib import Path |
||||
from typing import Any, Dict |
||||
|
||||
from pydolphinscheduler import configuration, tasks |
||||
from pydolphinscheduler.core.process_definition import ProcessDefinition |
||||
from pydolphinscheduler.core.task import Task |
||||
from pydolphinscheduler.exceptions import PyDSTaskNoFoundException |
||||
from pydolphinscheduler.utils.yaml_parser import YamlParser |
||||
|
||||
logger = logging.getLogger(__file__) |
||||
|
||||
KEY_PROCESS = "workflow" |
||||
KEY_TASK = "tasks" |
||||
KEY_TASK_TYPE = "task_type" |
||||
KEY_DEPS = "deps" |
||||
KEY_OP = "op" |
||||
|
||||
TASK_SPECIAL_KEYS = [KEY_TASK_TYPE, KEY_DEPS] |
||||
|
||||
|
||||
class ParseTool: |
||||
"""Enhanced parsing tools.""" |
||||
|
||||
@staticmethod |
||||
def parse_string_param_if_file(string_param: str, **kwargs): |
||||
"""Use $FILE{"data_path"} to load file from "data_path".""" |
||||
if string_param.startswith("$FILE"): |
||||
path = re.findall(r"\$FILE\{\"(.*?)\"\}", string_param)[0] |
||||
base_folder = kwargs.get("base_folder", ".") |
||||
path = ParseTool.get_possible_path(path, base_folder) |
||||
with open(path, "r") as read_file: |
||||
string_param = "".join(read_file) |
||||
return string_param |
||||
|
||||
@staticmethod |
||||
def parse_string_param_if_env(string_param: str, **kwargs): |
||||
"""Use $ENV{env_name} to load environment variable "env_name".""" |
||||
if "$ENV" in string_param: |
||||
key = re.findall(r"\$ENV\{(.*?)\}", string_param)[0] |
||||
env_value = os.environ.get(key, "$%s" % key) |
||||
string_param = string_param.replace("$ENV{%s}" % key, env_value) |
||||
return string_param |
||||
|
||||
@staticmethod |
||||
def parse_string_param_if_config(string_param: str, **kwargs): |
||||
"""Use ${CONFIG.var_name} to load variable "var_name" from configuration.""" |
||||
if "${CONFIG" in string_param: |
||||
key = re.findall(r"\$\{CONFIG\.(.*?)\}", string_param)[0] |
||||
if hasattr(configuration, key): |
||||
string_param = getattr(configuration, key) |
||||
else: |
||||
string_param = configuration.get_single_config(key) |
||||
|
||||
return string_param |
||||
|
||||
@staticmethod |
||||
def get_possible_path(file_path, base_folder): |
||||
"""Get file possible path. |
||||
|
||||
Return new path if file_path is not exists, but base_folder + file_path exists |
||||
""" |
||||
possible_path = file_path |
||||
if not Path(file_path).exists(): |
||||
new_path = Path(base_folder).joinpath(file_path) |
||||
if new_path.exists(): |
||||
possible_path = new_path |
||||
logger.info(f"{file_path} not exists, convert to {possible_path}") |
||||
|
||||
return possible_path |
||||
|
||||
|
||||
def get_task_cls(task_type) -> Task: |
||||
"""Get the task class object by task_type (case compatible).""" |
||||
# only get task class from tasks.__all__ |
||||
all_task_types = {type_.capitalize(): type_ for type_ in tasks.__all__} |
||||
task_type_cap = task_type.capitalize() |
||||
if task_type_cap not in all_task_types: |
||||
raise PyDSTaskNoFoundException("cant not find task %s" % task_type) |
||||
|
||||
standard_name = all_task_types[task_type_cap] |
||||
return getattr(tasks, standard_name) |
||||
|
||||
|
||||
class YamlProcess(YamlParser): |
||||
"""Yaml parser for create process. |
||||
|
||||
:param yaml_file: yaml file path. |
||||
|
||||
examples1 :: |
||||
|
||||
parser = YamlParser(yaml_file=...) |
||||
parser.create_process_definition() |
||||
|
||||
examples2 :: |
||||
|
||||
YamlParser(yaml_file=...).create_process_definition() |
||||
|
||||
""" |
||||
|
||||
_parse_rules = [ |
||||
ParseTool.parse_string_param_if_file, |
||||
ParseTool.parse_string_param_if_env, |
||||
ParseTool.parse_string_param_if_config, |
||||
] |
||||
|
||||
def __init__(self, yaml_file: str): |
||||
with open(yaml_file, "r") as f: |
||||
content = f.read() |
||||
|
||||
self._base_folder = Path(yaml_file).parent |
||||
content = self.prepare_refer_process(content) |
||||
super().__init__(content) |
||||
|
||||
def create_process_definition(self): |
||||
"""Create process main function.""" |
||||
# get process parameters with key "workflow" |
||||
process_params = self[KEY_PROCESS] |
||||
|
||||
# pop "run" parameter, used at the end |
||||
is_run = process_params.pop("run", False) |
||||
|
||||
# use YamlProcess._parse_rules to parse special value of yaml file |
||||
process_params = self.parse_params(process_params) |
||||
|
||||
process_name = process_params["name"] |
||||
logger.info(f"Create Process: {process_name}") |
||||
with ProcessDefinition(**process_params) as pd: |
||||
|
||||
# save dependencies between tasks |
||||
dependencies = {} |
||||
|
||||
# save name and task mapping |
||||
name2task = {} |
||||
|
||||
# get task datas with key "tasks" |
||||
for task_data in self[KEY_TASK]: |
||||
task = self.parse_task(task_data, name2task) |
||||
|
||||
deps = task_data.get(KEY_DEPS, []) |
||||
if deps: |
||||
dependencies[task.name] = deps |
||||
name2task[task.name] = task |
||||
|
||||
# build dependencies between task |
||||
for downstream_task_name, deps in dependencies.items(): |
||||
downstream_task = name2task[downstream_task_name] |
||||
for upstream_task_name in deps: |
||||
upstream_task = name2task[upstream_task_name] |
||||
upstream_task >> downstream_task |
||||
|
||||
pd.submit() |
||||
# if set is_run, run the process after submit |
||||
if is_run: |
||||
logger.info(f"run workflow: {pd}") |
||||
pd.run() |
||||
|
||||
return process_name |
||||
|
||||
def parse_params(self, params: Any): |
||||
"""Recursively resolves the parameter values. |
||||
|
||||
The function operates params only when it encounters a string; other types continue recursively. |
||||
""" |
||||
if isinstance(params, str): |
||||
for parse_rule in self._parse_rules: |
||||
params_ = params |
||||
params = parse_rule(params, base_folder=self._base_folder) |
||||
if params_ != params: |
||||
logger.info(f"parse {params_} -> {params}") |
||||
|
||||
elif isinstance(params, list): |
||||
for index in range(len(params)): |
||||
params[index] = self.parse_params(params[index]) |
||||
|
||||
elif isinstance(params, dict): |
||||
for key, value in params.items(): |
||||
params[key] = self.parse_params(value) |
||||
|
||||
return params |
||||
|
||||
@classmethod |
||||
def parse(cls, yaml_file: str): |
||||
"""Recursively resolves the parameter values. |
||||
|
||||
The function operates params only when it encounters a string; other types continue recursively. |
||||
""" |
||||
process_name = cls(yaml_file).create_process_definition() |
||||
return process_name |
||||
|
||||
def prepare_refer_process(self, content): |
||||
"""Allow YAML files to reference process derived from other YAML files.""" |
||||
process_paths = re.findall(r"\$WORKFLOW\{\"(.*?)\"\}", content) |
||||
for process_path in process_paths: |
||||
logger.info( |
||||
f"find special token {process_path}, load process form {process_path}" |
||||
) |
||||
possible_path = ParseTool.get_possible_path(process_path, self._base_folder) |
||||
process_name = YamlProcess.parse(possible_path) |
||||
content = content.replace('$WORKFLOW{"%s"}' % process_path, process_name) |
||||
|
||||
return content |
||||
|
||||
def parse_task(self, task_data: dict, name2task: Dict[str, Task]): |
||||
"""Parse various types of tasks. |
||||
|
||||
:param task_data: dict. |
||||
{ |
||||
"task_type": "Shell", |
||||
"params": {"name": "shell_task", "command":"ehco hellp"} |
||||
} |
||||
|
||||
:param name2task: Dict[str, Task]), mapping of task_name and task |
||||
|
||||
|
||||
Some task type have special parse func: |
||||
if task type is Switch, use parse_switch; |
||||
if task type is Condition, use parse_condition; |
||||
if task type is Dependent, use parse_dependent; |
||||
other, we pass all task_params as input to task class, like "task_cls(**task_params)". |
||||
""" |
||||
task_type = task_data["task_type"] |
||||
# get params without special key |
||||
task_params = {k: v for k, v in task_data.items() if k not in TASK_SPECIAL_KEYS} |
||||
|
||||
task_cls = get_task_cls(task_type) |
||||
|
||||
# use YamlProcess._parse_rules to parse special value of yaml file |
||||
task_params = self.parse_params(task_params) |
||||
|
||||
if task_cls == tasks.Switch: |
||||
task = self.parse_switch(task_params, name2task) |
||||
|
||||
elif task_cls == tasks.Condition: |
||||
task = self.parse_condition(task_params, name2task) |
||||
|
||||
elif task_cls == tasks.Dependent: |
||||
task = self.parse_dependent(task_params, name2task) |
||||
|
||||
else: |
||||
task = task_cls(**task_params) |
||||
logger.info(task_type, task) |
||||
return task |
||||
|
||||
def parse_switch(self, task_params, name2task): |
||||
"""Parse Switch Task. |
||||
|
||||
This is an example Yaml fragment of task_params |
||||
|
||||
name: switch |
||||
condition: |
||||
- ["${var} > 1", switch_child_1] |
||||
- switch_child_2 |
||||
""" |
||||
from pydolphinscheduler.tasks.switch import ( |
||||
Branch, |
||||
Default, |
||||
Switch, |
||||
SwitchCondition, |
||||
) |
||||
|
||||
condition_datas = task_params["condition"] |
||||
conditions = [] |
||||
for condition_data in condition_datas: |
||||
assert "task" in condition_data, "task must be in %s" % condition_data |
||||
task_name = condition_data["task"] |
||||
condition_string = condition_data.get("condition", None) |
||||
|
||||
# if condition_string is None, for example: {"task": "switch_child_2"}, set it to Default branch |
||||
if condition_string is None: |
||||
conditions.append(Default(task=name2task.get(task_name))) |
||||
|
||||
# if condition_string is not None, for example: |
||||
# {"task": "switch_child_2", "condition": "${var} > 1"} set it to Branch |
||||
else: |
||||
conditions.append( |
||||
Branch(condition_string, task=name2task.get(task_name)) |
||||
) |
||||
|
||||
switch = Switch( |
||||
name=task_params["name"], condition=SwitchCondition(*conditions) |
||||
) |
||||
return switch |
||||
|
||||
def parse_condition(self, task_params, name2task): |
||||
"""Parse Condition Task. |
||||
|
||||
This is an example Yaml fragment of task_params |
||||
|
||||
name: condition |
||||
success_task: success_branch |
||||
failed_task: fail_branch |
||||
OP: AND |
||||
groups: |
||||
- |
||||
OP: AND |
||||
groups: |
||||
- [pre_task_1, true] |
||||
- [pre_task_2, true] |
||||
- [pre_task_3, false] |
||||
- |
||||
OP: AND |
||||
groups: |
||||
- [pre_task_1, false] |
||||
- [pre_task_2, true] |
||||
- [pre_task_3, true] |
||||
|
||||
""" |
||||
from pydolphinscheduler.tasks.condition import ( |
||||
FAILURE, |
||||
SUCCESS, |
||||
And, |
||||
Condition, |
||||
Or, |
||||
) |
||||
|
||||
def get_op_cls(op): |
||||
cls = None |
||||
if op.lower() == "and": |
||||
cls = And |
||||
elif op.lower() == "or": |
||||
cls = Or |
||||
else: |
||||
raise Exception("OP must be in And or Or, but get: %s" % op) |
||||
return cls |
||||
|
||||
second_cond_ops = [] |
||||
for first_group in task_params["groups"]: |
||||
second_op = first_group["op"] |
||||
task_ops = [] |
||||
for condition_data in first_group["groups"]: |
||||
assert "task" in condition_data, "task must be in %s" % condition_data |
||||
assert "flag" in condition_data, "flag must be in %s" % condition_data |
||||
task_name = condition_data["task"] |
||||
flag = condition_data["flag"] |
||||
task = name2task[task_name] |
||||
|
||||
# for example: task = pre_task_1, flag = true |
||||
if flag: |
||||
task_ops.append(SUCCESS(task)) |
||||
else: |
||||
task_ops.append(FAILURE(task)) |
||||
|
||||
second_cond_ops.append(get_op_cls(second_op)(*task_ops)) |
||||
|
||||
first_op = task_params["op"] |
||||
cond_operator = get_op_cls(first_op)(*second_cond_ops) |
||||
|
||||
condition = Condition( |
||||
name=task_params["name"], |
||||
condition=cond_operator, |
||||
success_task=name2task[task_params["success_task"]], |
||||
failed_task=name2task[task_params["failed_task"]], |
||||
) |
||||
return condition |
||||
|
||||
def parse_dependent(self, task_params, name2task): |
||||
"""Parse Dependent Task. |
||||
|
||||
This is an example Yaml fragment of task_params |
||||
|
||||
name: dependent |
||||
denpendence: |
||||
OP: AND |
||||
groups: |
||||
- |
||||
OP: Or |
||||
groups: |
||||
- [pydolphin, task_dependent_external, task_1] |
||||
- [pydolphin, task_dependent_external, task_2] |
||||
- |
||||
OP: And |
||||
groups: |
||||
- [pydolphin, task_dependent_external, task_1, LAST_WEDNESDAY] |
||||
- [pydolphin, task_dependent_external, task_2, last24Hours] |
||||
|
||||
""" |
||||
from pydolphinscheduler.tasks.dependent import ( |
||||
And, |
||||
Dependent, |
||||
DependentDate, |
||||
DependentItem, |
||||
Or, |
||||
) |
||||
|
||||
def process_dependent_date(dependent_date): |
||||
"""Parse dependent date (Compatible with key and value of DependentDate).""" |
||||
dependent_date_upper = dependent_date.upper() |
||||
if hasattr(DependentDate, dependent_date_upper): |
||||
dependent_date = getattr(DependentDate, dependent_date_upper) |
||||
return dependent_date |
||||
|
||||
def get_op_cls(op): |
||||
cls = None |
||||
if op.lower() == "and": |
||||
cls = And |
||||
elif op.lower() == "or": |
||||
cls = Or |
||||
else: |
||||
raise Exception("OP must be in And or Or, but get: %s" % op) |
||||
return cls |
||||
|
||||
def create_dependent_item(source_items): |
||||
"""Parse dependent item. |
||||
|
||||
project_name: pydolphin |
||||
process_definition_name: task_dependent_external |
||||
dependent_task_name: task_1 |
||||
dependent_date: LAST_WEDNESDAY |
||||
""" |
||||
project_name = source_items["project_name"] |
||||
process_definition_name = source_items["process_definition_name"] |
||||
dependent_task_name = source_items["dependent_task_name"] |
||||
dependent_date = source_items.get("dependent_date", DependentDate.TODAY) |
||||
dependent_item = DependentItem( |
||||
project_name=project_name, |
||||
process_definition_name=process_definition_name, |
||||
dependent_task_name=dependent_task_name, |
||||
dependent_date=process_dependent_date(dependent_date), |
||||
) |
||||
|
||||
return dependent_item |
||||
|
||||
second_dependences = [] |
||||
for first_group in task_params["groups"]: |
||||
second_op = first_group[KEY_OP] |
||||
dependence_items = [] |
||||
for source_items in first_group["groups"]: |
||||
dependence_items.append(create_dependent_item(source_items)) |
||||
|
||||
second_dependences.append(get_op_cls(second_op)(*dependence_items)) |
||||
|
||||
first_op = task_params[KEY_OP] |
||||
dependence = get_op_cls(first_op)(*second_dependences) |
||||
|
||||
task = Dependent( |
||||
name=task_params["name"], |
||||
dependence=dependence, |
||||
) |
||||
return task |
||||
|
||||
|
||||
def create_process_definition(yaml_file): |
||||
"""CLI.""" |
||||
YamlProcess.parse(yaml_file) |
||||
@ -1,58 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# Setting about Java gateway server |
||||
java_gateway: |
||||
# The address of Python gateway server start. Set its value to `0.0.0.0` if your Python API run in different |
||||
# between Python gateway server. It could be be specific to other address like `127.0.0.1` or `localhost` |
||||
address: 127.0.0.1 |
||||
|
||||
# The port of Python gateway server start. Define which port you could connect to Python gateway server from |
||||
# Python API models. |
||||
port: 25333 |
||||
|
||||
# Whether automatically convert Python objects to Java Objects. Default value is ``True``. There is some |
||||
# performance lost when set to ``True`` but for now pydolphinscheduler do not handle the convert issue between |
||||
# java and Python, mark it as TODO item in the future. |
||||
auto_convert: true |
||||
|
||||
# Setting about dolphinscheduler default value, will use the value set below if property do not set, which |
||||
# including ``user``, ``workflow`` |
||||
default: |
||||
# Default value for dolphinscheduler's user object |
||||
user: |
||||
name: userPythonGateway |
||||
password: userPythonGateway |
||||
email: userPythonGateway@dolphinscheduler.com |
||||
tenant: tenant_pydolphin |
||||
phone: 11111111111 |
||||
state: 1 |
||||
# Default value for dolphinscheduler's workflow object |
||||
workflow: |
||||
project: project-pydolphin |
||||
tenant: tenant_pydolphin |
||||
user: userPythonGateway |
||||
queue: queuePythonGateway |
||||
worker_group: default |
||||
# Release state of workflow, default value is ``online`` which mean setting workflow online when it submits |
||||
# to Java gateway, if you want to set workflow offline set its value to ``offline`` |
||||
release_state: online |
||||
time_zone: Asia/Shanghai |
||||
# Warning type of the workflow, default value is ``NONE`` mean do not warn user in any cases of workflow state, |
||||
# change to ``FAILURE`` if you want to warn users when workflow failed. All available enum value are |
||||
# ``NONE``, ``SUCCESS``, ``FAILURE``, ``ALL`` |
||||
warning_type: NONE |
||||
@ -1,18 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Init examples package which provides users with pydolphinscheduler examples.""" |
||||
@ -1,55 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
""" |
||||
This example show you how to create workflows in batch mode. |
||||
|
||||
After this example run, we will create 10 workflows named `workflow:<workflow_num>`, and with 3 tasks |
||||
named `task:<task_num>-workflow:<workflow_num>` in each workflow. Task shape as below |
||||
|
||||
task:1-workflow:1 -> task:2-workflow:1 -> task:3-workflow:1 |
||||
|
||||
Each workflow is linear since we set `IS_CHAIN=True`, you could change task to parallel by set it to `False`. |
||||
""" |
||||
|
||||
from pydolphinscheduler.core.process_definition import ProcessDefinition |
||||
from pydolphinscheduler.tasks.shell import Shell |
||||
|
||||
NUM_WORKFLOWS = 10 |
||||
NUM_TASKS = 5 |
||||
# Make sure your tenant exists in your operator system |
||||
TENANT = "exists_tenant" |
||||
# Whether task should dependent on pre one or not |
||||
# False will create workflow with independent task, while True task will dependent on pre-task and dependence |
||||
# link like `pre_task -> current_task -> next_task`, default True |
||||
IS_CHAIN = True |
||||
|
||||
for wf in range(0, NUM_WORKFLOWS): |
||||
workflow_name = f"workflow:{wf}" |
||||
|
||||
with ProcessDefinition(name=workflow_name, tenant=TENANT) as pd: |
||||
for t in range(0, NUM_TASKS): |
||||
task_name = f"task:{t}-{workflow_name}" |
||||
command = f"echo This is task {task_name}" |
||||
task = Shell(name=task_name, command=command) |
||||
|
||||
if IS_CHAIN and t > 0: |
||||
pre_task_name = f"task:{t-1}-{workflow_name}" |
||||
pd.get_one_task_by_name(pre_task_name) >> task |
||||
|
||||
# We just submit workflow and task definition without set schedule time or run it manually |
||||
pd.submit() |
||||
@ -1,59 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# [start workflow_declare] |
||||
r""" |
||||
A example workflow for task condition. |
||||
|
||||
This example will create five task in single workflow, with four shell task and one condition task. Task |
||||
condition have one upstream which we declare explicit with syntax `parent >> condition`, and three downstream |
||||
automatically set dependence by condition task by passing parameter `condition`. The graph of this workflow |
||||
like: |
||||
pre_task_1 -> -> success_branch |
||||
\ / |
||||
pre_task_2 -> -> conditions -> |
||||
/ \ |
||||
pre_task_3 -> -> fail_branch |
||||
. |
||||
""" |
||||
|
||||
from pydolphinscheduler.core.process_definition import ProcessDefinition |
||||
from pydolphinscheduler.tasks.condition import FAILURE, SUCCESS, And, Condition |
||||
from pydolphinscheduler.tasks.shell import Shell |
||||
|
||||
with ProcessDefinition(name="task_condition_example", tenant="tenant_exists") as pd: |
||||
pre_task_1 = Shell(name="pre_task_1", command="echo pre_task_1") |
||||
pre_task_2 = Shell(name="pre_task_2", command="echo pre_task_2") |
||||
pre_task_3 = Shell(name="pre_task_3", command="echo pre_task_3") |
||||
cond_operator = And( |
||||
And( |
||||
SUCCESS(pre_task_1, pre_task_2), |
||||
FAILURE(pre_task_3), |
||||
), |
||||
) |
||||
|
||||
success_branch = Shell(name="success_branch", command="echo success_branch") |
||||
fail_branch = Shell(name="fail_branch", command="echo fail_branch") |
||||
|
||||
condition = Condition( |
||||
name="condition", |
||||
condition=cond_operator, |
||||
success_task=success_branch, |
||||
failed_task=fail_branch, |
||||
) |
||||
pd.submit() |
||||
# [end workflow_declare] |
||||
@ -1,95 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# [start workflow_declare] |
||||
""" |
||||
A example workflow for task datax. |
||||
|
||||
This example will create a workflow named `task_datax`. |
||||
`task_datax` is true workflow define and run task task_datax. |
||||
You can create data sources `first_mysql` and `first_mysql` through UI. |
||||
It creates a task to synchronize datax from the source database to the target database. |
||||
""" |
||||
|
||||
from pydolphinscheduler.core.process_definition import ProcessDefinition |
||||
from pydolphinscheduler.tasks.datax import CustomDataX, DataX |
||||
|
||||
# datax json template |
||||
JSON_TEMPLATE = { |
||||
"job": { |
||||
"content": [ |
||||
{ |
||||
"reader": { |
||||
"name": "mysqlreader", |
||||
"parameter": { |
||||
"username": "usr", |
||||
"password": "pwd", |
||||
"column": ["id", "name", "code", "description"], |
||||
"splitPk": "id", |
||||
"connection": [ |
||||
{ |
||||
"table": ["source_table"], |
||||
"jdbcUrl": ["jdbc:mysql://127.0.0.1:3306/source_db"], |
||||
} |
||||
], |
||||
}, |
||||
}, |
||||
"writer": { |
||||
"name": "mysqlwriter", |
||||
"parameter": { |
||||
"writeMode": "insert", |
||||
"username": "usr", |
||||
"password": "pwd", |
||||
"column": ["id", "name"], |
||||
"connection": [ |
||||
{ |
||||
"jdbcUrl": "jdbc:mysql://127.0.0.1:3306/target_db", |
||||
"table": ["target_table"], |
||||
} |
||||
], |
||||
}, |
||||
}, |
||||
} |
||||
], |
||||
"setting": { |
||||
"errorLimit": {"percentage": 0, "record": 0}, |
||||
"speed": {"channel": 1, "record": 1000}, |
||||
}, |
||||
} |
||||
} |
||||
|
||||
with ProcessDefinition( |
||||
name="task_datax_example", |
||||
tenant="tenant_exists", |
||||
) as pd: |
||||
# This task synchronizes the data in `t_ds_project` |
||||
# of `first_mysql` database to `target_project` of `second_mysql` database. |
||||
# You have to make sure data source named `first_mysql` and `second_mysql` exists |
||||
# in your environment. |
||||
task1 = DataX( |
||||
name="task_datax", |
||||
datasource_name="first_mysql", |
||||
datatarget_name="second_mysql", |
||||
sql="select id, name, code, description from source_table", |
||||
target_table="target_table", |
||||
) |
||||
|
||||
# You can custom json_template of datax to sync data. This task create a new |
||||
# datax job same as task1, transfer record from `first_mysql` to `second_mysql` |
||||
task2 = CustomDataX(name="task_custom_datax", json=str(JSON_TEMPLATE)) |
||||
pd.run() |
||||
# [end workflow_declare] |
||||
@ -1,74 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# [start workflow_declare] |
||||
r""" |
||||
A example workflow for task dependent. |
||||
|
||||
This example will create two workflows named `task_dependent` and `task_dependent_external`. |
||||
`task_dependent` is true workflow define and run task dependent, while `task_dependent_external` |
||||
define outside workflow and task from dependent. |
||||
|
||||
After this script submit, we would get workflow as below: |
||||
|
||||
task_dependent_external: |
||||
|
||||
task_1 |
||||
task_2 |
||||
task_3 |
||||
|
||||
task_dependent: |
||||
|
||||
task_dependent(this task dependent on task_dependent_external.task_1 and task_dependent_external.task_2). |
||||
""" |
||||
from pydolphinscheduler import configuration |
||||
from pydolphinscheduler.core.process_definition import ProcessDefinition |
||||
from pydolphinscheduler.tasks.dependent import And, Dependent, DependentItem, Or |
||||
from pydolphinscheduler.tasks.shell import Shell |
||||
|
||||
with ProcessDefinition( |
||||
name="task_dependent_external", |
||||
tenant="tenant_exists", |
||||
) as pd: |
||||
task_1 = Shell(name="task_1", command="echo task 1") |
||||
task_2 = Shell(name="task_2", command="echo task 2") |
||||
task_3 = Shell(name="task_3", command="echo task 3") |
||||
pd.submit() |
||||
|
||||
with ProcessDefinition( |
||||
name="task_dependent_example", |
||||
tenant="tenant_exists", |
||||
) as pd: |
||||
task = Dependent( |
||||
name="task_dependent", |
||||
dependence=And( |
||||
Or( |
||||
DependentItem( |
||||
project_name=configuration.WORKFLOW_PROJECT, |
||||
process_definition_name="task_dependent_external", |
||||
dependent_task_name="task_1", |
||||
), |
||||
DependentItem( |
||||
project_name=configuration.WORKFLOW_PROJECT, |
||||
process_definition_name="task_dependent_external", |
||||
dependent_task_name="task_2", |
||||
), |
||||
) |
||||
), |
||||
) |
||||
pd.submit() |
||||
# [end workflow_declare] |
||||
@ -1,52 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# [start workflow_declare] |
||||
"""A example workflow for task dvc.""" |
||||
|
||||
from pydolphinscheduler.core.process_definition import ProcessDefinition |
||||
from pydolphinscheduler.tasks import DVCDownload, DVCInit, DVCUpload |
||||
|
||||
repository = "git@github.com:<YOUR-NAME-OR-ORG>/dvc-data-repository-example.git" |
||||
|
||||
with ProcessDefinition( |
||||
name="task_dvc_example", |
||||
tenant="tenant_exists", |
||||
) as pd: |
||||
init_task = DVCInit(name="init_dvc", repository=repository, store_url="~/dvc_data") |
||||
upload_task = DVCUpload( |
||||
name="upload_data", |
||||
repository=repository, |
||||
data_path_in_dvc_repository="iris", |
||||
data_path_in_worker="~/source/iris", |
||||
version="v1", |
||||
message="upload iris data v1", |
||||
) |
||||
|
||||
download_task = DVCDownload( |
||||
name="download_data", |
||||
repository=repository, |
||||
data_path_in_dvc_repository="iris", |
||||
data_path_in_worker="~/target/iris", |
||||
version="v1", |
||||
) |
||||
|
||||
init_task >> upload_task >> download_task |
||||
|
||||
pd.run() |
||||
|
||||
# [end workflow_declare] |
||||
@ -1,33 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
# [start workflow_declare] |
||||
"""A example workflow for task flink.""" |
||||
|
||||
from pydolphinscheduler.core.process_definition import ProcessDefinition |
||||
from pydolphinscheduler.tasks.flink import DeployMode, Flink, ProgramType |
||||
|
||||
with ProcessDefinition(name="task_flink_example", tenant="tenant_exists") as pd: |
||||
task = Flink( |
||||
name="task_flink", |
||||
main_class="org.apache.flink.streaming.examples.wordcount.WordCount", |
||||
main_package="WordCount.jar", |
||||
program_type=ProgramType.JAVA, |
||||
deploy_mode=DeployMode.LOCAL, |
||||
) |
||||
pd.run() |
||||
# [end workflow_declare] |
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue