Browse Source

[Feature-8222][python] move examples into the scope of source package (#8340)

3.0.0/version-upgrade
Eric Gao 2 years ago committed by GitHub
parent
commit
49a1270ef5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/condition.rst
  2. 2
      dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/datax.rst
  3. 2
      dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/dependent.rst
  4. 2
      dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/flink.rst
  5. 2
      dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/map_reduce.rst
  6. 2
      dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/shell.rst
  7. 2
      dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/spark.rst
  8. 2
      dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/switch.rst
  9. 14
      dolphinscheduler-python/pydolphinscheduler/docs/source/tutorial.rst
  10. 18
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/__init__.py
  11. 0
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/bulk_create_example.py
  12. 0
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_condition_example.py
  13. 0
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_datax_example.py
  14. 0
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_dependent_example.py
  15. 0
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_flink_example.py
  16. 0
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_map_reduce_example.py
  17. 0
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_spark_example.py
  18. 0
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_switch_example.py
  19. 0
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial.py
  20. 7
      dolphinscheduler-python/pydolphinscheduler/tests/example/test_example.py
  21. 4
      dolphinscheduler-python/pydolphinscheduler/tests/testing/path.py

2
dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/condition.rst

@ -23,7 +23,7 @@ A condition task type's example and dive into information of **PyDolphinSchedule
Example
-------
.. literalinclude:: ../../../examples/task_condition_example.py
.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_condition_example.py
:start-after: [start workflow_declare]
:end-before: [end workflow_declare]

2
dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/datax.rst

@ -23,7 +23,7 @@ A DataX task type's example and dive into information of **PyDolphinScheduler**.
Example
-------
.. literalinclude:: ../../../examples/task_datax_example.py
.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_datax_example.py
:start-after: [start workflow_declare]
:end-before: [end workflow_declare]

2
dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/dependent.rst

@ -23,7 +23,7 @@ A dependent task type's example and dive into information of **PyDolphinSchedule
Example
-------
.. literalinclude:: ../../../examples/task_dependent_example.py
.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_dependent_example.py
:start-after: [start workflow_declare]
:end-before: [end workflow_declare]

2
dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/flink.rst

@ -23,7 +23,7 @@ A flink task type's example and dive into information of **PyDolphinScheduler**.
Example
-------
.. literalinclude:: ../../../examples/task_flink_example.py
.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_flink_example.py
:start-after: [start workflow_declare]
:end-before: [end workflow_declare]

2
dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/map_reduce.rst

@ -24,7 +24,7 @@ A Map Reduce task type's example and dive into information of **PyDolphinSchedul
Example
-------
.. literalinclude:: ../../../examples/task_map_reduce_example.py
.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_map_reduce_example.py
:start-after: [start workflow_declare]
:end-before: [end workflow_declare]

2
dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/shell.rst

@ -23,7 +23,7 @@ A shell task type's example and dive into information of **PyDolphinScheduler**.
Example
-------
.. literalinclude:: ../../../examples/tutorial.py
.. literalinclude:: ../../../src/pydolphinscheduler/examples/tutorial.py
:start-after: [start workflow_declare]
:end-before: [end task_relation_declare]

2
dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/spark.rst

@ -23,7 +23,7 @@ A spark task type's example and dive into information of **PyDolphinScheduler**.
Example
-------
.. literalinclude:: ../../../examples/task_spark_example.py
.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_spark_example.py
:start-after: [start workflow_declare]
:end-before: [end workflow_declare]

2
dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/switch.rst

@ -23,7 +23,7 @@ A switch task type's example and dive into information of **PyDolphinScheduler**
Example
-------
.. literalinclude:: ../../../examples/task_switch_example.py
.. literalinclude:: ../../../src/pydolphinscheduler/examples/task_switch_example.py
:start-after: [start workflow_declare]
:end-before: [end workflow_declare]

14
dolphinscheduler-python/pydolphinscheduler/docs/source/tutorial.rst

@ -29,7 +29,7 @@ Overview of Tutorial
Here have an overview of our tutorial, and it look a little complex but do not
worry about that because we explain this example below as detailed as possible.
.. literalinclude:: ../../examples/tutorial.py
.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py
:start-after: [start tutorial]
:end-before: [end tutorial]
@ -41,7 +41,7 @@ like other Python package. We just create a minimum demo here, so we just import
:class:`pydolphinscheduler.core.process_definition` and
:class:`pydolphinscheduler.tasks.shell`.
.. literalinclude:: ../../examples/tutorial.py
.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py
:start-after: [start package_import]
:end-before: [end package_import]
@ -60,7 +60,7 @@ interval and schedule start_time, and argument `tenant` which changing workflow'
task running user in the worker, :ref:`section tenant <concept:tenant>` in *PyDolphinScheduler*
:doc:`concept` page have more detail information.
.. literalinclude:: ../../examples/tutorial.py
.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py
:start-after: [start workflow_declare]
:end-before: [end workflow_declare]
@ -77,7 +77,7 @@ Here we declare four tasks, and bot of them are simple task of
Beside the argument `command`, we also need setting argument `name` for each task *(not
only shell task, `name` is required for each type of task)*.
.. literalinclude:: ../../examples/tutorial.py
.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py
:dedent: 0
:start-after: [start task_declare]
:end-before: [end task_declare]
@ -99,7 +99,7 @@ In this example, we set task `task_parent` is the upstream task of task
`task_child_one` and `task_child_two`, and task `task_union` is the downstream
task of both these two task.
.. literalinclude:: ../../examples/tutorial.py
.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py
:dedent: 0
:start-after: [start task_relation_declare]
:end-before: [end task_relation_declare]
@ -124,7 +124,7 @@ to the daemon, and set the schedule time we just declare in `process definition
Now, we could run the Python code like other Python script, for the basic usage run
:code:`python tutorial.py` to trigger and run it.
.. literalinclude:: ../../examples/tutorial.py
.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py
:dedent: 0
:start-after: [start submit_or_run]
:end-before: [end submit_or_run]
@ -142,7 +142,7 @@ After we run the tutorial code, you could login Apache DolphinScheduler web UI,
go and see the `DolphinScheduler project page`_. they is a new process definition be
created and named "Tutorial". It create by *PyDolphinScheduler* and the DAG graph as below
.. literalinclude:: ../../examples/tutorial.py
.. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py
:language: text
:lines: 24-28

18
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/__init__.py

@ -0,0 +1,18 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Init examples package which provides users with pydolphinscheduler examples."""

0
dolphinscheduler-python/pydolphinscheduler/examples/bulk_create_example.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/bulk_create_example.py

0
dolphinscheduler-python/pydolphinscheduler/examples/task_condition_example.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_condition_example.py

0
dolphinscheduler-python/pydolphinscheduler/examples/task_datax_example.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_datax_example.py

0
dolphinscheduler-python/pydolphinscheduler/examples/task_dependent_example.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_dependent_example.py

0
dolphinscheduler-python/pydolphinscheduler/examples/task_flink_example.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_flink_example.py

0
dolphinscheduler-python/pydolphinscheduler/examples/task_map_reduce_example.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_map_reduce_example.py

0
dolphinscheduler-python/pydolphinscheduler/examples/task_spark_example.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_spark_example.py

0
dolphinscheduler-python/pydolphinscheduler/examples/task_switch_example.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_switch_example.py

0
dolphinscheduler-python/pydolphinscheduler/examples/tutorial.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial.py

7
dolphinscheduler-python/pydolphinscheduler/tests/example/test_example.py

@ -96,8 +96,8 @@ def test_example_basic():
ex.suffix == ".py"
), f"We expect all examples is python script, but get {ex.name}."
# All except tutorial is end with keyword "_example"
if ex.stem != "tutorial":
# All except tutorial and __init__ is end with keyword "_example"
if ex.stem != "tutorial" and ex.stem != "__init__":
assert ex.stem.endswith(
"_example"
), f"We expect all examples script end with keyword '_example', but get {ex.stem}."
@ -159,6 +159,9 @@ def test_file_name_in_process_definition(mock_code_version, mock_submit, mock_st
"""
global process_definition_name
for ex in get_all_examples():
# Skip __init__ file
if ex.stem == "__init__":
continue
# Skip bulk_create_example check, cause it contain multiple workflow and
# without one named bulk_create_example
if ex.stem == "bulk_create_example":

4
dolphinscheduler-python/pydolphinscheduler/tests/testing/path.py

@ -23,7 +23,9 @@ from typing import Any, Generator
path_code_tasks = Path(__file__).parent.parent.parent.joinpath(
"src", "pydolphinscheduler", "tasks"
)
path_example = Path(__file__).parent.parent.parent.joinpath("examples")
path_example = Path(__file__).parent.parent.parent.joinpath(
"src", "pydolphinscheduler", "examples"
)
path_doc_tasks = Path(__file__).parent.parent.parent.joinpath("docs", "source", "tasks")

Loading…
Cancel
Save