Browse Source

[Feature-8222][python] move examples into the scope of source package (#8340)

3.0.0/version-upgrade
Eric Gao 3 years ago committed by GitHub
parent
commit
49a1270ef5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/condition.rst
  2. 2
      dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/datax.rst
  3. 2
      dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/dependent.rst
  4. 2
      dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/flink.rst
  5. 2
      dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/map_reduce.rst
  6. 2
      dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/shell.rst
  7. 2
      dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/spark.rst
  8. 2
      dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/switch.rst
  9. 14
      dolphinscheduler-python/pydolphinscheduler/docs/source/tutorial.rst
  10. 18
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/__init__.py
  11. 0
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/bulk_create_example.py
  12. 0
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_condition_example.py
  13. 0
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_datax_example.py
  14. 0
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_dependent_example.py
  15. 0
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_flink_example.py
  16. 0
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_map_reduce_example.py
  17. 0
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_spark_example.py
  18. 0
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_switch_example.py
  19. 0
      dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial.py
  20. 7
      dolphinscheduler-python/pydolphinscheduler/tests/example/test_example.py
  21. 4
      dolphinscheduler-python/pydolphinscheduler/tests/testing/path.py

2
dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/condition.rst

@ -23,7 +23,7 @@ A condition task type's example and dive into information of **PyDolphinSchedule
Example Example
------- -------
.. literalinclude:: ../../../examples/task_condition_example.py .. literalinclude:: ../../../src/pydolphinscheduler/examples/task_condition_example.py
:start-after: [start workflow_declare] :start-after: [start workflow_declare]
:end-before: [end workflow_declare] :end-before: [end workflow_declare]

2
dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/datax.rst

@ -23,7 +23,7 @@ A DataX task type's example and dive into information of **PyDolphinScheduler**.
Example Example
------- -------
.. literalinclude:: ../../../examples/task_datax_example.py .. literalinclude:: ../../../src/pydolphinscheduler/examples/task_datax_example.py
:start-after: [start workflow_declare] :start-after: [start workflow_declare]
:end-before: [end workflow_declare] :end-before: [end workflow_declare]

2
dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/dependent.rst

@ -23,7 +23,7 @@ A dependent task type's example and dive into information of **PyDolphinSchedule
Example Example
------- -------
.. literalinclude:: ../../../examples/task_dependent_example.py .. literalinclude:: ../../../src/pydolphinscheduler/examples/task_dependent_example.py
:start-after: [start workflow_declare] :start-after: [start workflow_declare]
:end-before: [end workflow_declare] :end-before: [end workflow_declare]

2
dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/flink.rst

@ -23,7 +23,7 @@ A flink task type's example and dive into information of **PyDolphinScheduler**.
Example Example
------- -------
.. literalinclude:: ../../../examples/task_flink_example.py .. literalinclude:: ../../../src/pydolphinscheduler/examples/task_flink_example.py
:start-after: [start workflow_declare] :start-after: [start workflow_declare]
:end-before: [end workflow_declare] :end-before: [end workflow_declare]

2
dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/map_reduce.rst

@ -24,7 +24,7 @@ A Map Reduce task type's example and dive into information of **PyDolphinSchedul
Example Example
------- -------
.. literalinclude:: ../../../examples/task_map_reduce_example.py .. literalinclude:: ../../../src/pydolphinscheduler/examples/task_map_reduce_example.py
:start-after: [start workflow_declare] :start-after: [start workflow_declare]
:end-before: [end workflow_declare] :end-before: [end workflow_declare]

2
dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/shell.rst

@ -23,7 +23,7 @@ A shell task type's example and dive into information of **PyDolphinScheduler**.
Example Example
------- -------
.. literalinclude:: ../../../examples/tutorial.py .. literalinclude:: ../../../src/pydolphinscheduler/examples/tutorial.py
:start-after: [start workflow_declare] :start-after: [start workflow_declare]
:end-before: [end task_relation_declare] :end-before: [end task_relation_declare]

2
dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/spark.rst

@ -23,7 +23,7 @@ A spark task type's example and dive into information of **PyDolphinScheduler**.
Example Example
------- -------
.. literalinclude:: ../../../examples/task_spark_example.py .. literalinclude:: ../../../src/pydolphinscheduler/examples/task_spark_example.py
:start-after: [start workflow_declare] :start-after: [start workflow_declare]
:end-before: [end workflow_declare] :end-before: [end workflow_declare]

2
dolphinscheduler-python/pydolphinscheduler/docs/source/tasks/switch.rst

@ -23,7 +23,7 @@ A switch task type's example and dive into information of **PyDolphinScheduler**
Example Example
------- -------
.. literalinclude:: ../../../examples/task_switch_example.py .. literalinclude:: ../../../src/pydolphinscheduler/examples/task_switch_example.py
:start-after: [start workflow_declare] :start-after: [start workflow_declare]
:end-before: [end workflow_declare] :end-before: [end workflow_declare]

14
dolphinscheduler-python/pydolphinscheduler/docs/source/tutorial.rst

@ -29,7 +29,7 @@ Overview of Tutorial
Here have an overview of our tutorial, and it look a little complex but do not Here have an overview of our tutorial, and it look a little complex but do not
worry about that because we explain this example below as detailed as possible. worry about that because we explain this example below as detailed as possible.
.. literalinclude:: ../../examples/tutorial.py .. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py
:start-after: [start tutorial] :start-after: [start tutorial]
:end-before: [end tutorial] :end-before: [end tutorial]
@ -41,7 +41,7 @@ like other Python package. We just create a minimum demo here, so we just import
:class:`pydolphinscheduler.core.process_definition` and :class:`pydolphinscheduler.core.process_definition` and
:class:`pydolphinscheduler.tasks.shell`. :class:`pydolphinscheduler.tasks.shell`.
.. literalinclude:: ../../examples/tutorial.py .. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py
:start-after: [start package_import] :start-after: [start package_import]
:end-before: [end package_import] :end-before: [end package_import]
@ -60,7 +60,7 @@ interval and schedule start_time, and argument `tenant` which changing workflow'
task running user in the worker, :ref:`section tenant <concept:tenant>` in *PyDolphinScheduler* task running user in the worker, :ref:`section tenant <concept:tenant>` in *PyDolphinScheduler*
:doc:`concept` page have more detail information. :doc:`concept` page have more detail information.
.. literalinclude:: ../../examples/tutorial.py .. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py
:start-after: [start workflow_declare] :start-after: [start workflow_declare]
:end-before: [end workflow_declare] :end-before: [end workflow_declare]
@ -77,7 +77,7 @@ Here we declare four tasks, and bot of them are simple task of
Beside the argument `command`, we also need setting argument `name` for each task *(not Beside the argument `command`, we also need setting argument `name` for each task *(not
only shell task, `name` is required for each type of task)*. only shell task, `name` is required for each type of task)*.
.. literalinclude:: ../../examples/tutorial.py .. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py
:dedent: 0 :dedent: 0
:start-after: [start task_declare] :start-after: [start task_declare]
:end-before: [end task_declare] :end-before: [end task_declare]
@ -99,7 +99,7 @@ In this example, we set task `task_parent` is the upstream task of task
`task_child_one` and `task_child_two`, and task `task_union` is the downstream `task_child_one` and `task_child_two`, and task `task_union` is the downstream
task of both these two task. task of both these two task.
.. literalinclude:: ../../examples/tutorial.py .. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py
:dedent: 0 :dedent: 0
:start-after: [start task_relation_declare] :start-after: [start task_relation_declare]
:end-before: [end task_relation_declare] :end-before: [end task_relation_declare]
@ -124,7 +124,7 @@ to the daemon, and set the schedule time we just declare in `process definition
Now, we could run the Python code like other Python script, for the basic usage run Now, we could run the Python code like other Python script, for the basic usage run
:code:`python tutorial.py` to trigger and run it. :code:`python tutorial.py` to trigger and run it.
.. literalinclude:: ../../examples/tutorial.py .. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py
:dedent: 0 :dedent: 0
:start-after: [start submit_or_run] :start-after: [start submit_or_run]
:end-before: [end submit_or_run] :end-before: [end submit_or_run]
@ -142,7 +142,7 @@ After we run the tutorial code, you could login Apache DolphinScheduler web UI,
go and see the `DolphinScheduler project page`_. they is a new process definition be go and see the `DolphinScheduler project page`_. they is a new process definition be
created and named "Tutorial". It create by *PyDolphinScheduler* and the DAG graph as below created and named "Tutorial". It create by *PyDolphinScheduler* and the DAG graph as below
.. literalinclude:: ../../examples/tutorial.py .. literalinclude:: ../../src/pydolphinscheduler/examples/tutorial.py
:language: text :language: text
:lines: 24-28 :lines: 24-28

18
dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/__init__.py

@ -0,0 +1,18 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Init examples package which provides users with pydolphinscheduler examples."""

0
dolphinscheduler-python/pydolphinscheduler/examples/bulk_create_example.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/bulk_create_example.py

0
dolphinscheduler-python/pydolphinscheduler/examples/task_condition_example.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_condition_example.py

0
dolphinscheduler-python/pydolphinscheduler/examples/task_datax_example.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_datax_example.py

0
dolphinscheduler-python/pydolphinscheduler/examples/task_dependent_example.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_dependent_example.py

0
dolphinscheduler-python/pydolphinscheduler/examples/task_flink_example.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_flink_example.py

0
dolphinscheduler-python/pydolphinscheduler/examples/task_map_reduce_example.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_map_reduce_example.py

0
dolphinscheduler-python/pydolphinscheduler/examples/task_spark_example.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_spark_example.py

0
dolphinscheduler-python/pydolphinscheduler/examples/task_switch_example.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/task_switch_example.py

0
dolphinscheduler-python/pydolphinscheduler/examples/tutorial.py → dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/examples/tutorial.py

7
dolphinscheduler-python/pydolphinscheduler/tests/example/test_example.py

@ -96,8 +96,8 @@ def test_example_basic():
ex.suffix == ".py" ex.suffix == ".py"
), f"We expect all examples is python script, but get {ex.name}." ), f"We expect all examples is python script, but get {ex.name}."
# All except tutorial is end with keyword "_example" # All except tutorial and __init__ is end with keyword "_example"
if ex.stem != "tutorial": if ex.stem != "tutorial" and ex.stem != "__init__":
assert ex.stem.endswith( assert ex.stem.endswith(
"_example" "_example"
), f"We expect all examples script end with keyword '_example', but get {ex.stem}." ), f"We expect all examples script end with keyword '_example', but get {ex.stem}."
@ -159,6 +159,9 @@ def test_file_name_in_process_definition(mock_code_version, mock_submit, mock_st
""" """
global process_definition_name global process_definition_name
for ex in get_all_examples(): for ex in get_all_examples():
# Skip __init__ file
if ex.stem == "__init__":
continue
# Skip bulk_create_example check, cause it contain multiple workflow and # Skip bulk_create_example check, cause it contain multiple workflow and
# without one named bulk_create_example # without one named bulk_create_example
if ex.stem == "bulk_create_example": if ex.stem == "bulk_create_example":

4
dolphinscheduler-python/pydolphinscheduler/tests/testing/path.py

@ -23,7 +23,9 @@ from typing import Any, Generator
path_code_tasks = Path(__file__).parent.parent.parent.joinpath( path_code_tasks = Path(__file__).parent.parent.parent.joinpath(
"src", "pydolphinscheduler", "tasks" "src", "pydolphinscheduler", "tasks"
) )
path_example = Path(__file__).parent.parent.parent.joinpath("examples") path_example = Path(__file__).parent.parent.parent.joinpath(
"src", "pydolphinscheduler", "examples"
)
path_doc_tasks = Path(__file__).parent.parent.parent.joinpath("docs", "source", "tasks") path_doc_tasks = Path(__file__).parent.parent.parent.joinpath("docs", "source", "tasks")

Loading…
Cancel
Save