Browse Source
* [python] Add task decorator for python function * Add decorator `@task` * Add a tutorial about it * Change tutorial doc and combine into traditional docs * Add sphinx-inline-tab for better view * revert not need change * Correct python function indent * Correct integration test3.0.0/version-upgrade
Jiajie Zhong
2 years ago
committed by
GitHub
16 changed files with 656 additions and 106 deletions
@ -0,0 +1,33 @@
|
||||
.. Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
.. Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
|
||||
Python Function Wrapper |
||||
======================= |
||||
|
||||
A decorator covert Python function into pydolphinscheduler's task. |
||||
|
||||
Example |
||||
------- |
||||
|
||||
.. literalinclude:: ../../../src/pydolphinscheduler/examples/tutorial_decorator.py |
||||
:start-after: [start tutorial] |
||||
:end-before: [end tutorial] |
||||
|
||||
Dive Into |
||||
--------- |
||||
|
||||
.. automodule:: pydolphinscheduler.tasks.func_wrap |
@ -0,0 +1,91 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
r""" |
||||
A tutorial example take you to experience pydolphinscheduler. |
||||
|
||||
After tutorial.py file submit to Apache DolphinScheduler server a DAG would be create, |
||||
and workflow DAG graph as below: |
||||
|
||||
--> task_child_one |
||||
/ \ |
||||
task_parent --> --> task_union |
||||
\ / |
||||
--> task_child_two |
||||
|
||||
it will instantiate and run all the task it have. |
||||
""" |
||||
|
||||
# [start tutorial] |
||||
# [start package_import] |
||||
# Import ProcessDefinition object to define your workflow attributes |
||||
from pydolphinscheduler.core.process_definition import ProcessDefinition |
||||
|
||||
# Import task Shell object cause we would create some shell tasks later |
||||
from pydolphinscheduler.tasks.func_wrap import task |
||||
|
||||
# [end package_import] |
||||
|
||||
|
||||
# [start task_declare] |
||||
@task |
||||
def task_parent(): |
||||
"""First task in this workflow.""" |
||||
print("echo hello pydolphinscheduler") |
||||
|
||||
|
||||
@task |
||||
def task_child_one(): |
||||
"""Child task will be run parallel after task ``task_parent`` finished.""" |
||||
print("echo 'child one'") |
||||
|
||||
|
||||
@task |
||||
def task_child_two(): |
||||
"""Child task will be run parallel after task ``task_parent`` finished.""" |
||||
print("echo 'child two'") |
||||
|
||||
|
||||
@task |
||||
def task_union(): |
||||
"""Last task in this workflow.""" |
||||
print("echo union") |
||||
|
||||
|
||||
# [end task_declare] |
||||
|
||||
|
||||
# [start workflow_declare] |
||||
with ProcessDefinition( |
||||
name="tutorial_decorator", |
||||
schedule="0 0 0 * * ? *", |
||||
start_time="2021-01-01", |
||||
tenant="tenant_exists", |
||||
) as pd: |
||||
# [end workflow_declare] |
||||
|
||||
# [start task_relation_declare] |
||||
task_group = [task_child_one(), task_child_two()] |
||||
task_parent().set_downstream(task_group) |
||||
|
||||
task_union() << task_group |
||||
# [end task_relation_declare] |
||||
|
||||
# [start submit_or_run] |
||||
pd.run() |
||||
# [end submit_or_run] |
||||
# [end tutorial] |
@ -0,0 +1,61 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Task function wrapper allows using decorator to create a task.""" |
||||
|
||||
import functools |
||||
import inspect |
||||
import itertools |
||||
import types |
||||
|
||||
from pydolphinscheduler.exceptions import PyDSParamException |
||||
from pydolphinscheduler.tasks.python import Python |
||||
|
||||
|
||||
def _get_func_str(func: types.FunctionType) -> str: |
||||
"""Get Python function string without indent from decorator. |
||||
|
||||
:param func: The function which wraps by decorator ``@task``. |
||||
""" |
||||
lines = inspect.getsourcelines(func)[0] |
||||
|
||||
src_strip = "" |
||||
lead_space_num = None |
||||
for line in lines: |
||||
if lead_space_num is None: |
||||
lead_space_num = sum(1 for _ in itertools.takewhile(str.isspace, line)) |
||||
if line.strip() == "@task": |
||||
continue |
||||
elif line.strip().startswith("@"): |
||||
raise PyDSParamException( |
||||
"Do no support other decorators for function ``task`` decorator." |
||||
) |
||||
src_strip += line[lead_space_num:] |
||||
return src_strip |
||||
|
||||
|
||||
def task(func: types.FunctionType): |
||||
"""Decorate which covert Python function into pydolphinscheduler task.""" |
||||
|
||||
@functools.wraps(func) |
||||
def wrapper(*args, **kwargs): |
||||
func_str = _get_func_str(func) |
||||
return Python( |
||||
name=kwargs.get("name", func.__name__), definition=func_str, *args, **kwargs |
||||
) |
||||
|
||||
return wrapper |
@ -0,0 +1,169 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Test module about function wrap task decorator.""" |
||||
|
||||
from unittest.mock import patch |
||||
|
||||
import pytest |
||||
|
||||
from pydolphinscheduler.core.process_definition import ProcessDefinition |
||||
from pydolphinscheduler.exceptions import PyDSParamException |
||||
from pydolphinscheduler.tasks.func_wrap import task |
||||
from tests.testing.decorator import foo as foo_decorator |
||||
from tests.testing.task import Task |
||||
|
||||
PD_NAME = "test_process_definition" |
||||
TASK_NAME = "test_task" |
||||
|
||||
|
||||
@patch( |
||||
"pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(12345, 1) |
||||
) |
||||
def test_single_task_outside(mock_code): |
||||
"""Test single decorator task which outside process definition.""" |
||||
|
||||
@task |
||||
def foo(): |
||||
print(TASK_NAME) |
||||
|
||||
with ProcessDefinition(PD_NAME) as pd: |
||||
foo() |
||||
|
||||
assert pd is not None and pd.name == PD_NAME |
||||
assert len(pd.tasks) == 1 |
||||
|
||||
pd_task = pd.tasks[12345] |
||||
assert pd_task.name == "foo" |
||||
assert pd_task.raw_script == "def foo():\n print(TASK_NAME)\nfoo()" |
||||
|
||||
|
||||
@patch( |
||||
"pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(12345, 1) |
||||
) |
||||
def test_single_task_inside(mock_code): |
||||
"""Test single decorator task which inside process definition.""" |
||||
with ProcessDefinition(PD_NAME) as pd: |
||||
|
||||
@task |
||||
def foo(): |
||||
print(TASK_NAME) |
||||
|
||||
foo() |
||||
|
||||
assert pd is not None and pd.name == PD_NAME |
||||
assert len(pd.tasks) == 1 |
||||
|
||||
pd_task = pd.tasks[12345] |
||||
assert pd_task.name == "foo" |
||||
assert pd_task.raw_script == "def foo():\n print(TASK_NAME)\nfoo()" |
||||
|
||||
|
||||
@patch( |
||||
"pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(12345, 1) |
||||
) |
||||
def test_addition_decorator_error(mock_code): |
||||
"""Test error when using task decorator to a function already have decorator.""" |
||||
|
||||
@task |
||||
@foo_decorator |
||||
def foo(): |
||||
print(TASK_NAME) |
||||
|
||||
with ProcessDefinition(PD_NAME) as pd: # noqa: F841 |
||||
with pytest.raises( |
||||
PyDSParamException, match="Do no support other decorators for.*" |
||||
): |
||||
foo() |
||||
|
||||
|
||||
@patch( |
||||
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||
side_effect=Task("test_func_wrap", "func_wrap").gen_code_and_version, |
||||
) |
||||
def test_multiple_tasks_outside(mock_code): |
||||
"""Test multiple decorator tasks which outside process definition.""" |
||||
|
||||
@task |
||||
def foo(): |
||||
print(TASK_NAME) |
||||
|
||||
@task |
||||
def bar(): |
||||
print(TASK_NAME) |
||||
|
||||
with ProcessDefinition(PD_NAME) as pd: |
||||
foo = foo() |
||||
bar = bar() |
||||
|
||||
foo >> bar |
||||
|
||||
assert pd is not None and pd.name == PD_NAME |
||||
assert len(pd.tasks) == 2 |
||||
|
||||
task_foo = pd.get_one_task_by_name("foo") |
||||
task_bar = pd.get_one_task_by_name("bar") |
||||
assert set(pd.task_list) == {task_foo, task_bar} |
||||
assert ( |
||||
task_foo is not None |
||||
and task_foo._upstream_task_codes == set() |
||||
and task_foo._downstream_task_codes.pop() == task_bar.code |
||||
) |
||||
assert ( |
||||
task_bar is not None |
||||
and task_bar._upstream_task_codes.pop() == task_foo.code |
||||
and task_bar._downstream_task_codes == set() |
||||
) |
||||
|
||||
|
||||
@patch( |
||||
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||
side_effect=Task("test_func_wrap", "func_wrap").gen_code_and_version, |
||||
) |
||||
def test_multiple_tasks_inside(mock_code): |
||||
"""Test multiple decorator tasks which inside process definition.""" |
||||
with ProcessDefinition(PD_NAME) as pd: |
||||
|
||||
@task |
||||
def foo(): |
||||
print(TASK_NAME) |
||||
|
||||
@task |
||||
def bar(): |
||||
print(TASK_NAME) |
||||
|
||||
foo = foo() |
||||
bar = bar() |
||||
|
||||
foo >> bar |
||||
|
||||
assert pd is not None and pd.name == PD_NAME |
||||
assert len(pd.tasks) == 2 |
||||
|
||||
task_foo = pd.get_one_task_by_name("foo") |
||||
task_bar = pd.get_one_task_by_name("bar") |
||||
assert set(pd.task_list) == {task_foo, task_bar} |
||||
assert ( |
||||
task_foo is not None |
||||
and task_foo._upstream_task_codes == set() |
||||
and task_foo._downstream_task_codes.pop() == task_bar.code |
||||
) |
||||
assert ( |
||||
task_bar is not None |
||||
and task_bar._upstream_task_codes.pop() == task_foo.code |
||||
and task_bar._downstream_task_codes == set() |
||||
) |
@ -0,0 +1,32 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Decorator module for testing module.""" |
||||
|
||||
import types |
||||
from functools import wraps |
||||
|
||||
|
||||
def foo(func: types.FunctionType): |
||||
"""Decorate which do nothing for testing module.""" |
||||
|
||||
@wraps(func) |
||||
def wrapper(): |
||||
print("foo decorator called.") |
||||
func() |
||||
|
||||
return wrapper |
Loading…
Reference in new issue