Browse Source
* [python] Add task decorator for python function * Add decorator `@task` * Add a tutorial about it * Change tutorial doc and combine into traditional docs * Add sphinx-inline-tab for better view * revert not need change * Correct python function indent * Correct integration test3.0.0/version-upgrade
Jiajie Zhong
3 years ago
committed by
GitHub
16 changed files with 656 additions and 106 deletions
@ -0,0 +1,33 @@ |
|||||||
|
.. Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
or more contributor license agreements. See the NOTICE file |
||||||
|
distributed with this work for additional information |
||||||
|
regarding copyright ownership. The ASF licenses this file |
||||||
|
to you under the Apache License, Version 2.0 (the |
||||||
|
"License"); you may not use this file except in compliance |
||||||
|
with the License. You may obtain a copy of the License at |
||||||
|
|
||||||
|
.. http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
|
||||||
|
.. Unless required by applicable law or agreed to in writing, |
||||||
|
software distributed under the License is distributed on an |
||||||
|
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
KIND, either express or implied. See the License for the |
||||||
|
specific language governing permissions and limitations |
||||||
|
under the License. |
||||||
|
|
||||||
|
Python Function Wrapper |
||||||
|
======================= |
||||||
|
|
||||||
|
A decorator covert Python function into pydolphinscheduler's task. |
||||||
|
|
||||||
|
Example |
||||||
|
------- |
||||||
|
|
||||||
|
.. literalinclude:: ../../../src/pydolphinscheduler/examples/tutorial_decorator.py |
||||||
|
:start-after: [start tutorial] |
||||||
|
:end-before: [end tutorial] |
||||||
|
|
||||||
|
Dive Into |
||||||
|
--------- |
||||||
|
|
||||||
|
.. automodule:: pydolphinscheduler.tasks.func_wrap |
@ -0,0 +1,91 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
r""" |
||||||
|
A tutorial example take you to experience pydolphinscheduler. |
||||||
|
|
||||||
|
After tutorial.py file submit to Apache DolphinScheduler server a DAG would be create, |
||||||
|
and workflow DAG graph as below: |
||||||
|
|
||||||
|
--> task_child_one |
||||||
|
/ \ |
||||||
|
task_parent --> --> task_union |
||||||
|
\ / |
||||||
|
--> task_child_two |
||||||
|
|
||||||
|
it will instantiate and run all the task it have. |
||||||
|
""" |
||||||
|
|
||||||
|
# [start tutorial] |
||||||
|
# [start package_import] |
||||||
|
# Import ProcessDefinition object to define your workflow attributes |
||||||
|
from pydolphinscheduler.core.process_definition import ProcessDefinition |
||||||
|
|
||||||
|
# Import task Shell object cause we would create some shell tasks later |
||||||
|
from pydolphinscheduler.tasks.func_wrap import task |
||||||
|
|
||||||
|
# [end package_import] |
||||||
|
|
||||||
|
|
||||||
|
# [start task_declare] |
||||||
|
@task |
||||||
|
def task_parent(): |
||||||
|
"""First task in this workflow.""" |
||||||
|
print("echo hello pydolphinscheduler") |
||||||
|
|
||||||
|
|
||||||
|
@task |
||||||
|
def task_child_one(): |
||||||
|
"""Child task will be run parallel after task ``task_parent`` finished.""" |
||||||
|
print("echo 'child one'") |
||||||
|
|
||||||
|
|
||||||
|
@task |
||||||
|
def task_child_two(): |
||||||
|
"""Child task will be run parallel after task ``task_parent`` finished.""" |
||||||
|
print("echo 'child two'") |
||||||
|
|
||||||
|
|
||||||
|
@task |
||||||
|
def task_union(): |
||||||
|
"""Last task in this workflow.""" |
||||||
|
print("echo union") |
||||||
|
|
||||||
|
|
||||||
|
# [end task_declare] |
||||||
|
|
||||||
|
|
||||||
|
# [start workflow_declare] |
||||||
|
with ProcessDefinition( |
||||||
|
name="tutorial_decorator", |
||||||
|
schedule="0 0 0 * * ? *", |
||||||
|
start_time="2021-01-01", |
||||||
|
tenant="tenant_exists", |
||||||
|
) as pd: |
||||||
|
# [end workflow_declare] |
||||||
|
|
||||||
|
# [start task_relation_declare] |
||||||
|
task_group = [task_child_one(), task_child_two()] |
||||||
|
task_parent().set_downstream(task_group) |
||||||
|
|
||||||
|
task_union() << task_group |
||||||
|
# [end task_relation_declare] |
||||||
|
|
||||||
|
# [start submit_or_run] |
||||||
|
pd.run() |
||||||
|
# [end submit_or_run] |
||||||
|
# [end tutorial] |
@ -0,0 +1,61 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Task function wrapper allows using decorator to create a task.""" |
||||||
|
|
||||||
|
import functools |
||||||
|
import inspect |
||||||
|
import itertools |
||||||
|
import types |
||||||
|
|
||||||
|
from pydolphinscheduler.exceptions import PyDSParamException |
||||||
|
from pydolphinscheduler.tasks.python import Python |
||||||
|
|
||||||
|
|
||||||
|
def _get_func_str(func: types.FunctionType) -> str: |
||||||
|
"""Get Python function string without indent from decorator. |
||||||
|
|
||||||
|
:param func: The function which wraps by decorator ``@task``. |
||||||
|
""" |
||||||
|
lines = inspect.getsourcelines(func)[0] |
||||||
|
|
||||||
|
src_strip = "" |
||||||
|
lead_space_num = None |
||||||
|
for line in lines: |
||||||
|
if lead_space_num is None: |
||||||
|
lead_space_num = sum(1 for _ in itertools.takewhile(str.isspace, line)) |
||||||
|
if line.strip() == "@task": |
||||||
|
continue |
||||||
|
elif line.strip().startswith("@"): |
||||||
|
raise PyDSParamException( |
||||||
|
"Do no support other decorators for function ``task`` decorator." |
||||||
|
) |
||||||
|
src_strip += line[lead_space_num:] |
||||||
|
return src_strip |
||||||
|
|
||||||
|
|
||||||
|
def task(func: types.FunctionType): |
||||||
|
"""Decorate which covert Python function into pydolphinscheduler task.""" |
||||||
|
|
||||||
|
@functools.wraps(func) |
||||||
|
def wrapper(*args, **kwargs): |
||||||
|
func_str = _get_func_str(func) |
||||||
|
return Python( |
||||||
|
name=kwargs.get("name", func.__name__), definition=func_str, *args, **kwargs |
||||||
|
) |
||||||
|
|
||||||
|
return wrapper |
@ -0,0 +1,169 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Test module about function wrap task decorator.""" |
||||||
|
|
||||||
|
from unittest.mock import patch |
||||||
|
|
||||||
|
import pytest |
||||||
|
|
||||||
|
from pydolphinscheduler.core.process_definition import ProcessDefinition |
||||||
|
from pydolphinscheduler.exceptions import PyDSParamException |
||||||
|
from pydolphinscheduler.tasks.func_wrap import task |
||||||
|
from tests.testing.decorator import foo as foo_decorator |
||||||
|
from tests.testing.task import Task |
||||||
|
|
||||||
|
PD_NAME = "test_process_definition" |
||||||
|
TASK_NAME = "test_task" |
||||||
|
|
||||||
|
|
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(12345, 1) |
||||||
|
) |
||||||
|
def test_single_task_outside(mock_code): |
||||||
|
"""Test single decorator task which outside process definition.""" |
||||||
|
|
||||||
|
@task |
||||||
|
def foo(): |
||||||
|
print(TASK_NAME) |
||||||
|
|
||||||
|
with ProcessDefinition(PD_NAME) as pd: |
||||||
|
foo() |
||||||
|
|
||||||
|
assert pd is not None and pd.name == PD_NAME |
||||||
|
assert len(pd.tasks) == 1 |
||||||
|
|
||||||
|
pd_task = pd.tasks[12345] |
||||||
|
assert pd_task.name == "foo" |
||||||
|
assert pd_task.raw_script == "def foo():\n print(TASK_NAME)\nfoo()" |
||||||
|
|
||||||
|
|
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(12345, 1) |
||||||
|
) |
||||||
|
def test_single_task_inside(mock_code): |
||||||
|
"""Test single decorator task which inside process definition.""" |
||||||
|
with ProcessDefinition(PD_NAME) as pd: |
||||||
|
|
||||||
|
@task |
||||||
|
def foo(): |
||||||
|
print(TASK_NAME) |
||||||
|
|
||||||
|
foo() |
||||||
|
|
||||||
|
assert pd is not None and pd.name == PD_NAME |
||||||
|
assert len(pd.tasks) == 1 |
||||||
|
|
||||||
|
pd_task = pd.tasks[12345] |
||||||
|
assert pd_task.name == "foo" |
||||||
|
assert pd_task.raw_script == "def foo():\n print(TASK_NAME)\nfoo()" |
||||||
|
|
||||||
|
|
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.core.task.Task.gen_code_and_version", return_value=(12345, 1) |
||||||
|
) |
||||||
|
def test_addition_decorator_error(mock_code): |
||||||
|
"""Test error when using task decorator to a function already have decorator.""" |
||||||
|
|
||||||
|
@task |
||||||
|
@foo_decorator |
||||||
|
def foo(): |
||||||
|
print(TASK_NAME) |
||||||
|
|
||||||
|
with ProcessDefinition(PD_NAME) as pd: # noqa: F841 |
||||||
|
with pytest.raises( |
||||||
|
PyDSParamException, match="Do no support other decorators for.*" |
||||||
|
): |
||||||
|
foo() |
||||||
|
|
||||||
|
|
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||||
|
side_effect=Task("test_func_wrap", "func_wrap").gen_code_and_version, |
||||||
|
) |
||||||
|
def test_multiple_tasks_outside(mock_code): |
||||||
|
"""Test multiple decorator tasks which outside process definition.""" |
||||||
|
|
||||||
|
@task |
||||||
|
def foo(): |
||||||
|
print(TASK_NAME) |
||||||
|
|
||||||
|
@task |
||||||
|
def bar(): |
||||||
|
print(TASK_NAME) |
||||||
|
|
||||||
|
with ProcessDefinition(PD_NAME) as pd: |
||||||
|
foo = foo() |
||||||
|
bar = bar() |
||||||
|
|
||||||
|
foo >> bar |
||||||
|
|
||||||
|
assert pd is not None and pd.name == PD_NAME |
||||||
|
assert len(pd.tasks) == 2 |
||||||
|
|
||||||
|
task_foo = pd.get_one_task_by_name("foo") |
||||||
|
task_bar = pd.get_one_task_by_name("bar") |
||||||
|
assert set(pd.task_list) == {task_foo, task_bar} |
||||||
|
assert ( |
||||||
|
task_foo is not None |
||||||
|
and task_foo._upstream_task_codes == set() |
||||||
|
and task_foo._downstream_task_codes.pop() == task_bar.code |
||||||
|
) |
||||||
|
assert ( |
||||||
|
task_bar is not None |
||||||
|
and task_bar._upstream_task_codes.pop() == task_foo.code |
||||||
|
and task_bar._downstream_task_codes == set() |
||||||
|
) |
||||||
|
|
||||||
|
|
||||||
|
@patch( |
||||||
|
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||||
|
side_effect=Task("test_func_wrap", "func_wrap").gen_code_and_version, |
||||||
|
) |
||||||
|
def test_multiple_tasks_inside(mock_code): |
||||||
|
"""Test multiple decorator tasks which inside process definition.""" |
||||||
|
with ProcessDefinition(PD_NAME) as pd: |
||||||
|
|
||||||
|
@task |
||||||
|
def foo(): |
||||||
|
print(TASK_NAME) |
||||||
|
|
||||||
|
@task |
||||||
|
def bar(): |
||||||
|
print(TASK_NAME) |
||||||
|
|
||||||
|
foo = foo() |
||||||
|
bar = bar() |
||||||
|
|
||||||
|
foo >> bar |
||||||
|
|
||||||
|
assert pd is not None and pd.name == PD_NAME |
||||||
|
assert len(pd.tasks) == 2 |
||||||
|
|
||||||
|
task_foo = pd.get_one_task_by_name("foo") |
||||||
|
task_bar = pd.get_one_task_by_name("bar") |
||||||
|
assert set(pd.task_list) == {task_foo, task_bar} |
||||||
|
assert ( |
||||||
|
task_foo is not None |
||||||
|
and task_foo._upstream_task_codes == set() |
||||||
|
and task_foo._downstream_task_codes.pop() == task_bar.code |
||||||
|
) |
||||||
|
assert ( |
||||||
|
task_bar is not None |
||||||
|
and task_bar._upstream_task_codes.pop() == task_foo.code |
||||||
|
and task_bar._downstream_task_codes == set() |
||||||
|
) |
@ -0,0 +1,32 @@ |
|||||||
|
# Licensed to the Apache Software Foundation (ASF) under one |
||||||
|
# or more contributor license agreements. See the NOTICE file |
||||||
|
# distributed with this work for additional information |
||||||
|
# regarding copyright ownership. The ASF licenses this file |
||||||
|
# to you under the Apache License, Version 2.0 (the |
||||||
|
# "License"); you may not use this file except in compliance |
||||||
|
# with the License. You may obtain a copy of the License at |
||||||
|
# |
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0 |
||||||
|
# |
||||||
|
# Unless required by applicable law or agreed to in writing, |
||||||
|
# software distributed under the License is distributed on an |
||||||
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||||
|
# KIND, either express or implied. See the License for the |
||||||
|
# specific language governing permissions and limitations |
||||||
|
# under the License. |
||||||
|
|
||||||
|
"""Decorator module for testing module.""" |
||||||
|
|
||||||
|
import types |
||||||
|
from functools import wraps |
||||||
|
|
||||||
|
|
||||||
|
def foo(func: types.FunctionType): |
||||||
|
"""Decorate which do nothing for testing module.""" |
||||||
|
|
||||||
|
@wraps(func) |
||||||
|
def wrapper(): |
||||||
|
print("foo decorator called.") |
||||||
|
func() |
||||||
|
|
||||||
|
return wrapper |
Loading…
Reference in new issue