Browse Source
* [python] Add task type python http * Fix unittest error * Fix UT error3.0.0/version-upgrade
Jiajie Zhong
3 years ago
committed by
GitHub
10 changed files with 435 additions and 33 deletions
@ -0,0 +1,115 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Task shell.""" |
||||
|
||||
from typing import Optional |
||||
|
||||
from pydolphinscheduler.constants import TaskType |
||||
from pydolphinscheduler.core.task import Task, TaskParams |
||||
|
||||
|
||||
class HttpMethod: |
||||
"""Constant of HTTP method.""" |
||||
|
||||
GET = "GET" |
||||
POST = "POST" |
||||
HEAD = "HEAD" |
||||
PUT = "PUT" |
||||
DELETE = "DELETE" |
||||
|
||||
|
||||
class HttpCheckCondition: |
||||
"""Constant of HTTP check condition. |
||||
|
||||
For now it contain four value: |
||||
- STATUS_CODE_DEFAULT: when http response code equal to 200, mark as success. |
||||
- STATUS_CODE_CUSTOM: when http response code equal to the code user define, mark as success. |
||||
- BODY_CONTAINS: when http response body contain text user define, mark as success. |
||||
- BODY_NOT_CONTAINS: when http response body do not contain text user define, mark as success. |
||||
""" |
||||
|
||||
STATUS_CODE_DEFAULT = "STATUS_CODE_DEFAULT" |
||||
STATUS_CODE_CUSTOM = "STATUS_CODE_CUSTOM" |
||||
BODY_CONTAINS = "BODY_CONTAINS" |
||||
BODY_NOT_CONTAINS = "BODY_NOT_CONTAINS" |
||||
|
||||
|
||||
class HttpTaskParams(TaskParams): |
||||
"""Parameter only for Http task types.""" |
||||
|
||||
def __init__( |
||||
self, |
||||
url: str, |
||||
http_method: Optional[str] = HttpMethod.GET, |
||||
http_params: Optional[str] = None, |
||||
http_check_condition: Optional[str] = HttpCheckCondition.STATUS_CODE_DEFAULT, |
||||
condition: Optional[str] = None, |
||||
connect_timeout: Optional[int] = 60000, |
||||
socket_timeout: Optional[int] = 60000, |
||||
*args, |
||||
**kwargs |
||||
): |
||||
super().__init__(*args, **kwargs) |
||||
self.url = url |
||||
if not hasattr(HttpMethod, http_method): |
||||
raise ValueError("Parameter http_method %s not support.", http_method) |
||||
self.http_method = http_method |
||||
self.http_params = http_params or [] |
||||
if not hasattr(HttpCheckCondition, http_check_condition): |
||||
raise ValueError( |
||||
"Parameter http_check_condition %s not support.", http_check_condition |
||||
) |
||||
self.http_check_condition = http_check_condition |
||||
if ( |
||||
http_check_condition != HttpCheckCondition.STATUS_CODE_DEFAULT |
||||
and condition is None |
||||
): |
||||
raise ValueError( |
||||
"Parameter condition must provider if http_check_condition not equal to STATUS_CODE_DEFAULT" |
||||
) |
||||
self.condition = condition |
||||
self.connect_timeout = connect_timeout |
||||
self.socket_timeout = socket_timeout |
||||
|
||||
|
||||
class Http(Task): |
||||
"""Task HTTP object, declare behavior for HTTP task to dolphinscheduler.""" |
||||
|
||||
def __init__( |
||||
self, |
||||
name: str, |
||||
url: str, |
||||
http_method: Optional[str] = HttpMethod.GET, |
||||
http_params: Optional[str] = None, |
||||
http_check_condition: Optional[str] = HttpCheckCondition.STATUS_CODE_DEFAULT, |
||||
condition: Optional[str] = None, |
||||
connect_timeout: Optional[int] = 60000, |
||||
socket_timeout: Optional[int] = 60000, |
||||
*args, |
||||
**kwargs |
||||
): |
||||
task_params = HttpTaskParams( |
||||
url=url, |
||||
http_method=http_method, |
||||
http_params=http_params, |
||||
http_check_condition=http_check_condition, |
||||
condition=condition, |
||||
connect_timeout=connect_timeout, |
||||
socket_timeout=socket_timeout, |
||||
) |
||||
super().__init__(name, TaskType.HTTP, task_params, *args, **kwargs) |
||||
@ -0,0 +1,47 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Task Python.""" |
||||
|
||||
import inspect |
||||
import types |
||||
from typing import Any |
||||
|
||||
from pydolphinscheduler.constants import TaskType |
||||
from pydolphinscheduler.core.task import Task, TaskParams |
||||
|
||||
|
||||
class PythonTaskParams(TaskParams): |
||||
"""Parameter only for Python task types.""" |
||||
|
||||
def __init__(self, raw_script: str, *args, **kwargs): |
||||
super().__init__(*args, **kwargs) |
||||
self.raw_script = raw_script |
||||
|
||||
|
||||
class Python(Task): |
||||
"""Task Python object, declare behavior for Python task to dolphinscheduler.""" |
||||
|
||||
def __init__(self, name: str, code: Any, *args, **kwargs): |
||||
if isinstance(code, str): |
||||
task_params = PythonTaskParams(raw_script=code) |
||||
elif isinstance(code, types.FunctionType): |
||||
py_function = inspect.getsource(code) |
||||
task_params = PythonTaskParams(raw_script=py_function) |
||||
else: |
||||
raise ValueError("Parameter code do not support % for now.", type(code)) |
||||
super().__init__(name, TaskType.PYTHON, task_params, *args, **kwargs) |
||||
@ -0,0 +1,112 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Test Task HTTP.""" |
||||
|
||||
from unittest.mock import patch |
||||
|
||||
import pytest |
||||
|
||||
from pydolphinscheduler.tasks.http import ( |
||||
Http, |
||||
HttpCheckCondition, |
||||
HttpMethod, |
||||
HttpTaskParams, |
||||
) |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"class_name, attrs", |
||||
[ |
||||
(HttpMethod, ("GET", "POST", "HEAD", "PUT", "DELETE")), |
||||
( |
||||
HttpCheckCondition, |
||||
( |
||||
"STATUS_CODE_DEFAULT", |
||||
"STATUS_CODE_CUSTOM", |
||||
"BODY_CONTAINS", |
||||
"BODY_NOT_CONTAINS", |
||||
), |
||||
), |
||||
], |
||||
) |
||||
def test_attr_exists(class_name, attrs): |
||||
"""Test weather class HttpMethod and HttpCheckCondition contain specific attribute.""" |
||||
assert all(hasattr(class_name, attr) for attr in attrs) |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"param", |
||||
[ |
||||
{"http_method": "http_method"}, |
||||
{"http_check_condition": "http_check_condition"}, |
||||
{"http_check_condition": HttpCheckCondition.STATUS_CODE_CUSTOM}, |
||||
{ |
||||
"http_check_condition": HttpCheckCondition.STATUS_CODE_CUSTOM, |
||||
"condition": None, |
||||
}, |
||||
], |
||||
) |
||||
def test_http_task_param_not_support_param(param): |
||||
"""Test HttpTaskParams not support parameter.""" |
||||
url = "https://www.apache.org" |
||||
with pytest.raises(ValueError, match="Parameter .*?"): |
||||
HttpTaskParams(url, **param) |
||||
|
||||
|
||||
def test_http_to_dict(): |
||||
"""Test task HTTP function to_dict.""" |
||||
code = 123 |
||||
version = 1 |
||||
name = "test_http_to_dict" |
||||
url = "https://www.apache.org" |
||||
expect = { |
||||
"code": code, |
||||
"name": name, |
||||
"version": 1, |
||||
"description": None, |
||||
"delayTime": 0, |
||||
"taskType": "HTTP", |
||||
"taskParams": { |
||||
"localParams": [], |
||||
"httpParams": [], |
||||
"url": url, |
||||
"httpMethod": "GET", |
||||
"httpCheckCondition": "STATUS_CODE_DEFAULT", |
||||
"condition": None, |
||||
"connectTimeout": 60000, |
||||
"socketTimeout": 60000, |
||||
"dependence": {}, |
||||
"resourceList": [], |
||||
"conditionResult": {"successNode": [""], "failedNode": [""]}, |
||||
"waitStartTimeout": {}, |
||||
}, |
||||
"flag": "YES", |
||||
"taskPriority": "MEDIUM", |
||||
"workerGroup": "default", |
||||
"failRetryTimes": 0, |
||||
"failRetryInterval": 1, |
||||
"timeoutFlag": "CLOSE", |
||||
"timeoutNotifyStrategy": None, |
||||
"timeout": 0, |
||||
} |
||||
with patch( |
||||
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||
return_value=(code, version), |
||||
): |
||||
http = Http(name, url) |
||||
assert http.to_dict() == expect |
||||
@ -0,0 +1,115 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Test Task python.""" |
||||
|
||||
|
||||
from unittest.mock import patch |
||||
|
||||
import pytest |
||||
|
||||
from pydolphinscheduler.tasks.python import Python, PythonTaskParams |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"name, value", |
||||
[ |
||||
("local_params", "local_params"), |
||||
("resource_list", "resource_list"), |
||||
("dependence", "dependence"), |
||||
("wait_start_timeout", "wait_start_timeout"), |
||||
("condition_result", "condition_result"), |
||||
], |
||||
) |
||||
def test_python_task_params_attr_setter(name, value): |
||||
"""Test python task parameters.""" |
||||
command = 'print("hello world.")' |
||||
python_task_params = PythonTaskParams(command) |
||||
assert command == python_task_params.raw_script |
||||
setattr(python_task_params, name, value) |
||||
assert value == getattr(python_task_params, name) |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"script_code", |
||||
[ |
||||
123, |
||||
("print", "hello world"), |
||||
], |
||||
) |
||||
def test_python_task_not_support_code(script_code): |
||||
"""Test python task parameters.""" |
||||
name = "not_support_code_type" |
||||
code = 123 |
||||
version = 1 |
||||
with patch( |
||||
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||
return_value=(code, version), |
||||
): |
||||
with pytest.raises(ValueError, match="Parameter code do not support .*?"): |
||||
Python(name, script_code) |
||||
|
||||
|
||||
def foo(): # noqa: D103 |
||||
print("hello world.") |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"name, script_code, raw", |
||||
[ |
||||
("string_define", 'print("hello world.")', 'print("hello world.")'), |
||||
( |
||||
"function_define", |
||||
foo, |
||||
'def foo(): # noqa: D103\n print("hello world.")\n', |
||||
), |
||||
], |
||||
) |
||||
def test_python_to_dict(name, script_code, raw): |
||||
"""Test task python function to_dict.""" |
||||
code = 123 |
||||
version = 1 |
||||
expect = { |
||||
"code": code, |
||||
"name": name, |
||||
"version": 1, |
||||
"description": None, |
||||
"delayTime": 0, |
||||
"taskType": "PYTHON", |
||||
"taskParams": { |
||||
"resourceList": [], |
||||
"localParams": [], |
||||
"rawScript": raw, |
||||
"dependence": {}, |
||||
"conditionResult": {"successNode": [""], "failedNode": [""]}, |
||||
"waitStartTimeout": {}, |
||||
}, |
||||
"flag": "YES", |
||||
"taskPriority": "MEDIUM", |
||||
"workerGroup": "default", |
||||
"failRetryTimes": 0, |
||||
"failRetryInterval": 1, |
||||
"timeoutFlag": "CLOSE", |
||||
"timeoutNotifyStrategy": None, |
||||
"timeout": 0, |
||||
} |
||||
with patch( |
||||
"pydolphinscheduler.core.task.Task.gen_code_and_version", |
||||
return_value=(code, version), |
||||
): |
||||
shell = Python(name, script_code) |
||||
assert shell.to_dict() == expect |
||||
Loading…
Reference in new issue