Browse Source
* [python] refactor yaml file parser * Change yaml parser package to ruamel.yaml * Refactor configuration.py module * And file.py to write file locally * Add more tests on it close: #8593 * Fix UT error * Remove pypyaml from tests * Fix file error when param create is False * Fix error logic * And tests to avoid regression3.0.0/version-upgrade
Jiajie Zhong
2 years ago
committed by
GitHub
15 changed files with 925 additions and 432 deletions
@ -0,0 +1,27 @@
|
||||
<!-- |
||||
Licensed to the Apache Software Foundation (ASF) under one |
||||
or more contributor license agreements. See the NOTICE file |
||||
distributed with this work for additional information |
||||
regarding copyright ownership. The ASF licenses this file |
||||
to you under the Apache License, Version 2.0 (the |
||||
"License"); you may not use this file except in compliance |
||||
with the License. You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
||||
Unless required by applicable law or agreed to in writing, |
||||
software distributed under the License is distributed on an |
||||
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
KIND, either express or implied. See the License for the |
||||
specific language governing permissions and limitations |
||||
under the License. |
||||
--> |
||||
|
||||
# UPDATING |
||||
|
||||
Updating is try to document non-backward compatible updates which notice users the detail changes about pydolphinscheduler. |
||||
It started after version 2.0.5 released |
||||
|
||||
## dev |
||||
|
||||
* Use package ``ruamel.yaml`` replace ``pyyaml`` for write yaml file with comment. |
||||
@ -0,0 +1,57 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""File util for pydolphinscheduler.""" |
||||
|
||||
from pathlib import Path |
||||
from typing import Optional |
||||
|
||||
|
||||
def write( |
||||
content: str, |
||||
to_path: str, |
||||
create: Optional[bool] = True, |
||||
overwrite: Optional[bool] = False, |
||||
) -> None: |
||||
"""Write configs dict to configuration file. |
||||
|
||||
:param content: The source string want to write to :param:`to_path`. |
||||
:param to_path: The path want to write content. |
||||
:param create: Whether create the file parent directory or not if it does not exist. |
||||
If set ``True`` will create file with :param:`to_path` if path not exists, otherwise |
||||
``False`` will not create. Default ``True``. |
||||
:param overwrite: Whether overwrite the file or not if it exists. If set ``True`` |
||||
will overwrite the exists content, otherwise ``False`` will not overwrite it. Default ``True``. |
||||
""" |
||||
path = Path(to_path) |
||||
if not path.parent.exists(): |
||||
if create: |
||||
path.parent.mkdir(parents=True) |
||||
else: |
||||
raise ValueError( |
||||
"Parent directory do not exists and set param `create` to `False`." |
||||
) |
||||
if not path.exists(): |
||||
with path.open(mode="w") as f: |
||||
f.write(content) |
||||
elif overwrite: |
||||
with path.open(mode="w") as f: |
||||
f.write(content) |
||||
else: |
||||
raise FileExistsError( |
||||
"File %s already exists and you choose not overwrite mode.", to_path |
||||
) |
||||
@ -1,85 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Path dict allow users access value by key chain, like `var.key1.key2.key3`.""" |
||||
|
||||
|
||||
# TODO maybe we should rewrite it by `collections.abc.MutableMapping` later, |
||||
# according to https://stackoverflow.com/q/3387691/7152658 |
||||
class PathDict(dict): |
||||
"""Path dict allow users access value by key chain, like `var.key1.key2.key3`.""" |
||||
|
||||
def __init__(self, original=None): |
||||
super().__init__() |
||||
if original is None: |
||||
pass |
||||
elif isinstance(original, dict): |
||||
for key in original: |
||||
self.__setitem__(key, original[key]) |
||||
else: |
||||
raise TypeError( |
||||
"Parameter original expected dict type but get %s", type(original) |
||||
) |
||||
|
||||
def __getitem__(self, key): |
||||
if "." not in key: |
||||
# try: |
||||
return dict.__getitem__(self, key) |
||||
# except KeyError: |
||||
# # cPickle would get error when key without value pairs, in this case we just skip it |
||||
# return |
||||
my_key, rest_of_key = key.split(".", 1) |
||||
target = dict.__getitem__(self, my_key) |
||||
if not isinstance(target, PathDict): |
||||
raise KeyError( |
||||
'Cannot get "%s" to (%s) as sub-key of "%s".' |
||||
% (rest_of_key, repr(target), my_key) |
||||
) |
||||
return target[rest_of_key] |
||||
|
||||
def __setitem__(self, key, value): |
||||
if "." in key: |
||||
my_key, rest_of_key = key.split(".", 1) |
||||
target = self.setdefault(my_key, PathDict()) |
||||
if not isinstance(target, PathDict): |
||||
raise KeyError( |
||||
'Cannot set "%s" from (%s) as sub-key of "%s"' |
||||
% (rest_of_key, repr(target), my_key) |
||||
) |
||||
target[rest_of_key] = value |
||||
else: |
||||
if isinstance(value, dict) and not isinstance(value, PathDict): |
||||
value = PathDict(value) |
||||
dict.__setitem__(self, key, value) |
||||
|
||||
def __contains__(self, key): |
||||
if "." not in key: |
||||
return dict.__contains__(self, key) |
||||
my_key, rest_of_key = key.split(".", 1) |
||||
target = dict.__getitem__(self, my_key) |
||||
if not isinstance(target, PathDict): |
||||
return False |
||||
return rest_of_key in target |
||||
|
||||
def setdefault(self, key, default): |
||||
"""Overwrite method dict.setdefault.""" |
||||
if key not in self: |
||||
self[key] = default |
||||
return self[key] |
||||
|
||||
__setattr__ = __setitem__ |
||||
__getattr__ = __getitem__ |
||||
@ -0,0 +1,169 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""YAML parser utils, parser yaml string to ``ruamel.yaml`` object and nested key dict.""" |
||||
|
||||
import copy |
||||
import io |
||||
from typing import Any, Dict, Optional |
||||
|
||||
from ruamel.yaml import YAML |
||||
from ruamel.yaml.comments import CommentedMap |
||||
|
||||
|
||||
class YamlParser: |
||||
"""A parser to parse Yaml file and provider easier way to access or change value. |
||||
|
||||
This parser provider delimiter string key to get or set :class:`ruamel.yaml.YAML` object |
||||
|
||||
For example, yaml config named ``test.yaml`` and its content as below: |
||||
|
||||
.. code-block:: yaml |
||||
|
||||
one: |
||||
two1: |
||||
three: value1 |
||||
two2: value2 |
||||
|
||||
you could get ``value1`` and ``value2`` by nested path |
||||
|
||||
.. code-block:: python |
||||
|
||||
yaml_parser = YamlParser("test.yaml") |
||||
|
||||
# Use function ``get`` to get value |
||||
value1 = yaml_parser.get("one.two1.three") |
||||
# Or use build-in ``__getitem__`` to get value |
||||
value2 = yaml_parser["one.two2"] |
||||
|
||||
or you could change ``value1`` to ``value3``, also change ``value2`` to ``value4`` by nested path assigned |
||||
|
||||
.. code-block:: python |
||||
|
||||
yaml_parser["one.two1.three"] = "value3" |
||||
yaml_parser["one.two2"] = "value4" |
||||
""" |
||||
|
||||
def __init__(self, content: str, delimiter: Optional[str] = "."): |
||||
self.src_parser = content |
||||
self._delimiter = delimiter |
||||
|
||||
@property |
||||
def src_parser(self) -> CommentedMap: |
||||
"""Get src_parser property.""" |
||||
return self._src_parser |
||||
|
||||
@src_parser.setter |
||||
def src_parser(self, content: str) -> None: |
||||
"""Set src_parser property.""" |
||||
self._yaml = YAML() |
||||
self._src_parser = self._yaml.load(content) |
||||
|
||||
def parse_nested_dict( |
||||
self, result: Dict, commented_map: CommentedMap, key: str |
||||
) -> None: |
||||
"""Parse :class:`ruamel.yaml.comments.CommentedMap` to nested dict using :param:`delimiter`.""" |
||||
if not isinstance(commented_map, CommentedMap): |
||||
return |
||||
for sub_key in set(commented_map.keys()): |
||||
next_key = f"{key}{self._delimiter}{sub_key}" |
||||
result[next_key] = commented_map[sub_key] |
||||
self.parse_nested_dict(result, commented_map[sub_key], next_key) |
||||
|
||||
@property |
||||
def dict_parser(self) -> Dict: |
||||
"""Get :class:`CommentedMap` to nested dict using :param:`delimiter` as key delimiter. |
||||
|
||||
Use Depth-First-Search get all nested key and value, and all key connect by :param:`delimiter`. |
||||
It make users could easier access or change :class:`CommentedMap` object. |
||||
|
||||
For example, yaml config named ``test.yaml`` and its content as below: |
||||
|
||||
.. code-block:: yaml |
||||
|
||||
one: |
||||
two1: |
||||
three: value1 |
||||
two2: value2 |
||||
|
||||
It could parser to nested dict as |
||||
|
||||
.. code-block:: python |
||||
|
||||
{ |
||||
"one": ordereddict([('two1', ordereddict([('three', 'value1')])), ('two2', 'value2')]), |
||||
"one.two1": ordereddict([('three', 'value1')]), |
||||
"one.two1.three": "value1", |
||||
"one.two2": "value2", |
||||
} |
||||
""" |
||||
res = dict() |
||||
src_parser_copy = copy.deepcopy(self.src_parser) |
||||
|
||||
base_keys = set(src_parser_copy.keys()) |
||||
if not base_keys: |
||||
return res |
||||
else: |
||||
for key in base_keys: |
||||
res[key] = src_parser_copy[key] |
||||
self.parse_nested_dict(res, src_parser_copy[key], key) |
||||
return res |
||||
|
||||
def __contains__(self, key) -> bool: |
||||
return key in self.dict_parser |
||||
|
||||
def __getitem__(self, key: str) -> Any: |
||||
return self.dict_parser[key] |
||||
|
||||
def __setitem__(self, key: str, val: Any) -> None: |
||||
if key not in self.dict_parser: |
||||
raise KeyError("Key %s do not exists.", key) |
||||
|
||||
mid = None |
||||
keys = key.split(self._delimiter) |
||||
for idx, k in enumerate(keys, 1): |
||||
if idx == len(keys): |
||||
mid[k] = val |
||||
else: |
||||
mid = mid[k] if mid else self.src_parser[k] |
||||
|
||||
def get(self, key: str) -> Any: |
||||
"""Get value by key, is call ``__getitem__``.""" |
||||
return self[key] |
||||
|
||||
def get_int(self, key: str) -> int: |
||||
"""Get value and covert it to int.""" |
||||
return int(self.get(key)) |
||||
|
||||
def get_bool(self, key: str) -> bool: |
||||
"""Get value and covert it to boolean.""" |
||||
val = self.get(key) |
||||
if isinstance(val, str): |
||||
return val.lower() in {"true", "t"} |
||||
elif isinstance(val, int): |
||||
return val != 0 |
||||
else: |
||||
return val |
||||
|
||||
def to_string(self) -> str: |
||||
"""Transfer :class:`YamlParser` to string object. |
||||
|
||||
It is useful when users want to output the :class:`YamlParser` object they change just now. |
||||
""" |
||||
buf = io.StringIO() |
||||
self._yaml.dump(self.src_parser, buf) |
||||
return buf.getvalue() |
||||
@ -0,0 +1,34 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Testing util about file operating.""" |
||||
|
||||
from pathlib import Path |
||||
from typing import Union |
||||
|
||||
|
||||
def get_file_content(path: Union[str, Path]) -> str: |
||||
"""Get file content in given path.""" |
||||
with open(path, mode="r") as f: |
||||
return f.read() |
||||
|
||||
|
||||
def delete_file(path: Union[str, Path]) -> None: |
||||
"""Delete file in given path.""" |
||||
path = Path(path).expanduser() if isinstance(path, str) else path.expanduser() |
||||
if path.exists(): |
||||
path.unlink() |
||||
@ -0,0 +1,85 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Test file utils.""" |
||||
|
||||
import shutil |
||||
from pathlib import Path |
||||
|
||||
import pytest |
||||
|
||||
from pydolphinscheduler.utils import file |
||||
from tests.testing.file import delete_file, get_file_content |
||||
|
||||
content = "test_content" |
||||
file_path = "/tmp/test/file/test_file_write.txt" |
||||
|
||||
|
||||
@pytest.fixture |
||||
def teardown_del_file(): |
||||
"""Teardown about delete file.""" |
||||
yield |
||||
delete_file(file_path) |
||||
|
||||
|
||||
@pytest.fixture |
||||
def setup_crt_first(): |
||||
"""Set up and teardown about create file first and then delete it.""" |
||||
file.write(content=content, to_path=file_path) |
||||
yield |
||||
delete_file(file_path) |
||||
|
||||
|
||||
def test_write_content(teardown_del_file): |
||||
"""Test function :func:`write` on write behavior with correct content.""" |
||||
assert not Path(file_path).exists() |
||||
file.write(content=content, to_path=file_path) |
||||
assert Path(file_path).exists() |
||||
assert content == get_file_content(file_path) |
||||
|
||||
|
||||
def test_write_not_create_parent(teardown_del_file): |
||||
"""Test function :func:`write` with parent not exists and do not create path.""" |
||||
file_test_dir = Path(file_path).parent |
||||
if file_test_dir.exists(): |
||||
shutil.rmtree(str(file_test_dir)) |
||||
assert not file_test_dir.exists() |
||||
with pytest.raises( |
||||
ValueError, |
||||
match="Parent directory do not exists and set param `create` to `False`", |
||||
): |
||||
file.write(content=content, to_path=file_path, create=False) |
||||
|
||||
|
||||
def test_write_overwrite(setup_crt_first): |
||||
"""Test success with file exists but set ``True`` to overwrite.""" |
||||
assert Path(file_path).exists() |
||||
|
||||
new_content = f"new_{content}" |
||||
file.write(content=new_content, to_path=file_path, overwrite=True) |
||||
assert new_content == get_file_content(file_path) |
||||
|
||||
|
||||
def test_write_overwrite_error(setup_crt_first): |
||||
"""Test error with file exists but set ``False`` to overwrite.""" |
||||
assert Path(file_path).exists() |
||||
|
||||
new_content = f"new_{content}" |
||||
with pytest.raises( |
||||
FileExistsError, match=".*already exists and you choose not overwrite mode\\." |
||||
): |
||||
file.write(content=new_content, to_path=file_path) |
||||
@ -1,201 +0,0 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Test utils.path_dict module.""" |
||||
|
||||
import copy |
||||
from typing import Dict, Tuple |
||||
|
||||
import pytest |
||||
|
||||
from pydolphinscheduler.utils.path_dict import PathDict |
||||
|
||||
src_dict_list = [ |
||||
# dict with one single level |
||||
{"a": 1}, |
||||
# dict with two levels, with same nested keys 'b' |
||||
{"a": 1, "b": 2, "c": {"d": 3}, "e": {"b": 4}}, |
||||
# dict with three levels, with same nested keys 'b' |
||||
{"a": 1, "b": 2, "c": {"d": 3}, "e": {"b": {"b": 4}, "f": 5}}, |
||||
# dict with specific key container |
||||
{ |
||||
"a": 1, |
||||
"a-b": 2, |
||||
}, |
||||
] |
||||
|
||||
|
||||
@pytest.mark.parametrize("org", src_dict_list) |
||||
def test_val_between_dict_and_path_dict(org: Dict): |
||||
"""Test path dict equal to original dict.""" |
||||
path_dict = PathDict(org) |
||||
assert org == dict(path_dict) |
||||
|
||||
|
||||
def test_path_dict_basic_attr_access(): |
||||
"""Test basic behavior of path dict. |
||||
|
||||
Including add by attribute, with simple, nested dict, and specific key dict. |
||||
""" |
||||
expect = copy.deepcopy(src_dict_list[2]) |
||||
path_dict = PathDict(expect) |
||||
|
||||
# Add node with one level |
||||
val = 3 |
||||
path_dict.f = val |
||||
expect.update({"f": val}) |
||||
assert expect == path_dict |
||||
|
||||
# Add node with multiple level |
||||
val = {"abc": 123} |
||||
path_dict.e.g = val |
||||
expect.update({"e": {"b": {"b": 4}, "f": 5, "g": val}}) |
||||
assert expect == path_dict |
||||
|
||||
# Specific key |
||||
expect = copy.deepcopy(src_dict_list[3]) |
||||
path_dict = PathDict(expect) |
||||
assert 1 == path_dict.a |
||||
assert 2 == getattr(path_dict, "a-b") |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"org, exists, not_exists", |
||||
[ |
||||
( |
||||
src_dict_list[0], |
||||
("a"), |
||||
("b", "a.b"), |
||||
), |
||||
( |
||||
src_dict_list[1], |
||||
("a", "b", "c", "e", "c.d", "e.b"), |
||||
("a.b", "c.e", "b.c", "b.e"), |
||||
), |
||||
( |
||||
src_dict_list[2], |
||||
("a", "b", "c", "e", "c.d", "e.b", "e.b.b", "e.b.b", "e.f"), |
||||
("a.b", "c.e", "b.c", "b.e", "b.b.f", "b.f"), |
||||
), |
||||
], |
||||
) |
||||
def test_path_dict_attr(org: Dict, exists: Tuple, not_exists: Tuple): |
||||
"""Test properties' integrity of path dict.""" |
||||
path_dict = PathDict(org) |
||||
assert all([hasattr(path_dict, path) for path in exists]) |
||||
# assert not any([hasattr(path_dict, path) for path in not_exists]) |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"org, path_get", |
||||
[ |
||||
( |
||||
src_dict_list[0], |
||||
{"a": 1}, |
||||
), |
||||
( |
||||
src_dict_list[1], |
||||
{ |
||||
"a": 1, |
||||
"b": 2, |
||||
"c": {"d": 3}, |
||||
"c.d": 3, |
||||
"e": {"b": 4}, |
||||
"e.b": 4, |
||||
}, |
||||
), |
||||
( |
||||
src_dict_list[2], |
||||
{ |
||||
"a": 1, |
||||
"b": 2, |
||||
"c": {"d": 3}, |
||||
"c.d": 3, |
||||
"e": {"b": {"b": 4}, "f": 5}, |
||||
"e.b": {"b": 4}, |
||||
"e.b.b": 4, |
||||
"e.f": 5, |
||||
}, |
||||
), |
||||
], |
||||
) |
||||
def test_path_dict_get(org: Dict, path_get: Dict): |
||||
"""Test path dict getter function.""" |
||||
path_dict = PathDict(org) |
||||
assert all([path_get[path] == path_dict.__getattr__(path) for path in path_get]) |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"org, path_set, expect", |
||||
[ |
||||
# Add not exists node |
||||
( |
||||
src_dict_list[0], |
||||
{"b": 2}, |
||||
{ |
||||
"a": 1, |
||||
"b": 2, |
||||
}, |
||||
), |
||||
# Overwrite exists node with different type of value |
||||
( |
||||
src_dict_list[0], |
||||
{"a": "b"}, |
||||
{"a": "b"}, |
||||
), |
||||
# Add multiple not exists node with variable types of value |
||||
( |
||||
src_dict_list[0], |
||||
{ |
||||
"b.c.d": 123, |
||||
"b.c.e": "a", |
||||
"b.f": {"g": 23, "h": "bc", "i": {"j": "k"}}, |
||||
}, |
||||
{ |
||||
"a": 1, |
||||
"b": { |
||||
"c": { |
||||
"d": 123, |
||||
"e": "a", |
||||
}, |
||||
"f": {"g": 23, "h": "bc", "i": {"j": "k"}}, |
||||
}, |
||||
}, |
||||
), |
||||
# Test complex original data |
||||
( |
||||
src_dict_list[2], |
||||
{ |
||||
"g": 12, |
||||
"c.h": 34, |
||||
}, |
||||
{ |
||||
"a": 1, |
||||
"b": 2, |
||||
"g": 12, |
||||
"c": {"d": 3, "h": 34}, |
||||
"e": {"b": {"b": 4}, "f": 5}, |
||||
}, |
||||
), |
||||
], |
||||
) |
||||
def test_path_dict_set(org: Dict, path_set: Dict, expect: Dict): |
||||
"""Test path dict setter function.""" |
||||
path_dict = PathDict(org) |
||||
for path in path_set: |
||||
path_dict.__setattr__(path, path_set[path]) |
||||
assert expect == path_dict |
||||
@ -0,0 +1,272 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Test utils.path_dict module.""" |
||||
|
||||
from typing import Dict |
||||
|
||||
import pytest |
||||
from ruamel.yaml import YAML |
||||
|
||||
from pydolphinscheduler.utils.yaml_parser import YamlParser |
||||
from tests.testing.path import path_default_config_yaml |
||||
|
||||
yaml = YAML() |
||||
|
||||
expects = [ |
||||
{ |
||||
# yaml.load("no need test") is a flag about skipping it because it to different to maintainer |
||||
"name": yaml.load("no need test"), |
||||
"name.family": ("Smith", "SmithEdit"), |
||||
"name.given": ("Alice", "AliceEdit"), |
||||
"name.mark": yaml.load("no need test"), |
||||
"name.mark.name_mark": yaml.load("no need test"), |
||||
"name.mark.name_mark.key": ("value", "valueEdit"), |
||||
}, |
||||
{ |
||||
# yaml.load("no need test") is a flag about skipping it because it to different to maintainer |
||||
"java_gateway": yaml.load("no need test"), |
||||
"java_gateway.address": ("127.0.0.1", "127.1.1.1"), |
||||
"java_gateway.port": (25333, 25555), |
||||
"java_gateway.auto_convert": (True, False), |
||||
"default": yaml.load("no need test"), |
||||
"default.user": yaml.load("no need test"), |
||||
"default.user.name": ("userPythonGateway", "userPythonGatewayEdit"), |
||||
"default.user.password": ("userPythonGateway", "userPythonGatewayEdit"), |
||||
"default.user.email": ( |
||||
"userPythonGateway@dolphinscheduler.com", |
||||
"userEdit@dolphinscheduler.com", |
||||
), |
||||
"default.user.tenant": ("tenant_pydolphin", "tenant_pydolphinEdit"), |
||||
"default.user.phone": (11111111111, 22222222222), |
||||
"default.user.state": (1, 0), |
||||
"default.workflow": yaml.load("no need test"), |
||||
"default.workflow.project": ("project-pydolphin", "project-pydolphinEdit"), |
||||
"default.workflow.tenant": ("tenant_pydolphin", "SmithEdit"), |
||||
"default.workflow.user": ("userPythonGateway", "SmithEdit"), |
||||
"default.workflow.queue": ("queuePythonGateway", "SmithEdit"), |
||||
"default.workflow.worker_group": ("default", "SmithEdit"), |
||||
"default.workflow.time_zone": ("Asia/Shanghai", "SmithEdit"), |
||||
}, |
||||
] |
||||
|
||||
param = [ |
||||
"""#example |
||||
name: |
||||
# details |
||||
family: Smith # very common |
||||
given: Alice # one of the siblings |
||||
mark: |
||||
name_mark: |
||||
key: value |
||||
""" |
||||
] |
||||
|
||||
with open(path_default_config_yaml, "r") as f: |
||||
param.append(f.read()) |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"src, delimiter, expect", |
||||
[ |
||||
( |
||||
param[0], |
||||
"|", |
||||
expects[0], |
||||
), |
||||
( |
||||
param[1], |
||||
"/", |
||||
expects[1], |
||||
), |
||||
], |
||||
) |
||||
def test_yaml_parser_specific_delimiter(src: str, delimiter: str, expect: Dict): |
||||
"""Test specific delimiter for :class:`YamlParser`.""" |
||||
|
||||
def ch_dl(key): |
||||
return key.replace(".", delimiter) |
||||
|
||||
yaml_parser = YamlParser(src, delimiter=delimiter) |
||||
assert all( |
||||
[ |
||||
expect[key][0] == yaml_parser[ch_dl(key)] |
||||
for key in expect |
||||
if expect[key] != "no need test" |
||||
] |
||||
) |
||||
assert all( |
||||
[ |
||||
expect[key][0] == yaml_parser.get(ch_dl(key)) |
||||
for key in expect |
||||
if expect[key] != "no need test" |
||||
] |
||||
) |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"src, expect", |
||||
[ |
||||
( |
||||
param[0], |
||||
expects[0], |
||||
), |
||||
( |
||||
param[1], |
||||
expects[1], |
||||
), |
||||
], |
||||
) |
||||
def test_yaml_parser_contains(src: str, expect: Dict): |
||||
"""Test magic function :func:`YamlParser.__contain__` also with `key in obj` syntax.""" |
||||
yaml_parser = YamlParser(src) |
||||
assert len(expect.keys()) == len( |
||||
yaml_parser.dict_parser.keys() |
||||
), "Parser keys length not equal to expect keys length" |
||||
assert all( |
||||
[key in yaml_parser for key in expect] |
||||
), "Parser keys not equal to expect keys" |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"src, expect", |
||||
[ |
||||
( |
||||
param[0], |
||||
expects[0], |
||||
), |
||||
( |
||||
param[1], |
||||
expects[1], |
||||
), |
||||
], |
||||
) |
||||
def test_yaml_parser_get(src: str, expect: Dict): |
||||
"""Test magic function :func:`YamlParser.__getitem__` also with `obj[key]` syntax.""" |
||||
yaml_parser = YamlParser(src) |
||||
assert all( |
||||
[ |
||||
expect[key][0] == yaml_parser[key] |
||||
for key in expect |
||||
if expect[key] != "no need test" |
||||
] |
||||
) |
||||
assert all( |
||||
[ |
||||
expect[key][0] == yaml_parser.get(key) |
||||
for key in expect |
||||
if expect[key] != "no need test" |
||||
] |
||||
) |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"src, expect", |
||||
[ |
||||
( |
||||
param[0], |
||||
expects[0], |
||||
), |
||||
( |
||||
param[1], |
||||
expects[1], |
||||
), |
||||
], |
||||
) |
||||
def test_yaml_parser_set(src: str, expect: Dict): |
||||
"""Test magic function :func:`YamlParser.__setitem__` also with `obj[key] = val` syntax.""" |
||||
yaml_parser = YamlParser(src) |
||||
for key in expect: |
||||
assert key in yaml_parser.dict_parser.keys() |
||||
if expect[key] == "no need test": |
||||
continue |
||||
assert expect[key][0] == yaml_parser.dict_parser[key] |
||||
assert expect[key][1] != yaml_parser.dict_parser[key] |
||||
|
||||
yaml_parser[key] = expect[key][1] |
||||
assert expect[key][0] != yaml_parser.dict_parser[key] |
||||
assert expect[key][1] == yaml_parser.dict_parser[key] |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"src, setter, expect", |
||||
[ |
||||
( |
||||
param[0], |
||||
{"name.mark.name_mark.key": "edit"}, |
||||
"""#example |
||||
name: |
||||
# details |
||||
family: Smith # very common |
||||
given: Alice # one of the siblings |
||||
mark: |
||||
name_mark: |
||||
key: edit |
||||
""", |
||||
), |
||||
( |
||||
param[0], |
||||
{ |
||||
"name.family": "SmithEdit", |
||||
"name.given": "AliceEdit", |
||||
"name.mark.name_mark.key": "edit", |
||||
}, |
||||
"""#example |
||||
name: |
||||
# details |
||||
family: SmithEdit # very common |
||||
given: AliceEdit # one of the siblings |
||||
mark: |
||||
name_mark: |
||||
key: edit |
||||
""", |
||||
), |
||||
], |
||||
) |
||||
def test_yaml_parser_to_string(src: str, setter: Dict, expect: str): |
||||
"""Test function :func:`YamlParser.to_string`.""" |
||||
yaml_parser = YamlParser(src) |
||||
for key, val in setter.items(): |
||||
yaml_parser[key] = val |
||||
|
||||
assert expect == yaml_parser.to_string() |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"src, key, expect", |
||||
[ |
||||
(param[1], "java_gateway.port", 25333), |
||||
(param[1], "default.user.phone", 11111111111), |
||||
(param[1], "default.user.state", 1), |
||||
], |
||||
) |
||||
def test_yaml_parser_get_int(src: str, key: str, expect: int): |
||||
"""Test function :func:`YamlParser.get_int`.""" |
||||
yaml_parser = YamlParser(src) |
||||
assert expect == yaml_parser.get_int(key) |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"src, key, expect", |
||||
[ |
||||
(param[1], "java_gateway.auto_convert", True), |
||||
], |
||||
) |
||||
def test_yaml_parser_get_bool(src: str, key: str, expect: bool): |
||||
"""Test function :func:`YamlParser.get_bool`.""" |
||||
yaml_parser = YamlParser(src) |
||||
assert expect == yaml_parser.get_bool(key) |
||||
Loading…
Reference in new issue