Browse Source
* [python] Add config mechanism and cli subcommand config * Add configuration.py mechanism for more easy change config and move some configs to it. It mechanism including configuration.py module and default_config.yaml file * Add `config` for cli subcommand allow users initialize, get, set configs close: #8344 * Change setup.py format3.0.0/version-upgrade
Jiajie Zhong
3 years ago
committed by
GitHub
24 changed files with 889 additions and 91 deletions
@ -0,0 +1,152 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Configuration module for pydolphinscheduler.""" |
||||
|
||||
import copy |
||||
import os |
||||
from pathlib import Path |
||||
from typing import Any, Dict |
||||
|
||||
import yaml |
||||
|
||||
from pydolphinscheduler.exceptions import PyDSConfException, PyDSParamException |
||||
from pydolphinscheduler.utils.path_dict import PathDict |
||||
|
||||
DEFAULT_CONFIG_PATH = Path(__file__).resolve().parent.joinpath("default_config.yaml") |
||||
|
||||
|
||||
def get_config_file_path() -> Path: |
||||
"""Get the path of pydolphinscheduler configuration file.""" |
||||
pyds_home = os.environ.get("PYDOLPHINSCHEDULER_HOME", "~/pydolphinscheduler") |
||||
config_file_path = Path(pyds_home).joinpath("config.yaml").expanduser() |
||||
return config_file_path |
||||
|
||||
|
||||
def read_yaml(path: str) -> Dict: |
||||
"""Read configs dict from configuration file. |
||||
|
||||
:param path: The path of configuration file. |
||||
""" |
||||
with open(path, "r") as f: |
||||
return yaml.safe_load(f) |
||||
|
||||
|
||||
def write_yaml(context: Dict, path: str) -> None: |
||||
"""Write configs dict to configuration file. |
||||
|
||||
:param context: The configs dict write to configuration file. |
||||
:param path: The path of configuration file. |
||||
""" |
||||
parent = Path(path).parent |
||||
if not parent.exists(): |
||||
parent.mkdir(parents=True) |
||||
with open(path, mode="w") as f: |
||||
f.write(yaml.dump(context)) |
||||
|
||||
|
||||
def default_yaml_config() -> Dict: |
||||
"""Get default configs in ``DEFAULT_CONFIG_PATH``.""" |
||||
with open(DEFAULT_CONFIG_PATH, "r") as f: |
||||
return yaml.safe_load(f) |
||||
|
||||
|
||||
def _whether_exists_config() -> bool: |
||||
"""Check whether config file already exists in :func:`get_config_file_path`.""" |
||||
return True if get_config_file_path().exists() else False |
||||
|
||||
|
||||
def get_all_configs(): |
||||
"""Get all configs from configuration file.""" |
||||
exists = _whether_exists_config() |
||||
if exists: |
||||
return read_yaml(str(get_config_file_path())) |
||||
else: |
||||
return default_yaml_config() |
||||
|
||||
|
||||
# Add configs as module variables to avoid read configuration multiple times when |
||||
# Get common configuration setting |
||||
# Set or get multiple configs in single time |
||||
configs = get_all_configs() |
||||
|
||||
|
||||
def init_config_file() -> None: |
||||
"""Initialize configuration file to :func:`get_config_file_path`.""" |
||||
if _whether_exists_config(): |
||||
raise PyDSConfException( |
||||
"Initialize configuration false to avoid overwrite configure by accident, file already exists " |
||||
"in %s, if you wan to overwrite the exists configure please remove the exists file manually.", |
||||
str(get_config_file_path()), |
||||
) |
||||
write_yaml(context=default_yaml_config(), path=str(get_config_file_path())) |
||||
|
||||
|
||||
def get_single_config(key: str) -> Any: |
||||
"""Get single config to configuration file. |
||||
|
||||
:param key: The config path want get. |
||||
""" |
||||
global configs |
||||
config_path_dict = PathDict(configs) |
||||
if key not in config_path_dict: |
||||
raise PyDSParamException( |
||||
"Configuration path %s do not exists. Can not get configuration.", key |
||||
) |
||||
return config_path_dict.__getattr__(key) |
||||
|
||||
|
||||
def set_single_config(key: str, value: Any) -> None: |
||||
"""Change single config to configuration file. |
||||
|
||||
:param key: The config path want change. |
||||
:param value: The new value want to set. |
||||
""" |
||||
global configs |
||||
config_path_dict = PathDict(configs) |
||||
if key not in config_path_dict: |
||||
raise PyDSParamException( |
||||
"Configuration path %s do not exists. Can not set configuration.", key |
||||
) |
||||
config_path_dict.__setattr__(key, value) |
||||
write_yaml(context=dict(config_path_dict), path=str(get_config_file_path())) |
||||
|
||||
|
||||
# Start Common Configuration Settings |
||||
path_configs = PathDict(copy.deepcopy(configs)) |
||||
|
||||
# Java Gateway Settings |
||||
JAVA_GATEWAY_ADDRESS = str(getattr(path_configs, "java_gateway.address")) |
||||
JAVA_GATEWAY_PORT = str(getattr(path_configs, "java_gateway.port")) |
||||
JAVA_GATEWAY_AUTO_CONVERT = str(getattr(path_configs, "java_gateway.auto_convert")) |
||||
|
||||
# User Settings |
||||
USER_NAME = str(getattr(path_configs, "default.user.name")) |
||||
USER_PASSWORD = str(getattr(path_configs, "default.user.password")) |
||||
USER_EMAIL = str(getattr(path_configs, "default.user.email")) |
||||
USER_PHONE = str(getattr(path_configs, "default.user.phone")) |
||||
USER_STATE = str(getattr(path_configs, "default.user.state")) |
||||
|
||||
# Workflow Settings |
||||
WORKFLOW_PROJECT = str(getattr(path_configs, "default.workflow.project")) |
||||
WORKFLOW_TENANT = str(getattr(path_configs, "default.workflow.tenant")) |
||||
WORKFLOW_USER = str(getattr(path_configs, "default.workflow.user")) |
||||
WORKFLOW_QUEUE = str(getattr(path_configs, "default.workflow.queue")) |
||||
WORKFLOW_WORKER_GROUP = str(getattr(path_configs, "default.workflow.worker_group")) |
||||
WORKFLOW_TIME_ZONE = str(getattr(path_configs, "default.workflow.time_zone")) |
||||
|
||||
# End Common Configuration Setting |
||||
@ -0,0 +1,43 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
java_gateway: |
||||
# The address of Python gateway server start. Set its value to `0.0.0.0` if your Python API run in different |
||||
# between Python gateway server. It could be be specific to other address like `127.0.0.1` or `localhost` |
||||
address: 127.0.0.1 |
||||
# The port of Python gateway server start. Define which port you could connect to Python gateway server from |
||||
# Python API side. |
||||
port: 25333 |
||||
|
||||
auto_convert: true |
||||
|
||||
default: |
||||
user: |
||||
name: userPythonGateway |
||||
# TODO simple set password same as username |
||||
password: userPythonGateway |
||||
email: userPythonGateway@dolphinscheduler.com |
||||
tenant: tenant_pydolphin |
||||
phone: 11111111111 |
||||
state: 1 |
||||
workflow: |
||||
project: project-pydolphin |
||||
tenant: tenant_pydolphin |
||||
user: userPythonGateway |
||||
queue: queuePythonGateway |
||||
worker_group: default |
||||
time_zone: Asia/Shanghai |
||||
@ -0,0 +1,85 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Path dict allow users access value by key chain, like `var.key1.key2.key3`.""" |
||||
|
||||
|
||||
# TODO maybe we should rewrite it by `collections.abc.MutableMapping` later, |
||||
# according to https://stackoverflow.com/q/3387691/7152658 |
||||
class PathDict(dict): |
||||
"""Path dict allow users access value by key chain, like `var.key1.key2.key3`.""" |
||||
|
||||
def __init__(self, original=None): |
||||
super().__init__() |
||||
if original is None: |
||||
pass |
||||
elif isinstance(original, dict): |
||||
for key in original: |
||||
self.__setitem__(key, original[key]) |
||||
else: |
||||
raise TypeError( |
||||
"Parameter original expected dict type but get %s", type(original) |
||||
) |
||||
|
||||
def __getitem__(self, key): |
||||
if "." not in key: |
||||
# try: |
||||
return dict.__getitem__(self, key) |
||||
# except KeyError: |
||||
# # cPickle would get error when key without value pairs, in this case we just skip it |
||||
# return |
||||
my_key, rest_of_key = key.split(".", 1) |
||||
target = dict.__getitem__(self, my_key) |
||||
if not isinstance(target, PathDict): |
||||
raise KeyError( |
||||
'Cannot get "%s" to (%s) as sub-key of "%s".' |
||||
% (rest_of_key, repr(target), my_key) |
||||
) |
||||
return target[rest_of_key] |
||||
|
||||
def __setitem__(self, key, value): |
||||
if "." in key: |
||||
my_key, rest_of_key = key.split(".", 1) |
||||
target = self.setdefault(my_key, PathDict()) |
||||
if not isinstance(target, PathDict): |
||||
raise KeyError( |
||||
'Cannot set "%s" from (%s) as sub-key of "%s"' |
||||
% (rest_of_key, repr(target), my_key) |
||||
) |
||||
target[rest_of_key] = value |
||||
else: |
||||
if isinstance(value, dict) and not isinstance(value, PathDict): |
||||
value = PathDict(value) |
||||
dict.__setitem__(self, key, value) |
||||
|
||||
def __contains__(self, key): |
||||
if "." not in key: |
||||
return dict.__contains__(self, key) |
||||
my_key, rest_of_key = key.split(".", 1) |
||||
target = dict.__getitem__(self, my_key) |
||||
if not isinstance(target, PathDict): |
||||
return False |
||||
return rest_of_key in target |
||||
|
||||
def setdefault(self, key, default): |
||||
"""Overwrite method dict.setdefault.""" |
||||
if key not in self: |
||||
self[key] = default |
||||
return self[key] |
||||
|
||||
__setattr__ = __setitem__ |
||||
__getattr__ = __getitem__ |
||||
@ -0,0 +1,201 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Test command line interface subcommand `config`.""" |
||||
|
||||
import os |
||||
from pathlib import Path |
||||
|
||||
import pytest |
||||
|
||||
from pydolphinscheduler.cli.commands import cli |
||||
from pydolphinscheduler.core.configuration import get_config_file_path |
||||
from tests.testing.cli import CliTestWrapper |
||||
from tests.testing.constants import DEV_MODE |
||||
|
||||
default_config_path = "~/pydolphinscheduler" |
||||
config_file = "config.yaml" |
||||
|
||||
|
||||
@pytest.fixture |
||||
def delete_tmp_config_file(): |
||||
"""Util for deleting temp configuration file after test finish.""" |
||||
yield |
||||
config_file_path = get_config_file_path() |
||||
config_file_path.unlink() |
||||
|
||||
|
||||
@pytest.mark.skipif( |
||||
DEV_MODE, |
||||
reason="Avoid delete ~/pydolphinscheduler/config.yaml by accident when test locally.", |
||||
) |
||||
@pytest.mark.parametrize( |
||||
"home", |
||||
[ |
||||
None, |
||||
"/tmp/pydolphinscheduler", |
||||
"/tmp/test_abc", |
||||
], |
||||
) |
||||
def test_config_init(delete_tmp_config_file, home): |
||||
"""Test command line interface `config --init`.""" |
||||
if home: |
||||
os.environ["PYDOLPHINSCHEDULER_HOME"] = home |
||||
config_path = home |
||||
else: |
||||
config_path = default_config_path |
||||
|
||||
path = Path(config_path).joinpath(config_file).expanduser() |
||||
assert not path.exists() |
||||
|
||||
cli_test = CliTestWrapper(cli, ["config", "--init"]) |
||||
cli_test.assert_success() |
||||
|
||||
assert path.exists() |
||||
# TODO We have a bug here, yaml dump do not support comment |
||||
# with path.open(mode="r") as cli_crt, open(path_default_config_yaml, "r") as src: |
||||
# assert src.read() == cli_crt.read() |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"key, expect", |
||||
[ |
||||
# We test each key in one single section |
||||
("java_gateway.address", "127.0.0.1"), |
||||
("default.user.name", "userPythonGateway"), |
||||
("default.workflow.project", "project-pydolphin"), |
||||
], |
||||
) |
||||
def test_config_get(delete_tmp_config_file, key: str, expect: str): |
||||
"""Test command line interface `config --get XXX`.""" |
||||
os.environ["PYDOLPHINSCHEDULER_HOME"] = "/tmp/pydolphinscheduler" |
||||
cli_test = CliTestWrapper(cli, ["config", "--init"]) |
||||
cli_test.assert_success() |
||||
|
||||
cli_test = CliTestWrapper(cli, ["config", "--get", key]) |
||||
cli_test.assert_success(output=f"{key} = {expect}", fuzzy=True) |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"keys, expects", |
||||
[ |
||||
# We test mix section keys |
||||
(("java_gateway.address", "java_gateway.port"), ("127.0.0.1", "25333")), |
||||
( |
||||
("java_gateway.auto_convert", "default.user.tenant"), |
||||
("True", "tenant_pydolphin"), |
||||
), |
||||
( |
||||
( |
||||
"java_gateway.port", |
||||
"default.user.state", |
||||
"default.workflow.worker_group", |
||||
), |
||||
("25333", "1", "default"), |
||||
), |
||||
], |
||||
) |
||||
def test_config_get_multiple(delete_tmp_config_file, keys: str, expects: str): |
||||
"""Test command line interface `config --get KEY1 --get KEY2 ...`.""" |
||||
os.environ["PYDOLPHINSCHEDULER_HOME"] = "/tmp/pydolphinscheduler" |
||||
cli_test = CliTestWrapper(cli, ["config", "--init"]) |
||||
cli_test.assert_success() |
||||
|
||||
get_args = ["config"] |
||||
for key in keys: |
||||
get_args.append("--get") |
||||
get_args.append(key) |
||||
cli_test = CliTestWrapper(cli, get_args) |
||||
|
||||
for idx, expect in enumerate(expects): |
||||
cli_test.assert_success(output=f"{keys[idx]} = {expect}", fuzzy=True) |
||||
|
||||
|
||||
# TODO fix command `config --set KEY VAL` |
||||
@pytest.mark.skip(reason="We still have bug in command `config --set KEY VAL`") |
||||
@pytest.mark.parametrize( |
||||
"key, value", |
||||
[ |
||||
# We test each key in one single section |
||||
("java_gateway.address", "127.1.1.1"), |
||||
("default.user.name", "editUserPythonGateway"), |
||||
("default.workflow.project", "edit-project-pydolphin"), |
||||
], |
||||
) |
||||
def test_config_set(delete_tmp_config_file, key: str, value: str): |
||||
"""Test command line interface `config --set KEY VALUE`.""" |
||||
os.environ["PYDOLPHINSCHEDULER_HOME"] = "/tmp/pydolphinscheduler" |
||||
cli_test = CliTestWrapper(cli, ["config", "--init"]) |
||||
cli_test.assert_success() |
||||
|
||||
# Make sure value do not exists first |
||||
cli_test = CliTestWrapper(cli, ["config", "--get", key]) |
||||
assert f"{key} = {value}" not in cli_test.result.output |
||||
|
||||
cli_test = CliTestWrapper(cli, ["config", "--set", key, value]) |
||||
cli_test.assert_success() |
||||
|
||||
cli_test = CliTestWrapper(cli, ["config", "--get", key]) |
||||
assert f"{key} = {value}" in cli_test.result.output |
||||
|
||||
|
||||
# TODO do not skip `config --set KEY1 VAL1 --set KEY2 VAL2` |
||||
@pytest.mark.skip( |
||||
reason="We still have bug in command `config --set KEY1 VAL1 --set KEY2 VAL2`" |
||||
) |
||||
@pytest.mark.parametrize( |
||||
"keys, values", |
||||
[ |
||||
# We test each key in mixture section |
||||
(("java_gateway.address", "java_gateway.port"), ("127.1.1.1", "25444")), |
||||
( |
||||
("java_gateway.auto_convert", "default.user.tenant"), |
||||
("False", "edit_tenant_pydolphin"), |
||||
), |
||||
( |
||||
( |
||||
"java_gateway.port", |
||||
"default.user.state", |
||||
"default.workflow.worker_group", |
||||
), |
||||
("25555", "0", "not-default"), |
||||
), |
||||
], |
||||
) |
||||
def test_config_set_multiple(delete_tmp_config_file, keys: str, values: str): |
||||
"""Test command line interface `config --set KEY1 VAL1 --set KEY2 VAL2`.""" |
||||
os.environ["PYDOLPHINSCHEDULER_HOME"] = "/tmp/pydolphinscheduler" |
||||
cli_test = CliTestWrapper(cli, ["config", "--init"]) |
||||
cli_test.assert_success() |
||||
|
||||
set_args = ["config"] |
||||
for idx, key in enumerate(keys): |
||||
# Make sure values do not exists first |
||||
cli_test = CliTestWrapper(cli, ["config", "--get", key]) |
||||
assert f"{key} = {values[idx]}" not in cli_test.result.output |
||||
|
||||
set_args.append("--set") |
||||
set_args.append(key) |
||||
set_args.append(values[idx]) |
||||
|
||||
cli_test = CliTestWrapper(cli, set_args) |
||||
cli_test.assert_success() |
||||
|
||||
for idx, key in enumerate(keys): |
||||
# Make sure values exists after `config --set` run |
||||
cli_test = CliTestWrapper(cli, ["config", "--get", key]) |
||||
assert f"{key} = {values[idx]}" in cli_test.result.output |
||||
@ -0,0 +1,45 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Test class :mod:`pydolphinscheduler.core.configuration`' method.""" |
||||
|
||||
import os |
||||
from pathlib import Path |
||||
|
||||
import pytest |
||||
|
||||
from pydolphinscheduler.core import configuration |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"env, expect", |
||||
[ |
||||
(None, "~/pydolphinscheduler"), |
||||
("/tmp/pydolphinscheduler", "/tmp/pydolphinscheduler"), |
||||
("/tmp/test_abc", "/tmp/test_abc"), |
||||
], |
||||
) |
||||
def test_get_config_file_path(env, expect): |
||||
"""Test get config file path method.""" |
||||
# Avoid env setting by other tests |
||||
os.environ.pop("PYDOLPHINSCHEDULER_HOME", None) |
||||
if env: |
||||
os.environ["PYDOLPHINSCHEDULER_HOME"] = env |
||||
assert ( |
||||
Path(expect).joinpath("config.yaml").expanduser() |
||||
== configuration.get_config_file_path() |
||||
) |
||||
@ -0,0 +1,39 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Test default config file.""" |
||||
|
||||
from typing import Dict |
||||
|
||||
import yaml |
||||
|
||||
from tests.testing.path import path_default_config_yaml |
||||
|
||||
|
||||
def nested_key_check(test_dict: Dict) -> None: |
||||
"""Test whether default configuration file exists specific character.""" |
||||
for key, val in test_dict.items(): |
||||
assert "." not in key, f"There is not allowed special character in key `{key}`." |
||||
if isinstance(val, dict): |
||||
nested_key_check(val) |
||||
|
||||
|
||||
def test_key_without_dot_delimiter(): |
||||
"""Test wrapper of whether default configuration file exists specific character.""" |
||||
with open(path_default_config_yaml, "r") as f: |
||||
default_config = yaml.safe_load(f) |
||||
nested_key_check(default_config) |
||||
@ -0,0 +1,201 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one |
||||
# or more contributor license agreements. See the NOTICE file |
||||
# distributed with this work for additional information |
||||
# regarding copyright ownership. The ASF licenses this file |
||||
# to you under the Apache License, Version 2.0 (the |
||||
# "License"); you may not use this file except in compliance |
||||
# with the License. You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, |
||||
# software distributed under the License is distributed on an |
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
||||
# KIND, either express or implied. See the License for the |
||||
# specific language governing permissions and limitations |
||||
# under the License. |
||||
|
||||
"""Test utils.path_dict module.""" |
||||
|
||||
import copy |
||||
from typing import Dict, Tuple |
||||
|
||||
import pytest |
||||
|
||||
from pydolphinscheduler.utils.path_dict import PathDict |
||||
|
||||
src_dict_list = [ |
||||
# dict with one single level |
||||
{"a": 1}, |
||||
# dict with two levels, with same nested keys 'b' |
||||
{"a": 1, "b": 2, "c": {"d": 3}, "e": {"b": 4}}, |
||||
# dict with three levels, with same nested keys 'b' |
||||
{"a": 1, "b": 2, "c": {"d": 3}, "e": {"b": {"b": 4}, "f": 5}}, |
||||
# dict with specific key container |
||||
{ |
||||
"a": 1, |
||||
"a-b": 2, |
||||
}, |
||||
] |
||||
|
||||
|
||||
@pytest.mark.parametrize("org", src_dict_list) |
||||
def test_val_between_dict_and_path_dict(org: Dict): |
||||
"""Test path dict equal to original dict.""" |
||||
path_dict = PathDict(org) |
||||
assert org == dict(path_dict) |
||||
|
||||
|
||||
def test_path_dict_basic_attr_access(): |
||||
"""Test basic behavior of path dict. |
||||
|
||||
Including add by attribute, with simple, nested dict, and specific key dict. |
||||
""" |
||||
expect = copy.deepcopy(src_dict_list[2]) |
||||
path_dict = PathDict(expect) |
||||
|
||||
# Add node with one level |
||||
val = 3 |
||||
path_dict.f = val |
||||
expect.update({"f": val}) |
||||
assert expect == path_dict |
||||
|
||||
# Add node with multiple level |
||||
val = {"abc": 123} |
||||
path_dict.e.g = val |
||||
expect.update({"e": {"b": {"b": 4}, "f": 5, "g": val}}) |
||||
assert expect == path_dict |
||||
|
||||
# Specific key |
||||
expect = copy.deepcopy(src_dict_list[3]) |
||||
path_dict = PathDict(expect) |
||||
assert 1 == path_dict.a |
||||
assert 2 == getattr(path_dict, "a-b") |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"org, exists, not_exists", |
||||
[ |
||||
( |
||||
src_dict_list[0], |
||||
("a"), |
||||
("b", "a.b"), |
||||
), |
||||
( |
||||
src_dict_list[1], |
||||
("a", "b", "c", "e", "c.d", "e.b"), |
||||
("a.b", "c.e", "b.c", "b.e"), |
||||
), |
||||
( |
||||
src_dict_list[2], |
||||
("a", "b", "c", "e", "c.d", "e.b", "e.b.b", "e.b.b", "e.f"), |
||||
("a.b", "c.e", "b.c", "b.e", "b.b.f", "b.f"), |
||||
), |
||||
], |
||||
) |
||||
def test_path_dict_attr(org: Dict, exists: Tuple, not_exists: Tuple): |
||||
"""Test properties' integrity of path dict.""" |
||||
path_dict = PathDict(org) |
||||
assert all([hasattr(path_dict, path) for path in exists]) |
||||
# assert not any([hasattr(path_dict, path) for path in not_exists]) |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"org, path_get", |
||||
[ |
||||
( |
||||
src_dict_list[0], |
||||
{"a": 1}, |
||||
), |
||||
( |
||||
src_dict_list[1], |
||||
{ |
||||
"a": 1, |
||||
"b": 2, |
||||
"c": {"d": 3}, |
||||
"c.d": 3, |
||||
"e": {"b": 4}, |
||||
"e.b": 4, |
||||
}, |
||||
), |
||||
( |
||||
src_dict_list[2], |
||||
{ |
||||
"a": 1, |
||||
"b": 2, |
||||
"c": {"d": 3}, |
||||
"c.d": 3, |
||||
"e": {"b": {"b": 4}, "f": 5}, |
||||
"e.b": {"b": 4}, |
||||
"e.b.b": 4, |
||||
"e.f": 5, |
||||
}, |
||||
), |
||||
], |
||||
) |
||||
def test_path_dict_get(org: Dict, path_get: Dict): |
||||
"""Test path dict getter function.""" |
||||
path_dict = PathDict(org) |
||||
assert all([path_get[path] == path_dict.__getattr__(path) for path in path_get]) |
||||
|
||||
|
||||
@pytest.mark.parametrize( |
||||
"org, path_set, expect", |
||||
[ |
||||
# Add not exists node |
||||
( |
||||
src_dict_list[0], |
||||
{"b": 2}, |
||||
{ |
||||
"a": 1, |
||||
"b": 2, |
||||
}, |
||||
), |
||||
# Overwrite exists node with different type of value |
||||
( |
||||
src_dict_list[0], |
||||
{"a": "b"}, |
||||
{"a": "b"}, |
||||
), |
||||
# Add multiple not exists node with variable types of value |
||||
( |
||||
src_dict_list[0], |
||||
{ |
||||
"b.c.d": 123, |
||||
"b.c.e": "a", |
||||
"b.f": {"g": 23, "h": "bc", "i": {"j": "k"}}, |
||||
}, |
||||
{ |
||||
"a": 1, |
||||
"b": { |
||||
"c": { |
||||
"d": 123, |
||||
"e": "a", |
||||
}, |
||||
"f": {"g": 23, "h": "bc", "i": {"j": "k"}}, |
||||
}, |
||||
}, |
||||
), |
||||
# Test complex original data |
||||
( |
||||
src_dict_list[2], |
||||
{ |
||||
"g": 12, |
||||
"c.h": 34, |
||||
}, |
||||
{ |
||||
"a": 1, |
||||
"b": 2, |
||||
"g": 12, |
||||
"c": {"d": 3, "h": 34}, |
||||
"e": {"b": {"b": 4}, "f": 5}, |
||||
}, |
||||
), |
||||
], |
||||
) |
||||
def test_path_dict_set(org: Dict, path_set: Dict, expect: Dict): |
||||
"""Test path dict setter function.""" |
||||
path_dict = PathDict(org) |
||||
for path in path_set: |
||||
path_dict.__setattr__(path, path_set[path]) |
||||
assert expect == path_dict |
||||
Loading…
Reference in new issue