From 4872d3fd481d68db58eecae6eaa2fa078146b7bf Mon Sep 17 00:00:00 2001 From: Egor Araslanov Date: Wed, 20 Nov 2024 17:54:55 +0500 Subject: [PATCH 1/4] ADCM-6125 Initial implementation of config management (core) --- adcm_aio_client/core/config/__init__.py | 0 adcm_aio_client/core/config/_base.py | 312 ++++++++++++++++++++++++ adcm_aio_client/core/config/errors.py | 10 + adcm_aio_client/core/config/types.py | 10 + tests/unit/test_config.py | 156 ++++++++++++ 5 files changed, 488 insertions(+) create mode 100644 adcm_aio_client/core/config/__init__.py create mode 100644 adcm_aio_client/core/config/_base.py create mode 100644 adcm_aio_client/core/config/errors.py create mode 100644 adcm_aio_client/core/config/types.py create mode 100644 tests/unit/test_config.py diff --git a/adcm_aio_client/core/config/__init__.py b/adcm_aio_client/core/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/adcm_aio_client/core/config/_base.py b/adcm_aio_client/core/config/_base.py new file mode 100644 index 0000000..04dd956 --- /dev/null +++ b/adcm_aio_client/core/config/_base.py @@ -0,0 +1,312 @@ +from abc import abstractmethod +from copy import deepcopy +from dataclasses import dataclass +from functools import partial +from typing import Any, Callable, Iterable, Self, Union, overload +import json + +from adcm_aio_client.core.config.errors import ParameterNotFoundError, ParameterTypeError +from adcm_aio_client.core.config.types import ( + AnyParameterName, + LevelNames, + ParameterDisplayName, + ParameterName, + ParameterValueOrNone, +) + +type SetValueCallback = Callable[[ParameterValueOrNone], Any] + +type SetNestedValueCallback = Callable[[LevelNames, ParameterValueOrNone], Any] +type SetActivationStateCallback = Callable[[LevelNames, bool], Any] + + +@dataclass(slots=True) +class Callbacks: + set_value: SetNestedValueCallback + set_activation_attribute: SetActivationStateCallback + + +class _ParametersGroup: + def __init__(self: Self, spec: dict, callbacks: Callbacks, previous_levels: LevelNames = ()) -> None: + self._spec = spec + self._previous_levels = previous_levels + self._callbacks = callbacks + self._names_mapping: dict[ParameterDisplayName, ParameterName] = {} + self._wrappers: dict[ParameterName, ParameterWrapper] = {} + + @property + @abstractmethod + def _current_config_level(self: Self) -> dict: ... + + @overload + def __getitem__[ExpectedType: "ParameterWrapper"]( + self: Self, item: tuple[AnyParameterName, type[ExpectedType]] + ) -> ExpectedType: ... + + @overload + def __getitem__(self: Self, item: AnyParameterName) -> "ParameterWrapper": ... + + def __getitem__[ExpectedType: "ParameterWrapper"]( + self: Self, item: AnyParameterName | tuple[AnyParameterName, type[ExpectedType]] + ) -> Union[ExpectedType, "ParameterWrapper"]: + if isinstance(item, str): + key = item + expected_type = None + else: + key, expected_type = item + + level_name = self._find_technical_name(display_name=key) + if not level_name: + level_name = key + + initialized_wrapper = self._wrappers.get(level_name) + + if not initialized_wrapper: + if level_name not in self._current_config_level: + message = f"No parameter with name {key} in configuration" + raise ParameterNotFoundError(message) + + # todo probably worth making it like "get_initialized_wrapper" and hide all cache work in here + initialized_wrapper = self._initialize_wrapper(level_name) + + self._wrappers[level_name] = initialized_wrapper + + if expected_type is not None and not isinstance(initialized_wrapper, expected_type): + message = f"Unexpected type of {key}: {type(initialized_wrapper)}.\nExpected: {expected_type}" + raise ParameterTypeError(message) + + return initialized_wrapper + + def _find_technical_name(self: Self, display_name: ParameterDisplayName) -> ParameterName | None: + cached_name = self._names_mapping.get(display_name) + if cached_name: + return cached_name + + for name, parameter_data in self._spec["properties"]: + if parameter_data.get("title") == display_name: + self._names_mapping[display_name] = name + return name + + return None + + def _get_parameter_spec(self: Self, name: ParameterName) -> dict: + value = self._spec[name] + if "oneOf" not in value: + return value + + # bald search, a lot may fail, + # but for more precise work with spec if require incapsulation in a separate handler class + return next(entry for entry in value["oneOf"] if entry.get("type") != "null") + + def _parameter_is_group(self: Self, parameter_spec: dict) -> bool: + return ( + # todo need to check group-like structures, because they are almost impossible to distinct from groups + parameter_spec.get("type") == "object" + and parameter_spec.get("additionalProperties") is False + and parameter_spec.get("default") == {} + ) + + def _initialize_wrapper(self: Self, name: ParameterName) -> "ParameterWrapper": + value = self._current_config_level[name] + spec = self._get_parameter_spec(name=name) + + is_group = isinstance(value, dict) and self._parameter_is_group(parameter_spec=spec) + + if is_group: + # value for groups isn't copied, + # because there isn't public interface for accessing it + they can be quite huge + level_data = value + previous_levels = (*self._previous_levels, name) + + is_activatable = spec["adcmMeta"].get("activation", {}).get("isAllowChange") + if is_activatable: + return ActivatableGroupWrapper( + config_level_data=level_data, spec=spec, callbacks=self._callbacks, previous_levels=previous_levels + ) + + return RegularGroupWrapper( + config_level_data=level_data, spec=spec, callbacks=self._callbacks, previous_levels=previous_levels + ) + + if isinstance(value, (dict, list)): + # simple failsafe for direct `value` edit + value = deepcopy(value) + + set_value_callback = partial(self._callbacks.set_value, (*self._previous_levels, name)) + + return ValueWrapper(value=value, set_value_callback=set_value_callback) + + +type ParameterWrapper = ValueWrapper | RegularGroupWrapper | ActivatableGroupWrapper + + +class ValueWrapper[InnerType: ParameterValueOrNone]: + __slots__ = ("_value", "_set_value") + + def __init__(self: Self, value: InnerType, set_value_callback: SetValueCallback) -> None: + self._value = value + self._set_value = set_value_callback + + @property + def value(self: Self) -> InnerType: + return self._value + + def set(self: Self, value: InnerType) -> Self: + self._set_value(value) + self._value = value + return self + + +class RegularGroupWrapper(_ParametersGroup): + def __init__( + self: Self, config_level_data: dict, spec: dict, callbacks: Callbacks, previous_levels: LevelNames + ) -> None: + super().__init__(spec=spec, callbacks=callbacks, previous_levels=previous_levels) + self._data = config_level_data + + @property + def _current_config_level(self: Self) -> dict: + return self._data + + +class ActivatableGroupWrapper(RegularGroupWrapper): + def activate(self: Self) -> Self: + # silencing check, because protocol is position-based + self._callbacks.set_activation_attribute(self._previous_levels, True) # noqa: FBT003 + return self + + def deactivate(self: Self) -> Self: + # silencing check, because protocol is position-based + self._callbacks.set_activation_attribute(self._previous_levels, False) # noqa: FBT003 + return self + + +class EditableConfig(_ParametersGroup): + def __init__( + self: Self, + data: dict, + spec: dict, + ) -> None: + self._initial_data = data + self._json_fields: set[tuple[ParameterName, ...]] = set() + self._convert_payload_formated_json_fields_inplace(self._initial_data["config"], prefix=()) + + self._changed_data = None + + super().__init__( + spec=spec, + callbacks=Callbacks( + set_value=self._set_parameter_value, set_activation_attribute=self._set_activation_attribute + ), + ) + + def to_payload(self: Self) -> dict: + payload = deepcopy(self._current_data) + self._convert_json_fields_to_payload_format_inplace(payload["config"]) + return payload + + @property + def _current_data(self: Self) -> dict: + if self._changed_data is not None: + return self._changed_data + + return self._initial_data + + @property + def _current_config_level(self: Self) -> dict: + return self._current_data["config"] + + def _set_parameter_value(self: Self, names: LevelNames, value: ParameterValueOrNone) -> None: + data = self._ensure_data_prepared_for_change() + set_nested_config_value(config=data["config"], level_names=names, value=value) + + # protocol is position-based now, so required to silence check + def _set_activation_attribute(self: Self, names: LevelNames, value: bool) -> None: # noqa: FBT001 + data = self._ensure_data_prepared_for_change() + + attribute_name = level_names_to_full_name(names) + + try: + data["adcmMeta"][attribute_name]["isActive"] = value + except KeyError as e: + message = ( + f"Failed to change activation attribute of {attribute_name}: not found in meta.\n" + "Either income data is incomplete or callback for this function is prepared incorrectly." + ) + raise RuntimeError(message) from e + + def _ensure_data_prepared_for_change(self: Self) -> dict: + if self._changed_data is None: + self._changed_data = deepcopy(self._initial_data) + + return self._changed_data + + def _convert_payload_formated_json_fields_inplace(self: Self, data: dict, prefix: LevelNames) -> None: + for key, value in data.items(): + parameter_spec = self._get_parameter_spec(key) + level_names = (*prefix, key) + if parameter_spec.get("format") == "json": + set_nested_config_value(data, level_names, self._json_value_from_payload_format(value)) + self._json_fields.add(level_names) + elif isinstance(value, dict) and self._parameter_is_group(parameter_spec): + self._convert_payload_formated_json_fields_inplace(data=value, prefix=level_names) + + def _convert_json_fields_to_payload_format_inplace(self: Self, data: dict) -> None: + for json_field_name in self._json_fields: + change_nested_config_value(data, json_field_name, self._json_value_to_payload_format) + + def _json_value_to_payload_format(self: Self, value: ParameterValueOrNone) -> str | None: + if value is None: + return None + + return json.dumps(value) + + def _json_value_from_payload_format(self: Self, value: str | None) -> ParameterValueOrNone: + if isinstance(value, str): + return json.loads(value) + + return None + + +# FOREIGN SECTION +# +# these functions are heavily inspired by configuration rework in ADCM (ADCM-6034) + +ROOT_PREFIX = "/" + + +def set_nested_config_value[T](config: dict[str, Any], level_names: LevelNames, value: T) -> T: + group, level_name = get_group_with_value(config=config, level_names=level_names) + group[level_name] = value + return value + + +def change_nested_config_value[T](config: dict[str, Any], level_names: LevelNames, func: Callable[[Any], T]) -> T: + group, level_name = get_group_with_value(config=config, level_names=level_names) + group[level_name] = func(group[level_name]) + return group[level_name] + + +def get_group_with_value(config: dict[str, Any], level_names: LevelNames) -> tuple[dict[str, Any], ParameterName]: + return _get_group_with_value(config=config, level_names=level_names) + + +def _get_group_with_value( + config: dict[str, Any], level_names: Iterable[ParameterName] +) -> tuple[dict[str, Any], ParameterName]: + level_name, *rest = level_names + if not rest: + return config, level_name + + return _get_group_with_value(config=config[level_name], level_names=rest) + + +def level_names_to_full_name(levels: LevelNames) -> str: + return ensure_full_name("/".join(levels)) + + +def ensure_full_name(name: str) -> str: + if not name.startswith(ROOT_PREFIX): + return f"{ROOT_PREFIX}{name}" + + return name diff --git a/adcm_aio_client/core/config/errors.py b/adcm_aio_client/core/config/errors.py new file mode 100644 index 0000000..009fe44 --- /dev/null +++ b/adcm_aio_client/core/config/errors.py @@ -0,0 +1,10 @@ +from adcm_aio_client.core.errors import ADCMClientError + + +class ConfigError(ADCMClientError): ... + + +class ParameterNotFoundError(ConfigError): ... + + +class ParameterTypeError(ConfigError): ... diff --git a/adcm_aio_client/core/config/types.py b/adcm_aio_client/core/config/types.py new file mode 100644 index 0000000..092ed26 --- /dev/null +++ b/adcm_aio_client/core/config/types.py @@ -0,0 +1,10 @@ +type ParameterName = str +type ParameterDisplayName = str +type AnyParameterName = str + +type LevelNames = tuple[ParameterName, ...] + + +type SimpleParameterValue = float | int | bool | str +type ComplexParameterValue = dict | list +type ParameterValueOrNone = SimpleParameterValue | ComplexParameterValue | None diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py new file mode 100644 index 0000000..27e2e99 --- /dev/null +++ b/tests/unit/test_config.py @@ -0,0 +1,156 @@ +from copy import deepcopy +import json + +import pytest + +from adcm_aio_client.core.config._base import ActivatableGroupWrapper, EditableConfig, RegularGroupWrapper, ValueWrapper + + +@pytest.fixture() +def example_config() -> tuple[dict, dict]: + regular_param_schema = {} + group_like_param_schema = { + "parameters": {}, + "additionalProperties": False, + "default": {}, + "adcmMeta": {}, + "type": "object", + } + json_like_param_schema = {"format": "json"} + + config = { + "config": { + "root_int": 100, + "root_list": ["first", "second", "third"], + "root_dict": {"k1": "v1", "k2": "v2"}, + "duplicate": "hehe", + "root_json": json.dumps({}), + "main": { + "inner_str": "evil", + "inner_dict": {"a": "b"}, + "inner_json": json.dumps({"complex": [], "jsonfield": 23, "server": "bestever"}), + "duplicate": 44, + }, + "optional_group": {"param": 44.44}, + "root_str": None, + }, + "adcmMeta": {"/optional_group": {"isActive": False}}, + } + schema = { + "parameters": { + **{key: regular_param_schema for key in config["config"]}, + "root_json": json_like_param_schema, + "main": group_like_param_schema + | { + "parameters": { + **{key: regular_param_schema for key in config["config"]["main"]}, + "inner_json": json_like_param_schema, + } + }, + "optional_group": group_like_param_schema + | {"properties": {"param": regular_param_schema}, "adcmMeta": {"activation": {"isAllowChange": True}}}, + } + } + + return config, schema + + +def test_config_edit_by_name(example_config: tuple[dict, dict]) -> None: + data, schema = example_config + + new_inner_json = { + "complex": [], + "jsonfield": 23, + "link": "do i look like a link to you?", + "arguments": ["-q", "something"], + } + new_root_json = ["now", "I am", "cool"] + + new_config = { + "root_int": 430, + "root_list": ["first", "second", "third", "best thing there is"], + "root_dict": None, + "duplicate": "hehe", + "root_json": json.dumps(new_root_json), + "main": { + "inner_str": "not the worst at least", + "inner_dict": {"a": "b", "additional": "keys", "are": "welcome"}, + "inner_json": json.dumps(new_inner_json), + "duplicate": 44, + }, + "optional_group": {"param": 44.44}, + "root_str": "newstring", + } + + # todo: + # - check no POST requests are performed + + config = EditableConfig(data=data, spec=schema) + + config_for_save = config.to_payload() + assert config_for_save == data + assert config_for_save is not data + + # Edit "root" values + + config["root_int", ValueWrapper].set(new_config["root_int"]) + + # inner type won't be checked (list), + # but here we pretend "to be 100% sure" it's `list`, not `None` + config["root_list", ValueWrapper].set([*config["root_list", ValueWrapper[list]].value, new_config["root_list"][-1]]) + + root_dict = config["root_dict"] + assert isinstance(root_dict, ValueWrapper) + assert isinstance(root_dict.value, dict) + root_dict.set(None) + assert root_dict.value is None + assert config["root_dict"] is None + + # Edit group ("nested") values + + assert isinstance(config["main"], RegularGroupWrapper) + config["main", RegularGroupWrapper]["inner_str", ValueWrapper].set(new_config["main"]["inner_str"]) + + main_group = config["main"] + assert isinstance(main_group, RegularGroupWrapper) + main_group["inner_dict", ValueWrapper].set( + {**main_group["inner_dict", ValueWrapper[dict]].value, "additional": "keys", "are": "welcome"} + ) + + activatable_group = config["optional_group"] + assert isinstance(activatable_group, ActivatableGroupWrapper) + activatable_group.activate() + + # Edit JSON field + + # change value separately and set + json_field = main_group["inner_json"] + assert isinstance(json_field, ValueWrapper) + assert isinstance(json_field.value, dict) + new_value = deepcopy(json_field.value) + new_value.pop("server") + new_value |= {"link": "do i look like a link to you?", "arguments": ["-q", "something"]} + json_field.set(new_value) + + # swap value type with direct set + assert isinstance(config["root_json", ValueWrapper[dict]].value, dict) + config["root_json", ValueWrapper].set(["now", "I am", "cool"]) + + # Type change specifics + + param = config["root_str"] + assert isinstance(param, ValueWrapper) + assert isinstance(param.value, str) + + param.set(None) + assert config["root_str"] is None + assert isinstance(param.value, str) + + param.set("newstring") + assert isinstance(config["root_str"], str) + + # Check all values are changed + + config_for_save = config.to_payload() + assert config_for_save["config"] == new_config + assert config_for_save["adcmMeta"] == {"/optional_group": {"isActive": True}} From 02b907e699ffc7aca522b8967ada79f8c2e8cda4 Mon Sep 17 00:00:00 2001 From: Egor Araslanov Date: Wed, 20 Nov 2024 19:00:32 +0500 Subject: [PATCH 2/4] ADCM-6125 Fixes and __getitem__ type detection enhancement --- adcm_aio_client/core/config/_base.py | 112 +++++++++++++++++--------- adcm_aio_client/core/config/errors.py | 3 + tests/unit/test_config.py | 24 +++--- 3 files changed, 86 insertions(+), 53 deletions(-) diff --git a/adcm_aio_client/core/config/_base.py b/adcm_aio_client/core/config/_base.py index 04dd956..d4b6278 100644 --- a/adcm_aio_client/core/config/_base.py +++ b/adcm_aio_client/core/config/_base.py @@ -5,7 +5,7 @@ from typing import Any, Callable, Iterable, Self, Union, overload import json -from adcm_aio_client.core.config.errors import ParameterNotFoundError, ParameterTypeError +from adcm_aio_client.core.config.errors import ParameterNotFoundError, ParameterTypeError, ParameterValueTypeError from adcm_aio_client.core.config.types import ( AnyParameterName, LevelNames, @@ -26,9 +26,27 @@ class Callbacks: set_activation_attribute: SetActivationStateCallback +class ValueWrapper[InnerType: ParameterValueOrNone]: + __slots__ = ("_value", "_set_value") + + def __init__(self: Self, value: InnerType, set_value_callback: SetValueCallback) -> None: + self._value = value + self._set_value = set_value_callback + + @property + def value(self: Self) -> InnerType: + return self._value + + def set(self: Self, value: InnerType) -> Self: + self._set_value(value) + self._value = value + return self + + class _ParametersGroup: def __init__(self: Self, spec: dict, callbacks: Callbacks, previous_levels: LevelNames = ()) -> None: - self._spec = spec + # for now we assume it's always there + self._spec = spec["parameters"] self._previous_levels = previous_levels self._callbacks = callbacks self._names_mapping: dict[ParameterDisplayName, ParameterName] = {} @@ -38,6 +56,11 @@ def __init__(self: Self, spec: dict, callbacks: Callbacks, previous_levels: Leve @abstractmethod def _current_config_level(self: Self) -> dict: ... + @overload + def __getitem__[InnerType: ParameterValueOrNone]( + self: Self, item: tuple[AnyParameterName, type[ValueWrapper], type[InnerType]] + ) -> ValueWrapper[InnerType]: ... + @overload def __getitem__[ExpectedType: "ParameterWrapper"]( self: Self, item: tuple[AnyParameterName, type[ExpectedType]] @@ -46,14 +69,23 @@ def __getitem__[ExpectedType: "ParameterWrapper"]( @overload def __getitem__(self: Self, item: AnyParameterName) -> "ParameterWrapper": ... - def __getitem__[ExpectedType: "ParameterWrapper"]( - self: Self, item: AnyParameterName | tuple[AnyParameterName, type[ExpectedType]] - ) -> Union[ExpectedType, "ParameterWrapper"]: + def __getitem__[ExpectedType: "ParameterWrapper", ValueType: ParameterValueOrNone]( + self: Self, + item: AnyParameterName + | tuple[AnyParameterName, type[ExpectedType]] + | tuple[AnyParameterName, type[ValueWrapper], type[ValueType]], + ) -> Union[ValueWrapper[ValueType], ExpectedType, "ParameterWrapper"]: + check_internal = False + internal_type = None if isinstance(item, str): key = item expected_type = None - else: + elif len(item) == 2: key, expected_type = item + else: + key, _, internal_type = item + expected_type = ValueWrapper + check_internal = True level_name = self._find_technical_name(display_name=key) if not level_name: @@ -71,9 +103,24 @@ def __getitem__[ExpectedType: "ParameterWrapper"]( self._wrappers[level_name] = initialized_wrapper - if expected_type is not None and not isinstance(initialized_wrapper, expected_type): - message = f"Unexpected type of {key}: {type(initialized_wrapper)}.\nExpected: {expected_type}" - raise ParameterTypeError(message) + if expected_type is not None: + if not isinstance(initialized_wrapper, expected_type): + message = f"Unexpected type of {key}: {type(initialized_wrapper)}.\nExpected: {expected_type}" + raise ParameterTypeError(message) + + if check_internal: + if not isinstance(initialized_wrapper, ValueWrapper): + message = f"Internal type can be checked only for ValueWrapper, not {type(initialized_wrapper)}" + raise ParameterTypeError(message) + + value = initialized_wrapper.value + if internal_type is None: + if value is not None: + message = f"Value expected to be None, not {value}" + raise ParameterValueTypeError(message) + elif not isinstance(value, internal_type): + message = f"Unexpected type of value of {key}: {type(value)}.\nExpected: {internal_type}" + raise ParameterValueTypeError(message) return initialized_wrapper @@ -82,15 +129,15 @@ def _find_technical_name(self: Self, display_name: ParameterDisplayName) -> Para if cached_name: return cached_name - for name, parameter_data in self._spec["properties"]: + for name, parameter_data in self._spec.items(): if parameter_data.get("title") == display_name: self._names_mapping[display_name] = name return name return None - def _get_parameter_spec(self: Self, name: ParameterName) -> dict: - value = self._spec[name] + def _get_parameter_spec(self: Self, name: ParameterName, parameters_spec: dict | None = None) -> dict: + value = (parameters_spec or self._spec)[name] if "oneOf" not in value: return value @@ -140,23 +187,6 @@ def _initialize_wrapper(self: Self, name: ParameterName) -> "ParameterWrapper": type ParameterWrapper = ValueWrapper | RegularGroupWrapper | ActivatableGroupWrapper -class ValueWrapper[InnerType: ParameterValueOrNone]: - __slots__ = ("_value", "_set_value") - - def __init__(self: Self, value: InnerType, set_value_callback: SetValueCallback) -> None: - self._value = value - self._set_value = set_value_callback - - @property - def value(self: Self) -> InnerType: - return self._value - - def set(self: Self, value: InnerType) -> Self: - self._set_value(value) - self._value = value - return self - - class RegularGroupWrapper(_ParametersGroup): def __init__( self: Self, config_level_data: dict, spec: dict, callbacks: Callbacks, previous_levels: LevelNames @@ -187,12 +217,6 @@ def __init__( data: dict, spec: dict, ) -> None: - self._initial_data = data - self._json_fields: set[tuple[ParameterName, ...]] = set() - self._convert_payload_formated_json_fields_inplace(self._initial_data["config"], prefix=()) - - self._changed_data = None - super().__init__( spec=spec, callbacks=Callbacks( @@ -200,6 +224,12 @@ def __init__( ), ) + self._initial_data = data + self._json_fields: set[tuple[ParameterName, ...]] = set() + self._convert_payload_formated_json_fields_inplace(self._spec, self._initial_data["config"], prefix=()) + + self._changed_data = None + def to_payload(self: Self) -> dict: payload = deepcopy(self._current_data) self._convert_json_fields_to_payload_format_inplace(payload["config"]) @@ -241,15 +271,19 @@ def _ensure_data_prepared_for_change(self: Self) -> dict: return self._changed_data - def _convert_payload_formated_json_fields_inplace(self: Self, data: dict, prefix: LevelNames) -> None: + def _convert_payload_formated_json_fields_inplace( + self: Self, parameters_spec: dict, data: dict, prefix: LevelNames + ) -> None: for key, value in data.items(): - parameter_spec = self._get_parameter_spec(key) + parameter_spec = self._get_parameter_spec(key, parameters_spec=parameters_spec) level_names = (*prefix, key) if parameter_spec.get("format") == "json": - set_nested_config_value(data, level_names, self._json_value_from_payload_format(value)) + set_nested_config_value(data, (key,), self._json_value_from_payload_format(value)) self._json_fields.add(level_names) elif isinstance(value, dict) and self._parameter_is_group(parameter_spec): - self._convert_payload_formated_json_fields_inplace(data=value, prefix=level_names) + self._convert_payload_formated_json_fields_inplace( + parameters_spec=parameters_spec[key]["parameters"], data=value, prefix=level_names + ) def _convert_json_fields_to_payload_format_inplace(self: Self, data: dict) -> None: for json_field_name in self._json_fields: diff --git a/adcm_aio_client/core/config/errors.py b/adcm_aio_client/core/config/errors.py index 009fe44..11365a1 100644 --- a/adcm_aio_client/core/config/errors.py +++ b/adcm_aio_client/core/config/errors.py @@ -8,3 +8,6 @@ class ParameterNotFoundError(ConfigError): ... class ParameterTypeError(ConfigError): ... + + +class ParameterValueTypeError(ConfigError): ... diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 27e2e99..26e24db 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -48,7 +48,7 @@ def example_config() -> tuple[dict, dict]: } }, "optional_group": group_like_param_schema - | {"properties": {"param": regular_param_schema}, "adcmMeta": {"activation": {"isAllowChange": True}}}, + | {"parameters": {"param": regular_param_schema}, "adcmMeta": {"activation": {"isAllowChange": True}}}, } } @@ -57,7 +57,6 @@ def example_config() -> tuple[dict, dict]: def test_config_edit_by_name(example_config: tuple[dict, dict]) -> None: data, schema = example_config - new_inner_json = { "complex": [], "jsonfield": 23, @@ -85,11 +84,12 @@ def test_config_edit_by_name(example_config: tuple[dict, dict]) -> None: # todo: # - check no POST requests are performed - config = EditableConfig(data=data, spec=schema) + # deepcopy, because we want our "source" to be intact + # and unchanged by json formating + config = EditableConfig(data=deepcopy(data), spec=schema) config_for_save = config.to_payload() assert config_for_save == data - assert config_for_save is not data # Edit "root" values @@ -97,14 +97,14 @@ def test_config_edit_by_name(example_config: tuple[dict, dict]) -> None: # inner type won't be checked (list), # but here we pretend "to be 100% sure" it's `list`, not `None` - config["root_list", ValueWrapper].set([*config["root_list", ValueWrapper[list]].value, new_config["root_list"][-1]]) + config["root_list", ValueWrapper].set([*config["root_list", ValueWrapper, list].value, new_config["root_list"][-1]]) root_dict = config["root_dict"] assert isinstance(root_dict, ValueWrapper) assert isinstance(root_dict.value, dict) root_dict.set(None) assert root_dict.value is None - assert config["root_dict"] is None + assert config["root_dict", ValueWrapper].value is None # Edit group ("nested") values @@ -114,7 +114,7 @@ def test_config_edit_by_name(example_config: tuple[dict, dict]) -> None: main_group = config["main"] assert isinstance(main_group, RegularGroupWrapper) main_group["inner_dict", ValueWrapper].set( - {**main_group["inner_dict", ValueWrapper[dict]].value, "additional": "keys", "are": "welcome"} + {**main_group["inner_dict", ValueWrapper, dict].value, "additional": "keys", "are": "welcome"} ) activatable_group = config["optional_group"] @@ -133,21 +133,17 @@ def test_config_edit_by_name(example_config: tuple[dict, dict]) -> None: json_field.set(new_value) # swap value type with direct set - assert isinstance(config["root_json", ValueWrapper[dict]].value, dict) + assert isinstance(config["root_json", ValueWrapper, dict].value, dict) config["root_json", ValueWrapper].set(["now", "I am", "cool"]) # Type change specifics param = config["root_str"] assert isinstance(param, ValueWrapper) - assert isinstance(param.value, str) - - param.set(None) - assert config["root_str"] is None - assert isinstance(param.value, str) + assert param.value is None param.set("newstring") - assert isinstance(config["root_str"], str) + assert isinstance(config["root_str", ValueWrapper].value, str) # Check all values are changed From 58521c3bdba809494e7e23af076554c0cd1f910f Mon Sep 17 00:00:00 2001 From: Egor Araslanov Date: Thu, 21 Nov 2024 11:48:03 +0500 Subject: [PATCH 3/4] ADCM-6125 Use prepared responses for test_config --- adcm_aio_client/core/config/_base.py | 6 +- tests/bundles/config_example_v1/config.yaml | 70 ++++ tests/unit/conftest.py | 3 + tests/unit/files/responses/.description | 2 + .../responses/test_config_example_config.json | 37 ++ .../test_config_example_config_schema.json | 324 ++++++++++++++++++ tests/unit/test_config.py | 48 +-- 7 files changed, 444 insertions(+), 46 deletions(-) create mode 100644 tests/bundles/config_example_v1/config.yaml create mode 100644 tests/unit/files/responses/.description create mode 100644 tests/unit/files/responses/test_config_example_config.json create mode 100644 tests/unit/files/responses/test_config_example_config_schema.json diff --git a/adcm_aio_client/core/config/_base.py b/adcm_aio_client/core/config/_base.py index d4b6278..40ca786 100644 --- a/adcm_aio_client/core/config/_base.py +++ b/adcm_aio_client/core/config/_base.py @@ -46,7 +46,7 @@ def set(self: Self, value: InnerType) -> Self: class _ParametersGroup: def __init__(self: Self, spec: dict, callbacks: Callbacks, previous_levels: LevelNames = ()) -> None: # for now we assume it's always there - self._spec = spec["parameters"] + self._spec = spec["properties"] self._previous_levels = previous_levels self._callbacks = callbacks self._names_mapping: dict[ParameterDisplayName, ParameterName] = {} @@ -165,7 +165,7 @@ def _initialize_wrapper(self: Self, name: ParameterName) -> "ParameterWrapper": level_data = value previous_levels = (*self._previous_levels, name) - is_activatable = spec["adcmMeta"].get("activation", {}).get("isAllowChange") + is_activatable = (spec["adcmMeta"].get("activation") or {}).get("isAllowChange") if is_activatable: return ActivatableGroupWrapper( config_level_data=level_data, spec=spec, callbacks=self._callbacks, previous_levels=previous_levels @@ -282,7 +282,7 @@ def _convert_payload_formated_json_fields_inplace( self._json_fields.add(level_names) elif isinstance(value, dict) and self._parameter_is_group(parameter_spec): self._convert_payload_formated_json_fields_inplace( - parameters_spec=parameters_spec[key]["parameters"], data=value, prefix=level_names + parameters_spec=parameters_spec[key]["properties"], data=value, prefix=level_names ) def _convert_json_fields_to_payload_format_inplace(self: Self, data: dict) -> None: diff --git a/tests/bundles/config_example_v1/config.yaml b/tests/bundles/config_example_v1/config.yaml new file mode 100644 index 0000000..ceba612 --- /dev/null +++ b/tests/bundles/config_example_v1/config.yaml @@ -0,0 +1,70 @@ +- type: cluster + name: Cluster With Config Example + version: 1 + description: | + This bundle is designed to provide sample of config, + not nessesary including all config types or combinations. + Don't change configs of existing objects in it, + add new service / component if you need. + +- type: service + name: with_json_fields_and_groups + version: 1.0 + + config: + - name: root_int + display_name: Integer At Root + type: integer + default: 100 + - name: root_list + display_name: List At Root + type: list + default: ["first", "second", "third"] + - name: root_dict + display_name: Map At Root + type: map + default: {"k1": "v1", "k2": "v2"} + required: false + - name: duplicate + display_name: Duplicate + type: string + default: "hehe" + - name: root_json + display_name: JSON At Root + type: json + default: {} + - name: main + display_name: Main Section + type: group + subs: + - name: inner_str + display_name: String In Group + type: string + default: "evil" + - name: inner_dict + display_name: Map In Group + type: map + default: {"a": "b"} + - name: inner_json + display_name: JSON In Group + type: json + default: {"complex": [], "jsonfield": 23, "server": "bestever"} + - name: duplicate + display_name: Integer In Group + type: integer + default: 44 + - name: optional_group + display_name: Optional Section + type: group + activatable: true + active: false + subs: + - name: param + display_name: Param In Activatable Group + type: float + default: 44.44 + required: false + - name: root_str + display_name: String At Root + type: string + required: false diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 6de824c..6b397dc 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -1,7 +1,10 @@ import pytest +from pathlib import Path from tests.unit.mocks.requesters import QueueRequester +FILES = Path(__file__).parent / "files" +RESPONSES = FILES / "responses" @pytest.fixture() def queue_requester() -> QueueRequester: diff --git a/tests/unit/files/responses/.description b/tests/unit/files/responses/.description new file mode 100644 index 0000000..837b946 --- /dev/null +++ b/tests/unit/files/responses/.description @@ -0,0 +1,2 @@ +This directory have samples of responses from ADCM +to use them as mock responses in unit tests. diff --git a/tests/unit/files/responses/test_config_example_config.json b/tests/unit/files/responses/test_config_example_config.json new file mode 100644 index 0000000..9a50a74 --- /dev/null +++ b/tests/unit/files/responses/test_config_example_config.json @@ -0,0 +1,37 @@ +{ + "id": 3, + "isCurrent": true, + "creationTime": "2024-11-21T06:38:58.517310Z", + "config": { + "main": { + "duplicate": 44, + "inner_str": "evil", + "inner_dict": { + "a": "b" + }, + "inner_json": "{\"server\": \"bestever\", \"complex\": [], \"jsonfield\": 23}" + }, + "root_int": 100, + "root_str": null, + "duplicate": "hehe", + "root_dict": { + "k1": "v1", + "k2": "v2" + }, + "root_json": "{}", + "root_list": [ + "first", + "second", + "third" + ], + "optional_group": { + "param": 44.44 + } + }, + "adcmMeta": { + "/optional_group": { + "isActive": false + } + }, + "description": "init" +} diff --git a/tests/unit/files/responses/test_config_example_config_schema.json b/tests/unit/files/responses/test_config_example_config_schema.json new file mode 100644 index 0000000..9b8b33d --- /dev/null +++ b/tests/unit/files/responses/test_config_example_config_schema.json @@ -0,0 +1,324 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Configuration", + "description": "", + "readOnly": false, + "adcmMeta": { + "isAdvanced": false, + "isInvisible": false, + "activation": null, + "synchronization": null, + "nullValue": null, + "isSecret": false, + "stringExtra": null, + "enumExtra": null + }, + "type": "object", + "properties": { + "root_int": { + "title": "Integer At Root", + "type": "integer", + "description": "", + "default": 100, + "readOnly": false, + "adcmMeta": { + "isAdvanced": false, + "isInvisible": false, + "activation": null, + "synchronization": null, + "isSecret": false, + "stringExtra": null, + "enumExtra": null + } + }, + "root_list": { + "title": "List At Root", + "type": "array", + "description": "", + "default": [ + "first", + "second", + "third" + ], + "readOnly": false, + "adcmMeta": { + "isAdvanced": false, + "isInvisible": false, + "activation": null, + "synchronization": null, + "isSecret": false, + "stringExtra": null, + "enumExtra": null + }, + "items": { + "type": "string", + "title": "", + "description": "", + "default": null, + "readOnly": false, + "adcmMeta": { + "isAdvanced": false, + "isInvisible": false, + "activation": null, + "synchronization": null, + "nullValue": null, + "isSecret": false, + "stringExtra": null, + "enumExtra": null + } + }, + "minItems": 1 + }, + "root_dict": { + "oneOf": [ + { + "title": "Map At Root", + "type": "object", + "description": "", + "default": { + "k1": "v1", + "k2": "v2" + }, + "readOnly": false, + "adcmMeta": { + "isAdvanced": false, + "isInvisible": false, + "activation": null, + "synchronization": null, + "isSecret": false, + "stringExtra": null, + "enumExtra": null + }, + "additionalProperties": true, + "properties": {} + }, + { + "type": "null" + } + ] + }, + "duplicate": { + "title": "Duplicate", + "type": "string", + "description": "", + "default": "hehe", + "readOnly": false, + "adcmMeta": { + "isAdvanced": false, + "isInvisible": false, + "activation": null, + "synchronization": null, + "isSecret": false, + "stringExtra": { + "isMultiline": false + }, + "enumExtra": null + }, + "minLength": 1 + }, + "root_json": { + "title": "JSON At Root", + "type": "string", + "description": "", + "default": "{}", + "readOnly": false, + "adcmMeta": { + "isAdvanced": false, + "isInvisible": false, + "activation": null, + "synchronization": null, + "isSecret": false, + "stringExtra": { + "isMultiline": true + }, + "enumExtra": null + }, + "format": "json", + "minLength": 1 + }, + "main": { + "title": "Main Section", + "type": "object", + "description": "", + "default": {}, + "readOnly": false, + "adcmMeta": { + "isAdvanced": false, + "isInvisible": false, + "activation": null, + "synchronization": null, + "isSecret": false, + "stringExtra": null, + "enumExtra": null + }, + "additionalProperties": false, + "properties": { + "inner_str": { + "title": "String In Group", + "type": "string", + "description": "", + "default": "evil", + "readOnly": false, + "adcmMeta": { + "isAdvanced": false, + "isInvisible": false, + "activation": null, + "synchronization": null, + "isSecret": false, + "stringExtra": { + "isMultiline": false + }, + "enumExtra": null + }, + "minLength": 1 + }, + "inner_dict": { + "title": "Map In Group", + "type": "object", + "description": "", + "default": { + "a": "b" + }, + "readOnly": false, + "adcmMeta": { + "isAdvanced": false, + "isInvisible": false, + "activation": null, + "synchronization": null, + "isSecret": false, + "stringExtra": null, + "enumExtra": null + }, + "additionalProperties": true, + "properties": {}, + "minProperties": 1 + }, + "inner_json": { + "title": "JSON In Group", + "type": "string", + "description": "", + "default": "{\"complex\": [], \"jsonfield\": 23, \"server\": \"bestever\"}", + "readOnly": false, + "adcmMeta": { + "isAdvanced": false, + "isInvisible": false, + "activation": null, + "synchronization": null, + "isSecret": false, + "stringExtra": { + "isMultiline": true + }, + "enumExtra": null + }, + "format": "json", + "minLength": 1 + }, + "duplicate": { + "title": "Integer In Group", + "type": "integer", + "description": "", + "default": 44, + "readOnly": false, + "adcmMeta": { + "isAdvanced": false, + "isInvisible": false, + "activation": null, + "synchronization": null, + "isSecret": false, + "stringExtra": null, + "enumExtra": null + } + } + }, + "required": [ + "inner_str", + "inner_dict", + "inner_json", + "duplicate" + ] + }, + "optional_group": { + "title": "Optional Section", + "type": "object", + "description": "", + "default": {}, + "readOnly": false, + "adcmMeta": { + "isAdvanced": false, + "isInvisible": false, + "activation": { + "isAllowChange": true + }, + "synchronization": null, + "isSecret": false, + "stringExtra": null, + "enumExtra": null + }, + "additionalProperties": false, + "properties": { + "param": { + "oneOf": [ + { + "title": "Param In Activatable Group", + "type": "number", + "description": "", + "default": 44.44, + "readOnly": false, + "adcmMeta": { + "isAdvanced": false, + "isInvisible": false, + "activation": null, + "synchronization": null, + "isSecret": false, + "stringExtra": null, + "enumExtra": null + } + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "param" + ] + }, + "root_str": { + "oneOf": [ + { + "title": "String At Root", + "type": "string", + "description": "", + "default": null, + "readOnly": false, + "adcmMeta": { + "isAdvanced": false, + "isInvisible": false, + "activation": null, + "synchronization": null, + "isSecret": false, + "stringExtra": { + "isMultiline": false + }, + "enumExtra": null + } + }, + { + "type": "null" + } + ] + } + }, + "additionalProperties": false, + "required": [ + "root_int", + "root_list", + "root_dict", + "duplicate", + "root_json", + "main", + "optional_group", + "root_str" + ] +} diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 26e24db..6f62eb1 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -4,53 +4,15 @@ import pytest from adcm_aio_client.core.config._base import ActivatableGroupWrapper, EditableConfig, RegularGroupWrapper, ValueWrapper +from tests.unit.conftest import RESPONSES + @pytest.fixture() def example_config() -> tuple[dict, dict]: - regular_param_schema = {} - group_like_param_schema = { - "parameters": {}, - "additionalProperties": False, - "default": {}, - "adcmMeta": {}, - "type": "object", - } - json_like_param_schema = {"format": "json"} - - config = { - "config": { - "root_int": 100, - "root_list": ["first", "second", "third"], - "root_dict": {"k1": "v1", "k2": "v2"}, - "duplicate": "hehe", - "root_json": json.dumps({}), - "main": { - "inner_str": "evil", - "inner_dict": {"a": "b"}, - "inner_json": json.dumps({"complex": [], "jsonfield": 23, "server": "bestever"}), - "duplicate": 44, - }, - "optional_group": {"param": 44.44}, - "root_str": None, - }, - "adcmMeta": {"/optional_group": {"isActive": False}}, - } - schema = { - "parameters": { - **{key: regular_param_schema for key in config["config"]}, - "root_json": json_like_param_schema, - "main": group_like_param_schema - | { - "parameters": { - **{key: regular_param_schema for key in config["config"]["main"]}, - "inner_json": json_like_param_schema, - } - }, - "optional_group": group_like_param_schema - | {"parameters": {"param": regular_param_schema}, "adcmMeta": {"activation": {"isAllowChange": True}}}, - } - } + config = json.loads((RESPONSES / "test_config_example_config.json").read_text()) + + schema = json.loads((RESPONSES / "test_config_example_config_schema.json").read_text()) return config, schema From 72092d35090005952c646347349a9fd32f4fcd44 Mon Sep 17 00:00:00 2001 From: Egor Araslanov Date: Thu, 21 Nov 2024 11:51:24 +0500 Subject: [PATCH 4/4] ADCM-6125 Formatting fixes --- tests/unit/conftest.py | 4 +++- tests/unit/test_config.py | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 6b397dc..3df732e 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -1,11 +1,13 @@ -import pytest from pathlib import Path +import pytest + from tests.unit.mocks.requesters import QueueRequester FILES = Path(__file__).parent / "files" RESPONSES = FILES / "responses" + @pytest.fixture() def queue_requester() -> QueueRequester: return QueueRequester() diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 6f62eb1..b7bc946 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -7,7 +7,6 @@ from tests.unit.conftest import RESPONSES - @pytest.fixture() def example_config() -> tuple[dict, dict]: config = json.loads((RESPONSES / "test_config_example_config.json").read_text())