From 0d76356ab17cd6ae76171e59e5ead91868937a57 Mon Sep 17 00:00:00 2001 From: Dmitrii Ovsyannikov Date: Fri, 20 Dec 2024 14:53:20 +0100 Subject: [PATCH] feat: BI-0 add new settings with fallbacks (#753) --- app/dl_control_api/dl_control_api/app.py | 4 +- .../dl_control_api/app_factory.py | 16 ++- app/dl_data_api/dl_data_api/app.py | 8 +- app/dl_data_api/dl_data_api/app_factory.py | 14 +- lib/dl_api_lib/dl_api_lib/app_settings.py | 80 +++++++---- lib/dl_api_lib/pyproject.toml | 1 + .../dl_auth_api_lib/settings.py | 2 +- lib/dl_settings/dl_settings/__init__.py | 8 +- lib/dl_settings/dl_settings/base/__init__.py | 10 +- lib/dl_settings/dl_settings/base/fallback.py | 136 ++++++++++++++++++ lib/dl_settings/dl_settings/base/typed.py | 39 ++--- .../unit/base/test_fallback.py | 130 +++++++++++++++++ .../dl_settings_tests/unit/base/test_typed.py | 58 +++++--- lib/dl_settings/pyproject.toml | 2 + tools/e2e/Taskfile.yaml | 2 +- 15 files changed, 417 insertions(+), 93 deletions(-) create mode 100644 lib/dl_settings/dl_settings/base/fallback.py create mode 100644 lib/dl_settings/dl_settings_tests/unit/base/test_fallback.py diff --git a/app/dl_control_api/dl_control_api/app.py b/app/dl_control_api/dl_control_api/app.py index 11bea4876..9f3e96a32 100644 --- a/app/dl_control_api/dl_control_api/app.py +++ b/app/dl_control_api/dl_control_api/app.py @@ -11,6 +11,7 @@ from dl_api_lib.app_settings import ( ControlApiAppSettingsOS, ControlApiAppTestingsSettings, + DeprecatedControlApiAppSettingsOS, ) from dl_api_lib.loader import ( ApiLibraryConfig, @@ -53,7 +54,8 @@ def create_app( def create_uwsgi_app() -> flask.Flask: preload_api_lib() - settings = load_settings_from_env_with_fallback(ControlApiAppSettingsOS) + deprecated_settings = load_settings_from_env_with_fallback(DeprecatedControlApiAppSettingsOS) + settings = ControlApiAppSettingsOS(fallback=deprecated_settings) load_api_lib( ApiLibraryConfig( api_connector_ep_names=settings.BI_API_CONNECTOR_WHITELIST, diff --git a/app/dl_control_api/dl_control_api/app_factory.py b/app/dl_control_api/dl_control_api/app_factory.py index 1695aa6eb..ede8091ef 100644 --- a/app/dl_control_api/dl_control_api/app_factory.py +++ b/app/dl_control_api/dl_control_api/app_factory.py @@ -18,6 +18,7 @@ from dl_api_lib.app_settings import ( ControlApiAppSettingsOS, ControlApiAppTestingsSettings, + ZitadelAuthSettingsOS, ) from dl_api_lib.connector_availability.base import ConnectorAvailabilityConfig from dl_cache_engine.primitives import CacheTTLConfig @@ -38,7 +39,9 @@ class AuthSetupResult: us_auth_mode: USAuthMode -class StandaloneControlApiSRFactoryBuilder(SRFactoryBuilder[ControlApiAppSettingsOS]): +class StandaloneControlApiSRFactoryBuilder( + SRFactoryBuilder[ControlApiAppSettingsOS] # type: ignore # ControlApiAppSettingsOS is not subtype of AppSettings due to migration to new settings +): def _get_required_services(self, settings: ControlApiAppSettingsOS) -> set[RequiredService]: return set() @@ -69,7 +72,8 @@ def _get_connector_availability(self, settings: ControlApiAppSettingsOS) -> Opti class StandaloneControlApiAppFactory( - ControlApiAppFactory[ControlApiAppSettingsOS], StandaloneControlApiSRFactoryBuilder + ControlApiAppFactory[ControlApiAppSettingsOS], # type: ignore # ControlApiAppSettingsOS is not subtype of AppSettings + StandaloneControlApiSRFactoryBuilder, ): def set_up_environment( self, @@ -87,13 +91,13 @@ def _setup_auth_middleware( ) -> AuthSetupResult: self._settings: ControlApiAppSettingsOS - if self._settings.AUTH is None or self._settings.AUTH.TYPE == "NONE": + if self._settings.AUTH is None or self._settings.AUTH.type == "NONE": return self._setup_auth_middleware_none(app=app, testing_app_settings=testing_app_settings) - if self._settings.AUTH.TYPE == "ZITADEL": + if self._settings.AUTH.type == "ZITADEL": return self._setup_auth_middleware_zitadel(app=app) - raise ValueError(f"Unknown auth type: {self._settings.AUTH.TYPE}") + raise ValueError(f"Unknown auth type: {self._settings.AUTH.type}") def _setup_auth_middleware_none( self, @@ -117,7 +121,7 @@ def _setup_auth_middleware_zitadel( self, app: flask.Flask, ) -> AuthSetupResult: - assert self._settings.AUTH is not None + assert isinstance(self._settings.AUTH, ZitadelAuthSettingsOS) import httpx diff --git a/app/dl_data_api/dl_data_api/app.py b/app/dl_data_api/dl_data_api/app.py index a718854de..32aecedeb 100644 --- a/app/dl_data_api/dl_data_api/app.py +++ b/app/dl_data_api/dl_data_api/app.py @@ -5,7 +5,10 @@ from aiohttp import web -from dl_api_lib.app_settings import DataApiAppSettingsOS +from dl_api_lib.app_settings import ( + DataApiAppSettingsOS, + DeprecatedDataApiAppSettingsOS, +) from dl_api_lib.loader import ( ApiLibraryConfig, load_api_lib, @@ -46,7 +49,8 @@ def create_app( async def create_gunicorn_app(start_selfcheck: bool = True) -> web.Application: preload_api_lib() - settings = load_settings_from_env_with_fallback(DataApiAppSettingsOS) + deprecated_settings = load_settings_from_env_with_fallback(DeprecatedDataApiAppSettingsOS) + settings = DataApiAppSettingsOS(fallback=deprecated_settings) load_api_lib( ApiLibraryConfig( api_connector_ep_names=settings.BI_API_CONNECTOR_WHITELIST, diff --git a/app/dl_data_api/dl_data_api/app_factory.py b/app/dl_data_api/dl_data_api/app_factory.py index c6a31d207..afc33c1ad 100644 --- a/app/dl_data_api/dl_data_api/app_factory.py +++ b/app/dl_data_api/dl_data_api/app_factory.py @@ -16,6 +16,7 @@ from dl_api_lib.app_settings import ( AppSettings, DataApiAppSettingsOS, + ZitadelAuthSettingsOS, ) from dl_api_lib.connector_availability.base import ConnectorAvailabilityConfig from dl_cache_engine.primitives import CacheTTLConfig @@ -73,7 +74,10 @@ def _get_connector_availability(self, settings: AppSettings) -> Optional[Connect return None -class StandaloneDataApiAppFactory(DataApiAppFactory[DataApiAppSettingsOS], StandaloneDataApiSRFactoryBuilder): +class StandaloneDataApiAppFactory( + DataApiAppFactory[DataApiAppSettingsOS], # type: ignore # DataApiAppSettingsOS is not subtype of AppSettings due to migration to new settings + StandaloneDataApiSRFactoryBuilder, +): @property def _is_public(self) -> bool: return False @@ -141,10 +145,10 @@ def set_up_environment( return result def _get_auth_middleware(self) -> Middleware: - if self._settings.AUTH is None or self._settings.AUTH.TYPE == "NONE": + if self._settings.AUTH is None or self._settings.AUTH.type == "NONE": return self._get_auth_middleware_none() - if self._settings.AUTH.TYPE == "ZITADEL": + if self._settings.AUTH.type == "ZITADEL": return self._get_auth_middleware_zitadel( ca_data=get_multiple_root_certificates( self._settings.CA_FILE_PATH, @@ -152,7 +156,7 @@ def _get_auth_middleware(self) -> Middleware: ), ) - raise ValueError(f"Unknown auth type: {self._settings.AUTH.TYPE}") + raise ValueError(f"Unknown auth type: {self._settings.AUTH.type}") def _get_auth_middleware_none( self, @@ -167,7 +171,7 @@ def _get_auth_middleware_zitadel( ca_data: bytes, ) -> Middleware: self._settings: DataApiAppSettingsOS - assert self._settings.AUTH is not None + assert isinstance(self._settings.AUTH, ZitadelAuthSettingsOS) import httpx diff --git a/lib/dl_api_lib/dl_api_lib/app_settings.py b/lib/dl_api_lib/dl_api_lib/app_settings.py index 629793f37..f2331da62 100644 --- a/lib/dl_api_lib/dl_api_lib/app_settings.py +++ b/lib/dl_api_lib/dl_api_lib/app_settings.py @@ -8,6 +8,7 @@ ) import attr +import pydantic from dl_api_commons.base_models import TenantDef from dl_api_lib.connector_availability.base import ConnectorAvailabilityConfig @@ -34,6 +35,7 @@ from dl_core.components.ids import FieldIdGeneratorType from dl_formula.parser.factory import ParserType from dl_pivot_pandas.pandas.constants import PIVOT_ENGINE_TYPE_PANDAS +import dl_settings @attr.s(frozen=True) @@ -304,44 +306,60 @@ class ControlApiAppTestingsSettings: fake_tenant: Optional[TenantDef] = attr.ib(default=None) -# TODO: move to dl_api_lib_os @attr.s(frozen=True) -class AuthSettingsOS(SettingsBase): - TYPE: str = s_attrib("TYPE") # type: ignore - BASE_URL: str = s_attrib("BASE_URL") # type: ignore - PROJECT_ID: str = s_attrib("PROJECT_ID") # type: ignore - CLIENT_ID: str = s_attrib("CLIENT_ID") # type: ignore - CLIENT_SECRET: str = s_attrib("CLIENT_SECRET", sensitive=True) # type: ignore - APP_CLIENT_ID: str = s_attrib("APP_CLIENT_ID") # type: ignore - APP_CLIENT_SECRET: str = s_attrib("APP_CLIENT_SECRET", sensitive=True) # type: ignore +class DeprecatedControlApiAppSettingsOS(ControlApiAppSettings): + ... @attr.s(frozen=True) -class AppSettingsOS(AppSettings): - AUTH: typing.Optional[AuthSettingsOS] = s_attrib( # type: ignore - "AUTH", - fallback_factory=( - lambda cfg: AuthSettingsOS( # type: ignore - TYPE=cfg.AUTH_TYPE, - BASE_URL=cfg.AUTH_BASE_URL, - PROJECT_ID=cfg.AUTH_PROJECT_ID, - CLIENT_ID=cfg.AUTH_CLIENT_ID, - CLIENT_SECRET=cfg.AUTH_CLIENT_SECRET, - APP_CLIENT_ID=cfg.AUTH_APP_CLIENT_ID, - APP_CLIENT_SECRET=cfg.AUTH_APP_CLIENT_SECRET, - ) - if is_setting_applicable(cfg, "AUTH_TYPE") - else None - ), - missing=None, - ) +class DeprecatedDataApiAppSettingsOS(DataApiAppSettings): + ... -@attr.s(frozen=True) -class ControlApiAppSettingsOS(AppSettingsOS, ControlApiAppSettings): +class BaseAuthSettingsOS(dl_settings.TypedBaseSettings): ... -@attr.s(frozen=True) -class DataApiAppSettingsOS(AppSettingsOS, DataApiAppSettings): +class NullAuthSettingsOS(BaseAuthSettingsOS): + ... + + +BaseAuthSettingsOS.register("NONE", NullAuthSettingsOS) + + +class ZitadelAuthSettingsOS(BaseAuthSettingsOS): + BASE_URL: str + PROJECT_ID: str + CLIENT_ID: str + CLIENT_SECRET: str = pydantic.Field(repr=False) + APP_CLIENT_ID: str + APP_CLIENT_SECRET: str = pydantic.Field(repr=False) + + +BaseAuthSettingsOS.register("ZITADEL", ZitadelAuthSettingsOS) + + +class AppSettingsOS( + dl_settings.WithFallbackGetAttr, + dl_settings.WithFallbackEnvSource, + dl_settings.BaseRootSettings, +): + AUTH: typing.Optional[dl_settings.TypedAnnotation[BaseAuthSettingsOS]] = None + + fallback_env_keys = { + "AUTH__TYPE": "AUTH_TYPE", + "AUTH__BASE_URL": "AUTH_BASE_URL", + "AUTH__PROJECT_ID": "AUTH_PROJECT_ID", + "AUTH__CLIENT_ID": "AUTH_CLIENT_ID", + "AUTH__CLIENT_SECRET": "AUTH_CLIENT_SECRET", + "AUTH__APP_CLIENT_ID": "AUTH_APP_CLIENT_ID", + "AUTH__APP_CLIENT_SECRET": "AUTH_APP_CLIENT_SECRET", + } + + +class ControlApiAppSettingsOS(AppSettingsOS): + ... + + +class DataApiAppSettingsOS(AppSettingsOS): ... diff --git a/lib/dl_api_lib/pyproject.toml b/lib/dl_api_lib/pyproject.toml index 16c5aff5d..1c6052ea8 100644 --- a/lib/dl_api_lib/pyproject.toml +++ b/lib/dl_api_lib/pyproject.toml @@ -40,6 +40,7 @@ dynamic-enum = {path = "../dynamic_enum"} typing-extensions = ">=4.9.0" dl-rls = {path = "../dl_rls"} dl-type-transformer = {path = "../dl_type_transformer"} +dl-settings = {path = "../dl_settings"} [tool.poetry.group.tests.dependencies] pytest = ">=7.2.2" diff --git a/lib/dl_auth_api_lib/dl_auth_api_lib/settings.py b/lib/dl_auth_api_lib/dl_auth_api_lib/settings.py index 3213bd8eb..947b1774d 100644 --- a/lib/dl_auth_api_lib/dl_auth_api_lib/settings.py +++ b/lib/dl_auth_api_lib/dl_auth_api_lib/settings.py @@ -3,7 +3,7 @@ import dl_settings -class BaseOAuthClient(dl_settings.TypedBaseModel): +class BaseOAuthClient(dl_settings.TypedBaseSettings): type: str = pydantic.Field(alias="auth_type") conn_type: str diff --git a/lib/dl_settings/dl_settings/__init__.py b/lib/dl_settings/dl_settings/__init__.py index c1fe44436..98bc24e8c 100644 --- a/lib/dl_settings/dl_settings/__init__.py +++ b/lib/dl_settings/dl_settings/__init__.py @@ -2,17 +2,21 @@ BaseRootSettings, BaseSettings, TypedAnnotation, - TypedBaseModel, + TypedBaseSettings, TypedDictAnnotation, TypedListAnnotation, + WithFallbackEnvSource, + WithFallbackGetAttr, ) __all__ = [ "BaseSettings", "BaseRootSettings", - "TypedBaseModel", + "TypedBaseSettings", "TypedAnnotation", "TypedListAnnotation", "TypedDictAnnotation", + "WithFallbackGetAttr", + "WithFallbackEnvSource", ] diff --git a/lib/dl_settings/dl_settings/base/__init__.py b/lib/dl_settings/dl_settings/base/__init__.py index 6bd60046e..5ca1711e2 100644 --- a/lib/dl_settings/dl_settings/base/__init__.py +++ b/lib/dl_settings/dl_settings/base/__init__.py @@ -1,10 +1,14 @@ +from .fallback import ( + WithFallbackEnvSource, + WithFallbackGetAttr, +) from .settings import ( BaseRootSettings, BaseSettings, ) from .typed import ( TypedAnnotation, - TypedBaseModel, + TypedBaseSettings, TypedDictAnnotation, TypedListAnnotation, ) @@ -13,8 +17,10 @@ __all__ = [ "BaseSettings", "BaseRootSettings", - "TypedBaseModel", + "TypedBaseSettings", "TypedAnnotation", "TypedListAnnotation", "TypedDictAnnotation", + "WithFallbackGetAttr", + "WithFallbackEnvSource", ] diff --git a/lib/dl_settings/dl_settings/base/fallback.py b/lib/dl_settings/dl_settings/base/fallback.py new file mode 100644 index 000000000..2b8f10206 --- /dev/null +++ b/lib/dl_settings/dl_settings/base/fallback.py @@ -0,0 +1,136 @@ +import logging +import typing +import warnings + +import pydantic_settings + +import dl_settings.base.settings as base_settings + + +LOGGER = logging.getLogger(__name__) + + +class WithFallbackGetAttr(base_settings.BaseSettings): + """ + Mixin to add fallback to deprecated settings from dl-configs + """ + + fallback: typing.Any = None + + def __getattr__(self, item: str) -> typing.Any: + try: + return super().__getattr__(item) # type: ignore # BaseSettings definitely has __getattr__, but mypy doesn't know about it + except AttributeError: + pass + + if self.fallback is None: + message = f"Setting '{item}' is not found in the settings and no fallback is provided" + LOGGER.warning(message) + raise AttributeError(message) + + LOGGER.warning(f"Setting '{item}' is not found in the settings, trying to fallback") + + try: + return getattr(self.fallback, item) + except AttributeError: + LOGGER.warning(f"Setting '{item}' is not found in the fallback using getattr") + + if item != item.upper(): + try: + return getattr(self.fallback, item.upper()) + except AttributeError: + LOGGER.exception(f"Setting '{item.upper()}' is not found in the fallback using getattr") + + if item != item.lower(): + try: + return getattr(self.fallback, item.lower()) + except AttributeError: + LOGGER.exception(f"Setting '{item.lower()}' is not found in the fallback using getattr") + + raise AttributeError(f"Setting '{item}' is not found in the settings and fallback") + + +class FallbackEnvSettingsSource(pydantic_settings.EnvSettingsSource): + def __init__( + self, + env_keys: dict[str, str], + settings_cls: type[pydantic_settings.BaseSettings], + case_sensitive: bool | None = None, + env_prefix: str | None = None, + env_nested_delimiter: str | None = None, + env_ignore_empty: bool | None = None, + env_parse_none_str: str | None = None, + env_parse_enums: bool | None = None, + ) -> None: + self._fallback_env_keys = env_keys + super().__init__( + settings_cls=settings_cls, + case_sensitive=case_sensitive, + env_prefix=env_prefix, + env_nested_delimiter=env_nested_delimiter, + env_ignore_empty=env_ignore_empty, + env_parse_none_str=env_parse_none_str, + env_parse_enums=env_parse_enums, + ) + + def _load_env_vars(self) -> typing.Mapping[str, str | None]: + result = dict(super()._load_env_vars()) + + for original_key, original_fallback_key in self._fallback_env_keys.items(): + key = original_key if self.case_sensitive else original_key.lower() + fallback_key = original_fallback_key if self.case_sensitive else original_fallback_key.lower() + + if fallback_key in result and key not in result: + warnings.warn( + message=f"Deprecated env var found: `{original_fallback_key}`. Use `{original_key}` instead", + category=DeprecationWarning, + stacklevel=1, + ) + result[key] = result[fallback_key] + + return result + + +class WithFallbackEnvSource(base_settings.BaseRootSettings): + """ + Mixin to add fallback to deprecated env vars. Should be redefined in the child class. + fallback_env_keys should be a dict where: + - keys: setting env var names + - values: fallback env var names + """ + + fallback_env_keys: typing.ClassVar[dict[str, str]] = {} + + @classmethod + def settings_customise_sources( + cls, + settings_cls: typing.Type[pydantic_settings.BaseSettings], + init_settings: pydantic_settings.PydanticBaseSettingsSource, + env_settings: pydantic_settings.PydanticBaseSettingsSource, + dotenv_settings: pydantic_settings.PydanticBaseSettingsSource, + file_secret_settings: pydantic_settings.PydanticBaseSettingsSource, + ) -> tuple[pydantic_settings.PydanticBaseSettingsSource, ...]: + assert isinstance(env_settings, pydantic_settings.EnvSettingsSource) + + return super().settings_customise_sources( + settings_cls=settings_cls, + init_settings=init_settings, + env_settings=FallbackEnvSettingsSource( + settings_cls=env_settings.settings_cls, + case_sensitive=env_settings.case_sensitive, + env_prefix=env_settings.env_prefix, + env_nested_delimiter=env_settings.env_nested_delimiter, + env_ignore_empty=env_settings.env_ignore_empty, + env_parse_none_str=env_settings.env_parse_none_str, + env_parse_enums=env_settings.env_parse_enums, + env_keys=cls.fallback_env_keys, + ), + dotenv_settings=dotenv_settings, + file_secret_settings=file_secret_settings, + ) + + +__all__ = [ + "WithFallbackGetAttr", + "WithFallbackEnvSource", +] diff --git a/lib/dl_settings/dl_settings/base/typed.py b/lib/dl_settings/dl_settings/base/typed.py index f7466a559..7bbaf507c 100644 --- a/lib/dl_settings/dl_settings/base/typed.py +++ b/lib/dl_settings/dl_settings/base/typed.py @@ -12,17 +12,8 @@ def __init__(cls, name: str, bases: tuple[type, ...], attrs: dict[str, typing.An cls._classes: dict[str, type[base_settings.BaseSettings]] = {} -class TypedBaseModel(base_settings.BaseSettings, metaclass=TypedMeta): - type: str = pydantic.Field(alias="type") - - @classmethod - def __get_type_field_name(cls) -> str: - alias = cls.model_fields["type"].alias - - if alias is None: - raise ValueError("Field 'type' must have alias") - - return alias +class TypedBaseSettings(base_settings.BaseSettings, metaclass=TypedMeta): + type: str @classmethod def register(cls, name: str, class_: typing.Type) -> None: @@ -42,7 +33,11 @@ def factory(cls, data: typing.Any) -> base_settings.BaseSettings: if not isinstance(data, dict): raise ValueError("Data must be dict") - class_name = data[cls.__get_type_field_name()] + type_key = cls.model_fields["type"].alias or "type" + if type_key not in data: + raise ValueError(f"Data must contain '{type_key}' key") + + class_name = data[type_key] if class_name not in cls._classes: raise ValueError(f"Unknown type: {class_name}") @@ -62,38 +57,34 @@ def dict_factory(cls, data: dict[str, typing.Any]) -> typing.Dict[str, base_sett if not isinstance(data, dict): raise ValueError("Data must be mapping for dict factory") - import logging - - logging.error(data) - return {key: cls.factory(value) for key, value in data.items()} -TypedBaseModelT = typing.TypeVar("TypedBaseModelT", bound=TypedBaseModel) +TypedBaseSettingsT = typing.TypeVar("TypedBaseSettingsT", bound=TypedBaseSettings) if typing.TYPE_CHECKING: - TypedAnnotation = typing.Annotated[TypedBaseModelT, ...] - TypedListAnnotation = typing.Annotated[list[TypedBaseModelT], ...] - TypedDictAnnotation = typing.Annotated[dict[str, TypedBaseModelT], ...] + TypedAnnotation = typing.Annotated[TypedBaseSettingsT, ...] + TypedListAnnotation = typing.Annotated[list[TypedBaseSettingsT], ...] + TypedDictAnnotation = typing.Annotated[dict[str, TypedBaseSettingsT], ...] else: class TypedAnnotation: - def __class_getitem__(cls, base_class: TypedBaseModelT) -> typing.Any: + def __class_getitem__(cls, base_class: TypedBaseSettingsT) -> typing.Any: return typing.Annotated[ pydantic.SerializeAsAny[base_class], pydantic.BeforeValidator(base_class.factory), ] class TypedListAnnotation: - def __class_getitem__(cls, base_class: TypedBaseModelT) -> typing.Any: + def __class_getitem__(cls, base_class: TypedBaseSettingsT) -> typing.Any: return typing.Annotated[ list[pydantic.SerializeAsAny[base_class]], pydantic.BeforeValidator(base_class.list_factory), ] class TypedDictAnnotation: - def __class_getitem__(cls, base_class: TypedBaseModelT) -> typing.Any: + def __class_getitem__(cls, base_class: TypedBaseSettingsT) -> typing.Any: return typing.Annotated[ dict[str, pydantic.SerializeAsAny[base_class]], pydantic.BeforeValidator(base_class.dict_factory), @@ -101,7 +92,7 @@ def __class_getitem__(cls, base_class: TypedBaseModelT) -> typing.Any: __all__ = [ - "TypedBaseModel", + "TypedBaseSettings", "TypedAnnotation", "TypedListAnnotation", "TypedDictAnnotation", diff --git a/lib/dl_settings/dl_settings_tests/unit/base/test_fallback.py b/lib/dl_settings/dl_settings_tests/unit/base/test_fallback.py new file mode 100644 index 000000000..d546bd59f --- /dev/null +++ b/lib/dl_settings/dl_settings_tests/unit/base/test_fallback.py @@ -0,0 +1,130 @@ +import warnings + +import attr +import pytest + +import dl_settings + + +def test_no_fallback_raises() -> None: + class Settings(dl_settings.BaseRootSettings): + ... + + settings = Settings() + + with pytest.raises(AttributeError): + assert settings.test # type: ignore + + +def test_default() -> None: + class Settings(dl_settings.WithFallbackGetAttr, dl_settings.BaseRootSettings): + test: str + + settings = Settings(test="value") + + assert settings.test == "value" + + +def test_no_fallback_map() -> None: + class Settings(dl_settings.WithFallbackGetAttr, dl_settings.BaseRootSettings): + ... + + settings = Settings() + + with pytest.raises(AttributeError): + assert settings.test + + +def test_fallback_getattr() -> None: + class Settings(dl_settings.WithFallbackGetAttr, dl_settings.BaseRootSettings): + ... + + @attr.s(auto_attribs=True) + class Fallback: + test: str + + fallback = Fallback(test="test") + settings = Settings(fallback=fallback) + + assert settings.test == "test" + + +def test_fallback_getattr_upper() -> None: + class Settings(dl_settings.WithFallbackGetAttr, dl_settings.BaseRootSettings): + ... + + @attr.s(auto_attribs=True) + class Fallback: + TEST: str + + fallback = Fallback(TEST="test") + settings = Settings(fallback=fallback) + + assert settings.test == "test" + + +def test_fallback_getattr_lower() -> None: + class Settings(dl_settings.WithFallbackGetAttr, dl_settings.BaseRootSettings): + ... + + @attr.s(auto_attribs=True) + class Fallback: + test: str + + fallback = Fallback(test="test") + settings = Settings(fallback=fallback) + + assert settings.TEST == "test" + + +@pytest.mark.parametrize( + "env_key, fallback_key", + [ + ("TEST_KEY", "ANOTHER_KEY"), + ("TEST_KEY", "another_key"), + ("test_key", "ANOTHER_KEY"), + ("test_key", "another_key"), + ], +) +def test_fallback_env_source( + env_key: str, + fallback_key: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + class Settings(dl_settings.WithFallbackEnvSource, dl_settings.BaseRootSettings): + TEST_KEY: str = NotImplemented + + fallback_env_keys = { + env_key: fallback_key, + } + + monkeypatch.setenv(fallback_key, "value") + + with warnings.catch_warnings(record=True) as w: + settings = Settings() + assert len(w) == 1 + assert issubclass(w[0].category, DeprecationWarning) + assert "Deprecated env var found" in str(w[0].message) + + assert settings.TEST_KEY == "value" + + +def test_fallback_env_source_nested( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class Nested(dl_settings.BaseSettings): + test: str = NotImplemented + + class Settings(dl_settings.WithFallbackEnvSource, dl_settings.BaseRootSettings): + nested: Nested = NotImplemented + + fallback_env_keys = { + "NESTED__TEST": "NESTED_TEST", + } + + monkeypatch.setenv("NESTED_TEST", "value") + + with warnings.catch_warnings(record=True): + settings = Settings() + + assert settings.nested.test == "value" diff --git a/lib/dl_settings/dl_settings_tests/unit/base/test_typed.py b/lib/dl_settings/dl_settings_tests/unit/base/test_typed.py index 3f9168e28..69094f411 100644 --- a/lib/dl_settings/dl_settings_tests/unit/base/test_typed.py +++ b/lib/dl_settings/dl_settings_tests/unit/base/test_typed.py @@ -7,7 +7,7 @@ def test_default_settings() -> None: - class Base(dl_settings.TypedBaseModel): + class Base(dl_settings.TypedBaseSettings): ... class Child(Base): @@ -24,7 +24,7 @@ class Child2(Base): def test_type_field_name_alias() -> None: - class Base(dl_settings.TypedBaseModel): + class Base(dl_settings.TypedBaseSettings): type: str = pydantic.Field(alias="test_type_field_name") class Child(Base): @@ -36,7 +36,7 @@ class Child(Base): def test_already_deseialized() -> None: - class Base(dl_settings.TypedBaseModel): + class Base(dl_settings.TypedBaseSettings): ... class Child(Base): @@ -49,7 +49,7 @@ class Child(Base): def test_not_a_dict_data() -> None: - class Base(dl_settings.TypedBaseModel): + class Base(dl_settings.TypedBaseSettings): ... class Child(Base): @@ -65,7 +65,7 @@ class Root(dl_settings.BaseSettings): def test_already_registered() -> None: - class Base(dl_settings.TypedBaseModel): + class Base(dl_settings.TypedBaseSettings): ... class Child(Base): @@ -77,10 +77,10 @@ class Child(Base): def test_not_subclass() -> None: - class Base(dl_settings.TypedBaseModel): + class Base(dl_settings.TypedBaseSettings): ... - class Base2(dl_settings.TypedBaseModel): + class Base2(dl_settings.TypedBaseSettings): ... class Child(Base2): @@ -91,7 +91,7 @@ class Child(Base2): def test_unknown_type() -> None: - class Base(dl_settings.TypedBaseModel): + class Base(dl_settings.TypedBaseSettings): ... with pytest.raises(ValueError): @@ -99,10 +99,10 @@ class Base(dl_settings.TypedBaseModel): def test_multiple_bases() -> None: - class Base(dl_settings.TypedBaseModel): + class Base(dl_settings.TypedBaseSettings): ... - class Base2(dl_settings.TypedBaseModel): + class Base2(dl_settings.TypedBaseSettings): ... class Child(Base): @@ -119,7 +119,7 @@ class Child2(Base2): def test_list_factory_default() -> None: - class Base(dl_settings.TypedBaseModel): + class Base(dl_settings.TypedBaseSettings): ... class Child(Base): @@ -133,7 +133,7 @@ class Child(Base): def test_list_factory_not_sequence() -> None: - class Base(dl_settings.TypedBaseModel): + class Base(dl_settings.TypedBaseSettings): ... class Child(Base): @@ -146,7 +146,7 @@ class Child(Base): def test_dict_factory_default() -> None: - class Base(dl_settings.TypedBaseModel): + class Base(dl_settings.TypedBaseSettings): ... class Child(Base): @@ -160,7 +160,7 @@ class Child(Base): def test_dict_factory_not_dict() -> None: - class Base(dl_settings.TypedBaseModel): + class Base(dl_settings.TypedBaseSettings): ... class Child(Base): @@ -173,7 +173,7 @@ class Child(Base): def test_annotation() -> None: - class Base(dl_settings.TypedBaseModel): + class Base(dl_settings.TypedBaseSettings): ... class Child(Base): @@ -189,8 +189,30 @@ class Root(dl_settings.BaseSettings): assert isinstance(root.child, Child) +def test_optional_annotation() -> None: + class Base(dl_settings.TypedBaseSettings): + ... + + class Child(Base): + ... + + Base.register("child", Child) + + class Root(dl_settings.BaseSettings): + child: typing.Optional[dl_settings.TypedAnnotation[Base]] = None + + root = Root.model_validate({"child": {"type": "child"}}) + assert isinstance(root.child, Child) + + root = Root.model_validate({"child": None}) + assert root.child is None + + root = Root.model_validate({}) + assert root.child is None + + def test_list_annotation() -> None: - class Base(dl_settings.TypedBaseModel): + class Base(dl_settings.TypedBaseSettings): ... class Child(Base): @@ -207,7 +229,7 @@ class Root(dl_settings.BaseSettings): def test_dict_annotation() -> None: - class Base(dl_settings.TypedBaseModel): + class Base(dl_settings.TypedBaseSettings): ... class Child(Base): @@ -226,7 +248,7 @@ class Root(dl_settings.BaseSettings): def test_dict_annotation_with_env( monkeypatch: pytest.MonkeyPatch, ) -> None: - class Base(dl_settings.TypedBaseModel): + class Base(dl_settings.TypedBaseSettings): ... class Child(Base): diff --git a/lib/dl_settings/pyproject.toml b/lib/dl_settings/pyproject.toml index 129900176..3b101f2a6 100644 --- a/lib/dl_settings/pyproject.toml +++ b/lib/dl_settings/pyproject.toml @@ -25,6 +25,8 @@ requires = [ ] [tool.pytest.ini_options] +# log_cli = true +# log_level = "DEBUG" minversion = "6.0" addopts = "-ra" testpaths = [] diff --git a/tools/e2e/Taskfile.yaml b/tools/e2e/Taskfile.yaml index 5d97a4982..6513047c7 100644 --- a/tools/e2e/Taskfile.yaml +++ b/tools/e2e/Taskfile.yaml @@ -36,7 +36,7 @@ tasks: dir: "{{.DATALENS_UI_FOLDER}}" desc: Checkout latest datalens-ui cmds: - - git fetch --all + - git fetch - git checkout main - git pull