feat: BI-0 add new settings with fallbacks (#753)

app/dl_control_api/dl_control_api/app.py

@@ -11,6 +11,7 @@
 from dl_api_lib.app_settings import (
     ControlApiAppSettingsOS,
     ControlApiAppTestingsSettings,
+    DeprecatedControlApiAppSettingsOS,
 )
 from dl_api_lib.loader import (
     ApiLibraryConfig,
@@ -53,7 +54,8 @@ def create_app(
 
 def create_uwsgi_app() -> flask.Flask:
     preload_api_lib()
-    settings = load_settings_from_env_with_fallback(ControlApiAppSettingsOS)
+    deprecated_settings = load_settings_from_env_with_fallback(DeprecatedControlApiAppSettingsOS)
+    settings = ControlApiAppSettingsOS(fallback=deprecated_settings)
     load_api_lib(
         ApiLibraryConfig(
             api_connector_ep_names=settings.BI_API_CONNECTOR_WHITELIST,

app/dl_control_api/dl_control_api/app_factory.py

@@ -18,6 +18,7 @@
 from dl_api_lib.app_settings import (
     ControlApiAppSettingsOS,
     ControlApiAppTestingsSettings,
+    ZitadelAuthSettingsOS,
 )
 from dl_api_lib.connector_availability.base import ConnectorAvailabilityConfig
 from dl_cache_engine.primitives import CacheTTLConfig
@@ -38,7 +39,9 @@ class AuthSetupResult:
     us_auth_mode: USAuthMode
 
 
-class StandaloneControlApiSRFactoryBuilder(SRFactoryBuilder[ControlApiAppSettingsOS]):
+class StandaloneControlApiSRFactoryBuilder(
+    SRFactoryBuilder[ControlApiAppSettingsOS]  # type: ignore # ControlApiAppSettingsOS is not subtype of AppSettings due to migration to new settings
+):
     def _get_required_services(self, settings: ControlApiAppSettingsOS) -> set[RequiredService]:
         return set()
 
@@ -69,7 +72,8 @@ def _get_connector_availability(self, settings: ControlApiAppSettingsOS) -> Opti
 
 
 class StandaloneControlApiAppFactory(
-    ControlApiAppFactory[ControlApiAppSettingsOS], StandaloneControlApiSRFactoryBuilder
+    ControlApiAppFactory[ControlApiAppSettingsOS],  # type: ignore # ControlApiAppSettingsOS is not subtype of AppSettings
+    StandaloneControlApiSRFactoryBuilder,
 ):
     def set_up_environment(
         self,
@@ -87,13 +91,13 @@ def _setup_auth_middleware(
     ) -> AuthSetupResult:
         self._settings: ControlApiAppSettingsOS
 
-        if self._settings.AUTH is None or self._settings.AUTH.TYPE == "NONE":
+        if self._settings.AUTH is None or self._settings.AUTH.type == "NONE":
             return self._setup_auth_middleware_none(app=app, testing_app_settings=testing_app_settings)
 
-        if self._settings.AUTH.TYPE == "ZITADEL":
+        if self._settings.AUTH.type == "ZITADEL":
             return self._setup_auth_middleware_zitadel(app=app)
 
-        raise ValueError(f"Unknown auth type: {self._settings.AUTH.TYPE}")
+        raise ValueError(f"Unknown auth type: {self._settings.AUTH.type}")
 
     def _setup_auth_middleware_none(
         self,
@@ -117,7 +121,7 @@ def _setup_auth_middleware_zitadel(
         self,
         app: flask.Flask,
     ) -> AuthSetupResult:
-        assert self._settings.AUTH is not None
+        assert isinstance(self._settings.AUTH, ZitadelAuthSettingsOS)
 
         import httpx
 

app/dl_data_api/dl_data_api/app.py

@@ -5,7 +5,10 @@
 
 from aiohttp import web
 
-from dl_api_lib.app_settings import DataApiAppSettingsOS
+from dl_api_lib.app_settings import (
+    DataApiAppSettingsOS,
+    DeprecatedDataApiAppSettingsOS,
+)
 from dl_api_lib.loader import (
     ApiLibraryConfig,
     load_api_lib,
@@ -46,7 +49,8 @@ def create_app(
 
 async def create_gunicorn_app(start_selfcheck: bool = True) -> web.Application:
     preload_api_lib()
-    settings = load_settings_from_env_with_fallback(DataApiAppSettingsOS)
+    deprecated_settings = load_settings_from_env_with_fallback(DeprecatedDataApiAppSettingsOS)
+    settings = DataApiAppSettingsOS(fallback=deprecated_settings)
     load_api_lib(
         ApiLibraryConfig(
             api_connector_ep_names=settings.BI_API_CONNECTOR_WHITELIST,

app/dl_data_api/dl_data_api/app_factory.py

@@ -16,6 +16,7 @@
 from dl_api_lib.app_settings import (
     AppSettings,
     DataApiAppSettingsOS,
+    ZitadelAuthSettingsOS,
 )
 from dl_api_lib.connector_availability.base import ConnectorAvailabilityConfig
 from dl_cache_engine.primitives import CacheTTLConfig
@@ -73,7 +74,10 @@ def _get_connector_availability(self, settings: AppSettings) -> Optional[Connect
         return None
 
 
-class StandaloneDataApiAppFactory(DataApiAppFactory[DataApiAppSettingsOS], StandaloneDataApiSRFactoryBuilder):
+class StandaloneDataApiAppFactory(
+    DataApiAppFactory[DataApiAppSettingsOS],  # type: ignore # DataApiAppSettingsOS is not subtype of AppSettings due to migration to new settings
+    StandaloneDataApiSRFactoryBuilder,
+):
     @property
     def _is_public(self) -> bool:
         return False
@@ -141,18 +145,18 @@ def set_up_environment(
         return result
 
     def _get_auth_middleware(self) -> Middleware:
-        if self._settings.AUTH is None or self._settings.AUTH.TYPE == "NONE":
+        if self._settings.AUTH is None or self._settings.AUTH.type == "NONE":
             return self._get_auth_middleware_none()
 
-        if self._settings.AUTH.TYPE == "ZITADEL":
+        if self._settings.AUTH.type == "ZITADEL":
             return self._get_auth_middleware_zitadel(
                 ca_data=get_multiple_root_certificates(
                     self._settings.CA_FILE_PATH,
                     *self._settings.EXTRA_CA_FILE_PATHS,
                 ),
             )
 
-        raise ValueError(f"Unknown auth type: {self._settings.AUTH.TYPE}")
+        raise ValueError(f"Unknown auth type: {self._settings.AUTH.type}")
 
     def _get_auth_middleware_none(
         self,
@@ -167,7 +171,7 @@ def _get_auth_middleware_zitadel(
         ca_data: bytes,
     ) -> Middleware:
         self._settings: DataApiAppSettingsOS
-        assert self._settings.AUTH is not None
+        assert isinstance(self._settings.AUTH, ZitadelAuthSettingsOS)
 
         import httpx
 

lib/dl_api_lib/dl_api_lib/app_settings.py

@@ -8,6 +8,7 @@
 )
 
 import attr
+import pydantic
 
 from dl_api_commons.base_models import TenantDef
 from dl_api_lib.connector_availability.base import ConnectorAvailabilityConfig
@@ -34,6 +35,7 @@
 from dl_core.components.ids import FieldIdGeneratorType
 from dl_formula.parser.factory import ParserType
 from dl_pivot_pandas.pandas.constants import PIVOT_ENGINE_TYPE_PANDAS
+import dl_settings
 
 
 @attr.s(frozen=True)
@@ -304,44 +306,60 @@ class ControlApiAppTestingsSettings:
     fake_tenant: Optional[TenantDef] = attr.ib(default=None)
 
 
-# TODO: move to dl_api_lib_os
 @attr.s(frozen=True)
-class AuthSettingsOS(SettingsBase):
-    TYPE: str = s_attrib("TYPE")  # type: ignore
-    BASE_URL: str = s_attrib("BASE_URL")  # type: ignore
-    PROJECT_ID: str = s_attrib("PROJECT_ID")  # type: ignore
-    CLIENT_ID: str = s_attrib("CLIENT_ID")  # type: ignore
-    CLIENT_SECRET: str = s_attrib("CLIENT_SECRET", sensitive=True)  # type: ignore
-    APP_CLIENT_ID: str = s_attrib("APP_CLIENT_ID")  # type: ignore
-    APP_CLIENT_SECRET: str = s_attrib("APP_CLIENT_SECRET", sensitive=True)  # type: ignore
+class DeprecatedControlApiAppSettingsOS(ControlApiAppSettings):
+    ...
 
 
 @attr.s(frozen=True)
-class AppSettingsOS(AppSettings):
-    AUTH: typing.Optional[AuthSettingsOS] = s_attrib(  # type: ignore
-        "AUTH",
-        fallback_factory=(
-            lambda cfg: AuthSettingsOS(  # type: ignore
-                TYPE=cfg.AUTH_TYPE,
-                BASE_URL=cfg.AUTH_BASE_URL,
-                PROJECT_ID=cfg.AUTH_PROJECT_ID,
-                CLIENT_ID=cfg.AUTH_CLIENT_ID,
-                CLIENT_SECRET=cfg.AUTH_CLIENT_SECRET,
-                APP_CLIENT_ID=cfg.AUTH_APP_CLIENT_ID,
-                APP_CLIENT_SECRET=cfg.AUTH_APP_CLIENT_SECRET,
-            )
-            if is_setting_applicable(cfg, "AUTH_TYPE")
-            else None
-        ),
-        missing=None,
-    )
+class DeprecatedDataApiAppSettingsOS(DataApiAppSettings):
+    ...
 
 
-@attr.s(frozen=True)
-class ControlApiAppSettingsOS(AppSettingsOS, ControlApiAppSettings):
+class BaseAuthSettingsOS(dl_settings.TypedBaseSettings):
     ...
 
 
-@attr.s(frozen=True)
-class DataApiAppSettingsOS(AppSettingsOS, DataApiAppSettings):
+class NullAuthSettingsOS(BaseAuthSettingsOS):
+    ...
+
+
+BaseAuthSettingsOS.register("NONE", NullAuthSettingsOS)
+
+
+class ZitadelAuthSettingsOS(BaseAuthSettingsOS):
+    BASE_URL: str
+    PROJECT_ID: str
+    CLIENT_ID: str
+    CLIENT_SECRET: str = pydantic.Field(repr=False)
+    APP_CLIENT_ID: str
+    APP_CLIENT_SECRET: str = pydantic.Field(repr=False)
+
+
+BaseAuthSettingsOS.register("ZITADEL", ZitadelAuthSettingsOS)
+
+
+class AppSettingsOS(
+    dl_settings.WithFallbackGetAttr,
+    dl_settings.WithFallbackEnvSource,
+    dl_settings.BaseRootSettings,
+):
+    AUTH: typing.Optional[dl_settings.TypedAnnotation[BaseAuthSettingsOS]] = None
+
+    fallback_env_keys = {
+        "AUTH__TYPE": "AUTH_TYPE",
+        "AUTH__BASE_URL": "AUTH_BASE_URL",
+        "AUTH__PROJECT_ID": "AUTH_PROJECT_ID",
+        "AUTH__CLIENT_ID": "AUTH_CLIENT_ID",
+        "AUTH__CLIENT_SECRET": "AUTH_CLIENT_SECRET",
+        "AUTH__APP_CLIENT_ID": "AUTH_APP_CLIENT_ID",
+        "AUTH__APP_CLIENT_SECRET": "AUTH_APP_CLIENT_SECRET",
+    }
+
+
+class ControlApiAppSettingsOS(AppSettingsOS):
+    ...
+
+
+class DataApiAppSettingsOS(AppSettingsOS):
     ...

lib/dl_api_lib/pyproject.toml

@@ -40,6 +40,7 @@ dynamic-enum = {path = "../dynamic_enum"}
 typing-extensions = ">=4.9.0"
 dl-rls = {path = "../dl_rls"}
 dl-type-transformer = {path = "../dl_type_transformer"}
+dl-settings = {path = "../dl_settings"}
 
 [tool.poetry.group.tests.dependencies]
 pytest = ">=7.2.2"

lib/dl_auth_api_lib/dl_auth_api_lib/settings.py

@@ -3,7 +3,7 @@
 import dl_settings
 
 
-class BaseOAuthClient(dl_settings.TypedBaseModel):
+class BaseOAuthClient(dl_settings.TypedBaseSettings):
     type: str = pydantic.Field(alias="auth_type")
     conn_type: str
 

lib/dl_settings/dl_settings/__init__.py

@@ -2,17 +2,21 @@
     BaseRootSettings,
     BaseSettings,
     TypedAnnotation,
-    TypedBaseModel,
+    TypedBaseSettings,
     TypedDictAnnotation,
     TypedListAnnotation,
+    WithFallbackEnvSource,
+    WithFallbackGetAttr,
 )
 
 
 __all__ = [
     "BaseSettings",
     "BaseRootSettings",
-    "TypedBaseModel",
+    "TypedBaseSettings",
     "TypedAnnotation",
     "TypedListAnnotation",
     "TypedDictAnnotation",
+    "WithFallbackGetAttr",
+    "WithFallbackEnvSource",
 ]

lib/dl_settings/dl_settings/base/__init__.py

@@ -1,10 +1,14 @@
+from .fallback import (
+    WithFallbackEnvSource,
+    WithFallbackGetAttr,
+)
 from .settings import (
     BaseRootSettings,
     BaseSettings,
 )
 from .typed import (
     TypedAnnotation,
-    TypedBaseModel,
+    TypedBaseSettings,
     TypedDictAnnotation,
     TypedListAnnotation,
 )
@@ -13,8 +17,10 @@
 __all__ = [
     "BaseSettings",
     "BaseRootSettings",
-    "TypedBaseModel",
+    "TypedBaseSettings",
     "TypedAnnotation",
     "TypedListAnnotation",
     "TypedDictAnnotation",
+    "WithFallbackGetAttr",
+    "WithFallbackEnvSource",
 ]