BI-4981: settings loading for RQE apps (#4)

lib/dl_core/dl_core/connection_executors/remote_query_executor/app_async.py

@@ -23,6 +23,7 @@
     jaeger_service_name_env_aware,
     use_jaeger_tracer,
 )
+from dl_configs.settings_loaders.loader_env import load_settings_from_env_with_fallback
 from dl_core.connection_executors.adapters.adapters_base import SyncDirectDBAdapter
 from dl_core.connection_executors.adapters.async_adapters_base import (
     AsyncDBAdapter,
@@ -47,10 +48,13 @@
 )
 from dl_core.connection_executors.remote_query_executor.crypto import get_hmac_hex_digest
 from dl_core.connection_executors.remote_query_executor.error_handler_rqe import RQEErrorHandler
+from dl_core.connection_executors.remote_query_executor.settings import RQESettings
 from dl_core.enums import RQEEventType
-from dl_core.loader import load_core_lib
+from dl_core.loader import (
+    CoreLibraryConfig,
+    load_core_lib,
+)
 from dl_core.logging_config import configure_logging
-from dl_core.utils import get_eqe_secret_key
 from dl_utils.aio import ContextVarExecutor
 
 
@@ -219,11 +223,7 @@ async def actual_middleware(request: web.Request, handler: Handler) -> web.Strea
     return actual_middleware
 
 
-def create_async_qe_app(hmac_key: Optional[bytes] = None) -> web.Application:
-    hmac_key = hmac_key or get_eqe_secret_key()
-    if not hmac_key:
-        raise Exception("No `hmac_key` set.")
-    assert isinstance(hmac_key, bytes)
+def create_async_qe_app(hmac_key: bytes) -> web.Application:
     req_id_service = RequestId(
         header_name=HEADER_REQUEST_ID,
         accept_logging_ctx=True,
@@ -260,7 +260,10 @@ def async_qe_main() -> None:
         use_jaeger_tracer=use_jaeger_tracer(),
         jaeger_service_name=jaeger_service_name_env_aware("bi-rqe-async"),
     )
-    load_core_lib()
+
+    settings = load_settings_from_env_with_fallback(RQESettings)
+    load_core_lib(core_lib_config=CoreLibraryConfig(core_connector_ep_names=settings.CORE_CONNECTOR_WHITELIST))
+
     try:
         parser = argparse.ArgumentParser(description="Process some integers.")
         parser.add_argument("--host", type=str)
@@ -270,4 +273,8 @@ def async_qe_main() -> None:
         LOGGER.exception("rqe-async args error: %r", err)
         sys.exit(-1)
         raise  # just-in-case
-    web.run_app(create_async_qe_app(), host=args.host, port=args.port, print=LOGGER.info)
+
+    hmac_key = settings.RQE_SECRET_KEY
+    if hmac_key is None:
+        raise Exception("No `hmac_key` set.")
+    web.run_app(create_async_qe_app(hmac_key.encode()), host=args.host, port=args.port, print=LOGGER.info)

lib/dl_core/dl_core/connection_executors/remote_query_executor/app_sync.py

@@ -27,6 +27,7 @@
     jaeger_service_name_env_aware,
     use_jaeger_tracer,
 )
+from dl_configs.settings_loaders.loader_env import load_settings_from_env_with_fallback
 from dl_core import profiling_middleware
 from dl_core.connection_executors.adapters.adapters_base import SyncDirectDBAdapter
 from dl_core.connection_executors.models.constants import HEADER_BODY_SIGNATURE
@@ -40,10 +41,13 @@
     SUPPORTED_ADAPTER_CLS,
 )
 from dl_core.connection_executors.remote_query_executor.crypto import get_hmac_hex_digest
+from dl_core.connection_executors.remote_query_executor.settings import RQESettings
 from dl_core.enums import RQEEventType
-from dl_core.loader import load_core_lib
+from dl_core.loader import (
+    CoreLibraryConfig,
+    load_core_lib,
+)
 from dl_core.logging_config import hook_configure_logging as _hook_configure_logging
-from dl_core.utils import get_eqe_secret_key
 
 
 if TYPE_CHECKING:
@@ -234,12 +238,13 @@ def _handle_exception(err: Exception) -> Tuple[flask.Response, int]:
         return flask.jsonify(ActionSerializer().serialize_exc(err)), 500
 
 
-def create_sync_app(hmac_key: Optional[bytes] = None) -> flask.Flask:
-    hmac_key = hmac_key or get_eqe_secret_key()
-    assert isinstance(hmac_key, bytes)
-    # Can't check `hmc_key` for nonemptiness here because this happens on import.
+def create_sync_app() -> flask.Flask:
+    settings = load_settings_from_env_with_fallback(RQESettings)
+    hmac_key = settings.RQE_SECRET_KEY
+    if hmac_key is None:
+        raise Exception("No `hmac_key` set.")
 
-    load_core_lib()
+    load_core_lib(core_lib_config=CoreLibraryConfig(core_connector_ep_names=settings.CORE_CONNECTOR_WHITELIST))
 
     app = flask.Flask(__name__)
     TracingMiddleware(
@@ -260,7 +265,7 @@ def create_sync_app(hmac_key: Optional[bytes] = None) -> flask.Flask:
     ).set_up(app)
     profiling_middleware.set_up(app, accept_outer_stages=True)
     BodySignatureValidator(
-        hmac_key=hmac_key,
+        hmac_key=hmac_key.encode(),
     ).set_up(app)
 
     app.add_url_rule("/ping", view_func=ping_view)

lib/dl_core/dl_core/connection_executors/remote_query_executor/settings.py

@@ -0,0 +1,16 @@
+from typing import Optional
+
+import attr
+
+from dl_configs.settings_loaders.meta_definition import s_attrib
+from dl_configs.utils import split_by_comma
+
+
+@attr.s(frozen=True)
+class RQESettings:
+    CORE_CONNECTOR_WHITELIST: Optional[list[str]] = s_attrib(  # type: ignore
+        "CORE_CONNECTOR_WHITELIST",
+        env_var_converter=lambda s: list(split_by_comma(s)),
+        missing=None,
+    )
+    RQE_SECRET_KEY: Optional[str] = s_attrib("RQE_SECRET_KEY", missing=None, sensitive=True)  # type: ignore

lib/dl_core/dl_core/utils.py

@@ -229,12 +229,6 @@ def fulfilled(cls, obj: _FUTURE_REF_TV) -> "FutureRef[_FUTURE_REF_TV]":
         return fr
 
 
-# TODO FIX: try to load in generic ways
-def get_eqe_secret_key() -> bytes:
-    effective_env = remap_env(os.environ)
-    return effective_env.get("RQE_SECRET_KEY", "").encode()
-
-
 def make_id() -> str:
     return shortuuid.uuid()