Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

BI-4981: settings loading for RQE apps #4

Merged
merged 4 commits into from
Oct 9, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
jaeger_service_name_env_aware,
use_jaeger_tracer,
)
from dl_configs.settings_loaders.loader_env import load_settings_from_env_with_fallback
from dl_core.connection_executors.adapters.adapters_base import SyncDirectDBAdapter
from dl_core.connection_executors.adapters.async_adapters_base import (
AsyncDBAdapter,
Expand All @@ -47,10 +48,13 @@
)
from dl_core.connection_executors.remote_query_executor.crypto import get_hmac_hex_digest
from dl_core.connection_executors.remote_query_executor.error_handler_rqe import RQEErrorHandler
from dl_core.connection_executors.remote_query_executor.settings import RQESettings
from dl_core.enums import RQEEventType
from dl_core.loader import load_core_lib
from dl_core.loader import (
CoreLibraryConfig,
load_core_lib,
)
from dl_core.logging_config import configure_logging
from dl_core.utils import get_eqe_secret_key
from dl_utils.aio import ContextVarExecutor


Expand Down Expand Up @@ -219,11 +223,7 @@ async def actual_middleware(request: web.Request, handler: Handler) -> web.Strea
return actual_middleware


def create_async_qe_app(hmac_key: Optional[bytes] = None) -> web.Application:
hmac_key = hmac_key or get_eqe_secret_key()
if not hmac_key:
raise Exception("No `hmac_key` set.")
assert isinstance(hmac_key, bytes)
def create_async_qe_app(hmac_key: bytes) -> web.Application:
req_id_service = RequestId(
header_name=HEADER_REQUEST_ID,
accept_logging_ctx=True,
Expand Down Expand Up @@ -260,7 +260,10 @@ def async_qe_main() -> None:
use_jaeger_tracer=use_jaeger_tracer(),
jaeger_service_name=jaeger_service_name_env_aware("bi-rqe-async"),
)
load_core_lib()

settings = load_settings_from_env_with_fallback(RQESettings)
load_core_lib(core_lib_config=CoreLibraryConfig(core_connector_ep_names=settings.CORE_CONNECTOR_WHITELIST))

try:
parser = argparse.ArgumentParser(description="Process some integers.")
parser.add_argument("--host", type=str)
Expand All @@ -270,4 +273,8 @@ def async_qe_main() -> None:
LOGGER.exception("rqe-async args error: %r", err)
sys.exit(-1)
raise # just-in-case
web.run_app(create_async_qe_app(), host=args.host, port=args.port, print=LOGGER.info)

hmac_key = settings.RQE_SECRET_KEY
if hmac_key is None:
raise Exception("No `hmac_key` set.")
web.run_app(create_async_qe_app(hmac_key.encode()), host=args.host, port=args.port, print=LOGGER.info)
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
jaeger_service_name_env_aware,
use_jaeger_tracer,
)
from dl_configs.settings_loaders.loader_env import load_settings_from_env_with_fallback
from dl_core import profiling_middleware
from dl_core.connection_executors.adapters.adapters_base import SyncDirectDBAdapter
from dl_core.connection_executors.models.constants import HEADER_BODY_SIGNATURE
Expand All @@ -40,10 +41,13 @@
SUPPORTED_ADAPTER_CLS,
)
from dl_core.connection_executors.remote_query_executor.crypto import get_hmac_hex_digest
from dl_core.connection_executors.remote_query_executor.settings import RQESettings
from dl_core.enums import RQEEventType
from dl_core.loader import load_core_lib
from dl_core.loader import (
CoreLibraryConfig,
load_core_lib,
)
from dl_core.logging_config import hook_configure_logging as _hook_configure_logging
from dl_core.utils import get_eqe_secret_key


if TYPE_CHECKING:
Expand Down Expand Up @@ -234,12 +238,13 @@ def _handle_exception(err: Exception) -> Tuple[flask.Response, int]:
return flask.jsonify(ActionSerializer().serialize_exc(err)), 500


def create_sync_app(hmac_key: Optional[bytes] = None) -> flask.Flask:
hmac_key = hmac_key or get_eqe_secret_key()
assert isinstance(hmac_key, bytes)
# Can't check `hmc_key` for nonemptiness here because this happens on import.
def create_sync_app() -> flask.Flask:
settings = load_settings_from_env_with_fallback(RQESettings)
hmac_key = settings.RQE_SECRET_KEY
if hmac_key is None:
raise Exception("No `hmac_key` set.")

load_core_lib()
load_core_lib(core_lib_config=CoreLibraryConfig(core_connector_ep_names=settings.CORE_CONNECTOR_WHITELIST))

app = flask.Flask(__name__)
TracingMiddleware(
Expand All @@ -260,7 +265,7 @@ def create_sync_app(hmac_key: Optional[bytes] = None) -> flask.Flask:
).set_up(app)
profiling_middleware.set_up(app, accept_outer_stages=True)
BodySignatureValidator(
hmac_key=hmac_key,
hmac_key=hmac_key.encode(),
).set_up(app)

app.add_url_rule("/ping", view_func=ping_view)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
from typing import Optional

import attr

from dl_configs.settings_loaders.meta_definition import s_attrib
from dl_configs.utils import split_by_comma


@attr.s(frozen=True)
class RQESettings:
CORE_CONNECTOR_WHITELIST: Optional[list[str]] = s_attrib( # type: ignore
"CORE_CONNECTOR_WHITELIST",
env_var_converter=lambda s: list(split_by_comma(s)),
missing=None,
)
RQE_SECRET_KEY: Optional[str] = s_attrib("RQE_SECRET_KEY", missing=None, sensitive=True) # type: ignore
6 changes: 0 additions & 6 deletions lib/dl_core/dl_core/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -229,12 +229,6 @@ def fulfilled(cls, obj: _FUTURE_REF_TV) -> "FutureRef[_FUTURE_REF_TV]":
return fr


# TODO FIX: try to load in generic ways
def get_eqe_secret_key() -> bytes:
effective_env = remap_env(os.environ)
return effective_env.get("RQE_SECRET_KEY", "").encode()


def make_id() -> str:
return shortuuid.uuid()

Expand Down
Loading