From a12ec70dee57242b1fb4e99c31332f2453ea9425 Mon Sep 17 00:00:00 2001 From: Grigorii Statsenko Date: Fri, 12 Apr 2024 09:56:03 -0300 Subject: [PATCH] Moved some request logging tools to classes to enable their customization (#425) --- lib/dl_api_commons/dl_api_commons/__init__.py | 8 - .../aio/middlewares/request_id.py | 10 +- .../flask/middlewares/request_id.py | 10 +- lib/dl_api_commons/dl_api_commons/logging.py | 417 +++++++++--------- .../dl_api_commons/logging_sentry.py | 4 +- .../unit/aio/test_common_logging.py | 21 +- .../aio/middlewares/json_body_middleware.py | 4 +- lib/dl_api_lib/dl_api_lib/api_decorators.py | 4 +- lib/dl_core/dl_core/united_storage_client.py | 4 +- lib/dl_core/dl_core/utils.py | 7 +- 10 files changed, 237 insertions(+), 252 deletions(-) diff --git a/lib/dl_api_commons/dl_api_commons/__init__.py b/lib/dl_api_commons/dl_api_commons/__init__.py index fc7b5014b..c6f140292 100644 --- a/lib/dl_api_commons/dl_api_commons/__init__.py +++ b/lib/dl_api_commons/dl_api_commons/__init__.py @@ -1,10 +1,5 @@ from __future__ import annotations -from dl_api_commons.logging import ( - clean_secret_data_in_headers, - log_request_end, - log_request_start, -) from dl_api_commons.request_id import ( make_uuid_from_parts, request_id_generator, @@ -14,7 +9,4 @@ __all__ = ( "make_uuid_from_parts", "request_id_generator", - "log_request_start", - "log_request_end", - "clean_secret_data_in_headers", ) diff --git a/lib/dl_api_commons/dl_api_commons/aio/middlewares/request_id.py b/lib/dl_api_commons/dl_api_commons/aio/middlewares/request_id.py index c031d685a..ed9d6ae52 100644 --- a/lib/dl_api_commons/dl_api_commons/aio/middlewares/request_id.py +++ b/lib/dl_api_commons/dl_api_commons/aio/middlewares/request_id.py @@ -11,7 +11,6 @@ import attr from dl_api_commons import ( - log_request_start, make_uuid_from_parts, request_id_generator, ) @@ -26,7 +25,7 @@ ) from dl_api_commons.logging import ( NON_TRANSITIVE_LOGGING_CTX_KEYS, - log_request_end, + RequestLogHelper, ) from dl_api_commons.reporting.profiler import DefaultReportingProfiler from dl_api_commons.reporting.registry import DefaultReportingRegistry @@ -35,6 +34,7 @@ LOGGER = logging.getLogger(__name__) +LOG_HELPER = RequestLogHelper(logger=LOGGER) @attr.s @@ -111,9 +111,7 @@ def dl_request_init(self, request: web.Request) -> DLRequestBase: else: log.context.put_to_context(ctx_key, ctx_val) - log_request_start( - logger=LOGGER, method=request.method, full_path=request.path_qs, headers=request.headers.items() - ) + LOG_HELPER.log_request_start(method=request.method, full_path=request.path_qs, headers=request.headers.items()) dl_request.init_temp_rci( RequestContextInfo.create( @@ -146,4 +144,4 @@ def dl_request_finilize(self, dl_request: DLRequestBase) -> None: dl_request.reporting_profiler.on_request_end() def on_request_end(self, method: str, full_path: str, status_code: int) -> None: - log_request_end(LOGGER, method, full_path, status_code) + LOG_HELPER.log_request_end(method=method, full_path=full_path, status_code=status_code) diff --git a/lib/dl_api_commons/dl_api_commons/flask/middlewares/request_id.py b/lib/dl_api_commons/dl_api_commons/flask/middlewares/request_id.py index 4167d74de..c9c92cf56 100644 --- a/lib/dl_api_commons/dl_api_commons/flask/middlewares/request_id.py +++ b/lib/dl_api_commons/dl_api_commons/flask/middlewares/request_id.py @@ -10,7 +10,6 @@ from flask import request from dl_api_commons import ( - log_request_start, make_uuid_from_parts, request_id_generator, ) @@ -24,13 +23,14 @@ ) from dl_api_commons.logging import ( NON_TRANSITIVE_LOGGING_CTX_KEYS, - log_request_end_extended, + RequestLogHelper, ) from dl_app_tools.log.context import put_to_context from dl_constants.api_constants import DLHeadersCommon LOGGER = logging.getLogger(__name__) +LOG_HELPER = RequestLogHelper(logger=LOGGER) @attr.s @@ -72,8 +72,7 @@ def _before_request(self) -> None: except Exception: # noqa LOGGER.exception("Can not parse logging context: %s", logging_ctx_header) - log_request_start( - logger=LOGGER, + LOG_HELPER.log_request_start( method=request.method, full_path=request.full_path, headers=request.headers.items(), @@ -120,8 +119,7 @@ def _post_log_request_id(response: flask.Response) -> flask.Response: user_name = rci.user_name user_id = rci.user_id - log_request_end_extended( - logger=LOGGER, + LOG_HELPER.log_request_end_extended( request_method=request.method, request_path=request.full_path, request_headers=dict(request.headers), diff --git a/lib/dl_api_commons/dl_api_commons/logging.py b/lib/dl_api_commons/dl_api_commons/logging.py index d935e0a83..434349903 100644 --- a/lib/dl_api_commons/dl_api_commons/logging.py +++ b/lib/dl_api_commons/dl_api_commons/logging.py @@ -29,192 +29,243 @@ } ) -SECRET_HEADERS: frozenset[str] = frozenset( - h.lower() - for h in ( - "Authorization", - "X-Us-Master-Token", - "Master-Token", - "X-DL-API-Key", - "X-Ya-Service-Ticket", - "X-Ya-User-Ticket", - ) -) -SECRET_HEADERS_PATTERNS: Sequence[Pattern] = (re.compile(r".*token.*", re.IGNORECASE),) - -SECRET_COOKIES: frozenset[str] = frozenset( - c.lower() - for c in ( - "Session_id", - "sessionid2", - "yc_session", - "iam_cookie", +class RequestObfuscator: + SECRET_HEADERS: ClassVar[frozenset[str]] = frozenset( + h.lower() + for h in ( + "Authorization", + "X-Us-Master-Token", + "Master-Token", + "X-DL-API-Key", + "X-Ya-Service-Ticket", + "X-Ya-User-Ticket", + ) + ) + SECRET_HEADERS_PATTERNS: ClassVar[Sequence[Pattern]] = (re.compile(r".*token.*", re.IGNORECASE),) + SECRET_COOKIES: ClassVar[frozenset[str]] = frozenset( + c.lower() + for c in ( + "Session_id", + "sessionid2", + "yc_session", + "iam_cookie", + ) + ) + SENSITIVE_KEY_NAMES: frozenset[str] = frozenset( + ( + "password", + "token", + "secret", + "private_key", + "cypher_text", + ) ) -) - - -def _is_secret_header(header_name: str) -> bool: - if header_name.lower() in SECRET_HEADERS: - return True - for pattern in SECRET_HEADERS_PATTERNS: - if pattern.match(header_name): + def _is_secret_header(self, header_name: str) -> bool: + if header_name.lower() in self.SECRET_HEADERS: return True - return False - + for pattern in self.SECRET_HEADERS_PATTERNS: + if pattern.match(header_name): + return True -def _obfuscate_value(secret_value: str) -> str: - repl_str = "" - if len(secret_value) > 8: - return secret_value[:3] + repl_str + secret_value[-3:] - else: - return repl_str + return False + def _obfuscate_value(self, secret_value: str) -> str: + repl_str = "" + if len(secret_value) > 8: + return secret_value[:3] + repl_str + secret_value[-3:] + else: + return repl_str -def _obfuscate_cookie_header_value(cookie_string: str) -> str: - cookie: SimpleCookie = SimpleCookie(cookie_string) - for cookie_name in cookie: - if cookie_name.lower() in SECRET_COOKIES: - repl_str = "" - cookie[cookie_name].set(cookie_name, repl_str, repl_str) - - return cookie.output(header="", sep=";").strip() + def _obfuscate_cookie_header_value(self, cookie_string: str) -> str: + cookie: SimpleCookie = SimpleCookie(cookie_string) + for cookie_name in cookie: + if cookie_name.lower() in self.SECRET_COOKIES: + repl_str = "" + cookie[cookie_name].set(cookie_name, repl_str, repl_str) + return cookie.output(header="", sep=";").strip() -def clean_secret_data_in_headers(headers: Iterable[tuple[str, str]]) -> Iterable[tuple[str, str]]: - return tuple( - ( - name, - _obfuscate_value(val) - if _is_secret_header(name) - else _obfuscate_cookie_header_value(val) - if name.lower() == "cookie" - else val, + def clean_secret_data_in_headers(self, headers: Iterable[tuple[str, str]]) -> Iterable[tuple[str, str]]: + return tuple( + ( + name, + self._obfuscate_value(val) + if self._is_secret_header(name) + else self._obfuscate_cookie_header_value(val) + if name.lower() == "cookie" + else val, + ) + for name, val in headers ) - for name, val in headers - ) - -def log_request_start(logger: logging.Logger, method: str, full_path: str, headers: Iterable[tuple[str, str]]) -> None: - clean_headers = clean_secret_data_in_headers(headers) + def mask_sensitive_fields_by_name_in_json_recursive( + self, source: Optional[dict[str, Any]], extra_sensitive_key_names: Iterable[str] = () + ) -> Optional[dict[str, Any]]: + if source is None: + return None - logger.info( - "Received request. method: %s, path: %s, headers: %s, pid: %s", - method.upper(), - full_path, - # Complexity to be compatible with previous version of logger - "{{{}}}".format(", ".join([f"{k!r}: {v!r}" for k, v in clean_headers])), - os.getpid(), - ) + all_sensitive_key_names = self.SENSITIVE_KEY_NAMES | frozenset(extra_sensitive_key_names) + + def process_value(key_name: Optional[str], value: Any) -> Any: + if value is None: + return None + + if isinstance(value, dict): + return { + nested_key: process_value(nested_key, nested_value) for nested_key, nested_value in value.items() + } + if isinstance( + value, + ( + list, + tuple, + ), + ): + return [process_value(key_name, nested_value) for nested_value in value] + if isinstance(value, str): + if key_name in all_sensitive_key_names: + return self._obfuscate_value(value) + return value + + if isinstance( + value, + ( + bool, + float, + int, + ), + ): + if key_name in all_sensitive_key_names: + LOGGER.error("Non-string type for sensitive field '%s': %s", key_name, type(value)) + return self._obfuscate_value(str(value)) + return value + + raise TypeError(f"Unexpected value type: {type}") + + return process_value(None, source) -def log_request_end(logger: logging.Logger, method: str, full_path: str, status_code: int) -> None: - logger.info( - "Response. method: %s, path: %s, status: %d", - method.upper(), - full_path, - status_code, - extra=dict( - event_code="http_response", - request_method=method, - request_path=full_path, - response_status=status_code, - ), - ) +@attr.s +class RequestLogHelper: + _logger: logging.Logger = attr.ib(kw_only=True) + _obfuscator: RequestObfuscator = attr.ib(kw_only=True, factory=RequestObfuscator) + + def log_request_start(self, method: str, full_path: str, headers: Iterable[tuple[str, str]]) -> None: + clean_headers = self._obfuscator.clean_secret_data_in_headers(headers) + + self._logger.info( + "Received request. method: %s, path: %s, headers: %s, pid: %s", + method.upper(), + full_path, + # Complexity to be compatible with previous version of logger + "{{{}}}".format(", ".join([f"{k!r}: {v!r}" for k, v in clean_headers])), + os.getpid(), + ) + def log_request_end(self, method: str, full_path: str, status_code: int) -> None: + self._logger.info( + "Response. method: %s, path: %s, status: %d", + method.upper(), + full_path, + status_code, + extra=dict( + event_code="http_response", + request_method=method, + request_path=full_path, + response_status=status_code, + ), + ) -# TODO FIX: Make more strict typing for headers -def _normalize_headers(headers: Any) -> Optional[dict[str, str]]: - if headers is None: - return None - if hasattr(headers, "items"): - headers = headers.items() - headers = sorted(headers) - headers = {normalize_header_name(key): value for key, value in headers} - - headers = dict(clean_secret_data_in_headers(headers.items())) - - return headers - - -# TODO CONSIDER: Create custom type for headers -def log_request_end_extended( - logger: logging.Logger, - request_method: str, - request_path: str, - request_headers: Optional[dict], - response_status: int, - response_headers: Optional[dict], - response_timing: Optional[float], - # ... - user_id: Optional[str] = None, - username: Optional[str] = None, - # extra extra for when they're not in the context. - # TODO: tenant_id - request_id: Optional[str] = None, - endpoint_code: Optional[str] = None, -) -> None: - """ - Response pre-return detailed (extended) logging. + # TODO FIX: Make more strict typing for headers + def _normalize_headers(self, headers: Any) -> Optional[dict[str, str]]: + if headers is None: + return None + if hasattr(headers, "items"): + headers = headers.items() + headers = sorted(headers) + headers = {normalize_header_name(key): value for key, value in headers} + + headers = dict(self._obfuscator.clean_secret_data_in_headers(headers.items())) + + return headers + + # TODO CONSIDER: Create custom type for headers + def log_request_end_extended( + self, + request_method: str, + request_path: str, + request_headers: Optional[dict], + response_status: int, + response_headers: Optional[dict], + response_timing: Optional[float], + # ... + user_id: Optional[str] = None, + username: Optional[str] = None, + # extra extra for when they're not in the context. + # TODO: tenant_id + request_id: Optional[str] = None, + endpoint_code: Optional[str] = None, + ) -> None: + """ + Response pre-return detailed (extended) logging. - :param logger: logger to send the log to. + :param request_method: HTTP method; uppercase, e.g. 'GET'. - :param request_method: HTTP method; uppercase, e.g. 'GET'. + :param request_path: HTTP path; should, generally, include the query string, + i.e. '/a/b?c=d&e=f'. - :param request_path: HTTP path; should, generally, include the query string, - i.e. '/a/b?c=d&e=f'. + :param request_headers: HTTP headers, unordered, normalized (lowercase + dash names, i.e. 'user-agent'), unduplicated, scrubbed. + Headers like 'remote-addr' should be authoritative. - :param request_headers: HTTP headers, unordered, normalized (lowercase - dash names, i.e. 'user-agent'), unduplicated, scrubbed. - Headers like 'remote-addr' should be authoritative. + :param user_id: primary identifier of the *authorized* initial user (authoritative); + e.g. '1120000000092758'. - :param user_id: primary identifier of the *authorized* initial user (authoritative); - e.g. '1120000000092758'. + :param username: readable identifier of the user, i.e. login, e.g. 'robot-datalens'. - :param username: readable identifier of the user, i.e. login, e.g. 'robot-datalens'. + :param response_status: HTTP status of the response; e.g. '502'. - :param response_status: HTTP status of the response; e.g. '502'. + :param response_headers: HTTP headers of the response, normalized. - :param response_headers: HTTP headers of the response, normalized. + :param response_timing: duration of the request handling, in seconds. - :param response_timing: duration of the request handling, in seconds. + :param request_id: unique request identifier; should be specified when not + available in context. - :param request_id: unique request identifier; should be specified when not - available in context. + :param endpoint_code: name of the handler; should be specified when not + available in context; use an empty string if it is not applicable. + """ + request_method = request_method.upper() + request_headers = self._normalize_headers(request_headers) + response_headers = self._normalize_headers(response_headers) + if response_timing is not None: + response_timing = round(response_timing, 4) - :param endpoint_code: name of the handler; should be specified when not - available in context; use an empty string if it is not applicable. - """ - request_method = request_method.upper() - request_headers = _normalize_headers(request_headers) - response_headers = _normalize_headers(response_headers) - if response_timing is not None: - response_timing = round(response_timing, 4) - - extra = dict( - event_code="http_response", - request_method=request_method, - request_path=request_path, - request_headers=request_headers, - user_id=user_id, - username=username, - response_status=response_status, - response_headers=response_headers, - response_timing=response_timing, - # Other possibilities: - # response_body_info=dict(body_piece=body[:max_size], body_size=body_size, ...), - # response_details=dict(...), - ) - if request_id is not None: - extra["request_id"] = request_id - if endpoint_code is not None: - extra["endpoint_code"] = endpoint_code + extra = dict( + event_code="http_response", + request_method=request_method, + request_path=request_path, + request_headers=request_headers, + user_id=user_id, + username=username, + response_status=response_status, + response_headers=response_headers, + response_timing=response_timing, + # Other possibilities: + # response_body_info=dict(body_piece=body[:max_size], body_size=body_size, ...), + # response_details=dict(...), + ) + if request_id is not None: + extra["request_id"] = request_id + if endpoint_code is not None: + extra["endpoint_code"] = endpoint_code - logger.info( - "Response. method: %s, path: %s, status: %d", request_method, request_path, response_status, extra=extra - ) + self._logger.info( + "Response. method: %s, path: %s, status: %d", request_method, request_path, response_status, extra=extra + ) class RequestLoggingContextController(metaclass=abc.ABCMeta): @@ -275,58 +326,6 @@ def format_dict(extra: dict[str, Any], separator: str = " ", *args: str, **kwarg return separator.join(parts) -def mask_sensitive_fields_by_name_in_json_recursive( - source: Optional[dict[str, Any]], extra_sensitive_key_names: Iterable[str] = () -) -> Optional[dict[str, Any]]: - if source is None: - return None - - all_sensitive_key_names: set[str] = { - "password", - "token", - "secret", - "private_key", - "cypher_text", - } - all_sensitive_key_names.update(extra_sensitive_key_names) - - def process_value(key_name: Optional[str], value: Any) -> Any: - if value is None: - return None - - if isinstance(value, dict): - return {nested_key: process_value(nested_key, nested_value) for nested_key, nested_value in value.items()} - if isinstance( - value, - ( - list, - tuple, - ), - ): - return [process_value(key_name, nested_value) for nested_value in value] - if isinstance(value, str): - if key_name in all_sensitive_key_names: - return _obfuscate_value(value) - return value - - if isinstance( - value, - ( - bool, - float, - int, - ), - ): - if key_name in all_sensitive_key_names: - LOGGER.error("Non-string type for sensitive field '%s': %s", key_name, type(value)) - return _obfuscate_value(str(value)) - return value - - raise TypeError(f"Unexpected value type: {type}") - - return process_value(None, source) - - class LogRequestLoggingContextController(RequestLoggingContextController): allowed_keys: ClassVar[tuple[str, ...]] = ( "request_id", diff --git a/lib/dl_api_commons/dl_api_commons/logging_sentry.py b/lib/dl_api_commons/dl_api_commons/logging_sentry.py index ab6bf703d..d8dc1d453 100644 --- a/lib/dl_api_commons/dl_api_commons/logging_sentry.py +++ b/lib/dl_api_commons/dl_api_commons/logging_sentry.py @@ -6,7 +6,7 @@ import attr -from dl_api_commons import clean_secret_data_in_headers +from dl_api_commons.logging import RequestObfuscator log = logging.getLogger() @@ -87,7 +87,7 @@ def cleanup_local_vars(local_vars: dict) -> None: def cleanup_event_headers(original_headers: dict[str, str]) -> dict[str, str]: return { name: value - for name, value in clean_secret_data_in_headers( + for name, value in RequestObfuscator().clean_secret_data_in_headers( ( original_name, original_value, diff --git a/lib/dl_api_commons/dl_api_commons_tests/unit/aio/test_common_logging.py b/lib/dl_api_commons/dl_api_commons_tests/unit/aio/test_common_logging.py index a6a7fcdb2..3cb350504 100644 --- a/lib/dl_api_commons/dl_api_commons_tests/unit/aio/test_common_logging.py +++ b/lib/dl_api_commons/dl_api_commons_tests/unit/aio/test_common_logging.py @@ -9,20 +9,19 @@ from multidict import CIMultiDict import pytest -from dl_api_commons import ( - clean_secret_data_in_headers, - log_request_end, - log_request_start, -) from dl_api_commons.aio.middlewares import request_id as aio_request_id from dl_api_commons.aio.middlewares.request_bootstrap import RequestBootstrap from dl_api_commons.flask.middlewares.context_var_middleware import ContextVarMiddleware from dl_api_commons.flask.middlewares.logging_context import RequestLoggingContextControllerMiddleWare from dl_api_commons.flask.middlewares.request_id import RequestIDService -from dl_api_commons.logging import mask_sensitive_fields_by_name_in_json_recursive +from dl_api_commons.logging import ( + RequestLogHelper, + RequestObfuscator, +) LOGGER = logging.getLogger(__name__) +LOG_HELPER = RequestLogHelper(logger=LOGGER) _EXPECTED_MASKED_HEADERS = ( ("SomeToken", ""), @@ -62,14 +61,13 @@ def test_headers_cleaning(): - assert _EXPECTED_MASKED_HEADERS == clean_secret_data_in_headers(_INPUT_HEADERS) + assert _EXPECTED_MASKED_HEADERS == RequestObfuscator().clean_secret_data_in_headers(_INPUT_HEADERS) def test_common_logging_request_start(caplog): caplog.set_level("INFO") caplog.clear() - log_request_start( - logger=LOGGER, + LOG_HELPER.log_request_start( method="get", full_path="/api/ololo/azaza?param=value", headers=( @@ -93,8 +91,7 @@ def test_common_logging_request_start(caplog): def test_common_logging_request_end(caplog): caplog.set_level("INFO") caplog.clear() - log_request_end( - logger=LOGGER, + LOG_HELPER.log_request_end( method="get", full_path="/unistat/?", status_code=200, @@ -265,5 +262,5 @@ async def handle(_: web.Request): ), ) def test_mask_sensitive_fields_by_name_recursive(source: dict[str, Any], expected_masked: dict[str, Any]): - actual_masked = mask_sensitive_fields_by_name_in_json_recursive(source) + actual_masked = RequestObfuscator().mask_sensitive_fields_by_name_in_json_recursive(source) assert actual_masked == expected_masked diff --git a/lib/dl_api_lib/dl_api_lib/aio/middlewares/json_body_middleware.py b/lib/dl_api_lib/dl_api_lib/aio/middlewares/json_body_middleware.py index f5c04f23f..1ee2d6fe9 100644 --- a/lib/dl_api_lib/dl_api_lib/aio/middlewares/json_body_middleware.py +++ b/lib/dl_api_lib/dl_api_lib/aio/middlewares/json_body_middleware.py @@ -9,7 +9,7 @@ from aiohttp.typedefs import Handler from dl_api_commons.aio.typing import AIOHTTPMiddleware -from dl_api_commons.logging import mask_sensitive_fields_by_name_in_json_recursive +from dl_api_commons.logging import RequestObfuscator from dl_api_lib.aio.aiohttp_wrappers import DSAPIRequest from dl_api_lib.app.data_api.resources.base import ( BaseView, @@ -41,7 +41,7 @@ async def actual_body_log_middleware(dl_request: DSAPIRequest, handler: Handler) dl_request.store_parsed_json_body(body) - dbg_body_data = mask_sensitive_fields_by_name_in_json_recursive(body) + dbg_body_data = RequestObfuscator().mask_sensitive_fields_by_name_in_json_recursive(body) dbg_body = json.dumps(dbg_body_data) url = dl_request.url # url with http->https override extra = dict( diff --git a/lib/dl_api_lib/dl_api_lib/api_decorators.py b/lib/dl_api_lib/dl_api_lib/api_decorators.py index 23b9937c7..0fc2c9da2 100644 --- a/lib/dl_api_lib/dl_api_lib/api_decorators.py +++ b/lib/dl_api_lib/dl_api_lib/api_decorators.py @@ -19,7 +19,7 @@ from marshmallow import ValidationError as MValidationError from werkzeug.exceptions import HTTPException -from dl_api_commons.logging import mask_sensitive_fields_by_name_in_json_recursive +from dl_api_commons.logging import RequestObfuscator from dl_api_lib import utils from dl_api_lib.error_handling import ( BIError, @@ -88,7 +88,7 @@ def wrapper(*args, **kwargs): # type: ignore # TODO: fix body = request.get_json() if body_schema is not None else None if LOGGER.isEnabledFor(logging.INFO): - dbg_body_data = mask_sensitive_fields_by_name_in_json_recursive(body) + dbg_body_data = RequestObfuscator().mask_sensitive_fields_by_name_in_json_recursive(body) dbg_body = json.dumps(dbg_body_data) url = request.url pfx = "http://" diff --git a/lib/dl_core/dl_core/united_storage_client.py b/lib/dl_core/dl_core/united_storage_client.py index a28307f3d..95af95536 100644 --- a/lib/dl_core/dl_core/united_storage_client.py +++ b/lib/dl_core/dl_core/united_storage_client.py @@ -27,7 +27,7 @@ TenantCommon, TenantDef, ) -from dl_api_commons.logging import mask_sensitive_fields_by_name_in_json_recursive +from dl_api_commons.logging import RequestObfuscator from dl_api_commons.tracing import get_current_tracing_headers from dl_api_commons.utils import ( get_retriable_requests_session, @@ -355,7 +355,7 @@ def _get_us_json_from_response(cls, response: ResponseAdapter) -> dict: "US error response on %s %s (%s): %s, folder_id: %s, org_id: %s, req_id: %s", response.request.method, response.request.relative_url, - mask_sensitive_fields_by_name_in_json_recursive(response.request.json), + RequestObfuscator().mask_sensitive_fields_by_name_in_json_recursive(response.request.json), response.content, # TODO: BI-4918 move to local injection and reuse bi_api_commons_ya_cloud.constants.DLHeadersYC response.request.get_header(DLHeadersCommon.FOLDER_ID.value), diff --git a/lib/dl_core/dl_core/utils.py b/lib/dl_core/dl_core/utils.py index 607d9a89e..5c4d5d47c 100644 --- a/lib/dl_core/dl_core/utils.py +++ b/lib/dl_core/dl_core/utils.py @@ -29,11 +29,11 @@ import shortuuid import sqlalchemy as sa -from dl_api_commons import clean_secret_data_in_headers from dl_api_commons.base_models import ( AuthData, RequestContextInfo, ) +from dl_api_commons.logging import RequestObfuscator from dl_api_commons.utils import ( stringify_dl_cookies, stringify_dl_headers, @@ -294,18 +294,19 @@ def raise_for_status_and_hide_secret_headers(response: ClientResponse) -> None: but hides secret data in headers for exception """ if not response.ok: + obfuscator = RequestObfuscator() # reason should always be not None for a started response assert response.reason is not None response.release() new_request_info = RequestInfo( response.request_info.url, response.request_info.method, - CIMultiDict(clean_secret_data_in_headers(_multidict_to_list(response.request_info.headers))), # type: ignore # 2024-01-24 # TODO: Argument 3 to "RequestInfo" has incompatible type "CIMultiDict[str]"; expected "CIMultiDictProxy[str]" [arg-type] + CIMultiDict(obfuscator.clean_secret_data_in_headers(_multidict_to_list(response.request_info.headers))), # type: ignore # 2024-01-24 # TODO: Argument 3 to "RequestInfo" has incompatible type "CIMultiDict[str]"; expected "CIMultiDictProxy[str]" [arg-type] ) raise ClientResponseError( new_request_info, response.history, status=response.status, message=response.reason, - headers=CIMultiDict(clean_secret_data_in_headers(_multidict_to_list(response.headers))), + headers=CIMultiDict(obfuscator.clean_secret_data_in_headers(_multidict_to_list(response.headers))), )