diff --git a/lib/dl_api_lib/dl_api_lib/app_settings.py b/lib/dl_api_lib/dl_api_lib/app_settings.py index 0f3f4eeb1..28b8f358b 100644 --- a/lib/dl_api_lib/dl_api_lib/app_settings.py +++ b/lib/dl_api_lib/dl_api_lib/app_settings.py @@ -88,29 +88,31 @@ class AppSettings: missing=None, ) - # RATE_LIMITER_ENABLED: bool = s_attrib("RATE_LIMITER_ENABLED", missing=False) # type: ignore # 2024-01-30 # TODO: Incompatible types in assignment (expression has type "Attribute[Any]", variable has type "bool") [assignment] - # RATE_LIMITER_REDIS: Optional[RedisSettings] = s_attrib( # type: ignore # 2024-01-30 # TODO: Incompatible types in assignment (expression has type "Attribute[Any]", variable has type "RedisSettings | None") [assignment] - # "RATE_LIMITER_REDIS", - # fallback_factory=( - # lambda cfg: RedisSettings( # type: ignore # 2024-01-30 # TODO: Unexpected keyword argument "MODE" for "RedisSettings" [call-arg] - # MODE=RedisMode.single_host, - # CLUSTER_NAME=cfg.RATE_LIMITER_REDIS_CLUSTER_NAME, - # HOSTS=_list_to_tuple(cfg.RATE_LIMITER_REDIS_HOSTS), - # PORT=cfg.RATE_LIMITER_REDIS_PORT, - # SSL=cfg.RATE_LIMITER_REDIS_SSL, - # DB=cfg.RATE_LIMITER_REDIS_DB, - # PASSWORD=required(str), - # ) - # if is_setting_applicable(cfg, "RATE_LIMITER_REDIS_DB") - # else None - # ), - # missing=None, - # ) - # RATE_LIMITER_CONFIG: Optional[RateLimiterConfig] = s_attrib( # type: ignore # 2024-01-30 # TODO: Incompatible types in assignment (expression has type "Attribute[Any]", variable has type "RateLimiterConfig") [assignment] - # "RATE_LIMITER", - # fallback_factory=lambda cfg: RateLimiterConfig.from_json(cfg.get("RATE_LIMITER")), - # missing=None, - # ) + RATE_LIMITER_ENABLED: bool = s_attrib("RATE_LIMITER_ENABLED", missing=False) # type: ignore # 2024-01-30 # TODO: Incompatible types in assignment (expression has type "Attribute[Any]", variable has type "bool") [assignment] + RATE_LIMITER_REDIS: Optional[RedisSettings] = s_attrib( # type: ignore # 2024-01-30 # TODO: Incompatible types in assignment (expression has type "Attribute[Any]", variable has type "RedisSettings | None") [assignment] + "RATE_LIMITER_REDIS", + fallback_factory=( + lambda cfg: RedisSettings( # type: ignore # 2024-01-30 # TODO: Unexpected keyword argument "MODE" for "RedisSettings" [call-arg] + MODE=RedisMode.single_host, + CLUSTER_NAME=cfg.RATE_LIMITER_REDIS_CLUSTER_NAME, + HOSTS=_list_to_tuple(cfg.RATE_LIMITER_REDIS_HOSTS), + PORT=cfg.RATE_LIMITER_REDIS_PORT, + SSL=cfg.RATE_LIMITER_REDIS_SSL, + DB=cfg.RATE_LIMITER_REDIS_DB, + PASSWORD=required(str), + SOCKET_TIMEOUT=0.1, + SOCKET_CONNECT_TIMEOUT=0.5, + ) + if is_setting_applicable(cfg, "RATE_LIMITER_REDIS_DB") + else None + ), + missing=None, + ) + RATE_LIMITER_CONFIG: Optional[RateLimiterConfig] = s_attrib( # type: ignore # 2024-01-30 # TODO: Incompatible types in assignment (expression has type "Attribute[Any]", variable has type "RateLimiterConfig") [assignment] + "RATE_LIMITER", + fallback_factory=lambda cfg: RateLimiterConfig.from_json(cfg.get("RATE_LIMITER")), + missing=None, + ) SAMPLES_CH_HOSTS: tuple[str, ...] = s_attrib( # type: ignore # 2024-01-30 # TODO: Incompatible types in assignment (expression has type "Attribute[Any]", variable has type "tuple[str, ...]") [assignment] "SAMPLES_CH_HOST", env_var_converter=split_by_comma, missing_factory=list ) diff --git a/lib/dl_configs/dl_configs/settings_submodels.py b/lib/dl_configs/dl_configs/settings_submodels.py index 7c0ea5751..2ef627e9a 100644 --- a/lib/dl_configs/dl_configs/settings_submodels.py +++ b/lib/dl_configs/dl_configs/settings_submodels.py @@ -25,6 +25,8 @@ class RedisSettings(SettingsBase): DB: int = s_attrib("DB") # type: ignore # 2024-01-24 # TODO: Incompatible types in assignment (expression has type "Attribute[Any]", variable has type "int") [assignment] PASSWORD: str = s_attrib("PASSWORD", sensitive=True, missing=None) # type: ignore # 2024-01-24 # TODO: Incompatible types in assignment (expression has type "Attribute[Any]", variable has type "str") [assignment] SSL: Optional[bool] = s_attrib("SSL", missing=None) # type: ignore # 2024-01-24 # TODO: Incompatible types in assignment (expression has type "Attribute[Any]", variable has type "bool | None") [assignment] + SOCKET_TIMEOUT: float = s_attrib("SOCKET_TIMEOUT", missing=0.0) # type: ignore # 2024-01-24 # TODO: Incompatible types in assignment (expression has type "Attribute[Any]", variable has type "float") [assignment] + SOCKET_CONNECT_TIMEOUT: float = s_attrib("SOCKET_CONNECT_TIMEOUT", missing=0.0) # type: ignore # 2024-01-24 # TODO: Incompatible types in assignment (expression has type "Attribute[Any]", variable has type "float") [assignment] def as_single_host_url(self) -> str: return make_url( diff --git a/lib/dl_rate_limiter/dl_rate_limiter/middlewares/aiohttp.py b/lib/dl_rate_limiter/dl_rate_limiter/middlewares/aiohttp.py index ea58d52c4..c05943246 100644 --- a/lib/dl_rate_limiter/dl_rate_limiter/middlewares/aiohttp.py +++ b/lib/dl_rate_limiter/dl_rate_limiter/middlewares/aiohttp.py @@ -28,6 +28,7 @@ async def process(self, request: aiohttp_web.Request, handler: AioHTTPHandler) - ) ) if not result: + logger.warning("Request was rate limited") return aiohttp_web.json_response( status=429, data={"description": "Too Many Requests"}, @@ -35,6 +36,7 @@ async def process(self, request: aiohttp_web.Request, handler: AioHTTPHandler) - except Exception: logger.exception("Failed to check request limit") + logger.info("No request limit was found") return await handler(request) diff --git a/lib/dl_rate_limiter/dl_rate_limiter/middlewares/flask.py b/lib/dl_rate_limiter/dl_rate_limiter/middlewares/flask.py index 9b20336a6..327ff4778 100644 --- a/lib/dl_rate_limiter/dl_rate_limiter/middlewares/flask.py +++ b/lib/dl_rate_limiter/dl_rate_limiter/middlewares/flask.py @@ -23,6 +23,7 @@ def process(self) -> None | flask.Response: ) ) if result is False: + logger.warning("Request was rate limited") return flask.Response( status=429, response={"description": "Too Many Requests"}, @@ -30,6 +31,7 @@ def process(self) -> None | flask.Response: except Exception: logger.exception("Failed to check request limit") + logger.info("No request limit was found") return None def set_up(self, app: flask.Flask) -> None: diff --git a/lib/dl_rate_limiter/dl_rate_limiter/request_rate_limiter.py b/lib/dl_rate_limiter/dl_rate_limiter/request_rate_limiter.py index 8bef989a9..da13eba07 100644 --- a/lib/dl_rate_limiter/dl_rate_limiter/request_rate_limiter.py +++ b/lib/dl_rate_limiter/dl_rate_limiter/request_rate_limiter.py @@ -63,10 +63,12 @@ class SyncRequestRateLimiter: _patterns: list[RequestPattern] = attr.ib(factory=list) def check_limit(self, request: Request) -> bool: + logger.info("Checking rate limit for Request(%s)", Request) for pattern in self._patterns: if not pattern.matches(request): continue + logger.info("Found match for Pattern(%s)", pattern) try: event_key = pattern.event_key_template.generate_key(request) except TemplateError: @@ -91,10 +93,12 @@ class AsyncRequestRateLimiter: _patterns: list[RequestPattern] = attr.ib(factory=list) async def check_limit(self, request: Request) -> bool: + logger.info("Checking rate limit for Request(%s)", Request) for pattern in self._patterns: if not pattern.matches(request): continue + logger.info("Found match for Pattern(%s)", pattern) try: event_key = pattern.event_key_template.generate_key(request) except TemplateError: