From 19cc1bdc57f5cc82e51a5a29e2b62be3a98d52cf Mon Sep 17 00:00:00 2001 From: Muhammad Luqman Date: Fri, 27 Oct 2023 07:49:25 +0700 Subject: [PATCH 1/3] feat: add exponential retry mechanism --- docs/guides/cookbook.rst | 124 ++++++++++++++++++------------ novu/api/base.py | 55 ++++++++++++- novu/api/blueprint.py | 7 +- novu/api/change.py | 7 +- novu/api/environment.py | 7 +- novu/api/event.py | 7 +- novu/api/execution_detail.py | 7 +- novu/api/feed.py | 7 +- novu/api/inbound_parse.py | 7 +- novu/api/integration.py | 7 +- novu/api/layout.py | 7 +- novu/api/message.py | 7 +- novu/api/notification.py | 7 +- novu/api/notification_group.py | 7 +- novu/api/notification_template.py | 7 +- novu/api/subscriber.py | 7 +- novu/api/tenant.py | 7 +- novu/api/topic.py | 7 +- poetry.lock | 26 ++++--- pyproject.toml | 1 + 20 files changed, 226 insertions(+), 92 deletions(-) diff --git a/docs/guides/cookbook.rst b/docs/guides/cookbook.rst index 145ab42f..2b50a701 100644 --- a/docs/guides/cookbook.rst +++ b/docs/guides/cookbook.rst @@ -1,48 +1,76 @@ -Cookbook -======== - -How to change API requests timeout? ------------------------------------ - -There are two ways to change the timeout of the :module:`requests` module that makes the API calls for you: - -* Via an environment variable named ``NOVU_PYTHON_REQUESTS_TIMEOUT``. - You can define it in seconds and it will overloads all timeouts in this package. -* Via the API constructor directly, passing :attr:`~novu.api.base.Api.requests_timeout` in keywords arguments during the initialisation. - -.. code-block:: python - - EventApi(...,...,request_timeout=60).trigger_bulk(...) - -How to take control over the session of ``requests``? ------------------------------------------------------ - -Sometime, you may want to setup advanced usage involving the use of a ``requests`` :class:`~requests.Session`. - -By default, a session is declared for every request you make to the API, but you can also -provide your own session and reuse it over and over again. - -To illustrate this use case, let's take the example of setting up an automatic retry mechanism on -requests when the API responds with a 502, 503 or 504 HTTP codes. - -To do this, before initializing the Event API, we will first instantiate a :class:`~requests.Session` from the -:module:`requests` module, then we will inject it into the keywords arguments of our API constructor. We finally -call the method we want to use and retry. - -.. code-block:: python - - from requests import Session - from requests.adapters import HTTPAdapter, Retry - - from novu.api import EventApi - - session = Session() - retries = Retry(total=5, backoff_factor=1, status_forcelist=[ 502, 503, 504 ]) - session.mount("https://api.novu.co", HTTPAdapter(max_retries=retries)) - - event_api = EventApi("https://api.novu.co/api/", "", session=session) - event_api.trigger( - name="", - recipients="", - payload={}, # Your Novu payload goes here - ) +Cookbook +======== + +How to change API requests timeout? +----------------------------------- + +There are two ways to change the timeout of the :module:`requests` module that makes the API calls for you: + +* Via an environment variable named ``NOVU_PYTHON_REQUESTS_TIMEOUT``. + You can define it in seconds and it will overloads all timeouts in this package. +* Via the API constructor directly, passing :attr:`~novu.api.base.Api.requests_timeout` in keywords arguments during the initialisation. + +.. code-block:: python + + EventApi(...,...,request_timeout=60).trigger_bulk(...) + +How to add retry mechanism for API call? +---------------------------------------- + +In order to enhance the resilience and reliability of the SDK, we provide Exponential Retry mechanism to retry failed API requests. + +This means that the waiting time between each retry is multiplied by a factor that increases with each retry. +For example, the first retry may wait for 1 second, the second retry may wait for 2 seconds, the third retry may wait for 4 seconds, and so on until it stops at nth retry. +You can configure this interval using ``initial_delay``, ``wait_min``, ``wait_max``, and ``retry_max`` attributes. + +This mechanism reduces the number of failed requests and prevents server overload by gradually increasing the waiting time between retries +and use an ``Idempotency-Key`` HTTP Header to ensure the idempotent processing of requests. + +To enable this feature, first create a :class:`~novu.api.base.RetryConfig` class and use it in API constructor. + +.. code-block:: python + + from novu.api import EventApi + from novu.api.base import RetryConfig + + retry_config = RetryConfig(initial_delay=1, wait_min=2, wait_max=100, retry_max=6) + + event_api = EventApi("https://api.novu.co", "", retry_config=retry_config) + event_api.trigger( + name="", + recipients="", + payload={}, # Your Novu payload goes here + ) + +How to take control over the session of ``requests``? +----------------------------------------------------- + +Sometime, you may want to setup advanced usage involving the use of a ``requests`` :class:`~requests.Session`. + +By default, a session is declared for every request you make to the API, but you can also +provide your own session and reuse it over and over again. + +To illustrate this use case, let's take the example of setting up an automatic retry mechanism on +requests when the API responds with a 502, 503 or 504 HTTP codes. + +To do this, before initializing the Event API, we will first instantiate a :class:`~requests.Session` from the +:module:`requests` module, then we will inject it into the keywords arguments of our API constructor. We finally +call the method we want to use and retry. + +.. code-block:: python + + from requests import Session + from requests.adapters import HTTPAdapter, Retry + + from novu.api import EventApi + + session = Session() + retries = Retry(total=5, backoff_factor=1, status_forcelist=[ 502, 503, 504 ]) + session.mount("https://api.novu.co", HTTPAdapter(max_retries=retries)) + + event_api = EventApi("https://api.novu.co", "", session=session) + event_api.trigger( + name="", + recipients="", + payload={}, # Your Novu payload goes here + ) diff --git a/novu/api/base.py b/novu/api/base.py index 491f01be..ea6c8e87 100644 --- a/novu/api/base.py +++ b/novu/api/base.py @@ -1,11 +1,15 @@ """This module is used to defined an abstract class for all reusable methods to communicate with the Novu API""" import copy +import dataclasses import logging import os +from datetime import datetime from json.decoder import JSONDecodeError from typing import Generic, List, Optional, Type, TypeVar, Union +from uuid import uuid4 import requests +from tenacity import retry, stop_after_attempt, wait_chain, wait_exponential, wait_fixed from novu.config import NovuConfig from novu.dto.base import CamelCaseDto @@ -17,6 +21,29 @@ _C_co = TypeVar("_C_co", bound=CamelCaseDto, covariant=True) +def retry_backoff(f): + """Decorator to handle retry mechanism based on user configuration""" + + def wrapper(*args): + retry_config: RetryConfig = args[0].retry_config + print(retry_config) + + if retry_config: + func = retry( + wait=wait_chain( + *[wait_fixed(retry_config.initial_delay)] + + [wait_exponential(min=retry_config.wait_min, max=retry_config.wait_max)] + ), + stop=stop_after_attempt(retry_config.retry_max), + )(f) + else: + func: f + + return func(*args) + + return wrapper + + class PaginationIterator(Generic[_C_co]): # pylint: disable=R0902 """The class is a generic iterator which allow to iterate directly on result without looking for pagination during handling. @@ -79,12 +106,32 @@ def __fetch_data(self): self.__index = 0 +@dataclasses.dataclass +class RetryConfig: # pylint: disable=R0903 + """Configuration Class to add Exponential Retry Mechanism""" + + initial_delay: int + """Initial delay for first retry""" + + wait_min: int + """Minimum time to wait""" + + wait_max: int + """Maximum time to wait""" + + retry_max: int + """Maximum number of retries""" + + class Api: # pylint: disable=R0903 """Base class for all API in the Novu client""" requests_timeout: int = 5 """This field allow you to change the :param:`~requests.request.timeout` params which is used during API calls.""" + retry_config: Optional[RetryConfig] = None + """This field allow you to add Exponential Retry Mechanism for API calls.""" + session: Optional[requests.Session] = None """This field allow you to use a :class:`~requests.Session` during API calls.""" @@ -93,6 +140,7 @@ def __init__( url: Optional[str] = None, api_key: Optional[str] = None, requests_timeout: Optional[int] = None, + retry_config: Optional[RetryConfig] = None, session: Optional[requests.Session] = None, ) -> None: config = NovuConfig() @@ -101,11 +149,13 @@ def __init__( api_key = api_key or config.api_key self._url = url - self._headers = {"Authorization": f"ApiKey {api_key}"} + self._headers = {"Authorization": f"ApiKey {api_key}", "Idempotency-Key": str(uuid4())} self.requests_timeout = requests_timeout or int(os.getenv("NOVU_PYTHON_REQUESTS_TIMEOUT", "5")) + self.retry_config = retry_config self.session = session + @retry_backoff def handle_request( self, method: str, @@ -130,6 +180,9 @@ def handle_request( Returns: Return parsed response. """ + print(datetime.now()) + print(self._headers) + if headers: _headers = copy.deepcopy(self._headers) _headers.update(copy.deepcopy(headers)) diff --git a/novu/api/blueprint.py b/novu/api/blueprint.py index 142570bc..ea890042 100644 --- a/novu/api/blueprint.py +++ b/novu/api/blueprint.py @@ -4,7 +4,7 @@ import requests -from novu.api.base import Api +from novu.api.base import Api, RetryConfig from novu.constants import BLUEPRINTS_ENDPOINT from novu.dto.blueprint import BlueprintDto, GroupedBlueprintDto @@ -17,9 +17,12 @@ def __init__( url: Optional[str] = None, api_key: Optional[str] = None, requests_timeout: Optional[int] = None, + retry_config: Optional[RetryConfig] = None, session: Optional[requests.Session] = None, ) -> None: - super().__init__(url=url, api_key=api_key, requests_timeout=requests_timeout, session=session) + super().__init__( + url=url, api_key=api_key, requests_timeout=requests_timeout, retry_config=retry_config, session=session + ) self._blueprint_url = f"{self._url}{BLUEPRINTS_ENDPOINT}" diff --git a/novu/api/change.py b/novu/api/change.py index c423eb1a..52224450 100644 --- a/novu/api/change.py +++ b/novu/api/change.py @@ -5,7 +5,7 @@ import requests -from novu.api.base import Api +from novu.api.base import Api, RetryConfig from novu.constants import CHANGES_ENDPOINT from novu.dto.change import ChangeDto, PaginatedChangeDto @@ -18,9 +18,12 @@ def __init__( url: Optional[str] = None, api_key: Optional[str] = None, requests_timeout: Optional[int] = None, + retry_config: Optional[RetryConfig] = None, session: Optional[requests.Session] = None, ) -> None: - super().__init__(url=url, api_key=api_key, requests_timeout=requests_timeout, session=session) + super().__init__( + url=url, api_key=api_key, requests_timeout=requests_timeout, retry_config=retry_config, session=session + ) self._change_url = f"{self._url}{CHANGES_ENDPOINT}" diff --git a/novu/api/environment.py b/novu/api/environment.py index b5e446af..e496c007 100644 --- a/novu/api/environment.py +++ b/novu/api/environment.py @@ -5,7 +5,7 @@ import requests -from novu.api.base import Api +from novu.api.base import Api, RetryConfig from novu.constants import ENVIRONMENTS_ENDPOINT from novu.dto import EnvironmentApiKeyDto, EnvironmentDto @@ -18,9 +18,12 @@ def __init__( url: Optional[str] = None, api_key: Optional[str] = None, requests_timeout: Optional[int] = None, + retry_config: Optional[RetryConfig] = None, session: Optional[requests.Session] = None, ) -> None: - super().__init__(url=url, api_key=api_key, requests_timeout=requests_timeout, session=session) + super().__init__( + url=url, api_key=api_key, requests_timeout=requests_timeout, retry_config=retry_config, session=session + ) self._environment_url = f"{self._url}{ENVIRONMENTS_ENDPOINT}" diff --git a/novu/api/event.py b/novu/api/event.py index de2983c8..9105e283 100644 --- a/novu/api/event.py +++ b/novu/api/event.py @@ -6,7 +6,7 @@ import requests -from novu.api.base import Api +from novu.api.base import Api, RetryConfig from novu.constants import EVENTS_ENDPOINT from novu.dto.event import EventDto, InputEventDto from novu.dto.topic import TriggerTopicDto @@ -20,9 +20,12 @@ def __init__( url: Optional[str] = None, api_key: Optional[str] = None, requests_timeout: Optional[int] = None, + retry_config: Optional[RetryConfig] = None, session: Optional[requests.Session] = None, ) -> None: - super().__init__(url=url, api_key=api_key, requests_timeout=requests_timeout, session=session) + super().__init__( + url=url, api_key=api_key, requests_timeout=requests_timeout, retry_config=retry_config, session=session + ) self._event_url = f"{self._url}{EVENTS_ENDPOINT}" diff --git a/novu/api/execution_detail.py b/novu/api/execution_detail.py index 9df373d4..ca569bad 100644 --- a/novu/api/execution_detail.py +++ b/novu/api/execution_detail.py @@ -6,7 +6,7 @@ import requests -from novu.api.base import Api +from novu.api.base import Api, RetryConfig from novu.constants import EXECUTION_DETAILS_ENDPOINT from novu.dto import ExecutionDetailDto @@ -19,9 +19,12 @@ def __init__( url: Optional[str] = None, api_key: Optional[str] = None, requests_timeout: Optional[int] = None, + retry_config: Optional[RetryConfig] = None, session: Optional[requests.Session] = None, ) -> None: - super().__init__(url=url, api_key=api_key, requests_timeout=requests_timeout, session=session) + super().__init__( + url=url, api_key=api_key, requests_timeout=requests_timeout, retry_config=retry_config, session=session + ) self._execution_detail_url = f"{self._url}{EXECUTION_DETAILS_ENDPOINT}" diff --git a/novu/api/feed.py b/novu/api/feed.py index 042a2ab9..92b47cbc 100644 --- a/novu/api/feed.py +++ b/novu/api/feed.py @@ -5,7 +5,7 @@ import requests -from novu.api.base import Api +from novu.api.base import Api, RetryConfig from novu.constants import FEEDS_ENDPOINT from novu.dto.feed import FeedDto @@ -18,9 +18,12 @@ def __init__( url: Optional[str] = None, api_key: Optional[str] = None, requests_timeout: Optional[int] = None, + retry_config: Optional[RetryConfig] = None, session: Optional[requests.Session] = None, ) -> None: - super().__init__(url=url, api_key=api_key, requests_timeout=requests_timeout, session=session) + super().__init__( + url=url, api_key=api_key, requests_timeout=requests_timeout, retry_config=retry_config, session=session + ) self._feed_url = f"{self._url}{FEEDS_ENDPOINT}" diff --git a/novu/api/inbound_parse.py b/novu/api/inbound_parse.py index 9f8e827a..fe2d5907 100644 --- a/novu/api/inbound_parse.py +++ b/novu/api/inbound_parse.py @@ -5,7 +5,7 @@ import requests -from novu.api.base import Api +from novu.api.base import Api, RetryConfig from novu.constants import INBOUND_PARSE_ENDPOINT @@ -17,9 +17,12 @@ def __init__( url: Optional[str] = None, api_key: Optional[str] = None, requests_timeout: Optional[int] = None, + retry_config: Optional[RetryConfig] = None, session: Optional[requests.Session] = None, ) -> None: - super().__init__(url=url, api_key=api_key, requests_timeout=requests_timeout, session=session) + super().__init__( + url=url, api_key=api_key, requests_timeout=requests_timeout, retry_config=retry_config, session=session + ) self._inbound_parse_url = f"{self._url}{INBOUND_PARSE_ENDPOINT}" diff --git a/novu/api/integration.py b/novu/api/integration.py index 21b10b03..079184b2 100644 --- a/novu/api/integration.py +++ b/novu/api/integration.py @@ -5,7 +5,7 @@ import requests -from novu.api.base import Api +from novu.api.base import Api, RetryConfig from novu.constants import INTEGRATIONS_ENDPOINT from novu.dto.integration import IntegrationChannelUsageDto, IntegrationDto from novu.enums import Channel, ProviderIdEnum @@ -19,9 +19,12 @@ def __init__( url: Optional[str] = None, api_key: Optional[str] = None, requests_timeout: Optional[int] = None, + retry_config: Optional[RetryConfig] = None, session: Optional[requests.Session] = None, ) -> None: - super().__init__(url=url, api_key=api_key, requests_timeout=requests_timeout, session=session) + super().__init__( + url=url, api_key=api_key, requests_timeout=requests_timeout, retry_config=retry_config, session=session + ) self._integration_url = f"{self._url}{INTEGRATIONS_ENDPOINT}" diff --git a/novu/api/layout.py b/novu/api/layout.py index 912320ad..853f4b4d 100644 --- a/novu/api/layout.py +++ b/novu/api/layout.py @@ -5,7 +5,7 @@ import requests -from novu.api.base import Api +from novu.api.base import Api, RetryConfig from novu.constants import LAYOUTS_ENDPOINT from novu.dto.layout import LayoutDto, PaginatedLayoutDto @@ -18,9 +18,12 @@ def __init__( url: Optional[str] = None, api_key: Optional[str] = None, requests_timeout: Optional[int] = None, + retry_config: Optional[RetryConfig] = None, session: Optional[requests.Session] = None, ) -> None: - super().__init__(url=url, api_key=api_key, requests_timeout=requests_timeout, session=session) + super().__init__( + url=url, api_key=api_key, requests_timeout=requests_timeout, retry_config=retry_config, session=session + ) self._layout_url = f"{self._url}{LAYOUTS_ENDPOINT}" diff --git a/novu/api/message.py b/novu/api/message.py index c19dc0df..6311637a 100644 --- a/novu/api/message.py +++ b/novu/api/message.py @@ -5,7 +5,7 @@ import requests -from novu.api.base import Api, PaginationIterator +from novu.api.base import Api, PaginationIterator, RetryConfig from novu.constants import MESSAGES_ENDPOINT from novu.dto.message import MessageDto, PaginatedMessageDto @@ -18,9 +18,12 @@ def __init__( url: Optional[str] = None, api_key: Optional[str] = None, requests_timeout: Optional[int] = None, + retry_config: Optional[RetryConfig] = None, session: Optional[requests.Session] = None, ) -> None: - super().__init__(url=url, api_key=api_key, requests_timeout=requests_timeout, session=session) + super().__init__( + url=url, api_key=api_key, requests_timeout=requests_timeout, retry_config=retry_config, session=session + ) self._message_url = f"{self._url}{MESSAGES_ENDPOINT}" diff --git a/novu/api/notification.py b/novu/api/notification.py index c24f64e1..0bf640fa 100644 --- a/novu/api/notification.py +++ b/novu/api/notification.py @@ -5,7 +5,7 @@ import requests -from novu.api.base import Api, PaginationIterator +from novu.api.base import Api, PaginationIterator, RetryConfig from novu.constants import NOTIFICATION_ENDPOINT from novu.dto.notification import ( ActivityGraphStatesDto, @@ -22,9 +22,12 @@ def __init__( url: Optional[str] = None, api_key: Optional[str] = None, requests_timeout: Optional[int] = None, + retry_config: Optional[RetryConfig] = None, session: Optional[requests.Session] = None, ) -> None: - super().__init__(url=url, api_key=api_key, requests_timeout=requests_timeout, session=session) + super().__init__( + url=url, api_key=api_key, requests_timeout=requests_timeout, retry_config=retry_config, session=session + ) self._notification_url = f"{self._url}{NOTIFICATION_ENDPOINT}" diff --git a/novu/api/notification_group.py b/novu/api/notification_group.py index 35ae3257..5449889b 100644 --- a/novu/api/notification_group.py +++ b/novu/api/notification_group.py @@ -6,7 +6,7 @@ import requests -from novu.api.base import Api +from novu.api.base import Api, RetryConfig from novu.constants import NOTIFICATION_GROUPS_ENDPOINT from novu.dto.notification_group import ( NotificationGroupDto, @@ -22,9 +22,12 @@ def __init__( url: Optional[str] = None, api_key: Optional[str] = None, requests_timeout: Optional[int] = None, + retry_config: Optional[RetryConfig] = None, session: Optional[requests.Session] = None, ) -> None: - super().__init__(url=url, api_key=api_key, requests_timeout=requests_timeout, session=session) + super().__init__( + url=url, api_key=api_key, requests_timeout=requests_timeout, retry_config=retry_config, session=session + ) self._notification_group_url = f"{self._url}{NOTIFICATION_GROUPS_ENDPOINT}" diff --git a/novu/api/notification_template.py b/novu/api/notification_template.py index 16bb7306..c132fe66 100644 --- a/novu/api/notification_template.py +++ b/novu/api/notification_template.py @@ -6,7 +6,7 @@ import requests -from novu.api.base import Api, PaginationIterator +from novu.api.base import Api, PaginationIterator, RetryConfig from novu.constants import NOTIFICATION_TEMPLATES_ENDPOINT from novu.dto.notification_template import ( NotificationTemplateDto, @@ -23,9 +23,12 @@ def __init__( url: Optional[str] = None, api_key: Optional[str] = None, requests_timeout: Optional[int] = None, + retry_config: Optional[RetryConfig] = None, session: Optional[requests.Session] = None, ) -> None: - super().__init__(url=url, api_key=api_key, requests_timeout=requests_timeout, session=session) + super().__init__( + url=url, api_key=api_key, requests_timeout=requests_timeout, retry_config=retry_config, session=session + ) self._notification_template_url = f"{self._url}{NOTIFICATION_TEMPLATES_ENDPOINT}" diff --git a/novu/api/subscriber.py b/novu/api/subscriber.py index e60956ba..0ec05997 100644 --- a/novu/api/subscriber.py +++ b/novu/api/subscriber.py @@ -5,7 +5,7 @@ import requests -from novu.api.base import Api, PaginationIterator +from novu.api.base import Api, PaginationIterator, RetryConfig from novu.constants import SUBSCRIBERS_ENDPOINT from novu.dto.subscriber import ( BulkResultSubscriberDto, @@ -24,9 +24,12 @@ def __init__( url: Optional[str] = None, api_key: Optional[str] = None, requests_timeout: Optional[int] = None, + retry_config: Optional[RetryConfig] = None, session: Optional[requests.Session] = None, ) -> None: - super().__init__(url=url, api_key=api_key, requests_timeout=requests_timeout, session=session) + super().__init__( + url=url, api_key=api_key, requests_timeout=requests_timeout, retry_config=retry_config, session=session + ) self._subscriber_url = f"{self._url}{SUBSCRIBERS_ENDPOINT}" diff --git a/novu/api/tenant.py b/novu/api/tenant.py index 9e4d2150..1888ab3f 100644 --- a/novu/api/tenant.py +++ b/novu/api/tenant.py @@ -5,7 +5,7 @@ import requests -from novu.api.base import Api, PaginationIterator +from novu.api.base import Api, PaginationIterator, RetryConfig from novu.constants import TENANTS_ENDPOINT from novu.dto.tenant import PaginatedTenantDto, TenantDto @@ -18,9 +18,12 @@ def __init__( url: Optional[str] = None, api_key: Optional[str] = None, requests_timeout: Optional[int] = None, + retry_config: Optional[RetryConfig] = None, session: Optional[requests.Session] = None, ) -> None: - super().__init__(url=url, api_key=api_key, requests_timeout=requests_timeout, session=session) + super().__init__( + url=url, api_key=api_key, requests_timeout=requests_timeout, retry_config=retry_config, session=session + ) self._tenant_url = f"{self._url}{TENANTS_ENDPOINT}" diff --git a/novu/api/topic.py b/novu/api/topic.py index 3e9ccbaa..287bb6d4 100644 --- a/novu/api/topic.py +++ b/novu/api/topic.py @@ -5,7 +5,7 @@ import requests -from novu.api.base import Api +from novu.api.base import Api, RetryConfig from novu.constants import TOPICS_ENDPOINT from novu.dto.topic import PaginatedTopicDto, TopicDto @@ -18,9 +18,12 @@ def __init__( url: Optional[str] = None, api_key: Optional[str] = None, requests_timeout: Optional[int] = None, + retry_config: Optional[RetryConfig] = None, session: Optional[requests.Session] = None, ) -> None: - super().__init__(url=url, api_key=api_key, requests_timeout=requests_timeout, session=session) + super().__init__( + url=url, api_key=api_key, requests_timeout=requests_timeout, retry_config=retry_config, session=session + ) self._topic_url = f"{self._url}{TOPICS_ENDPOINT}" diff --git a/poetry.lock b/poetry.lock index 6ef7cabe..860cd0b1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -962,7 +962,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -970,15 +969,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -995,7 +987,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1003,7 +994,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -1340,6 +1330,20 @@ files = [ [package.dependencies] pbr = ">=2.0.0,<2.1.0 || >2.1.0" +[[package]] +name = "tenacity" +version = "8.2.3" +description = "Retry code until it succeeds" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tenacity-8.2.3-py3-none-any.whl", hash = "sha256:ce510e327a630c9e1beaf17d42e6ffacc88185044ad85cf74c0a8887c6a0f88c"}, + {file = "tenacity-8.2.3.tar.gz", hash = "sha256:5398ef0d78e63f40007c1fb4c0bff96e1911394d2fa8d194f77619c05ff6cc8a"}, +] + +[package.extras] +doc = ["reno", "sphinx", "tornado (>=4.5)"] + [[package]] name = "toml" version = "0.10.2" @@ -1453,4 +1457,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "5fe72803e6dd238a9083886df81e36f4918cd4a6cf36e528c774a96e786ab50b" +content-hash = "b938d48cf3c6a65a38b129778447bc8d762704fe29973d9fcba15076bb1ce16d" diff --git a/pyproject.toml b/pyproject.toml index 45adb9d8..311a407c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,7 @@ classifiers = [ python = "^3.8" requests = "^2.28.2" +tenacity = "^8.2.3" [tool.poetry.group.dev.dependencies] bandit = "^1.7.4" From 0dec653d2615c8ee5bb890efdc5b43d7da636f7c Mon Sep 17 00:00:00 2001 From: im-perativa Date: Fri, 27 Oct 2023 07:55:40 +0700 Subject: [PATCH 2/3] fix: remove test print method --- novu/api/base.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/novu/api/base.py b/novu/api/base.py index ea6c8e87..61849038 100644 --- a/novu/api/base.py +++ b/novu/api/base.py @@ -3,7 +3,6 @@ import dataclasses import logging import os -from datetime import datetime from json.decoder import JSONDecodeError from typing import Generic, List, Optional, Type, TypeVar, Union from uuid import uuid4 @@ -26,7 +25,6 @@ def retry_backoff(f): def wrapper(*args): retry_config: RetryConfig = args[0].retry_config - print(retry_config) if retry_config: func = retry( @@ -180,8 +178,6 @@ def handle_request( Returns: Return parsed response. """ - print(datetime.now()) - print(self._headers) if headers: _headers = copy.deepcopy(self._headers) From 330327f2d85ce6f62c33ee8027461246cc1f9b90 Mon Sep 17 00:00:00 2001 From: im-perativa Date: Fri, 27 Oct 2023 09:16:24 +0700 Subject: [PATCH 3/3] test: add exponential retry test --- novu/api/base.py | 11 +++++++---- tests/api/test_base.py | 31 ++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/novu/api/base.py b/novu/api/base.py index 61849038..7efcc157 100644 --- a/novu/api/base.py +++ b/novu/api/base.py @@ -23,7 +23,7 @@ def retry_backoff(f): """Decorator to handle retry mechanism based on user configuration""" - def wrapper(*args): + def wrapper(*args, **kwargs): retry_config: RetryConfig = args[0].retry_config if retry_config: @@ -35,9 +35,9 @@ def wrapper(*args): stop=stop_after_attempt(retry_config.retry_max), )(f) else: - func: f + func = f - return func(*args) + return func(*args, **kwargs) return wrapper @@ -147,7 +147,10 @@ def __init__( api_key = api_key or config.api_key self._url = url - self._headers = {"Authorization": f"ApiKey {api_key}", "Idempotency-Key": str(uuid4())} + if retry_config: + self._headers = {"Authorization": f"ApiKey {api_key}", "Idempotency-Key": str(uuid4())} + else: + self._headers = {"Authorization": f"ApiKey {api_key}"} self.requests_timeout = requests_timeout or int(os.getenv("NOVU_PYTHON_REQUESTS_TIMEOUT", "5")) self.retry_config = retry_config diff --git a/tests/api/test_base.py b/tests/api/test_base.py index 0fd42a71..09495bf0 100644 --- a/tests/api/test_base.py +++ b/tests/api/test_base.py @@ -1,8 +1,10 @@ from unittest import TestCase, mock +from uuid import uuid4 from requests.exceptions import HTTPError +from tenacity import RetryError -from novu.api.base import Api +from novu.api.base import Api, RetryConfig from novu.config import NovuConfig from tests.factories import MockResponse @@ -98,3 +100,30 @@ def test_use_requests_session(self, mock_request: mock.MagicMock) -> None: params=None, timeout=5, ) + + @mock.patch("requests.request") + def test_use_retry_backoff_mechanism(self, mock_request: mock.MagicMock) -> None: + mock_request.return_value = MockResponse(500, raise_on_json_decode=True) + idempotency_key_header = str(uuid4()) + + api = Api(retry_config=RetryConfig(initial_delay=1, wait_min=2, wait_max=100, retry_max=4)) + + self.assertRaises( + RetryError, + lambda: api.handle_request("GET", api._url, headers={"Idempotency-Key": idempotency_key_header}), + ) + + mock_request.assert_called_with( + method="GET", + url="sample.novu.com", + headers={"Authorization": "ApiKey api-key", "Idempotency-Key": idempotency_key_header}, + json=None, + params=None, + timeout=5, + ) + + idempotency_key_headers = [req.kwargs["headers"]["Idempotency-Key"] for req in mock_request.call_args_list] + + self.assertEqual(idempotency_key_headers, [idempotency_key_header] * 4) + + self.assertEqual(4, mock_request.call_count)