diff --git a/src/pymedx/api.py b/src/pymedx/api.py index ca68aa2..a105667 100644 --- a/src/pymedx/api.py +++ b/src/pymedx/api.py @@ -1,6 +1,8 @@ """API module for PubMed.""" import datetime import itertools +import random +import time from typing import Any, Dict, Iterable, List, Union, cast @@ -50,6 +52,7 @@ def __init__( # Keep track of the rate limit self._rateLimit: int = 3 + self._maxRetries: int = 10 self._requestsMade: List[datetime.datetime] = [] self.parameters: Dict[str, Union[str, int, List[str]]] # Define the standard / default query parameters @@ -154,6 +157,23 @@ def _exceededRateLimit(self) -> bool: # than the rate limit return len(self._requestsMade) > self._rateLimit + def _wait_to_retry(self, attempt: int) -> None: + """ + Calculate and wait the appropriate amount of time before a retry. + + Parameters. + ---------- + attempt: int + The current attempt number. + """ + backoff_time = min( + 2**attempt, 32 + ) # Exponential backoff, capped at 32 seconds + + backoff_time += random.uniform(0, 1) # Add jitter + + time.sleep(backoff_time) + def _get( self, url: str, @@ -180,27 +200,38 @@ def _get( be parsed before returning, otherwise a string is returend """ - # Make sure the rate limit is not exceeded + attempt = 0 + while self._exceededRateLimit(): pass - # Set the response mode + while attempt < self._maxRetries: + try: + # Set the response mode + parameters["retmode"] = output - parameters["retmode"] = output + # Make the request to PubMed + response = requests.get(f"{BASE_URL}{url}", params=parameters) + # Check for any errors + response.raise_for_status() - # Make the request to PubMed - response = requests.get(f"{BASE_URL}{url}", params=parameters) - # Check for any errors - response.raise_for_status() + # Add this request to the list of requests made + self._requestsMade.append(datetime.datetime.now()) - # Add this request to the list of requests made - self._requestsMade.append(datetime.datetime.now()) + # Return the response + if output == "json": + return response.json() + else: + return response.text - # Return the response - if output == "json": - return response.json() - else: - return response.text + except Exception: + self._wait_to_retry(attempt) + attempt += 1 + + raise Exception( + f"Failed to retrieve data from {BASE_URL}{url} " + f"after {self._maxRetries} attempts" + ) def _getArticles( self, article_ids: List[str]