add exponential backoff

ArtemiyGranat · Jan 27, 2024 · 5cc27dd · 5cc27dd
1 parent 0346df2
commit 5cc27dd
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 2 deletions.
diff --git a/README.md b/README.md
@@ -7,3 +7,7 @@ rye shell
 rye sync
 rye run uvicorn --app-dir=backend/src main:app
 ```
+
+## TODO
+* [ ] Distributed scraping
+* [ ] ...
diff --git a/scraper/src/utils.py b/scraper/src/utils.py
@@ -7,7 +7,7 @@
 from imagehash import phash
 from PIL import Image
 from pydantic import BaseModel
-from tenacity import retry, stop_after_attempt
+from tenacity import retry, stop_after_attempt, wait_exponential
 
 import shared.entities as entities
 
@@ -18,7 +18,7 @@ class ScraperInfo(BaseModel):
     images_scraped: int
 
 
-@retry(stop=stop_after_attempt(7))
+@retry(stop=stop_after_attempt(7), wait=wait_exponential(multiplier=1, max=60))
 async def get_with_retry(url: str):
     return await ctx.http_client.get(url)