modify exp backoff implementation (#1513)

* Feature/tweak actions (#1507) * up * tweak actions * modify exp backoff impl --------- Co-authored-by: emrgnt-cmplxty <[email protected]>
SciPhi-AI · Oct 28, 2024 · f46d4bf · f46d4bf
1 parent 080d8cb
commit f46d4bf
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 7 deletions.
diff --git a/py/core/base/providers/embedding.py b/py/core/base/providers/embedding.py
@@ -14,6 +14,7 @@
     VectorSearchResult,
     default_embedding_prefixes,
 )
+import random
 from .base import Provider, ProviderConfig
 
 logger = logging.getLogger()
@@ -79,7 +80,7 @@ async def _execute_with_backoff_async(self, task: dict[str, Any]):
                 retries += 1
                 if retries == self.config.max_retries:
                     raise
-                await asyncio.sleep(backoff)
+                await asyncio.sleep(random.uniform(0, backoff))
                 backoff = min(backoff * 2, self.config.max_backoff)
 
     def _execute_with_backoff_sync(self, task: dict[str, Any]):
@@ -97,7 +98,7 @@ def _execute_with_backoff_sync(self, task: dict[str, Any]):
                 retries += 1
                 if retries == self.config.max_retries:
                     raise
-                time.sleep(backoff)
+                time.sleep(random.uniform(0, backoff))
                 backoff = min(backoff * 2, self.config.max_backoff)
 
     @abstractmethod

diff --git a/py/core/base/providers/llm.py b/py/core/base/providers/llm.py
@@ -4,7 +4,7 @@
 from abc import abstractmethod
 from concurrent.futures import ThreadPoolExecutor
 from typing import Any, AsyncGenerator, Generator, Optional
-
+import random
 from litellm import AuthenticationError
 
 from core.base.abstractions import (
@@ -67,7 +67,7 @@ async def _execute_with_backoff_async(self, task: dict[str, Any]):
                 retries += 1
                 if retries == self.config.max_retries:
                     raise
-                await asyncio.sleep(backoff)
+                await asyncio.sleep(random.uniform(0, backoff))
                 backoff = min(backoff * 2, self.config.max_backoff)
 
     async def _execute_with_backoff_async_stream(
@@ -90,7 +90,7 @@ async def _execute_with_backoff_async_stream(
                 retries += 1
                 if retries == self.config.max_retries:
                     raise
-                await asyncio.sleep(backoff)
+                await asyncio.sleep(random.uniform(0, backoff))
                 backoff = min(backoff * 2, self.config.max_backoff)
 
     def _execute_with_backoff_sync(self, task: dict[str, Any]):
@@ -106,7 +106,7 @@ def _execute_with_backoff_sync(self, task: dict[str, Any]):
                 retries += 1
                 if retries == self.config.max_retries:
                     raise
-                time.sleep(backoff)
+                time.sleep(random.uniform(0, backoff))
                 backoff = min(backoff * 2, self.config.max_backoff)
 
     def _execute_with_backoff_sync_stream(
@@ -125,7 +125,7 @@ def _execute_with_backoff_sync_stream(
                 retries += 1
                 if retries == self.config.max_retries:
                     raise
-                time.sleep(backoff)
+                time.sleep(random.uniform(0, backoff))
                 backoff = min(backoff * 2, self.config.max_backoff)
 
     @abstractmethod