From ab59d58fe6035a0e47f64d91171256c521f8acbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Bompard?= Date: Mon, 3 Jun 2024 15:59:08 +0200 Subject: [PATCH] Cap the retry interval in the crawler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Aurélien Bompard --- mirrormanager2/crawler/connector.py | 3 ++- mirrormanager2/crawler/constants.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/mirrormanager2/crawler/connector.py b/mirrormanager2/crawler/connector.py index fee09b004..53104851e 100755 --- a/mirrormanager2/crawler/connector.py +++ b/mirrormanager2/crawler/connector.py @@ -6,7 +6,7 @@ from mirrormanager2 import lib as mmlib -from .constants import RETRIES +from .constants import RETRIES, RETRIES_MAX_INTERVAL logger = logging.getLogger(__name__) @@ -74,6 +74,7 @@ def _get_file(self, url): backoff.expo, TryLater, max_tries=RETRIES, + max_value=RETRIES_MAX_INTERVAL, on_backoff=_on_backoff, on_giveup=_on_giveup, logger=None, # custom logging diff --git a/mirrormanager2/crawler/constants.py b/mirrormanager2/crawler/constants.py index afdc4c615..02463ac23 100755 --- a/mirrormanager2/crawler/constants.py +++ b/mirrormanager2/crawler/constants.py @@ -14,3 +14,4 @@ # Number of times to retry a connection RETRIES = 10 +RETRIES_MAX_INTERVAL = 10 # in seconds