feat(providers): added retry mechanism to google chat provider to ove…

…rcome rate limiting (#2572)

Co-authored-by: Shahar Glazner <[email protected]>

keep/providers/google_chat_provider/google_chat_provider.py

@@ -1,4 +1,7 @@
+import http
 import os
+import time
+
 import pydantic
 import dataclasses
 import requests
@@ -65,18 +68,32 @@ def _notify(self, message="", **kwargs: dict):
         if not message:
             raise ProviderException("Message is required")
 
+        def __send_message(url, body, headers, retries=3):
+            for attempt in range(retries):
+                try:
+                    resp = requests.post(url, json=body, headers=headers)
+                    if resp.status_code == http.HTTPStatus.OK:
+                        return resp
+
+                    self.logger.warning(f"Attempt {attempt + 1} failed with status code {resp.status_code}")
+
+                except requests.exceptions.RequestException as e:
+                    self.logger.error(f"Attempt {attempt + 1} failed: {e}")
+
+                if attempt < retries - 1:
+                    time.sleep(1)
+
+            raise requests.exceptions.RequestException(f"Failed to notify message after {retries} attempts")
+
         payload = {
             "text": message,
         }
 
-        requestHeaders = {"Content-Type": "application/json; charset=UTF-8"}
-
-        response = requests.post(webhook_url, json=payload, headers=requestHeaders)
+        request_headers = {"Content-Type": "application/json; charset=UTF-8"}
 
-        if not response.ok:
-            raise ProviderException(
-                f"Failed to notify message to Google Chat: {response.text}"
-            )
+        response = __send_message(webhook_url, body=payload, headers=request_headers)
+        if response.status_code != http.HTTPStatus.OK:
+            raise ProviderException(f"Failed to notify message to Google Chat: {response.text}")
 
         self.logger.debug("Alert message sent to Google Chat successfully")
         return "Alert message sent to Google Chat successfully"