diff --git a/keep/contextmanager/contextmanager.py b/keep/contextmanager/contextmanager.py
index 018b4e204..3283ea506 100644
--- a/keep/contextmanager/contextmanager.py
+++ b/keep/contextmanager/contextmanager.py
@@ -5,6 +5,7 @@
 import json5
 from pympler.asizeof import asizeof
 
+from keep.api.core.config import config
 from keep.api.core.db import get_last_workflow_execution_by_workflow_id, get_session
 from keep.api.logging import WorkflowLoggerAdapter
 from keep.api.models.alert import AlertDto
@@ -75,6 +76,13 @@ def __init__(
         self._api_key = None
         self.__loggers = {}
 
+    @property
+    def api_url(self):
+        """
+        The URL of the Keep API
+        """
+        return config("KEEP_API_URL")
+
     @property
     def api_key(self):
         # avoid circular import
diff --git a/keep/providers/base/base_provider.py b/keep/providers/base/base_provider.py
index 554c0d705..f92588d2e 100644
--- a/keep/providers/base/base_provider.py
+++ b/keep/providers/base/base_provider.py
@@ -570,6 +570,15 @@ def setup_webhook(
         """
         raise NotImplementedError("setup_webhook() method not implemented")
 
+    def clean_up(self):
+        """
+        Clean up the provider.
+
+        Raises:s
+            NotImplementedError: for providers who does not implement this method.
+        """
+        raise NotImplementedError("clean_up() method not implemented")
+
     @staticmethod
     def get_alert_schema() -> dict:
         """
diff --git a/keep/providers/pagerduty_provider/pagerduty_provider.py b/keep/providers/pagerduty_provider/pagerduty_provider.py
index d58ddff9a..c2a69b438 100644
--- a/keep/providers/pagerduty_provider/pagerduty_provider.py
+++ b/keep/providers/pagerduty_provider/pagerduty_provider.py
@@ -475,6 +475,41 @@ def _trigger_incident(
             )
             # This will give us a better error message in Keep workflows
             raise Exception(r.text) from e
+        
+    def clean_up(self):
+        """
+        Clean up the provider.
+        It will remove the webhook from PagerDuty if it exists.
+        """
+        self.logger.info("Cleaning up %s provider with id %s", self.PROVIDER_DISPLAY_NAME, self.provider_id)
+        keep_webhook_incidents_api_url = (
+                f"{self.context_manager.api_url}/incidents/event/{self.provider_type}?provider_id={self.provider_id}"
+            )
+        headers = self.__get_headers()
+        request = requests.get(self.SUBSCRIPTION_API_URL, headers=headers)
+        if not request.ok:
+            raise Exception("Could not get existing webhooks")
+        existing_webhooks = request.json().get("webhook_subscriptions", [])
+        webhook_exists = next(
+            iter(
+                [
+                    webhook
+                    for webhook in existing_webhooks
+                    if keep_webhook_incidents_api_url == webhook.get("delivery_method", {}).get("url", "")
+                ]
+            ),
+            False,
+        )
+        if webhook_exists:
+            self.logger.info("Webhook exists, removing it")
+            webhook_id = webhook_exists.get("id")
+            request = requests.delete(
+                f"{self.SUBSCRIPTION_API_URL}/{webhook_id}", headers=headers
+            )
+            if not request.ok:
+                raise Exception("Could not remove existing webhook")
+            self.logger.info("Webhook removed", extra={"webhook_id": webhook_id})
+
 
     def dispose(self):
         """
diff --git a/keep/providers/providers_service.py b/keep/providers/providers_service.py
index 30b801cf4..cf47f5266 100644
--- a/keep/providers/providers_service.py
+++ b/keep/providers/providers_service.py
@@ -236,34 +236,48 @@ def update_provider(
     def delete_provider(
         tenant_id: str, provider_id: str, session: Session, allow_provisioned=False
     ):
-        provider = session.exec(
+        provider_model: Provider = session.exec(
             select(Provider).where(
                 (Provider.tenant_id == tenant_id) & (Provider.id == provider_id)
             )
         ).one_or_none()
 
-        if not provider:
+        if not provider_model:
             raise HTTPException(404, detail="Provider not found")
 
-        if provider.provisioned and not allow_provisioned:
+        if provider_model.provisioned and not allow_provisioned:
             raise HTTPException(403, detail="Cannot delete a provisioned provider")
 
         context_manager = ContextManager(tenant_id=tenant_id)
         secret_manager = SecretManagerFactory.get_secret_manager(context_manager)
+        config = secret_manager.read_secret(provider_model.configuration_key, is_json=True)
+        provider = ProvidersFactory.get_provider(
+                context_manager, provider_model.id, provider_model.type, config
+            )
 
         try:
-            secret_manager.delete_secret(provider.configuration_key)
+            secret_manager.delete_secret(provider_model.configuration_key)
         except Exception:
             logger.exception("Failed to delete the provider secret")
 
-        if provider.consumer:
+        if provider_model.consumer:
             try:
                 event_subscriber = EventSubscriber.get_instance()
-                event_subscriber.remove_consumer(provider)
+                event_subscriber.remove_consumer(provider_model)
             except Exception:
                 logger.exception("Failed to unregister provider as a consumer")
 
-        session.delete(provider)
+        try:
+            provider.clean_up()
+        except NotImplementedError:
+            logger.info(
+                "Being deleted provider of type %s does not have a clean_up method",
+                provider_model.type
+            )
+        except Exception:
+            logger.exception(msg="Failed to clean up provider")
+
+        session.delete(provider_model)
         session.commit()
 
     @staticmethod