fix: provider last pull time

keep/api/consts.py

@@ -3,6 +3,9 @@
 from keep.api.models.db.preset import PresetDto, StaticPresetsId
 
 RUNNING_IN_CLOUD_RUN = os.environ.get("K_SERVICE") is not None
+PROVIDER_PULL_INTERVAL_DAYS = int(
+    os.environ.get("KEEP_PULL_INTERVAL", 7)
+)  # maximum once a week
 STATIC_PRESETS = {
     "feed": PresetDto(
         id=StaticPresetsId.FEED_PRESET_ID.value,

keep/api/core/db.py

@@ -493,6 +493,31 @@ def get_raw_workflow(tenant_id: str, workflow_id: str) -> str:
     return workflow.workflow_raw
 
 
+def update_provider_last_pull_time(tenant_id: str, provider_id: str):
+    extra = {"tenant_id": tenant_id, "provider_id": provider_id}
+    logger.info("Updating provider last pull time", extra=extra)
+    with Session(engine) as session:
+        provider = session.exec(
+            select(Provider).where(
+                Provider.tenant_id == tenant_id, Provider.id == provider_id
+            )
+        ).first()
+
+        if not provider:
+            logger.warning(
+                "Could not update provider last pull time since provider does not exist",
+                extra=extra,
+            )
+
+        try:
+            provider.last_pull_time = datetime.now(tz=timezone.utc)
+            session.commit()
+        except Exception:
+            logger.exception("Failed to update provider last pull time", extra=extra)
+            raise
+    logger.info("Successfully updated provider last pull time", extra=extra)
+
+
 def get_installed_providers(tenant_id: str) -> List[Provider]:
     with Session(engine) as session:
         providers = session.exec(

keep/api/models/db/migrations/versions/2024-08-05-13-09_4147d9e706c0.py

@@ -0,0 +1,28 @@
+"""Provider last pull time
+
+Revision ID: 4147d9e706c0
+Revises: 92f4f93f2140
+Create Date: 2024-08-05 13:09:18.851721
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "4147d9e706c0"
+down_revision = "92f4f93f2140"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("provider", sa.Column("last_pull_time", sa.DateTime(), nullable=True))
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("provider", "last_pull_time")
+    # ### end Alembic commands ###

keep/api/models/db/provider.py

@@ -20,6 +20,7 @@ class Provider(SQLModel, table=True):
         sa_column=Column(JSON)
     )  # scope name is key and value is either True if validated or string with error message, e.g: {"read": True, "write": "error message"}
     consumer: bool = False
+    last_pull_time: Optional[datetime]
 
     class Config:
         orm_mode = True

keep/api/models/provider.py

@@ -37,6 +37,7 @@ class Provider(BaseModel):
     methods: list[ProviderMethod] = []
     installed_by: str | None = None
     installation_time: datetime | None = None
+    last_pull_time: datetime | None = None
     docs: str | None = None
     tags: list[
         Literal["alert", "ticketing", "messaging", "data", "queue", "topology"]

keep/api/routes/preset.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import uuid
+from datetime import datetime, timezone
 
 from fastapi import (
     APIRouter,
@@ -13,10 +14,14 @@
 from pydantic import BaseModel
 from sqlmodel import Session, select
 
-from keep.api.consts import STATIC_PRESETS
+from keep.api.consts import PROVIDER_PULL_INTERVAL_DAYS, STATIC_PRESETS
 from keep.api.core.db import get_preset_by_name as get_preset_by_name_db
 from keep.api.core.db import get_presets as get_presets_db
-from keep.api.core.db import get_session, update_preset_options
+from keep.api.core.db import (
+    get_session,
+    update_preset_options,
+    update_provider_last_pull_time,
+)
 from keep.api.core.dependencies import AuthenticatedEntity, AuthVerifier
 from keep.api.models.alert import AlertDto
 from keep.api.models.db.preset import Preset, PresetDto, PresetOption
@@ -52,19 +57,28 @@ def pull_data_from_providers(
     )
 
     for provider in ProvidersFactory.get_installed_providers(tenant_id=tenant_id):
+        extra = {
+            "provider_type": provider.type,
+            "provider_id": provider.id,
+            "tenant_id": tenant_id,
+        }
+
+        if provider.last_pull_time is not None:
+            now = datetime.now(tz=timezone.utc)
+            if (now - provider.last_pull_time).days <= PROVIDER_PULL_INTERVAL_DAYS:
+                logger.info(
+                    "Skipping provider data pull",
+                    extra=extra,
+                )
+                continue
+
         provider_class = ProvidersFactory.get_provider(
             context_manager=context_manager,
             provider_id=provider.id,
             provider_type=provider.type,
             provider_config=provider.details,
         )
 
-        extra = {
-            "provider_type": provider.type,
-            "provider_id": provider.id,
-            "tenant_id": tenant_id,
-        }
-
         logger.info(
             f"Pulling alerts from provider {provider.type} ({provider.id})",
             extra=extra,
@@ -103,6 +117,7 @@ def pull_data_from_providers(
                 alert,
                 notify_client=False,
             )
+        update_provider_last_pull_time(tenant_id=tenant_id, provider_id=provider.id)
 
 
 @router.get(

keep/providers/providers_factory.py

@@ -400,6 +400,7 @@ def get_installed_providers(
             provider_copy.id = p.id
             provider_copy.installed_by = p.installed_by
             provider_copy.installation_time = p.installation_time
+            provider_copy.last_pull_time = p.last_pull_time
             try:
                 provider_auth = {"name": p.name}
                 if include_details: