diff --git a/.github/workflows/test-pr-e2e.yml b/.github/workflows/test-pr-e2e.yml index 6638ac00d..491745d45 100644 --- a/.github/workflows/test-pr-e2e.yml +++ b/.github/workflows/test-pr-e2e.yml @@ -138,14 +138,14 @@ jobs: verbose: true - name: Dump backend logs - if: failure() + if: always() run: | docker compose --project-directory . -f tests/e2e_tests/docker-compose-e2e-${{ matrix.db_type }}.yml logs keep-backend > backend_logs-${{ matrix.db_type }}.txt docker compose --project-directory . -f tests/e2e_tests/docker-compose-e2e-${{ matrix.db_type }}.yml logs keep-frontend > frontend_logs-${{ matrix.db_type }}.txt continue-on-error: true - name: Upload test artifacts on failure - if: failure() + if: always() uses: actions/upload-artifact@v3 with: name: test-artifacts diff --git a/keep/api/bl/enrichments.py b/keep/api/bl/enrichments.py index 443e30aa8..8ec09e1d0 100644 --- a/keep/api/bl/enrichments.py +++ b/keep/api/bl/enrichments.py @@ -8,7 +8,7 @@ from sqlmodel import Session from keep.api.core.db import enrich_alert as enrich_alert_db -from keep.api.core.db import get_enrichment, get_mapping_rule_by_id +from keep.api.core.db import get_enrichment_with_session, get_mapping_rule_by_id from keep.api.core.elastic import ElasticClient from keep.api.models.alert import AlertDto from keep.api.models.db.alert import AlertActionType @@ -415,7 +415,9 @@ def dispose_enrichments(self, fingerprint: str): Dispose of enrichments from the alert """ self.logger.debug("disposing enrichments", extra={"fingerprint": fingerprint}) - enrichments = get_enrichment(self.tenant_id, fingerprint) + enrichments = get_enrichment_with_session( + self.db_session, self.tenant_id, fingerprint + ) if not enrichments or not enrichments.enrichments: self.logger.debug( "no enrichments to dispose", extra={"fingerprint": fingerprint} diff --git a/keep/api/core/db.py b/keep/api/core/db.py index 524a530bb..023427c75 100644 --- a/keep/api/core/db.py +++ b/keep/api/core/db.py @@ -815,20 +815,7 @@ def count_alerts( def get_enrichment(tenant_id, fingerprint, refresh=False): with Session(engine) as session: - alert_enrichment = session.exec( - select(AlertEnrichment) - .where(AlertEnrichment.tenant_id == tenant_id) - .where(AlertEnrichment.alert_fingerprint == fingerprint) - ).first() - - if refresh: - try: - session.refresh(alert_enrichment) - except Exception: - logger.exception( - "Failed to refresh enrichment", - extra={"tenant_id": tenant_id, "fingerprint": fingerprint}, - ) + return get_enrichment_with_session(session, tenant_id, fingerprint, refresh) return alert_enrichment @@ -851,12 +838,20 @@ def get_enrichments( return result -def get_enrichment_with_session(session, tenant_id, fingerprint): +def get_enrichment_with_session(session, tenant_id, fingerprint, refresh=False): alert_enrichment = session.exec( select(AlertEnrichment) .where(AlertEnrichment.tenant_id == tenant_id) .where(AlertEnrichment.alert_fingerprint == fingerprint) ).first() + if refresh: + try: + session.refresh(alert_enrichment) + except Exception: + logger.exception( + "Failed to refresh enrichment", + extra={"tenant_id": tenant_id, "fingerprint": fingerprint}, + ) return alert_enrichment @@ -1360,7 +1355,10 @@ def assign_alert_to_group( enrich_alert( tenant_id, fingerprint, - enrichments={"group_expired": True}, + enrichments={ + "group_expired": True, + "status": AlertStatus.RESOLVED.value, # Shahar: expired groups should be resolved also in elasticsearch + }, action_type=AlertActionType.GENERIC_ENRICH, # TODO: is this a live code? action_callee="system", action_description="Enriched group with group_expired flag", @@ -2445,7 +2443,9 @@ def write_pmi_matrix_to_db(tenant_id: str, pmi_matrix_df: pd.DataFrame) -> bool: # Query for existing entries to differentiate between updates and inserts existing_entries = session.query(PMIMatrix).filter_by(tenant_id=tenant_id).all() - existing_entries_set = {(entry.fingerprint_i, entry.fingerprint_j) for entry in existing_entries} + existing_entries_set = { + (entry.fingerprint_i, entry.fingerprint_j) for entry in existing_entries + } for fingerprint_i in pmi_matrix_df.index: for fingerprint_j in pmi_matrix_df.columns: @@ -2455,7 +2455,7 @@ def write_pmi_matrix_to_db(tenant_id: str, pmi_matrix_df: pd.DataFrame) -> bool: "tenant_id": tenant_id, "fingerprint_i": fingerprint_i, "fingerprint_j": fingerprint_j, - "pmi": pmi + "pmi": pmi, } if (fingerprint_i, fingerprint_j) in existing_entries_set: @@ -2466,11 +2466,11 @@ def write_pmi_matrix_to_db(tenant_id: str, pmi_matrix_df: pd.DataFrame) -> bool: # Update existing records if pmi_entries_to_update: session.bulk_update_mappings(PMIMatrix, pmi_entries_to_update) - + # Insert new records if pmi_entries_to_insert: session.bulk_insert_mappings(PMIMatrix, pmi_entries_to_insert) - + session.commit() return True @@ -2510,69 +2510,6 @@ def get_pmi_values( return pmi_values -def get_alert_firing_time(tenant_id: str, fingerprint: str) -> timedelta: - with Session(engine) as session: - # Get the latest alert for this fingerprint - latest_alert = ( - session.query(Alert) - .filter(Alert.tenant_id == tenant_id) - .filter(Alert.fingerprint == fingerprint) - .order_by(Alert.timestamp.desc()) - .first() - ) - - if not latest_alert: - return timedelta() - - # Extract status from the event column - latest_status = latest_alert.event.get("status") - - # If the latest status is not 'firing', return 0 - if latest_status != "firing": - return timedelta() - - # Find the last time it wasn't firing - last_non_firing = ( - session.query(Alert) - .filter(Alert.tenant_id == tenant_id) - .filter(Alert.fingerprint == fingerprint) - .filter(func.json_extract(Alert.event, "$.status") != "firing") - .order_by(Alert.timestamp.desc()) - .first() - ) - - if last_non_firing: - # Find the next firing alert after the last non-firing alert - next_firing = ( - session.query(Alert) - .filter(Alert.tenant_id == tenant_id) - .filter(Alert.fingerprint == fingerprint) - .filter(Alert.timestamp > last_non_firing.timestamp) - .filter(func.json_extract(Alert.event, "$.status") == "firing") - .order_by(Alert.timestamp.asc()) - .first() - ) - if next_firing: - return datetime.now(tz=timezone.utc) - next_firing.timestamp.replace( - tzinfo=timezone.utc - ) - else: - # If no firing alert after the last non-firing, return 0 - return timedelta() - else: - # If all alerts are firing, use the earliest alert time - earliest_alert = ( - session.query(Alert) - .filter(Alert.tenant_id == tenant_id) - .filter(Alert.fingerprint == fingerprint) - .order_by(Alert.timestamp.asc()) - .first() - ) - return datetime.now(tz=timezone.utc) - earliest_alert.timestamp.replace( - tzinfo=timezone.utc - ) - - def update_incident_summary(incident_id: UUID, summary: str) -> Incident: with Session(engine) as session: incident = session.exec( diff --git a/keep/api/models/alert.py b/keep/api/models/alert.py index 4128c6c0e..333f95174 100644 --- a/keep/api/models/alert.py +++ b/keep/api/models/alert.py @@ -103,6 +103,7 @@ class AlertDto(BaseModel): status: AlertStatus severity: AlertSeverity lastReceived: str + firingStartTime: str | None = None environment: str = "undefined" isDuplicate: bool | None = None duplicateReason: str | None = None diff --git a/keep/api/routes/alerts.py b/keep/api/routes/alerts.py index dfcb08664..b9370a688 100644 --- a/keep/api/routes/alerts.py +++ b/keep/api/routes/alerts.py @@ -31,7 +31,12 @@ get_pusher_client, ) from keep.api.core.elastic import ElasticClient -from keep.api.models.alert import AlertDto, DeleteRequestBody, EnrichAlertRequestBody, UnEnrichAlertRequestBody +from keep.api.models.alert import ( + AlertDto, + DeleteRequestBody, + EnrichAlertRequestBody, + UnEnrichAlertRequestBody, +) from keep.api.models.db.alert import AlertActionType from keep.api.models.search_alert import SearchAlertsRequest from keep.api.tasks.process_event_task import process_event @@ -499,12 +504,11 @@ def enrich_alert( return {"status": "failed"} - @router.post( "/unenrich", description="Un-Enrich an alert", ) -def enrich_alert( +def unenrich_alert( enrich_data: UnEnrichAlertRequestBody, pusher_client: Pusher = Depends(get_pusher_client), authenticated_entity: AuthenticatedEntity = Depends(AuthVerifier(["write:alert"])), @@ -534,7 +538,9 @@ def enrich_alert( enrichement_bl = EnrichmentsBl(tenant_id) if "status" in enrich_data.enrichments: action_type = AlertActionType.STATUS_UNENRICH - action_description = f"Alert status was un-enriched by {authenticated_entity.email}" + action_description = ( + f"Alert status was un-enriched by {authenticated_entity.email}" + ) elif "note" in enrich_data.enrichments: action_type = AlertActionType.UNCOMMENT action_description = f"Comment removed by {authenticated_entity.email}" @@ -549,8 +555,9 @@ def enrich_alert( enrichments = enrichments_object.enrichments new_enrichments = { - key: value for key, value in enrichments.items() - if key not in enrich_data.enrichments + key: value + for key, value in enrichments.items() + if key not in enrich_data.enrichments } enrichement_bl.enrich_alert( @@ -559,7 +566,7 @@ def enrich_alert( action_type=action_type, action_callee=authenticated_entity.email, action_description=action_description, - force=True + force=True, ) alert = get_alerts_by_fingerprint( @@ -601,6 +608,7 @@ def enrich_alert( logger.exception("Failed to un-enrich alert", extra={"error": str(e)}) return {"status": "failed"} + @router.post( "/search", description="Search alerts", diff --git a/keep/api/tasks/process_event_task.py b/keep/api/tasks/process_event_task.py index a3ae023d6..dc613c2fe 100644 --- a/keep/api/tasks/process_event_task.py +++ b/keep/api/tasks/process_event_task.py @@ -14,12 +14,21 @@ # internals from keep.api.alert_deduplicator.alert_deduplicator import AlertDeduplicator from keep.api.bl.enrichments import EnrichmentsBl -from keep.api.core.db import get_all_presets, get_enrichment, get_session_sync +from keep.api.core.db import ( + get_alerts_by_fingerprint, + get_all_presets, + get_enrichment_with_session, + get_session_sync, +) from keep.api.core.dependencies import get_pusher_client from keep.api.core.elastic import ElasticClient from keep.api.models.alert import AlertDto, AlertStatus from keep.api.models.db.alert import Alert, AlertActionType, AlertAudit, AlertRaw from keep.api.models.db.preset import PresetDto +from keep.api.utils.enrichment_helpers import ( + calculated_start_firing_time, + convert_db_alerts_to_dto_alerts, +) from keep.providers.providers_factory import ProvidersFactory from keep.rulesengine.rulesengine import RulesEngine from keep.workflowmanager.workflowmanager import WorkflowManager @@ -86,6 +95,14 @@ def __save_to_db( enriched_formatted_events = [] for formatted_event in formatted_events: formatted_event.pushed = True + # calculate startFiring time + previous_alert = get_alerts_by_fingerprint( + tenant_id=tenant_id, fingerprint=formatted_event.fingerprint, limit=1 + ) + previous_alert = convert_db_alerts_to_dto_alerts(previous_alert) + formatted_event.firingStartTime = calculated_start_firing_time( + formatted_event, previous_alert + ) enrichments_bl = EnrichmentsBl(tenant_id, session) # Dispose enrichments that needs to be disposed @@ -127,11 +144,9 @@ def __save_to_db( "alert_hash": formatted_event.alert_hash, } if timestamp_forced is not None: - alert_args['timestamp'] = timestamp_forced + alert_args["timestamp"] = timestamp_forced - alert = Alert( - **alert_args - ) + alert = Alert(**alert_args) session.add(alert) audit = AlertAudit( tenant_id=tenant_id, @@ -156,8 +171,10 @@ def __save_to_db( except Exception: logger.exception("Failed to run mapping rules") - alert_enrichment = get_enrichment( - tenant_id=tenant_id, fingerprint=formatted_event.fingerprint + alert_enrichment = get_enrichment_with_session( + session=session, + tenant_id=tenant_id, + fingerprint=formatted_event.fingerprint, ) if alert_enrichment: for enrichment in alert_enrichment.enrichments: @@ -220,7 +237,6 @@ def __handle_formatted_events( "tenant_id": tenant_id, }, ) - pusher_client = get_pusher_client() # first, filter out any deduplicated events alert_deduplicator = AlertDeduplicator(tenant_id) @@ -316,7 +332,10 @@ def __handle_formatted_events( ) # Tell the client to poll alerts - if pusher_client and notify_client: + if notify_client: + pusher_client = get_pusher_client() + if not pusher_client: + return try: pusher_client.trigger( f"private-{tenant_id}", @@ -362,7 +381,7 @@ def __handle_formatted_events( logger.info("Noisy preset is noisy") preset_dto.should_do_noise_now = True # send with pusher - if pusher_client and notify_client: + if notify_client and pusher_client: try: pusher_client.trigger( f"private-{tenant_id}", @@ -395,6 +414,7 @@ def process_event( AlertDto | list[AlertDto] | dict ), # the event to process, either plain (generic) or from a specific provider notify_client: bool = True, + timestamp_forced: datetime.datetime | None = None, ): extra_dict = { "tenant_id": tenant_id, @@ -432,10 +452,12 @@ def process_event( # In case when provider_type is not set if isinstance(event, dict): event = [AlertDto(**event)] + raw_event = [raw_event] # Prepare the event for the digest if isinstance(event, AlertDto): event = [event] + raw_event = [raw_event] __internal_prepartion(event, fingerprint, api_key_name) __handle_formatted_events( @@ -446,6 +468,7 @@ def process_event( event, provider_id, notify_client, + timestamp_forced, ) except Exception: logger.exception("Error processing event", extra=extra_dict) diff --git a/keep/api/utils/enrichment_helpers.py b/keep/api/utils/enrichment_helpers.py index 6f3b332c2..7085626c0 100644 --- a/keep/api/utils/enrichment_helpers.py +++ b/keep/api/utils/enrichment_helpers.py @@ -3,7 +3,7 @@ from opentelemetry import trace -from keep.api.models.alert import AlertDto +from keep.api.models.alert import AlertDto, AlertStatus from keep.api.models.db.alert import Alert tracer = trace.get_tracer(__name__) @@ -40,14 +40,42 @@ def parse_and_enrich_deleted_and_assignees(alert: AlertDto, enrichments: dict): alert.assignee = assignee alert.enriched_fields = list( - filter(lambda x: not x.startswith("disposable_"), - list(enrichments.keys())) + filter(lambda x: not x.startswith("disposable_"), list(enrichments.keys())) ) if "assignees" in alert.enriched_fields: # User can't be un-assigned. Just re-assigned to someone else alert.enriched_fields.remove("assignees") +def calculated_start_firing_time( + alert: AlertDto, previous_alert: AlertDto | list[AlertDto] +) -> str: + """ + Calculate the start firing time of an alert based on the previous alert. + + Args: + alert (AlertDto): The alert to calculate the start firing time for. + previous_alert (AlertDto): The previous alert. + + Returns: + str: The calculated start firing time. + """ + # if the alert is not firing, there is no start firing time + if alert.status != AlertStatus.FIRING.value: + return None + # if this is the first alert, the start firing time is the same as the last received time + if not previous_alert: + return alert.lastReceived + elif isinstance(previous_alert, list): + previous_alert = previous_alert[0] + # else, if the previous alert was firing, the start firing time is the same as the previous alert + if previous_alert.status == AlertStatus.FIRING.value: + return previous_alert.firingStartTime + # else, if the previous alert was resolved, the start firing time is the same as the last received time + else: + return alert.lastReceived + + def convert_db_alerts_to_dto_alerts(alerts: list[Alert]) -> list[AlertDto]: """ Enriches the alerts with the enrichment data. diff --git a/keep/functions/__init__.py b/keep/functions/__init__.py index a4b05c843..8161a6ec2 100644 --- a/keep/functions/__init__.py +++ b/keep/functions/__init__.py @@ -11,7 +11,9 @@ from dateutil.parser import ParserError from keep.api.bl.enrichments import EnrichmentsBl -from keep.api.core.db import get_alert_firing_time, get_alerts_by_fingerprint +from keep.api.core.db import get_alerts_by_fingerprint +from keep.api.models.alert import AlertStatus +from keep.api.utils.enrichment_helpers import convert_db_alerts_to_dto_alerts _len = len _all = all @@ -261,7 +263,21 @@ def get_firing_time(alert: dict, time_unit: str, **kwargs) -> str: if not fingerprint: raise ValueError("fingerprint is required") - firing = get_alert_firing_time(tenant_id=tenant_id, fingerprint=fingerprint) + alert_from_db = get_alerts_by_fingerprint( + tenant_id=tenant_id, + fingerprint=fingerprint, + limit=1, + ) + if alert_from_db: + alert_dto = convert_db_alerts_to_dto_alerts(alert_from_db)[0] + # if the alert is not firing, there is no start firing time + if alert_dto.status != AlertStatus.FIRING.value: + return "0.00" + firing = datetime.datetime.now( + tz=datetime.timezone.utc + ) - datetime.datetime.fromisoformat(alert_dto.firingStartTime) + else: + return "0.00" if time_unit in ["m", "minutes"]: result = firing.total_seconds() / 60 diff --git a/tests/conftest.py b/tests/conftest.py index 53a0bcf34..47f04ca32 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,6 @@ import mysql.connector import pytest -import pytz from dotenv import find_dotenv, load_dotenv from pytest_docker.plugin import get_docker_services from sqlalchemy.orm import sessionmaker @@ -24,6 +23,7 @@ from keep.api.models.db.tenant import * from keep.api.models.db.user import * from keep.api.models.db.workflow import * +from keep.api.tasks.process_event_task import process_event from keep.api.utils.enrichment_helpers import convert_db_alerts_to_dto_alerts from keep.contextmanager.contextmanager import ContextManager @@ -150,6 +150,7 @@ def mysql_container(docker_ip, docker_services): @pytest.fixture def db_session(request): # Create a database connection + os.environ["DB_ECHO"] = "true" if ( request and hasattr(request, "param") @@ -184,6 +185,15 @@ def db_session(request): session.add_all(tenant_data) session.commit() # 2. Create some workflows + mock_raw_workflow = """workflow: +id: {} +actions: + - name: send-slack-message + provider: + type: console + with: + message: "mock" +""" workflow_data = [ Workflow( id="test-id-1", @@ -192,7 +202,7 @@ def db_session(request): description="test workflow", created_by="test@keephq.dev", interval=0, - workflow_raw="test workflow raw", + workflow_raw=mock_raw_workflow.format("test-id-1"), ), Workflow( id="test-id-2", @@ -201,7 +211,7 @@ def db_session(request): description="test workflow", created_by="test@keephq.dev", interval=0, - workflow_raw="test workflow raw", + workflow_raw=mock_raw_workflow.format("test-id-2"), ), WorkflowExecution( id="test-execution-id-1", @@ -376,7 +386,9 @@ def _setup_stress_alerts_no_elastic(num_alerts): "source": [ "source_{}".format(i % 10) ], # Cycle through 10 different sources - "service": "service_{}".format(i % 10), # Random of 10 different services + "service": "service_{}".format( + i % 10 + ), # Random of 10 different services "severity": random.choice( ["info", "warning", "critical"] ), # Alternate between 'critical' and 'warning' @@ -391,7 +403,7 @@ def _setup_stress_alerts_no_elastic(num_alerts): Alert( timestamp=random_timestamp, tenant_id=SINGLE_TENANT_UUID, - provider_type=detail['source'][0], + provider_type=detail["source"][0], provider_id="test_{}".format( i % 5 ), # Cycle through 5 different provider_ids @@ -403,11 +415,14 @@ def _setup_stress_alerts_no_elastic(num_alerts): db_session.commit() return alerts + return _setup_stress_alerts_no_elastic @pytest.fixture -def setup_stress_alerts(elastic_client, db_session, request, setup_stress_alerts_no_elastic): +def setup_stress_alerts( + elastic_client, db_session, request, setup_stress_alerts_no_elastic +): num_alerts = request.param.get( "num_alerts", 1000 ) # Default to 1000 alerts if not specified @@ -422,17 +437,26 @@ def setup_stress_alerts(elastic_client, db_session, request, setup_stress_alerts def create_alert(db_session): def _create_alert(fingerprint, status, timestamp, details=None): details = details or {} - alert = Alert( + random_name = "test-{}".format(fingerprint) + process_event( + ctx={"job_try": 1}, + trace_id="test", tenant_id=SINGLE_TENANT_UUID, - provider_type=details["source"][0] if details and "source" in details else "test", provider_id="test", - event={"fingerprint": fingerprint, "status": status.value, **details}, + provider_type=( + details["source"][0] if details and "source" in details else None + ), fingerprint=fingerprint, - alert_hash="test_hash", - timestamp=timestamp.replace(tzinfo=pytz.utc), + api_key_name="test", + event={ + "name": random_name, + "fingerprint": fingerprint, + "lastReceived": timestamp.isoformat(), + "status": status.value, + **details, + }, + notify_client=False, + timestamp_forced=timestamp, ) - db_session.add(alert) - db_session.commit() - return alert return _create_alert diff --git a/tests/test_functions.py b/tests/test_functions.py index 00e924ba8..47a850915 100644 --- a/tests/test_functions.py +++ b/tests/test_functions.py @@ -6,8 +6,10 @@ import pytz import keep.functions as functions +from keep.api.bl.enrichments import EnrichmentsBl from keep.api.core.dependencies import SINGLE_TENANT_UUID from keep.api.models.alert import AlertStatus +from keep.api.models.db.alert import AlertActionType @pytest.mark.parametrize( @@ -345,10 +347,10 @@ def test_get_firing_time_case1(create_alert): fingerprint = "fp1" base_time = datetime.datetime.now(tz=pytz.utc) - create_alert(fingerprint, AlertStatus.FIRING, base_time - timedelta(minutes=15)) - create_alert(fingerprint, AlertStatus.RESOLVED, base_time - timedelta(minutes=30)) - create_alert(fingerprint, AlertStatus.FIRING, base_time - timedelta(minutes=60)) create_alert(fingerprint, AlertStatus.FIRING, base_time - timedelta(minutes=90)) + create_alert(fingerprint, AlertStatus.FIRING, base_time - timedelta(minutes=60)) + create_alert(fingerprint, AlertStatus.RESOLVED, base_time - timedelta(minutes=30)) + create_alert(fingerprint, AlertStatus.FIRING, base_time - timedelta(minutes=15)) alert = {"fingerprint": fingerprint} result = functions.get_firing_time(alert, "m", tenant_id=SINGLE_TENANT_UUID) @@ -359,9 +361,9 @@ def test_get_firing_time_case2(create_alert): fingerprint = "fp2" base_time = datetime.datetime.now(tz=pytz.utc) - create_alert(fingerprint, AlertStatus.RESOLVED, base_time) - create_alert(fingerprint, AlertStatus.FIRING, base_time - timedelta(minutes=30)) create_alert(fingerprint, AlertStatus.RESOLVED, base_time - timedelta(minutes=90)) + create_alert(fingerprint, AlertStatus.FIRING, base_time - timedelta(minutes=30)) + create_alert(fingerprint, AlertStatus.RESOLVED, base_time) alert = {"fingerprint": fingerprint} assert functions.get_firing_time(alert, "m", tenant_id=SINGLE_TENANT_UUID) == "0.00" @@ -371,10 +373,10 @@ def test_get_firing_time_case3(create_alert): fingerprint = "fp3" base_time = datetime.datetime.now(tz=pytz.utc) - create_alert(fingerprint, AlertStatus.FIRING, base_time) - create_alert(fingerprint, AlertStatus.FIRING, base_time - timedelta(minutes=30)) - create_alert(fingerprint, AlertStatus.RESOLVED, base_time - timedelta(minutes=90)) create_alert(fingerprint, AlertStatus.FIRING, base_time - timedelta(minutes=120)) + create_alert(fingerprint, AlertStatus.RESOLVED, base_time - timedelta(minutes=90)) + create_alert(fingerprint, AlertStatus.FIRING, base_time - timedelta(minutes=30)) + create_alert(fingerprint, AlertStatus.FIRING, base_time) alert = {"fingerprint": fingerprint} result = functions.get_firing_time(alert, "m", tenant_id=SINGLE_TENANT_UUID) @@ -385,12 +387,12 @@ def test_get_firing_time_case4(create_alert): fingerprint = "fp4" base_time = datetime.datetime.now(tz=pytz.utc) - create_alert(fingerprint, AlertStatus.FIRING, base_time - timedelta(minutes=15)) - create_alert(fingerprint, AlertStatus.RESOLVED, base_time - timedelta(minutes=30)) - create_alert(fingerprint, AlertStatus.FIRING, base_time - timedelta(minutes=60)) - create_alert(fingerprint, AlertStatus.RESOLVED, base_time - timedelta(minutes=90)) - create_alert(fingerprint, AlertStatus.FIRING, base_time - timedelta(minutes=120)) create_alert(fingerprint, AlertStatus.FIRING, base_time - timedelta(minutes=150)) + create_alert(fingerprint, AlertStatus.FIRING, base_time - timedelta(minutes=120)) + create_alert(fingerprint, AlertStatus.RESOLVED, base_time - timedelta(minutes=90)) + create_alert(fingerprint, AlertStatus.FIRING, base_time - timedelta(minutes=60)) + create_alert(fingerprint, AlertStatus.RESOLVED, base_time - timedelta(minutes=30)) + create_alert(fingerprint, AlertStatus.FIRING, base_time - timedelta(minutes=15)) alert = {"fingerprint": fingerprint} result = functions.get_firing_time(alert, "m", tenant_id=SINGLE_TENANT_UUID) @@ -401,9 +403,9 @@ def test_get_firing_time_no_firing(create_alert): fingerprint = "fp5" base_time = datetime.datetime.now(tz=pytz.utc) - create_alert(fingerprint, AlertStatus.RESOLVED, base_time) - create_alert(fingerprint, AlertStatus.RESOLVED, base_time - timedelta(minutes=30)) create_alert(fingerprint, AlertStatus.RESOLVED, base_time - timedelta(minutes=60)) + create_alert(fingerprint, AlertStatus.RESOLVED, base_time - timedelta(minutes=30)) + create_alert(fingerprint, AlertStatus.RESOLVED, base_time) alert = {"fingerprint": fingerprint} assert functions.get_firing_time(alert, "m", tenant_id=SINGLE_TENANT_UUID) == "0.00" @@ -413,12 +415,12 @@ def test_get_firing_time_other_statuses(create_alert): fingerprint = "fp6" base_time = datetime.datetime.now(tz=pytz.utc) + create_alert(fingerprint, AlertStatus.RESOLVED, base_time - timedelta(minutes=90)) + create_alert(fingerprint, AlertStatus.SUPPRESSED, base_time - timedelta(minutes=60)) + create_alert(fingerprint, AlertStatus.FIRING, base_time - timedelta(minutes=45)) create_alert( fingerprint, AlertStatus.ACKNOWLEDGED, base_time - timedelta(minutes=30) ) - create_alert(fingerprint, AlertStatus.FIRING, base_time - timedelta(minutes=45)) - create_alert(fingerprint, AlertStatus.SUPPRESSED, base_time - timedelta(minutes=60)) - create_alert(fingerprint, AlertStatus.RESOLVED, base_time - timedelta(minutes=90)) alert = {"fingerprint": fingerprint} result = functions.get_firing_time(alert, "m", tenant_id=SINGLE_TENANT_UUID) @@ -429,16 +431,16 @@ def test_get_firing_time_minutes_and_seconds(create_alert): fingerprint = "fp7" base_time = datetime.datetime.now(tz=pytz.utc) - create_alert(fingerprint, AlertStatus.FIRING, base_time) + create_alert(fingerprint, AlertStatus.RESOLVED, base_time - timedelta(minutes=5)) create_alert( fingerprint, AlertStatus.FIRING, base_time - timedelta(minutes=2, seconds=30) ) - create_alert(fingerprint, AlertStatus.RESOLVED, base_time - timedelta(minutes=5)) + create_alert(fingerprint, AlertStatus.FIRING, base_time) alert = {"fingerprint": fingerprint} result = functions.get_firing_time(alert, "s", tenant_id=SINGLE_TENANT_UUID) assert ( - abs(float(result) - 150.0) < 1 + abs(float(result) - 150.0) < 5 # seconds ) # Allow for small time differences (150 seconds = 2.5 minutes) @@ -459,10 +461,11 @@ def test_first_time_with_since(create_alert): fingerprint = "fp2" base_time = datetime.datetime.now(tz=pytz.utc) - create_alert(fingerprint, AlertStatus.FIRING, base_time) create_alert( fingerprint, AlertStatus.FIRING, base_time - timedelta(minutes=24 * 60 + 1) ) + create_alert(fingerprint, AlertStatus.FIRING, base_time) + result = functions.is_first_time(fingerprint, "24h", tenant_id=SINGLE_TENANT_UUID) assert result == True @@ -479,3 +482,30 @@ def test_first_time_with_since(create_alert): assert result == False result = functions.is_first_time(fingerprint, "6h", tenant_id=SINGLE_TENANT_UUID) assert result == True + + +def test_firing_time_with_manual_resolve(create_alert): + fingerprint = "fp10" + base_time = datetime.datetime.now(tz=pytz.utc) + + # Alert fired 60 minutes ago + create_alert(fingerprint, AlertStatus.FIRING, base_time - timedelta(minutes=60)) + # It was manually resolved + enrichment_bl = EnrichmentsBl(tenant_id=SINGLE_TENANT_UUID) + enrichment_bl.enrich_alert( + fingerprint=fingerprint, + enrichments={"status": "resolved"}, + dispose_on_new_alert=True, + action_type=AlertActionType.GENERIC_ENRICH, + action_callee="tests", + action_description="tests", + ) + alert = {"fingerprint": fingerprint} + result = functions.get_firing_time(alert, "m", tenant_id=SINGLE_TENANT_UUID) + assert abs(float(result) - 0) < 1 # Allow for small time differences + + # Now its firing again, the firing time should be calculated from the last firing + create_alert(fingerprint, AlertStatus.FIRING, base_time - timedelta(minutes=30)) + # It should override the dispoable status, but show only the time since the last firing + result = functions.get_firing_time(alert, "m", tenant_id=SINGLE_TENANT_UUID) + assert abs(float(result) - 30) < 1 # Allow for small time differences diff --git a/tests/test_workflow_execution.py b/tests/test_workflow_execution.py index f5ac6bc44..f744f3aa4 100644 --- a/tests/test_workflow_execution.py +++ b/tests/test_workflow_execution.py @@ -157,6 +157,7 @@ def test_workflow_execution( base_time = datetime.now(tz=pytz.utc) # Create alerts with specified statuses and timestamps + alert_statuses.reverse() for time_diff, status in alert_statuses: alert_status = ( AlertStatus.FIRING if status == "firing" else AlertStatus.RESOLVED