From 7276ab4ccd199d888a86184a1bd6d0baa26a850d Mon Sep 17 00:00:00 2001 From: Shahar Glazner Date: Wed, 26 Jun 2024 17:31:54 +0300 Subject: [PATCH 1/4] feat: typo (#1294) From 46acbef26a306f3e4e416bf78b682e08c660fed1 Mon Sep 17 00:00:00 2001 From: Shahar Glazner Date: Fri, 18 Oct 2024 10:23:48 +0300 Subject: [PATCH 2/4] fix(deduplication): allow nested fields in dedup (#2232) --- keep/api/tasks/process_event_task.py | 8 +++++--- keep/providers/base/base_provider.py | 11 +++++++++-- pyproject.toml | 2 +- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/keep/api/tasks/process_event_task.py b/keep/api/tasks/process_event_task.py index 7630fbc5a..b7ef5d52f 100644 --- a/keep/api/tasks/process_event_task.py +++ b/keep/api/tasks/process_event_task.py @@ -317,7 +317,7 @@ def __handle_formatted_events( for key, value in enriched_formatted_event.dict().items(): if isinstance(value, dict): for nested_key in value.keys(): - fields.append(f"{key}_{nested_key}") + fields.append(f"{key}.{nested_key}") else: fields.append(key) @@ -326,7 +326,7 @@ def __handle_formatted_events( fields=fields, provider_id=enriched_formatted_event.providerId, provider_type=enriched_formatted_event.providerType, - session=session + session=session, ) logger.debug( @@ -386,7 +386,9 @@ def __handle_formatted_events( # Now we need to run the rules engine try: rules_engine = RulesEngine(tenant_id=tenant_id) - incidents: List[IncidentDto] = rules_engine.run_rules(enriched_formatted_events, session=session) + incidents: List[IncidentDto] = rules_engine.run_rules( + enriched_formatted_events, session=session + ) # TODO: Replace with incidents workflow triggers. Ticket: https://github.com/keephq/keep/issues/1527 # if new grouped incidents were created, we need to push them to the client diff --git a/keep/providers/base/base_provider.py b/keep/providers/base/base_provider.py index 0233b5837..9d2056636 100644 --- a/keep/providers/base/base_provider.py +++ b/keep/providers/base/base_provider.py @@ -395,10 +395,17 @@ def get_alert_fingerprint(alert: AlertDto, fingerprint_fields: list = []) -> str fingerprint = hashlib.sha256() event_dict = alert.dict() for fingerprint_field in fingerprint_fields: - fingerprint_field_value = event_dict.get(fingerprint_field, None) + keys = fingerprint_field.split(".") + fingerprint_field_value = event_dict + for key in keys: + if isinstance(fingerprint_field_value, dict): + fingerprint_field_value = fingerprint_field_value.get(key, None) + else: + fingerprint_field_value = None + break if isinstance(fingerprint_field_value, (list, dict)): fingerprint_field_value = json.dumps(fingerprint_field_value) - if fingerprint_field_value: + if fingerprint_field_value is not None: fingerprint.update(str(fingerprint_field_value).encode()) return fingerprint.hexdigest() diff --git a/pyproject.toml b/pyproject.toml index 1dfcc6d45..9c678e6cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "keep" -version = "0.26.0" +version = "0.26.1" description = "Alerting. for developers, by developers." authors = ["Keep Alerting LTD"] readme = "README.md" From ce997f0d42b8a52f6e6737b4c9e3719dd006aea2 Mon Sep 17 00:00:00 2001 From: Vladimir Filonov Date: Fri, 18 Oct 2024 12:25:34 +0400 Subject: [PATCH 3/4] /incident/{incident-id}/alerts now return only alerts with unique fingerprints --- keep/api/core/db.py | 29 ++++++++++++++++++++++++----- tests/test_rules_engine.py | 10 ++++++---- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/keep/api/core/db.py b/keep/api/core/db.py index 10a00c445..6bea7abb3 100644 --- a/keep/api/core/db.py +++ b/keep/api/core/db.py @@ -2838,21 +2838,38 @@ def get_incident_alerts_and_links_by_incident_id( tenant_id: str, incident_id: UUID | str, limit: Optional[int] = None, - offset: Optional[int] = None, + offset: Optional[int] = 0, session: Optional[Session] = None, include_unlinked: bool = False, ) -> tuple[List[tuple[Alert, AlertToIncident]], int]: with existed_or_new_session(session) as session: + + last_fingerprints_subquery = ( + session.query(Alert.fingerprint, func.max(Alert.timestamp).label("max_timestamp")) + .join(AlertToIncident, AlertToIncident.alert_id == Alert.id) + .filter( + AlertToIncident.tenant_id == tenant_id, + AlertToIncident.incident_id == incident_id, + ) + .group_by(Alert.fingerprint) + .subquery() + ) + query = ( session.query( Alert, AlertToIncident, ) + .select_from(last_fingerprints_subquery) + .outerjoin(Alert, and_( + last_fingerprints_subquery.c.fingerprint == Alert.fingerprint, + last_fingerprints_subquery.c.max_timestamp == Alert.timestamp, + + )) .join(AlertToIncident, AlertToIncident.alert_id == Alert.id) - .join(Incident, AlertToIncident.incident_id == Incident.id) .filter( AlertToIncident.tenant_id == tenant_id, - Incident.id == incident_id, + AlertToIncident.incident_id == incident_id, ) .order_by(col(Alert.timestamp).desc()) ) @@ -2863,8 +2880,10 @@ def get_incident_alerts_and_links_by_incident_id( total_count = query.count() - if limit and offset: - query = query.limit(limit).offset(offset) + if limit: + query = query.limit(limit) + if offset: + query = query.offset(offset) return query.all(), total_count diff --git a/tests/test_rules_engine.py b/tests/test_rules_engine.py index 7c032feeb..5abf1f5a9 100644 --- a/tests/test_rules_engine.py +++ b/tests/test_rules_engine.py @@ -385,6 +385,7 @@ def test_incident_resolution_on_all(db_session, create_alert): ) assert alert_count == 2 + # Same fingerprint create_alert( f"Something went wrong", AlertStatus.RESOLVED, @@ -409,7 +410,8 @@ def test_incident_resolution_on_all(db_session, create_alert): limit=10, offset=0, ) - assert alert_count == 3 + # Still 2 alerts, since 2 unique fingerprints + assert alert_count == 2 assert incident.status == IncidentStatus.FIRING.value create_alert( @@ -436,7 +438,7 @@ def test_incident_resolution_on_all(db_session, create_alert): limit=10, offset=0, ) - assert alert_count == 4 + assert alert_count == 2 assert incident.status == IncidentStatus.RESOLVED.value @@ -528,7 +530,7 @@ def test_incident_resolution_on_edge(db_session, create_alert, direction, second limit=10, offset=0, ) - assert alert_count == 3 + assert alert_count == 2 assert incident.status == IncidentStatus.FIRING.value create_alert( @@ -555,7 +557,7 @@ def test_incident_resolution_on_edge(db_session, create_alert, direction, second limit=10, offset=0, ) - assert alert_count == 4 + assert alert_count == 2 assert incident.status == IncidentStatus.RESOLVED.value # Next steps: From 13f706f1f10b2f4bf310361972b1faa6371e5916 Mon Sep 17 00:00:00 2001 From: Vladimir Filonov Date: Mon, 28 Oct 2024 23:30:29 +0400 Subject: [PATCH 4/4] FGix test after merging unique fingerprints count for incidents --- tests/test_incidents.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_incidents.py b/tests/test_incidents.py index b9758b638..f738d3913 100644 --- a/tests/test_incidents.py +++ b/tests/test_incidents.py @@ -124,7 +124,7 @@ def test_add_remove_alert_to_incidents(db_session, setup_stress_alerts_no_elasti incident_id=incident.id, tenant_id=incident.tenant_id, include_unlinked=True - )[0]) == 120 + )[0]) == 100 incident = get_incident_by_id(SINGLE_TENANT_UUID, incident.id)