From 98f037226a51f3338affba02020cfea84f3bfdfa Mon Sep 17 00:00:00 2001 From: Tal Borenstein Date: Tue, 3 Sep 2024 16:32:58 +0300 Subject: [PATCH 01/10] feat(provider): servicenow cmdb --- keep-ui/app/topology/custom-node.tsx | 2 +- keep-ui/app/topology/topology.tsx | 9 +- .../versions/2024-09-03-16-24_1a5eb7069f9a.py | 46 ++++++ keep/api/models/db/topology.py | 13 ++ keep/api/routes/preset.py | 4 +- keep/api/tasks/process_topology_task.py | 14 +- .../servicenow_provider.py | 144 +++++++++++++++--- 7 files changed, 202 insertions(+), 30 deletions(-) create mode 100644 keep/api/models/db/migrations/versions/2024-09-03-16-24_1a5eb7069f9a.py diff --git a/keep-ui/app/topology/custom-node.tsx b/keep-ui/app/topology/custom-node.tsx index f7891b498..5ba70b391 100644 --- a/keep-ui/app/topology/custom-node.tsx +++ b/keep-ui/app/topology/custom-node.tsx @@ -32,7 +32,7 @@ const CustomNode = ({ data }: { data: TopologyService }) => { return (
- {data.service} + {data.display_name ?? data.service} {alertCount > 0 && ( { - const node = reactFlowInstance?.getNode(nodeId); + let node = reactFlowInstance?.getNode(nodeId); + if (!node) { + // Maybe its by display name? + node = reactFlowInstance + ?.getNodes() + .find((n) => n.data.display_name === nodeId); + } if (node && reactFlowInstance) { reactFlowInstance.setCenter(node.position.x, node.position.y); } @@ -198,6 +204,7 @@ const TopologyPage = ({ None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("topologyservice", schema=None) as batch_op: + batch_op.add_column( + sa.Column("ip_address", sqlmodel.sql.sqltypes.AutoString(), nullable=True) + ) + batch_op.add_column( + sa.Column("mac_address", sqlmodel.sql.sqltypes.AutoString(), nullable=True) + ) + batch_op.add_column( + sa.Column("category", sqlmodel.sql.sqltypes.AutoString(), nullable=True) + ) + batch_op.add_column( + sa.Column("manufacturer", sqlmodel.sql.sqltypes.AutoString(), nullable=True) + ) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("topologyservice", schema=None) as batch_op: + batch_op.drop_column("manufacturer") + batch_op.drop_column("category") + batch_op.drop_column("mac_address") + batch_op.drop_column("ip_address") + # ### end Alembic commands ### diff --git a/keep/api/models/db/topology.py b/keep/api/models/db/topology.py index 7b802edbd..453790276 100644 --- a/keep/api/models/db/topology.py +++ b/keep/api/models/db/topology.py @@ -20,6 +20,11 @@ class TopologyService(SQLModel, table=True): application: Optional[str] email: Optional[str] slack: Optional[str] + ip_address: Optional[str] = None + mac_address: Optional[str] = None + category: Optional[str] = None + manufacturer: Optional[str] = None + updated_at: Optional[datetime] = Field( sa_column=Column( DateTime(timezone=True), @@ -84,6 +89,10 @@ class TopologyServiceDtoBase(BaseModel, extra="ignore"): application: Optional[str] = None email: Optional[str] = None slack: Optional[str] = None + ip_address: Optional[str] = None + mac_address: Optional[str] = None + category: Optional[str] = None + manufacturer: Optional[str] = None class TopologyServiceInDto(TopologyServiceDtoBase): @@ -116,6 +125,10 @@ def from_orm(cls, service: "TopologyService") -> "TopologyServiceDtoOut": application=service.application, email=service.email, slack=service.slack, + ip_address=service.ip_address, + mac_address=service.mac_address, + manufacturer=service.manufacturer, + category=service.category, dependencies=[ TopologyServiceDependencyDto( serviceId=dep.depends_on_service_id, diff --git a/keep/api/routes/preset.py b/keep/api/routes/preset.py index c05b6fcae..b10ab24e0 100644 --- a/keep/api/routes/preset.py +++ b/keep/api/routes/preset.py @@ -112,10 +112,10 @@ def pull_data_from_providers( f"Provider {provider.type} ({provider.id}) does not support topology data", extra=extra, ) - except Exception: + except Exception as e: logger.error( f"Unknown error pulling topology from provider {provider.type} ({provider.id})", - extra=extra, + extra={**extra, "error": str(e)}, ) # Even if we failed at processing some event, lets save the last pull time to not iterate this process over and over again. diff --git a/keep/api/tasks/process_topology_task.py b/keep/api/tasks/process_topology_task.py index 37788cbb0..88f282e97 100644 --- a/keep/api/tasks/process_topology_task.py +++ b/keep/api/tasks/process_topology_task.py @@ -65,11 +65,19 @@ def process_topology( # Then create the dependencies for service in topology_data: for dependency in service.dependencies: + service_id = service_to_keep_service_id_map.get(service.service) + depends_on_service_id = service_to_keep_service_id_map.get(dependency) + if not service_id or not depends_on_service_id: + logger.warning( + "Found a dangling service, skipping", + extra={"service": service.service, "dependency": dependency}, + ) + continue session.add( TopologyServiceDependency( - service_id=service_to_keep_service_id_map[service.service], - depends_on_service_id=service_to_keep_service_id_map[dependency], - protocol=service.dependencies[dependency], + service_id=service_id, + depends_on_service_id=depends_on_service_id, + protocol=service.dependencies.get(dependency, "unknown"), ) ) diff --git a/keep/providers/servicenow_provider/servicenow_provider.py b/keep/providers/servicenow_provider/servicenow_provider.py index 85f1475fb..46ed94447 100644 --- a/keep/providers/servicenow_provider/servicenow_provider.py +++ b/keep/providers/servicenow_provider/servicenow_provider.py @@ -1,6 +1,7 @@ """ ServicenowProvider is a class that implements the BaseProvider interface for Service Now updates. """ + import dataclasses import json @@ -8,10 +9,10 @@ import requests from requests.auth import HTTPBasicAuth -from keep.api.models.alert import AlertDto +from keep.api.models.db.topology import TopologyServiceInDto from keep.contextmanager.contextmanager import ContextManager from keep.exceptions.provider_exception import ProviderException -from keep.providers.base.base_provider import BaseProvider +from keep.providers.base.base_provider import BaseTopologyProvider from keep.providers.models.provider_config import ProviderConfig, ProviderScope @@ -45,7 +46,7 @@ class ServicenowProviderAuthConfig: ) -class ServicenowProvider(BaseProvider): +class ServicenowProvider(BaseTopologyProvider): """Manage ServiceNow tickets.""" PROVIDER_SCOPES = [ @@ -111,6 +112,119 @@ def validate_config(self): **self.config.authentication ) + def pull_topology(self) -> list[TopologyServiceInDto]: + # TODO: in scable, we'll need to use pagination around here + headers = {"Content-Type": "application/json", "Accept": "application/json"} + auth = ( + self.authentication_config.username, + self.authentication_config.password, + ) + topology = [] + self.logger.info( + "Pulling topology", extra={"tenant_id": self.context_manager.tenant_id} + ) + + self.logger.info("Pulling CMDB items") + fields = [ + "name", + "sys_id", + "ip_address", + "mac_address", + "owned_by.name" + "manufacturer.name", # Retrieve the name of the manufacturer + "short_description", + "environment", + ] + + # Set parameters for the request + cmdb_params = { + "sysparm_fields": ",".join(fields), + "sysparm_query": "active=true", + } + cmdb_response = requests.get( + f"{self.authentication_config.service_now_base_url}/api/now/table/cmdb_ci", + headers=headers, + auth=auth, + params=cmdb_params, + ) + + if not cmdb_response.ok: + self.logger.error( + "Failed to pull topology", + extra={ + "tenant_id": self.context_manager.tenant_id, + "status_code": cmdb_response.status_code, + }, + ) + return topology + + cmdb_data = cmdb_response.json().get("result", []) + self.logger.info( + "Pulling CMDB items completed", extra={"len_of_cmdb_items": len(cmdb_data)} + ) + + self.logger.info("Pulling relationship types") + relationship_types = {} + rel_type_response = requests.get( + f"{self.authentication_config.service_now_base_url}/api/now/table/cmdb_rel_type", + auth=auth, + headers=headers, + ) + if not rel_type_response.ok: + self.logger.error("Failed to get topology types") + else: + rel_type_json = rel_type_response.json() + for result in rel_type_json.get("result", []): + relationship_types[result.get("sys_id")] = result.get("sys_name") + self.logger.info("Pulling relationship types completed") + + self.logger.info("Pulling relationships") + relationships = {} + rel_response = requests.get( + f"{self.authentication_config.service_now_base_url}/api/now/table/cmdb_rel_ci", + auth=auth, + headers=headers, + ) + if not rel_response.ok: + self.logger.error("Failed to get topology relationships") + else: + rel_json = rel_response.json() + for relationship in rel_json.get("result", []): + parent_id = relationship.get("parent", {}).get("value") + child_id = relationship.get("child", {}).get("value") + relationship_type_id = relationship.get("type", {}).get("value") + relationship_type = relationship_types.get(relationship_type_id) + if parent_id not in relationships: + relationships[parent_id] = {} + relationships[parent_id][child_id] = relationship_type + self.logger.info("Pulling relationships completed") + + self.logger.info("Mixing up all topology data") + for entity in cmdb_data: + sys_id = entity.get("sys_id") + owned_by = entity.get("owned_by.name") + topology_service = TopologyServiceInDto( + source_provider_id=self.provider_id, + service=sys_id, + display_name=entity.get("name"), + description=entity.get("short_description"), + environment=entity.get("environment"), + team=owned_by, + dependencies=relationships.get(sys_id, {}), + ip_address=entity.get("ip_address"), + mac_address=entity.get("mac_address"), + ) + topology.append(topology_service) + + self.logger.info( + "Topology pulling completed", + extra={ + "tenant_id": self.context_manager.tenant_id, + "len_of_topology": len(topology), + }, + ) + return topology + def dispose(self): """ No need to dispose of anything, so just do nothing. @@ -152,9 +266,9 @@ def _notify(self, table_name: str, payload: dict = {}, **kwargs: dict): self.logger.info(f"Created ticket: {resp}") result = resp.get("result") # Add link to ticket - result[ - "link" - ] = f"{self.authentication_config.service_now_base_url}/now/nav/ui/classic/params/target/{table_name}.do%3Fsys_id%3D{result['sys_id']}" + result["link"] = ( + f"{self.authentication_config.service_now_base_url}/now/nav/ui/classic/params/target/{table_name}.do%3Fsys_id%3D{result['sys_id']}" + ) return result # if the instance is down due to hibranate you'll get 200 instead of 201 elif response.status_code == 200: @@ -226,21 +340,5 @@ def _notify_update(self, table_name: str, ticket_id: str, fingerprint: str): context_manager, provider_id="servicenow", config=config ) - # mock alert - context_manager = provider.context_manager - - alert = AlertDto.parse_obj( - json.loads( - '{"id": "4c54ce9a0d458b574d0aaa5fad23f44ce006e45bdf16fa65207cc6131979c000", "name": "Error in lambda", "status": "ALARM", "lastReceived": "2023-09-18 12:26:21.408000+00:00", "environment": "undefined", "isDuplicate": null, "duplicateReason": null, "service": null, "source": ["cloudwatch"], "message": null, "description": "Hey Shahar\\n\\nThis is a test alarm!", "severity": null, "pushed": true, "event_id": "3cbf2024-a1f0-42ac-9754-b9157c00b95e", "url": null, "AWSAccountId": "1234", "AlarmActions": ["arn:aws:sns:us-west-2:1234:Default_CloudWatch_Alarms_Topic"], "AlarmArn": "arn:aws:cloudwatch:us-west-2:1234:alarm:Error in lambda", "Trigger": {"MetricName": "Errors", "Namespace": "AWS/Lambda", "StatisticType": "Statistic", "Statistic": "AVERAGE", "Unit": null, "Dimensions": [{"value": "helloWorld", "name": "FunctionName"}], "Period": 300, "EvaluationPeriods": 1, "DatapointsToAlarm": 1, "ComparisonOperator": "GreaterThanThreshold", "Threshold": 0.0, "TreatMissingData": "missing", "EvaluateLowSampleCountPercentile": ""}, "Region": "US West (Oregon)", "InsufficientDataActions": [], "AlarmConfigurationUpdatedTimestamp": "2023-08-17T14:29:12.272+0000", "NewStateReason": "Setting state to ALARM for testing", "AlarmName": "Error in lambda", "NewStateValue": "ALARM", "OldStateValue": "INSUFFICIENT_DATA", "AlarmDescription": "Hey Shahar\\n\\nThis is a test alarm!", "OKActions": [], "StateChangeTime": "2023-09-18T12:26:21.408+0000", "trigger": "alert"}' - ) - ) - context_manager.set_event_context(alert) - r = provider.notify( - table_name="incident", - payload={ - "short_description": "My new incident", - "category": "software", - "created_by": "keep", - }, - ) + r = provider.pull_topology() print(r) From cddb6097815c769998a6124378f6db358aed59fc Mon Sep 17 00:00:00 2001 From: Tal Borenstein Date: Tue, 3 Sep 2024 16:42:12 +0300 Subject: [PATCH 02/10] feat: topology draggable --- keep-ui/app/topology/topology.tsx | 15 +++++++++++++++ keep/api/routes/topology.py | 5 +++++ 2 files changed, 20 insertions(+) diff --git a/keep-ui/app/topology/topology.tsx b/keep-ui/app/topology/topology.tsx index 70da6cc07..151a7f54d 100644 --- a/keep-ui/app/topology/topology.tsx +++ b/keep-ui/app/topology/topology.tsx @@ -9,6 +9,10 @@ import { ReactFlow, ReactFlowInstance, ReactFlowProvider, + applyNodeChanges, + applyEdgeChanges, + NodeChange, + EdgeChange, } from "@xyflow/react"; import dagre, { graphlib } from "@dagrejs/dagre"; import "@xyflow/react/dist/style.css"; @@ -90,6 +94,15 @@ const TopologyPage = ({ environment ); + const onNodesChange = useCallback( + (changes: NodeChange[]) => setNodes((nds) => applyNodeChanges(changes, nds)), + [] + ); + const onEdgesChange = useCallback( + (changes: EdgeChange[]) => setEdges((eds) => applyEdgeChanges(changes, eds)), + [] + ); + const onEdgeHover = (eventType: "enter" | "leave", edge: Edge) => { const newEdges = [...edges]; const currentEdge = newEdges.find((e) => e.id === edge.id); @@ -207,6 +220,8 @@ const TopologyPage = ({ defaultViewport={{ x: 0, y: 0, zoom: 2 }} fitView snapToGrid + onNodesChange={onNodesChange} + onEdgesChange={onEdgesChange} fitViewOptions={{ padding: 0.3 }} onEdgeMouseEnter={(_event, edge) => onEdgeHover("enter", edge)} onEdgeMouseLeave={(_event, edge) => onEdgeHover("leave", edge)} diff --git a/keep/api/routes/topology.py b/keep/api/routes/topology.py index 8e2be6678..5a9b88541 100644 --- a/keep/api/routes/topology.py +++ b/keep/api/routes/topology.py @@ -22,6 +22,7 @@ def get_topology_data( provider_id: Optional[str] = None, service_id: Optional[str] = None, environment: Optional[str] = None, + includeEmptyDeps: Optional[bool] = False, authenticated_entity: AuthenticatedEntity = Depends( IdentityManagerFactory.get_auth_verifier(["read:topology"]) ), @@ -43,6 +44,10 @@ def get_topology_data( topology_data = get_all_topology_data( tenant_id, provider_id, service_id, environment ) + if not includeEmptyDeps: + topology_data = [ + topology for topology in topology_data if topology.dependencies + ] return topology_data except Exception: logger.exception("Failed to get topology data") From e77c0874cd0aa9112d8919fc48cc7b3f18d891d2 Mon Sep 17 00:00:00 2001 From: Tal Borenstein Date: Tue, 3 Sep 2024 16:52:20 +0300 Subject: [PATCH 03/10] chore: tp beta is dynamic --- keep-ui/components/navbar/NoiseReductionLinks.tsx | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/keep-ui/components/navbar/NoiseReductionLinks.tsx b/keep-ui/components/navbar/NoiseReductionLinks.tsx index 3533cc2d8..ab6bc7f6e 100644 --- a/keep-ui/components/navbar/NoiseReductionLinks.tsx +++ b/keep-ui/components/navbar/NoiseReductionLinks.tsx @@ -10,11 +10,13 @@ import classNames from "classnames"; import { AILink } from "./AILink"; import { TbTopologyRing } from "react-icons/tb"; import { FaVolumeMute } from "react-icons/fa"; +import { useTopology } from "utils/hooks/useTopology"; type NoiseReductionLinksProps = { session: Session | null }; export const NoiseReductionLinks = ({ session }: NoiseReductionLinksProps) => { const isNOCRole = session?.userRole === "noc"; + const { topologyData } = useTopology(); if (isNOCRole) { return null; @@ -50,7 +52,12 @@ export const NoiseReductionLinks = ({ session }: NoiseReductionLinksProps) => {
  • - + Service Topology
    • From 1276767171c8167e1a31f7412edac0ec3a267083 Mon Sep 17 00:00:00 2001 From: Tal Borenstein Date: Wed, 4 Sep 2024 15:31:22 +0300 Subject: [PATCH 04/10] feat: working --- .../app/mapping/create-or-edit-mapping.tsx | 81 +++++++++++++---- keep-ui/app/mapping/mapping.tsx | 10 +-- keep-ui/app/mapping/models.tsx | 1 + keep-ui/app/mapping/rules-table.tsx | 5 ++ keep-ui/app/topology/custom-node.tsx | 67 +++++++++++++- keep-ui/app/topology/layout.tsx | 10 ++- keep-ui/app/topology/models.tsx | 3 + keep-ui/app/topology/page.tsx | 8 +- keep-ui/app/topology/topology.tsx | 9 +- .../components/navbar/NoiseReductionLinks.tsx | 6 +- keep/api/bl/enrichments_bl.py | 90 ++++++++++++------- keep/api/core/db.py | 12 +++ keep/api/models/db/mapping.py | 22 +++-- .../versions/2024-09-04-13-09_e6653be70b62.py | 34 +++++++ keep/api/routes/mapping.py | 34 ++++--- 15 files changed, 303 insertions(+), 89 deletions(-) create mode 100644 keep/api/models/db/migrations/versions/2024-09-04-13-09_e6653be70b62.py diff --git a/keep-ui/app/mapping/create-or-edit-mapping.tsx b/keep-ui/app/mapping/create-or-edit-mapping.tsx index 71097ba60..f911bf98a 100644 --- a/keep-ui/app/mapping/create-or-edit-mapping.tsx +++ b/keep-ui/app/mapping/create-or-edit-mapping.tsx @@ -11,6 +11,11 @@ import { Badge, Button, Icon, + TabGroup, + TabList, + Tab, + TabPanels, + TabPanel, } from "@tremor/react"; import { useSession } from "next-auth/react"; import { @@ -27,6 +32,7 @@ import { getApiURL } from "utils/apiUrl"; import { useMappings } from "utils/hooks/useMappingRules"; import { MappingRule } from "./models"; import { CreateableSearchSelect } from "@/components/ui/CreateableSearchSelect"; +import { useTopology } from "utils/hooks/useTopology"; interface Props { editRule: MappingRule | null; @@ -36,12 +42,15 @@ interface Props { export default function CreateOrEditMapping({ editRule, editCallback }: Props) { const { data: session } = useSession(); const { mutate } = useMappings(); + const [tabIndex, setTabIndex] = useState(0); const [mapName, setMapName] = useState(""); const [fileName, setFileName] = useState(""); + const [mappingType, setMappingType] = useState<"csv" | "topology">("csv"); const [mapDescription, setMapDescription] = useState(""); const [selectedLookupAttributes, setSelectedLookupAttributes] = useState< string[] >([]); + const { topologyData } = useTopology(); const [priority, setPriority] = useState(0); const editMode = editRule !== null; const inputFile = useRef(null); @@ -53,6 +62,8 @@ export default function CreateOrEditMapping({ editRule, editCallback }: Props) { setMapName(editRule.name); setFileName(editRule.file_name ? editRule.file_name : ""); setMapDescription(editRule.description ? editRule.description : ""); + setMappingType(editRule.type ? editRule.type : "csv"); + setTabIndex(editRule.type === "csv" ? 0 : 1); setSelectedLookupAttributes(editRule.matchers ? editRule.matchers : []); setPriority(editRule.priority); } @@ -83,6 +94,17 @@ export default function CreateOrEditMapping({ editRule, editCallback }: Props) { } }; + const updateMappingType = (index: number) => { + setTabIndex(index); + if (index === 0) { + setParsedData(null); + setMappingType("csv"); + } else { + setParsedData(topologyData!); + setMappingType("topology"); + } + }; + const readFile = (event: ChangeEvent) => { const file = event.target.files?.[0]; setFileName(file?.name || ""); @@ -123,8 +145,9 @@ export default function CreateOrEditMapping({ editRule, editCallback }: Props) { name: mapName, description: mapDescription, file_name: fileName, + type: mappingType, matchers: selectedLookupAttributes.map((attr) => attr.trim()), - rows: parsedData, + rows: mappingType === "csv" ? parsedData : null, }), }); if (response.ok) { @@ -154,8 +177,9 @@ export default function CreateOrEditMapping({ editRule, editCallback }: Props) { name: mapName, description: mapDescription, file_name: fileName, + type: mappingType, matchers: selectedLookupAttributes.map((attr) => attr.trim()), - rows: parsedData, + rows: mappingType === "csv" ? parsedData : null, }), }); if (response.ok) { @@ -229,20 +253,45 @@ export default function CreateOrEditMapping({ editRule, editCallback }: Props) {
    - - {!parsedData && ( - - {!editMode - ? "* Upload a CSV file to begin with creating a new mapping" - : ""} - - )} + updateMappingType(index)} + > + + CSV + + Topology + + + + + {mappingType === "csv" && ( + + )} + {!parsedData && ( + + {!editMode + ? "* Upload a CSV file to begin with creating a new mapping" + : ""} + + )} + + + +
    Mapping Schema
    diff --git a/keep-ui/app/mapping/mapping.tsx b/keep-ui/app/mapping/mapping.tsx index 3d65e3b37..ad86ae319 100644 --- a/keep-ui/app/mapping/mapping.tsx +++ b/keep-ui/app/mapping/mapping.tsx @@ -17,7 +17,7 @@ export default function Mapping() { return (
    -
    +

    Configure

    Add dynamic context to your alerts with mapping rules @@ -25,7 +25,7 @@ export default function Mapping() {

    -
    +
    {isLoading ? ( ) : mappings && mappings.length > 0 ? ( @@ -36,10 +36,8 @@ export default function Mapping() { title="Mapping rules does not exist" icon={MdWarning} > -

    No mapping rules found

    -

    - Configure new mapping rule using the mapping rules wizard. -

    + No mapping rules found. Configure new mapping rule using the + mapping rules wizard. )}
    diff --git a/keep-ui/app/mapping/models.tsx b/keep-ui/app/mapping/models.tsx index 1caa6f050..2d3ac7eee 100644 --- a/keep-ui/app/mapping/models.tsx +++ b/keep-ui/app/mapping/models.tsx @@ -11,6 +11,7 @@ export interface MappingRule { last_updated_at: Date; disabled: boolean; override: boolean; + type: "csv" | "topology"; condition?: string; matchers: string[]; rows: { [key: string]: any }[]; diff --git a/keep-ui/app/mapping/rules-table.tsx b/keep-ui/app/mapping/rules-table.tsx index f5ffe7529..0a126fa74 100644 --- a/keep-ui/app/mapping/rules-table.tsx +++ b/keep-ui/app/mapping/rules-table.tsx @@ -70,6 +70,11 @@ export default function RulesTable({ mappings, editCallback }: Props) { header: "Name", cell: (context) => context.row.original.name, }), + columnHelper.display({ + id: "type", + header: "Type", + cell: (context) => context.row.original.type, + }), columnHelper.display({ id: "description", header: "Description", diff --git a/keep-ui/app/topology/custom-node.tsx b/keep-ui/app/topology/custom-node.tsx index 5ba70b391..a416abcc3 100644 --- a/keep-ui/app/topology/custom-node.tsx +++ b/keep-ui/app/topology/custom-node.tsx @@ -1,4 +1,4 @@ -import React, { useEffect } from "react"; +import React, { useEffect, useState } from "react"; import { Handle, Position } from "@xyflow/react"; import { useAlerts } from "utils/hooks/useAlerts"; import { useAlertPolling } from "utils/hooks/usePusher"; @@ -12,6 +12,7 @@ const CustomNode = ({ data }: { data: TopologyService }) => { const { data: alerts, mutate } = useAllAlerts("feed"); const { data: pollAlerts } = useAlertPolling(); const router = useRouter(); + const [showDetails, setShowDetails] = useState(false); useEffect(() => { if (pollAlerts) { @@ -19,7 +20,9 @@ const CustomNode = ({ data }: { data: TopologyService }) => { } }, [pollAlerts, mutate]); - const relevantAlerts = alerts?.filter((alert) => alert.service === data.service); + const relevantAlerts = alerts?.filter( + (alert) => alert.service === data.service + ); const handleClick = () => { router.push( @@ -31,7 +34,11 @@ const CustomNode = ({ data }: { data: TopologyService }) => { const badgeColor = alertCount < THRESHOLD ? "bg-orange-500" : "bg-red-500"; return ( -
    +
    setShowDetails(true)} + onMouseLeave={() => setShowDetails(false)} + > {data.display_name ?? data.service} {alertCount > 0 && ( { {alertCount} )} + {showDetails && ( +
    + {data.service && ( +

    + Service: {data.service} +

    + )} + {data.display_name && ( +

    + Display Name: {data.display_name} +

    + )} + {data.description && ( +

    + Description: {data.description} +

    + )} + {data.team && ( +

    + Team: {data.team} +

    + )} + {data.application && ( +

    + Application: {data.application} +

    + )} + {data.email && ( +

    + Email: {data.email} +

    + )} + {data.slack && ( +

    + Slack: {data.slack} +

    + )} + {data.ip_address && ( +

    + IP Address: {data.ip_address} +

    + )} + {data.mac_address && ( +

    + MAC Address: {data.mac_address} +

    + )} + {data.manufacturer && ( +

    + Manufacturer: {data.manufacturer} +

    + )} +
    + )}
    diff --git a/keep-ui/app/topology/layout.tsx b/keep-ui/app/topology/layout.tsx index ab479c40f..cc1a91360 100644 --- a/keep-ui/app/topology/layout.tsx +++ b/keep-ui/app/topology/layout.tsx @@ -1,5 +1,13 @@ +import { Subtitle, Title } from "@tremor/react"; + export default function Layout({ children }: { children: any }) { return ( -
    {children}
    +
    + Service Topology + + Data describing the topology of components in your environment. + + {children} +
    ); } diff --git a/keep-ui/app/topology/models.tsx b/keep-ui/app/topology/models.tsx index ebd15ebed..4ee087158 100644 --- a/keep-ui/app/topology/models.tsx +++ b/keep-ui/app/topology/models.tsx @@ -17,4 +17,7 @@ export interface TopologyService { email?: string; slack?: string; dependencies: TopologyServiceDependency[]; + ip_address?: string; + mac_address?: string; + manufacturer?: string; } diff --git a/keep-ui/app/topology/page.tsx b/keep-ui/app/topology/page.tsx index 497f57a3f..abed94c59 100644 --- a/keep-ui/app/topology/page.tsx +++ b/keep-ui/app/topology/page.tsx @@ -1,13 +1,7 @@ -import { Title } from "@tremor/react"; import TopologyPage from "./topology"; export default function Page() { - return ( - <> - Service Topology - - - ); + return ; } export const metadata = { diff --git a/keep-ui/app/topology/topology.tsx b/keep-ui/app/topology/topology.tsx index 151a7f54d..d027690b6 100644 --- a/keep-ui/app/topology/topology.tsx +++ b/keep-ui/app/topology/topology.tsx @@ -95,11 +95,13 @@ const TopologyPage = ({ ); const onNodesChange = useCallback( - (changes: NodeChange[]) => setNodes((nds) => applyNodeChanges(changes, nds)), + (changes: NodeChange[]) => + setNodes((nds) => applyNodeChanges(changes, nds)), [] ); const onEdgesChange = useCallback( - (changes: EdgeChange[]) => setEdges((eds) => applyEdgeChanges(changes, eds)), + (changes: EdgeChange[]) => + setEdges((eds) => applyEdgeChanges(changes, eds)), [] ); @@ -202,7 +204,7 @@ const TopologyPage = ({ ); return ( - + {showSearch && (
    onEdgeHover("enter", edge)} onEdgeMouseLeave={(_event, edge) => onEdgeHover("leave", edge)} nodeTypes={{ customNode: CustomNode }} diff --git a/keep-ui/components/navbar/NoiseReductionLinks.tsx b/keep-ui/components/navbar/NoiseReductionLinks.tsx index ab6bc7f6e..b7799d42b 100644 --- a/keep-ui/components/navbar/NoiseReductionLinks.tsx +++ b/keep-ui/components/navbar/NoiseReductionLinks.tsx @@ -55,8 +55,10 @@ export const NoiseReductionLinks = ({ session }: NoiseReductionLinksProps) => { Service Topology diff --git a/keep/api/bl/enrichments_bl.py b/keep/api/bl/enrichments_bl.py index ead5d5259..f56b53fe6 100644 --- a/keep/api/bl/enrichments_bl.py +++ b/keep/api/bl/enrichments_bl.py @@ -8,7 +8,11 @@ from sqlmodel import Session from keep.api.core.db import enrich_alert as enrich_alert_db -from keep.api.core.db import get_enrichment_with_session, get_mapping_rule_by_id +from keep.api.core.db import ( + get_enrichment_with_session, + get_mapping_rule_by_id, + get_topology_data_by_dynamic_matcher, +) from keep.api.core.elastic import ElasticClient from keep.api.models.alert import AlertDto from keep.api.models.db.alert import AlertActionType @@ -264,41 +268,67 @@ def _check_alert_matches_rule(self, alert: AlertDto, rule: MappingRule) -> bool: ) # Apply enrichment to the alert - for row in rule.rows: - if any( - self._check_matcher(alert, row, matcher) for matcher in rule.matchers - ): - # Extract enrichments from the matched row - enrichments = { - key: value for key, value in row.items() if key not in rule.matchers - } + if rule.type == "topology": + matcher_value = {} + for matcher in rule.matchers: + matcher_value[matcher] = get_nested_attribute(alert, matcher) + topology_service = get_topology_data_by_dynamic_matcher( + self.tenant_id, matcher_value + ) + + if not topology_service: + self.logger.warning("No topology service found to match on") + enrichments = {} + else: + enrichments = topology_service.dict() + # Remove redundant fields + enrichments.pop("tenant_id", None) + enrichments.pop("id", None) + elif rule.type == "csv": + for row in rule.rows: + if any( + self._check_matcher(alert, row, matcher) + for matcher in rule.matchers + ): + # Extract enrichments from the matched row + enrichments = { + key: value + for key, value in row.items() + if key not in rule.matchers + } + break - # Enrich the alert with the matched data from the row - for key, value in enrichments.items(): + if enrichments: + # Enrich the alert with the matched data from the row + for key, value in enrichments.items(): + # It's not relevant to enrich if the value if empty + if value is not None: setattr(alert, key, value) - # Save the enrichments to the database - # SHAHAR: since when running this enrich_alert, the alert is not in elastic yet (its indexed after), - # enrich alert will fail to update the alert in elastic. - # hence should_exist = False - self.enrich_alert( - alert.fingerprint, - enrichments, - action_type=AlertActionType.MAPPING_RULE_ENRICH, - action_callee="system", - action_description="Alert enriched with mapping rule", - should_exist=False, - ) + # Save the enrichments to the database + # SHAHAR: since when running this enrich_alert, the alert is not in elastic yet (its indexed after), + # enrich alert will fail to update the alert in elastic. + # hence should_exist = False + self.enrich_alert( + alert.fingerprint, + enrichments, + action_type=AlertActionType.MAPPING_RULE_ENRICH, + action_callee="system", + action_description=f"Alert enriched with mapping from rule `{rule.name}`", + should_exist=False, + ) - self.logger.info( - "Alert enriched", - extra={"fingerprint": alert.fingerprint, "rule_id": rule.id}, - ) + self.logger.info( + "Alert enriched", + extra={"fingerprint": alert.fingerprint, "rule_id": rule.id}, + ) - return ( - True # Exit on first successful enrichment (assuming single match) - ) + return True # Exit on first successful enrichment (assuming single match) + self.logger.info( + "Alert was not enriched by mapping rule", + extra={"rule_id": rule.id, "alert_fingerprint": alert.fingerprint}, + ) return False @staticmethod diff --git a/keep/api/core/db.py b/keep/api/core/db.py index cfab9b586..c23195e1d 100644 --- a/keep/api/core/db.py +++ b/keep/api/core/db.py @@ -2743,6 +2743,18 @@ def get_all_topology_data( return service_dtos +def get_topology_data_by_dynamic_matcher( + tenant_id: str, matchers_value: dict[str, str] +) -> TopologyService | None: + with Session(engine) as session: + query = select(TopologyService).where(TopologyService.tenant_id == tenant_id) + for matcher in matchers_value: + query = query.where( + getattr(TopologyService, matcher) == matchers_value[matcher] + ) + return session.exec(query).first() + + def get_tags(tenant_id): with Session(engine) as session: tags = session.exec(select(Tag).where(Tag.tenant_id == tenant_id)).all() diff --git a/keep/api/models/db/mapping.py b/keep/api/models/db/mapping.py index 394951a97..2081656ea 100644 --- a/keep/api/models/db/mapping.py +++ b/keep/api/models/db/mapping.py @@ -1,7 +1,7 @@ from datetime import datetime, timezone -from typing import Optional +from typing import Literal, Optional -from pydantic import BaseModel +from pydantic import BaseModel, validator from sqlmodel import JSON, Column, Field, SQLModel @@ -18,12 +18,14 @@ class MappingRule(SQLModel, table=True): # Whether this rule should override existing attributes in the alert override: bool = Field(default=True) condition: Optional[str] = Field(max_length=2000) + # The type of this mapping rule + type: str = "csv" # The attributes to match against (e.g. ["service","region"]) matchers: list[str] = Field(sa_column=Column(JSON), nullable=False) # The rows of the CSV file [{service: "service1", region: "region1", ...}, ...] - rows: list[dict] = Field( + rows: Optional[list[dict]] = Field( sa_column=Column(JSON), - nullable=False, + nullable=True, ) # max_length=204800) updated_by: Optional[str] = Field(max_length=255, default=None) last_updated_at: datetime = Field(default_factory=datetime.utcnow) @@ -35,6 +37,7 @@ class MappRuleDtoBase(BaseModel): file_name: Optional[str] = None priority: int = 0 matchers: list[str] + type: Literal["csv", "topology"] = "csv" class MappingRuleDtoOut(MappRuleDtoBase, extra="ignore"): @@ -47,9 +50,10 @@ class MappingRuleDtoOut(MappRuleDtoBase, extra="ignore"): class MappingRuleDtoIn(MappRuleDtoBase): - rows: list[dict] - - -class MappingRuleDtoUpdate(MappRuleDtoBase): - id: int rows: Optional[list[dict]] = None + + @validator("type", pre=True, always=True) + def validate_type(cls, _type, values): + if _type == "csv" and not values.get("rows"): + raise ValueError("Mapping of type CSV cannot have empty rows") + return _type diff --git a/keep/api/models/db/migrations/versions/2024-09-04-13-09_e6653be70b62.py b/keep/api/models/db/migrations/versions/2024-09-04-13-09_e6653be70b62.py new file mode 100644 index 000000000..0fae6158b --- /dev/null +++ b/keep/api/models/db/migrations/versions/2024-09-04-13-09_e6653be70b62.py @@ -0,0 +1,34 @@ +"""mapping type + +Revision ID: e6653be70b62 +Revises: 1a5eb7069f9a +Create Date: 2024-09-04 13:09:14.958740 + +""" + +import sqlalchemy as sa +import sqlmodel +from alembic import op + +# revision identifiers, used by Alembic. +revision = "e6653be70b62" +down_revision = "1a5eb7069f9a" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("mappingrule", schema=None) as batch_op: + batch_op.add_column( + sa.Column("type", sqlmodel.sql.sqltypes.AutoString(), nullable=False) + ) + + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("mappingrule", schema=None) as batch_op: + batch_op.drop_column("type") + # ### end Alembic commands ### diff --git a/keep/api/routes/mapping.py b/keep/api/routes/mapping.py index 029adbf95..e5cfbc4a9 100644 --- a/keep/api/routes/mapping.py +++ b/keep/api/routes/mapping.py @@ -5,12 +5,8 @@ from sqlmodel import Session from keep.api.core.db import get_session -from keep.api.models.db.mapping import ( - MappingRule, - MappingRuleDtoIn, - MappingRuleDtoOut, - MappingRuleDtoUpdate, -) +from keep.api.models.db.mapping import MappingRule, MappingRuleDtoIn, MappingRuleDtoOut +from keep.api.models.db.topology import TopologyService from keep.identitymanager.authenticatedentity import AuthenticatedEntity from keep.identitymanager.identitymanagerfactory import IdentityManagerFactory @@ -38,9 +34,22 @@ def get_rules( if rules: for rule in rules: rule_dto = MappingRuleDtoOut(**rule.dict()) - rule_dto.attributes = [ - key for key in rule.rows[0].keys() if key not in rule.matchers - ] + + attributes = [] + if rule_dto.type == "csv": + attributes = [ + key for key in rule.rows[0].keys() if key not in rule.matchers + ] + elif rule_dto.type == "topology": + attributes = [ + field + for field in TopologyService.__fields__ + if field not in rule.matchers + and field != "tenant_id" + and field != "id" + ] + + rule_dto.attributes = attributes rules_dtos.append(rule_dto) return rules_dtos @@ -94,9 +103,10 @@ def delete_rule( return {"message": "Rule deleted successfully"} -@router.put("", description="Update an existing rule") +@router.put("/{rule_id}", description="Update an existing rule") def update_rule( - rule: MappingRuleDtoUpdate, + rule_id: int, + rule: MappingRuleDtoIn, authenticated_entity: AuthenticatedEntity = Depends( IdentityManagerFactory.get_auth_verifier(["write:rules"]) ), @@ -107,7 +117,7 @@ def update_rule( session.query(MappingRule) .filter( MappingRule.tenant_id == authenticated_entity.tenant_id, - MappingRule.id == rule.id, + MappingRule.id == rule_id, ) .first() ) From 68f206f7daa9b0d4eda387a5ca72d88d9624c69d Mon Sep 17 00:00:00 2001 From: Tal Borenstein Date: Wed, 4 Sep 2024 15:54:34 +0300 Subject: [PATCH 05/10] fix: styling --- keep-ui/app/alerts/alert-sidebar.tsx | 1 - .../app/mapping/create-or-edit-mapping.tsx | 2 +- keep-ui/app/topology/layout.tsx | 30 +++++++++++++++---- keep-ui/app/topology/topology.tsx | 27 ++++------------- 4 files changed, 31 insertions(+), 29 deletions(-) diff --git a/keep-ui/app/alerts/alert-sidebar.tsx b/keep-ui/app/alerts/alert-sidebar.tsx index 4603da55c..2f9195a20 100644 --- a/keep-ui/app/alerts/alert-sidebar.tsx +++ b/keep-ui/app/alerts/alert-sidebar.tsx @@ -112,7 +112,6 @@ const AlertSidebar = ({ isOpen, toggle, alert }: AlertSidebarProps) => { providerId={alert.providerId || ""} service={alert.service || ""} environment={"unknown"} - showSearch={false} />
    )} diff --git a/keep-ui/app/mapping/create-or-edit-mapping.tsx b/keep-ui/app/mapping/create-or-edit-mapping.tsx index f911bf98a..870a81a53 100644 --- a/keep-ui/app/mapping/create-or-edit-mapping.tsx +++ b/keep-ui/app/mapping/create-or-edit-mapping.tsx @@ -165,7 +165,7 @@ export default function CreateOrEditMapping({ editRule, editCallback }: Props) { const updateRule = async (e: FormEvent) => { e.preventDefault(); const apiUrl = getApiURL(); - const response = await fetch(`${apiUrl}/mapping`, { + const response = await fetch(`${apiUrl}/mapping/${editRule?.id}`, { method: "PUT", headers: { Authorization: `Bearer ${session?.accessToken}`, diff --git a/keep-ui/app/topology/layout.tsx b/keep-ui/app/topology/layout.tsx index cc1a91360..f9b2723f5 100644 --- a/keep-ui/app/topology/layout.tsx +++ b/keep-ui/app/topology/layout.tsx @@ -1,13 +1,31 @@ -import { Subtitle, Title } from "@tremor/react"; +"use client"; +import { Subtitle, TextInput, Title } from "@tremor/react"; +import { createContext, useState } from "react"; + +export const ServiceSearchContext = createContext(""); export default function Layout({ children }: { children: any }) { + const [serviceInput, setServiceInput] = useState(""); + return (
    - Service Topology - - Data describing the topology of components in your environment. - - {children} +
    +
    + Service Topology + + Data describing the topology of components in your environment. + +
    + +
    + + {children} +
    ); } diff --git a/keep-ui/app/topology/topology.tsx b/keep-ui/app/topology/topology.tsx index d027690b6..5fc749d5a 100644 --- a/keep-ui/app/topology/topology.tsx +++ b/keep-ui/app/topology/topology.tsx @@ -1,5 +1,5 @@ "use client"; -import React, { useCallback, useEffect, useState } from "react"; +import React, { useCallback, useContext, useEffect, useState } from "react"; import { Background, BackgroundVariant, @@ -33,12 +33,12 @@ import { useTopology } from "utils/hooks/useTopology"; import Loading from "app/loading"; import { EmptyStateCard } from "@/components/ui/EmptyStateCard"; import { useRouter } from "next/navigation"; +import { ServiceSearchContext } from "./layout"; interface Props { providerId?: string; service?: string; environment?: string; - showSearch?: boolean; } // Function to create a Dagre layout @@ -74,17 +74,12 @@ const getLayoutedElements = (nodes: any[], edges: any[]) => { return { nodes, edges }; }; -const TopologyPage = ({ - providerId, - service, - environment, - showSearch = true, -}: Props) => { +const TopologyPage = ({ providerId, service, environment }: Props) => { const router = useRouter(); // State for nodes and edges const [nodes, setNodes] = useState([]); const [edges, setEdges] = useState([]); - const [serviceInput, setServiceInput] = useState(""); + const serviceInput = useContext(ServiceSearchContext); const [reactFlowInstance, setReactFlowInstance] = useState>(); @@ -192,7 +187,7 @@ const TopologyPage = ({ return (
    - {showSearch && ( -
    - -
    - )} + Date: Wed, 4 Sep 2024 16:32:25 +0300 Subject: [PATCH 06/10] fix: build --- keep-ui/app/topology/layout.tsx | 5 ++--- keep-ui/app/topology/service-search-context.tsx | 3 +++ keep-ui/app/topology/topology.tsx | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) create mode 100644 keep-ui/app/topology/service-search-context.tsx diff --git a/keep-ui/app/topology/layout.tsx b/keep-ui/app/topology/layout.tsx index f9b2723f5..0a4ade6ce 100644 --- a/keep-ui/app/topology/layout.tsx +++ b/keep-ui/app/topology/layout.tsx @@ -1,8 +1,7 @@ "use client"; import { Subtitle, TextInput, Title } from "@tremor/react"; -import { createContext, useState } from "react"; - -export const ServiceSearchContext = createContext(""); +import { useState } from "react"; +import { ServiceSearchContext } from "./service-search-context"; export default function Layout({ children }: { children: any }) { const [serviceInput, setServiceInput] = useState(""); diff --git a/keep-ui/app/topology/service-search-context.tsx b/keep-ui/app/topology/service-search-context.tsx new file mode 100644 index 000000000..7d72545e9 --- /dev/null +++ b/keep-ui/app/topology/service-search-context.tsx @@ -0,0 +1,3 @@ +import { createContext } from "react"; + +export const ServiceSearchContext = createContext(""); diff --git a/keep-ui/app/topology/topology.tsx b/keep-ui/app/topology/topology.tsx index 5fc749d5a..478d252f1 100644 --- a/keep-ui/app/topology/topology.tsx +++ b/keep-ui/app/topology/topology.tsx @@ -17,7 +17,7 @@ import { import dagre, { graphlib } from "@dagrejs/dagre"; import "@xyflow/react/dist/style.css"; import CustomNode from "./custom-node"; -import { Card, TextInput } from "@tremor/react"; +import { Card } from "@tremor/react"; import { edgeLabelBgPaddingNoHover, edgeLabelBgStyleNoHover, @@ -33,7 +33,7 @@ import { useTopology } from "utils/hooks/useTopology"; import Loading from "app/loading"; import { EmptyStateCard } from "@/components/ui/EmptyStateCard"; import { useRouter } from "next/navigation"; -import { ServiceSearchContext } from "./layout"; +import { ServiceSearchContext } from "./service-search-context"; interface Props { providerId?: string; From 8fb4bf83b42f13aae815c4508357c7d9cb26fdb7 Mon Sep 17 00:00:00 2001 From: Tal Borenstein Date: Wed, 4 Sep 2024 16:55:55 +0300 Subject: [PATCH 07/10] docs: update --- docs/overview/enrichment/mapping.mdx | 26 +++++++++++++----- keep-ui/components/navbar/IncidentLinks.tsx | 22 +-------------- keep-ui/utils/hooks/useIncidents.ts | 1 - keep-ui/utils/hooks/useTopology.ts | 27 +++++++++++++++++++ keep/api/routes/preset.py | 2 +- keep/api/tasks/process_topology_task.py | 17 +++++++++++- .../datadog_provider/topology_mock.py | 2 +- 7 files changed, 65 insertions(+), 32 deletions(-) diff --git a/docs/overview/enrichment/mapping.mdx b/docs/overview/enrichment/mapping.mdx index 639555702..df140a2f0 100644 --- a/docs/overview/enrichment/mapping.mdx +++ b/docs/overview/enrichment/mapping.mdx @@ -4,28 +4,40 @@ title: "Mapping" # Alert Enrichment: Mapping -Keep's Alert Mapping enrichment feature provides a powerful mechanism for dynamically enhancing alert data by leveraging external data sources, such as CSV files. This feature allows for the matching of incoming alerts to specific records in a CSV file based on predefined attributes (matchers) and enriching those alerts with additional information from the matched records. +Keep's Alert Mapping enrichment feature provides a powerful mechanism for dynamically enhancing alert data by leveraging external data sources, such as CSV files and topology data. This feature allows for the matching of incoming alerts to specific records in a CSV file or topology data based on predefined attributes (matchers) and enriching those alerts with additional information from the matched records. ## Introduction -In complex monitoring environments, the need to enrich alert data with additional context is critical for effective alert analysis and response. Keep's Alert Mapping and Enrichment enables users to define rules that match alerts to rows in a CSV file, appending or modifying alert attributes with the values from matching rows. This process adds significant value to each alert, providing deeper insights and enabling more precise and informed decision-making. +In complex monitoring environments, the need to enrich alert data with additional context is critical for effective alert analysis and response. Keep's Alert Mapping and Enrichment enables users to define rules that match alerts to rows in a CSV file or topology data, appending or modifying alert attributes with the values from matching rows. This process adds significant value to each alert, providing deeper insights and enabling more precise and informed decision-making. ## How It Works +## Mapping with CSV Files + 1. **Rule Definition**: Users define mapping rules that specify which alert attributes (matchers) should be used for matching alerts to rows in a CSV file. 2. **CSV File Specification**: A CSV file is associated with each mapping rule. This file contains additional data that should be added to alerts matching the rule. 3. **Alert Matching**: When an alert is received, the system checks if it matches the conditions of any mapping rule based on the specified matchers. 4. **Data Enrichment**: If a match is found, the alert is enriched with additional data from the corresponding row in the CSV file. +## Mapping with Topology Data + +1. **Rule Definition**: Users define mapping rules that specify which alert attributes (matchers) should be used for matching alerts to topology data. +2. **Topology Data Specification**: Topology data is associated with each mapping rule. This data contains additional information about the components and their relationships in your environment. +3. **Alert Matching**: When an alert is received, the system checks if it matches the conditions of any mapping rule based on the specified matchers. +4. **Data Enrichment**: If a match is found, the alert is enriched with additional data from the corresponding topology data. + ## Practical Example Imagine you have a CSV file with columns representing different aspects of your infrastructure, such as `region`, `responsible_team`, and `severity_override`. By creating a mapping rule that matches alerts based on `service` and `region`, you can automatically enrich alerts with the responsible team and adjust severity based on the matched row in the CSV file. +Similarly, you can use topology data to enrich alerts. For example, if an alert is related to a specific service, you can use topology data to find related components and their statuses, providing a more comprehensive view of the issue. + ## Core Concepts -- **Matchers**: Attributes within the alert used to identify matching rows within the CSV file. Common matchers include identifiers like `service` or `region`. +- **Matchers**: Attributes within the alert used to identify matching rows within the CSV file or topology data. Common matchers include identifiers like `service` or `region`. - **CSV File**: A structured file containing rows of data. Each column represents a potential attribute that can be added to an alert. -- **Enrichment**: The process of adding new attributes or modifying existing ones in an alert based on the data from a matching CSV row. +- **Topology Data**: Information about the components and their relationships in your environment. This data can be used to enrich alerts with additional context. +- **Enrichment**: The process of adding new attributes or modifying existing ones in an alert based on the data from a matching CSV row or topology data. ## Creating a Mapping Rule @@ -35,13 +47,13 @@ To create an alert mapping and enrichment rule: -1. **Define the Matchers**: Specify which alert attributes will be used to match rows in the CSV file. -2. **Upload the CSV File**: Provide the CSV file containing the data for enrichment. +1. **Define the Matchers**: Specify which alert attributes will be used to match rows in the CSV file or topology data. +2. **Specify the Data Source**: Provide the CSV file or specify the topology data to be used for enrichment. 3. **Configure the Rule**: Set additional parameters, such as whether the rule should override existing alert attributes. ## Best Practices -- **Keep CSV Files Updated**: Regularly update the CSV files to reflect the current state of your infrastructure and operational data. +- **Keep CSV Files and Topology Data Updated**: Regularly update the CSV files and topology data to reflect the current state of your infrastructure and operational data. - **Use Specific Matchers**: Define matchers that are unique and relevant to ensure accurate matching. - **Monitor Rule Performance**: Review the application of mapping rules to ensure they are working as expected and adjust them as necessary. diff --git a/keep-ui/components/navbar/IncidentLinks.tsx b/keep-ui/components/navbar/IncidentLinks.tsx index 4533dd511..a78502337 100644 --- a/keep-ui/components/navbar/IncidentLinks.tsx +++ b/keep-ui/components/navbar/IncidentLinks.tsx @@ -17,7 +17,7 @@ const SHOW_N_INCIDENTS = 3; export const IncidentsLinks = ({ session }: IncidentsLinksProps) => { const isNOCRole = session?.userRole === "noc"; const { data: incidents, mutate } = useIncidents(); - usePollIncidents(mutate) + usePollIncidents(mutate); const currentPath = usePathname(); if (isNOCRole) { @@ -52,26 +52,6 @@ export const IncidentsLinks = ({ session }: IncidentsLinksProps) => { Incidents - {incidents?.items.slice(0, SHOW_N_INCIDENTS).map((incident) => ( -
  • - - - {incident.user_generated_name || incident.ai_generated_name} - -
    • - ))} - {/* {incidents && incidents.items.length > SHOW_N_INCIDENTS && ( -
  • -
    ...
    -
    • - )} */} ); diff --git a/keep-ui/utils/hooks/useIncidents.ts b/keep-ui/utils/hooks/useIncidents.ts index 849bf799f..e15e7e09c 100644 --- a/keep-ui/utils/hooks/useIncidents.ts +++ b/keep-ui/utils/hooks/useIncidents.ts @@ -1,4 +1,3 @@ -import { AlertDto } from "app/alerts/models"; import {IncidentDto, PaginatedIncidentAlertsDto, PaginatedIncidentsDto} from "app/incidents/model"; import { useSession } from "next-auth/react"; import useSWR, { SWRConfiguration } from "swr"; diff --git a/keep-ui/utils/hooks/useTopology.ts b/keep-ui/utils/hooks/useTopology.ts index 74d46c47b..ba959cd54 100644 --- a/keep-ui/utils/hooks/useTopology.ts +++ b/keep-ui/utils/hooks/useTopology.ts @@ -3,15 +3,24 @@ import { useSession } from "next-auth/react"; import useSWR from "swr"; import { getApiURL } from "utils/apiUrl"; import { fetcher } from "utils/fetcher"; +import { useWebsocket } from "./usePusher"; +import { useCallback, useEffect } from "react"; +import { toast } from "react-toastify"; const isNullOrUndefined = (value: any) => value === null || value === undefined; +interface TopologyUpdate { + providerType: string; + providerId: string; +} + export const useTopology = ( providerId?: string, service?: string, environment?: string ) => { const { data: session } = useSession(); + useTopologyPolling(); const apiUrl = getApiURL(); const url = !session @@ -34,3 +43,21 @@ export const useTopology = ( mutate, }; }; + +export const useTopologyPolling = () => { + const { bind, unbind } = useWebsocket(); + + const handleIncoming = useCallback((data: TopologyUpdate) => { + toast.success( + `Topology pulled from ${data.providerId} (${data.providerType})`, + { position: "top-right" } + ); + }, []); + + useEffect(() => { + bind("topology-update", handleIncoming); + return () => { + unbind("topology-update", handleIncoming); + }; + }, [bind, unbind, handleIncoming]); +}; diff --git a/keep/api/routes/preset.py b/keep/api/routes/preset.py index b10ab24e0..bb4cb3c00 100644 --- a/keep/api/routes/preset.py +++ b/keep/api/routes/preset.py @@ -105,7 +105,7 @@ def pull_data_from_providers( logger.info("Getting topology data", extra=extra) topology_data = provider_class.pull_topology() logger.info("Got topology data, processing", extra=extra) - process_topology(tenant_id, topology_data, provider.id) + process_topology(tenant_id, topology_data, provider.id, provider.type) logger.info("Processed topology data", extra=extra) except NotImplementedError: logger.warning( diff --git a/keep/api/tasks/process_topology_task.py b/keep/api/tasks/process_topology_task.py index 88f282e97..5d65de035 100644 --- a/keep/api/tasks/process_topology_task.py +++ b/keep/api/tasks/process_topology_task.py @@ -2,6 +2,7 @@ import logging from keep.api.core.db import get_session_sync +from keep.api.core.dependencies import get_pusher_client from keep.api.models.db.topology import ( TopologyService, TopologyServiceDependency, @@ -14,7 +15,10 @@ def process_topology( - tenant_id: str, topology_data: list[TopologyServiceInDto], provider_id: str + tenant_id: str, + topology_data: list[TopologyServiceInDto], + provider_id: str, + provider_type: str, ): extra = {"provider_id": provider_id, "tenant_id": tenant_id} if not topology_data: @@ -91,6 +95,17 @@ def process_topology( extra={**extra, "error": str(e)}, ) + try: + pusher_client = get_pusher_client() + if pusher_client: + pusher_client.trigger( + f"private-{tenant_id}", + "topology-update", + {"providerId": provider_id, "providerType": provider_type}, + ) + except Exception: + logger.exception("Failed to push topology update to the client") + logger.info( "Created new topology data", extra=extra, diff --git a/keep/providers/datadog_provider/topology_mock.py b/keep/providers/datadog_provider/topology_mock.py index 8ec5e443c..ea2100d64 100644 --- a/keep/providers/datadog_provider/topology_mock.py +++ b/keep/providers/datadog_provider/topology_mock.py @@ -37,4 +37,4 @@ topology_data = list(services.values()) print(topology_data) - process_topology("keep", topology_data, "datadog") + process_topology("keep", topology_data, "datadog", "datadog") From 324c367250a0672c6b762f9580ee92f46f9b23f7 Mon Sep 17 00:00:00 2001 From: Tal Borenstein Date: Wed, 4 Sep 2024 19:47:20 +0300 Subject: [PATCH 08/10] fix: fix --- keep-ui/app/mapping/mapping.tsx | 4 ++-- keep/api/tasks/process_event_task.py | 16 ++++++++++++++-- tests/test_enrichments.py | 8 ++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/keep-ui/app/mapping/mapping.tsx b/keep-ui/app/mapping/mapping.tsx index ad86ae319..dadb1509b 100644 --- a/keep-ui/app/mapping/mapping.tsx +++ b/keep-ui/app/mapping/mapping.tsx @@ -17,7 +17,7 @@ export default function Mapping() { return (
    -
    +

    Configure

    Add dynamic context to your alerts with mapping rules @@ -25,7 +25,7 @@ export default function Mapping() {

    -
    +
    {isLoading ? ( ) : mappings && mappings.length > 0 ? ( diff --git a/keep/api/tasks/process_event_task.py b/keep/api/tasks/process_event_task.py index ff05bd608..245d0326f 100644 --- a/keep/api/tasks/process_event_task.py +++ b/keep/api/tasks/process_event_task.py @@ -111,13 +111,25 @@ def __save_to_db( try: enrichments_bl.dispose_enrichments(formatted_event.fingerprint) except Exception: - logger.exception("Failed to dispose enrichments") + logger.exception( + "Failed to dispose enrichments", + extra={ + "tenant_id": tenant_id, + "fingerprint": formatted_event.fingerprint, + }, + ) # Post format enrichment try: formatted_event = enrichments_bl.run_extraction_rules(formatted_event) except Exception: - logger.exception("Failed to run post-formatting extraction rules") + logger.exception( + "Failed to run post-formatting extraction rules", + extra={ + "tenant_id": tenant_id, + "fingerprint": formatted_event.fingerprint, + }, + ) # Make sure the lastReceived is a valid date string # tb: we do this because `AlertDto` object lastReceived is a string and not a datetime object diff --git a/tests/test_enrichments.py b/tests/test_enrichments.py index 8ffe79850..c583a8620 100644 --- a/tests/test_enrichments.py +++ b/tests/test_enrichments.py @@ -249,6 +249,7 @@ def test_run_mapping_rules_applies(mock_session, mock_alert_dto): matchers=["name"], rows=[{"name": "Test Alert", "service": "new_service"}], disabled=False, + type="csv", ) mock_session.query.return_value.filter.return_value.filter.return_value.order_by.return_value.all.return_value = [ rule @@ -273,6 +274,7 @@ def test_run_mapping_rules_with_regex_match(mock_session, mock_alert_dto): {"name": "frontend-service", "service": "frontend_service"}, ], disabled=False, + type="csv", ) mock_session.query.return_value.filter.return_value.filter.return_value.order_by.return_value.all.return_value = [ rule @@ -316,6 +318,7 @@ def test_run_mapping_rules_no_match(mock_session, mock_alert_dto): {"name": "frontend-service", "service": "frontend_service"}, ], disabled=False, + type="csv", ) mock_session.query.return_value.filter.return_value.filter.return_value.order_by.return_value.all.return_value = [ rule @@ -341,6 +344,7 @@ def test_check_matcher_with_and_condition(mock_session, mock_alert_dto): matchers=["name && severity"], rows=[{"name": "Test Alert", "severity": "high", "service": "new_service"}], disabled=False, + type="csv", ) mock_session.query.return_value.filter.return_value.filter.return_value.order_by.return_value.all.return_value = [ rule @@ -380,6 +384,7 @@ def test_check_matcher_with_or_condition(mock_session, mock_alert_dto): {"severity": "high", "service": "high_severity_service"}, ], disabled=False, + type="csv", ) mock_session.query.return_value.filter.return_value.filter.return_value.order_by.return_value.all.return_value = [ rule @@ -435,6 +440,7 @@ def test_mapping_rule_with_elsatic(mock_session, mock_alert_dto, setup_alerts): {"severity": "high", "service": "high_severity_service"}, ], disabled=False, + type="csv", ) mock_session.query.return_value.filter.return_value.filter.return_value.order_by.return_value.all.return_value = [ rule @@ -462,6 +468,7 @@ def test_enrichment(client, db_session, test_app, mock_alert_dto, elastic_client ], name="new_rule", disabled=False, + type="csv", ) db_session.add(rule) db_session.commit() @@ -499,6 +506,7 @@ def test_disposable_enrichment(client, db_session, test_app, mock_alert_dto): ], name="new_rule", disabled=False, + type="csv", ) db_session.add(rule) db_session.commit() From e1f300e5af7b8dc7bb61897376d29a8155798a99 Mon Sep 17 00:00:00 2001 From: Tal Borenstein Date: Thu, 5 Sep 2024 15:51:06 +0300 Subject: [PATCH 09/10] fix: tests --- keep/api/bl/enrichments_bl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keep/api/bl/enrichments_bl.py b/keep/api/bl/enrichments_bl.py index f56b53fe6..728746939 100644 --- a/keep/api/bl/enrichments_bl.py +++ b/keep/api/bl/enrichments_bl.py @@ -268,6 +268,7 @@ def _check_alert_matches_rule(self, alert: AlertDto, rule: MappingRule) -> bool: ) # Apply enrichment to the alert + enrichments = {} if rule.type == "topology": matcher_value = {} for matcher in rule.matchers: @@ -278,7 +279,6 @@ def _check_alert_matches_rule(self, alert: AlertDto, rule: MappingRule) -> bool: if not topology_service: self.logger.warning("No topology service found to match on") - enrichments = {} else: enrichments = topology_service.dict() # Remove redundant fields From fb74fbf5fa816d3ec8accf0087337f9867e0fa3f Mon Sep 17 00:00:00 2001 From: Tal Borenstein Date: Sun, 8 Sep 2024 11:10:20 +0300 Subject: [PATCH 10/10] chore: add unit test --- keep/api/bl/enrichments_bl.py | 4 +-- tests/test_enrichments.py | 61 +++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 2 deletions(-) diff --git a/keep/api/bl/enrichments_bl.py b/keep/api/bl/enrichments_bl.py index 728746939..3cec23e0e 100644 --- a/keep/api/bl/enrichments_bl.py +++ b/keep/api/bl/enrichments_bl.py @@ -280,7 +280,7 @@ def _check_alert_matches_rule(self, alert: AlertDto, rule: MappingRule) -> bool: if not topology_service: self.logger.warning("No topology service found to match on") else: - enrichments = topology_service.dict() + enrichments = topology_service.dict(exclude_none=True) # Remove redundant fields enrichments.pop("tenant_id", None) enrichments.pop("id", None) @@ -294,7 +294,7 @@ def _check_alert_matches_rule(self, alert: AlertDto, rule: MappingRule) -> bool: enrichments = { key: value for key, value in row.items() - if key not in rule.matchers + if key not in rule.matchers and value is not None } break diff --git a/tests/test_enrichments.py b/tests/test_enrichments.py index c583a8620..74ea6878c 100644 --- a/tests/test_enrichments.py +++ b/tests/test_enrichments.py @@ -6,8 +6,10 @@ from keep.api.bl.enrichments_bl import EnrichmentsBl from keep.api.core.dependencies import SINGLE_TENANT_UUID from keep.api.models.alert import AlertDto +from keep.api.models.db.alert import AlertActionType from keep.api.models.db.extraction import ExtractionRule from keep.api.models.db.mapping import MappingRule +from keep.api.models.db.topology import TopologyService from tests.fixtures.client import client, setup_api_key, test_app # noqa @@ -556,3 +558,62 @@ def test_disposable_enrichment(client, db_session, test_app, mock_alert_dto): assert len(alerts) == 1 alert = alerts[0] assert alert["status"] == "firing" + + +def test_topology_mapping_rule_enrichment(mock_session, mock_alert_dto): + # Mock a TopologyService with dependencies to simulate the DB structure + mock_topology_service = TopologyService( + id=1, tenant_id="keep", service="test-service", display_name="Test Service" + ) + + # Create a mock MappingRule for topology + rule = MappingRule( + id=3, + tenant_id=SINGLE_TENANT_UUID, + priority=1, + matchers=["service"], + name="topology_rule", + disabled=False, + type="topology", + ) + + # Mock the session to return this topology mapping rule + mock_session.query.return_value.filter.return_value.all.return_value = [rule] + + # Initialize the EnrichmentsBl class with the mock session + enrichment_bl = EnrichmentsBl(tenant_id="test_tenant", db=mock_session) + + mock_alert_dto.service = "test-service" + + # Mock the get_topology_data_by_dynamic_matcher to return the mock topology service + with patch( + "keep.api.bl.enrichments_bl.get_topology_data_by_dynamic_matcher", + return_value=mock_topology_service, + ): + # Mock the enrichment database function so no actual DB actions occur + with patch( + "keep.api.bl.enrichments_bl.enrich_alert_db" + ) as mock_enrich_alert_db: + # Run the mapping rule logic for the topology + result_event = enrichment_bl.run_mapping_rules(mock_alert_dto) + + # Check that the topology enrichment was applied correctly + assert getattr(result_event, "display_name", None) == "Test Service" + + # Verify that the DB enrichment function was called correctly + mock_enrich_alert_db.assert_called_once_with( + "test_tenant", + mock_alert_dto.fingerprint, + { + "source_provider_id": "unknown", + "service": "test-service", + "environment": "unknown", + "display_name": "Test Service", + }, + action_callee="system", + action_type=AlertActionType.MAPPING_RULE_ENRICH, + action_description="Alert enriched with mapping from rule `topology_rule`", + session=mock_session, + force=False, + audit_enabled=True, + )