From efb788eb9f666ab0c972dc2a544de40317eef9ca Mon Sep 17 00:00:00 2001 From: shahargl Date: Wed, 25 Sep 2024 11:48:01 +0300 Subject: [PATCH] feat: cr fixes --- docs/mint.json | 1 + docs/overview/deduplication.mdx | 107 ++++ .../deduplication/DeduplicationSidebar.tsx | 478 ++++++++++-------- .../app/deduplication/DeduplicationTable.tsx | 2 +- 4 files changed, 383 insertions(+), 205 deletions(-) create mode 100644 docs/overview/deduplication.mdx diff --git a/docs/mint.json b/docs/mint.json index d06960ab6..191697a2b 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -45,6 +45,7 @@ ] }, "overview/maintenance-windows", + "overview/deduplication", "overview/examples" ] }, diff --git a/docs/overview/deduplication.mdx b/docs/overview/deduplication.mdx new file mode 100644 index 000000000..00d755ec5 --- /dev/null +++ b/docs/overview/deduplication.mdx @@ -0,0 +1,107 @@ +--- +title: "Alert Deduplication" +--- + +## Overview + +Alert deduplication is a crucial feature in Keep that helps reduce noise and streamline incident management by grouping similar alerts together. This process ensures that your team isn't overwhelmed by a flood of notifications for what is essentially the same issue, allowing for more efficient and focused incident response. + +## Glossary + +- **Deduplication Rule**: A set of criteria used to determine if alerts should be grouped together. +- **Partial Deduplication**: Correlates instances of alerts into single alerts, considering the case of the same alert with different statuses (e.g., firing and resolved). This is the default mode where specified fields are used to identify and group related alerts. +- **Fingerprint Fields**: Specific alert attributes used to identify similar alerts. +- **Full Deduplication**: A mode where alerts are considered identical if all fields match exactly (except those explicitly ignored). This helps avoid system overload by discarding duplicate alerts. +- **Ignore Fields**: In full deduplication mode, these are fields that are not considered when comparing alerts. + +## Deduplication Types + +### Partial Deduplication +Partial deduplication allows you to specify certain fields (fingerprint fields) that are used to identify similar alerts. Alerts with matching values in these specified fields are considered duplicates and are grouped together. This method is flexible and allows for fine-tuned control over how alerts are deduplicated. + +Every provider integrated with Keep comes with pre-built partial deduplication rule tailored to that provider's specific alert format and common use cases. +The default fingerprint fields defined using `FINGERPRINT_FIELDS` attributes in the provider code (e.g. [datadog provider](https://github.com/keephq/keep/blob/main/keep/providers/datadog_provider/datadog_provider.py#L188) or [gcp monitoring provder](https://github.com/keephq/keep/blob/main/keep/providers/gcpmonitoring_provider/gcpmonitoring_provider.py#L52)). + +### Full Deduplication +When full deduplication is enabled, Keep will also discard exact same events (excluding ignore fields). This mode considers all fields of an alert when determining duplicates, except for explicitly ignored fields. + +By default, exact similar events excluding lastReceived time are fully deduplicated and discarded. This helps prevent system overload from repeated identical alerts. + +## Real Examples of Alerts and Results + +### Example 1: Partial Deduplication + +**Rule** - Deduplicate based on 'service' and 'error_message' fields. + +```json +# alert 1 +{ + "service": "payment", + "error_message": "Database connection failed", + "severity": "high", + "lastReceived": "2023-05-01T10:00:00Z" +} +# alert 2 +{ + "service": "payment", + "error_message": "Database connection failed", + "severity": "critical", + "lastReceived": "2023-05-01T10:05:00Z" +} +# alert 3 +{ + "service": "auth", + "error_message": "Invalid token", + "severity": "medium", + "lastReceived": "2023-05-01T10:10:00Z" +} +``` + +**Result**: +- Alerts 1 and 2 are deduplicated into a single alert, fields are updated. +- Alert 3 remains separate as it has a different service and error message. + +### Example 2: Full Deduplication + +**Rule**: Full deduplication with 'timestamp' as an ignore field + +**Incoming Alerts**: + +```json + +# alert 1 +{ + service: "api", + error: "Rate limit exceeded", + user_id: "12345", + lastReceived: "2023-05-02T14:00:00Z" +} +# alert 2 (discarded as its identical) +{ + service: "api", + error: "Rate limit exceeded", + user_id: "12345", + lastReceived: "2023-05-02T14:01:00Z" +} +# alert 3 +{ + service: "api", + error: "Rate limit exceeded", + user_id: "67890", + lastReceived: "2023-05-02T14:02:00Z" +} +``` + +**Result**: +- Alerts 1 and 2 are deduplicated as they are identical except for the ignored timestamp field. +- Alert 3 remains separate due to the different user_id. + +## How It Works + +Keep's deduplication process follows these steps: + +1. **Alert Ingestion**: Every alert received by Keep is first ingested into the system. + +2. **Enrichment**: After ingestion, each alert undergoes an enrichment process. This step adds additional context or information to the alert, enhancing its value and usefulness. + +3. **Deduplication**: Following enrichment, Keep's alert deduplicator comes into play. It applies the defined deduplication rules to the enriched alerts. diff --git a/keep-ui/app/deduplication/DeduplicationSidebar.tsx b/keep-ui/app/deduplication/DeduplicationSidebar.tsx index fed236c89..049cd19d5 100644 --- a/keep-ui/app/deduplication/DeduplicationSidebar.tsx +++ b/keep-ui/app/deduplication/DeduplicationSidebar.tsx @@ -1,7 +1,17 @@ -import React, { Fragment, useEffect, useState, useMemo } from "react"; +import { Fragment, useEffect, useState, useMemo } from "react"; import { Dialog, Transition } from "@headlessui/react"; import { useForm, Controller, SubmitHandler } from "react-hook-form"; -import { Text, Button, TextInput, Callout, Badge, Switch } from "@tremor/react"; +import { + Text, + Button, + TextInput, + Callout, + Badge, + Switch, + Icon, + Title, + Card, +} from "@tremor/react"; import { IoMdClose } from "react-icons/io"; import { DeduplicationRule } from "app/deduplication/models"; import { useProviders } from "utils/hooks/useProviders"; @@ -9,7 +19,10 @@ import { useDeduplicationFields } from "utils/hooks/useDeduplicationRules"; import { GroupBase } from "react-select"; import Select from "@/components/ui/Select"; import MultiSelect from "@/components/ui/MultiSelect"; -import { ExclamationTriangleIcon } from "@heroicons/react/24/outline"; +import { + ExclamationTriangleIcon, + InformationCircleIcon, +} from "@heroicons/react/24/outline"; import { getApiURL } from "utils/apiUrl"; import { useSession } from "next-auth/react"; import { KeyedMutator } from "swr"; @@ -79,7 +92,6 @@ const DeduplicationSidebar: React.FC = ({ const ignoreFields = watch("ignore_fields"); const availableFields = useMemo(() => { - // todo: add default fields for each provider from the backend const defaultFields = [ "source", "service", @@ -145,8 +157,6 @@ const DeduplicationSidebar: React.FC = ({ url += `/${selectedDeduplicationRule.id}`; } - // Use POST if there's no selectedDeduplicationRule.id (it's a default rule or new rule) - // This ensures we always create a new rule for default rules const method = !selectedDeduplicationRule || !selectedDeduplicationRule.id ? "POST" @@ -206,24 +216,28 @@ const DeduplicationSidebar: React.FC = ({ leaveFrom="translate-x-0" leaveTo="translate-x-full" > - +
- - {selectedDeduplicationRule - ? "Edit Deduplication Rule" - : "Add Deduplication Rule"} - - Beta - - {selectedDeduplicationRule?.default && ( - - Default Rule +
+ + {selectedDeduplicationRule + ? "Edit Deduplication Rule" + : "Add Deduplication Rule"} + + Beta - )} - - + {selectedDeduplicationRule?.default && ( + + Default Rule + + )} + +
+
+ +
{selectedDeduplicationRule?.default && ( @@ -245,7 +259,8 @@ const DeduplicationSidebar: React.FC = ({

Learn more about deduplication rules @@ -258,196 +273,251 @@ const DeduplicationSidebar: React.FC = ({ onSubmit={handleSubmit(onFormSubmit)} className="mt-4 flex flex-col h-full" > -
-
- - Rule Name - - ( - + +
+
+ + Rule name + + ( + + )} /> - )} - /> -
-
- - Description - - ( - +
+ + Description + + ( + + )} /> - )} - /> -
-
- - Provider - - ( - > - {...field} - isDisabled={!!selectedDeduplicationRule?.default} - options={alertProviders.map((provider) => ({ - value: `${provider.type}_${provider.id}`, - label: - provider.details?.name || provider.id || "main", - logoUrl: `/icons/${provider.type}-icon.png`, - }))} - placeholder="Select provider" - onChange={(selectedOption) => { - if (selectedOption) { - const [providerType, providerId] = - selectedOption.value.split("_"); - setValue("provider_type", providerType); - setValue("provider_id", providerId as any); - } - }} - value={ - alertProviders.find( - (provider) => - `${provider.type}_${provider.id}` === - `${selectedProviderType}_${selectedProviderId}` - ) - ? ({ - value: `${selectedProviderType}_${selectedProviderId}`, - label: - alertProviders.find( - (provider) => - `${provider.type}_${provider.id}` === - `${selectedProviderType}_${selectedProviderId}` - )?.details?.name || - (selectedProviderId !== "null" && - selectedProviderId !== null - ? selectedProviderId - : "main"), - logoUrl: `/icons/${selectedProviderType}-icon.png`, - } as ProviderOption) - : null - } +
+
+ + Provider + + + + + Select the provider for which this deduplication + rule will apply. This determines the source of + alerts that will be processed by this rule. + + + + + ( + + > + {...field} + isDisabled={!!selectedDeduplicationRule?.default} + options={alertProviders.map((provider) => ({ + value: `${provider.type}_${provider.id}`, + label: + provider.details?.name || provider.id || "main", + logoUrl: `/icons/${provider.type}-icon.png`, + }))} + placeholder="Select provider" + onChange={(selectedOption) => { + if (selectedOption) { + const [providerType, providerId] = + selectedOption.value.split("_"); + setValue("provider_type", providerType); + setValue("provider_id", providerId as any); + } + }} + value={ + alertProviders.find( + (provider) => + `${provider.type}_${provider.id}` === + `${selectedProviderType}_${selectedProviderId}` + ) + ? ({ + value: `${selectedProviderType}_${selectedProviderId}`, + label: + alertProviders.find( + (provider) => + `${provider.type}_${provider.id}` === + `${selectedProviderType}_${selectedProviderId}` + )?.details?.name || + (selectedProviderId !== "null" && + selectedProviderId !== null + ? selectedProviderId + : "main"), + logoUrl: `/icons/${selectedProviderType}-icon.png`, + } as ProviderOption) + : null + } + /> + )} /> - )} - /> - {errors.provider_type && ( -

- {errors.provider_type.message} -

- )} -
-
- - Fingerprint Fields - - ( - ({ - value: fieldName, - label: fieldName, - }))} - placeholder="Select fingerprint fields" - value={field.value?.map((value: string) => ({ - value, - label: value, - }))} - onChange={(selectedOptions) => { - field.onChange( - selectedOptions.map( - (option: { value: string }) => option.value - ) - ); + {errors.provider_type && ( +

+ {errors.provider_type.message} +

+ )} +
+
+ + Fields to use for fingerprint + + + + + Fingerprint fields are used to identify and group + similar alerts. Choose fields that uniquely + identify an alert type, such as 'service', + 'error_type', or 'affected_component'. + + + + + - selectedProviderType - ? "No options" - : "Please choose provider to see available fields" - } + render={({ field }) => ( + ({ + value: fieldName, + label: fieldName, + }))} + placeholder="Select fingerprint fields" + value={field.value?.map((value: string) => ({ + value, + label: value, + }))} + onChange={(selectedOptions) => { + field.onChange( + selectedOptions.map( + (option: { value: string }) => option.value + ) + ); + }} + noOptionsMessage={() => + selectedProviderType + ? "No options" + : "Please choose provider to see available fields" + } + /> + )} /> - )} - /> - {errors.fingerprint_fields && ( -

- {errors.fingerprint_fields.message} -

- )} -
-
- - ( - + {errors.fingerprint_fields && ( +

+ {errors.fingerprint_fields.message} +

)} - /> - - Full Deduplication - - -
- {fullDeduplication && ( -
- - Ignore Fields - - ( - ({ - value: fieldName, - label: fieldName, - }))} - placeholder="Select ignore fields" - value={field.value?.map((value: string) => ({ - value, - label: value, - }))} - onChange={(selectedOptions) => { - field.onChange( - selectedOptions.map( - (option: { value: string }) => option.value - ) - ); - }} +
+
+ + ( + + )} /> - )} - /> - {errors.ignore_fields && ( -

- {errors.ignore_fields.message} -

+ + Full deduplication + + + + + 1. Full deduplication: Keep will discard events + if they are the same (excluding the 'Ignore + Fields'). +
+ 2. Partial deduplication (default): Uses + specified fields to correlate alerts. E.g., two + alerts with same 'service' and 'env' fields will + be deduped into one alert. + + + + + +
+ + {fullDeduplication && ( +
+ + Ignore fields + + ( + ({ + value: fieldName, + label: fieldName, + }))} + placeholder="Select ignore fields" + value={field.value?.map((value: string) => ({ + value, + label: value, + }))} + onChange={(selectedOptions) => { + field.onChange( + selectedOptions.map( + (option: { value: string }) => option.value + ) + ); + }} + /> + )} + /> + {errors.ignore_fields && ( +

+ {errors.ignore_fields.message} +

+ )} +
)}
- )} + {errors.root?.serverError && ( = ({ columnHelper.accessor("description", { header: "Name", cell: (info) => ( -
+
{info.getValue()}