From 8fc68ca45846d138000d78c690dc59554027ae10 Mon Sep 17 00:00:00 2001 From: Vladimir Filonov Date: Thu, 30 Jan 2025 15:03:15 +0400 Subject: [PATCH] feat: Provider Health functionality (#3169) --- keep-ui/app/(health)/health/modal.tsx | 164 ++++++++++++ keep-ui/app/(health)/health/page.tsx | 62 +++++ keep-ui/app/(health)/layout.tsx | 72 +++++ keep-ui/app/(keep)/ai/model.ts | 2 +- .../alerts/alert-create-incident-ai-modal.tsx | 2 +- .../(keep)/alerts/alert-table-facet-value.tsx | 2 +- .../(keep)/alerts/alerts-rules-builder.tsx | 4 +- .../DeduplicationPlaceholder.tsx | 7 +- .../deduplication/DeduplicationTable.tsx | 12 +- .../incidents/[id]/chat/incident-chat.tsx | 9 +- keep-ui/app/(keep)/incidents/page.tsx | 12 +- keep-ui/app/(keep)/providers/page.client.tsx | 7 +- .../app/(keep)/providers/provider-form.tsx | 248 ++++++++++-------- .../app/(keep)/providers/providers-tiles.tsx | 33 ++- keep-ui/app/(keep)/providers/providers.tsx | 2 + .../app/(keep)/settings/settings.client.tsx | 28 +- .../(keep)/topology/ui/map/service-node.tsx | 4 +- .../app/(keep)/workflows/workflow-tile.tsx | 3 +- keep-ui/app/api/healthcheck/route.ts | 10 +- keep-ui/app/frigade-provider.tsx | 6 +- keep-ui/middleware.ts | 2 +- keep-ui/shared/api/ApiClient.ts | 9 +- keep-ui/shared/lib/hooks/useApi.tsx | 9 +- keep-ui/types/auth.d.ts | 4 + keep-ui/utils/hooks/useProviders.ts | 14 + keep/api/models/provider.py | 1 + keep/api/routes/providers.py | 53 +++- keep/providers/base/base_provider.py | 91 ++++++- .../datadog_provider/datadog_provider.py | 5 +- keep/providers/providers_factory.py | 1 + keep/providers/providers_service.py | 51 ++++ 31 files changed, 761 insertions(+), 168 deletions(-) create mode 100644 keep-ui/app/(health)/health/modal.tsx create mode 100644 keep-ui/app/(health)/health/page.tsx create mode 100644 keep-ui/app/(health)/layout.tsx diff --git a/keep-ui/app/(health)/health/modal.tsx b/keep-ui/app/(health)/health/modal.tsx new file mode 100644 index 000000000..0a13282e8 --- /dev/null +++ b/keep-ui/app/(health)/health/modal.tsx @@ -0,0 +1,164 @@ +import React from "react"; +import Modal from "@/components/ui/Modal"; +import { useApi } from "@/shared/lib/hooks/useApi"; +import { + Badge, + BarChart, + Button, + Card, + DonutChart, + Subtitle, + Title, +} from "@tremor/react"; +import { CheckCircle2Icon } from "lucide-react"; + +interface ProviderHealthResultsModalProps { + handleClose: () => void; + isOpen: boolean; + healthResults: any; +} + +const ProviderHealthResultsModal = ({ + handleClose, + isOpen, + healthResults, +}: ProviderHealthResultsModalProps) => { + const api = useApi(); + + const handleModalClose = () => { + handleClose(); + }; + + return ( + +
+
+ + Spammy Alerts + {healthResults?.spammy?.length ? ( + <> + + Sorry to say, but looks like your alerts are spammy + + ) : ( + <> +
+ +
+ Everything is ok + + )} + + + Rules Quality + {healthResults?.rules?.unused ? ( + <> + + + {healthResults?.rules.unused} of your{" "} + {healthResults.rules.used + healthResults.rules.unused} alert + rules are not in use + + + ) : ( + <> +
+ +
+ Everything is ok + + )} + + + Actionable +
+ +
+ Everything is ok + + + + Topology coverage + {healthResults?.topology?.uncovered.length ? ( + <> + + + Not of your services are covered. Alerts are missing for: + {healthResults?.topology?.uncovered.map((service: any) => { + return ( + + {service.display_name + ? service.display_name + : service.service} + + ); + })} + + + ) : ( + <> +
+ +
+ Everything is ok + + )} + +
+ + + Want to improve your observability? + + +
+ + ); +}; + +export default ProviderHealthResultsModal; diff --git a/keep-ui/app/(health)/health/page.tsx b/keep-ui/app/(health)/health/page.tsx new file mode 100644 index 000000000..61fca4ef2 --- /dev/null +++ b/keep-ui/app/(health)/health/page.tsx @@ -0,0 +1,62 @@ +"use client"; + +import ProvidersTiles from "@/app/(keep)/providers/providers-tiles"; +import React, { useEffect, useState } from "react"; +import { defaultProvider, Provider } from "@/app/(keep)/providers/providers"; +import { useProvidersWithHealthCheck } from "@/utils/hooks/useProviders"; +import Loading from "@/app/(keep)/loading"; + +const useFetchProviders = () => { + const [providers, setProviders] = useState([]); + const { data, error, mutate } = useProvidersWithHealthCheck(); + + if (error) { + throw error; + } + + const isLocalhost: boolean = true; + + useEffect(() => { + if (data) { + const fetchedProviders = data.providers + .filter((provider: Provider) => { + return provider.health; + }) + .map((provider) => ({ + ...defaultProvider, + ...provider, + id: provider.type, + installed: provider.installed ?? false, + health: provider.health, + })); + + setProviders(fetchedProviders); + } + }, [data]); + + return { + providers, + error, + isLocalhost, + mutate, + }; +}; + +export default function ProviderHealthPage () { + const { providers, isLocalhost, mutate } = useFetchProviders(); + + if (!providers || providers.length <= 0) { + return ; + } + + return ( + <> + + + ); +} diff --git a/keep-ui/app/(health)/layout.tsx b/keep-ui/app/(health)/layout.tsx new file mode 100644 index 000000000..ec96858d3 --- /dev/null +++ b/keep-ui/app/(health)/layout.tsx @@ -0,0 +1,72 @@ +import { ReactNode } from "react"; +import { NextAuthProvider } from "../auth-provider"; +import { Mulish } from "next/font/google"; +import { ToastContainer } from "react-toastify"; +import Navbar from "components/navbar/Navbar"; +import { TopologyPollingContextProvider } from "@/app/(keep)/topology/model/TopologyPollingContext"; +import { FrigadeProvider } from "../frigade-provider"; +import { getConfig } from "@/shared/lib/server/getConfig"; +import { ConfigProvider } from "../config-provider"; +import { PHProvider } from "../posthog-provider"; +import dynamic from "next/dynamic"; +import ReadOnlyBanner from "../read-only-banner"; +import { auth } from "@/auth"; +import { ThemeScript, WatchUpdateTheme } from "@/shared/ui"; +import "@/app/globals.css"; +import "react-toastify/dist/ReactToastify.css"; + +const PostHogPageView = dynamic(() => import("@/shared/ui/PostHogPageView"), { + ssr: false, +}); + +// If loading a variable font, you don't need to specify the font weight +const mulish = Mulish({ + subsets: ["latin"], + display: "swap", +}); + +type RootLayoutProps = { + children: ReactNode; +}; + +export default async function RootLayout({ children }: RootLayoutProps) { + const config = getConfig(); + const session = await auth(); + + return ( + + + {/* ThemeScript must be the first thing to avoid flickering */} + + + + + + {/* @ts-ignore-error Server Component */} + + {/* https://discord.com/channels/752553802359505017/1068089513253019688/1117731746922893333 */} +
+ {/* Add the banner here, before the navbar */} + {config.READ_ONLY && } +
{children}
+ +
+ + + + + + + {/** footer */} + {process.env.GIT_COMMIT_HASH && + process.env.SHOW_BUILD_INFO !== "false" && ( +
+ Build: {process.env.GIT_COMMIT_HASH} +
+ Version: {process.env.KEEP_VERSION} +
+ )} + + + ); +} diff --git a/keep-ui/app/(keep)/ai/model.ts b/keep-ui/app/(keep)/ai/model.ts index 1fe4ed3f9..d1db8eee5 100644 --- a/keep-ui/app/(keep)/ai/model.ts +++ b/keep-ui/app/(keep)/ai/model.ts @@ -6,7 +6,7 @@ export interface AIConfig { algorithm: { name: string; description: string; - } + }; } export interface AIStats { diff --git a/keep-ui/app/(keep)/alerts/alert-create-incident-ai-modal.tsx b/keep-ui/app/(keep)/alerts/alert-create-incident-ai-modal.tsx index d22542b56..2cb28b41b 100644 --- a/keep-ui/app/(keep)/alerts/alert-create-incident-ai-modal.tsx +++ b/keep-ui/app/(keep)/alerts/alert-create-incident-ai-modal.tsx @@ -53,7 +53,7 @@ const CreateIncidentWithAIModal = ({ 20, 0, { id: "creation_time", desc: true }, - '', + "", {} ); diff --git a/keep-ui/app/(keep)/alerts/alert-table-facet-value.tsx b/keep-ui/app/(keep)/alerts/alert-table-facet-value.tsx index b6587c52a..fed735f44 100644 --- a/keep-ui/app/(keep)/alerts/alert-table-facet-value.tsx +++ b/keep-ui/app/(keep)/alerts/alert-table-facet-value.tsx @@ -31,7 +31,7 @@ export const FacetValue: React.FC = ({ 100, undefined, undefined, - '', + "", { revalidateOnFocus: false, } diff --git a/keep-ui/app/(keep)/alerts/alerts-rules-builder.tsx b/keep-ui/app/(keep)/alerts/alerts-rules-builder.tsx index 48f229e67..bffb0c741 100644 --- a/keep-ui/app/(keep)/alerts/alerts-rules-builder.tsx +++ b/keep-ui/app/(keep)/alerts/alerts-rules-builder.tsx @@ -504,8 +504,8 @@ export const AlertsRulesBuilder = ({ operators: getOperators(id), })) : customFields - ? customFields - : []; + ? customFields + : []; const onImportSQL = () => { setImportSQLOpen(true); diff --git a/keep-ui/app/(keep)/deduplication/DeduplicationPlaceholder.tsx b/keep-ui/app/(keep)/deduplication/DeduplicationPlaceholder.tsx index 902ade1f3..468262f98 100644 --- a/keep-ui/app/(keep)/deduplication/DeduplicationPlaceholder.tsx +++ b/keep-ui/app/(keep)/deduplication/DeduplicationPlaceholder.tsx @@ -15,7 +15,12 @@ export const DeduplicationPlaceholder = () => {
No Deduplications Yet - Alert deduplication is the first layer of denoising. It groups similar alerts from one source.
To connect alerts across sources into incidents, check Correlations + Alert deduplication is the first layer of denoising. It groups + similar alerts from one source. +
To connect alerts across sources into incidents, check{" "} + + Correlations + This page will become active once the first alerts are registered. diff --git a/keep-ui/app/(keep)/deduplication/DeduplicationTable.tsx b/keep-ui/app/(keep)/deduplication/DeduplicationTable.tsx index 2332b9d30..7643a26bd 100644 --- a/keep-ui/app/(keep)/deduplication/DeduplicationTable.tsx +++ b/keep-ui/app/(keep)/deduplication/DeduplicationTable.tsx @@ -120,7 +120,9 @@ export const DeduplicationTable: React.FC = ({ "Represents the percentage of alerts successfully deduplicated. Higher values indicate better deduplication efficiency, meaning fewer redundant alerts.", }; - function resolveDeleteButtonTooltip(deduplicationRule: DeduplicationRule): string { + function resolveDeleteButtonTooltip( + deduplicationRule: DeduplicationRule + ): string { if (deduplicationRule.default) { return "Cannot delete default rule"; } @@ -129,7 +131,7 @@ export const DeduplicationTable: React.FC = ({ return "Cannot delete provisioned rule."; } - return "Delete Rule" + return "Delete Rule"; } const DEDUPLICATION_TABLE_COLS = useMemo( @@ -276,10 +278,10 @@ export const DeduplicationTable: React.FC = ({ size="xs" variant="secondary" icon={TrashIcon} - tooltip={ - resolveDeleteButtonTooltip(info.row.original) + tooltip={resolveDeleteButtonTooltip(info.row.original)} + disabled={ + info.row.original.default || info.row.original.is_provisioned } - disabled={info.row.original.default || info.row.original.is_provisioned} onClick={(e) => handleDeleteRule(info.row.original, e)} />
diff --git a/keep-ui/app/(keep)/incidents/[id]/chat/incident-chat.tsx b/keep-ui/app/(keep)/incidents/[id]/chat/incident-chat.tsx index 4543f5116..290db5dcb 100644 --- a/keep-ui/app/(keep)/incidents/[id]/chat/incident-chat.tsx +++ b/keep-ui/app/(keep)/incidents/[id]/chat/incident-chat.tsx @@ -138,12 +138,9 @@ export function IncidentChat({ useIncidentActions(); const providersWithGetTrace = useMemo( () => - providers?.installed_providers - .filter( - (provider) => - provider.methods?.some((method) => method.func_name === "get_trace") - ) - .map((provider) => provider.id), + providers?.installed_providers.filter((provider) => + provider.methods?.some((method) => method.func_name === "get_trace") + ).map((provider) => provider.id), [providers] ); diff --git a/keep-ui/app/(keep)/incidents/page.tsx b/keep-ui/app/(keep)/incidents/page.tsx index 2b0cb7d1b..9b99032cf 100644 --- a/keep-ui/app/(keep)/incidents/page.tsx +++ b/keep-ui/app/(keep)/incidents/page.tsx @@ -2,7 +2,7 @@ import { IncidentList } from "@/features/incident-list"; import { getIncidents, GetIncidentsParams } from "@/entities/incidents/api"; import { PaginatedIncidentsDto } from "@/entities/incidents/model"; import { createServerApiClient } from "@/shared/api/server"; -import {DefaultIncidentFilters} from "@/entities/incidents/model/models"; +import { DefaultIncidentFilters } from "@/entities/incidents/model/models"; import { getInitialFacets, InitialFacetsData } from "@/features/filter/api"; const defaultIncidentsParams: GetIncidentsParams = { @@ -24,15 +24,19 @@ export default async function Page() { const tasks = [ getIncidents(api, defaultIncidentsParams, ), getInitialFacets(api, "incidents"), - ] + ]; const [_incidents, _facetsData] = await Promise.all(tasks); incidents = _incidents as PaginatedIncidentsDto; facetsData = _facetsData as InitialFacetsData; - } catch (error) { console.log(error); } - return ; + return ( + + ); } export const metadata = { diff --git a/keep-ui/app/(keep)/providers/page.client.tsx b/keep-ui/app/(keep)/providers/page.client.tsx index f2c4372b1..a599fc4c8 100644 --- a/keep-ui/app/(keep)/providers/page.client.tsx +++ b/keep-ui/app/(keep)/providers/page.client.tsx @@ -15,7 +15,7 @@ export const useFetchProviders = () => { const [installedProviders, setInstalledProviders] = useState([]); const [linkedProviders, setLinkedProviders] = useState([]); // Added state for linkedProviders const { data: config } = useConfig(); - const { data, error } = useProviders(); + const { data, error, mutate } = useProviders(); if (error) { throw error; @@ -100,6 +100,7 @@ export const useFetchProviders = () => { setInstalledProviders, error, isLocalhost, + mutate, }; }; @@ -114,6 +115,7 @@ export default function ProvidersPage({ linkedProviders, setInstalledProviders, isLocalhost, + mutate, } = useFetchProviders(); const { @@ -173,6 +175,7 @@ export default function ProvidersPage({ )} {linkedProviders?.length > 0 && ( @@ -180,6 +183,7 @@ export default function ProvidersPage({ providers={linkedProviders} linkedProvidersMode={true} isLocalhost={isLocalhost} + mutate={mutate} /> )} ); diff --git a/keep-ui/app/(keep)/providers/provider-form.tsx b/keep-ui/app/(keep)/providers/provider-form.tsx index c55d63291..8218dd1c1 100644 --- a/keep-ui/app/(keep)/providers/provider-form.tsx +++ b/keep-ui/app/(keep)/providers/provider-form.tsx @@ -67,24 +67,47 @@ import { import ProviderLogs from "./provider-logs"; import { DynamicImageProviderIcon } from "@/components/ui"; +type HealthResults = { + spammy: any[]; + rules: { + total: number; + used: number; + unused: number; + }; + topology: { + covered: any[]; + uncovered: any[]; + }; +}; + type ProviderFormProps = { provider: Provider; - onConnectChange?: (isConnecting: boolean, isConnected: boolean) => void; + onConnectChange?: ( + isConnecting: boolean, + isConnected: boolean, + healthResults: HealthResults | null + ) => void; closeModal: () => void; isProviderNameDisabled?: boolean; installedProvidersMode: boolean; isLocalhost?: boolean; + isHealthCheck?: boolean; + mutate: () => void; }; -function getInitialFormValues(provider: Provider) { +function getInitialFormValues(provider: Provider, isHealthCheck?: boolean) { const initialValues: ProviderFormData = { provider_id: provider.id, - install_webhook: provider.can_setup_webhook ?? false, + install_webhook: !isHealthCheck + ? (provider.can_setup_webhook ?? false) + : false, pulling_enabled: provider.pulling_enabled, }; Object.assign(initialValues, { - provider_name: provider.details?.name, + provider_name: + provider.details?.name || + (isHealthCheck ? `${provider.id} health check` : undefined), ...provider.details?.authentication, }); @@ -113,12 +136,13 @@ const ProviderForm = ({ isProviderNameDisabled, installedProvidersMode, isLocalhost, + isHealthCheck, + mutate }: ProviderFormProps) => { console.log("Loading the ProviderForm component"); - const { mutate } = useProviders(); const searchParams = useSearchParams(); const [formValues, setFormValues] = useState(() => - getInitialFormValues(provider) + getInitialFormValues(provider, isHealthCheck) ); const [formErrors, setFormErrors] = useState(null); const [inputErrors, setInputErrors] = useState({}); @@ -282,6 +306,7 @@ const ProviderForm = ({ function validate(data?: ProviderFormData) { let schema = zodSchema; + console.log(222, data) if (data) { schema = zodSchema.pick( Object.fromEntries(Object.keys(data).map((field) => [field, true])) @@ -399,13 +424,14 @@ const ProviderForm = ({ async function handleConnectClick() { if (!validate()) return; setIsLoading(true); - onConnectChange?.(true, false); - submit(`/providers/install`) + onConnectChange?.(true, false, null); + submit(isHealthCheck ? `/providers/healthcheck` : `/providers/install`) .then(async (data) => { console.log("Connect Result:", data); setIsLoading(false); - onConnectChange?.(false, true); + onConnectChange?.(false, true, data); if ( + !isHealthCheck && formValues.install_webhook && provider.can_setup_webhook && !isLocalhost @@ -418,7 +444,7 @@ const ProviderForm = ({ .catch((error) => { handleSubmitError(error); setIsLoading(false); - onConnectChange?.(false, false); + onConnectChange?.(false, false, null); }); } @@ -452,7 +478,7 @@ const ProviderForm = ({ config={providerNameFieldConfig} value={(formValues["provider_name"] ?? "").toString()} error={inputErrors["provider_name"]} - disabled={isProviderNameDisabled ?? false} + disabled={(isProviderNameDisabled || isHealthCheck) ?? false} title={ isProviderNameDisabled ? "This field is disabled because it is pre-filled from the workflow." @@ -516,37 +542,91 @@ const ProviderForm = ({ )}
- {provider.can_setup_webhook && !installedProvidersMode && ( -
-
- -
+ )} +
+ + {!isHealthCheck && + provider.can_setup_webhook && + installedProvidersMode && ( + <> +
- {isLocalhost && ( - - - - Webhook installation is disabled because Keep is running - without an external URL. -
-
- Click to learn more - - - - )} -
+ + )} - - - {provider.can_setup_webhook && installedProvidersMode && ( - <> -
- - -
- - - )} {provider.supports_webhook && ( @@ -813,7 +843,7 @@ const ProviderForm = ({ onClick={handleConnectClick} color="orange" > - Connect + {isHealthCheck ? `Check health` : `Connect`} )} diff --git a/keep-ui/app/(keep)/providers/providers-tiles.tsx b/keep-ui/app/(keep)/providers/providers-tiles.tsx index 29030fb9f..ae5c82c49 100644 --- a/keep-ui/app/(keep)/providers/providers-tiles.tsx +++ b/keep-ui/app/(keep)/providers/providers-tiles.tsx @@ -10,20 +10,27 @@ import "react-sliding-side-panel/lib/index.css"; import { useSearchParams } from "next/navigation"; import { QuestionMarkCircleIcon } from "@heroicons/react/24/outline"; import { Tooltip } from "@/shared/ui"; +import ProviderHealthResultsModal from "@/app/(health)/health/modal"; const ProvidersTiles = ({ providers, installedProvidersMode = false, linkedProvidersMode = false, isLocalhost = false, + isHealthCheck = false, + mutate, }: { providers: Providers; installedProvidersMode?: boolean; linkedProvidersMode?: boolean; isLocalhost?: boolean; + isHealthCheck?: boolean; + mutate: () => void; }) => { const searchParams = useSearchParams(); const [openPanel, setOpenPanel] = useState(false); + const [openHealthModal, setOpenHealthModal] = useState(false); + const [healthResults, setHealthResults] = useState({}); const [selectedProvider, setSelectedProvider] = useState( null ); @@ -57,8 +64,24 @@ const ProvidersTiles = ({ setSelectedProvider(null); }; - const handleConnecting = (isConnecting: boolean, isConnected: boolean) => { + const handleShowHealthModal = () => { + setOpenHealthModal(true); + }; + + const handleCloseHealthModal = () => { + setOpenHealthModal(false); + }; + + const handleConnecting = ( + isConnecting: boolean, + isConnected: boolean, + healthResults: any + ) => { if (isConnected) handleCloseModal(); + if (isConnected && isHealthCheck) { + setHealthResults(healthResults); + handleShowHealthModal(); + } }; const getSectionTitle = () => { @@ -134,9 +157,17 @@ const ProvidersTiles = ({ installedProvidersMode={installedProvidersMode} isProviderNameDisabled={installedProvidersMode} isLocalhost={isLocalhost} + isHealthCheck={isHealthCheck} + mutate={mutate} /> )} + + ); }; diff --git a/keep-ui/app/(keep)/providers/providers.tsx b/keep-ui/app/(keep)/providers/providers.tsx index b45afe59a..2da009803 100644 --- a/keep-ui/app/(keep)/providers/providers.tsx +++ b/keep-ui/app/(keep)/providers/providers.tsx @@ -129,6 +129,7 @@ export interface Provider { provisioned?: boolean; categories: TProviderCategory[]; coming_soon: boolean; + health: boolean; } export type Providers = Provider[]; @@ -149,6 +150,7 @@ export const defaultProvider: Provider = { pulling_enabled: true, categories: ["Others"], coming_soon: false, + health: false, }; export type ProviderFormKVData = Record[]; diff --git a/keep-ui/app/(keep)/settings/settings.client.tsx b/keep-ui/app/(keep)/settings/settings.client.tsx index bc83e3406..7fe3d6191 100644 --- a/keep-ui/app/(keep)/settings/settings.client.tsx +++ b/keep-ui/app/(keep)/settings/settings.client.tsx @@ -73,24 +73,24 @@ export default function SettingsPage() { newSelectedTab === "users" ? 0 : newSelectedTab === "webhook" - ? 1 - : newSelectedTab === "smtp" - ? 2 - : 0; + ? 1 + : newSelectedTab === "smtp" + ? 2 + : 0; const userSubTabIndex = newUserSubTab === "users" ? 0 : newUserSubTab === "groups" - ? 1 - : newUserSubTab === "roles" - ? 2 - : newUserSubTab === "permissions" - ? 3 - : newUserSubTab === "api-keys" - ? 4 - : newUserSubTab === "sso" - ? 5 - : 0; + ? 1 + : newUserSubTab === "roles" + ? 2 + : newUserSubTab === "permissions" + ? 3 + : newUserSubTab === "api-keys" + ? 4 + : newUserSubTab === "sso" + ? 5 + : 0; setTabIndex(tabIndex); setUserSubTabIndex(userSubTabIndex); setSelectedTab(newSelectedTab); diff --git a/keep-ui/app/(keep)/topology/ui/map/service-node.tsx b/keep-ui/app/(keep)/topology/ui/map/service-node.tsx index f93740784..ff4f927e2 100644 --- a/keep-ui/app/(keep)/topology/ui/map/service-node.tsx +++ b/keep-ui/app/(keep)/topology/ui/map/service-node.tsx @@ -100,9 +100,7 @@ export function ServiceNode({ data, selected }: NodeProps) { }, [showDetails]); const handleClick = () => { - router.push( - `/incidents?services=${encodeURIComponent(data.display_name)}` - ); + router.push(`/incidents?services=${encodeURIComponent(data.display_name)}`); }; const incidentsCount = data.incidents ?? 0; diff --git a/keep-ui/app/(keep)/workflows/workflow-tile.tsx b/keep-ui/app/(keep)/workflows/workflow-tile.tsx index a5f9d4a84..6aba0b731 100644 --- a/keep-ui/app/(keep)/workflows/workflow-tile.tsx +++ b/keep-ui/app/(keep)/workflows/workflow-tile.tsx @@ -165,7 +165,7 @@ function WorkflowTile({ workflow }: { workflow: Workflow }) { ?.filters?.find((f) => f.key === "source")?.value; const [fallBackIcon, setFallBackIcon] = useState(false); - const { providers } = useFetchProviders(); + const { providers, mutate} = useFetchProviders(); const { deleteWorkflow } = useWorkflowActions(); const { isRunning, @@ -537,6 +537,7 @@ function WorkflowTile({ workflow }: { workflow: Workflow }) { closeModal={handleCloseModal} installedProvidersMode={selectedProvider.installed} isProviderNameDisabled={true} + mutate={mutate} /> )} diff --git a/keep-ui/app/api/healthcheck/route.ts b/keep-ui/app/api/healthcheck/route.ts index 126270e90..a6c0ca05d 100644 --- a/keep-ui/app/api/healthcheck/route.ts +++ b/keep-ui/app/api/healthcheck/route.ts @@ -1,8 +1,8 @@ -import { NextResponse } from 'next/server' +import { NextResponse } from "next/server"; export async function GET() { return NextResponse.json({ - status: 'ok', - timestamp: new Date().toISOString() - }) -} \ No newline at end of file + status: "ok", + timestamp: new Date().toISOString(), + }); +} diff --git a/keep-ui/app/frigade-provider.tsx b/keep-ui/app/frigade-provider.tsx index d4980844a..05c6fbab8 100644 --- a/keep-ui/app/frigade-provider.tsx +++ b/keep-ui/app/frigade-provider.tsx @@ -2,7 +2,7 @@ import * as Frigade from "@frigade/react"; import { useHydratedSession as useSession } from "@/shared/lib/hooks/useHydratedSession"; -import {useConfig} from "@/utils/hooks/useConfig"; +import { useConfig } from "@/utils/hooks/useConfig"; export const FrigadeProvider = ({ children, }: { @@ -12,9 +12,7 @@ export const FrigadeProvider = ({ const { data: config } = useConfig(); if (!config || config.FRIGADE_DISABLED === "true") { - return <> - {children} - ; + return <>{children}; } return ( { } // If not authenticated and not on signin page, redirect to signin - if (!isAuthenticated && !pathname.startsWith("/signin")) { + if (!isAuthenticated && !pathname.startsWith("/signin") && !pathname.startsWith("/health")) { console.log("Redirecting to signin page because user is not authenticated"); return NextResponse.redirect(new URL("/signin", request.url)); } diff --git a/keep-ui/shared/api/ApiClient.ts b/keep-ui/shared/api/ApiClient.ts index 76be12887..e4291371e 100644 --- a/keep-ui/shared/api/ApiClient.ts +++ b/keep-ui/shared/api/ApiClient.ts @@ -5,6 +5,7 @@ import { getApiUrlFromConfig } from "@/shared/lib/getApiUrlFromConfig"; import { getApiURL } from "@/utils/apiUrl"; import * as Sentry from "@sentry/nextjs"; import { signOut as signOutClient } from "next-auth/react"; +import { GuestSession } from "@/types/auth"; const READ_ONLY_ALLOWED_METHODS = ["GET", "OPTIONS"]; const READ_ONLY_ALWAYS_ALLOWED_URLS = ["/alerts/audit"]; @@ -12,7 +13,7 @@ const READ_ONLY_ALWAYS_ALLOWED_URLS = ["/alerts/audit"]; export class ApiClient { private readonly isServer: boolean; constructor( - private readonly session: Session | null, + private readonly session: Session | GuestSession | null, private readonly config: InternalConfig | null ) { this.isServer = typeof window === "undefined"; @@ -26,6 +27,10 @@ export class ApiClient { if (!this.session || !this.session.accessToken) { throw new Error("No valid session or access token found"); } + // Guest session + if (this.session.accessToken === "unauthenticated") { + return {} + } return { Authorization: `Bearer ${this.session.accessToken}`, }; @@ -137,7 +142,7 @@ export class ApiClient { const response = await fetch(fullUrl, { ...requestInit, headers: { - ...this.getHeaders(), + ...(this.getHeaders() as HeadersInit), ...requestInit.headers, }, }); diff --git a/keep-ui/shared/lib/hooks/useApi.tsx b/keep-ui/shared/lib/hooks/useApi.tsx index 493372355..4c1cc9880 100644 --- a/keep-ui/shared/lib/hooks/useApi.tsx +++ b/keep-ui/shared/lib/hooks/useApi.tsx @@ -2,14 +2,19 @@ import { useConfig } from "@/utils/hooks/useConfig"; import { useHydratedSession as useSession } from "@/shared/lib/hooks/useHydratedSession"; import { useMemo } from "react"; import { ApiClient } from "@/shared/api/ApiClient"; +import { GuestSession } from "@/types/auth"; export function useApi() { const { data: config } = useConfig(); - const { data: session } = useSession(); + const { data: user_session, status } = useSession(); const api = useMemo(() => { + const session = status === "unauthenticated" ? { + accessToken: "unauthenticated" + } as GuestSession : user_session + return new ApiClient(session, config); - }, [session?.accessToken, config]); + }, [status, user_session?.accessToken, config]); return api; } diff --git a/keep-ui/types/auth.d.ts b/keep-ui/types/auth.d.ts index 6dd72458c..dae9cb0a7 100644 --- a/keep-ui/types/auth.d.ts +++ b/keep-ui/types/auth.d.ts @@ -34,3 +34,7 @@ declare module "next-auth/jwt" { role?: string; } } + +interface GuestSession { + accessToken: "unauthenticated"; +} diff --git a/keep-ui/utils/hooks/useProviders.ts b/keep-ui/utils/hooks/useProviders.ts index c12db4c4a..abe1f0ad5 100644 --- a/keep-ui/utils/hooks/useProviders.ts +++ b/keep-ui/utils/hooks/useProviders.ts @@ -2,6 +2,7 @@ import { SWRConfiguration } from "swr"; import { ProvidersResponse } from "@/app/(keep)/providers/providers"; import useSWRImmutable from "swr/immutable"; import { useApi } from "@/shared/lib/hooks/useApi"; +import {ApiClient} from "@/shared/api"; export const useProviders = ( options: SWRConfiguration = { revalidateOnFocus: false } @@ -14,3 +15,16 @@ export const useProviders = ( options ); }; + +export const useProvidersWithHealthCheck = ( + options: SWRConfiguration = { revalidateOnFocus: false } +) => { + const api = useApi(); + + return useSWRImmutable( + api.isReady() ? "/providers/healthcheck" : null, + (url) => api.get(url), + options + ); +}; + diff --git a/keep/api/models/provider.py b/keep/api/models/provider.py index ed3b1c23f..9c94d8860 100644 --- a/keep/api/models/provider.py +++ b/keep/api/models/provider.py @@ -53,3 +53,4 @@ class Provider(BaseModel): alertExample: dict | None = None default_fingerprint_fields: list[str] | None = None provisioned: bool = False + health: bool = False diff --git a/keep/api/routes/providers.py b/keep/api/routes/providers.py index 9992a6e25..18a6d28a5 100644 --- a/keep/api/routes/providers.py +++ b/keep/api/routes/providers.py @@ -4,7 +4,7 @@ import random import time import uuid -from typing import Callable, Optional +from typing import Callable, Optional, Dict, Any from fastapi import APIRouter, Body, Depends, HTTPException, Request from fastapi.encoders import jsonable_encoder @@ -754,3 +754,54 @@ def get_webhook_settings( ), webhookMarkdown=webhookMarkdown, ) + +@router.post("/healthcheck") +async def healthcheck_provider( + request: Request, +) -> Dict[str, Any]: + try: + provider_info = await request.json() + except Exception: + form_data = await request.form() + provider_info = dict(form_data) + + if not provider_info: + raise HTTPException(status_code=400, detail="No valid data provided") + + try: + provider_id = provider_info.pop("provider_id") + provider_type = provider_info.pop("provider_type", None) or provider_id + provider_name = f"{provider_type} healthcheck" + except KeyError as e: + raise HTTPException( + status_code=400, detail=f"Missing required field: {e.args[0]}" + ) + + for key, value in provider_info.items(): + if isinstance(value, UploadFile): + provider_info[key] = value.file.read().decode() + + provider = ProvidersService.prepare_provider( + provider_id, + provider_name, + provider_type, + provider_info, + ) + + result = provider.get_health_report() + return result + + +@router.get("/healthcheck") +def get_healthcheck_providers(): + logger.info("Getting all providers for healthcheck") + providers = ProvidersService.get_all_providers() + + healthcheck_providers = [provider for provider in providers if provider.health] + + is_localhost = _is_localhost() + + return { + "providers": healthcheck_providers, + "is_localhost": is_localhost, + } diff --git a/keep/providers/base/base_provider.py b/keep/providers/base/base_provider.py index 13283778b..aa1e09546 100644 --- a/keep/providers/base/base_provider.py +++ b/keep/providers/base/base_provider.py @@ -13,10 +13,13 @@ import os import re import uuid +from collections import Counter +from operator import attrgetter from typing import Literal, Optional import opentelemetry.trace as trace import requests +from dateutil.parser import parse from keep.api.bl.enrichments_bl import EnrichmentsBl from keep.api.core.db import ( @@ -36,6 +39,8 @@ tracer = trace.get_tracer(__name__) +SPAMMY_ALERTS_THRESHOLD_HOURS = 1 +SPAMMY_ALERTS_THRESHOLD = datetime.timedelta(hours=SPAMMY_ALERTS_THRESHOLD_HOURS) class BaseProvider(metaclass=abc.ABCMeta): OAUTH2_URL = None @@ -143,7 +148,7 @@ def dispose(self): raise NotImplementedError("dispose() method not implemented") @abc.abstractmethod - def validate_config(): + def validate_config(self): """ Validate provider configuration. """ @@ -785,6 +790,10 @@ def is_provisioned(self) -> bool: parser._parse_providers_from_env(self.context_manager) return self.config.name in self.context_manager.providers_context + @classmethod + def has_health_report(cls) -> bool: + return getattr(cls, "HAS_HEALTH_CHECK", False) + class BaseTopologyProvider(BaseProvider): def pull_topology(self) -> tuple[list[TopologyServiceInDto], dict]: @@ -860,3 +869,83 @@ def setup_incident_webhook( NotImplementedError: _description_ """ raise NotImplementedError("setup_webhook() method not implemented") + + +class ProviderHealthMixin: + + HAS_HEALTH_CHECK = True + + def get_health_report(self): + health = {} + + alerts = self.get_alerts() + + self.check_topology_coverage(alerts, health) + self.check_spammy_alerts(alerts, health) + self.check_alerting_rules(alerts, health) + + return health + + def check_topology_coverage(self, alerts, health): + if hasattr(self, "pull_topology"): + topology, _ = self.pull_topology() + uncovered_topology = copy.deepcopy(topology) + for alert in alerts: + uncovered_topology = list(filter(lambda t: not alert.service == t.service, uncovered_topology)) + + health["topology"] = { + "covered": [t for t in topology if t not in uncovered_topology], + "uncovered": uncovered_topology + } + + def check_alerting_rules(self, alerts, health): + if hasattr(self, "get_alerts_configuration"): + rules = self.get_alerts_configuration() + try: + rules = list(map(json.loads, rules)) + except json.JSONDecodeError: + pass + unused_rules = [] + compiled_patterns = [re.compile(rule['message']) for rule in rules] + matched_patterns = set() + + for alert in alerts: + for idx, pattern in enumerate(compiled_patterns): + if idx in matched_patterns: + continue + if pattern.search(alert.message): + matched_patterns.add(idx) + + health["rules"] = { + "total": len(rules), + "used": len(rules) - len(unused_rules), + "unused": len(unused_rules), + } + + def check_spammy_alerts(self, alerts, health): + sorter = sorted(alerts, key=attrgetter("fingerprint")) + alerts_per_fingerprint = itertools.groupby(sorter, key=attrgetter("fingerprint")) + spammy_alerts = [] + for fingerprint, fingerprint_alerts in alerts_per_fingerprint: + close_alerts = [] + + fingerprint_alerts = list(fingerprint_alerts) + + fingerprint_alerts.sort(key=attrgetter("lastReceived")) + # Iterate through alerts to check if some of them are too close + for i in range(len(fingerprint_alerts)): + for j in range(i + 1, len(fingerprint_alerts)): + if parse(fingerprint_alerts[j].lastReceived) - parse( + fingerprint_alerts[i].lastReceived) <= SPAMMY_ALERTS_THRESHOLD: + close_alerts.append((fingerprint_alerts[i], fingerprint_alerts[j])) + else: + break + + if len(close_alerts) > 2: + spammy_alerts.extend(fingerprint_alerts) + + timestamps = [parse(alert.lastReceived) for alert in spammy_alerts] + hours = [ts.strftime("%Y-%m-%d %H:00") for ts in timestamps] + hourly_alerts = Counter(hours) + health["spammy"] = [{"date": date, "value": value} for date, value in hourly_alerts.items()] + diff --git a/keep/providers/datadog_provider/datadog_provider.py b/keep/providers/datadog_provider/datadog_provider.py index 05bfbb0d9..73b642aab 100644 --- a/keep/providers/datadog_provider/datadog_provider.py +++ b/keep/providers/datadog_provider/datadog_provider.py @@ -13,6 +13,7 @@ import pydantic import requests + from datadog_api_client import ApiClient, Configuration from datadog_api_client.api_client import Endpoint from datadog_api_client.exceptions import ( @@ -36,7 +37,7 @@ from keep.api.models.alert import AlertDto, AlertSeverity, AlertStatus from keep.api.models.db.topology import TopologyServiceInDto from keep.contextmanager.contextmanager import ContextManager -from keep.providers.base.base_provider import BaseTopologyProvider +from keep.providers.base.base_provider import BaseTopologyProvider, ProviderHealthMixin from keep.providers.base.provider_exceptions import GetAlertException from keep.providers.datadog_provider.datadog_alert_format_description import ( DatadogAlertFormatDescription, @@ -105,7 +106,7 @@ class DatadogProviderAuthConfig: ) -class DatadogProvider(BaseTopologyProvider): +class DatadogProvider(BaseTopologyProvider, ProviderHealthMixin): """Pull/push alerts from Datadog.""" PROVIDER_CATEGORY = ["Monitoring"] diff --git a/keep/providers/providers_factory.py b/keep/providers/providers_factory.py index 522715ed3..d976e5eff 100644 --- a/keep/providers/providers_factory.py +++ b/keep/providers/providers_factory.py @@ -422,6 +422,7 @@ def get_all_providers(ignore_cache_file: bool = False) -> list[Provider]: default_fingerprint_fields=default_fingerprint_fields, categories=provider_class.PROVIDER_CATEGORY, coming_soon=provider_class.PROVIDER_COMING_SOON, + health=provider_class.has_health_report(), ) ) except ModuleNotFoundError: diff --git a/keep/providers/providers_service.py b/keep/providers/providers_service.py index cf47f5266..808683cc9 100644 --- a/keep/providers/providers_service.py +++ b/keep/providers/providers_service.py @@ -82,6 +82,57 @@ def validate_scopes( ) return validated_scopes + @staticmethod + def prepare_provider( + provider_id: str, + provider_name: str, + provider_type: str, + provider_config: Dict[str, Any], + validate_scopes: bool = True, + ) -> Dict[str, Any]: + provider_unique_id = uuid.uuid4().hex + logger.info( + "Installing provider", + extra={ + "provider_id": provider_id, + "provider_type": provider_type, + }, + ) + + config = { + "authentication": provider_config, + "name": provider_name, + } + tenant_id = None + context_manager = ContextManager(tenant_id=tenant_id) + try: + provider = ProvidersFactory.get_provider( + context_manager, provider_id, provider_type, config + ) + except Exception as e: + raise HTTPException(status_code=400, detail=str(e)) + + if validate_scopes: + ProvidersService.validate_scopes(provider) + + secret_manager = SecretManagerFactory.get_secret_manager(context_manager) + secret_name = f"{tenant_id}_{provider_type}_{provider_unique_id}" + secret_manager.write_secret( + secret_name=secret_name, + secret_value=json.dumps(config), + ) + + try: + secret_manager.delete_secret( + secret_name=secret_name, + ) + logger.warning("Secret deleted") + except Exception: + logger.exception("Failed to delete the secret") + pass + + return provider + @staticmethod def install_provider( tenant_id: str,