From 59e7759df1ae1ffadc72fef33dd43204dab8b5e4 Mon Sep 17 00:00:00 2001 From: Matvey Kukuy Date: Thu, 21 Nov 2024 13:45:25 +0200 Subject: [PATCH 1/3] chore(release): new version 0.29.3 (#2573) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 713a701e1..5002c864b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "keep" -version = "0.29.2" +version = "0.29.3" description = "Alerting. for developers, by developers." authors = ["Keep Alerting LTD"] readme = "README.md" From 4463c01b89e1e33182213c48743bcbcd21047f67 Mon Sep 17 00:00:00 2001 From: Furkan Pehlivan <65170388+pehlicd@users.noreply.github.com> Date: Thu, 21 Nov 2024 18:24:59 +0100 Subject: [PATCH 2/3] feat(providers): added retry mechanism to google chat provider to overcome rate limiting (#2572) Co-authored-by: Shahar Glazner --- .../google_chat_provider.py | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/keep/providers/google_chat_provider/google_chat_provider.py b/keep/providers/google_chat_provider/google_chat_provider.py index ba27d4624..7eef362e6 100644 --- a/keep/providers/google_chat_provider/google_chat_provider.py +++ b/keep/providers/google_chat_provider/google_chat_provider.py @@ -1,4 +1,7 @@ +import http import os +import time + import pydantic import dataclasses import requests @@ -65,18 +68,32 @@ def _notify(self, message="", **kwargs: dict): if not message: raise ProviderException("Message is required") + def __send_message(url, body, headers, retries=3): + for attempt in range(retries): + try: + resp = requests.post(url, json=body, headers=headers) + if resp.status_code == http.HTTPStatus.OK: + return resp + + self.logger.warning(f"Attempt {attempt + 1} failed with status code {resp.status_code}") + + except requests.exceptions.RequestException as e: + self.logger.error(f"Attempt {attempt + 1} failed: {e}") + + if attempt < retries - 1: + time.sleep(1) + + raise requests.exceptions.RequestException(f"Failed to notify message after {retries} attempts") + payload = { "text": message, } - requestHeaders = {"Content-Type": "application/json; charset=UTF-8"} - - response = requests.post(webhook_url, json=payload, headers=requestHeaders) + request_headers = {"Content-Type": "application/json; charset=UTF-8"} - if not response.ok: - raise ProviderException( - f"Failed to notify message to Google Chat: {response.text}" - ) + response = __send_message(webhook_url, body=payload, headers=request_headers) + if response.status_code != http.HTTPStatus.OK: + raise ProviderException(f"Failed to notify message to Google Chat: {response.text}") self.logger.debug("Alert message sent to Google Chat successfully") return "Alert message sent to Google Chat successfully" From ea748d01355b82905beb5d0ff51d9e67455f1924 Mon Sep 17 00:00:00 2001 From: Kirill Chernakov Date: Sun, 24 Nov 2024 13:32:01 +0400 Subject: [PATCH 3/3] fix: feed counter (#2587) --- keep-ui/app/(keep)/topology/api/index.ts | 29 +++++++++---------- .../app/(keep)/topology/model/useTopology.ts | 6 ++-- .../topology/model/useTopologyApplications.ts | 23 ++++++++------- keep-ui/utils/hooks/usePresets.ts | 8 +++-- keep-ui/utils/state.ts | 14 ++------- keep/api/core/db.py | 10 +++++-- keep/api/routes/preset.py | 4 +-- keep/api/tasks/process_event_task.py | 10 +++---- 8 files changed, 51 insertions(+), 53 deletions(-) diff --git a/keep-ui/app/(keep)/topology/api/index.ts b/keep-ui/app/(keep)/topology/api/index.ts index 2d4e3b7f5..5736af7b9 100644 --- a/keep-ui/app/(keep)/topology/api/index.ts +++ b/keep-ui/app/(keep)/topology/api/index.ts @@ -2,19 +2,16 @@ import { fetcher } from "@/utils/fetcher"; import { Session } from "next-auth"; import { TopologyApplication, TopologyService } from "../model/models"; -export function buildTopologyUrl( - apiUrl: string, - { - providerIds, - services, - environment, - }: { - providerIds?: string[]; - services?: string[]; - environment?: string; - } -) { - const baseUrl = `${apiUrl}/topology`; +export function buildTopologyUrl({ + providerIds, + services, + environment, +}: { + providerIds?: string[]; + services?: string[]; + environment?: string; +}) { + const baseUrl = `/topology`; const params = new URLSearchParams(); @@ -57,6 +54,8 @@ export function getTopology( if (!session) { return null; } - const url = buildTopologyUrl(apiUrl, { providerIds, services, environment }); - return fetcher(url, session.accessToken) as Promise; + const url = buildTopologyUrl({ providerIds, services, environment }); + return fetcher(apiUrl + url, session.accessToken) as Promise< + TopologyService[] + >; } diff --git a/keep-ui/app/(keep)/topology/model/useTopology.ts b/keep-ui/app/(keep)/topology/model/useTopology.ts index 24f616f21..b38ebff03 100644 --- a/keep-ui/app/(keep)/topology/model/useTopology.ts +++ b/keep-ui/app/(keep)/topology/model/useTopology.ts @@ -7,7 +7,7 @@ import { buildTopologyUrl } from "@/app/(keep)/topology/api"; import { useTopologyPollingContext } from "@/app/(keep)/topology/model/TopologyPollingContext"; import { useApiUrl } from "utils/hooks/useConfig"; -export const useTopologyBaseKey = () => `${useApiUrl()}/topology`; +export const TOPOLOGY_URL = `/topology`; type UseTopologyOptions = { providerIds?: string[]; @@ -37,11 +37,11 @@ export const useTopology = ( const url = !session ? null - : buildTopologyUrl(apiUrl!, { providerIds, services, environment }); + : buildTopologyUrl({ providerIds, services, environment }); const { data, error, mutate } = useSWR( url, - (url: string) => fetcher(url, session!.accessToken), + (url: string) => fetcher(apiUrl! + url, session!.accessToken), { fallbackData, ...options, diff --git a/keep-ui/app/(keep)/topology/model/useTopologyApplications.ts b/keep-ui/app/(keep)/topology/model/useTopologyApplications.ts index eb749d254..bfc211c65 100644 --- a/keep-ui/app/(keep)/topology/model/useTopologyApplications.ts +++ b/keep-ui/app/(keep)/topology/model/useTopologyApplications.ts @@ -4,14 +4,17 @@ import useSWR, { SWRConfiguration } from "swr"; import { fetcher } from "@/utils/fetcher"; import { useHydratedSession as useSession } from "@/shared/lib/hooks/useHydratedSession"; import { useCallback, useMemo } from "react"; -import { useTopologyBaseKey, useTopology } from "./useTopology"; +import { useTopology } from "./useTopology"; import { useRevalidateMultiple } from "@/utils/state"; +import { TOPOLOGY_URL } from "./useTopology"; type UseTopologyApplicationsOptions = { initialData?: TopologyApplication[]; options?: SWRConfiguration; }; +export const TOPOLOGY_APPLICATIONS_URL = `/topology/applications`; + export function useTopologyApplications( { initialData, options }: UseTopologyApplicationsOptions = { options: { @@ -21,13 +24,11 @@ export function useTopologyApplications( ) { const apiUrl = useApiUrl(); const { data: session } = useSession(); - const topologyBaseKey = useTopologyBaseKey(); const revalidateMultiple = useRevalidateMultiple(); const { topologyData, mutate: mutateTopology } = useTopology(); - const topologyApplicationsKey = `${apiUrl}/topology/applications`; const { data, error, isLoading, mutate } = useSWR( - !session ? null : topologyApplicationsKey, - (url: string) => fetcher(url, session!.accessToken), + !session ? null : TOPOLOGY_APPLICATIONS_URL, + (url: string) => fetcher(apiUrl + url, session!.accessToken), { fallbackData: initialData, ...options, @@ -48,7 +49,7 @@ export function useTopologyApplications( }); if (response.ok) { console.log("mutating on success"); - revalidateMultiple([topologyBaseKey, topologyApplicationsKey]); + revalidateMultiple([TOPOLOGY_URL, TOPOLOGY_APPLICATIONS_URL]); } else { // Rollback optimistic update on error throw new Error("Failed to add application", { @@ -58,7 +59,7 @@ export function useTopologyApplications( const json = await response.json(); return json as TopologyApplication; }, - [revalidateMultiple, session?.accessToken, topologyApplicationsKey] + [apiUrl, revalidateMultiple, session?.accessToken] ); const updateApplication = useCallback( @@ -98,7 +99,7 @@ export function useTopologyApplications( } ); if (response.ok) { - revalidateMultiple([topologyBaseKey, topologyApplicationsKey]); + revalidateMultiple([TOPOLOGY_URL, TOPOLOGY_APPLICATIONS_URL]); } else { // Rollback optimistic update on error mutate(applications, false); @@ -110,12 +111,12 @@ export function useTopologyApplications( return response; }, [ + apiUrl, applications, mutate, mutateTopology, revalidateMultiple, session?.accessToken, - topologyApplicationsKey, topologyData, ] ); @@ -152,7 +153,7 @@ export function useTopologyApplications( } ); if (response.ok) { - revalidateMultiple([topologyBaseKey, topologyApplicationsKey]); + revalidateMultiple([TOPOLOGY_URL, TOPOLOGY_APPLICATIONS_URL]); } else { // Rollback optimistic update on error mutate(applications, false); @@ -164,12 +165,12 @@ export function useTopologyApplications( return response; }, [ + apiUrl, applications, mutate, mutateTopology, revalidateMultiple, session?.accessToken, - topologyApplicationsKey, topologyData, ] ); diff --git a/keep-ui/utils/hooks/usePresets.ts b/keep-ui/utils/hooks/usePresets.ts index 6cc10873d..a3c72ec5e 100644 --- a/keep-ui/utils/hooks/usePresets.ts +++ b/keep-ui/utils/hooks/usePresets.ts @@ -9,6 +9,7 @@ import { useConfig } from "./useConfig"; import useSWRSubscription from "swr/subscription"; import { useWebsocket } from "./usePusher"; import { useSearchParams } from "next/navigation"; +import { useRevalidateMultiple } from "../state"; export const usePresets = (type?: string, useFilters?: boolean) => { const { data: session } = useSession(); @@ -31,6 +32,7 @@ export const usePresets = (type?: string, useFilters?: boolean) => { const presetsOrderRef = useRef(presetsOrderFromLS); const staticPresetsOrderRef = useRef(staticPresetsOrderFromLS); const { bind, unbind } = useWebsocket(); + const revalidateMultiple = useRevalidateMultiple(); useEffect(() => { presetsOrderRef.current = presetsOrderFromLS; @@ -88,6 +90,8 @@ export const usePresets = (type?: string, useFilters?: boolean) => { (_, { next }) => { const newPresets = (newPresets: Preset[]) => { updateLocalPresets(newPresets); + // update the presets aggregated endpoint for the sidebar + revalidateMultiple(["/preset"]); next(null, { presets: newPresets, isAsyncLoading: false, @@ -110,11 +114,11 @@ export const usePresets = (type?: string, useFilters?: boolean) => { return useSWR( () => session - ? `${apiUrl}/preset${ + ? `/preset${ useFilters && filters && isDashBoard ? `?${filters}` : "" }` : null, - (url) => fetcher(url, session?.accessToken), + (url) => fetcher(apiUrl + url, session?.accessToken), { ...options, onSuccess: (data) => { diff --git a/keep-ui/utils/state.ts b/keep-ui/utils/state.ts index 517ad49f9..ded186d6a 100644 --- a/keep-ui/utils/state.ts +++ b/keep-ui/utils/state.ts @@ -1,17 +1,9 @@ import { useSWRConfig } from "swr"; -type MutateArgs = [string, (data: any) => any]; - -export const mutateLocalMultiple = (args: MutateArgs[]) => { - const { cache } = useSWRConfig(); - args.forEach(([key, mutateFunction]) => { - const currentData = cache.get(key as string); - cache.set(key as string, mutateFunction(currentData)); - }); -}; - export const useRevalidateMultiple = () => { const { mutate } = useSWRConfig(); return (keys: string[]) => - mutate((key) => typeof key === "string" && keys.includes(key)); + mutate( + (key) => typeof key === "string" && keys.some((k) => key.startsWith(k)) + ); }; diff --git a/keep/api/core/db.py b/keep/api/core/db.py index f374117ee..9be0d25f7 100644 --- a/keep/api/core/db.py +++ b/keep/api/core/db.py @@ -40,6 +40,7 @@ from sqlalchemy.sql import exists, expression from sqlmodel import Session, SQLModel, col, or_, select, text +from keep.api.consts import STATIC_PRESETS from keep.api.core.db_utils import create_db_engine, get_json_extract_field # This import is required to create the tables @@ -2615,7 +2616,7 @@ def get_presets( return presets -def get_preset_by_name(tenant_id: str, preset_name: str) -> Preset: +def get_db_preset_by_name(tenant_id: str, preset_name: str) -> Preset | None: with Session(engine) as session: preset = session.exec( select(Preset) @@ -2624,8 +2625,7 @@ def get_preset_by_name(tenant_id: str, preset_name: str) -> Preset: ).first() return preset - -def get_all_presets(tenant_id: str) -> List[Preset]: +def get_db_presets(tenant_id: str) -> List[Preset]: with Session(engine) as session: presets = ( session.exec(select(Preset).where(Preset.tenant_id == tenant_id)) @@ -2634,6 +2634,10 @@ def get_all_presets(tenant_id: str) -> List[Preset]: ) return presets +def get_all_presets_dtos(tenant_id: str) -> List[PresetDto]: + presets = get_db_presets(tenant_id) + static_presets_dtos = list(STATIC_PRESETS.values()) + return [PresetDto(**preset.to_dict()) for preset in presets] + static_presets_dtos def get_dashboards(tenant_id: str, email=None) -> List[Dict[str, Any]]: with Session(engine) as session: diff --git a/keep/api/routes/preset.py b/keep/api/routes/preset.py index 1f3f9b22a..bd331806c 100644 --- a/keep/api/routes/preset.py +++ b/keep/api/routes/preset.py @@ -15,7 +15,7 @@ from sqlmodel import Session, select from keep.api.consts import PROVIDER_PULL_INTERVAL_DAYS, STATIC_PRESETS -from keep.api.core.db import get_preset_by_name as get_preset_by_name_db +from keep.api.core.db import get_db_preset_by_name from keep.api.core.db import get_presets as get_presets_db from keep.api.core.db import ( get_session, @@ -448,7 +448,7 @@ def get_preset_alerts( if preset_name in STATIC_PRESETS: preset = STATIC_PRESETS[preset_name] else: - preset = get_preset_by_name_db(tenant_id, preset_name) + preset = get_db_preset_by_name(tenant_id, preset_name) # if preset does not exist if not preset: raise HTTPException(404, "Preset not found") diff --git a/keep/api/tasks/process_event_task.py b/keep/api/tasks/process_event_task.py index a4bec5e8c..3809ea872 100644 --- a/keep/api/tasks/process_event_task.py +++ b/keep/api/tasks/process_event_task.py @@ -20,7 +20,7 @@ from keep.api.core.db import ( bulk_upsert_alert_fields, get_alerts_by_fingerprint, - get_all_presets, + get_all_presets_dtos, get_enrichment_with_session, get_session_sync, ) @@ -28,7 +28,6 @@ from keep.api.core.elastic import ElasticClient from keep.api.models.alert import AlertDto, AlertStatus, IncidentDto from keep.api.models.db.alert import Alert, AlertActionType, AlertAudit, AlertRaw -from keep.api.models.db.preset import PresetDto from keep.api.utils.enrichment_helpers import ( calculated_start_firing_time, convert_db_alerts_to_dto_alerts, @@ -443,12 +442,11 @@ def __handle_formatted_events( return try: - presets = get_all_presets(tenant_id) + presets = get_all_presets_dtos(tenant_id) rules_engine = RulesEngine(tenant_id=tenant_id) presets_do_update = [] - for preset in presets: + for preset_dto in presets: # filter the alerts based on the search query - preset_dto = PresetDto(**preset.to_dict()) filtered_alerts = rules_engine.filter_alerts( enriched_formatted_events, preset_dto.cel_query ) @@ -458,7 +456,7 @@ def __handle_formatted_events( presets_do_update.append(preset_dto) preset_dto.alerts_count = len(filtered_alerts) # update noisy - if preset.is_noisy: + if preset_dto.is_noisy: firing_filtered_alerts = list( filter( lambda alert: alert.status == AlertStatus.FIRING.value,