Merge pull request #1 from rajeshj11/feat-1779-main

feat: added the quality metrics stats logic in backend and minor modification on forntend

keep-ui/app/alerts/quality/alert-quality-table.tsx

@@ -1,27 +1,33 @@
 "use client"; // Add this line at the top to make this a Client Component
 
 import React, { useState, useEffect } from 'react';
-import { useFetchProviders } from 'app/providers/page.client';
 import { GenericTable } from '@/components/table/GenericTable';
+import { useAlertQualityMetrics } from 'utils/hooks/useAlertQuality';
+import { useProviders } from 'utils/hooks/useProviders';
+import { Providers } from 'app/providers/providers';
 
 interface ProviderAlertQuality {
-  providerName: string;
   alertsReceived: number;
   alertsCorrelatedToIncidentsPercentage: number;
-  alertsWithFieldFilledPercentage: number;
+  // alertsWithFieldFilledPercentage: number;
+  alertsWithSeverityPercentage: number;
 }
 
+interface Pagination {
+  limit: number;
+  offset: number;
+}
 const AlertQualityTable = () => {
-  const {installedProviders} = useFetchProviders();
-  const [data, setData] = useState<ProviderAlertQuality[]>([]);
-  const [rowCount, setRowCount] = useState<number>(0);
-  const [offset, setOffset] = useState<number>(0);
-  const [limit, setLimit] = useState<number>(10);
-
+  const {data: providersMeta} = useProviders(); 
+  const {data: alertsQualityMetrics, error} =  useAlertQualityMetrics()
+  const [pagination, setPagination] = useState<Pagination>({
+    limit: 25,
+    offset: 0,
+});
   const columns = [
     {
       header: 'Provider Name',
-      accessorKey: 'providerName',
+      accessorKey: 'display_name',
     },
     {
       header: 'Alerts Received',
@@ -33,53 +39,55 @@ const AlertQualityTable = () => {
       cell: (info: any) => `${info.getValue().toFixed(2)}%`,
     },
     {
-      header: '% of Alerts Having Field Filled',
-      accessorKey: 'alertsWithFieldFilledPercentage',
+      header: '% of Alerts Having Severity',//we are considering critical and warning as severe
+      accessorKey: 'alertsWithSeverityPercentage',
       cell: (info: any) => `${info.getValue().toFixed(2)}%`,
     },
   ];
 
-  useEffect(() => {
-    const fetchData = async () => {
-      try {
-        const transformedData = installedProviders.map((provider: any) => ({
-          providerName: `${provider.details.name}  (${provider.display_name})` || 'Unknown',
-          alertsReceived: provider.alertsReceived || 0,
-          alertsCorrelatedToIncidentsPercentage: provider.alertsCorrelatedToIncidentsPercentage * 100 || 0,
-          alertsWithFieldFilledPercentage: provider.alertsWithFieldFilledPercentage * 100 || 0,
-        }));
+  const finalData: Providers&ProviderAlertQuality[] = [];
+  const providers =  providersMeta?.providers;
 
-        setData(transformedData);
-        setRowCount(transformedData.length);
-      } catch (error) {
-        console.error('Failed to fetch data:', error);
-      }
-    };
 
-    fetchData();
-  }, [offset, limit]);
+  if (alertsQualityMetrics && providers) {
+    providers.forEach( provider => {
+      const  providerType = provider.type;
+      const alertQuality = alertsQualityMetrics[providerType];
+      const totalAlertsReceived = alertQuality?.total_alerts ?? 0;
+      const correlated_alerts = alertQuality?.correlated_alerts ?? 0;
+      const correltedPert = totalAlertsReceived && correlated_alerts ? (correlated_alerts/totalAlertsReceived)*100 : 0;
+      const severityPert = totalAlertsReceived ? ((alertQuality?.severity_count ?? 0)/totalAlertsReceived)*100 : 0
+      finalData.push({
+        ...provider,
+        alertsReceived: totalAlertsReceived,
+        alertsCorrelatedToIncidentsPercentage: correltedPert,
+        alertsWithSeverityPercentage: severityPert,
+      });
+    });
+  }
 
   const handlePaginationChange = (newLimit: number, newOffset: number) => {
-    setLimit(newLimit);
-    setOffset(newOffset);
+    setPagination({ limit: newLimit, offset: newOffset })
   };
 
+
   return (
     <div className="p-4">
         <h1 className="text-lg font-semibold text-gray-800 dark:text-gray-100 mb-4">
             Alert Quality Dashboard
         </h1>
-        <GenericTable
-            data={data}
+        {providers && alertsQualityMetrics && <GenericTable
+            data={finalData}
             columns={columns}
-            rowCount={rowCount}
-            offset={offset}
-            limit={limit}
+            rowCount={finalData?.length}
+            offset={pagination.offset}
+            limit={pagination.limit}
             onPaginationChange={handlePaginationChange}
+            dataFetchedAtOneGO={true}
             onRowClick={(row) => {
                 console.log('Row clicked:', row);
             }}
-        />
+        />}
     </div>
   );
 };

keep-ui/components/table/GenericTable.tsx

@@ -26,6 +26,7 @@ interface GenericTableProps<T> {
     limit: number;
     onPaginationChange: ( limit: number, offset: number ) => void;
     onRowClick?: (row: T) => void;
+    dataFetchedAtOneGO?: boolean
 }
 
 export function GenericTable<T>({
@@ -36,6 +37,7 @@ export function GenericTable<T>({
     limit,
     onPaginationChange,
     onRowClick,
+    dataFetchedAtOneGO,
 }: GenericTableProps<T>) {
     const [expanded, setExpanded] = useState<ExpandedState>({});
     const [pagination, setPagination] = useState({
@@ -60,9 +62,11 @@ export function GenericTable<T>({
         }
     }, [pagination]);
 
+    const finalData = (dataFetchedAtOneGO ? data.slice(pagination.pageSize * pagination.pageIndex, pagination.pageSize * (pagination.pageIndex + 1)) : data) as T[]
+
     const table = useReactTable({
         columns,
-        data,
+        data: finalData,
         state: { expanded, pagination },
         getCoreRowModel: getCoreRowModel(),
         manualPagination: true,

keep-ui/utils/hooks/useAlertQuality.ts

@@ -0,0 +1,17 @@
+import { useSession } from "next-auth/react";
+import { getApiURL } from "../apiUrl";
+import { SWRConfiguration } from "swr";
+import { fetcher } from "../fetcher";
+import useSWRImmutable from "swr/immutable";
+
+export const useAlertQualityMetrics = (options: SWRConfiguration = {}) => {
+  const { data: session } = useSession();
+  const apiUrl = getApiURL();
+
+  // TODO: Proper type needs to be defined.
+  return useSWRImmutable<Record<string, Record<string, any>>>(
+    () => (session ? `${apiUrl}/alerts/quality/metrics` : null),
+    (url) => fetcher(url, session?.accessToken),
+    options
+  );
+};

keep/api/core/db.py

@@ -19,7 +19,7 @@
 import validators
 from dotenv import find_dotenv, load_dotenv
 from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
-from sqlalchemy import and_, desc, null, update
+from sqlalchemy import and_, desc, null, update, func, case
 from sqlalchemy.dialects.mysql import insert as mysql_insert
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.dialects.sqlite import insert as sqlite_insert
@@ -3282,3 +3282,44 @@ def change_incident_status_by_id(
         updated = session.execute(stmt)
         session.commit()
         return updated.rowcount > 0
+
+def get_alerts_metrics_by_provider(
+    tenant_id: str,
+    start_date: Optional[datetime] = None, 
+    end_date: Optional[datetime] = None
+) -> Dict[str, Dict[str, Any]]:
+
+    # Set default dates to the last 30 days if not provided
+    if start_date is None:
+        start_date = datetime.now() - timedelta(days=30)
+    if end_date is None:
+        end_date = datetime.now()
+
+    #if the below query is not perfomring well, we can try to optimise the query using Venn Diagram or similar(for now we are using the below query)
+    with Session(engine) as session:
+        results = (
+            session.query(
+                Alert.provider_type,
+                func.count(Alert.id).label("total_alerts"),
+                func.sum(case([(AlertToIncident.alert_id.isnot(None), 1)], else_=0)).label("correlated_alerts"),
+                func.sum(case([(func.json_extract(Alert.event, '$.severity').in_(['critical', 'warning']), 1)], else_=0)).label("severity_count")
+            )
+            .outerjoin(AlertToIncident, Alert.id == AlertToIncident.alert_id)
+            .filter(
+                Alert.tenant_id == tenant_id,
+                Alert.timestamp >= start_date,
+                Alert.timestamp <= end_date,    
+                Alert.provider_type.isnot(None)
+            )
+            .group_by(Alert.provider_type)
+            .all()
+        )
+
+    return {
+        row.provider_type: {
+            "total_alerts": row.total_alerts,
+            "correlated_alerts": row.correlated_alerts,
+            "severity_count": row.severity_count,
+        }
+        for row in results
+    }      

keep/api/routes/alerts.py

@@ -25,7 +25,7 @@
 from keep.api.consts import KEEP_ARQ_QUEUE_BASIC
 from keep.api.core.config import config
 from keep.api.core.db import get_alert_audit as get_alert_audit_db
-from keep.api.core.db import get_alerts_by_fingerprint, get_enrichment, get_last_alerts
+from keep.api.core.db import get_alerts_by_fingerprint, get_enrichment, get_last_alerts, get_alerts_metrics_by_provider
 from keep.api.core.dependencies import get_pusher_client
 from keep.api.core.elastic import ElasticClient
 from keep.api.models.alert import (
@@ -744,3 +744,22 @@ def get_alert_audit(
 
     grouped_events = AlertAuditDto.from_orm_list(alert_audit)
     return grouped_events
+
+
+@router.get("/quality/metrics", description="Get alert quality")
+def get_alert_quality(
+    authenticated_entity: AuthenticatedEntity = Depends(
+        IdentityManagerFactory.get_auth_verifier(["read:alert"])
+    ),
+):
+    logger.info(
+        "Fetching alert quality metrics per provider",
+        extra={
+            "tenant_id": authenticated_entity.tenant_id,
+        },
+    )
+    db_alerts_quality = get_alerts_metrics_by_provider(
+        tenant_id=authenticated_entity.tenant_id
+    )
+
+    return db_alerts_quality