From 79a1938299a1cecfa09cb2c02496429e1aaae5fc Mon Sep 17 00:00:00 2001 From: Pankaj Bhageria Date: Wed, 8 Nov 2023 11:31:02 +0530 Subject: [PATCH] feature: add screen to view mirror activity --- .../[mirrorId]/aggregatedCountsByInterval.ts | 107 ++++++++++++++++++ ui/app/mirrors/edit/[mirrorId]/cdcDetails.tsx | 91 +++++++++++---- ui/app/mirrors/edit/[mirrorId]/cdcGraph.tsx | 101 +++++++++++++++++ ui/app/mirrors/edit/[mirrorId]/page.tsx | 33 +++++- ui/app/mirrors/edit/[mirrorId]/syncStatus.tsx | 5 +- ui/app/mirrors/status/timeline.tsx | 3 + 6 files changed, 308 insertions(+), 32 deletions(-) create mode 100644 ui/app/mirrors/edit/[mirrorId]/aggregatedCountsByInterval.ts create mode 100644 ui/app/mirrors/edit/[mirrorId]/cdcGraph.tsx create mode 100644 ui/app/mirrors/status/timeline.tsx diff --git a/ui/app/mirrors/edit/[mirrorId]/aggregatedCountsByInterval.ts b/ui/app/mirrors/edit/[mirrorId]/aggregatedCountsByInterval.ts new file mode 100644 index 0000000000..b1be46c7eb --- /dev/null +++ b/ui/app/mirrors/edit/[mirrorId]/aggregatedCountsByInterval.ts @@ -0,0 +1,107 @@ +type timestampType ={ + timestamp: string; + count: number; +} + +function aggregateCountsByInterval(timestamps: timestampType[], interval:string) { + let timeUnit; + switch (interval) { + case 'hour': + timeUnit = 'YYYY-MM-DD HH:00:00'; + break; + case '15min': + timeUnit = 'YYYY-MM-DD HH:mm'; + break; + case 'month': + timeUnit = 'YYYY-MM'; + break; + case 'day': + timeUnit = 'YYYY-MM-DD'; + break; + default: + throw new Error('Invalid interval provided'); + } + + // Create an object to store the aggregated counts + const aggregatedCounts :{ [key: string]: number } = {}; + + // Iterate through the timestamps and populate the aggregatedCounts object + for (let { timestamp, count } of timestamps) { + timestamp = roundUpToNearest15Minutes(timestamp); + const date = new Date(timestamp); + const formattedTimestamp = formatTimestamp(date, timeUnit); + + if (!aggregatedCounts[formattedTimestamp]) { + aggregatedCounts[formattedTimestamp] = 0; + } + + aggregatedCounts[formattedTimestamp] += count; + } + + + + // Create an array of intervals between the start and end timestamps + const intervals = []; + + let currentTimestamp = new Date(); + + if(interval === "15min"){ + currentTimestamp = roundUpToNearest15Minutes(currentTimestamp); + } + + while (intervals.length < 30) { + intervals.push(formatTimestamp(currentTimestamp, timeUnit)); + if (interval === 'hour') { + currentTimestamp.setHours(currentTimestamp.getHours() - 1); + } else if (interval === '15min') { + currentTimestamp.setMinutes(currentTimestamp.getMinutes() - 15); + } else if (interval === 'month') { + currentTimestamp.setMonth(currentTimestamp.getMonth() - 1); + } else if(interval === 'day'){ + currentTimestamp.setDate(currentTimestamp.getDate() - 1); + } + } + + // Populate the result array with intervals and counts + const resultArray = intervals.map((interval) => [interval, aggregatedCounts[interval] || 0]); + return resultArray; + } + + function roundUpToNearest15Minutes(date:Date) { + const minutes = date.getMinutes(); + const remainder = minutes % 15; + + if (remainder > 0) { + // Round up to the nearest 15 minutes + date.setMinutes(minutes + (15 - remainder)); + } + + // Reset seconds and milliseconds to zero to maintain the same time + date.setSeconds(0); + date.setMilliseconds(0); + + return date; + } + + // Helper function to format a timestamp + function formatTimestamp(date:Date, format:string) { + const year = date.getFullYear(); + const month = padZero(date.getMonth() + 1); // Months are zero-based + const day = padZero(date.getDate()); + const hour = padZero(date.getHours()); + const minutes = padZero(date.getMinutes()); + + return format + .replace('YYYY', year) + .replace('MM', month) + .replace('DD', day) + .replace('HH', hour) + .replace('mm', minutes); + } + + // Helper function to pad single digits with leading zeros + function padZero(number:number) { + return number < 10 ? `0${number}` : `${number}`; + } + +export default aggregateCountsByInterval \ No newline at end of file diff --git a/ui/app/mirrors/edit/[mirrorId]/cdcDetails.tsx b/ui/app/mirrors/edit/[mirrorId]/cdcDetails.tsx index bd20ad1851..4eb7b875d5 100644 --- a/ui/app/mirrors/edit/[mirrorId]/cdcDetails.tsx +++ b/ui/app/mirrors/edit/[mirrorId]/cdcDetails.tsx @@ -1,35 +1,76 @@ +'use client' +import { Label } from '@/lib/Label'; +import { Badge } from '@/lib/Badge'; +import { Icon } from '@/lib/Icon'; +import {Action} from '@/lib/Action'; import { FlowConnectionConfigs } from '@/grpc_generated/flow'; +import CdcGraph from './cdcGraph' +import moment from 'moment'; -type CDCDetailsProps = { - config: FlowConnectionConfigs | undefined; + +type SyncStatusRow = { + batchId: number; + startTime: Date; + endTime: Date | null; + numRows: number; }; -export default function CDCDetails({ config }: CDCDetailsProps) { - if (!config) { - return
No configuration provided
; - } +type props = { + syncs: SyncStatusRow[]; + mirrorConfig: FlowConnectionConfigs | undefined; + }; +function CdcDetails({ syncs,mirrorConfig }:props) { + + let lastSyncedAt = moment(syncs[0]?.startTime).fromNow(); + let rowsSynced = syncs.reduce((acc, sync) => acc + sync.numRows, 0) return ( -
-

CDC Details

-
- - - - - - - - - - - - - - - -
Source{config.source?.name || '-'}
Destination{config.destination?.name || '-'}
Flow Job Name{config.flowJobName}
+ <> +
+
+
+
+
+
+
+
+
+
+
+
+
+
{mirrorConfig?.source?.name}
+
+
+
+
{mirrorConfig?.destination?.name}
+
+
+
+
+
+
+
+
+
+
+
+
+ +
+ ); } + +function numberWithCommas(x:Number) { + return x.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ","); +} + +export default CdcDetails \ No newline at end of file diff --git a/ui/app/mirrors/edit/[mirrorId]/cdcGraph.tsx b/ui/app/mirrors/edit/[mirrorId]/cdcGraph.tsx new file mode 100644 index 0000000000..05b53aff3b --- /dev/null +++ b/ui/app/mirrors/edit/[mirrorId]/cdcGraph.tsx @@ -0,0 +1,101 @@ +import { Label } from '@/lib/Label'; +import { useState, useEffect } from 'react'; +import moment from 'moment'; + +type SyncStatusRow = { + batchId: number; + startTime: Date; + endTime: Date | null; + numRows: number; +}; + +import aggregateCountsByInterval from './aggregatedCountsByInterval' + +const aggregateTypeMap = { + "15min":" 15 mins", + "hour": "Hour", + "day": "Day", + "month": "Month" +} + + function CdcGraph({syncs}:{syncs:SyncStatusRow[]}) { + + let [aggregateType,setAggregateType] = useState('hour'); + let [counts,setCounts] = useState([]); + + + let rows = syncs.map((sync) => ({ + timestamp: sync.startTime, + count: sync.numRows, + })) + + useEffect(()=>{ + let counts = aggregateCountsByInterval(rows,aggregateType, undefined, new Date()); + counts = counts.slice(0,29) + counts = counts.reverse(); + setCounts(counts) + + },[aggregateType, rows]) + + return
+
+ {["15min","hour","day","month"].map((type)=>{ + return + })} +
+
+ +
+ {counts.map((count,i)=>)} +
+
+ } + + type filterButtonProps = { + aggregateType:String; + selectedAggregateType:String; + setAggregateType:Function; + }; + function FilterButton({aggregateType,selectedAggregateType,setAggregateType}:filterButtonProps){ + return + } + + type GraphBarProps = { + count: number | undefined; + label: string + }; + + + function formatGraphLabel(date:Date, aggregateType:String) { + switch (aggregateType) { + case "15min": + return moment(date).format('MMM Do HH:mm'); + case "hour": + return moment(date).format('MMM Do HH:mm'); + case "day": + return moment(date).format('MMM Do'); + case "month": + return moment(date).format('MMM yy'); + } + } + + function GraphBar({label,count}:GraphBarProps){ + let color = count && count >0 ? 'bg-green-500' : 'bg-gray-500'; + let classNames = `relative w-10 h-24 rounded ${color}`; + return
+
+
+
{label}
+
{count}
+
+
+
+ } + + + export default CdcGraph; diff --git a/ui/app/mirrors/edit/[mirrorId]/page.tsx b/ui/app/mirrors/edit/[mirrorId]/page.tsx index e2c5e6596a..e5ca9914ef 100644 --- a/ui/app/mirrors/edit/[mirrorId]/page.tsx +++ b/ui/app/mirrors/edit/[mirrorId]/page.tsx @@ -4,8 +4,10 @@ import { LayoutMain } from '@/lib/Layout'; import { GetFlowHttpAddressFromEnv } from '@/rpc/http'; import { redirect } from 'next/navigation'; import { Suspense } from 'react'; -import { CDCMirror } from './cdc'; +import CdcDetails from './cdcDetails'; import SyncStatus from './syncStatus'; +import prisma from '@/app/utils/prisma'; + export const dynamic = 'force-dynamic'; @@ -44,17 +46,36 @@ export default async function EditMirror({ redirect(`/mirrors/status/qrep/${mirrorId}`); } + let syncs = await prisma.cdc_batches.findMany({ + where: { + flow_name: mirrorId, + start_time: { + not: undefined, + }, + }, + orderBy: { + start_time: 'desc', + }, + }) + + const rows = syncs.map((sync) => ({ + batchId: sync.id, + startTime: sync.start_time, + endTime: sync.end_time, + numRows: sync.rows_in_batch, + })); + return (
{mirrorId}
}> {mirrorStatus.cdcStatus && ( - + )} +
+ {syncStatusChild} +
); -} +} \ No newline at end of file diff --git a/ui/app/mirrors/edit/[mirrorId]/syncStatus.tsx b/ui/app/mirrors/edit/[mirrorId]/syncStatus.tsx index 80cb35701c..562e9f9e1f 100644 --- a/ui/app/mirrors/edit/[mirrorId]/syncStatus.tsx +++ b/ui/app/mirrors/edit/[mirrorId]/syncStatus.tsx @@ -1,5 +1,6 @@ import prisma from '@/app/utils/prisma'; import { SyncStatusTable } from './syncStatusTable'; +import NewCdcDetails from './newCdcDetails'; type SyncStatusProps = { flowJobName: string | undefined; @@ -29,5 +30,7 @@ export default async function SyncStatus({ flowJobName }: SyncStatusProps) { numRows: sync.rows_in_batch, })); - return ; + return
+ +
; } diff --git a/ui/app/mirrors/status/timeline.tsx b/ui/app/mirrors/status/timeline.tsx new file mode 100644 index 0000000000..6941617820 --- /dev/null +++ b/ui/app/mirrors/status/timeline.tsx @@ -0,0 +1,3 @@ + +//React component to show a timeline of sync events. Each bar on the time line will +//represent the number of \ No newline at end of file