From b68f04a3ef95eee997902451f3e60986839c650c Mon Sep 17 00:00:00 2001 From: Pankaj B Date: Mon, 18 Dec 2023 16:19:21 +0530 Subject: [PATCH 1/6] render mirrors page on client --- ui/app/mirrors/page.tsx | 69 +++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 34 deletions(-) diff --git a/ui/app/mirrors/page.tsx b/ui/app/mirrors/page.tsx index 717af7b956..b3e53cbceb 100644 --- a/ui/app/mirrors/page.tsx +++ b/ui/app/mirrors/page.tsx @@ -1,3 +1,5 @@ +'use client'; + import { QRepConfig } from '@/grpc_generated/flow'; import { Button } from '@/lib/Button'; import { Header } from '@/lib/Header'; @@ -5,12 +7,13 @@ import { Icon } from '@/lib/Icon'; import { Label } from '@/lib/Label'; import { LayoutMain } from '@/lib/Layout'; import { Panel } from '@/lib/Panel'; +import { ProgressCircle } from '@/lib/ProgressCircle'; import Link from 'next/link'; -import { getTruePeer } from '../api/peers/route'; -import prisma from '../utils/prisma'; +import useSWR from 'swr'; import { CDCFlows, QRepFlows } from './tables'; export const dynamic = 'force-dynamic'; +const fetcher = (...args: [any]) => fetch(...args).then((res) => res.json()); const stringifyConfig = (flowArray: any[]) => { flowArray.forEach((flow) => { if (flow.config_proto) { @@ -19,29 +22,14 @@ const stringifyConfig = (flowArray: any[]) => { }); }; -export default async function Mirrors() { - let mirrors = await prisma.flows.findMany({ - distinct: 'name', - include: { - sourcePeer: true, - destinationPeer: true, - }, - }); - - const flows = mirrors.map((mirror) => { - let newMirror: any = { - ...mirror, - sourcePeer: getTruePeer(mirror.sourcePeer), - destinationPeer: getTruePeer(mirror.destinationPeer), - }; - return newMirror; - }); +export default function Mirrors() { + const { data: flows, error, isLoading } = useSWR('/api/mirrors', fetcher); - let cdcFlows = flows.filter((flow) => { + let cdcFlows = flows?.filter((flow: any) => { return !flow.query_string; }); - let qrepFlows = flows.filter((flow) => { + let qrepFlows = flows?.filter((flow: any) => { if (flow.config_proto && flow.query_string) { let config = QRepConfig.decode(flow.config_proto); const watermarkCol = config.watermarkColumn.toLowerCase(); @@ -50,7 +38,7 @@ export default async function Mirrors() { return false; }); - let xminFlows = flows.filter((flow) => { + let xminFlows = flows?.filter((flow: any) => { if (flow.config_proto && flow.query_string) { let config = QRepConfig.decode(flow.config_proto); return config.watermarkColumn.toLowerCase() === 'xmin'; @@ -58,9 +46,11 @@ export default async function Mirrors() { return false; }); - stringifyConfig(cdcFlows); - stringifyConfig(qrepFlows); - stringifyConfig(xminFlows); + if (!isLoading) { + stringifyConfig(cdcFlows); + stringifyConfig(qrepFlows); + stringifyConfig(xminFlows); + } return ( @@ -84,15 +74,26 @@ export default async function Mirrors() { Mirrors - - - - - - - - - + {isLoading && ( +
+ +
+ )} + {!isLoading && ( + + + + )} + {!isLoading && ( + + + + )} + {!isLoading && ( + + + + )} ); } From fd37ff7ca91c62e4e4939d0e280d140156ae20ed Mon Sep 17 00:00:00 2001 From: Pankaj B Date: Mon, 18 Dec 2023 16:40:16 +0530 Subject: [PATCH 2/6] add mirrors api --- ui/app/api/mirrors/route.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 ui/app/api/mirrors/route.ts diff --git a/ui/app/api/mirrors/route.ts b/ui/app/api/mirrors/route.ts new file mode 100644 index 0000000000..ab5d1efde2 --- /dev/null +++ b/ui/app/api/mirrors/route.ts @@ -0,0 +1,21 @@ +import { getTruePeer } from '@/app/api/peers/route'; +import prisma from '@/app/utils/prisma'; +export async function GET(request: Request) { + const mirrors = await prisma.flows.findMany({ + distinct: 'name', + include: { + sourcePeer: true, + destinationPeer: true, + }, + }); + + const flows = mirrors?.map((mirror) => { + let newMirror: any = { + ...mirror, + sourcePeer: getTruePeer(mirror.sourcePeer), + destinationPeer: getTruePeer(mirror.destinationPeer), + }; + return newMirror; + }); + return new Response(JSON.stringify(flows)); +} From b168f59e38b9ae9d8ff009f4a307dc0a12b0d679 Mon Sep 17 00:00:00 2001 From: Pankaj B Date: Mon, 18 Dec 2023 21:55:21 +0530 Subject: [PATCH 3/6] cleanup --- ui/app/api/mirrors/route.ts | 5 ++- ui/app/api/peers/getTruePeer.ts | 60 +++++++++++++++++++++++++++++++++ ui/app/api/peers/route.ts | 55 ++---------------------------- 3 files changed, 67 insertions(+), 53 deletions(-) create mode 100644 ui/app/api/peers/getTruePeer.ts diff --git a/ui/app/api/mirrors/route.ts b/ui/app/api/mirrors/route.ts index ab5d1efde2..6b7e8f4253 100644 --- a/ui/app/api/mirrors/route.ts +++ b/ui/app/api/mirrors/route.ts @@ -1,5 +1,8 @@ -import { getTruePeer } from '@/app/api/peers/route'; +import { getTruePeer } from '@/app/api/peers/getTruePeer'; import prisma from '@/app/utils/prisma'; + +export const dynamic = 'force-dynamic'; + export async function GET(request: Request) { const mirrors = await prisma.flows.findMany({ distinct: 'name', diff --git a/ui/app/api/peers/getTruePeer.ts b/ui/app/api/peers/getTruePeer.ts new file mode 100644 index 0000000000..1af4155dec --- /dev/null +++ b/ui/app/api/peers/getTruePeer.ts @@ -0,0 +1,60 @@ +import { CatalogPeer } from '@/app/dto/PeersDTO'; +import { + BigqueryConfig, + EventHubConfig, + EventHubGroupConfig, + Peer, + PostgresConfig, + S3Config, + SnowflakeConfig, + SqlServerConfig, +} from '@/grpc_generated/peers'; + +export const getTruePeer = (peer: CatalogPeer) => { + const newPeer: Peer = { + name: peer.name, + type: peer.type, + }; + const options = peer.options; + let config: + | BigqueryConfig + | SnowflakeConfig + | PostgresConfig + | EventHubConfig + | S3Config + | SqlServerConfig + | EventHubGroupConfig; + switch (peer.type) { + case 0: + config = BigqueryConfig.decode(options); + newPeer.bigqueryConfig = config; + break; + case 1: + config = SnowflakeConfig.decode(options); + newPeer.snowflakeConfig = config; + break; + case 3: + config = PostgresConfig.decode(options); + newPeer.postgresConfig = config; + break; + case 4: + config = EventHubConfig.decode(options); + newPeer.eventhubConfig = config; + break; + case 5: + config = S3Config.decode(options); + newPeer.s3Config = config; + break; + case 6: + config = SqlServerConfig.decode(options); + newPeer.sqlserverConfig = config; + break; + case 7: + config = EventHubGroupConfig.decode(options); + newPeer.eventhubGroupConfig = config; + break; + default: + return newPeer; + } + return newPeer; +}; diff --git a/ui/app/api/peers/route.ts b/ui/app/api/peers/route.ts index c865979efe..03aa98ae4a 100644 --- a/ui/app/api/peers/route.ts +++ b/ui/app/api/peers/route.ts @@ -1,3 +1,4 @@ +import { getTruePeer } from '@/app/api/peers/getTruePeer'; import { CatalogPeer, PeerConfig, @@ -8,13 +9,10 @@ import prisma from '@/app/utils/prisma'; import { BigqueryConfig, DBType, - EventHubConfig, - EventHubGroupConfig, Peer, PostgresConfig, S3Config, SnowflakeConfig, - SqlServerConfig, } from '@/grpc_generated/peers'; import { CreatePeerRequest, @@ -63,6 +61,8 @@ const constructPeer = ( } }; +export const dynamic = 'force-dynamic'; + export async function POST(request: Request) { const body = await request.json(); console.log('POST Validate Peer:', body); @@ -117,55 +117,6 @@ export async function POST(request: Request) { } } -export const getTruePeer = (peer: CatalogPeer) => { - const newPeer: Peer = { - name: peer.name, - type: peer.type, - }; - const options = peer.options; - let config: - | BigqueryConfig - | SnowflakeConfig - | PostgresConfig - | EventHubConfig - | S3Config - | SqlServerConfig - | EventHubGroupConfig; - switch (peer.type) { - case 0: - config = BigqueryConfig.decode(options); - newPeer.bigqueryConfig = config; - break; - case 1: - config = SnowflakeConfig.decode(options); - newPeer.snowflakeConfig = config; - break; - case 3: - config = PostgresConfig.decode(options); - newPeer.postgresConfig = config; - break; - case 4: - config = EventHubConfig.decode(options); - newPeer.eventhubConfig = config; - break; - case 5: - config = S3Config.decode(options); - newPeer.s3Config = config; - break; - case 6: - config = SqlServerConfig.decode(options); - newPeer.sqlserverConfig = config; - break; - case 7: - config = EventHubGroupConfig.decode(options); - newPeer.eventhubGroupConfig = config; - break; - default: - return newPeer; - } - return newPeer; -}; - // GET all the peers from the database export async function GET(request: Request) { const peers = await prisma.peers.findMany(); From 101305a7230345e37be19465f4caf34b8cd6458f Mon Sep 17 00:00:00 2001 From: Pankaj B Date: Mon, 18 Dec 2023 22:15:35 +0530 Subject: [PATCH 4/6] cleanup --- ui/app/mirrors/page.tsx | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/ui/app/mirrors/page.tsx b/ui/app/mirrors/page.tsx index b3e53cbceb..e4cb5318a6 100644 --- a/ui/app/mirrors/page.tsx +++ b/ui/app/mirrors/page.tsx @@ -23,13 +23,20 @@ const stringifyConfig = (flowArray: any[]) => { }; export default function Mirrors() { - const { data: flows, error, isLoading } = useSWR('/api/mirrors', fetcher); + const { + data: flows, + error, + isLoading, + }: { data: [any]; error: any; isLoading: boolean } = useSWR( + '/api/mirrors', + fetcher + ); - let cdcFlows = flows?.filter((flow: any) => { + let cdcFlows = flows?.filter((flow) => { return !flow.query_string; }); - let qrepFlows = flows?.filter((flow: any) => { + let qrepFlows = flows?.filter((flow) => { if (flow.config_proto && flow.query_string) { let config = QRepConfig.decode(flow.config_proto); const watermarkCol = config.watermarkColumn.toLowerCase(); @@ -38,7 +45,7 @@ export default function Mirrors() { return false; }); - let xminFlows = flows?.filter((flow: any) => { + let xminFlows = flows?.filter((flow) => { if (flow.config_proto && flow.query_string) { let config = QRepConfig.decode(flow.config_proto); return config.watermarkColumn.toLowerCase() === 'xmin'; From 07cf5bb1d831299522fc42becc8e678fe24c52e8 Mon Sep 17 00:00:00 2001 From: Pankaj B Date: Mon, 18 Dec 2023 23:11:00 +0530 Subject: [PATCH 5/6] fix stringify config, move it to api --- ui/app/api/mirrors/route.ts | 11 ++++++++++- ui/app/mirrors/page.tsx | 13 ------------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/ui/app/api/mirrors/route.ts b/ui/app/api/mirrors/route.ts index 6b7e8f4253..f9ca2f1911 100644 --- a/ui/app/api/mirrors/route.ts +++ b/ui/app/api/mirrors/route.ts @@ -3,6 +3,15 @@ import prisma from '@/app/utils/prisma'; export const dynamic = 'force-dynamic'; +const stringifyConfig = (flowArray: any[]) => { + flowArray.forEach((flow) => { + if (flow.config_proto) { + flow.config_proto = new TextDecoder().decode(flow.config_proto); + } + }); + return flowArray; +}; + export async function GET(request: Request) { const mirrors = await prisma.flows.findMany({ distinct: 'name', @@ -20,5 +29,5 @@ export async function GET(request: Request) { }; return newMirror; }); - return new Response(JSON.stringify(flows)); + return new Response(JSON.stringify(stringifyConfig(flows))); } diff --git a/ui/app/mirrors/page.tsx b/ui/app/mirrors/page.tsx index e4cb5318a6..b6ed706de9 100644 --- a/ui/app/mirrors/page.tsx +++ b/ui/app/mirrors/page.tsx @@ -14,13 +14,6 @@ import { CDCFlows, QRepFlows } from './tables'; export const dynamic = 'force-dynamic'; const fetcher = (...args: [any]) => fetch(...args).then((res) => res.json()); -const stringifyConfig = (flowArray: any[]) => { - flowArray.forEach((flow) => { - if (flow.config_proto) { - flow.config_proto = new TextDecoder().decode(flow.config_proto); - } - }); -}; export default function Mirrors() { const { @@ -53,12 +46,6 @@ export default function Mirrors() { return false; }); - if (!isLoading) { - stringifyConfig(cdcFlows); - stringifyConfig(qrepFlows); - stringifyConfig(xminFlows); - } - return ( From 8439d9252a702827f4dacbb1550446825071fc16 Mon Sep 17 00:00:00 2001 From: Pankaj B Date: Mon, 18 Dec 2023 23:32:19 +0530 Subject: [PATCH 6/6] centre loader --- ui/app/mirrors/page.tsx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ui/app/mirrors/page.tsx b/ui/app/mirrors/page.tsx index b6ed706de9..7cfe894294 100644 --- a/ui/app/mirrors/page.tsx +++ b/ui/app/mirrors/page.tsx @@ -69,9 +69,11 @@ export default function Mirrors() { {isLoading && ( -
- -
+ +
+ +
+ )} {!isLoading && (