From 54ab279f8c2cb949c1983005f9d651b20e5b8230 Mon Sep 17 00:00:00 2001 From: Amogh Bharadwaj Date: Tue, 12 Mar 2024 19:04:23 +0530 Subject: [PATCH 1/2] Plethora of UI Improvements (#1470) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## UI Usability Changes This PR performs a handful of improvements, fixes and changes across the board for our UI. ### Create Mirror Page - Moves `Snapshot Number of Tables In Parallel` , `CDC Staging Path` ,`Snapshot Staging Path` to **Advanced Settings** - Publication name is now a dropdown field. ![Screenshot 2024-03-11 at 11 15 31 PM](https://github.com/PeerDB-io/peerdb/assets/65964360/f7ed924d-0ef1-4834-b686-2636d9b0faf6) - Increased font size of info tips - Table picker now shows the size of every table Screenshot 2024-03-10 at 6 59 42 PM Screenshot 2024-03-10 at 6 01 11 PM ### Edit Mirror Page - Added a banner regarding adding tables. This banner pops up only when atleast one table has been selected for addition. - Made Edit and Back button float to reduce scrolling hassle Screenshot 2024-03-10 at 6 59 20 PM ### Create Peer Pages - Dropdown now has options for RDS, Azure and GCP - Errors now show in toast bars - Added link to corresponding PeerDB doc. Screenshot 2024-03-10 at 7 46 11 PM Screenshot 2024-03-10 at 7 46 30 PM --- flow/cmd/peer_data.go | 40 ++++++++++++- protos/route.proto | 9 +++ ui/app/api/peers/publications/route.ts | 22 +++++++ ui/app/api/peers/schemas/route.ts | 5 -- ui/app/dto/MirrorsDTO.ts | 1 + ui/app/dto/PeersDTO.ts | 4 ++ ui/app/mirrors/[mirrorId]/edit/page.tsx | 52 +++++++++------- ui/app/mirrors/create/cdc/cdc.tsx | 29 ++++++++- ui/app/mirrors/create/cdc/fields.tsx | 64 ++++++++++++++++++-- ui/app/mirrors/create/cdc/guide.tsx | 18 ++++-- ui/app/mirrors/create/cdc/schemabox.tsx | 7 +++ ui/app/mirrors/create/handlers.ts | 16 +++++ ui/app/mirrors/create/helpers/cdc.ts | 40 ++++++++----- ui/app/mirrors/create/helpers/common.ts | 9 +-- ui/app/mirrors/create/schema.ts | 8 +-- ui/app/peers/create/[peerType]/handlers.ts | 26 ++++---- ui/app/peers/create/[peerType]/page.tsx | 70 ++++++++++++++-------- ui/app/peers/create/page.tsx | 1 + ui/app/utils/titlecase.ts | 11 ++++ ui/components/InfoPopover.tsx | 24 +++++++- ui/components/PeerComponent.tsx | 6 ++ ui/components/PeerForms/PostgresForm.tsx | 4 +- ui/components/ResyncDialog.tsx | 2 +- ui/components/SelectSource.tsx | 21 ++++--- ui/lib/Button/Button.styles.ts | 10 ++++ ui/public/svgs/azurepg.svg | 1 + ui/public/svgs/gcp.svg | 1 + ui/public/svgs/rds.svg | 16 +++++ 28 files changed, 404 insertions(+), 113 deletions(-) create mode 100644 ui/app/api/peers/publications/route.ts create mode 100644 ui/app/utils/titlecase.ts create mode 100644 ui/public/svgs/azurepg.svg create mode 100644 ui/public/svgs/gcp.svg create mode 100644 ui/public/svgs/rds.svg diff --git a/flow/cmd/peer_data.go b/flow/cmd/peer_data.go index 7ec28329e2..b61936dc2b 100644 --- a/flow/cmd/peer_data.go +++ b/flow/cmd/peer_data.go @@ -92,7 +92,8 @@ func (h *FlowRequestHandler) GetTablesInSchema( CASE WHEN con.contype = 'p' OR t.relreplident = 'i' OR t.relreplident = 'f' THEN true ELSE false - END AS can_mirror + END AS can_mirror, + pg_size_pretty(pg_total_relation_size(t.oid)) :: text AS table_size FROM pg_class t LEFT JOIN @@ -108,6 +109,7 @@ func (h *FlowRequestHandler) GetTablesInSchema( can_mirror DESC; `, req.SchemaName) if err != nil { + slog.Info("failed to fetch publications", slog.Any("error", err)) return &protos.SchemaTablesResponse{Tables: nil}, err } @@ -116,10 +118,15 @@ func (h *FlowRequestHandler) GetTablesInSchema( for rows.Next() { var table pgtype.Text var hasPkeyOrReplica pgtype.Bool - err := rows.Scan(&table, &hasPkeyOrReplica) + var tableSize pgtype.Text + err := rows.Scan(&table, &hasPkeyOrReplica, &tableSize) if err != nil { return &protos.SchemaTablesResponse{Tables: nil}, err } + var sizeOfTable string + if tableSize.Valid { + sizeOfTable = tableSize.String + } canMirror := false if hasPkeyOrReplica.Valid && hasPkeyOrReplica.Bool { canMirror = true @@ -128,8 +135,14 @@ func (h *FlowRequestHandler) GetTablesInSchema( tables = append(tables, &protos.TableResponse{ TableName: table.String, CanMirror: canMirror, + TableSize: sizeOfTable, }) } + + if err := rows.Err(); err != nil { + slog.Info("failed to fetch publications", slog.Any("error", err)) + return &protos.SchemaTablesResponse{Tables: nil}, err + } return &protos.SchemaTablesResponse{Tables: tables}, nil } @@ -325,3 +338,26 @@ func (h *FlowRequestHandler) GetStatInfo( StatData: statInfoRows, }, nil } + +func (h *FlowRequestHandler) GetPublications( + ctx context.Context, + req *protos.PostgresPeerActivityInfoRequest, +) (*protos.PeerPublicationsResponse, error) { + tunnel, peerConn, err := h.getConnForPGPeer(ctx, req.PeerName) + if err != nil { + return &protos.PeerPublicationsResponse{PublicationNames: nil}, err + } + defer tunnel.Close() + defer peerConn.Close(ctx) + + rows, err := peerConn.Query(ctx, "select pubname from pg_publication;") + if err != nil { + return &protos.PeerPublicationsResponse{PublicationNames: nil}, err + } + + publications, err := pgx.CollectRows[string](rows, pgx.RowTo) + if err != nil { + return &protos.PeerPublicationsResponse{PublicationNames: nil}, err + } + return &protos.PeerPublicationsResponse{PublicationNames: publications}, nil +} diff --git a/protos/route.proto b/protos/route.proto index 316459d78f..4d2c601fad 100644 --- a/protos/route.proto +++ b/protos/route.proto @@ -109,6 +109,10 @@ message PeerSchemasResponse { repeated string schemas = 1; } +message PeerPublicationsResponse { + repeated string publication_names = 1; +} + message SchemaTablesRequest { string peer_name = 1; string schema_name = 2; @@ -121,6 +125,7 @@ message SchemaTablesResponse { message TableResponse { string table_name = 1; bool can_mirror = 2; + string table_size = 3; } message AllTablesResponse { @@ -265,6 +270,10 @@ service FlowService { option (google.api.http) = { get: "/v1/peers/schemas" }; } + rpc GetPublications(PostgresPeerActivityInfoRequest) returns (PeerPublicationsResponse) { + option (google.api.http) = { get: "/v1/peers/publications" }; + } + rpc GetTablesInSchema(SchemaTablesRequest) returns (SchemaTablesResponse) { option (google.api.http) = { get: "/v1/peers/tables" }; } diff --git a/ui/app/api/peers/publications/route.ts b/ui/app/api/peers/publications/route.ts new file mode 100644 index 0000000000..64dcd3dcf6 --- /dev/null +++ b/ui/app/api/peers/publications/route.ts @@ -0,0 +1,22 @@ +import { UPublicationsResponse } from '@/app/dto/PeersDTO'; +import { PeerPublicationsResponse } from '@/grpc_generated/route'; +import { GetFlowHttpAddressFromEnv } from '@/rpc/http'; + +export async function POST(request: Request) { + const body = await request.json(); + const { peerName } = body; + const flowServiceAddr = GetFlowHttpAddressFromEnv(); + try { + const publicationList: PeerPublicationsResponse = await fetch( + `${flowServiceAddr}/v1/peers/publications?peer_name=${peerName}` + ).then((res) => { + return res.json(); + }); + let response: UPublicationsResponse = { + publicationNames: publicationList.publicationNames, + }; + return new Response(JSON.stringify(response)); + } catch (e) { + console.log(e); + } +} diff --git a/ui/app/api/peers/schemas/route.ts b/ui/app/api/peers/schemas/route.ts index 0c701cb713..9a1c10856e 100644 --- a/ui/app/api/peers/schemas/route.ts +++ b/ui/app/api/peers/schemas/route.ts @@ -14,11 +14,6 @@ export async function POST(request: Request) { let response: USchemasResponse = { schemas: schemaList.schemas, }; - if (schemaList.message === 'no rows in result set') { - response = { - schemas: [], - }; - } return new Response(JSON.stringify(response)); } catch (e) { console.log(e); diff --git a/ui/app/dto/MirrorsDTO.ts b/ui/app/dto/MirrorsDTO.ts index 33234a557d..9b3724902a 100644 --- a/ui/app/dto/MirrorsDTO.ts +++ b/ui/app/dto/MirrorsDTO.ts @@ -26,6 +26,7 @@ export type TableMapRow = { exclude: Set; selected: boolean; canMirror: boolean; + tableSize: string; }; export type SyncStatusRow = { diff --git a/ui/app/dto/PeersDTO.ts b/ui/app/dto/PeersDTO.ts index a015439d83..23b0243b4f 100644 --- a/ui/app/dto/PeersDTO.ts +++ b/ui/app/dto/PeersDTO.ts @@ -56,3 +56,7 @@ export type SlotLagPoint = { updatedAt: string; slotSize?: string; }; + +export type UPublicationsResponse = { + publicationNames: string[]; +}; diff --git a/ui/app/mirrors/[mirrorId]/edit/page.tsx b/ui/app/mirrors/[mirrorId]/edit/page.tsx index 4cff1b4538..3b0240a0c6 100644 --- a/ui/app/mirrors/[mirrorId]/edit/page.tsx +++ b/ui/app/mirrors/[mirrorId]/edit/page.tsx @@ -11,6 +11,7 @@ import { Label } from '@/lib/Label'; import { RowWithTextField } from '@/lib/Layout'; import { ProgressCircle } from '@/lib/ProgressCircle'; import { TextField } from '@/lib/TextField'; +import { Callout } from '@tremor/react'; import { useRouter } from 'next/navigation'; import { useCallback, useEffect, useMemo, useState } from 'react'; import { ToastContainer, toast } from 'react-toastify'; @@ -18,6 +19,7 @@ import 'react-toastify/dist/ReactToastify.css'; import TableMapping from '../../create/cdc/tablemapping'; import { reformattedTableMapping } from '../../create/handlers'; import { blankCDCSetting } from '../../create/helpers/common'; +import * as styles from '../../create/styles'; type EditMirrorProps = { params: { mirrorId: string }; }; @@ -120,8 +122,6 @@ const EditMirror = ({ params: { mirrorId } }: EditMirrorProps) => { return (
- - {'Pull Batch Size'} } @@ -174,6 +174,22 @@ const EditMirror = ({ params: { mirrorId } }: EditMirrorProps) => { } /> + + {!isNotPaused && rows.some((row) => row.selected) && ( + + CDC will be put on hold until initial load for these added tables have + been completed. +

+ The replication slot will grow during this period. + + )} + { omitAdditionalTablesMapping={omitAdditionalTablesMapping} /> - {isNotPaused ? ( - - ) : ( - + {isNotPaused && ( + + Mirror can only be edited while paused. + )} -
+ +
+ -
diff --git a/ui/app/mirrors/create/cdc/cdc.tsx b/ui/app/mirrors/create/cdc/cdc.tsx index 25c15e58f4..cb9ea4a381 100644 --- a/ui/app/mirrors/create/cdc/cdc.tsx +++ b/ui/app/mirrors/create/cdc/cdc.tsx @@ -2,8 +2,9 @@ import { DBType } from '@/grpc_generated/peers'; import { Button } from '@/lib/Button'; import { Icon } from '@/lib/Icon'; -import { Dispatch, SetStateAction, useMemo, useState } from 'react'; +import { Dispatch, SetStateAction, useEffect, useMemo, useState } from 'react'; import { CDCConfig, MirrorSetter, TableMapRow } from '../../../dto/MirrorsDTO'; +import { fetchPublications } from '../handlers'; import { MirrorSetting } from '../helpers/common'; import CDCField from './fields'; import TableMapping from './tablemapping'; @@ -36,6 +37,7 @@ export default function CDCConfigForm({ rows, setRows, }: MirrorConfigProps) { + const [publications, setPublications] = useState(); const [show, setShow] = useState(false); const handleChange = (val: string | boolean, setting: MirrorSetting) => { let stateVal: string | boolean = val; @@ -64,7 +66,26 @@ export default function CDCConfigForm({ return true; }; - if (mirrorConfig.source != undefined && mirrorConfig.destination != undefined) + const optionsForField = (setting: MirrorSetting) => { + switch (setting.label) { + case 'Publication Name': + return publications; + default: + return []; + } + }; + + useEffect(() => { + fetchPublications(mirrorConfig.source?.name || '').then((pubs) => { + setPublications(pubs); + }); + }, [mirrorConfig.source?.name]); + + if ( + mirrorConfig.source != undefined && + mirrorConfig.destination != undefined && + publications != undefined + ) return ( <> {normalSettings.map((setting, id) => { @@ -74,6 +95,7 @@ export default function CDCConfigForm({ key={id} handleChange={handleChange} setting={setting} + options={optionsForField(setting)} /> ) ); @@ -101,9 +123,10 @@ export default function CDCConfigForm({ advancedSettings.map((setting, id) => { return ( ); })} diff --git a/ui/app/mirrors/create/cdc/fields.tsx b/ui/app/mirrors/create/cdc/fields.tsx index acbb47764b..8bf802ffa1 100644 --- a/ui/app/mirrors/create/cdc/fields.tsx +++ b/ui/app/mirrors/create/cdc/fields.tsx @@ -1,21 +1,29 @@ 'use client'; +import SelectTheme from '@/app/styles/select'; import { RequiredIndicator } from '@/components/RequiredIndicator'; import { Label } from '@/lib/Label'; -import { RowWithSwitch, RowWithTextField } from '@/lib/Layout'; +import { RowWithSelect, RowWithSwitch, RowWithTextField } from '@/lib/Layout'; import { Switch } from '@/lib/Switch'; import { TextField } from '@/lib/TextField'; +import ReactSelect from 'react-select'; import { InfoPopover } from '../../../../components/InfoPopover'; import { MirrorSetting } from '../helpers/common'; interface FieldProps { setting: MirrorSetting; handleChange: (val: string | boolean, setting: MirrorSetting) => void; + options?: string[]; } -const CDCField = ({ setting, handleChange }: FieldProps) => { +const CDCField = ({ setting, handleChange, options }: FieldProps) => { return setting.type === 'switch' ? ( {setting.label}} + label={ + + } action={
{ onCheckedChange={(state: boolean) => handleChange(state, setting)} /> {setting.tips && ( - + + )} +
+ } + /> + ) : setting.type === 'select' ? ( + + {setting.label} + {RequiredIndicator(setting.required)} + + } + action={ +
+
+ + val && handleChange(val.option, setting) + } + options={options?.map((option) => ({ option, label: option }))} + getOptionLabel={(option) => option.label} + getOptionValue={(option) => option.option} + theme={SelectTheme} + /> +
+ {setting.tips && ( + )}
} @@ -59,7 +109,11 @@ const CDCField = ({ setting, handleChange }: FieldProps) => { } /> {setting.tips && ( - + )}
} diff --git a/ui/app/mirrors/create/cdc/guide.tsx b/ui/app/mirrors/create/cdc/guide.tsx index 041632e378..b27204b991 100644 --- a/ui/app/mirrors/create/cdc/guide.tsx +++ b/ui/app/mirrors/create/cdc/guide.tsx @@ -1,10 +1,11 @@ +import TitleCase from '@/app/utils/titlecase'; import { Label } from '@/lib/Label'; import Link from 'next/link'; const GuideForDestinationSetup = ({ - dstPeerType: peerType, + createPeerType: peerType, }: { - dstPeerType: string; + createPeerType: string; }) => { const linkForDst = () => { switch (peerType) { @@ -12,18 +13,25 @@ const GuideForDestinationSetup = ({ return 'https://docs.peerdb.io/connect/snowflake'; case 'BIGQUERY': return 'https://docs.peerdb.io/connect/bigquery'; + case 'RDS POSTGRESQL': + case 'POSTGRESQL': + return 'https://docs.peerdb.io/connect/rds_postgres'; + case 'AZURE FLEXIBLE POSTGRESQL': + return 'https://docs.peerdb.io/connect/azure_flexible_server_postgres'; + case 'GOOGLE CLOUD POSTGRESQL': + return 'https://docs.peerdb.io/connect/cloudsql_postgres'; default: - return 'https://docs.peerdb.io/'; + return ''; } }; - if (peerType != 'SNOWFLAKE' && peerType != 'BIGQUERY') { + if (linkForDst() == '') { return <>; } return ( diff --git a/ui/app/mirrors/create/cdc/schemabox.tsx b/ui/app/mirrors/create/cdc/schemabox.tsx index 0c1d7e9a19..4ec403f8ee 100644 --- a/ui/app/mirrors/create/cdc/schemabox.tsx +++ b/ui/app/mirrors/create/cdc/schemabox.tsx @@ -237,6 +237,13 @@ const SchemaBox = ({ > {row.source} + } action={ diff --git a/ui/app/mirrors/create/handlers.ts b/ui/app/mirrors/create/handlers.ts index beeb2aaf5d..282fe70a34 100644 --- a/ui/app/mirrors/create/handlers.ts +++ b/ui/app/mirrors/create/handlers.ts @@ -1,6 +1,7 @@ import { UCreateMirrorResponse } from '@/app/dto/MirrorsDTO'; import { UColumnsResponse, + UPublicationsResponse, USchemasResponse, UTablesAllResponse, UTablesResponse, @@ -298,6 +299,7 @@ export const fetchTables = async ( exclude: new Set(), selected: false, canMirror: tableObject.canMirror, + tableSize: tableObject.tableSize, }); } } @@ -367,3 +369,17 @@ export const handleValidateCDC = async ( notify('CDC Mirror is valid', true); setLoading(false); }; + +export const fetchPublications = async (peerName: string) => { + const publicationsRes: UPublicationsResponse = await fetch( + '/api/peers/publications', + { + method: 'POST', + body: JSON.stringify({ + peerName, + }), + cache: 'no-store', + } + ).then((res) => res.json()); + return publicationsRes.publicationNames; +}; diff --git a/ui/app/mirrors/create/helpers/cdc.ts b/ui/app/mirrors/create/helpers/cdc.ts index 10ab07a951..0d70e3fac1 100644 --- a/ui/app/mirrors/create/helpers/cdc.ts +++ b/ui/app/mirrors/create/helpers/cdc.ts @@ -11,6 +11,7 @@ export const cdcSettings: MirrorSetting[] = [ tips: 'Specify if you want initial load to happen for your tables.', type: 'switch', default: true, + required: true, }, { label: 'Pull Batch Size', @@ -35,6 +36,7 @@ export const cdcSettings: MirrorSetting[] = [ helpfulLink: 'https://docs.peerdb.io/metrics/important_cdc_configs', type: 'number', default: '60', + required: true, }, { label: 'Publication Name', @@ -43,7 +45,11 @@ export const cdcSettings: MirrorSetting[] = [ ...curr, publicationName: (value as string) || '', })), - tips: 'If set, PeerDB will use this publication for the mirror.', + type: 'select', + tips: 'PeerDB requires a publication associated with the tables you wish to sync.', + helpfulLink: + 'https://www.postgresql.org/docs/current/sql-createpublication.html', + command: 'CREATE PUBLICATION FOR ALL TABLES;', }, { label: 'Replication Slot Name', @@ -59,33 +65,36 @@ export const cdcSettings: MirrorSetting[] = [ stateHandler: (value, setter) => setter((curr: CDCConfig) => ({ ...curr, - snapshotNumRowsPerPartition: parseInt(value as string, 10) || 500000, + snapshotNumRowsPerPartition: parseInt(value as string, 10) || 1000000, })), - tips: 'PeerDB splits up table data into partitions for increased performance. This setting controls the number of rows per partition. The default value is 500000.', - default: '500000', + tips: 'PeerDB splits up table data into partitions for increased performance. This setting controls the number of rows per partition. The default value is 1000000.', + default: '1000000', type: 'number', + advanced: true, }, { - label: 'Snapshot Maximum Parallel Workers', + label: 'Parallelism for Initial Load', stateHandler: (value, setter) => setter((curr: CDCConfig) => ({ ...curr, - snapshotMaxParallelWorkers: parseInt(value as string, 10) || 1, + snapshotMaxParallelWorkers: parseInt(value as string, 10) || 4, })), - tips: 'PeerDB spins up parallel threads for each partition. This setting controls the number of partitions to sync in parallel. The default value is 1.', - default: '1', + tips: 'PeerDB spins up parallel threads for each partition in initial load. This setting controls the number of partitions to sync in parallel. The default value is 4.', + default: '4', type: 'number', + required: true, }, { label: 'Snapshot Number of Tables In Parallel', stateHandler: (value, setter) => setter((curr: CDCConfig) => ({ ...curr, - snapshotNumTablesInParallel: parseInt(value as string, 10) || 4, + snapshotNumTablesInParallel: parseInt(value as string, 10) || 1, })), - tips: 'Specify the number of tables to sync perform initial load for, in parallel. The default value is 4.', - default: '4', + tips: 'Specify the number of tables to sync perform initial load for, in parallel. The default value is 1.', + default: '1', type: 'number', + advanced: true, }, { label: 'Snapshot Staging Path', @@ -95,6 +104,7 @@ export const cdcSettings: MirrorSetting[] = [ snapshotStagingPath: value as string | '', })), tips: 'You can specify staging path for Snapshot sync mode AVRO. For Snowflake as destination peer, this must be either empty or an S3 bucket URL. For BigQuery, this must be either empty or an existing GCS bucket name. In both cases, if empty, the local filesystem will be used.', + advanced: true, }, { label: 'CDC Staging Path', @@ -104,24 +114,26 @@ export const cdcSettings: MirrorSetting[] = [ cdcStagingPath: (value as string) || '', })), tips: 'You can specify staging path for CDC sync mode AVRO. For Snowflake as destination peer, this must be either empty or an S3 bucket URL. For BigQuery, this must be either empty or an existing GCS bucket name. In both cases, if empty, the local filesystem will be used.', + advanced: true, }, { label: 'Soft Delete', stateHandler: (value, setter) => setter((curr: CDCConfig) => ({ ...curr, - softDelete: (value as boolean) || false, + softDelete: (value as boolean) ?? true, })), tips: 'Allows you to mark some records as deleted without actual erasure from the database', - default: false, + default: true, type: 'switch', + required: true, }, { label: 'Initial Copy Only', stateHandler: (value, setter) => setter((curr: CDCConfig) => ({ ...curr, - initialSnapshotOnly: (value as boolean) || false, + initialSnapshotOnly: (value as boolean) ?? false, })), tips: 'If set, PeerDB will only perform initial load and will not perform CDC sync.', type: 'switch', diff --git a/ui/app/mirrors/create/helpers/common.ts b/ui/app/mirrors/create/helpers/common.ts index 5d1b8d9a93..34233f24fc 100644 --- a/ui/app/mirrors/create/helpers/common.ts +++ b/ui/app/mirrors/create/helpers/common.ts @@ -13,6 +13,7 @@ export interface MirrorSetting { helpfulLink?: string; default?: string | number | boolean; advanced?: boolean; // whether it should come under an 'Advanced' section + command?: string; } export const blankCDCSetting: FlowConnectionConfigs = { @@ -23,12 +24,12 @@ export const blankCDCSetting: FlowConnectionConfigs = { maxBatchSize: 1000000, doInitialSnapshot: true, publicationName: '', - snapshotNumRowsPerPartition: 500000, - snapshotMaxParallelWorkers: 1, - snapshotNumTablesInParallel: 4, + snapshotNumRowsPerPartition: 1000000, + snapshotMaxParallelWorkers: 4, + snapshotNumTablesInParallel: 1, snapshotStagingPath: '', cdcStagingPath: '', - softDelete: false, + softDelete: true, replicationSlotName: '', resync: false, softDeleteColName: '', diff --git a/ui/app/mirrors/create/schema.ts b/ui/app/mirrors/create/schema.ts index 3ca4408315..7b615e7c34 100644 --- a/ui/app/mirrors/create/schema.ts +++ b/ui/app/mirrors/create/schema.ts @@ -63,9 +63,9 @@ export const cdcSchema = z.object({ .optional(), replicationSlotName: z .string({ - invalid_type_error: 'Publication name must be a string', + invalid_type_error: 'Replication slot name must be a string', }) - .max(255, 'Publication name must be less than 255 characters') + .max(255, 'Replication slot name must be less than 255 characters') .optional(), snapshotNumRowsPerPartition: z .number({ @@ -76,10 +76,10 @@ export const cdcSchema = z.object({ .optional(), snapshotMaxParallelWorkers: z .number({ - invalid_type_error: 'Snapshot max workers must be a number', + invalid_type_error: 'Initial load parallelism must be a number', }) .int() - .min(1, 'Snapshot max workers must be a positive integer') + .min(1, 'Initial load parallelism must be a positive integer') .optional(), snapshotNumTablesInParallel: z .number({ diff --git a/ui/app/peers/create/[peerType]/handlers.ts b/ui/app/peers/create/[peerType]/handlers.ts index e43164aa96..423b996aec 100644 --- a/ui/app/peers/create/[peerType]/handlers.ts +++ b/ui/app/peers/create/[peerType]/handlers.ts @@ -17,20 +17,20 @@ import { const validateFields = ( type: string, config: PeerConfig, - setMessage: Dispatch>, + notify: (msg: string) => void, name?: string ): boolean => { const peerNameValid = peerNameSchema.safeParse(name); if (!peerNameValid.success) { const peerNameErr = peerNameValid.error.issues[0].message; - setMessage({ ok: false, msg: peerNameErr }); + notify(peerNameErr); return false; } if (type === 'S3') { const s3Valid = S3Validation(config as S3Config); if (s3Valid.length > 0) { - setMessage({ ok: false, msg: s3Valid }); + notify(s3Valid); return false; } } @@ -61,9 +61,9 @@ const validateFields = ( validationErr = 'Unsupported peer type ' + type; } if (validationErr) { - setMessage({ ok: false, msg: validationErr }); + notify(validationErr); return false; - } else setMessage({ ok: true, msg: '' }); + } return true; }; @@ -71,11 +71,11 @@ const validateFields = ( export const handleValidate = async ( type: string, config: PeerConfig, - setMessage: Dispatch>, + notify: (msg: string, success?: boolean) => void, setLoading: Dispatch>, name?: string ) => { - const isValid = validateFields(type, config, setMessage, name); + const isValid = validateFields(type, config, notify, name); if (!isValid) return; setLoading(true); const valid: UValidatePeerResponse = await fetch('/api/peers/', { @@ -89,11 +89,11 @@ export const handleValidate = async ( cache: 'no-store', }).then((res) => res.json()); if (!valid.valid) { - setMessage({ ok: false, msg: valid.message }); + notify(valid.message); setLoading(false); return; } - setMessage({ ok: true, msg: 'Peer is valid' }); + notify('Peer is valid', true); setLoading(false); }; @@ -108,12 +108,12 @@ const S3Validation = (config: S3Config): string => { export const handleCreate = async ( type: string, config: PeerConfig, - setMessage: Dispatch>, + notify: (msg: string) => void, setLoading: Dispatch>, route: RouteCallback, name?: string ) => { - let isValid = validateFields(type, config, setMessage, name); + let isValid = validateFields(type, config, notify, name); if (!isValid) return; setLoading(true); const createdPeer: UCreatePeerResponse = await fetch('/api/peers/', { @@ -127,11 +127,11 @@ export const handleCreate = async ( cache: 'no-store', }).then((res) => res.json()); if (!createdPeer.created) { - setMessage({ ok: false, msg: createdPeer.message }); + notify(createdPeer.message); setLoading(false); return; } - setMessage({ ok: true, msg: 'Peer created successfully' }); + route(); setLoading(false); }; diff --git a/ui/app/peers/create/[peerType]/page.tsx b/ui/app/peers/create/[peerType]/page.tsx index b0693075b8..4aa958756a 100644 --- a/ui/app/peers/create/[peerType]/page.tsx +++ b/ui/app/peers/create/[peerType]/page.tsx @@ -7,6 +7,7 @@ import PostgresForm from '@/components/PeerForms/PostgresForm'; import S3Form from '@/components/PeerForms/S3Form'; import SnowflakeForm from '@/components/PeerForms/SnowflakeForm'; +import TitleCase from '@/app/utils/titlecase'; import { Button } from '@/lib/Button'; import { ButtonGroup } from '@/lib/ButtonGroup'; import { Label } from '@/lib/Label'; @@ -17,6 +18,8 @@ import { Tooltip } from '@/lib/Tooltip'; import Link from 'next/link'; import { useRouter } from 'next/navigation'; import { useState } from 'react'; +import { ToastContainer, toast } from 'react-toastify'; +import 'react-toastify/dist/ReactToastify.css'; import { handleCreate, handleValidate } from './handlers'; import { clickhouseSetting } from './helpers/ch'; import { getBlankSetting } from './helpers/common'; @@ -27,23 +30,47 @@ type CreateConfigProps = { params: { peerType: string }; }; +const notify = (msg: string, success?: boolean) => { + if (success) { + toast.success(msg, { + position: 'bottom-center', + autoClose: 1000, + }); + } else { + toast.error(msg, { + position: 'bottom-center', + }); + } +}; + export default function CreateConfig({ params: { peerType }, }: CreateConfigProps) { const router = useRouter(); - const dbType = peerType; - const blankSetting = getBlankSetting(dbType); + const blankSetting = getBlankSetting(peerType); const [name, setName] = useState(''); const [config, setConfig] = useState(blankSetting); - const [formMessage, setFormMessage] = useState<{ ok: boolean; msg: string }>({ - ok: true, - msg: '', - }); const [loading, setLoading] = useState(false); - const configComponentMap = (dbType: string) => { - switch (dbType) { - case 'POSTGRES': - return ; + const peerLabel = peerType.toUpperCase().replaceAll('%20', ' '); + const getDBType = () => { + if (peerType.includes('POSTGRESQL')) { + return 'POSTGRES'; + } + return peerType; + }; + + const configComponentMap = (peerType: string) => { + if (peerType.includes('POSTGRESQL')) { + return ( + + ); + } + + switch (peerType) { case 'SNOWFLAKE': return ; case 'BIGQUERY': @@ -76,11 +103,10 @@ export default function CreateConfig({ > - + Configuration - {configComponentMap(dbType)} + {configComponentMap(peerType)} } diff --git a/ui/components/SelectSource.tsx b/ui/components/SelectSource.tsx index 4aefe67082..e77d1b5e4d 100644 --- a/ui/components/SelectSource.tsx +++ b/ui/components/SelectSource.tsx @@ -1,5 +1,6 @@ 'use client'; import SelectTheme from '@/app/styles/select'; +import TitleCase from '@/app/utils/titlecase'; import { DBType } from '@/grpc_generated/peers'; import Image from 'next/image'; import { Dispatch, SetStateAction } from 'react'; @@ -11,12 +12,12 @@ interface SelectSourceProps { setPeerType: Dispatch>; } -function SourceLabel({ value }: { value: string }) { - const peerLogo = DBTypeToImageMapping(value); +function SourceLabel({ value, label }: { value: string; label: string }) { + const peerLogo = DBTypeToImageMapping(label); return (
peer -
{value}
+
{TitleCase(label)}
); } @@ -29,7 +30,7 @@ export default function SelectSource({ .filter( (value): value is string => typeof value === 'string' && - (value === 'POSTGRES' || + (value === 'POSTGRESQL' || value === 'SNOWFLAKE' || value === 'BIGQUERY' || value === 'S3' || @@ -37,13 +38,19 @@ export default function SelectSource({ ) .map((value) => ({ label: value, value })); + dbTypes.push( + { value: 'POSTGRESQL', label: 'POSTGRESQL' }, + { value: 'POSTGRESQL', label: 'RDS POSTGRESQL' }, + { value: 'POSTGRESQL', label: 'GOOGLE CLOUD POSTGRESQL' }, + { value: 'POSTGRESQL', label: 'AZURE FLEXIBLE POSTGRESQL' } + ); return ( opt.value === peerType)} - onChange={(val, _) => val && setPeerType(val.value)} + defaultValue={dbTypes.find((opt) => opt.label === peerType)} + onChange={(val, _) => val && setPeerType(val.label)} formatOptionLabel={SourceLabel} theme={SelectTheme} /> diff --git a/ui/lib/Button/Button.styles.ts b/ui/lib/Button/Button.styles.ts index 8336f45346..d74e612afb 100644 --- a/ui/lib/Button/Button.styles.ts +++ b/ui/lib/Button/Button.styles.ts @@ -97,6 +97,15 @@ const peerSolidStyle = css` --background-color-focus: ${({ theme }) => theme.colors.accent.fill.normal}; `; +const blueSolidStyle = css` + --focus-border-color: ${({ theme }) => theme.colors.accent.border.normal}; + --text-color: ${({ theme }) => theme.colors.special.fixed.white}; + --background-color-default: ${({ theme, $loading }) => + $loading ? 'rgba(74, 176, 240,0.8)' : 'rgba(74, 176, 240,0.8)'}; + --background-color-hover: ${({ theme }) => 'rgba(74, 176, 240,1)'}; + --background-color-focus: ${({ theme }) => theme.colors.accent.fill.normal}; +`; + const destructiveSolidStyle = css` --focus-border-color: ${({ theme }) => theme.colors.accent.border.normal}; --text-color: ${({ theme }) => theme.colors.special.fixed.white}; @@ -138,6 +147,7 @@ const variants = { peer: peerSolidStyle, normalBorderless: normalBorderlessStyle, destructiveBorderless: destructiveBorderlessStyle, + blue: blueSolidStyle, } as const; export type ButtonVariant = keyof typeof variants; diff --git a/ui/public/svgs/azurepg.svg b/ui/public/svgs/azurepg.svg new file mode 100644 index 0000000000..811a6c7397 --- /dev/null +++ b/ui/public/svgs/azurepg.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/ui/public/svgs/gcp.svg b/ui/public/svgs/gcp.svg new file mode 100644 index 0000000000..81b7d24547 --- /dev/null +++ b/ui/public/svgs/gcp.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/ui/public/svgs/rds.svg b/ui/public/svgs/rds.svg new file mode 100644 index 0000000000..f384cdad89 --- /dev/null +++ b/ui/public/svgs/rds.svg @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file From 2d3790a6bed763c10670136d4520d6f2e521805f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Tue, 12 Mar 2024 14:11:57 +0000 Subject: [PATCH 2/2] Optimize slot lag graph (#1473) 1. past month gets 8640 points, only send back at most 720 (randomly selected) 2. transmit numbers; formatting happens on client 3. always show HH:mm in timestamp --- ui/app/api/peers/slots/[name]/route.ts | 40 ++++++++---------- ui/app/dto/PeersDTO.ts | 4 +- ui/app/peers/[peerName]/lagGraph.tsx | 56 ++++++++++++++------------ 3 files changed, 49 insertions(+), 51 deletions(-) diff --git a/ui/app/api/peers/slots/[name]/route.ts b/ui/app/api/peers/slots/[name]/route.ts index 7ba1d943f0..78b3c8615f 100644 --- a/ui/app/api/peers/slots/[name]/route.ts +++ b/ui/app/api/peers/slots/[name]/route.ts @@ -30,30 +30,22 @@ export async function GET( break; } - const lagPoints = await prisma.peer_slot_size.findMany({ - select: { - updated_at: true, - slot_size: true, - }, - where: { - slot_name: context.params.name, - updated_at: { - gte: new Date(Date.now() - forThePastThisMuchTime), - }, - }, - }); + const lagPoints = await prisma.$queryRaw< + { updated_at: Date; slot_size: bigint }[] + >` + select updated_at, slot_size + from peerdb_stats.peer_slot_size + where slot_size is not null + and slot_name = ${context.params.name} + and updated_at > ${new Date(Date.now() - forThePastThisMuchTime)} + order by random() + limit 720 + `; - // convert slot_size to string - const stringedLagPoints: SlotLagPoint[] = lagPoints.map((lagPoint) => { - return { - // human readable - updatedAt: - lagPoint.updated_at.toDateString() + - ' ' + - lagPoint.updated_at.toLocaleTimeString(), - slotSize: lagPoint.slot_size?.toString(), - }; - }); + const slotLagPoints: SlotLagPoint[] = lagPoints.map((lagPoint) => ({ + updatedAt: +lagPoint.updated_at, + slotSize: Number(lagPoint.slot_size) / 1000, + })); - return NextResponse.json(stringedLagPoints); + return NextResponse.json(slotLagPoints); } diff --git a/ui/app/dto/PeersDTO.ts b/ui/app/dto/PeersDTO.ts index 23b0243b4f..bbf5fbccde 100644 --- a/ui/app/dto/PeersDTO.ts +++ b/ui/app/dto/PeersDTO.ts @@ -53,8 +53,8 @@ export type CatalogPeer = { export type PeerSetter = React.Dispatch>; export type SlotLagPoint = { - updatedAt: string; - slotSize?: string; + updatedAt: number; + slotSize: number; }; export type UPublicationsResponse = { diff --git a/ui/app/peers/[peerName]/lagGraph.tsx b/ui/app/peers/[peerName]/lagGraph.tsx index b495aaf6f3..12b0c2b49b 100644 --- a/ui/app/peers/[peerName]/lagGraph.tsx +++ b/ui/app/peers/[peerName]/lagGraph.tsx @@ -5,19 +5,25 @@ import { formatGraphLabel, timeOptions } from '@/app/utils/graph'; import { Label } from '@/lib/Label'; import { ProgressCircle } from '@/lib/ProgressCircle/ProgressCircle'; import { LineChart } from '@tremor/react'; -import { useCallback, useEffect, useMemo, useState } from 'react'; +import { useCallback, useEffect, useState } from 'react'; import ReactSelect from 'react-select'; import { useLocalStorage } from 'usehooks-ts'; function LagGraph({ slotNames }: { slotNames: string[] }) { - const [lagPoints, setLagPoints] = useState([]); + const [mounted, setMounted] = useState(false); + const [lagPoints, setLagPoints] = useState< + { time: string; 'Lag in GB': number }[] + >([]); const [defaultSlot, setDefaultSlot] = useLocalStorage('defaultSlot', ''); const [selectedSlot, setSelectedSlot] = useState(defaultSlot); + const [loading, setLoading] = useState(false); let [timeSince, setTimeSince] = useState('hour'); const fetchLagPoints = useCallback(async () => { if (selectedSlot == '') { return; } + + setLoading(true); const pointsRes = await fetch( `/api/peers/slots/${selectedSlot}?timeSince=${timeSince}`, { @@ -25,7 +31,15 @@ function LagGraph({ slotNames }: { slotNames: string[] }) { } ); const points: SlotLagPoint[] = await pointsRes.json(); - setLagPoints(points); + setLagPoints( + points + .sort((x, y) => x.updatedAt - y.updatedAt) + .map((data) => ({ + time: formatGraphLabel(new Date(data.updatedAt!), 'hour'), + 'Lag in GB': data.slotSize, + })) + ); + setLoading(false); }, [selectedSlot, timeSince]); const handleChange = (val: string) => { @@ -33,23 +47,8 @@ function LagGraph({ slotNames }: { slotNames: string[] }) { setSelectedSlot(val); }; - const graphValues = useMemo(() => { - let lagDataDot = lagPoints.map((point) => [ - point.updatedAt, - point.slotSize, - ]); - return lagDataDot.map((data) => ({ - time: formatGraphLabel(new Date(data[0]!), timeSince), - 'Lag in GB': parseInt(data[1] || '0', 10) / 1000, - })); - }, [lagPoints, timeSince]); - - const [mounted, setMounted] = useState(false); useEffect(() => { setMounted(true); - }, []); - - useEffect(() => { fetchLagPoints(); }, [fetchLagPoints]); @@ -105,13 +104,20 @@ function LagGraph({ slotNames }: { slotNames: string[] }) { theme={SelectTheme} /> - + {loading ? ( +
+ + +
+ ) : ( + + )} ); }