From cad7eb194a6f4ab0e34d5f5add34d766377c9f06 Mon Sep 17 00:00:00 2001 From: Amogh Bharadwaj Date: Mon, 6 May 2024 23:58:40 +0530 Subject: [PATCH] PeerDB: For queues, move sync interval to advanced, default 10 mins (#1672) For queue destination peers: Moves sync interval field in create cdc mirror UI to advanced settings and defaults to 10 minutes. Functionally tested --- ui/app/mirrors/create/cdc/cdc.tsx | 46 ++++++++++++++++++------- ui/app/mirrors/create/helpers/cdc.ts | 23 +++++++------ ui/app/mirrors/create/helpers/common.ts | 8 ++++- 3 files changed, 52 insertions(+), 25 deletions(-) diff --git a/ui/app/mirrors/create/cdc/cdc.tsx b/ui/app/mirrors/create/cdc/cdc.tsx index b4581b9cf0..2ef76be5be 100644 --- a/ui/app/mirrors/create/cdc/cdc.tsx +++ b/ui/app/mirrors/create/cdc/cdc.tsx @@ -5,7 +5,7 @@ import { Icon } from '@/lib/Icon'; import { Dispatch, SetStateAction, useEffect, useMemo, useState } from 'react'; import { CDCConfig, MirrorSetter, TableMapRow } from '../../../dto/MirrorsDTO'; import { IsQueuePeer, fetchPublications } from '../handlers'; -import { MirrorSetting } from '../helpers/common'; +import { AdvancedSettingType, MirrorSetting } from '../helpers/common'; import CDCField from './fields'; import TableMapping from './tablemapping'; @@ -46,12 +46,32 @@ export default function CDCConfigForm({ }; const normalSettings = useMemo(() => { - return settings.filter((setting) => setting.advanced != true); - }, [settings]); + return settings!.filter( + (setting) => + !( + (IsQueuePeer(mirrorConfig.destination?.type) && + setting.advanced === AdvancedSettingType.QUEUE) || + setting.advanced === AdvancedSettingType.ALL + ) + ); + }, [settings, mirrorConfig.destination?.type]); const advancedSettings = useMemo(() => { - return settings.filter((setting) => setting.advanced == true); - }, [settings]); + return settings! + .map((setting) => { + if ( + IsQueuePeer(mirrorConfig.destination?.type) && + setting.advanced === AdvancedSettingType.QUEUE + ) { + setting.stateHandler(600, setter); + return { ...setting, default: 600 }; + } + if (setting.advanced === AdvancedSettingType.ALL) { + return setting; + } + }) + .filter((setting) => setting !== undefined); + }, [settings, mirrorConfig.destination?.type, setter]); const paramDisplayCondition = (setting: MirrorSetting) => { const label = setting.label.toLowerCase(); @@ -91,15 +111,15 @@ export default function CDCConfigForm({ if (mirrorConfig.source != undefined && mirrorConfig.destination != undefined) return ( <> - {normalSettings.map((setting, id) => { + {normalSettings!.map((setting, id) => { return ( - paramDisplayCondition(setting) && ( + paramDisplayCondition(setting!) && ( {show && - advancedSettings.map((setting, id) => { + advancedSettings!.map((setting, id) => { return ( - paramDisplayCondition(setting) && ( + paramDisplayCondition(setting!) && ( ) ); diff --git a/ui/app/mirrors/create/helpers/cdc.ts b/ui/app/mirrors/create/helpers/cdc.ts index 1d190f78a7..a480840047 100644 --- a/ui/app/mirrors/create/helpers/cdc.ts +++ b/ui/app/mirrors/create/helpers/cdc.ts @@ -1,6 +1,6 @@ import { TypeSystem } from '@/grpc_generated/flow'; import { CDCConfig } from '../../../dto/MirrorsDTO'; -import { MirrorSetting } from './common'; +import { AdvancedSettingType, MirrorSetting } from './common'; export const cdcSettings: MirrorSetting[] = [ { label: 'Initial Copy', @@ -24,7 +24,7 @@ export const cdcSettings: MirrorSetting[] = [ tips: 'The number of rows PeerDB will pull from source at a time. If left empty, the default value is 1,000,000 rows.', type: 'number', default: '1000000', - advanced: true, + advanced: AdvancedSettingType.ALL, }, { label: 'Sync Interval (Seconds)', @@ -38,6 +38,7 @@ export const cdcSettings: MirrorSetting[] = [ type: 'number', default: '60', required: true, + advanced: AdvancedSettingType.QUEUE, }, { label: 'Publication Name', @@ -71,7 +72,7 @@ export const cdcSettings: MirrorSetting[] = [ tips: 'PeerDB splits up table data into partitions for increased performance. This setting controls the number of rows per partition. The default value is 1000000.', default: '1000000', type: 'number', - advanced: true, + advanced: AdvancedSettingType.ALL, }, { label: 'Parallelism for Initial Load', @@ -95,7 +96,7 @@ export const cdcSettings: MirrorSetting[] = [ tips: 'Specify the number of tables to sync perform initial load for, in parallel. The default value is 1.', default: '1', type: 'number', - advanced: true, + advanced: AdvancedSettingType.ALL, }, { label: 'Snapshot Staging Path', @@ -105,7 +106,7 @@ export const cdcSettings: MirrorSetting[] = [ snapshotStagingPath: value as string | '', })), tips: 'You can specify staging path for Snapshot sync mode AVRO. For Snowflake as destination peer, this must be either empty or an S3 bucket URL. For BigQuery, this must be either empty or an existing GCS bucket name. In both cases, if empty, the local filesystem will be used.', - advanced: true, + advanced: AdvancedSettingType.ALL, }, { label: 'CDC Staging Path', @@ -115,7 +116,7 @@ export const cdcSettings: MirrorSetting[] = [ cdcStagingPath: (value as string) || '', })), tips: 'You can specify staging path for CDC sync mode AVRO. For Snowflake as destination peer, this must be either empty or an S3 bucket URL. For BigQuery, this must be either empty or an existing GCS bucket name. In both cases, if empty, the local filesystem will be used.', - advanced: true, + advanced: AdvancedSettingType.ALL, }, { label: 'Soft Delete', @@ -138,7 +139,7 @@ export const cdcSettings: MirrorSetting[] = [ })), tips: 'If set, PeerDB will only perform initial load and will not perform CDC sync.', type: 'switch', - advanced: true, + advanced: AdvancedSettingType.ALL, }, { label: 'Script', @@ -148,7 +149,7 @@ export const cdcSettings: MirrorSetting[] = [ script: (value as string) || '', })), tips: 'Associate PeerDB script with this mirror.', - advanced: true, + advanced: AdvancedSettingType.ALL, }, { label: 'Use Postgres type system', @@ -160,7 +161,7 @@ export const cdcSettings: MirrorSetting[] = [ type: 'switch', default: false, tips: 'Decide if PeerDB should use native Postgres types directly', - advanced: true, + advanced: AdvancedSettingType.ALL, }, { label: 'Synced-At Column Name', @@ -170,7 +171,7 @@ export const cdcSettings: MirrorSetting[] = [ syncedAtColName: value as string | '', })), tips: 'A field to set the name of PeerDBs synced_at column. If not set, a default name will be set', - advanced: true, + advanced: AdvancedSettingType.ALL, }, { label: 'Soft Delete Column Name', @@ -180,6 +181,6 @@ export const cdcSettings: MirrorSetting[] = [ softDeleteColName: value as string | '', })), tips: 'A field to set the name of PeerDBs soft delete column.', - advanced: true, + advanced: AdvancedSettingType.ALL, }, ]; diff --git a/ui/app/mirrors/create/helpers/common.ts b/ui/app/mirrors/create/helpers/common.ts index 4237bc2176..c63278a4a9 100644 --- a/ui/app/mirrors/create/helpers/common.ts +++ b/ui/app/mirrors/create/helpers/common.ts @@ -1,5 +1,11 @@ import { FlowConnectionConfigs, TypeSystem } from '@/grpc_generated/flow'; +export enum AdvancedSettingType { + QUEUE = 'queue', + ALL = 'all', + NONE = 'none', +} + export interface MirrorSetting { label: string; stateHandler: (value: any, setter: any) => void; @@ -8,7 +14,7 @@ export interface MirrorSetting { tips?: string; helpfulLink?: string; default?: string | number | boolean; - advanced?: boolean; // whether it should come under an 'Advanced' section + advanced?: AdvancedSettingType; // whether it should come under an 'Advanced' section command?: string; }