From 6731e24dbc1009bcdda97302ee02bd9f0a4b5926 Mon Sep 17 00:00:00 2001 From: Amogh-Bharadwaj Date: Mon, 27 Nov 2023 15:00:52 +0530 Subject: [PATCH] remove sync mode fields from create mirror --- ui/app/mirrors/create/cdc/cdc.tsx | 73 ++------------------------- ui/app/mirrors/create/handlers.ts | 16 ++++++ ui/app/mirrors/create/helpers/cdc.ts | 29 +---------- ui/app/mirrors/create/helpers/qrep.ts | 17 +------ ui/app/mirrors/create/qrep/qrep.tsx | 43 +++------------- ui/components/InfoPopover.tsx | 11 ++-- 6 files changed, 37 insertions(+), 152 deletions(-) diff --git a/ui/app/mirrors/create/cdc/cdc.tsx b/ui/app/mirrors/create/cdc/cdc.tsx index 58ae9afd5a..94b786c5c8 100644 --- a/ui/app/mirrors/create/cdc/cdc.tsx +++ b/ui/app/mirrors/create/cdc/cdc.tsx @@ -3,11 +3,10 @@ import { RequiredIndicator } from '@/components/RequiredIndicator'; import { QRepSyncMode } from '@/grpc_generated/flow'; import { DBType } from '@/grpc_generated/peers'; import { Label } from '@/lib/Label'; -import { RowWithSelect, RowWithSwitch, RowWithTextField } from '@/lib/Layout'; +import { RowWithSwitch, RowWithTextField } from '@/lib/Layout'; import { Switch } from '@/lib/Switch'; import { TextField } from '@/lib/TextField'; import { Dispatch, SetStateAction } from 'react'; -import ReactSelect from 'react-select'; import { InfoPopover } from '../../../../components/InfoPopover'; import { CDCConfig, MirrorSetter, TableMapRow } from '../../../dto/MirrorsDTO'; import { MirrorSetting } from '../helpers/common'; @@ -21,17 +20,13 @@ interface MirrorConfigProps { setRows: Dispatch>; } -const SyncModeOptions = ['AVRO', 'Copy with Binary'].map((value) => ({ - label: value, - value, -})); - export const defaultSyncMode = (dtype: DBType | undefined) => { switch (dtype) { case DBType.POSTGRES: return 'Copy with Binary'; case DBType.SNOWFLAKE: case DBType.BIGQUERY: + case DBType.S3: return 'AVRO'; default: return 'Copy with Binary'; @@ -45,32 +40,17 @@ export default function CDCConfigForm({ rows, setRows, }: MirrorConfigProps) { - const setToDefault = (setting: MirrorSetting) => { - const destinationPeerType = mirrorConfig.destination?.type; - return ( - setting.label.includes('Sync') && - (destinationPeerType === DBType.POSTGRES || - destinationPeerType === DBType.SNOWFLAKE) - ); - }; const handleChange = (val: string | boolean, setting: MirrorSetting) => { let stateVal: string | boolean | QRepSyncMode = val; - if (setting.label.includes('Sync Mode')) { - stateVal = - val === 'AVRO' - ? QRepSyncMode.QREP_SYNC_MODE_STORAGE_AVRO - : QRepSyncMode.QREP_SYNC_MODE_MULTI_INSERT; - } setting.stateHandler(stateVal, setter); }; + const paramDisplayCondition = (setting: MirrorSetting) => { const label = setting.label.toLowerCase(); if ( (label.includes('snapshot') && mirrorConfig.doInitialCopy !== true) || - (label.includes('snapshot staging') && - mirrorConfig.snapshotSyncMode?.toString() !== '1') || - (label.includes('cdc staging') && - mirrorConfig.cdcSyncMode?.toString() !== '1') + (label.includes('staging path') && + defaultSyncMode(mirrorConfig.destination?.type) !== 'AVRO') ) { return false; } @@ -115,49 +95,6 @@ export default function CDCConfigForm({ } /> - ) : setting.type === 'select' ? ( - - {setting.label} - {RequiredIndicator(setting.required)} - - } - action={ -
- - val && handleChange(val.value, setting) - } - isDisabled={setToDefault(setting)} - value={ - setToDefault(setting) - ? SyncModeOptions.find( - (opt) => - opt.value === - defaultSyncMode(mirrorConfig.destination?.type) - ) - : undefined - } - options={SyncModeOptions} - /> - {setting.tips && ( - - )} -
- } - /> ) : ( - setter((curr: CDCConfig) => ({ - ...curr, - snapshotSyncMode: - (value as QRepSyncMode) || QRepSyncMode.QREP_SYNC_MODE_MULTI_INSERT, - })), - tips: 'Specify whether you want the sync mode for initial load to be via SQL or by staging AVRO files. The default mode is SQL.', - default: 'SQL', - type: 'select', - }, - { - label: 'CDC Sync Mode', - stateHandler: (value, setter) => - setter((curr: CDCConfig) => ({ - ...curr, - cdcSyncMode: - (value as QRepSyncMode) || QRepSyncMode.QREP_SYNC_MODE_MULTI_INSERT, - })), - tips: 'Specify whether you want the sync mode for CDC to be via SQL or by staging AVRO files. The default mode is SQL.', - default: 'SQL', - type: 'select', - }, { label: 'Snapshot Staging Path', stateHandler: (value, setter) => @@ -94,7 +69,7 @@ export const cdcSettings: MirrorSetting[] = [ ...curr, snapshotStagingPath: value as string | '', })), - tips: 'You can specify staging path if you have set the Snapshot sync mode as AVRO. For Snowflake as destination peer, this must be either empty or an S3 bucket URL.', + tips: 'You can specify staging path for Snapshot sync mode AVRO. For Snowflake as destination peer, this must be either empty or an S3 bucket URL. For BigQuery, this must be either empty or an existing GCS bucket name. In both cases, if empty, the local filesystem will be used.', }, { label: 'CDC Staging Path', @@ -103,7 +78,7 @@ export const cdcSettings: MirrorSetting[] = [ ...curr, cdcStagingPath: (value as string) || '', })), - tips: 'You can specify staging path if you have set the CDC sync mode as AVRO. For Snowflake as destination peer, this must be either empty or an S3 bucket url', + tips: 'You can specify staging path for CDC sync mode AVRO. For Snowflake as destination peer, this must be either empty or an S3 bucket URL. For BigQuery, this must be either empty or an existing GCS bucket name. In both cases, if empty, the local filesystem will be used.', }, { label: 'Soft Delete', diff --git a/ui/app/mirrors/create/helpers/qrep.ts b/ui/app/mirrors/create/helpers/qrep.ts index b0fba435f7..654d5c7ff3 100644 --- a/ui/app/mirrors/create/helpers/qrep.ts +++ b/ui/app/mirrors/create/helpers/qrep.ts @@ -1,6 +1,5 @@ import { QRepConfig, - QRepSyncMode, QRepWriteMode, QRepWriteType, } from '@/grpc_generated/flow'; @@ -71,19 +70,6 @@ export const qrepSettings: MirrorSetting[] = [ default: '4', type: 'number', }, - { - label: 'Sync Mode', - stateHandler: (value, setter) => - setter((curr: QRepConfig) => ({ - ...curr, - syncMode: - (value as QRepSyncMode) || QRepSyncMode.QREP_SYNC_MODE_MULTI_INSERT, - })), - tips: 'Specify whether you want the sync mode to be via SQL or by staging AVRO files. The default mode is SQL.', - default: 'SQL', - type: 'select', - }, - { label: 'Staging Path', stateHandler: (value, setter) => @@ -91,8 +77,7 @@ export const qrepSettings: MirrorSetting[] = [ ...curr, stagingPath: (value as string) || '', })), - tips: `You can specify staging path if you have set the sync mode as AVRO. For Snowflake as destination peer. - If this starts with gs:// then it will be written to GCS. + tips: `You can specify staging path for sync mode AVRO. For Snowflake as destination peer: If this starts with s3:// then it will be written to S3. If nothing is specified then it will be written to local disk.`, }, diff --git a/ui/app/mirrors/create/qrep/qrep.tsx b/ui/app/mirrors/create/qrep/qrep.tsx index 098c940c65..993e7bc1aa 100644 --- a/ui/app/mirrors/create/qrep/qrep.tsx +++ b/ui/app/mirrors/create/qrep/qrep.tsx @@ -51,24 +51,11 @@ export default function QRepConfigForm({ { value: string; label: string }[] >([]); const [loading, setLoading] = useState(false); - const setToDefault = (setting: MirrorSetting) => { - const destinationPeerType = mirrorConfig.destinationPeer?.type; - return ( - setting.label.includes('Sync') && - (destinationPeerType === DBType.POSTGRES || - destinationPeerType === DBType.SNOWFLAKE) - ); - }; const handleChange = (val: string | boolean, setting: MirrorSetting) => { let stateVal: string | boolean | QRepSyncMode | QRepWriteType | string[] = val; - if (setting.label.includes('Sync Mode')) { - stateVal = - val === 'AVRO' - ? QRepSyncMode.QREP_SYNC_MODE_STORAGE_AVRO - : QRepSyncMode.QREP_SYNC_MODE_MULTI_INSERT; - } else if (setting.label.includes('Write Type')) { + if (setting.label.includes('Write Type')) { switch (val) { case 'Upsert': stateVal = QRepWriteType.QREP_WRITE_MODE_UPSERT; @@ -86,6 +73,7 @@ export default function QRepConfigForm({ } setting.stateHandler(stateVal, setter); }; + const paramDisplayCondition = (setting: MirrorSetting) => { const label = setting.label.toLowerCase(); if ( @@ -93,7 +81,7 @@ export default function QRepConfigForm({ mirrorConfig.writeMode?.writeType != QRepWriteType.QREP_WRITE_MODE_UPSERT) || (label.includes('staging') && - mirrorConfig.syncMode?.toString() !== '1') || + defaultSyncMode(mirrorConfig.destinationPeer?.type) !== 'AVRO') || (label.includes('watermark column') && xmin) || (label.includes('initial copy') && xmin) ) { @@ -200,30 +188,13 @@ export default function QRepConfigForm({ }} >
- {setting.label.includes('Sync') || - setting.label.includes('Write') ? ( + {setting.label.includes('Write') ? ( + placeholder='Select a write mode' + onChange={(val) => val && handleChange(val.value, setting) } - isDisabled={setToDefault(setting)} - defaultValue={ - setToDefault(setting) - ? ((mode) => - SyncModes.find((opt) => opt.value === mode) || - WriteModes.find((opt) => opt.value === mode))( - defaultSyncMode( - mirrorConfig.destinationPeer?.type - ) - ) - : undefined - } - options={ - setting.label.includes('Sync') - ? SyncModes - : WriteModes - } + options={WriteModes} /> ) : (
-

- {tips} -

+ {tips.split('.').map((sentence, index) => ( +

+ {sentence.trim()} +

+ ))} {link && (