From 34e408e426d0802f6308103aa9431e1b7a2459a5 Mon Sep 17 00:00:00 2001 From: Amogh-Bharadwaj Date: Mon, 29 Apr 2024 21:55:24 +0530 Subject: [PATCH 1/3] add peerdb columns to ui, fix for bq cdc --- flow/connectors/bigquery/bigquery.go | 7 ++++--- ui/app/mirrors/create/helpers/cdc.ts | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/flow/connectors/bigquery/bigquery.go b/flow/connectors/bigquery/bigquery.go index 7c9d66ab93..40682a5f75 100644 --- a/flow/connectors/bigquery/bigquery.go +++ b/flow/connectors/bigquery/bigquery.go @@ -639,9 +639,10 @@ func (c *BigQueryConnector) SetupNormalizedTable( if softDeleteColName != "" { columns = append(columns, &bigquery.FieldSchema{ - Name: softDeleteColName, - Type: bigquery.BooleanFieldType, - Repeated: false, + Name: softDeleteColName, + Type: bigquery.BooleanFieldType, + Repeated: false, + DefaultValueExpression: "false", }) } diff --git a/ui/app/mirrors/create/helpers/cdc.ts b/ui/app/mirrors/create/helpers/cdc.ts index c18f483f30..1d190f78a7 100644 --- a/ui/app/mirrors/create/helpers/cdc.ts +++ b/ui/app/mirrors/create/helpers/cdc.ts @@ -162,4 +162,24 @@ export const cdcSettings: MirrorSetting[] = [ tips: 'Decide if PeerDB should use native Postgres types directly', advanced: true, }, + { + label: 'Synced-At Column Name', + stateHandler: (value, setter) => + setter((curr: CDCConfig) => ({ + ...curr, + syncedAtColName: value as string | '', + })), + tips: 'A field to set the name of PeerDBs synced_at column. If not set, a default name will be set', + advanced: true, + }, + { + label: 'Soft Delete Column Name', + stateHandler: (value, setter) => + setter((curr: CDCConfig) => ({ + ...curr, + softDeleteColName: value as string | '', + })), + tips: 'A field to set the name of PeerDBs soft delete column.', + advanced: true, + }, ]; From 6d06ea206ef2d02d9fd54d9118b20ad4aebdc763 Mon Sep 17 00:00:00 2001 From: Amogh-Bharadwaj Date: Mon, 29 Apr 2024 22:22:30 +0530 Subject: [PATCH 2/3] gate to bq and sf --- ui/app/mirrors/create/cdc/cdc.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ui/app/mirrors/create/cdc/cdc.tsx b/ui/app/mirrors/create/cdc/cdc.tsx index dc34c1dfd9..b4581b9cf0 100644 --- a/ui/app/mirrors/create/cdc/cdc.tsx +++ b/ui/app/mirrors/create/cdc/cdc.tsx @@ -70,7 +70,10 @@ export default function CDCConfigForm({ label.includes('snapshot'))) || ((mirrorConfig.source?.type !== DBType.POSTGRES || mirrorConfig.destination?.type !== DBType.POSTGRES) && - label.includes('type system')) + label.includes('type system')) || + (mirrorConfig.destination?.type !== DBType.BIGQUERY && + mirrorConfig.destination?.type !== DBType.SNOWFLAKE && + label.includes('column name')) ) { return false; } From 87079dc13f8b0c0985fc299b4f385512a410fe79 Mon Sep 17 00:00:00 2001 From: Amogh-Bharadwaj Date: Tue, 30 Apr 2024 12:49:03 +0530 Subject: [PATCH 3/3] fix test --- flow/e2e/bigquery/peer_flow_bq_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flow/e2e/bigquery/peer_flow_bq_test.go b/flow/e2e/bigquery/peer_flow_bq_test.go index 206f71ea27..da4f036b0a 100644 --- a/flow/e2e/bigquery/peer_flow_bq_test.go +++ b/flow/e2e/bigquery/peer_flow_bq_test.go @@ -1231,7 +1231,7 @@ func (s PeerFlowE2ETestSuiteBQ) Test_Soft_Delete_UD_Same_Batch() { s.bqHelper.Config.DatasetId, dstName) numNewRows, err := s.bqHelper.RunInt64Query(newerSyncedAtQuery) require.NoError(s.t, err) - require.Equal(s.t, int64(0), numNewRows) + require.Equal(s.t, int64(1), numNewRows) } func (s PeerFlowE2ETestSuiteBQ) Test_Soft_Delete_Insert_After_Delete() {