From c52084981d82b1d7633b6ddbcb6f8aec0209fad4 Mon Sep 17 00:00:00 2001 From: Amogh-Bharadwaj Date: Thu, 7 Mar 2024 20:20:13 +0530 Subject: [PATCH 1/4] remove query box, truncate step --- flow/connectors/postgres/qrep.go | 9 --------- ui/app/mirrors/create/handlers.ts | 1 + ui/app/mirrors/create/qrep/snowflakeQrep.tsx | 15 --------------- 3 files changed, 1 insertion(+), 24 deletions(-) diff --git a/flow/connectors/postgres/qrep.go b/flow/connectors/postgres/qrep.go index df82fb5bf4..041c3ceb6c 100644 --- a/flow/connectors/postgres/qrep.go +++ b/flow/connectors/postgres/qrep.go @@ -497,15 +497,6 @@ func (c *PostgresConnector) SetupQRepMetadataTables(ctx context.Context, config } c.logger.Info("Setup metadata table.") - if config.WriteMode != nil && - config.WriteMode.WriteType == protos.QRepWriteType_QREP_WRITE_MODE_OVERWRITE { - _, err = c.conn.Exec(ctx, - "TRUNCATE TABLE "+config.DestinationTableIdentifier) - if err != nil { - return fmt.Errorf("failed to TRUNCATE table before query replication: %w", err) - } - } - return nil } diff --git a/ui/app/mirrors/create/handlers.ts b/ui/app/mirrors/create/handlers.ts index 450a5ad98d..01c8a11b8f 100644 --- a/ui/app/mirrors/create/handlers.ts +++ b/ui/app/mirrors/create/handlers.ts @@ -205,6 +205,7 @@ export const handleCreateQRep = async ( } if (config.sourcePeer?.snowflakeConfig) { + config.query = 'SELECT * FROM ' + config.watermarkTable; if (config.watermarkTable == '') { notify('Please fill in the source table'); return; diff --git a/ui/app/mirrors/create/qrep/snowflakeQrep.tsx b/ui/app/mirrors/create/qrep/snowflakeQrep.tsx index daf812c2a6..1a83420eb9 100644 --- a/ui/app/mirrors/create/qrep/snowflakeQrep.tsx +++ b/ui/app/mirrors/create/qrep/snowflakeQrep.tsx @@ -13,7 +13,6 @@ import { MirrorSetter } from '../../types'; import { fetchAllTables } from '../handlers'; import { MirrorSetting, blankSnowflakeQRepSetting } from '../helpers/common'; import { snowflakeQRepSettings } from '../helpers/qrep'; -import QRepQuery from './query'; interface SnowflakeQRepProps { mirrorConfig: QRepConfig; @@ -82,20 +81,6 @@ export default function SnowflakeQRepForm({ }, [setter]); return ( <> - - { - setter((curr) => ({ - ...curr, - query: val, - })); - }} - /> {mirrorConfig.sourcePeer?.name ? ( snowflakeQRepSettings.map((setting, id) => { return ( From 42b3ec731f06f34df06f23b3c4deb67fd4ec1237 Mon Sep 17 00:00:00 2001 From: Amogh-Bharadwaj Date: Thu, 7 Mar 2024 20:24:41 +0530 Subject: [PATCH 2/4] change skip condition for truncate --- flow/connectors/postgres/qrep.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/flow/connectors/postgres/qrep.go b/flow/connectors/postgres/qrep.go index 041c3ceb6c..8b2110a481 100644 --- a/flow/connectors/postgres/qrep.go +++ b/flow/connectors/postgres/qrep.go @@ -497,6 +497,16 @@ func (c *PostgresConnector) SetupQRepMetadataTables(ctx context.Context, config } c.logger.Info("Setup metadata table.") + if config.WriteMode != nil && + config.WriteMode.WriteType == protos.QRepWriteType_QREP_WRITE_MODE_OVERWRITE && + config.DestinationPeer.Type != protos.DBType_SNOWFLAKE { + _, err = c.conn.Exec(ctx, + "TRUNCATE TABLE "+config.DestinationTableIdentifier) + if err != nil { + return fmt.Errorf("failed to TRUNCATE table before query replication: %w", err) + } + } + return nil } From 3dd01b304ffe66901bb3342f07918e5ba6aa150f Mon Sep 17 00:00:00 2001 From: Amogh-Bharadwaj Date: Thu, 7 Mar 2024 20:25:12 +0530 Subject: [PATCH 3/4] fix --- flow/connectors/postgres/qrep.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flow/connectors/postgres/qrep.go b/flow/connectors/postgres/qrep.go index 8b2110a481..f918c2ca3b 100644 --- a/flow/connectors/postgres/qrep.go +++ b/flow/connectors/postgres/qrep.go @@ -499,7 +499,7 @@ func (c *PostgresConnector) SetupQRepMetadataTables(ctx context.Context, config if config.WriteMode != nil && config.WriteMode.WriteType == protos.QRepWriteType_QREP_WRITE_MODE_OVERWRITE && - config.DestinationPeer.Type != protos.DBType_SNOWFLAKE { + config.SourcePeer.Type != protos.DBType_SNOWFLAKE { _, err = c.conn.Exec(ctx, "TRUNCATE TABLE "+config.DestinationTableIdentifier) if err != nil { From 131cda94d324680f12092c0f3c1e032f75bfc0c0 Mon Sep 17 00:00:00 2001 From: Amogh-Bharadwaj Date: Thu, 7 Mar 2024 20:38:23 +0530 Subject: [PATCH 4/4] setupwatermark default true --- ui/app/mirrors/create/helpers/common.ts | 2 +- ui/app/mirrors/create/helpers/qrep.ts | 2 +- ui/app/mirrors/create/page.tsx | 11 ----------- 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/ui/app/mirrors/create/helpers/common.ts b/ui/app/mirrors/create/helpers/common.ts index cc005cfc34..04db03cad8 100644 --- a/ui/app/mirrors/create/helpers/common.ts +++ b/ui/app/mirrors/create/helpers/common.ts @@ -71,6 +71,6 @@ export const blankSnowflakeQRepSetting = { } as QRepWriteMode, stagingPath: '', numRowsPerPartition: 100000, - setupWatermarkTableOnDestination: false, + setupWatermarkTableOnDestination: true, initialCopyOnly: true, }; diff --git a/ui/app/mirrors/create/helpers/qrep.ts b/ui/app/mirrors/create/helpers/qrep.ts index ad9e1a0f44..64472d94c0 100644 --- a/ui/app/mirrors/create/helpers/qrep.ts +++ b/ui/app/mirrors/create/helpers/qrep.ts @@ -32,7 +32,7 @@ export const qrepSettings: MirrorSetting[] = [ stateHandler: (value, setter) => setter((curr: QRepConfig) => ({ ...curr, - setupWatermarkTableOnDestination: (value as boolean) || false, + setupWatermarkTableOnDestination: (value as boolean) || true, })), tips: 'Specify if you want to create the watermark table on the destination as-is, can be used for some queries.', type: 'switch', diff --git a/ui/app/mirrors/create/page.tsx b/ui/app/mirrors/create/page.tsx index d1d6e1e287..fbf49286f7 100644 --- a/ui/app/mirrors/create/page.tsx +++ b/ui/app/mirrors/create/page.tsx @@ -82,17 +82,6 @@ export default function CreateMirrors() { .then((res) => { setPeers(res); }); - - if (mirrorType === 'Query Replication' || mirrorType === 'XMIN') { - if (mirrorType === 'XMIN') { - setConfig((curr) => { - return { ...curr, setupWatermarkTableOnDestination: true }; - }); - } else - setConfig((curr) => { - return { ...curr, setupWatermarkTableOnDestination: false }; - }); - } }, [mirrorType]); let listMirrorsPage = () => {