From b80d43462126bf2aea23fa9f0966937d2227200e Mon Sep 17 00:00:00 2001 From: Amogh Bharadwaj Date: Fri, 10 May 2024 00:15:23 +0530 Subject: [PATCH] Snowflake merge statement: filter out empty peerdb data (#1708) This prevents NormaliseFlow from erroring when we truncate rows > 16MB for PG to Snowflake CDC mirrors --- flow/connectors/snowflake/snowflake.go | 1 + 1 file changed, 1 insertion(+) diff --git a/flow/connectors/snowflake/snowflake.go b/flow/connectors/snowflake/snowflake.go index 27bb69d07f..a0fcc1754d 100644 --- a/flow/connectors/snowflake/snowflake.go +++ b/flow/connectors/snowflake/snowflake.go @@ -44,6 +44,7 @@ const ( SELECT _PEERDB_UID,_PEERDB_TIMESTAMP,TO_VARIANT(PARSE_JSON(_PEERDB_DATA)) %s,_PEERDB_RECORD_TYPE, _PEERDB_MATCH_DATA,_PEERDB_BATCH_ID,_PEERDB_UNCHANGED_TOAST_COLUMNS FROM _PEERDB_INTERNAL.%s WHERE _PEERDB_BATCH_ID = %d AND + _PEERDB_DATA != '' AND _PEERDB_DESTINATION_TABLE_NAME = ? ), FLATTENED AS (SELECT _PEERDB_UID,_PEERDB_TIMESTAMP,_PEERDB_RECORD_TYPE,_PEERDB_MATCH_DATA,_PEERDB_BATCH_ID, _PEERDB_UNCHANGED_TOAST_COLUMNS,%s