From fdff1a7d0463cbcb8727f12eab17ca065d9b4cdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philip=20Dub=C3=A9?= Date: Tue, 19 Mar 2024 10:54:16 +0000 Subject: [PATCH] Snowflake: remove ON_ERROR=CONTINUE (#1494) This was added before truncation was implemented There are still situations where errors can occur, such as timestamps in the year 20020, but we shouldn't be dropping those records Co-authored-by: Kaushik Iska Co-authored-by: Amogh Bharadwaj --- flow/connectors/snowflake/qrep_avro_consolidate.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/flow/connectors/snowflake/qrep_avro_consolidate.go b/flow/connectors/snowflake/qrep_avro_consolidate.go index 707afd878c..8d39ecf0b8 100644 --- a/flow/connectors/snowflake/qrep_avro_consolidate.go +++ b/flow/connectors/snowflake/qrep_avro_consolidate.go @@ -118,14 +118,9 @@ func getTransformSQL(colNames []string, colTypes []string, syncedAtCol string) ( // copy to either the actual destination table or a tempTable func (s *SnowflakeAvroConsolidateHandler) getCopyTransformation(copyDstTable string) string { - copyOpts := []string{ - "FILE_FORMAT = (TYPE = AVRO)", - "PURGE = TRUE", - "ON_ERROR = 'CONTINUE'", - } transformationSQL, columnsSQL := getTransformSQL(s.allColNames, s.allColTypes, s.config.SyncedAtColName) - return fmt.Sprintf("COPY INTO %s(%s) FROM (SELECT %s FROM @%s) %s", - copyDstTable, columnsSQL, transformationSQL, s.stage, strings.Join(copyOpts, ",")) + return fmt.Sprintf("COPY INTO %s(%s) FROM (SELECT %s FROM @%s) FILE_FORMAT=(TYPE=AVRO), PURGE=TRUE", + copyDstTable, columnsSQL, transformationSQL, s.stage) } func (s *SnowflakeAvroConsolidateHandler) handleAppendMode(ctx context.Context) error {