Snowflake: remove ON_ERROR=CONTINUE (#1494)

This was added before truncation was implemented There are still situations where errors can occur, such as timestamps in the year 20020, but we shouldn't be dropping those records Co-authored-by: Kaushik Iska <[email protected]> Co-authored-by: Amogh Bharadwaj <[email protected]>
PeerDB-io · Mar 19, 2024 · fdff1a7 · fdff1a7
1 parent 9abd6b6
commit fdff1a7
Showing 1 changed file with 2 additions and 7 deletions.
diff --git a/flow/connectors/snowflake/qrep_avro_consolidate.go b/flow/connectors/snowflake/qrep_avro_consolidate.go
@@ -118,14 +118,9 @@ func getTransformSQL(colNames []string, colTypes []string, syncedAtCol string) (
 
 // copy to either the actual destination table or a tempTable
 func (s *SnowflakeAvroConsolidateHandler) getCopyTransformation(copyDstTable string) string {
-	copyOpts := []string{
-		"FILE_FORMAT = (TYPE = AVRO)",
-		"PURGE = TRUE",
-		"ON_ERROR = 'CONTINUE'",
-	}
 	transformationSQL, columnsSQL := getTransformSQL(s.allColNames, s.allColTypes, s.config.SyncedAtColName)
-	return fmt.Sprintf("COPY INTO %s(%s) FROM (SELECT %s FROM @%s) %s",
-		copyDstTable, columnsSQL, transformationSQL, s.stage, strings.Join(copyOpts, ","))
+	return fmt.Sprintf("COPY INTO %s(%s) FROM (SELECT %s FROM @%s) FILE_FORMAT=(TYPE=AVRO), PURGE=TRUE",
+		copyDstTable, columnsSQL, transformationSQL, s.stage)
 }
 
 func (s *SnowflakeAvroConsolidateHandler) handleAppendMode(ctx context.Context) error {