Skip to content

Commit

Permalink
Snowflake: remove ON_ERROR=CONTINUE
Browse files Browse the repository at this point in the history
This was added before truncation was implemented

There are still situations where errors can occur,
such as timestamps in the year 20020,
but we shouldn't be dropping those records
  • Loading branch information
serprex committed Mar 16, 2024
1 parent 78a8cdb commit f2e4f4c
Showing 1 changed file with 2 additions and 7 deletions.
9 changes: 2 additions & 7 deletions flow/connectors/snowflake/qrep_avro_consolidate.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,14 +118,9 @@ func getTransformSQL(colNames []string, colTypes []string, syncedAtCol string) (

// copy to either the actual destination table or a tempTable
func (s *SnowflakeAvroConsolidateHandler) getCopyTransformation(copyDstTable string) string {
copyOpts := []string{
"FILE_FORMAT = (TYPE = AVRO)",
"PURGE = TRUE",
"ON_ERROR = 'CONTINUE'",
}
transformationSQL, columnsSQL := getTransformSQL(s.allColNames, s.allColTypes, s.config.SyncedAtColName)
return fmt.Sprintf("COPY INTO %s(%s) FROM (SELECT %s FROM @%s) %s",
copyDstTable, columnsSQL, transformationSQL, s.stage, strings.Join(copyOpts, ","))
return fmt.Sprintf("COPY INTO %s(%s) FROM (SELECT %s FROM @%s) FILE_FORMAT=(TYPE=AVRO), PURGE=TRUE",
copyDstTable, columnsSQL, transformationSQL, s.stage)
}

func (s *SnowflakeAvroConsolidateHandler) handleAppendMode(ctx context.Context) error {
Expand Down

0 comments on commit f2e4f4c

Please sign in to comment.