Skip to content

Commit

Permalink
BigQuery: Skip merge if no primary keys present (#1290)
Browse files Browse the repository at this point in the history
Merge statements currently are syntactically incorrect for tables
without a primary key. This case is ideally not seen as tables without
pkey/replica identity full are filtered out the UI level but in branches
where we rely on signals for adding tables, this case can go unchecked
and normalize flow fails.

This PR skips a table during merge if there are no primary keys for it.
  • Loading branch information
Amogh-Bharadwaj authored Feb 14, 2024
1 parent 293ee58 commit 51f4256
Showing 1 changed file with 6 additions and 2 deletions.
8 changes: 6 additions & 2 deletions flow/connectors/bigquery/bigquery.go
Original file line number Diff line number Diff line change
Expand Up @@ -629,12 +629,16 @@ func (c *BigQueryConnector) NormalizeRecords(req *model.NormalizeRecordsRequest)
if err != nil {
return nil, fmt.Errorf("couldn't get tablename to unchanged cols mapping: %w", err)
}

// append all the statements to one list
c.logger.Info(fmt.Sprintf("merge raw records to corresponding tables: %s %s %v",
c.datasetID, rawTableName, distinctTableNames))

for _, tableName := range distinctTableNames {
normalizeTableSchema := req.TableNameSchemaMapping[tableName]
if len(normalizeTableSchema.PrimaryKeyColumns) == 0 {
c.logger.Warn(fmt.Sprintf("skipping merge for table %s as it has no primary key", tableName))
continue
}
unchangedToastColumns := tableNametoUnchangedToastCols[tableName]
dstDatasetTable, _ := c.convertToDatasetTable(tableName)
mergeGen := &mergeStmtGenerator{
Expand All @@ -645,7 +649,7 @@ func (c *BigQueryConnector) NormalizeRecords(req *model.NormalizeRecordsRequest)
},
dstTableName: tableName,
dstDatasetTable: dstDatasetTable,
normalizedTableSchema: req.TableNameSchemaMapping[tableName],
normalizedTableSchema: normalizeTableSchema,
syncBatchID: req.SyncBatchID,
normalizeBatchID: normBatchID,
peerdbCols: &protos.PeerDBColumns{
Expand Down

0 comments on commit 51f4256

Please sign in to comment.