Skip to content

Commit

Permalink
add set limit
Browse files Browse the repository at this point in the history
  • Loading branch information
iskakaushik committed Nov 15, 2023
1 parent 04b1969 commit 53d8615
Showing 1 changed file with 1 addition and 4 deletions.
5 changes: 1 addition & 4 deletions flow/connectors/snowflake/snowflake.go
Original file line number Diff line number Diff line change
Expand Up @@ -757,15 +757,12 @@ func (c *SnowflakeConnector) NormalizeRecords(req *model.NormalizeRecordsRequest

var totalRowsAffected int64 = 0
g, gCtx := errgroup.WithContext(c.ctx)
sem := make(chan struct{}, 8) // semaphore to limit parallel merges
g.SetLimit(8) // limit parallel merges to 8

for _, destinationTableName := range destinationTableNames {
sem <- struct{}{} // block if semaphore is full
tableName := destinationTableName // local variable for the closure

g.Go(func() error {
defer func() { <-sem }() // release semaphore

rowsAffected, err := c.generateAndExecuteMergeStatement(
gCtx,
tableName,
Expand Down

0 comments on commit 53d8615

Please sign in to comment.