Skip to content

Commit

Permalink
fix some error (#1063)
Browse files Browse the repository at this point in the history
  • Loading branch information
iskakaushik authored Jan 11, 2024
1 parent be09741 commit 0c73650
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 5 deletions.
2 changes: 1 addition & 1 deletion flow/connectors/bigquery/merge_stmt_generator.go
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ func (m *mergeStmtGenerator) generateMergeStmt(unchangedToastColumns []string) s
func (m *mergeStmtGenerator) generateMergeStmts() []string {
// TODO (kaushik): This is so that the statement size for individual merge statements
// doesn't exceed the limit. We should make this configurable.
const batchSize = 8
const batchSize = 4
partitions := utils.ArrayChunks(m.unchangedToastColumns, batchSize)

mergeStmts := make([]string, 0, len(partitions))
Expand Down
9 changes: 5 additions & 4 deletions flow/connectors/utils/array.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,14 @@ func ArrayMinus(first []string, second []string) []string {
}

func ArrayChunks[T any](slice []T, size int) [][]T {
var chunks [][]T
var partitions [][]T

for size < len(slice) {
chunks = append(chunks, slice[:size])
slice = slice[size:]
slice, partitions = slice[size:], append(partitions, slice[0:size])
}

// Add the last remaining values
return append(chunks, slice)
partitions = append(partitions, slice)

return partitions
}

0 comments on commit 0c73650

Please sign in to comment.