Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Flowable: Use errgroup in replicateQRep #1390

Merged
merged 5 commits into from
Feb 28, 2024
Merged

Flowable: Use errgroup in replicateQRep #1390

Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
89bc6e1
qrep flowable errgroup
Amogh-Bharadwaj Feb 27, 2024
e114960
refactor replicateQRepPartition
Amogh-Bharadwaj Feb 28, 2024
496783e
minor change
Amogh-Bharadwaj Feb 28, 2024
82b6eec
more refactoring
Amogh-Bharadwaj Feb 28, 2024
4fb3a7b
minor change
Amogh-Bharadwaj Feb 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 11 additions & 17 deletions flow/activities/flowable.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -637,31 +637,25 @@ func (a *FlowableActivity) replicateQRepPartition(ctx context.Context,

var stream *model.QRecordStream
bufferSize := shared.FetchAndChannelSize
var wg sync.WaitGroup

var goroutineErr error = nil
errGroup, errCtx := errgroup.WithContext(ctx)
Amogh-Bharadwaj marked this conversation as resolved.
Show resolved Hide resolved
if config.SourcePeer.Type == protos.DBType_POSTGRES {
stream = model.NewQRecordStream(bufferSize)
Amogh-Bharadwaj marked this conversation as resolved.
Show resolved Hide resolved
wg.Add(1)

go func() {
errGroup.Go(func() error {
pgConn := srcConn.(*connpostgres.PostgresConnector)
tmp, err := pgConn.PullQRepRecordStream(ctx, config, partition, stream)
tmp, err := pgConn.PullQRepRecordStream(errCtx, config, partition, stream)
numRecords := int64(tmp)
if err != nil {
a.Alerter.LogFlowError(ctx, config.FlowJobName, err)
logger.Error("failed to pull records", slog.Any("error", err))
goroutineErr = err
return fmt.Errorf("failed to pull records: %v", err)
Amogh-Bharadwaj marked this conversation as resolved.
Show resolved Hide resolved
} else {
err = monitoring.UpdatePullEndTimeAndRowsForPartition(ctx,
err = monitoring.UpdatePullEndTimeAndRowsForPartition(errCtx,
a.CatalogPool, runUUID, partition, numRecords)
if err != nil {
logger.Error(err.Error())
goroutineErr = err
}
}
wg.Done()
}()
return nil
})
} else {
recordBatch, err := srcConn.PullQRepRecords(ctx, config, partition)
if err != nil {
Expand Down Expand Up @@ -692,10 +686,10 @@ func (a *FlowableActivity) replicateQRepPartition(ctx context.Context,
logger.Info("no records to push for partition " + partition.PartitionId)
Amogh-Bharadwaj marked this conversation as resolved.
Show resolved Hide resolved
pullCancel()
} else {
wg.Wait()
if goroutineErr != nil {
a.Alerter.LogFlowError(ctx, config.FlowJobName, goroutineErr)
return goroutineErr
err = errGroup.Wait()
if err != nil {
a.Alerter.LogFlowError(ctx, config.FlowJobName, err)
return err
}

err := monitoring.UpdateRowsSyncedForPartition(ctx, a.CatalogPool, rowsSynced, runUUID, partition)
Expand Down
Loading