Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Flowable: Use errgroup in replicateQRep #1390

Merged
merged 5 commits into from
Feb 28, 2024
Merged

Flowable: Use errgroup in replicateQRep #1390

Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
89bc6e1
qrep flowable errgroup
Amogh-Bharadwaj Feb 27, 2024
e114960
refactor replicateQRepPartition
Amogh-Bharadwaj Feb 28, 2024
496783e
minor change
Amogh-Bharadwaj Feb 28, 2024
82b6eec
more refactoring
Amogh-Bharadwaj Feb 28, 2024
4fb3a7b
minor change
Amogh-Bharadwaj Feb 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 33 additions & 18 deletions flow/activities/flowable.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -613,9 +613,7 @@ func (a *FlowableActivity) replicateQRepPartition(ctx context.Context,
return fmt.Errorf("failed to update start time for partition: %w", err)
}

pullCtx, pullCancel := context.WithCancel(ctx)
defer pullCancel()
srcConn, err := connectors.GetQRepPullConnector(pullCtx, config.SourcePeer)
srcConn, err := connectors.GetQRepPullConnector(ctx, config.SourcePeer)
if err != nil {
a.Alerter.LogFlowError(ctx, config.FlowJobName, err)
return fmt.Errorf("failed to get qrep source connector: %w", err)
Expand All @@ -638,6 +636,7 @@ func (a *FlowableActivity) replicateQRepPartition(ctx context.Context,
var stream *model.QRecordStream
bufferSize := shared.FetchAndChannelSize
errGroup, errCtx := errgroup.WithContext(ctx)
Amogh-Bharadwaj marked this conversation as resolved.
Show resolved Hide resolved
var rowsSynced int
if config.SourcePeer.Type == protos.DBType_POSTGRES {
stream = model.NewQRecordStream(bufferSize)
Amogh-Bharadwaj marked this conversation as resolved.
Show resolved Hide resolved
errGroup.Go(func() error {
Expand All @@ -656,6 +655,21 @@ func (a *FlowableActivity) replicateQRepPartition(ctx context.Context,
}
return nil
})

errGroup.Go(func() error {
rowsSynced, err = dstConn.SyncQRepRecords(ctx, config, partition, stream)
if err != nil {
a.Alerter.LogFlowError(ctx, config.FlowJobName, err)
return fmt.Errorf("failed to sync records: %w", err)
}
return nil
})

err = errGroup.Wait()
if err != nil {
a.Alerter.LogFlowError(ctx, config.FlowJobName, err)
return err
}
} else {
recordBatch, err := srcConn.PullQRepRecords(ctx, config, partition)
if err != nil {
Expand All @@ -674,30 +688,31 @@ func (a *FlowableActivity) replicateQRepPartition(ctx context.Context,
a.Alerter.LogFlowError(ctx, config.FlowJobName, err)
return fmt.Errorf("failed to convert to qrecord stream: %w", err)
}
}

rowsSynced, err := dstConn.SyncQRepRecords(ctx, config, partition, stream)
if err != nil {
a.Alerter.LogFlowError(ctx, config.FlowJobName, err)
return fmt.Errorf("failed to sync records: %w", err)
}

if rowsSynced == 0 {
logger.Info("no records to push for partition " + partition.PartitionId)
pullCancel()
} else {
err = errGroup.Wait()
rowsSynced, err = dstConn.SyncQRepRecords(ctx, config, partition, stream)
if err != nil {
a.Alerter.LogFlowError(ctx, config.FlowJobName, err)
return err
return fmt.Errorf("failed to sync records: %w", err)
}

if rowsSynced == 0 {
logger.Info("no records to push for partition " + partition.PartitionId)
} else {
err = errGroup.Wait()
serprex marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
a.Alerter.LogFlowError(ctx, config.FlowJobName, err)
return err
}
}
}

logger.Info(fmt.Sprintf("pushed %d records", rowsSynced))

if rowsSynced > 0 {
err := monitoring.UpdateRowsSyncedForPartition(ctx, a.CatalogPool, rowsSynced, runUUID, partition)
if err != nil {
return err
}

logger.Info(fmt.Sprintf("pushed %d records", rowsSynced))
}

err = monitoring.UpdateEndTimeForPartition(ctx, a.CatalogPool, runUUID, partition)
Expand Down
Loading