Skip to content

Commit

Permalink
fix xmin ci
Browse files Browse the repository at this point in the history
Also fix poorly named method after marker changed from `txid_current()` to `txid_snapshot_min(txid_current_snapsnot())`

Remove special casing for xmin in qrep code
  • Loading branch information
serprex committed Dec 5, 2023
1 parent 53235aa commit 88b9008
Show file tree
Hide file tree
Showing 5 changed files with 11 additions and 15 deletions.
6 changes: 0 additions & 6 deletions flow/activities/flowable.go
Original file line number Diff line number Diff line change
Expand Up @@ -764,12 +764,6 @@ func (a *FlowableActivity) QRepWaitUntilNewRows(ctx context.Context,
waitBetweenBatches = time.Duration(config.WaitBetweenBatchesSeconds) * time.Second
}

if config.WatermarkColumn == "xmin" {
// for xmin we ignore the wait between batches, as seq scan time is
// extremely slow.
waitBetweenBatches = 10 * time.Second
}

srcConn, err := connectors.GetQRepPullConnector(ctx, config.SourcePeer)
if err != nil {
return fmt.Errorf("failed to get qrep source connector: %w", err)
Expand Down
9 changes: 2 additions & 7 deletions flow/connectors/postgres/qrep.go
Original file line number Diff line number Diff line change
Expand Up @@ -274,11 +274,6 @@ func (c *PostgresConnector) getMinMaxValues(

func (c *PostgresConnector) CheckForUpdatedMaxValue(config *protos.QRepConfig,
last *protos.QRepPartition) (bool, error) {
// for xmin lets always assume there are updates
if config.WatermarkColumn == "xmin" {
return true, nil
}

tx, err := c.pool.Begin(c.ctx)
if err != nil {
return false, fmt.Errorf("unable to begin transaction for getting max value: %w", err)
Expand Down Expand Up @@ -571,9 +566,9 @@ func (c *PostgresConnector) PullXminRecordStream(

var numRecords int
if partition.Range != nil {
numRecords, currentSnapshotXmin, err = executor.ExecuteAndProcessQueryStreamGettingCurrentTxid(stream, query, oldxid)
numRecords, currentSnapshotXmin, err = executor.ExecuteAndProcessQueryStreamGettingCurrentSnapshotXmin(stream, query, oldxid)
} else {
numRecords, currentSnapshotXmin, err = executor.ExecuteAndProcessQueryStreamGettingCurrentTxid(stream, query)
numRecords, currentSnapshotXmin, err = executor.ExecuteAndProcessQueryStreamGettingCurrentSnapshotXmin(stream, query)
}
if err != nil {
return 0, currentSnapshotXmin, err
Expand Down
2 changes: 1 addition & 1 deletion flow/connectors/postgres/qrep_query_executor.go
Original file line number Diff line number Diff line change
Expand Up @@ -325,7 +325,7 @@ func (qe *QRepQueryExecutor) ExecuteAndProcessQueryStream(
return totalRecordsFetched, err
}

func (qe *QRepQueryExecutor) ExecuteAndProcessQueryStreamGettingCurrentTxid(
func (qe *QRepQueryExecutor) ExecuteAndProcessQueryStreamGettingCurrentSnapshotXmin(
stream *model.QRecordStream,
query string,
args ...interface{},
Expand Down
2 changes: 1 addition & 1 deletion flow/e2e/snowflake/qrep_flow_sf_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -208,7 +208,7 @@ func (s *PeerFlowE2ETestSuiteSF) Test_Complete_QRep_Flow_Avro_SF_Upsert_XMIN() {
qrepConfig.WatermarkColumn = "xmin"
s.NoError(err)

e2e.RunQrepFlowWorkflow(env, qrepConfig)
e2e.RunXminFlowWorkflow(env, qrepConfig)

// Verify workflow completes without error
s.True(env.IsWorkflowCompleted())
Expand Down
7 changes: 7 additions & 0 deletions flow/e2e/test_utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ func RegisterWorkflowsAndActivities(env *testsuite.TestWorkflowEnvironment) {
env.RegisterWorkflow(peerflow.SnapshotFlowWorkflow)
env.RegisterWorkflow(peerflow.NormalizeFlowWorkflow)
env.RegisterWorkflow(peerflow.QRepFlowWorkflow)
env.RegisterWorkflow(peerflow.XminFlowWorkflow)
env.RegisterWorkflow(peerflow.QRepPartitionWorkflow)
env.RegisterActivity(&activities.FlowableActivity{})
env.RegisterActivity(&activities.SnapshotActivity{})
Expand Down Expand Up @@ -303,6 +304,12 @@ func RunQrepFlowWorkflow(env *testsuite.TestWorkflowEnvironment, config *protos.
env.ExecuteWorkflow(peerflow.QRepFlowWorkflow, config, state)
}

func RunXminFlowWorkflow(env *testsuite.TestWorkflowEnvironment, config *protos.QRepConfig) {
state := peerflow.NewQRepFlowState()
time.Sleep(5 * time.Second)
env.ExecuteWorkflow(peerflow.XminFlowWorkflow, config, state)
}

func GetOwnersSchema() *model.QRecordSchema {
return &model.QRecordSchema{
Fields: []*model.QField{
Expand Down

0 comments on commit 88b9008

Please sign in to comment.