Skip to content

Commit

Permalink
Wait for at least one record or PKM before returning even after timeout
Browse files Browse the repository at this point in the history
  • Loading branch information
iskakaushik committed Nov 16, 2023
1 parent d0b4f20 commit ce67c82
Show file tree
Hide file tree
Showing 2 changed files with 21 additions and 4 deletions.
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,7 @@ tmp/
private/
nexus/server/tests/assets/*.json
nexus/server/tests/results/actual/

# go workspace
go.work
go.work.sum
21 changes: 17 additions & 4 deletions flow/connectors/postgres/cdc.go
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,7 @@ func (p *PostgresCDCSource) consumeStream(
}()

tablePKeyLastSeen := make(map[model.TableWithPkey]int)
receivedPKM := false

addRecord := func(rec model.Record) {
records.AddRecord(rec)
Expand Down Expand Up @@ -245,8 +246,15 @@ func (p *PostgresCDCSource) consumeStream(
cancel()
if err != nil && !p.commitLock {
if pgconn.Timeout(err) {
log.Infof("Idle timeout reached, returning currently accumulated records - %d", len(localRecords))
return nil
log.Infof("Idle timeout reached")
if receivedPKM || len(localRecords) > 0 {
log.Infof("Returning currently accumulated records - %d", len(localRecords))
return nil
} else {
log.Infof("No records or pkms received, reseting timemout and continuing to wait...")
nextStandbyMessageDeadline = time.Now().Add(standbyMessageTimeout)
continue
}
} else {
return fmt.Errorf("ReceiveMessage failed: %w", err)
}
Expand Down Expand Up @@ -285,8 +293,13 @@ func (p *PostgresCDCSource) consumeStream(
return fmt.Errorf("ParseXLogData failed: %w", err)
}

log.Debugf("XLogData => WALStart %s ServerWALEnd %s ServerTime %s\n",
xld.WALStart, xld.ServerWALEnd, xld.ServerTime)
if !receivedPKM {
receivedPKM = true
log.WithField("flowName", req.FlowJobName).Infof(
"Received first PKM. XLogData => WALStart %s ServerWALEnd %s ServerTime %s\n",
xld.WALStart, xld.ServerWALEnd, xld.ServerTime)
}

rec, err := p.processMessage(records, xld)

if err != nil {
Expand Down

0 comments on commit ce67c82

Please sign in to comment.