From e9a53f95a1bf4e39cb96d0da2fbdd0e0eaf25fb0 Mon Sep 17 00:00:00 2001 From: sweexordious Date: Fri, 1 Dec 2023 00:10:56 +0100 Subject: [PATCH] feat: orchestrator not failing and requeing failed nonces --- cmd/blobstream/orchestrator/cmd.go | 2 +- ...rt_orchestrator_after_validator_created.sh | 6 ++- helpers/retrier.go | 4 +- orchestrator/orchestrator.go | 39 +++++++++++++------ 4 files changed, 35 insertions(+), 16 deletions(-) diff --git a/cmd/blobstream/orchestrator/cmd.go b/cmd/blobstream/orchestrator/cmd.go index 83768757..ba1349f3 100644 --- a/cmd/blobstream/orchestrator/cmd.go +++ b/cmd/blobstream/orchestrator/cmd.go @@ -110,7 +110,7 @@ func Start() *cobra.Command { // creating the p2p querier p2pQuerier := p2p.NewQuerier(dht, logger) - retrier := helpers.NewRetrier(logger, 6, time.Minute) + retrier := helpers.NewRetrier(logger, 5, 30*time.Second) defer func() { for _, f := range stopFuncs { diff --git a/e2e/scripts/start_orchestrator_after_validator_created.sh b/e2e/scripts/start_orchestrator_after_validator_created.sh index 8f8fe0bb..5188bdd3 100644 --- a/e2e/scripts/start_orchestrator_after_validator_created.sh +++ b/e2e/scripts/start_orchestrator_after_validator_created.sh @@ -54,7 +54,8 @@ then --grpc.insecure \ --p2p.nickname=key \ --p2p.listen-addr="${P2P_LISTEN}" \ - --evm.passphrase=123 + --evm.passphrase=123 \ + --log.level debug else # to give time for the bootstrappers to be up sleep 5s @@ -66,5 +67,6 @@ else --grpc.insecure \ --p2p.listen-addr="${P2P_LISTEN}" \ --p2p.bootstrappers="${P2P_BOOTSTRAPPERS}" \ - --evm.passphrase=123 + --evm.passphrase=123 \ + --log.level debug fi diff --git a/helpers/retrier.go b/helpers/retrier.go index 3819fa44..9a05e82a 100644 --- a/helpers/retrier.go +++ b/helpers/retrier.go @@ -37,13 +37,13 @@ func (r Retrier) Retry(ctx context.Context, retryMethod func() error) error { case <-ctx.Done(): return ctx.Err() case <-nextTick.C: - r.logger.Info("retrying", "retry_number", i, "retries_left", r.retriesNumber-i) + r.logger.Debug("retrying", "retry_number", i, "retries_left", r.retriesNumber-i) err = retryMethod() if err == nil { r.logger.Info("succeeded", "retries_number", i) return nil } - r.logger.Error("failed attempt", "retry", i, "err", err) + r.logger.Debug("failed attempt", "retry", i, "err", err) } } return err diff --git a/orchestrator/orchestrator.go b/orchestrator/orchestrator.go index 02ac174d..5f150a75 100644 --- a/orchestrator/orchestrator.go +++ b/orchestrator/orchestrator.go @@ -26,6 +26,10 @@ import ( coretypes "github.com/tendermint/tendermint/types" ) +// RequeueWindow the number of nonces that we want to re-enqueue if we can't process them even after retry. +// After this window is elapsed, the nonce is discarded. +const RequeueWindow = 50 + type Orchestrator struct { Logger tmlog.Logger // maybe use a more general interface @@ -69,18 +73,16 @@ func (orch Orchestrator) Start(ctx context.Context) { // used to send a signal when the nonces processor wants to notify the nonces enqueuing services to stop. signalChan := make(chan struct{}) - withCancel, cancel := context.WithCancel(ctx) - wg := &sync.WaitGroup{} // go routine to listen for new attestation nonces wg.Add(1) go func() { defer wg.Done() - err := orch.StartNewEventsListener(withCancel, noncesQueue, signalChan) + err := orch.StartNewEventsListener(ctx, noncesQueue, signalChan) if err != nil { orch.Logger.Error("error listening to new attestations", "err", err) - cancel() + return } orch.Logger.Error("stopping listening to new attestations") }() @@ -89,10 +91,10 @@ func (orch Orchestrator) Start(ctx context.Context) { wg.Add(1) go func() { defer wg.Done() - err := orch.ProcessNonces(withCancel, noncesQueue, signalChan) + err := orch.ProcessNonces(ctx, noncesQueue, signalChan) if err != nil { orch.Logger.Error("error processing attestations", "err", err) - cancel() + return } orch.Logger.Error("stopping processing attestations") }() @@ -101,10 +103,10 @@ func (orch Orchestrator) Start(ctx context.Context) { wg.Add(1) go func() { defer wg.Done() - err := orch.EnqueueMissingEvents(withCancel, noncesQueue, signalChan) + err := orch.EnqueueMissingEvents(ctx, noncesQueue, signalChan) if err != nil { orch.Logger.Error("error enqueuing missing attestations", "err", err) - cancel() + return } }() @@ -237,7 +239,7 @@ func (orch Orchestrator) EnqueueMissingEvents( func (orch Orchestrator) ProcessNonces( ctx context.Context, - noncesQueue <-chan uint64, + noncesQueue chan uint64, signalChan chan<- struct{}, ) error { for { @@ -252,14 +254,29 @@ func (orch Orchestrator) ProcessNonces( if err := orch.Retrier.Retry(ctx, func() error { return orch.Process(ctx, nonce) }); err != nil { - close(signalChan) - return err + orch.Logger.Error("error processing nonce even after retrying", "err", err.Error()) + go orch.MaybeRequeue(ctx, noncesQueue, nonce) } } } } } +// MaybeRequeue requeue the nonce to be re-processed subsequently if it's recent. +func (orch Orchestrator) MaybeRequeue(ctx context.Context, noncesQueue chan<- uint64, nonce uint64) { + latestNonce, err := orch.AppQuerier.QueryLatestAttestationNonce(ctx) + if err != nil { + orch.Logger.Debug("error requeuing nonce", "nonce", nonce, "err", err.Error()) + return + } + if latestNonce <= RequeueWindow || nonce >= latestNonce-RequeueWindow { + orch.Logger.Debug("requeuing nonce", "nonce", nonce) + noncesQueue <- nonce + } else { + orch.Logger.Debug("nonce is too old, will not retry it in the future", "nonce", nonce) + } +} + func (orch Orchestrator) Process(ctx context.Context, nonce uint64) error { att, err := orch.AppQuerier.QueryAttestationByNonce(ctx, nonce) if err != nil {