Skip to content

Commit

Permalink
Minor fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
mateusz-sekara committed Mar 7, 2024
1 parent af33aa8 commit 3a8f100
Show file tree
Hide file tree
Showing 2 changed files with 36 additions and 28 deletions.
4 changes: 2 additions & 2 deletions core/services/ocr2/plugins/ccip/ccipcommit/ocr2_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -492,7 +492,7 @@ func TestCommitReportingPlugin_ShouldAcceptFinalizedReport(t *testing.T) {
assert.NoError(t, err)

chainHealthCheck := ccipcachemocks.NewChainHealthcheck(t)
chainHealthCheck.On("IsHealthy", ctx).Return(nil)
chainHealthCheck.On("IsHealthy", ctx).Return(true, nil)
p.chainHealthcheck = chainHealthCheck

shouldAccept, err := p.ShouldAcceptFinalizedReport(ctx, types.ReportTimestamp{}, encodedReport)
Expand Down Expand Up @@ -531,7 +531,7 @@ func TestCommitReportingPlugin_ShouldTransmitAcceptedReport(t *testing.T) {
p.lggr = logger.TestLogger(t)

chainHealthCheck := ccipcachemocks.NewChainHealthcheck(t)
chainHealthCheck.On("ForceIsHealthy", ctx).Return(nil)
chainHealthCheck.On("ForceIsHealthy", ctx).Return(true, nil).Maybe()
p.chainHealthcheck = chainHealthCheck

t.Run("should transmit when report is not stale", func(t *testing.T) {
Expand Down
60 changes: 34 additions & 26 deletions core/services/ocr2/plugins/ccip/internal/cache/chain_health.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,32 +96,32 @@ func newChainHealthcheckWithCustomEviction(
}
}

func (a chainHealthcheck) IsHealthy(ctx context.Context) (bool, error) {
return a.isHealthy(ctx, false)
func (c *chainHealthcheck) IsHealthy(ctx context.Context) (bool, error) {
return c.isHealthy(ctx, false)
}

func (a chainHealthcheck) ForceIsHealthy(ctx context.Context) (bool, error) {
return a.isHealthy(ctx, true)
func (c *chainHealthcheck) ForceIsHealthy(ctx context.Context) (bool, error) {
return c.isHealthy(ctx, true)
}

func (a chainHealthcheck) isHealthy(ctx context.Context, forceRmnRefresh bool) (bool, error) {
func (c *chainHealthcheck) isHealthy(ctx context.Context, forceRmnRefresh bool) (bool, error) {
// Verify if flag is raised to indicate that the chain is not healthy
// If set to false then immediately return false without checking the chain
if healthy, found := a.cache.Get(a.globalStatusKey); found && !healthy.(bool) {
if healthy, found := c.cache.Get(c.globalStatusKey); found && !healthy.(bool) {
return false, nil
}

if healthy, err := a.checkIfReadersAreHealthy(ctx); err != nil {
if healthy, err := c.checkIfReadersAreHealthy(ctx); err != nil {
return false, err
} else if !healthy {
a.cache.Set(a.globalStatusKey, false, a.globalStatusExpiration)
c.cache.Set(c.globalStatusKey, false, c.globalStatusExpiration)
return healthy, nil
}

if healthy, err := a.checkIfRMNsAreHealthy(ctx, forceRmnRefresh); err != nil {
if healthy, err := c.checkIfRMNsAreHealthy(ctx, forceRmnRefresh); err != nil {
return false, err
} else if !healthy {
a.cache.Set(a.globalStatusKey, false, a.globalStatusExpiration)
c.cache.Set(c.globalStatusKey, false, c.globalStatusExpiration)
return healthy, nil
}
return true, nil
Expand All @@ -130,49 +130,53 @@ func (a chainHealthcheck) isHealthy(ctx context.Context, forceRmnRefresh bool) (
// checkIfReadersAreHealthy checks if the source and destination chains are healthy by calling underlying LogPoller
// These calls are very cheap, because doesn't require any communication with database or RPC, so we don't have
// to cache the result of these calls.
func (a chainHealthcheck) checkIfReadersAreHealthy(ctx context.Context) (bool, error) {
sourceChainHealthy, err := a.onRamp.IsSourceChainHealthy(ctx)
func (c *chainHealthcheck) checkIfReadersAreHealthy(ctx context.Context) (bool, error) {
sourceChainHealthy, err := c.onRamp.IsSourceChainHealthy(ctx)
if err != nil {
return false, errors.Wrap(err, "onRamp IsSourceChainHealthy errored")
}

destChainHealthy, err := a.commitStore.IsDestChainHealthy(ctx)
destChainHealthy, err := c.commitStore.IsDestChainHealthy(ctx)
if err != nil {
return false, errors.Wrap(err, "commitStore IsDestChainHealthy errored")
}

if !sourceChainHealthy || !destChainHealthy {
a.lggr.Errorf("Source or destination chain is unhealthy", "sourceChainHealthy", sourceChainHealthy, "destChainHealthy", destChainHealthy)
c.lggr.Criticalw(
"Source or destination chain is unhealthy",
"sourceChainHealthy", sourceChainHealthy,
"destChainHealthy", destChainHealthy,
)
}
return sourceChainHealthy && destChainHealthy, nil
}

func (a chainHealthcheck) checkIfRMNsAreHealthy(ctx context.Context, forceFetch bool) (bool, error) {
func (c *chainHealthcheck) checkIfRMNsAreHealthy(ctx context.Context, forceFetch bool) (bool, error) {
if !forceFetch {
if healthy, found := a.cache.Get(a.rmnStatusKey); found {
if healthy, found := c.cache.Get(c.rmnStatusKey); found {
return healthy.(bool), nil
}
}

healthy, err := a.fetchRMNCurseState(ctx)
healthy, err := c.fetchRMNCurseState(ctx)
if err != nil {
return false, err
}

a.cache.Set(a.rmnStatusKey, healthy, a.rmnStatusExpiration)
c.cache.Set(c.rmnStatusKey, healthy, c.rmnStatusExpiration)
return healthy, nil
}

func (a chainHealthcheck) fetchRMNCurseState(ctx context.Context) (bool, error) {
func (c *chainHealthcheck) fetchRMNCurseState(ctx context.Context) (bool, error) {
var (
eg = new(errgroup.Group)
isDown bool
isCursed bool
eg = new(errgroup.Group)
isDestDown bool
isSourceCursed bool
)

eg.Go(func() error {
var err error
isDown, err = a.commitStore.IsDown(ctx)
isDestDown, err = c.commitStore.IsDown(ctx)
if err != nil {
return errors.Wrap(err, "commitStore isDown check errored")
}
Expand All @@ -181,7 +185,7 @@ func (a chainHealthcheck) fetchRMNCurseState(ctx context.Context) (bool, error)

eg.Go(func() error {
var err error
isCursed, err = a.onRamp.IsSourceCursed(ctx)
isSourceCursed, err = c.onRamp.IsSourceCursed(ctx)
if err != nil {
return errors.Wrap(err, "onRamp isSourceCursed errored")
}
Expand All @@ -192,8 +196,12 @@ func (a chainHealthcheck) fetchRMNCurseState(ctx context.Context) (bool, error)
return false, err
}

if isDown || isCursed {
a.lggr.Errorw("Source chain is cursed or CommitStore is down", "isDown", isDown, "isCursed", isCursed)
if isDestDown || isSourceCursed {
c.lggr.Criticalw(
"Source chain is cursed or CommitStore is down",
"isDestDown", isDestDown,
"isSourceCursed", isSourceCursed,
)
return false, nil
}
return true, nil
Expand Down

0 comments on commit 3a8f100

Please sign in to comment.