Skip to content

Commit

Permalink
Fix reorg test flake (#1542)
Browse files Browse the repository at this point in the history
For some weird reasons, the health check call in nodes doesn't show
finality violated error consistently across all the nodes. Due to which
this test
TestSmokeCCIPReorgAboveFinalityAtDestination/Above_finality_reorg_in_destination_chain
flakes often in CI.

Adjusting the expectation to have at least half of the nodes detects
violation instead of every node.
CI Failure examples:
https://github.com/smartcontractkit/chainlink/actions/runs/11894867899/job/33143516553

https://github.com/smartcontractkit/chainlink/actions/runs/11894232425/job/33141414438?pr=15282
  • Loading branch information
b-gopalswami authored Nov 20, 2024
1 parent 228298c commit 9f39f87
Showing 1 changed file with 10 additions and 7 deletions.
17 changes: 10 additions & 7 deletions integration-tests/ccip-tests/smoke/ccip_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package smoke

import (
"fmt"
"math"
"math/big"
"strings"
"testing"
Expand Down Expand Up @@ -892,7 +893,7 @@ func TestSmokeCCIPReorgBelowFinality(t *testing.T) {

// Test creates above finality reorg at destination and
// expects ccip transactions in-flight and the one initiated after reorg
// doesn't go through and verifies every node is able to detect reorg.
// doesn't go through and verifies f+1 nodes are able to detect reorg.
// Note: LogPollInterval interval is set as 1s to detect the reorg immediately
func TestSmokeCCIPReorgAboveFinalityAtDestination(t *testing.T) {
t.Parallel()
Expand All @@ -903,7 +904,7 @@ func TestSmokeCCIPReorgAboveFinalityAtDestination(t *testing.T) {

// Test creates above finality reorg at destination and
// expects ccip transactions in-flight doesn't go through, the transaction initiated after reorg
// shouldn't even get initiated and verifies every node is able to detect reorg.
// shouldn't even get initiated and verifies f+1 nodes are able to detect reorg.
// Note: LogPollInterval interval is set as 1s to detect the reorg immediately
func TestSmokeCCIPReorgAboveFinalityAtSource(t *testing.T) {
t.Parallel()
Expand Down Expand Up @@ -1059,11 +1060,13 @@ func performAboveFinalityReorgAndValidate(t *testing.T, network string) {
logPollerName = fmt.Sprintf("EVM.%d.LogPoller", lane.SourceChain.GetChainID())
rs.RunReorg(rs.SrcClient, int(rs.Cfg.SrcFinalityDepth)+rs.Cfg.FinalityDelta, network, 2*time.Second)
}
clNodes := setUpOutput.Env.CLNodes
// assert every node is detecting the reorg (LogPollInterval is set as 1s for faster detection)
// DON is 3F+1, finding f+1 from the given number of nodes in the environment
fPlus1Nodes := int(math.Ceil(float64(len(setUpOutput.Env.CLNodes)-1)/3)) + 1
// assert at least f+1 nodes is detecting the reorg (LogPollInterval is set as 1s for faster detection)
// additional context: Commit requires 2f+1 observations, so f+1 nodes need to detect it in order to force the entire DON to stop processing messages.
nodesDetectedViolation := make(map[string]bool)
assert.Eventually(t, func() bool {
for _, node := range clNodes {
for _, node := range setUpOutput.Env.CLNodes {
if _, ok := nodesDetectedViolation[node.ChainlinkClient.URL()]; ok {
continue
}
Expand All @@ -1076,8 +1079,8 @@ func performAboveFinalityReorgAndValidate(t *testing.T, network string) {
}
}
}
return len(nodesDetectedViolation) == len(clNodes)
}, 5*time.Minute, 5*time.Second, "Reorg above finality depth is not detected by every node")
return len(nodesDetectedViolation) >= fPlus1Nodes
}, 3*time.Minute, 20*time.Second, "Reorg above finality depth is not detected by f+1 nodes")
log.Debug().Interface("Nodes", nodesDetectedViolation).Msg("Violation detection details")
// send another request and verify it fails
err = lane.SendRequests(1, gasLimit)
Expand Down

0 comments on commit 9f39f87

Please sign in to comment.