Increase timeout further to find what works on CI

f3_test.go

@@ -29,11 +29,11 @@ import (
 )
 
 const (
-	conditionCheckInterval  = 500 * time.Millisecond
-	conditionCheckTimeout   = time.Minute
+	eventualCheckInterval   = 100 * time.Millisecond
+	eventualCheckTimeout    = 2 * time.Minute
 	manifestPublishInterval = time.Second
-	advanceClockEvery       = 50 * time.Millisecond
-	advanceClockBy          = time.Second
+	advanceClockEvery       = 10 * time.Millisecond
+	advanceClockBy          = time.Second / 2
 )
 
 func init() {
@@ -47,8 +47,8 @@ func init() {
 func TestF3Simple(t *testing.T) {
 	t.Parallel()
 	env := newTestEnvironment(t).withNodes(2).start()
-	env.requireInstanceEventually(5, conditionCheckTimeout, true)
-	env.requireEpochFinalizedEventually(env.manifest.BootstrapEpoch+10, conditionCheckTimeout)
+	env.requireInstanceEventually(5, eventualCheckTimeout, true)
+	env.requireEpochFinalizedEventually(env.manifest.BootstrapEpoch, eventualCheckTimeout)
 }
 
 func TestF3WithLookback(t *testing.T) {
@@ -57,7 +57,7 @@ func TestF3WithLookback(t *testing.T) {
 		withNodes(2).
 		withManifest(func(m *manifest.Manifest) { m.EC.HeadLookback = 20 }).
 		start()
-	env.requireInstanceEventually(5, conditionCheckTimeout, true)
+	env.requireInstanceEventually(5, eventualCheckTimeout, true)
 
 	headEpoch := env.ec.GetCurrentHead()
 	cert, err := env.nodes[0].f3.GetLatestCert(env.testCtx)
@@ -86,19 +86,19 @@ func TestF3WithLookback(t *testing.T) {
 		env.clock.Add(env.manifest.EC.Period / 10)
 	}
 
-	env.requireInstanceEventually(5, conditionCheckTimeout, true)
+	env.requireInstanceEventually(5, eventualCheckTimeout, true)
 }
 
 func TestF3PauseResumeCatchup(t *testing.T) {
 	t.Parallel()
 	env := newTestEnvironment(t).withNodes(3).start()
-	env.requireInstanceEventually(1, conditionCheckTimeout, true)
+	env.requireInstanceEventually(1, eventualCheckTimeout, true)
 
 	// Pausing two nodes should pause the network.
 	env.pauseNode(1)
 	env.pauseNode(2)
 
-	env.requireF3NotRunningEventually(conditionCheckTimeout, nodeMatchers.byID(1, 2))
+	env.requireF3NotRunningEventually(eventualCheckTimeout, nodeMatchers.byID(1, 2))
 
 	// Wait until node 0 stops progressing
 	env.whileAdvancingClock(func() {
@@ -107,30 +107,30 @@ func TestF3PauseResumeCatchup(t *testing.T) {
 			env.clock.Add(10 * env.manifest.EC.Period)
 			newInstance := env.nodes[0].currentGpbftInstance()
 			return oldInstance == newInstance
-		}, conditionCheckTimeout, conditionCheckInterval)
+		}, eventualCheckTimeout, eventualCheckInterval)
 	})
 
 	// Resuming node 1 should continue agreeing on instances.
 	env.resumeNode(1)
-	env.requireF3RunningEventually(conditionCheckTimeout, nodeMatchers.byID(1))
+	env.requireF3RunningEventually(eventualCheckTimeout, nodeMatchers.byID(1))
 
 	// Wait until we're far enough that pure GPBFT catchup should be impossible.
 	targetInstance := env.nodes[1].currentGpbftInstance() + env.manifest.CommitteeLookback + 1
-	env.requireInstanceEventually(targetInstance, conditionCheckTimeout, false)
+	env.requireInstanceEventually(targetInstance, eventualCheckTimeout, false)
 
 	env.resumeNode(2)
-	env.requireF3RunningEventually(conditionCheckTimeout, nodeMatchers.byID(2))
+	env.requireF3RunningEventually(eventualCheckTimeout, nodeMatchers.byID(2))
 
 	// Everyone should catch up eventually
-	env.requireInstanceEventually(targetInstance, conditionCheckTimeout, true)
+	env.requireInstanceEventually(targetInstance, eventualCheckTimeout, true)
 
 	// Pause the "good" node.
 	node0failInstance := env.nodes[0].currentGpbftInstance()
 	env.pauseNode(0)
-	env.requireF3NotRunningEventually(conditionCheckTimeout, nodeMatchers.byID(0))
+	env.requireF3NotRunningEventually(eventualCheckTimeout, nodeMatchers.byID(0))
 
 	// We should be able to make progress with the remaining nodes.
-	env.requireInstanceEventually(node0failInstance+3, conditionCheckTimeout, false)
+	env.requireInstanceEventually(node0failInstance+3, eventualCheckTimeout, false)
 }
 
 func TestF3FailRecover(t *testing.T) {
@@ -153,15 +153,15 @@ func TestF3FailRecover(t *testing.T) {
 	env.injectDatastoreFailures(0, dsFailureFunc)
 
 	env.start()
-	env.requireInstanceEventually(1, conditionCheckTimeout, true)
+	env.requireInstanceEventually(1, eventualCheckTimeout, true)
 
 	// Inject a single write failure. This should prevent us from storing a single decision
 	// decision.
 	failDsWrite.Store(true)
 
 	// We should proceed anyways (catching up via the certificate exchange protocol).
 	oldInstance := env.nodes[0].currentGpbftInstance()
-	env.requireInstanceEventually(oldInstance+3, conditionCheckTimeout, true)
+	env.requireInstanceEventually(oldInstance+3, eventualCheckTimeout, true)
 }
 
 func TestF3DynamicManifest_WithoutChanges(t *testing.T) {
@@ -171,7 +171,7 @@ func TestF3DynamicManifest_WithoutChanges(t *testing.T) {
 	env.start()
 	prev := env.nodes[0].f3.Manifest()
 
-	env.requireInstanceEventually(5, conditionCheckTimeout, false)
+	env.requireInstanceEventually(5, eventualCheckTimeout, false)
 	// no changes in manifest
 	require.Equal(t, prev, env.nodes[0].f3.Manifest())
 	env.requireConsistentManifest(true)
@@ -182,7 +182,7 @@ func TestF3DynamicManifest_WithRebootstrap(t *testing.T) {
 	env := newTestEnvironment(t).withNodes(2).withDynamicManifest().start()
 
 	prevManifest := env.nodes[0].f3.Manifest()
-	env.requireInstanceEventually(3, conditionCheckTimeout, true)
+	env.requireInstanceEventually(3, eventualCheckTimeout, true)
 
 	for range 2 {
 		nd := env.addNode()
@@ -195,7 +195,7 @@ func TestF3DynamicManifest_WithRebootstrap(t *testing.T) {
 	}
 
 	env.updateManifest()
-	env.requireManifestPropagatedEventually(conditionCheckTimeout)
+	env.requireManifestPropagatedEventually(eventualCheckTimeout)
 
 	// check that it rebootstrapped and has a new base epoch.
 	targetBaseEpoch := env.manifest.BootstrapEpoch - env.manifest.EC.Finality
@@ -207,9 +207,9 @@ func TestF3DynamicManifest_WithRebootstrap(t *testing.T) {
 				return false
 			}
 			return c.ECChain.Base().Epoch == targetBaseEpoch
-		}, conditionCheckTimeout, conditionCheckInterval, "Base epoch on node 0 not reached in time. Environment: %s", env)
+		}, eventualCheckTimeout, eventualCheckInterval, "Base epoch on node 0 not reached in time. Environment: %s", env)
 	})
-	env.requireInstanceEventually(3, conditionCheckTimeout, false)
+	env.requireInstanceEventually(3, eventualCheckTimeout, false)
 	require.NotEqual(t, prevManifest, env.nodes[0].f3.Manifest())
 	env.requireConsistentManifest(false)
 
@@ -225,8 +225,8 @@ func TestF3DynamicManifest_WithPauseAndRebootstrap(t *testing.T) {
 	t.Parallel()
 	env := newTestEnvironment(t).withNodes(2).withDynamicManifest().start()
 
-	env.requireInstanceEventually(10, conditionCheckTimeout, true)
-	env.requireEpochFinalizedEventually(env.manifest.BootstrapEpoch+10, conditionCheckTimeout)
+	env.requireInstanceEventually(10, eventualCheckTimeout, true)
+	env.requireEpochFinalizedEventually(env.manifest.BootstrapEpoch+10, eventualCheckTimeout)
 	cert0, err := env.nodes[0].f3.GetCert(env.testCtx, 0)
 	require.NoError(t, err)
 	FirstFinalizedEpochBeforeRebootstrap := cert0.ECChain.Base().Epoch
@@ -235,17 +235,17 @@ func TestF3DynamicManifest_WithPauseAndRebootstrap(t *testing.T) {
 	env.manifest.Pause = true
 	env.updateManifest()
 	// check that it paused
-	env.requireF3NotRunningEventually(conditionCheckTimeout, nodeMatchers.all)
+	env.requireF3NotRunningEventually(eventualCheckTimeout, nodeMatchers.all)
 
 	env.manifest.BootstrapEpoch += 10
 	env.manifest.Pause = false
 	env.updateManifest()
-	env.requireManifestPropagatedEventually(conditionCheckTimeout)
+	env.requireManifestPropagatedEventually(eventualCheckTimeout)
 	env.requireConsistentManifest(true)
-	env.requireF3RunningEventually(conditionCheckTimeout, nodeMatchers.all)
+	env.requireF3RunningEventually(eventualCheckTimeout, nodeMatchers.all)
 
-	env.requireInstanceEventually(3, conditionCheckTimeout, true)
-	env.requireEpochFinalizedEventually(env.manifest.BootstrapEpoch+10, conditionCheckTimeout)
+	env.requireInstanceEventually(3, eventualCheckTimeout, true)
+	env.requireEpochFinalizedEventually(env.manifest.BootstrapEpoch+10, eventualCheckTimeout)
 
 	// Now check that we have the correct base for certificate 0.
 	cert0, err = env.nodes[0].f3.GetCert(env.testCtx, 0)
@@ -258,8 +258,8 @@ func TestF3DynamicManifest_RebootstrapWithCompression(t *testing.T) {
 	t.Parallel()
 	env := newTestEnvironment(t).withNodes(2).withDynamicManifest().start()
 
-	env.requireInstanceEventually(10, conditionCheckTimeout, true)
-	env.requireEpochFinalizedEventually(env.manifest.BootstrapEpoch+10, conditionCheckTimeout)
+	env.requireInstanceEventually(10, eventualCheckTimeout, true)
+	env.requireEpochFinalizedEventually(env.manifest.BootstrapEpoch+10, eventualCheckTimeout)
 	cert0, err := env.nodes[0].f3.GetCert(env.testCtx, 0)
 	require.NoError(t, err)
 	FirstFinalizedEpochBeforeRebootstrap := cert0.ECChain.Base().Epoch
@@ -268,18 +268,18 @@ func TestF3DynamicManifest_RebootstrapWithCompression(t *testing.T) {
 	env.manifest.Pause = true
 	env.updateManifest()
 	// check that it paused
-	env.requireF3NotRunningEventually(conditionCheckTimeout, nodeMatchers.all)
+	env.requireF3NotRunningEventually(eventualCheckTimeout, nodeMatchers.all)
 
 	env.manifest.BootstrapEpoch += 10
 	env.manifest.Pause = false
 	env.manifest.PubSub.CompressionEnabled = true
 	env.updateManifest()
-	env.requireManifestPropagatedEventually(conditionCheckTimeout)
+	env.requireManifestPropagatedEventually(eventualCheckTimeout)
 	env.requireConsistentManifest(true)
-	env.requireF3RunningEventually(conditionCheckTimeout, nodeMatchers.all)
+	env.requireF3RunningEventually(eventualCheckTimeout, nodeMatchers.all)
 
-	env.requireInstanceEventually(3, conditionCheckTimeout, true)
-	env.requireEpochFinalizedEventually(env.manifest.BootstrapEpoch+10, conditionCheckTimeout)
+	env.requireInstanceEventually(3, eventualCheckTimeout, true)
+	env.requireEpochFinalizedEventually(env.manifest.BootstrapEpoch+10, eventualCheckTimeout)
 
 	// Now check that we have the correct base for certificate 0.
 	cert0, err = env.nodes[0].f3.GetCert(env.testCtx, 0)
@@ -294,13 +294,13 @@ func TestF3LateBootstrap(t *testing.T) {
 
 	// Wait till we're "caught up".
 	bootstrapInstances := uint64(env.manifest.EC.Finality/(gpbft.ChainDefaultLen-1)) + 1
-	env.requireInstanceEventually(bootstrapInstances, conditionCheckTimeout, true)
+	env.requireInstanceEventually(bootstrapInstances, eventualCheckTimeout, true)
 
 	// Wait until we've finalized a distant epoch. Once we do, our EC will forget the historical
 	// chain (importantly, forget the bootstrap power table).
 	targetEpoch := 2*env.manifest.EC.Finality + env.manifest.BootstrapEpoch
 	env.clock.Add(time.Duration(targetEpoch/3) * env.manifest.EC.Period) // Fast-forward to 1/3 of the target epoch for faster test e2e time.
-	env.requireEpochFinalizedEventually(targetEpoch, conditionCheckTimeout)
+	env.requireEpochFinalizedEventually(targetEpoch, eventualCheckTimeout)
 
 	// Now update the manifest with the initial power-table CID.
 	cert0, err := env.nodes[0].f3.GetCert(env.testCtx, 0)
@@ -318,7 +318,7 @@ func TestF3LateBootstrap(t *testing.T) {
 
 	// Wait for it to finish starting.
 	env.whileAdvancingClock(func() {
-		require.Eventually(t, newNode.IsRunning, conditionCheckTimeout, conditionCheckInterval)
+		require.Eventually(t, newNode.IsRunning, eventualCheckTimeout, eventualCheckInterval)
 	})
 
 	// It should eventually catch up.
@@ -327,7 +327,7 @@ func TestF3LateBootstrap(t *testing.T) {
 			cert, err := newNode.GetLatestCert(env.testCtx)
 			require.NoError(t, err)
 			return cert != nil && cert.ECChain.Head().Epoch > targetEpoch
-		}, conditionCheckTimeout, conditionCheckInterval)
+		}, eventualCheckTimeout, eventualCheckInterval)
 	})
 }
 
@@ -513,7 +513,7 @@ func (e *testEnv) requireManifestPropagatedEventually(timeout time.Duration) {
 				}
 			}
 			return true
-		}, timeout, conditionCheckInterval, "Manifest did not propagate in time. Environment: %s", e)
+		}, timeout, eventualCheckInterval, "Manifest did not propagate in time. Environment: %s", e)
 	})
 }
 
@@ -539,7 +539,7 @@ func (e *testEnv) requireInstanceEventually(instanceNumber uint64, timeout time.
 					}
 				}
 				return true
-			}, timeout, conditionCheckInterval, "Instance %d not reached in time. Environment: %s", instanceNumber, e)
+			}, timeout, eventualCheckInterval, "Instance %d not reached in time. Environment: %s", instanceNumber, e)
 		},
 	)
 }
@@ -588,7 +588,7 @@ func (e *testEnv) requireEpochFinalizedEventually(epoch int64, timeout time.Dura
 					}
 				}
 				return false
-			}, timeout, conditionCheckInterval, "Epoch %d not reached in time. Environment: %s", epoch, e)
+			}, timeout, eventualCheckInterval, "Epoch %d not reached in time. Environment: %s", epoch, e)
 		},
 	)
 }
@@ -676,7 +676,7 @@ func (e *testEnv) requireF3RunningEventually(timeout time.Duration, matcher node
 				}
 			}
 			return true
-		}, timeout, conditionCheckInterval, "F3 not running in time. Environment: %s", e)
+		}, timeout, eventualCheckInterval, "F3 not running in time. Environment: %s", e)
 	})
 }
 
@@ -693,7 +693,7 @@ func (e *testEnv) requireF3NotRunningEventually(timeout time.Duration, matcher n
 				}
 			}
 			return true
-		}, timeout, conditionCheckInterval, "F3 still running after timeout. Environment: %s", e)
+		}, timeout, eventualCheckInterval, "F3 still running after timeout. Environment: %s", e)
 	})
 }
 
@@ -746,7 +746,7 @@ func (e *testEnv) start() *testEnv {
 	}
 
 	// wait for nodes to initialize
-	e.requireF3RunningEventually(conditionCheckTimeout, nodeMatchers.all)
+	e.requireF3RunningEventually(eventualCheckTimeout, nodeMatchers.all)
 
 	// If it exists, start the manifest sender
 	if e.manifestSender != nil {