chore: do not wait forever for required depth

CHANGELOG.md

@@ -47,6 +47,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 * [#136](https://github.com/babylonlabs-io/vigilante/pull/136) rate limit activations
 * [#141](https://github.com/babylonlabs-io/vigilante/pull/141) decrement tracked delegations in atomic slasher
 * [#143](https://github.com/babylonlabs-io/vigilante/pull/143) adds nlreturn linter rule
+* [#145](https://github.com/babylonlabs-io/vigilante/pull/145) do not wait forever
 
 ## v0.18.0
 

btcstaking-tracker/stakingeventwatcher/stakingeventwatcher.go

@@ -656,7 +656,11 @@ func (sew *StakingEventWatcher) checkBtcForStakingTx() {
 
 		go func() {
 			defer sew.activationLimiter.Release(1)
-			sew.activateBtcDelegation(txHash, proof, details.Block.BlockHash(), params.ConfirmationTimeBlocks)
+			done := make(chan struct{})
+
+			go sew.activateBtcDelegation(txHash, proof, details.Block.BlockHash(), params.ConfirmationTimeBlocks, done)
+
+			<-done
 		}()
 	}
 }
@@ -667,7 +671,10 @@ func (sew *StakingEventWatcher) activateBtcDelegation(
 	proof *btcctypes.BTCSpvProof,
 	inclusionBlockHash chainhash.Hash,
 	requiredDepth uint32,
+	done chan struct{},
 ) {
+	var once sync.Once
+	defer once.Do(func() { close(done) })
 	sew.metrics.NumberOfActivationInProgress.Inc()
 	defer sew.metrics.NumberOfActivationInProgress.Dec()
 
@@ -677,7 +684,11 @@ func (sew *StakingEventWatcher) activateBtcDelegation(
 	defer sew.latency("activateBtcDelegation")()
 	defer sew.inProgressTracker.RemoveDelegation(stakingTxHash)
 
-	sew.waitForRequiredDepth(ctx, stakingTxHash, &inclusionBlockHash, requiredDepth)
+	if err := sew.waitForRequiredDepth(ctx, stakingTxHash, &inclusionBlockHash, requiredDepth); err != nil {
+		sew.logger.Warnf("exceeded waiting for required depth, will try later: err %v", err)
+
+		return
+	}
 
 	defer sew.latency("activateDelegationRPC")()
 
@@ -701,9 +712,13 @@ func (sew *StakingEventWatcher) activateBtcDelegation(
 
 		sew.metrics.ReportedActivateDelegationsCounter.Inc()
 
-		sew.pendingTracker.RemoveDelegation(stakingTxHash)
+		once.Do(func() {
+			close(done)
+		})
 
+		sew.pendingTracker.RemoveDelegation(stakingTxHash)
 		sew.metrics.NumberOfVerifiedDelegations.Dec()
+
 		sew.logger.Debugf("staking tx activated %s", stakingTxHash.String())
 
 		return nil
@@ -724,11 +739,12 @@ func (sew *StakingEventWatcher) waitForRequiredDepth(
 	stakingTxHash chainhash.Hash,
 	inclusionBlockHash *chainhash.Hash,
 	requiredDepth uint32,
-) {
+) error {
 	defer sew.latency("waitForRequiredDepth")()
 
 	var depth uint32
-	_ = retry.Do(func() error {
+
+	return retry.Do(func() error {
 		var err error
 		depth, err = sew.babylonNodeAdapter.QueryHeaderDepth(inclusionBlockHash)
 		if err != nil {
@@ -749,7 +765,7 @@ func (sew *StakingEventWatcher) waitForRequiredDepth(
 		return nil
 	},
 		retry.Context(ctx),
-		retryForever,
+		retry.Attempts(10),
 		fixedDelyTypeWithJitter,
 		retry.MaxDelay(sew.cfg.RetrySubmitUnbondingTxInterval),
 		retry.MaxJitter(sew.cfg.RetryJitter),