[test] Add retry logic to make Test_BlockByNumber from eth_api_test.go more consistent

[Unique-Divine]

The Ask

Test_BlockByNumber is one of the newer tests added to eth/rpc/backend suite. It's doing a request for the current block number and sometimes is off by 1. We should write the logic to account for this case of being off by 1 or 2 blocks.

Context - Test Logs

I[2024-10-01|17:02:33.365] Calling EVMIndexerService CancelFunc         indexer=evm
130
I[2024-10-01|17:02:33.365] Stopping indexer goroutine                   indexer=evm
131
I[2024-10-01|17:02:33.365] Closing EVMTxIndexer DB                      indexer=evm
132
--- FAIL: TestSuite_RunAll (35.53s)
133
    cases.go:35: setting up integration test suite
134
    network.go:150: acquiring test network lock
135
    network.go:168: preparing test network...
136
    network.go:455: starting test network...
137
    start_node.go:134: Set EVM indexer
138
    start_node.go:157: Set up Ethereum JSON-RPC client objects
139
    start_node.go:167: Expose typed methods for each namespace
140
    network.go:461: started validator 0
141
    network.go:469: started test network at height: 48
142
    --- FAIL: TestSuite_RunAll/Test_BlockByNumber (0.00s)
143
        eth_api_test.go:119: 
144
            	Error Trace:	/home/runner/work/nibiru/nibiru/eth/rpc/rpcapi/eth_api_test.go:119
145
            	Error:      	Received unexpected error:
146
            	            	not found
147
            	Test:       	TestSuite_RunAll/Test_BlockByNumber
148
        panic.go:261: test panicked: runtime error: invalid memory address or nil pointer dereference
149
            goroutine 2909 [running]:
150
            runtime/debug.Stack()
151
            	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/debug/stack.go:24 +0x67
152
            github.com/stretchr/testify/suite.failOnPanic(0xc000c0bba0, {0x4125fc0, 0x7b89a30})
153
            	/home/runner/go/pkg/mod/github.com/stretchr/[email protected]/suite/suite.go:89 +0x5b
154
            github.com/stretchr/testify/suite.Run.func1.1()
155
            	/home/runner/go/pkg/mod/github.com/stretchr/[email protected]/suite/suite.go:188 +0x351
156
            panic({0x4125fc0?, 0x7b89a30?})
157
            	/opt/hostedtoolcache/go/1.21.13/x64/src/runtime/panic.go:920 +0x270
158
            github.com/ethereum/go-ethereum/core/types.(*Block).SanityCheck(...)
159
            	/home/runner/go/pkg/mod/github.com/!nibiru!chain/[email protected]/core/types/block.go:334
160
            github.com/NibiruChain/nibiru/v2/eth/rpc/rpcapi_test.(*NodeSuite).Test_BlockByNumber(0xc0017acfc0)
161
            	/home/runner/work/nibiru/nibiru/eth/rpc/rpcapi/eth_api_test.go:120 +0x1eb
162
            reflect.Value.call({0xc0018d5080?, 0xc0024182f8?, 0x13?}, {0x45746e2, 0x4}, {0xc001229db0, 0x1, 0x1?})
163
            	/opt/hostedtoolcache/go/1.21.13/x64/src/reflect/value.go:596 +0x14ce
164
            reflect.Value.Call({0xc0018d5080?, 0xc0024182f8?, 0xc0017acfc0?}, {0xc001b09db0, 0x1, 0x1})
165
            	/opt/hostedtoolcache/go/1.21.13/x64/src/reflect/value.go:380 +0xb6
166
            github.com/stretchr/testify/suite.Run.func1(0xc000c0bba0)
167
            	/home/runner/go/pkg/mod/github.com/stretchr/[email protected]/suite/suite.go:202 +0x767
168
            testing.tRunner(0xc000c0bba0, 0xc001cf6bd0)
169
            	/opt/hostedtoolcache/go/1.21.13/x64/src/testing/testing.go:1595 +0x262
170
            created by testing.(*T).Run in goroutine 51
171
            	/opt/hostedtoolcache/go/1.21.13/x64/src/testing/testing.go:1648 +0x846
172
    --- FAIL: TestSuite_RunAll/Test_BlockNumber (0.00s)
173
        eth_api_test.go:110: 
174
            	Error Trace:	/home/runner/work/nibiru/nibiru/eth/rpc/rpcapi/eth_api_test.go:110
175
            	Error:      	Not equal: 
176
            	            	expected: 51
177
            	            	actual  : 50
178
            	Test:       	TestSuite_RunAll/Test_BlockNumber
179
    eth_api_test.go:436: tearing down integration test suite
180
    network.go:593: cleaning up test network...
181
    validator_node.go:172: ✅ Successfully shut down JSON-RPC server
182
    validator_node.go:180: ✅ Successfully shut down EVMTxIndexerService
183
    network.go:635: finished cleaning up test network
184
    network.go:590: released test network lock
185
E[2024-10-01|17:02:33.368] failed to add event topic to event bus       topic=event error="topic already registered"
186
FAIL
187
coverage: 12.2% of statements
188
FAIL	github.com/NibiruChain/nibiru/v2/eth/rpc/rpcapi	36.231s