test: improves e2e tests performance by reading the block metadata in…

…stead of the entire block data (#3672)

Closes #3667

I also used this PR to make a change to E2ESimple to address a flaky
behavior. The sequence count of the txclient has been increased to 10 to
ensure there are sufficient transactions at the end of the test.

test/e2e/benchmark/benchmark.go

@@ -148,18 +148,18 @@ func (b *BenchmarkTest) CheckResults(expectedBlockSizeBytes int64) error {
 		}
 	}
 
-	log.Println("Reading blockchain")
-	blockchain, err := testnode.ReadBlockchain(context.Background(),
+	log.Println("Reading blockchain headers")
+	blockchain, err := testnode.ReadBlockchainHeaders(context.Background(),
 		b.Node(0).AddressRPC())
-	testnet.NoError("failed to read blockchain", err)
+	testnet.NoError("failed to read blockchain headers", err)
 
 	targetSizeReached := false
 	maxBlockSize := int64(0)
-	for _, block := range blockchain {
-		if appconsts.LatestVersion != block.Version.App {
-			return fmt.Errorf("expected app version %d, got %d", appconsts.LatestVersion, block.Version.App)
+	for _, blockMeta := range blockchain {
+		if appconsts.LatestVersion != blockMeta.Header.Version.App {
+			return fmt.Errorf("expected app version %d, got %d", appconsts.LatestVersion, blockMeta.Header.Version.App)
 		}
-		size := int64(block.Size())
+		size := int64(blockMeta.BlockSize)
 		if size > maxBlockSize {
 			maxBlockSize = size
 		}

test/e2e/simple.go

@@ -31,7 +31,8 @@ func E2ESimple(logger *log.Logger) error {
 	logger.Println("Creating txsim")
 	endpoints, err := testNet.RemoteGRPCEndpoints()
 	testnet.NoError("failed to get remote gRPC endpoints", err)
-	err = testNet.CreateTxClient("txsim", testnet.TxsimVersion, 10, "100-2000", 1, testnet.DefaultResources, endpoints[0])
+	err = testNet.CreateTxClient("txsim", testnet.TxsimVersion, 10,
+		"100-2000", 100, testnet.DefaultResources, endpoints[0])
 	testnet.NoError("failed to create tx client", err)
 
 	logger.Println("Setting up testnets")
@@ -43,15 +44,17 @@ func E2ESimple(logger *log.Logger) error {
 	logger.Println("Waiting for 30 seconds to produce blocks")
 	time.Sleep(30 * time.Second)
 
-	blockchain, err := testnode.ReadBlockchain(context.Background(), testNet.Node(0).AddressRPC())
-	testnet.NoError("failed to read blockchain", err)
+	logger.Println("Reading blockchain headers")
+	blockchain, err := testnode.ReadBlockchainHeaders(context.Background(), testNet.Node(0).AddressRPC())
+	testnet.NoError("failed to read blockchain headers", err)
 
 	totalTxs := 0
-	for _, block := range blockchain {
-		if appconsts.LatestVersion != block.Version.App {
-			return fmt.Errorf("expected app version %d, got %d in block %d", appconsts.LatestVersion, block.Version.App, block.Height)
+	for _, blockMeta := range blockchain {
+		version := blockMeta.Header.Version.App
+		if appconsts.LatestVersion != version {
+			return fmt.Errorf("expected app version %d, got %d in blockMeta %d", appconsts.LatestVersion, version, blockMeta.Header.Height)
 		}
-		totalTxs += len(block.Data.Txs)
+		totalTxs += blockMeta.NumTxs
 	}
 	if totalTxs < 10 {
 		return fmt.Errorf("expected at least 10 transactions, got %d", totalTxs)

test/util/testnode/read.go

@@ -39,6 +39,63 @@ func ReadBlockchain(ctx context.Context, rpcAddress string) ([]*types.Block, err
 	return ReadBlockHeights(ctx, rpcAddress, 1, status.SyncInfo.LatestBlockHeight)
 }
 
+// ReadBlockchainHeaders retrieves the blockchain headers from height 1 up to
+// latest available height from the node at rpcAddress and returns it.
+// The headers are returned in ascending order (lowest first).
+func ReadBlockchainHeaders(ctx context.Context, rpcAddress string) ([]*types.BlockMeta, error) {
+	client, err := http.New(rpcAddress, "/websocket")
+	if err != nil {
+		return nil, err
+	}
+
+	// fetch the latest height
+	resp, err := client.Status(ctx)
+	if err != nil {
+		return nil, err
+	}
+	maxHeight := resp.SyncInfo.LatestBlockHeight
+
+	blockHeaders := make([]*types.BlockMeta, 0)
+	// fetch headers up to maxHeight
+	lastFetchedHeight := int64(0)
+	for {
+		// BlockchainInfo may apply a limit on the range of blocks to fetch,
+		// so we need to request them iteratively.
+		// note that block headers returned by BlockchainInfo are in descending
+		// order (highest first).
+
+		res, err := client.BlockchainInfo(ctx, 1, maxHeight)
+		if err != nil {
+			return nil, err
+		}
+
+		blockHeaders = append(blockHeaders, res.BlockMetas...)
+
+		lastFetchedHeight = res.BlockMetas[len(res.BlockMetas)-1].Header.Height
+
+		// fetch until the first block
+		if lastFetchedHeight <= 1 {
+			break
+		}
+
+		// set the new maxHeight to fetch the next batch of headers
+		maxHeight = lastFetchedHeight - 1
+
+	}
+
+	// reverse the order of headers to be ascending (lowest first).
+	reverseSlice(blockHeaders)
+
+	return blockHeaders, nil
+}
+
+// reverseSlice reverses the order of elements in a slice in place.
+func reverseSlice[T any](s []T) {
+	for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {
+		s[i], s[j] = s[j], s[i]
+	}
+}
+
 func ReadBlockHeights(ctx context.Context, rpcAddress string, fromHeight, toHeight int64) ([]*types.Block, error) {
 	client, err := http.New(rpcAddress, "/websocket")
 	if err != nil {

test/util/testnode/read_test.go

@@ -0,0 +1,62 @@
+package testnode
+
+import (
+	"context"
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestReverseSlice(t *testing.T) {
+	tests := []struct {
+		input    interface{}
+		expected interface{}
+	}{
+		{[]int{1, 2, 3, 4, 5}, []int{5, 4, 3, 2, 1}},
+		{[]string{"a", "b", "c", "d"}, []string{"d", "c", "b", "a"}},
+		{[]int{1, 2}, []int{2, 1}},
+		{[]int{1}, []int{1}},
+		{[]string{}, []string{}},
+	}
+
+	for _, tt := range tests {
+		switch v := tt.input.(type) {
+		case []int:
+			// reverseSlice modifies the input slice, so we need to make a copy
+			original := make([]int, len(tt.input.([]int)))
+			copy(original, tt.input.([]int))
+			reverseSlice(v)
+			require.True(t, reflect.DeepEqual(v, tt.expected), "reverseSlice(%v) = %v, want %v", original, tt.input, tt.expected)
+		case []string:
+			// reverseSlice modifies the input slice, so we need to make a copy
+			original := make([]string, len(tt.input.([]string)))
+			copy(original, tt.input.([]string))
+			reverseSlice(v)
+			require.True(t, reflect.DeepEqual(v, tt.expected), "reverseSlice(%v) = %v, want %v", original, tt.input, tt.expected)
+		}
+	}
+}
+
+func TestReadBlockchainHeaders(t *testing.T) {
+	cfg := DefaultConfig()
+	cctx, rpcAddr, _ := NewNetwork(t, cfg)
+	// wait for 30 blocks to be produced
+	err := cctx.WaitForBlocks(30)
+	require.NoError(t, err)
+
+	// fetch headers
+	headers, err := ReadBlockchainHeaders(context.Background(), rpcAddr)
+	require.NoError(t, err)
+	// we should have at least 30 headers
+	require.True(t, len(headers) >= 30)
+
+	// check that the headers are in ascending order, starting from 1
+	i := int64(1)
+	for _, header := range headers {
+		got := header.Header.Height
+		require.Equal(t, i, got,
+			"expected height %d, got %d", i, got)
+		i++
+	}
+}