test: minor version upgrade e2e testing (#2797)

This PR introduces a test for checking compatibility and the running of
upgrades of minor versions.

It reads all minor versions tagged in git. Each node begins on a random
version.

Each node then individually performs upgrades, going down and spinning
back up on the new randomly chosen version (can be downgrades as well)

Currently we only check that their is no app version mismatch and the
network is able to continually build blocks

Co-authored-by: Evan Forbes <[email protected]>
Co-authored-by: Rootul P <[email protected]>

Makefile

@@ -1,6 +1,7 @@
 VERSION := $(shell echo $(shell git describe --tags 2>/dev/null || git log -1 --format='%h') | sed 's/^v//')
 COMMIT := $(shell git log -1 --format='%H')
 DOCKER := $(shell which docker)
+ALL_VERSIONS := $(shell git tag -l)
 DOCKER_BUF := $(DOCKER) run --rm -v $(CURDIR):/workspace --workdir /workspace bufbuild/buf
 IMAGE := ghcr.io/tendermint/docker-build-proto:latest
 DOCKER_PROTO_BUILDER := docker run -v $(shell pwd):/workspace --workdir /workspace $(IMAGE)
@@ -115,10 +116,10 @@ test-short:
 	@go test ./... -short -timeout 1m
 .PHONY: test-short
 
-## test-e2e: Run end to end tests via knuu.
+## test-e2e: Run end to end tests via knuu. This command requires a kube/config file to configure kubernetes.
 test-e2e:
-	@echo "--> Running e2e tests on version: $(shell git rev-parse --short HEAD)"
-	@KNUU_NAMESPACE=test E2E_VERSION=$(shell git rev-parse --short HEAD) E2E=true go test ./test/e2e/... -timeout 10m -v
+	@echo "--> Running end to end tests"
+	@KNUU_NAMESPACE=test KNUU_TIMEOUT=20m E2E_VERSIONS="$(ALL_VERSIONS)" E2E=true go test ./test/e2e/... -timeout 20m -v
 .PHONY: test-e2e
 
 ## test-race: Run tests in race mode.

test/e2e/node.go

@@ -50,7 +50,7 @@ func NewNode(
 	if err != nil {
 		return nil, err
 	}
-	err = instance.SetImage(fmt.Sprintf("%s:%s", dockerSrcURL, version))
+	err = instance.SetImage(DockerImageName(version))
 	if err != nil {
 		return nil, err
 	}
@@ -260,3 +260,11 @@ func (n *Node) Start() error {
 	n.grpcProxyPort = grpcProxyPort
 	return nil
 }
+
+func (n *Node) Upgrade(version string) error {
+	return n.Instance.SetImageInstant(DockerImageName(version))
+}
+
+func DockerImageName(version string) string {
+	return fmt.Sprintf("%s:%s", dockerSrcURL, version)
+}

test/e2e/simple_test.go

@@ -22,13 +22,18 @@ var latestVersion = "latest"
 // and MsgSends over 30 seconds and then asserts that at least 10 transactions were
 // committed.
 func TestE2ESimple(t *testing.T) {
-	if os.Getenv("E2E") == "" {
+	if os.Getenv("E2E") != "true" {
 		t.Skip("skipping e2e test")
 	}
 
-	if os.Getenv("E2E_VERSION") != "" {
-		latestVersion = os.Getenv("E2E_VERSION")
+	if os.Getenv("E2E_VERSIONS") != "" {
+		versionsStr := os.Getenv("E2E_VERSIONS")
+		versions := ParseVersions(versionsStr)
+		if len(versions) > 0 {
+			latestVersion = versions.GetLatest().String()
+		}
 	}
+	t.Log("Running simple e2e test", "version", latestVersion)
 
 	testnet, err := New(t.Name(), seed)
 	require.NoError(t, err)

test/e2e/upgrade_test.go

@@ -0,0 +1,160 @@
+package e2e
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math/rand"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/celestiaorg/celestia-app/app"
+	"github.com/celestiaorg/celestia-app/app/encoding"
+	"github.com/celestiaorg/celestia-app/test/txsim"
+	"github.com/celestiaorg/knuu/pkg/knuu"
+	"github.com/stretchr/testify/require"
+	"github.com/tendermint/tendermint/rpc/client/http"
+)
+
+// This will only run tests within the v1 major release cycle
+const MajorVersion = 1
+
+func TestMinorVersionCompatibility(t *testing.T) {
+	if os.Getenv("E2E") != "true" {
+		t.Skip("skipping e2e test")
+	}
+
+	if os.Getenv("E2E_VERSIONS") == "" {
+		t.Skip("skipping e2e test: E2E_VERSIONS not set")
+	}
+
+	versionStr := os.Getenv("E2E_VERSIONS")
+	versions := ParseVersions(versionStr).FilterMajor(MajorVersion).FilterOutReleaseCandidates()
+	if len(versions) == 0 {
+		t.Skip("skipping e2e test: no versions to test")
+	}
+	numNodes := 4
+	r := rand.New(rand.NewSource(seed))
+	t.Log("Running minor version compatibility test", "versions", versions)
+
+	testnet, err := New(t.Name(), seed)
+	require.NoError(t, err)
+	t.Cleanup(testnet.Cleanup)
+
+	// preload all docker images
+	preloader, err := knuu.NewPreloader()
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = preloader.EmptyImages() })
+	for _, v := range versions {
+		err := preloader.AddImage(DockerImageName(v.String()))
+		require.NoError(t, err)
+	}
+
+	for i := 0; i < numNodes; i++ {
+		// each node begins with a random version within the same major version set
+		v := versions.Random(r).String()
+		t.Log("Starting node", "node", i, "version", v)
+		require.NoError(t, testnet.CreateGenesisNode(v, 10000000))
+	}
+
+	kr, err := testnet.CreateAccount("alice", 1e12)
+	require.NoError(t, err)
+
+	require.NoError(t, testnet.Setup())
+	require.NoError(t, testnet.Start())
+
+	sequences := txsim.NewBlobSequence(txsim.NewRange(200, 4000), txsim.NewRange(1, 3)).Clone(5)
+	sequences = append(sequences, txsim.NewSendSequence(4, 1000, 100).Clone(5)...)
+
+	errCh := make(chan error)
+	encCfg := encoding.MakeConfig(app.ModuleEncodingRegisters...)
+	opts := txsim.DefaultOptions().WithSeed(seed)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	go func() {
+		errCh <- txsim.Run(ctx, testnet.GRPCEndpoints()[0], kr, encCfg, opts, sequences...)
+	}()
+
+	for i := 0; i < len(versions)*2; i++ {
+		// FIXME: skip the first node because we need them available to
+		// submit txs
+		if i%numNodes == 0 {
+			continue
+		}
+		client, err := testnet.Node(i % numNodes).Client()
+		require.NoError(t, err)
+		heightBefore, err := getHeight(ctx, client, time.Second)
+		require.NoError(t, err)
+		newVersion := versions.Random(r).String()
+		t.Log("Upgrading node", "node", i%numNodes, "version", newVersion)
+		err = testnet.Node(i % numNodes).Upgrade(newVersion)
+		require.NoError(t, err)
+		// wait for the node to reach two more heights
+		err = waitForHeight(ctx, client, heightBefore+2, 30*time.Second)
+		require.NoError(t, err)
+	}
+
+	heights := make([]int64, 4)
+	for i := 0; i < numNodes; i++ {
+		client, err := testnet.Node(i).Client()
+		require.NoError(t, err)
+		heights[i], err = getHeight(ctx, client, time.Second)
+		require.NoError(t, err)
+	}
+
+	t.Log("checking that all nodes are at the same height")
+	const maxPermissableDiff = 2
+	for i := 0; i < len(heights); i++ {
+		for j := i + 1; j < len(heights); j++ {
+			diff := heights[i] - heights[j]
+			if diff > maxPermissableDiff {
+				t.Fatalf("node %d is behind node %d by %d blocks", j, i, diff)
+			}
+		}
+	}
+
+	// end the tx sim
+	cancel()
+
+	err = <-errCh
+	require.True(t, errors.Is(err, context.Canceled), err.Error())
+}
+
+func getHeight(ctx context.Context, client *http.HTTP, period time.Duration) (int64, error) {
+	timer := time.NewTimer(period)
+	ticker := time.NewTicker(100 * time.Millisecond)
+	for {
+		select {
+		case <-timer.C:
+			return 0, fmt.Errorf("failed to get height after %.2f seconds", period.Seconds())
+		case <-ticker.C:
+			status, err := client.Status(ctx)
+			if err == nil {
+				return status.SyncInfo.LatestBlockHeight, nil
+			}
+			if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+				return 0, err
+			}
+		}
+	}
+}
+
+func waitForHeight(ctx context.Context, client *http.HTTP, height int64, period time.Duration) error {
+	timer := time.NewTimer(period)
+	ticker := time.NewTicker(100 * time.Millisecond)
+	for {
+		select {
+		case <-timer.C:
+			return fmt.Errorf("failed to reach height %d in %.2f seconds", height, period.Seconds())
+		case <-ticker.C:
+			status, err := client.Status(ctx)
+			if err != nil {
+				return err
+			}
+			if status.SyncInfo.LatestBlockHeight >= height {
+				return nil
+			}
+		}
+	}
+}

test/e2e/versions.go

@@ -0,0 +1,116 @@
+package e2e
+
+import (
+	"fmt"
+	"math/rand"
+	"sort"
+	"strings"
+)
+
+type Version struct {
+	Major uint64
+	Minor uint64
+	Patch uint64
+	IsRC  bool
+	RC    uint64
+}
+
+func (v Version) String() string {
+	if v.IsRC {
+		return fmt.Sprintf("v%d.%d.%d-rc%d", v.Major, v.Minor, v.Patch, v.RC)
+	}
+	return fmt.Sprintf("v%d.%d.%d", v.Major, v.Minor, v.Patch)
+}
+
+func (v Version) IsGreater(v2 Version) bool {
+	if v.Major != v2.Major {
+		return v.Major > v2.Major
+	}
+	if v.Minor != v2.Minor {
+		return v.Minor > v2.Minor
+	}
+	if v.Patch != v2.Patch {
+		return v.Patch > v2.Patch
+	}
+	if v.IsRC != v2.IsRC {
+		return !v.IsRC
+	}
+	return v.RC > v2.RC
+}
+
+type VersionSet []Version
+
+func ParseVersions(versionStr string) VersionSet {
+	versions := strings.Split(versionStr, " ")
+	output := make(VersionSet, 0, len(versions))
+	for _, v := range versions {
+		var major, minor, patch, rc uint64
+		isRC := false
+		if strings.Contains(v, "rc") {
+			_, err := fmt.Sscanf(v, "v%d.%d.%d-rc%d", &major, &minor, &patch, &rc)
+			isRC = true
+			if err != nil {
+				continue
+			}
+		} else {
+			_, err := fmt.Sscanf(v, "v%d.%d.%d", &major, &minor, &patch)
+			if err != nil {
+				continue
+			}
+		}
+		output = append(output, Version{major, minor, patch, isRC, rc})
+	}
+	return output
+}
+
+func (v VersionSet) FilterMajor(majorVersion uint64) VersionSet {
+	output := make(VersionSet, 0, len(v))
+	for _, version := range v {
+		if version.Major == majorVersion {
+			output = append(output, version)
+		}
+	}
+	return output
+}
+
+func (v VersionSet) FilterOutReleaseCandidates() VersionSet {
+	output := make(VersionSet, 0, len(v))
+	for _, version := range v {
+		if version.IsRC {
+			continue
+		}
+		output = append(output, version)
+	}
+	return output
+}
+
+func (v VersionSet) GetLatest() Version {
+	latest := Version{}
+	for _, version := range v {
+		if version.IsGreater(latest) {
+			latest = version
+		}
+	}
+	return latest
+}
+
+func (v VersionSet) Order() {
+	sort.Slice(v, func(i, j int) bool {
+		return v[j].IsGreater(v[i])
+	})
+}
+
+func (v VersionSet) Random(r *rand.Rand) Version {
+	if len(v) == 0 {
+		panic("there are no versions to pick from")
+	}
+	return v[r.Intn(len(v))]
+}
+
+func (v VersionSet) String() string {
+	output := make([]string, len(v))
+	for i, version := range v {
+		output[i] = version.String()
+	}
+	return strings.Join(output, "\t")
+}

test/e2e/versions_test.go

@@ -0,0 +1,49 @@
+package e2e_test
+
+import (
+	"testing"
+
+	"github.com/celestiaorg/celestia-app/test/e2e"
+	"github.com/stretchr/testify/require"
+)
+
+func TestVersionParsing(t *testing.T) {
+	versionStr := "v1.3.0 v1.1.0 v1.2.0-rc0"
+	versions := e2e.ParseVersions(versionStr)
+	require.Len(t, versions, 3)
+	require.Len(t, versions.FilterOutReleaseCandidates(), 2)
+	require.Equal(t, versions.GetLatest(), e2e.Version{1, 3, 0, false, 0})
+}
+
+// Test case with multiple major versions and filtering out a single major version
+func TestFilterMajorVersions(t *testing.T) {
+	versionStr := "v2.0.0 v1.1.0 v2.1.0-rc0 v1.2.0 v2.2.0 v1.3.0"
+	versions := e2e.ParseVersions(versionStr)
+	require.Len(t, versions, 6)
+	require.Len(t, versions.FilterMajor(1), 3)
+}
+
+// Test case to check the Order function
+func TestOrder(t *testing.T) {
+	versionStr := "v1.3.0 v1.1.0 v1.2.0-rc0 v1.4.0 v1.2.1 v2.0.0"
+	versions := e2e.ParseVersions(versionStr)
+	versions.Order()
+	require.Equal(t, versions[0], e2e.Version{1, 1, 0, false, 0})
+	require.Equal(t, versions[1], e2e.Version{1, 2, 0, true, 0})
+	require.Equal(t, versions[2], e2e.Version{1, 2, 1, false, 0})
+	require.Equal(t, versions[3], e2e.Version{1, 3, 0, false, 0})
+	require.Equal(t, versions[4], e2e.Version{1, 4, 0, false, 0})
+	require.Equal(t, versions[5], e2e.Version{2, 0, 0, false, 0})
+	for i := len(versions) - 1; i > 0; i-- {
+		require.True(t, versions[i].IsGreater(versions[i-1]))
+	}
+}
+
+func TestOrderOfReleaseCandidates(t *testing.T) {
+	versionsStr := "v1.0.0 v1.0.0-rc0 v1.0.0-rc1"
+	versions := e2e.ParseVersions(versionsStr)
+	versions.Order()
+	require.Equal(t, versions[0], e2e.Version{1, 0, 0, true, 0})
+	require.Equal(t, versions[1], e2e.Version{1, 0, 0, true, 1})
+	require.Equal(t, versions[2], e2e.Version{1, 0, 0, false, 0})
+}