test: minor version upgrade e2e testing

Makefile

@@ -115,10 +115,12 @@ test-short:
 	@go test ./... -short -timeout 1m
 .PHONY: test-short
 
-## test-e2e: Run end to end tests via knuu.
+## test-e2e: Run end to end tests via knuu. This requires a kube/config file to configure the kubernetes. Without
+## this, the test will not work
 test-e2e:
-	@echo "--> Running e2e tests on version: $(shell git rev-parse --short HEAD)"
-	@KNUU_NAMESPACE=test E2E_VERSION=$(shell git rev-parse --short HEAD) E2E=true go test ./test/e2e/... -timeout 10m -v
+	@export E2E_VERSIONS=$(git tag -l)
+	@echo "--> Running end to end tests"
+	@KNUU_NAMESPACE=test E2E=true go test ./test/e2e/... -timeout 10m -v
 .PHONY: test-e2e
 
 ## test-race: Run tests in race mode.

test/e2e/node.go

@@ -50,7 +50,7 @@ func NewNode(
 	if err != nil {
 		return nil, err
 	}
-	err = instance.SetImage(fmt.Sprintf("%s:%s", dockerSrcURL, version))
+	err = instance.SetImage(DockerImageName(version))
 	if err != nil {
 		return nil, err
 	}
@@ -260,3 +260,11 @@ func (n *Node) Start() error {
 	n.grpcProxyPort = grpcProxyPort
 	return nil
 }
+
+func (n *Node) Upgrade(version string) error {
+	return n.Instance.SetImageInstant(DockerImageName(version))
+}
+
+func DockerImageName(version string) string {
+	return fmt.Sprintf("%s:%s", dockerSrcURL, version)
+}

test/e2e/simple_test.go

@@ -22,13 +22,16 @@ var latestVersion = "latest"
 // and MsgSends over 30 seconds and then asserts that at least 10 transactions were
 // committed.
 func TestE2ESimple(t *testing.T) {
-	if os.Getenv("E2E") == "" {
+	if os.Getenv("E2E") != "true" {
 		t.Skip("skipping e2e test")
 	}
 
-	if os.Getenv("E2E_VERSION") != "" {
-		latestVersion = os.Getenv("E2E_VERSION")
+	if os.Getenv("E2E_VERSIONS") != "" {
+		versionsStr := os.Getenv("E2E_VERSIONS")
+		versions := ParseVersions(versionsStr)
+		latestVersion = versions.GetLatest().String()
 	}
+	t.Log("Running simple e2e test", "version", latestVersion)
 
 	testnet, err := New(t.Name(), seed)
 	require.NoError(t, err)

test/e2e/upgrade_test.go

@@ -0,0 +1,156 @@
+package e2e
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math/rand"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/celestiaorg/celestia-app/app"
+	"github.com/celestiaorg/celestia-app/app/encoding"
+	"github.com/celestiaorg/celestia-app/test/txsim"
+	"github.com/celestiaorg/knuu/pkg/knuu"
+	"github.com/stretchr/testify/require"
+	"github.com/tendermint/tendermint/rpc/client/http"
+)
+
+// This will only run tests within the v1 major release cycle
+const MajorVersion = 1
+
+func TestMinorVersionCompatibility(t *testing.T) {
+	if os.Getenv("E2E") != "true" {
+		t.Skip("skipping e2e test")
+	}
+
+	if os.Getenv("E2E_VERSIONS") == "" {
+		t.Skip("skipping e2e test: E2E_VERSION not set")
+	}
+
+	versionStr := os.Getenv("E2E_VERSIONS")
+	versions := ParseVersions(versionStr).FilterMajor(MajorVersion).FilterOutReleaseCandidates()
+	numNodes := 4
+	r := rand.New(rand.NewSource(seed))
+	t.Log("Running minor version compatibility test", "versions", versions)
+
+	testnet, err := New(t.Name(), seed)
+	require.NoError(t, err)
+	t.Cleanup(testnet.Cleanup)
+
+	// preload all docker images
+	preloader, err := knuu.NewPreloader()
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = preloader.EmptyImages() })
+	for _, v := range versions {
+		err := preloader.AddImage(DockerImageName(v.String()))
+		require.NoError(t, err)
+	}
+
+	for i := 0; i < numNodes; i++ {
+		// each node begins with a random version within the same major version set
+		v := versions.Random(r).String()
+		t.Log("Starting node", "node", i, "version", v)
+		require.NoError(t, testnet.CreateGenesisNode(v, 10000000))
+	}
+
+	kr, err := testnet.CreateAccount("alice", 1e12)
+	require.NoError(t, err)
+
+	require.NoError(t, testnet.Setup())
+	require.NoError(t, testnet.Start())
+
+	sequences := txsim.NewBlobSequence(txsim.NewRange(200, 4000), txsim.NewRange(1, 3)).Clone(5)
+	sequences = append(sequences, txsim.NewSendSequence(4, 1000, 100).Clone(5)...)
+
+	errCh := make(chan error)
+	encCfg := encoding.MakeConfig(app.ModuleEncodingRegisters...)
+	opts := txsim.DefaultOptions().WithSeed(seed)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	go func() {
+		errCh <- txsim.Run(ctx, testnet.GRPCEndpoints()[0], kr, encCfg, opts, sequences...)
+	}()
+
+	for i := 0; i < len(versions)*2; i++ {
+		// FIXME: skip the first node because we need them available to
+		// submit txs
+		if i%numNodes == 0 {
+			continue
+		}
+		client, err := testnet.Node(i % numNodes).Client()
+		require.NoError(t, err)
+		heightBefore, err := getHeight(ctx, client, time.Second)
+		require.NoError(t, err)
+		newVersion := versions.Random(r).String()
+		t.Log("Upgrading node", "node", i%numNodes, "version", newVersion)
+		err = testnet.Node(i % numNodes).Upgrade(newVersion)
+		require.NoError(t, err)
+		// wait for the node to reach two more heights
+		err = waitForHeight(ctx, client, heightBefore+2, 30*time.Second)
+		require.NoError(t, err)
+	}
+
+	heights := make([]int64, 4)
+	for i := 0; i < numNodes; i++ {
+		client, err := testnet.Node(i).Client()
+		require.NoError(t, err)
+		heights[i], err = getHeight(ctx, client, time.Second)
+		require.NoError(t, err)
+	}
+
+	const maxPermissableDiff = 2
+	for i := 0; i < len(heights); i++ {
+		for j := i + 1; j < len(heights); j++ {
+			diff := heights[i] - heights[j]
+			if diff > maxPermissableDiff {
+				t.Fatalf("node %d is behind node %d by %d blocks", j, i, diff)
+			}
+		}
+	}
+
+	// end the tx sim
+	cancel()
+
+	err = <-errCh
+	require.True(t, errors.Is(err, context.Canceled), err.Error())
+}
+
+func getHeight(ctx context.Context, client *http.HTTP, period time.Duration) (int64, error) {
+	timer := time.NewTimer(period)
+	ticker := time.NewTicker(100 * time.Millisecond)
+	for {
+		select {
+		case <-timer.C:
+			return 0, fmt.Errorf("failed to get height after %.2f seconds", period.Seconds())
+		case <-ticker.C:
+			status, err := client.Status(ctx)
+			if err == nil {
+				return status.SyncInfo.LatestBlockHeight, nil
+			}
+			if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+				return 0, err
+			}
+		}
+	}
+}
+
+func waitForHeight(ctx context.Context, client *http.HTTP, height int64, period time.Duration) error {
+	timer := time.NewTimer(period)
+	ticker := time.NewTicker(100 * time.Millisecond)
+	for {
+		select {
+		case <-timer.C:
+			return fmt.Errorf("failed to reach height %d in %.2f seconds", height, period.Seconds())
+		case <-ticker.C:
+			status, err := client.Status(ctx)
+			if err != nil {
+				return err
+			}
+			if status.SyncInfo.LatestBlockHeight >= height {
+				return nil
+			}
+		}
+	}
+}

test/e2e/versions.go

@@ -0,0 +1,113 @@
+package e2e
+
+import (
+	"fmt"
+	"math/rand"
+	"sort"
+	"strings"
+)
+
+type Version struct {
+	Major uint64
+	Minor uint64
+	Patch uint64
+	IsRC  bool
+	RC    uint64
+}
+
+func (v Version) String() string {
+	if v.IsRC {
+		return fmt.Sprintf("v%d.%d.%d-rc%d", v.Major, v.Minor, v.Patch, v.RC)
+	}
+	return fmt.Sprintf("v%d.%d.%d", v.Major, v.Minor, v.Patch)
+}
+
+func (v Version) IsGreater(v2 Version) bool {
+	if v.Major != v2.Major {
+		return v.Major > v2.Major
+	}
+	if v.Minor != v2.Minor {
+		return v.Minor > v2.Minor
+	}
+	if v.Patch != v2.Patch {
+		return v.Patch > v2.Patch
+	}
+	if v.IsRC != v2.IsRC {
+		return !v.IsRC
+	}
+	return v.RC > v2.RC
+}
+
+type VersionSet []Version
+
+func ParseVersions(versionStr string) VersionSet {
+	versions := strings.Split(versionStr, "\n")
+	output := make(VersionSet, 0, len(versions))
+	for _, v := range versions {
+		var major, minor, patch, rc uint64
+		isRC := false
+		if strings.Contains(v, "rc") {
+			_, err := fmt.Sscanf(v, "v%d.%d.%d-rc%d", &major, &minor, &patch, &rc)
+			isRC = true
+			if err != nil {
+				continue
+			}
+		} else {
+			_, err := fmt.Sscanf(v, "v%d.%d.%d", &major, &minor, &patch)
+			if err != nil {
+				continue
+			}
+		}
+		output = append(output, Version{major, minor, patch, isRC, rc})
+	}
+	return output
+}
+
+func (v VersionSet) FilterMajor(majorVersion uint64) VersionSet {
+	output := make(VersionSet, 0, len(v))
+	for _, version := range v {
+		if version.Major == majorVersion {
+			output = append(output, version)
+		}
+	}
+	return output
+}
+
+func (v VersionSet) FilterOutReleaseCandidates() VersionSet {
+	output := make(VersionSet, 0, len(v))
+	for _, version := range v {
+		if version.IsRC {
+			continue
+		}
+		output = append(output, version)
+	}
+	return output
+}
+
+func (v VersionSet) GetLatest() Version {
+	latest := Version{}
+	for _, version := range v {
+		if version.IsGreater(latest) {
+			latest = version
+		}
+	}
+	return latest
+}
+
+func (v VersionSet) Order() {
+	sort.Slice(v, func(i, j int) bool {
+		return v[j].IsGreater(v[i])
+	})
+}
+
+func (v VersionSet) Random(r *rand.Rand) Version {
+	return v[r.Intn(len(v))]
+}
+
+func (v VersionSet) String() string {
+	output := make([]string, len(v))
+	for i, version := range v {
+		output[i] = version.String()
+	}
+	return strings.Join(output, "\t")
+}

test/e2e/versions_test.go

@@ -0,0 +1,49 @@
+package e2e_test
+
+import (
+	"testing"
+
+	"github.com/celestiaorg/celestia-app/test/e2e"
+	"github.com/stretchr/testify/require"
+)
+
+func TestVersionParsing(t *testing.T) {
+	versionStr := "v1.3.0\nv1.1.0\nv1.2.0-rc0"
+	versions := e2e.ParseVersions(versionStr)
+	require.Len(t, versions, 3)
+	require.Len(t, versions.FilterOutReleaseCandidates(), 2)
+	require.Equal(t, versions.GetLatest(), e2e.Version{1, 3, 0, false, 0})
+}
+
+// Test case with multiple major versions and filtering out a single major version
+func TestFilterMajorVersions(t *testing.T) {
+	versionStr := "v2.0.0\nv1.1.0\nv2.1.0-rc0\nv1.2.0\nv2.2.0\nv1.3.0"
+	versions := e2e.ParseVersions(versionStr)
+	require.Len(t, versions, 6)
+	require.Len(t, versions.FilterMajor(1), 3)
+}
+
+// Test case to check the Order function
+func TestOrder(t *testing.T) {
+	versionStr := "v1.3.0\nv1.1.0\nv1.2.0-rc0\nv1.4.0\nv1.2.1\nv2.0.0"
+	versions := e2e.ParseVersions(versionStr)
+	versions.Order()
+	require.Equal(t, versions[0], e2e.Version{1, 1, 0, false, 0})
+	require.Equal(t, versions[1], e2e.Version{1, 2, 0, true, 0})
+	require.Equal(t, versions[2], e2e.Version{1, 2, 1, false, 0})
+	require.Equal(t, versions[3], e2e.Version{1, 3, 0, false, 0})
+	require.Equal(t, versions[4], e2e.Version{1, 4, 0, false, 0})
+	require.Equal(t, versions[5], e2e.Version{2, 0, 0, false, 0})
+	for i := len(versions) - 1; i > 0; i-- {
+		require.True(t, versions[i].IsGreater(versions[i-1]))
+	}
+}
+
+func TestOrderOfReleaseCandidates(t *testing.T) {
+	versionsStr := "v1.0.0\nv1.0.0-rc0\nv1.0.0-rc1\n"
+	versions := e2e.ParseVersions(versionsStr)
+	versions.Order()
+	require.Equal(t, versions[0], e2e.Version{1, 0, 0, true, 0})
+	require.Equal(t, versions[1], e2e.Version{1, 0, 0, true, 1})
+	require.Equal(t, versions[2], e2e.Version{1, 0, 0, false, 0})
+}