Merge branch 'main' into dependabot/docker/golang-1.21.4-alpine3.18

Makefile

@@ -1,6 +1,7 @@
 VERSION := $(shell echo $(shell git describe --tags 2>/dev/null || git log -1 --format='%h') | sed 's/^v//')
 COMMIT := $(shell git log -1 --format='%H')
 DOCKER := $(shell which docker)
+ALL_VERSIONS := $(shell git tag -l)
 DOCKER_BUF := $(DOCKER) run --rm -v $(CURDIR):/workspace --workdir /workspace bufbuild/buf
 IMAGE := ghcr.io/tendermint/docker-build-proto:latest
 DOCKER_PROTO_BUILDER := docker run -v $(shell pwd):/workspace --workdir /workspace $(IMAGE)
@@ -115,10 +116,10 @@ test-short:
 	@go test ./... -short -timeout 1m
 .PHONY: test-short
 
-## test-e2e: Run end to end tests via knuu.
+## test-e2e: Run end to end tests via knuu. This command requires a kube/config file to configure kubernetes.
 test-e2e:
-	@echo "--> Running e2e tests on version: $(shell git rev-parse --short HEAD)"
-	@KNUU_NAMESPACE=test E2E_VERSION=$(shell git rev-parse --short HEAD) E2E=true go test ./test/e2e/... -timeout 10m -v
+	@echo "--> Running end to end tests"
+	@KNUU_NAMESPACE=test KNUU_TIMEOUT=20m E2E_VERSIONS="$(ALL_VERSIONS)" E2E=true go test ./test/e2e/... -timeout 20m -v
 .PHONY: test-e2e
 
 ## test-race: Run tests in race mode.

app/app.go

@@ -85,6 +85,7 @@ import (
 	"github.com/celestiaorg/celestia-app/app/ante"
 	"github.com/celestiaorg/celestia-app/app/encoding"
 	"github.com/celestiaorg/celestia-app/pkg/appconsts"
+	v2 "github.com/celestiaorg/celestia-app/pkg/appconsts/v2"
 	"github.com/celestiaorg/celestia-app/pkg/proof"
 	blobmodule "github.com/celestiaorg/celestia-app/x/blob"
 	blobmodulekeeper "github.com/celestiaorg/celestia-app/x/blob/keeper"
@@ -239,23 +240,21 @@ type App struct {
 }
 
 // New returns a reference to an initialized celestia app.
+//
+// NOTE: upgradeHeight refers specifically to the height that
+// a node will upgrade from v1 to v2. It will be deprecated in v3
+// in place for a dynamically signalling scheme
 func New(
 	logger log.Logger,
 	db dbm.DB,
 	traceStore io.Writer,
 	loadLatest bool,
 	invCheckPeriod uint,
 	encodingConfig encoding.Config,
-	upgradeSchedule map[string]upgrade.Schedule,
+	upgradeHeight int64,
 	appOpts servertypes.AppOptions,
 	baseAppOptions ...func(*baseapp.BaseApp),
 ) *App {
-	for _, schedule := range upgradeSchedule {
-		if err := schedule.ValidateVersions(supportedVersions); err != nil {
-			panic(err)
-		}
-	}
-
 	appCodec := encodingConfig.Codec
 	cdc := encodingConfig.Amino
 	interfaceRegistry := encodingConfig.InterfaceRegistry
@@ -334,7 +333,7 @@ func New(
 	)
 
 	app.FeeGrantKeeper = feegrantkeeper.NewKeeper(appCodec, keys[feegrant.StoreKey], app.AccountKeeper)
-	app.UpgradeKeeper = upgrade.NewKeeper(keys[upgrade.StoreKey], upgradeSchedule)
+	app.UpgradeKeeper = upgrade.NewKeeper(keys[upgrade.StoreKey], upgradeHeight)
 
 	app.BlobstreamKeeper = *bsmodulekeeper.NewKeeper(
 		appCodec,
@@ -574,14 +573,10 @@ func (app *App) BeginBlocker(ctx sdk.Context, req abci.RequestBeginBlock) abci.R
 // EndBlocker application updates every end block
 func (app *App) EndBlocker(ctx sdk.Context, req abci.RequestEndBlock) abci.ResponseEndBlock {
 	res := app.mm.EndBlock(ctx, req)
-	if app.UpgradeKeeper.ShouldUpgrade() {
-		newAppVersion := app.UpgradeKeeper.GetNextAppVersion()
-		app.SetProtocolVersion(newAppVersion)
-		_, err := app.mm.RunMigrations(ctx, app.configurator, GetModuleVersion(newAppVersion))
-		if err != nil {
-			panic(err)
-		}
-		app.UpgradeKeeper.MarkUpgradeComplete()
+	// NOTE: this is a specific feature for upgrading to v2 as v3 and onward is expected
+	// to be coordinated through the signalling protocol
+	if app.UpgradeKeeper.ShouldUpgrade(req.Height) {
+		app.SetProtocolVersion(v2.Version)
 	}
 	return res
 }

app/default_overrides_test.go

@@ -9,6 +9,7 @@ import (
 	distributiontypes "github.com/cosmos/cosmos-sdk/x/distribution/types"
 	govtypes "github.com/cosmos/cosmos-sdk/x/gov/types/v1"
 	"github.com/stretchr/testify/assert"
+	tmcfg "github.com/tendermint/tendermint/config"
 )
 
 // Test_newGovModule verifies that the gov module's genesis state has defaults
@@ -63,3 +64,28 @@ func TestDefaultAppConfig(t *testing.T) {
 	assert.Equal(t, uint32(2), cfg.StateSync.SnapshotKeepRecent)
 	assert.Equal(t, "0.1utia", cfg.MinGasPrices)
 }
+
+func TestDefaultConsensusConfig(t *testing.T) {
+	got := DefaultConsensusConfig()
+
+	t.Run("mempool overrides", func(t *testing.T) {
+		want := tmcfg.MempoolConfig{
+			// defaults
+			Broadcast:             tmcfg.DefaultMempoolConfig().Broadcast,
+			CacheSize:             tmcfg.DefaultMempoolConfig().CacheSize,
+			KeepInvalidTxsInCache: tmcfg.DefaultMempoolConfig().KeepInvalidTxsInCache,
+			Recheck:               tmcfg.DefaultMempoolConfig().Recheck,
+			RootDir:               tmcfg.DefaultMempoolConfig().RootDir,
+			Size:                  tmcfg.DefaultMempoolConfig().Size,
+			WalPath:               tmcfg.DefaultMempoolConfig().WalPath,
+
+			// overrides
+			MaxTxBytes:   7_897_088,
+			MaxTxsBytes:  39_485_440,
+			TTLDuration:  75 * time.Second,
+			TTLNumBlocks: 5,
+			Version:      "v1",
+		}
+		assert.Equal(t, want, *got.Mempool)
+	})
+}

app/prepare_proposal.go

@@ -7,7 +7,6 @@ import (
 	"github.com/celestiaorg/celestia-app/pkg/da"
 	"github.com/celestiaorg/celestia-app/pkg/shares"
 	"github.com/celestiaorg/celestia-app/pkg/square"
-	"github.com/celestiaorg/celestia-app/x/upgrade"
 	"github.com/cosmos/cosmos-sdk/telemetry"
 	abci "github.com/tendermint/tendermint/abci/types"
 	core "github.com/tendermint/tendermint/proto/tendermint/types"
@@ -59,27 +58,6 @@ func (app *App) PrepareProposal(req abci.RequestPrepareProposal) abci.ResponsePr
 		txs = make([][]byte, 0)
 	} else {
 		txs = FilterTxs(app.Logger(), sdkCtx, handler, app.txConfig, req.BlockData.Txs)
-
-		// TODO: this would be improved if we only attempted the upgrade in the first round of the
-		// height to still allow transactions to pass through without being delayed from trying
-		// to coordinate the upgrade height
-		if newVersion, ok := app.UpgradeKeeper.ShouldProposeUpgrade(req.ChainId, req.Height); ok && newVersion > app.GetBaseApp().AppVersion() {
-			upgradeTx, err := upgrade.NewMsgVersionChange(app.txConfig, newVersion)
-			if err != nil {
-				panic(err)
-			}
-			// the upgrade transaction must be the first transaction in the block
-			txs = append([][]byte{upgradeTx}, txs...)
-
-			// because we are adding bytes, we need to check that we are not going over the limit
-			// if we are, we continually prune the last tx (the lowest paying blobTx).
-			size := sizeOf(txs)
-			for size > int(req.BlockDataSize) {
-				lastTx := txs[len(txs)-1]
-				txs = txs[:len(txs)-1]
-				size -= len(lastTx)
-			}
-		}
 	}
 
 	// build the square from the set of valid and prioritised transactions.
@@ -124,11 +102,3 @@ func (app *App) PrepareProposal(req abci.RequestPrepareProposal) abci.ResponsePr
 		},
 	}
 }
-
-func sizeOf(txs [][]byte) int {
-	size := 0
-	for _, tx := range txs {
-		size += len(tx)
-	}
-	return size
-}

app/process_proposal.go

@@ -11,7 +11,6 @@ import (
 	"github.com/celestiaorg/celestia-app/pkg/shares"
 	"github.com/celestiaorg/celestia-app/pkg/square"
 	blobtypes "github.com/celestiaorg/celestia-app/x/blob/types"
-	"github.com/celestiaorg/celestia-app/x/upgrade"
 	"github.com/cosmos/cosmos-sdk/telemetry"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	abci "github.com/tendermint/tendermint/abci/types"
@@ -76,27 +75,6 @@ func (app *App) ProcessProposal(req abci.RequestProcessProposal) (resp abci.Resp
 				return reject()
 			}
 
-			if appVersion, ok := upgrade.IsUpgradeMsg(msgs); ok {
-				if idx != 0 {
-					logInvalidPropBlock(app.Logger(), req.Header, fmt.Sprintf("upgrade message %d is not the first transaction", idx))
-					return reject()
-				}
-
-				if !IsSupported(appVersion) {
-					logInvalidPropBlock(app.Logger(), req.Header, fmt.Sprintf("block proposes an unsupported app version %d", appVersion))
-					return reject()
-				}
-
-				// app version must always increase
-				if appVersion <= app.GetBaseApp().AppVersion() {
-					logInvalidPropBlock(app.Logger(), req.Header, fmt.Sprintf("block proposes an app version %d that is not greater than the current app version %d", appVersion, app.GetBaseApp().AppVersion()))
-					return reject()
-				}
-
-				// we don't need to pass this message through the ante handler
-				continue
-			}
-
 			// we need to increment the sequence for every transaction so that
 			// the signature check below is accurate. this error only gets hit
 			// if the account in question doens't exist.
@@ -159,7 +137,7 @@ func (app *App) ProcessProposal(req abci.RequestProcessProposal) (resp abci.Resp
 	// are identical and that square layout is consistent. This also means that the share commitment rules
 	// have been followed and thus each blobs share commitment should be valid
 	if !bytes.Equal(dah.Hash(), req.Header.DataHash) {
-		logInvalidPropBlock(app.Logger(), req.Header, "proposed data root differs from calculated data root")
+		logInvalidPropBlock(app.Logger(), req.Header, fmt.Sprintf("proposed data root %X differs from calculated data root %X", req.Header.DataHash, dah.Hash()))
 		return reject()
 	}
 

cmd/celestia-appd/cmd/root.go

@@ -4,6 +4,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"strconv"
 
 	bscmd "github.com/celestiaorg/celestia-app/x/blobstream/client"
 
@@ -21,6 +22,7 @@ import (
 	"github.com/cosmos/cosmos-sdk/client/flags"
 	"github.com/cosmos/cosmos-sdk/client/keys"
 	"github.com/cosmos/cosmos-sdk/client/rpc"
+	"github.com/cosmos/cosmos-sdk/client/snapshot"
 	"github.com/cosmos/cosmos-sdk/server"
 	serverconfig "github.com/cosmos/cosmos-sdk/server/config"
 	servertypes "github.com/cosmos/cosmos-sdk/server/types"
@@ -44,6 +46,8 @@ const (
 
 	// FlagLogToFile specifies whether to log to file or not.
 	FlagLogToFile = "log-to-file"
+
+	UpgradeHeightFlag = "v2-upgrade-height"
 )
 
 // NewRootCmd creates a new root command for celestia-appd. It is called once in the
@@ -149,10 +153,14 @@ func initRootCmd(rootCmd *cobra.Command, encodingConfig encoding.Config) {
 		keys.Commands(app.DefaultNodeHome),
 		bscmd.VerifyCmd(),
 	)
+	rootCmd.AddCommand(
+		snapshot.Cmd(NewAppServer),
+	)
 }
 
 func addModuleInitFlags(startCmd *cobra.Command) {
 	crisis.AddModuleInitFlags(startCmd)
+	startCmd.Flags().Int64(UpgradeHeightFlag, 0, "Upgrade height to switch from v1 to v2. Must be coordinated amongst all validators")
 }
 
 func queryCommand() *cobra.Command {
@@ -229,11 +237,20 @@ func NewAppServer(logger log.Logger, db dbm.DB, traceStore io.Writer, appOpts se
 		panic(err)
 	}
 
+	var upgradeHeight int64
+	upgradeHeightStr, ok := appOpts.Get(UpgradeHeightFlag).(string)
+	if ok {
+		upgradeHeight, err = strconv.ParseInt(upgradeHeightStr, 10, 64)
+		if err != nil {
+			panic(err)
+		}
+	}
+
 	return app.New(
 		logger, db, traceStore, true,
 		cast.ToUint(appOpts.Get(server.FlagInvCheckPeriod)),
 		encoding.MakeConfig(app.ModuleEncodingRegisters...), // Ideally, we would reuse the one created by NewRootCmd.
-		nil,
+		upgradeHeight,
 		appOpts,
 		baseapp.SetPruning(pruningOpts),
 		baseapp.SetMinGasPrices(cast.ToString(appOpts.Get(server.FlagMinGasPrices))),
@@ -256,13 +273,13 @@ func createAppAndExport(
 	encCfg.Codec = codec.NewProtoCodec(encCfg.InterfaceRegistry)
 	var capp *app.App
 	if height != -1 {
-		capp = app.New(logger, db, traceStore, false, uint(1), encCfg, nil, appOpts)
+		capp = app.New(logger, db, traceStore, false, uint(1), encCfg, 0, appOpts)
 
 		if err := capp.LoadHeight(height); err != nil {
 			return servertypes.ExportedApp{}, err
 		}
 	} else {
-		capp = app.New(logger, db, traceStore, true, uint(1), encCfg, nil, appOpts)
+		capp = app.New(logger, db, traceStore, true, uint(1), encCfg, 0, appOpts)
 	}
 
 	return capp.ExportAppStateAndValidators(forZeroHeight, jailWhiteList)

docker/Dockerfile_txsim

@@ -1,5 +1,5 @@
 # Stage 1: generate celestia-appd binary
-FROM --platform=$BUILDPLATFORM docker.io/golang:1.21.3-alpine3.18 as builder
+FROM --platform=$BUILDPLATFORM docker.io/golang:1.21.4-alpine3.18 as builder
 
 ARG TARGETOS
 ARG TARGETARCH

go.mod

@@ -25,7 +25,7 @@ require (
 
 require (
 	cosmossdk.io/errors v1.0.0-beta.7
-	cosmossdk.io/math v1.1.2
+	cosmossdk.io/math v1.2.0
 	github.com/celestiaorg/blobstream-contracts/v3 v3.1.0
 	github.com/celestiaorg/rsmt2d v0.11.0
 	github.com/cosmos/cosmos-proto v1.0.0-alpha8
@@ -54,9 +54,9 @@ require (
 	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
-	github.com/go-openapi/jsonreference v0.20.1 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.3 // indirect
-	github.com/google/gnostic v0.5.7-v3refs // indirect
+	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/s2a-go v0.1.4 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
@@ -84,12 +84,12 @@ require (
 	golang.org/x/tools v0.13.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	k8s.io/api v0.27.3 // indirect
-	k8s.io/apimachinery v0.27.3 // indirect
-	k8s.io/client-go v0.27.3 // indirect
-	k8s.io/klog/v2 v2.90.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
-	k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect
+	k8s.io/api v0.28.2 // indirect
+	k8s.io/apimachinery v0.28.2 // indirect
+	k8s.io/client-go v0.28.2 // indirect
+	k8s.io/klog/v2 v2.100.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
+	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect
 	rsc.io/tmplfunc v0.0.3 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
@@ -113,7 +113,7 @@ require (
 	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
 	github.com/bgentry/speakeasy v0.1.0 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect
-	github.com/celestiaorg/knuu v0.8.2
+	github.com/celestiaorg/knuu v0.9.0
 	github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 // indirect
 	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect