Merge branch 'main' into use_file_input

.github/workflows/nightly.yml

@@ -27,14 +27,5 @@ jobs:
       - name: Run e2e tests
         run: make test-e2e
 
-  test-fuzz:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: actions/setup-go@v4
-        with:
-          go-version: ${{ env.GO_VERSION }}
-
-      - name: Run fuzz tests
-        run: make test-fuzz
+  test:
+    uses: ./.github/workflows/test.yml

.github/workflows/test.yml

@@ -59,3 +59,15 @@ jobs:
 
       - name: Run tests in race mode
         run: make test-race
+
+  test-fuzz:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v4
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: Run fuzz tests
+        run: make test-fuzz

Dockerfile

@@ -1,5 +1,5 @@
 # stage 1 Generate celestia-appd Binary
-FROM --platform=$BUILDPLATFORM docker.io/golang:1.21.3-alpine3.18 as builder
+FROM --platform=$BUILDPLATFORM docker.io/golang:1.21.4-alpine3.18 as builder
 
 ARG TARGETOS
 ARG TARGETARCH

Makefile

@@ -1,6 +1,7 @@
 VERSION := $(shell echo $(shell git describe --tags 2>/dev/null || git log -1 --format='%h') | sed 's/^v//')
 COMMIT := $(shell git log -1 --format='%H')
 DOCKER := $(shell which docker)
+ALL_VERSIONS := $(shell git tag -l)
 DOCKER_BUF := $(DOCKER) run --rm -v $(CURDIR):/workspace --workdir /workspace bufbuild/buf
 IMAGE := ghcr.io/tendermint/docker-build-proto:latest
 DOCKER_PROTO_BUILDER := docker run -v $(shell pwd):/workspace --workdir /workspace $(IMAGE)
@@ -115,10 +116,10 @@ test-short:
 	@go test ./... -short -timeout 1m
 .PHONY: test-short
 
-## test-e2e: Run end to end tests via knuu.
+## test-e2e: Run end to end tests via knuu. This command requires a kube/config file to configure kubernetes.
 test-e2e:
-	@echo "--> Running e2e tests on version: $(shell git rev-parse --short HEAD)"
-	@KNUU_NAMESPACE=test E2E_VERSION=$(shell git rev-parse --short HEAD) E2E=true go test ./test/e2e/... -timeout 10m -v
+	@echo "--> Running end to end tests"
+	@KNUU_NAMESPACE=test KNUU_TIMEOUT=20m E2E_VERSIONS="$(ALL_VERSIONS)" E2E=true go test ./test/e2e/... -timeout 20m -v
 .PHONY: test-e2e
 
 ## test-race: Run tests in race mode.

app/app.go

@@ -85,6 +85,7 @@ import (
 	"github.com/celestiaorg/celestia-app/app/ante"
 	"github.com/celestiaorg/celestia-app/app/encoding"
 	"github.com/celestiaorg/celestia-app/pkg/appconsts"
+	v2 "github.com/celestiaorg/celestia-app/pkg/appconsts/v2"
 	"github.com/celestiaorg/celestia-app/pkg/proof"
 	blobmodule "github.com/celestiaorg/celestia-app/x/blob"
 	blobmodulekeeper "github.com/celestiaorg/celestia-app/x/blob/keeper"
@@ -239,23 +240,21 @@ type App struct {
 }
 
 // New returns a reference to an initialized celestia app.
+//
+// NOTE: upgradeHeight refers specifically to the height that
+// a node will upgrade from v1 to v2. It will be deprecated in v3
+// in place for a dynamically signalling scheme
 func New(
 	logger log.Logger,
 	db dbm.DB,
 	traceStore io.Writer,
 	loadLatest bool,
 	invCheckPeriod uint,
 	encodingConfig encoding.Config,
-	upgradeSchedule map[string]upgrade.Schedule,
+	upgradeHeight int64,
 	appOpts servertypes.AppOptions,
 	baseAppOptions ...func(*baseapp.BaseApp),
 ) *App {
-	for _, schedule := range upgradeSchedule {
-		if err := schedule.ValidateVersions(supportedVersions); err != nil {
-			panic(err)
-		}
-	}
-
 	appCodec := encodingConfig.Codec
 	cdc := encodingConfig.Amino
 	interfaceRegistry := encodingConfig.InterfaceRegistry
@@ -334,7 +333,7 @@ func New(
 	)
 
 	app.FeeGrantKeeper = feegrantkeeper.NewKeeper(appCodec, keys[feegrant.StoreKey], app.AccountKeeper)
-	app.UpgradeKeeper = upgrade.NewKeeper(keys[upgrade.StoreKey], upgradeSchedule)
+	app.UpgradeKeeper = upgrade.NewKeeper(keys[upgrade.StoreKey], upgradeHeight)
 
 	app.BlobstreamKeeper = *bsmodulekeeper.NewKeeper(
 		appCodec,
@@ -574,14 +573,10 @@ func (app *App) BeginBlocker(ctx sdk.Context, req abci.RequestBeginBlock) abci.R
 // EndBlocker application updates every end block
 func (app *App) EndBlocker(ctx sdk.Context, req abci.RequestEndBlock) abci.ResponseEndBlock {
 	res := app.mm.EndBlock(ctx, req)
-	if app.UpgradeKeeper.ShouldUpgrade() {
-		newAppVersion := app.UpgradeKeeper.GetNextAppVersion()
-		app.SetProtocolVersion(newAppVersion)
-		_, err := app.mm.RunMigrations(ctx, app.configurator, GetModuleVersion(newAppVersion))
-		if err != nil {
-			panic(err)
-		}
-		app.UpgradeKeeper.MarkUpgradeComplete()
+	// NOTE: this is a specific feature for upgrading to v2 as v3 and onward is expected
+	// to be coordinated through the signalling protocol
+	if app.UpgradeKeeper.ShouldUpgrade(req.Height) {
+		app.SetProtocolVersion(v2.Version)
 	}
 	return res
 }

app/default_overrides_test.go

@@ -9,6 +9,7 @@ import (
 	distributiontypes "github.com/cosmos/cosmos-sdk/x/distribution/types"
 	govtypes "github.com/cosmos/cosmos-sdk/x/gov/types/v1"
 	"github.com/stretchr/testify/assert"
+	tmcfg "github.com/tendermint/tendermint/config"
 )
 
 // Test_newGovModule verifies that the gov module's genesis state has defaults
@@ -63,3 +64,28 @@ func TestDefaultAppConfig(t *testing.T) {
 	assert.Equal(t, uint32(2), cfg.StateSync.SnapshotKeepRecent)
 	assert.Equal(t, "0.1utia", cfg.MinGasPrices)
 }
+
+func TestDefaultConsensusConfig(t *testing.T) {
+	got := DefaultConsensusConfig()
+
+	t.Run("mempool overrides", func(t *testing.T) {
+		want := tmcfg.MempoolConfig{
+			// defaults
+			Broadcast:             tmcfg.DefaultMempoolConfig().Broadcast,
+			CacheSize:             tmcfg.DefaultMempoolConfig().CacheSize,
+			KeepInvalidTxsInCache: tmcfg.DefaultMempoolConfig().KeepInvalidTxsInCache,
+			Recheck:               tmcfg.DefaultMempoolConfig().Recheck,
+			RootDir:               tmcfg.DefaultMempoolConfig().RootDir,
+			Size:                  tmcfg.DefaultMempoolConfig().Size,
+			WalPath:               tmcfg.DefaultMempoolConfig().WalPath,
+
+			// overrides
+			MaxTxBytes:   7_897_088,
+			MaxTxsBytes:  39_485_440,
+			TTLDuration:  75 * time.Second,
+			TTLNumBlocks: 5,
+			Version:      "v1",
+		}
+		assert.Equal(t, want, *got.Mempool)
+	})
+}

app/prepare_proposal.go

@@ -7,7 +7,6 @@ import (
 	"github.com/celestiaorg/celestia-app/pkg/da"
 	"github.com/celestiaorg/celestia-app/pkg/shares"
 	"github.com/celestiaorg/celestia-app/pkg/square"
-	"github.com/celestiaorg/celestia-app/x/upgrade"
 	"github.com/cosmos/cosmos-sdk/telemetry"
 	abci "github.com/tendermint/tendermint/abci/types"
 	core "github.com/tendermint/tendermint/proto/tendermint/types"
@@ -59,27 +58,6 @@ func (app *App) PrepareProposal(req abci.RequestPrepareProposal) abci.ResponsePr
 		txs = make([][]byte, 0)
 	} else {
 		txs = FilterTxs(app.Logger(), sdkCtx, handler, app.txConfig, req.BlockData.Txs)
-
-		// TODO: this would be improved if we only attempted the upgrade in the first round of the
-		// height to still allow transactions to pass through without being delayed from trying
-		// to coordinate the upgrade height
-		if newVersion, ok := app.UpgradeKeeper.ShouldProposeUpgrade(req.ChainId, req.Height); ok && newVersion > app.GetBaseApp().AppVersion() {
-			upgradeTx, err := upgrade.NewMsgVersionChange(app.txConfig, newVersion)
-			if err != nil {
-				panic(err)
-			}
-			// the upgrade transaction must be the first transaction in the block
-			txs = append([][]byte{upgradeTx}, txs...)
-
-			// because we are adding bytes, we need to check that we are not going over the limit
-			// if we are, we continually prune the last tx (the lowest paying blobTx).
-			size := sizeOf(txs)
-			for size > int(req.BlockDataSize) {
-				lastTx := txs[len(txs)-1]
-				txs = txs[:len(txs)-1]
-				size -= len(lastTx)
-			}
-		}
 	}
 
 	// build the square from the set of valid and prioritised transactions.
@@ -124,11 +102,3 @@ func (app *App) PrepareProposal(req abci.RequestPrepareProposal) abci.ResponsePr
 		},
 	}
 }
-
-func sizeOf(txs [][]byte) int {
-	size := 0
-	for _, tx := range txs {
-		size += len(tx)
-	}
-	return size
-}

app/process_proposal.go

@@ -11,7 +11,6 @@ import (
 	"github.com/celestiaorg/celestia-app/pkg/shares"
 	"github.com/celestiaorg/celestia-app/pkg/square"
 	blobtypes "github.com/celestiaorg/celestia-app/x/blob/types"
-	"github.com/celestiaorg/celestia-app/x/upgrade"
 	"github.com/cosmos/cosmos-sdk/telemetry"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	abci "github.com/tendermint/tendermint/abci/types"
@@ -76,27 +75,6 @@ func (app *App) ProcessProposal(req abci.RequestProcessProposal) (resp abci.Resp
 				return reject()
 			}
 
-			if appVersion, ok := upgrade.IsUpgradeMsg(msgs); ok {
-				if idx != 0 {
-					logInvalidPropBlock(app.Logger(), req.Header, fmt.Sprintf("upgrade message %d is not the first transaction", idx))
-					return reject()
-				}
-
-				if !IsSupported(appVersion) {
-					logInvalidPropBlock(app.Logger(), req.Header, fmt.Sprintf("block proposes an unsupported app version %d", appVersion))
-					return reject()
-				}
-
-				// app version must always increase
-				if appVersion <= app.GetBaseApp().AppVersion() {
-					logInvalidPropBlock(app.Logger(), req.Header, fmt.Sprintf("block proposes an app version %d that is not greater than the current app version %d", appVersion, app.GetBaseApp().AppVersion()))
-					return reject()
-				}
-
-				// we don't need to pass this message through the ante handler
-				continue
-			}
-
 			// we need to increment the sequence for every transaction so that
 			// the signature check below is accurate. this error only gets hit
 			// if the account in question doens't exist.
@@ -159,7 +137,7 @@ func (app *App) ProcessProposal(req abci.RequestProcessProposal) (resp abci.Resp
 	// are identical and that square layout is consistent. This also means that the share commitment rules
 	// have been followed and thus each blobs share commitment should be valid
 	if !bytes.Equal(dah.Hash(), req.Header.DataHash) {
-		logInvalidPropBlock(app.Logger(), req.Header, "proposed data root differs from calculated data root")
+		logInvalidPropBlock(app.Logger(), req.Header, fmt.Sprintf("proposed data root %X differs from calculated data root %X", req.Header.DataHash, dah.Hash()))
 		return reject()
 	}
 

cmd/celestia-appd/cmd/download-genesis.go → cmd/celestia-appd/cmd/download_genesis.go

@@ -12,6 +12,9 @@ import (
 	"github.com/spf13/cobra"
 )
 
+// chainIDToSha256 is a map of chainID to the SHA-256 hash of the genesis file for that chain ID.
+// To add a new chain-id, download the genesis file from the networks repo and compute the SHA-256 hash.
+// Add the chain-id and hash to this map.
 var chainIDToSha256 = map[string]string{
 	"celestia":   "9727aac9bbfb021ce7fc695a92f901986421283a891b89e0af97bc9fad187793",
 	"mocha-4":    "0846b99099271b240b638a94e17a6301423b5e4047f6558df543d6e91db7e575",
@@ -75,11 +78,7 @@ func getChainIDOrDefault(args []string) string {
 
 // isKnownChainID returns true if the chainID is known.
 func isKnownChainID(chainID string) bool {
-	knownChainIDs := []string{
-		"arabica-10", // testnet
-		"mocha-4",    // testnet
-		"celestia",   // mainnet
-	}
+	knownChainIDs := getKeys(chainIDToSha256)
 	return contains(knownChainIDs, chainID)
 }
 
@@ -127,3 +126,10 @@ func computeSha256(filepath string) (string, error) {
 
 	return hex.EncodeToString(hasher.Sum(nil)), nil
 }
+
+func getKeys(m map[string]string) (result []string) {
+	for key := range m {
+		result = append(result, key)
+	}
+	return result
+}

cmd/celestia-appd/cmd/download_genesis_test.go

@@ -0,0 +1,27 @@
+package cmd
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_isKnownChainID(t *testing.T) {
+	type testCase struct {
+		chainID string
+		want    bool
+	}
+	testCases := []testCase{
+		{"celestia", true},
+		{"mocha-4", true},
+		{"arabica-10", true},
+		{"foo", false},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.chainID, func(t *testing.T) {
+			got := isKnownChainID(tc.chainID)
+			assert.Equal(t, tc.want, got)
+		})
+	}
+}