diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 7097d57c6b..fbd09f5799 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -9,10 +9,3 @@ # global owners * @celestiaorg/celestia-core - -# directory owners -# NOTE: the directory owners should include the global owners unless the global -# owner is fully deferring ownership to the directory owner -docs @liamsi @celestiaorg/celestia-core -specs @liamsi @celestiaorg/celestia-core -x/blobstream @rach-id @evan-forbes diff --git a/.github/auto_request_review.yml b/.github/auto_request_review.yml index a00957c0e3..32ab61b10e 100644 --- a/.github/auto_request_review.yml +++ b/.github/auto_request_review.yml @@ -1,11 +1,11 @@ # More info at https://github.com/necojackarc/auto-request-review reviewers: - # The default reviewers defaults: - # Example of Github Team. Github team must have write access to repo. - # NOTE: This assigned the team itself, not members of the team. The Github - # team auto PR assignment will then turn this into individuals - - team:celestia-core # This is the Github Team + - cmwaters + - evan-forbes + - ninabarbakadze + - rach-id + - rootulp options: ignore_draft: true diff --git a/.github/workflows/ci-release.yml b/.github/workflows/ci-release.yml index 5fe828d636..65b50b48b2 100644 --- a/.github/workflows/ci-release.yml +++ b/.github/workflows/ci-release.yml @@ -9,6 +9,8 @@ on: - "v*" tags: - "v*" + release: + types: [published] pull_request: jobs: @@ -22,6 +24,7 @@ jobs: uses: ./.github/workflows/test.yml goreleaser: + if: github.event_name == 'release' uses: ./.github/workflows/goreleaser.yml permissions: write-all secrets: diff --git a/.github/workflows/goreleaser.yml b/.github/workflows/goreleaser.yml index dd265479e9..61a9161918 100644 --- a/.github/workflows/goreleaser.yml +++ b/.github/workflows/goreleaser.yml @@ -7,12 +7,15 @@ on: jobs: goreleaser-check: + if: github.event_name == 'release' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - run: git fetch --force --tags + - name: Set GORELEASER_CURRENT_TAG in GitHub env + run: echo "GORELEASER_CURRENT_TAG=${{ github.event.release.tag_name }}" >> $GITHUB_ENV - uses: actions/setup-go@v5 with: go-version-file: 'go.mod' @@ -21,18 +24,21 @@ jobs: echo 'GITHUB_TOKEN=${{secrets.GORELEASER_ACCESS_TOKEN}}' >> .release-env - name: Check the .goreleaser.yaml config file run: make goreleaser-check + env: + GORELEASER_CURRENT_TAG: ${{ env.GORELEASER_CURRENT_TAG }} goreleaser: + if: github.event_name == 'release' needs: goreleaser-check runs-on: ubuntu-latest - if: | - (github.event_name == 'push' && contains(github.ref, 'refs/tags/')) permissions: write-all steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - run: git fetch --force --tags + - name: Set GORELEASER_CURRENT_TAG in GitHub env + run: echo "GORELEASER_CURRENT_TAG=${{ github.event.release.tag_name }}" >> $GITHUB_ENV - uses: actions/setup-go@v5 with: go-version-file: 'go.mod' @@ -41,3 +47,5 @@ jobs: echo 'GITHUB_TOKEN=${{secrets.GORELEASER_ACCESS_TOKEN}}' >> .release-env - name: Create prebuilt binaries and attach them to the GitHub release run: make prebuilt-binary + env: + GORELEASER_CURRENT_TAG: ${{ env.GORELEASER_CURRENT_TAG }} diff --git a/Makefile b/Makefile index c734b1e596..7e7a511f45 100644 --- a/Makefile +++ b/Makefile @@ -1,20 +1,4 @@ -# GIT_TAG is an environment variable that is set to the latest git tag on the -# current commit with the following example priority: v2.2.0, v2.2.0-mocha, -# v2.2.0-arabica, v2.2.0-rc0, v2.2.0-beta, v2.2.0-alpha. If no tag points to the -# current commit, git describe is used. The priority in this command is -# necessary because `git tag --sort=-creatordate` only works for annotated tags -# with metadata. Git tags created via GitHub releases are not annotated and do -# not have metadata like creatordate. Therefore, this command is a hacky attempt -# to get the most recent tag on the current commit according to Celestia's -# testnet versioning scheme + SemVer. -GIT_TAG := $(shell git tag --points-at HEAD --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$$' \ - || git tag --points-at HEAD --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+-mocha$$' \ - || git tag --points-at HEAD --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+-arabica$$' \ - || git tag --points-at HEAD --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]*$$' \ - || git tag --points-at HEAD --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+-(beta)$$' \ - || git tag --points-at HEAD --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+-(alpha)$$' \ - || git describe --tags) -VERSION := $(shell echo $(GIT_TAG) | sed 's/^v//') +VERSION := $(shell echo $(shell git describe --tags 2>/dev/null || git log -1 --format='%h') | sed 's/^v//') COMMIT := $(shell git rev-parse --short HEAD) DOCKER := $(shell which docker) DOCKER_BUF := $(DOCKER) run --rm -v $(CURDIR):/workspace --workdir /workspace bufbuild/buf @@ -41,13 +25,14 @@ BUILD_FLAGS := -tags "ledger" -ldflags '$(ldflags)' ## help: Get more info on make commands. help: Makefile @echo " Choose a command run in "$(PROJECTNAME)":" - @sed -n 's/^##//p' $< | column -t -s ':' | sed -e 's/^/ /' + @sed -n 's/^##//p' $< | sort | column -t -s ':' | sed -e 's/^/ /' .PHONY: help ## build: Build the celestia-appd binary into the ./build directory. build: mod @cd ./cmd/celestia-appd @mkdir -p build/ + @echo "--> Building build/celestia-appd" @go build $(BUILD_FLAGS) -o build/ ./cmd/celestia-appd .PHONY: build @@ -95,25 +80,34 @@ proto-format: @$(DOCKER_PROTO_BUILDER) find . -name '*.proto' -path "./proto/*" -exec clang-format -i {} \; .PHONY: proto-format -## build-docker: Build the celestia-appd docker image from the current branch. Requires docker. build-docker: @echo "--> Building Docker image" $(DOCKER) build -t celestiaorg/celestia-app -f docker/Dockerfile . .PHONY: build-docker -## build-ghcr-docker: Build the celestia-appd docker image from the last commit. Requires docker. +## docker-build: Build the celestia-appd docker image from the current branch. Requires docker. +docker-build: build-docker +.PHONY: docker-build + build-ghcr-docker: @echo "--> Building Docker image" $(DOCKER) build -t ghcr.io/celestiaorg/celestia-app:$(COMMIT) -f docker/Dockerfile . .PHONY: build-ghcr-docker -## publish-ghcr-docker: Publish the celestia-appd docker image. Requires docker. +## docker-build-ghcr: Build the celestia-appd docker image from the last commit. Requires docker. +docker-build-ghcr: build-ghcr-docker +.PHONY: docker-build-ghcr + publish-ghcr-docker: # Make sure you are logged in and authenticated to the ghcr.io registry. @echo "--> Publishing Docker image" $(DOCKER) push ghcr.io/celestiaorg/celestia-app:$(COMMIT) .PHONY: publish-ghcr-docker +## docker-publish: Publish the celestia-appd docker image. Requires docker. +docker-publish: publish-ghcr-docker +.PHONY: docker-publish + ## lint: Run all linters; golangci-lint, markdownlint, hadolint, yamllint. lint: @echo "--> Running golangci-lint" @@ -127,14 +121,16 @@ lint: @yamllint --no-warnings . -c .yamllint.yml .PHONY: lint -## markdown-link-check: Check all markdown links. markdown-link-check: @echo "--> Running markdown-link-check" @find . -name \*.md -print0 | xargs -0 -n1 markdown-link-check .PHONY: markdown-link-check +## lint-links: Check all markdown links. +lint-links: markdown-link-check +.PHONY: lint-links + -## fmt: Format files per linters golangci-lint and markdownlint. fmt: @echo "--> Running golangci-lint --fix" @golangci-lint run --fix @@ -142,6 +138,10 @@ fmt: @markdownlint --fix --quiet --config .markdownlint.yaml . .PHONY: fmt +## lint-fix: Format files per linters golangci-lint and markdownlint. +lint-fix: fmt +.PHONY: lint-fix + ## test: Run tests. test: @echo "--> Running tests" @@ -220,7 +220,7 @@ goreleaser-check: docker run \ --rm \ --env CGO_ENABLED=1 \ - --env GORELEASER_CURRENT_TAG=${GIT_TAG} \ + --env GORELEASER_CURRENT_TAG=${GORELEASER_CURRENT_TAG} \ --env-file .release-env \ -v /var/run/docker.sock:/var/run/docker.sock \ -v `pwd`:/go/src/$(PACKAGE_NAME) \ @@ -229,7 +229,6 @@ goreleaser-check: check .PHONY: goreleaser-check -## prebuilt-binary: Create prebuilt binaries and attach them to GitHub release. Requires Docker. prebuilt-binary: @if [ ! -f ".release-env" ]; then \ echo "A .release-env file was not found but is required to create prebuilt binaries. This command is expected to be run in CI where a .release-env file exists. If you need to run this command locally to attach binaries to a release, you need to create a .release-env file with a Github token (classic) that has repo:public_repo scope."; \ @@ -238,7 +237,7 @@ prebuilt-binary: docker run \ --rm \ --env CGO_ENABLED=1 \ - --env GORELEASER_CURRENT_TAG=${GIT_TAG} \ + --env GORELEASER_CURRENT_TAG=${GORELEASER_CURRENT_TAG} \ --env-file .release-env \ -v /var/run/docker.sock:/var/run/docker.sock \ -v `pwd`:/go/src/$(PACKAGE_NAME) \ @@ -247,7 +246,10 @@ prebuilt-binary: release --clean .PHONY: prebuilt-binary -## check-bbr: Check if your system uses BBR congestion control algorithm. Only works on Linux. +## goreleaser: Create prebuilt binaries and attach them to GitHub release. Requires Docker. +goreleaser: prebuilt-binary +.PHONY: goreleaser + check-bbr: @echo "Checking if BBR is enabled..." @if [ "$$(sysctl net.ipv4.tcp_congestion_control | awk '{print $$3}')" != "bbr" ]; then \ @@ -257,7 +259,10 @@ check-bbr: fi .PHONY: check-bbr -## enable-bbr: Enable BBR congestion control algorithm. Only works on Linux. +## bbr-check: Check if your system uses BBR congestion control algorithm. Only works on Linux. +bbr-check: check-bbr +.PHONY: bbr-check + enable-bbr: @echo "Configuring system to use BBR..." @if [ "$(sysctl net.ipv4.tcp_congestion_control | awk '{print $3}')" != "bbr" ]; then \ @@ -273,7 +278,10 @@ enable-bbr: fi .PHONY: enable-bbr -## disable-bbr: Disable BBR congestion control algorithm and revert to default. +## bbr-enable: Enable BBR congestion control algorithm. Only works on Linux. +bbr-enable: enable-bbr +.PHONY: bbr-enable + disable-bbr: @echo "Disabling BBR and reverting to default congestion control algorithm..." @if [ "$$(sysctl net.ipv4.tcp_congestion_control | awk '{print $$3}')" = "bbr" ]; then \ @@ -289,7 +297,10 @@ disable-bbr: fi .PHONY: disable-bbr -## enable-mptcp: Enable mptcp over multiple ports (not interfaces). Only works on Linux Kernel 5.6 and above. +## bbr-disable: Disable BBR congestion control algorithm and revert to default. +bbr-disable: disable-bbr +.PHONY: bbr-disable + enable-mptcp: @echo "Configuring system to use mptcp..." @sudo sysctl -w net.mptcp.enabled=1 @@ -300,10 +311,12 @@ enable-mptcp: @echo "net.mptcp.mptcp_path_manager=ndiffports" | sudo tee -a /etc/sysctl.conf @echo "net.mptcp.mptcp_ndiffports=16" | sudo tee -a /etc/sysctl.conf @echo "MPTCP configuration complete and persistent!" - .PHONY: enable-mptcp -## disable-mptcp: Disables mptcp over multiple ports. Only works on Linux Kernel 5.6 and above. +## mptcp-enable: Enable mptcp over multiple ports (not interfaces). Only works on Linux Kernel 5.6 and above. +mptcp-enable: enable-mptcp +.PHONY: mptcp-enable + disable-mptcp: @echo "Disabling MPTCP..." @sudo sysctl -w net.mptcp.enabled=0 @@ -313,9 +326,11 @@ disable-mptcp: @sudo sed -i '/net.mptcp.mptcp_path_manager=ndiffports/d' /etc/sysctl.conf @sudo sed -i '/net.mptcp.mptcp_ndiffports=16/d' /etc/sysctl.conf @echo "MPTCP configuration reverted!" - .PHONY: disable-mptcp +## mptcp-disable: Disable mptcp over multiple ports. Only works on Linux Kernel 5.6 and above. +mptcp-disable: disable-mptcp + CONFIG_FILE ?= ${HOME}/.celestia-app/config/config.toml SEND_RECV_RATE ?= 10485760 # 10 MiB @@ -331,10 +346,3 @@ configure-v3: sed -i "s/ttl-num-blocks = .*/ttl-num-blocks = 12/" $(CONFIG_FILE); \ fi .PHONY: configure-v3 - - -## debug-version: Print the git tag and version. -debug-version: - @echo "GIT_TAG: $(GIT_TAG)" - @echo "VERSION: $(VERSION)" -.PHONY: debug-version diff --git a/README.md b/README.md index 44906140b4..b0421e3217 100644 --- a/README.md +++ b/README.md @@ -33,11 +33,11 @@ node | | | | ## Install -### Source +### From source 1. [Install Go](https://go.dev/doc/install) 1.23.1 1. Clone this repo -1. Install the celestia-app CLI +1. Install the celestia-appd binary ```shell make install @@ -77,25 +77,19 @@ See for more information. ## Usage -First, make sure that the [BBR](https://www.ietf.org/archive/id/draft-cardwell-iccrg-bbr-congestion-control-01.html) ("Bottleneck Bandwidth and Round-trip propagation time") congestion control algorithm is enabled in the -system's kernel. The result should contain `bbr`: +> [!WARNING] +> The celestia-appd binary doesn't support signing with Ledger hardware wallets on Windows and OpenBSD. -```sh -sysctl net.ipv4.tcp_congestion_control -``` +### Prerequisites -If not, enable it on Linux by calling the `make enable-bbr` or by running: +Enable the [BBR](https://www.ietf.org/archive/id/draft-cardwell-iccrg-bbr-congestion-control-01.html) ("Bottleneck Bandwidth and Round-trip propagation time") congestion control algorithm. -```sh -sudo modprobe tcp_bbr -net.core.default_qdisc=fq -net.ipv4.tcp_congestion_control=bbr -sudo sysctl -p -``` +```shell +# Check if BBR is enabled. +make bbr-check -```sh -# Print help -celestia-appd --help +# If BBR is not enabled then enable it. +make bbr-enable ``` ### Environment variables @@ -104,13 +98,26 @@ celestia-appd --help |-----------------|-------------------------------------------------------------------|----------------------------------------------------------|----------| | `CELESTIA_HOME` | Where the application directory (`.celestia-app`) should be saved | [User home directory](https://pkg.go.dev/os#UserHomeDir) | Optional | -### Create your own single node devnet +### Using celestia-appd + +```sh +# Print help. +celestia-appd --help + +# Create config files for a new chain named "test". +celestia-appd init test + +# Start the consensus node. +celestia-appd start +``` + +### Create a single node local testnet ```sh -# Start a single node devnet +# Start a single node local testnet. ./scripts/single-node.sh -# Publish blob data to the local devnet +# Publish blob data to the local testnet. celestia-appd tx blob pay-for-blob 0x00010203040506070809 0x48656c6c6f2c20576f726c6421 \ --chain-id private \ --from validator \ @@ -119,12 +126,18 @@ celestia-appd tx blob pay-for-blob 0x00010203040506070809 0x48656c6c6f2c20576f72 --yes ``` -> [!NOTE] -> The celestia-appd binary doesn't support signing with Ledger hardware wallets on Windows and OpenBSD. - ### Usage as a library -If import celestia-app as a Go module, you may need to add some Go module `replace` directives to avoid type incompatibilities. Please see the `replace` directive in [go.mod](./go.mod) for inspiration. +If you import celestia-app as a Go module, you may need to add some Go module `replace` directives to avoid type incompatibilities. Please see the `replace` directive in [go.mod](./go.mod) for inspiration. + +### Usage in tests + +If you are running celestia-app in tests, you may want to override the `timeout_commit` to produce blocks faster. By default, a celestia-app chain with app version >= 3 will produce blocks every ~6 seconds. To produce blocks faster, you can override the `timeout_commit` with the `--timeout-commit` flag. + +```shell +# Start celestia-appd with a one second timeout commit. +celestia-appd start --timeout-commit 1s +``` ## Contributing @@ -153,13 +166,13 @@ make build # Build and install the celestia-appd binary into the $GOPATH/bin directory. make install -# Run tests +# Run tests. make test -# Format code with linters (this assumes golangci-lint and markdownlint are installed) -make fmt +# Format code with linters (this assumes golangci-lint and markdownlint are installed). +make lint-fix -# Regenerate Protobuf files (this assumes Docker is running) +# Regenerate Protobuf files (this assumes Docker is running). make proto-gen ``` diff --git a/app/app.go b/app/app.go index eadd034ea1..87aeacb661 100644 --- a/app/app.go +++ b/app/app.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "slices" + "time" "github.com/celestiaorg/celestia-app/v3/app/ante" "github.com/celestiaorg/celestia-app/v3/app/encoding" @@ -170,6 +171,10 @@ type App struct { // upgradeHeightV2 is used as a coordination mechanism for the height-based // upgrade from v1 to v2. upgradeHeightV2 int64 + // timeoutCommit is used to override the default timeoutCommit. This is + // useful for testing purposes and should not be used on public networks + // (Arabica, Mocha, or Mainnet Beta). + timeoutCommit time.Duration // MsgGateKeeper is used to define which messages are accepted for a given // app version. MsgGateKeeper *ante.MsgVersioningGateKeeper @@ -188,6 +193,7 @@ func New( invCheckPeriod uint, encodingConfig encoding.Config, upgradeHeightV2 int64, + timeoutCommit time.Duration, appOpts servertypes.AppOptions, baseAppOptions ...func(*baseapp.BaseApp), ) *App { @@ -214,6 +220,7 @@ func New( tkeys: tkeys, memKeys: memKeys, upgradeHeightV2: upgradeHeightV2, + timeoutCommit: timeoutCommit, } app.ParamsKeeper = initParamsKeeper(appCodec, encodingConfig.Amino, keys[paramstypes.StoreKey], tkeys[paramstypes.TStoreKey]) @@ -481,7 +488,7 @@ func (app *App) EndBlocker(ctx sdk.Context, req abci.RequestEndBlock) abci.Respo app.SignalKeeper.ResetTally(ctx) } } - res.Timeouts.TimeoutCommit = appconsts.GetTimeoutCommit(currentVersion) + res.Timeouts.TimeoutCommit = app.getTimeoutCommit(currentVersion) res.Timeouts.TimeoutPropose = appconsts.GetTimeoutPropose(currentVersion) return res } @@ -539,8 +546,8 @@ func (app *App) Info(req abci.RequestInfo) abci.ResponseInfo { app.mountKeysAndInit(resp.AppVersion) } + resp.Timeouts.TimeoutCommit = app.getTimeoutCommit(resp.AppVersion) resp.Timeouts.TimeoutPropose = appconsts.GetTimeoutPropose(resp.AppVersion) - resp.Timeouts.TimeoutCommit = appconsts.GetTimeoutCommit(resp.AppVersion) return resp } @@ -565,7 +572,7 @@ func (app *App) InitChain(req abci.RequestInitChain) (res abci.ResponseInitChain app.SetInitialAppVersionInConsensusParams(ctx, appVersion) app.SetAppVersion(ctx, appVersion) } - res.Timeouts.TimeoutCommit = appconsts.GetTimeoutCommit(appVersion) + res.Timeouts.TimeoutCommit = app.getTimeoutCommit(appVersion) res.Timeouts.TimeoutPropose = appconsts.GetTimeoutPropose(appVersion) return res } @@ -849,3 +856,13 @@ func (app *App) OfferSnapshot(req abci.RequestOfferSnapshot) abci.ResponseOfferS func isSupportedAppVersion(appVersion uint64) bool { return appVersion == v1 || appVersion == v2 || appVersion == v3 } + +// getTimeoutCommit returns the timeoutCommit if a user has overridden it via the +// --timeout-commit flag. Otherwise, it returns the default timeout commit based +// on the app version. +func (app *App) getTimeoutCommit(appVersion uint64) time.Duration { + if app.timeoutCommit != 0 { + return app.timeoutCommit + } + return appconsts.GetTimeoutCommit(appVersion) +} diff --git a/app/app_test.go b/app/app_test.go index ae1c6089f5..eee1271ad4 100644 --- a/app/app_test.go +++ b/app/app_test.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/celestiaorg/celestia-app/v3/app" "github.com/celestiaorg/celestia-app/v3/app/encoding" @@ -29,9 +30,10 @@ func TestNew(t *testing.T) { invCheckPeriod := uint(1) encodingConfig := encoding.MakeConfig(app.ModuleEncodingRegisters...) upgradeHeight := int64(0) + timeoutCommit := time.Second appOptions := NoopAppOptions{} - got := app.New(logger, db, traceStore, invCheckPeriod, encodingConfig, upgradeHeight, appOptions) + got := app.New(logger, db, traceStore, invCheckPeriod, encodingConfig, upgradeHeight, timeoutCommit, appOptions) t.Run("initializes ICAHostKeeper", func(t *testing.T) { assert.NotNil(t, got.ICAHostKeeper) @@ -65,8 +67,9 @@ func TestInitChain(t *testing.T) { invCheckPeriod := uint(1) encodingConfig := encoding.MakeConfig(app.ModuleEncodingRegisters...) upgradeHeight := int64(0) + timeoutCommit := time.Second appOptions := NoopAppOptions{} - testApp := app.New(logger, db, traceStore, invCheckPeriod, encodingConfig, upgradeHeight, appOptions) + testApp := app.New(logger, db, traceStore, invCheckPeriod, encodingConfig, upgradeHeight, timeoutCommit, appOptions) genesisState, _, _ := util.GenesisStateWithSingleValidator(testApp, "account") appStateBytes, err := json.MarshalIndent(genesisState, "", " ") require.NoError(t, err) @@ -103,7 +106,7 @@ func TestInitChain(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - application := app.New(logger, db, traceStore, invCheckPeriod, encodingConfig, upgradeHeight, appOptions) + application := app.New(logger, db, traceStore, invCheckPeriod, encodingConfig, upgradeHeight, timeoutCommit, appOptions) if tc.wantPanic { assert.Panics(t, func() { application.InitChain(tc.request) }) } else { @@ -160,6 +163,7 @@ func createTestApp(t *testing.T) *app.App { db := tmdb.NewMemDB() config := encoding.MakeConfig(app.ModuleEncodingRegisters...) upgradeHeight := int64(3) + timeoutCommit := time.Second snapshotDir := filepath.Join(t.TempDir(), "data", "snapshots") t.Cleanup(func() { err := os.RemoveAll(snapshotDir) @@ -174,7 +178,7 @@ func createTestApp(t *testing.T) *app.App { snapshotStore, err := snapshots.NewStore(snapshotDB, snapshotDir) require.NoError(t, err) baseAppOption := baseapp.SetSnapshot(snapshotStore, snapshottypes.NewSnapshotOptions(10, 10)) - testApp := app.New(log.NewNopLogger(), db, nil, 0, config, upgradeHeight, util.EmptyAppOptions{}, baseAppOption) + testApp := app.New(log.NewNopLogger(), db, nil, 0, config, upgradeHeight, timeoutCommit, util.EmptyAppOptions{}, baseAppOption) require.NoError(t, err) response := testApp.Info(abci.RequestInfo{}) require.Equal(t, uint64(0), response.AppVersion) diff --git a/app/check_tx.go b/app/check_tx.go index f76b37880f..68ff288caa 100644 --- a/app/check_tx.go +++ b/app/check_tx.go @@ -3,6 +3,9 @@ package app import ( "fmt" + "cosmossdk.io/errors" + + apperr "github.com/celestiaorg/celestia-app/v3/app/errors" "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" blobtypes "github.com/celestiaorg/celestia-app/v3/x/blob/types" blobtx "github.com/celestiaorg/go-square/v2/tx" @@ -15,6 +18,14 @@ import ( // transactions that contain blobs. func (app *App) CheckTx(req abci.RequestCheckTx) abci.ResponseCheckTx { tx := req.Tx + + // all txs must be less than or equal to the max tx size limit + maxTxSize := appconsts.MaxTxSize(app.AppVersion()) + currentTxSize := len(tx) + if currentTxSize > maxTxSize { + return sdkerrors.ResponseCheckTxWithEvents(errors.Wrapf(apperr.ErrTxExceedsMaxSize, "tx size %d bytes is larger than the application's configured MaxTxSize of %d bytes for version %d", currentTxSize, maxTxSize, app.AppVersion()), 0, 0, []abci.Event{}, false) + } + // check if the transaction contains blobs btx, isBlob, err := blobtx.UnmarshalBlobTx(tx) if isBlob && err != nil { diff --git a/app/errors/errors.go b/app/errors/errors.go new file mode 100644 index 0000000000..9bbff18b42 --- /dev/null +++ b/app/errors/errors.go @@ -0,0 +1,12 @@ +package errors + +import ( + "cosmossdk.io/errors" +) + +const AppErrorsCodespace = "app" + +// general application errors +var ( + ErrTxExceedsMaxSize = errors.Register(AppErrorsCodespace, 11142, "exceeds max tx size limit") +) diff --git a/app/test/big_blob_test.go b/app/test/big_blob_test.go index cede1ae65d..bbce473016 100644 --- a/app/test/big_blob_test.go +++ b/app/test/big_blob_test.go @@ -7,6 +7,7 @@ import ( "github.com/celestiaorg/celestia-app/v3/app" "github.com/celestiaorg/celestia-app/v3/app/encoding" + apperrors "github.com/celestiaorg/celestia-app/v3/app/errors" "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" "github.com/celestiaorg/celestia-app/v3/pkg/user" "github.com/celestiaorg/celestia-app/v3/test/util/testfactory" @@ -55,7 +56,7 @@ func (s *BigBlobSuite) SetupSuite() { require.NoError(t, cctx.WaitForNextBlock()) } -// TestErrBlobsTooLarge verifies that submitting a 2 MiB blob hits ErrBlobsTooLarge. +// TestErrBlobsTooLarge verifies that submitting a ~1.9 MiB blob hits ErrBlobsTooLarge. func (s *BigBlobSuite) TestErrBlobsTooLarge() { t := s.T() @@ -67,8 +68,8 @@ func (s *BigBlobSuite) TestErrBlobsTooLarge() { } testCases := []testCase{ { - name: "2 mebibyte blob", - blob: newBlobWithSize(2 * mebibyte), + name: "~ 1.9 MiB blob", + blob: newBlobWithSize(2_000_000), want: blobtypes.ErrBlobsTooLarge.ABCICode(), }, } @@ -88,3 +89,38 @@ func (s *BigBlobSuite) TestErrBlobsTooLarge() { }) } } + +// TestBlobExceedsMaxTxSize verifies that submitting a 2 MiB blob hits ErrTxExceedsMaxSize. +func (s *BigBlobSuite) TestBlobExceedsMaxTxSize() { + t := s.T() + + type testCase struct { + name string + blob *share.Blob + expectedCode uint32 + expectedErr string + } + testCases := []testCase{ + { + name: "2 MiB blob", + blob: newBlobWithSize(2097152), + expectedCode: apperrors.ErrTxExceedsMaxSize.ABCICode(), + expectedErr: apperrors.ErrTxExceedsMaxSize.Error(), + }, + } + + txClient, err := testnode.NewTxClientFromContext(s.cctx) + require.NoError(t, err) + + for _, tc := range testCases { + s.Run(tc.name, func() { + subCtx, cancel := context.WithTimeout(s.cctx.GoContext(), 30*time.Second) + defer cancel() + res, err := txClient.SubmitPayForBlob(subCtx, []*share.Blob{tc.blob}, user.SetGasLimitAndGasPrice(1e9, appconsts.DefaultMinGasPrice)) + require.Error(t, err) + require.Nil(t, res) + code := err.(*user.BroadcastTxError).Code + require.Equal(t, tc.expectedCode, code, err.Error(), tc.expectedErr) + }) + } +} diff --git a/app/test/check_tx_test.go b/app/test/check_tx_test.go index aba7720547..87c7cc9e01 100644 --- a/app/test/check_tx_test.go +++ b/app/test/check_tx_test.go @@ -11,6 +11,7 @@ import ( "github.com/celestiaorg/celestia-app/v3/app" "github.com/celestiaorg/celestia-app/v3/app/encoding" + apperr "github.com/celestiaorg/celestia-app/v3/app/errors" "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" "github.com/celestiaorg/celestia-app/v3/pkg/user" testutil "github.com/celestiaorg/celestia-app/v3/test/util" @@ -32,7 +33,7 @@ func TestCheckTx(t *testing.T) { ns1, err := share.NewV0Namespace(bytes.Repeat([]byte{1}, share.NamespaceVersionZeroIDSize)) require.NoError(t, err) - accs := []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k"} + accs := []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m"} testApp, kr := testutil.SetupTestAppWithGenesisValSet(app.DefaultConsensusParams(), accs...) testApp.Commit() @@ -173,11 +174,11 @@ func TestCheckTx(t *testing.T) { expectedABCICode: abci.CodeTypeOK, }, { - name: "10,000,000 byte blob", + name: "2,000,000 byte blob", checkType: abci.CheckTxType_New, getTx: func() []byte { signer := createSigner(t, kr, accs[9], encCfg.TxConfig, 10) - _, blobs := blobfactory.RandMsgPayForBlobsWithSigner(tmrand.NewRand(), signer.Account(accs[9]).Address().String(), 10_000_000, 1) + _, blobs := blobfactory.RandMsgPayForBlobsWithSigner(tmrand.NewRand(), signer.Account(accs[9]).Address().String(), 2_000_000, 1) tx, _, err := signer.CreatePayForBlobs(accs[9], blobs, opts...) require.NoError(t, err) return tx @@ -217,6 +218,32 @@ func TestCheckTx(t *testing.T) { }, expectedABCICode: abci.CodeTypeOK, }, + { + name: "v1 blob over 2MiB", + checkType: abci.CheckTxType_New, + getTx: func() []byte { + signer := createSigner(t, kr, accs[11], encCfg.TxConfig, 12) + blob, err := share.NewV1Blob(share.RandomBlobNamespace(), bytes.Repeat([]byte{1}, 2097152), signer.Account(accs[11]).Address()) + require.NoError(t, err) + blobTx, _, err := signer.CreatePayForBlobs(accs[11], []*share.Blob{blob}, opts...) + require.NoError(t, err) + return blobTx + }, + expectedABCICode: apperr.ErrTxExceedsMaxSize.ABCICode(), + }, + { + name: "v0 blob over 2MiB", + checkType: abci.CheckTxType_New, + getTx: func() []byte { + signer := createSigner(t, kr, accs[12], encCfg.TxConfig, 13) + blob, err := share.NewV0Blob(share.RandomBlobNamespace(), bytes.Repeat([]byte{1}, 2097152)) + require.NoError(t, err) + blobTx, _, err := signer.CreatePayForBlobs(accs[12], []*share.Blob{blob}, opts...) + require.NoError(t, err) + return blobTx + }, + expectedABCICode: apperr.ErrTxExceedsMaxSize.ABCICode(), + }, } for _, tt := range tests { diff --git a/app/test/consistent_apphash_test.go b/app/test/consistent_apphash_test.go index 9a108e4dd1..2acc1030ae 100644 --- a/app/test/consistent_apphash_test.go +++ b/app/test/consistent_apphash_test.go @@ -62,6 +62,10 @@ type appHashTest struct { expectedAppHash []byte } +func DefaultTxOpts() []user.TxOption { + return blobfactory.FeeTxOpts(10_000_000) +} + // TestConsistentAppHash executes all state machine messages on all app versions, generates an app hash, // and compares it against a previously generated hash from the same set of transactions. // App hashes across different commits should be consistent. @@ -377,7 +381,7 @@ func createEncodedBlobTx(t *testing.T, signer *user.Signer, accountAddresses []s blobTx := blobTx{ author: senderAcc.Name(), blobs: []*share.Blob{blob}, - txOptions: blobfactory.DefaultTxOpts(), + txOptions: DefaultTxOpts(), } encodedBlobTx, _, err := signer.CreatePayForBlobs(blobTx.author, blobTx.blobs, blobTx.txOptions...) require.NoError(t, err) @@ -429,7 +433,7 @@ func deterministicKeyRing(cdc codec.Codec) (keyring.Keyring, []types.PubKey) { func processSdkMessages(signer *user.Signer, sdkMessages []sdk.Msg) ([][]byte, error) { encodedTxs := make([][]byte, 0, len(sdkMessages)) for _, msg := range sdkMessages { - encodedTx, err := signer.CreateTx([]sdk.Msg{msg}, blobfactory.DefaultTxOpts()...) + encodedTx, err := signer.CreateTx([]sdk.Msg{msg}, DefaultTxOpts()...) if err != nil { return nil, err } diff --git a/app/test/prepare_proposal_test.go b/app/test/prepare_proposal_test.go index f274d9da6a..b167419a15 100644 --- a/app/test/prepare_proposal_test.go +++ b/app/test/prepare_proposal_test.go @@ -51,7 +51,6 @@ func TestPrepareProposalPutsPFBsAtEnd(t *testing.T) { accnts[:numBlobTxs], infos[:numBlobTxs], testfactory.Repeat([]*share.Blob{protoBlob}, numBlobTxs), - blobfactory.DefaultTxOpts()..., ) normalTxs := testutil.SendTxsWithAccounts( @@ -109,7 +108,6 @@ func TestPrepareProposalFiltering(t *testing.T) { testfactory.RandomBlobNamespaces(tmrand.NewRand(), 3), [][]int{{100}, {1000}, {420}}, ), - blobfactory.DefaultTxOpts()..., ) // create 3 MsgSend transactions that are signed with valid account numbers @@ -173,22 +171,6 @@ func TestPrepareProposalFiltering(t *testing.T) { // 3 transactions over MaxTxSize limit largeTxs := coretypes.Txs(testutil.SendTxsWithAccounts(t, testApp, encConf.TxConfig, kr, 1000, accounts[0], accounts[:3], testutil.ChainID, user.SetMemo(largeString))).ToSliceOfBytes() - // 3 blobTxs over MaxTxSize limit - largeBlobTxs := blobfactory.ManyMultiBlobTx( - t, - encConf.TxConfig, - kr, - testutil.ChainID, - accounts[:3], - infos[:3], - blobfactory.NestedBlobs( - t, - testfactory.RandomBlobNamespaces(tmrand.NewRand(), 3), - [][]int{{100}, {1000}, {420}}, - ), - user.SetMemo(largeString), - ) - type test struct { name string txs func() [][]byte @@ -243,9 +225,9 @@ func TestPrepareProposalFiltering(t *testing.T) { { name: "blobTxs and sendTxs that exceed MaxTxSize limit", txs: func() [][]byte { - return append(largeTxs, largeBlobTxs...) // All txs are over MaxTxSize limit + return largeTxs // All txs are over MaxTxSize limit }, - prunedTxs: append(largeTxs, largeBlobTxs...), + prunedTxs: largeTxs, }, } diff --git a/app/test/process_proposal_test.go b/app/test/process_proposal_test.go index 33b1d406c5..093116a85b 100644 --- a/app/test/process_proposal_test.go +++ b/app/test/process_proposal_test.go @@ -3,7 +3,6 @@ package app_test import ( "bytes" "fmt" - "strings" "testing" "time" @@ -51,26 +50,6 @@ func TestProcessProposal(t *testing.T) { testfactory.RandomBlobNamespaces(tmrand.NewRand(), 4), [][]int{{100}, {1000}, {420}, {300}}, ), - blobfactory.DefaultTxOpts()..., - ) - - largeMemo := strings.Repeat("a", appconsts.MaxTxSize(appconsts.LatestVersion)) - - // create 2 single blobTxs that include a large memo making the transaction - // larger than the configured max tx size - largeBlobTxs := blobfactory.ManyMultiBlobTx( - t, enc, kr, testutil.ChainID, accounts[3:], infos[3:], - blobfactory.NestedBlobs( - t, - testfactory.RandomBlobNamespaces(tmrand.NewRand(), 4), - [][]int{{100}, {1000}, {420}, {300}}, - ), - user.SetMemo(largeMemo)) - - // create 1 large sendTx that includes a large memo making the - // transaction over the configured max tx size limit - largeSendTx := testutil.SendTxsWithAccounts( - t, testApp, enc, kr, 1000, accounts[0], accounts[1:2], testutil.ChainID, user.SetMemo(largeMemo), ) // create 3 MsgSend transactions that are signed with valid account numbers @@ -349,26 +328,6 @@ func TestProcessProposal(t *testing.T) { appVersion: v3.Version, expectedResult: abci.ResponseProcessProposal_REJECT, }, - { - name: "blob txs larger than configured max tx size", - input: validData(), - mutator: func(d *tmproto.Data) { - d.Txs = append(d.Txs, largeBlobTxs...) - d.Hash = calculateNewDataHash(t, d.Txs) - }, - appVersion: appconsts.LatestVersion, - expectedResult: abci.ResponseProcessProposal_REJECT, - }, - { - name: "send tx larger than configured max tx size", - input: validData(), - mutator: func(d *tmproto.Data) { - d.Txs = append(coretypes.Txs(largeSendTx).ToSliceOfBytes(), d.Txs...) - d.Hash = calculateNewDataHash(t, d.Txs) - }, - appVersion: appconsts.LatestVersion, - expectedResult: abci.ResponseProcessProposal_REJECT, - }, } for _, tt := range tests { diff --git a/app/test/upgrade_test.go b/app/test/upgrade_test.go index b90e146c67..13525d0d14 100644 --- a/app/test/upgrade_test.go +++ b/app/test/upgrade_test.go @@ -253,7 +253,7 @@ func SetupTestAppWithUpgradeHeight(t *testing.T, upgradeHeight int64) (*app.App, db := dbm.NewMemDB() encCfg := encoding.MakeConfig(app.ModuleEncodingRegisters...) - testApp := app.New(log.NewNopLogger(), db, nil, 0, encCfg, upgradeHeight, util.EmptyAppOptions{}) + testApp := app.New(log.NewNopLogger(), db, nil, 0, encCfg, upgradeHeight, 0, util.EmptyAppOptions{}) genesis := genesis.NewDefaultGenesis(). WithChainID(appconsts.TestChainID). WithValidators(genesis.NewDefaultValidator(testnode.DefaultValidatorAccountName)). diff --git a/cmd/celestia-appd/cmd/app_exporter.go b/cmd/celestia-appd/cmd/app_exporter.go index 8ab082c2b2..fa5c89d71f 100644 --- a/cmd/celestia-appd/cmd/app_exporter.go +++ b/cmd/celestia-appd/cmd/app_exporter.go @@ -20,7 +20,7 @@ func appExporter( appOptions servertypes.AppOptions, ) (servertypes.ExportedApp, error) { config := encoding.MakeConfig(app.ModuleEncodingRegisters...) - application := app.New(logger, db, traceStore, uint(1), config, 0, appOptions) + application := app.New(logger, db, traceStore, uint(1), config, 0, 0, appOptions) if height != -1 { if err := application.LoadHeight(height); err != nil { return servertypes.ExportedApp{}, err diff --git a/cmd/celestia-appd/cmd/app_server.go b/cmd/celestia-appd/cmd/app_server.go index b2bfecf844..62ab48b816 100644 --- a/cmd/celestia-appd/cmd/app_server.go +++ b/cmd/celestia-appd/cmd/app_server.go @@ -44,10 +44,13 @@ func NewAppServer(logger log.Logger, db dbm.DB, traceStore io.Writer, appOptions } return app.New( - logger, db, traceStore, + logger, + db, + traceStore, cast.ToUint(appOptions.Get(server.FlagInvCheckPeriod)), encoding.MakeConfig(app.ModuleEncodingRegisters...), cast.ToInt64(appOptions.Get(UpgradeHeightFlag)), + cast.ToDuration(appOptions.Get(TimeoutCommitFlag)), appOptions, baseapp.SetPruning(pruningOpts), baseapp.SetMinGasPrices(cast.ToString(appOptions.Get(server.FlagMinGasPrices))), diff --git a/cmd/celestia-appd/cmd/root.go b/cmd/celestia-appd/cmd/root.go index 581ce3592e..78d0163479 100644 --- a/cmd/celestia-appd/cmd/root.go +++ b/cmd/celestia-appd/cmd/root.go @@ -37,6 +37,9 @@ const ( // UpgradeHeightFlag is the flag to specify the upgrade height for v1 to v2 // application upgrade. UpgradeHeightFlag = "v2-upgrade-height" + + // TimeoutCommit is a flag that can be used to override the timeout_commit. + TimeoutCommitFlag = "timeout-commit" ) // NewRootCmd creates a new root command for celestia-appd. @@ -125,7 +128,7 @@ func initRootCommand(rootCommand *cobra.Command, encodingConfig encoding.Config) ) // Add the following commands to the rootCommand: start, tendermint, export, version, and rollback. - addCommands(rootCommand, app.DefaultNodeHome, NewAppServer, appExporter, addModuleInitFlags) + addCommands(rootCommand, app.DefaultNodeHome, NewAppServer, appExporter, addStartFlags) } // setDefaultConsensusParams sets the default consensus parameters for the @@ -136,9 +139,11 @@ func setDefaultConsensusParams(command *cobra.Command) error { return server.SetCmdServerContext(command, ctx) } -func addModuleInitFlags(startCmd *cobra.Command) { +// addStartFlags adds flags to the start command. +func addStartFlags(startCmd *cobra.Command) { crisis.AddModuleInitFlags(startCmd) startCmd.Flags().Int64(UpgradeHeightFlag, 0, "Upgrade height to switch from v1 to v2. Must be coordinated amongst all validators") + startCmd.Flags().Duration(TimeoutCommitFlag, 0, "Override the application configured timeout_commit. Note: only for testing purposes.") } // replaceLogger optionally replaces the logger with a file logger if the flag diff --git a/cmd/celestia-appd/cmd/start.go b/cmd/celestia-appd/cmd/start.go index de97d4dfa0..057ae4422a 100644 --- a/cmd/celestia-appd/cmd/start.go +++ b/cmd/celestia-appd/cmd/start.go @@ -14,6 +14,7 @@ import ( "strings" "time" + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/client/flags" "github.com/cosmos/cosmos-sdk/codec" @@ -117,6 +118,24 @@ is performed. Note, when enabled, gRPC will also be automatically enabled. return err } + switch clientCtx.ChainID { + case appconsts.ArabicaChainID: + serverCtx.Logger.Info(fmt.Sprintf("Since the chainID is %v, configuring the default v2 upgrade height to %v", appconsts.ArabicaChainID, appconsts.ArabicaUpgradeHeightV2)) + serverCtx.Viper.SetDefault(UpgradeHeightFlag, appconsts.ArabicaUpgradeHeightV2) + case appconsts.MochaChainID: + serverCtx.Logger.Info(fmt.Sprintf("Since the chainID is %v, configuring the default v2 upgrade height to %v", appconsts.MochaChainID, appconsts.MochaUpgradeHeightV2)) + serverCtx.Viper.SetDefault(UpgradeHeightFlag, appconsts.MochaUpgradeHeightV2) + case appconsts.MainnetChainID: + serverCtx.Logger.Info(fmt.Sprintf("Since the chainID is %v, configuring the default v2 upgrade height to %v", appconsts.MainnetChainID, appconsts.MainnetUpgradeHeightV2)) + serverCtx.Viper.SetDefault(UpgradeHeightFlag, appconsts.MainnetUpgradeHeightV2) + default: + serverCtx.Logger.Info(fmt.Sprintf("No default value exists for the v2 upgrade height when the chainID is %v", clientCtx.ChainID)) + } + + if contains(appconsts.PublicNetworks, clientCtx.ChainID) && serverCtx.Viper.GetDuration(TimeoutCommitFlag) != 0 { + return fmt.Errorf("the --timeout-commit flag was used on %v but it is unsupported on public networks: %v. The --timeout-commit flag should only be used on private testnets", clientCtx.ChainID, strings.Join(appconsts.PublicNetworks, ", ")) + } + withTM, _ := cmd.Flags().GetBool(flagWithTendermint) if !withTM { serverCtx.Logger.Info("starting ABCI without Tendermint") diff --git a/go.mod b/go.mod index 6ddc9b7eb3..0e57fc6ea1 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/celestiaorg/blobstream-contracts/v3 v3.1.0 github.com/celestiaorg/go-square v1.1.1 github.com/celestiaorg/go-square/v2 v2.1.0 - github.com/celestiaorg/knuu v0.16.1 + github.com/celestiaorg/knuu v0.16.2 github.com/celestiaorg/nmt v0.22.2 github.com/celestiaorg/rsmt2d v0.14.0 github.com/cometbft/cometbft-db v1.0.1 @@ -36,7 +36,7 @@ require ( google.golang.org/grpc v1.68.0 google.golang.org/protobuf v1.35.2 gopkg.in/yaml.v2 v2.4.0 - k8s.io/apimachinery v0.31.3 + k8s.io/apimachinery v0.31.4 ) require ( @@ -227,13 +227,13 @@ require ( go.opentelemetry.io/otel/trace v1.30.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.27.0 // indirect - golang.org/x/net v0.29.0 // indirect + golang.org/x/crypto v0.31.0 // indirect + golang.org/x/net v0.31.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect - golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.25.0 // indirect - golang.org/x/term v0.24.0 // indirect - golang.org/x/text v0.18.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/api v0.169.0 // indirect google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect diff --git a/go.sum b/go.sum index dc62f3ea98..cc534f4e95 100644 --- a/go.sum +++ b/go.sum @@ -325,8 +325,8 @@ github.com/celestiaorg/go-square v1.1.1 h1:Cy3p8WVspVcyOqHM8BWFuuYPwMitO1pYGe+Im github.com/celestiaorg/go-square v1.1.1/go.mod h1:1EXMErhDrWJM8B8V9hN7dqJ2kUTClfwdqMOmF9yQUa0= github.com/celestiaorg/go-square/v2 v2.1.0 h1:ECIvYEeHIWiIJGDCJxQNtzqm5DmnBly7XGhSpLsl+Lw= github.com/celestiaorg/go-square/v2 v2.1.0/go.mod h1:n3ztrh8CBjWOD6iWYMo3pPOlQIgzLK9yrnqMPcNo6g8= -github.com/celestiaorg/knuu v0.16.1 h1:EOR/c9kvc0jZet/mma2qwAdlvEbl94bW9cC8FItkyBE= -github.com/celestiaorg/knuu v0.16.1/go.mod h1:y20nUmVWVgbzxBKHqmbwp3C0ZJ9J9ovCg1ylHo85hdQ= +github.com/celestiaorg/knuu v0.16.2 h1:OOl2xMf+5ryWtPcSF+M6gmw4YQPEh6lsCA/lTDl8fwI= +github.com/celestiaorg/knuu v0.16.2/go.mod h1:nB7IGCR984YKEDW+j5xHPOidYfbO4DoZI1rDKijvF5E= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4/go.mod h1:fzuHnhzj1pUygGz+1ZkB3uQbEUL4htqCGJ4Qs2LwMZA= github.com/celestiaorg/nmt v0.22.2 h1:JmOMtZL9zWAed1hiwb9DDs+ELcKp/ZQZ3rPverge/V8= @@ -1405,8 +1405,8 @@ golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= -golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1514,8 +1514,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= -golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= +golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1557,8 +1557,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1669,14 +1669,14 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= -golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1687,8 +1687,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= -golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1761,8 +1761,8 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= -golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= +golang.org/x/tools v0.27.0 h1:qEKojBykQkQ4EynWy4S8Weg69NumxKdn40Fce3uc/8o= +golang.org/x/tools v0.27.0/go.mod h1:sUi0ZgbwW9ZPAq26Ekut+weQPR5eIM6GQLQ1Yjm1H0Q= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2065,8 +2065,8 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= k8s.io/api v0.30.2 h1:+ZhRj+28QT4UOH+BKznu4CBgPWgkXO7XAvMcMl0qKvI= k8s.io/api v0.30.2/go.mod h1:ULg5g9JvOev2dG0u2hig4Z7tQ2hHIuS+m8MNZ+X6EmI= -k8s.io/apimachinery v0.31.3 h1:6l0WhcYgasZ/wk9ktLq5vLaoXJJr5ts6lkaQzgeYPq4= -k8s.io/apimachinery v0.31.3/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/apimachinery v0.31.4 h1:8xjE2C4CzhYVm9DGf60yohpNUh5AEBnPxCryPBECmlM= +k8s.io/apimachinery v0.31.4/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= k8s.io/client-go v0.30.2 h1:sBIVJdojUNPDU/jObC+18tXWcTJVcwyqS9diGdWHk50= k8s.io/client-go v0.30.2/go.mod h1:JglKSWULm9xlJLx4KCkfLLQ7XwtlbflV6uFFSHTMgVs= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= diff --git a/pkg/appconsts/chain_ids.go b/pkg/appconsts/chain_ids.go index f26b4816f7..ed0de9769f 100644 --- a/pkg/appconsts/chain_ids.go +++ b/pkg/appconsts/chain_ids.go @@ -2,6 +2,10 @@ package appconsts const ( ArabicaChainID = "arabica-11" + MochaChainID = "mocha-4" + MainnetChainID = "celestia" // TestChainID is the chain ID used for testing. TestChainID = "test" ) + +var PublicNetworks = []string{ArabicaChainID, MochaChainID, MainnetChainID} diff --git a/pkg/appconsts/upgrade_heights.go b/pkg/appconsts/upgrade_heights.go new file mode 100644 index 0000000000..13ee0963f0 --- /dev/null +++ b/pkg/appconsts/upgrade_heights.go @@ -0,0 +1,13 @@ +package appconsts + +const ( + // ArabicaUpgradeHeightV2 is the block height at which the arabica-11 + // upgraded from app version 1 to 2. + ArabicaUpgradeHeightV2 = 1751707 + // MochaUpgradeHeightV2 is the block height at which the mocha-4 upgraded + // from app version 1 to 2. + MochaUpgradeHeightV2 = 2585031 + // MainnetUpgradeHeightV2 is the block height at which the celestia upgraded + // from app version 1 to 2. + MainnetUpgradeHeightV2 = 2371495 +) diff --git a/scripts/single-node.sh b/scripts/single-node.sh index 14c24e9eb8..c2159fe70e 100755 --- a/scripts/single-node.sh +++ b/scripts/single-node.sh @@ -102,6 +102,7 @@ startCelestiaApp() { --api.enable \ --grpc.enable \ --grpc-web.enable \ + --timeout-commit 1s \ --force-no-bbr # no need to require BBR usage on a local node } diff --git a/specs/src/cat_pool.md b/specs/src/cat_pool.md index a71dc27cc3..ccc5caa74a 100644 --- a/specs/src/cat_pool.md +++ b/specs/src/cat_pool.md @@ -41,7 +41,7 @@ Both `SeenTx` and `WantTx` contain the sha256 hash of the raw transaction bytes. Both messages are sent across a new channel with the ID: `byte(0x31)`. This enables cross compatibility as discussed in greater detail below. > **Note:** -> The term `SeenTx` is used over the more common `HasTx` because the transaction pool contains sophisticated eviction logic. TTL's, higher priority transactions and reCheckTx may mean that a transaction pool *had* a transaction but does not have it any more. Semantically it's more appropriate to use `SeenTx` to imply not the presence of a transaction but that the node has seen it and dealt with it accordingly. +> The term `SeenTx` is used over the more common `HasTx` because the transaction pool contains sophisticated eviction logic. TTLs, higher priority transactions and reCheckTx may mean that a transaction pool *had* a transaction but does not have it any more. Semantically it's more appropriate to use `SeenTx` to imply not the presence of a transaction but that the node has seen it and dealt with it accordingly. ## Outbound logic @@ -88,7 +88,7 @@ Upon receiving a `SeenTx` message: - If the node does not have the transaction but recently evicted it, it MAY choose to rerequest the transaction if it has adequate resources now to process it. - If the node has not seen the transaction or does not have any pending requests for that transaction, it can do one of two things: - It MAY immediately request the tx from the peer with a `WantTx`. - - If the node is connected to the peer specified in `FROM`, it is likely, from a non-byzantine peer, that the node will also shortly receive the transaction from the peer. It MAY wait for a `Txs` message for a bounded amount of time but MUST eventually send a `WantMsg` message to either the original peer or any other peer that *has* the specified transaction. + - If the node is connected to the peer specified in `FROM`, it is likely, from a non-byzantine peer, that the node will also shortly receive the transaction from the peer. It MAY wait for a `Txs` message for a bounded amount of time but MUST eventually send a `WantTx` message to either the original peer or any other peer that *has* the specified transaction. Upon receiving a `WantTx` message: diff --git a/specs/src/data_square_layout.md b/specs/src/data_square_layout.md index 70e476250f..e7eb64d226 100644 --- a/specs/src/data_square_layout.md +++ b/specs/src/data_square_layout.md @@ -39,7 +39,7 @@ The order of blobs in a namespace is dictated by the priority of the PFBs that p Transactions can pay fees for a blob to be included in the same block as the transaction itself. It may seem natural to bundle the `MsgPayForBlobs` transaction that pays for a number of blobs with these blobs (which is the case in other blockchains with native execution, e.g. calldata in Ethereum transactions or OP_RETURN data in Bitcoin transactions), however this would mean that processes validating the state of the Celestia network would need to download all blob data. PayForBlob transactions must therefore only include a commitment to (i.e. some hash of) the blob they pay fees for. If implemented naively (e.g. with a simple hash of the blob, or a simple binary Merkle tree root of the blob), this can lead to a data availability problem, as there are no guarantees that the data behind these commitments is actually part of the block data. -To that end, we impose some additional rules onto _blobs only_: blobs must be placed is a way such that both the transaction sender and the block producer can be held accountable—a necessary property for e.g. fee burning. Accountable in this context means that +To that end, we impose some additional rules onto _blobs only_: blobs must be placed in a way such that both the transaction sender and the block producer can be held accountable—a necessary property for e.g. fee burning. Accountable in this context means that 1. The transaction sender must pay sufficient fees for blob inclusion. 1. The block proposer cannot claim that a blob was included when it was not (which implies that a transaction and the blob it pays for must be included in the same block). In addition all blobs must be accompanied by a PayForBlob transaction. diff --git a/test/e2e/benchmark/benchmark.go b/test/e2e/benchmark/benchmark.go index 5af7b8539e..95e713bd69 100644 --- a/test/e2e/benchmark/benchmark.go +++ b/test/e2e/benchmark/benchmark.go @@ -24,7 +24,7 @@ type BenchmarkTest struct { // NewBenchmarkTest wraps around testnet.New to create a new benchmark test. // It may modify genesis consensus parameters based on manifest. -func NewBenchmarkTest(name string, manifest *Manifest) (*BenchmarkTest, error) { +func NewBenchmarkTest(logger *log.Logger, name string, manifest *Manifest) (*BenchmarkTest, error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -42,8 +42,8 @@ func NewBenchmarkTest(name string, manifest *Manifest) (*BenchmarkTest, error) { log.Printf("Knuu initialized with scope %s", kn.Scope) - testNet, err := testnet.New(kn, testnet.Options{ - Grafana: testnet.GetGrafanaInfoFromEnvVar(), + testNet, err := testnet.New(logger, kn, testnet.Options{ + Grafana: testnet.GetGrafanaInfoFromEnvVar(logger), ChainID: manifest.ChainID, GenesisModifiers: manifest.GetGenesisModifiers(), }) @@ -70,7 +70,7 @@ func (b *BenchmarkTest) SetupNodes() error { } } // obtain the GRPC endpoints of the validators - gRPCEndpoints, err := b.RemoteGRPCEndpoints(ctx) + gRPCEndpoints, err := b.RemoteGRPCEndpoints() testnet.NoError("failed to get validators GRPC endpoints", err) log.Println("validators GRPC endpoints", gRPCEndpoints) diff --git a/test/e2e/benchmark/throughput.go b/test/e2e/benchmark/throughput.go index fb3fdca6ba..b0fb34dd30 100644 --- a/test/e2e/benchmark/throughput.go +++ b/test/e2e/benchmark/throughput.go @@ -90,7 +90,7 @@ func TwoNodeSimple(logger *log.Logger) error { DisableBBR: true, } - benchTest, err := NewBenchmarkTest(testName, &manifest) + benchTest, err := NewBenchmarkTest(logger, testName, &manifest) testnet.NoError("failed to create benchmark test", err) ctx, cancel := context.WithCancel(context.Background()) @@ -113,8 +113,8 @@ func TwoNodeSimple(logger *log.Logger) error { func runBenchmarkTest(logger *log.Logger, testName string, manifest Manifest) error { logger.Println("Running", testName) manifest.ChainID = manifest.summary() - log.Println("ChainID: ", manifest.ChainID) - benchTest, err := NewBenchmarkTest(testName, &manifest) + logger.Println("ChainID: ", manifest.ChainID) + benchTest, err := NewBenchmarkTest(logger, testName, &manifest) testnet.NoError("failed to create benchmark test", err) ctx, cancel := context.WithCancel(context.Background()) diff --git a/test/e2e/major_upgrade_v2.go b/test/e2e/major_upgrade_v2.go index 3e44eb0bc3..03383124ae 100644 --- a/test/e2e/major_upgrade_v2.go +++ b/test/e2e/major_upgrade_v2.go @@ -35,7 +35,7 @@ func MajorUpgradeToV2(logger *log.Logger) error { logger.Printf("Knuu initialized with scope %s", kn.Scope) logger.Println("Creating testnet") - testNet, err := testnet.New(kn, testnet.Options{}) + testNet, err := testnet.New(logger, kn, testnet.Options{}) testnet.NoError("failed to create testnet", err) defer testNet.Cleanup(ctx) @@ -60,7 +60,7 @@ func MajorUpgradeToV2(logger *log.Logger) error { } logger.Println("Creating txsim") - endpoints, err := testNet.RemoteGRPCEndpoints(ctx) + endpoints, err := testNet.RemoteGRPCEndpoints() testnet.NoError("failed to get remote gRPC endpoints", err) upgradeSchedule := map[int64]uint64{} err = testNet.CreateTxClient(ctx, "txsim", testnet.TxsimVersion, 1, "100-2000", 100, testnet.DefaultResources, endpoints[0], upgradeSchedule) diff --git a/test/e2e/major_upgrade_v3.go b/test/e2e/major_upgrade_v3.go index f1e39d5d41..37f147ca1b 100644 --- a/test/e2e/major_upgrade_v3.go +++ b/test/e2e/major_upgrade_v3.go @@ -34,7 +34,7 @@ func MajorUpgradeToV3(logger *log.Logger) error { logger.Printf("Knuu initialized with scope %s", kn.Scope) logger.Println("Creating testnet") - testNet, err := testnet.New(kn, testnet.Options{ + testNet, err := testnet.New(logger, kn, testnet.Options{ ChainID: appconsts.TestChainID, }) testnet.NoError("failed to create testnet", err) @@ -62,7 +62,7 @@ func MajorUpgradeToV3(logger *log.Logger) error { } logger.Println("Creating txsim") - endpoints, err := testNet.RemoteGRPCEndpoints(ctx) + endpoints, err := testNet.RemoteGRPCEndpoints() testnet.NoError("failed to get remote gRPC endpoints", err) upgradeSchedule := map[int64]uint64{ upgradeHeightV3: v3.Version, diff --git a/test/e2e/minor_version_compatibility.go b/test/e2e/minor_version_compatibility.go index bb81d5e021..ab9b0580e4 100644 --- a/test/e2e/minor_version_compatibility.go +++ b/test/e2e/minor_version_compatibility.go @@ -50,7 +50,7 @@ func MinorVersionCompatibility(logger *log.Logger) error { kn.HandleStopSignal(ctx) logger.Printf("Knuu initialized with scope %s", kn.Scope) - testNet, err := testnet.New(kn, testnet.Options{Seed: seed}) + testNet, err := testnet.New(logger, kn, testnet.Options{Seed: seed}) testnet.NoError("failed to create testnet", err) defer testNet.Cleanup(ctx) @@ -76,7 +76,7 @@ func MinorVersionCompatibility(logger *log.Logger) error { } logger.Println("Creating txsim") - endpoints, err := testNet.RemoteGRPCEndpoints(ctx) + endpoints, err := testNet.RemoteGRPCEndpoints() testnet.NoError("failed to get remote gRPC endpoints", err) upgradeSchedule := map[int64]uint64{} err = testNet.CreateTxClient(ctx, "txsim", testnet.TxsimVersion, 1, "100-2000", 100, testnet.DefaultResources, endpoints[0], upgradeSchedule) diff --git a/test/e2e/readme.md b/test/e2e/readme.md index ab0dbe8264..913ee84862 100644 --- a/test/e2e/readme.md +++ b/test/e2e/readme.md @@ -30,7 +30,7 @@ make test-e2e E2ESimple **Optional parameters**: -- `KNUUU_TIMEOUT` can be used to override the default timeout of 60 minutes for the tests. +- `KNUU_TIMEOUT` can be used to override the default timeout of 60 minutes for the tests. ## Observation @@ -56,7 +56,7 @@ This will back up your default kubernetes configuration. If you use a different ### Install minikube -Minikube is required to be installed on your machine. If you have a linux machine, follow the [minikube docs](https://kubernetes.io/fr/docs/tasks/tools/install-minikube/). If you're on macOS ARM, this [tutorial](https://devopscube.com/minikube-mac/) can be helpful to run it using qemu. +Minikube is required to be installed on your machine. If you have a linux machine, follow the [minikube docs](https://kubernetes.io/docs/tasks/tools/install-minikube/). If you're on macOS ARM, this [tutorial](https://devopscube.com/minikube-mac/) can be helpful to run it using qemu. ### Create namespace diff --git a/test/e2e/simple.go b/test/e2e/simple.go index 8dab0278eb..50a0faab46 100644 --- a/test/e2e/simple.go +++ b/test/e2e/simple.go @@ -30,7 +30,7 @@ func E2ESimple(logger *log.Logger) error { kn.HandleStopSignal(ctx) logger.Printf("Knuu initialized with scope %s", kn.Scope) - testNet, err := testnet.New(kn, testnet.Options{}) + testNet, err := testnet.New(logger, kn, testnet.Options{}) testnet.NoError("failed to create testnet", err) defer testNet.Cleanup(ctx) @@ -44,7 +44,7 @@ func E2ESimple(logger *log.Logger) error { testNet.CreateGenesisNodes(ctx, 4, latestVersion, 10000000, 0, testnet.DefaultResources, true)) logger.Println("Creating txsim") - endpoints, err := testNet.RemoteGRPCEndpoints(ctx) + endpoints, err := testNet.RemoteGRPCEndpoints() testnet.NoError("failed to get remote gRPC endpoints", err) upgradeSchedule := map[int64]uint64{} err = testNet.CreateTxClient(ctx, "txsim", testnet.TxsimVersion, 10, "100-2000", 100, testnet.DefaultResources, endpoints[0], upgradeSchedule) diff --git a/test/e2e/testnet/node.go b/test/e2e/testnet/node.go index cf96ee905d..e640fa97cb 100644 --- a/test/e2e/testnet/node.go +++ b/test/e2e/testnet/node.go @@ -4,11 +4,11 @@ package testnet import ( "context" "fmt" + "log" "os" "path/filepath" serverconfig "github.com/cosmos/cosmos-sdk/server/config" - "github.com/rs/zerolog/log" "github.com/tendermint/tendermint/config" "github.com/tendermint/tendermint/crypto" "github.com/tendermint/tendermint/p2p" @@ -55,13 +55,15 @@ type Node struct { // FIXME: This does not work currently with the reverse proxy // grpcProxyHost string traceProxyHost string + + logger *log.Logger } // PullRoundStateTraces retrieves the round state traces from a node. // It will save them to the provided path. func (n *Node) PullRoundStateTraces(path string) ([]trace.Event[schema.RoundState], error) { addr := n.AddressTracing() - log.Info().Str("Address", addr).Msg("Pulling round state traces") + n.logger.Println("Pulling round state traces", "address", addr) err := trace.GetTable(addr, schema.RoundState{}.Table(), path) if err != nil { @@ -74,7 +76,7 @@ func (n *Node) PullRoundStateTraces(path string) ([]trace.Event[schema.RoundStat // It will save them to the provided path. func (n *Node) PullBlockSummaryTraces(path string) ([]trace.Event[schema.BlockSummary], error) { addr := n.AddressTracing() - log.Info().Str("Address", addr).Msg("Pulling block summary traces") + n.logger.Println("Pulling block summary traces", "address", addr) err := trace.GetTable(addr, schema.BlockSummary{}.Table(), path) if err != nil { @@ -97,6 +99,7 @@ type Resources struct { func NewNode( ctx context.Context, + logger *log.Logger, name string, version string, startHeight int64, @@ -178,6 +181,7 @@ func NewNode( NetworkKey: networkKey, SelfDelegation: selfDelegation, sidecars: sidecars, + logger: logger, }, nil } @@ -199,11 +203,13 @@ func (n *Node) Init(ctx context.Context, genesis *types.GenesisDoc, peers []stri } // Initialize file directories - rootDir := os.TempDir() - nodeDir := filepath.Join(rootDir, n.Name) - log.Info().Str("name", n.Name). - Str("directory", nodeDir). - Msg("Creating validator's config and data directories") + tmpDir, err := os.MkdirTemp("", "e2e_test_") + if err != nil { + return fmt.Errorf("failed to create temp dir: %w", err) + } + defer os.RemoveAll(tmpDir) + nodeDir := filepath.Join(tmpDir, n.Name) + n.logger.Println("Creating validator's config and data directories", "name", n.Name, "directory", nodeDir) for _, dir := range []string{ filepath.Join(nodeDir, "config"), filepath.Join(nodeDir, "data"), @@ -214,7 +220,7 @@ func (n *Node) Init(ctx context.Context, genesis *types.GenesisDoc, peers []stri } // Create and write the config file - cfg, err := MakeConfig(ctx, n, configOptions...) + cfg, err := MakeConfig(n, peers, configOptions...) if err != nil { return fmt.Errorf("making config: %w", err) } @@ -253,12 +259,6 @@ func (n *Node) Init(ctx context.Context, genesis *types.GenesisDoc, peers []stri pvStatePath := filepath.Join(nodeDir, "data", "priv_validator_state.json") (privval.NewFilePV(n.SignerKey, pvKeyPath, pvStatePath)).Save() - addrBookFile := filepath.Join(nodeDir, "config", "addrbook.json") - err = WriteAddressBook(peers, addrBookFile) - if err != nil { - return fmt.Errorf("writing address book: %w", err) - } - if err := n.Instance.Build().Commit(ctx); err != nil { return fmt.Errorf("committing instance: %w", err) } @@ -278,12 +278,9 @@ func (n *Node) Init(ctx context.Context, genesis *types.GenesisDoc, peers []stri // AddressP2P returns a P2P endpoint address for the node. This is used for // populating the address book. This will look something like: // 3314051954fc072a0678ec0cbac690ad8676ab98@61.108.66.220:26656 -func (n Node) AddressP2P(ctx context.Context, withID bool) string { - ip, err := n.Instance.Network().GetIP(ctx) - if err != nil { - panic(err) - } - addr := fmt.Sprintf("%v:%d", ip, p2pPort) +func (n Node) AddressP2P(withID bool) string { + hostName := n.Instance.Network().HostName() + addr := fmt.Sprintf("%v:%d", hostName, p2pPort) if withID { addr = fmt.Sprintf("%x@%v", n.NetworkKey.PubKey().Address().Bytes(), addr) } @@ -304,33 +301,24 @@ func (n Node) AddressRPC() string { // } // RemoteAddressGRPC retrieves the gRPC endpoint address of a node within the cluster. -func (n Node) RemoteAddressGRPC(ctx context.Context) (string, error) { - ip, err := n.Instance.Network().GetIP(ctx) - if err != nil { - return "", err - } - return fmt.Sprintf("%s:%d", ip, grpcPort), nil +func (n Node) RemoteAddressGRPC() (string, error) { + hostName := n.Instance.Network().HostName() + return fmt.Sprintf("%s:%d", hostName, grpcPort), nil } // RemoteAddressRPC retrieves the RPC endpoint address of a node within the cluster. -func (n Node) RemoteAddressRPC(ctx context.Context) (string, error) { - ip, err := n.Instance.Network().GetIP(ctx) - if err != nil { - return "", err - } - return fmt.Sprintf("%s:%d", ip, rpcPort), nil +func (n Node) RemoteAddressRPC() (string, error) { + hostName := n.Instance.Network().HostName() + return fmt.Sprintf("%s:%d", hostName, rpcPort), nil } func (n Node) AddressTracing() string { return n.traceProxyHost } -func (n Node) RemoteAddressTracing(ctx context.Context) (string, error) { - ip, err := n.Instance.Network().GetIP(ctx) - if err != nil { - return "", err - } - return fmt.Sprintf("http://%s:26661", ip), nil +func (n Node) RemoteAddressTracing() (string, error) { + hostName := n.Instance.Network().HostName() + return fmt.Sprintf("http://%s:26661", hostName), nil } func (n Node) IsValidator() bool { @@ -338,7 +326,7 @@ func (n Node) IsValidator() bool { } func (n Node) Client() (*http.HTTP, error) { - log.Debug().Str("RPC Address", n.AddressRPC()).Msg("Creating HTTP client for node") + n.logger.Println("Creating HTTP client for node", "rpc_address", n.AddressRPC()) return http.New(n.AddressRPC(), "/websocket") } diff --git a/test/e2e/testnet/setup.go b/test/e2e/testnet/setup.go index d7db215c34..33b64973c6 100644 --- a/test/e2e/testnet/setup.go +++ b/test/e2e/testnet/setup.go @@ -1,7 +1,6 @@ package testnet import ( - "context" "fmt" "strings" "time" @@ -13,15 +12,14 @@ import ( "github.com/tendermint/tendermint/p2p/pex" ) -func MakeConfig(ctx context.Context, node *Node, opts ...Option) (*config.Config, error) { +func MakeConfig(node *Node, peers []string, opts ...Option) (*config.Config, error) { cfg := app.DefaultConsensusConfig() cfg.TxIndex.Indexer = "kv" cfg.Consensus.TimeoutPropose = config.DefaultConsensusConfig().TimeoutPropose cfg.Consensus.TimeoutCommit = config.DefaultConsensusConfig().TimeoutCommit cfg.Moniker = node.Name cfg.RPC.ListenAddress = "tcp://0.0.0.0:26657" - cfg.P2P.ExternalAddress = fmt.Sprintf("tcp://%v", node.AddressP2P(ctx, false)) - cfg.P2P.PersistentPeers = strings.Join(node.InitialPeers, ",") + cfg.P2P.PersistentPeers = strings.Join(peers, ",") cfg.Instrumentation.Prometheus = true for _, opt := range opts { diff --git a/test/e2e/testnet/testnet.go b/test/e2e/testnet/testnet.go index 7cc74fb011..31ff18cabf 100644 --- a/test/e2e/testnet/testnet.go +++ b/test/e2e/testnet/testnet.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "log" "os" "path/filepath" "time" @@ -16,7 +17,6 @@ import ( "github.com/celestiaorg/knuu/pkg/preloader" "github.com/cosmos/cosmos-sdk/crypto/hd" "github.com/cosmos/cosmos-sdk/crypto/keyring" - "github.com/rs/zerolog/log" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" ) @@ -26,13 +26,17 @@ const ( ) type Testnet struct { - seed int64 - nodes []*Node - genesis *genesis.Genesis - keygen *keyGenerator - grafana *GrafanaInfo - txClients []*TxSim - knuu *knuu.Knuu + seed int64 + nodes []*Node + genesis *genesis.Genesis + keygen *keyGenerator + grafana *GrafanaInfo + txClients []*TxSim + knuu *knuu.Knuu + chainID string + genesisHash string + + logger *log.Logger } type Options struct { @@ -42,15 +46,18 @@ type Options struct { GenesisModifiers []genesis.Modifier } -func New(knuu *knuu.Knuu, opts Options) (*Testnet, error) { +func New(logger *log.Logger, knuu *knuu.Knuu, opts Options) (*Testnet, error) { opts.setDefaults() return &Testnet{ - seed: opts.Seed, - nodes: make([]*Node, 0), - genesis: genesis.NewDefaultGenesis().WithChainID(opts.ChainID).WithModifiers(opts.GenesisModifiers...), - keygen: newKeyGenerator(opts.Seed), - grafana: opts.Grafana, - knuu: knuu, + seed: opts.Seed, + nodes: make([]*Node, 0), + genesis: genesis.NewDefaultGenesis().WithChainID(opts.ChainID).WithModifiers(opts.GenesisModifiers...), + keygen: newKeyGenerator(opts.Seed), + grafana: opts.Grafana, + knuu: knuu, + chainID: opts.ChainID, + genesisHash: "", + logger: logger, }, nil } @@ -74,7 +81,7 @@ func (t *Testnet) SetConsensusMaxBlockSize(size int64) { func (t *Testnet) CreateGenesisNode(ctx context.Context, version string, selfDelegation, upgradeHeightV2 int64, resources Resources, disableBBR bool) error { signerKey := t.keygen.Generate(ed25519Type) networkKey := t.keygen.Generate(ed25519Type) - node, err := NewNode(ctx, fmt.Sprintf("val%d", len(t.nodes)), version, 0, selfDelegation, nil, signerKey, networkKey, upgradeHeightV2, resources, t.grafana, t.knuu, disableBBR) + node, err := NewNode(ctx, t.logger, fmt.Sprintf("val%d", len(t.nodes)), version, 0, selfDelegation, nil, signerKey, networkKey, upgradeHeightV2, resources, t.grafana, t.knuu, disableBBR) if err != nil { return err } @@ -107,15 +114,10 @@ func (t *Testnet) CreateTxClients(ctx context.Context, name := fmt.Sprintf("txsim%d", i) err := t.CreateTxClient(ctx, name, version, sequences, blobRange, blobPerSequence, resources, grpcEndpoint, upgradeSchedule) if err != nil { - log.Err(err).Str("name", name). - Str("grpc endpoint", grpcEndpoint). - Msg("txsim creation failed") + t.logger.Println("txsim creation failed", "name", name, "grpc_endpoint", grpcEndpoint, "error", err) return err } - log.Info(). - Str("name", name). - Str("grpc endpoint", grpcEndpoint). - Msg("txsim created") + t.logger.Println("txsim created", "name", name, "grpc_endpoint", grpcEndpoint) } return nil } @@ -143,8 +145,12 @@ func (t *Testnet) CreateTxClient( grpcEndpoint string, upgradeSchedule map[int64]uint64, ) error { - txsimKeyringDir := filepath.Join(os.TempDir(), name) - defer os.RemoveAll(txsimKeyringDir) + tmpDir, err := os.MkdirTemp("", "e2e_test_") + if err != nil { + return fmt.Errorf("failed to create temp dir: %w", err) + } + defer os.RemoveAll(tmpDir) + txsimKeyringDir := filepath.Join(tmpDir, name) config := encoding.MakeConfig(app.ModuleEncodingRegisters...).Codec txsimKeyring, err := keyring.New(app.Name, keyring.BackendTest, txsimKeyringDir, nil, config) @@ -182,29 +188,22 @@ func (t *Testnet) CreateTxClient( } } - txsim, err := CreateTxClient(ctx, name, version, grpcEndpoint, t.seed, blobSequences, blobRange, blobPerSequence, 1, resources, txsimKeyringDir, t.knuu, upgradeSchedule) + txsim, err := CreateTxClient(ctx, t.logger, name, version, grpcEndpoint, t.seed, blobSequences, blobRange, blobPerSequence, 1, resources, remoteRootDir, t.knuu, upgradeSchedule) if err != nil { - log.Err(err). - Str("name", name). - Msg("error creating txsim") + t.logger.Println("error creating txsim", "name", name, "error", err) return err } err = txsim.Instance.Build().Commit(ctx) if err != nil { - log.Err(err). - Str("name", name). - Msg("error committing txsim") + t.logger.Println("error committing txsim", "name", name, "error", err) return err } // copy over the keyring directory to the txsim instance - err = txsim.Instance.Storage().AddFolder(txsimKeyringDir, txsimKeyringDir, "10001:10001") + err = txsim.Instance.Storage().AddFolder(txsimKeyringDir, remoteRootDir, "10001:10001") if err != nil { - log.Err(err). - Str("directory", txsimKeyringDir). - Str("name", name). - Msg("error adding keyring dir to txsim") + t.logger.Println("error adding keyring dir to txsim", "directory", txsimKeyringDir, "name", name, "error", err) return err } @@ -216,14 +215,10 @@ func (t *Testnet) StartTxClients(ctx context.Context) error { for _, txsim := range t.txClients { err := txsim.Instance.Execution().StartAsync(ctx) if err != nil { - log.Err(err). - Str("name", txsim.Name). - Msg("txsim failed to start") + t.logger.Println("txsim failed to start", "name", txsim.Name, "error", err) return err } - log.Info(). - Str("name", txsim.Name). - Msg("txsim started") + t.logger.Println("txsim started", "name", txsim.Name) } // wait for txsims to start for _, txsim := range t.txClients { @@ -271,17 +266,14 @@ func (t *Testnet) CreateAccount(name string, tokens int64, txsimKeyringDir strin return nil, err } - log.Info(). - Str("name", name). - Str("pk", pk.String()). - Msg("txsim account created and added to genesis") + t.logger.Println("txsim account created and added to genesis", "name", name, "pk", pk.String()) return kr, nil } func (t *Testnet) CreateNode(ctx context.Context, version string, startHeight, upgradeHeight int64, resources Resources, disableBBR bool) error { signerKey := t.keygen.Generate(ed25519Type) networkKey := t.keygen.Generate(ed25519Type) - node, err := NewNode(ctx, fmt.Sprintf("val%d", len(t.nodes)), version, startHeight, 0, nil, signerKey, networkKey, upgradeHeight, resources, t.grafana, t.knuu, disableBBR) + node, err := NewNode(ctx, t.logger, fmt.Sprintf("val%d", len(t.nodes)), version, startHeight, 0, nil, signerKey, networkKey, upgradeHeight, resources, t.grafana, t.knuu, disableBBR) if err != nil { return err } @@ -297,11 +289,11 @@ func (t *Testnet) Setup(ctx context.Context, configOpts ...Option) error { for _, node := range t.nodes { // nodes are initialized with the addresses of all other - // nodes in their addressbook + // nodes as trusted peers peers := make([]string, 0, len(t.nodes)-1) for _, peer := range t.nodes { if peer.Name != node.Name { - peers = append(peers, peer.AddressP2P(ctx, true)) + peers = append(peers, peer.AddressP2P(true)) } } @@ -333,10 +325,10 @@ func (t *Testnet) RPCEndpoints() []string { // RemoteGRPCEndpoints retrieves the gRPC endpoint addresses of the // testnet's validator nodes. -func (t *Testnet) RemoteGRPCEndpoints(ctx context.Context) ([]string, error) { +func (t *Testnet) RemoteGRPCEndpoints() ([]string, error) { grpcEndpoints := make([]string, len(t.nodes)) for idx, node := range t.nodes { - grpcEP, err := node.RemoteAddressGRPC(ctx) + grpcEP, err := node.RemoteAddressGRPC() if err != nil { return nil, err } @@ -355,10 +347,10 @@ func (t *Testnet) GetGenesisValidators() []genesis.Validator { // RemoteRPCEndpoints retrieves the RPC endpoint addresses of the testnet's // validator nodes. -func (t *Testnet) RemoteRPCEndpoints(ctx context.Context) ([]string, error) { +func (t *Testnet) RemoteRPCEndpoints() ([]string, error) { rpcEndpoints := make([]string, len(t.nodes)) for idx, node := range t.nodes { - grpcEP, err := node.RemoteAddressRPC(ctx) + grpcEP, err := node.RemoteAddressRPC() if err != nil { return nil, err } @@ -367,6 +359,8 @@ func (t *Testnet) RemoteRPCEndpoints(ctx context.Context) ([]string, error) { return rpcEndpoints, nil } +const maxSyncAttempts = 20 + // WaitToSync waits for the started nodes to sync with the network and move // past the genesis block. func (t *Testnet) WaitToSync(ctx context.Context) error { @@ -378,29 +372,28 @@ func (t *Testnet) WaitToSync(ctx context.Context) error { } for _, node := range genesisNodes { - log.Info().Str("name", node.Name).Msg( - "waiting for node to sync") + t.logger.Println("waiting for node to sync", "name", node.Name) client, err := node.Client() if err != nil { return fmt.Errorf("failed to initialize client for node %s: %w", node.Name, err) } - for i := 0; i < 10; i++ { + var lastErr error + for i := 0; i < maxSyncAttempts; i++ { resp, err := client.Status(ctx) + lastErr = err if err == nil { - if resp.SyncInfo.LatestBlockHeight > 0 { - log.Info().Int("attempts", i).Str("name", node.Name).Msg( - "node has synced") + if resp != nil && resp.SyncInfo.LatestBlockHeight > 0 { + t.logger.Println("node has synced", "name", node.Name, "attempts", i, "latest_block_height", resp.SyncInfo.LatestBlockHeight) break } + t.logger.Println("node status retrieved but not synced yet, waiting...", "name", node.Name, "attempt", i) } else { - err = errors.New("error getting status") + t.logger.Println("error getting status, retrying...", "name", node.Name, "attempt", i, "error", err) } - if i == 9 { - return fmt.Errorf("failed to start node %s: %w", node.Name, err) + if i == maxSyncAttempts-1 { + return fmt.Errorf("timed out waiting for node %s to sync: %w", node.Name, lastErr) } - log.Info().Str("name", node.Name).Int("attempt", i).Msg( - "node is not synced yet, waiting...") - time.Sleep(time.Duration(i) * time.Second) + time.Sleep(time.Second * time.Duration(1<