From c4c7e947fcf7d11e8a59ec6e7996fa02b81d60ac Mon Sep 17 00:00:00 2001 From: Rootul P Date: Mon, 30 Oct 2023 11:54:04 +0100 Subject: [PATCH 1/5] feat(addrbook): skip peers with errors (#2783) Closes https://github.com/celestiaorg/celestia-app/issues/2784 Part of https://github.com/celestiaorg/networks/issues/513 --- cmd/celestia-appd/cmd/addrbook.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/celestia-appd/cmd/addrbook.go b/cmd/celestia-appd/cmd/addrbook.go index f363ed3626..296b72e152 100644 --- a/cmd/celestia-appd/cmd/addrbook.go +++ b/cmd/celestia-appd/cmd/addrbook.go @@ -37,11 +37,13 @@ func addrbookCommand() *cobra.Command { } address, err := p2p.NewNetAddressString(line) if err != nil { - return err + fmt.Printf("Error parsing %s: %s\n", line, err) + continue } err = book.AddAddress(address, address) if err != nil { - return err + fmt.Printf("Error adding %s: %s\n", address, err) + continue } } From 0cc6a543b6a67cc7464d109be23421bc5e67d4f4 Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Mon, 30 Oct 2023 15:40:08 +0100 Subject: [PATCH 2/5] fix: nightly e2e workflow (#2670) The workflow wasn't correctly pulling the latest version. This fixes that. It also updates the default to "latest" --- .github/workflows/nightly.yml | 1 + Makefile | 5 ++--- test/e2e/simple_test.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 87420c62c9..d19ea9b1b3 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -3,6 +3,7 @@ on: schedule: # runs every day at 6am UTC - cron: "0 6 * * *" + workflow_dispatch: env: GO_VERSION: '1.21.1' diff --git a/Makefile b/Makefile index 54a313c499..f9a9375de4 100644 --- a/Makefile +++ b/Makefile @@ -117,9 +117,8 @@ test-short: ## test-e2e: Run end to end tests via knuu. test-e2e: - @version=$(git rev-parse --short HEAD) - @echo "--> Running e2e tests on version: $version" - @KNUU_NAMESPACE=test E2E_VERSION=$version E2E=true go test ./test/e2e/... -timeout 30m + @echo "--> Running e2e tests on version: $(shell git rev-parse --short HEAD)" + @KNUU_NAMESPACE=test E2E_VERSION=$(shell git rev-parse --short HEAD) E2E=true go test ./test/e2e/... -timeout 10m -v .PHONY: test-e2e ## test-race: Run tests in race mode. diff --git a/test/e2e/simple_test.go b/test/e2e/simple_test.go index 71433ad69d..dc668ad3b9 100644 --- a/test/e2e/simple_test.go +++ b/test/e2e/simple_test.go @@ -16,7 +16,7 @@ import ( const seed = 42 -var latestVersion = "v1.0.0-rc12" +var latestVersion = "latest" // This test runs a simple testnet with 4 validators. It submits both MsgPayForBlobs // and MsgSends over 30 seconds and then asserts that at least 10 transactions were From 1f0a521c63126356d3aa47baef55bf501376352f Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Mon, 30 Oct 2023 15:40:31 +0100 Subject: [PATCH 3/5] ci: add fuzz tests to nightly run (#2787) --- .github/workflows/nightly.yml | 15 +++++++++++++-- Makefile | 4 ++++ scripts/test_fuzz.sh | 10 ++++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) create mode 100755 scripts/test_fuzz.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index d19ea9b1b3..40c64da4a2 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -9,7 +9,7 @@ env: GO_VERSION: '1.21.1' jobs: - e2e-tests: + test-e2e: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -24,6 +24,17 @@ jobs: run: | mkdir -p $HOME/.kube echo "${KUBECONFIG_FILE}" > $HOME/.kube/config - - name: Run e2e tests run: make test-e2e + + test-fuzz: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v4 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Run fuzz tests + run: make test-fuzz diff --git a/Makefile b/Makefile index f9a9375de4..27dbcec62d 100644 --- a/Makefile +++ b/Makefile @@ -141,6 +141,10 @@ test-coverage: @export VERSION=$(VERSION); bash -x scripts/test_cover.sh .PHONY: test-coverage +test-fuzz: + bash -x scripts/test_fuzz.sh +.PHONY: test-fuzz + ## txsim-install: Install the tx simulator. txsim-install: @echo "--> Installing tx simulator" diff --git a/scripts/test_fuzz.sh b/scripts/test_fuzz.sh new file mode 100755 index 0000000000..4d65c4f06a --- /dev/null +++ b/scripts/test_fuzz.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -e + +echo "Running fuzz tests..." +# manually specify the tests to fuzz since go toolchain doesn't support +# fuzzing multiple packages with multiple fuzz tests +go test -fuzz=FuzzNewInfoByte -fuzztime 1m ./pkg/shares +go test -fuzz=FuzzValidSequenceLen -fuzztime 1m ./pkg/shares +go test -fuzz=FuzzSquare -fuzztime 5m ./pkg/square +go test -fuzz=FuzzPFBGasEstimation -fuzztime 3m ./x/blob/types From e6b64a6012ea5c30ab12fae5901cca3ba20c7724 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Tue, 31 Oct 2023 11:25:51 +0100 Subject: [PATCH 4/5] test: Ledger support (#2782) Closes https://github.com/celestiaorg/celestia-app/issues/2225 Note: this test won't fail when Goreleaser creates binaries that break Ledger support. However, this may catch when we bump a dependency that breaks Ledger support. --- test/ledger/ledger_test.go | 62 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 test/ledger/ledger_test.go diff --git a/test/ledger/ledger_test.go b/test/ledger/ledger_test.go new file mode 100644 index 0000000000..eeda5314d9 --- /dev/null +++ b/test/ledger/ledger_test.go @@ -0,0 +1,62 @@ +package ledger + +import ( + "bytes" + "os" + "os/exec" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLedgerSupport(t *testing.T) { + if testing.Short() { + t.Skip("skipping ledger support test in short mode.") + } + + type testCase struct { + name string + ledger bool + want string + } + + testCases := []testCase{ + { + name: "ledger support enabled", + ledger: true, + want: "Error: failed to generate ledger key: failed to retrieve device: ledger nano S: LedgerHID device (idx 0) not found. Ledger LOCKED OR Other Program/Web Browser may have control of device.\n", + }, + { + name: "ledger support disabled", + ledger: false, + want: "Error: failed to generate ledger key: failed to retrieve device: ledger nano S: support for ledger devices is not available in this executable\n", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var cmd *exec.Cmd + if tc.ledger { + // Generate the binary with Ledger support + cmd = exec.Command("go", "build", "-tags", "ledger", "-o", "celestia-appd", "../../cmd/celestia-appd") + } else { + // Generate the binary without Ledger support + cmd = exec.Command("go", "build", "-o", "celestia-appd", "../../cmd/celestia-appd") + } + err := cmd.Run() + require.NoError(t, err) + + // Clean up the binary + defer os.Remove("celestia-appd") + + // Run the binary + cmd = exec.Command("./celestia-appd", "keys", "add", "test-key-name", "--ledger") + var out bytes.Buffer + cmd.Stderr = &out + err = cmd.Run() + require.Error(t, err) + assert.Equal(t, tc.want, out.String()) + }) + } +} From c34a93d79419a75e077103c3f1785c64f854641c Mon Sep 17 00:00:00 2001 From: Rootul P Date: Tue, 31 Oct 2023 15:26:22 +0100 Subject: [PATCH 5/5] feat: `download-genesis` command (#2791) Closes https://github.com/celestiaorg/celestia-app/issues/2777 Note for reviewers: I'd rather not backport this PR to v1.x b/c it's a new feature and not a bug fix. I wanted to have it ready in case ppl do want it on v1.x ## Testing Works for all 3 known networks. Complains on an unknown network ```shell $ ./build/celestia-appd download-genesis mocha-4 Downloading genesis file for mocha-4 to /Users/rootulp/.celestia-app/config/genesis.json Downloaded genesis file for mocha-4 to /Users/rootulp/.celestia-app/config/genesis.json SHA-256 hash verified for mocha-4 $ ./build/celestia-appd download-genesis celestia Downloading genesis file for celestia to /Users/rootulp/.celestia-app/config/genesis.json Downloaded genesis file for celestia to /Users/rootulp/.celestia-app/config/genesis.json SHA-256 hash verified for celestia $ ./build/celestia-appd download-genesis arabica-10 Downloading genesis file for arabica-10 to /Users/rootulp/.celestia-app/config/genesis.json Downloaded genesis file for arabica-10 to /Users/rootulp/.celestia-app/config/genesis.json SHA-256 hash verified for arabica-10 $ ./build/celestia-appd download-genesis foo Error: unknown chain-id: foo. Must be: celestia, mocha-4, or arabica-10. ``` I verified the chain ID actually differs for each file: ```shell $ ./build/celestia-appd download-genesis mocha-4 && cat ~/.celestia-app/config/genesis.json | jq ."chain_id" Downloading genesis file for mocha-4 to /Users/rootulp/.celestia-app/config/genesis.json Downloaded genesis file for mocha-4 to /Users/rootulp/.celestia-app/config/genesis.json SHA-256 hash verified for mocha-4 "mocha-4" $ ./build/celestia-appd download-genesis celestia && cat ~/.celestia-app/config/genesis.json | jq ."chain_id" Downloading genesis file for celestia to /Users/rootulp/.celestia-app/config/genesis.json Downloaded genesis file for celestia to /Users/rootulp/.celestia-app/config/genesis.json SHA-256 hash verified for celestia "celestia" ``` I tampered with the hard-coded Arabica hash and it correctly identified the mismatch: ```shell $ ./build/celestia-appd download-genesis arabica-10 Downloading genesis file for arabica-10 to /Users/rootulp/.celestia-app/config/genesis.json Downloaded genesis file for arabica-10 to /Users/rootulp/.celestia-app/config/genesis.json Error: sha256 hash mismatch: got fad0a187669f7a2c11bb07f9dc27140d66d2448b7193e186312713856f28e3e1, expected fad0a187669f7a2c11bb07f9dc27140d66d2448b7193e186312713856f28e3e2 ``` --------- Co-authored-by: Sanaz Taheri <35961250+staheri14@users.noreply.github.com> --- cmd/celestia-appd/cmd/download-genesis.go | 128 ++++++++++++++++++++++ cmd/celestia-appd/cmd/root.go | 1 + 2 files changed, 129 insertions(+) create mode 100644 cmd/celestia-appd/cmd/download-genesis.go diff --git a/cmd/celestia-appd/cmd/download-genesis.go b/cmd/celestia-appd/cmd/download-genesis.go new file mode 100644 index 0000000000..ced81d352f --- /dev/null +++ b/cmd/celestia-appd/cmd/download-genesis.go @@ -0,0 +1,128 @@ +package cmd + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "os" + + "github.com/cosmos/cosmos-sdk/server" + "github.com/spf13/cobra" +) + +var chainIDToSha256 = map[string]string{ + "celestia": "9727aac9bbfb021ce7fc695a92f901986421283a891b89e0af97bc9fad187793", + "mocha-4": "0846b99099271b240b638a94e17a6301423b5e4047f6558df543d6e91db7e575", + "arabica-10": "fad0a187669f7a2c11bb07f9dc27140d66d2448b7193e186312713856f28e3e1", +} + +func downloadGenesisCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "download-genesis [chain-id]", + Short: "Download genesis file from https://github.com/celestiaorg/networks", + Long: "Download genesis file from https://github.com/celestiaorg/networks.\n" + + "The first argument should be a known chain-id. Ex. celestia, mocha-4, or arabica-10.\n" + + "If no argument is provided, defaults to celestia.\n", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + chainID := getChainIDOrDefault(args) + if !isKnownChainID(chainID) { + return fmt.Errorf("unknown chain-id: %s. Must be: celestia, mocha-4, or arabica-10", chainID) + } + outputFile := server.GetServerContextFromCmd(cmd).Config.GenesisFile() + fmt.Printf("Downloading genesis file for %s to %s\n", chainID, outputFile) + + url := fmt.Sprintf("https://raw.githubusercontent.com/celestiaorg/networks/master/%s/genesis.json", chainID) + if err := downloadFile(outputFile, url); err != nil { + return fmt.Errorf("error downloading / persisting the genesis file: %s", err) + } + fmt.Printf("Downloaded genesis file for %s to %s\n", chainID, outputFile) + + // Compute SHA-256 hash of the downloaded file + hash, err := computeSha256(outputFile) + if err != nil { + return fmt.Errorf("error computing sha256 hash: %s", err) + } + + // Compare computed hash against known hash + knownHash, ok := chainIDToSha256[chainID] + if !ok { + return fmt.Errorf("unknown chain-id: %s", chainID) + } + + if hash != knownHash { + return fmt.Errorf("sha256 hash mismatch: got %s, expected %s", hash, knownHash) + } + + fmt.Printf("SHA-256 hash verified for %s\n", chainID) + return nil + }, + } + + return cmd +} + +// getChainIDOrDefault returns the chainID from the command line arguments. If +// none is provided, defaults to celestia (mainnet). +func getChainIDOrDefault(args []string) string { + if len(args) == 1 { + return args[0] + } + return "celestia" +} + +// isKnownChainID returns true if the chainID is known. +func isKnownChainID(chainID string) bool { + knownChainIDs := []string{ + "arabica-10", // testnet + "mocha-4", // testnet + "celestia", // mainnet + } + return contains(knownChainIDs, chainID) +} + +// contains checks if a string is present in a slice. +func contains(slice []string, s string) bool { + for _, v := range slice { + if v == s { + return true + } + } + return false +} + +// downloadFile will download a URL to a local file. +func downloadFile(filepath string, url string) error { + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + + out, err := os.Create(filepath) + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, resp.Body) + return err +} + +// computeSha256 computes the SHA-256 hash of a file. +func computeSha256(filepath string) (string, error) { + f, err := os.Open(filepath) + if err != nil { + return "", err + } + defer f.Close() + + hasher := sha256.New() + if _, err := io.Copy(hasher, f); err != nil { + return "", err + } + + return hex.EncodeToString(hasher.Sum(nil)), nil +} diff --git a/cmd/celestia-appd/cmd/root.go b/cmd/celestia-appd/cmd/root.go index be95396665..12fea2ed71 100644 --- a/cmd/celestia-appd/cmd/root.go +++ b/cmd/celestia-appd/cmd/root.go @@ -136,6 +136,7 @@ func initRootCmd(rootCmd *cobra.Command, encodingConfig encoding.Config) { config.Cmd(), commands.CompactGoLevelDBCmd, addrbookCommand(), + downloadGenesisCommand(), ) server.AddCommands(rootCmd, app.DefaultNodeHome, NewAppServer, createAppAndExport, addModuleInitFlags)