From a28b9e7160cc44568a82534f4b0558c1d88d167c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nina=20/=20=E1=83=9C=E1=83=98=E1=83=9C=E1=83=90?= Date: Fri, 16 Aug 2024 14:15:37 +0200 Subject: [PATCH 01/85] refactor: signer to use txstatus (#3767) ## Overview Fixes #3366 Opens - https://github.com/celestiaorg/celestia-core/issues/1453 and https://github.com/celestiaorg/celestia-core/issues/1454 --- app/test/big_blob_test.go | 3 +- app/test/prepare_proposal_context_test.go | 8 ++- app/test/square_size_test.go | 8 ++- app/test/std_sdk_test.go | 8 ++- pkg/user/tx_client.go | 85 +++++++++++++++-------- pkg/user/tx_client_test.go | 59 +++++++++++----- test/txsim/account.go | 2 +- x/blobstream/integration_test.go | 8 ++- x/signal/legacy_test.go | 19 +++-- 9 files changed, 143 insertions(+), 57 deletions(-) diff --git a/app/test/big_blob_test.go b/app/test/big_blob_test.go index 41934c6379..ad219c2e3d 100644 --- a/app/test/big_blob_test.go +++ b/app/test/big_blob_test.go @@ -83,7 +83,8 @@ func (s *BigBlobSuite) TestErrBlobsTooLarge() { res, err := txClient.SubmitPayForBlob(subCtx, []*share.Blob{tc.blob}, user.SetGasLimitAndGasPrice(1e9, appconsts.DefaultMinGasPrice)) require.Error(t, err) require.NotNil(t, res) - require.Equal(t, tc.want, res.Code, res.Logs) + // FIXME: assert RawLog once TxStatus supports it. + require.Equal(t, tc.want, res.Code) }) } } diff --git a/app/test/prepare_proposal_context_test.go b/app/test/prepare_proposal_context_test.go index dbec6798b3..7eb14bf9a8 100644 --- a/app/test/prepare_proposal_context_test.go +++ b/app/test/prepare_proposal_context_test.go @@ -12,6 +12,7 @@ import ( "github.com/cosmos/cosmos-sdk/crypto/hd" "github.com/cosmos/cosmos-sdk/crypto/keyring" sdk "github.com/cosmos/cosmos-sdk/types" + sdktx "github.com/cosmos/cosmos-sdk/types/tx" vestingtypes "github.com/cosmos/cosmos-sdk/x/auth/vesting/types" banktypes "github.com/cosmos/cosmos-sdk/x/bank/types" "github.com/stretchr/testify/assert" @@ -80,8 +81,13 @@ func TestTimeInPrepareProposalContext(t *testing.T) { msgs, _ := tt.msgFunc() res, err := txClient.SubmitTx(cctx.GoContext(), msgs, user.SetGasLimit(1000000), user.SetFee(2000)) require.NoError(t, err) + // FIXME: Temporary way of querying the raw log. + // TxStatus will natively support this in the future. + serviceClient := sdktx.NewServiceClient(cctx.GRPCClient) + getTxResp, err := serviceClient.GetTx(cctx.GoContext(), &sdktx.GetTxRequest{Hash: res.TxHash}) + require.NoError(t, err) require.NotNil(t, res) - assert.Equal(t, abci.CodeTypeOK, res.Code, res.RawLog) + assert.Equal(t, abci.CodeTypeOK, res.Code, getTxResp.TxResponse.RawLog) }) } } diff --git a/app/test/square_size_test.go b/app/test/square_size_test.go index e8082b17f5..653f4b4ae4 100644 --- a/app/test/square_size_test.go +++ b/app/test/square_size_test.go @@ -17,6 +17,7 @@ import ( "github.com/celestiaorg/celestia-app/v3/test/util/testnode" blobtypes "github.com/celestiaorg/celestia-app/v3/x/blob/types" sdk "github.com/cosmos/cosmos-sdk/types" + sdktx "github.com/cosmos/cosmos-sdk/types/tx" v1 "github.com/cosmos/cosmos-sdk/x/gov/types/v1" oldgov "github.com/cosmos/cosmos-sdk/x/gov/types/v1beta1" "github.com/cosmos/cosmos-sdk/x/params/types/proposal" @@ -173,7 +174,12 @@ func (s *SquareSizeIntegrationTest) setBlockSizeParams(t *testing.T, squareSize, res, err := txClient.SubmitTx(s.cctx.GoContext(), []sdk.Msg{msg}, blobfactory.DefaultTxOpts()...) require.NoError(t, err) - require.Equal(t, res.Code, abci.CodeTypeOK, res.RawLog) + // FIXME: Temporary way of querying the raw log. + // TxStatus will natively support this in the future. + serviceClient := sdktx.NewServiceClient(s.cctx.GRPCClient) + getTxResp, err := serviceClient.GetTx(s.cctx.GoContext(), &sdktx.GetTxRequest{Hash: res.TxHash}) + require.NoError(t, err) + require.Equal(t, res.Code, abci.CodeTypeOK, getTxResp.TxResponse.RawLog) require.NoError(t, s.cctx.WaitForNextBlock()) diff --git a/app/test/std_sdk_test.go b/app/test/std_sdk_test.go index 1ef3766e8a..ca4b401768 100644 --- a/app/test/std_sdk_test.go +++ b/app/test/std_sdk_test.go @@ -21,6 +21,7 @@ import ( "github.com/cosmos/cosmos-sdk/crypto/keyring" "github.com/cosmos/cosmos-sdk/testutil/mock" sdk "github.com/cosmos/cosmos-sdk/types" + sdktx "github.com/cosmos/cosmos-sdk/types/tx" vestingtypes "github.com/cosmos/cosmos-sdk/x/auth/vesting/types" banktypes "github.com/cosmos/cosmos-sdk/x/bank/types" disttypes "github.com/cosmos/cosmos-sdk/x/distribution/types" @@ -325,8 +326,13 @@ func (s *StandardSDKIntegrationTestSuite) TestStandardSDK() { } else { require.NoError(t, err) } + // FIXME: Temporary way of querying the raw log. + // TxStatus will natively support this in the future. + serviceClient := sdktx.NewServiceClient(s.cctx.GRPCClient) + getTxResp, err := serviceClient.GetTx(s.cctx.GoContext(), &sdktx.GetTxRequest{Hash: res.TxHash}) + require.NoError(t, err) require.NotNil(t, res) - assert.Equal(t, tt.expectedCode, res.Code, res.RawLog) + assert.Equal(t, tt.expectedCode, res.Code, getTxResp.TxResponse.RawLog) }) } } diff --git a/pkg/user/tx_client.go b/pkg/user/tx_client.go index 0d14128421..75863f1293 100644 --- a/pkg/user/tx_client.go +++ b/pkg/user/tx_client.go @@ -27,6 +27,7 @@ import ( "github.com/celestiaorg/celestia-app/v3/app" "github.com/celestiaorg/celestia-app/v3/app/encoding" apperrors "github.com/celestiaorg/celestia-app/v3/app/errors" + "github.com/celestiaorg/celestia-app/v3/app/grpc/tx" "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" "github.com/celestiaorg/celestia-app/v3/x/blob/types" "github.com/celestiaorg/celestia-app/v3/x/minfee" @@ -39,6 +40,14 @@ const ( type Option func(client *TxClient) +// TxResponse is a response from the chain after +// a transaction has been submitted. +type TxResponse struct { + Height int64 + TxHash string + Code uint32 +} + // WithGasMultiplier is a functional option allows to configure the gas multiplier. func WithGasMultiplier(multiplier float64) Option { return func(c *TxClient) { @@ -200,19 +209,25 @@ func SetupTxClient( // SubmitPayForBlob forms a transaction from the provided blobs, signs it, and submits it to the chain. // TxOptions may be provided to set the fee and gas limit. -func (client *TxClient) SubmitPayForBlob(ctx context.Context, blobs []*share.Blob, opts ...TxOption) (*sdktypes.TxResponse, error) { +func (client *TxClient) SubmitPayForBlob(ctx context.Context, blobs []*share.Blob, opts ...TxOption) (*TxResponse, error) { resp, err := client.BroadcastPayForBlob(ctx, blobs, opts...) - if err != nil { - return resp, err + if err != nil && resp != nil { + return &TxResponse{Code: resp.Code, TxHash: resp.TxHash}, fmt.Errorf("failed to broadcast pay for blob: %v", err) + } else if err != nil { + return &TxResponse{}, fmt.Errorf("failed to broadcast pay for blob: %v", err) } return client.ConfirmTx(ctx, resp.TxHash) } -func (client *TxClient) SubmitPayForBlobWithAccount(ctx context.Context, account string, blobs []*share.Blob, opts ...TxOption) (*sdktypes.TxResponse, error) { +// SubmitPayForBlobWithAccount forms a transaction from the provided blobs, signs it with the provided account, and submits it to the chain. +// TxOptions may be provided to set the fee and gas limit. +func (client *TxClient) SubmitPayForBlobWithAccount(ctx context.Context, account string, blobs []*share.Blob, opts ...TxOption) (*TxResponse, error) { resp, err := client.BroadcastPayForBlobWithAccount(ctx, account, blobs, opts...) - if err != nil { - return resp, err + if err != nil && resp != nil { + return &TxResponse{Code: resp.Code, TxHash: resp.TxHash}, fmt.Errorf("failed to broadcast pay for blob with account: %v", err) + } else if err != nil { + return &TxResponse{}, fmt.Errorf("failed to broadcast pay for blob with account: %v", err) } return client.ConfirmTx(ctx, resp.TxHash) @@ -253,10 +268,12 @@ func (client *TxClient) BroadcastPayForBlobWithAccount(ctx context.Context, acco // SubmitTx forms a transaction from the provided messages, signs it, and submits it to the chain. TxOptions // may be provided to set the fee and gas limit. -func (client *TxClient) SubmitTx(ctx context.Context, msgs []sdktypes.Msg, opts ...TxOption) (*sdktypes.TxResponse, error) { +func (client *TxClient) SubmitTx(ctx context.Context, msgs []sdktypes.Msg, opts ...TxOption) (*TxResponse, error) { resp, err := client.BroadcastTx(ctx, msgs, opts...) - if err != nil { - return resp, err + if err != nil && resp != nil { + return &TxResponse{Code: resp.Code, TxHash: resp.TxHash}, fmt.Errorf("failed to broadcast tx: %v", err) + } else if err != nil { + return &TxResponse{}, fmt.Errorf("failed to broadcast tx: %v", err) } return client.ConfirmTx(ctx, resp.TxHash) @@ -414,32 +431,44 @@ func (client *TxClient) retryBroadcastingTx(ctx context.Context, txBytes []byte) // ConfirmTx periodically pings the provided node for the commitment of a transaction by its // hash. It will continually loop until the context is cancelled, the tx is found or an error // is encountered. -func (client *TxClient) ConfirmTx(ctx context.Context, txHash string) (*sdktypes.TxResponse, error) { - txClient := sdktx.NewServiceClient(client.grpc) +func (client *TxClient) ConfirmTx(ctx context.Context, txHash string) (*TxResponse, error) { + txClient := tx.NewTxClient(client.grpc) pollTicker := time.NewTicker(client.pollTime) defer pollTicker.Stop() for { - resp, err := txClient.GetTx(ctx, &sdktx.GetTxRequest{Hash: txHash}) - if err == nil { - if resp.TxResponse.Code != 0 { - return resp.TxResponse, fmt.Errorf("tx was included but failed with code %d: %s", resp.TxResponse.Code, resp.TxResponse.RawLog) - } - return resp.TxResponse, nil - } - // FIXME: this is a relatively brittle of working out whether to retry or not. The tx might be not found for other - // reasons. It may have been removed from the mempool at a later point. We should build an endpoint that gives the - // signer more information on the status of their transaction and then update the logic here - if !strings.Contains(err.Error(), "not found") { - return &sdktypes.TxResponse{}, err + resp, err := txClient.TxStatus(ctx, &tx.TxStatusRequest{TxId: txHash}) + if err != nil { + return &TxResponse{}, err } - // Wait for the next round. - select { - case <-ctx.Done(): - return &sdktypes.TxResponse{}, ctx.Err() - case <-pollTicker.C: + if err == nil && resp != nil { + switch resp.Status { + // FIXME: replace hardcoded status with constants + case "PENDING": + // Continue polling if the transaction is still pending + select { + case <-ctx.Done(): + return &TxResponse{}, ctx.Err() + case <-pollTicker.C: + continue + } + case "COMMITTED": + txResponse := &TxResponse{ + Height: resp.Height, + TxHash: txHash, + Code: resp.ExecutionCode, + } + if resp.ExecutionCode != 0 { + return txResponse, fmt.Errorf("tx was included but failed with code %d: %s", resp.ExecutionCode, resp.Status) + } + return txResponse, nil + case "EVICTED": + return &TxResponse{}, fmt.Errorf("tx: %s was evicted from the mempool", txHash) + default: + return &TxResponse{}, fmt.Errorf("unknown tx: %s", txHash) + } } } } diff --git a/pkg/user/tx_client_test.go b/pkg/user/tx_client_test.go index 0c4821b2c4..1dcfc16822 100644 --- a/pkg/user/tx_client_test.go +++ b/pkg/user/tx_client_test.go @@ -6,6 +6,7 @@ import ( "time" sdk "github.com/cosmos/cosmos-sdk/types" + sdktx "github.com/cosmos/cosmos-sdk/types/tx" bank "github.com/cosmos/cosmos-sdk/x/bank/types" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -30,9 +31,10 @@ func TestTxClientTestSuite(t *testing.T) { type TxClientTestSuite struct { suite.Suite - ctx testnode.Context - encCfg encoding.Config - txClient *user.TxClient + ctx testnode.Context + encCfg encoding.Config + txClient *user.TxClient + serviceClient sdktx.ServiceClient } func (suite *TxClientTestSuite) SetupSuite() { @@ -45,6 +47,9 @@ func (suite *TxClientTestSuite) SetupSuite() { suite.Require().NoError(err) suite.txClient, err = user.SetupTxClient(suite.ctx.GoContext(), suite.ctx.Keyring, suite.ctx.GRPCClient, suite.encCfg, user.WithGasMultiplier(1.2)) suite.Require().NoError(err) + // FIXME: Temporary way of querying the raw log. + // TxStatus will natively support this in the future. + suite.serviceClient = sdktx.NewServiceClient(suite.ctx.GRPCClient) } func (suite *TxClientTestSuite) TestSubmitPayForBlob() { @@ -57,8 +62,10 @@ func (suite *TxClientTestSuite) TestSubmitPayForBlob() { t.Run("submit blob without provided fee and gas limit", func(t *testing.T) { resp, err := suite.txClient.SubmitPayForBlob(subCtx, blobs) require.NoError(t, err) + getTxResp, err := suite.serviceClient.GetTx(subCtx, &sdktx.GetTxRequest{Hash: resp.TxHash}) + require.NoError(t, err) require.EqualValues(t, 0, resp.Code) - require.Greater(t, resp.GasWanted, int64(0)) + require.Greater(t, getTxResp.TxResponse.GasWanted, int64(0)) }) t.Run("submit blob with provided fee and gas limit", func(t *testing.T) { @@ -66,15 +73,19 @@ func (suite *TxClientTestSuite) TestSubmitPayForBlob() { gas := user.SetGasLimit(1e6) resp, err := suite.txClient.SubmitPayForBlob(subCtx, blobs, fee, gas) require.NoError(t, err) + getTxResp, err := suite.serviceClient.GetTx(subCtx, &sdktx.GetTxRequest{Hash: resp.TxHash}) + require.NoError(t, err) require.EqualValues(t, 0, resp.Code) - require.EqualValues(t, resp.GasWanted, 1e6) + require.EqualValues(t, getTxResp.TxResponse.GasWanted, 1e6) }) t.Run("submit blob with different account", func(t *testing.T) { resp, err := suite.txClient.SubmitPayForBlobWithAccount(subCtx, "c", blobs, user.SetFee(1e6), user.SetGasLimit(1e6)) require.NoError(t, err) + getTxResp, err := suite.serviceClient.GetTx(subCtx, &sdktx.GetTxRequest{Hash: resp.TxHash}) + require.NoError(t, err) require.EqualValues(t, 0, resp.Code) - require.EqualValues(t, resp.GasWanted, 1e6) + require.EqualValues(t, getTxResp.TxResponse.GasWanted, 1e6) }) t.Run("try submit a blob with an account that doesn't exist", func(t *testing.T) { @@ -96,14 +107,18 @@ func (suite *TxClientTestSuite) TestSubmitTx() { resp, err := suite.txClient.SubmitTx(suite.ctx.GoContext(), []sdk.Msg{msg}) require.NoError(t, err) require.Equal(t, abci.CodeTypeOK, resp.Code) - require.Greater(t, resp.GasWanted, int64(0)) + getTxResp, err := suite.serviceClient.GetTx(suite.ctx.GoContext(), &sdktx.GetTxRequest{Hash: resp.TxHash}) + require.NoError(t, err) + require.Greater(t, getTxResp.TxResponse.GasWanted, int64(0)) }) t.Run("submit tx with provided gas limit", func(t *testing.T) { resp, err := suite.txClient.SubmitTx(suite.ctx.GoContext(), []sdk.Msg{msg}, gasLimitOption) require.NoError(t, err) require.Equal(t, abci.CodeTypeOK, resp.Code) - require.EqualValues(t, gasLimit, resp.GasWanted) + getTxResp, err := suite.serviceClient.GetTx(suite.ctx.GoContext(), &sdktx.GetTxRequest{Hash: resp.TxHash}) + require.NoError(t, err) + require.EqualValues(t, int64(gasLimit), getTxResp.TxResponse.GasWanted) }) t.Run("submit tx with provided fee", func(t *testing.T) { @@ -116,7 +131,9 @@ func (suite *TxClientTestSuite) TestSubmitTx() { resp, err := suite.txClient.SubmitTx(suite.ctx.GoContext(), []sdk.Msg{msg}, feeOption, gasLimitOption) require.NoError(t, err) require.Equal(t, abci.CodeTypeOK, resp.Code) - require.EqualValues(t, gasLimit, resp.GasWanted) + getTxResp, err := suite.serviceClient.GetTx(suite.ctx.GoContext(), &sdktx.GetTxRequest{Hash: resp.TxHash}) + require.NoError(t, err) + require.EqualValues(t, int64(gasLimit), getTxResp.TxResponse.GasWanted) }) t.Run("submit tx with a different account", func(t *testing.T) { @@ -145,7 +162,12 @@ func (suite *TxClientTestSuite) TestConfirmTx() { t.Run("deadline exceeded when the context times out", func(t *testing.T) { ctx, cancel := context.WithTimeout(suite.ctx.GoContext(), time.Second) defer cancel() - _, err := suite.txClient.ConfirmTx(ctx, "E32BD15CAF57AF15D17B0D63CF4E63A9835DD1CEBB059C335C79586BC3013728") + + msg := bank.NewMsgSend(suite.txClient.DefaultAddress(), testnode.RandomAddress().(sdk.AccAddress), sdk.NewCoins(sdk.NewInt64Coin(app.BondDenom, 10))) + resp, err := suite.txClient.BroadcastTx(ctx, []sdk.Msg{msg}) + require.NoError(t, err) + + _, err = suite.txClient.ConfirmTx(ctx, resp.TxHash) require.Error(t, err) require.Contains(t, err.Error(), context.DeadlineExceeded.Error()) }) @@ -153,8 +175,8 @@ func (suite *TxClientTestSuite) TestConfirmTx() { t.Run("should error when tx is not found", func(t *testing.T) { ctx, cancel := context.WithTimeout(suite.ctx.GoContext(), 5*time.Second) defer cancel() - _, err := suite.txClient.ConfirmTx(ctx, "not found tx") - require.Error(t, err) + _, err := suite.txClient.ConfirmTx(ctx, "E32BD15CAF57AF15D17B0D63CF4E63A9835DD1CEBB059C335C79586BC3013728") + require.Contains(t, err.Error(), "unknown tx: E32BD15CAF57AF15D17B0D63CF4E63A9835DD1CEBB059C335C79586BC3013728") }) t.Run("should success when tx is found immediately", func(t *testing.T) { @@ -165,9 +187,9 @@ func (suite *TxClientTestSuite) TestConfirmTx() { require.NotNil(t, resp) ctx, cancel := context.WithTimeout(suite.ctx.GoContext(), 30*time.Second) defer cancel() - resp, err = suite.txClient.ConfirmTx(ctx, resp.TxHash) + confirmTxResp, err := suite.txClient.ConfirmTx(ctx, resp.TxHash) require.NoError(t, err) - require.Equal(t, abci.CodeTypeOK, resp.Code) + require.Equal(t, abci.CodeTypeOK, confirmTxResp.Code) }) t.Run("should error when tx is found with a non-zero error code", func(t *testing.T) { @@ -178,9 +200,9 @@ func (suite *TxClientTestSuite) TestConfirmTx() { resp, err := suite.txClient.BroadcastTx(suite.ctx.GoContext(), []sdk.Msg{msg}, fee, gas) require.NoError(t, err) require.NotNil(t, resp) - resp, err = suite.txClient.ConfirmTx(suite.ctx.GoContext(), resp.TxHash) + confirmTxResp, err := suite.txClient.ConfirmTx(suite.ctx.GoContext(), resp.TxHash) require.Error(t, err) - require.NotEqual(t, abci.CodeTypeOK, resp.Code) + require.NotEqual(t, abci.CodeTypeOK, confirmTxResp.Code) }) } @@ -221,8 +243,11 @@ func (suite *TxClientTestSuite) TestGasConsumption() { amountDeducted := balanceBefore - balanceAfter - utiaToSend require.Equal(t, int64(fee), amountDeducted) + res, err := suite.serviceClient.GetTx(suite.ctx.GoContext(), &sdktx.GetTxRequest{Hash: resp.TxHash}) + require.NoError(t, err) + // verify that the amount deducted does not depend on the actual gas used. - gasUsedBasedDeduction := resp.GasUsed * gasPrice + gasUsedBasedDeduction := res.TxResponse.GasUsed * gasPrice require.NotEqual(t, gasUsedBasedDeduction, amountDeducted) // The gas used based deduction should be less than the fee because the fee is 1 TIA. require.Less(t, gasUsedBasedDeduction, int64(fee)) diff --git a/test/txsim/account.go b/test/txsim/account.go index e01264c68d..9796f60b04 100644 --- a/test/txsim/account.go +++ b/test/txsim/account.go @@ -240,7 +240,7 @@ func (am *AccountManager) Submit(ctx context.Context, op Operation) error { } var ( - res *types.TxResponse + res *user.TxResponse err error ) if len(op.Blobs) > 0 { diff --git a/x/blobstream/integration_test.go b/x/blobstream/integration_test.go index 08a96ecd56..9aaa91709d 100644 --- a/x/blobstream/integration_test.go +++ b/x/blobstream/integration_test.go @@ -11,6 +11,7 @@ import ( "github.com/celestiaorg/celestia-app/v3/test/util/testnode" blobstreamtypes "github.com/celestiaorg/celestia-app/v3/x/blobstream/types" sdk "github.com/cosmos/cosmos-sdk/types" + sdktx "github.com/cosmos/cosmos-sdk/types/tx" staking "github.com/cosmos/cosmos-sdk/x/staking/types" gethcommon "github.com/ethereum/go-ethereum/common" "github.com/stretchr/testify/require" @@ -89,8 +90,13 @@ func (s *BlobstreamIntegrationSuite) TestBlobstream() { } else { require.Error(t, err) } + // FIXME: Temporary way of querying the raw log. + // TxStatus will natively support this in the future. + serviceClient := sdktx.NewServiceClient(s.cctx.GRPCClient) + getTxResp, err := serviceClient.GetTx(s.cctx.GoContext(), &sdktx.GetTxRequest{Hash: res.TxHash}) + require.NoError(t, err) require.NotNil(t, res) - require.Equal(t, tt.expectedTxCode, res.Code, res.RawLog) + require.Equal(t, tt.expectedTxCode, res.Code, getTxResp.TxResponse.RawLog) }) } } diff --git a/x/signal/legacy_test.go b/x/signal/legacy_test.go index 3c202cd50f..2516c66dc8 100644 --- a/x/signal/legacy_test.go +++ b/x/signal/legacy_test.go @@ -14,6 +14,7 @@ import ( "github.com/celestiaorg/celestia-app/v3/test/util/testnode" "github.com/cosmos/cosmos-sdk/crypto/keyring" sdk "github.com/cosmos/cosmos-sdk/types" + sdktx "github.com/cosmos/cosmos-sdk/types/tx" authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" v1 "github.com/cosmos/cosmos-sdk/x/gov/types/v1" v1beta1 "github.com/cosmos/cosmos-sdk/x/gov/types/v1beta1" @@ -54,6 +55,7 @@ type LegacyUpgradeTestSuite struct { mut sync.Mutex accountCounter int + serviceClient sdktx.ServiceClient } // SetupSuite inits a standard chain, with the only exception being a @@ -91,6 +93,9 @@ func (s *LegacyUpgradeTestSuite) SetupSuite() { // Set the gov module address s.govModuleAddress = acc.GetAddress().String() + // FIXME: Temporary way of querying the raw log. + // TxStatus will natively support this in the future. + s.serviceClient = sdktx.NewServiceClient(s.cctx.GRPCClient) } func (s *LegacyUpgradeTestSuite) unusedAccount() string { @@ -127,7 +132,7 @@ func (s *LegacyUpgradeTestSuite) TestLegacyGovUpgradeFailure() { res, err := signer.SubmitTx(subCtx, []sdk.Msg{msg}, blobfactory.DefaultTxOpts()...) require.Error(t, err) // As the type is not registered, the message will fail with unable to resolve type URL - require.EqualValues(t, 2, res.Code, res.RawLog) + require.EqualValues(t, 2, res.Code) } // TestNewGovUpgradeFailure verifies that a transaction with a @@ -156,7 +161,7 @@ func (s *LegacyUpgradeTestSuite) TestNewGovUpgradeFailure() { res, err := signer.SubmitTx(subCtx, []sdk.Msg{msg}, blobfactory.DefaultTxOpts()...) require.Error(t, err) // As the type is not registered, the message will fail with unable to resolve type URL - require.EqualValues(t, 2, res.Code, res.RawLog) + require.EqualValues(t, 2, res.Code) } func (s *LegacyUpgradeTestSuite) TestIBCUpgradeFailure() { @@ -178,14 +183,16 @@ func (s *LegacyUpgradeTestSuite) TestIBCUpgradeFailure() { require.NoError(t, err) // submit the transaction and wait a block for it to be included - signer, err := testnode.NewTxClientFromContext(s.cctx) + txClient, err := testnode.NewTxClientFromContext(s.cctx) require.NoError(t, err) subCtx, cancel := context.WithTimeout(s.cctx.GoContext(), time.Minute) defer cancel() - res, err := signer.SubmitTx(subCtx, []sdk.Msg{msg}, blobfactory.DefaultTxOpts()...) + res, err := txClient.SubmitTx(subCtx, []sdk.Msg{msg}, blobfactory.DefaultTxOpts()...) require.Error(t, err) - require.EqualValues(t, 9, res.Code, res.RawLog) // we're only submitting the tx, so we expect everything to work - assert.Contains(t, res.RawLog, "ibc upgrade proposal not supported") + getTxResp, err := s.serviceClient.GetTx(subCtx, &sdktx.GetTxRequest{Hash: res.TxHash}) + require.NoError(t, err) + require.EqualValues(t, 9, res.Code, getTxResp.TxResponse.RawLog) // we're only submitting the tx, so we expect everything to work + assert.Contains(t, getTxResp.TxResponse.RawLog, "ibc upgrade proposal not supported") } func getAddress(account string, kr keyring.Keyring) sdk.AccAddress { From 02b604de49ecd9873a83d2011641bf30723e1a8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nina=20/=20=E1=83=9C=E1=83=98=E1=83=9C=E1=83=90?= Date: Mon, 19 Aug 2024 13:57:44 +0200 Subject: [PATCH 02/85] feat: add error log to txstatus (#3788) ## Overview - [x] Bump celestia-core verison including refactored TxStatus work - [x] Extend the rpc response to return the error log - [x] Update TxClient with extended TxStatus struct --------- Co-authored-by: Callum Waters --- app/grpc/tx/server.go | 1 + app/grpc/tx/tx.pb.go | 99 +++++++++++++++++------ app/test/big_blob_test.go | 3 +- app/test/prepare_proposal_context_test.go | 2 - app/test/square_size_test.go | 2 - app/test/std_sdk_test.go | 2 - go.mod | 2 +- go.sum | 4 +- pkg/user/tx_client.go | 37 ++++----- pkg/user/tx_client_test.go | 13 ++- proto/celestia/core/v1/tx/tx.proto | 4 +- test/interchain/go.mod | 4 +- test/interchain/go.sum | 8 +- x/blobstream/integration_test.go | 2 - x/signal/legacy_test.go | 11 +-- 15 files changed, 122 insertions(+), 72 deletions(-) diff --git a/app/grpc/tx/server.go b/app/grpc/tx/server.go index 93b3c9da36..417b7c088b 100644 --- a/app/grpc/tx/server.go +++ b/app/grpc/tx/server.go @@ -76,6 +76,7 @@ func (s *txServer) TxStatus(ctx context.Context, req *TxStatusRequest) (*TxStatu Height: resTx.Height, Index: resTx.Index, ExecutionCode: resTx.ExecutionCode, + Error: resTx.Error, Status: resTx.Status, }, nil } diff --git a/app/grpc/tx/tx.pb.go b/app/grpc/tx/tx.pb.go index c962ec2dd5..de589600d2 100644 --- a/app/grpc/tx/tx.pb.go +++ b/app/grpc/tx/tx.pb.go @@ -82,8 +82,10 @@ type TxStatusResponse struct { // and returns whether it was successful or errored. A non zero // execution code indicated an error. ExecutionCode uint32 `protobuf:"varint,3,opt,name=execution_code,json=executionCode,proto3" json:"execution_code,omitempty"` + // error log for failed transactions. + Error string `protobuf:"bytes,4,opt,name=error,proto3" json:"error,omitempty"` // status is the status of the transaction. - Status string `protobuf:"bytes,4,opt,name=status,proto3" json:"status,omitempty"` + Status string `protobuf:"bytes,5,opt,name=status,proto3" json:"status,omitempty"` } func (m *TxStatusResponse) Reset() { *m = TxStatusResponse{} } @@ -140,6 +142,13 @@ func (m *TxStatusResponse) GetExecutionCode() uint32 { return 0 } +func (m *TxStatusResponse) GetError() string { + if m != nil { + return m.Error + } + return "" +} + func (m *TxStatusResponse) GetStatus() string { if m != nil { return m.Status @@ -155,28 +164,29 @@ func init() { func init() { proto.RegisterFile("celestia/core/v1/tx/tx.proto", fileDescriptor_7d8b070565b0dcb6) } var fileDescriptor_7d8b070565b0dcb6 = []byte{ - // 322 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x51, 0xcd, 0x4a, 0x03, 0x31, - 0x18, 0x6c, 0xfa, 0x87, 0x06, 0xaa, 0x92, 0x8a, 0x2c, 0xa5, 0x84, 0x52, 0x5a, 0xe9, 0xc5, 0x84, - 0xea, 0x1b, 0xe8, 0xa9, 0xd7, 0xb5, 0x27, 0x2f, 0x65, 0xbb, 0x1b, 0xb6, 0x81, 0xba, 0x89, 0x9b, - 0x6f, 0x4b, 0x40, 0x8a, 0xa0, 0x2f, 0x20, 0xf8, 0x52, 0x1e, 0x0b, 0x5e, 0x3c, 0x4a, 0xeb, 0x83, - 0x48, 0xd3, 0x76, 0x05, 0x29, 0x78, 0x08, 0x64, 0x32, 0x93, 0x6f, 0x32, 0x13, 0xdc, 0x0c, 0xc5, - 0x54, 0x18, 0x90, 0x01, 0x0f, 0x55, 0x2a, 0xf8, 0xac, 0xcf, 0xc1, 0x72, 0xb0, 0x4c, 0xa7, 0x0a, - 0x14, 0xa9, 0xef, 0x58, 0xb6, 0x66, 0xd9, 0xac, 0xcf, 0xc0, 0x36, 0x9a, 0xb1, 0x52, 0xf1, 0x54, - 0xf0, 0x40, 0x4b, 0x1e, 0x24, 0x89, 0x82, 0x00, 0xa4, 0x4a, 0xcc, 0xe6, 0x4a, 0xfb, 0x1c, 0x1f, - 0x0f, 0xed, 0x2d, 0x04, 0x90, 0x19, 0x5f, 0x3c, 0x64, 0xc2, 0x00, 0xa9, 0xe3, 0x0a, 0xd8, 0x91, - 0x8c, 0x3c, 0xd4, 0x42, 0xbd, 0x43, 0xbf, 0x0c, 0x76, 0x10, 0xb5, 0x9f, 0xf0, 0xc9, 0xaf, 0xce, - 0x68, 0x95, 0x18, 0x41, 0xce, 0x70, 0x75, 0x22, 0x64, 0x3c, 0x01, 0xa7, 0x2c, 0xf9, 0x5b, 0x44, - 0x4e, 0x71, 0x45, 0x26, 0x91, 0xb0, 0x5e, 0xb1, 0x85, 0x7a, 0x35, 0x7f, 0x03, 0x48, 0x17, 0x1f, - 0x09, 0x2b, 0xc2, 0x6c, 0xed, 0x3e, 0x0a, 0x55, 0x24, 0xbc, 0x92, 0xa3, 0x6b, 0xf9, 0xe9, 0x8d, - 0x8a, 0xdc, 0x50, 0xe3, 0x6c, 0xbc, 0xb2, 0xb3, 0xdf, 0xa2, 0xcb, 0x17, 0x84, 0x8b, 0x43, 0x4b, - 0xe6, 0xf8, 0x60, 0xf7, 0x0e, 0xd2, 0x61, 0x7b, 0xf2, 0xb2, 0x3f, 0x71, 0x1a, 0xdd, 0x7f, 0x54, - 0x9b, 0x30, 0xed, 0xce, 0xf3, 0xc7, 0xf7, 0x5b, 0x91, 0x92, 0x26, 0xdf, 0x57, 0xf1, 0xa3, 0x6b, - 0x64, 0x7e, 0x3d, 0x78, 0x5f, 0x52, 0xb4, 0x58, 0x52, 0xf4, 0xb5, 0xa4, 0xe8, 0x75, 0x45, 0x0b, - 0x8b, 0x15, 0x2d, 0x7c, 0xae, 0x68, 0xe1, 0x8e, 0xc7, 0x12, 0x26, 0xd9, 0x98, 0x85, 0xea, 0x3e, - 0x9f, 0xa0, 0xd2, 0x38, 0xdf, 0x5f, 0x04, 0x5a, 0xf3, 0xf5, 0x8a, 0x53, 0x1d, 0x72, 0xb0, 0xe3, - 0xaa, 0xfb, 0x80, 0xab, 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xdb, 0x2a, 0x51, 0xff, 0xd3, 0x01, - 0x00, 0x00, + // 337 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x51, 0xcd, 0x4a, 0x33, 0x31, + 0x14, 0x6d, 0xfa, 0xc7, 0xf7, 0x05, 0xfa, 0x7d, 0x92, 0x8a, 0x0c, 0xa5, 0x0c, 0xa5, 0xb4, 0xd2, + 0x8d, 0x13, 0xaa, 0x6f, 0xa0, 0xab, 0x6e, 0xc7, 0xae, 0xdc, 0x94, 0xe9, 0xcc, 0x65, 0x1a, 0xa8, + 0x93, 0x31, 0xb9, 0x53, 0x02, 0xd2, 0x8d, 0xbe, 0x80, 0x20, 0xbe, 0x93, 0xcb, 0x82, 0x1b, 0x97, + 0xd2, 0xfa, 0x20, 0x32, 0x99, 0xb6, 0x82, 0x14, 0x5c, 0x04, 0x72, 0xee, 0x39, 0x37, 0x27, 0xf7, + 0x5c, 0xda, 0x0e, 0x61, 0x0e, 0x1a, 0x45, 0xc0, 0x43, 0xa9, 0x80, 0x2f, 0x86, 0x1c, 0x0d, 0x47, + 0xe3, 0xa5, 0x4a, 0xa2, 0x64, 0xcd, 0x1d, 0xeb, 0xe5, 0xac, 0xb7, 0x18, 0x7a, 0x68, 0x5a, 0xed, + 0x58, 0xca, 0x78, 0x0e, 0x3c, 0x48, 0x05, 0x0f, 0x92, 0x44, 0x62, 0x80, 0x42, 0x26, 0xba, 0x68, + 0xe9, 0x9e, 0xd2, 0xff, 0x63, 0x73, 0x8d, 0x01, 0x66, 0xda, 0x87, 0xbb, 0x0c, 0x34, 0xb2, 0x26, + 0xad, 0xa1, 0x99, 0x88, 0xc8, 0x21, 0x1d, 0x32, 0xf8, 0xeb, 0x57, 0xd1, 0x8c, 0xa2, 0xee, 0x0b, + 0xa1, 0x47, 0xdf, 0x42, 0x9d, 0xca, 0x44, 0x03, 0x3b, 0xa1, 0xf5, 0x19, 0x88, 0x78, 0x86, 0x56, + 0x5a, 0xf1, 0xb7, 0x88, 0x1d, 0xd3, 0x9a, 0x48, 0x22, 0x30, 0x4e, 0xb9, 0x43, 0x06, 0x0d, 0xbf, + 0x00, 0xac, 0x4f, 0xff, 0x81, 0x81, 0x30, 0xcb, 0xed, 0x27, 0xa1, 0x8c, 0xc0, 0xa9, 0x58, 0xba, + 0xb1, 0xaf, 0x5e, 0xc9, 0x08, 0xf2, 0x66, 0x50, 0x4a, 0x2a, 0xa7, 0x6a, 0xed, 0x0b, 0x90, 0x5b, + 0x69, 0x6b, 0xee, 0xd4, 0x6c, 0x79, 0x8b, 0xce, 0x1f, 0x09, 0x2d, 0x8f, 0x0d, 0x5b, 0xd2, 0x3f, + 0xbb, 0xdf, 0xb1, 0x9e, 0x77, 0x20, 0x06, 0xef, 0xc7, 0x94, 0xad, 0xfe, 0x2f, 0xaa, 0x62, 0xc4, + 0x6e, 0xef, 0xe1, 0xed, 0xf3, 0xb9, 0xec, 0xb2, 0x36, 0x3f, 0x94, 0xfc, 0xbd, 0x0d, 0x6a, 0x79, + 0x39, 0x7a, 0x5d, 0xbb, 0x64, 0xb5, 0x76, 0xc9, 0xc7, 0xda, 0x25, 0x4f, 0x1b, 0xb7, 0xb4, 0xda, + 0xb8, 0xa5, 0xf7, 0x8d, 0x5b, 0xba, 0xe1, 0xb1, 0xc0, 0x59, 0x36, 0xf5, 0x42, 0x79, 0xbb, 0x7f, + 0x41, 0xaa, 0x78, 0x7f, 0x3f, 0x0b, 0xd2, 0x94, 0xe7, 0x27, 0x56, 0x69, 0xc8, 0xd1, 0x4c, 0xeb, + 0x76, 0x2f, 0x17, 0x5f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x5a, 0x2f, 0x1b, 0xd1, 0xea, 0x01, 0x00, + 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -324,6 +334,13 @@ func (m *TxStatusResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { copy(dAtA[i:], m.Status) i = encodeVarintTx(dAtA, i, uint64(len(m.Status))) i-- + dAtA[i] = 0x2a + } + if len(m.Error) > 0 { + i -= len(m.Error) + copy(dAtA[i:], m.Error) + i = encodeVarintTx(dAtA, i, uint64(len(m.Error))) + i-- dAtA[i] = 0x22 } if m.ExecutionCode != 0 { @@ -383,6 +400,10 @@ func (m *TxStatusResponse) Size() (n int) { if m.ExecutionCode != 0 { n += 1 + sovTx(uint64(m.ExecutionCode)) } + l = len(m.Error) + if l > 0 { + n += 1 + l + sovTx(uint64(l)) + } l = len(m.Status) if l > 0 { n += 1 + l + sovTx(uint64(l)) @@ -565,6 +586,38 @@ func (m *TxStatusResponse) Unmarshal(dAtA []byte) error { } } case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Error", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTx + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthTx + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthTx + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Error = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 5: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Status", wireType) } diff --git a/app/test/big_blob_test.go b/app/test/big_blob_test.go index ad219c2e3d..ecb257031f 100644 --- a/app/test/big_blob_test.go +++ b/app/test/big_blob_test.go @@ -83,8 +83,7 @@ func (s *BigBlobSuite) TestErrBlobsTooLarge() { res, err := txClient.SubmitPayForBlob(subCtx, []*share.Blob{tc.blob}, user.SetGasLimitAndGasPrice(1e9, appconsts.DefaultMinGasPrice)) require.Error(t, err) require.NotNil(t, res) - // FIXME: assert RawLog once TxStatus supports it. - require.Equal(t, tc.want, res.Code) + require.Equal(t, tc.want, res.Code, err.Error()) }) } } diff --git a/app/test/prepare_proposal_context_test.go b/app/test/prepare_proposal_context_test.go index 7eb14bf9a8..d01fad5a9d 100644 --- a/app/test/prepare_proposal_context_test.go +++ b/app/test/prepare_proposal_context_test.go @@ -81,8 +81,6 @@ func TestTimeInPrepareProposalContext(t *testing.T) { msgs, _ := tt.msgFunc() res, err := txClient.SubmitTx(cctx.GoContext(), msgs, user.SetGasLimit(1000000), user.SetFee(2000)) require.NoError(t, err) - // FIXME: Temporary way of querying the raw log. - // TxStatus will natively support this in the future. serviceClient := sdktx.NewServiceClient(cctx.GRPCClient) getTxResp, err := serviceClient.GetTx(cctx.GoContext(), &sdktx.GetTxRequest{Hash: res.TxHash}) require.NoError(t, err) diff --git a/app/test/square_size_test.go b/app/test/square_size_test.go index 653f4b4ae4..d030b89093 100644 --- a/app/test/square_size_test.go +++ b/app/test/square_size_test.go @@ -174,8 +174,6 @@ func (s *SquareSizeIntegrationTest) setBlockSizeParams(t *testing.T, squareSize, res, err := txClient.SubmitTx(s.cctx.GoContext(), []sdk.Msg{msg}, blobfactory.DefaultTxOpts()...) require.NoError(t, err) - // FIXME: Temporary way of querying the raw log. - // TxStatus will natively support this in the future. serviceClient := sdktx.NewServiceClient(s.cctx.GRPCClient) getTxResp, err := serviceClient.GetTx(s.cctx.GoContext(), &sdktx.GetTxRequest{Hash: res.TxHash}) require.NoError(t, err) diff --git a/app/test/std_sdk_test.go b/app/test/std_sdk_test.go index ca4b401768..8d3e02aa3d 100644 --- a/app/test/std_sdk_test.go +++ b/app/test/std_sdk_test.go @@ -326,8 +326,6 @@ func (s *StandardSDKIntegrationTestSuite) TestStandardSDK() { } else { require.NoError(t, err) } - // FIXME: Temporary way of querying the raw log. - // TxStatus will natively support this in the future. serviceClient := sdktx.NewServiceClient(s.cctx.GRPCClient) getTxResp, err := serviceClient.GetTx(s.cctx.GoContext(), &sdktx.GetTxRequest{Hash: res.TxHash}) require.NoError(t, err) diff --git a/go.mod b/go.mod index 5f1fab11a6..8375b680f5 100644 --- a/go.mod +++ b/go.mod @@ -252,5 +252,5 @@ replace ( github.com/cosmos/ledger-cosmos-go => github.com/cosmos/ledger-cosmos-go v0.12.4 github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 - github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.39.0-tm-v0.34.29 + github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.40.0-tm-v0.34.29 ) diff --git a/go.sum b/go.sum index 034dabe1c1..d5be2573b4 100644 --- a/go.sum +++ b/go.sum @@ -318,8 +318,8 @@ github.com/celestiaorg/bittwister v0.0.0-20231213180407-65cdbaf5b8c7 h1:nxplQi8w github.com/celestiaorg/bittwister v0.0.0-20231213180407-65cdbaf5b8c7/go.mod h1:1EF5MfOxVf0WC51Gb7pJ6bcZxnXKNAf9pqWtjgPBAYc= github.com/celestiaorg/blobstream-contracts/v3 v3.1.0 h1:h1Y4V3EMQ2mFmNtWt2sIhZIuyASInj1a9ExI8xOsTOw= github.com/celestiaorg/blobstream-contracts/v3 v3.1.0/go.mod h1:x4DKyfKOSv1ZJM9NwV+Pw01kH2CD7N5zTFclXIVJ6GQ= -github.com/celestiaorg/celestia-core v1.39.0-tm-v0.34.29 h1:9Co/2peu4+9S6KMVNPFS0NTI/RYIRirNpM4N7dmi9ak= -github.com/celestiaorg/celestia-core v1.39.0-tm-v0.34.29/go.mod h1:5jJ5magtH7gQOwSYfS/m5fliIS7irKunLV7kLNaD8o0= +github.com/celestiaorg/celestia-core v1.40.0-tm-v0.34.29 h1:J79TAjizxwIvm7/k+WI3PPH1aFj4AjOSjajoq5UzAwI= +github.com/celestiaorg/celestia-core v1.40.0-tm-v0.34.29/go.mod h1:5jJ5magtH7gQOwSYfS/m5fliIS7irKunLV7kLNaD8o0= github.com/celestiaorg/cosmos-sdk v1.24.1-sdk-v0.46.16 h1:SeQ7Y/CyOcUMKo7mQiexaj/pZ/xIgyuZFIwYZwpSkWE= github.com/celestiaorg/cosmos-sdk v1.24.1-sdk-v0.46.16/go.mod h1:Bpl1LSWiDpQumgOhhMTZBMopqa0j7fRasIhvTZB44P0= github.com/celestiaorg/go-square/v2 v2.0.0-rc2 h1:4D+ASgZGYVCsffc2uhPagACrvNiLZu9/CqNYvnlHCgg= diff --git a/pkg/user/tx_client.go b/pkg/user/tx_client.go index 75863f1293..b2c5b750a3 100644 --- a/pkg/user/tx_client.go +++ b/pkg/user/tx_client.go @@ -31,6 +31,7 @@ import ( "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" "github.com/celestiaorg/celestia-app/v3/x/blob/types" "github.com/celestiaorg/celestia-app/v3/x/minfee" + "github.com/tendermint/tendermint/rpc/core" ) const ( @@ -211,10 +212,8 @@ func SetupTxClient( // TxOptions may be provided to set the fee and gas limit. func (client *TxClient) SubmitPayForBlob(ctx context.Context, blobs []*share.Blob, opts ...TxOption) (*TxResponse, error) { resp, err := client.BroadcastPayForBlob(ctx, blobs, opts...) - if err != nil && resp != nil { - return &TxResponse{Code: resp.Code, TxHash: resp.TxHash}, fmt.Errorf("failed to broadcast pay for blob: %v", err) - } else if err != nil { - return &TxResponse{}, fmt.Errorf("failed to broadcast pay for blob: %v", err) + if err != nil { + return parseTxResponse(resp, fmt.Errorf("failed to broadcast pay for blob: %v", err)) } return client.ConfirmTx(ctx, resp.TxHash) @@ -224,10 +223,8 @@ func (client *TxClient) SubmitPayForBlob(ctx context.Context, blobs []*share.Blo // TxOptions may be provided to set the fee and gas limit. func (client *TxClient) SubmitPayForBlobWithAccount(ctx context.Context, account string, blobs []*share.Blob, opts ...TxOption) (*TxResponse, error) { resp, err := client.BroadcastPayForBlobWithAccount(ctx, account, blobs, opts...) - if err != nil && resp != nil { - return &TxResponse{Code: resp.Code, TxHash: resp.TxHash}, fmt.Errorf("failed to broadcast pay for blob with account: %v", err) - } else if err != nil { - return &TxResponse{}, fmt.Errorf("failed to broadcast pay for blob with account: %v", err) + if err != nil { + return parseTxResponse(resp, fmt.Errorf("failed to broadcast pay for blob with account: %v", err)) } return client.ConfirmTx(ctx, resp.TxHash) @@ -270,10 +267,8 @@ func (client *TxClient) BroadcastPayForBlobWithAccount(ctx context.Context, acco // may be provided to set the fee and gas limit. func (client *TxClient) SubmitTx(ctx context.Context, msgs []sdktypes.Msg, opts ...TxOption) (*TxResponse, error) { resp, err := client.BroadcastTx(ctx, msgs, opts...) - if err != nil && resp != nil { - return &TxResponse{Code: resp.Code, TxHash: resp.TxHash}, fmt.Errorf("failed to broadcast tx: %v", err) - } else if err != nil { - return &TxResponse{}, fmt.Errorf("failed to broadcast tx: %v", err) + if err != nil { + return parseTxResponse(resp, fmt.Errorf("failed to broadcast tx: %v", err)) } return client.ConfirmTx(ctx, resp.TxHash) @@ -445,8 +440,7 @@ func (client *TxClient) ConfirmTx(ctx context.Context, txHash string) (*TxRespon if err == nil && resp != nil { switch resp.Status { - // FIXME: replace hardcoded status with constants - case "PENDING": + case core.TxStatusPending: // Continue polling if the transaction is still pending select { case <-ctx.Done(): @@ -454,18 +448,18 @@ func (client *TxClient) ConfirmTx(ctx context.Context, txHash string) (*TxRespon case <-pollTicker.C: continue } - case "COMMITTED": + case core.TxStatusCommitted: txResponse := &TxResponse{ Height: resp.Height, TxHash: txHash, Code: resp.ExecutionCode, } if resp.ExecutionCode != 0 { - return txResponse, fmt.Errorf("tx was included but failed with code %d: %s", resp.ExecutionCode, resp.Status) + return txResponse, fmt.Errorf("tx was committed but failed with code %d: %s", resp.ExecutionCode, resp.Error) } return txResponse, nil - case "EVICTED": - return &TxResponse{}, fmt.Errorf("tx: %s was evicted from the mempool", txHash) + case core.TxStatusEvicted: + return &TxResponse{TxHash: txHash}, fmt.Errorf("tx: %s was evicted from the mempool", txHash) default: return &TxResponse{}, fmt.Errorf("unknown tx: %s", txHash) } @@ -575,6 +569,13 @@ func (client *TxClient) getAccountNameFromMsgs(msgs []sdktypes.Msg) (string, err return record.Name, nil } +func parseTxResponse(resp *sdktypes.TxResponse, err error) (*TxResponse, error) { + if resp != nil { + return &TxResponse{Code: resp.Code, TxHash: resp.TxHash}, err + } + return &TxResponse{}, err +} + // Signer exposes the tx clients underlying signer func (client *TxClient) Signer() *Signer { return client.signer diff --git a/pkg/user/tx_client_test.go b/pkg/user/tx_client_test.go index 1dcfc16822..c416ba2d8e 100644 --- a/pkg/user/tx_client_test.go +++ b/pkg/user/tx_client_test.go @@ -19,6 +19,7 @@ import ( "github.com/celestiaorg/celestia-app/v3/pkg/user" "github.com/celestiaorg/celestia-app/v3/test/util/blobfactory" "github.com/celestiaorg/celestia-app/v3/test/util/testnode" + "github.com/cosmos/cosmos-sdk/x/authz" ) func TestTxClientTestSuite(t *testing.T) { @@ -47,8 +48,6 @@ func (suite *TxClientTestSuite) SetupSuite() { suite.Require().NoError(err) suite.txClient, err = user.SetupTxClient(suite.ctx.GoContext(), suite.ctx.Keyring, suite.ctx.GRPCClient, suite.encCfg, user.WithGasMultiplier(1.2)) suite.Require().NoError(err) - // FIXME: Temporary way of querying the raw log. - // TxStatus will natively support this in the future. suite.serviceClient = sdktx.NewServiceClient(suite.ctx.GRPCClient) } @@ -179,6 +178,16 @@ func (suite *TxClientTestSuite) TestConfirmTx() { require.Contains(t, err.Error(), "unknown tx: E32BD15CAF57AF15D17B0D63CF4E63A9835DD1CEBB059C335C79586BC3013728") }) + t.Run("should return error log when execution fails", func(t *testing.T) { + innerMsg := bank.NewMsgSend(testnode.RandomAddress().(sdk.AccAddress), testnode.RandomAddress().(sdk.AccAddress), sdk.NewCoins(sdk.NewInt64Coin(app.BondDenom, 10))) + msg := authz.NewMsgExec(suite.txClient.DefaultAddress(), []sdk.Msg{innerMsg}) + resp, err := suite.txClient.BroadcastTx(suite.ctx.GoContext(), []sdk.Msg{&msg}, fee, gas) + require.NoError(t, err) + _, err = suite.txClient.ConfirmTx(suite.ctx.GoContext(), resp.TxHash) + require.Error(t, err) + require.Contains(t, err.Error(), "authorization not found") + }) + t.Run("should success when tx is found immediately", func(t *testing.T) { addr := suite.txClient.DefaultAddress() msg := bank.NewMsgSend(addr, testnode.RandomAddress().(sdk.AccAddress), sdk.NewCoins(sdk.NewInt64Coin(app.BondDenom, 10))) diff --git a/proto/celestia/core/v1/tx/tx.proto b/proto/celestia/core/v1/tx/tx.proto index f4a8301606..0da698ad84 100644 --- a/proto/celestia/core/v1/tx/tx.proto +++ b/proto/celestia/core/v1/tx/tx.proto @@ -33,6 +33,8 @@ message TxStatusResponse { // and returns whether it was successful or errored. A non zero // execution code indicated an error. uint32 execution_code = 3; + // error log for failed transactions. + string error = 4; // status is the status of the transaction. - string status = 4; + string status = 5; } \ No newline at end of file diff --git a/test/interchain/go.mod b/test/interchain/go.mod index 8bdd1bf22d..bbcd6956b1 100644 --- a/test/interchain/go.mod +++ b/test/interchain/go.mod @@ -33,7 +33,7 @@ require ( github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect github.com/bgentry/speakeasy v0.1.0 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect - github.com/celestiaorg/nmt v0.21.0 // indirect + github.com/celestiaorg/nmt v0.22.0 // indirect github.com/centrifuge/go-substrate-rpc-client/v4 v4.0.10 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect @@ -229,5 +229,5 @@ replace ( github.com/docker/docker => github.com/docker/docker v24.0.1+incompatible github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 - github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.38.0-tm-v0.34.29 + github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.40.0-tm-v0.34.29 ) diff --git a/test/interchain/go.sum b/test/interchain/go.sum index 3f306fe454..c24352cb35 100644 --- a/test/interchain/go.sum +++ b/test/interchain/go.sum @@ -249,12 +249,12 @@ github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOF github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce h1:YtWJF7RHm2pYCvA5t0RPmAaLUhREsKuKd+SLhxFbFeQ= github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o= -github.com/celestiaorg/celestia-core v1.38.0-tm-v0.34.29 h1:HwbA4OegRvXX0aNchBA7Cmu+oIxnH7xRcOhISuDP0ak= -github.com/celestiaorg/celestia-core v1.38.0-tm-v0.34.29/go.mod h1:MyElURdWAOJkOp84WZnfEUJ+OLvTwOOHG2lbK9E8XRI= +github.com/celestiaorg/celestia-core v1.40.0-tm-v0.34.29 h1:J79TAjizxwIvm7/k+WI3PPH1aFj4AjOSjajoq5UzAwI= +github.com/celestiaorg/celestia-core v1.40.0-tm-v0.34.29/go.mod h1:5jJ5magtH7gQOwSYfS/m5fliIS7irKunLV7kLNaD8o0= github.com/celestiaorg/cosmos-sdk v1.24.0-sdk-v0.46.16 h1:AlBZS4WykzrwfcNbKD+yQQM1RTMz7lYDC1NS7ClAidM= github.com/celestiaorg/cosmos-sdk v1.24.0-sdk-v0.46.16/go.mod h1:Bpl1LSWiDpQumgOhhMTZBMopqa0j7fRasIhvTZB44P0= -github.com/celestiaorg/nmt v0.21.0 h1:81MBqxNn3orByoiCtdNVjwi5WsLgMkzHwP02ZMhTBHM= -github.com/celestiaorg/nmt v0.21.0/go.mod h1:ia/EpCk0enD5yO5frcxoNoFToz2Ghtk2i+blmCRjIY8= +github.com/celestiaorg/nmt v0.22.0 h1:AGtfmBiVgreR1KkIV5R7XFNeMp/H4IUDLlBbLjZZ3zk= +github.com/celestiaorg/nmt v0.22.0/go.mod h1:ia/EpCk0enD5yO5frcxoNoFToz2Ghtk2i+blmCRjIY8= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= diff --git a/x/blobstream/integration_test.go b/x/blobstream/integration_test.go index 9aaa91709d..c4a7b6d9a6 100644 --- a/x/blobstream/integration_test.go +++ b/x/blobstream/integration_test.go @@ -90,8 +90,6 @@ func (s *BlobstreamIntegrationSuite) TestBlobstream() { } else { require.Error(t, err) } - // FIXME: Temporary way of querying the raw log. - // TxStatus will natively support this in the future. serviceClient := sdktx.NewServiceClient(s.cctx.GRPCClient) getTxResp, err := serviceClient.GetTx(s.cctx.GoContext(), &sdktx.GetTxRequest{Hash: res.TxHash}) require.NoError(t, err) diff --git a/x/signal/legacy_test.go b/x/signal/legacy_test.go index 2516c66dc8..aa53038cec 100644 --- a/x/signal/legacy_test.go +++ b/x/signal/legacy_test.go @@ -14,7 +14,6 @@ import ( "github.com/celestiaorg/celestia-app/v3/test/util/testnode" "github.com/cosmos/cosmos-sdk/crypto/keyring" sdk "github.com/cosmos/cosmos-sdk/types" - sdktx "github.com/cosmos/cosmos-sdk/types/tx" authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" v1 "github.com/cosmos/cosmos-sdk/x/gov/types/v1" v1beta1 "github.com/cosmos/cosmos-sdk/x/gov/types/v1beta1" @@ -55,7 +54,6 @@ type LegacyUpgradeTestSuite struct { mut sync.Mutex accountCounter int - serviceClient sdktx.ServiceClient } // SetupSuite inits a standard chain, with the only exception being a @@ -93,9 +91,6 @@ func (s *LegacyUpgradeTestSuite) SetupSuite() { // Set the gov module address s.govModuleAddress = acc.GetAddress().String() - // FIXME: Temporary way of querying the raw log. - // TxStatus will natively support this in the future. - s.serviceClient = sdktx.NewServiceClient(s.cctx.GRPCClient) } func (s *LegacyUpgradeTestSuite) unusedAccount() string { @@ -189,10 +184,8 @@ func (s *LegacyUpgradeTestSuite) TestIBCUpgradeFailure() { defer cancel() res, err := txClient.SubmitTx(subCtx, []sdk.Msg{msg}, blobfactory.DefaultTxOpts()...) require.Error(t, err) - getTxResp, err := s.serviceClient.GetTx(subCtx, &sdktx.GetTxRequest{Hash: res.TxHash}) - require.NoError(t, err) - require.EqualValues(t, 9, res.Code, getTxResp.TxResponse.RawLog) // we're only submitting the tx, so we expect everything to work - assert.Contains(t, getTxResp.TxResponse.RawLog, "ibc upgrade proposal not supported") + require.EqualValues(t, 9, res.Code) // we're only submitting the tx, so we expect everything to work + assert.Contains(t, err.Error(), "ibc upgrade proposal not supported") } func getAddress(account string, kr keyring.Keyring) sdk.AccAddress { From 02f0825baeaf2a4aa1efd4c0ccc0a9d48350b3be Mon Sep 17 00:00:00 2001 From: Rootul P Date: Tue, 20 Aug 2024 13:11:15 -0400 Subject: [PATCH 03/85] ci: fix goreleaser check deprecation warning (#3795) Closes https://github.com/celestiaorg/celestia-app/issues/3793 by resolving this [deprecation notice](https://goreleaser.com/deprecations/#snapshotname_template) --- .goreleaser.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 950143a3f1..c8ce5cff64 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -108,7 +108,7 @@ archives: checksum: name_template: "checksums.txt" snapshot: - name_template: "{{ incpatch .Version }}-next" + version_template: "{{ incpatch .Version }}-next" changelog: sort: asc filters: From 9c9e77a2853f4f90c9fa3808ad2f3401fd5bd989 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Tue, 20 Aug 2024 14:32:28 -0400 Subject: [PATCH 04/85] refactor: remove unnecessary `config.Seal()` (#3786) - Add unit tests to cosmos SDK config set up - Remove an unnecessary `config.Seal` invocation that has been around for a [long time](https://github.com/celestiaorg/celestia-app/commit/5e4a1dc56e42e7341b6da9526430b12dcc7a6691#diff-9e0425d31b7cf4072a746feb0c6bc1e0045b639e01dd0b5ac2f08e8c2952c886R120) but isn't necessary because the `init()` command always sets and seals the config. --- app/config.go | 13 ------------- app/sdk_config.go | 15 +++++++++++++++ app/sdk_config_test.go | 18 ++++++++++++++++++ cmd/celestia-appd/cmd/root.go | 4 ---- 4 files changed, 33 insertions(+), 17 deletions(-) delete mode 100644 app/config.go create mode 100644 app/sdk_config.go create mode 100644 app/sdk_config_test.go diff --git a/app/config.go b/app/config.go deleted file mode 100644 index c6cd322e3b..0000000000 --- a/app/config.go +++ /dev/null @@ -1,13 +0,0 @@ -package app - -import ( - sdk "github.com/cosmos/cosmos-sdk/types" -) - -func init() { - cfg := sdk.GetConfig() - cfg.SetBech32PrefixForAccount(Bech32PrefixAccAddr, Bech32PrefixAccPub) - cfg.SetBech32PrefixForValidator(Bech32PrefixValAddr, Bech32PrefixValPub) - cfg.SetBech32PrefixForConsensusNode(Bech32PrefixConsAddr, Bech32PrefixConsPub) - cfg.Seal() -} diff --git a/app/sdk_config.go b/app/sdk_config.go new file mode 100644 index 0000000000..dd74de158d --- /dev/null +++ b/app/sdk_config.go @@ -0,0 +1,15 @@ +package app + +import sdk "github.com/cosmos/cosmos-sdk/types" + +func init() { + setCosmosSDKConfig() +} + +func setCosmosSDKConfig() { + config := sdk.GetConfig() + config.SetBech32PrefixForAccount(Bech32PrefixAccAddr, Bech32PrefixAccPub) + config.SetBech32PrefixForValidator(Bech32PrefixValAddr, Bech32PrefixValPub) + config.SetBech32PrefixForConsensusNode(Bech32PrefixConsAddr, Bech32PrefixConsPub) + config.Seal() +} diff --git a/app/sdk_config_test.go b/app/sdk_config_test.go new file mode 100644 index 0000000000..f3179096df --- /dev/null +++ b/app/sdk_config_test.go @@ -0,0 +1,18 @@ +package app + +import ( + "testing" + + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/stretchr/testify/assert" +) + +func Test_setCosmosSDKConfig(t *testing.T) { + config := sdk.GetConfig() + assert.Equal(t, Bech32PrefixAccAddr, config.GetBech32AccountAddrPrefix()) + assert.Equal(t, Bech32PrefixAccPub, config.GetBech32AccountPubPrefix()) + assert.Equal(t, Bech32PrefixValAddr, config.GetBech32ValidatorAddrPrefix()) + assert.Equal(t, Bech32PrefixValPub, config.GetBech32ValidatorPubPrefix()) + assert.Equal(t, Bech32PrefixConsAddr, config.GetBech32ConsensusAddrPrefix()) + assert.Equal(t, Bech32PrefixConsPub, config.GetBech32ConsensusPubPrefix()) +} diff --git a/cmd/celestia-appd/cmd/root.go b/cmd/celestia-appd/cmd/root.go index bfb052fc8a..a1fef093ad 100644 --- a/cmd/celestia-appd/cmd/root.go +++ b/cmd/celestia-appd/cmd/root.go @@ -16,7 +16,6 @@ import ( "github.com/cosmos/cosmos-sdk/server" serverconfig "github.com/cosmos/cosmos-sdk/server/config" simdcmd "github.com/cosmos/cosmos-sdk/simapp/simd/cmd" - sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/x/auth/types" banktypes "github.com/cosmos/cosmos-sdk/x/bank/types" "github.com/cosmos/cosmos-sdk/x/crisis" @@ -100,9 +99,6 @@ func NewRootCmd() *cobra.Command { // initRootCommand performs a bunch of side-effects on the root command. func initRootCommand(rootCommand *cobra.Command, encodingConfig encoding.Config) { - config := sdk.GetConfig() - config.Seal() - rootCommand.AddCommand( genutilcli.InitCmd(app.ModuleBasics, app.DefaultNodeHome), genutilcli.CollectGenTxsCmd(banktypes.GenesisBalancesIterator{}, app.DefaultNodeHome), From 5a06415d69418f0052d4ccbaf2363dfc399a2fee Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 21 Aug 2024 10:32:48 +0200 Subject: [PATCH 05/85] chore(deps): Bump github.com/cosmos/gogoproto from 1.5.0 to 1.7.0 (#3784) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/cosmos/gogoproto](https://github.com/cosmos/gogoproto) from 1.5.0 to 1.7.0.
Release notes

Sourced from github.com/cosmos/gogoproto's releases.

v1.7.0

What's Changed

Full Changelog: https://github.com/cosmos/gogoproto/compare/v1.6.0...v1.7.0

v1.6.0

What's Changed

Full Changelog: https://github.com/cosmos/gogoproto/compare/v1.5.0...v1.6.0

Changelog

Sourced from github.com/cosmos/gogoproto's changelog.

v1.7.0 - 2024-08-12

Improvements

  • #145 Remove x/exp dependency for greater compatibility.
  • #144 Change proto.Message and jsonpb.Anyresolver to aliased types to allow different implementations of the same interface.

v1.6.0 - 2024-08-08

Features

  • #142 Update code generator to make grpc ServiceDesc public.
Commits
  • 70f82eb refactor: increase gogoproto compatibility (#145)
  • a2d130f refactor: use structural type for AnyResolver & Message (#144)
  • 5312ce8 feat: make serviceDesc public (#142)
  • 0923950 chore: bump bufbuild/buf-setup-action from 1.35.0 to 1.36.0 (#141)
  • dbc6930 chore: bump amannn/action-semantic-pull-request from 5.5.2 to 5.5.3 (#136)
  • 8cd96de chore: bump bufbuild/buf-setup-action from 1.31.0 to 1.35.0 (#139)
  • 89ccff4 chore: improve any compat error (#138)
  • 6eec973 docs: Update CHANGELOG.md for v1.5.0 (#132)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/cosmos/gogoproto&package-manager=go_modules&previous-version=1.5.0&new-version=1.7.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8375b680f5..b6b9b48739 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/celestiaorg/rsmt2d v0.14.0 github.com/cosmos/cosmos-proto v1.0.0-beta.5 github.com/cosmos/cosmos-sdk v0.46.16 - github.com/cosmos/gogoproto v1.5.0 + github.com/cosmos/gogoproto v1.7.0 github.com/cosmos/ibc-apps/middleware/packet-forward-middleware/v6 v6.1.2 github.com/cosmos/ibc-go/v6 v6.2.2 github.com/ethereum/go-ethereum v1.14.7 diff --git a/go.sum b/go.sum index d5be2573b4..cbcbd380ad 100644 --- a/go.sum +++ b/go.sum @@ -419,8 +419,8 @@ github.com/cosmos/cosmos-proto v1.0.0-beta.5/go.mod h1:hQGLpiIUloJBMdQMMWb/4wRAp github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y= github.com/cosmos/go-bip39 v1.0.0 h1:pcomnQdrdH22njcAatO0yWojsUnCO3y2tNoV1cb6hHY= github.com/cosmos/go-bip39 v1.0.0/go.mod h1:RNJv0H/pOIVgxw6KS7QeX2a0Uo0aKUlfhZ4xuwvCdJw= -github.com/cosmos/gogoproto v1.5.0 h1:SDVwzEqZDDBoslaeZg+dGE55hdzHfgUA40pEanMh52o= -github.com/cosmos/gogoproto v1.5.0/go.mod h1:iUM31aofn3ymidYG6bUR5ZFrk+Om8p5s754eMUcyp8I= +github.com/cosmos/gogoproto v1.7.0 h1:79USr0oyXAbxg3rspGh/m4SWNyoz/GLaAh0QlCe2fro= +github.com/cosmos/gogoproto v1.7.0/go.mod h1:yWChEv5IUEYURQasfyBW5ffkMHR/90hiHgbNgrtp4j0= github.com/cosmos/gorocksdb v1.2.0 h1:d0l3jJG8M4hBouIZq0mDUHZ+zjOx044J3nGRskwTb4Y= github.com/cosmos/gorocksdb v1.2.0/go.mod h1:aaKvKItm514hKfNJpUJXnnOWeBnk2GL4+Qw9NHizILw= github.com/cosmos/iavl v0.19.6 h1:XY78yEeNPrEYyNCKlqr9chrwoeSDJ0bV2VjocTk//OU= From 0113f49bc74baffbfa06bcfc3786af4bd83d117e Mon Sep 17 00:00:00 2001 From: Rootul P Date: Wed, 21 Aug 2024 16:32:06 -0400 Subject: [PATCH 06/85] refactor: testnode utilities (#3785) ## Motivation I was looking through testnode code while working on a separate PR and these minor renames + moves made the code a little easier to read and understand. --- test/util/genesis/files.go | 28 ++++----- test/util/testnode/comet_node.go | 47 +++++++++++++++ .../{full_node_test.go => comet_node_test.go} | 4 +- test/util/testnode/config.go | 9 +++ test/util/testnode/full_node.go | 58 ------------------- test/util/testnode/logger.go | 24 ++++++++ test/util/testnode/network.go | 40 +++---------- test/util/testnode/node_interaction_api.go | 34 +++++------ test/util/testnode/rpc_client.go | 56 ++++-------------- test/util/testnode/utils.go | 48 +++++++++++++++ 10 files changed, 179 insertions(+), 169 deletions(-) create mode 100644 test/util/testnode/comet_node.go rename test/util/testnode/{full_node_test.go => comet_node_test.go} (97%) delete mode 100644 test/util/testnode/full_node.go create mode 100644 test/util/testnode/logger.go diff --git a/test/util/genesis/files.go b/test/util/genesis/files.go index c8de7eec03..ff702d3c15 100644 --- a/test/util/genesis/files.go +++ b/test/util/genesis/files.go @@ -11,50 +11,50 @@ import ( "github.com/tendermint/tendermint/privval" ) -// InitFiles initializes the files for a new tendermint node with the provided +// InitFiles initializes the files for a new Comet node with the provided // genesis. It will use the validatorIndex to save the validator's consensus // key. func InitFiles( - dir string, - tmCfg *config.Config, - g *Genesis, + rootDir string, + tmConfig *config.Config, + genesis *Genesis, validatorIndex int, -) (string, error) { - val, has := g.Validator(validatorIndex) +) (basePath string, err error) { + val, has := genesis.Validator(validatorIndex) if !has { return "", fmt.Errorf("validator %d not found", validatorIndex) } - basePath := filepath.Join(dir, ".celestia-app") - tmCfg.SetRoot(basePath) + basePath = filepath.Join(rootDir, ".celestia-app") + tmConfig.SetRoot(basePath) // save the genesis file configPath := filepath.Join(basePath, "config") - err := os.MkdirAll(configPath, os.ModePerm) + err = os.MkdirAll(configPath, os.ModePerm) if err != nil { return "", err } - gDoc, err := g.Export() + genesisDoc, err := genesis.Export() if err != nil { return "", fmt.Errorf("exporting genesis: %w", err) } - err = gDoc.SaveAs(tmCfg.GenesisFile()) + err = genesisDoc.SaveAs(tmConfig.GenesisFile()) if err != nil { return "", err } - pvStateFile := tmCfg.PrivValidatorStateFile() + pvStateFile := tmConfig.PrivValidatorStateFile() if err := tmos.EnsureDir(filepath.Dir(pvStateFile), 0o777); err != nil { return "", err } - pvKeyFile := tmCfg.PrivValidatorKeyFile() + pvKeyFile := tmConfig.PrivValidatorKeyFile() if err := tmos.EnsureDir(filepath.Dir(pvKeyFile), 0o777); err != nil { return "", err } filePV := privval.NewFilePV(val.ConsensusKey, pvKeyFile, pvStateFile) filePV.Save() - nodeKeyFile := tmCfg.NodeKeyFile() + nodeKeyFile := tmConfig.NodeKeyFile() if err := tmos.EnsureDir(filepath.Dir(nodeKeyFile), 0o777); err != nil { return "", err } diff --git a/test/util/testnode/comet_node.go b/test/util/testnode/comet_node.go new file mode 100644 index 0000000000..da0e566b37 --- /dev/null +++ b/test/util/testnode/comet_node.go @@ -0,0 +1,47 @@ +package testnode + +import ( + "path/filepath" + + "github.com/cosmos/cosmos-sdk/client/flags" + servertypes "github.com/cosmos/cosmos-sdk/server/types" + "github.com/tendermint/tendermint/node" + "github.com/tendermint/tendermint/p2p" + "github.com/tendermint/tendermint/privval" + "github.com/tendermint/tendermint/proxy" + tmdb "github.com/tendermint/tm-db" +) + +// NewCometNode creates a ready to use comet node that operates a single +// validator celestia-app network. It expects that all configuration files are +// already initialized and saved to the baseDir. +func NewCometNode(baseDir string, config *UniversalTestingConfig) (*node.Node, servertypes.Application, error) { + logger := NewLogger(config) + dbPath := filepath.Join(config.TmConfig.RootDir, "data") + db, err := tmdb.NewGoLevelDB("application", dbPath) + if err != nil { + return nil, nil, err + } + + config.AppOptions.Set(flags.FlagHome, baseDir) + + app := config.AppCreator(logger, db, nil, config.AppOptions) + + nodeKey, err := p2p.LoadOrGenNodeKey(config.TmConfig.NodeKeyFile()) + if err != nil { + return nil, nil, err + } + + cometNode, err := node.NewNode( + config.TmConfig, + privval.LoadOrGenFilePV(config.TmConfig.PrivValidatorKeyFile(), config.TmConfig.PrivValidatorStateFile()), + nodeKey, + proxy.NewLocalClientCreator(app), + node.DefaultGenesisDocProviderFunc(config.TmConfig), + node.DefaultDBProvider, + node.DefaultMetricsProvider(config.TmConfig.Instrumentation), + logger, + ) + + return cometNode, app, err +} diff --git a/test/util/testnode/full_node_test.go b/test/util/testnode/comet_node_test.go similarity index 97% rename from test/util/testnode/full_node_test.go rename to test/util/testnode/comet_node_test.go index 5a62602b65..a011992f79 100644 --- a/test/util/testnode/full_node_test.go +++ b/test/util/testnode/comet_node_test.go @@ -11,7 +11,7 @@ import ( "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" "github.com/celestiaorg/celestia-app/v3/test/util/genesis" blobtypes "github.com/celestiaorg/celestia-app/v3/x/blob/types" - appns "github.com/celestiaorg/go-square/v2/share" + "github.com/celestiaorg/go-square/v2/share" "github.com/cosmos/cosmos-sdk/client/flags" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -72,7 +72,7 @@ func (s *IntegrationTestSuite) Test_verifyTimeIotaMs() { func (s *IntegrationTestSuite) TestPostData() { require := s.Require() - _, err := s.cctx.PostData(s.accounts[0], flags.BroadcastBlock, appns.RandomBlobNamespace(), tmrand.Bytes(kibibyte)) + _, err := s.cctx.PostData(s.accounts[0], flags.BroadcastBlock, share.RandomBlobNamespace(), tmrand.Bytes(kibibyte)) require.NoError(err) } diff --git a/test/util/testnode/config.go b/test/util/testnode/config.go index 947df61fa9..7062735a4c 100644 --- a/test/util/testnode/config.go +++ b/test/util/testnode/config.go @@ -202,3 +202,12 @@ func CustomAppCreator(minGasPrice string) srvtypes.AppCreator { ) } } + +// DefaultAppConfig wraps the default config described in the server +func DefaultAppConfig() *srvconfig.Config { + appCfg := srvconfig.DefaultConfig() + appCfg.GRPC.Address = fmt.Sprintf("127.0.0.1:%d", mustGetFreePort()) + appCfg.API.Address = fmt.Sprintf("tcp://127.0.0.1:%d", mustGetFreePort()) + appCfg.MinGasPrices = fmt.Sprintf("%v%s", appconsts.DefaultMinGasPrice, appconsts.BondDenom) + return appCfg +} diff --git a/test/util/testnode/full_node.go b/test/util/testnode/full_node.go deleted file mode 100644 index 2a7b6e12c5..0000000000 --- a/test/util/testnode/full_node.go +++ /dev/null @@ -1,58 +0,0 @@ -package testnode - -import ( - "os" - "path/filepath" - - "github.com/cosmos/cosmos-sdk/client/flags" - srvtypes "github.com/cosmos/cosmos-sdk/server/types" - "github.com/tendermint/tendermint/libs/log" - "github.com/tendermint/tendermint/node" - "github.com/tendermint/tendermint/p2p" - "github.com/tendermint/tendermint/privval" - "github.com/tendermint/tendermint/proxy" - dbm "github.com/tendermint/tm-db" -) - -// NewCometNode creates a ready to use comet node that operates a single -// validator celestia-app network. It expects that all configuration files are -// already initialized and saved to the baseDir. -func NewCometNode(baseDir string, cfg *UniversalTestingConfig) (*node.Node, srvtypes.Application, error) { - logger := newLogger(cfg) - dbPath := filepath.Join(cfg.TmConfig.RootDir, "data") - db, err := dbm.NewGoLevelDB("application", dbPath) - if err != nil { - return nil, nil, err - } - - cfg.AppOptions.Set(flags.FlagHome, baseDir) - - app := cfg.AppCreator(logger, db, nil, cfg.AppOptions) - - nodeKey, err := p2p.LoadOrGenNodeKey(cfg.TmConfig.NodeKeyFile()) - if err != nil { - return nil, nil, err - } - - tmNode, err := node.NewNode( - cfg.TmConfig, - privval.LoadOrGenFilePV(cfg.TmConfig.PrivValidatorKeyFile(), cfg.TmConfig.PrivValidatorStateFile()), - nodeKey, - proxy.NewLocalClientCreator(app), - node.DefaultGenesisDocProviderFunc(cfg.TmConfig), - node.DefaultDBProvider, - node.DefaultMetricsProvider(cfg.TmConfig.Instrumentation), - logger, - ) - - return tmNode, app, err -} - -func newLogger(cfg *UniversalTestingConfig) log.Logger { - if cfg.SuppressLogs { - return log.NewNopLogger() - } - logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout)) - logger = log.NewFilter(logger, log.AllowError()) - return logger -} diff --git a/test/util/testnode/logger.go b/test/util/testnode/logger.go new file mode 100644 index 0000000000..36f80e249b --- /dev/null +++ b/test/util/testnode/logger.go @@ -0,0 +1,24 @@ +package testnode + +import ( + "os" + + "github.com/tendermint/tendermint/libs/log" +) + +func NewLogger(config *UniversalTestingConfig) log.Logger { + if config.SuppressLogs { + return log.NewNopLogger() + } + logger := log.NewTMLogger(log.NewSyncWriter(os.Stdout)) + switch config.TmConfig.LogLevel { + case "error": + return log.NewFilter(logger, log.AllowError()) + case "info": + return log.NewFilter(logger, log.AllowInfo()) + case "debug": + return log.NewFilter(logger, log.AllowDebug()) + default: + return logger + } +} diff --git a/test/util/testnode/network.go b/test/util/testnode/network.go index 34a013a524..79a1dc7d69 100644 --- a/test/util/testnode/network.go +++ b/test/util/testnode/network.go @@ -2,7 +2,6 @@ package testnode import ( "context" - "net" "testing" "github.com/celestiaorg/celestia-app/v3/test/util/genesis" @@ -16,14 +15,14 @@ import ( // accessed via the returned client.Context or via the returned rpc and grpc // addresses. Configured genesis options will be applied after all accounts have // been initialized. -func NewNetwork(t testing.TB, cfg *Config) (cctx Context, rpcAddr, grpcAddr string) { +func NewNetwork(t testing.TB, config *Config) (cctx Context, rpcAddr, grpcAddr string) { t.Helper() // initialize the genesis file and validator files for the first validator. - baseDir, err := genesis.InitFiles(t.TempDir(), cfg.TmConfig, cfg.Genesis, 0) + baseDir, err := genesis.InitFiles(t.TempDir(), config.TmConfig, config.Genesis, 0) require.NoError(t, err) - tmNode, app, err := NewCometNode(baseDir, &cfg.UniversalTestingConfig) + tmNode, app, err := NewCometNode(baseDir, &config.UniversalTestingConfig) require.NoError(t, err) ctx, cancel := context.WithCancel(context.Background()) @@ -31,15 +30,15 @@ func NewNetwork(t testing.TB, cfg *Config) (cctx Context, rpcAddr, grpcAddr stri cancel() }) - cctx = NewContext(ctx, cfg.Genesis.Keyring(), cfg.TmConfig, cfg.Genesis.ChainID, cfg.AppConfig.API.Address) + cctx = NewContext(ctx, config.Genesis.Keyring(), config.TmConfig, config.Genesis.ChainID, config.AppConfig.API.Address) cctx, stopNode, err := StartNode(tmNode, cctx) require.NoError(t, err) - cctx, cleanupGRPC, err := StartGRPCServer(app, cfg.AppConfig, cctx) + cctx, cleanupGRPC, err := StartGRPCServer(app, config.AppConfig, cctx) require.NoError(t, err) - apiServer, err := StartAPIServer(app, *cfg.AppConfig, cctx) + apiServer, err := StartAPIServer(app, *config.AppConfig, cctx) require.NoError(t, err) t.Cleanup(func() { @@ -64,30 +63,5 @@ func NewNetwork(t testing.TB, cfg *Config) (cctx Context, rpcAddr, grpcAddr stri } }) - return cctx, cfg.TmConfig.RPC.ListenAddress, cfg.AppConfig.GRPC.Address -} - -// getFreePort returns a free port and optionally an error. -func getFreePort() (int, error) { - a, err := net.ResolveTCPAddr("tcp", "localhost:0") - if err != nil { - return 0, err - } - - l, err := net.ListenTCP("tcp", a) - if err != nil { - return 0, err - } - defer l.Close() - return l.Addr().(*net.TCPAddr).Port, nil -} - -// mustGetFreePort returns a free port. Panics if no free ports are available or -// an error is encountered. -func mustGetFreePort() int { - port, err := getFreePort() - if err != nil { - panic(err) - } - return port + return cctx, config.TmConfig.RPC.ListenAddress, config.AppConfig.GRPC.Address } diff --git a/test/util/testnode/node_interaction_api.go b/test/util/testnode/node_interaction_api.go index 3695ee2a99..162840c819 100644 --- a/test/util/testnode/node_interaction_api.go +++ b/test/util/testnode/node_interaction_api.go @@ -30,28 +30,28 @@ const ( ) type Context struct { - rootCtx context.Context + goContext context.Context client.Context apiAddress string } -func NewContext(goCtx context.Context, kr keyring.Keyring, tmCfg *tmconfig.Config, chainID, apiAddress string) Context { - ecfg := encoding.MakeConfig(app.ModuleEncodingRegisters...) - cctx := client.Context{}. - WithKeyring(kr). - WithHomeDir(tmCfg.RootDir). +func NewContext(goContext context.Context, keyring keyring.Keyring, tmConfig *tmconfig.Config, chainID, apiAddress string) Context { + config := encoding.MakeConfig(app.ModuleEncodingRegisters...) + clientContext := client.Context{}. + WithKeyring(keyring). + WithHomeDir(tmConfig.RootDir). WithChainID(chainID). - WithInterfaceRegistry(ecfg.InterfaceRegistry). - WithCodec(ecfg.Codec). - WithLegacyAmino(ecfg.Amino). - WithTxConfig(ecfg.TxConfig). + WithInterfaceRegistry(config.InterfaceRegistry). + WithCodec(config.Codec). + WithLegacyAmino(config.Amino). + WithTxConfig(config.TxConfig). WithAccountRetriever(authtypes.AccountRetriever{}) - return Context{rootCtx: goCtx, Context: cctx, apiAddress: apiAddress} + return Context{goContext: goContext, Context: clientContext, apiAddress: apiAddress} } func (c *Context) GoContext() context.Context { - return c.rootCtx + return c.goContext } // GenesisTime returns the genesis block time. @@ -92,7 +92,7 @@ func (c *Context) WaitForHeightWithTimeout(h int64, t time.Duration) (int64, err ticker := time.NewTicker(time.Second) defer ticker.Stop() - ctx, cancel := context.WithTimeout(c.rootCtx, t) + ctx, cancel := context.WithTimeout(c.goContext, t) defer cancel() var ( @@ -102,8 +102,8 @@ func (c *Context) WaitForHeightWithTimeout(h int64, t time.Duration) (int64, err for { select { case <-ctx.Done(): - if c.rootCtx.Err() != nil { - return latestHeight, c.rootCtx.Err() + if c.goContext.Err() != nil { + return latestHeight, c.goContext.Err() } return latestHeight, fmt.Errorf("timeout (%v) exceeded waiting for network to reach height %d. Got to height %d", t, h, latestHeight) case <-ticker.C: @@ -123,7 +123,7 @@ func (c *Context) WaitForTimestampWithTimeout(t time.Time, d time.Duration) (tim ticker := time.NewTicker(time.Second) defer ticker.Stop() - ctx, cancel := context.WithTimeout(c.rootCtx, d) + ctx, cancel := context.WithTimeout(c.goContext, d) defer cancel() var latestTimestamp time.Time @@ -193,7 +193,7 @@ func (c *Context) WaitForTx(hashHexStr string, blocks int) (*rpctypes.ResultTx, return nil, err } - ctx, cancel := context.WithTimeout(c.rootCtx, DefaultTimeout) + ctx, cancel := context.WithTimeout(c.goContext, DefaultTimeout) defer cancel() for { diff --git a/test/util/testnode/rpc_client.go b/test/util/testnode/rpc_client.go index 835ced0eb5..c6299af62a 100644 --- a/test/util/testnode/rpc_client.go +++ b/test/util/testnode/rpc_client.go @@ -1,13 +1,10 @@ package testnode import ( - "fmt" - "os" "path" "strings" "time" - "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" "github.com/cosmos/cosmos-sdk/codec" "github.com/cosmos/cosmos-sdk/server/api" srvconfig "github.com/cosmos/cosmos-sdk/server/config" @@ -24,35 +21,33 @@ import ( // performs no operation. var noOpCleanup = func() error { return nil } -// StartNode starts the tendermint node along with a local core rpc client. The -// rpc is returned via the client.Context. The function returned should be +// StartNode starts the Comet node along with a local core RPC client. The +// RPC is returned via the client.Context. The function returned should be // called during cleanup to teardown the node, core client, along with canceling // the internal context.Context in the returned Context. -func StartNode(tmNode *node.Node, cctx Context) (Context, func() error, error) { - if err := tmNode.Start(); err != nil { +func StartNode(cometNode *node.Node, cctx Context) (Context, func() error, error) { + if err := cometNode.Start(); err != nil { return cctx, noOpCleanup, err } - - coreClient := local.New(tmNode) - - cctx.Context = cctx.WithClient(coreClient) + client := local.New(cometNode) + cctx.Context = cctx.WithClient(client) cleanup := func() error { - err := tmNode.Stop() + err := cometNode.Stop() if err != nil { return err } - tmNode.Wait() + cometNode.Wait() if err = removeDir(path.Join([]string{cctx.HomeDir, "config"}...)); err != nil { return err } - return removeDir(path.Join([]string{cctx.HomeDir, tmNode.Config().DBPath}...)) + return removeDir(path.Join([]string{cctx.HomeDir, cometNode.Config().DBPath}...)) } return cctx, cleanup, nil } -// StartGRPCServer starts the grpc server using the provided application and -// config. A grpc client connection to that server is also added to the client +// StartGRPCServer starts the GRPC server using the provided application and +// config. A GRPC client connection to that server is also added to the client // context. The returned function should be used to shutdown the server. func StartGRPCServer(app srvtypes.Application, appCfg *srvconfig.Config, cctx Context) (Context, func() error, error) { emptycleanup := func() error { return nil } @@ -91,35 +86,6 @@ func StartGRPCServer(app srvtypes.Application, appCfg *srvconfig.Config, cctx Co }, nil } -// DefaultAppConfig wraps the default config described in the server -func DefaultAppConfig() *srvconfig.Config { - appCfg := srvconfig.DefaultConfig() - appCfg.GRPC.Address = fmt.Sprintf("127.0.0.1:%d", mustGetFreePort()) - appCfg.API.Address = fmt.Sprintf("tcp://127.0.0.1:%d", mustGetFreePort()) - appCfg.MinGasPrices = fmt.Sprintf("%v%s", appconsts.DefaultMinGasPrice, appconsts.BondDenom) - return appCfg -} - -// removeDir removes the directory `rootDir`. -// The main use of this is to reduce the flakiness of the CI when it's unable to delete -// the config folder of the tendermint node. -// This will manually go over the files contained inside the provided `rootDir` -// and delete them one by one. -func removeDir(rootDir string) error { - dir, err := os.ReadDir(rootDir) - if err != nil { - return err - } - for _, d := range dir { - path := path.Join([]string{rootDir, d.Name()}...) - err := os.RemoveAll(path) - if err != nil { - return err - } - } - return os.RemoveAll(rootDir) -} - func StartAPIServer(app srvtypes.Application, appCfg srvconfig.Config, cctx Context) (*api.Server, error) { apiSrv := api.New(cctx.Context, log.NewNopLogger()) app.RegisterAPIRoutes(apiSrv, appCfg.API) diff --git a/test/util/testnode/utils.go b/test/util/testnode/utils.go index bf71aaa951..b17c0db6c8 100644 --- a/test/util/testnode/utils.go +++ b/test/util/testnode/utils.go @@ -3,6 +3,9 @@ package testnode import ( "context" "encoding/hex" + "net" + "os" + "path" "github.com/celestiaorg/celestia-app/v3/app" "github.com/celestiaorg/celestia-app/v3/app/encoding" @@ -88,3 +91,48 @@ func GenerateAccounts(count int) []string { } return accs } + +// getFreePort returns a free port and optionally an error. +func getFreePort() (int, error) { + a, err := net.ResolveTCPAddr("tcp", "localhost:0") + if err != nil { + return 0, err + } + + l, err := net.ListenTCP("tcp", a) + if err != nil { + return 0, err + } + defer l.Close() + return l.Addr().(*net.TCPAddr).Port, nil +} + +// mustGetFreePort returns a free port. Panics if no free ports are available or +// an error is encountered. +func mustGetFreePort() int { + port, err := getFreePort() + if err != nil { + panic(err) + } + return port +} + +// removeDir removes the directory `rootDir`. +// The main use of this is to reduce the flakiness of the CI when it's unable to delete +// the config folder of the tendermint node. +// This will manually go over the files contained inside the provided `rootDir` +// and delete them one by one. +func removeDir(rootDir string) error { + dir, err := os.ReadDir(rootDir) + if err != nil { + return err + } + for _, d := range dir { + path := path.Join([]string{rootDir, d.Name()}...) + err := os.RemoveAll(path) + if err != nil { + return err + } + } + return os.RemoveAll(rootDir) +} From fd36203b50ec19973e232a31d4a6bd86ead21f1b Mon Sep 17 00:00:00 2001 From: Rootul P Date: Wed, 21 Aug 2024 17:26:44 -0400 Subject: [PATCH 07/85] chore: remove make test-interchain (#3800) I forgot to remove this Makefile target in https://github.com/celestiaorg/celestia-app/pull/3776 --- Makefile | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Makefile b/Makefile index 448677db94..65711cfecd 100644 --- a/Makefile +++ b/Makefile @@ -163,13 +163,6 @@ test-fuzz: bash -x scripts/test_fuzz.sh .PHONY: test-fuzz -## test-interchain: Run interchain tests in verbose mode. Requires Docker. -test-interchain: - @echo "Reminder: this test uses the latest celestia-app Docker image. If you would like to test recent code changes, re-build the Docker image by running: make build-docker" - @echo "--> Running interchain tests" - @cd ./test/interchain && go test . -v -.PHONY: test-interchain - ## txsim-install: Install the tx simulator. txsim-install: @echo "--> Installing tx simulator" From b948c08c545049809aeb14e1e0e02e7367fb0bfc Mon Sep 17 00:00:00 2001 From: Evan Forbes <42654277+evan-forbes@users.noreply.github.com> Date: Thu, 22 Aug 2024 11:45:01 -0500 Subject: [PATCH 08/85] feat: require bbr by default (#3774) ## Overview This PR adds a requirement to use bbr by default. This requirement can be bypassed by using a flag. Warnings are returned when installing, and there is a makefile command that will configure the system to use bbr. ### ~~~ Note for Reviewers ~~~ Most of this PR is copying over the start command from the sdk. Alternatively, we could simply modify the sdk. That would not require copy past forking, but the trend has been that every change that we make to the sdk eventually gets moved here, so I figured we should just skip to doing that. This also gives us finer grained control in the future. I'm definitely not tied to this approach, and if we'd rather change our fork, then I'm happy go do that instead. closes #3745 --------- Co-authored-by: Rootul P Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- Makefile | 27 +- README.md | 16 + cmd/celestia-appd/cmd/root.go | 2 +- cmd/celestia-appd/cmd/start.go | 632 +++++++++++++++++++++++++++++++++ scripts/single-node.sh | 3 +- 5 files changed, 677 insertions(+), 3 deletions(-) create mode 100644 cmd/celestia-appd/cmd/start.go diff --git a/Makefile b/Makefile index 65711cfecd..8af2c10c00 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ build: mod .PHONY: build ## install: Build and install the celestia-appd binary into the $GOPATH/bin directory. -install: go.sum +install: go.sum check-bbr @echo "--> Installing celestia-appd" @go install $(BUILD_FLAGS) ./cmd/celestia-appd .PHONY: install @@ -205,3 +205,28 @@ prebuilt-binary: ghcr.io/goreleaser/goreleaser-cross:${GOLANG_CROSS_VERSION} \ release --clean .PHONY: prebuilt-binary + +## check-bbr: Check if your system uses BBR congestion control algorithm. Only works on Linux. +check-bbr: + @echo "Checking if BBR is enabled..." + @if [ "$(sysctl net.ipv4.tcp_congestion_control | awk '{print $3}')" != "bbr" ]; then \ + echo "WARNING: BBR is not enabled. Please enable BBR for optimal performance. Call make enable-bbr or see Usage section in the README."; \ + else \ + echo "BBR is enabled."; \ + fi +.PHONY: check-bbr + +## enable-bbr: Enable BBR congestion control algorithm. Only works on Linux. +enable-bbr: + @echo "Configuring system to use BBR..." + @if [ "$(sysctl net.ipv4.tcp_congestion_control | awk '{print $3}')" != "bbr" ]; then \ + echo "BBR is not enabled. Configuring BBR..."; \ + sudo modprobe tcp_bbr; \ + echo "net.core.default_qdisc=fq" | sudo tee -a /etc/sysctl.conf; \ + echo "net.ipv4.tcp_congestion_control=bbr" | sudo tee -a /etc/sysctl.conf; \ + sudo sysctl -p; \ + echo "BBR has been enabled."; \ + else \ + echo "BBR is already enabled."; \ + fi +.PHONY: enable-bbr \ No newline at end of file diff --git a/README.md b/README.md index 58ac9da858..f1f94636c7 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,22 @@ See for more information. ## Usage +First, make sure that the [BBR](https://www.ietf.org/archive/id/draft-cardwell-iccrg-bbr-congestion-control-01.html) ("Bottleneck Bandwidth and Round-trip propagation time") congestion control algorithm is enabled in the +system's kernel. The result should contain `bbr`: + +```sh +sysctl net.ipv4.tcp_congestion_control +``` + +If not, enable it on Linux by calling the `make use-bbr` or by running: + +```sh +sudo modprobe tcp_bbr +net.core.default_qdisc=fq +net.ipv4.tcp_congestion_control=bbr +sudo sysctl -p +``` + ```sh # Print help celestia-appd --help diff --git a/cmd/celestia-appd/cmd/root.go b/cmd/celestia-appd/cmd/root.go index a1fef093ad..f8d57040fb 100644 --- a/cmd/celestia-appd/cmd/root.go +++ b/cmd/celestia-appd/cmd/root.go @@ -122,7 +122,7 @@ func initRootCommand(rootCommand *cobra.Command, encodingConfig encoding.Config) ) // Add the following commands to the rootCommand: start, tendermint, export, version, and rollback. - server.AddCommands(rootCommand, app.DefaultNodeHome, NewAppServer, appExporter, addModuleInitFlags) + addCommands(rootCommand, app.DefaultNodeHome, NewAppServer, appExporter, addModuleInitFlags) } // setDefaultConsensusParams sets the default consensus parameters for the diff --git a/cmd/celestia-appd/cmd/start.go b/cmd/celestia-appd/cmd/start.go new file mode 100644 index 0000000000..04c64173c3 --- /dev/null +++ b/cmd/celestia-appd/cmd/start.go @@ -0,0 +1,632 @@ +package cmd + +// NOTE: This file was copy paste forked from the sdk in order to modify the +// start command flag. + +import ( + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime/pprof" + "strings" + "time" + + "github.com/cosmos/cosmos-sdk/client" + "github.com/cosmos/cosmos-sdk/client/flags" + "github.com/cosmos/cosmos-sdk/codec" + pruningtypes "github.com/cosmos/cosmos-sdk/pruning/types" + "github.com/cosmos/cosmos-sdk/server" + "github.com/cosmos/cosmos-sdk/server/api" + serverconfig "github.com/cosmos/cosmos-sdk/server/config" + servergrpc "github.com/cosmos/cosmos-sdk/server/grpc" + "github.com/cosmos/cosmos-sdk/server/rosetta" + crgserver "github.com/cosmos/cosmos-sdk/server/rosetta/lib/server" + srvrtypes "github.com/cosmos/cosmos-sdk/server/types" + "github.com/cosmos/cosmos-sdk/telemetry" + sdktypes "github.com/cosmos/cosmos-sdk/types" + "github.com/cosmos/cosmos-sdk/version" + "github.com/spf13/cobra" + tmserver "github.com/tendermint/tendermint/abci/server" + cmtcmd "github.com/tendermint/tendermint/cmd/cometbft/commands" + tmos "github.com/tendermint/tendermint/libs/os" + "github.com/tendermint/tendermint/node" + "github.com/tendermint/tendermint/p2p" + "github.com/tendermint/tendermint/privval" + "github.com/tendermint/tendermint/proxy" + "github.com/tendermint/tendermint/rpc/client/local" + dbm "github.com/tendermint/tm-db" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +const ( + // Tendermint full-node start flags + flagWithTendermint = "with-tendermint" + flagAddress = "address" + flagTransport = "transport" + flagTraceStore = "trace-store" + flagCPUProfile = "cpu-profile" + + FlagForceNoBBR = "force-no-bbr" + + // gRPC-related flags + flagGRPCOnly = "grpc-only" + flagGRPCEnable = "grpc.enable" + flagGRPCAddress = "grpc.address" + flagGRPCWebEnable = "grpc-web.enable" + flagGRPCWebAddress = "grpc-web.address" +) + +// startCmd runs the service passed in, either stand-alone or in-process with +// Tendermint. +func startCmd(appCreator srvrtypes.AppCreator, defaultNodeHome string) *cobra.Command { + cmd := &cobra.Command{ + Use: "start", + Short: "Run the full node", + Long: `Run the full node application with Tendermint in or out of process. By +default, the application will run with Tendermint in process. + +Pruning options can be provided via the '--pruning' flag or alternatively with '--pruning-keep-recent', and +'pruning-interval' together. + +For '--pruning' the options are as follows: + +default: the last 362880 states are kept, pruning at 10 block intervals +nothing: all historic states will be saved, nothing will be deleted (i.e. archiving node) +everything: 2 latest states will be kept; pruning at 10 block intervals. +custom: allow pruning options to be manually specified through 'pruning-keep-recent', and 'pruning-interval' + +Node halting configurations exist in the form of two flags: '--halt-height' and '--halt-time'. During +the ABCI Commit phase, the node will check if the current block height is greater than or equal to +the halt-height or if the current block time is greater than or equal to the halt-time. If so, the +node will attempt to gracefully shutdown and the block will not be committed. In addition, the node +will not be able to commit subsequent blocks. + +For profiling and benchmarking purposes, CPU profiling can be enabled via the '--cpu-profile' flag +which accepts a path for the resulting pprof file. + +The node may be started in a 'query only' mode where only the gRPC and JSON HTTP +API services are enabled via the 'grpc-only' flag. In this mode, Tendermint is +bypassed and can be used when legacy queries are needed after an on-chain upgrade +is performed. Note, when enabled, gRPC will also be automatically enabled. +`, + PreRunE: func(cmd *cobra.Command, _ []string) error { + serverCtx := server.GetServerContextFromCmd(cmd) + + // Bind flags to the Context's Viper so the app construction can set + // options accordingly. + if err := serverCtx.Viper.BindPFlags(cmd.Flags()); err != nil { + return err + } + + _, err := server.GetPruningOptionsFromFlags(serverCtx.Viper) + return err + }, + RunE: func(cmd *cobra.Command, _ []string) error { + err := checkBBR(cmd) + if err != nil { + return err + } + + serverCtx := server.GetServerContextFromCmd(cmd) + clientCtx, err := client.GetClientQueryContext(cmd) + if err != nil { + return err + } + + withTM, _ := cmd.Flags().GetBool(flagWithTendermint) + if !withTM { + serverCtx.Logger.Info("starting ABCI without Tendermint") + return wrapCPUProfile(serverCtx, func() error { + return startStandAlone(serverCtx, appCreator) + }) + } + + // amino is needed here for backwards compatibility of REST routes + err = wrapCPUProfile(serverCtx, func() error { + return startInProcess(serverCtx, clientCtx, appCreator) + }) + errCode, ok := err.(server.ErrorCode) + if !ok { + return err + } + + serverCtx.Logger.Debug(fmt.Sprintf("received quit signal: %d", errCode.Code)) + return nil + }, + } + + cmd.Flags().String(flags.FlagHome, defaultNodeHome, "The application home directory") + cmd.Flags().Bool(flagWithTendermint, true, "Run abci app embedded in-process with tendermint") + cmd.Flags().String(flagAddress, "tcp://0.0.0.0:26658", "Listen address") + cmd.Flags().String(flagTransport, "socket", "Transport protocol: socket, grpc") + cmd.Flags().String(flagTraceStore, "", "Enable KVStore tracing to an output file") + cmd.Flags().String(server.FlagMinGasPrices, "", "Minimum gas prices to accept for transactions; Any fee in a tx must meet this minimum (e.g. 0.01photino;0.0001stake)") + cmd.Flags().IntSlice(server.FlagUnsafeSkipUpgrades, []int{}, "Skip a set of upgrade heights to continue the old binary") + cmd.Flags().Uint64(server.FlagHaltHeight, 0, "Block height at which to gracefully halt the chain and shutdown the node") + cmd.Flags().Uint64(server.FlagHaltTime, 0, "Minimum block time (in Unix seconds) at which to gracefully halt the chain and shutdown the node") + cmd.Flags().Bool(server.FlagInterBlockCache, true, "Enable inter-block caching") + cmd.Flags().String(flagCPUProfile, "", "Enable CPU profiling and write to the provided file") + cmd.Flags().Bool(server.FlagTrace, false, "Provide full stack traces for errors in ABCI Log") + cmd.Flags().String(server.FlagPruning, pruningtypes.PruningOptionDefault, "Pruning strategy (default|nothing|everything|custom)") + cmd.Flags().Uint64(server.FlagPruningKeepRecent, 0, "Number of recent heights to keep on disk (ignored if pruning is not 'custom')") + cmd.Flags().Uint64(server.FlagPruningInterval, 0, "Height interval at which pruned heights are removed from disk (ignored if pruning is not 'custom')") + cmd.Flags().Uint(server.FlagInvCheckPeriod, 0, "Assert registered invariants every N blocks") + cmd.Flags().Uint64(server.FlagMinRetainBlocks, 0, "Minimum block height offset during ABCI commit to prune Tendermint blocks") + cmd.Flags().Bool(FlagForceNoBBR, false, "bypass the requirement to use bbr locally") + + cmd.Flags().Bool(server.FlagAPIEnable, false, "Define if the API server should be enabled") + cmd.Flags().Bool(server.FlagAPISwagger, false, "Define if swagger documentation should automatically be registered (Note: the API must also be enabled)") + cmd.Flags().String(server.FlagAPIAddress, serverconfig.DefaultAPIAddress, "the API server address to listen on") + cmd.Flags().Uint(server.FlagAPIMaxOpenConnections, 1000, "Define the number of maximum open connections") + cmd.Flags().Uint(server.FlagRPCReadTimeout, 10, "Define the Tendermint RPC read timeout (in seconds)") + cmd.Flags().Uint(server.FlagRPCWriteTimeout, 0, "Define the Tendermint RPC write timeout (in seconds)") + cmd.Flags().Uint(server.FlagRPCMaxBodyBytes, 1000000, "Define the Tendermint maximum response body (in bytes)") + cmd.Flags().Bool(server.FlagAPIEnableUnsafeCORS, false, "Define if CORS should be enabled (unsafe - use it at your own risk)") + + cmd.Flags().Bool(flagGRPCOnly, false, "Start the node in gRPC query only mode (no Tendermint process is started)") + cmd.Flags().Bool(flagGRPCEnable, true, "Define if the gRPC server should be enabled") + cmd.Flags().String(flagGRPCAddress, serverconfig.DefaultGRPCAddress, "the gRPC server address to listen on") + + cmd.Flags().Bool(flagGRPCWebEnable, true, "Define if the gRPC-Web server should be enabled. (Note: gRPC must also be enabled)") + cmd.Flags().String(flagGRPCWebAddress, serverconfig.DefaultGRPCWebAddress, "The gRPC-Web server address to listen on") + + cmd.Flags().Uint64(server.FlagStateSyncSnapshotInterval, 0, "State sync snapshot interval") + cmd.Flags().Uint32(server.FlagStateSyncSnapshotKeepRecent, 2, "State sync snapshot to keep") + + cmd.Flags().Bool(server.FlagDisableIAVLFastNode, false, "Disable fast node for IAVL tree") + + // add support for all Tendermint-specific command line options + cmtcmd.AddNodeFlags(cmd) + return cmd +} + +func startStandAlone(ctx *server.Context, appCreator srvrtypes.AppCreator) error { + addr := ctx.Viper.GetString(flagAddress) + transport := ctx.Viper.GetString(flagTransport) + home := ctx.Viper.GetString(flags.FlagHome) + + db, err := openDB(home, server.GetAppDBBackend(ctx.Viper)) + if err != nil { + return err + } + + traceWriterFile := ctx.Viper.GetString(flagTraceStore) + traceWriter, err := openTraceWriter(traceWriterFile) + if err != nil { + return err + } + + app := appCreator(ctx.Logger, db, traceWriter, ctx.Viper) + + config, err := serverconfig.GetConfig(ctx.Viper) + if err != nil { + return err + } + + _, err = startTelemetry(config) + if err != nil { + return err + } + + svr, err := tmserver.NewServer(addr, transport, app) + if err != nil { + return fmt.Errorf("error creating listener: %v", err) + } + + svr.SetLogger(ctx.Logger.With("module", "abci-server")) + + err = svr.Start() + if err != nil { + tmos.Exit(err.Error()) + } + + defer func() { + if err = svr.Stop(); err != nil { + tmos.Exit(err.Error()) + } + + if err = app.Close(); err != nil { + tmos.Exit(err.Error()) + } + }() + + // Wait for SIGINT or SIGTERM signal + return server.WaitForQuitSignals() +} + +func startInProcess(ctx *server.Context, clientCtx client.Context, appCreator srvrtypes.AppCreator) error { + cfg := ctx.Config + home := cfg.RootDir + + db, err := openDB(home, server.GetAppDBBackend(ctx.Viper)) + if err != nil { + return err + } + + traceWriterFile := ctx.Viper.GetString(flagTraceStore) + traceWriter, err := openTraceWriter(traceWriterFile) + if err != nil { + return err + } + + config, err := serverconfig.GetConfig(ctx.Viper) + if err != nil { + return err + } + + if err := config.ValidateBasic(); err != nil { + return err + } + + app := appCreator(ctx.Logger, db, traceWriter, ctx.Viper) + + nodeKey, err := p2p.LoadOrGenNodeKey(cfg.NodeKeyFile()) + if err != nil { + return err + } + + genDocProvider := node.DefaultGenesisDocProviderFunc(cfg) + + var ( + tmNode *node.Node + gRPCOnly = ctx.Viper.GetBool(flagGRPCOnly) + ) + + if gRPCOnly { + ctx.Logger.Info("starting node in gRPC only mode; Tendermint is disabled") + config.GRPC.Enable = true + } else { + ctx.Logger.Info("starting node with ABCI Tendermint in-process") + + tmNode, err = node.NewNode( + cfg, + privval.LoadOrGenFilePV(cfg.PrivValidatorKeyFile(), cfg.PrivValidatorStateFile()), + nodeKey, + proxy.NewLocalClientCreator(app), + genDocProvider, + node.DefaultDBProvider, + node.DefaultMetricsProvider(cfg.Instrumentation), + ctx.Logger, + ) + if err != nil { + return err + } + if err := tmNode.Start(); err != nil { + return err + } + } + + // Add the tx service to the gRPC router. We only need to register this + // service if API or gRPC is enabled, and avoid doing so in the general + // case, because it spawns a new local tendermint RPC client. + if (config.API.Enable || config.GRPC.Enable) && tmNode != nil { + // re-assign for making the client available below + // do not use := to avoid shadowing clientCtx + clientCtx = clientCtx.WithClient(local.New(tmNode)) + + app.RegisterTxService(clientCtx) + app.RegisterTendermintService(clientCtx) + + if a, ok := app.(srvrtypes.ApplicationQueryService); ok { + a.RegisterNodeService(clientCtx) + } + } + + metrics, err := startTelemetry(config) + if err != nil { + return err + } + + var apiSrv *api.Server + if config.API.Enable { + genDoc, err := genDocProvider() + if err != nil { + return err + } + + clientCtx := clientCtx.WithHomeDir(home).WithChainID(genDoc.ChainID) + + if config.GRPC.Enable { + _, _, err := net.SplitHostPort(config.GRPC.Address) + if err != nil { + return err + } + + maxSendMsgSize := config.GRPC.MaxSendMsgSize + if maxSendMsgSize == 0 { + maxSendMsgSize = serverconfig.DefaultGRPCMaxSendMsgSize + } + + maxRecvMsgSize := config.GRPC.MaxRecvMsgSize + if maxRecvMsgSize == 0 { + maxRecvMsgSize = serverconfig.DefaultGRPCMaxRecvMsgSize + } + + // If grpc is enabled, configure grpc client for grpc gateway. + grpcClient, err := grpc.NewClient( + config.GRPC.Address, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultCallOptions( + grpc.ForceCodec(codec.NewProtoCodec(clientCtx.InterfaceRegistry).GRPCCodec()), + grpc.MaxCallRecvMsgSize(maxRecvMsgSize), + grpc.MaxCallSendMsgSize(maxSendMsgSize), + ), + ) + if err != nil { + return err + } + + clientCtx = clientCtx.WithGRPCClient(grpcClient) + ctx.Logger.Debug("grpc client assigned to client context", "target", config.GRPC.Address) + } + + apiSrv = api.New(clientCtx, ctx.Logger.With("module", "api-server")) + app.RegisterAPIRoutes(apiSrv, config.API) + if config.Telemetry.Enabled { + apiSrv.SetTelemetry(metrics) + } + errCh := make(chan error) + + go func() { + if err := apiSrv.Start(config); err != nil { + errCh <- err + } + }() + + select { + case err := <-errCh: + return err + + case <-time.After(srvrtypes.ServerStartTime): // assume server started successfully + } + } + + var ( + grpcSrv *grpc.Server + grpcWebSrv *http.Server + ) + + if config.GRPC.Enable { + grpcSrv, err = servergrpc.StartGRPCServer(clientCtx, app, config.GRPC) + if err != nil { + return err + } + defer grpcSrv.Stop() + if config.GRPCWeb.Enable { + grpcWebSrv, err = servergrpc.StartGRPCWeb(grpcSrv, config) + if err != nil { + ctx.Logger.Error("failed to start grpc-web http server: ", err) + return err + } + defer func() { + if err := grpcWebSrv.Close(); err != nil { + ctx.Logger.Error("failed to close grpc-web http server: ", err) + } + }() + } + } + + // At this point it is safe to block the process if we're in gRPC only mode as + // we do not need to start Rosetta or handle any Tendermint related processes. + if gRPCOnly { + // wait for signal capture and gracefully return + return server.WaitForQuitSignals() + } + + var rosettaSrv crgserver.Server + if config.Rosetta.Enable { + offlineMode := config.Rosetta.Offline + + // If GRPC is not enabled rosetta cannot work in online mode, so it works in + // offline mode. + if !config.GRPC.Enable { + offlineMode = true + } + + minGasPrices, err := sdktypes.ParseDecCoins(config.MinGasPrices) + if err != nil { + ctx.Logger.Error("failed to parse minimum-gas-prices: ", err) + return err + } + + conf := &rosetta.Config{ + Blockchain: config.Rosetta.Blockchain, + Network: config.Rosetta.Network, + TendermintRPC: ctx.Config.RPC.ListenAddress, + GRPCEndpoint: config.GRPC.Address, + Addr: config.Rosetta.Address, + Retries: config.Rosetta.Retries, + Offline: offlineMode, + GasToSuggest: config.Rosetta.GasToSuggest, + EnableFeeSuggestion: config.Rosetta.EnableFeeSuggestion, + GasPrices: minGasPrices.Sort(), + Codec: clientCtx.Codec.(*codec.ProtoCodec), + InterfaceRegistry: clientCtx.InterfaceRegistry, + } + + rosettaSrv, err = rosetta.ServerFromConfig(conf) + if err != nil { + return err + } + + errCh := make(chan error) + go func() { + if err := rosettaSrv.Start(); err != nil { + errCh <- err + } + }() + + select { + case err := <-errCh: + return err + + case <-time.After(srvrtypes.ServerStartTime): // assume server started successfully + } + } + + defer func() { + if tmNode != nil && tmNode.IsRunning() { + _ = tmNode.Stop() + _ = app.Close() + } + + if apiSrv != nil { + _ = apiSrv.Close() + } + + ctx.Logger.Info("exiting...") + }() + + // wait for signal capture and gracefully return + return server.WaitForQuitSignals() +} + +func startTelemetry(cfg serverconfig.Config) (*telemetry.Metrics, error) { + if !cfg.Telemetry.Enabled { + return nil, nil + } + return telemetry.New(cfg.Telemetry) +} + +// wrapCPUProfile runs callback in a goroutine, then wait for quit signals. +func wrapCPUProfile(ctx *server.Context, callback func() error) error { + if cpuProfile := ctx.Viper.GetString(flagCPUProfile); cpuProfile != "" { + f, err := os.Create(cpuProfile) + if err != nil { + return err + } + + ctx.Logger.Info("starting CPU profiler", "profile", cpuProfile) + if err := pprof.StartCPUProfile(f); err != nil { + return err + } + + defer func() { + ctx.Logger.Info("stopping CPU profiler", "profile", cpuProfile) + pprof.StopCPUProfile() + if err := f.Close(); err != nil { + ctx.Logger.Info("failed to close cpu-profile file", "profile", cpuProfile, "err", err.Error()) + } + }() + } + + errCh := make(chan error) + go func() { + errCh <- callback() + }() + + select { + case err := <-errCh: + return err + + case <-time.After(srvrtypes.ServerStartTime): + } + + return server.WaitForQuitSignals() +} + +func addCommands( + rootCmd *cobra.Command, + defaultNodeHome string, + appCreator srvrtypes.AppCreator, + appExport srvrtypes.AppExporter, + addStartFlags srvrtypes.ModuleInitFlags, +) { + tendermintCmd := &cobra.Command{ + Use: "tendermint", + Aliases: []string{"comet", "cometbft"}, + Short: "Tendermint subcommands", + } + + tendermintCmd.AddCommand( + server.ShowNodeIDCmd(), + server.ShowValidatorCmd(), + server.ShowAddressCmd(), + server.VersionCmd(), + cmtcmd.ResetAllCmd, + cmtcmd.ResetStateCmd, + server.BootstrapStateCmd(appCreator), + ) + + startCmd := startCmd(appCreator, defaultNodeHome) + addStartFlags(startCmd) + + rootCmd.AddCommand( + startCmd, + tendermintCmd, + server.ExportCmd(appExport, defaultNodeHome), + version.NewVersionCommand(), + server.NewRollbackCmd(appCreator, defaultNodeHome), + ) +} + +// checkBBR checks is bbr is configured to be used as a congestion control algo. +func checkBBR(command *cobra.Command) error { + const ( + errorMsg = ` +// The BBR congestion control algorithm does not appear to be enabled in this +// system's kernel. This is important for the p2p stack to be performant. +// +// to enable bbr call: +// +sudo modprobe tcp_bbr +net.core.default_qdisc=fq +net.ipv4.tcp_congestion_control=bbr +sudo sysctl -p +// +// and can be verified to be running using +// +sysctl net.ipv4.tcp_congestion_control + +// This might not work for all systems, you might have to search online to +// figure out how to enable bbr for your system. +// +// While this node will get worse performance using something other than bbr, +// If you need to bypass this block use the "--force-no-bbr true" flag. + ` + ) + + noBBRErr := errors.New(errorMsg) + + forceNoBBR, err := command.Flags().GetBool(FlagForceNoBBR) + if err != nil { + return noBBRErr + } + if forceNoBBR { + return nil + } + + cmd := exec.Command("sysctl", "net.ipv4.tcp_congestion_control") + output, err := cmd.Output() + if err != nil { + return err + } + + if !strings.Contains(string(output), "bbr") { + return noBBRErr + } + + return nil +} + +func openDB(rootDir string, backendType dbm.BackendType) (dbm.DB, error) { + dataDir := filepath.Join(rootDir, "data") + return dbm.NewDB("application", backendType, dataDir) +} + +func openTraceWriter(traceWriterFile string) (w io.Writer, err error) { + if traceWriterFile == "" { + return nil, nil + } + return os.OpenFile( + traceWriterFile, + os.O_WRONLY|os.O_APPEND|os.O_CREATE, + 0o666, + ) +} diff --git a/scripts/single-node.sh b/scripts/single-node.sh index 982f3430f1..d7d19b4f07 100755 --- a/scripts/single-node.sh +++ b/scripts/single-node.sh @@ -102,7 +102,8 @@ startCelestiaApp() { --api.enable \ --grpc.enable \ --grpc-web.enable \ - --v2-upgrade-height 3 + --v2-upgrade-height 3 \ + --force-no-bbr // no need to require BBR usage on a local node } if [ -f $GENESIS_FILE ]; then From ad250dde754082f68d33a4c4435c01e97125d3ec Mon Sep 17 00:00:00 2001 From: Rootul P Date: Thu, 22 Aug 2024 12:55:56 -0400 Subject: [PATCH 09/85] fix: state sync on main (#3811) Forward port of https://github.com/celestiaorg/celestia-app/pull/3808 --- app/app.go | 27 +++++++++++++++++++++ app/app_test.go | 44 ++++++++++++++++++++++++++++++++++ scripts/arabica.sh | 59 ++++++++++++++++++++++++++++++++++++++++++++++ scripts/mocha.sh | 59 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 189 insertions(+) create mode 100644 scripts/arabica.sh create mode 100644 scripts/mocha.sh diff --git a/app/app.go b/app/app.go index 9f012dcba3..71c519305e 100644 --- a/app/app.go +++ b/app/app.go @@ -570,6 +570,7 @@ func (app *App) InitChain(req abci.RequestInitChain) (res abci.ResponseInitChain // mountKeysAndInit mounts the keys for the provided app version and then // invokes baseapp.Init(). func (app *App) mountKeysAndInit(appVersion uint64) { + app.BaseApp.Logger().Debug(fmt.Sprintf("mounting KV stores for app version %v", appVersion)) app.MountKVStores(app.versionedKeys(appVersion)) // Invoke load latest version for its side-effect of invoking baseapp.Init() @@ -784,3 +785,29 @@ func (app *App) InitializeAppVersion(ctx sdk.Context) { app.SetAppVersion(ctx, appVersion) } } + +// OfferSnapshot is a wrapper around the baseapp's OfferSnapshot method. It is +// needed to mount stores for the appropriate app version. +func (app *App) OfferSnapshot(req abci.RequestOfferSnapshot) abci.ResponseOfferSnapshot { + if app.IsSealed() { + // If the app is sealed, keys have already been mounted so this can + // delegate to the baseapp's OfferSnapshot. + return app.BaseApp.OfferSnapshot(req) + } + + if app.upgradeHeightV2 == 0 { + app.Logger().Debug("v2 upgrade height not set, assuming app version 2") + app.mountKeysAndInit(v2) + return app.BaseApp.OfferSnapshot(req) + } + + if req.Snapshot.Height >= uint64(app.upgradeHeightV2) { + app.Logger().Debug("snapshot height is greater than or equal to upgrade height, assuming app version 2") + app.mountKeysAndInit(v2) + return app.BaseApp.OfferSnapshot(req) + } + + app.Logger().Debug("snapshot height is less than upgrade height, assuming app version 1") + app.mountKeysAndInit(v1) + return app.BaseApp.OfferSnapshot(req) +} diff --git a/app/app_test.go b/app/app_test.go index e1189e5599..5e99ec522c 100644 --- a/app/app_test.go +++ b/app/app_test.go @@ -6,7 +6,12 @@ import ( "github.com/celestiaorg/celestia-app/v3/app" "github.com/celestiaorg/celestia-app/v3/app/encoding" "github.com/celestiaorg/celestia-app/v3/x/minfee" + "github.com/cosmos/cosmos-sdk/baseapp" + "github.com/cosmos/cosmos-sdk/snapshots" + snapshottypes "github.com/cosmos/cosmos-sdk/snapshots/types" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/libs/log" tmdb "github.com/tendermint/tm-db" ) @@ -47,6 +52,45 @@ func TestNew(t *testing.T) { }) } +func TestOfferSnapshot(t *testing.T) { + logger := log.NewNopLogger() + db := tmdb.NewMemDB() + traceStore := &NoopWriter{} + invCheckPeriod := uint(1) + encodingConfig := encoding.MakeConfig(app.ModuleEncodingRegisters...) + upgradeHeight := int64(0) + appOptions := NoopAppOptions{} + snapshotOption := getSnapshotOption(t) + app := app.New(logger, db, traceStore, invCheckPeriod, encodingConfig, upgradeHeight, appOptions, snapshotOption) + + t.Run("should return ACCEPT", func(t *testing.T) { + request := abci.RequestOfferSnapshot{ + Snapshot: &abci.Snapshot{ + Height: 0x1b07ec, + Format: 0x2, + Chunks: 0x1, + Hash: []uint8{0xaf, 0xa5, 0xe, 0x16, 0x45, 0x4, 0x2e, 0x45, 0xd3, 0x49, 0xdf, 0x83, 0x2a, 0x57, 0x9d, 0x64, 0xc8, 0xad, 0xa5, 0xb, 0x65, 0x1b, 0x46, 0xd6, 0xc3, 0x85, 0x6, 0x51, 0xd7, 0x45, 0x8e, 0xb8}, + Metadata: []uint8{0xa, 0x20, 0xaf, 0xa5, 0xe, 0x16, 0x45, 0x4, 0x2e, 0x45, 0xd3, 0x49, 0xdf, 0x83, 0x2a, 0x57, 0x9d, 0x64, 0xc8, 0xad, 0xa5, 0xb, 0x65, 0x1b, 0x46, 0xd6, 0xc3, 0x85, 0x6, 0x51, 0xd7, 0x45, 0x8e, 0xb8}, + }, + AppHash: []byte("apphash"), + } + want := abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_ACCEPT} + got := app.OfferSnapshot(request) + assert.Equal(t, want, got) + }) +} + +func getSnapshotOption(t *testing.T) func(*baseapp.BaseApp) { + snapshotDir := t.TempDir() + snapshotDB, err := tmdb.NewDB("metadata", tmdb.GoLevelDBBackend, t.TempDir()) + require.NoError(t, err) + snapshotStore, err := snapshots.NewStore(snapshotDB, snapshotDir) + require.NoError(t, err) + interval := uint64(10) + keepRecent := uint32(10) + return baseapp.SetSnapshot(snapshotStore, snapshottypes.NewSnapshotOptions(interval, keepRecent)) +} + // NoopWriter is a no-op implementation of a writer. type NoopWriter struct{} diff --git a/scripts/arabica.sh b/scripts/arabica.sh new file mode 100644 index 0000000000..831c785245 --- /dev/null +++ b/scripts/arabica.sh @@ -0,0 +1,59 @@ +#!/bin/sh + +# This script starts a consensus node on Arabica and state syncs to the tip of +# the chain. + +# Stop script execution if an error is encountered +set -o errexit +# Stop script execution if an undefined variable is used +set -o nounset + +CHAIN_ID="arabica-11" +NODE_NAME="node-name" +SEEDS="827583022cc6ce65cf762115642258f937c954cd@validator-1.celestia-arabica-11.com:26656,74e42b39f512f844492ff09e30af23d54579b7bc@validator-2.celestia-arabica-11.com:26656,00d577159b2eb1f524ef9c37cb389c020a2c38d2@validator-3.celestia-arabica-11.com:26656,b2871b6dc2e18916d07264af0e87c456c2bba04f@validator-4.celestia-arabica-11.com:26656" +RPC="https://rpc.celestia-arabica-11.com:443" + +CELESTIA_APP_HOME="${HOME}/.celestia-app" +CELESTIA_APP_VERSION=$(celestia-appd version 2>&1) + +echo "celestia-app home: ${CELESTIA_APP_HOME}" +echo "celestia-app version: ${CELESTIA_APP_VERSION}" +echo "" + +# Ask the user for confirmation before deleting the existing celestia-app home +# directory. +read -p "Are you sure you want to delete: $CELESTIA_APP_HOME? [y/n] " response + +# Check the user's response +if [ "$response" != "y" ]; then + # Exit if the user did not respond with "y" + echo "You must delete $CELESTIA_APP_HOME to continue." + exit 1 +fi + +echo "Deleting $CELESTIA_APP_HOME..." +rm -r "$CELESTIA_APP_HOME" + +echo "Initializing config files..." +celestia-appd init ${NODE_NAME} --chain-id ${CHAIN_ID} > /dev/null 2>&1 # Hide output to reduce terminal noise + +echo "Settings seeds in config.toml..." +sed -i.bak -e "s/^seeds *=.*/seeds = \"$SEEDS\"/" $CELESTIA_APP_HOME/config/config.toml + +# LATEST_HEIGHT=$(curl -s $RPC/block | jq -r .result.block.header.height); +BLOCK_HEIGHT=1751700 +TRUST_HASH=$(curl -s "$RPC/block?height=$BLOCK_HEIGHT" | jq -r .result.block_id.hash) + +echo "Block height: $BLOCK_HEIGHT" +echo "Trust hash: $TRUST_HASH" +echo "Enabling state sync in config.toml..." +sed -i.bak -E "s|^(enable[[:space:]]+=[[:space:]]+).*$|\1true| ; \ +s|^(rpc_servers[[:space:]]+=[[:space:]]+).*$|\1\"$RPC,$RPC\"| ; \ +s|^(trust_height[[:space:]]+=[[:space:]]+).*$|\1$BLOCK_HEIGHT| ; \ +s|^(trust_hash[[:space:]]+=[[:space:]]+).*$|\1\"$TRUST_HASH\"|" $HOME/.celestia-app/config/config.toml + +echo "Downloading genesis file..." +celestia-appd download-genesis ${CHAIN_ID} + +echo "Starting celestia-appd..." +celestia-appd start --v2-upgrade-height 1751707 diff --git a/scripts/mocha.sh b/scripts/mocha.sh new file mode 100644 index 0000000000..d64921b2a8 --- /dev/null +++ b/scripts/mocha.sh @@ -0,0 +1,59 @@ +#!/bin/sh + +# This script starts a consensus node on Mocha and state syncs to the tip of the +# chain. + +# Stop script execution if an error is encountered +set -o errexit +# Stop script execution if an undefined variable is used +set -o nounset + +CHAIN_ID="mocha-4" +NODE_NAME="node-name" +SEEDS="ee9f90974f85c59d3861fc7f7edb10894f6ac3c8@seed-mocha.pops.one:26656,258f523c96efde50d5fe0a9faeea8a3e83be22ca@seed.mocha-4.celestia.aviaone.com:20279,5d0bf034d6e6a8b5ee31a2f42f753f1107b3a00e@celestia-testnet-seed.itrocket.net:11656,7da0fb48d6ef0823bc9770c0c8068dd7c89ed4ee@celest-test-seed.theamsolutions.info:443" +RPC="https://celestia-testnet-rpc.itrocket.net:443" + +CELESTIA_APP_HOME="${HOME}/.celestia-app" +CELESTIA_APP_VERSION=$(celestia-appd version 2>&1) + +echo "celestia-app home: ${CELESTIA_APP_HOME}" +echo "celestia-app version: ${CELESTIA_APP_VERSION}" +echo "" + +# Ask the user for confirmation before deleting the existing celestia-app home +# directory. +read -p "Are you sure you want to delete: $CELESTIA_APP_HOME? [y/n] " response + +# Check the user's response +if [ "$response" != "y" ]; then + # Exit if the user did not respond with "y" + echo "You must delete $CELESTIA_APP_HOME to continue." + exit 1 +fi + +echo "Deleting $CELESTIA_APP_HOME..." +rm -r "$CELESTIA_APP_HOME" + +echo "Initializing config files..." +celestia-appd init ${NODE_NAME} --chain-id ${CHAIN_ID} > /dev/null 2>&1 # Hide output to reduce terminal noise + +echo "Settings seeds in config.toml..." +sed -i.bak -e "s/^seeds *=.*/seeds = \"$SEEDS\"/" $CELESTIA_APP_HOME/config/config.toml + +LATEST_HEIGHT=$(curl -s $RPC/block | jq -r .result.block.header.height); +BLOCK_HEIGHT=$((LATEST_HEIGHT - 2000)); \ +TRUST_HASH=$(curl -s "$RPC/block?height=$BLOCK_HEIGHT" | jq -r .result.block_id.hash) + +echo "Block height: $BLOCK_HEIGHT" +echo "Trust hash: $TRUST_HASH" +echo "Enabling state sync in config.toml..." +sed -i.bak -E "s|^(enable[[:space:]]+=[[:space:]]+).*$|\1true| ; \ +s|^(rpc_servers[[:space:]]+=[[:space:]]+).*$|\1\"$RPC,$RPC\"| ; \ +s|^(trust_height[[:space:]]+=[[:space:]]+).*$|\1$BLOCK_HEIGHT| ; \ +s|^(trust_hash[[:space:]]+=[[:space:]]+).*$|\1\"$TRUST_HASH\"|" $HOME/.celestia-app/config/config.toml + +echo "Downloading genesis file..." +celestia-appd download-genesis ${CHAIN_ID} > /dev/null 2>&1 # Hide output to reduce terminal noise + +echo "Starting celestia-appd..." +celestia-appd start --v2-upgrade-height 2585031 From 13f30980ddd0c52bca17908a193dbb291eefb76a Mon Sep 17 00:00:00 2001 From: Rootul P Date: Thu, 22 Aug 2024 13:02:18 -0400 Subject: [PATCH 10/85] ci: use goreleaser-cross for goreleaser-check command (#3806) Closes https://github.com/celestiaorg/celestia-app/issues/3805 [goreleaser v2.2.0](https://github.com/goreleaser/goreleaser/releases/tag/v2.2.0) deprecated the `name_template` field in favor of `version_template`. Our goreleaser-check command used a GH action that was using goreleaser v2.2.0 so we fixed the deprecation in https://github.com/celestiaorg/celestia-app/pull/3797 but the make prebuilt-binary command uses goreleaser-cross which was using a previous version of goreleaser that fails for the new field. This PR updates the goreleaser-check command to use the goreleaser-cross Docker image so that the check command and prebuilt-binary command uses the same version. Note: we'll need to update `name_template` to `version_template` when we upgrade to Go 1.23.0 because that will use goreleaser-cross 1.23.0 which bumped to goreleaser 2.22.0. See https://github.com/goreleaser/goreleaser-cross/pull/77 --- .github/workflows/goreleaser.yml | 16 +++++++++++----- .goreleaser.yaml | 2 +- Makefile | 17 +++++++++++++++++ 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/.github/workflows/goreleaser.yml b/.github/workflows/goreleaser.yml index f25d738a65..dd265479e9 100644 --- a/.github/workflows/goreleaser.yml +++ b/.github/workflows/goreleaser.yml @@ -9,12 +9,18 @@ jobs: goreleaser-check: runs-on: ubuntu-latest steps: - - name: checkout - uses: actions/checkout@v4 - - uses: goreleaser/goreleaser-action@v6 + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - run: git fetch --force --tags + - uses: actions/setup-go@v5 with: - version: latest - args: check + go-version-file: 'go.mod' + - name: Create .release-env file + run: |- + echo 'GITHUB_TOKEN=${{secrets.GORELEASER_ACCESS_TOKEN}}' >> .release-env + - name: Check the .goreleaser.yaml config file + run: make goreleaser-check goreleaser: needs: goreleaser-check diff --git a/.goreleaser.yaml b/.goreleaser.yaml index c8ce5cff64..950143a3f1 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -108,7 +108,7 @@ archives: checksum: name_template: "checksums.txt" snapshot: - version_template: "{{ incpatch .Version }}-next" + name_template: "{{ incpatch .Version }}-next" changelog: sort: asc filters: diff --git a/Makefile b/Makefile index 8af2c10c00..d291c6d2b3 100644 --- a/Makefile +++ b/Makefile @@ -189,6 +189,23 @@ adr-gen: @curl -sSL https://raw.githubusercontent.com/celestiaorg/.github/main/adr-template.md > docs/architecture/adr-template.md .PHONY: adr-gen +## goreleaser-check: Check the .goreleaser.yaml config file. +goreleaser-check: + @if [ ! -f ".release-env" ]; then \ + echo "A .release-env file was not found but is required to create prebuilt binaries. This command is expected to be run in CI where a .release-env file exists. If you need to run this command locally to attach binaries to a release, you need to create a .release-env file with a Github token (classic) that has repo:public_repo scope."; \ + exit 1;\ + fi + docker run \ + --rm \ + -e CGO_ENABLED=1 \ + --env-file .release-env \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v `pwd`:/go/src/$(PACKAGE_NAME) \ + -w /go/src/$(PACKAGE_NAME) \ + ghcr.io/goreleaser/goreleaser-cross:${GOLANG_CROSS_VERSION} \ + check +.PHONY: goreleaser-check + ## prebuilt-binary: Create prebuilt binaries and attach them to GitHub release. Requires Docker. prebuilt-binary: @if [ ! -f ".release-env" ]; then \ From 359f993610f8076506dbe90e8822b547c48bbfe2 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Tue, 27 Aug 2024 12:46:38 -0400 Subject: [PATCH 11/85] docs: release guide (#3821) Closes https://github.com/celestiaorg/celestia-app/issues/3521 --------- Co-authored-by: Callum Waters --- docs/maintainers/release-guide.md | 38 +++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 docs/maintainers/release-guide.md diff --git a/docs/maintainers/release-guide.md b/docs/maintainers/release-guide.md new file mode 100644 index 0000000000..7260e95192 --- /dev/null +++ b/docs/maintainers/release-guide.md @@ -0,0 +1,38 @@ +# Release Guide + +The target audience for this guide is maintainers of this repo. In general, the release process is as follows: + +1. Create a release candidate +1. Test the release candidate + 1. If the release candidate is not satisfactory, go back to step 1 + 1. If the release candidate is satisfactory, create an official release + +## Release Candidate + +### Creating a release candidate + +1. Navigate to . +1. Choose a version tag based on [Semantic Versioning](https://semver.org/). Include the `-rc` suffix followed by the next integer. RCs start at 0. +1. Change the target branch to `v1.x` or `v2.x` based on the version you're releasing. +1. Click **Generate release notes**. +1. Toggle on the **Set as a pre-relase** checkbox. +1. **Publish release**. + +### After creating the release candidate + +1. Wait until CI passes on the release and verify that prebuilt binaries were attached to the release. +1. Create a PR to bump the celestia-app dependency in [celestia-node](https://github.com/celestiaorg/celestia-node). +1. [Optional] Start a testnet via auto-devops that uses the release candidate. Confirm it works. +1. [Optional] Use the release candidate to sync from genesis. Confirm it works. + +## Official Release + +Follow the [creating a release candidate](#creating-a-release-candidate) section with the following considerations: + +- The version tag should not include the `-rc` suffix. +- The release notes should contain an **Upgrade Notice** section with notable changes for node operators or library consumers. + +After creating the release: + +1. Wait until CI passes on the release and verify that prebuilt binaries were attached to the release. +1. Create a PR to bump the celestia-app dependency in [celestia-node](https://github.com/celestiaorg/celestia-node). From 2f74d9b84353b714295d3179b6df6c8315ffb813 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Tue, 27 Aug 2024 13:05:04 -0400 Subject: [PATCH 12/85] docs: release notes (#3794) Closes https://github.com/celestiaorg/celestia-app/issues/3791 --------- Co-authored-by: Sanaz Taheri <35961250+staheri14@users.noreply.github.com> Co-authored-by: Callum Waters --- docs/release-notes/pending-release.md | 11 +++++++++++ docs/release-notes/release-notes.md | 22 ++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 docs/release-notes/pending-release.md create mode 100644 docs/release-notes/release-notes.md diff --git a/docs/release-notes/pending-release.md b/docs/release-notes/pending-release.md new file mode 100644 index 0000000000..80c7c228eb --- /dev/null +++ b/docs/release-notes/pending-release.md @@ -0,0 +1,11 @@ +# Pending Nelease + +## v3.0.0 [Unreleased] + +### Node Operators + +- Consensus node operators should enable the BBR (Bottleneck Bandwidth and Round-trip propagation time) congestion control algorithm. See [#3774](https://github.com/celestiaorg/celestia-app/pull/3774). + +### Library Consumers + +- Namespace and share constants in the `appconsts` package were moved to [celestiaorg/go-square](https://github.com/celestiaorg/go-square). See [#3765](https://github.com/celestiaorg/celestia-app/pull/3765). diff --git a/docs/release-notes/release-notes.md b/docs/release-notes/release-notes.md new file mode 100644 index 0000000000..5cc49d1010 --- /dev/null +++ b/docs/release-notes/release-notes.md @@ -0,0 +1,22 @@ +# Release Notes + +This guide provides notes for major version releases. These notes may be helpful for users when upgrading from previous major versions. + +## [v2.0.0](https://github.com/celestiaorg/celestia-app/releases/tag/v2.0.0) + +### Node Operators + +If you are a consensus node operator, please follow the communication channels listed under [network upgrades](https://docs.celestia.org/nodes/participate#network-upgrades) to learn when this release is recommended for each network (e.g. Mocha, Mainnet Beta). + +Consensus node operators are expected to upgrade to this release _prior_ to the Lemongrass hardfork if they intend to continue participating in the network. The command used to start the [consensus node](https://docs.celestia.org/nodes/consensus-node#start-the-consensus-node) or [validator node](https://docs.celestia.org/nodes/validator-node#run-the-validator-node) will accept an additional `--v2-upgrade-height` flag. See [this table](https://docs.celestia.org/nodes/hardfork-process#lemongrass-hardfork) for upgrade heights for each network. + +Consensus node operators should enable the BBR (Bottleneck Bandwidth and Round-trip propagation time) congestion control algorithm. See [#3812](https://github.com/celestiaorg/celestia-app/pull/3812). + +### Library Consumers + +If you are a library consumer, a number of the Go APIs have changed since celestia-app v1.x.x. Some of the notable changes are: + +- Code pertaining to the original data square was extracted to [celestiaorg/go-square](https://github.com/celestiaorg/go-square). + - celestia-app v1.x had a shares package. celestia-app v2.x uses [go-square/shares](https://github.com/celestiaorg/go-square/tree/c8242f96a844956f8d1c60e5511104deed8bc361/shares) + - celestia-app v1.x had a blob.types package with `CreateCommitment` function. celestia-app v2.x uses `CreateCommitment` function from the [go-square/inclusion](https://github.com/celestiaorg/go-square/tree/c8242f96a844956f8d1c60e5511104deed8bc361/inclusion). +- celestia-app v1.x had a lot of functionality included in the signer. celestia-app v2.x splits a txClient from the signer. See [#3433](https://github.com/celestiaorg/celestia-app/pull/3433). From b9ff23d4484cdf3673b953a0e01800a79a620e46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nina=20/=20=E1=83=9C=E1=83=98=E1=83=9C=E1=83=90?= Date: Wed, 28 Aug 2024 17:41:07 +0200 Subject: [PATCH 13/85] chore: tx status forward port (#3816) ## Overview Forward-porting some work that was only addressed on v2.x branch --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- app/test/big_blob_test.go | 5 ++- app/test/std_sdk_test.go | 16 +++++--- pkg/user/tx_client.go | 66 +++++++++++++++++++++++--------- pkg/user/tx_client_test.go | 7 ++-- x/blobstream/integration_test.go | 16 +++++--- x/signal/legacy_test.go | 23 ++++++----- 6 files changed, 89 insertions(+), 44 deletions(-) diff --git a/app/test/big_blob_test.go b/app/test/big_blob_test.go index ecb257031f..cede1ae65d 100644 --- a/app/test/big_blob_test.go +++ b/app/test/big_blob_test.go @@ -82,8 +82,9 @@ func (s *BigBlobSuite) TestErrBlobsTooLarge() { defer cancel() res, err := txClient.SubmitPayForBlob(subCtx, []*share.Blob{tc.blob}, user.SetGasLimitAndGasPrice(1e9, appconsts.DefaultMinGasPrice)) require.Error(t, err) - require.NotNil(t, res) - require.Equal(t, tc.want, res.Code, err.Error()) + require.Nil(t, res) + code := err.(*user.BroadcastTxError).Code + require.Equal(t, tc.want, code, err.Error()) }) } } diff --git a/app/test/std_sdk_test.go b/app/test/std_sdk_test.go index 8d3e02aa3d..75eb932b3f 100644 --- a/app/test/std_sdk_test.go +++ b/app/test/std_sdk_test.go @@ -317,20 +317,26 @@ func (s *StandardSDKIntegrationTestSuite) TestStandardSDK() { // sign and submit the transactions for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + serviceClient := sdktx.NewServiceClient(s.cctx.GRPCClient) msgs, signer := tt.msgFunc() txClient, err := user.SetupTxClient(s.cctx.GoContext(), s.cctx.Keyring, s.cctx.GRPCClient, s.ecfg, user.WithDefaultAccount(signer)) require.NoError(t, err) res, err := txClient.SubmitTx(s.cctx.GoContext(), msgs, blobfactory.DefaultTxOpts()...) if tt.expectedCode != abci.CodeTypeOK { require.Error(t, err) + require.Nil(t, res) + txHash := err.(*user.ExecutionError).TxHash + code := err.(*user.ExecutionError).Code + getTxResp, err := serviceClient.GetTx(s.cctx.GoContext(), &sdktx.GetTxRequest{Hash: txHash}) + require.NoError(t, err) + assert.Equal(t, tt.expectedCode, code, getTxResp.TxResponse.RawLog) } else { require.NoError(t, err) + require.NotNil(t, res) + getTxResp, err := serviceClient.GetTx(s.cctx.GoContext(), &sdktx.GetTxRequest{Hash: res.TxHash}) + require.NoError(t, err) + assert.Equal(t, tt.expectedCode, res.Code, getTxResp.TxResponse.RawLog) } - serviceClient := sdktx.NewServiceClient(s.cctx.GRPCClient) - getTxResp, err := serviceClient.GetTx(s.cctx.GoContext(), &sdktx.GetTxRequest{Hash: res.TxHash}) - require.NoError(t, err) - require.NotNil(t, res) - assert.Equal(t, tt.expectedCode, res.Code, getTxResp.TxResponse.RawLog) }) } } diff --git a/pkg/user/tx_client.go b/pkg/user/tx_client.go index b2c5b750a3..a403a91991 100644 --- a/pkg/user/tx_client.go +++ b/pkg/user/tx_client.go @@ -22,6 +22,7 @@ import ( sdktx "github.com/cosmos/cosmos-sdk/types/tx" paramtypes "github.com/cosmos/cosmos-sdk/x/params/types/proposal" abci "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/rpc/core" "google.golang.org/grpc" "github.com/celestiaorg/celestia-app/v3/app" @@ -31,7 +32,6 @@ import ( "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" "github.com/celestiaorg/celestia-app/v3/x/blob/types" "github.com/celestiaorg/celestia-app/v3/x/minfee" - "github.com/tendermint/tendermint/rpc/core" ) const ( @@ -44,11 +44,36 @@ type Option func(client *TxClient) // TxResponse is a response from the chain after // a transaction has been submitted. type TxResponse struct { + // Height is the block height at which the transaction was included on-chain. Height int64 TxHash string Code uint32 } +// BroadcastTxError is an error that occurs when broadcasting a transaction. +type BroadcastTxError struct { + TxHash string + Code uint32 + // ErrorLog is the error output of the app's logger + ErrorLog string +} + +func (e *BroadcastTxError) Error() string { + return fmt.Sprintf("broadcast tx error: %s", e.ErrorLog) +} + +// ExecutionError is an error that occurs when a transaction gets executed. +type ExecutionError struct { + TxHash string + Code uint32 + // ErrorLog is the error output of the app's logger + ErrorLog string +} + +func (e *ExecutionError) Error() string { + return fmt.Sprintf("tx execution failed with code %d: %s", e.Code, e.ErrorLog) +} + // WithGasMultiplier is a functional option allows to configure the gas multiplier. func WithGasMultiplier(multiplier float64) Option { return func(c *TxClient) { @@ -213,7 +238,7 @@ func SetupTxClient( func (client *TxClient) SubmitPayForBlob(ctx context.Context, blobs []*share.Blob, opts ...TxOption) (*TxResponse, error) { resp, err := client.BroadcastPayForBlob(ctx, blobs, opts...) if err != nil { - return parseTxResponse(resp, fmt.Errorf("failed to broadcast pay for blob: %v", err)) + return nil, err } return client.ConfirmTx(ctx, resp.TxHash) @@ -224,7 +249,7 @@ func (client *TxClient) SubmitPayForBlob(ctx context.Context, blobs []*share.Blo func (client *TxClient) SubmitPayForBlobWithAccount(ctx context.Context, account string, blobs []*share.Blob, opts ...TxOption) (*TxResponse, error) { resp, err := client.BroadcastPayForBlobWithAccount(ctx, account, blobs, opts...) if err != nil { - return parseTxResponse(resp, fmt.Errorf("failed to broadcast pay for blob with account: %v", err)) + return nil, err } return client.ConfirmTx(ctx, resp.TxHash) @@ -268,7 +293,7 @@ func (client *TxClient) BroadcastPayForBlobWithAccount(ctx context.Context, acco func (client *TxClient) SubmitTx(ctx context.Context, msgs []sdktypes.Msg, opts ...TxOption) (*TxResponse, error) { resp, err := client.BroadcastTx(ctx, msgs, opts...) if err != nil { - return parseTxResponse(resp, fmt.Errorf("failed to broadcast tx: %v", err)) + return nil, err } return client.ConfirmTx(ctx, resp.TxHash) @@ -354,7 +379,12 @@ func (client *TxClient) broadcastTx(ctx context.Context, txBytes []byte, signer } return client.retryBroadcastingTx(ctx, txBytes) } - return resp.TxResponse, fmt.Errorf("tx failed with code %d: %s", resp.TxResponse.Code, resp.TxResponse.RawLog) + broadcastTxErr := &BroadcastTxError{ + TxHash: resp.TxResponse.TxHash, + Code: resp.TxResponse.Code, + ErrorLog: resp.TxResponse.RawLog, + } + return resp.TxResponse, broadcastTxErr } // after the transaction has been submitted, we can increment the @@ -435,16 +465,16 @@ func (client *TxClient) ConfirmTx(ctx context.Context, txHash string) (*TxRespon for { resp, err := txClient.TxStatus(ctx, &tx.TxStatusRequest{TxId: txHash}) if err != nil { - return &TxResponse{}, err + return nil, err } - if err == nil && resp != nil { + if resp != nil { switch resp.Status { case core.TxStatusPending: // Continue polling if the transaction is still pending select { case <-ctx.Done(): - return &TxResponse{}, ctx.Err() + return nil, ctx.Err() case <-pollTicker.C: continue } @@ -454,14 +484,19 @@ func (client *TxClient) ConfirmTx(ctx context.Context, txHash string) (*TxRespon TxHash: txHash, Code: resp.ExecutionCode, } - if resp.ExecutionCode != 0 { - return txResponse, fmt.Errorf("tx was committed but failed with code %d: %s", resp.ExecutionCode, resp.Error) + if resp.ExecutionCode != abci.CodeTypeOK { + executionErr := &ExecutionError{ + TxHash: txHash, + Code: resp.ExecutionCode, + ErrorLog: resp.Error, + } + return nil, executionErr } return txResponse, nil case core.TxStatusEvicted: - return &TxResponse{TxHash: txHash}, fmt.Errorf("tx: %s was evicted from the mempool", txHash) + return nil, fmt.Errorf("tx was evicted from the mempool") default: - return &TxResponse{}, fmt.Errorf("unknown tx: %s", txHash) + return nil, fmt.Errorf("unknown tx: %s", txHash) } } } @@ -569,13 +604,6 @@ func (client *TxClient) getAccountNameFromMsgs(msgs []sdktypes.Msg) (string, err return record.Name, nil } -func parseTxResponse(resp *sdktypes.TxResponse, err error) (*TxResponse, error) { - if resp != nil { - return &TxResponse{Code: resp.Code, TxHash: resp.TxHash}, err - } - return &TxResponse{}, err -} - // Signer exposes the tx clients underlying signer func (client *TxClient) Signer() *Signer { return client.signer diff --git a/pkg/user/tx_client_test.go b/pkg/user/tx_client_test.go index c416ba2d8e..1588c3a060 100644 --- a/pkg/user/tx_client_test.go +++ b/pkg/user/tx_client_test.go @@ -7,6 +7,7 @@ import ( sdk "github.com/cosmos/cosmos-sdk/types" sdktx "github.com/cosmos/cosmos-sdk/types/tx" + "github.com/cosmos/cosmos-sdk/x/authz" bank "github.com/cosmos/cosmos-sdk/x/bank/types" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" @@ -19,7 +20,6 @@ import ( "github.com/celestiaorg/celestia-app/v3/pkg/user" "github.com/celestiaorg/celestia-app/v3/test/util/blobfactory" "github.com/celestiaorg/celestia-app/v3/test/util/testnode" - "github.com/cosmos/cosmos-sdk/x/authz" ) func TestTxClientTestSuite(t *testing.T) { @@ -209,9 +209,10 @@ func (suite *TxClientTestSuite) TestConfirmTx() { resp, err := suite.txClient.BroadcastTx(suite.ctx.GoContext(), []sdk.Msg{msg}, fee, gas) require.NoError(t, err) require.NotNil(t, resp) - confirmTxResp, err := suite.txClient.ConfirmTx(suite.ctx.GoContext(), resp.TxHash) + _, err = suite.txClient.ConfirmTx(suite.ctx.GoContext(), resp.TxHash) require.Error(t, err) - require.NotEqual(t, abci.CodeTypeOK, confirmTxResp.Code) + code := err.(*user.ExecutionError).Code + require.NotEqual(t, abci.CodeTypeOK, code) }) } diff --git a/x/blobstream/integration_test.go b/x/blobstream/integration_test.go index c4a7b6d9a6..a8f19bf4b6 100644 --- a/x/blobstream/integration_test.go +++ b/x/blobstream/integration_test.go @@ -81,20 +81,26 @@ func (s *BlobstreamIntegrationSuite) TestBlobstream() { // sign and submit the transactions for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + serviceClient := sdktx.NewServiceClient(s.cctx.GRPCClient) msgs, _ := tt.msgFunc() txClient, err := user.SetupTxClient(s.cctx.GoContext(), s.cctx.Keyring, s.cctx.GRPCClient, s.ecfg) require.NoError(t, err) res, err := txClient.SubmitTx(s.cctx.GoContext(), msgs, blobfactory.DefaultTxOpts()...) if tt.expectedTxCode == abci.CodeTypeOK { require.NoError(t, err) + require.NotNil(t, res) + getTxResp, err := serviceClient.GetTx(s.cctx.GoContext(), &sdktx.GetTxRequest{Hash: res.TxHash}) + require.NoError(t, err) + require.Equal(t, tt.expectedTxCode, res.Code, getTxResp.TxResponse.RawLog) } else { require.Error(t, err) + require.Nil(t, res) + txHash := err.(*user.ExecutionError).TxHash + code := err.(*user.ExecutionError).Code + getTxResp, err := serviceClient.GetTx(s.cctx.GoContext(), &sdktx.GetTxRequest{Hash: txHash}) + require.NoError(t, err) + require.Equal(t, tt.expectedTxCode, code, getTxResp.TxResponse.RawLog) } - serviceClient := sdktx.NewServiceClient(s.cctx.GRPCClient) - getTxResp, err := serviceClient.GetTx(s.cctx.GoContext(), &sdktx.GetTxRequest{Hash: res.TxHash}) - require.NoError(t, err) - require.NotNil(t, res) - require.Equal(t, tt.expectedTxCode, res.Code, getTxResp.TxResponse.RawLog) }) } } diff --git a/x/signal/legacy_test.go b/x/signal/legacy_test.go index aa53038cec..21f2695aa0 100644 --- a/x/signal/legacy_test.go +++ b/x/signal/legacy_test.go @@ -8,6 +8,7 @@ import ( "github.com/celestiaorg/celestia-app/v3/app" "github.com/celestiaorg/celestia-app/v3/app/encoding" + "github.com/celestiaorg/celestia-app/v3/pkg/user" testutil "github.com/celestiaorg/celestia-app/v3/test/util" "github.com/celestiaorg/celestia-app/v3/test/util/blobfactory" "github.com/celestiaorg/celestia-app/v3/test/util/genesis" @@ -120,14 +121,14 @@ func (s *LegacyUpgradeTestSuite) TestLegacyGovUpgradeFailure() { require.NoError(t, err) // submit the transaction and wait a block for it to be included - signer, err := testnode.NewTxClientFromContext(s.cctx) + txClient, err := testnode.NewTxClientFromContext(s.cctx) require.NoError(t, err) subCtx, cancel := context.WithTimeout(s.cctx.GoContext(), time.Minute) defer cancel() - res, err := signer.SubmitTx(subCtx, []sdk.Msg{msg}, blobfactory.DefaultTxOpts()...) - require.Error(t, err) + _, err = txClient.SubmitTx(subCtx, []sdk.Msg{msg}, blobfactory.DefaultTxOpts()...) + code := err.(*user.BroadcastTxError).Code // As the type is not registered, the message will fail with unable to resolve type URL - require.EqualValues(t, 2, res.Code) + require.EqualValues(t, 2, code, err.Error()) } // TestNewGovUpgradeFailure verifies that a transaction with a @@ -149,14 +150,15 @@ func (s *LegacyUpgradeTestSuite) TestNewGovUpgradeFailure() { require.NoError(t, err) // submit the transaction and wait a block for it to be included - signer, err := testnode.NewTxClientFromContext(s.cctx) + txClient, err := testnode.NewTxClientFromContext(s.cctx) require.NoError(t, err) subCtx, cancel := context.WithTimeout(s.cctx.GoContext(), time.Minute) defer cancel() - res, err := signer.SubmitTx(subCtx, []sdk.Msg{msg}, blobfactory.DefaultTxOpts()...) - require.Error(t, err) + _, err = txClient.SubmitTx(subCtx, []sdk.Msg{msg}, blobfactory.DefaultTxOpts()...) // As the type is not registered, the message will fail with unable to resolve type URL - require.EqualValues(t, 2, res.Code) + require.Error(t, err) + code := err.(*user.BroadcastTxError).Code + require.EqualValues(t, 2, code, err.Error()) } func (s *LegacyUpgradeTestSuite) TestIBCUpgradeFailure() { @@ -182,9 +184,10 @@ func (s *LegacyUpgradeTestSuite) TestIBCUpgradeFailure() { require.NoError(t, err) subCtx, cancel := context.WithTimeout(s.cctx.GoContext(), time.Minute) defer cancel() - res, err := txClient.SubmitTx(subCtx, []sdk.Msg{msg}, blobfactory.DefaultTxOpts()...) + _, err = txClient.SubmitTx(subCtx, []sdk.Msg{msg}, blobfactory.DefaultTxOpts()...) require.Error(t, err) - require.EqualValues(t, 9, res.Code) // we're only submitting the tx, so we expect everything to work + code := err.(*user.ExecutionError).Code + require.EqualValues(t, 9, code) // we're only submitting the tx, so we expect everything to work assert.Contains(t, err.Error(), "ibc upgrade proposal not supported") } From 8bfa2513cc829f6c45ffb7563525e9b6fc8b1223 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Thu, 29 Aug 2024 05:44:40 -0400 Subject: [PATCH 14/85] docs: guide to modify param via governance (#3832) I created a doc to prepare for https://github.com/celestiaorg/celestia-app/issues/3827 --- docs/maintainers/modify-param.md | 48 ++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 docs/maintainers/modify-param.md diff --git a/docs/maintainers/modify-param.md b/docs/maintainers/modify-param.md new file mode 100644 index 0000000000..47fff0d757 --- /dev/null +++ b/docs/maintainers/modify-param.md @@ -0,0 +1,48 @@ +# Modify Param + +This doc will guide you through the process of modifying a parameter via governance. + +## Prerequisites + +```shell +# Verify the current parameter value +$ celestia-appd query params subspace icahost AllowMessages +key: AllowMessages +subspace: icahost +value: '["*"]' +``` + +## Steps + +```shell +# Create a proposal.json file +echo '{"title": "Modify ICA host allow messages", "description": "Modify ICA host allow messages", "changes": [{"subspace": "icahost", "key": "AllowMessages", "value": ["/ibc.applications.transfer.v1.MsgTransfer","/cosmos.bank.v1beta1.MsgSend","/cosmos.staking.v1beta1.MsgDelegate","/cosmos.staking.v1beta1.MsgBeginRedelegate","/cosmos.staking.v1beta1.MsgUndelegate","/cosmos.staking.v1beta1.MsgCancelUnbondingDelegation","/cosmos.distribution.v1beta1.MsgSetWithdrawAddress","/cosmos.distribution.v1beta1.MsgWithdrawDelegatorReward","/cosmos.distribution.v1beta1.MsgFundCommunityPool","/cosmos.gov.v1.MsgVote","/cosmos.feegrant.v1beta1.MsgGrantAllowance","/cosmos.feegrant.v1beta1.MsgRevokeAllowance"]}], "deposit": "10000000000utia"}' > proposal.json + +# Export a variable for the key that will be used to submit the proposal +export FROM="validator" +export FEES="210000utia" +export GAS="auto" +export GAS_ADJUSTMENT="1.5" + +# Submit the proposal +celestia-appd tx gov submit-legacy-proposal param-change proposal.json --from $FROM --fees $FEES --gas $GAS --gas-adjustment $GAS_ADJUSTMENT --yes + +# Query the proposals +celestia-appd query gov proposals --output json | jq . + +# Export a variable for the relevant proposal ID based on the output from the previous command +export PROPOSAL_ID=1 + +# Vote yes on the proposal +celestia-appd tx gov vote $PROPOSAL_ID yes --from $FROM --fees $FEES --gas $GAS --gas-adjustment $GAS_ADJUSTMENT --yes +``` + +## After the proposal passes + +```shell +# Verify the parameter value changed +$ celestia-appd query params subspace icahost AllowMessages +key: AllowMessages +subspace: icahost +value: '["/ibc.applications.transfer.v1.MsgTransfer","/cosmos.bank.v1beta1.MsgSend","/cosmos.staking.v1beta1.MsgDelegate","/cosmos.staking.v1beta1.MsgBeginRedelegate","/cosmos.staking.v1beta1.MsgUndelegate","/cosmos.staking.v1beta1.MsgCancelUnbondingDelegation","/cosmos.distribution.v1beta1.MsgSetWithdrawAddress","/cosmos.distribution.v1beta1.MsgWithdrawDelegatorReward","/cosmos.distribution.v1beta1.MsgFundCommunityPool","/cosmos.gov.v1.MsgVote","/cosmos.feegrant.v1beta1.MsgGrantAllowance","/cosmos.feegrant.v1beta1.MsgRevokeAllowance"]' +``` From 62aae1fc05d505446c9c90b95fb5049e7053f4e9 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Thu, 29 Aug 2024 05:45:23 -0400 Subject: [PATCH 15/85] chore(scripts): remove hardcoded height for arabica.sh (#3831) --- scripts/arabica.sh | 5 +++-- scripts/mocha.sh | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) mode change 100644 => 100755 scripts/arabica.sh mode change 100644 => 100755 scripts/mocha.sh diff --git a/scripts/arabica.sh b/scripts/arabica.sh old mode 100644 new mode 100755 index 831c785245..fe88e26c89 --- a/scripts/arabica.sh +++ b/scripts/arabica.sh @@ -40,10 +40,11 @@ celestia-appd init ${NODE_NAME} --chain-id ${CHAIN_ID} > /dev/null 2>&1 # Hide o echo "Settings seeds in config.toml..." sed -i.bak -e "s/^seeds *=.*/seeds = \"$SEEDS\"/" $CELESTIA_APP_HOME/config/config.toml -# LATEST_HEIGHT=$(curl -s $RPC/block | jq -r .result.block.header.height); -BLOCK_HEIGHT=1751700 +LATEST_HEIGHT=$(curl -s $RPC/block | jq -r .result.block.header.height); +BLOCK_HEIGHT=$((LATEST_HEIGHT - 2000)); \ TRUST_HASH=$(curl -s "$RPC/block?height=$BLOCK_HEIGHT" | jq -r .result.block_id.hash) +echo "Latest height: $LATEST_HEIGHT" echo "Block height: $BLOCK_HEIGHT" echo "Trust hash: $TRUST_HASH" echo "Enabling state sync in config.toml..." diff --git a/scripts/mocha.sh b/scripts/mocha.sh old mode 100644 new mode 100755 index d64921b2a8..5447eb9e3b --- a/scripts/mocha.sh +++ b/scripts/mocha.sh @@ -44,6 +44,7 @@ LATEST_HEIGHT=$(curl -s $RPC/block | jq -r .result.block.header.height); BLOCK_HEIGHT=$((LATEST_HEIGHT - 2000)); \ TRUST_HASH=$(curl -s "$RPC/block?height=$BLOCK_HEIGHT" | jq -r .result.block_id.hash) +echo "Latest height: $LATEST_HEIGHT" echo "Block height: $BLOCK_HEIGHT" echo "Trust hash: $TRUST_HASH" echo "Enabling state sync in config.toml..." From 62b6c5746e3957c4114391b70d593bad17d485f3 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Fri, 30 Aug 2024 10:05:27 -0400 Subject: [PATCH 16/85] docs(specs): remove root level specification page (#3822) Closes https://github.com/celestiaorg/celestia-app/issues/3751 --- specs/src/SUMMARY.md | 45 +- specs/src/ante_handler.md | 13 + specs/src/ante_handler_v1.md | 23 + specs/src/ante_handler_v2.md | 25 + specs/src/block_proposer.md | 24 + specs/src/block_validity_rules.md | 66 ++ specs/src/cat_pool.md | 100 +++ specs/src/consensus.md | 717 +++++++++++++++++ specs/src/data_square_layout.md | 62 ++ specs/src/data_structures.md | 417 ++++++++++ .../figures/blob_share_commitment.svg | 0 .../figures/block_data_structures.dot | 0 .../figures/block_data_structures.svg | 0 specs/src/{specs => }/figures/data_root.svg | 0 .../gas_consumption/100kib_pfb_trace.png | Bin .../msg_create_validator_trace.png | Bin .../gas_consumption/msg_send_trace.png | Bin .../pfb_with_two_single_share_blobs_trace.png | Bin .../single_share_pfb_trace.png | Bin specs/src/{specs => }/figures/namespace.dot | 0 specs/src/{specs => }/figures/namespace.svg | 0 specs/src/{specs => }/figures/rs2d.svg | 0 specs/src/{specs => }/figures/rs2d_extend.svg | 0 .../{specs => }/figures/rs2d_extending.svg | 0 .../figures/rs2d_originaldata_blob.svg | 0 .../figures/rs2d_originaldata_reserved.svg | 0 .../{specs => }/figures/rs2d_quadrants.svg | 0 specs/src/{specs => }/figures/rs2d_row.svg | 0 .../figures/share_continuation.dot | 0 .../figures/share_continuation.svg | 0 specs/src/{specs => }/figures/share_start.dot | 0 specs/src/{specs => }/figures/share_start.svg | 0 .../src/{specs => }/figures/square_layout.svg | 0 .../src/{specs => }/figures/subtree_width.svg | 0 .../transaction_share_continuation.dot | 0 .../transaction_share_continuation.svg | 0 .../figures/transaction_share_start.dot | 0 .../figures/transaction_share_start.svg | 0 specs/src/fraud_proofs.md | 25 + specs/src/multisig.md | 15 + specs/src/namespace.md | 115 +++ specs/src/networking.md | 111 +++ specs/src/parameters.md | 6 + specs/src/parameters_v1.md | 67 ++ specs/src/parameters_v2.md | 71 ++ specs/src/{specs => }/proto/consensus.proto | 0 specs/src/{specs => }/proto/types.proto | 0 specs/src/{specs => }/proto/wire.proto | 0 specs/src/public_key_cryptography.md | 68 ++ specs/src/resource_pricing.md | 254 +++++++ specs/src/shares.md | 121 +++ specs/src/specs/ante_handler.md | 12 - specs/src/specs/ante_handler_v1.md | 22 - specs/src/specs/ante_handler_v2.md | 24 - specs/src/specs/block_proposer.md | 25 +- specs/src/specs/block_validity_rules.md | 65 -- specs/src/specs/cat_pool.md | 101 +-- specs/src/specs/consensus.md | 718 +----------------- specs/src/specs/data_square_layout.md | 61 -- specs/src/specs/data_structures.md | 416 ---------- specs/src/specs/fraud_proofs.md | 24 - specs/src/specs/index.md | 18 - specs/src/specs/multisig.md | 14 - specs/src/specs/namespace.md | 114 --- specs/src/specs/networking.md | 110 --- specs/src/specs/parameters.md | 5 - specs/src/specs/parameters_v1.md | 66 -- specs/src/specs/parameters_v2.md | 70 -- specs/src/specs/public_key_cryptography.md | 67 -- specs/src/specs/resource_pricing.md | 253 ------ specs/src/specs/shares.md | 120 --- specs/src/specs/state_machine_modules.md | 5 - specs/src/specs/state_machine_modules_v1.md | 29 - specs/src/specs/state_machine_modules_v2.md | 32 - specs/src/state_machine_modules.md | 6 + specs/src/state_machine_modules_v1.md | 30 + specs/src/state_machine_modules_v2.md | 33 + x/blob/README.md | 16 +- x/blobstream/overview.md | 2 +- 79 files changed, 2401 insertions(+), 2402 deletions(-) create mode 100644 specs/src/ante_handler.md create mode 100644 specs/src/ante_handler_v1.md create mode 100644 specs/src/ante_handler_v2.md create mode 100644 specs/src/block_proposer.md create mode 100644 specs/src/block_validity_rules.md create mode 100644 specs/src/cat_pool.md create mode 100644 specs/src/consensus.md create mode 100644 specs/src/data_square_layout.md create mode 100644 specs/src/data_structures.md rename specs/src/{specs => }/figures/blob_share_commitment.svg (100%) rename specs/src/{specs => }/figures/block_data_structures.dot (100%) rename specs/src/{specs => }/figures/block_data_structures.svg (100%) rename specs/src/{specs => }/figures/data_root.svg (100%) rename specs/src/{specs => }/figures/gas_consumption/100kib_pfb_trace.png (100%) rename specs/src/{specs => }/figures/gas_consumption/msg_create_validator_trace.png (100%) rename specs/src/{specs => }/figures/gas_consumption/msg_send_trace.png (100%) rename specs/src/{specs => }/figures/gas_consumption/pfb_with_two_single_share_blobs_trace.png (100%) rename specs/src/{specs => }/figures/gas_consumption/single_share_pfb_trace.png (100%) rename specs/src/{specs => }/figures/namespace.dot (100%) rename specs/src/{specs => }/figures/namespace.svg (100%) rename specs/src/{specs => }/figures/rs2d.svg (100%) rename specs/src/{specs => }/figures/rs2d_extend.svg (100%) rename specs/src/{specs => }/figures/rs2d_extending.svg (100%) rename specs/src/{specs => }/figures/rs2d_originaldata_blob.svg (100%) rename specs/src/{specs => }/figures/rs2d_originaldata_reserved.svg (100%) rename specs/src/{specs => }/figures/rs2d_quadrants.svg (100%) rename specs/src/{specs => }/figures/rs2d_row.svg (100%) rename specs/src/{specs => }/figures/share_continuation.dot (100%) rename specs/src/{specs => }/figures/share_continuation.svg (100%) rename specs/src/{specs => }/figures/share_start.dot (100%) rename specs/src/{specs => }/figures/share_start.svg (100%) rename specs/src/{specs => }/figures/square_layout.svg (100%) rename specs/src/{specs => }/figures/subtree_width.svg (100%) rename specs/src/{specs => }/figures/transaction_share_continuation.dot (100%) rename specs/src/{specs => }/figures/transaction_share_continuation.svg (100%) rename specs/src/{specs => }/figures/transaction_share_start.dot (100%) rename specs/src/{specs => }/figures/transaction_share_start.svg (100%) create mode 100644 specs/src/fraud_proofs.md create mode 100644 specs/src/multisig.md create mode 100644 specs/src/namespace.md create mode 100644 specs/src/networking.md create mode 100644 specs/src/parameters.md create mode 100644 specs/src/parameters_v1.md create mode 100644 specs/src/parameters_v2.md rename specs/src/{specs => }/proto/consensus.proto (100%) rename specs/src/{specs => }/proto/types.proto (100%) rename specs/src/{specs => }/proto/wire.proto (100%) create mode 100644 specs/src/public_key_cryptography.md create mode 100644 specs/src/resource_pricing.md create mode 100644 specs/src/shares.md delete mode 100644 specs/src/specs/index.md create mode 100644 specs/src/state_machine_modules.md create mode 100644 specs/src/state_machine_modules_v1.md create mode 100644 specs/src/state_machine_modules_v2.md diff --git a/specs/src/SUMMARY.md b/specs/src/SUMMARY.md index 9f27074a92..f5aa23cec1 100644 --- a/specs/src/SUMMARY.md +++ b/specs/src/SUMMARY.md @@ -2,26 +2,25 @@ Celestia App Specifications -- [Specification](./specs/index.md) - - [Data Structures](./specs/data_structures.md) - - [Namespace](./specs/namespace.md) - - [Shares](./specs/shares.md) - - [Consensus](./specs/consensus.md) - - [CAT Pool](./specs/cat_pool.md) - - [Block Proposer](./specs/block_proposer.md) - - [Block Validity Rules](./specs/block_validity_rules.md) - - [AnteHandler](./specs/ante_handler.md) - - [AnteHandler v1](./specs/ante_handler_v1.md) - - [AnteHandler v2](./specs/ante_handler_v2.md) - - [Fraud Proofs](./specs/fraud_proofs.md) - - [Networking](./specs/networking.md) - - [Public-Key Cryptography](./specs/public_key_cryptography.md) - - [Data Square Layout](./specs/data_square_layout.md) - - [Resource Pricing](./specs/resource_pricing.md) - - [Multisig](./specs/multisig.md) -- [State Machine Modules](./specs/state_machine_modules.md) - - [State Machine Modules v1](./specs/state_machine_modules_v1.md) - - [State Machine Modules v2](./specs/state_machine_modules_v2.md) -- [Parameters](./specs/parameters.md) - - [Parameters v1](./specs/parameters_v1.md) - - [Parameters v2](./specs/parameters_v2.md) +- [Data Structures](./data_structures.md) +- [Namespace](./namespace.md) +- [Shares](./shares.md) +- [Consensus](./consensus.md) + - [CAT Pool](./cat_pool.md) +- [Block Proposer](./block_proposer.md) +- [Block Validity Rules](./block_validity_rules.md) +- [AnteHandler](./ante_handler.md) + - [AnteHandler v1](./ante_handler_v1.md) + - [AnteHandler v2](./ante_handler_v2.md) +- [Fraud Proofs](./fraud_proofs.md) +- [Networking](./networking.md) +- [Public-Key Cryptography](./public_key_cryptography.md) +- [Data Square Layout](./data_square_layout.md) +- [Resource Pricing](./resource_pricing.md) +- [Multisig](./multisig.md) +- [State Machine Modules](./state_machine_modules.md) + - [State Machine Modules v1](./state_machine_modules_v1.md) + - [State Machine Modules v2](./state_machine_modules_v2.md) +- [Parameters](./parameters.md) + - [Parameters v1](./parameters_v1.md) + - [Parameters v2](./parameters_v2.md) diff --git a/specs/src/ante_handler.md b/specs/src/ante_handler.md new file mode 100644 index 0000000000..2ebe759a79 --- /dev/null +++ b/specs/src/ante_handler.md @@ -0,0 +1,13 @@ +# AnteHandler + +Celestia makes use of a Cosmos SDK [AnteHandler](https://github.com/cosmos/cosmos-sdk/blob/v0.46.15/x/auth/spec/03_antehandlers.md) in order to reject decodable sdk.Txs that do not meet certain criteria. The AnteHandler is invoked at multiple times during the transaction lifecycle: + +1. `CheckTx` prior to the transaction entering the mempool +1. `PrepareProposal` when the block proposer includes the transaction in a block proposal +1. `ProcessProposal` when validators validate the transaction in a block proposal +1. `DeliverTx` when full nodes execute the transaction in a decided block + +The AnteHandler is defined in `app/ante/ante.go`. The app version impacts AnteHandler behavior. See: + +- [AnteHandler v1](./ante_handler_v1.md) +- [AnteHandler v2](./ante_handler_v2.md) diff --git a/specs/src/ante_handler_v1.md b/specs/src/ante_handler_v1.md new file mode 100644 index 0000000000..52109f67f8 --- /dev/null +++ b/specs/src/ante_handler_v1.md @@ -0,0 +1,23 @@ +# AnteHandler v1 + +The AnteHandler chains together several decorators to ensure the following criteria are met for app version 1: + +- The tx does not contain any [extension options](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L119-L122). +- The tx passes `ValidateBasic()`. +- The tx's [timeout_height](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L115-L117) has not been reached if one is specified. +- The tx's [memo](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L110-L113) is <= the max memo characters where [`MaxMemoCharacters = 256`](). +- The tx's [gas_limit](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L211-L213) is > the gas consumed based on the tx's size where [`TxSizeCostPerByte = 10`](https://github.com/cosmos/cosmos-sdk/blob/a429238fc267da88a8548bfebe0ba7fb28b82a13/x/auth/README.md?plain=1#L232). +- The tx's feepayer has enough funds to pay fees for the tx. The tx's feepayer is the feegranter (if specified) or the tx's first signer. Note the [feegrant](https://github.com/cosmos/cosmos-sdk/blob/v0.46.15/x/feegrant/README.md) module is enabled. +- The tx's count of signatures <= the max number of signatures. The max number of signatures is [`TxSigLimit = 7`](https://github.com/cosmos/cosmos-sdk/blob/a429238fc267da88a8548bfebe0ba7fb28b82a13/x/auth/README.md?plain=1#L231). +- The tx's [gas_limit](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L211-L213) is > the gas consumed based on the tx's signatures. +- The tx's [signatures](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/types/tx/signing/signature.go#L10-L26) are valid. For each signature, ensure that the signature's sequence number (a.k.a nonce) matches the account sequence number of the signer. +- The tx's [gas_limit](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L211-L213) is > the gas consumed based on the blob size(s). Since blobs are charged based on the number of shares they occupy, the gas consumed is calculated as follows: `gasToConsume = sharesNeeded(blob) * bytesPerShare * gasPerBlobByte`. Where `bytesPerShare` is a global constant (an alias for [`ShareSize = 512`](https://github.com/celestiaorg/celestia-app/blob/c90e61d5a2d0c0bd0e123df4ab416f6f0d141b7f/pkg/appconsts/global_consts.go#L27-L28)) and `gasPerBlobByte` is a governance parameter that can be modified (the [`DefaultGasPerBlobByte = 8`](https://github.com/celestiaorg/celestia-app/blob/c90e61d5a2d0c0bd0e123df4ab416f6f0d141b7f/pkg/appconsts/initial_consts.go#L16-L18)). +- The tx's total blob size is <= the max blob size. The max blob size is derived from the maximum valid square size. The max valid square size is the minimum of: `GovMaxSquareSize` and `SquareSizeUpperBound`. +- The tx does not contain a message of type [MsgSubmitProposal](https://github.com/cosmos/cosmos-sdk/blob/d6d929843bbd331b885467475bcb3050788e30ca/proto/cosmos/gov/v1/tx.proto#L33-L43) with zero proposal messages. +- The tx is not an IBC packet or update message that has already been processed. + +In addition to the above criteria, the AnteHandler also has a number of side-effects: + +- Tx fees are deducted from the tx's feepayer and added to the fee collector module account. +- Tx priority is calculated based on the smallest denomination of gas price in the tx and set in context. +- The nonce of all tx signers is incremented by 1. diff --git a/specs/src/ante_handler_v2.md b/specs/src/ante_handler_v2.md new file mode 100644 index 0000000000..a752654cd3 --- /dev/null +++ b/specs/src/ante_handler_v2.md @@ -0,0 +1,25 @@ +# AnteHandler v2 + +The AnteHandler chains together several decorators to ensure the following criteria are met for app version 2: + +- The tx does not contain any messages that are unsupported by the current app version. See `MsgVersioningGateKeeper`. +- The tx does not contain any [extension options](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L119-L122). +- The tx passes `ValidateBasic()`. +- The tx's [timeout_height](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L115-L117) has not been reached if one is specified. +- The tx's [memo](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L110-L113) is <= the max memo characters where [`MaxMemoCharacters = 256`](). +- The tx's [gas_limit](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L211-L213) is > the gas consumed based on the tx's size where [`TxSizeCostPerByte = 10`](https://github.com/cosmos/cosmos-sdk/blob/a429238fc267da88a8548bfebe0ba7fb28b82a13/x/auth/README.md?plain=1#L232). +- The tx's feepayer has enough funds to pay fees for the tx. The tx's feepayer is the feegranter (if specified) or the tx's first signer. Note the [feegrant](https://github.com/cosmos/cosmos-sdk/blob/v0.46.15/x/feegrant/README.md) module is enabled. +- The tx's gas price is >= the network minimum gas price where [`NetworkMinGasPrice = 0.000001` utia](https://github.com/celestiaorg/celestia-app/blob/8caa5807df8d15477554eba953bd056ae72d4503/pkg/appconsts/v2/app_consts.go#L9). +- The tx's count of signatures <= the max number of signatures. The max number of signatures is [`TxSigLimit = 7`](https://github.com/cosmos/cosmos-sdk/blob/a429238fc267da88a8548bfebe0ba7fb28b82a13/x/auth/README.md?plain=1#L231). +- The tx's [gas_limit](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L211-L213) is > the gas consumed based on the tx's signatures. +- The tx's [signatures](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/types/tx/signing/signature.go#L10-L26) are valid. For each signature, ensure that the signature's sequence number (a.k.a nonce) matches the account sequence number of the signer. +- The tx's [gas_limit](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L211-L213) is > the gas consumed based on the blob size(s). Since blobs are charged based on the number of shares they occupy, the gas consumed is calculated as follows: `gasToConsume = sharesNeeded(blob) * bytesPerShare * gasPerBlobByte`. Where `bytesPerShare` is a global constant (an alias for [`ShareSize = 512`](https://github.com/celestiaorg/celestia-app/blob/c90e61d5a2d0c0bd0e123df4ab416f6f0d141b7f/pkg/appconsts/global_consts.go#L27-L28)) and `gasPerBlobByte` is a governance parameter that can be modified (the [`DefaultGasPerBlobByte = 8`](https://github.com/celestiaorg/celestia-app/blob/c90e61d5a2d0c0bd0e123df4ab416f6f0d141b7f/pkg/appconsts/initial_consts.go#L16-L18)). +- The tx's total blob share count is <= the max blob share count. The max blob share count is derived from the maximum valid square size. The max valid square size is the minimum of: `GovMaxSquareSize` and `SquareSizeUpperBound`. +- The tx does not contain a message of type [MsgSubmitProposal](https://github.com/cosmos/cosmos-sdk/blob/d6d929843bbd331b885467475bcb3050788e30ca/proto/cosmos/gov/v1/tx.proto#L33-L43) with zero proposal messages. +- The tx is not an IBC packet or update message that has already been processed. + +In addition to the above criteria, the AnteHandler also has a number of side-effects: + +- Tx fees are deducted from the tx's feepayer and added to the fee collector module account. +- Tx priority is calculated based on the smallest denomination of gas price in the tx and set in context. +- The nonce of all tx signers is incremented by 1. diff --git a/specs/src/block_proposer.md b/specs/src/block_proposer.md new file mode 100644 index 0000000000..213449bbd9 --- /dev/null +++ b/specs/src/block_proposer.md @@ -0,0 +1,24 @@ +# Honest Block Proposer + + + +This document describes the tasks of an honest block proposer to assemble a new block. Performing these actions is not enforced by the [consensus rules](./consensus.md), so long as a valid block is produced. + +## Deciding on a Block Size + +Before [arranging available data into shares](./data_structures.md#arranging-available-data-into-shares), the size of the original data's square must be determined. + +There are two restrictions on the original data's square size: + +1. It must be at most [`AVAILABLE_DATA_ORIGINAL_SQUARE_MAX`](./consensus.md#constants). +1. It must be a power of 2. + +With these restrictions in mind, the block proposer performs the following actions: + +1. Collect as many transactions and blobs from the mempool as possible, such that the total number of shares is at most [`AVAILABLE_DATA_ORIGINAL_SQUARE_MAX`](./consensus.md#constants). +1. Compute the smallest square size that is a power of 2 that can fit the number of shares. +1. Attempt to lay out the collected transactions and blobs in the current square. + 1. If the square is too small to fit all transactions and blobs (which may happen [due to needing to insert padding between blobs](./data_square_layout.md)) and the square size is smaller than [`AVAILABLE_DATA_ORIGINAL_SQUARE_MAX`](./consensus.md#constants), double the size of the square and repeat the above step. + 1. If the square is too small to fit all transactions and blobs (which may happen [due to needing to insert padding between blobs](./data_square_layout.md)) and the square size is at [`AVAILABLE_DATA_ORIGINAL_SQUARE_MAX`](./consensus.md#constants), drop the transactions and blobs until the data fits within the square. + +Note: the maximum padding shares between blobs should be at most twice the number of blob shares. Doubling the square size (i.e. quadrupling the number of shares in the square) should thus only have to happen at most once. diff --git a/specs/src/block_validity_rules.md b/specs/src/block_validity_rules.md new file mode 100644 index 0000000000..b7ae2f2df4 --- /dev/null +++ b/specs/src/block_validity_rules.md @@ -0,0 +1,66 @@ +# Block Validity Rules + +## Introduction + +Unlike most blockchains, Celestia derives most of its functionality from +stateless commitments to data rather than stateful transitions. This means that +the protocol relies heavily on block validity rules. Notably, resource +constrained light clients must be able to detect when a subset of these validity +rules have not been followed in order to avoid making an honest majority +assumption on the consensus network. This has a significant impact on their +design. More information on how light clients can check the invalidity of a +block can be found in the [Fraud Proofs](./fraud_proofs.md) spec. + +> **Note** Celestia relies on CometBFT (formerly tendermint) for consensus, +> meaning that it has single slot finality and is fork-free. Therefore, in order +> to ensure that an invalid block is never committed to, each validator must +> check that each block follows all validity rules before voting. If over two +> thirds of the voting power colludes to break a validity rule, then fraud +> proofs are created for light clients. After light clients verify fraud proofs, +> they halt. + +## Validity Rules + +Before any Celestia specific validation is performed, all CometBFT [block +validation +rules](https://github.com/cometbft/cometbft/blob/v0.34.28/spec/core/data_structures.md#block) +must be followed. + +Notably, this includes verifying data availability. Consensus nodes verify data +availability by simply downloading the entire block. + +> **Note** Light clients only sample a fraction of the block. More details on +> how sampling actually works can be found in the seminal ["Fraud and Data +> Availability Proofs: Maximising Light Client Security and Scaling Blockchains +> with Dishonest Majorities"](https://arxiv.org/abs/1809.09044) and in the +> [`celestia-node`](https://github.com/celestiaorg/celestia-node) repo. + +Celestia specific validity rules can be categorized into multiple groups: + +### Block Rules + +1. In `Block.Data.Txs`, all `BlobTx` transactions must be ordered after non-`BlobTx` transactions. + +### Transaction Validity Rules + +#### App Version 1 + +There is no validity rule that transactions must be decodable so the following rules only apply to transactions that are decodable. + +1. Decodable transactions must pass all [AnteHandler](./ante_handler.md) checks. +1. Decodable non-`BlobTx` transactions must not contain a `MsgPayForBlobs` message. +1. Decodable `BlobTx` transactions must be valid according to the [BlobTx validity rules](../../x/blob/README.md#validity-rules). + +#### App Version 2 + +1. All transactions must be decodable. +1. All transactions must pass all [AnteHandler](./ante_handler.md) checks. +1. Non-`BlobTx` transactions must not contain a `MsgPayForBlobs` message. +1. `BlobTx` transactions must be valid according to the [BlobTx validity rules](../../x/blob/README.md#validity-rules). + +### Data Root Construction + +The data root must be calculated from a correctly constructed data square per the [data square layout](./data_square_layout.md) rules. + +Figure 1: Erasure Encoding Figure 2: rsmt2d Figure 3: Data Root diff --git a/specs/src/cat_pool.md b/specs/src/cat_pool.md new file mode 100644 index 0000000000..22ad999aec --- /dev/null +++ b/specs/src/cat_pool.md @@ -0,0 +1,100 @@ +# Content Addressable Transaction Pool Specification + +- 01.12.2022 | Initial specification (@cmwaters) +- 09.12.2022 | Add Push/Pull mechanics (@cmwaters) + +## Outline + +This document specifies the properties, design and implementation of a content addressable transaction pool (CAT). This protocol is intended as an alternative to the FIFO and Priority mempools currently built-in to the Tendermint consensus protocol. The term content-addressable here, indicates that each transaction is identified by a smaller, unique tag (in this case a sha256 hash). These tags are broadcast among the transactions as a means of more compactly indicating which peers have which transactions. Tracking what each peer has aims at reducing the amount of duplication. In a network without content tracking, a peer may receive as many duplicate transactions as peers connected to. The tradeoff here therefore is that the transactions are significantly larger than the tag such that the sum of the data saved sending what would be duplicated transactions is larger than the sum of sending each peer a tag. + +## Purpose + +The objective of such a protocol is to transport transactions from the author (usually a client) to a proposed block, optimizing both latency and throughput i.e. how quickly can a transaction be proposed (and committed) and how many transactions can be transported into a block at once. + +Typically the mempool serves to receive inbound transactions via an RPC endpoint, gossip them to all nodes in the network (regardless of whether they are capable of proposing a block or not), and stage groups of transactions to both consensus and the application to be included in a block. + +## Assumptions + +The following are assumptions inherited from existing Tendermint mempool protocols: + +- `CheckTx` should be seen as a simple gatekeeper to what transactions enter the pool to be gossiped and staged. It is non-deterministic: one node may reject a transaction that another node keeps. +- Applications implementing `CheckTx` are responsible for replay protection (i.e. the same transaction being present in multiple blocks). The mempool ensures that within the same block, no duplicate transactions can exist. +- The underlying p2p layer guarantees eventually reliable broadcast. A transaction need only be sent once to eventually reach the target peer. + +## Messages + +The CAT protocol extends on the existing mempool implementations by introducing two new protobuf messages: + +```protobuf +message SeenTx { + bytes tx_key = 1; + optional string from = 2; +} + +message WantTx { + bytes tx_key = 1; +} +``` + +Both `SeenTx` and `WantTx` contain the sha256 hash of the raw transaction bytes. `SeenTx` also contains an optional `p2p.ID` that corresponds to the peer that the node received the tx from. The only validation for both is that the byte slice of the `tx_key` MUST have a length of 32. + +Both messages are sent across a new channel with the ID: `byte(0x31)`. This enables cross compatibility as discussed in greater detail below. + +> **Note:** +> The term `SeenTx` is used over the more common `HasTx` because the transaction pool contains sophisticated eviction logic. TTL's, higher priority transactions and reCheckTx may mean that a transaction pool *had* a transaction but does not have it any more. Semantically it's more appropriate to use `SeenTx` to imply not the presence of a transaction but that the node has seen it and dealt with it accordingly. + +## Outbound logic + +A node in the protocol has two distinct modes: "broadcast" and "request/response". When a node receives a transaction via RPC (or specifically through `CheckTx`), it assumed that it is the only recipient from that client and thus will immediately send that transaction, after validation, to all connected peers. Afterwards, only "request/response" is used to disseminate that transaction to everyone else. + +> **Note:** +> Given that one can configure a mempool to switch off broadcast, there are no guarantees when a client submits a transaction via RPC and no error is returned that it will find its way into a proposers transaction pool. + +A `SeenTx` is broadcasted to ALL nodes upon receiving a "new" transaction from a peer. The transaction pool does not need to track every unique inbound transaction, therefore "new" is identified as: + +- The node does not currently have the transaction +- The node did not recently reject the transacton or has recently seen the same transaction committed (subject to the size of the cache) +- The node did not recently evict the transaction (subject to the size of the cache) + +Given this criteria, it is feasible, yet unlikely that a node receives two `SeenTx` messages from the same peer for the same transaction. + +A `SeenTx` MAY be sent for each transaction currently in the transaction pool when a connection with a peer is first established. This acts as a mechanism for syncing pool state across peers. + +The `SeenTx` message MUST only be broadcasted after validation and storage. Although it is possible that a node later drops a transaction under load shedding, a `SeenTx` should give as strong guarantees as possible that the node can be relied upon by others that don't yet have the transcation to obtain it. + +> **Note:** +> Inbound transactions submitted via the RPC do not trigger a `SeenTx` message as it is assumed that the node is the first to see the transaction and by gossiping it to others it is implied that the node has seen the transaction. + +A `WantTx` message is always sent point to point and never broadcasted. A `WantTx` MUST only be sent after receiving a `SeenTx` message from that peer. There is one exception which is that a `WantTx` MAY also be sent by a node after receiving an identical `WantTx` message from a peer that had previously received the nodes `SeenTx` but which after the lapse in time, did no longer exist in the nodes transaction pool. This provides an optional synchronous method for communicating that a node no longer has a transaction rather than relying on the defaulted asynchronous approach which is to wait for a period of time and try again with a new peer. + +`WantTx` must be tracked. A node SHOULD not send multiple `WantTx`s to multiple peers for the same transaction at once but wait for a period that matches the expected network latency before rerequesting the transaction to another peer. + +## Inbound logic + +Transaction pools are solely run in-memory; thus when a node stops, all transactions are discarded. To avoid the scenario where a node restarts and does not receive transactions because other nodes recorded a `SeenTx` message from their previous run, each transaction pool should track peer state based **per connection** and not per `NodeID`. + +Upon receiving a `Txs` message: + +- Check whether it is in response to a request or simply an unsolicited broadcast +- Validate the tx against current resources and the applications `CheckTx` +- If rejected or evicted, mark accordingly +- If successful, send a `SeenTx` message to all connected peers excluding the original sender. If it was from an initial broadcast, the `SeenTx` should populate the `From` field with the `p2p.ID` of the recipient else if it is in response to a request `From` should remain empty. + +Upon receiving a `SeenTx` message: + +- It should mark the peer as having seen the message. +- If the node has recently rejected that transaction, it SHOULD ignore the message. +- If the node already has the transaction, it SHOULD ignore the message. +- If the node does not have the transaction but recently evicted it, it MAY choose to rerequest the transaction if it has adequate resources now to process it. +- If the node has not seen the transaction or does not have any pending requests for that transaction, it can do one of two things: + - It MAY immediately request the tx from the peer with a `WantTx`. + - If the node is connected to the peer specified in `FROM`, it is likely, from a non-byzantine peer, that the node will also shortly receive the transaction from the peer. It MAY wait for a `Txs` message for a bounded amount of time but MUST eventually send a `WantMsg` message to either the original peer or any other peer that *has* the specified transaction. + +Upon receiving a `WantTx` message: + +- If it has the transaction, it MUST respond with a `Txs` message containing that transaction. +- If it does not have the transaction, it MAY respond with an identical `WantTx` or rely on the timeout of the peer that requested the transaction to eventually ask another peer. + +## Compatibility + +CAT has Go API compatibility with the existing two mempool implementations. It implements both the `Reactor` interface required by Tendermint's P2P layer and the `Mempool` interface used by `consensus` and `rpc`. CAT is currently network compatible with existing implementations (by using another channel), but the protocol is unaware that it is communicating with a different mempool and that `SeenTx` and `WantTx` messages aren't reaching those peers thus it is recommended that the entire network use CAT. diff --git a/specs/src/consensus.md b/specs/src/consensus.md new file mode 100644 index 0000000000..69dc17f136 --- /dev/null +++ b/specs/src/consensus.md @@ -0,0 +1,717 @@ +# Consensus Rules + + + +## System Parameters + +### Units + +| name | SI | value | description | +|------|-------|---------|---------------------| +| `1u` | `1u` | `10**0` | `1` unit. | +| `2u` | `k1u` | `10**3` | `1000` units. | +| `3u` | `M1u` | `10**6` | `1000000` units. | +| `4u` | `G1u` | `10**9` | `1000000000` units. | + +### Constants + +| name | type | value | unit | description | +|-----------------------------------------|----------|--------------|---------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `AVAILABLE_DATA_ORIGINAL_SQUARE_MAX` | `uint64` | | `share` | Maximum number of rows/columns of the original data [shares](data_structures.md#share) in [square layout](data_structures.md#arranging-available-data-into-shares). | +| `AVAILABLE_DATA_ORIGINAL_SQUARE_TARGET` | `uint64` | | `share` | Target number of rows/columns of the original data [shares](data_structures.md#share) in [square layout](data_structures.md#arranging-available-data-into-shares). | +| `BLOCK_TIME` | `uint64` | | second | Block time, in seconds. | +| `CHAIN_ID` | `string` | `"Celestia"` | | Chain ID. Each chain assigns itself a (unique) ID. | +| `GENESIS_COIN_COUNT` | `uint64` | `10**8` | `4u` | `(= 100000000)` Number of coins at genesis. | +| `MAX_GRAFFITI_BYTES` | `uint64` | `32` | `byte` | Maximum size of transaction graffiti, in bytes. | +| `MAX_VALIDATORS` | `uint16` | `64` | | Maximum number of active validators. | +| `NAMESPACE_VERSION_SIZE` | `int` | `1` | `byte` | Size of namespace version in bytes. | +| `NAMESPACE_ID_SIZE` | `int` | `28` | `byte` | Size of namespace ID in bytes. | +| `NAMESPACE_SIZE` | `int` | `29` | `byte` | Size of namespace in bytes. | +| `NAMESPACE_ID_MAX_RESERVED` | `uint64` | `255` | | Value of maximum reserved namespace (inclusive). 1 byte worth of IDs. | +| `SEQUENCE_BYTES` | `uint64` | `4` | `byte` | The number of bytes used to store the sequence length in the first share of a sequence | +| `SHARE_INFO_BYTES` | `uint64` | `1` | `byte` | The number of bytes used for [share](data_structures.md#share) information | +| `SHARE_RESERVED_BYTES` | `uint64` | `4` | `byte` | The number of bytes used to store the index of the first transaction in a transaction share. Must be able to represent any integer up to and including `SHARE_SIZE - 1`. | +| `SHARE_SIZE` | `uint64` | `512` | `byte` | Size of transaction and blob [shares](data_structures.md#share), in bytes. | +| `STATE_SUBTREE_RESERVED_BYTES` | `uint64` | `1` | `byte` | Number of bytes reserved to identify state subtrees. | +| `UNBONDING_DURATION` | `uint32` | | `block` | Duration, in blocks, for unbonding a validator or delegation. | +| `v1.Version` | `uint64` | `1` | | First version of the application. Breaking changes (hard forks) must update this parameter. | +| `v2.Version` | `uint64` | `2` | | Second version of the application. Breaking changes (hard forks) must update this parameter. | +| `VERSION_BLOCK` | `uint64` | `1` | | Version of the Celestia chain. Breaking changes (hard forks) must update this parameter. | + +### Rewards and Penalties + +| name | type | value | unit | description | +|--------------------------|----------|-------------|--------|---------------------------------------------------------| +| `SECONDS_PER_YEAR` | `uint64` | `31536000` | second | Seconds per year. Omit leap seconds. | +| `TARGET_ANNUAL_ISSUANCE` | `uint64` | `2 * 10**6` | `4u` | `(= 2000000)` Target number of coins to issue per year. | + +## Leader Selection + +Refer to the CometBFT specifications for [proposer selection procedure](https://docs.cometbft.com/v0.34/spec/consensus/proposer-selection). + +## Fork Choice + +The Tendermint consensus protocol is fork-free by construction under an honest majority of stake assumption. + +If a block has a [valid commit](#blocklastcommit), it is part of the canonical chain. If equivocation evidence is detected for more than 1/3 of voting power, the node must halt. See [proof of fork accountability](https://docs.cometbft.com/v0.34/spec/consensus/consensus#proof-of-fork-accountability). + +## Block Validity + +The validity of a newly-seen block, `block`, is determined by two components, detailed in subsequent sections: + +1. [Block structure](#block-structure): whether the block header is valid, and data in a block is arranged into a valid and matching data root (i.e. syntax). +1. [State transition](#state-transitions): whether the application of transactions in the block produces a matching and valid state root (i.e. semantics). + +Pseudocode in this section is not in any specific language and should be interpreted as being in a neutral and sane language. + +## Block Structure + +Before executing [state transitions](#state-transitions), the structure of the [block](./data_structures.md#block) must be verified. + +The following block fields are acquired from the network and parsed (i.e. [deserialized](./data_structures.md#serialization)). If they cannot be parsed, the block is ignored but is not explicitly considered invalid by consensus rules. Further implications of ignoring a block are found in the [networking spec](./networking.md). + +1. [block.header](./data_structures.md#header) +1. [block.availableDataHeader](./data_structures.md#availabledataheader) +1. [block.lastCommit](./data_structures.md#commit) + +If the above fields are parsed successfully, the available data `block.availableData` is acquired in erasure-coded form as [a list of share rows](./networking.md#availabledata), then parsed. If it cannot be parsed, the block is ignored but not explicitly invalid, as above. + +### `block.header` + +The [block header](./data_structures.md#header) `block.header` (`header` for short) is the first thing that is downloaded from the new block, and commits to everything inside the block in some way. For previous block `prev` (if `prev` is not known, then the block is ignored), and previous block header `prev.header`, the following checks must be `true`: + +`availableDataOriginalSquareSize` is computed as described [here](./data_structures.md#header). + +1. `header.height` == `prev.header.height + 1`. +1. `header.timestamp` > `prev.header.timestamp`. +1. `header.lastHeaderHash` == the [header hash](./data_structures.md#header) of `prev`. +1. `header.lastCommitHash` == the [hash](./data_structures.md#hashing) of `lastCommit`. +1. `header.consensusHash` == the value computed [here](./data_structures.md#consensus-parameters). +1. `header.stateCommitment` == the root of the state, computed [with the application of all state transitions in this block](#state-transitions). +1. `availableDataOriginalSquareSize` <= [`AVAILABLE_DATA_ORIGINAL_SQUARE_MAX`](#constants). +1. `header.availableDataRoot` == the [Merkle root](./data_structures.md#binary-merkle-tree) of the tree with the row and column roots of `block.availableDataHeader` as leaves. +1. `header.proposerAddress` == the [leader](#leader-selection) for `header.height`. + +### `block.availableDataHeader` + +The [available data header](./data_structures.md#availabledataheader) `block.availableDataHeader` (`availableDataHeader` for short) is then processed. This commits to the available data, which is only downloaded after the [consensus commit](#blocklastcommit) is processed. The following checks must be `true`: + +1. Length of `availableDataHeader.rowRoots` == `availableDataOriginalSquareSize * 2`. +1. Length of `availableDataHeader.colRoots` == `availableDataOriginalSquareSize * 2`. +1. The length of each element in `availableDataHeader.rowRoots` and `availableDataHeader.colRoots` must be [`32`](./data_structures.md#hashing). + +### `block.lastCommit` + +The last [commit](./data_structures.md#commit) `block.lastCommit` (`lastCommit` for short) is processed next. This is the Tendermint commit (i.e. polka of votes) _for the previous block_. For previous block `prev` and previous block header `prev.header`, the following checks must be `true`: + +1. `lastCommit.height` == `prev.header.height`. +1. `lastCommit.round` >= `1`. +1. `lastCommit.headerHash` == the [header hash](./data_structures.md#header) of `prev`. +1. Length of `lastCommit.signatures` <= [`MAX_VALIDATORS`](#constants). +1. Each of `lastCommit.signatures` must be a valid [CommitSig](./data_structures.md#commitsig) +1. The sum of the votes for `prev` in `lastCommit` must be at least 2/3 (rounded up) of the voting power of `prev`'s next validator set. + +### `block.availableData` + +The block's [available data](./data_structures.md#availabledata) (analogous to transactions in contemporary blockchain designs) `block.availableData` (`availableData` for short) is finally processed. The [list of share rows](./networking.md#availabledata) is parsed into the [actual data structures](./data_structures.md#availabledata) using the reverse of [the process to encode available data into shares](./data_structures.md#arranging-available-data-into-shares); if parsing fails here, the block is invalid. + +Once parsed, the following checks must be `true`: + +1. The commitments of the [erasure-coded extended](./data_structures.md#2d-reed-solomon-encoding-scheme) `availableData` must match those in `header.availableDataHeader`. Implicitly, this means that both rows and columns must be ordered lexicographically by namespace since they are committed to in a [Namespace Merkle Tree](data_structures.md#namespace-merkle-tree). +1. Length of `availableData.intermediateStateRootData` == length of `availableData.transactionData` + length of `availableData.payForBlobData` + 2. (Two additional state transitions are the [begin](#begin-block) and [end block](#end-block) implicit transitions.) + +## State Transitions + +Once the basic structure of the block [has been validated](#block-structure), state transitions must be applied to compute the new state and state root. + +For this section, the variable `state` represents the [state tree](./data_structures.md#state), with `state.accounts[k]`, `state.inactiveValidatorSet[k]`, `state.activeValidatorSet[k]`, and `state.delegationSet[k]` being shorthand for the leaf in the state tree in the [accounts, inactive validator set, active validator set, and delegation set subtrees](./data_structures.md#state) with [pre-hashed key](./data_structures.md#state) `k`. E.g. `state.accounts[a]` is shorthand for `state[(ACCOUNTS_SUBTREE_ID << 8*(32-STATE_SUBTREE_RESERVED_BYTES)) | ((-1 >> 8*STATE_SUBTREE_RESERVED_BYTES) & hash(a))]`. + +State transitions are applied in the following order: + +1. [Begin block](#begin-block). +1. [Transactions](#blockavailabledatatransactiondata). +1. [End block](#end-block). + +### `block.availableData.transactionData` + +Transactions are applied to the state. Note that _transactions_ mutate the state (essentially, the validator set and minimal balances), while _blobs_ do not. + +`block.availableData.transactionData` is simply a list of [WrappedTransaction](./data_structures.md#wrappedtransaction)s. For each wrapped transaction in this list, `wrappedTransaction`, with index `i` (starting from `0`), the following checks must be `true`: + +1. `wrappedTransaction.index` == `i`. + +For `wrappedTransaction`'s [transaction](./data_structures.md#transaction) `transaction`, the following checks must be `true`: + +1. `transaction.signature` must be a [valid signature](./data_structures.md#public-key-cryptography) over `transaction.signedTransactionData`. + +Finally, each `wrappedTransaction` is processed depending on [its transaction type](./data_structures.md#signedtransactiondata). These are specified in the next subsections, where `tx` is short for `transaction.signedTransactionData`, and `sender` is the recovered signing [address](./data_structures.md#address). We will define a few helper functions: + +```py +tipCost(y, z) = y * z +totalCost(x, y, z) = x + tipCost(y, z) +``` + +where `x` above is the amount of coins sent by the transaction authorizer, `y` above is the tip rate set in the transaction, and `z` above is the measure of the block space used by the transaction (i.e. size in bytes). + +Four additional helper functions are defined to manage the [validator queue](./data_structures.md#validator): + +1. `findFromQueue(power)`, which returns the address of the last validator in the [validator queue](./data_structures.md#validator) with voting power greater than or equal to `power`, or `0` if the queue is empty or no validators in the queue have at least `power` voting power. +1. `parentFromQueue(address)`, which returns the address of the parent in the validator queue of the validator with address `address`, or `0` if `address` is not in the queue or is the head of the queue. +1. `validatorQueueInsert`, defined as + +```py +function validatorQueueInsert(validator) + # Insert the new validator into the linked list + parent = findFromQueue(validator.votingPower) + if parent != 0 + if state.accounts[parent].status == AccountStatus.ValidatorBonded + validator.next = state.activeValidatorSet[parent].next + state.activeValidatorSet[parent].next = sender + else + validator.next = state.inactiveValidatorSet[parent].next + state.inactiveValidatorSet[parent].next = sender + else + validator.next = state.validatorQueueHead + state.validatorQueueHead = sender +``` + + +4. `validatorQueueRemove`, defined as + +```py +function validatorQueueRemove(validator, sender) + # Remove existing validator from the linked list + parent = parentFromQueue(sender) + if parent != 0 + if state.accounts[parent].status == AccountStatus.ValidatorBonded + state.activeValidatorSet[parent].next = validator.next + validator.next = 0 + else + state.inactiveValidatorSet[parent].next = validator.next + validator.next = 0 + else + state.validatorQueueHead = validator.next + validator.next = 0 +``` + +Note that light clients cannot perform a linear search through a linked list, and are instead provided logarithmic proofs (e.g. in the case of `parentFromQueue`, a proof to the parent is provided, which should have `address` as its next validator). + +In addition, three helper functions to manage the [blob paid list](./data_structures.md#blobpaid): + +1. `findFromBlobPaidList(start)`, which returns the transaction ID of the last transaction in the [blob paid list](./data_structures.md#blobpaid) with `finish` greater than `start`, or `0` if the list is empty or no transactions in the list have at least `start` `finish`. +1. `parentFromBlobPaidList(txid)`, which returns the transaction ID of the parent in the blob paid list of the transaction with ID `txid`, or `0` if `txid` is not in the list or is the head of the list. +1. `blobPaidListInsert`, defined as + +```py +function blobPaidListInsert(tx, txid) + # Insert the new transaction into the linked list + parent = findFromBlobPaidList(tx.blobStartIndex) + state.blobsPaid[txid].start = tx.blobStartIndex + numShares = ceil(tx.blobSize / SHARE_SIZE) + state.blobsPaid[txid].finish = tx.blobStartIndex + numShares - 1 + if parent != 0 + state.blobsPaid[txid].next = state.blobsPaid[parent].next + state.blobsPaid[parent].next = txid + else + state.blobsPaid[txid].next = state.blobPaidHead + state.blobPaidHead = txid +``` + +We define a helper function to compute F1 entries: + +```py +function compute_new_entry(reward, power) + if power == 0 + return 0 + return reward // power +``` + +After applying a transaction, the new state state root is computed. + +#### SignedTransactionDataTransfer + +```py +bytesPaid = len(tx) +``` + +The following checks must be `true`: + +1. `tx.type` == [`TransactionType.Transfer`](./data_structures.md#signedtransactiondata). +1. `totalCost(tx.amount, tx.fee.tipRate, bytesPaid)` <= `state.accounts[sender].balance`. +1. `tx.nonce` == `state.accounts[sender].nonce + 1`. + +Apply the following to the state: + +```py +state.accounts[sender].nonce += 1 + +state.accounts[sender].balance -= totalCost(tx.amount, tx.fee.tipRate, bytesPaid) +state.accounts[tx.to].balance += tx.amount + +state.activeValidatorSet.proposerBlockReward += tipCost(bytesPaid) +``` + +#### SignedTransactionDataMsgPayForData + +```py +bytesPaid = len(tx) + tx.blobSize +currentStartFinish = state.blobsPaid[findFromBlobPaidList(tx.blobStartIndex)] +parentStartFinish = state.blobsPaid[parentFromBlobPaidList(findFromBlobPaidList(tx.blobStartIndex))] +``` + +The following checks must be `true`: + +1. `tx.type` == [`TransactionType.MsgPayForData`](./data_structures.md#signedtransactiondata). +1. `totalCost(0, tx.fee.tipRate, bytesPaid)` <= `state.accounts[sender].balance`. +1. `tx.nonce` == `state.accounts[sender].nonce + 1`. +1. The `ceil(tx.blobSize / SHARE_SIZE)` shares starting at index `tx.blobStartIndex` must: + 1. Have namespace `tx.blobNamespace`. +1. `tx.blobShareCommitment` == computed as described [here](./data_structures.md#signedtransactiondatamsgpayfordata). +1. `parentStartFinish.finish` < `tx.blobStartIndex`. +1. `currentStartFinish.start` == `0` or `currentStartFinish.start` > `tx.blobStartIndex + ceil(tx.blobSize / SHARE_SIZE)`. + +Apply the following to the state: + +```py +state.accounts[sender].nonce += 1 +state.accounts[sender].balance -= totalCost(tx.amount, tx.fee.tipRate, bytesPaid) + +blobPaidListInsert(tx, id(tx)) + +state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) +``` + +#### SignedTransactionDataCreateValidator + +```py +bytesPaid = len(tx) +``` + +The following checks must be `true`: + +1. `tx.type` == [`TransactionType.CreateValidator`](./data_structures.md#signedtransactiondata). +1. `totalCost(0, tx.fee.tipRate, bytesPaid)` <= `state.accounts[sender].balance`. +1. `tx.nonce` == `state.accounts[sender].nonce + 1`. +1. `tx.commissionRate.denominator > 0`. +1. `tx.commissionRate.numerator <= tx.commissionRate.denominator`. +1. `state.accounts[sender].status` == `AccountStatus.None`. + +Apply the following to the state: + +```py +state.accounts[sender].nonce += 1 +state.accounts[sender].balance -= totalCost(0, tx.fee.tipRate, bytesPaid) +state.accounts[sender].status = AccountStatus.ValidatorQueued + +validator = new Validator +validator.commissionRate = tx.commissionRate +validator.delegatedCount = 0 +validator.votingPower = 0 +validator.pendingRewards = 0 +validator.latestEntry = PeriodEntry(0) +validator.unbondingHeight = 0 +validator.isSlashed = false + +validatorQueueInsert(validator) + +state.inactiveValidatorSet[sender] = validator + +state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) +``` + +#### SignedTransactionDataBeginUnbondingValidator + +```py +bytesPaid = len(tx) +``` + +The following checks must be `true`: + +1. `tx.type` == [`TransactionType.BeginUnbondingValidator`](./data_structures.md#signedtransactiondata). +1. `totalCost(0, tx.fee.tipRate, bytesPaid)` <= `state.accounts[sender].balance`. +1. `tx.nonce` == `state.accounts[sender].nonce + 1`. +1. `state.accounts[sender].status` == `AccountStatus.ValidatorQueued` or `state.accounts[sender].status` == `AccountStatus.ValidatorBonded`. + +Apply the following to the state: + +```py +state.accounts[sender].nonce += 1 +state.accounts[sender].balance -= totalCost(0, tx.fee.tipRate, bytesPaid) +state.accounts[sender].status = ValidatorStatus.Unbonding + +if state.accounts[sender].status == AccountStatus.ValidatorQueued + validator = state.inactiveValidatorSet[sender] +else if state.accounts[sender].status == AccountStatus.ValidatorBonded + validator = state.activeValidatorSet[sender] + delete state.activeValidatorSet[sender] + +validator.unbondingHeight = block.height + 1 +validator.latestEntry += compute_new_entry(validator.pendingRewards, validator.votingPower) +validator.pendingRewards = 0 + +validatorQueueRemove(validator, sender) + +state.inactiveValidatorSet[sender] = validator + +state.activeValidatorSet.activeVotingPower -= validator.votingPower + +state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) +``` + +#### SignedTransactionDataUnbondValidator + +```py +bytesPaid = len(tx) +``` + +The following checks must be `true`: + +1. `tx.type` == [`TransactionType.UnbondValidator`](./data_structures.md#signedtransactiondata). +1. `totalCost(0, tx.fee.tipRate, bytesPaid)` <= `state.accounts[sender].balance`. +1. `tx.nonce` == `state.accounts[sender].nonce + 1`. +1. `state.accounts[sender].status` == `AccountStatus.ValidatorUnbonding`. +1. `state.inactiveValidatorSet[sender].unbondingHeight + UNBONDING_DURATION` < `block.height`. + +Apply the following to the state: + +```py +validator = state.inactiveValidatorSet[sender] + +state.accounts[sender].nonce += 1 +state.accounts[sender].balance -= totalCost(0, tx.fee.tipRate, bytesPaid) +state.accounts[sender].status = AccountStatus.ValidatorUnbonded + +state.accounts[sender].balance += validator.commissionRewards + +state.inactiveValidatorSet[sender] = validator + +if validator.delegatedCount == 0 + state.accounts[sender].status = AccountStatus.None + delete state.inactiveValidatorSet[sender] + +state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) +``` + +#### SignedTransactionDataCreateDelegation + +```py +bytesPaid = len(tx) +``` + +The following checks must be `true`: + +1. `tx.type` == [`TransactionType.CreateDelegation`](./data_structures.md#signedtransactiondata). +1. `totalCost(tx.amount, tx.fee.tipRate, bytesPaid)` <= `state.accounts[sender].balance`. +1. `state.accounts[tx.to].status` == `AccountStatus.ValidatorQueued` or `state.accounts[tx.to].status` == `AccountStatus.ValidatorBonded`. +1. `tx.nonce` == `state.accounts[sender].nonce + 1`. +1. `state.accounts[sender].status` == `AccountStatus.None`. + +Apply the following to the state: + +```py +state.accounts[sender].nonce += 1 +state.accounts[sender].balance -= totalCost(tx.amount, tx.fee.tipRate, bytesPaid) +state.accounts[sender].status = AccountStatus.DelegationBonded + +if state.accounts[tx.to].status == AccountStatus.ValidatorQueued + validator = state.inactiveValidatorSet[tx.to] +else if state.accounts[tx.to].status == AccountStatus.ValidatorBonded + validator = state.activeValidatorSet[tx.to] + +delegation = new Delegation +delegation.status = DelegationStatus.Bonded +delegation.validator = tx.to +delegation.stakedBalance = tx.amount +delegation.beginEntry = validator.latestEntry +delegation.endEntry = PeriodEntry(0) +delegation.unbondingHeight = 0 + +validator.latestEntry += compute_new_entry(validator.pendingRewards, validator.votingPower) +validator.pendingRewards = 0 +validator.delegatedCount += 1 +validator.votingPower += tx.amount + +# Update the validator in the linked list by first removing then inserting +validatorQueueRemove(validator, delegation.validator) +validatorQueueInsert(validator) + +state.delegationSet[sender] = delegation + +if state.accounts[tx.to].status == AccountStatus.ValidatorQueued + state.inactiveValidatorSet[tx.to] = validator +else if state.accounts[tx.to].status == AccountStatus.ValidatorBonded + state.activeValidatorSet[tx.to] = validator + state.activeValidatorSet.activeVotingPower += tx.amount + +state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) +``` + +#### SignedTransactionDataBeginUnbondingDelegation + +```py +bytesPaid = len(tx) +``` + +The following checks must be `true`: + +1. `tx.type` == [`TransactionType.BeginUnbondingDelegation`](./data_structures.md#signedtransactiondata). +1. `totalCost(0, tx.fee.tipRate, bytesPaid)` <= `state.accounts[sender].balance`. +1. `tx.nonce` == `state.accounts[sender].nonce + 1`. +1. `state.accounts[sender].status` == `AccountStatus.DelegationBonded`. + +Apply the following to the state: + +```py +state.accounts[sender].nonce += 1 +state.accounts[sender].balance -= totalCost(0, tx.fee.tipRate, bytesPaid) +state.accounts[sender].status = AccountStatus.DelegationUnbonding + +delegation = state.delegationSet[sender] + +if state.accounts[delegation.validator].status == AccountStatus.ValidatorQueued || + state.accounts[delegation.validator].status == AccountStatus.ValidatorUnbonding || + state.accounts[delegation.validator].status == AccountStatus.ValidatorUnbonded + validator = state.inactiveValidatorSet[delegation.validator] +else if state.accounts[delegation.validator].status == AccountStatus.ValidatorBonded + validator = state.activeValidatorSet[delegation.validator] + +delegation.status = DelegationStatus.Unbonding +delegation.endEntry = validator.latestEntry +delegation.unbondingHeight = block.height + 1 + +validator.latestEntry += compute_new_entry(validator.pendingRewards, validator.votingPower) +validator.pendingRewards = 0 +validator.delegatedCount -= 1 +validator.votingPower -= delegation.stakedBalance + +# Update the validator in the linked list by first removing then inserting +# Only do this if the validator is actually in the queue (i.e. bonded or queued) +if state.accounts[delegation.validator].status == AccountStatus.ValidatorBonded || + state.accounts[delegation.validator].status == AccountStatus.ValidatorQueued + validatorQueueRemove(validator, delegation.validator) + validatorQueueInsert(validator) + +state.delegationSet[sender] = delegation + +if state.accounts[delegation.validator].status == AccountStatus.ValidatorQueued || + state.accounts[delegation.validator].status == AccountStatus.ValidatorUnbonding || + state.accounts[delegation.validator].status == AccountStatus.ValidatorUnbonded + state.inactiveValidatorSet[delegation.validator] = validator +else if state.accounts[delegation.validator].status == AccountStatus.ValidatorBonded + state.activeValidatorSet[delegation.validator] = validator + state.activeValidatorSet.activeVotingPower -= delegation.stakedBalance + +state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) +``` + +#### SignedTransactionDataUnbondDelegation + +```py +bytesPaid = len(tx) +``` + +The following checks must be `true`: + +1. `tx.type` == [`TransactionType.UnbondDelegation`](./data_structures.md#signedtransactiondata). +1. `totalCost(0, bytesPaid)` <= `state.accounts[sender].balance`. +1. `tx.nonce` == `state.accounts[sender].nonce + 1`. +1. `state.accounts[sender].status` == `AccountStatus.DelegationUnbonding`. +1. `state.delegationSet[sender].unbondingHeight + UNBONDING_DURATION` < `block.height`. + +Apply the following to the state: + +```py +delegation = state.accounts[sender].delegationInfo + +state.accounts[sender].nonce += 1 +state.accounts[sender].balance -= totalCost(0, tx.fee.tipRate, bytesPaid) +state.accounts[sender].status = None + +# Return the delegated stake +state.accounts[sender].balance += delegation.stakedBalance +# Also disperse rewards (commission has already been levied) +state.accounts[sender].balance += delegation.stakedBalance * (delegation.endEntry - delegation.beginEntry) + +if state.accounts[delegation.validator].status == AccountStatus.ValidatorQueued || + state.accounts[delegation.validator].status == AccountStatus.ValidatorUnbonding + state.accounts[delegation.validator].status == AccountStatus.ValidatorUnbonded + validator = state.inactiveValidatorSet[delegation.validator] +else if state.accounts[delegation.validator].status == AccountStatus.ValidatorBonded + validator = state.activeValidatorSet[delegation.validator] + +if validator.delegatedCount == 0 && + state.accounts[delegation.validator].status == AccountStatus.ValidatorUnbonded + state.accounts[delegation.validator].status = AccountStatus.None + delete state.inactiveValidatorSet[delegation.validator] + +delete state.accounts[sender].delegationInfo + +state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) +``` + +#### SignedTransactionDataBurn + +```py +bytesPaid = len(tx) +``` + +The following checks must be `true`: + +1. `tx.type` == [`TransactionType.Burn`](./data_structures.md#signedtransactiondata). +1. `totalCost(tx.amount, bytesPaid)` <= `state.accounts[sender].balance`. +1. `tx.nonce` == `state.accounts[sender].nonce + 1`. + +Apply the following to the state: + +```py +state.accounts[sender].nonce += 1 +state.accounts[sender].balance -= totalCost(tx.amount, tx.fee.tipRate, bytesPaid) + +state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) +``` + +#### SignedTransactionRedelegateCommission + +```py +bytesPaid = len(tx) +``` + +The following checks must be `true`: + +1. `tx.type` == [`TransactionType.RedelegateCommission`](./data_structures.md#signedtransactiondata). +1. `totalCost(0, tx.fee.tipRate, bytesPaid)` <= `state.accounts[sender].balance`. +1. `tx.nonce` == `state.accounts[sender].nonce + 1`. +1. `state.accounts[tx.to].status` == `AccountStatus.DelegationBonded`. +1. `state.accounts[sender].status` == `AccountStatus.ValidatorBonded`. + +Apply the following to the state: + +```py +state.accounts[sender].nonce += 1 +state.accounts[sender].balance -= totalCost(0, tx.fee.tipRate, bytesPaid) + +delegation = state.delegationSet[tx.to] +validator = state.activeValidatorSet[delegation.validator] + +# Force-redelegate pending rewards for delegation +pendingRewards = delegation.stakedBalance * (validator.latestEntry - delegation.beginEntry) +delegation.stakedBalance += pendingRewards +delegation.beginEntry = validator.latestEntry + +validator.latestEntry += compute_new_entry(validator.pendingRewards, validator.votingPower) +validator.pendingRewards = 0 + +# Assign pending commission rewards to delegation +commissionRewards = validator.commissionRewards +delegation.stakedBalance += commissionRewards +validator.commissionRewards = 0 + +# Update voting power +validator.votingPower += pendingRewards + commissionRewards +state.activeValidatorSet.activeVotingPower += pendingRewards + commissionRewards + +state.delegationSet[tx.to] = delegation +state.activeValidatorSet[delegation.validator] = validator + +state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) +``` + +#### SignedTransactionRedelegateReward + +```py +bytesPaid = len(tx) +``` + +The following checks must be `true`: + +1. `tx.type` == [`TransactionType.RedelegateReward`](./data_structures.md#signedtransactiondata). +1. `totalCost(0, tx.fee.tipRate, bytesPaid)` <= `state.accounts[sender].balance`. +1. `tx.nonce` == `state.accounts[sender].nonce + 1`. +1. `state.accounts[sender].status` == `AccountStatus.DelegationBonded`. +1. `state.accounts[state.delegationSet[sender].validator].status` == `AccountStatus.ValidatorBonded`. + +Apply the following to the state: + +```py +state.accounts[sender].nonce += 1 +state.accounts[sender].balance -= totalCost(0, tx.fee.tipRate, bytesPaid) + +delegation = state.delegationSet[sender] +validator = state.activeValidatorSet[delegation.validator] + +# Redelegate pending rewards for delegation +pendingRewards = delegation.stakedBalance * (validator.latestEntry - delegation.beginEntry) +delegation.stakedBalance += pendingRewards +delegation.beginEntry = validator.latestEntry + +validator.latestEntry += compute_new_entry(validator.pendingRewards, validator.votingPower) +validator.pendingRewards = 0 + +# Update voting power +validator.votingPower += pendingRewards +state.activeValidatorSet.activeVotingPower += pendingRewards + +state.delegationSet[sender] = delegation +state.activeValidatorSet[delegation.validator] = validator + +state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) +``` + +#### Begin Block + +At the beginning of the block, rewards are distributed to the block proposer. + +Apply the following to the state: + +```py +proposer = state.activeValidatorSet[block.header.proposerAddress] + +# Compute block subsidy and save to state for use in end block. +rewardFactor = (TARGET_ANNUAL_ISSUANCE * BLOCK_TIME) / (SECONDS_PER_YEAR * sqrt(GENESIS_COIN_COUNT)) +blockReward = rewardFactor * sqrt(state.activeValidatorSet.activeVotingPower) +state.activeValidatorSet.proposerBlockReward = blockReward + +# Save proposer's initial voting power to state for use in end block. +state.activeValidatorSet.proposerInitialVotingPower = proposer.votingPower + +state.activeValidatorSet[block.header.proposerAddress] = proposer +``` + +#### End Block + +Apply the following to the state: + +```py +account = state.accounts[block.header.proposerAddress] + +if account.status == AccountStatus.ValidatorUnbonding + account.status == AccountStatus.ValidatorUnbonded + proposer = state.inactiveValidatorSet[block.header.proposerAddress] +else if account.status == AccountStatus.ValidatorBonded + proposer = state.activeValidatorSet[block.header.proposerAddress] + +# Flush the outstanding pending rewards. +proposer.latestEntry += compute_new_entry(proposer.pendingRewards, proposer.votingPower) +proposer.pendingRewards = 0 + +blockReward = state.activeValidatorSet.proposerBlockReward +commissionReward = proposer.commissionRate.numerator * blockReward // proposer.commissionRate.denominator +proposer.commissionRewards += commissionReward +proposer.pendingRewards += blockReward - commissionReward + +# Even though the voting power hasn't changed yet, we consider this a period change. +proposer.latestEntry += compute_new_entry(proposer.pendingRewards, state.activeValidatorSet.proposerInitialVotingPower) +proposer.pendingRewards = 0 + +if account.status == AccountStatus.ValidatorUnbonding + account.status == AccountStatus.ValidatorUnbonded + state.inactiveValidatorSet[block.header.proposerAddress] = proposer +else if account.status == AccountStatus.ValidatorBonded + state.activeValidatorSet[block.header.proposerAddress] = proposer +``` + +At the end of a block, the top `MAX_VALIDATORS` validators by voting power with voting power _greater than_ zero are or become active (bonded). For newly-bonded validators, the entire validator object is moved to the active validators subtree and their status is changed to bonded. For previously-bonded validators that are no longer in the top `MAX_VALIDATORS` validators begin unbonding. + +Bonding validators is simply setting their status to `AccountStatus.ValidatorBonded`. The logic for validator unbonding is found [here](#signedtransactiondatabeginunbondingvalidator), minus transaction sender updates (nonce, balance, and fee). + +This end block implicit state transition is a single state transition, and [only has a single intermediate state root](#blockavailabledata) associated with it. diff --git a/specs/src/data_square_layout.md b/specs/src/data_square_layout.md new file mode 100644 index 0000000000..70e476250f --- /dev/null +++ b/specs/src/data_square_layout.md @@ -0,0 +1,62 @@ +# Data Square Layout + + + +## Preamble + +Celestia uses [a data availability scheme](https://arxiv.org/abs/1809.09044) that allows nodes to determine whether a block's data was published without downloading the whole block. The core of this scheme is arranging data in a two-dimensional matrix of [shares](./shares.md), then applying erasure coding to each row and column. This document describes the rationale for how data—transactions, blobs, and other data—[is actually arranged](./data_structures.md#arranging-available-data-into-shares). Familiarity with the [originally proposed data layout format](https://arxiv.org/abs/1809.09044) is assumed. + +## Layout Rationale + +Block data consists of: + +1. Standard cosmos-SDK transactions: (which are often represented internally as the [`sdk.Tx` interface](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/types/tx_msg.go#L42-L50)) as described in [cosmos-sdk ADR020](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/docs/architecture/adr-020-protobuf-transaction-encoding.md) + 1. These transactions contain protobuf encoded [`sdk.Msg`](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/types/tx_msg.go#L14-L26)s, which get executed atomically (if one fails they all fail) to update the Celestia state. The complete list of modules, which define the `sdk.Msg`s that the state machine is capable of handling, can be found in the [state machine modules spec](./state_machine_modules.md). Examples include standard cosmos-sdk module messages such as [MsgSend](https://github.com/cosmos/cosmos-sdk/blob/f71df80e93bffbf7ce5fbd519c6154a2ee9f991b/proto/cosmos/bank/v1beta1/tx.proto#L21-L32)), and celestia specific module messages such as [`MsgPayForBlobs`](https://github.com/celestiaorg/celestia-app/blob/v1.0.0-rc2/proto/celestia/blob/v1/tx.proto#L16-L31) +1. Blobs: binary large objects which do not modify the Celestia state, but which are intended for a Celestia application identified with a provided namespace. + +We want to arrange this data into a `k * k` matrix of fixed-sized [shares](./shares.md), which will later be committed to in [Namespace Merkle Trees (NMTs)](https://github.com/celestiaorg/nmt/blob/v0.16.0/docs/spec/nmt.md) so that individual shares in this matrix can be proven to belong to a single data root. `k` must always be a power of 2 (e.g. 1, 2, 4, 8, 16, 32, etc.) as this is optimal for the erasure coding algorithm. + +The simplest way we can imagine arranging block data is to simply serialize it all in no particular order, split it into fixed-sized shares, then arrange those shares into the `k * k` matrix in row-major order. However, this naive scheme can be improved in a number of ways, described below. + +First, we impose some ground rules: + +1. Data must be ordered by namespace. This makes queries into a NMT commitment of that data more efficient. +1. Since non-blob data are not naturally intended for particular namespaces, we assign [reserved namespaces](./namespace.md#Reserved-Namespaces) for them. A range of namespaces is reserved for this purpose, starting from the lowest possible namespace. +1. By construction, the above two rules mean that non-blob data always precedes blob data in the row-major matrix, even when considering single rows or columns. +1. Data with different namespaces must not be in the same share. This might cause a small amount of wasted block space, but makes the NMT easier to reason about in general since leaves are guaranteed to belong to a single namespace. + +Given these rules, a square may look as follows: + +![square_layout](./figures/square_layout.svg) + +Padding is addressed in the [padding section](#padding). Namespace C contains two blobs of two shares each while Namespace D contains one blob of three shares. + +### Ordering + +The order of blobs in a namespace is dictated by the priority of the PFBs that paid for the blob. A PFB with greater priority will have all blobs in that namespace strictly before a PFB with less priority. Priority is determined by the `gas-price` of the transaction (`fee`/`gas`). + +## Blob Share Commitment Rules + +Transactions can pay fees for a blob to be included in the same block as the transaction itself. It may seem natural to bundle the `MsgPayForBlobs` transaction that pays for a number of blobs with these blobs (which is the case in other blockchains with native execution, e.g. calldata in Ethereum transactions or OP_RETURN data in Bitcoin transactions), however this would mean that processes validating the state of the Celestia network would need to download all blob data. PayForBlob transactions must therefore only include a commitment to (i.e. some hash of) the blob they pay fees for. If implemented naively (e.g. with a simple hash of the blob, or a simple binary Merkle tree root of the blob), this can lead to a data availability problem, as there are no guarantees that the data behind these commitments is actually part of the block data. + +To that end, we impose some additional rules onto _blobs only_: blobs must be placed is a way such that both the transaction sender and the block producer can be held accountable—a necessary property for e.g. fee burning. Accountable in this context means that + +1. The transaction sender must pay sufficient fees for blob inclusion. +1. The block proposer cannot claim that a blob was included when it was not (which implies that a transaction and the blob it pays for must be included in the same block). In addition all blobs must be accompanied by a PayForBlob transaction. + +Specifically, a `MsgPayForBlobs` must include a `ShareCommitment` over the contents of each blob it is paying for. If the transaction sender knows 1) `k`, the size of the matrix, 2) the starting location of their blob in a row, and 3) the length of the blob (they know this since they are sending the blob), then they can actually compute a sequence of roots to _subtrees in the row NMTs_. Taking _the simple Merkle root of these subtree roots_ provides us with the `ShareCommitment` that gets included in `MsgPayForBlobs`. Using subtree roots instead of all the leafs makes blob inclusion proofs smaller. + +![subtree roots](./figures/blob_share_commitment.svg) + +Understanding 1) and 2) would usually require interaction with the block proposer. To make the possible starting locations of blobs sufficiently predictable and to make `ShareCommitment` independent of `k`, we impose an additional rule. The blob must start at a multiple of the `SubtreeWidth`. + +The `SubtreeWidth` is calculated as the length of the blob in shares, divided by the [`SubtreeRootThreshold`](https://github.com/celestiaorg/celestia-app/blob/v1.0.0-rc2/pkg/appconsts/v1/app_consts.go#L6) and rounded up to the nearest power of 2 ([implementation here](https://github.com/celestiaorg/celestia-app/blob/v1.0.0-rc2/pkg/shares/non_interactive_defaults.go#L94-L116)). If the output is greater than the minimum square size that the blob can fit in (i.e. a blob of 15 shares has a minimum square size of 4) then we take that minimum value. This `SubtreeWidth` is used as the width of the first mountain in the [Merkle Mountain Range](https://docs.grin.mw/wiki/chain-state/merkle-mountain-range/) that would all together represent the `ShareCommitment` over the blob. + +![subtree root width](./figures/subtree_width.svg) + +The `SubtreeRootThreshold` is an arbitrary versioned protocol constant that aims to put a soft limit on the number of subtree roots included in a blob inclusion proof, as described in [ADR013](../../docs/architecture/adr-013-non-interactive-default-rules-for-zero-padding.md). A higher `SubtreeRootThreshold` means less padding and more tightly packed squares but also means greater blob inclusion proof sizes. +With the above constraint, we can compute subtree roots deterministically. For example, a blob of 172 shares and `SubtreeRootThreshold` (SRT) = 64, must start on a share index that is a multiple of 4 because 172/64 = 3. 3 rounded up to the nearest power of 2 is 4. In this case, there will be a maximum of 3 shares of padding between blobs (more on padding below). The maximum subtree width in shares for the first mountain in the Merkle range will be 4 (The actual mountain range would be 43 subtree roots of 4 shares each). The `ShareCommitment` is then the Merkle tree over the peaks of the mountain range. + +## Padding + +Given these rules whereby blobs in their share format can't be directly appended one after the other, we use padding shares to fill the gaps. These are shares with a particular format (see [padding](./shares.md#padding)). Padding always comes after all the blobs in the namespace. The padding at the end of the reserved namespace and at the end of the square are special in that they belong to unique namespaces. All other padding shares use the namespace of the blob before it in the data square. diff --git a/specs/src/data_structures.md b/specs/src/data_structures.md new file mode 100644 index 0000000000..a7a8bba805 --- /dev/null +++ b/specs/src/data_structures.md @@ -0,0 +1,417 @@ +# Data Structures + + + +## Data Structures Overview + +![fig: Block data structures.](./figures/block_data_structures.svg) + +## Type Aliases + +| name | type | +|-----------------------------|-----------------------------| +| `Amount` | `uint64` | +| `Graffiti` | `byte[MAX_GRAFFITI_BYTES]` | +| [`HashDigest`](#hashdigest) | `byte[32]` | +| `Height` | `int64` | +| `Nonce` | `uint64` | +| `Round` | `int32` | +| `StateSubtreeID` | `byte` | +| [`Timestamp`](#timestamp) | `google.protobuf.Timestamp` | +| `VotingPower` | `uint64` | + +## Blockchain Data Structures + +### Block + +Blocks are the top-level data structure of the Celestia blockchain. + +| name | type | description | +|-----------------------|---------------------------------------------|-----------------------------------------------------------------------| +| `header` | [Header](#header) | Block header. Contains primarily identification info and commitments. | +| `availableDataHeader` | [AvailableDataHeader](#availabledataheader) | Header of available data. Contains commitments to erasure-coded data. | +| `availableData` | [AvailableData](#availabledata) | Data that is erasure-coded for availability. | +| `lastCommit` | [Commit](#commit) | Previous block's Tendermint commit. | + +### Header + +Block header, which is fully downloaded by both full clients and light clients. + +| name | type | description | +|-----------------------------------|---------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `version` | [ConsensusVersion](#consensusversion) | The consensus version struct. | +| `chainID` | `string` | The `CHAIN_ID`. | +| `height` | [Height](#type-aliases) | Block height. The genesis block is at height `1`. | +| `timestamp` | [Timestamp](#timestamp) | Timestamp of this block. | +| `lastHeaderHash` | [HashDigest](#hashdigest) | Previous block's header hash. | +| `lastCommitHash` | [HashDigest](#hashdigest) | Previous block's Tendermint commit hash. | +| `consensusHash` | [HashDigest](#hashdigest) | Hash of [consensus parameters](#consensus-parameters) for this block. | +| `AppHash` | [HashDigest](#hashdigest) | The [state root](#state) after the previous block's transactions are applied. | +| `availableDataOriginalSharesUsed` | `uint64` | The number of shares used in the [original data square](#arranging-available-data-into-shares) that are not [tail padding](./consensus.md#reserved-namespace-ids). | +| `availableDataRoot` | [HashDigest](#hashdigest) | Root of [commitments to erasure-coded data](#availabledataheader). | +| `proposerAddress` | [Address](#address) | Address of this block's proposer. | + +The size of the [original data square](#arranging-available-data-into-shares), `availableDataOriginalSquareSize`, isn't explicitly declared in the block header. Instead, it is implicitly computed as the smallest power of 2 whose square is at least `availableDataOriginalSharesUsed` (in other words, the smallest power of 4 that is at least `availableDataOriginalSharesUsed`). + +The header hash is the [hash](#hashing) of the [serialized](#serialization) header. + +### AvailableDataHeader + +| name | type | description | +|------------|-------------------------------|----------------------------------------| +| `rowRoots` | [HashDigest](#hashdigest)`[]` | Commitments to all erasure-coded data. | +| `colRoots` | [HashDigest](#hashdigest)`[]` | Commitments to all erasure-coded data. | + +The number of row/column roots of the original data [shares](data_structures.md#share) in [square layout](#arranging-available-data-into-shares) for this block. The `availableDataRoot` of the [header](#header) is computed using the compact row and column roots as described [here](#2d-reed-solomon-encoding-scheme). + +The number of row and column roots is each `availableDataOriginalSquareSize * 2`, and must be a power of 2. Note that the minimum `availableDataOriginalSquareSize` is 1 (not 0), therefore the number of row and column roots are each at least 2. + +Implementations can prune rows containing only [tail padding](./consensus.md#reserved-namespace-ids) as they are implicitly available. + +### AvailableData + +Data that is [erasure-coded](#erasure-coding) for [data availability checks](https://arxiv.org/abs/1809.09044). + +| name | type | description | +|------------------|-----------------------------------|-----------------------------------------------------------------------------------------------------------------------| +| `transactions` | [Transaction](#transaction) | Transactions are ordinary Cosmos SDK transactions. For example: they may modify the validator set and token balances. | +| `payForBlobData` | [PayForBlobData](#payforblobdata) | PayForBlob data. Transactions that pay for blobs to be included. | +| `blobData` | [BlobData](#blobdata) | Blob data is arbitrary user submitted data that will be published to the Celestia blockchain. | + +### Commit + +| name | type | description | +|--------------|-----------------------------|------------------------------------| +| `height` | [Height](#type-aliases) | Block height. | +| `round` | [Round](#type-aliases) | Round. Incremented on view change. | +| `headerHash` | [HashDigest](#hashdigest) | Header hash of the previous block. | +| `signatures` | [CommitSig](#commitsig)`[]` | List of signatures. | + +### Timestamp + +Timestamp is a [type alias](#type-aliases). + +Celestia uses [`google.protobuf.Timestamp`](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#google.protobuf.Timestamp) to represent time. + +### HashDigest + +HashDigest is a [type alias](#type-aliases). + +Output of the [hashing](#hashing) function. Exactly 256 bits (32 bytes) long. + +### TransactionFee + +| name | type | description | +|-----------|----------|------------------------------------| +| `tipRate` | `uint64` | The tip rate for this transaction. | + +Abstraction over transaction fees. + +### Address + +Celestia supports [secp256k1](https://en.bitcoin.it/wiki/Secp256k1) keys where [addresses](https://github.com/cosmos/cosmos-sdk/blob/main/docs/architecture/adr-028-public-key-addresses.md) are 20 bytes in length. + +| name | type | description | +|--------------|------------|-------------------------------------------------------------------------| +| `AccAddress` | `[20]byte` | AccAddress a wrapper around bytes meant to represent an account address | + +### CommitSig + +```C++ +enum CommitFlag : uint8_t { + CommitFlagAbsent = 1, + CommitFlagCommit = 2, + CommitFlagNil = 3, +}; +``` + +| name | type | description | +|--------------------|-------------------------|-------------| +| `commitFlag` | `CommitFlag` | | +| `validatorAddress` | [Address](#address) | | +| `timestamp` | [Timestamp](#timestamp) | | +| `signature` | [Signature](#signature) | | + +### Signature + +| name | type | description | +|------|------------|-----------------------------| +| `r` | `byte[32]` | `r` value of the signature. | +| `s` | `byte[32]` | `s` value of signature. | + +## ConsensusVersion + +| name | type | description | +|---------|----------|----------------------| +| `block` | `uint64` | The `VERSION_BLOCK`. | +| `app` | `uint64` | The app version. | + +## Serialization + +Objects that are committed to or signed over require a canonical serialization. This is done using a deterministic (and thus, bijective) variant of protobuf defined [here](https://github.com/cosmos/cosmos-sdk/blob/master/docs/architecture/adr-027-deterministic-protobuf-serialization.md). + +Note: there are two requirements for a serialization scheme, should this need to be changed: + +1. Must be bijective. +1. Serialization must include the length of dynamic structures (e.g. arrays with variable length). + +## Hashing + + + +All protocol-level hashing is done using SHA-2-256 as defined in [FIPS 180-4](https://doi.org/10.6028/NIST.FIPS.180-4). SHA-2-256 outputs a digest that is 256 bits (i.e. 32 bytes) long. + + +Libraries implementing SHA-2-256 are available in Go () and Rust (). + +Unless otherwise indicated explicitly, objects are first [serialized](#serialization) before being hashed. + +## Merkle Trees + +Merkle trees are used to authenticate various pieces of data across the Celestia stack, including transactions, blobs, the validator set, etc. This section provides an overview of the different tree types used, and specifies how to construct them. + +### Binary Merkle Tree + +Binary Merkle trees are constructed in the same fashion as described in [Certificate Transparency (RFC-6962)](https://tools.ietf.org/html/rfc6962), except for using [a different hashing function](#hashing). Leaves are hashed once to get leaf node values and internal node values are the hash of the concatenation of their children (either leaf nodes or other internal nodes). + +Nodes contain a single field: + +| name | type | description | +|------|---------------------------|-------------| +| `v` | [HashDigest](#hashdigest) | Node value. | + +The base case (an empty tree) is defined as the [hash](#hashing) of the empty string: + +```C++ +node.v = 0xe3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +``` + +For leaf node `node` of leaf data `d`: + +```C++ +node.v = h(0x00, serialize(d)) +``` + +For internal node `node` with children `l` and `r`: + +```C++ +node.v = h(0x01, l.v, r.v) +``` + +Note that rather than duplicating the last node if there are an odd number of nodes (the [Bitcoin design](https://github.com/bitcoin/bitcoin/blob/5961b23898ee7c0af2626c46d5d70e80136578d3/src/consensus/merkle.cpp#L9-L43)), trees are allowed to be imbalanced. In other words, the height of each leaf may be different. For an example, see Section 2.1.3 of [Certificate Transparency (RFC-6962)](https://tools.ietf.org/html/rfc6962#section-2.1.3). + +Leaves and internal nodes are hashed differently: the one-byte `0x00` is prepended for leaf nodes while `0x01` is prepended for internal nodes. This avoids a second-preimage attack [where internal nodes are presented as leaves](https://en.wikipedia.org/wiki/Merkle_tree#Second_preimage_attack) trees with leaves at different heights. + +#### BinaryMerkleTreeInclusionProof + +| name | type | description | +|------------|-------------------------------|-----------------------------------------------------------------| +| `siblings` | [HashDigest](#hashdigest)`[]` | Sibling hash values, ordered starting from the leaf's neighbor. | + +A proof for a leaf in a [binary Merkle tree](#binary-merkle-tree), as per Section 2.1.1 of [Certificate Transparency (RFC-6962)](https://tools.ietf.org/html/rfc6962#section-2.1.1). + +### Namespace Merkle Tree + + + +[Shares](./shares.md) in Celestia are associated with a provided _namespace_. The Namespace Merkle Tree (NMT) is a variation of the [Merkle Interval Tree](https://eprint.iacr.org/2018/642), which is itself an extension of the [Merkle Sum Tree](https://bitcointalk.org/index.php?topic=845978.0). It allows for compact proofs around the inclusion or exclusion of shares with particular namespace IDs. + + +Nodes contain three fields: + +| name | type | description | +|---------|-----------------------------|-----------------------------------------------| +| `n_min` | [Namespace](./namespace.md) | Min namespace in subtree rooted at this node. | +| `n_max` | [Namespace](./namespace.md) | Max namespace in subtree rooted at this node. | +| `v` | [HashDigest](#hashdigest) | Node value. | + +The base case (an empty tree) is defined as: + +```C++ +node.n_min = 0x0000000000000000 +node.n_max = 0x0000000000000000 +node.v = 0xe3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +``` + +For leaf node `node` of [share](./shares.md) data `d`: + +```C++ +node.n_min = d.namespace +node.n_max = d.namespace +node.v = h(0x00, d.namespace, d.rawData) +``` + +The `namespace` blob field here is the namespace of the leaf, which is a [`NAMESPACE_SIZE`](consensus.md#system-parameters)-long byte array. + +Leaves in an NMT **must** be lexicographically sorted by namespace in ascending order. + +For internal node `node` with children `l` and `r`: + +```C++ +node.n_min = min(l.n_min, r.n_min) +if l.n_min == PARITY_SHARE_NAMESPACE + node.n_max = PARITY_SHARE_NAMESPACE +else if r.n_min == PARITY_SHARE_NAMESPACE + node.n_max = l.n_max +else + node.n_max = max(l.n_max, r.n_max) +node.v = h(0x01, l.n_min, l.n_max, l.v, r.n_min, r.n_max, r.v) +``` + +Note that the above snippet leverages the property that leaves are sorted by namespace: if `l.n_min` is [`PARITY_SHARE_NAMESPACE`](consensus.md#reserved-state-subtree-ids), so must `{l,r}.n_max`. By construction, either both the min and max namespace of a node will be [`PARITY_SHARE_NAMESPACE`](consensus.md#reserved-state-subtree-ids), or neither will: if `r.n_min` is [`PARITY_SHARE_NAMESPACE`](consensus.md#reserved-state-subtree-ids), so must `r.n_max`. + +For some intuition: the min and max namespace for subtree roots with at least one non-parity leaf (which includes the root of an NMT, as [the right half of an NMT as used in Celestia will be parity shares](#2d-reed-solomon-encoding-scheme)) _ignore_ the namespace ID for the parity leaves. Subtree roots with _only parity leaves_ have their min and max namespace ID set to [`PARITY_SHARE_NAMESPACE`](consensus.md#reserved-state-subtree-ids). This allows for shorter proofs into the tree than if the namespace ID of parity shares was not ignored (which would cause the max namespace ID of the root to always be [`PARITY_SHARE_NAMESPACE`](consensus.md#reserved-state-subtree-ids)). + +A compact commitment can be computed by taking the [hash](#hashing) of the [serialized](#serialization) root node. + +#### NamespaceMerkleTreeInclusionProof + +| name | type | description | +|-----------------|---------------------------------|-----------------------------------------------------------------| +| `siblingValues` | [HashDigest](#hashdigest)`[]` | Sibling hash values, ordered starting from the leaf's neighbor. | +| `siblingMins` | [Namespace](./namespace.md)`[]` | Sibling min namespace IDs. | +| `siblingMaxes` | [Namespace](./namespace.md)`[]` | Sibling max namespace IDs. | + +When verifying an NMT proof, the root hash is checked by reconstructing the root node `root_node` with the computed `root_node.v` (computed as with a [plain Merkle proof](#binarymerkletreeinclusionproof)) and the provided `rootNamespaceMin` and `rootNamespaceMax` as the `root_node.n_min` and `root_node.n_max`, respectively. + +## Erasure Coding + +In order to enable trust-minimized light clients (i.e. light clients that do not rely on an honest majority of validating state assumption), it is critical that light clients can determine whether the data in each block is _available_ or not, without downloading the whole block itself. The technique used here was formally described in the paper [Fraud and Data Availability Proofs: Maximising Light Client Security and Scaling Blockchains with Dishonest Majorities](https://arxiv.org/abs/1809.09044). + +The remainder of the subsections below specify the [2D Reed-Solomon erasure coding scheme](#2d-reed-solomon-encoding-scheme) used, along with the format of [shares](./shares.md) and how [available data](#available-data) is arranged into shares. + +### Reed-Solomon Erasure Coding + +Note that while data is laid out in a two-dimensional square, rows and columns are erasure coded using a standard one-dimensional encoding. + +Reed-Solomon erasure coding is used as the underlying coding scheme. The parameters are: + +- 16-bit Galois field +- [`availableDataOriginalSquareSize`](#header) original pieces (maximum of [`AVAILABLE_DATA_ORIGINAL_SQUARE_MAX`](./consensus.md#constants)) +- [`availableDataOriginalSquareSize`](#header) parity pieces (maximum of [`AVAILABLE_DATA_ORIGINAL_SQUARE_MAX`](./consensus.md#constants)) (i.e `availableDataOriginalSquareSize * 2` total pieces), for an erasure efficiency of 50%. In other words, any 50% of the pieces from the `availableDataOriginalSquareSize * 2` total pieces are enough to recover the original data. +- [`SHARE_SIZE`](./consensus.md#constants) bytes per piece + +Note that [`availableDataOriginalSquareSize`](#header) may vary each block, and [is decided by the block proposer of that block](./block_proposer.md#deciding-on-a-block-size). [Leopard-RS](https://github.com/catid/leopard) is a C library that implements the above scheme with quasilinear runtime. + +### 2D Reed-Solomon Encoding Scheme + +The 2-dimensional data layout is described in this section. The roots of [NMTs](#namespace-merkle-tree) for each row and column across four quadrants of data in a `2k * 2k` matrix of shares, `Q0` to `Q3` (shown below), must be computed. In other words, `2k` row roots and `2k` column roots must be computed. The row and column roots are stored in the `availableDataCommitments` of the [AvailableDataHeader](#availabledataheader). + +![fig: RS2D encoding: data quadrants.](./figures/rs2d_quadrants.svg) + +The data of `Q0` is the original data, and the remaining quadrants are parity data. Setting `k = availableDataOriginalSquareSize`, the original data first must be split into [shares](./shares.md) and [arranged into a `k * k` matrix](#arranging-available-data-into-shares). Then the parity data can be computed. + +Where `A -> B` indicates that `B` is computed using [erasure coding](#reed-solomon-erasure-coding) from `A`: + +- `Q0 -> Q1` for each row in `Q0` and `Q1` +- `Q0 -> Q2` for each column in `Q0` and `Q2` +- `Q2 -> Q3` for each row in `Q2` and `Q3` + +Note that the parity data in `Q3` will be identical if it is vertically extended from `Q1` or horizontally extended from `Q2`. + +![fig: RS2D encoding: extending data.](./figures/rs2d_extending.svg) + +As an example, the parity data in the second column of `Q2` (in striped purple) is computed by [extending](#reed-solomon-erasure-coding) the original data in the second column of `Q0` (in solid blue). + +![fig: RS2D encoding: extending a column.](./figures/rs2d_extend.svg) + +Now that all four quadrants of the `2k * 2k` matrix are filled, the row and column roots can be computed. To do so, each row/column is used as the leaves of a [NMT](#namespace-merkle-tree), for which the compact root is computed (i.e. an extra hash operation over the NMT root is used to produce a single [HashDigest](#hashdigest)). In this example, the fourth row root value is computed as the NMT root of the fourth row of `Q0` and the fourth row of `Q1` as leaves. + +![fig: RS2D encoding: a row root.](./figures/rs2d_row.svg) + +Finally, the `availableDataRoot` of the block [Header](#header) is computed as the Merkle root of the [binary Merkle tree](#binary-merkle-tree) with the row and column roots as leaves, in that order. + +![fig: Available data root.](./figures/data_root.svg) + +### Arranging Available Data Into Shares + +The previous sections described how some original data, arranged into a `k * k` matrix, can be extended into a `2k * 2k` matrix and committed to with NMT roots. This section specifies how [available data](#available-data) (which includes [transactions](#transaction), PayForBlob transactions, and [blobs](#blobdata)) is arranged into the matrix in the first place. + +Note that each [share](./shares.md) only has a single namespace, and that the list of concatenated shares is lexicographically ordered by namespace. + +Then, + +1. For each of `transactionData`, `intermediateStateRootData`, PayForBlob transactions, [serialize](#serialization): + 1. For each request in the list: + 1. [Serialize](#serialization) the request (individually). + 1. Compute the length of each serialized request, [serialize the length](#serialization), and prepend the serialized request with its serialized length. + 1. Split up the length/request pairs into [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants)`-`[`SHARE_RESERVED_BYTES`](./consensus.md#constants)-byte chunks. + 1. Create a [share](./shares.md) out of each chunk. This data has a _reserved_ namespace ID, so the first [`NAMESPACE_SIZE`](./consensus.md#constants)`+`[`SHARE_RESERVED_BYTES`](./consensus.md#constants) bytes for these shares must be set specially. +1. Concatenate the lists of shares in the order: transactions, intermediate state roots, PayForBlob transactions. + +These shares are arranged in the [first quadrant](#2d-reed-solomon-encoding-scheme) (`Q0`) of the `availableDataOriginalSquareSize*2 * availableDataOriginalSquareSize*2` available data matrix in _row-major_ order. In the example below, each reserved data element takes up exactly one share. + +![fig: Original data: reserved.](./figures/rs2d_originaldata_reserved.svg) + +Each blob in the list `blobData`: + +1. [Serialize](#serialization) the blob (individually). +1. Compute the length of each serialized blob, [serialize the length](#serialization), and prepend the serialized blob with its serialized length. +1. Split up the length/blob pairs into [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_SIZE`](./consensus.md#constants)-byte chunks. +1. Create a [share](./shares.md) out of each chunk. The first [`NAMESPACE_SIZE`](./consensus.md#constants) bytes for these shares is set to the namespace. + +For each blob, it is placed in the available data matrix, with row-major order, as follows: + +1. Place the first share of the blob at the next unused location in the matrix, then place the remaining shares in the following locations. + +Transactions [must commit to a Merkle root of a list of hashes](#transaction) that are each guaranteed (assuming the block is valid) to be subtree roots in one or more of the row NMTs. For additional info, see [the rationale document](./data_square_layout.md) for this section. + +However, with only the rule above, interaction between the block producer and transaction sender may be required to compute a commitment to the blob the transaction sender can sign over. To remove interaction, blobs can optionally be laid out using a non-interactive default: + +1. Place the first share of the blob at the next unused location in the matrix whose column is aligned with the largest power of 2 that is not larger than the blob length or [`availableDataOriginalSquareSize`](#header), then place the remaining shares in the following locations **unless** there are insufficient unused locations in the row. +1. If there are insufficient unused locations in the row, place the first share of the blob at the first column of the next row. Then place the remaining shares in the following locations. By construction, any blob whose length is greater than [`availableDataOriginalSquareSize`](#header) will be placed in this way. + +In the example below, two blobs (of lengths 2 and 1, respectively) are placed using the aforementioned default non-interactive rules. + +![fig: original data blob](./figures/rs2d_originaldata_blob.svg) + +The blob share commitment rules may introduce empty shares that do not belong to any blob (in the example above, the top-right share is empty). These are zeroes with namespace ID equal to the either [`TAIL_TRANSACTION_PADDING_NAMESPACE_ID`](./consensus.md#constants) if between a request with a reserved namespace ID and a blob, or the namespace ID of the previous blob if succeeded by a blob. See the [data square layout](./data_square_layout.md) for more info. + +## Available Data + +### Transaction + +Celestia transactions are Cosmos SDK [transactions](https://github.com/cosmos/cosmos-sdk/blob/v0.46.15/docs/core/transactions.md). + +### PayForBlobData + +### IndexWrapper + +IndexWrapper are wrappers around PayForBlob transactions. They include additional metadata by the block proposer that is committed to in the [available data matrix](#arranging-available-data-into-shares). + +| name | type | description | +|-----------------|------------|--------------------------------------------------------------------------------------------------------------------------------------------------------| +| `tx` | `bytes` | Actual transaction. | +| `share_indexes` | `[]uint32` | Share indexes (in row-major order) of the first share for each blob this transaction pays for. Needed for light verification of proper blob inclusion. | +| `type_id` | `string` | Type ID of the IndexWrapper transaction type. This is used for encoding and decoding IndexWrapper transactions. It is always set to `"INDX"`. | + +### BlobData + +| name | type | description | +|---------|-------------------|----------------| +| `blobs` | [Blob](#blob)`[]` | List of blobs. | + +#### Blob + +| name | type | description | +|---------------|------------------------------|----------------------------| +| `namespaceID` | [NamespaceID](#type-aliases) | Namespace ID of this blob. | +| `rawData` | `byte[]` | Raw blob bytes. | + +## State + +The state of the Celestia chain is intentionally restricted to containing only account balances and the validator set metadata. Similar to other Cosmos SDK based chains, the state of the Celestia chain is maintained in a [multistore](https://github.com/cosmos/cosmos-sdk/blob/v0.46.15/docs/core/store.md#multistore). The root of the application state is committed to in the [block header](#header) via the `AppHash`. + +## Consensus Parameters + +Various [consensus parameters](consensus.md#system-parameters) are committed to in the block header, such as limits and constants. + +| name | type | description | +|----------------------------------|---------------------------------------|-------------------------------------------| +| `version` | [ConsensusVersion](#consensusversion) | The consensus version struct. | +| `chainID` | `string` | The `CHAIN_ID`. | +| `shareSize` | `uint64` | The `SHARE_SIZE`. | +| `shareReservedBytes` | `uint64` | The `SHARE_RESERVED_BYTES`. | +| `availableDataOriginalSquareMax` | `uint64` | The `AVAILABLE_DATA_ORIGINAL_SQUARE_MAX`. | + +In order to compute the `consensusHash` field in the [block header](#header), the above list of parameters is [hashed](#hashing). diff --git a/specs/src/specs/figures/blob_share_commitment.svg b/specs/src/figures/blob_share_commitment.svg similarity index 100% rename from specs/src/specs/figures/blob_share_commitment.svg rename to specs/src/figures/blob_share_commitment.svg diff --git a/specs/src/specs/figures/block_data_structures.dot b/specs/src/figures/block_data_structures.dot similarity index 100% rename from specs/src/specs/figures/block_data_structures.dot rename to specs/src/figures/block_data_structures.dot diff --git a/specs/src/specs/figures/block_data_structures.svg b/specs/src/figures/block_data_structures.svg similarity index 100% rename from specs/src/specs/figures/block_data_structures.svg rename to specs/src/figures/block_data_structures.svg diff --git a/specs/src/specs/figures/data_root.svg b/specs/src/figures/data_root.svg similarity index 100% rename from specs/src/specs/figures/data_root.svg rename to specs/src/figures/data_root.svg diff --git a/specs/src/specs/figures/gas_consumption/100kib_pfb_trace.png b/specs/src/figures/gas_consumption/100kib_pfb_trace.png similarity index 100% rename from specs/src/specs/figures/gas_consumption/100kib_pfb_trace.png rename to specs/src/figures/gas_consumption/100kib_pfb_trace.png diff --git a/specs/src/specs/figures/gas_consumption/msg_create_validator_trace.png b/specs/src/figures/gas_consumption/msg_create_validator_trace.png similarity index 100% rename from specs/src/specs/figures/gas_consumption/msg_create_validator_trace.png rename to specs/src/figures/gas_consumption/msg_create_validator_trace.png diff --git a/specs/src/specs/figures/gas_consumption/msg_send_trace.png b/specs/src/figures/gas_consumption/msg_send_trace.png similarity index 100% rename from specs/src/specs/figures/gas_consumption/msg_send_trace.png rename to specs/src/figures/gas_consumption/msg_send_trace.png diff --git a/specs/src/specs/figures/gas_consumption/pfb_with_two_single_share_blobs_trace.png b/specs/src/figures/gas_consumption/pfb_with_two_single_share_blobs_trace.png similarity index 100% rename from specs/src/specs/figures/gas_consumption/pfb_with_two_single_share_blobs_trace.png rename to specs/src/figures/gas_consumption/pfb_with_two_single_share_blobs_trace.png diff --git a/specs/src/specs/figures/gas_consumption/single_share_pfb_trace.png b/specs/src/figures/gas_consumption/single_share_pfb_trace.png similarity index 100% rename from specs/src/specs/figures/gas_consumption/single_share_pfb_trace.png rename to specs/src/figures/gas_consumption/single_share_pfb_trace.png diff --git a/specs/src/specs/figures/namespace.dot b/specs/src/figures/namespace.dot similarity index 100% rename from specs/src/specs/figures/namespace.dot rename to specs/src/figures/namespace.dot diff --git a/specs/src/specs/figures/namespace.svg b/specs/src/figures/namespace.svg similarity index 100% rename from specs/src/specs/figures/namespace.svg rename to specs/src/figures/namespace.svg diff --git a/specs/src/specs/figures/rs2d.svg b/specs/src/figures/rs2d.svg similarity index 100% rename from specs/src/specs/figures/rs2d.svg rename to specs/src/figures/rs2d.svg diff --git a/specs/src/specs/figures/rs2d_extend.svg b/specs/src/figures/rs2d_extend.svg similarity index 100% rename from specs/src/specs/figures/rs2d_extend.svg rename to specs/src/figures/rs2d_extend.svg diff --git a/specs/src/specs/figures/rs2d_extending.svg b/specs/src/figures/rs2d_extending.svg similarity index 100% rename from specs/src/specs/figures/rs2d_extending.svg rename to specs/src/figures/rs2d_extending.svg diff --git a/specs/src/specs/figures/rs2d_originaldata_blob.svg b/specs/src/figures/rs2d_originaldata_blob.svg similarity index 100% rename from specs/src/specs/figures/rs2d_originaldata_blob.svg rename to specs/src/figures/rs2d_originaldata_blob.svg diff --git a/specs/src/specs/figures/rs2d_originaldata_reserved.svg b/specs/src/figures/rs2d_originaldata_reserved.svg similarity index 100% rename from specs/src/specs/figures/rs2d_originaldata_reserved.svg rename to specs/src/figures/rs2d_originaldata_reserved.svg diff --git a/specs/src/specs/figures/rs2d_quadrants.svg b/specs/src/figures/rs2d_quadrants.svg similarity index 100% rename from specs/src/specs/figures/rs2d_quadrants.svg rename to specs/src/figures/rs2d_quadrants.svg diff --git a/specs/src/specs/figures/rs2d_row.svg b/specs/src/figures/rs2d_row.svg similarity index 100% rename from specs/src/specs/figures/rs2d_row.svg rename to specs/src/figures/rs2d_row.svg diff --git a/specs/src/specs/figures/share_continuation.dot b/specs/src/figures/share_continuation.dot similarity index 100% rename from specs/src/specs/figures/share_continuation.dot rename to specs/src/figures/share_continuation.dot diff --git a/specs/src/specs/figures/share_continuation.svg b/specs/src/figures/share_continuation.svg similarity index 100% rename from specs/src/specs/figures/share_continuation.svg rename to specs/src/figures/share_continuation.svg diff --git a/specs/src/specs/figures/share_start.dot b/specs/src/figures/share_start.dot similarity index 100% rename from specs/src/specs/figures/share_start.dot rename to specs/src/figures/share_start.dot diff --git a/specs/src/specs/figures/share_start.svg b/specs/src/figures/share_start.svg similarity index 100% rename from specs/src/specs/figures/share_start.svg rename to specs/src/figures/share_start.svg diff --git a/specs/src/specs/figures/square_layout.svg b/specs/src/figures/square_layout.svg similarity index 100% rename from specs/src/specs/figures/square_layout.svg rename to specs/src/figures/square_layout.svg diff --git a/specs/src/specs/figures/subtree_width.svg b/specs/src/figures/subtree_width.svg similarity index 100% rename from specs/src/specs/figures/subtree_width.svg rename to specs/src/figures/subtree_width.svg diff --git a/specs/src/specs/figures/transaction_share_continuation.dot b/specs/src/figures/transaction_share_continuation.dot similarity index 100% rename from specs/src/specs/figures/transaction_share_continuation.dot rename to specs/src/figures/transaction_share_continuation.dot diff --git a/specs/src/specs/figures/transaction_share_continuation.svg b/specs/src/figures/transaction_share_continuation.svg similarity index 100% rename from specs/src/specs/figures/transaction_share_continuation.svg rename to specs/src/figures/transaction_share_continuation.svg diff --git a/specs/src/specs/figures/transaction_share_start.dot b/specs/src/figures/transaction_share_start.dot similarity index 100% rename from specs/src/specs/figures/transaction_share_start.dot rename to specs/src/figures/transaction_share_start.dot diff --git a/specs/src/specs/figures/transaction_share_start.svg b/specs/src/figures/transaction_share_start.svg similarity index 100% rename from specs/src/specs/figures/transaction_share_start.svg rename to specs/src/figures/transaction_share_start.svg diff --git a/specs/src/fraud_proofs.md b/specs/src/fraud_proofs.md new file mode 100644 index 0000000000..2ec0802b00 --- /dev/null +++ b/specs/src/fraud_proofs.md @@ -0,0 +1,25 @@ +# Fraud Proofs + +## Bad Encoding Fraud Proofs + +In order for data availability sampling to work, light clients must be convinced +that erasure encoded parity data was encoded correctly. For light clients, this +is ultimately enforced via [bad encoding fraud proofs +(BEFPs)](https://github.com/celestiaorg/celestia-node/blob/v0.11.0-rc3/docs/adr/adr-006-fraud-service.md#detailed-design). +Consensus nodes must verify this themselves before considering a block valid. +This is done automatically by verifying the data root of the header, since that +requires reconstructing the square from the block data, performing the erasure +encoding, calculating the data root using that representation, and then +comparing the data root found in the header. + +## Blob Inclusion + +TODO + +## State + +State fraud proofs allow light clients to avoid making an honest majority assumption for +state validity. While these are not incorporated into the protocol as of v1.0.0, +there are example implementations that can be found in +[Rollkit](https://github.com/rollkit/rollkit). More info in +[rollkit-ADR009](https://github.com/rollkit/rollkit/blob/4fd97ba8b8352771f2e66454099785d06fd0c31b/docs/lazy-adr/adr-009-state-fraud-proofs.md). diff --git a/specs/src/multisig.md b/specs/src/multisig.md new file mode 100644 index 0000000000..55ecf7f7b8 --- /dev/null +++ b/specs/src/multisig.md @@ -0,0 +1,15 @@ +# Multisig + +Celestia inherits support for Multisig accounts from the Cosmos SDK. Multisig accounts behave similarly to regular accounts with the added requirement that a threshold of signatures is needed to authorize a transaction. + +The maximum number of signatures allowed for a multisig account is determined by the parameter `auth.TxSigLimit` (see [parameters](./parameters.md)). The threshold and list of signers for a multisig account are set at the time of creation and can be viewed in the `pubkey` field of a key. For example: + +```shell +$ celestia-appd keys show multisig +- address: celestia17rehcgutjfra8zhjl8675t8hhw8wsavzzutv06 + name: multisig + pubkey: '{"@type":"/cosmos.crypto.multisig.LegacyAminoPubKey","threshold":2,"public_keys":[{"@type":"/cosmos.crypto.secp256k1.PubKey","key":"AxMTEFDH8oyBPIH+d2MKfCIY1yAsEd0HVekoPaAOiu9c"},{"@type":"/cosmos.crypto.secp256k1.PubKey","key":"Ax0ANkTPWcCDWy9O2TcUXw90Z0DxnX2zqPvhi4VJPUl5"},{"@type":"/cosmos.crypto.secp256k1.PubKey","key":"AlUwWCGLzhclCMEKc2YLEap9H8JT5tWq1kB8BagU1TVH"}]}' + type: multi +``` + +Please see the [Cosmos SDK docs](https://docs.cosmos.network/main/user/run-node/multisig-guide#step-by-step-guide-to-multisig-transactions) for more information on how to use multisig accounts. diff --git a/specs/src/namespace.md b/specs/src/namespace.md new file mode 100644 index 0000000000..7f0098cdc4 --- /dev/null +++ b/specs/src/namespace.md @@ -0,0 +1,115 @@ +# Namespace + + + +## Abstract + +One of Celestia's core data structures is the namespace. +When a user submits a transaction encapsulating a `MsgPayForBlobs` message to Celestia, they MUST associate each blob with exactly one namespace. +After their transaction has been included in a block, the namespace enables users to take an interest in a subset of the blobs published to Celestia by allowing the user to query for blobs by namespace. + +In order to enable efficient retrieval of blobs by namespace, Celestia makes use of a [Namespaced Merkle Tree](https://github.com/celestiaorg/nmt). +See section 5.2 of the [LazyLedger whitepaper](https://arxiv.org/pdf/1905.09274.pdf) for more details. + +## Overview + +A namespace is composed of two fields: [version](#version) and [id](#id). +A namespace is encoded as a byte slice with the version and id concatenated. + +![namespace](./figures/namespace.svg) + +### Version + +The namespace version is an 8-bit unsigned integer that indicates the version of the namespace. +The version is used to determine the format of the namespace and +is encoded as a single byte. +A new namespace version MUST be introduced if the namespace format changes in a backwards incompatible way. + +Below we explain supported user-specifiable namespace versions, +however, we note that Celestia MAY utilize other namespace versions for internal use. +For more details, see the [Reserved Namespaces](#reserved-namespaces) section. + +#### Version 0 + +The only supported user-specifiable namespace version is `0`. +A namespace with version `0` MUST contain an id with a prefix of 18 leading `0` bytes. +The remaining 10 bytes of the id are user-specified. +Below, we provide examples of valid and invalid encoded user-supplied namespaces with version `0`. + +```go +// Valid encoded namespaces +0x0000000000000000000000000000000000000001010101010101010101 // valid blob namespace +0x0000000000000000000000000000000000000011111111111111111111 // valid blob namespace + +// Invalid encoded namespaces +0x0000000000000000000000000111111111111111111111111111111111 // invalid because it does not have 18 leading 0 bytes +0x1000000000000000000000000000000000000000000000000000000000 // invalid because it does not have version 0 +0x1111111111111111111111111111111111111111111111111111111111 // invalid because it does not have version 0 +``` + +Any change in the number of leading `0` bytes in the id of a namespace with version `0` is considered a backwards incompatible change and MUST be introduced as a new namespace version. + +### ID + +The namespace ID is a 28 byte identifier that uniquely identifies a namespace. +The ID is encoded as a byte slice of length 28. + + +## Reserved Namespaces + +Celestia reserves some namespaces for protocol use. +These namespaces are called "reserved namespaces". +Reserved namespaces are used to arrange the contents of the [data square](./data_square_layout.md). +Applications MUST NOT use reserved namespaces for their blob data. +Reserved namespaces fall into two categories: _Primary_ and _Secondary_. + +- Primary: Namespaces with values less than or equal to `0x00000000000000000000000000000000000000000000000000000000FF`. Primary namespaces always have a version of `0`. +- Secondary: Namespaces with values greater than or equal to `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00`. +Secondary namespaces always have a version of `255` (`0xFF`) so that they are placed after all user specifiable namespaces in a sorted data square. +The `PARITY_SHARE_NAMESPACE` uses version `255` (`0xFF`) to enable more efficient proof generation within the context of [nmt](https://github.com/celestiaorg/nmt), where it is used in conjunction with the `IgnoreMaxNamespace` feature. +The `TAIL_PADDING_NAMESPACE` uses the version `255` to ensure that padding shares are always placed at the end of the Celestia data square even if a new user-specifiable version is introduced. + +Below is a list of the current reserved namespaces. +For additional information on the significance and application of the reserved namespaces, please refer to the [Data Square Layout](./data_square_layout.md) specifications. + +| name | type | category | value | description | +|--------------------------------------|-------------|-----------|----------------------------------------------------------------|----------------------------------------------------------------------------| +| `TRANSACTION_NAMESPACE` | `Namespace` | Primary | `0x0000000000000000000000000000000000000000000000000000000001` | Namespace for ordinary Cosmos SDK transactions. | +| `INTERMEDIATE_STATE_ROOT_NAMESPACE` | `Namespace` | Primary | `0x0000000000000000000000000000000000000000000000000000000002` | Namespace for intermediate state roots (not currently utilized). | +| `PAY_FOR_BLOB_NAMESPACE` | `Namespace` | Primary | `0x0000000000000000000000000000000000000000000000000000000004` | Namespace for transactions that contain a PayForBlob. | +| `PRIMARY_RESERVED_PADDING_NAMESPACE` | `Namespace` | Primary | `0x00000000000000000000000000000000000000000000000000000000FF` | Namespace for padding after all primary reserved namespaces. | +| `MAX_PRIMARY_RESERVED_NAMESPACE` | `Namespace` | Primary | `0x00000000000000000000000000000000000000000000000000000000FF` | Namespace for the highest primary reserved namespace. | +| `MIN_SECONDARY_RESERVED_NAMESPACE` | `Namespace` | Secondary | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00` | Namespace for the lowest secondary reserved namespace. | +| `TAIL_PADDING_NAMESPACE` | `Namespace` | Secondary | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE` | Namespace for padding after all blobs to fill up the original data square. | +| `PARITY_SHARE_NAMESPACE` | `Namespace` | Secondary | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF` | Namespace for parity shares. | + +## Assumptions and Considerations + +Applications MUST refrain from using the [reserved namespaces](#reserved-namespaces) for their blob data. + +Celestia does not ensure the prevention of non-reserved namespace collisions. +Consequently, two distinct applications might use the same namespace. +It is the responsibility of these applications to be cautious and manage the implications and consequences arising from such namespace collisions. +Among the potential consequences is the _Woods Attack_, as elaborated in this forum post: [Woods Attack on Celestia](https://forum.celestia.org/t/woods-attack-on-celestia/59). + +## Implementation + +See the [namespace implementation in go-square](https://github.com/celestiaorg/go-square/blob/4209e67d30fef7de29af2f2712104871c173543a/share/namespace.go). +For the most recent version, which may not reflect the current specifications, refer to [the latest namespace code](https://github.com/celestiaorg/go-square/blob/main/share/namespace.go). + +## Go Definition + +```go +type Namespace struct { + Version uint8 + ID []byte +} +``` + +## References + +1. [ADR-014](../../docs/architecture/adr-014-versioned-namespaces.md) +1. [ADR-015](../../docs/architecture/adr-015-namespace-id-size.md) +1. [Namespaced Merkle Tree](https://github.com/celestiaorg/nmt) +1. [LazyLedger whitepaper](https://arxiv.org/pdf/1905.09274.pdf) +1. [Data Square Layout](./data_square_layout.md) diff --git a/specs/src/networking.md b/specs/src/networking.md new file mode 100644 index 0000000000..8f58da5be1 --- /dev/null +++ b/specs/src/networking.md @@ -0,0 +1,111 @@ +# Networking + + + +## Wire Format + +### AvailableData + +| name | type | description | +|---------------------|-------------------------------------------|---------------| +| `availableDataRows` | [AvailableDataRow](#availabledatarow)`[]` | List of rows. | + +### AvailableDataRow + +| name | type | description | +|----------|-----------------------------------------|------------------| +| `shares` | [Share](./data_structures.md#share)`[]` | Shares in a row. | + +### ConsensusProposal + +Defined as `ConsensusProposal`: + +```protobuf +{{#include ./proto/consensus.proto:ConsensusProposal}} +``` + +When receiving a new block proposal `proposal` from the network, the following steps are performed in order. _Must_ indicates that peers must be blacklisted (to prevent DoS attacks) and _should_ indicates that the network blob can simply be ignored. + +1. `proposal.type` must be a `SignedMsgType`. +1. `proposal.round` is processed identically to Tendermint. +1. `proposal.pol_round` is processed identically to Tendermint. +1. `proposal.header` must be well-formed. +1. `proposal.header.version.block` must be [`VERSION_BLOCK`](./consensus.md#constants). +1. `proposal.header.version.app` must be a supported app version. +1. `proposal.header.height` should be previous known height + 1. +1. `proposal.header.chain_id` must be [`CHAIN_ID`](./consensus.md#constants). +1. `proposal.header.time` is processed identically to Tendermint. +1. `proposal.header.last_header_hash` must be previous block's header hash. +1. `proposal.header.last_commit_hash` must be the previous block's commit hash. +1. `proposal.header.consensus_hash` must be the hash of [consensus parameters](./data_structures.md#header). +1. `proposal.header.state_commitment` must be the state root after applying the previous block's transactions. +1. `proposal.header.available_data_original_shares_used` must be at most [`AVAILABLE_DATA_ORIGINAL_SQUARE_MAX ** 2`](./consensus.md#constants). +1. `proposal.header.available_data_root` must be the [root](./data_structures.md#availabledataheader) of `proposal.da_header`. +1. `proposal.header.proposer_address` must be the [correct leader](./consensus.md#leader-selection). +1. `proposal.da_header` must be well-formed. +1. The number of elements in `proposal.da_header.row_roots` and `proposal.da_header.row_roots` must be equal. +1. The number of elements in `proposal.da_header.row_roots` must be the same as computed [here](./data_structures.md#header). +1. `proposal.proposer_signature` must be a valid [digital signature](./data_structures.md#public-key-cryptography) over the header hash of `proposal.header` that recovers to `proposal.header.proposer_address`. +1. For full nodes, `proposal.da_header` must be the result of computing the roots of the shares (received separately). +1. For light nodes, `proposal.da_header` should be sampled from for availability. + +### MsgWirePayForData + +Defined as `MsgWirePayForData`: + +```protobuf +{{#include ./proto/wire.proto:MsgWirePayForData}} +``` + +Accepting a `MsgWirePayForData` into the mempool requires different logic than other transactions in Celestia, since it leverages the paradigm of block proposers being able to malleate transaction data. Unlike [SignedTransactionDataMsgPayForData](./data_structures.md#signedtransactiondatamsgpayfordata) (the canonical data type that is included in blocks and committed to with a data root in the block header), each `MsgWirePayForData` (the over-the-wire representation of the same) has potentially multiple signatures. + +Transaction senders who want to pay for a blob will create a [SignedTransactionDataMsgPayForData](./data_structures.md#signedtransactiondatamsgpayfordata) object, `stx`, filling in the `stx.blobShareCommitment` field based on the [blob share commitment rules](./data_square_layout.md#blob-share-commitment-rules), then signing it to get a [transaction](./data_structures.md#transaction) `tx`. + +Receiving a `MsgWirePayForData` object from the network follows the reverse process: verify using the [blob share commitment rules](./data_square_layout.md#blob-share-commitment-rules) that the signature is valid. + +## Invalid Erasure Coding + +If a malicious block producer incorrectly computes the 2D Reed-Solomon code for a block's data, a fraud proof for this can be presented. We assume that the light clients have the [AvailableDataHeader](./data_structures.md#availabledataheader) and the [Header](./data_structures.md#header) for each block. Hence, given a [ShareProof](#shareproof), they can verify if the `rowRoot` or `colRoot` specified by `isCol` and `position` commits to the corresponding [Share](./data_structures.md#share). Similarly, given the `height` of a block, they can access all elements within the [AvailableDataHeader](./data_structures.md#availabledataheader) and the [Header](./data_structures.md#header) of the block. + +### ShareProof + +| name | type | description | +|------------|---------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------| +| `share` | [Share](./data_structures.md#share) | The share. | +| `proof` | [NamespaceMerkleTreeInclusionProof](./data_structures.md#namespacemerkletreeinclusionproof) | The Merkle proof of the share in the offending row or column root. | +| `isCol` | `bool` | A Boolean indicating if the proof is from a row root or column root; `false` if it is a row root. | +| `position` | `uint64` | The index of the share in the offending row or column. | + +### BadEncodingFraudProof + +Defined as `BadEncodingFraudProof`: + +```protobuf +{{#include ./proto/types.proto:BadEncodingFraudProof}} +``` + +| name | type | description | +|---------------|---------------------------------------------|-----------------------------------------------------------------------------------| +| `height` | [Height](./data_structures.md#type-aliases) | Height of the block with the offending row or column. | +| `shareProofs` | [ShareProof](#shareproof)`[]` | The available shares in the offending row or column. | +| `isCol` | `bool` | A Boolean indicating if it is an offending row or column; `false` if it is a row. | +| `position` | `uint64` | The index of the offending row or column in the square. | + +## Invalid State Update + +If a malicious block producer incorrectly computes the state, a fraud proof for this can be presented. We assume that the light clients have the [AvailableDataHeader](./data_structures.md#availabledataheader) and the [Header](./data_structures.md#header) for each block. Hence, given a [ShareProof](#shareproof), they can verify if the `rowRoot` or `colRoot` specified by `isCol` and `position` commits to the corresponding [Share](./data_structures.md#share). Similarly, given the `height` of a block, they can access all elements within the [AvailableDataHeader](./data_structures.md#availabledataheader) and the [Header](./data_structures.md#header) of the block. + +### StateFraudProof + +Defined as `StateFraudProof`: + +```protobuf +{{#include ./proto/types.proto:StateFraudProof}} +``` + +| name | type | description | +|-----------------------------|-------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `height` | [Height](./data_structures.md#type-aliases) | Height of the block with the intermediate state roots. Subtracting one from `height` gives the height of the block with the transactions. | +| `transactionShareProofs` | [ShareProof](#shareproof)`[]` | `isCol` of type `bool` must be `false`. | +| `isrShareProofs` | [ShareProof](#shareproof)`[]` | `isCol` of type `bool` must be `false`. | +| `index` | `uint64` | Index for connecting the [WrappedIntermediateStateRoot](./data_structures.md#wrappedintermediatestateroot) and [WrappedTransaction](./data_structures.md#wrappedtransaction) after shares are parsed. | diff --git a/specs/src/parameters.md b/specs/src/parameters.md new file mode 100644 index 0000000000..4dcecce93d --- /dev/null +++ b/specs/src/parameters.md @@ -0,0 +1,6 @@ +# Parameters + +The parameters in the application depend on the app version: + +- [Parameters v1](./parameters_v1.md) +- [Parameters v2](./parameters_v2.md) diff --git a/specs/src/parameters_v1.md b/specs/src/parameters_v1.md new file mode 100644 index 0000000000..436b8cb471 --- /dev/null +++ b/specs/src/parameters_v1.md @@ -0,0 +1,67 @@ +# Parameters v1 + +The parameters below represent the parameters for app version 1. + +Note that not all of these parameters are changeable via governance. This list +also includes parameter that require a hardfork to change due to being manually +hardcoded in the application or they are blocked by the `x/paramfilter` module. + +## Global parameters + +| Parameter | Default | Summary | Changeable via Governance | +|-------------------|---------|------------------------------------------------------------------------------------------------------------------------|---------------------------| +| MaxBlockSizeBytes | 100MiB | Hardcoded value in CometBFT for the protobuf encoded block. | False | +| MaxSquareSize | 128 | Hardcoded maximum square size determined per shares per row or column for the original data square (not yet extended). | False | + +## Module parameters + +| Module.Parameter | Default | Summary | Changeable via Governance | +|-----------------------------------------------|---------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------| +| auth.MaxMemoCharacters | 256 | Largest allowed size for a memo in bytes. | True | +| auth.SigVerifyCostED25519 | 590 | Gas used to verify Ed25519 signature. | True | +| auth.SigVerifyCostSecp256k1 | 1000 | Gas used to verify secp256k1 signature. | True | +| auth.TxSigLimit | 7 | Max number of signatures allowed in a multisig transaction. | True | +| auth.TxSizeCostPerByte | 10 | Gas used per transaction byte. | True | +| bank.SendEnabled | true | Allow transfers. | False | +| blob.GasPerBlobByte | 8 | Gas used per blob byte. | True | +| blob.GovMaxSquareSize | 64 | Governance parameter for the maximum square size determined per shares per row or column for the original data square (not yet extended)s. If larger than MaxSquareSize, MaxSquareSize is used. | True | +| blobstream.DataCommitmentWindow | 400 | Number of blocks that are included in a signed batch (DataCommitment). | True | +| consensus.block.MaxBytes | 1974272 bytes (~1.88 MiB) | Governance parameter for the maximum size of the protobuf encoded block. | True | +| consensus.block.MaxGas | -1 | Maximum gas allowed per block (-1 is infinite). | True | +| consensus.block.TimeIotaMs | 1000 | Minimum time added to the time in the header each block. | False | +| consensus.evidence.MaxAgeDuration | 1814400000000000 (21 days) | The maximum age of evidence before it is considered invalid in nanoseconds. This value should be identical to the unbonding period. | True | +| consensus.evidence.MaxAgeNumBlocks | 120960 | The maximum number of blocks before evidence is considered invalid. This value will stop CometBFT from pruning block data. | True | +| consensus.evidence.MaxBytes | 1MiB | Maximum size in bytes used by evidence in a given block. | True | +| consensus.validator.PubKeyTypes | Ed25519 | The type of public key used by validators. | False | +| consensus.Version.AppVersion | 1 | Determines protocol rules used for a given height. Incremented by the application upon an upgrade. | True | +| distribution.BaseProposerReward | 0 | Reward in the mint denomination for proposing a block. | True | +| distribution.BonusProposerReward | 0 | Extra reward in the mint denomination for proposers based on the voting power included in the commit. | True | +| distribution.CommunityTax | 0.02 (2%) | Percentage of the inflation sent to the community pool. | True | +| distribution.WithdrawAddrEnabled | true | Enables delegators to withdraw funds to a different address. | True | +| gov.DepositParams.MaxDepositPeriod | 604800000000000 (1 week) | Maximum period for token holders to deposit on a proposal in nanoseconds. | True | +| gov.DepositParams.MinDeposit | 10_000_000_000 utia (10,000 TIA) | Minimum deposit for a proposal to enter voting period. | True | +| gov.TallyParams.Quorum | 0.334 (33.4%) | Minimum percentage of total stake needed to vote for a result to be considered valid. | True | +| gov.TallyParams.Threshold | 0.50 (50%) | Minimum proportion of Yes votes for proposal to pass. | True | +| gov.TallyParams.VetoThreshold | 0.334 (33.4%) | Minimum value of Veto votes to Total votes ratio for proposal to be vetoed. | True | +| gov.VotingParams.VotingPeriod | 604800000000000 (1 week) | Duration of the voting period in nanoseconds. | True | +| ibc.ClientGenesis.AllowedClients | []string{"06-solomachine", "07-tendermint"} | List of allowed IBC light clients. | True | +| ibc.ConnectionGenesis.MaxExpectedTimePerBlock | 7500000000000 (75 seconds) | Maximum expected time per block in nanoseconds under normal operation. | True | +| ibc.Transfer.ReceiveEnabled | true | Enable receiving tokens via IBC. | True | +| ibc.Transfer.SendEnabled | true | Enable sending tokens via IBC. | True | +| mint.BondDenom | utia | Denomination that is inflated and sent to the distribution module account. | False | +| mint.DisinflationRate | 0.10 (10%) | The rate at which the inflation rate decreases each year. | False | +| mint.InitialInflationRate | 0.08 (8%) | The inflation rate the network starts at. | False | +| mint.TargetInflationRate | 0.015 (1.5%) | The inflation rate that the network aims to stabilize at. | False | +| slashing.DowntimeJailDuration | 1 min | Duration of time a validator must stay jailed. | True | +| slashing.MinSignedPerWindow | 0.75 (75%) | The percentage of SignedBlocksWindow that must be signed not to get jailed. | True | +| slashing.SignedBlocksWindow | 5000 | The range of blocks used to count for downtime. | True | +| slashing.SlashFractionDoubleSign | 0.02 (2%) | Percentage slashed after a validator is jailed for double signing. | True | +| slashing.SlashFractionDowntime | 0.00 (0%) | Percentage slashed after a validator is jailed for downtime. | True | +| staking.BondDenom | utia | Bondable coin denomination. | False | +| staking.HistoricalEntries | 10000 | Number of historical entries to persist in store. | True | +| staking.MaxEntries | 7 | Maximum number of entries in the redelegation queue. | True | +| staking.MaxValidators | 100 | Maximum number of validators. | True | +| staking.MinCommissionRate | 0.05 (5%) | Minimum commission rate used by all validators. | True | +| staking.UnbondingTime | 1814400 (21 days) | Duration of time for unbonding in seconds. | False | + +Note: none of the mint module parameters are governance modifiable because they have been converted into hardcoded constants. See the x/mint README.md for more details. diff --git a/specs/src/parameters_v2.md b/specs/src/parameters_v2.md new file mode 100644 index 0000000000..7555dd8761 --- /dev/null +++ b/specs/src/parameters_v2.md @@ -0,0 +1,71 @@ +# Parameters v2 + +The parameters below represent the parameters for app version 2. + +Note that not all of these parameters are changeable via governance. This list +also includes parameter that require a hardfork to change due to being manually +hardcoded in the application or they are blocked by the `x/paramfilter` module. + +## Global parameters + +| Parameter | Default | Summary | Changeable via Governance | +|-------------------|---------|------------------------------------------------------------------------------------------------------------------------|---------------------------| +| MaxBlockSizeBytes | 100MiB | Hardcoded value in CometBFT for the protobuf encoded block. | False | +| MaxSquareSize | 128 | Hardcoded maximum square size determined per shares per row or column for the original data square (not yet extended). | False | + +## Module parameters + +| Module.Parameter | Default | Summary | Changeable via Governance | +|-----------------------------------------------|---------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------|---------------------------| +| auth.MaxMemoCharacters | 256 | Largest allowed size for a memo in bytes. | True | +| auth.SigVerifyCostED25519 | 590 | Gas used to verify Ed25519 signature. | True | +| auth.SigVerifyCostSecp256k1 | 1000 | Gas used to verify secp256k1 signature. | True | +| auth.TxSigLimit | 7 | Max number of signatures allowed in a multisig transaction. | True | +| auth.TxSizeCostPerByte | 10 | Gas used per transaction byte. | True | +| bank.SendEnabled | true | Allow transfers. | False | +| blob.GasPerBlobByte | 8 | Gas used per blob byte. | True | +| blob.GovMaxSquareSize | 64 | Governance parameter for the maximum square size of the original data square. | True | +| consensus.block.MaxBytes | 1974272 bytes (~1.88 MiB) | Governance parameter for the maximum size of the protobuf encoded block. | True | +| consensus.block.MaxGas | -1 | Maximum gas allowed per block (-1 is infinite). | True | +| consensus.block.TimeIotaMs | 1000 | Minimum time added to the time in the header each block. | False | +| consensus.evidence.MaxAgeDuration | 1814400000000000 (21 days) | The maximum age of evidence before it is considered invalid in nanoseconds. This value should be identical to the unbonding period. | True | +| consensus.evidence.MaxAgeNumBlocks | 120960 | The maximum number of blocks before evidence is considered invalid. This value will stop CometBFT from pruning block data. | True | +| consensus.evidence.MaxBytes | 1MiB | Maximum size in bytes used by evidence in a given block. | True | +| consensus.validator.PubKeyTypes | Ed25519 | The type of public key used by validators. | False | +| consensus.Version.AppVersion | 2 | Determines protocol rules used for a given height. Incremented by the application upon an upgrade. | True | +| distribution.BaseProposerReward | 0 | Reward in the mint denomination for proposing a block. | True | +| distribution.BonusProposerReward | 0 | Extra reward in the mint denomination for proposers based on the voting power included in the commit. | True | +| distribution.CommunityTax | 0.02 (2%) | Percentage of the inflation sent to the community pool. | True | +| distribution.WithdrawAddrEnabled | true | Enables delegators to withdraw funds to a different address. | True | +| gov.DepositParams.MaxDepositPeriod | 604800000000000 (1 week) | Maximum period for token holders to deposit on a proposal in nanoseconds. | True | +| gov.DepositParams.MinDeposit | 10_000_000_000 utia (10,000 TIA) | Minimum deposit for a proposal to enter voting period. | True | +| gov.TallyParams.Quorum | 0.334 (33.4%) | Minimum percentage of total stake needed to vote for a result to be considered valid. | True | +| gov.TallyParams.Threshold | 0.50 (50%) | Minimum proportion of Yes votes for proposal to pass. | True | +| gov.TallyParams.VetoThreshold | 0.334 (33.4%) | Minimum value of Veto votes to Total votes ratio for proposal to be vetoed. | True | +| gov.VotingParams.VotingPeriod | 604800000000000 (1 week) | Duration of the voting period in nanoseconds. | True | +| ibc.ClientGenesis.AllowedClients | []string{"06-solomachine", "07-tendermint"} | List of allowed IBC light clients. | True | +| ibc.ConnectionGenesis.MaxExpectedTimePerBlock | 7500000000000 (75 seconds) | Maximum expected time per block in nanoseconds under normal operation. | True | +| ibc.Transfer.ReceiveEnabled | true | Enable receiving tokens via IBC. | True | +| ibc.Transfer.SendEnabled | true | Enable sending tokens via IBC. | True | +| icahost.HostEnabled | True | Enables or disables the Inter-Chain Accounts host module. | True | +| icahost.AllowMessages | [icaAllowMessages] | Defines a list of sdk message typeURLs allowed to be executed on a host chain. | True | +| minfee.NetworkMinGasPrice | 0.000001 utia | All transactions must have a gas price greater than or equal to this value. | True | +| mint.BondDenom | utia | Denomination that is inflated and sent to the distribution module account. | False | +| mint.DisinflationRate | 0.10 (10%) | The rate at which the inflation rate decreases each year. | False | +| mint.InitialInflationRate | 0.08 (8%) | The inflation rate the network starts at. | False | +| mint.TargetInflationRate | 0.015 (1.5%) | The inflation rate that the network aims to stabilize at. | False | +| slashing.DowntimeJailDuration | 1 min | Duration of time a validator must stay jailed. | True | +| slashing.MinSignedPerWindow | 0.75 (75%) | The percentage of SignedBlocksWindow that must be signed not to get jailed. | True | +| slashing.SignedBlocksWindow | 5000 | The range of blocks used to count for downtime. | True | +| slashing.SlashFractionDoubleSign | 0.02 (2%) | Percentage slashed after a validator is jailed for double signing. | True | +| slashing.SlashFractionDowntime | 0.00 (0%) | Percentage slashed after a validator is jailed for downtime. | True | +| staking.BondDenom | utia | Bondable coin denomination. | False | +| staking.HistoricalEntries | 10000 | Number of historical entries to persist in store. | True | +| staking.MaxEntries | 7 | Maximum number of entries in the redelegation queue. | True | +| staking.MaxValidators | 100 | Maximum number of validators. | True | +| staking.MinCommissionRate | 0.05 (5%) | Minimum commission rate used by all validators. | True | +| staking.UnbondingTime | 1814400 (21 days) | Duration of time for unbonding in seconds. | False | + +Note: none of the mint module parameters are governance modifiable because they have been converted into hardcoded constants. See the x/mint README.md for more details. + +[icaAllowMessages]: https://github.com/rootulp/celestia-app/blob/8caa5807df8d15477554eba953bd056ae72d4503/app/ica_host.go#L3-L18 diff --git a/specs/src/specs/proto/consensus.proto b/specs/src/proto/consensus.proto similarity index 100% rename from specs/src/specs/proto/consensus.proto rename to specs/src/proto/consensus.proto diff --git a/specs/src/specs/proto/types.proto b/specs/src/proto/types.proto similarity index 100% rename from specs/src/specs/proto/types.proto rename to specs/src/proto/types.proto diff --git a/specs/src/specs/proto/wire.proto b/specs/src/proto/wire.proto similarity index 100% rename from specs/src/specs/proto/wire.proto rename to specs/src/proto/wire.proto diff --git a/specs/src/public_key_cryptography.md b/specs/src/public_key_cryptography.md new file mode 100644 index 0000000000..50cae12d69 --- /dev/null +++ b/specs/src/public_key_cryptography.md @@ -0,0 +1,68 @@ +# Public-Key Cryptography + + + +Consensus-critical data is authenticated using [ECDSA](https://www.secg.org/sec1-v2.pdf) with the curves: [Secp256k1](https://en.bitcoin.it/wiki/Secp256k1) or [Ed25519](https://en.wikipedia.org/wiki/EdDSA#Ed25519). + +## Secp256k1 + +The Secp256k1 key type is used by accounts that submit transactions to be included in Celestia. + +### Libraries + +A highly-optimized library is available in C (), with wrappers in Go () and Rust (). + +### Public-keys + +Secp256k1 public keys can be compressed to 257-bits (or 33 bytes) per the format described [here](https://github.com/cosmos/cosmos-sdk/blob/v0.46.15/docs/basics/accounts.md#public-keys). + +### Addresses + +Cosmos [addresses](https://github.com/cosmos/cosmos-sdk/blob/v0.46.15/docs/basics/accounts.md#addresses) are 20 bytes in length. + +### Signatures + + +Deterministic signatures ([RFC-6979](https://tools.ietf.org/rfc/rfc6979.txt)) should be used when signing, but this is not enforced at the protocol level as it cannot be for Secp256k1 signatures. + + +Signatures are represented as the `r` and `s` (each 32 bytes) values of the signature. `r` and `s` take on their usual meaning (see: [SEC 1, 4.1.3 Signing Operation](https://www.secg.org/sec1-v2.pdf)). Signatures are encoded with protobuf as described [here](https://github.com/cosmos/cosmos-sdk/blob/v0.46.15/docs/core/encoding.md). + +### Human Readable Encoding + +In front-ends addresses are prefixed with the [Bech32](https://en.bitcoin.it/wiki/Bech32) prefix `celestia`. For example, a valid address is `celestia1kj39jkzqlr073t42am9d8pd40tgudc3e2kj9yf`. + +## Ed25519 + +The Ed25519 key type is used by validators. + + +### Libraries + +- [crypto/ed25519](https://pkg.go.dev/crypto/ed25519) +- [cometbft crypto/ed25519](https://pkg.go.dev/github.com/cometbft/cometbft@v0.37.0/crypto/ed25519) + +### Public Keys + +Ed25519 public keys are 32 bytes in length. They often appear in validator configuration files (e.g. `genesis.json`) base64 encoded: + +```json + "pub_key": { + "type": "tendermint/PubKeyEd25519", + "value": "DMEMMj1+thrkUCGocbvvKzXeaAtRslvX9MWtB+smuIA=" + } +``` + + +### Addresses + +Ed25519 addresses are the first 20-bytes of the SHA256 hash of the raw 32-byte public key: + +```go +address = SHA256(pubkey)[:20] +``` + + +### Signatures + +Ed25519 signatures are 64 bytes in length. diff --git a/specs/src/resource_pricing.md b/specs/src/resource_pricing.md new file mode 100644 index 0000000000..525bad661e --- /dev/null +++ b/specs/src/resource_pricing.md @@ -0,0 +1,254 @@ +# Resource Pricing + +For all standard cosmos-sdk transactions (staking, IBC, etc), Celestia utilizes +the [default cosmos-sdk mechanisms](https://github.com/cosmos/cosmos-sdk/blob/v0.46.15/docs/basics/gas-fees.md) for pricing resources. This involves +incrementing a gas counter during transaction execution each time the state is +read from/written to, or when specific costly operations occur such as signature +verification or inclusion of data. + +```go +// GasMeter interface to track gas consumption +type GasMeter interface { + GasConsumed() Gas + GasConsumedToLimit() Gas + GasRemaining() Gas + Limit() Gas + ConsumeGas(amount Gas, descriptor string) + RefundGas(amount Gas, descriptor string) + IsPastLimit() bool + IsOutOfGas() bool + String() string +} +``` + +We can see how this gas meter is used in practice by looking at the store. +Notice where gas is consumed each time we write or read, specifically a flat +cost for initiating the action followed by a prorated cost for the amount of +data read or written. + +```go +// Implements KVStore. +func (gs *Store) Get(key []byte) (value []byte) { + gs.gasMeter.ConsumeGas(gs.gasConfig.ReadCostFlat, types.GasReadCostFlatDesc) + value = gs.parent.Get(key) + + // TODO overflow-safe math? + gs.gasMeter.ConsumeGas(gs.gasConfig.ReadCostPerByte*types.Gas(len(key)), types.GasReadPerByteDesc) + gs.gasMeter.ConsumeGas(gs.gasConfig.ReadCostPerByte*types.Gas(len(value)), types.GasReadPerByteDesc) + + return value +} + +// Implements KVStore. +func (gs *Store) Set(key []byte, value []byte) { + types.AssertValidKey(key) + types.AssertValidValue(value) + gs.gasMeter.ConsumeGas(gs.gasConfig.WriteCostFlat, types.GasWriteCostFlatDesc) + // TODO overflow-safe math? + gs.gasMeter.ConsumeGas(gs.gasConfig.WriteCostPerByte*types.Gas(len(key)), types.GasWritePerByteDesc) + gs.gasMeter.ConsumeGas(gs.gasConfig.WriteCostPerByte*types.Gas(len(value)), types.GasWritePerByteDesc) + gs.parent.Set(key, value) +} +``` + +The configuration for the gas meter used by Celestia is as follows. + +```go +// KVGasConfig returns a default gas config for KVStores. +func KVGasConfig() GasConfig { + return GasConfig{ + HasCost: 1000, + DeleteCost: 1000, + ReadCostFlat: 1000, + ReadCostPerByte: 3, + WriteCostFlat: 2000, + WriteCostPerByte: 30, + IterNextCostFlat: 30, + } +} + +// TransientGasConfig returns a default gas config for TransientStores. +func TransientGasConfig() GasConfig { + return GasConfig{ + HasCost: 100, + DeleteCost: 100, + ReadCostFlat: 100, + ReadCostPerByte: 0, + WriteCostFlat: 200, + WriteCostPerByte: 3, + IterNextCostFlat: 3, + } +} +``` + +Two notable gas consumption events that are not Celestia specific are the total +bytes used for a transaction and the verification of the signature + +```go +func (cgts ConsumeTxSizeGasDecorator) AnteHandle(ctx sdk.Context, tx sdk.Tx, simulate bool, next sdk.AnteHandler) (sdk.Context, error) { + sigTx, ok := tx.(authsigning.SigVerifiableTx) + if !ok { + return ctx, sdkerrors.Wrap(sdkerrors.ErrTxDecode, "invalid tx type") + } + params := cgts.ak.GetParams(ctx) + + ctx.GasMeter().ConsumeGas(params.TxSizeCostPerByte*sdk.Gas(len(ctx.TxBytes())), "txSize") + ... +} + +// DefaultSigVerificationGasConsumer is the default implementation of SignatureVerificationGasConsumer. It consumes gas +// for signature verification based upon the public key type. The cost is fetched from the given params and is matched +// by the concrete type. +func DefaultSigVerificationGasConsumer( + meter sdk.GasMeter, sig signing.SignatureV2, params types.Params, +) error { + pubkey := sig.PubKey + switch pubkey := pubkey.(type) { + case *ed25519.PubKey: + meter.ConsumeGas(params.SigVerifyCostED25519, "ante verify: ed25519") + return sdkerrors.Wrap(sdkerrors.ErrInvalidPubKey, "ED25519 public keys are unsupported") + + case *secp256k1.PubKey: + meter.ConsumeGas(params.SigVerifyCostSecp256k1, "ante verify: secp256k1") + return nil + + case *secp256r1.PubKey: + meter.ConsumeGas(params.SigVerifyCostSecp256r1(), "ante verify: secp256r1") + return nil + + case multisig.PubKey: + multisignature, ok := sig.Data.(*signing.MultiSignatureData) + if !ok { + return fmt.Errorf("expected %T, got, %T", &signing.MultiSignatureData{}, sig.Data) + } + err := ConsumeMultisignatureVerificationGas(meter, multisignature, pubkey, params, sig.Sequence) + if err != nil { + return err + } + return nil + + default: + return sdkerrors.Wrapf(sdkerrors.ErrInvalidPubKey, "unrecognized public key type: %T", pubkey) + } +} +``` + +Since gas is consumed in this fashion and many of the cosmos-sdk transactions +are composable, any given transaction can have a large window of possible gas +consumption. For example, vesting accounts use more bytes of state than a normal +account, so more gas is consumed each time a vesting account is read from or +updated. + +## Parameters + +There are four parameters that can be modified via governance to modify gas +usage. + +| Parameter | Default Value | Description | Changeable via Governance | +|-----------------|---------------|-----------------------------------------|---------------------------| +| consensus/max_gas | -1 | The maximum gas allowed in a block. Default of -1 means this value is not capped. | True | +| auth/tx_size_cost_per_byte | 10 | Gas used per each byte used by the transaction. | True | +| auth/sig_verify_cost_secp256k1 | 1000 | Gas used per verifying a secp256k1 signature | True | +| blob/gas_per_blob_byte | 8 | Gas used per byte used by blob. Note that this value is applied to all encoding overhead, meaning things like the padding of the remaining share and namespace. See PFB gas estimation section for more details. | True | + +## Gas Limit + +The gas limit must be included in each transaction. If the transaction exceeds +this gas limit during the execution of the transaction, then the transaction +will fail. + +> Note: When a transaction is submitted to the mempool, the transaction is not +> fully executed. This can lead to a transaction getting accepted by the mempool +> and eventually included in a block, yet failing because the transaction ends +> up exceeding the gas limit. + +Fees are not currently refunded. While users can specify a gas price, the total +fee is then calculated by simply multiplying the gas limit by the gas price. The +entire fee is then deducted from the transaction no matter what. + +## Fee market + +By default, Celestia's consensus nodes prioritize transactions in their mempools +based on gas price. In version 1, there was no enforced minimum gas price, which +allowed each consensus node to independently set its own minimum gas price in +`app.toml`. This even permitted a gas price of 0, thereby creating the +possibility of secondary markets. In version 2, Celestia introduces a network +minimum gas price, a consensus constant, unaffected by individual node +configurations. Although nodes retain the freedom to increase gas prices +locally, all transactions in a block must be greater than or equal to the network +minimum threshold. If a block is proposed that contains a tx with a gas price +below the network min gas price, the block will be rejected as invalid. + +## Estimating PFB cost + +Generally, the gas used by a PFB transaction involves a static "fixed cost" and +a dynamic cost based on the size of each blob involved in the transaction. + +> Note: For a general use case of a normal account submitting a PFB, the static +> costs can be treated as such. However, due to the description above of how gas +> works in the cosmos-sdk this is not always the case. Notably, if we use a +> vesting account or the `feegrant` modules, then these static costs change. + +The "fixed cost" is an approximation of the gas consumed by operations outside +the function `GasToConsume` (for example, signature verification, tx size, read +access to accounts), which has a default value of 65,000. + +> Note: the first transaction sent by an account (sequence number == 0) has an +> additional one time gas cost of 10,000. If this is the case, this should be +> accounted for. + +Each blob in the PFB contributes to the total gas cost based on its size. The +function `GasToConsume` calculates the total gas consumed by all the blobs +involved in a PFB, where each blob's gas cost is computed by first determining +how many shares are needed to store the blob size. Then, it computes the product +of the number of shares, the number of bytes per share, and the `gasPerByte` +parameter. Finally, it adds a static amount per blob. + +The gas cost per blob byte and gas cost per transaction byte are parameters that +could potentially be adjusted through the system's governance mechanisms. Hence, +actual costs may vary depending on the current settings of these parameters. + +## Tracing Gas Consumption + +This figure plots each instance of the gas meter being incremented as a colored +dot over the execution lifecycle of a given transaction. The y-axis is units of +gas and the x-axis is cumulative gas consumption. The legend shows which color +indicates what the cause of the gas consumption was. + +This code used to trace gas consumption can be found in the `tools/gasmonitor` of the branch for [#2131](https://github.com/celestiaorg/celestia-app/pull/2131), and the script to generate the plots below can be found [here](https://gist.github.com/evan-forbes/948c8cf574f2f50b101c89a95ee1d43c) (warning: this script will not be maintained). + +### MsgSend + +Here we can see the gas consumption trace of a common send transaction for +1`utia` + +![MsgSend](./figures/gas_consumption/msg_send_trace.png) + +### MsgCreateValidator + +Here we examine a more complex transaction. + +![MsgCreateValidator](./figures/gas_consumption/msg_create_validator_trace.png) + +### PFB with One Single Share Blob + +![MsgPayForBlobs Single +Share](./figures/gas_consumption/single_share_pfb_trace.png) + +### PFB with Two Single Share Blobs + +This PFB transaction contains two single share blobs. Notice the gas cost for +`pay for blob` is double what it is above due to two shares being used, and +there is also additional cost from `txSize` since the transaction itself is +larger to accommodate the second set of metadata in the PFB. + +![MsgPayForBlobs with Two +Blobs](./figures/gas_consumption/pfb_with_two_single_share_blobs_trace.png) + +### 100KiB Single Blob PFB + +Here we can see how the cost of a PFB with a large blob (100KiB) is quickly dominated by +the cost of the blob. + +![MsgPayForBlobs with One Large +Blob](./figures/gas_consumption/100kib_pfb_trace.png) diff --git a/specs/src/shares.md b/specs/src/shares.md new file mode 100644 index 0000000000..ce84a8950a --- /dev/null +++ b/specs/src/shares.md @@ -0,0 +1,121 @@ +# Shares + + + +## Abstract + +All available data in a Celestia [block](./data_structures.md#block) is split into fixed-size data chunks known as "shares". Shares are the atomic unit of the Celestia data square. The shares in a Celestia block are eventually [erasure-coded](./data_structures.md#erasure-coding) and committed to in [Namespace Merkle trees](./data_structures.md#namespace-merkle-tree) (also see [NMT spec](https://github.com/celestiaorg/nmt/blob/master/docs/spec/nmt.md)). + +## Terms + +- **Blob**: User specified data (e.g. a roll-up block) that is associated with exactly one namespace. Blob data are opaque bytes of data that are included in the block but do not impact Celestia's state. +- **Share**: A fixed-size data chunk that is associated with exactly one namespace. +- **Share sequence**: A share sequence is a contiguous set of shares that contain semantically relevant data. A share sequence MUST contain one or more shares. When a [blob](../../x/blob/README.md) is split into shares, it is written to one share sequence. As a result, all shares in a share sequence are typically parsed together because the original blob data may have been split across share boundaries. All transactions in the [`TRANSACTION_NAMESPACE`](./namespace.md#reserved-namespaces) are contained in one share sequence. All transactions in the [`PAY_FOR_BLOB_NAMESPACE`](./namespace.md#reserved-namespaces) are contained in one share sequence. + +## Overview + +User submitted transactions are split into shares (see [share splitting](#share-splitting)) and arranged in a `k * k` matrix (see [arranging available data into shares](./data_structures.md#arranging-available-data-into-shares)) prior to the erasure coding step. Shares in the `k * k` matrix are ordered by namespace and have a common [share format](#share-format). + +[Padding](#padding) shares are added to the `k * k` matrix to ensure: + +1. Blob sequences start on an index that conforms to [blob share commitment rules](./data_square_layout.md#blob-share-commitment-rules) (see [namespace padding share](#namespace-padding-share) and [reserved padding share](#primary-reserved-padding-share)) +1. The number of shares in the matrix is a perfect square (see [tail padding share](#tail-padding-share)) + +## Share Format + +Every share has a fixed size [`SHARE_SIZE`](./consensus.md#constants). The share format below is consistent for all shares: + +- The first [`NAMESPACE_VERSION_SIZE`](./consensus.md#constants) bytes of a share's raw data is the namespace version of that share (denoted by "namespace version" in the figure below). +- The next [`NAMESPACE_ID_SIZE`](./consensus.md#constants) bytes of a share's raw data is the namespace ID of that share (denoted by "namespace id" in the figure below). +- The next [`SHARE_INFO_BYTES`](./consensus.md#constants) bytes are for share information (denoted by "info byte" in the figure below) with the following structure: + - The first 7 bits represent the [share version](#share-version) in big endian form (initially, this will be `0000000` for version `0`); + - The last bit is a sequence start indicator. The indicator is `1` if this share is the first share in a sequence or `0` if this share is a continuation share in a sequence. +- If this share is the first share in a sequence, it will include the length of the sequence in bytes. The next [`SEQUENCE_BYTES`](./consensus.md#constants) represent a big-endian uint32 value (denoted by "sequence length" in the figure below). This length is placed immediately after the `SHARE_INFO_BYTES` field. It's important to note that shares that are not the first share in a sequence do not contain this field. +- The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_SIZE`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants)`-`[`SEQUENCE_BYTES`](./consensus.md#constants) bytes (if first share) or [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_SIZE`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants) bytes (if continuation share) are raw data (denoted by "blob1" in the figure below). Typically raw data is the blob payload that user's submit in a [BlobTx](../../x/blob/README.md). However, raw data can also be transaction data (see [transaction shares](#transaction-shares) below). +- If there is insufficient raw data to fill the share, the remaining bytes are filled with `0`. + +First share in a sequence: + +![figure 1: share start](./figures/share_start.svg) + +Continuation share in a sequence: + +![figure 2: share continuation](./figures/share_continuation.svg) + +Since raw data that exceeds [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_SIZE`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants) `-` [`SEQUENCE_BYTES`](./consensus.md#constants) bytes will span more than one share, developers MAY choose to encode additional metadata in their raw blob data prior to inclusion in a Celestia block. For example, Celestia transaction shares encode additional metadata in the form of "reserved bytes". + +### Share Version + +The share version is a 7-bit big-endian unsigned integer that is used to indicate the version of the [share format](#share-format). The only supported share version is `0`. A new share version MUST be introduced if the share format changes in a way that is not backwards compatible. + +## Transaction Shares + +In order for clients to parse shares in the middle of a sequence without downloading antecedent shares, Celestia encodes additional metadata in the shares associated with reserved namespaces. At the time of writing this only applies to the [`TRANSACTION_NAMESPACE`](./namespace.md#reserved-namespaces) and [`PAY_FOR_BLOB_NAMESPACE`](./namespace.md#reserved-namespaces). This share structure is often referred to as "compact shares" to differentiate from the share structure defined above for all shares. It conforms to the common [share format](#share-format) with one additional field, the "reserved bytes" field, which is described below: + +- Every transaction share includes [`SHARE_RESERVED_BYTES`](./consensus.md#constants) bytes that contain the index of the starting byte of the length of the [canonically serialized](./consensus.md#serialization) first transaction that starts in the share, or `0` if there is none, as a binary big endian `uint32`. Denoted by "reserved bytes" in the figure below. The [`SHARE_RESERVED_BYTES`](./consensus.md#constants) are placed immediately after the `SEQUENCE_BYTES` if this is the first share in a sequence or immediately after the `SHARE_INFO_BYTES` if this is a continuation share in a sequence. +- The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_SIZE`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants)`-`[`SEQUENCE_BYTES`](./consensus.md#constants)`-`[`SHARE_RESERVED_BYTES`](./consensus.md#constants) bytes (if first share) or [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_SIZE`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants)`-`[`SHARE_RESERVED_BYTES`](./consensus.md#constants) bytes (if continuation share) are transaction or PayForBlob transaction data (denoted by "tx1" and "tx2" in the figure below). Each transaction or PayForBlob transaction is prefixed with a [varint](https://developers.google.com/protocol-buffers/docs/encoding) of the length of that unit (denoted by "len(tx1)" and "len(tx2)" in the figure below). +- If there is insufficient transaction or PayForBlob transaction data to fill the share, the remaining bytes are filled with `0`. + +First share in a sequence: + +![figure 3: transaction share start](./figures/transaction_share_start.svg) + +where reserved bytes would be `38` as a binary big endian `uint32` (`[0b00000000, 0b00000000, 0b00000000, 0b00100110]`). + +Continuation share in a sequence: + +![figure 4: transaction share continuation](./figures/transaction_share_continuation.svg) + +where reserved bytes would be `80` as a binary big endian `uint32` (`[0b00000000, 0b00000000, 0b00000000, 0b01010000]`). + +## Padding + +Padding shares vary based on namespace but they conform to the [share format](#share-format) described above. + +- The first [`NAMESPACE_VERSION_SIZE`](./consensus.md#constants) bytes of a share's raw data is the namespace version of that share (initially, this will be `0`). +- The next [`NAMESPACE_ID_SIZE`](./consensus.md#constants) bytes of a share's raw data is the namespace ID of that share. This varies based on the type of padding share. +- The next [`SHARE_INFO_BYTES`](./consensus.md#constants) bytes are for share information. + - The first 7 bits represent the [share version](#share-version) in big endian form (initially, this will be `0000000` for version `0`); + - The last bit is a sequence start indicator. The indicator is always `1`. +- The next [`SEQUENCE_BYTES`](./consensus.md#constants) contain a big endian `uint32` of value `0`. +- The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_SIZE`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants)`-`[`SEQUENCE_BYTES`](./consensus.md#constants) bytes are filled with `0`. + +### Namespace Padding Share + +A namespace padding share uses the namespace of the blob that precedes it in the data square so that the data square can retain the property that all shares are ordered by namespace. +A namespace padding share acts as padding between blobs so that the subsequent blob begins at an index that conforms to the [blob share commitment rules](./data_square_layout.md#blob-share-commitment-rules). Clients MAY ignore the contents of these shares because they don't contain any significant data. + +### Primary Reserved Padding Share + +Primary reserved padding shares use the [`PRIMARY_RESERVED_PADDING_NAMESPACE`](./namespace.md#reserved-namespaces). Primary reserved padding shares are placed after shares in the primary reserved namespace range so that the first blob can start at an index that conforms to blob share commitment rules. Clients MAY ignore the contents of these shares because they don't contain any significant data. + +### Tail Padding Share + +Tail padding shares use the [`TAIL_PADDING_NAMESPACE`](./namespace.md#reserved-namespaces). Tail padding shares are placed after the last blob in the data square so that the number of shares in the data square is a perfect square. Clients MAY ignore the contents of these shares because they don't contain any significant data. + +## Parity Share + +Parity shares use the [`PARITY_SHARE_NAMESPACE`](./namespace.md#reserved-namespaces). Parity shares are the output of the erasure coding step of the data square construction process. They occupy quadrants Q1, Q2, and Q3 of the extended data square and are used to reconstruct the original data square (Q0). Bytes carry no special meaning. + +## Share Splitting + +Share splitting is the process of converting a blob into a share sequence. The process is as follows: + +1. Create a new share and populate the prefix of the share with the blob's namespace and [share version](#share-version). Set the sequence start indicator to `1`. Write the blob length as the sequence length. Write the blob's data into the share until the share is full. +1. If there is more data to write, create a new share (a.k.a continuation share) and populate the prefix of the share with the blob's namespace and [share version](#share-version). Set the sequence start indicator to `0`. Write the remaining blob data into the share until the share is full. +1. Repeat the previous step until all blob data has been written. +1. If the last share is not full, fill the remainder of the share with `0`. + +## Assumptions and Considerations + +- Shares are assumed to be byte slices of length 512. Parsing shares of a different length WILL result in an error. + +## Implementation + +See [go-square/shares](https://github.com/celestiaorg/go-square/tree/be3c2801e902a0f90f694c062b9c4e6a7e01154e/shares). + +## References + +1. [ADR-012](../../docs/architecture/adr-012-sequence-length-encoding.md) +1. [ADR-014](../../docs/architecture/adr-014-versioned-namespaces.md) +1. [ADR-015](../../docs/architecture/adr-015-namespace-id-size.md) diff --git a/specs/src/specs/ante_handler.md b/specs/src/specs/ante_handler.md index 2ebe759a79..12710bdb85 100644 --- a/specs/src/specs/ante_handler.md +++ b/specs/src/specs/ante_handler.md @@ -1,13 +1 @@ # AnteHandler - -Celestia makes use of a Cosmos SDK [AnteHandler](https://github.com/cosmos/cosmos-sdk/blob/v0.46.15/x/auth/spec/03_antehandlers.md) in order to reject decodable sdk.Txs that do not meet certain criteria. The AnteHandler is invoked at multiple times during the transaction lifecycle: - -1. `CheckTx` prior to the transaction entering the mempool -1. `PrepareProposal` when the block proposer includes the transaction in a block proposal -1. `ProcessProposal` when validators validate the transaction in a block proposal -1. `DeliverTx` when full nodes execute the transaction in a decided block - -The AnteHandler is defined in `app/ante/ante.go`. The app version impacts AnteHandler behavior. See: - -- [AnteHandler v1](./ante_handler_v1.md) -- [AnteHandler v2](./ante_handler_v2.md) diff --git a/specs/src/specs/ante_handler_v1.md b/specs/src/specs/ante_handler_v1.md index 52109f67f8..a5c3cb3fdf 100644 --- a/specs/src/specs/ante_handler_v1.md +++ b/specs/src/specs/ante_handler_v1.md @@ -1,23 +1 @@ # AnteHandler v1 - -The AnteHandler chains together several decorators to ensure the following criteria are met for app version 1: - -- The tx does not contain any [extension options](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L119-L122). -- The tx passes `ValidateBasic()`. -- The tx's [timeout_height](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L115-L117) has not been reached if one is specified. -- The tx's [memo](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L110-L113) is <= the max memo characters where [`MaxMemoCharacters = 256`](). -- The tx's [gas_limit](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L211-L213) is > the gas consumed based on the tx's size where [`TxSizeCostPerByte = 10`](https://github.com/cosmos/cosmos-sdk/blob/a429238fc267da88a8548bfebe0ba7fb28b82a13/x/auth/README.md?plain=1#L232). -- The tx's feepayer has enough funds to pay fees for the tx. The tx's feepayer is the feegranter (if specified) or the tx's first signer. Note the [feegrant](https://github.com/cosmos/cosmos-sdk/blob/v0.46.15/x/feegrant/README.md) module is enabled. -- The tx's count of signatures <= the max number of signatures. The max number of signatures is [`TxSigLimit = 7`](https://github.com/cosmos/cosmos-sdk/blob/a429238fc267da88a8548bfebe0ba7fb28b82a13/x/auth/README.md?plain=1#L231). -- The tx's [gas_limit](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L211-L213) is > the gas consumed based on the tx's signatures. -- The tx's [signatures](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/types/tx/signing/signature.go#L10-L26) are valid. For each signature, ensure that the signature's sequence number (a.k.a nonce) matches the account sequence number of the signer. -- The tx's [gas_limit](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L211-L213) is > the gas consumed based on the blob size(s). Since blobs are charged based on the number of shares they occupy, the gas consumed is calculated as follows: `gasToConsume = sharesNeeded(blob) * bytesPerShare * gasPerBlobByte`. Where `bytesPerShare` is a global constant (an alias for [`ShareSize = 512`](https://github.com/celestiaorg/celestia-app/blob/c90e61d5a2d0c0bd0e123df4ab416f6f0d141b7f/pkg/appconsts/global_consts.go#L27-L28)) and `gasPerBlobByte` is a governance parameter that can be modified (the [`DefaultGasPerBlobByte = 8`](https://github.com/celestiaorg/celestia-app/blob/c90e61d5a2d0c0bd0e123df4ab416f6f0d141b7f/pkg/appconsts/initial_consts.go#L16-L18)). -- The tx's total blob size is <= the max blob size. The max blob size is derived from the maximum valid square size. The max valid square size is the minimum of: `GovMaxSquareSize` and `SquareSizeUpperBound`. -- The tx does not contain a message of type [MsgSubmitProposal](https://github.com/cosmos/cosmos-sdk/blob/d6d929843bbd331b885467475bcb3050788e30ca/proto/cosmos/gov/v1/tx.proto#L33-L43) with zero proposal messages. -- The tx is not an IBC packet or update message that has already been processed. - -In addition to the above criteria, the AnteHandler also has a number of side-effects: - -- Tx fees are deducted from the tx's feepayer and added to the fee collector module account. -- Tx priority is calculated based on the smallest denomination of gas price in the tx and set in context. -- The nonce of all tx signers is incremented by 1. diff --git a/specs/src/specs/ante_handler_v2.md b/specs/src/specs/ante_handler_v2.md index a752654cd3..9ebb88a269 100644 --- a/specs/src/specs/ante_handler_v2.md +++ b/specs/src/specs/ante_handler_v2.md @@ -1,25 +1 @@ # AnteHandler v2 - -The AnteHandler chains together several decorators to ensure the following criteria are met for app version 2: - -- The tx does not contain any messages that are unsupported by the current app version. See `MsgVersioningGateKeeper`. -- The tx does not contain any [extension options](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L119-L122). -- The tx passes `ValidateBasic()`. -- The tx's [timeout_height](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L115-L117) has not been reached if one is specified. -- The tx's [memo](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L110-L113) is <= the max memo characters where [`MaxMemoCharacters = 256`](). -- The tx's [gas_limit](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L211-L213) is > the gas consumed based on the tx's size where [`TxSizeCostPerByte = 10`](https://github.com/cosmos/cosmos-sdk/blob/a429238fc267da88a8548bfebe0ba7fb28b82a13/x/auth/README.md?plain=1#L232). -- The tx's feepayer has enough funds to pay fees for the tx. The tx's feepayer is the feegranter (if specified) or the tx's first signer. Note the [feegrant](https://github.com/cosmos/cosmos-sdk/blob/v0.46.15/x/feegrant/README.md) module is enabled. -- The tx's gas price is >= the network minimum gas price where [`NetworkMinGasPrice = 0.000001` utia](https://github.com/celestiaorg/celestia-app/blob/8caa5807df8d15477554eba953bd056ae72d4503/pkg/appconsts/v2/app_consts.go#L9). -- The tx's count of signatures <= the max number of signatures. The max number of signatures is [`TxSigLimit = 7`](https://github.com/cosmos/cosmos-sdk/blob/a429238fc267da88a8548bfebe0ba7fb28b82a13/x/auth/README.md?plain=1#L231). -- The tx's [gas_limit](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L211-L213) is > the gas consumed based on the tx's signatures. -- The tx's [signatures](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/types/tx/signing/signature.go#L10-L26) are valid. For each signature, ensure that the signature's sequence number (a.k.a nonce) matches the account sequence number of the signer. -- The tx's [gas_limit](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L211-L213) is > the gas consumed based on the blob size(s). Since blobs are charged based on the number of shares they occupy, the gas consumed is calculated as follows: `gasToConsume = sharesNeeded(blob) * bytesPerShare * gasPerBlobByte`. Where `bytesPerShare` is a global constant (an alias for [`ShareSize = 512`](https://github.com/celestiaorg/celestia-app/blob/c90e61d5a2d0c0bd0e123df4ab416f6f0d141b7f/pkg/appconsts/global_consts.go#L27-L28)) and `gasPerBlobByte` is a governance parameter that can be modified (the [`DefaultGasPerBlobByte = 8`](https://github.com/celestiaorg/celestia-app/blob/c90e61d5a2d0c0bd0e123df4ab416f6f0d141b7f/pkg/appconsts/initial_consts.go#L16-L18)). -- The tx's total blob share count is <= the max blob share count. The max blob share count is derived from the maximum valid square size. The max valid square size is the minimum of: `GovMaxSquareSize` and `SquareSizeUpperBound`. -- The tx does not contain a message of type [MsgSubmitProposal](https://github.com/cosmos/cosmos-sdk/blob/d6d929843bbd331b885467475bcb3050788e30ca/proto/cosmos/gov/v1/tx.proto#L33-L43) with zero proposal messages. -- The tx is not an IBC packet or update message that has already been processed. - -In addition to the above criteria, the AnteHandler also has a number of side-effects: - -- Tx fees are deducted from the tx's feepayer and added to the fee collector module account. -- Tx priority is calculated based on the smallest denomination of gas price in the tx and set in context. -- The nonce of all tx signers is incremented by 1. diff --git a/specs/src/specs/block_proposer.md b/specs/src/specs/block_proposer.md index e53acec673..a29ade0863 100644 --- a/specs/src/specs/block_proposer.md +++ b/specs/src/specs/block_proposer.md @@ -1,24 +1 @@ -# Honest Block Proposer - - - -This document describes the tasks of an honest block proposer to assemble a new block. Performing these actions is not enforced by the [consensus rules](./consensus.md), so long as a valid block is produced. - -## Deciding on a Block Size - -Before [arranging available data into shares](./data_structures.md#arranging-available-data-into-shares), the size of the original data's square must be determined. - -There are two restrictions on the original data's square size: - -1. It must be at most [`AVAILABLE_DATA_ORIGINAL_SQUARE_MAX`](./consensus.md#constants). -1. It must be a power of 2. - -With these restrictions in mind, the block proposer performs the following actions: - -1. Collect as many transactions and blobs from the mempool as possible, such that the total number of shares is at most [`AVAILABLE_DATA_ORIGINAL_SQUARE_MAX`](./consensus.md#constants). -1. Compute the smallest square size that is a power of 2 that can fit the number of shares. -1. Attempt to lay out the collected transactions and blobs in the current square. - 1. If the square is too small to fit all transactions and blobs (which may happen [due to needing to insert padding between blobs](../specs/data_square_layout.md)) and the square size is smaller than [`AVAILABLE_DATA_ORIGINAL_SQUARE_MAX`](./consensus.md#constants), double the size of the square and repeat the above step. - 1. If the square is too small to fit all transactions and blobs (which may happen [due to needing to insert padding between blobs](../specs/data_square_layout.md)) and the square size is at [`AVAILABLE_DATA_ORIGINAL_SQUARE_MAX`](./consensus.md#constants), drop the transactions and blobs until the data fits within the square. - -Note: the maximum padding shares between blobs should be at most twice the number of blob shares. Doubling the square size (i.e. quadrupling the number of shares in the square) should thus only have to happen at most once. +# Block Proposer diff --git a/specs/src/specs/block_validity_rules.md b/specs/src/specs/block_validity_rules.md index 858d4a8708..204d783573 100644 --- a/specs/src/specs/block_validity_rules.md +++ b/specs/src/specs/block_validity_rules.md @@ -1,66 +1 @@ # Block Validity Rules - -## Introduction - -Unlike most blockchains, Celestia derives most of its functionality from -stateless commitments to data rather than stateful transitions. This means that -the protocol relies heavily on block validity rules. Notably, resource -constrained light clients must be able to detect when a subset of these validity -rules have not been followed in order to avoid making an honest majority -assumption on the consensus network. This has a significant impact on their -design. More information on how light clients can check the invalidity of a -block can be found in the [Fraud Proofs](./fraud_proofs.md) spec. - -> **Note** Celestia relies on CometBFT (formerly tendermint) for consensus, -> meaning that it has single slot finality and is fork-free. Therefore, in order -> to ensure that an invalid block is never committed to, each validator must -> check that each block follows all validity rules before voting. If over two -> thirds of the voting power colludes to break a validity rule, then fraud -> proofs are created for light clients. After light clients verify fraud proofs, -> they halt. - -## Validity Rules - -Before any Celestia specific validation is performed, all CometBFT [block -validation -rules](https://github.com/cometbft/cometbft/blob/v0.34.28/spec/core/data_structures.md#block) -must be followed. - -Notably, this includes verifying data availability. Consensus nodes verify data -availability by simply downloading the entire block. - -> **Note** Light clients only sample a fraction of the block. More details on -> how sampling actually works can be found in the seminal ["Fraud and Data -> Availability Proofs: Maximising Light Client Security and Scaling Blockchains -> with Dishonest Majorities"](https://arxiv.org/abs/1809.09044) and in the -> [`celestia-node`](https://github.com/celestiaorg/celestia-node) repo. - -Celestia specific validity rules can be categorized into multiple groups: - -### Block Rules - -1. In `Block.Data.Txs`, all `BlobTx` transactions must be ordered after non-`BlobTx` transactions. - -### Transaction Validity Rules - -#### App Version 1 - -There is no validity rule that transactions must be decodable so the following rules only apply to transactions that are decodable. - -1. Decodable transactions must pass all [AnteHandler](./ante_handler.md) checks. -1. Decodable non-`BlobTx` transactions must not contain a `MsgPayForBlobs` message. -1. Decodable `BlobTx` transactions must be valid according to the [BlobTx validity rules](../../../x/blob/README.md#validity-rules). - -#### App Version 2 - -1. All transactions must be decodable. -1. All transactions must pass all [AnteHandler](./ante_handler.md) checks. -1. Non-`BlobTx` transactions must not contain a `MsgPayForBlobs` message. -1. `BlobTx` transactions must be valid according to the [BlobTx validity rules](../../../x/blob/README.md#validity-rules). - -### Data Root Construction - -The data root must be calculated from a correctly constructed data square per the [data square layout rules](./data_square_layout.md) - -Figure 1: Erasure Encoding Figure 2: rsmt2d Figure 3: Data Root diff --git a/specs/src/specs/cat_pool.md b/specs/src/specs/cat_pool.md index 22ad999aec..eb6559023a 100644 --- a/specs/src/specs/cat_pool.md +++ b/specs/src/specs/cat_pool.md @@ -1,100 +1 @@ -# Content Addressable Transaction Pool Specification - -- 01.12.2022 | Initial specification (@cmwaters) -- 09.12.2022 | Add Push/Pull mechanics (@cmwaters) - -## Outline - -This document specifies the properties, design and implementation of a content addressable transaction pool (CAT). This protocol is intended as an alternative to the FIFO and Priority mempools currently built-in to the Tendermint consensus protocol. The term content-addressable here, indicates that each transaction is identified by a smaller, unique tag (in this case a sha256 hash). These tags are broadcast among the transactions as a means of more compactly indicating which peers have which transactions. Tracking what each peer has aims at reducing the amount of duplication. In a network without content tracking, a peer may receive as many duplicate transactions as peers connected to. The tradeoff here therefore is that the transactions are significantly larger than the tag such that the sum of the data saved sending what would be duplicated transactions is larger than the sum of sending each peer a tag. - -## Purpose - -The objective of such a protocol is to transport transactions from the author (usually a client) to a proposed block, optimizing both latency and throughput i.e. how quickly can a transaction be proposed (and committed) and how many transactions can be transported into a block at once. - -Typically the mempool serves to receive inbound transactions via an RPC endpoint, gossip them to all nodes in the network (regardless of whether they are capable of proposing a block or not), and stage groups of transactions to both consensus and the application to be included in a block. - -## Assumptions - -The following are assumptions inherited from existing Tendermint mempool protocols: - -- `CheckTx` should be seen as a simple gatekeeper to what transactions enter the pool to be gossiped and staged. It is non-deterministic: one node may reject a transaction that another node keeps. -- Applications implementing `CheckTx` are responsible for replay protection (i.e. the same transaction being present in multiple blocks). The mempool ensures that within the same block, no duplicate transactions can exist. -- The underlying p2p layer guarantees eventually reliable broadcast. A transaction need only be sent once to eventually reach the target peer. - -## Messages - -The CAT protocol extends on the existing mempool implementations by introducing two new protobuf messages: - -```protobuf -message SeenTx { - bytes tx_key = 1; - optional string from = 2; -} - -message WantTx { - bytes tx_key = 1; -} -``` - -Both `SeenTx` and `WantTx` contain the sha256 hash of the raw transaction bytes. `SeenTx` also contains an optional `p2p.ID` that corresponds to the peer that the node received the tx from. The only validation for both is that the byte slice of the `tx_key` MUST have a length of 32. - -Both messages are sent across a new channel with the ID: `byte(0x31)`. This enables cross compatibility as discussed in greater detail below. - -> **Note:** -> The term `SeenTx` is used over the more common `HasTx` because the transaction pool contains sophisticated eviction logic. TTL's, higher priority transactions and reCheckTx may mean that a transaction pool *had* a transaction but does not have it any more. Semantically it's more appropriate to use `SeenTx` to imply not the presence of a transaction but that the node has seen it and dealt with it accordingly. - -## Outbound logic - -A node in the protocol has two distinct modes: "broadcast" and "request/response". When a node receives a transaction via RPC (or specifically through `CheckTx`), it assumed that it is the only recipient from that client and thus will immediately send that transaction, after validation, to all connected peers. Afterwards, only "request/response" is used to disseminate that transaction to everyone else. - -> **Note:** -> Given that one can configure a mempool to switch off broadcast, there are no guarantees when a client submits a transaction via RPC and no error is returned that it will find its way into a proposers transaction pool. - -A `SeenTx` is broadcasted to ALL nodes upon receiving a "new" transaction from a peer. The transaction pool does not need to track every unique inbound transaction, therefore "new" is identified as: - -- The node does not currently have the transaction -- The node did not recently reject the transacton or has recently seen the same transaction committed (subject to the size of the cache) -- The node did not recently evict the transaction (subject to the size of the cache) - -Given this criteria, it is feasible, yet unlikely that a node receives two `SeenTx` messages from the same peer for the same transaction. - -A `SeenTx` MAY be sent for each transaction currently in the transaction pool when a connection with a peer is first established. This acts as a mechanism for syncing pool state across peers. - -The `SeenTx` message MUST only be broadcasted after validation and storage. Although it is possible that a node later drops a transaction under load shedding, a `SeenTx` should give as strong guarantees as possible that the node can be relied upon by others that don't yet have the transcation to obtain it. - -> **Note:** -> Inbound transactions submitted via the RPC do not trigger a `SeenTx` message as it is assumed that the node is the first to see the transaction and by gossiping it to others it is implied that the node has seen the transaction. - -A `WantTx` message is always sent point to point and never broadcasted. A `WantTx` MUST only be sent after receiving a `SeenTx` message from that peer. There is one exception which is that a `WantTx` MAY also be sent by a node after receiving an identical `WantTx` message from a peer that had previously received the nodes `SeenTx` but which after the lapse in time, did no longer exist in the nodes transaction pool. This provides an optional synchronous method for communicating that a node no longer has a transaction rather than relying on the defaulted asynchronous approach which is to wait for a period of time and try again with a new peer. - -`WantTx` must be tracked. A node SHOULD not send multiple `WantTx`s to multiple peers for the same transaction at once but wait for a period that matches the expected network latency before rerequesting the transaction to another peer. - -## Inbound logic - -Transaction pools are solely run in-memory; thus when a node stops, all transactions are discarded. To avoid the scenario where a node restarts and does not receive transactions because other nodes recorded a `SeenTx` message from their previous run, each transaction pool should track peer state based **per connection** and not per `NodeID`. - -Upon receiving a `Txs` message: - -- Check whether it is in response to a request or simply an unsolicited broadcast -- Validate the tx against current resources and the applications `CheckTx` -- If rejected or evicted, mark accordingly -- If successful, send a `SeenTx` message to all connected peers excluding the original sender. If it was from an initial broadcast, the `SeenTx` should populate the `From` field with the `p2p.ID` of the recipient else if it is in response to a request `From` should remain empty. - -Upon receiving a `SeenTx` message: - -- It should mark the peer as having seen the message. -- If the node has recently rejected that transaction, it SHOULD ignore the message. -- If the node already has the transaction, it SHOULD ignore the message. -- If the node does not have the transaction but recently evicted it, it MAY choose to rerequest the transaction if it has adequate resources now to process it. -- If the node has not seen the transaction or does not have any pending requests for that transaction, it can do one of two things: - - It MAY immediately request the tx from the peer with a `WantTx`. - - If the node is connected to the peer specified in `FROM`, it is likely, from a non-byzantine peer, that the node will also shortly receive the transaction from the peer. It MAY wait for a `Txs` message for a bounded amount of time but MUST eventually send a `WantMsg` message to either the original peer or any other peer that *has* the specified transaction. - -Upon receiving a `WantTx` message: - -- If it has the transaction, it MUST respond with a `Txs` message containing that transaction. -- If it does not have the transaction, it MAY respond with an identical `WantTx` or rely on the timeout of the peer that requested the transaction to eventually ask another peer. - -## Compatibility - -CAT has Go API compatibility with the existing two mempool implementations. It implements both the `Reactor` interface required by Tendermint's P2P layer and the `Mempool` interface used by `consensus` and `rpc`. CAT is currently network compatible with existing implementations (by using another channel), but the protocol is unaware that it is communicating with a different mempool and that `SeenTx` and `WantTx` messages aren't reaching those peers thus it is recommended that the entire network use CAT. +# CAT Pool diff --git a/specs/src/specs/consensus.md b/specs/src/specs/consensus.md index 69dc17f136..5cfc2c950c 100644 --- a/specs/src/specs/consensus.md +++ b/specs/src/specs/consensus.md @@ -1,717 +1 @@ -# Consensus Rules - - - -## System Parameters - -### Units - -| name | SI | value | description | -|------|-------|---------|---------------------| -| `1u` | `1u` | `10**0` | `1` unit. | -| `2u` | `k1u` | `10**3` | `1000` units. | -| `3u` | `M1u` | `10**6` | `1000000` units. | -| `4u` | `G1u` | `10**9` | `1000000000` units. | - -### Constants - -| name | type | value | unit | description | -|-----------------------------------------|----------|--------------|---------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `AVAILABLE_DATA_ORIGINAL_SQUARE_MAX` | `uint64` | | `share` | Maximum number of rows/columns of the original data [shares](data_structures.md#share) in [square layout](data_structures.md#arranging-available-data-into-shares). | -| `AVAILABLE_DATA_ORIGINAL_SQUARE_TARGET` | `uint64` | | `share` | Target number of rows/columns of the original data [shares](data_structures.md#share) in [square layout](data_structures.md#arranging-available-data-into-shares). | -| `BLOCK_TIME` | `uint64` | | second | Block time, in seconds. | -| `CHAIN_ID` | `string` | `"Celestia"` | | Chain ID. Each chain assigns itself a (unique) ID. | -| `GENESIS_COIN_COUNT` | `uint64` | `10**8` | `4u` | `(= 100000000)` Number of coins at genesis. | -| `MAX_GRAFFITI_BYTES` | `uint64` | `32` | `byte` | Maximum size of transaction graffiti, in bytes. | -| `MAX_VALIDATORS` | `uint16` | `64` | | Maximum number of active validators. | -| `NAMESPACE_VERSION_SIZE` | `int` | `1` | `byte` | Size of namespace version in bytes. | -| `NAMESPACE_ID_SIZE` | `int` | `28` | `byte` | Size of namespace ID in bytes. | -| `NAMESPACE_SIZE` | `int` | `29` | `byte` | Size of namespace in bytes. | -| `NAMESPACE_ID_MAX_RESERVED` | `uint64` | `255` | | Value of maximum reserved namespace (inclusive). 1 byte worth of IDs. | -| `SEQUENCE_BYTES` | `uint64` | `4` | `byte` | The number of bytes used to store the sequence length in the first share of a sequence | -| `SHARE_INFO_BYTES` | `uint64` | `1` | `byte` | The number of bytes used for [share](data_structures.md#share) information | -| `SHARE_RESERVED_BYTES` | `uint64` | `4` | `byte` | The number of bytes used to store the index of the first transaction in a transaction share. Must be able to represent any integer up to and including `SHARE_SIZE - 1`. | -| `SHARE_SIZE` | `uint64` | `512` | `byte` | Size of transaction and blob [shares](data_structures.md#share), in bytes. | -| `STATE_SUBTREE_RESERVED_BYTES` | `uint64` | `1` | `byte` | Number of bytes reserved to identify state subtrees. | -| `UNBONDING_DURATION` | `uint32` | | `block` | Duration, in blocks, for unbonding a validator or delegation. | -| `v1.Version` | `uint64` | `1` | | First version of the application. Breaking changes (hard forks) must update this parameter. | -| `v2.Version` | `uint64` | `2` | | Second version of the application. Breaking changes (hard forks) must update this parameter. | -| `VERSION_BLOCK` | `uint64` | `1` | | Version of the Celestia chain. Breaking changes (hard forks) must update this parameter. | - -### Rewards and Penalties - -| name | type | value | unit | description | -|--------------------------|----------|-------------|--------|---------------------------------------------------------| -| `SECONDS_PER_YEAR` | `uint64` | `31536000` | second | Seconds per year. Omit leap seconds. | -| `TARGET_ANNUAL_ISSUANCE` | `uint64` | `2 * 10**6` | `4u` | `(= 2000000)` Target number of coins to issue per year. | - -## Leader Selection - -Refer to the CometBFT specifications for [proposer selection procedure](https://docs.cometbft.com/v0.34/spec/consensus/proposer-selection). - -## Fork Choice - -The Tendermint consensus protocol is fork-free by construction under an honest majority of stake assumption. - -If a block has a [valid commit](#blocklastcommit), it is part of the canonical chain. If equivocation evidence is detected for more than 1/3 of voting power, the node must halt. See [proof of fork accountability](https://docs.cometbft.com/v0.34/spec/consensus/consensus#proof-of-fork-accountability). - -## Block Validity - -The validity of a newly-seen block, `block`, is determined by two components, detailed in subsequent sections: - -1. [Block structure](#block-structure): whether the block header is valid, and data in a block is arranged into a valid and matching data root (i.e. syntax). -1. [State transition](#state-transitions): whether the application of transactions in the block produces a matching and valid state root (i.e. semantics). - -Pseudocode in this section is not in any specific language and should be interpreted as being in a neutral and sane language. - -## Block Structure - -Before executing [state transitions](#state-transitions), the structure of the [block](./data_structures.md#block) must be verified. - -The following block fields are acquired from the network and parsed (i.e. [deserialized](./data_structures.md#serialization)). If they cannot be parsed, the block is ignored but is not explicitly considered invalid by consensus rules. Further implications of ignoring a block are found in the [networking spec](./networking.md). - -1. [block.header](./data_structures.md#header) -1. [block.availableDataHeader](./data_structures.md#availabledataheader) -1. [block.lastCommit](./data_structures.md#commit) - -If the above fields are parsed successfully, the available data `block.availableData` is acquired in erasure-coded form as [a list of share rows](./networking.md#availabledata), then parsed. If it cannot be parsed, the block is ignored but not explicitly invalid, as above. - -### `block.header` - -The [block header](./data_structures.md#header) `block.header` (`header` for short) is the first thing that is downloaded from the new block, and commits to everything inside the block in some way. For previous block `prev` (if `prev` is not known, then the block is ignored), and previous block header `prev.header`, the following checks must be `true`: - -`availableDataOriginalSquareSize` is computed as described [here](./data_structures.md#header). - -1. `header.height` == `prev.header.height + 1`. -1. `header.timestamp` > `prev.header.timestamp`. -1. `header.lastHeaderHash` == the [header hash](./data_structures.md#header) of `prev`. -1. `header.lastCommitHash` == the [hash](./data_structures.md#hashing) of `lastCommit`. -1. `header.consensusHash` == the value computed [here](./data_structures.md#consensus-parameters). -1. `header.stateCommitment` == the root of the state, computed [with the application of all state transitions in this block](#state-transitions). -1. `availableDataOriginalSquareSize` <= [`AVAILABLE_DATA_ORIGINAL_SQUARE_MAX`](#constants). -1. `header.availableDataRoot` == the [Merkle root](./data_structures.md#binary-merkle-tree) of the tree with the row and column roots of `block.availableDataHeader` as leaves. -1. `header.proposerAddress` == the [leader](#leader-selection) for `header.height`. - -### `block.availableDataHeader` - -The [available data header](./data_structures.md#availabledataheader) `block.availableDataHeader` (`availableDataHeader` for short) is then processed. This commits to the available data, which is only downloaded after the [consensus commit](#blocklastcommit) is processed. The following checks must be `true`: - -1. Length of `availableDataHeader.rowRoots` == `availableDataOriginalSquareSize * 2`. -1. Length of `availableDataHeader.colRoots` == `availableDataOriginalSquareSize * 2`. -1. The length of each element in `availableDataHeader.rowRoots` and `availableDataHeader.colRoots` must be [`32`](./data_structures.md#hashing). - -### `block.lastCommit` - -The last [commit](./data_structures.md#commit) `block.lastCommit` (`lastCommit` for short) is processed next. This is the Tendermint commit (i.e. polka of votes) _for the previous block_. For previous block `prev` and previous block header `prev.header`, the following checks must be `true`: - -1. `lastCommit.height` == `prev.header.height`. -1. `lastCommit.round` >= `1`. -1. `lastCommit.headerHash` == the [header hash](./data_structures.md#header) of `prev`. -1. Length of `lastCommit.signatures` <= [`MAX_VALIDATORS`](#constants). -1. Each of `lastCommit.signatures` must be a valid [CommitSig](./data_structures.md#commitsig) -1. The sum of the votes for `prev` in `lastCommit` must be at least 2/3 (rounded up) of the voting power of `prev`'s next validator set. - -### `block.availableData` - -The block's [available data](./data_structures.md#availabledata) (analogous to transactions in contemporary blockchain designs) `block.availableData` (`availableData` for short) is finally processed. The [list of share rows](./networking.md#availabledata) is parsed into the [actual data structures](./data_structures.md#availabledata) using the reverse of [the process to encode available data into shares](./data_structures.md#arranging-available-data-into-shares); if parsing fails here, the block is invalid. - -Once parsed, the following checks must be `true`: - -1. The commitments of the [erasure-coded extended](./data_structures.md#2d-reed-solomon-encoding-scheme) `availableData` must match those in `header.availableDataHeader`. Implicitly, this means that both rows and columns must be ordered lexicographically by namespace since they are committed to in a [Namespace Merkle Tree](data_structures.md#namespace-merkle-tree). -1. Length of `availableData.intermediateStateRootData` == length of `availableData.transactionData` + length of `availableData.payForBlobData` + 2. (Two additional state transitions are the [begin](#begin-block) and [end block](#end-block) implicit transitions.) - -## State Transitions - -Once the basic structure of the block [has been validated](#block-structure), state transitions must be applied to compute the new state and state root. - -For this section, the variable `state` represents the [state tree](./data_structures.md#state), with `state.accounts[k]`, `state.inactiveValidatorSet[k]`, `state.activeValidatorSet[k]`, and `state.delegationSet[k]` being shorthand for the leaf in the state tree in the [accounts, inactive validator set, active validator set, and delegation set subtrees](./data_structures.md#state) with [pre-hashed key](./data_structures.md#state) `k`. E.g. `state.accounts[a]` is shorthand for `state[(ACCOUNTS_SUBTREE_ID << 8*(32-STATE_SUBTREE_RESERVED_BYTES)) | ((-1 >> 8*STATE_SUBTREE_RESERVED_BYTES) & hash(a))]`. - -State transitions are applied in the following order: - -1. [Begin block](#begin-block). -1. [Transactions](#blockavailabledatatransactiondata). -1. [End block](#end-block). - -### `block.availableData.transactionData` - -Transactions are applied to the state. Note that _transactions_ mutate the state (essentially, the validator set and minimal balances), while _blobs_ do not. - -`block.availableData.transactionData` is simply a list of [WrappedTransaction](./data_structures.md#wrappedtransaction)s. For each wrapped transaction in this list, `wrappedTransaction`, with index `i` (starting from `0`), the following checks must be `true`: - -1. `wrappedTransaction.index` == `i`. - -For `wrappedTransaction`'s [transaction](./data_structures.md#transaction) `transaction`, the following checks must be `true`: - -1. `transaction.signature` must be a [valid signature](./data_structures.md#public-key-cryptography) over `transaction.signedTransactionData`. - -Finally, each `wrappedTransaction` is processed depending on [its transaction type](./data_structures.md#signedtransactiondata). These are specified in the next subsections, where `tx` is short for `transaction.signedTransactionData`, and `sender` is the recovered signing [address](./data_structures.md#address). We will define a few helper functions: - -```py -tipCost(y, z) = y * z -totalCost(x, y, z) = x + tipCost(y, z) -``` - -where `x` above is the amount of coins sent by the transaction authorizer, `y` above is the tip rate set in the transaction, and `z` above is the measure of the block space used by the transaction (i.e. size in bytes). - -Four additional helper functions are defined to manage the [validator queue](./data_structures.md#validator): - -1. `findFromQueue(power)`, which returns the address of the last validator in the [validator queue](./data_structures.md#validator) with voting power greater than or equal to `power`, or `0` if the queue is empty or no validators in the queue have at least `power` voting power. -1. `parentFromQueue(address)`, which returns the address of the parent in the validator queue of the validator with address `address`, or `0` if `address` is not in the queue or is the head of the queue. -1. `validatorQueueInsert`, defined as - -```py -function validatorQueueInsert(validator) - # Insert the new validator into the linked list - parent = findFromQueue(validator.votingPower) - if parent != 0 - if state.accounts[parent].status == AccountStatus.ValidatorBonded - validator.next = state.activeValidatorSet[parent].next - state.activeValidatorSet[parent].next = sender - else - validator.next = state.inactiveValidatorSet[parent].next - state.inactiveValidatorSet[parent].next = sender - else - validator.next = state.validatorQueueHead - state.validatorQueueHead = sender -``` - - -4. `validatorQueueRemove`, defined as - -```py -function validatorQueueRemove(validator, sender) - # Remove existing validator from the linked list - parent = parentFromQueue(sender) - if parent != 0 - if state.accounts[parent].status == AccountStatus.ValidatorBonded - state.activeValidatorSet[parent].next = validator.next - validator.next = 0 - else - state.inactiveValidatorSet[parent].next = validator.next - validator.next = 0 - else - state.validatorQueueHead = validator.next - validator.next = 0 -``` - -Note that light clients cannot perform a linear search through a linked list, and are instead provided logarithmic proofs (e.g. in the case of `parentFromQueue`, a proof to the parent is provided, which should have `address` as its next validator). - -In addition, three helper functions to manage the [blob paid list](./data_structures.md#blobpaid): - -1. `findFromBlobPaidList(start)`, which returns the transaction ID of the last transaction in the [blob paid list](./data_structures.md#blobpaid) with `finish` greater than `start`, or `0` if the list is empty or no transactions in the list have at least `start` `finish`. -1. `parentFromBlobPaidList(txid)`, which returns the transaction ID of the parent in the blob paid list of the transaction with ID `txid`, or `0` if `txid` is not in the list or is the head of the list. -1. `blobPaidListInsert`, defined as - -```py -function blobPaidListInsert(tx, txid) - # Insert the new transaction into the linked list - parent = findFromBlobPaidList(tx.blobStartIndex) - state.blobsPaid[txid].start = tx.blobStartIndex - numShares = ceil(tx.blobSize / SHARE_SIZE) - state.blobsPaid[txid].finish = tx.blobStartIndex + numShares - 1 - if parent != 0 - state.blobsPaid[txid].next = state.blobsPaid[parent].next - state.blobsPaid[parent].next = txid - else - state.blobsPaid[txid].next = state.blobPaidHead - state.blobPaidHead = txid -``` - -We define a helper function to compute F1 entries: - -```py -function compute_new_entry(reward, power) - if power == 0 - return 0 - return reward // power -``` - -After applying a transaction, the new state state root is computed. - -#### SignedTransactionDataTransfer - -```py -bytesPaid = len(tx) -``` - -The following checks must be `true`: - -1. `tx.type` == [`TransactionType.Transfer`](./data_structures.md#signedtransactiondata). -1. `totalCost(tx.amount, tx.fee.tipRate, bytesPaid)` <= `state.accounts[sender].balance`. -1. `tx.nonce` == `state.accounts[sender].nonce + 1`. - -Apply the following to the state: - -```py -state.accounts[sender].nonce += 1 - -state.accounts[sender].balance -= totalCost(tx.amount, tx.fee.tipRate, bytesPaid) -state.accounts[tx.to].balance += tx.amount - -state.activeValidatorSet.proposerBlockReward += tipCost(bytesPaid) -``` - -#### SignedTransactionDataMsgPayForData - -```py -bytesPaid = len(tx) + tx.blobSize -currentStartFinish = state.blobsPaid[findFromBlobPaidList(tx.blobStartIndex)] -parentStartFinish = state.blobsPaid[parentFromBlobPaidList(findFromBlobPaidList(tx.blobStartIndex))] -``` - -The following checks must be `true`: - -1. `tx.type` == [`TransactionType.MsgPayForData`](./data_structures.md#signedtransactiondata). -1. `totalCost(0, tx.fee.tipRate, bytesPaid)` <= `state.accounts[sender].balance`. -1. `tx.nonce` == `state.accounts[sender].nonce + 1`. -1. The `ceil(tx.blobSize / SHARE_SIZE)` shares starting at index `tx.blobStartIndex` must: - 1. Have namespace `tx.blobNamespace`. -1. `tx.blobShareCommitment` == computed as described [here](./data_structures.md#signedtransactiondatamsgpayfordata). -1. `parentStartFinish.finish` < `tx.blobStartIndex`. -1. `currentStartFinish.start` == `0` or `currentStartFinish.start` > `tx.blobStartIndex + ceil(tx.blobSize / SHARE_SIZE)`. - -Apply the following to the state: - -```py -state.accounts[sender].nonce += 1 -state.accounts[sender].balance -= totalCost(tx.amount, tx.fee.tipRate, bytesPaid) - -blobPaidListInsert(tx, id(tx)) - -state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) -``` - -#### SignedTransactionDataCreateValidator - -```py -bytesPaid = len(tx) -``` - -The following checks must be `true`: - -1. `tx.type` == [`TransactionType.CreateValidator`](./data_structures.md#signedtransactiondata). -1. `totalCost(0, tx.fee.tipRate, bytesPaid)` <= `state.accounts[sender].balance`. -1. `tx.nonce` == `state.accounts[sender].nonce + 1`. -1. `tx.commissionRate.denominator > 0`. -1. `tx.commissionRate.numerator <= tx.commissionRate.denominator`. -1. `state.accounts[sender].status` == `AccountStatus.None`. - -Apply the following to the state: - -```py -state.accounts[sender].nonce += 1 -state.accounts[sender].balance -= totalCost(0, tx.fee.tipRate, bytesPaid) -state.accounts[sender].status = AccountStatus.ValidatorQueued - -validator = new Validator -validator.commissionRate = tx.commissionRate -validator.delegatedCount = 0 -validator.votingPower = 0 -validator.pendingRewards = 0 -validator.latestEntry = PeriodEntry(0) -validator.unbondingHeight = 0 -validator.isSlashed = false - -validatorQueueInsert(validator) - -state.inactiveValidatorSet[sender] = validator - -state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) -``` - -#### SignedTransactionDataBeginUnbondingValidator - -```py -bytesPaid = len(tx) -``` - -The following checks must be `true`: - -1. `tx.type` == [`TransactionType.BeginUnbondingValidator`](./data_structures.md#signedtransactiondata). -1. `totalCost(0, tx.fee.tipRate, bytesPaid)` <= `state.accounts[sender].balance`. -1. `tx.nonce` == `state.accounts[sender].nonce + 1`. -1. `state.accounts[sender].status` == `AccountStatus.ValidatorQueued` or `state.accounts[sender].status` == `AccountStatus.ValidatorBonded`. - -Apply the following to the state: - -```py -state.accounts[sender].nonce += 1 -state.accounts[sender].balance -= totalCost(0, tx.fee.tipRate, bytesPaid) -state.accounts[sender].status = ValidatorStatus.Unbonding - -if state.accounts[sender].status == AccountStatus.ValidatorQueued - validator = state.inactiveValidatorSet[sender] -else if state.accounts[sender].status == AccountStatus.ValidatorBonded - validator = state.activeValidatorSet[sender] - delete state.activeValidatorSet[sender] - -validator.unbondingHeight = block.height + 1 -validator.latestEntry += compute_new_entry(validator.pendingRewards, validator.votingPower) -validator.pendingRewards = 0 - -validatorQueueRemove(validator, sender) - -state.inactiveValidatorSet[sender] = validator - -state.activeValidatorSet.activeVotingPower -= validator.votingPower - -state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) -``` - -#### SignedTransactionDataUnbondValidator - -```py -bytesPaid = len(tx) -``` - -The following checks must be `true`: - -1. `tx.type` == [`TransactionType.UnbondValidator`](./data_structures.md#signedtransactiondata). -1. `totalCost(0, tx.fee.tipRate, bytesPaid)` <= `state.accounts[sender].balance`. -1. `tx.nonce` == `state.accounts[sender].nonce + 1`. -1. `state.accounts[sender].status` == `AccountStatus.ValidatorUnbonding`. -1. `state.inactiveValidatorSet[sender].unbondingHeight + UNBONDING_DURATION` < `block.height`. - -Apply the following to the state: - -```py -validator = state.inactiveValidatorSet[sender] - -state.accounts[sender].nonce += 1 -state.accounts[sender].balance -= totalCost(0, tx.fee.tipRate, bytesPaid) -state.accounts[sender].status = AccountStatus.ValidatorUnbonded - -state.accounts[sender].balance += validator.commissionRewards - -state.inactiveValidatorSet[sender] = validator - -if validator.delegatedCount == 0 - state.accounts[sender].status = AccountStatus.None - delete state.inactiveValidatorSet[sender] - -state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) -``` - -#### SignedTransactionDataCreateDelegation - -```py -bytesPaid = len(tx) -``` - -The following checks must be `true`: - -1. `tx.type` == [`TransactionType.CreateDelegation`](./data_structures.md#signedtransactiondata). -1. `totalCost(tx.amount, tx.fee.tipRate, bytesPaid)` <= `state.accounts[sender].balance`. -1. `state.accounts[tx.to].status` == `AccountStatus.ValidatorQueued` or `state.accounts[tx.to].status` == `AccountStatus.ValidatorBonded`. -1. `tx.nonce` == `state.accounts[sender].nonce + 1`. -1. `state.accounts[sender].status` == `AccountStatus.None`. - -Apply the following to the state: - -```py -state.accounts[sender].nonce += 1 -state.accounts[sender].balance -= totalCost(tx.amount, tx.fee.tipRate, bytesPaid) -state.accounts[sender].status = AccountStatus.DelegationBonded - -if state.accounts[tx.to].status == AccountStatus.ValidatorQueued - validator = state.inactiveValidatorSet[tx.to] -else if state.accounts[tx.to].status == AccountStatus.ValidatorBonded - validator = state.activeValidatorSet[tx.to] - -delegation = new Delegation -delegation.status = DelegationStatus.Bonded -delegation.validator = tx.to -delegation.stakedBalance = tx.amount -delegation.beginEntry = validator.latestEntry -delegation.endEntry = PeriodEntry(0) -delegation.unbondingHeight = 0 - -validator.latestEntry += compute_new_entry(validator.pendingRewards, validator.votingPower) -validator.pendingRewards = 0 -validator.delegatedCount += 1 -validator.votingPower += tx.amount - -# Update the validator in the linked list by first removing then inserting -validatorQueueRemove(validator, delegation.validator) -validatorQueueInsert(validator) - -state.delegationSet[sender] = delegation - -if state.accounts[tx.to].status == AccountStatus.ValidatorQueued - state.inactiveValidatorSet[tx.to] = validator -else if state.accounts[tx.to].status == AccountStatus.ValidatorBonded - state.activeValidatorSet[tx.to] = validator - state.activeValidatorSet.activeVotingPower += tx.amount - -state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) -``` - -#### SignedTransactionDataBeginUnbondingDelegation - -```py -bytesPaid = len(tx) -``` - -The following checks must be `true`: - -1. `tx.type` == [`TransactionType.BeginUnbondingDelegation`](./data_structures.md#signedtransactiondata). -1. `totalCost(0, tx.fee.tipRate, bytesPaid)` <= `state.accounts[sender].balance`. -1. `tx.nonce` == `state.accounts[sender].nonce + 1`. -1. `state.accounts[sender].status` == `AccountStatus.DelegationBonded`. - -Apply the following to the state: - -```py -state.accounts[sender].nonce += 1 -state.accounts[sender].balance -= totalCost(0, tx.fee.tipRate, bytesPaid) -state.accounts[sender].status = AccountStatus.DelegationUnbonding - -delegation = state.delegationSet[sender] - -if state.accounts[delegation.validator].status == AccountStatus.ValidatorQueued || - state.accounts[delegation.validator].status == AccountStatus.ValidatorUnbonding || - state.accounts[delegation.validator].status == AccountStatus.ValidatorUnbonded - validator = state.inactiveValidatorSet[delegation.validator] -else if state.accounts[delegation.validator].status == AccountStatus.ValidatorBonded - validator = state.activeValidatorSet[delegation.validator] - -delegation.status = DelegationStatus.Unbonding -delegation.endEntry = validator.latestEntry -delegation.unbondingHeight = block.height + 1 - -validator.latestEntry += compute_new_entry(validator.pendingRewards, validator.votingPower) -validator.pendingRewards = 0 -validator.delegatedCount -= 1 -validator.votingPower -= delegation.stakedBalance - -# Update the validator in the linked list by first removing then inserting -# Only do this if the validator is actually in the queue (i.e. bonded or queued) -if state.accounts[delegation.validator].status == AccountStatus.ValidatorBonded || - state.accounts[delegation.validator].status == AccountStatus.ValidatorQueued - validatorQueueRemove(validator, delegation.validator) - validatorQueueInsert(validator) - -state.delegationSet[sender] = delegation - -if state.accounts[delegation.validator].status == AccountStatus.ValidatorQueued || - state.accounts[delegation.validator].status == AccountStatus.ValidatorUnbonding || - state.accounts[delegation.validator].status == AccountStatus.ValidatorUnbonded - state.inactiveValidatorSet[delegation.validator] = validator -else if state.accounts[delegation.validator].status == AccountStatus.ValidatorBonded - state.activeValidatorSet[delegation.validator] = validator - state.activeValidatorSet.activeVotingPower -= delegation.stakedBalance - -state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) -``` - -#### SignedTransactionDataUnbondDelegation - -```py -bytesPaid = len(tx) -``` - -The following checks must be `true`: - -1. `tx.type` == [`TransactionType.UnbondDelegation`](./data_structures.md#signedtransactiondata). -1. `totalCost(0, bytesPaid)` <= `state.accounts[sender].balance`. -1. `tx.nonce` == `state.accounts[sender].nonce + 1`. -1. `state.accounts[sender].status` == `AccountStatus.DelegationUnbonding`. -1. `state.delegationSet[sender].unbondingHeight + UNBONDING_DURATION` < `block.height`. - -Apply the following to the state: - -```py -delegation = state.accounts[sender].delegationInfo - -state.accounts[sender].nonce += 1 -state.accounts[sender].balance -= totalCost(0, tx.fee.tipRate, bytesPaid) -state.accounts[sender].status = None - -# Return the delegated stake -state.accounts[sender].balance += delegation.stakedBalance -# Also disperse rewards (commission has already been levied) -state.accounts[sender].balance += delegation.stakedBalance * (delegation.endEntry - delegation.beginEntry) - -if state.accounts[delegation.validator].status == AccountStatus.ValidatorQueued || - state.accounts[delegation.validator].status == AccountStatus.ValidatorUnbonding - state.accounts[delegation.validator].status == AccountStatus.ValidatorUnbonded - validator = state.inactiveValidatorSet[delegation.validator] -else if state.accounts[delegation.validator].status == AccountStatus.ValidatorBonded - validator = state.activeValidatorSet[delegation.validator] - -if validator.delegatedCount == 0 && - state.accounts[delegation.validator].status == AccountStatus.ValidatorUnbonded - state.accounts[delegation.validator].status = AccountStatus.None - delete state.inactiveValidatorSet[delegation.validator] - -delete state.accounts[sender].delegationInfo - -state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) -``` - -#### SignedTransactionDataBurn - -```py -bytesPaid = len(tx) -``` - -The following checks must be `true`: - -1. `tx.type` == [`TransactionType.Burn`](./data_structures.md#signedtransactiondata). -1. `totalCost(tx.amount, bytesPaid)` <= `state.accounts[sender].balance`. -1. `tx.nonce` == `state.accounts[sender].nonce + 1`. - -Apply the following to the state: - -```py -state.accounts[sender].nonce += 1 -state.accounts[sender].balance -= totalCost(tx.amount, tx.fee.tipRate, bytesPaid) - -state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) -``` - -#### SignedTransactionRedelegateCommission - -```py -bytesPaid = len(tx) -``` - -The following checks must be `true`: - -1. `tx.type` == [`TransactionType.RedelegateCommission`](./data_structures.md#signedtransactiondata). -1. `totalCost(0, tx.fee.tipRate, bytesPaid)` <= `state.accounts[sender].balance`. -1. `tx.nonce` == `state.accounts[sender].nonce + 1`. -1. `state.accounts[tx.to].status` == `AccountStatus.DelegationBonded`. -1. `state.accounts[sender].status` == `AccountStatus.ValidatorBonded`. - -Apply the following to the state: - -```py -state.accounts[sender].nonce += 1 -state.accounts[sender].balance -= totalCost(0, tx.fee.tipRate, bytesPaid) - -delegation = state.delegationSet[tx.to] -validator = state.activeValidatorSet[delegation.validator] - -# Force-redelegate pending rewards for delegation -pendingRewards = delegation.stakedBalance * (validator.latestEntry - delegation.beginEntry) -delegation.stakedBalance += pendingRewards -delegation.beginEntry = validator.latestEntry - -validator.latestEntry += compute_new_entry(validator.pendingRewards, validator.votingPower) -validator.pendingRewards = 0 - -# Assign pending commission rewards to delegation -commissionRewards = validator.commissionRewards -delegation.stakedBalance += commissionRewards -validator.commissionRewards = 0 - -# Update voting power -validator.votingPower += pendingRewards + commissionRewards -state.activeValidatorSet.activeVotingPower += pendingRewards + commissionRewards - -state.delegationSet[tx.to] = delegation -state.activeValidatorSet[delegation.validator] = validator - -state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) -``` - -#### SignedTransactionRedelegateReward - -```py -bytesPaid = len(tx) -``` - -The following checks must be `true`: - -1. `tx.type` == [`TransactionType.RedelegateReward`](./data_structures.md#signedtransactiondata). -1. `totalCost(0, tx.fee.tipRate, bytesPaid)` <= `state.accounts[sender].balance`. -1. `tx.nonce` == `state.accounts[sender].nonce + 1`. -1. `state.accounts[sender].status` == `AccountStatus.DelegationBonded`. -1. `state.accounts[state.delegationSet[sender].validator].status` == `AccountStatus.ValidatorBonded`. - -Apply the following to the state: - -```py -state.accounts[sender].nonce += 1 -state.accounts[sender].balance -= totalCost(0, tx.fee.tipRate, bytesPaid) - -delegation = state.delegationSet[sender] -validator = state.activeValidatorSet[delegation.validator] - -# Redelegate pending rewards for delegation -pendingRewards = delegation.stakedBalance * (validator.latestEntry - delegation.beginEntry) -delegation.stakedBalance += pendingRewards -delegation.beginEntry = validator.latestEntry - -validator.latestEntry += compute_new_entry(validator.pendingRewards, validator.votingPower) -validator.pendingRewards = 0 - -# Update voting power -validator.votingPower += pendingRewards -state.activeValidatorSet.activeVotingPower += pendingRewards - -state.delegationSet[sender] = delegation -state.activeValidatorSet[delegation.validator] = validator - -state.activeValidatorSet.proposerBlockReward += tipCost(tx.fee.tipRate, bytesPaid) -``` - -#### Begin Block - -At the beginning of the block, rewards are distributed to the block proposer. - -Apply the following to the state: - -```py -proposer = state.activeValidatorSet[block.header.proposerAddress] - -# Compute block subsidy and save to state for use in end block. -rewardFactor = (TARGET_ANNUAL_ISSUANCE * BLOCK_TIME) / (SECONDS_PER_YEAR * sqrt(GENESIS_COIN_COUNT)) -blockReward = rewardFactor * sqrt(state.activeValidatorSet.activeVotingPower) -state.activeValidatorSet.proposerBlockReward = blockReward - -# Save proposer's initial voting power to state for use in end block. -state.activeValidatorSet.proposerInitialVotingPower = proposer.votingPower - -state.activeValidatorSet[block.header.proposerAddress] = proposer -``` - -#### End Block - -Apply the following to the state: - -```py -account = state.accounts[block.header.proposerAddress] - -if account.status == AccountStatus.ValidatorUnbonding - account.status == AccountStatus.ValidatorUnbonded - proposer = state.inactiveValidatorSet[block.header.proposerAddress] -else if account.status == AccountStatus.ValidatorBonded - proposer = state.activeValidatorSet[block.header.proposerAddress] - -# Flush the outstanding pending rewards. -proposer.latestEntry += compute_new_entry(proposer.pendingRewards, proposer.votingPower) -proposer.pendingRewards = 0 - -blockReward = state.activeValidatorSet.proposerBlockReward -commissionReward = proposer.commissionRate.numerator * blockReward // proposer.commissionRate.denominator -proposer.commissionRewards += commissionReward -proposer.pendingRewards += blockReward - commissionReward - -# Even though the voting power hasn't changed yet, we consider this a period change. -proposer.latestEntry += compute_new_entry(proposer.pendingRewards, state.activeValidatorSet.proposerInitialVotingPower) -proposer.pendingRewards = 0 - -if account.status == AccountStatus.ValidatorUnbonding - account.status == AccountStatus.ValidatorUnbonded - state.inactiveValidatorSet[block.header.proposerAddress] = proposer -else if account.status == AccountStatus.ValidatorBonded - state.activeValidatorSet[block.header.proposerAddress] = proposer -``` - -At the end of a block, the top `MAX_VALIDATORS` validators by voting power with voting power _greater than_ zero are or become active (bonded). For newly-bonded validators, the entire validator object is moved to the active validators subtree and their status is changed to bonded. For previously-bonded validators that are no longer in the top `MAX_VALIDATORS` validators begin unbonding. - -Bonding validators is simply setting their status to `AccountStatus.ValidatorBonded`. The logic for validator unbonding is found [here](#signedtransactiondatabeginunbondingvalidator), minus transaction sender updates (nonce, balance, and fee). - -This end block implicit state transition is a single state transition, and [only has a single intermediate state root](#blockavailabledata) associated with it. +# Consensus diff --git a/specs/src/specs/data_square_layout.md b/specs/src/specs/data_square_layout.md index 7960e6e88c..4be6fe18f6 100644 --- a/specs/src/specs/data_square_layout.md +++ b/specs/src/specs/data_square_layout.md @@ -1,62 +1 @@ # Data Square Layout - - - -## Preamble - -Celestia uses [a data availability scheme](https://arxiv.org/abs/1809.09044) that allows nodes to determine whether a block's data was published without downloading the whole block. The core of this scheme is arranging data in a two-dimensional matrix of [shares](./shares.md), then applying erasure coding to each row and column. This document describes the rationale for how data—transactions, blobs, and other data—[is actually arranged](./data_structures.md#arranging-available-data-into-shares). Familiarity with the [originally proposed data layout format](https://arxiv.org/abs/1809.09044) is assumed. - -## Layout Rationale - -Block data consists of: - -1. Standard cosmos-SDK transactions: (which are often represented internally as the [`sdk.Tx` interface](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/types/tx_msg.go#L42-L50)) as described in [cosmos-sdk ADR020](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/docs/architecture/adr-020-protobuf-transaction-encoding.md) - 1. These transactions contain protobuf encoded [`sdk.Msg`](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/types/tx_msg.go#L14-L26)s, which get executed atomically (if one fails they all fail) to update the Celestia state. The complete list of modules, which define the `sdk.Msg`s that the state machine is capable of handling, can be found in the [state machine modules spec](../specs/state_machine_modules.md). Examples include standard cosmos-sdk module messages such as [MsgSend](https://github.com/cosmos/cosmos-sdk/blob/f71df80e93bffbf7ce5fbd519c6154a2ee9f991b/proto/cosmos/bank/v1beta1/tx.proto#L21-L32)), and celestia specific module messages such as [`MsgPayForBlobs`](https://github.com/celestiaorg/celestia-app/blob/v1.0.0-rc2/proto/celestia/blob/v1/tx.proto#L16-L31) -1. Blobs: binary large objects which do not modify the Celestia state, but which are intended for a Celestia application identified with a provided namespace. - -We want to arrange this data into a `k * k` matrix of fixed-sized [shares](../specs/shares.md), which will later be committed to in [Namespace Merkle Trees (NMTs)](https://github.com/celestiaorg/nmt/blob/v0.16.0/docs/spec/nmt.md) so that individual shares in this matrix can be proven to belong to a single data root. `k` must always be a power of 2 (e.g. 1, 2, 4, 8, 16, 32, etc.) as this is optimal for the erasure coding algorithm. - -The simplest way we can imagine arranging block data is to simply serialize it all in no particular order, split it into fixed-sized shares, then arrange those shares into the `k * k` matrix in row-major order. However, this naive scheme can be improved in a number of ways, described below. - -First, we impose some ground rules: - -1. Data must be ordered by namespace. This makes queries into a NMT commitment of that data more efficient. -1. Since non-blob data are not naturally intended for particular namespaces, we assign [reserved namespaces](./namespace.md#Reserved-Namespaces) for them. A range of namespaces is reserved for this purpose, starting from the lowest possible namespace. -1. By construction, the above two rules mean that non-blob data always precedes blob data in the row-major matrix, even when considering single rows or columns. -1. Data with different namespaces must not be in the same share. This might cause a small amount of wasted block space, but makes the NMT easier to reason about in general since leaves are guaranteed to belong to a single namespace. - -Given these rules, a square may look as follows: - -![square_layout](./figures/square_layout.svg) - -Padding is addressed in the [padding section](#padding). Namespace C contains two blobs of two shares each while Namespace D contains one blob of three shares. - -### Ordering - -The order of blobs in a namespace is dictated by the priority of the PFBs that paid for the blob. A PFB with greater priority will have all blobs in that namespace strictly before a PFB with less priority. Priority is determined by the `gas-price` of the transaction (`fee`/`gas`). - -## Blob Share Commitment Rules - -Transactions can pay fees for a blob to be included in the same block as the transaction itself. It may seem natural to bundle the `MsgPayForBlobs` transaction that pays for a number of blobs with these blobs (which is the case in other blockchains with native execution, e.g. calldata in Ethereum transactions or OP_RETURN data in Bitcoin transactions), however this would mean that processes validating the state of the Celestia network would need to download all blob data. PayForBlob transactions must therefore only include a commitment to (i.e. some hash of) the blob they pay fees for. If implemented naively (e.g. with a simple hash of the blob, or a simple binary Merkle tree root of the blob), this can lead to a data availability problem, as there are no guarantees that the data behind these commitments is actually part of the block data. - -To that end, we impose some additional rules onto _blobs only_: blobs must be placed is a way such that both the transaction sender and the block producer can be held accountable—a necessary property for e.g. fee burning. Accountable in this context means that - -1. The transaction sender must pay sufficient fees for blob inclusion. -1. The block proposer cannot claim that a blob was included when it was not (which implies that a transaction and the blob it pays for must be included in the same block). In addition all blobs must be accompanied by a PayForBlob transaction. - -Specifically, a `MsgPayForBlobs` must include a `ShareCommitment` over the contents of each blob it is paying for. If the transaction sender knows 1) `k`, the size of the matrix, 2) the starting location of their blob in a row, and 3) the length of the blob (they know this since they are sending the blob), then they can actually compute a sequence of roots to _subtrees in the row NMTs_. Taking _the simple Merkle root of these subtree roots_ provides us with the `ShareCommitment` that gets included in `MsgPayForBlobs`. Using subtree roots instead of all the leafs makes blob inclusion proofs smaller. - -![subtree roots](./figures/blob_share_commitment.svg) - -Understanding 1) and 2) would usually require interaction with the block proposer. To make the possible starting locations of blobs sufficiently predictable and to make `ShareCommitment` independent of `k`, we impose an additional rule. The blob must start at a multiple of the `SubtreeWidth`. - -The `SubtreeWidth` is calculated as the length of the blob in shares, divided by the [`SubtreeRootThreshold`](https://github.com/celestiaorg/celestia-app/blob/v1.0.0-rc2/pkg/appconsts/v1/app_consts.go#L6) and rounded up to the nearest power of 2 ([implementation here](https://github.com/celestiaorg/celestia-app/blob/v1.0.0-rc2/pkg/shares/non_interactive_defaults.go#L94-L116)). If the output is greater than the minimum square size that the blob can fit in (i.e. a blob of 15 shares has a minimum square size of 4) then we take that minimum value. This `SubtreeWidth` is used as the width of the first mountain in the [Merkle Mountain Range](https://docs.grin.mw/wiki/chain-state/merkle-mountain-range/) that would all together represent the `ShareCommitment` over the blob. - -![subtree root width](./figures/subtree_width.svg) - -The `SubtreeRootThreshold` is an arbitrary versioned protocol constant that aims to put a soft limit on the number of subtree roots included in a blob inclusion proof, as described in [ADR013](../../../docs/architecture/adr-013-non-interactive-default-rules-for-zero-padding.md). A higher `SubtreeRootThreshold` means less padding and more tightly packed squares but also means greater blob inclusion proof sizes. -With the above constraint, we can compute subtree roots deterministically. For example, a blob of 172 shares and `SubtreeRootThreshold` (SRT) = 64, must start on a share index that is a multiple of 4 because 172/64 = 3. 3 rounded up to the nearest power of 2 is 4. In this case, there will be a maximum of 3 shares of padding between blobs (more on padding below). The maximum subtree width in shares for the first mountain in the Merkle range will be 4 (The actual mountain range would be 43 subtree roots of 4 shares each). The `ShareCommitment` is then the Merkle tree over the peaks of the mountain range. - -## Padding - -Given these rules whereby blobs in their share format can't be directly appended one after the other, we use padding shares to fill the gaps. These are shares with a particular format (see [padding](./shares.md#padding)). Padding always comes after all the blobs in the namespace. The padding at the end of the reserved namespace and at the end of the square are special in that they belong to unique namespaces. All other padding shares use the namespace of the blob before it in the data square. diff --git a/specs/src/specs/data_structures.md b/specs/src/specs/data_structures.md index 745e06f8cb..c73ca16086 100644 --- a/specs/src/specs/data_structures.md +++ b/specs/src/specs/data_structures.md @@ -1,417 +1 @@ # Data Structures - - - -## Data Structures Overview - -![fig: Block data structures.](./figures/block_data_structures.svg) - -## Type Aliases - -| name | type | -|-----------------------------|-----------------------------| -| `Amount` | `uint64` | -| `Graffiti` | `byte[MAX_GRAFFITI_BYTES]` | -| [`HashDigest`](#hashdigest) | `byte[32]` | -| `Height` | `int64` | -| `Nonce` | `uint64` | -| `Round` | `int32` | -| `StateSubtreeID` | `byte` | -| [`Timestamp`](#timestamp) | `google.protobuf.Timestamp` | -| `VotingPower` | `uint64` | - -## Blockchain Data Structures - -### Block - -Blocks are the top-level data structure of the Celestia blockchain. - -| name | type | description | -|-----------------------|---------------------------------------------|-----------------------------------------------------------------------| -| `header` | [Header](#header) | Block header. Contains primarily identification info and commitments. | -| `availableDataHeader` | [AvailableDataHeader](#availabledataheader) | Header of available data. Contains commitments to erasure-coded data. | -| `availableData` | [AvailableData](#availabledata) | Data that is erasure-coded for availability. | -| `lastCommit` | [Commit](#commit) | Previous block's Tendermint commit. | - -### Header - -Block header, which is fully downloaded by both full clients and light clients. - -| name | type | description | -|-----------------------------------|---------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `version` | [ConsensusVersion](#consensusversion) | The consensus version struct. | -| `chainID` | `string` | The `CHAIN_ID`. | -| `height` | [Height](#type-aliases) | Block height. The genesis block is at height `1`. | -| `timestamp` | [Timestamp](#timestamp) | Timestamp of this block. | -| `lastHeaderHash` | [HashDigest](#hashdigest) | Previous block's header hash. | -| `lastCommitHash` | [HashDigest](#hashdigest) | Previous block's Tendermint commit hash. | -| `consensusHash` | [HashDigest](#hashdigest) | Hash of [consensus parameters](#consensus-parameters) for this block. | -| `AppHash` | [HashDigest](#hashdigest) | The [state root](#state) after the previous block's transactions are applied. | -| `availableDataOriginalSharesUsed` | `uint64` | The number of shares used in the [original data square](#arranging-available-data-into-shares) that are not [tail padding](./consensus.md#reserved-namespace-ids). | -| `availableDataRoot` | [HashDigest](#hashdigest) | Root of [commitments to erasure-coded data](#availabledataheader). | -| `proposerAddress` | [Address](#address) | Address of this block's proposer. | - -The size of the [original data square](#arranging-available-data-into-shares), `availableDataOriginalSquareSize`, isn't explicitly declared in the block header. Instead, it is implicitly computed as the smallest power of 2 whose square is at least `availableDataOriginalSharesUsed` (in other words, the smallest power of 4 that is at least `availableDataOriginalSharesUsed`). - -The header hash is the [hash](#hashing) of the [serialized](#serialization) header. - -### AvailableDataHeader - -| name | type | description | -|------------|-------------------------------|----------------------------------------| -| `rowRoots` | [HashDigest](#hashdigest)`[]` | Commitments to all erasure-coded data. | -| `colRoots` | [HashDigest](#hashdigest)`[]` | Commitments to all erasure-coded data. | - -The number of row/column roots of the original data [shares](data_structures.md#share) in [square layout](#arranging-available-data-into-shares) for this block. The `availableDataRoot` of the [header](#header) is computed using the compact row and column roots as described [here](#2d-reed-solomon-encoding-scheme). - -The number of row and column roots is each `availableDataOriginalSquareSize * 2`, and must be a power of 2. Note that the minimum `availableDataOriginalSquareSize` is 1 (not 0), therefore the number of row and column roots are each at least 2. - -Implementations can prune rows containing only [tail padding](./consensus.md#reserved-namespace-ids) as they are implicitly available. - -### AvailableData - -Data that is [erasure-coded](#erasure-coding) for [data availability checks](https://arxiv.org/abs/1809.09044). - -| name | type | description | -|------------------|-----------------------------------|-----------------------------------------------------------------------------------------------------------------------| -| `transactions` | [Transaction](#transaction) | Transactions are ordinary Cosmos SDK transactions. For example: they may modify the validator set and token balances. | -| `payForBlobData` | [PayForBlobData](#payforblobdata) | PayForBlob data. Transactions that pay for blobs to be included. | -| `blobData` | [BlobData](#blobdata) | Blob data is arbitrary user submitted data that will be published to the Celestia blockchain. | - -### Commit - -| name | type | description | -|--------------|-----------------------------|------------------------------------| -| `height` | [Height](#type-aliases) | Block height. | -| `round` | [Round](#type-aliases) | Round. Incremented on view change. | -| `headerHash` | [HashDigest](#hashdigest) | Header hash of the previous block. | -| `signatures` | [CommitSig](#commitsig)`[]` | List of signatures. | - -### Timestamp - -Timestamp is a [type alias](#type-aliases). - -Celestia uses [`google.protobuf.Timestamp`](https://developers.google.com/protocol-buffers/docs/reference/google.protobuf#google.protobuf.Timestamp) to represent time. - -### HashDigest - -HashDigest is a [type alias](#type-aliases). - -Output of the [hashing](#hashing) function. Exactly 256 bits (32 bytes) long. - -### TransactionFee - -| name | type | description | -|-----------|----------|------------------------------------| -| `tipRate` | `uint64` | The tip rate for this transaction. | - -Abstraction over transaction fees. - -### Address - -Celestia supports [secp256k1](https://en.bitcoin.it/wiki/Secp256k1) keys where [addresses](https://github.com/cosmos/cosmos-sdk/blob/main/docs/architecture/adr-028-public-key-addresses.md) are 20 bytes in length. - -| name | type | description | -|--------------|------------|-------------------------------------------------------------------------| -| `AccAddress` | `[20]byte` | AccAddress a wrapper around bytes meant to represent an account address | - -### CommitSig - -```C++ -enum CommitFlag : uint8_t { - CommitFlagAbsent = 1, - CommitFlagCommit = 2, - CommitFlagNil = 3, -}; -``` - -| name | type | description | -|--------------------|-------------------------|-------------| -| `commitFlag` | `CommitFlag` | | -| `validatorAddress` | [Address](#address) | | -| `timestamp` | [Timestamp](#timestamp) | | -| `signature` | [Signature](#signature) | | - -### Signature - -| name | type | description | -|------|------------|-----------------------------| -| `r` | `byte[32]` | `r` value of the signature. | -| `s` | `byte[32]` | `s` value of signature. | - -## ConsensusVersion - -| name | type | description | -|---------|----------|----------------------| -| `block` | `uint64` | The `VERSION_BLOCK`. | -| `app` | `uint64` | The app version. | - -## Serialization - -Objects that are committed to or signed over require a canonical serialization. This is done using a deterministic (and thus, bijective) variant of protobuf defined [here](https://github.com/cosmos/cosmos-sdk/blob/master/docs/architecture/adr-027-deterministic-protobuf-serialization.md). - -Note: there are two requirements for a serialization scheme, should this need to be changed: - -1. Must be bijective. -1. Serialization must include the length of dynamic structures (e.g. arrays with variable length). - -## Hashing - - - -All protocol-level hashing is done using SHA-2-256 as defined in [FIPS 180-4](https://doi.org/10.6028/NIST.FIPS.180-4). SHA-2-256 outputs a digest that is 256 bits (i.e. 32 bytes) long. - - -Libraries implementing SHA-2-256 are available in Go () and Rust (). - -Unless otherwise indicated explicitly, objects are first [serialized](#serialization) before being hashed. - -## Merkle Trees - -Merkle trees are used to authenticate various pieces of data across the Celestia stack, including transactions, blobs, the validator set, etc. This section provides an overview of the different tree types used, and specifies how to construct them. - -### Binary Merkle Tree - -Binary Merkle trees are constructed in the same fashion as described in [Certificate Transparency (RFC-6962)](https://tools.ietf.org/html/rfc6962), except for using [a different hashing function](#hashing). Leaves are hashed once to get leaf node values and internal node values are the hash of the concatenation of their children (either leaf nodes or other internal nodes). - -Nodes contain a single field: - -| name | type | description | -|------|---------------------------|-------------| -| `v` | [HashDigest](#hashdigest) | Node value. | - -The base case (an empty tree) is defined as the [hash](#hashing) of the empty string: - -```C++ -node.v = 0xe3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -``` - -For leaf node `node` of leaf data `d`: - -```C++ -node.v = h(0x00, serialize(d)) -``` - -For internal node `node` with children `l` and `r`: - -```C++ -node.v = h(0x01, l.v, r.v) -``` - -Note that rather than duplicating the last node if there are an odd number of nodes (the [Bitcoin design](https://github.com/bitcoin/bitcoin/blob/5961b23898ee7c0af2626c46d5d70e80136578d3/src/consensus/merkle.cpp#L9-L43)), trees are allowed to be imbalanced. In other words, the height of each leaf may be different. For an example, see Section 2.1.3 of [Certificate Transparency (RFC-6962)](https://tools.ietf.org/html/rfc6962#section-2.1.3). - -Leaves and internal nodes are hashed differently: the one-byte `0x00` is prepended for leaf nodes while `0x01` is prepended for internal nodes. This avoids a second-preimage attack [where internal nodes are presented as leaves](https://en.wikipedia.org/wiki/Merkle_tree#Second_preimage_attack) trees with leaves at different heights. - -#### BinaryMerkleTreeInclusionProof - -| name | type | description | -|------------|-------------------------------|-----------------------------------------------------------------| -| `siblings` | [HashDigest](#hashdigest)`[]` | Sibling hash values, ordered starting from the leaf's neighbor. | - -A proof for a leaf in a [binary Merkle tree](#binary-merkle-tree), as per Section 2.1.1 of [Certificate Transparency (RFC-6962)](https://tools.ietf.org/html/rfc6962#section-2.1.1). - -### Namespace Merkle Tree - - - -[Shares](./shares.md) in Celestia are associated with a provided _namespace_. The Namespace Merkle Tree (NMT) is a variation of the [Merkle Interval Tree](https://eprint.iacr.org/2018/642), which is itself an extension of the [Merkle Sum Tree](https://bitcointalk.org/index.php?topic=845978.0). It allows for compact proofs around the inclusion or exclusion of shares with particular namespace IDs. - - -Nodes contain three fields: - -| name | type | description | -|---------|-----------------------------|-----------------------------------------------| -| `n_min` | [Namespace](./namespace.md) | Min namespace in subtree rooted at this node. | -| `n_max` | [Namespace](./namespace.md) | Max namespace in subtree rooted at this node. | -| `v` | [HashDigest](#hashdigest) | Node value. | - -The base case (an empty tree) is defined as: - -```C++ -node.n_min = 0x0000000000000000 -node.n_max = 0x0000000000000000 -node.v = 0xe3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -``` - -For leaf node `node` of [share](./shares.md) data `d`: - -```C++ -node.n_min = d.namespace -node.n_max = d.namespace -node.v = h(0x00, d.namespace, d.rawData) -``` - -The `namespace` blob field here is the namespace of the leaf, which is a [`NAMESPACE_SIZE`](consensus.md#system-parameters)-long byte array. - -Leaves in an NMT **must** be lexicographically sorted by namespace in ascending order. - -For internal node `node` with children `l` and `r`: - -```C++ -node.n_min = min(l.n_min, r.n_min) -if l.n_min == PARITY_SHARE_NAMESPACE - node.n_max = PARITY_SHARE_NAMESPACE -else if r.n_min == PARITY_SHARE_NAMESPACE - node.n_max = l.n_max -else - node.n_max = max(l.n_max, r.n_max) -node.v = h(0x01, l.n_min, l.n_max, l.v, r.n_min, r.n_max, r.v) -``` - -Note that the above snippet leverages the property that leaves are sorted by namespace: if `l.n_min` is [`PARITY_SHARE_NAMESPACE`](consensus.md#reserved-state-subtree-ids), so must `{l,r}.n_max`. By construction, either both the min and max namespace of a node will be [`PARITY_SHARE_NAMESPACE`](consensus.md#reserved-state-subtree-ids), or neither will: if `r.n_min` is [`PARITY_SHARE_NAMESPACE`](consensus.md#reserved-state-subtree-ids), so must `r.n_max`. - -For some intuition: the min and max namespace for subtree roots with at least one non-parity leaf (which includes the root of an NMT, as [the right half of an NMT as used in Celestia will be parity shares](#2d-reed-solomon-encoding-scheme)) _ignore_ the namespace ID for the parity leaves. Subtree roots with _only parity leaves_ have their min and max namespace ID set to [`PARITY_SHARE_NAMESPACE`](consensus.md#reserved-state-subtree-ids). This allows for shorter proofs into the tree than if the namespace ID of parity shares was not ignored (which would cause the max namespace ID of the root to always be [`PARITY_SHARE_NAMESPACE`](consensus.md#reserved-state-subtree-ids)). - -A compact commitment can be computed by taking the [hash](#hashing) of the [serialized](#serialization) root node. - -#### NamespaceMerkleTreeInclusionProof - -| name | type | description | -|-----------------|---------------------------------|-----------------------------------------------------------------| -| `siblingValues` | [HashDigest](#hashdigest)`[]` | Sibling hash values, ordered starting from the leaf's neighbor. | -| `siblingMins` | [Namespace](./namespace.md)`[]` | Sibling min namespace IDs. | -| `siblingMaxes` | [Namespace](./namespace.md)`[]` | Sibling max namespace IDs. | - -When verifying an NMT proof, the root hash is checked by reconstructing the root node `root_node` with the computed `root_node.v` (computed as with a [plain Merkle proof](#binarymerkletreeinclusionproof)) and the provided `rootNamespaceMin` and `rootNamespaceMax` as the `root_node.n_min` and `root_node.n_max`, respectively. - -## Erasure Coding - -In order to enable trust-minimized light clients (i.e. light clients that do not rely on an honest majority of validating state assumption), it is critical that light clients can determine whether the data in each block is _available_ or not, without downloading the whole block itself. The technique used here was formally described in the paper [Fraud and Data Availability Proofs: Maximising Light Client Security and Scaling Blockchains with Dishonest Majorities](https://arxiv.org/abs/1809.09044). - -The remainder of the subsections below specify the [2D Reed-Solomon erasure coding scheme](#2d-reed-solomon-encoding-scheme) used, along with the format of [shares](./shares.md) and how [available data](#available-data) is arranged into shares. - -### Reed-Solomon Erasure Coding - -Note that while data is laid out in a two-dimensional square, rows and columns are erasure coded using a standard one-dimensional encoding. - -Reed-Solomon erasure coding is used as the underlying coding scheme. The parameters are: - -- 16-bit Galois field -- [`availableDataOriginalSquareSize`](#header) original pieces (maximum of [`AVAILABLE_DATA_ORIGINAL_SQUARE_MAX`](./consensus.md#constants)) -- [`availableDataOriginalSquareSize`](#header) parity pieces (maximum of [`AVAILABLE_DATA_ORIGINAL_SQUARE_MAX`](./consensus.md#constants)) (i.e `availableDataOriginalSquareSize * 2` total pieces), for an erasure efficiency of 50%. In other words, any 50% of the pieces from the `availableDataOriginalSquareSize * 2` total pieces are enough to recover the original data. -- [`SHARE_SIZE`](./consensus.md#constants) bytes per piece - -Note that [`availableDataOriginalSquareSize`](#header) may vary each block, and [is decided by the block proposer of that block](./block_proposer.md#deciding-on-a-block-size). [Leopard-RS](https://github.com/catid/leopard) is a C library that implements the above scheme with quasilinear runtime. - -### 2D Reed-Solomon Encoding Scheme - -The 2-dimensional data layout is described in this section. The roots of [NMTs](#namespace-merkle-tree) for each row and column across four quadrants of data in a `2k * 2k` matrix of shares, `Q0` to `Q3` (shown below), must be computed. In other words, `2k` row roots and `2k` column roots must be computed. The row and column roots are stored in the `availableDataCommitments` of the [AvailableDataHeader](#availabledataheader). - -![fig: RS2D encoding: data quadrants.](./figures/rs2d_quadrants.svg) - -The data of `Q0` is the original data, and the remaining quadrants are parity data. Setting `k = availableDataOriginalSquareSize`, the original data first must be split into [shares](./shares.md) and [arranged into a `k * k` matrix](#arranging-available-data-into-shares). Then the parity data can be computed. - -Where `A -> B` indicates that `B` is computed using [erasure coding](#reed-solomon-erasure-coding) from `A`: - -- `Q0 -> Q1` for each row in `Q0` and `Q1` -- `Q0 -> Q2` for each column in `Q0` and `Q2` -- `Q2 -> Q3` for each row in `Q2` and `Q3` - -Note that the parity data in `Q3` will be identical if it is vertically extended from `Q1` or horizontally extended from `Q2`. - -![fig: RS2D encoding: extending data.](./figures/rs2d_extending.svg) - -As an example, the parity data in the second column of `Q2` (in striped purple) is computed by [extending](#reed-solomon-erasure-coding) the original data in the second column of `Q0` (in solid blue). - -![fig: RS2D encoding: extending a column.](./figures/rs2d_extend.svg) - -Now that all four quadrants of the `2k * 2k` matrix are filled, the row and column roots can be computed. To do so, each row/column is used as the leaves of a [NMT](#namespace-merkle-tree), for which the compact root is computed (i.e. an extra hash operation over the NMT root is used to produce a single [HashDigest](#hashdigest)). In this example, the fourth row root value is computed as the NMT root of the fourth row of `Q0` and the fourth row of `Q1` as leaves. - -![fig: RS2D encoding: a row root.](./figures/rs2d_row.svg) - -Finally, the `availableDataRoot` of the block [Header](#header) is computed as the Merkle root of the [binary Merkle tree](#binary-merkle-tree) with the row and column roots as leaves, in that order. - -![fig: Available data root.](./figures/data_root.svg) - -### Arranging Available Data Into Shares - -The previous sections described how some original data, arranged into a `k * k` matrix, can be extended into a `2k * 2k` matrix and committed to with NMT roots. This section specifies how [available data](#available-data) (which includes [transactions](#transaction), PayForBlob transactions, and [blobs](#blobdata)) is arranged into the matrix in the first place. - -Note that each [share](./shares.md) only has a single namespace, and that the list of concatenated shares is lexicographically ordered by namespace. - -Then, - -1. For each of `transactionData`, `intermediateStateRootData`, PayForBlob transactions, [serialize](#serialization): - 1. For each request in the list: - 1. [Serialize](#serialization) the request (individually). - 1. Compute the length of each serialized request, [serialize the length](#serialization), and prepend the serialized request with its serialized length. - 1. Split up the length/request pairs into [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_ID_BYTES`](./consensus.md#constants)`-`[`SHARE_RESERVED_BYTES`](./consensus.md#constants)-byte chunks. - 1. Create a [share](./shares.md) out of each chunk. This data has a _reserved_ namespace ID, so the first [`NAMESPACE_SIZE`](./consensus.md#constants)`+`[`SHARE_RESERVED_BYTES`](./consensus.md#constants) bytes for these shares must be set specially. -1. Concatenate the lists of shares in the order: transactions, intermediate state roots, PayForBlob transactions. - -These shares are arranged in the [first quadrant](#2d-reed-solomon-encoding-scheme) (`Q0`) of the `availableDataOriginalSquareSize*2 * availableDataOriginalSquareSize*2` available data matrix in _row-major_ order. In the example below, each reserved data element takes up exactly one share. - -![fig: Original data: reserved.](./figures/rs2d_originaldata_reserved.svg) - -Each blob in the list `blobData`: - -1. [Serialize](#serialization) the blob (individually). -1. Compute the length of each serialized blob, [serialize the length](#serialization), and prepend the serialized blob with its serialized length. -1. Split up the length/blob pairs into [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_SIZE`](./consensus.md#constants)-byte chunks. -1. Create a [share](./shares.md) out of each chunk. The first [`NAMESPACE_SIZE`](./consensus.md#constants) bytes for these shares is set to the namespace. - -For each blob, it is placed in the available data matrix, with row-major order, as follows: - -1. Place the first share of the blob at the next unused location in the matrix, then place the remaining shares in the following locations. - -Transactions [must commit to a Merkle root of a list of hashes](#transaction) that are each guaranteed (assuming the block is valid) to be subtree roots in one or more of the row NMTs. For additional info, see [the rationale document](../specs/data_square_layout.md) for this section. - -However, with only the rule above, interaction between the block producer and transaction sender may be required to compute a commitment to the blob the transaction sender can sign over. To remove interaction, blobs can optionally be laid out using a non-interactive default: - -1. Place the first share of the blob at the next unused location in the matrix whose column is aligned with the largest power of 2 that is not larger than the blob length or [`availableDataOriginalSquareSize`](#header), then place the remaining shares in the following locations **unless** there are insufficient unused locations in the row. -1. If there are insufficient unused locations in the row, place the first share of the blob at the first column of the next row. Then place the remaining shares in the following locations. By construction, any blob whose length is greater than [`availableDataOriginalSquareSize`](#header) will be placed in this way. - -In the example below, two blobs (of lengths 2 and 1, respectively) are placed using the aforementioned default non-interactive rules. - -![fig: original data blob](./figures/rs2d_originaldata_blob.svg) - -The blob share commitment rules may introduce empty shares that do not belong to any blob (in the example above, the top-right share is empty). These are zeroes with namespace ID equal to the either [`TAIL_TRANSACTION_PADDING_NAMESPACE_ID`](./consensus.md#constants) if between a request with a reserved namespace ID and a blob, or the namespace ID of the previous blob if succeeded by a blob. See the [rationale doc](../specs/data_square_layout.md) for more info. - -## Available Data - -### Transaction - -Celestia transactions are Cosmos SDK [transactions](https://github.com/cosmos/cosmos-sdk/blob/v0.46.15/docs/core/transactions.md). - -### PayForBlobData - -### IndexWrapper - -IndexWrapper are wrappers around PayForBlob transactions. They include additional metadata by the block proposer that is committed to in the [available data matrix](#arranging-available-data-into-shares). - -| name | type | description | -|-----------------|------------|--------------------------------------------------------------------------------------------------------------------------------------------------------| -| `tx` | `bytes` | Actual transaction. | -| `share_indexes` | `[]uint32` | Share indexes (in row-major order) of the first share for each blob this transaction pays for. Needed for light verification of proper blob inclusion. | -| `type_id` | `string` | Type ID of the IndexWrapper transaction type. This is used for encoding and decoding IndexWrapper transactions. It is always set to `"INDX"`. | - -### BlobData - -| name | type | description | -|---------|-------------------|----------------| -| `blobs` | [Blob](#blob)`[]` | List of blobs. | - -#### Blob - -| name | type | description | -|---------------|------------------------------|----------------------------| -| `namespaceID` | [NamespaceID](#type-aliases) | Namespace ID of this blob. | -| `rawData` | `byte[]` | Raw blob bytes. | - -## State - -The state of the Celestia chain is intentionally restricted to containing only account balances and the validator set metadata. Similar to other Cosmos SDK based chains, the state of the Celestia chain is maintained in a [multistore](https://github.com/cosmos/cosmos-sdk/blob/v0.46.15/docs/core/store.md#multistore). The root of the application state is committed to in the [block header](#header) via the `AppHash`. - -## Consensus Parameters - -Various [consensus parameters](consensus.md#system-parameters) are committed to in the block header, such as limits and constants. - -| name | type | description | -|----------------------------------|---------------------------------------|-------------------------------------------| -| `version` | [ConsensusVersion](#consensusversion) | The consensus version struct. | -| `chainID` | `string` | The `CHAIN_ID`. | -| `shareSize` | `uint64` | The `SHARE_SIZE`. | -| `shareReservedBytes` | `uint64` | The `SHARE_RESERVED_BYTES`. | -| `availableDataOriginalSquareMax` | `uint64` | The `AVAILABLE_DATA_ORIGINAL_SQUARE_MAX`. | - -In order to compute the `consensusHash` field in the [block header](#header), the above list of parameters is [hashed](#hashing). diff --git a/specs/src/specs/fraud_proofs.md b/specs/src/specs/fraud_proofs.md index 2ec0802b00..8f33083bce 100644 --- a/specs/src/specs/fraud_proofs.md +++ b/specs/src/specs/fraud_proofs.md @@ -1,25 +1 @@ # Fraud Proofs - -## Bad Encoding Fraud Proofs - -In order for data availability sampling to work, light clients must be convinced -that erasure encoded parity data was encoded correctly. For light clients, this -is ultimately enforced via [bad encoding fraud proofs -(BEFPs)](https://github.com/celestiaorg/celestia-node/blob/v0.11.0-rc3/docs/adr/adr-006-fraud-service.md#detailed-design). -Consensus nodes must verify this themselves before considering a block valid. -This is done automatically by verifying the data root of the header, since that -requires reconstructing the square from the block data, performing the erasure -encoding, calculating the data root using that representation, and then -comparing the data root found in the header. - -## Blob Inclusion - -TODO - -## State - -State fraud proofs allow light clients to avoid making an honest majority assumption for -state validity. While these are not incorporated into the protocol as of v1.0.0, -there are example implementations that can be found in -[Rollkit](https://github.com/rollkit/rollkit). More info in -[rollkit-ADR009](https://github.com/rollkit/rollkit/blob/4fd97ba8b8352771f2e66454099785d06fd0c31b/docs/lazy-adr/adr-009-state-fraud-proofs.md). diff --git a/specs/src/specs/index.md b/specs/src/specs/index.md deleted file mode 100644 index d7bda658a9..0000000000 --- a/specs/src/specs/index.md +++ /dev/null @@ -1,18 +0,0 @@ -# Specification - -- [Data Structures](./data_structures.md) -- [Namespace](./namespace.md) -- [Shares](./shares.md) -- [Consensus](./consensus.md) - - [CAT Pool](./cat_pool.md) -- [Block Proposer](./block_proposer.md) -- [Block Validity Rules](./block_validity_rules.md) -- [AnteHandler](./ante_handler.md) - - [AnteHandler v1](./ante_handler_v1.md) - - [AnteHandler v2](./ante_handler_v2.md) -- [Fraud Proofs](./fraud_proofs.md) -- [Networking](./networking.md) -- [Public-Key Cryptography](./public_key_cryptography.md) -- [Data Square Layout](./data_square_layout.md) -- [Resource Pricing](./resource_pricing.md) -- [Multisig](./multisig.md) diff --git a/specs/src/specs/multisig.md b/specs/src/specs/multisig.md index 55ecf7f7b8..206d574852 100644 --- a/specs/src/specs/multisig.md +++ b/specs/src/specs/multisig.md @@ -1,15 +1 @@ # Multisig - -Celestia inherits support for Multisig accounts from the Cosmos SDK. Multisig accounts behave similarly to regular accounts with the added requirement that a threshold of signatures is needed to authorize a transaction. - -The maximum number of signatures allowed for a multisig account is determined by the parameter `auth.TxSigLimit` (see [parameters](./parameters.md)). The threshold and list of signers for a multisig account are set at the time of creation and can be viewed in the `pubkey` field of a key. For example: - -```shell -$ celestia-appd keys show multisig -- address: celestia17rehcgutjfra8zhjl8675t8hhw8wsavzzutv06 - name: multisig - pubkey: '{"@type":"/cosmos.crypto.multisig.LegacyAminoPubKey","threshold":2,"public_keys":[{"@type":"/cosmos.crypto.secp256k1.PubKey","key":"AxMTEFDH8oyBPIH+d2MKfCIY1yAsEd0HVekoPaAOiu9c"},{"@type":"/cosmos.crypto.secp256k1.PubKey","key":"Ax0ANkTPWcCDWy9O2TcUXw90Z0DxnX2zqPvhi4VJPUl5"},{"@type":"/cosmos.crypto.secp256k1.PubKey","key":"AlUwWCGLzhclCMEKc2YLEap9H8JT5tWq1kB8BagU1TVH"}]}' - type: multi -``` - -Please see the [Cosmos SDK docs](https://docs.cosmos.network/main/user/run-node/multisig-guide#step-by-step-guide-to-multisig-transactions) for more information on how to use multisig accounts. diff --git a/specs/src/specs/namespace.md b/specs/src/specs/namespace.md index 2042595452..aae590a94d 100644 --- a/specs/src/specs/namespace.md +++ b/specs/src/specs/namespace.md @@ -1,115 +1 @@ # Namespace - - - -## Abstract - -One of Celestia's core data structures is the namespace. -When a user submits a transaction encapsulating a `MsgPayForBlobs` message to Celestia, they MUST associate each blob with exactly one namespace. -After their transaction has been included in a block, the namespace enables users to take an interest in a subset of the blobs published to Celestia by allowing the user to query for blobs by namespace. - -In order to enable efficient retrieval of blobs by namespace, Celestia makes use of a [Namespaced Merkle Tree](https://github.com/celestiaorg/nmt). -See section 5.2 of the [LazyLedger whitepaper](https://arxiv.org/pdf/1905.09274.pdf) for more details. - -## Overview - -A namespace is composed of two fields: [version](#version) and [id](#id). -A namespace is encoded as a byte slice with the version and id concatenated. - -![namespace](./figures/namespace.svg) - -### Version - -The namespace version is an 8-bit unsigned integer that indicates the version of the namespace. -The version is used to determine the format of the namespace and -is encoded as a single byte. -A new namespace version MUST be introduced if the namespace format changes in a backwards incompatible way. - -Below we explain supported user-specifiable namespace versions, -however, we note that Celestia MAY utilize other namespace versions for internal use. -For more details, see the [Reserved Namespaces](#reserved-namespaces) section. - -#### Version 0 - -The only supported user-specifiable namespace version is `0`. -A namespace with version `0` MUST contain an id with a prefix of 18 leading `0` bytes. -The remaining 10 bytes of the id are user-specified. -Below, we provide examples of valid and invalid encoded user-supplied namespaces with version `0`. - -```go -// Valid encoded namespaces -0x0000000000000000000000000000000000000001010101010101010101 // valid blob namespace -0x0000000000000000000000000000000000000011111111111111111111 // valid blob namespace - -// Invalid encoded namespaces -0x0000000000000000000000000111111111111111111111111111111111 // invalid because it does not have 18 leading 0 bytes -0x1000000000000000000000000000000000000000000000000000000000 // invalid because it does not have version 0 -0x1111111111111111111111111111111111111111111111111111111111 // invalid because it does not have version 0 -``` - -Any change in the number of leading `0` bytes in the id of a namespace with version `0` is considered a backwards incompatible change and MUST be introduced as a new namespace version. - -### ID - -The namespace ID is a 28 byte identifier that uniquely identifies a namespace. -The ID is encoded as a byte slice of length 28. - - -## Reserved Namespaces - -Celestia reserves some namespaces for protocol use. -These namespaces are called "reserved namespaces". -Reserved namespaces are used to arrange the contents of the [data square](./data_square_layout.md). -Applications MUST NOT use reserved namespaces for their blob data. -Reserved namespaces fall into two categories: _Primary_ and _Secondary_. - -- Primary: Namespaces with values less than or equal to `0x00000000000000000000000000000000000000000000000000000000FF`. Primary namespaces always have a version of `0`. -- Secondary: Namespaces with values greater than or equal to `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00`. -Secondary namespaces always have a version of `255` (`0xFF`) so that they are placed after all user specifiable namespaces in a sorted data square. -The `PARITY_SHARE_NAMESPACE` uses version `255` (`0xFF`) to enable more efficient proof generation within the context of [nmt](https://github.com/celestiaorg/nmt), where it is used in conjunction with the `IgnoreMaxNamespace` feature. -The `TAIL_PADDING_NAMESPACE` uses the version `255` to ensure that padding shares are always placed at the end of the Celestia data square even if a new user-specifiable version is introduced. - -Below is a list of the current reserved namespaces. -For additional information on the significance and application of the reserved namespaces, please refer to the [Data Square Layout](./data_square_layout.md) specifications. - -| name | type | category | value | description | -|--------------------------------------|-------------|-----------|----------------------------------------------------------------|----------------------------------------------------------------------------| -| `TRANSACTION_NAMESPACE` | `Namespace` | Primary | `0x0000000000000000000000000000000000000000000000000000000001` | Namespace for ordinary Cosmos SDK transactions. | -| `INTERMEDIATE_STATE_ROOT_NAMESPACE` | `Namespace` | Primary | `0x0000000000000000000000000000000000000000000000000000000002` | Namespace for intermediate state roots (not currently utilized). | -| `PAY_FOR_BLOB_NAMESPACE` | `Namespace` | Primary | `0x0000000000000000000000000000000000000000000000000000000004` | Namespace for transactions that contain a PayForBlob. | -| `PRIMARY_RESERVED_PADDING_NAMESPACE` | `Namespace` | Primary | `0x00000000000000000000000000000000000000000000000000000000FF` | Namespace for padding after all primary reserved namespaces. | -| `MAX_PRIMARY_RESERVED_NAMESPACE` | `Namespace` | Primary | `0x00000000000000000000000000000000000000000000000000000000FF` | Namespace for the highest primary reserved namespace. | -| `MIN_SECONDARY_RESERVED_NAMESPACE` | `Namespace` | Secondary | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF00` | Namespace for the lowest secondary reserved namespace. | -| `TAIL_PADDING_NAMESPACE` | `Namespace` | Secondary | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE` | Namespace for padding after all blobs to fill up the original data square. | -| `PARITY_SHARE_NAMESPACE` | `Namespace` | Secondary | `0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF` | Namespace for parity shares. | - -## Assumptions and Considerations - -Applications MUST refrain from using the [reserved namespaces](#reserved-namespaces) for their blob data. - -Celestia does not ensure the prevention of non-reserved namespace collisions. -Consequently, two distinct applications might use the same namespace. -It is the responsibility of these applications to be cautious and manage the implications and consequences arising from such namespace collisions. -Among the potential consequences is the _Woods Attack_, as elaborated in this forum post: [Woods Attack on Celestia](https://forum.celestia.org/t/woods-attack-on-celestia/59). - -## Implementation - -See the [namespace implementation in go-square](https://github.com/celestiaorg/go-square/blob/4209e67d30fef7de29af2f2712104871c173543a/share/namespace.go). -For the most recent version, which may not reflect the current specifications, refer to [the latest namespace code](https://github.com/celestiaorg/go-square/blob/main/share/namespace.go). - -## Go Definition - -```go -type Namespace struct { - Version uint8 - ID []byte -} -``` - -## References - -1. [ADR-014](../../../docs/architecture/adr-014-versioned-namespaces.md) -1. [ADR-015](../../../docs/architecture/adr-015-namespace-id-size.md) -1. [Namespaced Merkle Tree](https://github.com/celestiaorg/nmt) -1. [LazyLedger whitepaper](https://arxiv.org/pdf/1905.09274.pdf) -1. [Data Square Layout](./data_square_layout.md) diff --git a/specs/src/specs/networking.md b/specs/src/specs/networking.md index 454b56c197..048579a4c1 100644 --- a/specs/src/specs/networking.md +++ b/specs/src/specs/networking.md @@ -1,111 +1 @@ # Networking - - - -## Wire Format - -### AvailableData - -| name | type | description | -|---------------------|-------------------------------------------|---------------| -| `availableDataRows` | [AvailableDataRow](#availabledatarow)`[]` | List of rows. | - -### AvailableDataRow - -| name | type | description | -|----------|-----------------------------------------|------------------| -| `shares` | [Share](./data_structures.md#share)`[]` | Shares in a row. | - -### ConsensusProposal - -Defined as `ConsensusProposal`: - -```protobuf -{{#include ./proto/consensus.proto:ConsensusProposal}} -``` - -When receiving a new block proposal `proposal` from the network, the following steps are performed in order. _Must_ indicates that peers must be blacklisted (to prevent DoS attacks) and _should_ indicates that the network blob can simply be ignored. - -1. `proposal.type` must be a `SignedMsgType`. -1. `proposal.round` is processed identically to Tendermint. -1. `proposal.pol_round` is processed identically to Tendermint. -1. `proposal.header` must be well-formed. -1. `proposal.header.version.block` must be [`VERSION_BLOCK`](./consensus.md#constants). -1. `proposal.header.version.app` must be a supported app version. -1. `proposal.header.height` should be previous known height + 1. -1. `proposal.header.chain_id` must be [`CHAIN_ID`](./consensus.md#constants). -1. `proposal.header.time` is processed identically to Tendermint. -1. `proposal.header.last_header_hash` must be previous block's header hash. -1. `proposal.header.last_commit_hash` must be the previous block's commit hash. -1. `proposal.header.consensus_hash` must be the hash of [consensus parameters](./data_structures.md#header). -1. `proposal.header.state_commitment` must be the state root after applying the previous block's transactions. -1. `proposal.header.available_data_original_shares_used` must be at most [`AVAILABLE_DATA_ORIGINAL_SQUARE_MAX ** 2`](./consensus.md#constants). -1. `proposal.header.available_data_root` must be the [root](./data_structures.md#availabledataheader) of `proposal.da_header`. -1. `proposal.header.proposer_address` must be the [correct leader](./consensus.md#leader-selection). -1. `proposal.da_header` must be well-formed. -1. The number of elements in `proposal.da_header.row_roots` and `proposal.da_header.row_roots` must be equal. -1. The number of elements in `proposal.da_header.row_roots` must be the same as computed [here](./data_structures.md#header). -1. `proposal.proposer_signature` must be a valid [digital signature](./data_structures.md#public-key-cryptography) over the header hash of `proposal.header` that recovers to `proposal.header.proposer_address`. -1. For full nodes, `proposal.da_header` must be the result of computing the roots of the shares (received separately). -1. For light nodes, `proposal.da_header` should be sampled from for availability. - -### MsgWirePayForData - -Defined as `MsgWirePayForData`: - -```protobuf -{{#include ./proto/wire.proto:MsgWirePayForData}} -``` - -Accepting a `MsgWirePayForData` into the mempool requires different logic than other transactions in Celestia, since it leverages the paradigm of block proposers being able to malleate transaction data. Unlike [SignedTransactionDataMsgPayForData](./data_structures.md#signedtransactiondatamsgpayfordata) (the canonical data type that is included in blocks and committed to with a data root in the block header), each `MsgWirePayForData` (the over-the-wire representation of the same) has potentially multiple signatures. - -Transaction senders who want to pay for a blob will create a [SignedTransactionDataMsgPayForData](./data_structures.md#signedtransactiondatamsgpayfordata) object, `stx`, filling in the `stx.blobShareCommitment` field [based on the blob share commitmentrules](../specs/data_square_layout.md#blob-share-commitment-rules), then signing it to get a [transaction](./data_structures.md#transaction) `tx`. - -Receiving a `MsgWirePayForData` object from the network follows the reverse process: verify using the [blob share commitmentrules](../specs/data_square_layout.md#blob-share-commitment-rules) that the signature is valid. - -## Invalid Erasure Coding - -If a malicious block producer incorrectly computes the 2D Reed-Solomon code for a block's data, a fraud proof for this can be presented. We assume that the light clients have the [AvailableDataHeader](./data_structures.md#availabledataheader) and the [Header](./data_structures.md#header) for each block. Hence, given a [ShareProof](#shareproof), they can verify if the `rowRoot` or `colRoot` specified by `isCol` and `position` commits to the corresponding [Share](./data_structures.md#share). Similarly, given the `height` of a block, they can access all elements within the [AvailableDataHeader](./data_structures.md#availabledataheader) and the [Header](./data_structures.md#header) of the block. - -### ShareProof - -| name | type | description | -|------------|---------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------| -| `share` | [Share](./data_structures.md#share) | The share. | -| `proof` | [NamespaceMerkleTreeInclusionProof](./data_structures.md#namespacemerkletreeinclusionproof) | The Merkle proof of the share in the offending row or column root. | -| `isCol` | `bool` | A Boolean indicating if the proof is from a row root or column root; `false` if it is a row root. | -| `position` | `uint64` | The index of the share in the offending row or column. | - -### BadEncodingFraudProof - -Defined as `BadEncodingFraudProof`: - -```protobuf -{{#include ./proto/types.proto:BadEncodingFraudProof}} -``` - -| name | type | description | -|---------------|---------------------------------------------|-----------------------------------------------------------------------------------| -| `height` | [Height](./data_structures.md#type-aliases) | Height of the block with the offending row or column. | -| `shareProofs` | [ShareProof](#shareproof)`[]` | The available shares in the offending row or column. | -| `isCol` | `bool` | A Boolean indicating if it is an offending row or column; `false` if it is a row. | -| `position` | `uint64` | The index of the offending row or column in the square. | - -## Invalid State Update - -If a malicious block producer incorrectly computes the state, a fraud proof for this can be presented. We assume that the light clients have the [AvailableDataHeader](./data_structures.md#availabledataheader) and the [Header](./data_structures.md#header) for each block. Hence, given a [ShareProof](#shareproof), they can verify if the `rowRoot` or `colRoot` specified by `isCol` and `position` commits to the corresponding [Share](./data_structures.md#share). Similarly, given the `height` of a block, they can access all elements within the [AvailableDataHeader](./data_structures.md#availabledataheader) and the [Header](./data_structures.md#header) of the block. - -### StateFraudProof - -Defined as `StateFraudProof`: - -```protobuf -{{#include ./proto/types.proto:StateFraudProof}} -``` - -| name | type | description | -|-----------------------------|-------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `height` | [Height](./data_structures.md#type-aliases) | Height of the block with the intermediate state roots. Subtracting one from `height` gives the height of the block with the transactions. | -| `transactionShareProofs` | [ShareProof](#shareproof)`[]` | `isCol` of type `bool` must be `false`. | -| `isrShareProofs` | [ShareProof](#shareproof)`[]` | `isCol` of type `bool` must be `false`. | -| `index` | `uint64` | Index for connecting the [WrappedIntermediateStateRoot](./data_structures.md#wrappedintermediatestateroot) and [WrappedTransaction](./data_structures.md#wrappedtransaction) after shares are parsed. | diff --git a/specs/src/specs/parameters.md b/specs/src/specs/parameters.md index 4dcecce93d..04f9547d32 100644 --- a/specs/src/specs/parameters.md +++ b/specs/src/specs/parameters.md @@ -1,6 +1 @@ # Parameters - -The parameters in the application depend on the app version: - -- [Parameters v1](./parameters_v1.md) -- [Parameters v2](./parameters_v2.md) diff --git a/specs/src/specs/parameters_v1.md b/specs/src/specs/parameters_v1.md index 436b8cb471..eae5f5081a 100644 --- a/specs/src/specs/parameters_v1.md +++ b/specs/src/specs/parameters_v1.md @@ -1,67 +1 @@ # Parameters v1 - -The parameters below represent the parameters for app version 1. - -Note that not all of these parameters are changeable via governance. This list -also includes parameter that require a hardfork to change due to being manually -hardcoded in the application or they are blocked by the `x/paramfilter` module. - -## Global parameters - -| Parameter | Default | Summary | Changeable via Governance | -|-------------------|---------|------------------------------------------------------------------------------------------------------------------------|---------------------------| -| MaxBlockSizeBytes | 100MiB | Hardcoded value in CometBFT for the protobuf encoded block. | False | -| MaxSquareSize | 128 | Hardcoded maximum square size determined per shares per row or column for the original data square (not yet extended). | False | - -## Module parameters - -| Module.Parameter | Default | Summary | Changeable via Governance | -|-----------------------------------------------|---------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------| -| auth.MaxMemoCharacters | 256 | Largest allowed size for a memo in bytes. | True | -| auth.SigVerifyCostED25519 | 590 | Gas used to verify Ed25519 signature. | True | -| auth.SigVerifyCostSecp256k1 | 1000 | Gas used to verify secp256k1 signature. | True | -| auth.TxSigLimit | 7 | Max number of signatures allowed in a multisig transaction. | True | -| auth.TxSizeCostPerByte | 10 | Gas used per transaction byte. | True | -| bank.SendEnabled | true | Allow transfers. | False | -| blob.GasPerBlobByte | 8 | Gas used per blob byte. | True | -| blob.GovMaxSquareSize | 64 | Governance parameter for the maximum square size determined per shares per row or column for the original data square (not yet extended)s. If larger than MaxSquareSize, MaxSquareSize is used. | True | -| blobstream.DataCommitmentWindow | 400 | Number of blocks that are included in a signed batch (DataCommitment). | True | -| consensus.block.MaxBytes | 1974272 bytes (~1.88 MiB) | Governance parameter for the maximum size of the protobuf encoded block. | True | -| consensus.block.MaxGas | -1 | Maximum gas allowed per block (-1 is infinite). | True | -| consensus.block.TimeIotaMs | 1000 | Minimum time added to the time in the header each block. | False | -| consensus.evidence.MaxAgeDuration | 1814400000000000 (21 days) | The maximum age of evidence before it is considered invalid in nanoseconds. This value should be identical to the unbonding period. | True | -| consensus.evidence.MaxAgeNumBlocks | 120960 | The maximum number of blocks before evidence is considered invalid. This value will stop CometBFT from pruning block data. | True | -| consensus.evidence.MaxBytes | 1MiB | Maximum size in bytes used by evidence in a given block. | True | -| consensus.validator.PubKeyTypes | Ed25519 | The type of public key used by validators. | False | -| consensus.Version.AppVersion | 1 | Determines protocol rules used for a given height. Incremented by the application upon an upgrade. | True | -| distribution.BaseProposerReward | 0 | Reward in the mint denomination for proposing a block. | True | -| distribution.BonusProposerReward | 0 | Extra reward in the mint denomination for proposers based on the voting power included in the commit. | True | -| distribution.CommunityTax | 0.02 (2%) | Percentage of the inflation sent to the community pool. | True | -| distribution.WithdrawAddrEnabled | true | Enables delegators to withdraw funds to a different address. | True | -| gov.DepositParams.MaxDepositPeriod | 604800000000000 (1 week) | Maximum period for token holders to deposit on a proposal in nanoseconds. | True | -| gov.DepositParams.MinDeposit | 10_000_000_000 utia (10,000 TIA) | Minimum deposit for a proposal to enter voting period. | True | -| gov.TallyParams.Quorum | 0.334 (33.4%) | Minimum percentage of total stake needed to vote for a result to be considered valid. | True | -| gov.TallyParams.Threshold | 0.50 (50%) | Minimum proportion of Yes votes for proposal to pass. | True | -| gov.TallyParams.VetoThreshold | 0.334 (33.4%) | Minimum value of Veto votes to Total votes ratio for proposal to be vetoed. | True | -| gov.VotingParams.VotingPeriod | 604800000000000 (1 week) | Duration of the voting period in nanoseconds. | True | -| ibc.ClientGenesis.AllowedClients | []string{"06-solomachine", "07-tendermint"} | List of allowed IBC light clients. | True | -| ibc.ConnectionGenesis.MaxExpectedTimePerBlock | 7500000000000 (75 seconds) | Maximum expected time per block in nanoseconds under normal operation. | True | -| ibc.Transfer.ReceiveEnabled | true | Enable receiving tokens via IBC. | True | -| ibc.Transfer.SendEnabled | true | Enable sending tokens via IBC. | True | -| mint.BondDenom | utia | Denomination that is inflated and sent to the distribution module account. | False | -| mint.DisinflationRate | 0.10 (10%) | The rate at which the inflation rate decreases each year. | False | -| mint.InitialInflationRate | 0.08 (8%) | The inflation rate the network starts at. | False | -| mint.TargetInflationRate | 0.015 (1.5%) | The inflation rate that the network aims to stabilize at. | False | -| slashing.DowntimeJailDuration | 1 min | Duration of time a validator must stay jailed. | True | -| slashing.MinSignedPerWindow | 0.75 (75%) | The percentage of SignedBlocksWindow that must be signed not to get jailed. | True | -| slashing.SignedBlocksWindow | 5000 | The range of blocks used to count for downtime. | True | -| slashing.SlashFractionDoubleSign | 0.02 (2%) | Percentage slashed after a validator is jailed for double signing. | True | -| slashing.SlashFractionDowntime | 0.00 (0%) | Percentage slashed after a validator is jailed for downtime. | True | -| staking.BondDenom | utia | Bondable coin denomination. | False | -| staking.HistoricalEntries | 10000 | Number of historical entries to persist in store. | True | -| staking.MaxEntries | 7 | Maximum number of entries in the redelegation queue. | True | -| staking.MaxValidators | 100 | Maximum number of validators. | True | -| staking.MinCommissionRate | 0.05 (5%) | Minimum commission rate used by all validators. | True | -| staking.UnbondingTime | 1814400 (21 days) | Duration of time for unbonding in seconds. | False | - -Note: none of the mint module parameters are governance modifiable because they have been converted into hardcoded constants. See the x/mint README.md for more details. diff --git a/specs/src/specs/parameters_v2.md b/specs/src/specs/parameters_v2.md index 7555dd8761..0aa1503bd2 100644 --- a/specs/src/specs/parameters_v2.md +++ b/specs/src/specs/parameters_v2.md @@ -1,71 +1 @@ # Parameters v2 - -The parameters below represent the parameters for app version 2. - -Note that not all of these parameters are changeable via governance. This list -also includes parameter that require a hardfork to change due to being manually -hardcoded in the application or they are blocked by the `x/paramfilter` module. - -## Global parameters - -| Parameter | Default | Summary | Changeable via Governance | -|-------------------|---------|------------------------------------------------------------------------------------------------------------------------|---------------------------| -| MaxBlockSizeBytes | 100MiB | Hardcoded value in CometBFT for the protobuf encoded block. | False | -| MaxSquareSize | 128 | Hardcoded maximum square size determined per shares per row or column for the original data square (not yet extended). | False | - -## Module parameters - -| Module.Parameter | Default | Summary | Changeable via Governance | -|-----------------------------------------------|---------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------|---------------------------| -| auth.MaxMemoCharacters | 256 | Largest allowed size for a memo in bytes. | True | -| auth.SigVerifyCostED25519 | 590 | Gas used to verify Ed25519 signature. | True | -| auth.SigVerifyCostSecp256k1 | 1000 | Gas used to verify secp256k1 signature. | True | -| auth.TxSigLimit | 7 | Max number of signatures allowed in a multisig transaction. | True | -| auth.TxSizeCostPerByte | 10 | Gas used per transaction byte. | True | -| bank.SendEnabled | true | Allow transfers. | False | -| blob.GasPerBlobByte | 8 | Gas used per blob byte. | True | -| blob.GovMaxSquareSize | 64 | Governance parameter for the maximum square size of the original data square. | True | -| consensus.block.MaxBytes | 1974272 bytes (~1.88 MiB) | Governance parameter for the maximum size of the protobuf encoded block. | True | -| consensus.block.MaxGas | -1 | Maximum gas allowed per block (-1 is infinite). | True | -| consensus.block.TimeIotaMs | 1000 | Minimum time added to the time in the header each block. | False | -| consensus.evidence.MaxAgeDuration | 1814400000000000 (21 days) | The maximum age of evidence before it is considered invalid in nanoseconds. This value should be identical to the unbonding period. | True | -| consensus.evidence.MaxAgeNumBlocks | 120960 | The maximum number of blocks before evidence is considered invalid. This value will stop CometBFT from pruning block data. | True | -| consensus.evidence.MaxBytes | 1MiB | Maximum size in bytes used by evidence in a given block. | True | -| consensus.validator.PubKeyTypes | Ed25519 | The type of public key used by validators. | False | -| consensus.Version.AppVersion | 2 | Determines protocol rules used for a given height. Incremented by the application upon an upgrade. | True | -| distribution.BaseProposerReward | 0 | Reward in the mint denomination for proposing a block. | True | -| distribution.BonusProposerReward | 0 | Extra reward in the mint denomination for proposers based on the voting power included in the commit. | True | -| distribution.CommunityTax | 0.02 (2%) | Percentage of the inflation sent to the community pool. | True | -| distribution.WithdrawAddrEnabled | true | Enables delegators to withdraw funds to a different address. | True | -| gov.DepositParams.MaxDepositPeriod | 604800000000000 (1 week) | Maximum period for token holders to deposit on a proposal in nanoseconds. | True | -| gov.DepositParams.MinDeposit | 10_000_000_000 utia (10,000 TIA) | Minimum deposit for a proposal to enter voting period. | True | -| gov.TallyParams.Quorum | 0.334 (33.4%) | Minimum percentage of total stake needed to vote for a result to be considered valid. | True | -| gov.TallyParams.Threshold | 0.50 (50%) | Minimum proportion of Yes votes for proposal to pass. | True | -| gov.TallyParams.VetoThreshold | 0.334 (33.4%) | Minimum value of Veto votes to Total votes ratio for proposal to be vetoed. | True | -| gov.VotingParams.VotingPeriod | 604800000000000 (1 week) | Duration of the voting period in nanoseconds. | True | -| ibc.ClientGenesis.AllowedClients | []string{"06-solomachine", "07-tendermint"} | List of allowed IBC light clients. | True | -| ibc.ConnectionGenesis.MaxExpectedTimePerBlock | 7500000000000 (75 seconds) | Maximum expected time per block in nanoseconds under normal operation. | True | -| ibc.Transfer.ReceiveEnabled | true | Enable receiving tokens via IBC. | True | -| ibc.Transfer.SendEnabled | true | Enable sending tokens via IBC. | True | -| icahost.HostEnabled | True | Enables or disables the Inter-Chain Accounts host module. | True | -| icahost.AllowMessages | [icaAllowMessages] | Defines a list of sdk message typeURLs allowed to be executed on a host chain. | True | -| minfee.NetworkMinGasPrice | 0.000001 utia | All transactions must have a gas price greater than or equal to this value. | True | -| mint.BondDenom | utia | Denomination that is inflated and sent to the distribution module account. | False | -| mint.DisinflationRate | 0.10 (10%) | The rate at which the inflation rate decreases each year. | False | -| mint.InitialInflationRate | 0.08 (8%) | The inflation rate the network starts at. | False | -| mint.TargetInflationRate | 0.015 (1.5%) | The inflation rate that the network aims to stabilize at. | False | -| slashing.DowntimeJailDuration | 1 min | Duration of time a validator must stay jailed. | True | -| slashing.MinSignedPerWindow | 0.75 (75%) | The percentage of SignedBlocksWindow that must be signed not to get jailed. | True | -| slashing.SignedBlocksWindow | 5000 | The range of blocks used to count for downtime. | True | -| slashing.SlashFractionDoubleSign | 0.02 (2%) | Percentage slashed after a validator is jailed for double signing. | True | -| slashing.SlashFractionDowntime | 0.00 (0%) | Percentage slashed after a validator is jailed for downtime. | True | -| staking.BondDenom | utia | Bondable coin denomination. | False | -| staking.HistoricalEntries | 10000 | Number of historical entries to persist in store. | True | -| staking.MaxEntries | 7 | Maximum number of entries in the redelegation queue. | True | -| staking.MaxValidators | 100 | Maximum number of validators. | True | -| staking.MinCommissionRate | 0.05 (5%) | Minimum commission rate used by all validators. | True | -| staking.UnbondingTime | 1814400 (21 days) | Duration of time for unbonding in seconds. | False | - -Note: none of the mint module parameters are governance modifiable because they have been converted into hardcoded constants. See the x/mint README.md for more details. - -[icaAllowMessages]: https://github.com/rootulp/celestia-app/blob/8caa5807df8d15477554eba953bd056ae72d4503/app/ica_host.go#L3-L18 diff --git a/specs/src/specs/public_key_cryptography.md b/specs/src/specs/public_key_cryptography.md index 50cae12d69..5234fcb002 100644 --- a/specs/src/specs/public_key_cryptography.md +++ b/specs/src/specs/public_key_cryptography.md @@ -1,68 +1 @@ # Public-Key Cryptography - - - -Consensus-critical data is authenticated using [ECDSA](https://www.secg.org/sec1-v2.pdf) with the curves: [Secp256k1](https://en.bitcoin.it/wiki/Secp256k1) or [Ed25519](https://en.wikipedia.org/wiki/EdDSA#Ed25519). - -## Secp256k1 - -The Secp256k1 key type is used by accounts that submit transactions to be included in Celestia. - -### Libraries - -A highly-optimized library is available in C (), with wrappers in Go () and Rust (). - -### Public-keys - -Secp256k1 public keys can be compressed to 257-bits (or 33 bytes) per the format described [here](https://github.com/cosmos/cosmos-sdk/blob/v0.46.15/docs/basics/accounts.md#public-keys). - -### Addresses - -Cosmos [addresses](https://github.com/cosmos/cosmos-sdk/blob/v0.46.15/docs/basics/accounts.md#addresses) are 20 bytes in length. - -### Signatures - - -Deterministic signatures ([RFC-6979](https://tools.ietf.org/rfc/rfc6979.txt)) should be used when signing, but this is not enforced at the protocol level as it cannot be for Secp256k1 signatures. - - -Signatures are represented as the `r` and `s` (each 32 bytes) values of the signature. `r` and `s` take on their usual meaning (see: [SEC 1, 4.1.3 Signing Operation](https://www.secg.org/sec1-v2.pdf)). Signatures are encoded with protobuf as described [here](https://github.com/cosmos/cosmos-sdk/blob/v0.46.15/docs/core/encoding.md). - -### Human Readable Encoding - -In front-ends addresses are prefixed with the [Bech32](https://en.bitcoin.it/wiki/Bech32) prefix `celestia`. For example, a valid address is `celestia1kj39jkzqlr073t42am9d8pd40tgudc3e2kj9yf`. - -## Ed25519 - -The Ed25519 key type is used by validators. - - -### Libraries - -- [crypto/ed25519](https://pkg.go.dev/crypto/ed25519) -- [cometbft crypto/ed25519](https://pkg.go.dev/github.com/cometbft/cometbft@v0.37.0/crypto/ed25519) - -### Public Keys - -Ed25519 public keys are 32 bytes in length. They often appear in validator configuration files (e.g. `genesis.json`) base64 encoded: - -```json - "pub_key": { - "type": "tendermint/PubKeyEd25519", - "value": "DMEMMj1+thrkUCGocbvvKzXeaAtRslvX9MWtB+smuIA=" - } -``` - - -### Addresses - -Ed25519 addresses are the first 20-bytes of the SHA256 hash of the raw 32-byte public key: - -```go -address = SHA256(pubkey)[:20] -``` - - -### Signatures - -Ed25519 signatures are 64 bytes in length. diff --git a/specs/src/specs/resource_pricing.md b/specs/src/specs/resource_pricing.md index 525bad661e..7b2572cdb6 100644 --- a/specs/src/specs/resource_pricing.md +++ b/specs/src/specs/resource_pricing.md @@ -1,254 +1 @@ # Resource Pricing - -For all standard cosmos-sdk transactions (staking, IBC, etc), Celestia utilizes -the [default cosmos-sdk mechanisms](https://github.com/cosmos/cosmos-sdk/blob/v0.46.15/docs/basics/gas-fees.md) for pricing resources. This involves -incrementing a gas counter during transaction execution each time the state is -read from/written to, or when specific costly operations occur such as signature -verification or inclusion of data. - -```go -// GasMeter interface to track gas consumption -type GasMeter interface { - GasConsumed() Gas - GasConsumedToLimit() Gas - GasRemaining() Gas - Limit() Gas - ConsumeGas(amount Gas, descriptor string) - RefundGas(amount Gas, descriptor string) - IsPastLimit() bool - IsOutOfGas() bool - String() string -} -``` - -We can see how this gas meter is used in practice by looking at the store. -Notice where gas is consumed each time we write or read, specifically a flat -cost for initiating the action followed by a prorated cost for the amount of -data read or written. - -```go -// Implements KVStore. -func (gs *Store) Get(key []byte) (value []byte) { - gs.gasMeter.ConsumeGas(gs.gasConfig.ReadCostFlat, types.GasReadCostFlatDesc) - value = gs.parent.Get(key) - - // TODO overflow-safe math? - gs.gasMeter.ConsumeGas(gs.gasConfig.ReadCostPerByte*types.Gas(len(key)), types.GasReadPerByteDesc) - gs.gasMeter.ConsumeGas(gs.gasConfig.ReadCostPerByte*types.Gas(len(value)), types.GasReadPerByteDesc) - - return value -} - -// Implements KVStore. -func (gs *Store) Set(key []byte, value []byte) { - types.AssertValidKey(key) - types.AssertValidValue(value) - gs.gasMeter.ConsumeGas(gs.gasConfig.WriteCostFlat, types.GasWriteCostFlatDesc) - // TODO overflow-safe math? - gs.gasMeter.ConsumeGas(gs.gasConfig.WriteCostPerByte*types.Gas(len(key)), types.GasWritePerByteDesc) - gs.gasMeter.ConsumeGas(gs.gasConfig.WriteCostPerByte*types.Gas(len(value)), types.GasWritePerByteDesc) - gs.parent.Set(key, value) -} -``` - -The configuration for the gas meter used by Celestia is as follows. - -```go -// KVGasConfig returns a default gas config for KVStores. -func KVGasConfig() GasConfig { - return GasConfig{ - HasCost: 1000, - DeleteCost: 1000, - ReadCostFlat: 1000, - ReadCostPerByte: 3, - WriteCostFlat: 2000, - WriteCostPerByte: 30, - IterNextCostFlat: 30, - } -} - -// TransientGasConfig returns a default gas config for TransientStores. -func TransientGasConfig() GasConfig { - return GasConfig{ - HasCost: 100, - DeleteCost: 100, - ReadCostFlat: 100, - ReadCostPerByte: 0, - WriteCostFlat: 200, - WriteCostPerByte: 3, - IterNextCostFlat: 3, - } -} -``` - -Two notable gas consumption events that are not Celestia specific are the total -bytes used for a transaction and the verification of the signature - -```go -func (cgts ConsumeTxSizeGasDecorator) AnteHandle(ctx sdk.Context, tx sdk.Tx, simulate bool, next sdk.AnteHandler) (sdk.Context, error) { - sigTx, ok := tx.(authsigning.SigVerifiableTx) - if !ok { - return ctx, sdkerrors.Wrap(sdkerrors.ErrTxDecode, "invalid tx type") - } - params := cgts.ak.GetParams(ctx) - - ctx.GasMeter().ConsumeGas(params.TxSizeCostPerByte*sdk.Gas(len(ctx.TxBytes())), "txSize") - ... -} - -// DefaultSigVerificationGasConsumer is the default implementation of SignatureVerificationGasConsumer. It consumes gas -// for signature verification based upon the public key type. The cost is fetched from the given params and is matched -// by the concrete type. -func DefaultSigVerificationGasConsumer( - meter sdk.GasMeter, sig signing.SignatureV2, params types.Params, -) error { - pubkey := sig.PubKey - switch pubkey := pubkey.(type) { - case *ed25519.PubKey: - meter.ConsumeGas(params.SigVerifyCostED25519, "ante verify: ed25519") - return sdkerrors.Wrap(sdkerrors.ErrInvalidPubKey, "ED25519 public keys are unsupported") - - case *secp256k1.PubKey: - meter.ConsumeGas(params.SigVerifyCostSecp256k1, "ante verify: secp256k1") - return nil - - case *secp256r1.PubKey: - meter.ConsumeGas(params.SigVerifyCostSecp256r1(), "ante verify: secp256r1") - return nil - - case multisig.PubKey: - multisignature, ok := sig.Data.(*signing.MultiSignatureData) - if !ok { - return fmt.Errorf("expected %T, got, %T", &signing.MultiSignatureData{}, sig.Data) - } - err := ConsumeMultisignatureVerificationGas(meter, multisignature, pubkey, params, sig.Sequence) - if err != nil { - return err - } - return nil - - default: - return sdkerrors.Wrapf(sdkerrors.ErrInvalidPubKey, "unrecognized public key type: %T", pubkey) - } -} -``` - -Since gas is consumed in this fashion and many of the cosmos-sdk transactions -are composable, any given transaction can have a large window of possible gas -consumption. For example, vesting accounts use more bytes of state than a normal -account, so more gas is consumed each time a vesting account is read from or -updated. - -## Parameters - -There are four parameters that can be modified via governance to modify gas -usage. - -| Parameter | Default Value | Description | Changeable via Governance | -|-----------------|---------------|-----------------------------------------|---------------------------| -| consensus/max_gas | -1 | The maximum gas allowed in a block. Default of -1 means this value is not capped. | True | -| auth/tx_size_cost_per_byte | 10 | Gas used per each byte used by the transaction. | True | -| auth/sig_verify_cost_secp256k1 | 1000 | Gas used per verifying a secp256k1 signature | True | -| blob/gas_per_blob_byte | 8 | Gas used per byte used by blob. Note that this value is applied to all encoding overhead, meaning things like the padding of the remaining share and namespace. See PFB gas estimation section for more details. | True | - -## Gas Limit - -The gas limit must be included in each transaction. If the transaction exceeds -this gas limit during the execution of the transaction, then the transaction -will fail. - -> Note: When a transaction is submitted to the mempool, the transaction is not -> fully executed. This can lead to a transaction getting accepted by the mempool -> and eventually included in a block, yet failing because the transaction ends -> up exceeding the gas limit. - -Fees are not currently refunded. While users can specify a gas price, the total -fee is then calculated by simply multiplying the gas limit by the gas price. The -entire fee is then deducted from the transaction no matter what. - -## Fee market - -By default, Celestia's consensus nodes prioritize transactions in their mempools -based on gas price. In version 1, there was no enforced minimum gas price, which -allowed each consensus node to independently set its own minimum gas price in -`app.toml`. This even permitted a gas price of 0, thereby creating the -possibility of secondary markets. In version 2, Celestia introduces a network -minimum gas price, a consensus constant, unaffected by individual node -configurations. Although nodes retain the freedom to increase gas prices -locally, all transactions in a block must be greater than or equal to the network -minimum threshold. If a block is proposed that contains a tx with a gas price -below the network min gas price, the block will be rejected as invalid. - -## Estimating PFB cost - -Generally, the gas used by a PFB transaction involves a static "fixed cost" and -a dynamic cost based on the size of each blob involved in the transaction. - -> Note: For a general use case of a normal account submitting a PFB, the static -> costs can be treated as such. However, due to the description above of how gas -> works in the cosmos-sdk this is not always the case. Notably, if we use a -> vesting account or the `feegrant` modules, then these static costs change. - -The "fixed cost" is an approximation of the gas consumed by operations outside -the function `GasToConsume` (for example, signature verification, tx size, read -access to accounts), which has a default value of 65,000. - -> Note: the first transaction sent by an account (sequence number == 0) has an -> additional one time gas cost of 10,000. If this is the case, this should be -> accounted for. - -Each blob in the PFB contributes to the total gas cost based on its size. The -function `GasToConsume` calculates the total gas consumed by all the blobs -involved in a PFB, where each blob's gas cost is computed by first determining -how many shares are needed to store the blob size. Then, it computes the product -of the number of shares, the number of bytes per share, and the `gasPerByte` -parameter. Finally, it adds a static amount per blob. - -The gas cost per blob byte and gas cost per transaction byte are parameters that -could potentially be adjusted through the system's governance mechanisms. Hence, -actual costs may vary depending on the current settings of these parameters. - -## Tracing Gas Consumption - -This figure plots each instance of the gas meter being incremented as a colored -dot over the execution lifecycle of a given transaction. The y-axis is units of -gas and the x-axis is cumulative gas consumption. The legend shows which color -indicates what the cause of the gas consumption was. - -This code used to trace gas consumption can be found in the `tools/gasmonitor` of the branch for [#2131](https://github.com/celestiaorg/celestia-app/pull/2131), and the script to generate the plots below can be found [here](https://gist.github.com/evan-forbes/948c8cf574f2f50b101c89a95ee1d43c) (warning: this script will not be maintained). - -### MsgSend - -Here we can see the gas consumption trace of a common send transaction for -1`utia` - -![MsgSend](./figures/gas_consumption/msg_send_trace.png) - -### MsgCreateValidator - -Here we examine a more complex transaction. - -![MsgCreateValidator](./figures/gas_consumption/msg_create_validator_trace.png) - -### PFB with One Single Share Blob - -![MsgPayForBlobs Single -Share](./figures/gas_consumption/single_share_pfb_trace.png) - -### PFB with Two Single Share Blobs - -This PFB transaction contains two single share blobs. Notice the gas cost for -`pay for blob` is double what it is above due to two shares being used, and -there is also additional cost from `txSize` since the transaction itself is -larger to accommodate the second set of metadata in the PFB. - -![MsgPayForBlobs with Two -Blobs](./figures/gas_consumption/pfb_with_two_single_share_blobs_trace.png) - -### 100KiB Single Blob PFB - -Here we can see how the cost of a PFB with a large blob (100KiB) is quickly dominated by -the cost of the blob. - -![MsgPayForBlobs with One Large -Blob](./figures/gas_consumption/100kib_pfb_trace.png) diff --git a/specs/src/specs/shares.md b/specs/src/specs/shares.md index ee469f25f7..806e046c43 100644 --- a/specs/src/specs/shares.md +++ b/specs/src/specs/shares.md @@ -1,121 +1 @@ # Shares - - - -## Abstract - -All available data in a Celestia [block](./data_structures.md#block) is split into fixed-size data chunks known as "shares". Shares are the atomic unit of the Celestia data square. The shares in a Celestia block are eventually [erasure-coded](./data_structures.md#erasure-coding) and committed to in [Namespace Merkle trees](./data_structures.md#namespace-merkle-tree) (also see [NMT spec](https://github.com/celestiaorg/nmt/blob/master/docs/spec/nmt.md)). - -## Terms - -- **Blob**: User specified data (e.g. a roll-up block) that is associated with exactly one namespace. Blob data are opaque bytes of data that are included in the block but do not impact Celestia's state. -- **Share**: A fixed-size data chunk that is associated with exactly one namespace. -- **Share sequence**: A share sequence is a contiguous set of shares that contain semantically relevant data. A share sequence MUST contain one or more shares. When a [blob](../../../x/blob/README.md) is split into shares, it is written to one share sequence. As a result, all shares in a share sequence are typically parsed together because the original blob data may have been split across share boundaries. All transactions in the [`TRANSACTION_NAMESPACE`](./namespace.md#reserved-namespaces) are contained in one share sequence. All transactions in the [`PAY_FOR_BLOB_NAMESPACE`](./namespace.md#reserved-namespaces) are contained in one share sequence. - -## Overview - -User submitted transactions are split into shares (see [share splitting](#share-splitting)) and arranged in a `k * k` matrix (see [arranging available data into shares](./data_structures.md#arranging-available-data-into-shares)) prior to the erasure coding step. Shares in the `k * k` matrix are ordered by namespace and have a common [share format](#share-format). - -[Padding](#padding) shares are added to the `k * k` matrix to ensure: - -1. Blob sequences start on an index that conforms to [blob share commitment rules](./data_square_layout.md#blob-share-commitment-rules) (see [namespace padding share](#namespace-padding-share) and [reserved padding share](#primary-reserved-padding-share)) -1. The number of shares in the matrix is a perfect square (see [tail padding share](#tail-padding-share)) - -## Share Format - -Every share has a fixed size [`SHARE_SIZE`](./consensus.md#constants). The share format below is consistent for all shares: - -- The first [`NAMESPACE_VERSION_SIZE`](./consensus.md#constants) bytes of a share's raw data is the namespace version of that share (denoted by "namespace version" in the figure below). -- The next [`NAMESPACE_ID_SIZE`](./consensus.md#constants) bytes of a share's raw data is the namespace ID of that share (denoted by "namespace id" in the figure below). -- The next [`SHARE_INFO_BYTES`](./consensus.md#constants) bytes are for share information (denoted by "info byte" in the figure below) with the following structure: - - The first 7 bits represent the [share version](#share-version) in big endian form (initially, this will be `0000000` for version `0`); - - The last bit is a sequence start indicator. The indicator is `1` if this share is the first share in a sequence or `0` if this share is a continuation share in a sequence. -- If this share is the first share in a sequence, it will include the length of the sequence in bytes. The next [`SEQUENCE_BYTES`](./consensus.md#constants) represent a big-endian uint32 value (denoted by "sequence length" in the figure below). This length is placed immediately after the `SHARE_INFO_BYTES` field. It's important to note that shares that are not the first share in a sequence do not contain this field. -- The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_SIZE`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants)`-`[`SEQUENCE_BYTES`](./consensus.md#constants) bytes (if first share) or [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_SIZE`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants) bytes (if continuation share) are raw data (denoted by "blob1" in the figure below). Typically raw data is the blob payload that user's submit in a [BlobTx](../../../x/blob/README.md). However, raw data can also be transaction data (see [transaction shares](#transaction-shares) below). -- If there is insufficient raw data to fill the share, the remaining bytes are filled with `0`. - -First share in a sequence: - -![figure 1: share start](./figures/share_start.svg) - -Continuation share in a sequence: - -![figure 2: share continuation](./figures/share_continuation.svg) - -Since raw data that exceeds [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_SIZE`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants) `-` [`SEQUENCE_BYTES`](./consensus.md#constants) bytes will span more than one share, developers MAY choose to encode additional metadata in their raw blob data prior to inclusion in a Celestia block. For example, Celestia transaction shares encode additional metadata in the form of "reserved bytes". - -### Share Version - -The share version is a 7-bit big-endian unsigned integer that is used to indicate the version of the [share format](#share-format). The only supported share version is `0`. A new share version MUST be introduced if the share format changes in a way that is not backwards compatible. - -## Transaction Shares - -In order for clients to parse shares in the middle of a sequence without downloading antecedent shares, Celestia encodes additional metadata in the shares associated with reserved namespaces. At the time of writing this only applies to the [`TRANSACTION_NAMESPACE`](./namespace.md#reserved-namespaces) and [`PAY_FOR_BLOB_NAMESPACE`](./namespace.md#reserved-namespaces). This share structure is often referred to as "compact shares" to differentiate from the share structure defined above for all shares. It conforms to the common [share format](#share-format) with one additional field, the "reserved bytes" field, which is described below: - -- Every transaction share includes [`SHARE_RESERVED_BYTES`](./consensus.md#constants) bytes that contain the index of the starting byte of the length of the [canonically serialized](./consensus.md#serialization) first transaction that starts in the share, or `0` if there is none, as a binary big endian `uint32`. Denoted by "reserved bytes" in the figure below. The [`SHARE_RESERVED_BYTES`](./consensus.md#constants) are placed immediately after the `SEQUENCE_BYTES` if this is the first share in a sequence or immediately after the `SHARE_INFO_BYTES` if this is a continuation share in a sequence. -- The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_SIZE`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants)`-`[`SEQUENCE_BYTES`](./consensus.md#constants)`-`[`SHARE_RESERVED_BYTES`](./consensus.md#constants) bytes (if first share) or [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_SIZE`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants)`-`[`SHARE_RESERVED_BYTES`](./consensus.md#constants) bytes (if continuation share) are transaction or PayForBlob transaction data (denoted by "tx1" and "tx2" in the figure below). Each transaction or PayForBlob transaction is prefixed with a [varint](https://developers.google.com/protocol-buffers/docs/encoding) of the length of that unit (denoted by "len(tx1)" and "len(tx2)" in the figure below). -- If there is insufficient transaction or PayForBlob transaction data to fill the share, the remaining bytes are filled with `0`. - -First share in a sequence: - -![figure 3: transaction share start](./figures/transaction_share_start.svg) - -where reserved bytes would be `38` as a binary big endian `uint32` (`[0b00000000, 0b00000000, 0b00000000, 0b00100110]`). - -Continuation share in a sequence: - -![figure 4: transaction share continuation](./figures/transaction_share_continuation.svg) - -where reserved bytes would be `80` as a binary big endian `uint32` (`[0b00000000, 0b00000000, 0b00000000, 0b01010000]`). - -## Padding - -Padding shares vary based on namespace but they conform to the [share format](#share-format) described above. - -- The first [`NAMESPACE_VERSION_SIZE`](./consensus.md#constants) bytes of a share's raw data is the namespace version of that share (initially, this will be `0`). -- The next [`NAMESPACE_ID_SIZE`](./consensus.md#constants) bytes of a share's raw data is the namespace ID of that share. This varies based on the type of padding share. -- The next [`SHARE_INFO_BYTES`](./consensus.md#constants) bytes are for share information. - - The first 7 bits represent the [share version](#share-version) in big endian form (initially, this will be `0000000` for version `0`); - - The last bit is a sequence start indicator. The indicator is always `1`. -- The next [`SEQUENCE_BYTES`](./consensus.md#constants) contain a big endian `uint32` of value `0`. -- The remaining [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_SIZE`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants)`-`[`SEQUENCE_BYTES`](./consensus.md#constants) bytes are filled with `0`. - -### Namespace Padding Share - -A namespace padding share uses the namespace of the blob that precedes it in the data square so that the data square can retain the property that all shares are ordered by namespace. -A namespace padding share acts as padding between blobs so that the subsequent blob begins at an index that conforms to the [blob share commitment rules](./data_square_layout.md#blob-share-commitment-rules). Clients MAY ignore the contents of these shares because they don't contain any significant data. - -### Primary Reserved Padding Share - -Primary reserved padding shares use the [`PRIMARY_RESERVED_PADDING_NAMESPACE`](./namespace.md#reserved-namespaces). Primary reserved padding shares are placed after shares in the primary reserved namespace range so that the first blob can start at an index that conforms to blob share commitment rules. Clients MAY ignore the contents of these shares because they don't contain any significant data. - -### Tail Padding Share - -Tail padding shares use the [`TAIL_PADDING_NAMESPACE`](./namespace.md#reserved-namespaces). Tail padding shares are placed after the last blob in the data square so that the number of shares in the data square is a perfect square. Clients MAY ignore the contents of these shares because they don't contain any significant data. - -## Parity Share - -Parity shares use the [`PARITY_SHARE_NAMESPACE`](./namespace.md#reserved-namespaces). Parity shares are the output of the erasure coding step of the data square construction process. They occupy quadrants Q1, Q2, and Q3 of the extended data square and are used to reconstruct the original data square (Q0). Bytes carry no special meaning. - -## Share Splitting - -Share splitting is the process of converting a blob into a share sequence. The process is as follows: - -1. Create a new share and populate the prefix of the share with the blob's namespace and [share version](#share-version). Set the sequence start indicator to `1`. Write the blob length as the sequence length. Write the blob's data into the share until the share is full. -1. If there is more data to write, create a new share (a.k.a continuation share) and populate the prefix of the share with the blob's namespace and [share version](#share-version). Set the sequence start indicator to `0`. Write the remaining blob data into the share until the share is full. -1. Repeat the previous step until all blob data has been written. -1. If the last share is not full, fill the remainder of the share with `0`. - -## Assumptions and Considerations - -- Shares are assumed to be byte slices of length 512. Parsing shares of a different length WILL result in an error. - -## Implementation - -See [go-square/shares](https://github.com/celestiaorg/go-square/tree/be3c2801e902a0f90f694c062b9c4e6a7e01154e/shares). - -## References - -1. [ADR-012](../../../docs/architecture/adr-012-sequence-length-encoding.md) -1. [ADR-014](../../../docs/architecture/adr-014-versioned-namespaces.md) -1. [ADR-015](../../../docs/architecture/adr-015-namespace-id-size.md) diff --git a/specs/src/specs/state_machine_modules.md b/specs/src/specs/state_machine_modules.md index 3889c42472..58c122c799 100644 --- a/specs/src/specs/state_machine_modules.md +++ b/specs/src/specs/state_machine_modules.md @@ -1,6 +1 @@ # State Machine Modules - -Celestia app is built using the cosmos-sdk, and follows standard cosmos-sdk module structure. The modules used in the application vary based on app version: - -- [State Machine Modules v1](state_machine_modules_v1.md) -- [State Machine Modules v2](state_machine_modules_v2.md) diff --git a/specs/src/specs/state_machine_modules_v1.md b/specs/src/specs/state_machine_modules_v1.md index eef6d4df54..ec8a32dd68 100644 --- a/specs/src/specs/state_machine_modules_v1.md +++ b/specs/src/specs/state_machine_modules_v1.md @@ -1,30 +1 @@ # State Machine Modules v1 - -The modules used in app version 1 are: - -## `celestia-app` modules - -- [blob](https://github.com/celestiaorg/celestia-app/blob/main/x/blob/README.md) -- [blobstream](https://github.com/celestiaorg/celestia-app/blob/main/x/blobstream/README.md) -- [mint](https://github.com/celestiaorg/celestia-app/blob/main/x/mint/README.md) -- [paramfilter](https://github.com/celestiaorg/celestia-app/blob/main/x/paramfilter/README.md) -- [tokenfilter](https://github.com/celestiaorg/celestia-app/blob/main/x/tokenfilter/README.md) - -## `cosmos-sdk` modules - -- [auth](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/auth/spec/README.md) -- [authz](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/authz/spec/README.md) -- [bank](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/bank/spec/README.md) -- [capability](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/capability/spec/README.md) -- [crisis](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/crisis/spec/README.md) -- [distribution](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/distribution/spec/README.md) -- [evidence](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/evidence/spec/README.md) -- [feegrant](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/feegrant/spec/README.md) -- [genutil](https://github.com/celestiaorg/cosmos-sdk/tree/v1.14.0-sdk-v0.46.11/x/genutil) (no spec) -- [gov](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/gov/spec/README.md) -- [ibc](https://github.com/cosmos/ibc/blob/f990a7f96eb7753c2fabbd49ed50b64d3a807629/README.md) -- [params](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/params/spec/README.md) -- [slashing](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/slashing/spec/README.md) -- [staking](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/staking/spec/README.md) -- [transfer](https://github.com/cosmos/ibc/blob/f990a7f96eb7753c2fabbd49ed50b64d3a807629/spec/app/ics-020-fungible-token-transfer/README.md) -- [vesting](https://github.com/celestiaorg/cosmos-sdk/tree/v1.14.0-sdk-v0.46.11/x/auth/vesting) (no spec) diff --git a/specs/src/specs/state_machine_modules_v2.md b/specs/src/specs/state_machine_modules_v2.md index 25fea4947e..ce9a0bb852 100644 --- a/specs/src/specs/state_machine_modules_v2.md +++ b/specs/src/specs/state_machine_modules_v2.md @@ -1,33 +1 @@ # State Machine Modules v2 - -The modules used in app version 2 are: - -## `celestia-app` modules - -- [blob](https://github.com/celestiaorg/celestia-app/blob/main/x/blob/README.md) -- [minfee](https://github.com/celestiaorg/celestia-app/blob/main/x/minfee/README.md) -- [mint](https://github.com/celestiaorg/celestia-app/blob/main/x/mint/README.md) -- [paramfilter](https://github.com/celestiaorg/celestia-app/blob/main/x/paramfilter/README.md) -- [signal](https://github.com/celestiaorg/celestia-app/blob/main/x/signal/README.md) -- [tokenfilter](https://github.com/celestiaorg/celestia-app/blob/main/x/tokenfilter/README.md) - -## `cosmos-sdk` modules - -- [auth](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/auth/spec/README.md) -- [authz](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/authz/spec/README.md) -- [bank](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/bank/spec/README.md) -- [capability](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/capability/spec/README.md) -- [crisis](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/crisis/spec/README.md) -- [distribution](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/distribution/spec/README.md) -- [evidence](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/evidence/spec/README.md) -- [feegrant](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/feegrant/spec/README.md) -- [genutil](https://github.com/celestiaorg/cosmos-sdk/tree/v1.14.0-sdk-v0.46.11/x/genutil) (no spec) -- [gov](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/gov/spec/README.md) -- [ibc](https://github.com/cosmos/ibc/blob/f990a7f96eb7753c2fabbd49ed50b64d3a807629/README.md) -- [interchain accounts](https://github.com/cosmos/ibc/blob/2921c5cec7b18e4ef77677e16a6b693051ae3b35/spec/app/ics-027-interchain-accounts/README.md) -- [packetforwardmiddleware](https://github.com/cosmos/ibc-apps/blob/main/middleware/packet-forward-middleware/README.md) -- [params](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/params/spec/README.md) -- [slashing](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/slashing/spec/README.md) -- [staking](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/staking/spec/README.md) -- [transfer](https://github.com/cosmos/ibc/blob/f990a7f96eb7753c2fabbd49ed50b64d3a807629/spec/app/ics-020-fungible-token-transfer/README.md) -- [vesting](https://github.com/celestiaorg/cosmos-sdk/tree/v1.14.0-sdk-v0.46.11/x/auth/vesting) (no spec) diff --git a/specs/src/state_machine_modules.md b/specs/src/state_machine_modules.md new file mode 100644 index 0000000000..3889c42472 --- /dev/null +++ b/specs/src/state_machine_modules.md @@ -0,0 +1,6 @@ +# State Machine Modules + +Celestia app is built using the cosmos-sdk, and follows standard cosmos-sdk module structure. The modules used in the application vary based on app version: + +- [State Machine Modules v1](state_machine_modules_v1.md) +- [State Machine Modules v2](state_machine_modules_v2.md) diff --git a/specs/src/state_machine_modules_v1.md b/specs/src/state_machine_modules_v1.md new file mode 100644 index 0000000000..eef6d4df54 --- /dev/null +++ b/specs/src/state_machine_modules_v1.md @@ -0,0 +1,30 @@ +# State Machine Modules v1 + +The modules used in app version 1 are: + +## `celestia-app` modules + +- [blob](https://github.com/celestiaorg/celestia-app/blob/main/x/blob/README.md) +- [blobstream](https://github.com/celestiaorg/celestia-app/blob/main/x/blobstream/README.md) +- [mint](https://github.com/celestiaorg/celestia-app/blob/main/x/mint/README.md) +- [paramfilter](https://github.com/celestiaorg/celestia-app/blob/main/x/paramfilter/README.md) +- [tokenfilter](https://github.com/celestiaorg/celestia-app/blob/main/x/tokenfilter/README.md) + +## `cosmos-sdk` modules + +- [auth](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/auth/spec/README.md) +- [authz](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/authz/spec/README.md) +- [bank](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/bank/spec/README.md) +- [capability](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/capability/spec/README.md) +- [crisis](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/crisis/spec/README.md) +- [distribution](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/distribution/spec/README.md) +- [evidence](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/evidence/spec/README.md) +- [feegrant](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/feegrant/spec/README.md) +- [genutil](https://github.com/celestiaorg/cosmos-sdk/tree/v1.14.0-sdk-v0.46.11/x/genutil) (no spec) +- [gov](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/gov/spec/README.md) +- [ibc](https://github.com/cosmos/ibc/blob/f990a7f96eb7753c2fabbd49ed50b64d3a807629/README.md) +- [params](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/params/spec/README.md) +- [slashing](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/slashing/spec/README.md) +- [staking](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/staking/spec/README.md) +- [transfer](https://github.com/cosmos/ibc/blob/f990a7f96eb7753c2fabbd49ed50b64d3a807629/spec/app/ics-020-fungible-token-transfer/README.md) +- [vesting](https://github.com/celestiaorg/cosmos-sdk/tree/v1.14.0-sdk-v0.46.11/x/auth/vesting) (no spec) diff --git a/specs/src/state_machine_modules_v2.md b/specs/src/state_machine_modules_v2.md new file mode 100644 index 0000000000..25fea4947e --- /dev/null +++ b/specs/src/state_machine_modules_v2.md @@ -0,0 +1,33 @@ +# State Machine Modules v2 + +The modules used in app version 2 are: + +## `celestia-app` modules + +- [blob](https://github.com/celestiaorg/celestia-app/blob/main/x/blob/README.md) +- [minfee](https://github.com/celestiaorg/celestia-app/blob/main/x/minfee/README.md) +- [mint](https://github.com/celestiaorg/celestia-app/blob/main/x/mint/README.md) +- [paramfilter](https://github.com/celestiaorg/celestia-app/blob/main/x/paramfilter/README.md) +- [signal](https://github.com/celestiaorg/celestia-app/blob/main/x/signal/README.md) +- [tokenfilter](https://github.com/celestiaorg/celestia-app/blob/main/x/tokenfilter/README.md) + +## `cosmos-sdk` modules + +- [auth](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/auth/spec/README.md) +- [authz](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/authz/spec/README.md) +- [bank](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/bank/spec/README.md) +- [capability](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/capability/spec/README.md) +- [crisis](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/crisis/spec/README.md) +- [distribution](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/distribution/spec/README.md) +- [evidence](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/evidence/spec/README.md) +- [feegrant](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/feegrant/spec/README.md) +- [genutil](https://github.com/celestiaorg/cosmos-sdk/tree/v1.14.0-sdk-v0.46.11/x/genutil) (no spec) +- [gov](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/gov/spec/README.md) +- [ibc](https://github.com/cosmos/ibc/blob/f990a7f96eb7753c2fabbd49ed50b64d3a807629/README.md) +- [interchain accounts](https://github.com/cosmos/ibc/blob/2921c5cec7b18e4ef77677e16a6b693051ae3b35/spec/app/ics-027-interchain-accounts/README.md) +- [packetforwardmiddleware](https://github.com/cosmos/ibc-apps/blob/main/middleware/packet-forward-middleware/README.md) +- [params](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/params/spec/README.md) +- [slashing](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/slashing/spec/README.md) +- [staking](https://github.com/celestiaorg/cosmos-sdk/blob/v1.14.0-sdk-v0.46.11/x/staking/spec/README.md) +- [transfer](https://github.com/cosmos/ibc/blob/f990a7f96eb7753c2fabbd49ed50b64d3a807629/spec/app/ics-020-fungible-token-transfer/README.md) +- [vesting](https://github.com/celestiaorg/cosmos-sdk/tree/v1.14.0-sdk-v0.46.11/x/auth/vesting) (no spec) diff --git a/x/blob/README.md b/x/blob/README.md index c39250b33b..3f1e0484ae 100644 --- a/x/blob/README.md +++ b/x/blob/README.md @@ -13,9 +13,9 @@ To use the blob module, users create and submit a `BlobTx` that is composed of: After the `BlobTx` is submitted to the network, a block producer separates the `sdk.Tx` from the blob(s). Both components get included in the -[data square](../../specs/src/specs/data_square_layout.md) in different namespaces: +[data square](../../specs/src/data_square_layout.md) in different namespaces: -1. The `sdk.Tx` and some metadata about the separated blobs gets included in the `PayForBlobNamespace` (one of the [reserved namespaces](../../specs/src/specs/namespace.md#reserved-namespaces)). +1. The `sdk.Tx` and some metadata about the separated blobs gets included in the `PayForBlobNamespace` (one of the [reserved namespaces](../../specs/src/namespace.md#reserved-namespaces)). 1. The blob(s) get included in the namespace specified by each blob. After a block has been created, the user can verify that their data was included @@ -89,10 +89,9 @@ message MsgPayForBlobs { The share commitment is the commitment to share encoded blobs. It can be used for cheap inclusion checks for some data by light clients. More information and -rational can be found in the [data square layout -specs](../../specs/src/specs/data_square_layout.md). +rational can be found in the [data square layout](../../specs/src/data_square_layout.md) specification. -1. Split the blob into shares of size [`shareSize`](../../specs/src/specs/data_structures.md#consensus-parameters) +1. Split the blob into shares of size [`shareSize`](../../specs/src/data_structures.md#consensus-parameters) 1. Determine the [`SubtreeWidth`](https://github.com/celestiaorg/celestia-app/blob/v1.0.0-rc2/pkg/shares/non_interactive_defaults.go#L94-L116) by dividing the length in shares by the `SubtreeRootThreshold`. @@ -106,8 +105,7 @@ specs](../../specs/src/specs/data_square_layout.md). See [`CreateCommitment`](https://github.com/celestiaorg/celestia-app/blob/v1.0.0-rc2/x/blob/types/payforblob.go#L169-L236) -for an implementation. See [data square -layout](../../specs/src/specs/data_square_layout.md) and +for an implementation. See [data square layout](../../specs/src/data_square_layout.md) and [ADR013](../../docs/architecture/adr-013-non-interactive-default-rules-for-zero-padding.md) for details on the rational of the square layout. @@ -145,9 +143,7 @@ each PFB, to be included in a block must follow a set of validity rules. When a block producer is preparing a block, they must perform an extra step for `BlobTx`s so that end-users can find the blob shares relevant to their submitted `BlobTx`. In particular, block proposers wrap the `BlobTx` in the PFB namespace -with the index of the first share of the blob in the data square. See [Blob -share commitment -rules](../../specs/src/specs/data_square_layout.md#blob-share-commitment-rules) +with the index of the first share of the blob in the data square. See [Blob share commitment rules](../../specs/src/data_square_layout.md#blob-share-commitment-rules) for more details. Since `BlobTx`s can contain multiple blobs, the `sdk.Tx` portion of the `BlobTx` diff --git a/x/blobstream/overview.md b/x/blobstream/overview.md index afd3b4c1f8..bfe07a59e4 100644 --- a/x/blobstream/overview.md +++ b/x/blobstream/overview.md @@ -99,7 +99,7 @@ These signatures are then verified in the smart contract using the [`verifySig() ## Security assumptions -The security of the Blobstream relies on an honest majority of the Celestia validator set. This assumption indicates that more than 2/3s of the voting power follows each [block validity rule](../../specs/src/specs/block_validity_rules.md). Additionally, over 2/3s of the voting power sign valid validator set updates and data commitments, as outlined above. +The security of the Blobstream relies on an honest majority of the Celestia validator set. This assumption indicates that more than 2/3s of the voting power follows each [block validity rule](../../specs/src/block_validity_rules.md). Additionally, over 2/3s of the voting power sign valid validator set updates and data commitments, as outlined above. If more than 1/3rd of the validator set stops running their orchestrators, then the Blobstream halts. And, if more than 2/3rds sign invalid data, then the Blobstream contract will commit to invalid data. The only recovery from such a state is to revert to social consensus, potentially slashing the guilty validators and redeploying the smart contracts. From 43344db99ffd2893bce1eb4c72b4a00e31887f62 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Fri, 30 Aug 2024 16:38:16 -0400 Subject: [PATCH 17/85] feat(tools): block ETA (#3829) I used this tool to predict the arrival time of the v2 activation height on Mocha. --- tools/blocketa/README.md | 18 +++++++++ tools/blocketa/main.go | 81 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 tools/blocketa/README.md create mode 100644 tools/blocketa/main.go diff --git a/tools/blocketa/README.md b/tools/blocketa/README.md new file mode 100644 index 0000000000..c4968e2c22 --- /dev/null +++ b/tools/blocketa/README.md @@ -0,0 +1,18 @@ +# Block ETA (Estimated Time of Arrival) + +blocketa is a tool that estimates the time of arrival of a block height. + +## Usage + +```shell +$ go run main.go https://celestia-mocha-rpc.publicnode.com:443 2585031 +chainID: mocha-4 +currentHeight: 2580660 +currentTime: 2024-08-28 02:46:32.933542677 +0000 UTC +diffInBlockHeight: 4371 +diffInTime: 14h37m50.55s +arrivalTime: 2024-08-28 17:24:23.483542677 +0000 UTC +``` + +> [!NOTE] +> The block time is currently hard-coded. If you're running this for a network with a different block time, you'll need to update the `blockTime` constant in the main.go file. You can use https://www.mintscan.io/celestia/block/ or the blocktime tool. diff --git a/tools/blocketa/main.go b/tools/blocketa/main.go new file mode 100644 index 0000000000..54c21c66a1 --- /dev/null +++ b/tools/blocketa/main.go @@ -0,0 +1,81 @@ +package main + +import ( + "context" + "fmt" + "os" + "strconv" + "time" + + "github.com/tendermint/tendermint/rpc/client/http" +) + +const ( + // blockTime is the observed average time between blocks. You can update this + // value based on the block time on https://www.mintscan.io/celestia/block/ or + // the output from the blocktime tool. + blockTime = 12.05 // seconds between blocks for Mocha + + // exampleArabicaRPC is an example node RPC endpoint for the Arabica testnet. + exampleArabicaRPC = "https://rpc.celestia-arabica-11.com:443" + + // exampleMochaRPC is an example node RPC endpoint for the Mocha testnet. + exampleMochaRPC = "https://celestia-mocha-rpc.publicnode.com:443" + + // exampleArabicaHeight is an example block height for the Arabica testnet. + exampleArabicaHeight = 1751707 + + // exampleMochaHeight is an example block height for the Mocha testnet. + exampleMochaHeight = 2585031 +) + +func main() { + if err := Run(); err != nil { + fmt.Printf("ERROR: %s", err.Error()) + } +} + +func Run() error { + if len(os.Args) < 3 { + fmt.Printf("Usage: %s \n", os.Args[0]) + fmt.Printf("Example: %s %s %v\n", os.Args[0], exampleArabicaRPC, exampleArabicaHeight) + fmt.Printf("Example: %s %s %v\n", os.Args[0], exampleMochaRPC, exampleMochaHeight) + return nil + } + + _, nodeRPC, targetBlockHeightArg := os.Args[0], os.Args[1], os.Args[2] + targetBlockHeight, err := strconv.ParseInt(targetBlockHeightArg, 10, 64) + if err != nil { + return err + } + c, err := http.New(nodeRPC, "/websocket") + if err != nil { + return err + } + resp, err := c.Status(context.Background()) + if err != nil { + return err + } + chainID := resp.NodeInfo.Network + currentHeight := resp.SyncInfo.LatestBlockHeight + currentTime := resp.SyncInfo.LatestBlockTime + + if currentHeight >= targetBlockHeight { + return fmt.Errorf("current height %v is already after target height %v", currentHeight, targetBlockHeight) + } + diffInBlockHeight := targetBlockHeight - currentHeight + diffInSeconds := blockTime * float64(diffInBlockHeight) + diffInTime, err := time.ParseDuration(fmt.Sprintf("%vs", diffInSeconds)) + if err != nil { + return err + } + arrivalTime := currentTime.Add(diffInTime) + + fmt.Printf("chainID: %v\n", chainID) + fmt.Printf("currentHeight: %v\n", currentHeight) + fmt.Printf("currentTime: %v\n", currentTime.String()) + fmt.Printf("diffInBlockHeight: %v\n", diffInBlockHeight) + fmt.Printf("diffInTime: %v\n", diffInTime) + fmt.Printf("arrivalTime: %v\n", arrivalTime) + return nil +} From 2931f15d27ee54424d425a5abfa76f6678e3ea7e Mon Sep 17 00:00:00 2001 From: Rootul P Date: Fri, 30 Aug 2024 19:23:30 -0400 Subject: [PATCH 18/85] ci(goreleaser): fix release candidate suffix (#3833) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prevents https://github.com/celestiaorg/celestia-app/issues/3819 from happening again. Also adds a document so other maintainers can generate + upload prebuilt binaries if they don't work again in the future. The current `-rc*` [prerelease_suffix](https://goreleaser.com/customization/git/#git) was not working because goreleaser treated v2.1.2-rc1 as more recent than v2.1.2 in the [failing workflow](https://github.com/celestiaorg/celestia-app/actions/runs/10523365886/job/29157983992#step:6:68). This resolves by removing the `*`. ## Testing 1. Worked on my fork: https://github.com/rootulp/celestia-app/releases/tag/v2.1.2 2. Worked on this repo: https://github.com/celestiaorg/celestia-app/releases/tag/v2.1.2 ### Before ``` • git state commit=48173df3dc78f9348eedb3796f29ef9e9e5dc584 branch=HEAD current_tag=v2.1.2-rc1 previous_tag=v2.1.2-rc0 dirty=true ``` ### After ``` • git state commit=48173df3dc78f9348eedb3796f29ef9e9e5dc584 branch=HEAD current_tag=v2.1.2 previous_tag=v2.1.2-rc0 dirty=true ``` It is curious that it thinks the previous tag is v2.1.2-rc0 and not v2.1.2-rc1 but I couldn't fix with various [tag_sort](https://goreleaser.com/customization/git/?h=tag_sort#git) options. I think this only impacts Goreleaser changelog generation which we don't use b/c we use Github's auto generate release notes feature. --- .goreleaser.yaml | 2 +- docs/maintainers/prebuilt-binaries.md | 49 +++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 docs/maintainers/prebuilt-binaries.md diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 950143a3f1..f36e8e0435 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -118,4 +118,4 @@ changelog: release: prerelease: auto git: - prerelease_suffix: "-rc*" + prerelease_suffix: "-rc" diff --git a/docs/maintainers/prebuilt-binaries.md b/docs/maintainers/prebuilt-binaries.md new file mode 100644 index 0000000000..9be52d76bc --- /dev/null +++ b/docs/maintainers/prebuilt-binaries.md @@ -0,0 +1,49 @@ +# Prebuilt binaries + +Prebuilt binaries are attached to each release via [GoReleaser](https://goreleaser.com/) which runs in Github Actions. If GoReleaser failed to attach prebuilt binaries, you may want to attach them manually by following the steps below. + +## Prerequisites + +1. Create a Github token (classic) that has `repo:public_repo` scope via: + + ```shell + export GORELEASER_ACCESS_TOKEN= + echo "GITHUB_TOKEN=${GORELEASER_ACCESS_TOKEN}" >> .release-env + ``` + +## Steps + +1. [Optional] If you need to make any code changes to fix the issue that occured when CI tried to generate and attach the prebuilt binaries, then you likely need to skip validation when running GoReleaser locally. To skip validation, modify the Makefile command like so: + + ```diff + ## prebuilt-binary: Create prebuilt binaries and attach them to GitHub release. Requires Docker. + prebuilt-binary: + @if [ ! -f ".release-env" ]; then \ + echo "A .release-env file was not found but is required to create prebuilt binaries. This command is expected to be run in CI where a .release-env file exists. If you need to run this command locally to attach binaries to a release, you need to create a .release-env file with a Github token (classic) that has repo:public_repo scope."; \ + exit 1;\ + fi + docker run \ + --rm \ + -e CGO_ENABLED=1 \ + --env-file .release-env \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v `pwd`:/go/src/$(PACKAGE_NAME) \ + -w /go/src/$(PACKAGE_NAME) \ + ghcr.io/goreleaser/goreleaser-cross:${GOLANG_CROSS_VERSION} \ + - release --clean + + release --clean --skip=validate + .PHONY: prebuilt-binary + ``` + +1. Before proceeding, test your change by running `make prebuilt-binary` and verifying that prebuilt binaries are attached to a release on your Github fork. +1. Modify `.goreleaser.yaml` so that you can upload assets to the main repository: + + ```diff + release: + + github: + + owner: celestiaorg + + name: celestia-app + ``` + +1. Run `make prebuilt-binary` to generate and attach the prebuilt binaries. +1. Verify the assets were attached to the release on the main repository. From 588ea980e89a020b46f96b19b95149f1c3a79030 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Sep 2024 17:56:49 +0200 Subject: [PATCH 19/85] chore(deps): Bump google.golang.org/grpc from 1.65.0 to 1.66.0 (#3834) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.65.0 to 1.66.0.
Release notes

Sourced from google.golang.org/grpc's releases.

Release 1.66.0

New Features

  • metadata: stabilize ValueFromIncomingContext (#7368)
  • client: stabilize the WaitForStateChange and GetState methods, which were previously experimental. (#7425)
  • xds: Implement ADS flow control mechanism (#7458)
  • balancer/rls: Add metrics for data cache and picker internals (#7484, #7495)
  • xds: LRS load reports now include the total_issued_requests field. (#7544)

Bug Fixes

  • grpc: Clients now return status code INTERNAL instead of UNIMPLEMENTED when the server uses an unsupported compressor. This is consistent with the gRPC compression spec. (#7461)
  • transport: Fix a bug which could result in writes busy looping when the underlying conn.Write returns errors (#7394)
  • client: fix race that could lead to orphaned connections and associated resources. (#7390)
  • xds: use locality from the connected address for load reporting with pick_first (#7378)
    • without this fix, if a priority contains multiple localities with pick_first, load was reported for the wrong locality
  • client: prevent hanging during ClientConn.Close() when the network is unreachable (#7540)

Performance Improvements

  • transport: double buffering is avoided when using an http connect proxy and the target server waits for client to send the first message. (#7424)
  • codec: Implement a new Codec which uses buffer recycling for encoded message (#7356)
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=google.golang.org/grpc&package-manager=go_modules&previous-version=1.65.0&new-version=1.66.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 18 +++++++++--------- go.sum | 40 ++++++++++++++++++++-------------------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/go.mod b/go.mod index b6b9b48739..79efbfa194 100644 --- a/go.mod +++ b/go.mod @@ -30,8 +30,8 @@ require ( github.com/tendermint/tendermint v0.34.29 github.com/tendermint/tm-db v0.6.7 golang.org/x/exp v0.0.0-20240213143201-ec583247a57a - google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 - google.golang.org/grpc v1.65.0 + google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 + google.golang.org/grpc v1.66.0 google.golang.org/protobuf v1.34.2 gopkg.in/yaml.v2 v2.4.0 ) @@ -218,17 +218,17 @@ require ( go.opentelemetry.io/proto/otlp v1.1.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.23.0 // indirect - golang.org/x/net v0.25.0 // indirect - golang.org/x/oauth2 v0.20.0 // indirect + golang.org/x/crypto v0.24.0 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/oauth2 v0.21.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/term v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/term v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/api v0.169.0 // indirect google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index cbcbd380ad..8b9c945120 100644 --- a/go.sum +++ b/go.sum @@ -1433,8 +1433,8 @@ golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1542,8 +1542,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1569,8 +1569,8 @@ golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= -golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= -golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1699,14 +1699,14 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= -golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1717,8 +1717,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1791,8 +1791,8 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= -golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1978,10 +1978,10 @@ google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= -google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw= -google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 h1:+rdxYoE3E5htTEWIe15GlN6IfvbURM//Jt0mmkmm6ZU= +google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117/go.mod h1:OimBR/bc1wPO9iV4NC2bpyjy3VnAwZh5EBPQdtaE5oo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -2024,8 +2024,8 @@ google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= -google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= From 4b34f8b3c707f6ceacbee6afe0d061b29645ce4b Mon Sep 17 00:00:00 2001 From: Rootul P Date: Thu, 5 Sep 2024 08:29:56 -0400 Subject: [PATCH 20/85] feat(tools): more examples for blocketa and blockheight (#3842) ## Description 1. Add more examples for both tools `blockheight` and `blocketa` 2. Use the times + heights for the v2 activation height on each network --- tools/blocketa/main.go | 11 +++++++++-- tools/blockheight/main.go | 26 ++++++++++++++++++++------ 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/tools/blocketa/main.go b/tools/blocketa/main.go index 54c21c66a1..7706ffd431 100644 --- a/tools/blocketa/main.go +++ b/tools/blocketa/main.go @@ -14,7 +14,7 @@ const ( // blockTime is the observed average time between blocks. You can update this // value based on the block time on https://www.mintscan.io/celestia/block/ or // the output from the blocktime tool. - blockTime = 12.05 // seconds between blocks for Mocha + blockTime = 11.75 // seconds between blocks on Mainnet Beta. // exampleArabicaRPC is an example node RPC endpoint for the Arabica testnet. exampleArabicaRPC = "https://rpc.celestia-arabica-11.com:443" @@ -22,11 +22,17 @@ const ( // exampleMochaRPC is an example node RPC endpoint for the Mocha testnet. exampleMochaRPC = "https://celestia-mocha-rpc.publicnode.com:443" + // exampleMainnetHeight is an example node RPC endpoint for Mainnet Beta. + exampleMainnetRPC = "https://celestia-rpc.publicnode.com:443" + // exampleArabicaHeight is an example block height for the Arabica testnet. exampleArabicaHeight = 1751707 // exampleMochaHeight is an example block height for the Mocha testnet. exampleMochaHeight = 2585031 + + // exampleMainnetHeight is an example block height for Mainnet Beta. + exampleMainnetHeight = 2371495 ) func main() { @@ -40,6 +46,7 @@ func Run() error { fmt.Printf("Usage: %s \n", os.Args[0]) fmt.Printf("Example: %s %s %v\n", os.Args[0], exampleArabicaRPC, exampleArabicaHeight) fmt.Printf("Example: %s %s %v\n", os.Args[0], exampleMochaRPC, exampleMochaHeight) + fmt.Printf("Example: %s %s %v\n", os.Args[0], exampleMainnetRPC, exampleMainnetHeight) return nil } @@ -65,7 +72,7 @@ func Run() error { } diffInBlockHeight := targetBlockHeight - currentHeight diffInSeconds := blockTime * float64(diffInBlockHeight) - diffInTime, err := time.ParseDuration(fmt.Sprintf("%vs", diffInSeconds)) + diffInTime, err := time.ParseDuration(fmt.Sprintf("%.0fs", diffInSeconds)) if err != nil { return err } diff --git a/tools/blockheight/main.go b/tools/blockheight/main.go index c20342aec9..e309b29b6f 100644 --- a/tools/blockheight/main.go +++ b/tools/blockheight/main.go @@ -14,13 +14,25 @@ const ( // blockTime is the observed average time between blocks. You can update this // value based on the block time on https://www.mintscan.io/celestia/block/ or // the output from the blocktime tool. - blockTime = 11.30 // seconds between blocks for Arabica + blockTime = 11.75 // seconds between blocks on Mainnet Beta. - // exampleNodeRPC is an example node RPC endpoint for the Arabica testnet. - exampleNodeRPC = "https://rpc.celestia-arabica-11.com:443" + // exampleArabicaRPC is an example node RPC endpoint for the Arabica testnet. + exampleArabicaRPC = "https://rpc.celestia-arabica-11.com:443" - // targetTime is an example target time for the block height prediction. - targetTime = "2024-08-14T14:00:00" + // exampleMochaRPC is an example node RPC endpoint for the Mocha testnet. + exampleMochaRPC = "https://celestia-mocha-rpc.publicnode.com:443" + + // exampleMainnetHeight is an example node RPC endpoint for Mainnet Beta. + exampleMainnetRPC = "https://celestia-rpc.publicnode.com:443" + + // exampleArabicaTime is an example target time for the block height prediction. + exampleArabicaTime = "2024-08-19T14:00:00" + + // exampleMochaTime is an example target time for the block height prediction. + exampleMochaTime = "2024-08-28T14:00:00" + + // exampleMainnetTime is an example target time for the block height prediction. + exampleMainnetTime = "2024-09-18T14:00:00" // layout is the expected time format for targetTime. layout = "2006-01-02T15:04:05" @@ -35,7 +47,9 @@ func main() { func Run() error { if len(os.Args) < 3 { fmt.Printf("Usage: %s \n", os.Args[0]) - fmt.Printf("Example: %s %s %s\n", os.Args[0], exampleNodeRPC, targetTime) + fmt.Printf("Example: %s %s %s\n", os.Args[0], exampleArabicaRPC, exampleArabicaTime) + fmt.Printf("Example: %s %s %s\n", os.Args[0], exampleMochaRPC, exampleMochaTime) + fmt.Printf("Example: %s %s %s\n", os.Args[0], exampleMainnetRPC, exampleMainnetTime) return nil } From 2c74ea5c79345fdc03c74c7389d26f2bf579c578 Mon Sep 17 00:00:00 2001 From: Oleg Kovalov Date: Tue, 10 Sep 2024 14:08:25 +0200 Subject: [PATCH 21/85] chore(go.mod): bump grafana/pyroscope-go deps (#3849) On Go 1.23 I got such failures: ``` % make build --> Updating go.mod --> Updating go.mod in ./test/interchain # github.com/grafana/pyroscope-go/godeltaprof/internal/pprof ../../../go/pkg/mod/github.com/grafana/pyroscope-go/godeltaprof@v0.1.6/internal/pprof/delta_mutex.go:30:20: undefined: runtime_cyclesPerSecond ../../../go/pkg/mod/github.com/grafana/pyroscope-go/godeltaprof@v0.1.6/internal/pprof/proto.go:320:8: undefined: runtime_expandFinalInlineFrame make: *** [build] Error 1 ``` Due to recent change in Go 1.23 where Go team removed some internal things. Simple bump works fine. --- go.mod | 6 +++--- go.sum | 13 ++++++------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 79efbfa194..1de8165f96 100644 --- a/go.mod +++ b/go.mod @@ -121,8 +121,8 @@ require ( github.com/gorilla/handlers v1.5.2 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/grafana/otel-profiling-go v0.5.1 // indirect - github.com/grafana/pyroscope-go v1.1.1 // indirect - github.com/grafana/pyroscope-go/godeltaprof v0.1.6 // indirect + github.com/grafana/pyroscope-go v1.1.2 // indirect + github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect github.com/gtank/merlin v0.1.1 // indirect @@ -146,7 +146,7 @@ require ( github.com/joho/godotenv v1.5.1 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.6 // indirect + github.com/klauspost/compress v1.17.8 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/klauspost/reedsolomon v1.12.1 // indirect github.com/lib/pq v1.10.7 // indirect diff --git a/go.sum b/go.sum index 8b9c945120..b8775d02cd 100644 --- a/go.sum +++ b/go.sum @@ -779,10 +779,10 @@ github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWm github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grafana/otel-profiling-go v0.5.1 h1:stVPKAFZSa7eGiqbYuG25VcqYksR6iWvF3YH66t4qL8= github.com/grafana/otel-profiling-go v0.5.1/go.mod h1:ftN/t5A/4gQI19/8MoWurBEtC6gFw8Dns1sJZ9W4Tls= -github.com/grafana/pyroscope-go v1.1.1 h1:PQoUU9oWtO3ve/fgIiklYuGilvsm8qaGhlY4Vw6MAcQ= -github.com/grafana/pyroscope-go v1.1.1/go.mod h1:Mw26jU7jsL/KStNSGGuuVYdUq7Qghem5P8aXYXSXG88= -github.com/grafana/pyroscope-go/godeltaprof v0.1.6 h1:nEdZ8louGAplSvIJi1HVp7kWvFvdiiYg3COLlTwJiFo= -github.com/grafana/pyroscope-go/godeltaprof v0.1.6/go.mod h1:Tk376Nbldo4Cha9RgiU7ik8WKFkNpfds98aUzS8omLE= +github.com/grafana/pyroscope-go v1.1.2 h1:7vCfdORYQMCxIzI3NlYAs3FcBP760+gWuYWOyiVyYx8= +github.com/grafana/pyroscope-go v1.1.2/go.mod h1:HSSmHo2KRn6FasBA4vK7BMiQqyQq8KSuBKvrhkXxYPU= +github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg= +github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.2.2/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI= @@ -925,9 +925,8 @@ github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= -github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= -github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= -github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= From 226ad616b6100a6ad7a81306c86219b85dc44a91 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Wed, 11 Sep 2024 09:41:34 +0200 Subject: [PATCH 22/85] fix: show bbr error message on `celestia-appd start` (#3851) Closes https://github.com/celestiaorg/celestia-app/issues/3845 ## Before ``` $ celestia-appd start Error: exit status 1 ``` ## After ``` $ celestia-appd start The BBR (Bottleneck Bandwidth and Round-trip propagation time) congestion control algorithm is not enabled in this system's kernel. BBR is important for the performance of the p2p stack. To enable BBR: sudo modprobe tcp_bbr net.core.default_qdisc=fq net.ipv4.tcp_congestion_control=bbr sudo sysctl -p Then verify BBR is enabled: sysctl net.ipv4.tcp_congestion_control This node will get worse p2p performance using a different congestion control algorithm. If you need to bypass this check use the --force-no-bbr flag. Error: failed to execute 'sysctl net.ipv4.tcp_congestion_control' exit status 1 ``` --- cmd/celestia-appd/cmd/start.go | 38 ++++++++++++++-------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/cmd/celestia-appd/cmd/start.go b/cmd/celestia-appd/cmd/start.go index 04c64173c3..763f1b6cd5 100644 --- a/cmd/celestia-appd/cmd/start.go +++ b/cmd/celestia-appd/cmd/start.go @@ -4,7 +4,6 @@ package cmd // start command flag. import ( - "errors" "fmt" "io" "net" @@ -566,37 +565,30 @@ func addCommands( ) } -// checkBBR checks is bbr is configured to be used as a congestion control algo. +// checkBBR checks if BBR is enabled. func checkBBR(command *cobra.Command) error { const ( - errorMsg = ` -// The BBR congestion control algorithm does not appear to be enabled in this -// system's kernel. This is important for the p2p stack to be performant. -// -// to enable bbr call: -// + warning = ` +The BBR (Bottleneck Bandwidth and Round-trip propagation time) congestion control algorithm is not enabled in this system's kernel. +BBR is important for the performance of the p2p stack. + +To enable BBR: sudo modprobe tcp_bbr net.core.default_qdisc=fq net.ipv4.tcp_congestion_control=bbr sudo sysctl -p -// -// and can be verified to be running using -// + +Then verify BBR is enabled: sysctl net.ipv4.tcp_congestion_control -// This might not work for all systems, you might have to search online to -// figure out how to enable bbr for your system. -// -// While this node will get worse performance using something other than bbr, -// If you need to bypass this block use the "--force-no-bbr true" flag. - ` +This node will get worse p2p performance using a different congestion control algorithm. +If you need to bypass this check use the --force-no-bbr flag. +` ) - noBBRErr := errors.New(errorMsg) - forceNoBBR, err := command.Flags().GetBool(FlagForceNoBBR) if err != nil { - return noBBRErr + return err } if forceNoBBR { return nil @@ -605,11 +597,13 @@ sysctl net.ipv4.tcp_congestion_control cmd := exec.Command("sysctl", "net.ipv4.tcp_congestion_control") output, err := cmd.Output() if err != nil { - return err + fmt.Print(warning) + return fmt.Errorf("failed to execute 'sysctl net.ipv4.tcp_congestion_control' %w", err) } if !strings.Contains(string(output), "bbr") { - return noBBRErr + fmt.Print(warning) + return fmt.Errorf("BBR not enabled because output %v does not contain 'bbr'", string(output)) } return nil From 0d0b9e76be630f4a833b876150a0eddd54a79aa4 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Wed, 11 Sep 2024 19:02:11 +0200 Subject: [PATCH 23/85] ci: request reviews from all celestia-core members (#3856) Closes https://github.com/celestiaorg/celestia-app/issues/3850 I'm not sure if this explicitly assigns all celestia-core members as reviewers or just the entire group. --- .github/auto_request_review.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/auto_request_review.yml b/.github/auto_request_review.yml index fcaed3c592..a00957c0e3 100644 --- a/.github/auto_request_review.yml +++ b/.github/auto_request_review.yml @@ -12,7 +12,3 @@ options: ignored_keywords: - DO NOT REVIEW enable_group_assignment: false - - # Randomly pick reviewers up to this number. - # Do not set this option if you'd like to assign all matching reviewers. - number_of_reviewers: 2 From 2c53ddbe0bf0d6aa0e944f34fbd5aae34581f742 Mon Sep 17 00:00:00 2001 From: Sanaz Taheri <35961250+staheri14@users.noreply.github.com> Date: Wed, 11 Sep 2024 12:21:57 -0700 Subject: [PATCH 24/85] test!: adds the ability to set or unset bbr in e2e tests (#3817) Closes #3815 by disabling BBR in e2e tests. --- docs/maintainers/release-guide.md | 2 +- test/e2e/benchmark/benchmark.go | 2 +- test/e2e/benchmark/manifest.go | 10 ++++++++-- test/e2e/benchmark/throughput.go | 2 ++ test/e2e/major_upgrade_v2.go | 3 ++- test/e2e/minor_version_compatibility.go | 4 +++- test/e2e/simple.go | 4 +++- test/e2e/testnet/node.go | 4 ++++ test/e2e/testnet/testnet.go | 16 ++++++++++------ tools/blocketa/README.md | 2 +- 10 files changed, 35 insertions(+), 14 deletions(-) diff --git a/docs/maintainers/release-guide.md b/docs/maintainers/release-guide.md index 7260e95192..33d181aaab 100644 --- a/docs/maintainers/release-guide.md +++ b/docs/maintainers/release-guide.md @@ -27,7 +27,7 @@ The target audience for this guide is maintainers of this repo. In general, the ## Official Release -Follow the [creating a release candidate](#creating-a-release-candidate) section with the following considerations: +Follow the [creating a release candidate](#creating-a-release-candidate) section with the following considerations: - The version tag should not include the `-rc` suffix. - The release notes should contain an **Upgrade Notice** section with notable changes for node operators or library consumers. diff --git a/test/e2e/benchmark/benchmark.go b/test/e2e/benchmark/benchmark.go index 456f8da0dd..e34d322bb8 100644 --- a/test/e2e/benchmark/benchmark.go +++ b/test/e2e/benchmark/benchmark.go @@ -38,7 +38,7 @@ func (b *BenchmarkTest) SetupNodes() error { testnet.NoError("failed to create genesis nodes", b.CreateGenesisNodes(b.manifest.Validators, b.manifest.CelestiaAppVersion, b.manifest.SelfDelegation, - b.manifest.UpgradeHeight, b.manifest.ValidatorResource)) + b.manifest.UpgradeHeight, b.manifest.ValidatorResource, b.manifest.DisableBBR)) // enable latency if specified in the manifest if b.manifest.EnableLatency { diff --git a/test/e2e/benchmark/manifest.go b/test/e2e/benchmark/manifest.go index f130069657..64b319fcc8 100644 --- a/test/e2e/benchmark/manifest.go +++ b/test/e2e/benchmark/manifest.go @@ -78,6 +78,8 @@ type Manifest struct { UpgradeHeight int64 GovMaxSquareSize int64 + + DisableBBR bool } func (m *Manifest) GetGenesisModifiers() []genesis.Modifier { @@ -103,12 +105,16 @@ func (m *Manifest) summary() string { if m.EnableLatency { latency = 1 } + bbr := 1 + if m.DisableBBR { + bbr = 0 + } maxBlockMB := m.MaxBlockBytes / testnet.MB - summary := fmt.Sprintf("v%d-t%d-b%d-bw%dmb-tc%d-tp%d-l%d-%s-%dmb", + summary := fmt.Sprintf("v%d-t%d-b%d-bw%dmb-tc%d-tp%d-l%d-%s-br%d-%dmb", m.Validators, m.TxClients, m.BlobSequences, m.PerPeerBandwidth/testnet.MB, m.TimeoutCommit/time.Second, m.TimeoutPropose/time.Second, - latency, m.Mempool, maxBlockMB) + latency, m.Mempool, bbr, maxBlockMB) if len(summary) > 50 { return summary[:50] } diff --git a/test/e2e/benchmark/throughput.go b/test/e2e/benchmark/throughput.go index b919093d8a..b0392468ce 100644 --- a/test/e2e/benchmark/throughput.go +++ b/test/e2e/benchmark/throughput.go @@ -51,6 +51,7 @@ var bigBlockManifest = Manifest{ TestDuration: 5 * time.Minute, LocalTracingType: "local", PushTrace: true, + DisableBBR: true, } func TwoNodeSimple(logger *log.Logger) error { @@ -88,6 +89,7 @@ func TwoNodeSimple(logger *log.Logger) error { DownloadTraces: false, TestDuration: 3 * time.Minute, TxClients: 2, + DisableBBR: true, } benchTest, err := NewBenchmarkTest(testName, &manifest) diff --git a/test/e2e/major_upgrade_v2.go b/test/e2e/major_upgrade_v2.go index f595be542d..9797aa9c26 100644 --- a/test/e2e/major_upgrade_v2.go +++ b/test/e2e/major_upgrade_v2.go @@ -43,7 +43,8 @@ func MajorUpgradeToV2(logger *log.Logger) error { logger.Println("Creating genesis nodes") for i := 0; i < numNodes; i++ { - err := testNet.CreateGenesisNode(latestVersion, 10000000, upgradeHeight, testnet.DefaultResources) + err := testNet.CreateGenesisNode(latestVersion, 10000000, + upgradeHeight, testnet.DefaultResources, true) testnet.NoError("failed to create genesis node", err) } diff --git a/test/e2e/minor_version_compatibility.go b/test/e2e/minor_version_compatibility.go index d0fd6314e9..9c09b57895 100644 --- a/test/e2e/minor_version_compatibility.go +++ b/test/e2e/minor_version_compatibility.go @@ -55,7 +55,9 @@ func MinorVersionCompatibility(logger *log.Logger) error { // each node begins with a random version within the same major version set v := versions.Random(r).String() logger.Println("Starting node", "node", i, "version", v) - testnet.NoError("failed to create genesis node", testNet.CreateGenesisNode(v, 10000000, 0, testnet.DefaultResources)) + testnet.NoError("failed to create genesis node", + testNet.CreateGenesisNode(v, 10000000, 0, + testnet.DefaultResources, false)) } logger.Println("Creating txsim") diff --git a/test/e2e/simple.go b/test/e2e/simple.go index 6673129f41..12e493de49 100644 --- a/test/e2e/simple.go +++ b/test/e2e/simple.go @@ -26,7 +26,9 @@ func E2ESimple(logger *log.Logger) error { defer testNet.Cleanup() logger.Println("Creating testnet validators") - testnet.NoError("failed to create genesis nodes", testNet.CreateGenesisNodes(4, latestVersion, 10000000, 0, testnet.DefaultResources)) + testnet.NoError("failed to create genesis nodes", + testNet.CreateGenesisNodes(4, latestVersion, 10000000, 0, + testnet.DefaultResources, true)) logger.Println("Creating txsim") endpoints, err := testNet.RemoteGRPCEndpoints() diff --git a/test/e2e/testnet/node.go b/test/e2e/testnet/node.go index addbc98066..270cd545b7 100644 --- a/test/e2e/testnet/node.go +++ b/test/e2e/testnet/node.go @@ -97,6 +97,7 @@ func NewNode( upgradeHeight int64, resources Resources, grafana *GrafanaInfo, + disableBBR bool, ) (*Node, error) { instance, err := knuu.NewInstance(name) if err != nil { @@ -148,6 +149,9 @@ func NewNode( return nil, err } args := []string{"start", fmt.Sprintf("--home=%s", remoteRootDir), "--rpc.laddr=tcp://0.0.0.0:26657"} + if disableBBR { + args = append(args, "--force-no-bbr") + } if upgradeHeight != 0 { args = append(args, fmt.Sprintf("--v2-upgrade-height=%d", upgradeHeight)) } diff --git a/test/e2e/testnet/testnet.go b/test/e2e/testnet/testnet.go index 4b3beebe99..76d0bc15df 100644 --- a/test/e2e/testnet/testnet.go +++ b/test/e2e/testnet/testnet.go @@ -54,12 +54,12 @@ func (t *Testnet) SetConsensusMaxBlockSize(size int64) { t.genesis.ConsensusParams.Block.MaxBytes = size } -func (t *Testnet) CreateGenesisNode(version string, selfDelegation, upgradeHeight int64, resources Resources) error { +func (t *Testnet) CreateGenesisNode(version string, selfDelegation, upgradeHeight int64, resources Resources, disableBBR bool) error { signerKey := t.keygen.Generate(ed25519Type) networkKey := t.keygen.Generate(ed25519Type) node, err := NewNode(fmt.Sprintf("val%d", len(t.nodes)), version, 0, selfDelegation, nil, signerKey, networkKey, upgradeHeight, resources, - t.grafana) + t.grafana, disableBBR) if err != nil { return err } @@ -70,9 +70,9 @@ func (t *Testnet) CreateGenesisNode(version string, selfDelegation, upgradeHeigh return nil } -func (t *Testnet) CreateGenesisNodes(num int, version string, selfDelegation, upgradeHeight int64, resources Resources) error { +func (t *Testnet) CreateGenesisNodes(num int, version string, selfDelegation, upgradeHeight int64, resources Resources, disableBBR bool) error { for i := 0; i < num; i++ { - if err := t.CreateGenesisNode(version, selfDelegation, upgradeHeight, resources); err != nil { + if err := t.CreateGenesisNode(version, selfDelegation, upgradeHeight, resources, disableBBR); err != nil { return err } } @@ -232,12 +232,12 @@ func (t *Testnet) CreateAccount(name string, tokens int64, txsimKeyringDir strin return kr, nil } -func (t *Testnet) CreateNode(version string, startHeight, upgradeHeight int64, resources Resources) error { +func (t *Testnet) CreateNode(version string, startHeight, upgradeHeight int64, resources Resources, disableBBR bool) error { signerKey := t.keygen.Generate(ed25519Type) networkKey := t.keygen.Generate(ed25519Type) node, err := NewNode(fmt.Sprintf("val%d", len(t.nodes)), version, startHeight, 0, nil, signerKey, networkKey, upgradeHeight, resources, - t.grafana) + t.grafana, disableBBR) if err != nil { return err } @@ -383,8 +383,12 @@ func (t *Testnet) StartNodes() error { for _, node := range genesisNodes { err := node.WaitUntilStartedAndForwardPorts() if err != nil { + log.Err(err).Str("name", node.Name).Str("version", + node.Version).Msg("failed to start and forward ports") return fmt.Errorf("node %s failed to start: %w", node.Name, err) } + log.Info().Str("name", node.Name).Str("version", + node.Version).Msg("started and ports forwarded") } return nil } diff --git a/tools/blocketa/README.md b/tools/blocketa/README.md index c4968e2c22..f7de7539b5 100644 --- a/tools/blocketa/README.md +++ b/tools/blocketa/README.md @@ -15,4 +15,4 @@ arrivalTime: 2024-08-28 17:24:23.483542677 +0000 UTC ``` > [!NOTE] -> The block time is currently hard-coded. If you're running this for a network with a different block time, you'll need to update the `blockTime` constant in the main.go file. You can use https://www.mintscan.io/celestia/block/ or the blocktime tool. +> The block time is currently hard-coded. If you're running this for a network with a different block time, you'll need to update the `blockTime` constant in the main.go file. You can use [https://www.mintscan.io/celestia/block](https://www.mintscan.io/celestia/block/) or the blocktime tool. From b4e12ca036ec85cbed7e6b09cf60866a2f6fa164 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Thu, 12 Sep 2024 10:03:49 +0200 Subject: [PATCH 25/85] chore: remove empty spec pages (#3863) Follow up to https://github.com/celestiaorg/celestia-app/pull/3822. Not sure how I missed this but there were mostly empty files in the nested `specs/` directory. I think a previous invocation of `mdbook serve` auto-generated them but they're not used. --- specs/src/specs/ante_handler.md | 1 - specs/src/specs/ante_handler_v1.md | 1 - specs/src/specs/ante_handler_v2.md | 1 - specs/src/specs/block_proposer.md | 1 - specs/src/specs/block_validity_rules.md | 1 - specs/src/specs/cat_pool.md | 1 - specs/src/specs/consensus.md | 1 - specs/src/specs/data_square_layout.md | 1 - specs/src/specs/data_structures.md | 1 - specs/src/specs/fraud_proofs.md | 1 - specs/src/specs/multisig.md | 1 - specs/src/specs/namespace.md | 1 - specs/src/specs/networking.md | 1 - specs/src/specs/parameters.md | 1 - specs/src/specs/parameters_v1.md | 1 - specs/src/specs/parameters_v2.md | 1 - specs/src/specs/public_key_cryptography.md | 1 - specs/src/specs/resource_pricing.md | 1 - specs/src/specs/shares.md | 1 - specs/src/specs/state_machine_modules.md | 1 - specs/src/specs/state_machine_modules_v1.md | 1 - specs/src/specs/state_machine_modules_v2.md | 1 - 22 files changed, 22 deletions(-) delete mode 100644 specs/src/specs/ante_handler.md delete mode 100644 specs/src/specs/ante_handler_v1.md delete mode 100644 specs/src/specs/ante_handler_v2.md delete mode 100644 specs/src/specs/block_proposer.md delete mode 100644 specs/src/specs/block_validity_rules.md delete mode 100644 specs/src/specs/cat_pool.md delete mode 100644 specs/src/specs/consensus.md delete mode 100644 specs/src/specs/data_square_layout.md delete mode 100644 specs/src/specs/data_structures.md delete mode 100644 specs/src/specs/fraud_proofs.md delete mode 100644 specs/src/specs/multisig.md delete mode 100644 specs/src/specs/namespace.md delete mode 100644 specs/src/specs/networking.md delete mode 100644 specs/src/specs/parameters.md delete mode 100644 specs/src/specs/parameters_v1.md delete mode 100644 specs/src/specs/parameters_v2.md delete mode 100644 specs/src/specs/public_key_cryptography.md delete mode 100644 specs/src/specs/resource_pricing.md delete mode 100644 specs/src/specs/shares.md delete mode 100644 specs/src/specs/state_machine_modules.md delete mode 100644 specs/src/specs/state_machine_modules_v1.md delete mode 100644 specs/src/specs/state_machine_modules_v2.md diff --git a/specs/src/specs/ante_handler.md b/specs/src/specs/ante_handler.md deleted file mode 100644 index 12710bdb85..0000000000 --- a/specs/src/specs/ante_handler.md +++ /dev/null @@ -1 +0,0 @@ -# AnteHandler diff --git a/specs/src/specs/ante_handler_v1.md b/specs/src/specs/ante_handler_v1.md deleted file mode 100644 index a5c3cb3fdf..0000000000 --- a/specs/src/specs/ante_handler_v1.md +++ /dev/null @@ -1 +0,0 @@ -# AnteHandler v1 diff --git a/specs/src/specs/ante_handler_v2.md b/specs/src/specs/ante_handler_v2.md deleted file mode 100644 index 9ebb88a269..0000000000 --- a/specs/src/specs/ante_handler_v2.md +++ /dev/null @@ -1 +0,0 @@ -# AnteHandler v2 diff --git a/specs/src/specs/block_proposer.md b/specs/src/specs/block_proposer.md deleted file mode 100644 index a29ade0863..0000000000 --- a/specs/src/specs/block_proposer.md +++ /dev/null @@ -1 +0,0 @@ -# Block Proposer diff --git a/specs/src/specs/block_validity_rules.md b/specs/src/specs/block_validity_rules.md deleted file mode 100644 index 204d783573..0000000000 --- a/specs/src/specs/block_validity_rules.md +++ /dev/null @@ -1 +0,0 @@ -# Block Validity Rules diff --git a/specs/src/specs/cat_pool.md b/specs/src/specs/cat_pool.md deleted file mode 100644 index eb6559023a..0000000000 --- a/specs/src/specs/cat_pool.md +++ /dev/null @@ -1 +0,0 @@ -# CAT Pool diff --git a/specs/src/specs/consensus.md b/specs/src/specs/consensus.md deleted file mode 100644 index 5cfc2c950c..0000000000 --- a/specs/src/specs/consensus.md +++ /dev/null @@ -1 +0,0 @@ -# Consensus diff --git a/specs/src/specs/data_square_layout.md b/specs/src/specs/data_square_layout.md deleted file mode 100644 index 4be6fe18f6..0000000000 --- a/specs/src/specs/data_square_layout.md +++ /dev/null @@ -1 +0,0 @@ -# Data Square Layout diff --git a/specs/src/specs/data_structures.md b/specs/src/specs/data_structures.md deleted file mode 100644 index c73ca16086..0000000000 --- a/specs/src/specs/data_structures.md +++ /dev/null @@ -1 +0,0 @@ -# Data Structures diff --git a/specs/src/specs/fraud_proofs.md b/specs/src/specs/fraud_proofs.md deleted file mode 100644 index 8f33083bce..0000000000 --- a/specs/src/specs/fraud_proofs.md +++ /dev/null @@ -1 +0,0 @@ -# Fraud Proofs diff --git a/specs/src/specs/multisig.md b/specs/src/specs/multisig.md deleted file mode 100644 index 206d574852..0000000000 --- a/specs/src/specs/multisig.md +++ /dev/null @@ -1 +0,0 @@ -# Multisig diff --git a/specs/src/specs/namespace.md b/specs/src/specs/namespace.md deleted file mode 100644 index aae590a94d..0000000000 --- a/specs/src/specs/namespace.md +++ /dev/null @@ -1 +0,0 @@ -# Namespace diff --git a/specs/src/specs/networking.md b/specs/src/specs/networking.md deleted file mode 100644 index 048579a4c1..0000000000 --- a/specs/src/specs/networking.md +++ /dev/null @@ -1 +0,0 @@ -# Networking diff --git a/specs/src/specs/parameters.md b/specs/src/specs/parameters.md deleted file mode 100644 index 04f9547d32..0000000000 --- a/specs/src/specs/parameters.md +++ /dev/null @@ -1 +0,0 @@ -# Parameters diff --git a/specs/src/specs/parameters_v1.md b/specs/src/specs/parameters_v1.md deleted file mode 100644 index eae5f5081a..0000000000 --- a/specs/src/specs/parameters_v1.md +++ /dev/null @@ -1 +0,0 @@ -# Parameters v1 diff --git a/specs/src/specs/parameters_v2.md b/specs/src/specs/parameters_v2.md deleted file mode 100644 index 0aa1503bd2..0000000000 --- a/specs/src/specs/parameters_v2.md +++ /dev/null @@ -1 +0,0 @@ -# Parameters v2 diff --git a/specs/src/specs/public_key_cryptography.md b/specs/src/specs/public_key_cryptography.md deleted file mode 100644 index 5234fcb002..0000000000 --- a/specs/src/specs/public_key_cryptography.md +++ /dev/null @@ -1 +0,0 @@ -# Public-Key Cryptography diff --git a/specs/src/specs/resource_pricing.md b/specs/src/specs/resource_pricing.md deleted file mode 100644 index 7b2572cdb6..0000000000 --- a/specs/src/specs/resource_pricing.md +++ /dev/null @@ -1 +0,0 @@ -# Resource Pricing diff --git a/specs/src/specs/shares.md b/specs/src/specs/shares.md deleted file mode 100644 index 806e046c43..0000000000 --- a/specs/src/specs/shares.md +++ /dev/null @@ -1 +0,0 @@ -# Shares diff --git a/specs/src/specs/state_machine_modules.md b/specs/src/specs/state_machine_modules.md deleted file mode 100644 index 58c122c799..0000000000 --- a/specs/src/specs/state_machine_modules.md +++ /dev/null @@ -1 +0,0 @@ -# State Machine Modules diff --git a/specs/src/specs/state_machine_modules_v1.md b/specs/src/specs/state_machine_modules_v1.md deleted file mode 100644 index ec8a32dd68..0000000000 --- a/specs/src/specs/state_machine_modules_v1.md +++ /dev/null @@ -1 +0,0 @@ -# State Machine Modules v1 diff --git a/specs/src/specs/state_machine_modules_v2.md b/specs/src/specs/state_machine_modules_v2.md deleted file mode 100644 index ce9a0bb852..0000000000 --- a/specs/src/specs/state_machine_modules_v2.md +++ /dev/null @@ -1 +0,0 @@ -# State Machine Modules v2 From 3aeb3c5a25cd8cce7560721aa4eebbf5d029b878 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Thu, 12 Sep 2024 14:53:05 +0200 Subject: [PATCH 26/85] chore(scripts): mainnet v2 upgrade height (#3868) Modify the mainnet.sh script to provide the `--v2-upgrade-height` and use state sync. --- scripts/mainnet.sh | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/scripts/mainnet.sh b/scripts/mainnet.sh index 317fe5dbdc..aa92e13326 100755 --- a/scripts/mainnet.sh +++ b/scripts/mainnet.sh @@ -1,5 +1,8 @@ #!/bin/sh +# This script starts a consensus node on Mainnet Beta and state syncs to the tip +# of the chain. + # Stop script execution if an error is encountered set -o errexit # Stop script execution if an undefined variable is used @@ -10,6 +13,7 @@ NODE_NAME="node-name" SEEDS="e6116822e1a5e283d8a85d3ec38f4d232274eaf3@consensus-full-seed-1.celestia-bootstrap.net:26656,cf7ac8b19ff56a9d47c75551bd4864883d1e24b5@consensus-full-seed-2.celestia-bootstrap.net:26656" CELESTIA_APP_HOME="${HOME}/.celestia-app" CELESTIA_APP_VERSION=$(celestia-appd version 2>&1) +RPC="https://celestia-rpc.polkachu.com:443" echo "celestia-app home: ${CELESTIA_APP_HOME}" echo "celestia-app version: ${CELESTIA_APP_VERSION}" @@ -35,10 +39,20 @@ celestia-appd init ${NODE_NAME} --chain-id ${CHAIN_ID} > /dev/null 2>&1 # Hide o echo "Settings seeds in config.toml..." sed -i.bak -e "s/^seeds *=.*/seeds = \"$SEEDS\"/" $CELESTIA_APP_HOME/config/config.toml +LATEST_HEIGHT=$(curl -s $RPC/block | jq -r .result.block.header.height); +BLOCK_HEIGHT=$((LATEST_HEIGHT - 2000)); \ +TRUST_HASH=$(curl -s "$RPC/block?height=$BLOCK_HEIGHT" | jq -r .result.block_id.hash) + +echo "Block height: $BLOCK_HEIGHT" +echo "Trust hash: $TRUST_HASH" +echo "Enabling state sync in config.toml..." +sed -i.bak -E "s|^(enable[[:space:]]+=[[:space:]]+).*$|\1true| ; \ +s|^(rpc_servers[[:space:]]+=[[:space:]]+).*$|\1\"$RPC,$RPC\"| ; \ +s|^(trust_height[[:space:]]+=[[:space:]]+).*$|\1$BLOCK_HEIGHT| ; \ +s|^(trust_hash[[:space:]]+=[[:space:]]+).*$|\1\"$TRUST_HASH\"|" $HOME/.celestia-app/config/config.toml + echo "Downloading genesis file..." celestia-appd download-genesis ${CHAIN_ID} > /dev/null 2>&1 # Hide output to reduce terminal noise -echo "Starting celestia-appd in the background and piping logs to mainnet.log" -nohup celestia-appd start > "${HOME}/mainnet.log" 2>&1 & - -echo "You can check the node's status via: celestia-appd status" +echo "Starting celestia-appd..." +celestia-appd start --v2-upgrade-height 2371495 From 07411e2e8eda7f58defb1dca139f936e35b50f9d Mon Sep 17 00:00:00 2001 From: Rootul P Date: Thu, 12 Sep 2024 23:48:48 +0200 Subject: [PATCH 27/85] docs(specs): add PFM param (#3864) Closes https://github.com/celestiaorg/celestia-app/issues/3862 --- specs/src/parameters_v2.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/src/parameters_v2.md b/specs/src/parameters_v2.md index 7555dd8761..9bd3e63f25 100644 --- a/specs/src/parameters_v2.md +++ b/specs/src/parameters_v2.md @@ -54,6 +54,7 @@ hardcoded in the application or they are blocked by the `x/paramfilter` module. | mint.DisinflationRate | 0.10 (10%) | The rate at which the inflation rate decreases each year. | False | | mint.InitialInflationRate | 0.08 (8%) | The inflation rate the network starts at. | False | | mint.TargetInflationRate | 0.015 (1.5%) | The inflation rate that the network aims to stabilize at. | False | +| packetfowardmiddleware.FeePercentage | 0 | % of the forwarded packet amount which will be subtracted and distributed to the community pool. | True | | slashing.DowntimeJailDuration | 1 min | Duration of time a validator must stay jailed. | True | | slashing.MinSignedPerWindow | 0.75 (75%) | The percentage of SignedBlocksWindow that must be signed not to get jailed. | True | | slashing.SignedBlocksWindow | 5000 | The range of blocks used to count for downtime. | True | From 628fb99e87aebf26f75e823895b6af3876f4a905 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Mon, 16 Sep 2024 19:00:50 +0200 Subject: [PATCH 28/85] fix: block sync on mocha on main (#3846) Closes https://github.com/celestiaorg/celestia-app/issues/3840 Forward port of https://github.com/celestiaorg/celestia-app/pull/3843 ## Testing I can state sync and block sync on Arabica and Mocha ```shell ./scripts/arabica.sh ./scripts/arabica-block-sync.sh ./scripts/mocha.sh ./scripts/mocha-block-sync.sh ``` --- app/app.go | 35 ++++++++++++--------- app/app_test.go | 59 +++++++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 +-- scripts/arabica-block-sync.sh | 46 +++++++++++++++++++++++++++ scripts/arabica.sh | 2 +- scripts/mocha-block-sync.sh | 46 +++++++++++++++++++++++++++ scripts/mocha.sh | 2 +- 8 files changed, 177 insertions(+), 19 deletions(-) create mode 100755 scripts/arabica-block-sync.sh create mode 100755 scripts/mocha-block-sync.sh diff --git a/app/app.go b/app/app.go index 71c519305e..9f8bdf4408 100644 --- a/app/app.go +++ b/app/app.go @@ -541,17 +541,8 @@ func (app *App) Info(req abci.RequestInfo) abci.ResponseInfo { // // Side-effect: calls baseapp.Init() func (app *App) InitChain(req abci.RequestInitChain) (res abci.ResponseInitChain) { - // genesis must always contain the consensus params. The validator set however is derived from the - // initial genesis state. The genesis must always contain a non zero app version which is the initial - // version that the chain starts on - if req.ConsensusParams == nil || req.ConsensusParams.Version == nil { - panic("no consensus params set") - } - if req.ConsensusParams.Version.AppVersion == 0 { - panic("app version 0 is not accepted. Please set an app version in the genesis") - } + req = setDefaultAppVersion(req) appVersion := req.ConsensusParams.Version.AppVersion - // mount the stores for the provided app version if it has not already been mounted if app.AppVersion() == 0 && !app.IsSealed() { app.mountKeysAndInit(appVersion) @@ -567,10 +558,26 @@ func (app *App) InitChain(req abci.RequestInitChain) (res abci.ResponseInitChain return res } +// setDefaultAppVersion sets the default app version in the consensus params if +// it was 0. This is needed because chains (e.x. mocha-4) did not explicitly set +// an app version in genesis.json. +func setDefaultAppVersion(req abci.RequestInitChain) abci.RequestInitChain { + if req.ConsensusParams == nil { + panic("no consensus params set") + } + if req.ConsensusParams.Version == nil { + panic("no version set in consensus params") + } + if req.ConsensusParams.Version.AppVersion == 0 { + req.ConsensusParams.Version.AppVersion = v1 + } + return req +} + // mountKeysAndInit mounts the keys for the provided app version and then // invokes baseapp.Init(). func (app *App) mountKeysAndInit(appVersion uint64) { - app.BaseApp.Logger().Debug(fmt.Sprintf("mounting KV stores for app version %v", appVersion)) + app.BaseApp.Logger().Info(fmt.Sprintf("mounting KV stores for app version %v", appVersion)) app.MountKVStores(app.versionedKeys(appVersion)) // Invoke load latest version for its side-effect of invoking baseapp.Init() @@ -585,9 +592,9 @@ func (app *App) InitChainer(ctx sdk.Context, req abci.RequestInitChain) abci.Res if err := tmjson.Unmarshal(req.AppStateBytes, &genesisState); err != nil { panic(err) } - - app.UpgradeKeeper.SetModuleVersionMap(ctx, app.manager.GetVersionMap(req.ConsensusParams.Version.AppVersion)) - return app.manager.InitGenesis(ctx, app.appCodec, genesisState, req.ConsensusParams.Version.AppVersion) + appVersion := req.ConsensusParams.Version.AppVersion + app.UpgradeKeeper.SetModuleVersionMap(ctx, app.manager.GetVersionMap(appVersion)) + return app.manager.InitGenesis(ctx, app.appCodec, genesisState, appVersion) } // LoadHeight loads a particular height diff --git a/app/app_test.go b/app/app_test.go index 5e99ec522c..16bca43ec7 100644 --- a/app/app_test.go +++ b/app/app_test.go @@ -1,10 +1,13 @@ package app_test import ( + "encoding/json" "testing" "github.com/celestiaorg/celestia-app/v3/app" "github.com/celestiaorg/celestia-app/v3/app/encoding" + "github.com/celestiaorg/celestia-app/v3/test/util" + "github.com/celestiaorg/celestia-app/v3/test/util/testnode" "github.com/celestiaorg/celestia-app/v3/x/minfee" "github.com/cosmos/cosmos-sdk/baseapp" "github.com/cosmos/cosmos-sdk/snapshots" @@ -13,6 +16,7 @@ import ( "github.com/stretchr/testify/require" abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/libs/log" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" tmdb "github.com/tendermint/tm-db" ) @@ -52,6 +56,61 @@ func TestNew(t *testing.T) { }) } +func TestInitChain(t *testing.T) { + logger := log.NewNopLogger() + db := tmdb.NewMemDB() + traceStore := &NoopWriter{} + invCheckPeriod := uint(1) + encodingConfig := encoding.MakeConfig(app.ModuleEncodingRegisters...) + upgradeHeight := int64(0) + appOptions := NoopAppOptions{} + testApp := app.New(logger, db, traceStore, invCheckPeriod, encodingConfig, upgradeHeight, appOptions) + genesisState, _, _ := util.GenesisStateWithSingleValidator(testApp, "account") + appStateBytes, err := json.MarshalIndent(genesisState, "", " ") + require.NoError(t, err) + genesis := testnode.DefaultConfig().Genesis + + type testCase struct { + name string + request abci.RequestInitChain + wantPanic bool + } + testCases := []testCase{ + { + name: "should panic if consensus params not set", + request: abci.RequestInitChain{}, + wantPanic: true, + }, + { + name: "should not panic on a genesis that does not contain an app version", + request: abci.RequestInitChain{ + Time: genesis.GenesisTime, + ChainId: genesis.ChainID, + ConsensusParams: &abci.ConsensusParams{ + Block: &abci.BlockParams{}, + Evidence: &genesis.ConsensusParams.Evidence, + Validator: &genesis.ConsensusParams.Validator, + Version: &tmproto.VersionParams{}, // explicitly set to empty to remove app version., + }, + AppStateBytes: appStateBytes, + InitialHeight: 0, + }, + wantPanic: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + application := app.New(logger, db, traceStore, invCheckPeriod, encodingConfig, upgradeHeight, appOptions) + if tc.wantPanic { + assert.Panics(t, func() { application.InitChain(tc.request) }) + } else { + assert.NotPanics(t, func() { application.InitChain(tc.request) }) + } + }) + } +} + func TestOfferSnapshot(t *testing.T) { logger := log.NewNopLogger() db := tmdb.NewMemDB() diff --git a/go.mod b/go.mod index 1de8165f96..74e0c3422d 100644 --- a/go.mod +++ b/go.mod @@ -252,5 +252,5 @@ replace ( github.com/cosmos/ledger-cosmos-go => github.com/cosmos/ledger-cosmos-go v0.12.4 github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 - github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.40.0-tm-v0.34.29 + github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.41.0-tm-v0.34.29 ) diff --git a/go.sum b/go.sum index b8775d02cd..cf7c5943f8 100644 --- a/go.sum +++ b/go.sum @@ -318,8 +318,8 @@ github.com/celestiaorg/bittwister v0.0.0-20231213180407-65cdbaf5b8c7 h1:nxplQi8w github.com/celestiaorg/bittwister v0.0.0-20231213180407-65cdbaf5b8c7/go.mod h1:1EF5MfOxVf0WC51Gb7pJ6bcZxnXKNAf9pqWtjgPBAYc= github.com/celestiaorg/blobstream-contracts/v3 v3.1.0 h1:h1Y4V3EMQ2mFmNtWt2sIhZIuyASInj1a9ExI8xOsTOw= github.com/celestiaorg/blobstream-contracts/v3 v3.1.0/go.mod h1:x4DKyfKOSv1ZJM9NwV+Pw01kH2CD7N5zTFclXIVJ6GQ= -github.com/celestiaorg/celestia-core v1.40.0-tm-v0.34.29 h1:J79TAjizxwIvm7/k+WI3PPH1aFj4AjOSjajoq5UzAwI= -github.com/celestiaorg/celestia-core v1.40.0-tm-v0.34.29/go.mod h1:5jJ5magtH7gQOwSYfS/m5fliIS7irKunLV7kLNaD8o0= +github.com/celestiaorg/celestia-core v1.41.0-tm-v0.34.29 h1:hRdTxe+Dz6kiqifRZCC9qYQiTJME7CzAZodrTHlhhnk= +github.com/celestiaorg/celestia-core v1.41.0-tm-v0.34.29/go.mod h1:H6vjzdoqTt4qmbf11z1Lnc9YLUp/B8ITEQLhU92ghqQ= github.com/celestiaorg/cosmos-sdk v1.24.1-sdk-v0.46.16 h1:SeQ7Y/CyOcUMKo7mQiexaj/pZ/xIgyuZFIwYZwpSkWE= github.com/celestiaorg/cosmos-sdk v1.24.1-sdk-v0.46.16/go.mod h1:Bpl1LSWiDpQumgOhhMTZBMopqa0j7fRasIhvTZB44P0= github.com/celestiaorg/go-square/v2 v2.0.0-rc2 h1:4D+ASgZGYVCsffc2uhPagACrvNiLZu9/CqNYvnlHCgg= diff --git a/scripts/arabica-block-sync.sh b/scripts/arabica-block-sync.sh new file mode 100755 index 0000000000..1b911ccdfb --- /dev/null +++ b/scripts/arabica-block-sync.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +# This script starts a consensus node on Arabica and block syncs from genesis to +# the tip of the chain. + +# Stop script execution if an error is encountered +set -o errexit +# Stop script execution if an undefined variable is used +set -o nounset + +CHAIN_ID="arabica-11" +NODE_NAME="node-name" +SEEDS="827583022cc6ce65cf762115642258f937c954cd@validator-1.celestia-arabica-11.com:26656,74e42b39f512f844492ff09e30af23d54579b7bc@validator-2.celestia-arabica-11.com:26656,00d577159b2eb1f524ef9c37cb389c020a2c38d2@validator-3.celestia-arabica-11.com:26656,b2871b6dc2e18916d07264af0e87c456c2bba04f@validator-4.celestia-arabica-11.com:26656" + +CELESTIA_APP_HOME="${HOME}/.celestia-app" +CELESTIA_APP_VERSION=$(celestia-appd version 2>&1) + +echo "celestia-app home: ${CELESTIA_APP_HOME}" +echo "celestia-app version: ${CELESTIA_APP_VERSION}" +echo "" + +# Ask the user for confirmation before deleting the existing celestia-app home +# directory. +read -p "Are you sure you want to delete: $CELESTIA_APP_HOME? [y/n] " response + +# Check the user's response +if [ "$response" != "y" ]; then + # Exit if the user did not respond with "y" + echo "You must delete $CELESTIA_APP_HOME to continue." + exit 1 +fi + +echo "Deleting $CELESTIA_APP_HOME..." +rm -r "$CELESTIA_APP_HOME" + +echo "Initializing config files..." +celestia-appd init ${NODE_NAME} --chain-id ${CHAIN_ID} > /dev/null 2>&1 # Hide output to reduce terminal noise + +echo "Settings seeds in config.toml..." +sed -i.bak -e "s/^seeds *=.*/seeds = \"$SEEDS\"/" $CELESTIA_APP_HOME/config/config.toml + +echo "Downloading genesis file..." +celestia-appd download-genesis ${CHAIN_ID} + +echo "Starting celestia-appd..." +celestia-appd start --v2-upgrade-height 1751707 --force-no-bbr diff --git a/scripts/arabica.sh b/scripts/arabica.sh index fe88e26c89..d09b9d8441 100755 --- a/scripts/arabica.sh +++ b/scripts/arabica.sh @@ -57,4 +57,4 @@ echo "Downloading genesis file..." celestia-appd download-genesis ${CHAIN_ID} echo "Starting celestia-appd..." -celestia-appd start --v2-upgrade-height 1751707 +celestia-appd start --v2-upgrade-height 1751707 --force-no-bbr diff --git a/scripts/mocha-block-sync.sh b/scripts/mocha-block-sync.sh new file mode 100755 index 0000000000..3f8795dd6e --- /dev/null +++ b/scripts/mocha-block-sync.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +# This script starts a consensus node on Mocha and block syncs from genesis to +# the tip of the chain. This is expected to take a few weeks. + +# Stop script execution if an error is encountered +set -o errexit +# Stop script execution if an undefined variable is used +set -o nounset + +CHAIN_ID="mocha-4" +NODE_NAME="node-name" +SEEDS="ee9f90974f85c59d3861fc7f7edb10894f6ac3c8@seed-mocha.pops.one:26656,258f523c96efde50d5fe0a9faeea8a3e83be22ca@seed.mocha-4.celestia.aviaone.com:20279,5d0bf034d6e6a8b5ee31a2f42f753f1107b3a00e@celestia-testnet-seed.itrocket.net:11656,7da0fb48d6ef0823bc9770c0c8068dd7c89ed4ee@celest-test-seed.theamsolutions.info:443" + +CELESTIA_APP_HOME="${HOME}/.celestia-app" +CELESTIA_APP_VERSION=$(celestia-appd version 2>&1) + +echo "celestia-app home: ${CELESTIA_APP_HOME}" +echo "celestia-app version: ${CELESTIA_APP_VERSION}" +echo "" + +# Ask the user for confirmation before deleting the existing celestia-app home +# directory. +read -p "Are you sure you want to delete: $CELESTIA_APP_HOME? [y/n] " response + +# Check the user's response +if [ "$response" != "y" ]; then + # Exit if the user did not respond with "y" + echo "You must delete $CELESTIA_APP_HOME to continue." + exit 1 +fi + +echo "Deleting $CELESTIA_APP_HOME..." +rm -r "$CELESTIA_APP_HOME" + +echo "Initializing config files..." +celestia-appd init ${NODE_NAME} --chain-id ${CHAIN_ID} > /dev/null 2>&1 # Hide output to reduce terminal noise + +echo "Settings seeds in config.toml..." +sed -i.bak -e "s/^seeds *=.*/seeds = \"$SEEDS\"/" $CELESTIA_APP_HOME/config/config.toml + +echo "Downloading genesis file..." +celestia-appd download-genesis ${CHAIN_ID} > /dev/null 2>&1 # Hide output to reduce terminal noise + +echo "Starting celestia-appd..." +celestia-appd start --v2-upgrade-height 2585031 --force-no-bbr diff --git a/scripts/mocha.sh b/scripts/mocha.sh index 5447eb9e3b..56c0e89480 100755 --- a/scripts/mocha.sh +++ b/scripts/mocha.sh @@ -57,4 +57,4 @@ echo "Downloading genesis file..." celestia-appd download-genesis ${CHAIN_ID} > /dev/null 2>&1 # Hide output to reduce terminal noise echo "Starting celestia-appd..." -celestia-appd start --v2-upgrade-height 2585031 +celestia-appd start --v2-upgrade-height 2585031 --force-no-bbr From 07ebd7b4696f0e3137df5026afb937c1624742e2 Mon Sep 17 00:00:00 2001 From: Moji Date: Tue, 17 Sep 2024 16:34:02 +0200 Subject: [PATCH 29/85] refactor(e2e): migrate the tests to the new knuu (#3873) Closes #3780 Since this PR points to a branch on a fork, `GetLatestVersion` does not fetch the latest commit hash to the main; instead, it fetches the latest commit of the forked repo. So it is tested with the hard coded version `07411e2` which is, currently, the latest commit on the main. The `E2ESimple` test passes, did not test the benchmarks which also were not needed for this purpose. --------- Co-authored-by: CHAMI Rachid --- go.mod | 40 +++---- go.sum | 97 +++++++---------- test/e2e/benchmark/benchmark.go | 61 +++++------ test/e2e/benchmark/throughput.go | 32 +++--- test/e2e/main.go | 2 +- test/e2e/major_upgrade_v2.go | 34 +++--- test/e2e/minor_version_compatibility.go | 31 +++--- test/e2e/simple.go | 26 ++--- test/e2e/testnet/defaults.go | 10 +- test/e2e/testnet/node.go | 136 ++++++++++++++---------- test/e2e/testnet/setup.go | 5 +- test/e2e/testnet/testnet.go | 129 +++++++++++----------- test/e2e/testnet/txsimNode.go | 21 ++-- 13 files changed, 318 insertions(+), 306 deletions(-) diff --git a/go.mod b/go.mod index 74e0c3422d..12ea474f6a 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( cosmossdk.io/math v1.3.0 github.com/celestiaorg/blobstream-contracts/v3 v3.1.0 github.com/celestiaorg/go-square/v2 v2.0.0-rc2 - github.com/celestiaorg/knuu v0.14.0 + github.com/celestiaorg/knuu v0.15.2 github.com/celestiaorg/nmt v0.22.1 github.com/celestiaorg/rsmt2d v0.14.0 github.com/cosmos/cosmos-proto v1.0.0-beta.5 @@ -34,6 +34,7 @@ require ( google.golang.org/grpc v1.66.0 google.golang.org/protobuf v1.34.2 gopkg.in/yaml.v2 v2.4.0 + k8s.io/apimachinery v0.30.2 ) require ( @@ -83,16 +84,13 @@ require ( github.com/dgraph-io/badger/v2 v2.2007.4 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect - github.com/distribution/reference v0.5.0 // indirect - github.com/docker/docker v26.1.5+incompatible // indirect - github.com/docker/go-connections v0.4.1-0.20210727194412-58542c764a11 // indirect - github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/dvsekhvalnov/jose2go v1.6.0 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/ethereum/c-kzg-4844 v1.0.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/go-kit/kit v0.12.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect @@ -103,7 +101,7 @@ require ( github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/go-playground/validator/v10 v10.11.2 // indirect - github.com/goccy/go-json v0.10.2 // indirect + github.com/goccy/go-json v0.10.3 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect github.com/gogo/gateway v1.1.0 // indirect github.com/golang/glog v1.2.1 // indirect @@ -143,11 +141,10 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jmhodges/levigo v1.0.0 // indirect - github.com/joho/godotenv v1.5.1 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.8 // indirect - github.com/klauspost/cpuid/v2 v2.2.7 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/klauspost/reedsolomon v1.12.1 // indirect github.com/lib/pq v1.10.7 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect @@ -160,20 +157,18 @@ require ( github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0 // indirect github.com/minio/highwayhash v1.0.2 // indirect github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/minio-go/v7 v7.0.70 // indirect + github.com/minio/minio-go/v7 v7.0.74 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect - github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/spdystream v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mtibben/percent v0.2.1 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/onsi/ginkgo v1.16.5 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 // indirect github.com/pkg/errors v0.9.1 // indirect @@ -210,21 +205,19 @@ require ( go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect go.opentelemetry.io/otel v1.26.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.18.0 // indirect go.opentelemetry.io/otel/metric v1.26.0 // indirect go.opentelemetry.io/otel/sdk v1.26.0 // indirect go.opentelemetry.io/otel/trace v1.26.0 // indirect - go.opentelemetry.io/proto/otlp v1.1.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.24.0 // indirect - golang.org/x/net v0.26.0 // indirect + golang.org/x/crypto v0.26.0 // indirect + golang.org/x/net v0.28.0 // indirect golang.org/x/oauth2 v0.21.0 // indirect - golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.21.0 // indirect - golang.org/x/term v0.21.0 // indirect - golang.org/x/text v0.16.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.23.0 // indirect + golang.org/x/term v0.23.0 // indirect + golang.org/x/text v0.17.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/api v0.169.0 // indirect google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect @@ -232,9 +225,8 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/api v0.28.2 // indirect - k8s.io/apimachinery v0.28.2 // indirect - k8s.io/client-go v0.28.2 // indirect + k8s.io/api v0.30.2 // indirect + k8s.io/client-go v0.30.2 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect diff --git a/go.sum b/go.sum index cf7c5943f8..4f3bfca260 100644 --- a/go.sum +++ b/go.sum @@ -213,7 +213,6 @@ github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3 github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo= github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= @@ -324,8 +323,8 @@ github.com/celestiaorg/cosmos-sdk v1.24.1-sdk-v0.46.16 h1:SeQ7Y/CyOcUMKo7mQiexaj github.com/celestiaorg/cosmos-sdk v1.24.1-sdk-v0.46.16/go.mod h1:Bpl1LSWiDpQumgOhhMTZBMopqa0j7fRasIhvTZB44P0= github.com/celestiaorg/go-square/v2 v2.0.0-rc2 h1:4D+ASgZGYVCsffc2uhPagACrvNiLZu9/CqNYvnlHCgg= github.com/celestiaorg/go-square/v2 v2.0.0-rc2/go.mod h1:eeaU8f8jBpk3ZS/gaDZIlTesJR2F51QAmveNzWH6aEU= -github.com/celestiaorg/knuu v0.14.0 h1:96uaDHTzlTfhDLrAiygq9Ewow7UzOzGAbUvMwws1S4A= -github.com/celestiaorg/knuu v0.14.0/go.mod h1:5x/+tlLebBSfLmmSBm2ps6aLjnKLn5bOaZpUfI5FpsA= +github.com/celestiaorg/knuu v0.15.2 h1:l6MrFTfDfrNtbvOLSjD/YzRpaaBFIIvlWIYXRNNL1/s= +github.com/celestiaorg/knuu v0.15.2/go.mod h1:hTKu4uGH/Dx7eaYgT26coKx/vbALEQdLdYOwNxkAIG0= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4/go.mod h1:fzuHnhzj1pUygGz+1ZkB3uQbEUL4htqCGJ4Qs2LwMZA= github.com/celestiaorg/nmt v0.22.1 h1:t7fqoP5MJ8mBns5DB2XjfcPxQpS3CKMkY+v+BEkDxYc= @@ -404,8 +403,6 @@ github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJ github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= -github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= -github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -478,14 +475,10 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUn github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= -github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v26.1.5+incompatible h1:NEAxTwEjxV6VbBMBoGG3zPqbiJosIApZjxlbrG9q3/g= -github.com/docker/docker v26.1.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.1-0.20210727194412-58542c764a11 h1:IPrmumsT9t5BS7XcPhgsCTlkWbYg80SEXUzDpReaU6Y= github.com/docker/go-connections v0.4.1-0.20210727194412-58542c764a11/go.mod h1:a6bNUGTbQBsY6VRHTr4h/rkOXjl244DyRD0tx3fgq4Q= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -567,6 +560,8 @@ github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= @@ -620,8 +615,8 @@ github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= -github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= -github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -793,8 +788,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 h1:Wqo399gCIufwto+VfwCSvsnfGpF/w5E9CNxSwbpD6No= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0/go.mod h1:qmOFXW2epJhM0qSnUUYpldc7gVz2KMQwJ/QYCDIa7XU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/gtank/merlin v0.1.1-0.20191105220539-8318aed1a79f/go.mod h1:T86dnYJhcGOh5BjZFCJWTDeTK7XW8uE+E21Cy/bIQ+s= @@ -894,8 +887,6 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= -github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= -github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= @@ -925,12 +916,12 @@ github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= -github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= -github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= -github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= +github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/reedsolomon v1.12.1 h1:NhWgum1efX1x58daOBGCFWcxtEhOhXKKl1HAPQUp03Q= @@ -1008,8 +999,8 @@ github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.70 h1:1u9NtMgfK1U42kUxcsl5v0yj6TEOPR497OAQxpJnn2g= -github.com/minio/minio-go/v7 v7.0.70/go.mod h1:4yBA8v80xGA30cfM3fz0DKYMXunWl/AV/6tWEs9ryzo= +github.com/minio/minio-go/v7 v7.0.74 h1:fTo/XlPBTSpo3BAMshlwKL5RspXRv9us5UeHEGYCFe0= +github.com/minio/minio-go/v7 v7.0.74/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -1030,12 +1021,8 @@ github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8oh github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= -github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= -github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= -github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -1044,8 +1031,6 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= @@ -1055,6 +1040,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/grpc-proxy v0.0.0-20181017164139-0f1106ef9c76/go.mod h1:x5OoJHDHqxHS801UIuhqGl6QdSAEJvtausosHSdazIo= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= @@ -1087,8 +1074,8 @@ github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= -github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/onsi/gomega v1.31.0 h1:54UJxxj6cPInHS3a35wm6BK/F9nHYueZ1NVujHDrnXE= +github.com/onsi/gomega v1.31.0/go.mod h1:DW9aCi7U6Yi40wNVAvT6kzFnEVEI5n3DloYBiKiT6zk= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= @@ -1197,8 +1184,8 @@ github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= +github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -1216,7 +1203,6 @@ github.com/shirou/gopsutil v3.21.6+incompatible h1:mmZtAlWSd8U2HeRTjswbnDLPxqsEo github.com/shirou/gopsutil v3.21.6+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= @@ -1375,10 +1361,6 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1: go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 h1:t6wl9SPayj+c7lEIFgm4ooDBZVb01IhLB4InpomhRw8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0/go.mod h1:iSDOcsnSA5INXzZtwaBPrKp/lWu/V14Dd+llD0oI2EA= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0 h1:1wp/gyxsuYtuE/JFxsQRtcCDtMrO2qMvlfXALU5wkzI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.26.0/go.mod h1:gbTHmghkGgqxMomVQQMur1Nba4M0MQ8AYThXDUjsJ38= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.18.0 h1:hSWWvDjXHVLq9DkmB+77fl8v7+t+yYiS+eNkiplDK54= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.18.0/go.mod h1:zG7KQql1WjZCaUJd+L/ReSYx4bjbYJxg5ws9ws+mYes= go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= @@ -1391,8 +1373,6 @@ go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+ go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.opentelemetry.io/proto/otlp v1.1.0 h1:2Di21piLrCqJ3U3eXGCTPHE9R8Nh+0uglSnOyxikMeI= -go.opentelemetry.io/proto/otlp v1.1.0/go.mod h1:GpBHCBWiqvVLDqmHZsoMM3C5ySeKTC7ej/RNTae6MdY= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -1432,8 +1412,8 @@ golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1541,8 +1521,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1584,8 +1564,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1698,14 +1678,14 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= +golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= +golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1716,8 +1696,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1790,8 +1770,8 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= +golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2082,10 +2062,7 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= -gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -2095,12 +2072,12 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= -k8s.io/api v0.28.2 h1:9mpl5mOb6vXZvqbQmankOfPIGiudghwCoLl1EYfUZbw= -k8s.io/api v0.28.2/go.mod h1:RVnJBsjU8tcMq7C3iaRSGMeaKt2TWEUXcpIt/90fjEg= -k8s.io/apimachinery v0.28.2 h1:KCOJLrc6gu+wV1BYgwik4AF4vXOlVJPdiqn0yAWWwXQ= -k8s.io/apimachinery v0.28.2/go.mod h1:RdzF87y/ngqk9H4z3EL2Rppv5jj95vGS/HaFXrLDApU= -k8s.io/client-go v0.28.2 h1:DNoYI1vGq0slMBN/SWKMZMw0Rq+0EQW6/AK4v9+3VeY= -k8s.io/client-go v0.28.2/go.mod h1:sMkApowspLuc7omj1FOSUxSoqjr+d5Q0Yc0LOFnYFJY= +k8s.io/api v0.30.2 h1:+ZhRj+28QT4UOH+BKznu4CBgPWgkXO7XAvMcMl0qKvI= +k8s.io/api v0.30.2/go.mod h1:ULg5g9JvOev2dG0u2hig4Z7tQ2hHIuS+m8MNZ+X6EmI= +k8s.io/apimachinery v0.30.2 h1:fEMcnBj6qkzzPGSVsAZtQThU62SmQ4ZymlXRC5yFSCg= +k8s.io/apimachinery v0.30.2/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/client-go v0.30.2 h1:sBIVJdojUNPDU/jObC+18tXWcTJVcwyqS9diGdWHk50= +k8s.io/client-go v0.30.2/go.mod h1:JglKSWULm9xlJLx4KCkfLLQ7XwtlbflV6uFFSHTMgVs= k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= diff --git a/test/e2e/benchmark/benchmark.go b/test/e2e/benchmark/benchmark.go index e34d322bb8..81b2748cb1 100644 --- a/test/e2e/benchmark/benchmark.go +++ b/test/e2e/benchmark/benchmark.go @@ -7,10 +7,11 @@ import ( "log" "time" + "github.com/tendermint/tendermint/pkg/trace" + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" "github.com/celestiaorg/celestia-app/v3/test/e2e/testnet" "github.com/celestiaorg/celestia-app/v3/test/util/testnode" - "github.com/tendermint/tendermint/pkg/trace" ) type BenchmarkTest struct { @@ -20,7 +21,7 @@ type BenchmarkTest struct { func NewBenchmarkTest(name string, manifest *Manifest) (*BenchmarkTest, error) { // create a new testnet - testNet, err := testnet.New(name, seed, + testNet, err := testnet.New(context.Background(), name, seed, testnet.GetGrafanaInfoFromEnvVar(), manifest.ChainID, manifest.GetGenesisModifiers()...) if err != nil { @@ -35,28 +36,27 @@ func NewBenchmarkTest(name string, manifest *Manifest) (*BenchmarkTest, error) { // There will be manifest.Validators validators and manifest.TxClients tx clients. // Each tx client connects to one validator. If TxClients are fewer than Validators, some validators will not have a tx client. func (b *BenchmarkTest) SetupNodes() error { + ctx := context.Background() testnet.NoError("failed to create genesis nodes", - b.CreateGenesisNodes(b.manifest.Validators, + b.CreateGenesisNodes(ctx, b.manifest.Validators, b.manifest.CelestiaAppVersion, b.manifest.SelfDelegation, b.manifest.UpgradeHeight, b.manifest.ValidatorResource, b.manifest.DisableBBR)) // enable latency if specified in the manifest if b.manifest.EnableLatency { for _, node := range b.Nodes() { - if err := node.Instance.EnableBitTwister(); err != nil { - return fmt.Errorf("failed to enable bit twister: %v", err) - } + node.EnableNetShaper() } } // obtain the GRPC endpoints of the validators - gRPCEndpoints, err := b.RemoteGRPCEndpoints() + gRPCEndpoints, err := b.RemoteGRPCEndpoints(ctx) testnet.NoError("failed to get validators GRPC endpoints", err) log.Println("validators GRPC endpoints", gRPCEndpoints) // create tx clients and point them to the validators log.Println("Creating tx clients") - err = b.CreateTxClients(b.manifest.TxClientVersion, + err = b.CreateTxClients(ctx, b.manifest.TxClientVersion, b.manifest.BlobSequences, b.manifest.BlobSizes, b.manifest.BlobsPerSeq, @@ -64,7 +64,7 @@ func (b *BenchmarkTest) SetupNodes() error { testnet.NoError("failed to create tx clients", err) log.Println("Setting up testnet") - testnet.NoError("failed to setup testnet", b.Setup( + testnet.NoError("failed to setup testnet", b.Setup(ctx, testnet.WithPerPeerBandwidth(b.manifest.PerPeerBandwidth), testnet.WithTimeoutPropose(b.manifest.TimeoutPropose), testnet.WithTimeoutCommit(b.manifest.TimeoutCommit), @@ -78,21 +78,18 @@ func (b *BenchmarkTest) SetupNodes() error { log.Println("reading trace push config") if pushConfig, err := trace.GetPushConfigFromEnv(); err == nil { log.Print("Setting up trace push config") + envVars := map[string]string{ + trace.PushBucketName: pushConfig.BucketName, + trace.PushRegion: pushConfig.Region, + trace.PushAccessKey: pushConfig.AccessKey, + trace.PushKey: pushConfig.SecretKey, + trace.PushDelay: fmt.Sprintf("%d", pushConfig.PushDelay), + } for _, node := range b.Nodes() { - if err = node.Instance.SetEnvironmentVariable(trace.PushBucketName, pushConfig.BucketName); err != nil { - return fmt.Errorf("failed to set TRACE_PUSH_BUCKET_NAME: %v", err) - } - if err = node.Instance.SetEnvironmentVariable(trace.PushRegion, pushConfig.Region); err != nil { - return fmt.Errorf("failed to set TRACE_PUSH_REGION: %v", err) - } - if err = node.Instance.SetEnvironmentVariable(trace.PushAccessKey, pushConfig.AccessKey); err != nil { - return fmt.Errorf("failed to set TRACE_PUSH_ACCESS_KEY: %v", err) - } - if err = node.Instance.SetEnvironmentVariable(trace.PushKey, pushConfig.SecretKey); err != nil { - return fmt.Errorf("failed to set TRACE_PUSH_SECRET_KEY: %v", err) - } - if err = node.Instance.SetEnvironmentVariable(trace.PushDelay, fmt.Sprintf("%d", pushConfig.PushDelay)); err != nil { - return fmt.Errorf("failed to set TRACE_PUSH_DELAY: %v", err) + for key, value := range envVars { + if err = node.Instance.Build().SetEnvironmentVariable(key, value); err != nil { + return fmt.Errorf("failed to set %s: %v", key, err) + } } } } @@ -101,20 +98,22 @@ func (b *BenchmarkTest) SetupNodes() error { } // Run runs the benchmark test for the specified duration in the manifest. -func (b *BenchmarkTest) Run() error { +func (b *BenchmarkTest) Run(ctx context.Context) error { log.Println("Starting benchmark testnet") log.Println("Starting nodes") - err := b.StartNodes() - if err != nil { + if err := b.StartNodes(ctx); err != nil { return fmt.Errorf("failed to start testnet: %v", err) } // add latency if specified in the manifest if b.manifest.EnableLatency { for _, node := range b.Nodes() { - if err = node.Instance.SetLatencyAndJitter(b.manifest.LatencyParams. - Latency, b.manifest.LatencyParams.Jitter); err != nil { + err := node.SetLatencyAndJitter( + b.manifest.LatencyParams.Latency, + b.manifest.LatencyParams.Jitter, + ) + if err != nil { return fmt.Errorf("failed to set latency and jitter: %v", err) } } @@ -122,15 +121,13 @@ func (b *BenchmarkTest) Run() error { // wait for the nodes to sync log.Println("Waiting for nodes to sync") - err = b.WaitToSync() - if err != nil { + if err := b.WaitToSync(ctx); err != nil { return err } // start tx clients log.Println("Starting tx clients") - err = b.StartTxClients() - if err != nil { + if err := b.StartTxClients(ctx); err != nil { return fmt.Errorf("failed to start tx clients: %v", err) } diff --git a/test/e2e/benchmark/throughput.go b/test/e2e/benchmark/throughput.go index b0392468ce..337d700525 100644 --- a/test/e2e/benchmark/throughput.go +++ b/test/e2e/benchmark/throughput.go @@ -1,11 +1,13 @@ package main import ( + "context" "log" "time" "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" "github.com/celestiaorg/celestia-app/v3/test/e2e/testnet" + "k8s.io/apimachinery/pkg/api/resource" ) const ( @@ -17,16 +19,16 @@ var bigBlockManifest = Manifest{ Validators: 2, TxClients: 2, ValidatorResource: testnet.Resources{ - MemoryRequest: "12Gi", - MemoryLimit: "12Gi", - CPU: "8", - Volume: "20Gi", + MemoryRequest: resource.MustParse("12Gi"), + MemoryLimit: resource.MustParse("12Gi"), + CPU: resource.MustParse("8"), + Volume: resource.MustParse("20Gi"), }, TxClientsResource: testnet.Resources{ - MemoryRequest: "1Gi", - MemoryLimit: "3Gi", - CPU: "2", - Volume: "1Gi", + MemoryRequest: resource.MustParse("1Gi"), + MemoryLimit: resource.MustParse("3Gi"), + CPU: resource.MustParse("2"), + Volume: resource.MustParse("1Gi"), }, SelfDelegation: 10000000, // @TODO Update the CelestiaAppVersion and TxClientVersion to the latest @@ -95,14 +97,17 @@ func TwoNodeSimple(logger *log.Logger) error { benchTest, err := NewBenchmarkTest(testName, &manifest) testnet.NoError("failed to create benchmark test", err) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + defer func() { log.Print("Cleaning up testnet") - benchTest.Cleanup() + benchTest.Cleanup(ctx) }() testnet.NoError("failed to setup nodes", benchTest.SetupNodes()) - testnet.NoError("failed to run the benchmark test", benchTest.Run()) + testnet.NoError("failed to run the benchmark test", benchTest.Run(ctx)) testnet.NoError("failed to check results", benchTest.CheckResults(1*testnet.MB)) @@ -116,13 +121,16 @@ func runBenchmarkTest(logger *log.Logger, testName string, manifest Manifest) er benchTest, err := NewBenchmarkTest(testName, &manifest) testnet.NoError("failed to create benchmark test", err) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + defer func() { log.Print("Cleaning up testnet") - benchTest.Cleanup() + benchTest.Cleanup(ctx) }() testnet.NoError("failed to setup nodes", benchTest.SetupNodes()) - testnet.NoError("failed to run the benchmark test", benchTest.Run()) + testnet.NoError("failed to run the benchmark test", benchTest.Run(ctx)) expectedBlockSize := int64(0.90 * float64(manifest.MaxBlockBytes)) testnet.NoError("failed to check results", benchTest.CheckResults(expectedBlockSize)) diff --git a/test/e2e/main.go b/test/e2e/main.go index c133e00a6d..0928779dbb 100644 --- a/test/e2e/main.go +++ b/test/e2e/main.go @@ -10,7 +10,7 @@ const ( seed = 42 ) -type TestFunc func(*log.Logger) error +type TestFunc func(logger *log.Logger) error type Test struct { Name string diff --git a/test/e2e/major_upgrade_v2.go b/test/e2e/major_upgrade_v2.go index 9797aa9c26..db21d65306 100644 --- a/test/e2e/major_upgrade_v2.go +++ b/test/e2e/major_upgrade_v2.go @@ -12,52 +12,50 @@ import ( v1 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v1" v2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" "github.com/celestiaorg/celestia-app/v3/test/e2e/testnet" - "github.com/celestiaorg/knuu/pkg/knuu" "github.com/tendermint/tendermint/rpc/client/http" ) func MajorUpgradeToV2(logger *log.Logger) error { - latestVersion, err := testnet.GetLatestVersion() - testnet.NoError("failed to get latest version", err) - - logger.Println("Running major upgrade to v2 test", "version", latestVersion) - numNodes := 4 upgradeHeight := int64(10) ctx, cancel := context.WithCancel(context.Background()) defer cancel() logger.Println("Creating testnet") - testNet, err := testnet.New("runMajorUpgradeToV2", seed, nil, "test") + testNet, err := testnet.New(ctx, "runMajorUpgradeToV2", seed, nil, "test") testnet.NoError("failed to create testnet", err) - defer testNet.Cleanup() + defer testNet.Cleanup(ctx) + + latestVersion, err := testnet.GetLatestVersion() + testnet.NoError("failed to get latest version", err) + + logger.Println("Running major upgrade to v2 test", "version", latestVersion) testNet.SetConsensusParams(app.DefaultInitialConsensusParams()) - preloader, err := knuu.NewPreloader() + preloader, err := testNet.NewPreloader() testnet.NoError("failed to create preloader", err) - defer func() { _ = preloader.EmptyImages() }() - testnet.NoError("failed to add image", preloader.AddImage(testnet.DockerImageName(latestVersion))) + defer func() { _ = preloader.EmptyImages(ctx) }() + testnet.NoError("failed to add image", preloader.AddImage(ctx, testnet.DockerImageName(latestVersion))) logger.Println("Creating genesis nodes") for i := 0; i < numNodes; i++ { - err := testNet.CreateGenesisNode(latestVersion, 10000000, - upgradeHeight, testnet.DefaultResources, true) + err := testNet.CreateGenesisNode(ctx, latestVersion, 10000000, upgradeHeight, testnet.DefaultResources, true) testnet.NoError("failed to create genesis node", err) } logger.Println("Creating txsim") - endpoints, err := testNet.RemoteGRPCEndpoints() + endpoints, err := testNet.RemoteGRPCEndpoints(ctx) testnet.NoError("failed to get remote gRPC endpoints", err) - err = testNet.CreateTxClient("txsim", testnet.TxsimVersion, 1, "100-2000", 100, testnet.DefaultResources, endpoints[0]) + err = testNet.CreateTxClient(ctx, "txsim", testnet.TxsimVersion, 1, "100-2000", 100, testnet.DefaultResources, endpoints[0]) testnet.NoError("failed to create tx client", err) logger.Println("Setting up testnet") - testnet.NoError("Failed to setup testnet", testNet.Setup()) + testnet.NoError("Failed to setup testnet", testNet.Setup(ctx)) logger.Println("Starting testnet") - testnet.NoError("Failed to start testnet", testNet.Start()) + testnet.NoError("Failed to start testnet", testNet.Start(ctx)) heightBefore := upgradeHeight - 1 for i := 0; i < numNodes; i++ { @@ -91,7 +89,7 @@ func MajorUpgradeToV2(logger *log.Logger) error { return fmt.Errorf("failed to get height: %w", err) } - if err := node.Upgrade(latestVersion); err != nil { + if err := node.Upgrade(ctx, latestVersion); err != nil { return fmt.Errorf("failed to restart node: %w", err) } diff --git a/test/e2e/minor_version_compatibility.go b/test/e2e/minor_version_compatibility.go index 9c09b57895..c13d93f23c 100644 --- a/test/e2e/minor_version_compatibility.go +++ b/test/e2e/minor_version_compatibility.go @@ -15,7 +15,6 @@ import ( v1 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v1" v2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" "github.com/celestiaorg/celestia-app/v3/test/e2e/testnet" - "github.com/celestiaorg/knuu/pkg/knuu" ) func MinorVersionCompatibility(logger *log.Logger) error { @@ -32,45 +31,45 @@ func MinorVersionCompatibility(logger *log.Logger) error { r := rand.New(rand.NewSource(seed)) logger.Println("Running minor version compatibility test", "versions", versions) - testNet, err := testnet.New("runMinorVersionCompatibility", seed, nil, "test") - testnet.NoError("failed to create testnet", err) - ctx, cancel := context.WithCancel(context.Background()) defer cancel() - defer testNet.Cleanup() + testNet, err := testnet.New(ctx, "runMinorVersionCompatibility", seed, nil, "test") + testnet.NoError("failed to create testnet", err) + + defer testNet.Cleanup(ctx) testNet.SetConsensusParams(app.DefaultInitialConsensusParams()) // preload all docker images - preloader, err := knuu.NewPreloader() + preloader, err := testNet.NewPreloader() testnet.NoError("failed to create preloader", err) - defer func() { _ = preloader.EmptyImages() }() + defer func() { _ = preloader.EmptyImages(ctx) }() for _, v := range versions { - testnet.NoError("failed to add image", preloader.AddImage(testnet.DockerImageName(v.String()))) + testnet.NoError("failed to add image", preloader.AddImage(ctx, testnet.DockerImageName(v.String()))) } for i := 0; i < numNodes; i++ { // each node begins with a random version within the same major version set v := versions.Random(r).String() logger.Println("Starting node", "node", i, "version", v) + testnet.NoError("failed to create genesis node", - testNet.CreateGenesisNode(v, 10000000, 0, - testnet.DefaultResources, false)) + testNet.CreateGenesisNode(ctx, v, 10000000, 0, testnet.DefaultResources, false)) } logger.Println("Creating txsim") - endpoints, err := testNet.RemoteGRPCEndpoints() + endpoints, err := testNet.RemoteGRPCEndpoints(ctx) testnet.NoError("failed to get remote gRPC endpoints", err) - err = testNet.CreateTxClient("txsim", testnet.TxsimVersion, 1, "100-2000", 100, testnet.DefaultResources, endpoints[0]) + err = testNet.CreateTxClient(ctx, "txsim", testnet.TxsimVersion, 1, "100-2000", 100, testnet.DefaultResources, endpoints[0]) testnet.NoError("failed to create tx client", err) // start the testnet logger.Println("Setting up testnet") - testnet.NoError("Failed to setup testnet", testNet.Setup()) + testnet.NoError("Failed to setup testnet", testNet.Setup(ctx)) logger.Println("Starting testnet") - testnet.NoError("Failed to start testnet", testNet.Start()) + testnet.NoError("Failed to start testnet", testNet.Start(ctx)) for i := 0; i < len(versions)*2; i++ { // FIXME: skip the first node because we need them available to @@ -86,10 +85,10 @@ func MinorVersionCompatibility(logger *log.Logger) error { newVersion := versions.Random(r).String() logger.Println("Upgrading node", "node", i%numNodes+1, "version", newVersion) - testnet.NoError("failed to upgrade node", testNet.Node(i%numNodes).Upgrade(newVersion)) + testnet.NoError("failed to upgrade node", testNet.Node(i%numNodes).Upgrade(ctx, newVersion)) time.Sleep(10 * time.Second) // wait for the node to reach two more heights - testnet.NoError("failed to wait for height", waitForHeight(ctx, client, heightBefore+2, 30*time.Second)) + testnet.NoError("failed to wait for height", waitForHeight(ctx, client, heightBefore+2, time.Minute)) } heights := make([]int64, 4) diff --git a/test/e2e/simple.go b/test/e2e/simple.go index 12e493de49..7999c726a0 100644 --- a/test/e2e/simple.go +++ b/test/e2e/simple.go @@ -15,39 +15,39 @@ import ( // and MsgSends over 30 seconds and then asserts that at least 10 transactions were // committed. func E2ESimple(logger *log.Logger) error { - latestVersion, err := testnet.GetLatestVersion() - testnet.NoError("failed to get latest version", err) + ctx := context.Background() + testNet, err := testnet.New(ctx, "E2ESimple", seed, nil, "test") + testnet.NoError("failed to create testnet", err) - logger.Println("Running simple e2e test", "version", latestVersion) + defer testNet.Cleanup(ctx) - testNet, err := testnet.New("E2ESimple", seed, nil, "test") - testnet.NoError("failed to create testnet", err) + latestVersion, err := testnet.GetLatestVersion() + testnet.NoError("failed to get latest version", err) - defer testNet.Cleanup() + logger.Println("Running E2ESimple test", "version", latestVersion) logger.Println("Creating testnet validators") testnet.NoError("failed to create genesis nodes", - testNet.CreateGenesisNodes(4, latestVersion, 10000000, 0, - testnet.DefaultResources, true)) + testNet.CreateGenesisNodes(ctx, 4, latestVersion, 10000000, 0, testnet.DefaultResources, true)) logger.Println("Creating txsim") - endpoints, err := testNet.RemoteGRPCEndpoints() + endpoints, err := testNet.RemoteGRPCEndpoints(ctx) testnet.NoError("failed to get remote gRPC endpoints", err) - err = testNet.CreateTxClient("txsim", testnet.TxsimVersion, 10, + err = testNet.CreateTxClient(ctx, "txsim", testnet.TxsimVersion, 10, "100-2000", 100, testnet.DefaultResources, endpoints[0]) testnet.NoError("failed to create tx client", err) logger.Println("Setting up testnets") - testnet.NoError("failed to setup testnets", testNet.Setup()) + testnet.NoError("failed to setup testnets", testNet.Setup(ctx)) logger.Println("Starting testnets") - testnet.NoError("failed to start testnets", testNet.Start()) + testnet.NoError("failed to start testnets", testNet.Start(ctx)) logger.Println("Waiting for 30 seconds to produce blocks") time.Sleep(30 * time.Second) logger.Println("Reading blockchain headers") - blockchain, err := testnode.ReadBlockchainHeaders(context.Background(), testNet.Node(0).AddressRPC()) + blockchain, err := testnode.ReadBlockchainHeaders(ctx, testNet.Node(0).AddressRPC()) testnet.NoError("failed to read blockchain headers", err) totalTxs := 0 diff --git a/test/e2e/testnet/defaults.go b/test/e2e/testnet/defaults.go index 1ea4bbb585..6c6da2cbf9 100644 --- a/test/e2e/testnet/defaults.go +++ b/test/e2e/testnet/defaults.go @@ -1,10 +1,12 @@ package testnet +import "k8s.io/apimachinery/pkg/api/resource" + var DefaultResources = Resources{ - MemoryRequest: "400Mi", - MemoryLimit: "400Mi", - CPU: "300m", - Volume: "1Gi", + MemoryRequest: resource.MustParse("400Mi"), + MemoryLimit: resource.MustParse("400Mi"), + CPU: resource.MustParse("300m"), + Volume: resource.MustParse("1Gi"), } const ( diff --git a/test/e2e/testnet/node.go b/test/e2e/testnet/node.go index 270cd545b7..14293da5fc 100644 --- a/test/e2e/testnet/node.go +++ b/test/e2e/testnet/node.go @@ -2,12 +2,11 @@ package testnet import ( + "context" "fmt" "os" "path/filepath" - "github.com/celestiaorg/celestia-app/v3/test/util/genesis" - "github.com/celestiaorg/knuu/pkg/knuu" serverconfig "github.com/cosmos/cosmos-sdk/server/config" "github.com/rs/zerolog/log" "github.com/tendermint/tendermint/config" @@ -18,6 +17,13 @@ import ( "github.com/tendermint/tendermint/privval" "github.com/tendermint/tendermint/rpc/client/http" "github.com/tendermint/tendermint/types" + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/celestiaorg/celestia-app/v3/test/util/genesis" + "github.com/celestiaorg/knuu/pkg/instance" + "github.com/celestiaorg/knuu/pkg/knuu" + "github.com/celestiaorg/knuu/pkg/sidecars/netshaper" + "github.com/celestiaorg/knuu/pkg/sidecars/observability" ) const ( @@ -41,7 +47,9 @@ type Node struct { SignerKey crypto.PrivKey NetworkKey crypto.PrivKey SelfDelegation int64 - Instance *knuu.Instance + Instance *instance.Instance + sidecars []instance.SidecarManager + netShaper *netshaper.NetShaper // a reference to the netshaper sidecar rpcProxyHost string // FIXME: This does not work currently with the reverse proxy @@ -80,16 +88,17 @@ func (n *Node) PullBlockSummaryTraces(path string) ([]trace.Event[schema.BlockSu // Resources defines the resource requirements for a Node. type Resources struct { // MemoryRequest specifies the initial memory allocation for the Node. - MemoryRequest string + MemoryRequest resource.Quantity // MemoryLimit specifies the maximum memory allocation for the Node. - MemoryLimit string + MemoryLimit resource.Quantity // CPU specifies the CPU allocation for the Node. - CPU string + CPU resource.Quantity // Volume specifies the storage volume allocation for the Node. - Volume string + Volume resource.Quantity } func NewNode( + ctx context.Context, name, version string, startHeight, selfDelegation int64, peers []string, @@ -97,54 +106,52 @@ func NewNode( upgradeHeight int64, resources Resources, grafana *GrafanaInfo, + kn *knuu.Knuu, disableBBR bool, ) (*Node, error) { - instance, err := knuu.NewInstance(name) + knInstance, err := kn.NewInstance(name) if err != nil { return nil, err } - err = instance.SetImage(DockerImageName(version)) + err = knInstance.Build().SetImage(ctx, DockerImageName(version)) if err != nil { return nil, err } - if err := instance.AddPortTCP(rpcPort); err != nil { - return nil, err - } - if err := instance.AddPortTCP(p2pPort); err != nil { - return nil, err - } - if err := instance.AddPortTCP(grpcPort); err != nil { - return nil, err - } - if err := instance.AddPortTCP(tracingPort); err != nil { - return nil, err + for _, port := range []int{rpcPort, p2pPort, grpcPort, tracingPort} { + if err := knInstance.Network().AddPortTCP(port); err != nil { + return nil, err + } } + var sidecars []instance.SidecarManager if grafana != nil { + obsySc := observability.New() + // add support for metrics - if err := instance.SetPrometheusEndpoint(prometheusPort, fmt.Sprintf("knuu-%s", knuu.Scope()), "1m"); err != nil { + if err := obsySc.SetPrometheusEndpoint(prometheusPort, fmt.Sprintf("knuu-%s", kn.Scope), "1m"); err != nil { return nil, fmt.Errorf("setting prometheus endpoint: %w", err) } - if err := instance.SetJaegerEndpoint(14250, 6831, 14268); err != nil { + if err := obsySc.SetJaegerEndpoint(14250, 6831, 14268); err != nil { return nil, fmt.Errorf("error setting jaeger endpoint: %v", err) } - if err := instance.SetOtlpExporter(grafana.Endpoint, grafana.Username, grafana.Token); err != nil { + if err := obsySc.SetOtlpExporter(grafana.Endpoint, grafana.Username, grafana.Token); err != nil { return nil, fmt.Errorf("error setting otlp exporter: %v", err) } - if err := instance.SetJaegerExporter("jaeger-collector.jaeger-cluster.svc.cluster.local:14250"); err != nil { + if err := obsySc.SetJaegerExporter("jaeger-collector.jaeger-cluster.svc.cluster.local:14250"); err != nil { return nil, fmt.Errorf("error setting jaeger exporter: %v", err) } + sidecars = append(sidecars, obsySc) } - err = instance.SetMemory(resources.MemoryRequest, resources.MemoryLimit) + err = knInstance.Resources().SetMemory(resources.MemoryRequest, resources.MemoryLimit) if err != nil { return nil, err } - err = instance.SetCPU(resources.CPU) + err = knInstance.Resources().SetCPU(resources.CPU) if err != nil { return nil, err } - err = instance.AddVolumeWithOwner(remoteRootDir, resources.Volume, 10001) + err = knInstance.Storage().AddVolumeWithOwner(remoteRootDir, resources.Volume, 10001) if err != nil { return nil, err } @@ -156,24 +163,36 @@ func NewNode( args = append(args, fmt.Sprintf("--v2-upgrade-height=%d", upgradeHeight)) } - err = instance.SetArgs(args...) - if err != nil { + if err := knInstance.Build().SetArgs(args...); err != nil { return nil, err } return &Node{ Name: name, - Instance: instance, + Instance: knInstance, Version: version, StartHeight: startHeight, InitialPeers: peers, SignerKey: signerKey, NetworkKey: networkKey, SelfDelegation: selfDelegation, + sidecars: sidecars, }, nil } -func (n *Node) Init(genesis *types.GenesisDoc, peers []string, configOptions ...Option) error { +func (n *Node) EnableNetShaper() { + n.netShaper = netshaper.New() + n.sidecars = append(n.sidecars, n.netShaper) +} + +func (n *Node) SetLatencyAndJitter(latency, jitter int64) error { + if n.netShaper == nil { + return fmt.Errorf("netshaper is not enabled") + } + return n.netShaper.SetLatencyAndJitter(latency, jitter) +} + +func (n *Node) Init(ctx context.Context, genesis *types.GenesisDoc, peers []string, configOptions ...Option) error { if len(peers) == 0 { return fmt.Errorf("no peers provided") } @@ -194,7 +213,7 @@ func (n *Node) Init(genesis *types.GenesisDoc, peers []string, configOptions ... } // Create and write the config file - cfg, err := MakeConfig(n, configOptions...) + cfg, err := MakeConfig(ctx, n, configOptions...) if err != nil { return fmt.Errorf("making config: %w", err) } @@ -239,12 +258,17 @@ func (n *Node) Init(genesis *types.GenesisDoc, peers []string, configOptions ... return fmt.Errorf("writing address book: %w", err) } - err = n.Instance.Commit() - if err != nil { + if err := n.Instance.Build().Commit(ctx); err != nil { return fmt.Errorf("committing instance: %w", err) } - if err = n.Instance.AddFolder(nodeDir, remoteRootDir, "10001:10001"); err != nil { + for _, sc := range n.sidecars { + if err := n.Instance.Sidecars().Add(ctx, sc); err != nil { + return fmt.Errorf("adding sidecar: %w", err) + } + } + + if err = n.Instance.Storage().AddFolder(nodeDir, remoteRootDir, "10001:10001"); err != nil { return fmt.Errorf("copying over node %s directory: %w", n.Name, err) } return nil @@ -253,8 +277,8 @@ func (n *Node) Init(genesis *types.GenesisDoc, peers []string, configOptions ... // AddressP2P returns a P2P endpoint address for the node. This is used for // populating the address book. This will look something like: // 3314051954fc072a0678ec0cbac690ad8676ab98@61.108.66.220:26656 -func (n Node) AddressP2P(withID bool) string { - ip, err := n.Instance.GetIP() +func (n Node) AddressP2P(ctx context.Context, withID bool) string { + ip, err := n.Instance.Network().GetIP(ctx) if err != nil { panic(err) } @@ -279,8 +303,8 @@ func (n Node) AddressRPC() string { // } // RemoteAddressGRPC retrieves the gRPC endpoint address of a node within the cluster. -func (n Node) RemoteAddressGRPC() (string, error) { - ip, err := n.Instance.GetIP() +func (n Node) RemoteAddressGRPC(ctx context.Context) (string, error) { + ip, err := n.Instance.Network().GetIP(ctx) if err != nil { return "", err } @@ -288,8 +312,8 @@ func (n Node) RemoteAddressGRPC() (string, error) { } // RemoteAddressRPC retrieves the RPC endpoint address of a node within the cluster. -func (n Node) RemoteAddressRPC() (string, error) { - ip, err := n.Instance.GetIP() +func (n Node) RemoteAddressRPC(ctx context.Context) (string, error) { + ip, err := n.Instance.Network().GetIP(ctx) if err != nil { return "", err } @@ -300,8 +324,8 @@ func (n Node) AddressTracing() string { return n.traceProxyHost } -func (n Node) RemoteAddressTracing() (string, error) { - ip, err := n.Instance.GetIP() +func (n Node) RemoteAddressTracing(ctx context.Context) (string, error) { + ip, err := n.Instance.Network().GetIP(ctx) if err != nil { return "", err } @@ -317,24 +341,25 @@ func (n Node) Client() (*http.HTTP, error) { return http.New(n.AddressRPC(), "/websocket") } -func (n *Node) Start() error { - if err := n.StartAsync(); err != nil { +func (n *Node) Start(ctx context.Context) error { + if err := n.StartAsync(ctx); err != nil { return err } - return n.WaitUntilStartedAndForwardPorts() + return n.WaitUntilStartedAndCreateProxy(ctx) } -func (n *Node) StartAsync() error { - return n.Instance.StartAsync() +func (n *Node) StartAsync(ctx context.Context) error { + return n.Instance.Execution().StartAsync(ctx) } -func (n *Node) WaitUntilStartedAndForwardPorts() error { - if err := n.Instance.WaitInstanceIsRunning(); err != nil { +func (n *Node) WaitUntilStartedAndCreateProxy(ctx context.Context) error { + if err := n.Instance.Execution().WaitInstanceIsRunning(ctx); err != nil { return err } - err, rpcProxyHost := n.Instance.AddHost(rpcPort) + // TODO: It is recommended to use AddHostWithReadyCheck for the proxy + rpcProxyHost, err := n.Instance.Network().AddHost(ctx, rpcPort) if err != nil { return err } @@ -347,7 +372,8 @@ func (n *Node) WaitUntilStartedAndForwardPorts() error { // } // n.grpcProxyHost = grpcProxyHost - err, traceProxyHost := n.Instance.AddHost(tracingPort) + // TODO: It is recommended to use AddHostWithReadyCheck for the proxy + traceProxyHost, err := n.Instance.Network().AddHost(ctx, tracingPort) if err != nil { return err } @@ -368,12 +394,12 @@ func (n *Node) GenesisValidator() genesis.Validator { } } -func (n *Node) Upgrade(version string) error { - if err := n.Instance.SetImageInstant(DockerImageName(version)); err != nil { +func (n *Node) Upgrade(ctx context.Context, version string) error { + if err := n.Instance.Execution().UpgradeImage(ctx, DockerImageName(version)); err != nil { return err } - return n.Instance.WaitInstanceIsRunning() + return n.Instance.Execution().WaitInstanceIsRunning(ctx) } func DockerImageName(version string) string { diff --git a/test/e2e/testnet/setup.go b/test/e2e/testnet/setup.go index 620b46cb66..d7db215c34 100644 --- a/test/e2e/testnet/setup.go +++ b/test/e2e/testnet/setup.go @@ -1,6 +1,7 @@ package testnet import ( + "context" "fmt" "strings" "time" @@ -12,14 +13,14 @@ import ( "github.com/tendermint/tendermint/p2p/pex" ) -func MakeConfig(node *Node, opts ...Option) (*config.Config, error) { +func MakeConfig(ctx context.Context, node *Node, opts ...Option) (*config.Config, error) { cfg := app.DefaultConsensusConfig() cfg.TxIndex.Indexer = "kv" cfg.Consensus.TimeoutPropose = config.DefaultConsensusConfig().TimeoutPropose cfg.Consensus.TimeoutCommit = config.DefaultConsensusConfig().TimeoutCommit cfg.Moniker = node.Name cfg.RPC.ListenAddress = "tcp://0.0.0.0:26657" - cfg.P2P.ExternalAddress = fmt.Sprintf("tcp://%v", node.AddressP2P(false)) + cfg.P2P.ExternalAddress = fmt.Sprintf("tcp://%v", node.AddressP2P(ctx, false)) cfg.P2P.PersistentPeers = strings.Join(node.InitialPeers, ",") cfg.Instrumentation.Prometheus = true diff --git a/test/e2e/testnet/testnet.go b/test/e2e/testnet/testnet.go index 76d0bc15df..6b4ff136ed 100644 --- a/test/e2e/testnet/testnet.go +++ b/test/e2e/testnet/testnet.go @@ -13,6 +13,7 @@ import ( "github.com/celestiaorg/celestia-app/v3/app/encoding" "github.com/celestiaorg/celestia-app/v3/test/util/genesis" "github.com/celestiaorg/knuu/pkg/knuu" + "github.com/celestiaorg/knuu/pkg/preloader" "github.com/cosmos/cosmos-sdk/crypto/hd" "github.com/cosmos/cosmos-sdk/crypto/keyring" "github.com/rs/zerolog/log" @@ -26,26 +27,43 @@ type Testnet struct { keygen *keyGenerator grafana *GrafanaInfo txClients []*TxSim + knuu *knuu.Knuu } -func New(name string, seed int64, grafana *GrafanaInfo, chainID string, +func New(ctx context.Context, name string, seed int64, grafana *GrafanaInfo, chainID string, genesisModifiers ...genesis.Modifier) ( *Testnet, error, ) { identifier := fmt.Sprintf("%s_%s", name, time.Now().Format("20060102_150405")) - if err := knuu.InitializeWithScope(identifier); err != nil { + kn, err := knuu.New(ctx, knuu.Options{ + Scope: identifier, + ProxyEnabled: true, + // if the tests timeout, pass the timeout option + // Timeout: 120 * time.Minute, + }) + if err != nil { return nil, err } + log.Info().Str("scope", kn.Scope).Msg("Knuu initialized") + return &Testnet{ seed: seed, nodes: make([]*Node, 0), genesis: genesis.NewDefaultGenesis().WithChainID(chainID).WithModifiers(genesisModifiers...), keygen: newKeyGenerator(seed), grafana: grafana, + knuu: kn, }, nil } +func (t *Testnet) NewPreloader() (*preloader.Preloader, error) { + if t.knuu == nil { + return nil, errors.New("knuu is not initialized") + } + return preloader.New(t.knuu.SystemDependencies) +} + func (t *Testnet) SetConsensusParams(params *tmproto.ConsensusParams) { t.genesis.WithConsensusParams(params) } @@ -54,12 +72,14 @@ func (t *Testnet) SetConsensusMaxBlockSize(size int64) { t.genesis.ConsensusParams.Block.MaxBytes = size } -func (t *Testnet) CreateGenesisNode(version string, selfDelegation, upgradeHeight int64, resources Resources, disableBBR bool) error { +func (t *Testnet) CreateGenesisNode(ctx context.Context, version string, selfDelegation, upgradeHeight int64, resources Resources, disableBBR bool) error { signerKey := t.keygen.Generate(ed25519Type) networkKey := t.keygen.Generate(ed25519Type) - node, err := NewNode(fmt.Sprintf("val%d", len(t.nodes)), version, 0, - selfDelegation, nil, signerKey, networkKey, upgradeHeight, resources, - t.grafana, disableBBR) + node, err := NewNode(ctx, + fmt.Sprintf("val%d", len(t.nodes)), version, 0, + selfDelegation, nil, signerKey, networkKey, + upgradeHeight, resources, t.grafana, t.knuu, disableBBR, + ) if err != nil { return err } @@ -70,16 +90,17 @@ func (t *Testnet) CreateGenesisNode(version string, selfDelegation, upgradeHeigh return nil } -func (t *Testnet) CreateGenesisNodes(num int, version string, selfDelegation, upgradeHeight int64, resources Resources, disableBBR bool) error { +func (t *Testnet) CreateGenesisNodes(ctx context.Context, num int, version string, selfDelegation, upgradeHeight int64, resources Resources, disableBBR bool) error { for i := 0; i < num; i++ { - if err := t.CreateGenesisNode(version, selfDelegation, upgradeHeight, resources, disableBBR); err != nil { + if err := t.CreateGenesisNode(ctx, version, selfDelegation, upgradeHeight, resources, disableBBR); err != nil { return err } } return nil } -func (t *Testnet) CreateTxClients(version string, +func (t *Testnet) CreateTxClients(ctx context.Context, + version string, sequences int, blobRange string, blobPerSequence int, @@ -88,7 +109,7 @@ func (t *Testnet) CreateTxClients(version string, ) error { for i, grpcEndpoint := range grpcEndpoints { name := fmt.Sprintf("txsim%d", i) - err := t.CreateTxClient(name, version, sequences, + err := t.CreateTxClient(ctx, name, version, sequences, blobRange, blobPerSequence, resources, grpcEndpoint) if err != nil { log.Err(err).Str("name", name). @@ -114,8 +135,9 @@ func (t *Testnet) CreateTxClients(version string, // pollTime: time in seconds between each sequence // resources: resources to be allocated to the txsim // grpcEndpoint: grpc endpoint of the node to which the txsim will connect and send transactions -func (t *Testnet) CreateTxClient(name, - version string, +func (t *Testnet) CreateTxClient( + ctx context.Context, + name, version string, sequences int, blobRange string, blobPerSequence int, @@ -135,15 +157,15 @@ func (t *Testnet) CreateTxClient(name, } // Create a txsim node using the key stored in the txsimKeyringDir - txsim, err := CreateTxClient(name, version, grpcEndpoint, t.seed, - sequences, blobRange, blobPerSequence, 1, resources, txsimRootDir) + txsim, err := CreateTxClient(ctx, name, version, grpcEndpoint, t.seed, + sequences, blobRange, blobPerSequence, 1, resources, txsimRootDir, t.knuu) if err != nil { log.Err(err). Str("name", name). Msg("error creating txsim") return err } - err = txsim.Instance.Commit() + err = txsim.Instance.Build().Commit(ctx) if err != nil { log.Err(err). Str("name", name). @@ -152,7 +174,7 @@ func (t *Testnet) CreateTxClient(name, } // copy over the keyring directory to the txsim instance - err = txsim.Instance.AddFolder(txsimKeyringDir, txsimRootDir, "10001:10001") + err = txsim.Instance.Storage().AddFolder(txsimKeyringDir, txsimRootDir, "10001:10001") if err != nil { log.Err(err). Str("directory", txsimKeyringDir). @@ -165,9 +187,9 @@ func (t *Testnet) CreateTxClient(name, return nil } -func (t *Testnet) StartTxClients() error { +func (t *Testnet) StartTxClients(ctx context.Context) error { for _, txsim := range t.txClients { - err := txsim.Instance.StartWithoutWait() + err := txsim.Instance.Execution().StartAsync(ctx) if err != nil { log.Err(err). Str("name", txsim.Name). @@ -180,7 +202,7 @@ func (t *Testnet) StartTxClients() error { } // wait for txsims to start for _, txsim := range t.txClients { - err := txsim.Instance.WaitInstanceIsRunning() + err := txsim.Instance.Execution().WaitInstanceIsRunning(ctx) if err != nil { return fmt.Errorf("txsim %s failed to run: %w", txsim.Name, err) } @@ -232,12 +254,14 @@ func (t *Testnet) CreateAccount(name string, tokens int64, txsimKeyringDir strin return kr, nil } -func (t *Testnet) CreateNode(version string, startHeight, upgradeHeight int64, resources Resources, disableBBR bool) error { +func (t *Testnet) CreateNode(ctx context.Context, version string, startHeight, upgradeHeight int64, resources Resources, disableBBR bool) error { signerKey := t.keygen.Generate(ed25519Type) networkKey := t.keygen.Generate(ed25519Type) - node, err := NewNode(fmt.Sprintf("val%d", len(t.nodes)), version, - startHeight, 0, nil, signerKey, networkKey, upgradeHeight, resources, - t.grafana, disableBBR) + node, err := NewNode(ctx, + fmt.Sprintf("val%d", len(t.nodes)), version, + startHeight, 0, nil, signerKey, networkKey, + upgradeHeight, resources, t.grafana, t.knuu, disableBBR, + ) if err != nil { return err } @@ -245,7 +269,7 @@ func (t *Testnet) CreateNode(version string, startHeight, upgradeHeight int64, r return nil } -func (t *Testnet) Setup(configOpts ...Option) error { +func (t *Testnet) Setup(ctx context.Context, configOpts ...Option) error { genesis, err := t.genesis.Export() if err != nil { return err @@ -257,11 +281,11 @@ func (t *Testnet) Setup(configOpts ...Option) error { peers := make([]string, 0, len(t.nodes)-1) for _, peer := range t.nodes { if peer.Name != node.Name { - peers = append(peers, peer.AddressP2P(true)) + peers = append(peers, peer.AddressP2P(ctx, true)) } } - err := node.Init(genesis, peers, configOpts...) + err := node.Init(ctx, genesis, peers, configOpts...) if err != nil { return err } @@ -289,10 +313,10 @@ func (t *Testnet) RPCEndpoints() []string { // RemoteGRPCEndpoints retrieves the gRPC endpoint addresses of the // testnet's validator nodes. -func (t *Testnet) RemoteGRPCEndpoints() ([]string, error) { +func (t *Testnet) RemoteGRPCEndpoints(ctx context.Context) ([]string, error) { grpcEndpoints := make([]string, len(t.nodes)) for idx, node := range t.nodes { - grpcEP, err := node.RemoteAddressGRPC() + grpcEP, err := node.RemoteAddressGRPC(ctx) if err != nil { return nil, err } @@ -311,10 +335,10 @@ func (t *Testnet) GetGenesisValidators() []genesis.Validator { // RemoteRPCEndpoints retrieves the RPC endpoint addresses of the testnet's // validator nodes. -func (t *Testnet) RemoteRPCEndpoints() ([]string, error) { +func (t *Testnet) RemoteRPCEndpoints(ctx context.Context) ([]string, error) { rpcEndpoints := make([]string, len(t.nodes)) for idx, node := range t.nodes { - grpcEP, err := node.RemoteAddressRPC() + grpcEP, err := node.RemoteAddressRPC(ctx) if err != nil { return nil, err } @@ -325,7 +349,7 @@ func (t *Testnet) RemoteRPCEndpoints() ([]string, error) { // WaitToSync waits for the started nodes to sync with the network and move // past the genesis block. -func (t *Testnet) WaitToSync() error { +func (t *Testnet) WaitToSync(ctx context.Context) error { genesisNodes := make([]*Node, 0) for _, node := range t.nodes { if node.StartHeight == 0 { @@ -340,7 +364,7 @@ func (t *Testnet) WaitToSync() error { return fmt.Errorf("failed to initialize client for node %s: %w", node.Name, err) } for i := 0; i < 10; i++ { - resp, err := client.Status(context.Background()) + resp, err := client.Status(ctx) if err == nil { if resp.SyncInfo.LatestBlockHeight > 0 { log.Info().Int("attempts", i).Str("name", node.Name).Msg( @@ -361,10 +385,10 @@ func (t *Testnet) WaitToSync() error { return nil } -// StartNodes starts the testnet nodes and forwards the ports. +// StartNodes starts the testnet nodes and setup proxies. // It does not wait for the nodes to produce blocks. // For that, use WaitToSync. -func (t *Testnet) StartNodes() error { +func (t *Testnet) StartNodes(ctx context.Context) error { genesisNodes := make([]*Node, 0) for _, node := range t.nodes { if node.StartHeight == 0 { @@ -373,15 +397,15 @@ func (t *Testnet) StartNodes() error { } // start genesis nodes asynchronously for _, node := range genesisNodes { - err := node.StartAsync() + err := node.StartAsync(ctx) if err != nil { return fmt.Errorf("node %s failed to start: %w", node.Name, err) } } - log.Info().Msg("forwarding ports for genesis nodes") + log.Info().Msg("create endpoint proxies for genesis nodes") // wait for instances to be running for _, node := range genesisNodes { - err := node.WaitUntilStartedAndForwardPorts() + err := node.WaitUntilStartedAndCreateProxy(ctx) if err != nil { log.Err(err).Str("name", node.Name).Str("version", node.Version).Msg("failed to start and forward ports") @@ -393,40 +417,25 @@ func (t *Testnet) StartNodes() error { return nil } -func (t *Testnet) Start() error { - // start nodes and forward ports - err := t.StartNodes() +func (t *Testnet) Start(ctx context.Context) error { + // start nodes and setup proxies + err := t.StartNodes(ctx) if err != nil { return err } // wait for nodes to sync log.Info().Msg("waiting for genesis nodes to sync") - err = t.WaitToSync() + err = t.WaitToSync(ctx) if err != nil { return err } - return t.StartTxClients() + return t.StartTxClients(ctx) } -func (t *Testnet) Cleanup() { - // cleanup txsim - for _, txsim := range t.txClients { - err := txsim.Instance.Destroy() - if err != nil { - log.Err(err). - Str("name", txsim.Name). - Msg("txsim failed to cleanup") - } - } - // cleanup nodes - for _, node := range t.nodes { - err := node.Instance.Destroy() - if err != nil { - log.Err(err). - Str("name", node.Name). - Msg("node failed to cleanup") - } +func (t *Testnet) Cleanup(ctx context.Context) { + if err := t.knuu.CleanUp(ctx); err != nil { + log.Err(err).Msg("failed to cleanup knuu") } } diff --git a/test/e2e/testnet/txsimNode.go b/test/e2e/testnet/txsimNode.go index 994d87a831..66bd005f3e 100644 --- a/test/e2e/testnet/txsimNode.go +++ b/test/e2e/testnet/txsimNode.go @@ -2,8 +2,10 @@ package testnet import ( + "context" "fmt" + "github.com/celestiaorg/knuu/pkg/instance" "github.com/celestiaorg/knuu/pkg/knuu" "github.com/rs/zerolog/log" ) @@ -18,10 +20,11 @@ func txsimDockerImageName(version string) string { type TxSim struct { Name string - Instance *knuu.Instance + Instance *instance.Instance } func CreateTxClient( + ctx context.Context, name, version string, endpoint string, seed int64, @@ -31,8 +34,9 @@ func CreateTxClient( pollTime int, resources Resources, volumePath string, + knuu *knuu.Knuu, ) (*TxSim, error) { - instance, err := knuu.NewInstance(name) + txIns, err := knuu.NewInstance(name) if err != nil { return nil, err } @@ -41,7 +45,7 @@ func CreateTxClient( Str("name", name). Str("image", image). Msg("setting image for tx client") - err = instance.SetImage(image) + err = txIns.Build().SetImage(ctx, image) if err != nil { log.Err(err). Str("name", name). @@ -49,15 +53,15 @@ func CreateTxClient( Msg("failed to set image for tx client") return nil, err } - err = instance.SetMemory(resources.MemoryRequest, resources.MemoryLimit) + err = txIns.Resources().SetMemory(resources.MemoryRequest, resources.MemoryLimit) if err != nil { return nil, err } - err = instance.SetCPU(resources.CPU) + err = txIns.Resources().SetCPU(resources.CPU) if err != nil { return nil, err } - err = instance.AddVolumeWithOwner(volumePath, resources.Volume, 10001) + err = txIns.Storage().AddVolumeWithOwner(volumePath, resources.Volume, 10001) if err != nil { return nil, err } @@ -71,13 +75,12 @@ func CreateTxClient( fmt.Sprintf("-s %s ", blobRange), } - err = instance.SetArgs(args...) - if err != nil { + if err := txIns.Build().SetArgs(args...); err != nil { return nil, err } return &TxSim{ Name: name, - Instance: instance, + Instance: txIns, }, nil } From 228a32d02ac3be8bdf1889826912fe1fe32eb1cc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 17 Sep 2024 14:46:03 +0000 Subject: [PATCH 30/85] chore(deps): Bump google.golang.org/grpc from 1.66.0 to 1.66.2 (#3869) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [//]: # (dependabot-start) ⚠️ **Dependabot is rebasing this PR** ⚠️ Rebasing might not happen immediately, so don't worry if this takes some time. Note: if you make any changes to this PR yourself, they will take precedence over the rebase. --- [//]: # (dependabot-end) Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.66.0 to 1.66.2.
Release notes

Sourced from google.golang.org/grpc's releases.

Release 1.66.2

Dependencies

  • Remove unintentional dependency on the testing package (#7579)
  • Remove unintentional dependency on the flate package (#7595)

Bug Fixes

  • client: fix a bug that prevented memory reuse after handling unary RPCs (#7571)
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=google.golang.org/grpc&package-manager=go_modules&previous-version=1.66.0&new-version=1.66.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 12ea474f6a..ec2d06001b 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/tendermint/tm-db v0.6.7 golang.org/x/exp v0.0.0-20240213143201-ec583247a57a google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 - google.golang.org/grpc v1.66.0 + google.golang.org/grpc v1.66.2 google.golang.org/protobuf v1.34.2 gopkg.in/yaml.v2 v2.4.0 k8s.io/apimachinery v0.30.2 diff --git a/go.sum b/go.sum index 4f3bfca260..c98a7c818e 100644 --- a/go.sum +++ b/go.sum @@ -2003,8 +2003,8 @@ google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= -google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/grpc v1.66.2 h1:3QdXkuq3Bkh7w+ywLdLvM56cmGvQHUMZpiCzt6Rqaoo= +google.golang.org/grpc v1.66.2/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= From 306c58745d135d31c3777a1af2f58d50adbd32c8 Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Tue, 17 Sep 2024 17:30:07 +0200 Subject: [PATCH 31/85] feat: support v3 app (#3870) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds v3 as an app version. --------- Co-authored-by: Rootul P Co-authored-by: nina / ნინა --- app/app.go | 8 +- app/check_tx.go | 5 +- app/module/versioned_ibc_module_test.go | 2 +- app/modules.go | 68 +++++++---- app/prepare_proposal.go | 40 +++++-- app/process_proposal.go | 48 +++++--- app/test/std_sdk_test.go | 3 +- app/test/upgrade_test.go | 151 +++++++++++++++++++++--- go.mod | 3 +- go.sum | 6 +- pkg/appconsts/v3/app_consts.go | 7 ++ pkg/appconsts/versioned_consts.go | 4 +- x/blob/types/blob_tx.go | 6 +- x/blob/types/blob_tx_test.go | 2 +- x/signal/integration_test.go | 17 +-- 15 files changed, 283 insertions(+), 87 deletions(-) create mode 100644 pkg/appconsts/v3/app_consts.go diff --git a/app/app.go b/app/app.go index 9f8bdf4408..d31d33fcbf 100644 --- a/app/app.go +++ b/app/app.go @@ -12,6 +12,7 @@ import ( "github.com/celestiaorg/celestia-app/v3/app/posthandler" appv1 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v1" appv2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" + appv3 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v3" "github.com/celestiaorg/celestia-app/v3/pkg/proof" blobkeeper "github.com/celestiaorg/celestia-app/v3/x/blob/keeper" blobtypes "github.com/celestiaorg/celestia-app/v3/x/blob/types" @@ -108,6 +109,7 @@ var maccPerms = map[string][]string{ const ( v1 = appv1.Version v2 = appv2.Version + v3 = appv3.Version DefaultInitialVersion = v1 ) @@ -340,11 +342,11 @@ func New( packetforwardkeeper.DefaultForwardTransferPacketTimeoutTimestamp, // forward timeout packetforwardkeeper.DefaultRefundTransferPacketTimeoutTimestamp, // refund timeout ) - // PacketForwardMiddleware is used only for version 2. - transferStack = module.NewVersionedIBCModule(packetForwardMiddleware, transferStack, v2, v2) + // PacketForwardMiddleware is used only for version >= 2. + transferStack = module.NewVersionedIBCModule(packetForwardMiddleware, transferStack, v2, v3) // Token filter wraps packet forward middleware and is thus the first module in the transfer stack. tokenFilterMiddelware := tokenfilter.NewIBCMiddleware(transferStack) - transferStack = module.NewVersionedIBCModule(tokenFilterMiddelware, transferStack, v1, v2) + transferStack = module.NewVersionedIBCModule(tokenFilterMiddelware, transferStack, v1, v3) app.EvidenceKeeper = *evidencekeeper.NewKeeper( appCodec, diff --git a/app/check_tx.go b/app/check_tx.go index 0867cd2436..f76b37880f 100644 --- a/app/check_tx.go +++ b/app/check_tx.go @@ -41,9 +41,8 @@ func (app *App) CheckTx(req abci.RequestCheckTx) abci.ResponseCheckTx { switch req.Type { // new transactions must be checked in their entirety case abci.CheckTxType_New: - // FIXME: we have a hardcoded subtree root threshold here. This is because we can't access - // the app version because the context is not initialized - err := blobtypes.ValidateBlobTx(app.txConfig, btx, appconsts.DefaultSubtreeRootThreshold) + appVersion := app.AppVersion() + err := blobtypes.ValidateBlobTx(app.txConfig, btx, appconsts.SubtreeRootThreshold(appVersion), appVersion) if err != nil { return sdkerrors.ResponseCheckTxWithEvents(err, 0, 0, []abci.Event{}, false) } diff --git a/app/module/versioned_ibc_module_test.go b/app/module/versioned_ibc_module_test.go index 8d2fb23d0e..57f94ec9f9 100644 --- a/app/module/versioned_ibc_module_test.go +++ b/app/module/versioned_ibc_module_test.go @@ -23,7 +23,7 @@ func TestVersionedIBCModule(t *testing.T) { mockWrappedModule := mocks.NewMockIBCModule(ctrl) mockNextModule := mocks.NewMockIBCModule(ctrl) - versionedModule := module.NewVersionedIBCModule(mockWrappedModule, mockNextModule, 2, 2) + versionedModule := module.NewVersionedIBCModule(mockWrappedModule, mockNextModule, 2, 3) testCases := []struct { name string diff --git a/app/modules.go b/app/modules.go index a89f418205..d629a99073 100644 --- a/app/modules.go +++ b/app/modules.go @@ -96,75 +96,75 @@ func (app *App) setupModuleManager(skipGenesisInvariants bool) error { app.manager, err = module.NewManager([]module.VersionedModule{ { Module: genutil.NewAppModule(app.AccountKeeper, app.StakingKeeper, app.BaseApp.DeliverTx, app.txConfig), - FromVersion: v1, ToVersion: v2, + FromVersion: v1, ToVersion: v3, }, { Module: auth.NewAppModule(app.appCodec, app.AccountKeeper, nil), - FromVersion: v1, ToVersion: v2, + FromVersion: v1, ToVersion: v3, }, { Module: vesting.NewAppModule(app.AccountKeeper, app.BankKeeper), - FromVersion: v1, ToVersion: v2, + FromVersion: v1, ToVersion: v3, }, { Module: bank.NewAppModule(app.appCodec, app.BankKeeper, app.AccountKeeper), - FromVersion: v1, ToVersion: v2, + FromVersion: v1, ToVersion: v3, }, { Module: capability.NewAppModule(app.appCodec, *app.CapabilityKeeper), - FromVersion: v1, ToVersion: v2, + FromVersion: v1, ToVersion: v3, }, { Module: feegrantmodule.NewAppModule(app.appCodec, app.AccountKeeper, app.BankKeeper, app.FeeGrantKeeper, app.interfaceRegistry), - FromVersion: v1, ToVersion: v2, + FromVersion: v1, ToVersion: v3, }, { Module: crisis.NewAppModule(&app.CrisisKeeper, skipGenesisInvariants), - FromVersion: v1, ToVersion: v2, + FromVersion: v1, ToVersion: v3, }, { Module: gov.NewAppModule(app.appCodec, app.GovKeeper, app.AccountKeeper, app.BankKeeper), - FromVersion: v1, ToVersion: v2, + FromVersion: v1, ToVersion: v3, }, { Module: mint.NewAppModule(app.appCodec, app.MintKeeper, app.AccountKeeper), - FromVersion: v1, ToVersion: v2, + FromVersion: v1, ToVersion: v3, }, { Module: slashing.NewAppModule(app.appCodec, app.SlashingKeeper, app.AccountKeeper, app.BankKeeper, app.StakingKeeper), - FromVersion: v1, ToVersion: v2, + FromVersion: v1, ToVersion: v3, }, { Module: distr.NewAppModule(app.appCodec, app.DistrKeeper, app.AccountKeeper, app.BankKeeper, app.StakingKeeper), - FromVersion: v1, ToVersion: v2, + FromVersion: v1, ToVersion: v3, }, { Module: staking.NewAppModule(app.appCodec, app.StakingKeeper, app.AccountKeeper, app.BankKeeper), - FromVersion: v1, ToVersion: v2, + FromVersion: v1, ToVersion: v3, }, { Module: evidence.NewAppModule(app.EvidenceKeeper), - FromVersion: v1, ToVersion: v2, + FromVersion: v1, ToVersion: v3, }, { Module: authzmodule.NewAppModule(app.appCodec, app.AuthzKeeper, app.AccountKeeper, app.BankKeeper, app.interfaceRegistry), - FromVersion: v1, ToVersion: v2, + FromVersion: v1, ToVersion: v3, }, { Module: ibc.NewAppModule(app.IBCKeeper), - FromVersion: v1, ToVersion: v2, + FromVersion: v1, ToVersion: v3, }, { Module: params.NewAppModule(app.ParamsKeeper), - FromVersion: v1, ToVersion: v2, + FromVersion: v1, ToVersion: v3, }, { Module: transfer.NewAppModule(app.TransferKeeper), - FromVersion: v1, ToVersion: v2, + FromVersion: v1, ToVersion: v3, }, { Module: blob.NewAppModule(app.appCodec, app.BlobKeeper), - FromVersion: v1, ToVersion: v2, + FromVersion: v1, ToVersion: v3, }, { Module: blobstream.NewAppModule(app.appCodec, app.BlobstreamKeeper), @@ -172,19 +172,19 @@ func (app *App) setupModuleManager(skipGenesisInvariants bool) error { }, { Module: signal.NewAppModule(app.SignalKeeper), - FromVersion: v2, ToVersion: v2, + FromVersion: v2, ToVersion: v3, }, { Module: minfee.NewAppModule(app.ParamsKeeper), - FromVersion: v2, ToVersion: v2, + FromVersion: v2, ToVersion: v3, }, { Module: packetforward.NewAppModule(app.PacketForwardKeeper), - FromVersion: v2, ToVersion: v2, + FromVersion: v2, ToVersion: v3, }, { Module: ica.NewAppModule(nil, &app.ICAHostKeeper), - FromVersion: v2, ToVersion: v2, + FromVersion: v2, ToVersion: v3, }, }) if err != nil { @@ -303,7 +303,7 @@ func allStoreKeys() []string { // versionedStoreKeys returns the store keys for each app version. func versionedStoreKeys() map[uint64][]string { return map[uint64][]string{ - 1: { + v1: { authtypes.StoreKey, authzkeeper.StoreKey, banktypes.StoreKey, @@ -321,7 +321,7 @@ func versionedStoreKeys() map[uint64][]string { stakingtypes.StoreKey, upgradetypes.StoreKey, }, - 2: { + v2: { authtypes.StoreKey, authzkeeper.StoreKey, banktypes.StoreKey, @@ -341,6 +341,26 @@ func versionedStoreKeys() map[uint64][]string { stakingtypes.StoreKey, upgradetypes.StoreKey, }, + v3: { // same as v2 + authtypes.StoreKey, + authzkeeper.StoreKey, + banktypes.StoreKey, + blobtypes.StoreKey, + capabilitytypes.StoreKey, + distrtypes.StoreKey, + evidencetypes.StoreKey, + feegrant.StoreKey, + govtypes.StoreKey, + ibchost.StoreKey, + ibctransfertypes.StoreKey, + icahosttypes.StoreKey, + minttypes.StoreKey, + packetforwardtypes.StoreKey, + signaltypes.StoreKey, + slashingtypes.StoreKey, + stakingtypes.StoreKey, + upgradetypes.StoreKey, + }, } } diff --git a/app/prepare_proposal.go b/app/prepare_proposal.go index 78049a1ad4..a34c64bd7d 100644 --- a/app/prepare_proposal.go +++ b/app/prepare_proposal.go @@ -1,13 +1,16 @@ package app import ( + "fmt" "time" "github.com/celestiaorg/celestia-app/v3/app/ante" "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" "github.com/celestiaorg/celestia-app/v3/pkg/da" - square "github.com/celestiaorg/go-square/v2" - "github.com/celestiaorg/go-square/v2/share" + shares "github.com/celestiaorg/go-square/shares" + square "github.com/celestiaorg/go-square/square" + squarev2 "github.com/celestiaorg/go-square/v2" + sharev2 "github.com/celestiaorg/go-square/v2/share" "github.com/cosmos/cosmos-sdk/telemetry" abci "github.com/tendermint/tendermint/abci/types" core "github.com/tendermint/tendermint/proto/tendermint/types" @@ -27,7 +30,7 @@ func (app *App) PrepareProposal(req abci.RequestPrepareProposal) abci.ResponsePr Height: req.Height, Time: req.Time, Version: version.Consensus{ - App: app.BaseApp.AppVersion(), + App: app.AppVersion(), }, }) handler := ante.NewAnteHandler( @@ -47,10 +50,31 @@ func (app *App) PrepareProposal(req abci.RequestPrepareProposal) abci.ResponsePr // Build the square from the set of valid and prioritised transactions. // The txs returned are the ones used in the square and block. - dataSquare, txs, err := square.Build(txs, - app.MaxEffectiveSquareSize(sdkCtx), - appconsts.SubtreeRootThreshold(app.GetBaseApp().AppVersion()), + var ( + dataSquareBytes [][]byte + err error + size uint64 ) + switch app.AppVersion() { + case v3: + var dataSquare squarev2.Square + dataSquare, txs, err = squarev2.Build(txs, + app.MaxEffectiveSquareSize(sdkCtx), + appconsts.SubtreeRootThreshold(app.GetBaseApp().AppVersion()), + ) + dataSquareBytes = sharev2.ToBytes(dataSquare) + size = uint64(dataSquare.Size()) + case v2, v1: + var dataSquare square.Square + dataSquare, txs, err = square.Build(txs, + app.MaxEffectiveSquareSize(sdkCtx), + appconsts.SubtreeRootThreshold(app.GetBaseApp().AppVersion()), + ) + dataSquareBytes = shares.ToBytes(dataSquare) + size = uint64(dataSquare.Size()) + default: + err = fmt.Errorf("unsupported app version: %d", app.AppVersion()) + } if err != nil { panic(err) } @@ -58,7 +82,7 @@ func (app *App) PrepareProposal(req abci.RequestPrepareProposal) abci.ResponsePr // Erasure encode the data square to create the extended data square (eds). // Note: uses the nmt wrapper to construct the tree. See // pkg/wrapper/nmt_wrapper.go for more information. - eds, err := da.ExtendShares(share.ToBytes(dataSquare)) + eds, err := da.ExtendShares(dataSquareBytes) if err != nil { app.Logger().Error( "failure to erasure the data square while creating a proposal block", @@ -84,7 +108,7 @@ func (app *App) PrepareProposal(req abci.RequestPrepareProposal) abci.ResponsePr return abci.ResponsePrepareProposal{ BlockData: &core.Data{ Txs: txs, - SquareSize: uint64(dataSquare.Size()), + SquareSize: size, Hash: dah.Hash(), // also known as the data root }, } diff --git a/app/process_proposal.go b/app/process_proposal.go index d1b367179c..fc10bb88e2 100644 --- a/app/process_proposal.go +++ b/app/process_proposal.go @@ -9,8 +9,10 @@ import ( "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" "github.com/celestiaorg/celestia-app/v3/pkg/da" blobtypes "github.com/celestiaorg/celestia-app/v3/x/blob/types" - "github.com/celestiaorg/go-square/v2" - "github.com/celestiaorg/go-square/v2/share" + shares "github.com/celestiaorg/go-square/shares" + square "github.com/celestiaorg/go-square/square" + squarev2 "github.com/celestiaorg/go-square/v2" + sharev2 "github.com/celestiaorg/go-square/v2/share" blobtx "github.com/celestiaorg/go-square/v2/tx" "github.com/cosmos/cosmos-sdk/telemetry" sdk "github.com/cosmos/cosmos-sdk/types" @@ -108,7 +110,7 @@ func (app *App) ProcessProposal(req abci.RequestProcessProposal) (resp abci.Resp // - that the sizes match // - that the namespaces match between blob and PFB // - that the share commitment is correct - if err := blobtypes.ValidateBlobTx(app.txConfig, blobTx, subtreeRootThreshold); err != nil { + if err := blobtypes.ValidateBlobTx(app.txConfig, blobTx, subtreeRootThreshold, app.AppVersion()); err != nil { logInvalidPropBlockError(app.Logger(), req.Header, fmt.Sprintf("invalid blob tx %d", idx), err) return reject() } @@ -122,24 +124,40 @@ func (app *App) ProcessProposal(req abci.RequestProcessProposal) (resp abci.Resp } - // Construct the data square from the block's transactions - dataSquare, err := square.Construct( - req.BlockData.Txs, - app.MaxEffectiveSquareSize(sdkCtx), - subtreeRootThreshold, + var ( + dataSquareBytes [][]byte + err error ) - if err != nil { - logInvalidPropBlockError(app.Logger(), req.Header, "failure to compute data square from transactions:", err) + + switch app.AppVersion() { + case v3: + var dataSquare squarev2.Square + dataSquare, err = squarev2.Construct(req.BlockData.Txs, app.MaxEffectiveSquareSize(sdkCtx), subtreeRootThreshold) + dataSquareBytes = sharev2.ToBytes(dataSquare) + // Assert that the square size stated by the proposer is correct + if uint64(dataSquare.Size()) != req.BlockData.SquareSize { + logInvalidPropBlock(app.Logger(), req.Header, "proposed square size differs from calculated square size") + return reject() + } + case v2, v1: + var dataSquare square.Square + dataSquare, err = square.Construct(req.BlockData.Txs, app.MaxEffectiveSquareSize(sdkCtx), subtreeRootThreshold) + dataSquareBytes = shares.ToBytes(dataSquare) + // Assert that the square size stated by the proposer is correct + if uint64(dataSquare.Size()) != req.BlockData.SquareSize { + logInvalidPropBlock(app.Logger(), req.Header, "proposed square size differs from calculated square size") + return reject() + } + default: + logInvalidPropBlock(app.Logger(), req.Header, "unsupported app version") return reject() } - - // Assert that the square size stated by the proposer is correct - if uint64(dataSquare.Size()) != req.BlockData.SquareSize { - logInvalidPropBlock(app.Logger(), req.Header, "proposed square size differs from calculated square size") + if err != nil { + logInvalidPropBlockError(app.Logger(), req.Header, "failure to compute data square from transactions:", err) return reject() } - eds, err := da.ExtendShares(share.ToBytes(dataSquare)) + eds, err := da.ExtendShares(dataSquareBytes) if err != nil { logInvalidPropBlockError(app.Logger(), req.Header, "failure to erasure the data square", err) return reject() diff --git a/app/test/std_sdk_test.go b/app/test/std_sdk_test.go index 75eb932b3f..cddb02c9a8 100644 --- a/app/test/std_sdk_test.go +++ b/app/test/std_sdk_test.go @@ -8,6 +8,7 @@ import ( "github.com/celestiaorg/celestia-app/v3/app" "github.com/celestiaorg/celestia-app/v3/app/encoding" "github.com/celestiaorg/celestia-app/v3/app/grpc/tx" + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" v2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" "github.com/celestiaorg/celestia-app/v3/pkg/user" "github.com/celestiaorg/celestia-app/v3/test/util/blobfactory" @@ -307,7 +308,7 @@ func (s *StandardSDKIntegrationTestSuite) TestStandardSDK() { name: "signal a version change", msgFunc: func() (msgs []sdk.Msg, signer string) { valAccount := s.getValidatorAccount() - msg := signal.NewMsgSignalVersion(valAccount, 2) + msg := signal.NewMsgSignalVersion(valAccount, appconsts.LatestVersion+1) return []sdk.Msg{msg}, s.getValidatorName() }, expectedCode: abci.CodeTypeOK, diff --git a/app/test/upgrade_test.go b/app/test/upgrade_test.go index 55de8fc486..3180de8c4e 100644 --- a/app/test/upgrade_test.go +++ b/app/test/upgrade_test.go @@ -1,21 +1,28 @@ package app_test import ( - "encoding/json" "fmt" "strings" "testing" - "time" app "github.com/celestiaorg/celestia-app/v3/app" "github.com/celestiaorg/celestia-app/v3/app/encoding" + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" v1 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v1" v2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" + v3 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v3" + "github.com/celestiaorg/celestia-app/v3/pkg/user" "github.com/celestiaorg/celestia-app/v3/test/util" + "github.com/celestiaorg/celestia-app/v3/test/util/genesis" + "github.com/celestiaorg/celestia-app/v3/test/util/testnode" blobstreamtypes "github.com/celestiaorg/celestia-app/v3/x/blobstream/types" "github.com/celestiaorg/celestia-app/v3/x/minfee" - "github.com/cosmos/cosmos-sdk/crypto/keyring" + "github.com/celestiaorg/celestia-app/v3/x/signal" + signaltypes "github.com/celestiaorg/celestia-app/v3/x/signal/types" + "github.com/celestiaorg/go-square/v2/share" + "github.com/celestiaorg/go-square/v2/tx" sdk "github.com/cosmos/cosmos-sdk/types" + authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" "github.com/cosmos/cosmos-sdk/x/params/types/proposal" packetforwardtypes "github.com/cosmos/ibc-apps/middleware/packet-forward-middleware/v6/packetforward/types" icahosttypes "github.com/cosmos/ibc-go/v6/modules/apps/27-interchain-accounts/host/types" @@ -27,9 +34,118 @@ import ( dbm "github.com/tendermint/tm-db" ) -// TestAppUpgrades verifies that the all module's params are overridden during an +func TestAppUpgradeV3(t *testing.T) { + if testing.Short() { + t.Skip("skipping TestAppUpgradeV3 in short mode") + } + testApp, genesis := SetupTestAppWithUpgradeHeight(t, 3) + upgradeFromV1ToV2(t, testApp) + + ctx := testApp.NewContext(true, tmproto.Header{}) + validators := testApp.StakingKeeper.GetAllValidators(ctx) + valAddr, err := sdk.ValAddressFromBech32(validators[0].OperatorAddress) + require.NoError(t, err) + record, err := genesis.Keyring().Key(testnode.DefaultValidatorAccountName) + require.NoError(t, err) + accAddr, err := record.GetAddress() + require.NoError(t, err) + encCfg := encoding.MakeConfig(app.ModuleEncodingRegisters...) + resp, err := testApp.AccountKeeper.Account(ctx, &authtypes.QueryAccountRequest{ + Address: accAddr.String(), + }) + require.NoError(t, err) + var account authtypes.AccountI + err = encCfg.InterfaceRegistry.UnpackAny(resp.Account, &account) + require.NoError(t, err) + + signer, err := user.NewSigner( + genesis.Keyring(), encCfg.TxConfig, testApp.GetChainID(), v3.Version, + user.NewAccount(testnode.DefaultValidatorAccountName, account.GetAccountNumber(), account.GetSequence()), + ) + require.NoError(t, err) + + upgradeTx, err := signer.CreateTx( + []sdk.Msg{ + signaltypes.NewMsgSignalVersion(valAddr, 3), + signaltypes.NewMsgTryUpgrade(accAddr), + }, + user.SetGasLimitAndGasPrice(100_000, appconsts.DefaultMinGasPrice), + ) + require.NoError(t, err) + testApp.BeginBlock(abci.RequestBeginBlock{ + Header: tmproto.Header{ + ChainID: genesis.ChainID, + Height: 3, + Version: tmversion.Consensus{App: 2}, + }, + }) + + deliverTxResp := testApp.DeliverTx(abci.RequestDeliverTx{ + Tx: upgradeTx, + }) + require.Equal(t, abci.CodeTypeOK, deliverTxResp.Code, deliverTxResp.Log) + + endBlockResp := testApp.EndBlock(abci.RequestEndBlock{ + Height: 3, + }) + require.Equal(t, v2.Version, endBlockResp.ConsensusParamUpdates.Version.AppVersion) + testApp.Commit() + require.NoError(t, signer.IncrementSequence(testnode.DefaultValidatorAccountName)) + + ctx = testApp.NewContext(true, tmproto.Header{}) + getUpgradeResp, err := testApp.SignalKeeper.GetUpgrade(ctx, &signaltypes.QueryGetUpgradeRequest{}) + require.NoError(t, err) + require.Equal(t, v3.Version, getUpgradeResp.Upgrade.AppVersion) + + // brace yourselfs, this part may take a while + initialHeight := int64(4) + for height := initialHeight; height < initialHeight+signal.DefaultUpgradeHeightDelay; height++ { + _ = testApp.BeginBlock(abci.RequestBeginBlock{ + Header: tmproto.Header{ + Height: height, + Version: tmversion.Consensus{App: 2}, + }, + }) + + endBlockResp = testApp.EndBlock(abci.RequestEndBlock{ + Height: 3 + signal.DefaultUpgradeHeightDelay, + }) + + _ = testApp.Commit() + } + require.Equal(t, v3.Version, endBlockResp.ConsensusParamUpdates.Version.AppVersion) + + // confirm that an authored blob tx works + blob, err := share.NewV1Blob(share.RandomBlobNamespace(), []byte("hello world"), accAddr.Bytes()) + require.NoError(t, err) + blobTxBytes, _, err := signer.CreatePayForBlobs( + testnode.DefaultValidatorAccountName, + []*share.Blob{blob}, + user.SetGasLimitAndGasPrice(200_000, appconsts.DefaultMinGasPrice), + ) + require.NoError(t, err) + blobTx, _, err := tx.UnmarshalBlobTx(blobTxBytes) + require.NoError(t, err) + + _ = testApp.BeginBlock(abci.RequestBeginBlock{ + Header: tmproto.Header{ + ChainID: genesis.ChainID, + Height: initialHeight + signal.DefaultUpgradeHeightDelay, + Version: tmversion.Consensus{App: 3}, + }, + }) + + deliverTxResp = testApp.DeliverTx(abci.RequestDeliverTx{ + Tx: blobTx.Tx, + }) + require.Equal(t, abci.CodeTypeOK, deliverTxResp.Code, deliverTxResp.Log) + + _ = testApp.EndBlock(abci.RequestEndBlock{}) +} + +// TestAppUpgradeV2 verifies that the all module's params are overridden during an // upgrade from v1 -> v2 and the app version changes correctly. -func TestAppUpgrades(t *testing.T) { +func TestAppUpgradeV2(t *testing.T) { NetworkMinGasPriceDec, err := sdk.NewDecFromStr(fmt.Sprintf("%f", v2.NetworkMinGasPrice)) require.NoError(t, err) @@ -122,19 +238,18 @@ func TestBlobstreamRemovedInV2(t *testing.T) { require.Error(t, err) } -func SetupTestAppWithUpgradeHeight(t *testing.T, upgradeHeight int64) (*app.App, keyring.Keyring) { +func SetupTestAppWithUpgradeHeight(t *testing.T, upgradeHeight int64) (*app.App, *genesis.Genesis) { t.Helper() db := dbm.NewMemDB() - chainID := "test_chain" encCfg := encoding.MakeConfig(app.ModuleEncodingRegisters...) testApp := app.New(log.NewNopLogger(), db, nil, 0, encCfg, upgradeHeight, util.EmptyAppOptions{}) - genesisState, _, kr := util.GenesisStateWithSingleValidator(testApp, "account") - stateBytes, err := json.MarshalIndent(genesisState, "", " ") + genesis := genesis.NewDefaultGenesis(). + WithValidators(genesis.NewDefaultValidator(testnode.DefaultValidatorAccountName)). + WithConsensusParams(app.DefaultInitialConsensusParams()) + genDoc, err := genesis.Export() require.NoError(t, err) - infoResp := testApp.Info(abci.RequestInfo{}) - require.EqualValues(t, 0, infoResp.AppVersion) - cp := app.DefaultInitialConsensusParams() + cp := genDoc.ConsensusParams abciParams := &abci.ConsensusParams{ Block: &abci.BlockParams{ MaxBytes: cp.Block.MaxBytes, @@ -147,23 +262,23 @@ func SetupTestAppWithUpgradeHeight(t *testing.T, upgradeHeight int64) (*app.App, _ = testApp.InitChain( abci.RequestInitChain{ - Time: time.Now(), + Time: genDoc.GenesisTime, Validators: []abci.ValidatorUpdate{}, ConsensusParams: abciParams, - AppStateBytes: stateBytes, - ChainId: chainID, + AppStateBytes: genDoc.AppState, + ChainId: genDoc.ChainID, }, ) // assert that the chain starts with version provided in genesis - infoResp = testApp.Info(abci.RequestInfo{}) + infoResp := testApp.Info(abci.RequestInfo{}) require.EqualValues(t, app.DefaultInitialConsensusParams().Version.AppVersion, infoResp.AppVersion) - supportedVersions := []uint64{v1.Version, v2.Version} + supportedVersions := []uint64{v1.Version, v2.Version, v3.Version} require.Equal(t, supportedVersions, testApp.SupportedVersions()) _ = testApp.Commit() - return testApp, kr + return testApp, genesis } func upgradeFromV1ToV2(t *testing.T, testApp *app.App) { diff --git a/go.mod b/go.mod index ec2d06001b..bf9aaa9178 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,8 @@ require ( cosmossdk.io/errors v1.0.1 cosmossdk.io/math v1.3.0 github.com/celestiaorg/blobstream-contracts/v3 v3.1.0 - github.com/celestiaorg/go-square/v2 v2.0.0-rc2 + github.com/celestiaorg/go-square v1.0.0 + github.com/celestiaorg/go-square/v2 v2.0.0 github.com/celestiaorg/knuu v0.15.2 github.com/celestiaorg/nmt v0.22.1 github.com/celestiaorg/rsmt2d v0.14.0 diff --git a/go.sum b/go.sum index c98a7c818e..b0c0f27e3d 100644 --- a/go.sum +++ b/go.sum @@ -321,8 +321,10 @@ github.com/celestiaorg/celestia-core v1.41.0-tm-v0.34.29 h1:hRdTxe+Dz6kiqifRZCC9 github.com/celestiaorg/celestia-core v1.41.0-tm-v0.34.29/go.mod h1:H6vjzdoqTt4qmbf11z1Lnc9YLUp/B8ITEQLhU92ghqQ= github.com/celestiaorg/cosmos-sdk v1.24.1-sdk-v0.46.16 h1:SeQ7Y/CyOcUMKo7mQiexaj/pZ/xIgyuZFIwYZwpSkWE= github.com/celestiaorg/cosmos-sdk v1.24.1-sdk-v0.46.16/go.mod h1:Bpl1LSWiDpQumgOhhMTZBMopqa0j7fRasIhvTZB44P0= -github.com/celestiaorg/go-square/v2 v2.0.0-rc2 h1:4D+ASgZGYVCsffc2uhPagACrvNiLZu9/CqNYvnlHCgg= -github.com/celestiaorg/go-square/v2 v2.0.0-rc2/go.mod h1:eeaU8f8jBpk3ZS/gaDZIlTesJR2F51QAmveNzWH6aEU= +github.com/celestiaorg/go-square v1.0.0 h1:pb1leaUi2WWSpri6ubNJOCUWdAoPr6AYGlT19F/Y/4k= +github.com/celestiaorg/go-square v1.0.0/go.mod h1:XMv5SGCeGSkynW2OOsedugaW/rQlvzxGzWGxTKsyYOU= +github.com/celestiaorg/go-square/v2 v2.0.0 h1:U5QV8/de5lc7glosfgyHhcxbFwNuwU4+6aYZ2RgjM04= +github.com/celestiaorg/go-square/v2 v2.0.0/go.mod h1:y0BolG0tRM7UN1sAQyDDUkT+aMJPwFIjviVvnCB62C0= github.com/celestiaorg/knuu v0.15.2 h1:l6MrFTfDfrNtbvOLSjD/YzRpaaBFIIvlWIYXRNNL1/s= github.com/celestiaorg/knuu v0.15.2/go.mod h1:hTKu4uGH/Dx7eaYgT26coKx/vbALEQdLdYOwNxkAIG0= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= diff --git a/pkg/appconsts/v3/app_consts.go b/pkg/appconsts/v3/app_consts.go new file mode 100644 index 0000000000..504360864a --- /dev/null +++ b/pkg/appconsts/v3/app_consts.go @@ -0,0 +1,7 @@ +package v3 + +const ( + Version uint64 = 3 + SquareSizeUpperBound int = 128 + SubtreeRootThreshold int = 64 +) diff --git a/pkg/appconsts/versioned_consts.go b/pkg/appconsts/versioned_consts.go index 67c3c8a8f2..a8d01fdedf 100644 --- a/pkg/appconsts/versioned_consts.go +++ b/pkg/appconsts/versioned_consts.go @@ -2,11 +2,11 @@ package appconsts import ( v1 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v1" - v2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" + v3 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v3" ) const ( - LatestVersion = v2.Version + LatestVersion = v3.Version ) // SubtreeRootThreshold works as a target upper bound for the number of subtree diff --git a/x/blob/types/blob_tx.go b/x/blob/types/blob_tx.go index 1f6273f10c..8089e247ef 100644 --- a/x/blob/types/blob_tx.go +++ b/x/blob/types/blob_tx.go @@ -3,6 +3,7 @@ package types import ( "bytes" + v3 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v3" "github.com/celestiaorg/go-square/v2/inclusion" "github.com/celestiaorg/go-square/v2/share" "github.com/celestiaorg/go-square/v2/tx" @@ -35,7 +36,7 @@ func NewV1Blob(ns share.Namespace, data []byte, signer sdk.AccAddress) (*share.B // ValidateBlobTx performs stateless checks on the BlobTx to ensure that the // blobs attached to the transaction are valid. -func ValidateBlobTx(txcfg client.TxEncodingConfig, bTx *tx.BlobTx, subtreeRootThreshold int) error { +func ValidateBlobTx(txcfg client.TxEncodingConfig, bTx *tx.BlobTx, subtreeRootThreshold int, appVersion uint64) error { if bTx == nil { return ErrNoBlobs } @@ -79,6 +80,9 @@ func ValidateBlobTx(txcfg client.TxEncodingConfig, bTx *tx.BlobTx, subtreeRootTh // If share version is 1, assert that the signer in the blob // matches the signer in the msgPFB. if blob.ShareVersion() == share.ShareVersionOne { + if appVersion < v3.Version { + return ErrUnsupportedShareVersion.Wrapf("share version %d is not supported in %d. Supported from v3 onwards", blob.ShareVersion(), appVersion) + } if !bytes.Equal(blob.Signer(), signer) { return ErrInvalidBlobSigner.Wrapf("blob signer %s does not match msgPFB signer %s", sdk.AccAddress(blob.Signer()).String(), msgPFB.Signer) } diff --git a/x/blob/types/blob_tx_test.go b/x/blob/types/blob_tx_test.go index 1a2d8295dc..a1867e2d92 100644 --- a/x/blob/types/blob_tx_test.go +++ b/x/blob/types/blob_tx_test.go @@ -251,7 +251,7 @@ func TestValidateBlobTx(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := types.ValidateBlobTx(encCfg.TxConfig, tt.getTx(), appconsts.DefaultSubtreeRootThreshold) + err := types.ValidateBlobTx(encCfg.TxConfig, tt.getTx(), appconsts.DefaultSubtreeRootThreshold, appconsts.LatestVersion) if tt.expectedErr != nil { assert.ErrorIs(t, err, tt.expectedErr, tt.name) } diff --git a/x/signal/integration_test.go b/x/signal/integration_test.go index 434b9cf882..fb016e0813 100644 --- a/x/signal/integration_test.go +++ b/x/signal/integration_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/celestiaorg/celestia-app/v3/app" + v2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" testutil "github.com/celestiaorg/celestia-app/v3/test/util" "github.com/celestiaorg/celestia-app/v3/x/signal" "github.com/celestiaorg/celestia-app/v3/x/signal/types" @@ -19,17 +20,19 @@ import ( // simulates an upgrade scenario with a single validator which signals for the version change, checks the quorum // has been reached and then calls TryUpgrade, asserting that the upgrade module returns the new app version func TestUpgradeIntegration(t *testing.T) { - app, _ := testutil.SetupTestAppWithGenesisValSet(app.DefaultConsensusParams()) + cp := app.DefaultConsensusParams() + cp.Version.AppVersion = v2.Version + app, _ := testutil.SetupTestAppWithGenesisValSet(cp) ctx := sdk.NewContext(app.CommitMultiStore(), tmtypes.Header{ Version: tmversion.Consensus{ - App: 1, + App: v2.Version, }, }, false, tmlog.NewNopLogger()) goCtx := sdk.WrapSDKContext(ctx) ctx = sdk.UnwrapSDKContext(goCtx) res, err := app.SignalKeeper.VersionTally(goCtx, &types.QueryVersionTallyRequest{ - Version: 2, + Version: 3, }) require.NoError(t, err) require.EqualValues(t, 0, res.VotingPower) @@ -40,12 +43,12 @@ func TestUpgradeIntegration(t *testing.T) { _, err = app.SignalKeeper.SignalVersion(ctx, &types.MsgSignalVersion{ ValidatorAddress: valAddr.String(), - Version: 2, + Version: 3, }) require.NoError(t, err) res, err = app.SignalKeeper.VersionTally(goCtx, &types.QueryVersionTallyRequest{ - Version: 2, + Version: 3, }) require.NoError(t, err) require.EqualValues(t, 1, res.VotingPower) @@ -65,7 +68,7 @@ func TestUpgradeIntegration(t *testing.T) { // returns an error because an upgrade is pending. _, err = app.SignalKeeper.SignalVersion(ctx, &types.MsgSignalVersion{ ValidatorAddress: valAddr.String(), - Version: 3, + Version: 4, }) require.Error(t, err) require.ErrorIs(t, err, types.ErrUpgradePending) @@ -78,5 +81,5 @@ func TestUpgradeIntegration(t *testing.T) { shouldUpgrade, version = app.SignalKeeper.ShouldUpgrade(ctx) require.True(t, shouldUpgrade) - require.EqualValues(t, 2, version) + require.EqualValues(t, 3, version) } From 6137f2dd120b3a96fe633ba811cb0b1fc8146644 Mon Sep 17 00:00:00 2001 From: Darryl PAU Date: Wed, 18 Sep 2024 13:31:24 -0400 Subject: [PATCH 32/85] docs: fix broken link (#3887) https://github.com/celestiaorg/celestia-app/issues/3885 All fixed. :=) --- pkg/wrapper/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/wrapper/README.md b/pkg/wrapper/README.md index e58188bd7a..cacbd55dda 100644 --- a/pkg/wrapper/README.md +++ b/pkg/wrapper/README.md @@ -76,9 +76,9 @@ One namespace ID is located in the first `NamespaceIDSize` bytes, while the othe ## References - Namespaced Merkle tree specifications: -- Celestia original data square specification: -- Celestia constants: -- Celestia reserved namespace IDs: +- Celestia original data square specification: +- Celestia constants: +- Celestia reserved namespace IDs: [nmtlink]: https://github.com/celestiaorg/nmt/blob/master/docs/spec/nmt.md [nmtwrapper-link]: https://github.com/celestiaorg/celestia-app/blob/main/pkg/wrapper/nmt_wrapper.go From 0f1a99be5f9a8f47015677949adfbebe4102f1eb Mon Sep 17 00:00:00 2001 From: Darryl PAU Date: Fri, 20 Sep 2024 04:54:20 -0400 Subject: [PATCH 33/85] docs: fix upgrade doc link (#3888) fix https://github.com/celestiaorg/celestia-app/pull/3887?notification_referrer_id=NT_kwDOClGbmbUxMjQ2MzY5MTMwMToxNzMxMjA0MDk#pullrequestreview-2313343476 but Im not sure If I found the right link here cc @rach-id --- docs/release-notes/release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/release-notes/release-notes.md b/docs/release-notes/release-notes.md index 5cc49d1010..c5031566e5 100644 --- a/docs/release-notes/release-notes.md +++ b/docs/release-notes/release-notes.md @@ -8,7 +8,7 @@ This guide provides notes for major version releases. These notes may be helpful If you are a consensus node operator, please follow the communication channels listed under [network upgrades](https://docs.celestia.org/nodes/participate#network-upgrades) to learn when this release is recommended for each network (e.g. Mocha, Mainnet Beta). -Consensus node operators are expected to upgrade to this release _prior_ to the Lemongrass hardfork if they intend to continue participating in the network. The command used to start the [consensus node](https://docs.celestia.org/nodes/consensus-node#start-the-consensus-node) or [validator node](https://docs.celestia.org/nodes/validator-node#run-the-validator-node) will accept an additional `--v2-upgrade-height` flag. See [this table](https://docs.celestia.org/nodes/hardfork-process#lemongrass-hardfork) for upgrade heights for each network. +Consensus node operators are expected to upgrade to this release _prior_ to the Lemongrass hardfork if they intend to continue participating in the network. The command used to start the [consensus node](https://docs.celestia.org/nodes/consensus-node#start-the-consensus-node) or [validator node](https://docs.celestia.org/nodes/validator-node#run-the-validator-node) will accept an additional `--v2-upgrade-height` flag. See [this table](https://docs.celestia.org/nodes/network-upgrade-process#lemongrass-network-upgrade) for upgrade heights for each network. Consensus node operators should enable the BBR (Bottleneck Bandwidth and Round-trip propagation time) congestion control algorithm. See [#3812](https://github.com/celestiaorg/celestia-app/pull/3812). From 857d107a79f989b35ebbf9365979743e8e155554 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nina=20/=20=E1=83=9C=E1=83=98=E1=83=9C=E1=83=90?= Date: Fri, 20 Sep 2024 16:06:38 +0200 Subject: [PATCH 34/85] refactor!: version gas scheduler variables (#3735) ## Overview - Fixes #3404 - Fixes #3748 - Fixes #3762 --------- Co-authored-by: Rootul P --- app/ante/ante.go | 2 +- app/ante/min_fee_test.go | 11 +- app/ante/tx_size.go | 151 ++++++++++++++++++ app/ante/tx_size_test.go | 197 ++++++++++++++++++++++++ app/test/std_sdk_test.go | 3 +- app/test/upgrade_test.go | 2 +- pkg/appconsts/initial_consts.go | 9 +- pkg/appconsts/v2/app_consts.go | 3 - pkg/appconsts/v3/app_consts.go | 2 + pkg/appconsts/versioned_consts.go | 15 +- pkg/appconsts/versioned_consts_test.go | 80 ++++++---- specs/src/parameters_v3.md | 72 +++++++++ x/blob/README.md | 1 + x/blob/ante/ante.go | 10 +- x/blob/ante/ante_test.go | 80 +++++++--- x/blob/keeper/gas_test.go | 15 +- x/blob/keeper/genesis_test.go | 3 +- x/blob/keeper/grpc_query_params_test.go | 3 +- x/blob/keeper/keeper.go | 11 +- x/blob/keeper/keeper_test.go | 6 +- x/blob/keeper/params_test.go | 3 +- x/blob/types/payforblob.go | 6 +- x/minfee/grpc_query_test.go | 4 +- x/minfee/params.go | 4 +- 24 files changed, 596 insertions(+), 97 deletions(-) create mode 100644 app/ante/tx_size.go create mode 100644 app/ante/tx_size_test.go create mode 100644 specs/src/parameters_v3.md diff --git a/app/ante/ante.go b/app/ante/ante.go index dd65fdd120..89b6c1d050 100644 --- a/app/ante/ante.go +++ b/app/ante/ante.go @@ -42,7 +42,7 @@ func NewAnteHandler( ante.NewValidateMemoDecorator(accountKeeper), // Ensure the tx's gas limit is > the gas consumed based on the tx size. // Side effect: consumes gas from the gas meter. - ante.NewConsumeGasForTxSizeDecorator(accountKeeper), + NewConsumeGasForTxSizeDecorator(accountKeeper), // Ensure the feepayer (fee granter or first signer) has enough funds to pay for the tx. // Ensure the gas price >= network min gas price if app version >= 2. // Side effect: deducts fees from the fee payer. Sets the tx priority in context. diff --git a/app/ante/min_fee_test.go b/app/ante/min_fee_test.go index 30912813c4..fc66e11a67 100644 --- a/app/ante/min_fee_test.go +++ b/app/ante/min_fee_test.go @@ -9,7 +9,6 @@ import ( "github.com/celestiaorg/celestia-app/v3/app/ante" "github.com/celestiaorg/celestia-app/v3/app/encoding" "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" - v2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" "github.com/celestiaorg/celestia-app/v3/test/util/testnode" "github.com/celestiaorg/celestia-app/v3/x/minfee" "github.com/cosmos/cosmos-sdk/codec" @@ -58,7 +57,7 @@ func TestValidateTxFee(t *testing.T) { { name: "bad tx; fee below required minimum", fee: sdk.NewCoins(sdk.NewInt64Coin(appconsts.BondDenom, feeAmount-1)), - gasLimit: uint64(float64(feeAmount) / v2.NetworkMinGasPrice), + gasLimit: uint64(float64(feeAmount) / appconsts.DefaultNetworkMinGasPrice), appVersion: uint64(2), isCheckTx: false, expErr: true, @@ -66,7 +65,7 @@ func TestValidateTxFee(t *testing.T) { { name: "good tx; fee equal to required minimum", fee: sdk.NewCoins(sdk.NewInt64Coin(appconsts.BondDenom, feeAmount)), - gasLimit: uint64(float64(feeAmount) / v2.NetworkMinGasPrice), + gasLimit: uint64(float64(feeAmount) / appconsts.DefaultNetworkMinGasPrice), appVersion: uint64(2), isCheckTx: false, expErr: false, @@ -74,7 +73,7 @@ func TestValidateTxFee(t *testing.T) { { name: "good tx; fee above required minimum", fee: sdk.NewCoins(sdk.NewInt64Coin(appconsts.BondDenom, feeAmount+1)), - gasLimit: uint64(float64(feeAmount) / v2.NetworkMinGasPrice), + gasLimit: uint64(float64(feeAmount) / appconsts.DefaultNetworkMinGasPrice), appVersion: uint64(2), isCheckTx: false, expErr: false, @@ -82,7 +81,7 @@ func TestValidateTxFee(t *testing.T) { { name: "good tx; with no fee (v1)", fee: sdk.NewCoins(sdk.NewInt64Coin(appconsts.BondDenom, feeAmount)), - gasLimit: uint64(float64(feeAmount) / v2.NetworkMinGasPrice), + gasLimit: uint64(float64(feeAmount) / appconsts.DefaultNetworkMinGasPrice), appVersion: uint64(1), isCheckTx: false, expErr: false, @@ -143,7 +142,7 @@ func TestValidateTxFee(t *testing.T) { ctx = ctx.WithMinGasPrices(sdk.DecCoins{validatorMinGasPriceCoin}) - networkMinGasPriceDec, err := sdk.NewDecFromStr(fmt.Sprintf("%f", v2.NetworkMinGasPrice)) + networkMinGasPriceDec, err := sdk.NewDecFromStr(fmt.Sprintf("%f", appconsts.DefaultNetworkMinGasPrice)) require.NoError(t, err) subspace, _ := paramsKeeper.GetSubspace(minfee.ModuleName) diff --git a/app/ante/tx_size.go b/app/ante/tx_size.go new file mode 100644 index 0000000000..356f6f2646 --- /dev/null +++ b/app/ante/tx_size.go @@ -0,0 +1,151 @@ +package ante + +import ( + "encoding/hex" + + "cosmossdk.io/errors" + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" + v2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" + "github.com/cosmos/cosmos-sdk/codec/legacy" + "github.com/cosmos/cosmos-sdk/crypto/keys/multisig" + "github.com/cosmos/cosmos-sdk/crypto/keys/secp256k1" + cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types" + sdk "github.com/cosmos/cosmos-sdk/types" + sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" + "github.com/cosmos/cosmos-sdk/types/tx/signing" + "github.com/cosmos/cosmos-sdk/x/auth/ante" + "github.com/cosmos/cosmos-sdk/x/auth/migrations/legacytx" + authsigning "github.com/cosmos/cosmos-sdk/x/auth/signing" + auth "github.com/cosmos/cosmos-sdk/x/auth/types" +) + +var ( + // Simulation signature values used to estimate gas consumption. + key = make([]byte, secp256k1.PubKeySize) + simSecp256k1Pubkey = &secp256k1.PubKey{Key: key} + simSecp256k1Sig [64]byte +) + +func init() { + // Decodes a valid hex string into a sepc256k1Pubkey for use in transaction simulation + bz, _ := hex.DecodeString("035AD6810A47F073553FF30D2FCC7E0D3B1C0B74B61A1AAA2582344037151E143A") + copy(key, bz) + simSecp256k1Pubkey.Key = key +} + +// ConsumeTxSizeGasDecorator will take in parameters and consume gas proportional +// to the size of tx before calling next AnteHandler. Note, the gas costs will be +// slightly over estimated due to the fact that any given signing account may need +// to be retrieved from state. +// +// CONTRACT: If simulate=true, then signatures must either be completely filled +// in or empty. +// CONTRACT: To use this decorator, signatures of transaction must be represented +// as legacytx.StdSignature otherwise simulate mode will incorrectly estimate gas cost. + +// The code was copied from celestia's fork of the cosmos-sdk: +// https://github.com/celestiaorg/cosmos-sdk/blob/release/v0.46.x-celestia/x/auth/ante/basic.go +// In app versions v2 and below, the txSizeCostPerByte used for gas cost estimation is taken from the auth module. +// In app v3 and above, the versioned constant appconsts.TxSizeCostPerByte is used. +type ConsumeTxSizeGasDecorator struct { + ak ante.AccountKeeper +} + +func NewConsumeGasForTxSizeDecorator(ak ante.AccountKeeper) ConsumeTxSizeGasDecorator { + return ConsumeTxSizeGasDecorator{ + ak: ak, + } +} + +func (cgts ConsumeTxSizeGasDecorator) AnteHandle(ctx sdk.Context, tx sdk.Tx, simulate bool, next sdk.AnteHandler) (sdk.Context, error) { + sigTx, ok := tx.(authsigning.SigVerifiableTx) + if !ok { + return ctx, errors.Wrap(sdkerrors.ErrTxDecode, "invalid tx type") + } + params := cgts.ak.GetParams(ctx) + + consumeGasForTxSize(ctx, sdk.Gas(len(ctx.TxBytes())), params) + + // simulate gas cost for signatures in simulate mode + if simulate { + // in simulate mode, each element should be a nil signature + sigs, err := sigTx.GetSignaturesV2() + if err != nil { + return ctx, err + } + n := len(sigs) + + for i, signer := range sigTx.GetSigners() { + // if signature is already filled in, no need to simulate gas cost + if i < n && !isIncompleteSignature(sigs[i].Data) { + continue + } + + var pubkey cryptotypes.PubKey + + acc := cgts.ak.GetAccount(ctx, signer) + + // use placeholder simSecp256k1Pubkey if sig is nil + if acc == nil || acc.GetPubKey() == nil { + pubkey = simSecp256k1Pubkey + } else { + pubkey = acc.GetPubKey() + } + + // use stdsignature to mock the size of a full signature + simSig := legacytx.StdSignature{ //nolint:staticcheck // this will be removed when proto is ready + Signature: simSecp256k1Sig[:], + PubKey: pubkey, + } + + sigBz := legacy.Cdc.MustMarshal(simSig) + txBytes := sdk.Gas(len(sigBz) + 6) + + // If the pubkey is a multi-signature pubkey, then we estimate for the maximum + // number of signers. + if _, ok := pubkey.(*multisig.LegacyAminoPubKey); ok { + txBytes *= params.TxSigLimit + } + + consumeGasForTxSize(ctx, txBytes, params) + } + } + + return next(ctx, tx, simulate) +} + +// isIncompleteSignature tests whether SignatureData is fully filled in for simulation purposes +func isIncompleteSignature(data signing.SignatureData) bool { + if data == nil { + return true + } + + switch data := data.(type) { + case *signing.SingleSignatureData: + return len(data.Signature) == 0 + case *signing.MultiSignatureData: + if len(data.Signatures) == 0 { + return true + } + for _, s := range data.Signatures { + if isIncompleteSignature(s) { + return true + } + } + } + + return false +} + +// consumeGasForTxSize consumes gas based on the size of the transaction. +// It uses different parameters depending on the app version. +func consumeGasForTxSize(ctx sdk.Context, txBytes uint64, params auth.Params) { + // For app v2 and below we should get txSizeCostPerByte from auth module + if ctx.BlockHeader().Version.App <= v2.Version { + ctx.GasMeter().ConsumeGas(params.TxSizeCostPerByte*txBytes, "txSize") + } else { + // From v3 onwards, we should get txSizeCostPerByte from appconsts + txSizeCostPerByte := appconsts.TxSizeCostPerByte(ctx.BlockHeader().Version.App) + ctx.GasMeter().ConsumeGas(txSizeCostPerByte*txBytes, "txSize") + } +} diff --git a/app/ante/tx_size_test.go b/app/ante/tx_size_test.go new file mode 100644 index 0000000000..6b6bdbd80b --- /dev/null +++ b/app/ante/tx_size_test.go @@ -0,0 +1,197 @@ +package ante_test + +import ( + "fmt" + "strings" + "testing" + + "github.com/celestiaorg/celestia-app/v3/app" + "github.com/celestiaorg/celestia-app/v3/app/ante" + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" + v2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" + testutil "github.com/celestiaorg/celestia-app/v3/test/util" + "github.com/cosmos/cosmos-sdk/client" + "github.com/cosmos/cosmos-sdk/client/tx" + cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types" + "github.com/cosmos/cosmos-sdk/crypto/types/multisig" + "github.com/cosmos/cosmos-sdk/simapp" + "github.com/cosmos/cosmos-sdk/testutil/testdata" + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/cosmos/cosmos-sdk/types/tx/signing" + xauthsigning "github.com/cosmos/cosmos-sdk/x/auth/signing" + authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" + "github.com/stretchr/testify/require" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" + "github.com/tendermint/tendermint/proto/tendermint/version" +) + +const TxSizeCostPerByte = 8 + +func setup() (*app.App, sdk.Context, client.Context, error) { + app, _, _ := testutil.NewTestAppWithGenesisSet(app.DefaultConsensusParams()) + ctx := app.NewContext(false, tmproto.Header{}) + params := authtypes.DefaultParams() + // Override default with a different TxSizeCostPerByte value for testing + params.TxSizeCostPerByte = TxSizeCostPerByte + app.AccountKeeper.SetParams(ctx, params) + ctx = ctx.WithBlockHeight(1) + + // Set up TxConfig. + encodingConfig := simapp.MakeTestEncodingConfig() + // We're using TestMsg encoding in the test, so register it here. + encodingConfig.Amino.RegisterConcrete(&testdata.TestMsg{}, "testdata.TestMsg", nil) + testdata.RegisterInterfaces(encodingConfig.InterfaceRegistry) + + clientCtx := client.Context{}. + WithTxConfig(encodingConfig.TxConfig) + + return app, ctx, clientCtx, nil +} + +func TestConsumeGasForTxSize(t *testing.T) { + app, ctx, clientCtx, err := setup() + require.NoError(t, err) + var txBuilder client.TxBuilder + + // keys and addresses + priv1, _, addr1 := testdata.KeyTestPubAddr() + + // msg and signatures + msg := testdata.NewTestMsg(addr1) + feeAmount := testdata.NewTestFeeAmount() + gasLimit := testdata.NewTestGasLimit() + + cgtsd := ante.NewConsumeGasForTxSizeDecorator(app.AccountKeeper) + antehandler := sdk.ChainAnteDecorators(cgtsd) + + testCases := []struct { + version uint64 + name string + sigV2 signing.SignatureV2 + }{ + {v2.Version, "SingleSignatureData v2", signing.SignatureV2{PubKey: priv1.PubKey()}}, + {v2.Version, "MultiSignatureData v2", signing.SignatureV2{PubKey: priv1.PubKey(), Data: multisig.NewMultisig(2)}}, + {appconsts.LatestVersion, fmt.Sprintf("SingleSignatureData v%d", appconsts.LatestVersion), signing.SignatureV2{PubKey: priv1.PubKey()}}, + {appconsts.LatestVersion, fmt.Sprintf("MultiSignatureData v%d", appconsts.LatestVersion), signing.SignatureV2{PubKey: priv1.PubKey(), Data: multisig.NewMultisig(2)}}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // set the version + ctx = app.NewContext(false, tmproto.Header{Version: version.Consensus{ + App: tc.version, + }}) + + txBuilder = clientCtx.TxConfig.NewTxBuilder() + require.NoError(t, txBuilder.SetMsgs(msg)) + txBuilder.SetFeeAmount(feeAmount) + txBuilder.SetGasLimit(gasLimit) + txBuilder.SetMemo(strings.Repeat("01234567890", 10)) + + privs, accNums, accSeqs := []cryptotypes.PrivKey{priv1}, []uint64{0}, []uint64{0} + tx, err := createTestTx(txBuilder, clientCtx, privs, accNums, accSeqs, ctx.ChainID()) + require.NoError(t, err) + + txBytes, err := clientCtx.TxConfig.TxJSONEncoder()(tx) + require.Nil(t, err, "Cannot marshal tx: %v", err) + + // expected TxSizeCostPerByte is different for each version + var txSizeCostPerByte uint64 + if tc.version == v2.Version { + txSizeCostPerByte = TxSizeCostPerByte + } else { + txSizeCostPerByte = appconsts.TxSizeCostPerByte(tc.version) + } + + expectedGas := sdk.Gas(len(txBytes)) * txSizeCostPerByte + + // set suite.ctx with TxBytes manually + ctx = ctx.WithTxBytes(txBytes) + + // track how much gas is necessary to retrieve parameters + beforeGas := ctx.GasMeter().GasConsumed() + app.AccountKeeper.GetParams(ctx) + afterGas := ctx.GasMeter().GasConsumed() + expectedGas += afterGas - beforeGas + + beforeGas = ctx.GasMeter().GasConsumed() + ctx, err = antehandler(ctx, tx, false) + require.Nil(t, err, "ConsumeTxSizeGasDecorator returned error: %v", err) + + // require that decorator consumes expected amount of gas + consumedGas := ctx.GasMeter().GasConsumed() - beforeGas + require.Equal(t, expectedGas, consumedGas, "Decorator did not consume the correct amount of gas") + + // simulation must not underestimate gas of this decorator even with nil signatures + txBuilder, err := clientCtx.TxConfig.WrapTxBuilder(tx) + require.NoError(t, err) + require.NoError(t, txBuilder.SetSignatures(tc.sigV2)) + tx = txBuilder.GetTx() + + simTxBytes, err := clientCtx.TxConfig.TxJSONEncoder()(tx) + require.Nil(t, err, "Cannot marshal tx: %v", err) + // require that simulated tx is smaller than tx with signatures + require.True(t, len(simTxBytes) < len(txBytes), "simulated tx still has signatures") + + // Set suite.ctx with smaller simulated TxBytes manually + ctx = ctx.WithTxBytes(simTxBytes) + + beforeSimGas := ctx.GasMeter().GasConsumed() + + // run antehandler with simulate=true + ctx, err = antehandler(ctx, tx, true) + consumedSimGas := ctx.GasMeter().GasConsumed() - beforeSimGas + + // require that antehandler passes and does not underestimate decorator cost + require.Nil(t, err, "ConsumeTxSizeGasDecorator returned error: %v", err) + require.True(t, consumedSimGas >= expectedGas, "Simulate mode underestimates gas on AnteDecorator. Simulated cost: %d, expected cost: %d", consumedSimGas, expectedGas) + }) + } +} + +// createTestTx creates a test tx given multiple inputs. +func createTestTx(txBuilder client.TxBuilder, clientCtx client.Context, privs []cryptotypes.PrivKey, accNums []uint64, accSeqs []uint64, chainID string) (xauthsigning.Tx, error) { + // First round: we gather all the signer infos. We use the "set empty + // signature" hack to do that. + sigsV2 := make([]signing.SignatureV2, 0, len(privs)) + for i, priv := range privs { + sigV2 := signing.SignatureV2{ + PubKey: priv.PubKey(), + Data: &signing.SingleSignatureData{ + SignMode: clientCtx.TxConfig.SignModeHandler().DefaultMode(), + Signature: nil, + }, + Sequence: accSeqs[i], + } + + sigsV2 = append(sigsV2, sigV2) + } + err := txBuilder.SetSignatures(sigsV2...) + if err != nil { + return nil, err + } + + // Second round: all signer infos are set, so each signer can sign. + sigsV2 = []signing.SignatureV2{} + for i, priv := range privs { + signerData := xauthsigning.SignerData{ + ChainID: chainID, + AccountNumber: accNums[i], + Sequence: accSeqs[i], + } + sigV2, err := tx.SignWithPrivKey( + clientCtx.TxConfig.SignModeHandler().DefaultMode(), signerData, + txBuilder, priv, clientCtx.TxConfig, accSeqs[i]) + if err != nil { + return nil, err + } + + sigsV2 = append(sigsV2, sigV2) + } + err = txBuilder.SetSignatures(sigsV2...) + if err != nil { + return nil, err + } + + return txBuilder.GetTx(), nil +} diff --git a/app/test/std_sdk_test.go b/app/test/std_sdk_test.go index cddb02c9a8..dc54a030d0 100644 --- a/app/test/std_sdk_test.go +++ b/app/test/std_sdk_test.go @@ -9,7 +9,6 @@ import ( "github.com/celestiaorg/celestia-app/v3/app/encoding" "github.com/celestiaorg/celestia-app/v3/app/grpc/tx" "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" - v2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" "github.com/celestiaorg/celestia-app/v3/pkg/user" "github.com/celestiaorg/celestia-app/v3/test/util/blobfactory" "github.com/celestiaorg/celestia-app/v3/test/util/testfactory" @@ -350,7 +349,7 @@ func (s *StandardSDKIntegrationTestSuite) TestGRPCQueries() { require.NoError(t, err) got, err := resp.NetworkMinGasPrice.Float64() require.NoError(t, err) - assert.Equal(t, v2.NetworkMinGasPrice, got) + assert.Equal(t, appconsts.DefaultNetworkMinGasPrice, got) }) t.Run("testnode can query local min gas price", func(t *testing.T) { serviceClient := nodeservice.NewServiceClient(s.cctx.GRPCClient) diff --git a/app/test/upgrade_test.go b/app/test/upgrade_test.go index 3180de8c4e..459a33aefc 100644 --- a/app/test/upgrade_test.go +++ b/app/test/upgrade_test.go @@ -146,7 +146,7 @@ func TestAppUpgradeV3(t *testing.T) { // TestAppUpgradeV2 verifies that the all module's params are overridden during an // upgrade from v1 -> v2 and the app version changes correctly. func TestAppUpgradeV2(t *testing.T) { - NetworkMinGasPriceDec, err := sdk.NewDecFromStr(fmt.Sprintf("%f", v2.NetworkMinGasPrice)) + NetworkMinGasPriceDec, err := sdk.NewDecFromStr(fmt.Sprintf("%f", appconsts.DefaultNetworkMinGasPrice)) require.NoError(t, err) tests := []struct { diff --git a/pkg/appconsts/initial_consts.go b/pkg/appconsts/initial_consts.go index 1f97adf951..18cafc969d 100644 --- a/pkg/appconsts/initial_consts.go +++ b/pkg/appconsts/initial_consts.go @@ -17,10 +17,6 @@ const ( // maximum number of bytes allowed in a valid block. DefaultMaxBytes = DefaultGovMaxSquareSize * DefaultGovMaxSquareSize * share.ContinuationSparseShareContentSize - // DefaultGasPerBlobByte is the default gas cost deducted per byte of blob - // included in a PayForBlobs txn - DefaultGasPerBlobByte = 8 - // DefaultMinGasPrice is the default min gas price that gets set in the app.toml file. // The min gas price acts as a filter. Transactions below that limit will not pass // a nodes `CheckTx` and thus not be proposed by that node. @@ -30,6 +26,11 @@ const ( // to unbond in a proof of stake system. Any validator within this // time can be subject to slashing under conditions of misbehavior. DefaultUnbondingTime = 3 * 7 * 24 * time.Hour + + // DefaultNetworkMinGasPrice is used by x/minfee to prevent transactions from being + // included in a block if they specify a gas price lower than this. + // Only applies to app version >= 2 + DefaultNetworkMinGasPrice = 0.000001 // utia ) var DefaultUpperBoundMaxBytes = DefaultSquareSizeUpperBound * DefaultSquareSizeUpperBound * share.ContinuationSparseShareContentSize diff --git a/pkg/appconsts/v2/app_consts.go b/pkg/appconsts/v2/app_consts.go index 2ef7a4075c..d5829e84c5 100644 --- a/pkg/appconsts/v2/app_consts.go +++ b/pkg/appconsts/v2/app_consts.go @@ -4,7 +4,4 @@ const ( Version uint64 = 2 SquareSizeUpperBound int = 128 SubtreeRootThreshold int = 64 - // NetworkMinGasPrice is used by x/minfee to prevent transactions from being - // included in a block if they specify a gas price lower than this. - NetworkMinGasPrice float64 = 0.000001 // utia ) diff --git a/pkg/appconsts/v3/app_consts.go b/pkg/appconsts/v3/app_consts.go index 504360864a..3fa92f49a0 100644 --- a/pkg/appconsts/v3/app_consts.go +++ b/pkg/appconsts/v3/app_consts.go @@ -4,4 +4,6 @@ const ( Version uint64 = 3 SquareSizeUpperBound int = 128 SubtreeRootThreshold int = 64 + TxSizeCostPerByte uint64 = 10 + GasPerBlobByte uint32 = 8 ) diff --git a/pkg/appconsts/versioned_consts.go b/pkg/appconsts/versioned_consts.go index a8d01fdedf..7bb1088a8b 100644 --- a/pkg/appconsts/versioned_consts.go +++ b/pkg/appconsts/versioned_consts.go @@ -1,7 +1,6 @@ package appconsts import ( - v1 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v1" v3 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v3" ) @@ -18,15 +17,25 @@ const ( // // The rationale for this value is described in more detail in ADR-013. func SubtreeRootThreshold(_ uint64) int { - return v1.SubtreeRootThreshold + return v3.SubtreeRootThreshold } // SquareSizeUpperBound imposes an upper bound on the max effective square size. func SquareSizeUpperBound(_ uint64) int { - return v1.SquareSizeUpperBound + return v3.SquareSizeUpperBound +} + +func TxSizeCostPerByte(_ uint64) uint64 { + return v3.TxSizeCostPerByte +} + +func GasPerBlobByte(_ uint64) uint32 { + return v3.GasPerBlobByte } var ( DefaultSubtreeRootThreshold = SubtreeRootThreshold(LatestVersion) DefaultSquareSizeUpperBound = SquareSizeUpperBound(LatestVersion) + DefaultTxSizeCostPerByte = TxSizeCostPerByte(LatestVersion) + DefaultGasPerBlobByte = GasPerBlobByte(LatestVersion) ) diff --git a/pkg/appconsts/versioned_consts_test.go b/pkg/appconsts/versioned_consts_test.go index 6fb5cfc48d..da29d71241 100644 --- a/pkg/appconsts/versioned_consts_test.go +++ b/pkg/appconsts/versioned_consts_test.go @@ -1,7 +1,6 @@ package appconsts_test import ( - "fmt" "testing" "github.com/stretchr/testify/require" @@ -9,52 +8,69 @@ import ( "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" v1 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v1" v2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" + v3 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v3" ) -func TestSubtreeRootThreshold(t *testing.T) { +func TestVersionedConsts(t *testing.T) { testCases := []struct { - version uint64 - expected int + name string + version uint64 + expectedConstant interface{} + got interface{} }{ { - version: v1.Version, - expected: v1.SubtreeRootThreshold, + name: "SubtreeRootThreshold v1", + version: v1.Version, + expectedConstant: v1.SubtreeRootThreshold, + got: appconsts.SubtreeRootThreshold(v1.Version), }, { - version: v2.Version, - expected: v2.SubtreeRootThreshold, + name: "SubtreeRootThreshold v2", + version: v2.Version, + expectedConstant: v2.SubtreeRootThreshold, + got: appconsts.SubtreeRootThreshold(v2.Version), + }, + { + name: "SubtreeRootThreshold v3", + version: v3.Version, + expectedConstant: v3.SubtreeRootThreshold, + got: appconsts.SubtreeRootThreshold(v3.Version), + }, + { + name: "SquareSizeUpperBound v1", + version: v1.Version, + expectedConstant: v1.SquareSizeUpperBound, + got: appconsts.SquareSizeUpperBound(v1.Version), + }, + { + name: "SquareSizeUpperBound v2", + version: v2.Version, + expectedConstant: v2.SquareSizeUpperBound, + got: appconsts.SquareSizeUpperBound(v2.Version), + }, + { + name: "SquareSizeUpperBound v3", + version: v3.Version, + expectedConstant: v3.SquareSizeUpperBound, + got: appconsts.SquareSizeUpperBound(v3.Version), }, - } - - for _, tc := range testCases { - name := fmt.Sprintf("version %v", tc.version) - t.Run(name, func(t *testing.T) { - got := appconsts.SubtreeRootThreshold(tc.version) - require.Equal(t, tc.expected, got) - }) - } -} - -func TestSquareSizeUpperBound(t *testing.T) { - testCases := []struct { - version uint64 - expected int - }{ { - version: v1.Version, - expected: v1.SquareSizeUpperBound, + name: "TxSizeCostPerByte v3", + version: v3.Version, + expectedConstant: v3.TxSizeCostPerByte, + got: appconsts.TxSizeCostPerByte(v3.Version), }, { - version: v2.Version, - expected: v2.SquareSizeUpperBound, + name: "GasPerBlobByte v3", + version: v3.Version, + expectedConstant: v3.GasPerBlobByte, + got: appconsts.GasPerBlobByte(v3.Version), }, } for _, tc := range testCases { - name := fmt.Sprintf("version %v", tc.version) - t.Run(name, func(t *testing.T) { - got := appconsts.SquareSizeUpperBound(tc.version) - require.Equal(t, tc.expected, got) + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.expectedConstant, tc.got) }) } } diff --git a/specs/src/parameters_v3.md b/specs/src/parameters_v3.md new file mode 100644 index 0000000000..491e4834c1 --- /dev/null +++ b/specs/src/parameters_v3.md @@ -0,0 +1,72 @@ +# Parameters v3 + +The parameters below represent the parameters for app version 3. + +Note that not all of these parameters are changeable via governance. This list +also includes parameter that require a hardfork to change due to being manually +hardcoded in the application or they are blocked by the `x/paramfilter` module. + +## Global parameters + +| Parameter | Default | Summary | Changeable via Governance | +|-------------------|---------|------------------------------------------------------------------------------------------------------------------------|---------------------------| +| MaxBlockSizeBytes | 100MiB | Hardcoded value in CometBFT for the protobuf encoded block. | False | +| MaxSquareSize | 128 | Hardcoded maximum square size determined per shares per row or column for the original data square (not yet extended). | False | + +## Module parameters + +| Module.Parameter | Default | Summary | Changeable via Governance | +|-----------------------------------------------|---------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------|---------------------------| +| auth.MaxMemoCharacters | 256 | Largest allowed size for a memo in bytes. | True | +| auth.SigVerifyCostED25519 | 590 | Gas used to verify Ed25519 signature. | True | +| auth.SigVerifyCostSecp256k1 | 1000 | Gas used to verify secp256k1 signature. | True | +| auth.TxSigLimit | 7 | Max number of signatures allowed in a multisig transaction. | True | +| auth.TxSizeCostPerByte | 10 | Gas used per transaction byte. | False | +| bank.SendEnabled | true | Allow transfers. | False | +| blob.GasPerBlobByte | 8 | Gas used per blob byte. | False | +| blob.GovMaxSquareSize | 64 | Governance parameter for the maximum square size of the original data square. | True | +| consensus.block.MaxBytes | 1974272 bytes (~1.88 MiB) | Governance parameter for the maximum size of the protobuf encoded block. | True | +| consensus.block.MaxGas | -1 | Maximum gas allowed per block (-1 is infinite). | True | +| consensus.block.TimeIotaMs | 1000 | Minimum time added to the time in the header each block. | False | +| consensus.evidence.MaxAgeDuration | 1814400000000000 (21 days) | The maximum age of evidence before it is considered invalid in nanoseconds. This value should be identical to the unbonding period. | True | +| consensus.evidence.MaxAgeNumBlocks | 120960 | The maximum number of blocks before evidence is considered invalid. This value will stop CometBFT from pruning block data. | True | +| consensus.evidence.MaxBytes | 1MiB | Maximum size in bytes used by evidence in a given block. | True | +| consensus.validator.PubKeyTypes | Ed25519 | The type of public key used by validators. | False | +| consensus.Version.AppVersion | 2 | Determines protocol rules used for a given height. Incremented by the application upon an upgrade. | True | +| distribution.BaseProposerReward | 0 | Reward in the mint denomination for proposing a block. | True | +| distribution.BonusProposerReward | 0 | Extra reward in the mint denomination for proposers based on the voting power included in the commit. | True | +| distribution.CommunityTax | 0.02 (2%) | Percentage of the inflation sent to the community pool. | True | +| distribution.WithdrawAddrEnabled | true | Enables delegators to withdraw funds to a different address. | True | +| gov.DepositParams.MaxDepositPeriod | 604800000000000 (1 week) | Maximum period for token holders to deposit on a proposal in nanoseconds. | True | +| gov.DepositParams.MinDeposit | 10_000_000_000 utia (10,000 TIA) | Minimum deposit for a proposal to enter voting period. | True | +| gov.TallyParams.Quorum | 0.334 (33.4%) | Minimum percentage of total stake needed to vote for a result to be considered valid. | True | +| gov.TallyParams.Threshold | 0.50 (50%) | Minimum proportion of Yes votes for proposal to pass. | True | +| gov.TallyParams.VetoThreshold | 0.334 (33.4%) | Minimum value of Veto votes to Total votes ratio for proposal to be vetoed. | True | +| gov.VotingParams.VotingPeriod | 604800000000000 (1 week) | Duration of the voting period in nanoseconds. | True | +| ibc.ClientGenesis.AllowedClients | []string{"06-solomachine", "07-tendermint"} | List of allowed IBC light clients. | True | +| ibc.ConnectionGenesis.MaxExpectedTimePerBlock | 7500000000000 (75 seconds) | Maximum expected time per block in nanoseconds under normal operation. | True | +| ibc.Transfer.ReceiveEnabled | true | Enable receiving tokens via IBC. | True | +| ibc.Transfer.SendEnabled | true | Enable sending tokens via IBC. | True | +| icahost.HostEnabled | True | Enables or disables the Inter-Chain Accounts host module. | True | +| icahost.AllowMessages | [icaAllowMessages] | Defines a list of sdk message typeURLs allowed to be executed on a host chain. | True | +| minfee.NetworkMinGasPrice | 0.000001 utia | All transactions must have a gas price greater than or equal to this value. | True | +| mint.BondDenom | utia | Denomination that is inflated and sent to the distribution module account. | False | +| mint.DisinflationRate | 0.10 (10%) | The rate at which the inflation rate decreases each year. | False | +| mint.InitialInflationRate | 0.08 (8%) | The inflation rate the network starts at. | False | +| mint.TargetInflationRate | 0.015 (1.5%) | The inflation rate that the network aims to stabilize at. | False | +| packetfowardmiddleware.FeePercentage | 0 | % of the forwarded packet amount which will be subtracted and distributed to the community pool. | True | +| slashing.DowntimeJailDuration | 1 min | Duration of time a validator must stay jailed. | True | +| slashing.MinSignedPerWindow | 0.75 (75%) | The percentage of SignedBlocksWindow that must be signed not to get jailed. | True | +| slashing.SignedBlocksWindow | 5000 | The range of blocks used to count for downtime. | True | +| slashing.SlashFractionDoubleSign | 0.02 (2%) | Percentage slashed after a validator is jailed for double signing. | True | +| slashing.SlashFractionDowntime | 0.00 (0%) | Percentage slashed after a validator is jailed for downtime. | True | +| staking.BondDenom | utia | Bondable coin denomination. | False | +| staking.HistoricalEntries | 10000 | Number of historical entries to persist in store. | True | +| staking.MaxEntries | 7 | Maximum number of entries in the redelegation queue. | True | +| staking.MaxValidators | 100 | Maximum number of validators. | True | +| staking.MinCommissionRate | 0.05 (5%) | Minimum commission rate used by all validators. | True | +| staking.UnbondingTime | 1814400 (21 days) | Duration of time for unbonding in seconds. | False | + +Note: none of the mint module parameters are governance modifiable because they have been converted into hardcoded constants. See the x/mint README.md for more details. + +[icaAllowMessages]: https://github.com/rootulp/celestia-app/blob/8caa5807df8d15477554eba953bd056ae72d4503/app/ica_host.go#L3-L18 diff --git a/x/blob/README.md b/x/blob/README.md index 3f1e0484ae..efa1f74858 100644 --- a/x/blob/README.md +++ b/x/blob/README.md @@ -49,6 +49,7 @@ message Params { `GasPerBlobByte` is the amount of gas that is consumed per byte of blob data when a `MsgPayForBlobs` is processed. Currently, the default value is 8. This value is set below that of normal transaction gas consumption, which is 10. +`GasPerBlobByte` was a governance-modifiable parameter in v1 and v2. In app v3 and above, it is a versioned parameter, meaning it can only be changed through hard fork upgrades. #### `GovMaxSquareSize` diff --git a/x/blob/ante/ante.go b/x/blob/ante/ante.go index fe58eae87c..3c5249b291 100644 --- a/x/blob/ante/ante.go +++ b/x/blob/ante/ante.go @@ -1,6 +1,8 @@ package ante import ( + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" + v2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" "github.com/celestiaorg/celestia-app/v3/x/blob/types" "cosmossdk.io/errors" @@ -33,8 +35,12 @@ func (d MinGasPFBDecorator) AnteHandle(ctx sdk.Context, tx sdk.Tx, simulate bool // NOTE: here we assume only one PFB per transaction if pfb, ok := m.(*types.MsgPayForBlobs); ok { if gasPerByte == 0 { - // lazily fetch the gas per byte param - gasPerByte = d.k.GasPerBlobByte(ctx) + if ctx.BlockHeader().Version.App <= v2.Version { + // lazily fetch the gas per byte param + gasPerByte = d.k.GasPerBlobByte(ctx) + } else { + gasPerByte = appconsts.GasPerBlobByte(ctx.BlockHeader().Version.App) + } } gasToConsume := pfb.Gas(gasPerByte) if gasToConsume > txGas { diff --git a/x/blob/ante/ante_test.go b/x/blob/ante/ante_test.go index e1f16e2b7a..6be76d77d9 100644 --- a/x/blob/ante/ante_test.go +++ b/x/blob/ante/ante_test.go @@ -1,15 +1,20 @@ package ante_test import ( + "fmt" "testing" "github.com/celestiaorg/celestia-app/v3/app" "github.com/celestiaorg/celestia-app/v3/app/encoding" + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" + v2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" ante "github.com/celestiaorg/celestia-app/v3/x/blob/ante" blob "github.com/celestiaorg/celestia-app/v3/x/blob/types" "github.com/celestiaorg/go-square/v2/share" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/stretchr/testify/require" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" + "github.com/tendermint/tendermint/proto/tendermint/version" ) const ( @@ -22,8 +27,9 @@ func TestPFBAnteHandler(t *testing.T) { testCases := []struct { name string pfb *blob.MsgPayForBlobs - txGas uint64 + txGas func(uint32) uint32 gasConsumed uint64 + versions []uint64 wantErr bool }{ { @@ -32,8 +38,11 @@ func TestPFBAnteHandler(t *testing.T) { // 1 share = 512 bytes = 5120 gas BlobSizes: []uint32{uint32(share.AvailableBytesFromSparseShares(1))}, }, - txGas: share.ShareSize * testGasPerBlobByte, + txGas: func(testGasPerBlobByte uint32) uint32 { + return share.ShareSize * testGasPerBlobByte + }, gasConsumed: 0, + versions: []uint64{v2.Version, appconsts.LatestVersion}, wantErr: false, }, { @@ -41,8 +50,11 @@ func TestPFBAnteHandler(t *testing.T) { pfb: &blob.MsgPayForBlobs{ BlobSizes: []uint32{uint32(share.AvailableBytesFromSparseShares(1)), uint32(share.AvailableBytesFromSparseShares(2))}, }, - txGas: 3 * share.ShareSize * testGasPerBlobByte, + txGas: func(testGasPerBlobByte uint32) uint32 { + return 3 * share.ShareSize * testGasPerBlobByte + }, gasConsumed: 0, + versions: []uint64{v2.Version, appconsts.LatestVersion}, wantErr: false, }, { @@ -51,8 +63,11 @@ func TestPFBAnteHandler(t *testing.T) { // 2 share = 1024 bytes = 10240 gas BlobSizes: []uint32{uint32(share.AvailableBytesFromSparseShares(1) + 1)}, }, - txGas: 2*share.ShareSize*testGasPerBlobByte - 1, + txGas: func(testGasPerBlobByte uint32) uint32 { + return 2*share.ShareSize*testGasPerBlobByte - 1 + }, gasConsumed: 0, + versions: []uint64{v2.Version, appconsts.LatestVersion}, wantErr: true, }, { @@ -60,8 +75,11 @@ func TestPFBAnteHandler(t *testing.T) { pfb: &blob.MsgPayForBlobs{ BlobSizes: []uint32{uint32(share.AvailableBytesFromSparseShares(1)), uint32(share.AvailableBytesFromSparseShares(2))}, }, - txGas: 3*share.ShareSize*testGasPerBlobByte - 1, + txGas: func(testGasPerBlobByte uint32) uint32 { + return 3*share.ShareSize*testGasPerBlobByte - 1 + }, gasConsumed: 0, + versions: []uint64{v2.Version, appconsts.LatestVersion}, wantErr: true, }, { @@ -70,8 +88,11 @@ func TestPFBAnteHandler(t *testing.T) { // 1 share = 512 bytes = 5120 gas BlobSizes: []uint32{uint32(share.AvailableBytesFromSparseShares(1))}, }, - txGas: share.ShareSize*testGasPerBlobByte + 10000 - 1, + txGas: func(testGasPerBlobByte uint32) uint32 { + return share.ShareSize*testGasPerBlobByte + 10000 - 1 + }, gasConsumed: 10000, + versions: []uint64{v2.Version, appconsts.LatestVersion}, wantErr: true, }, { @@ -80,26 +101,43 @@ func TestPFBAnteHandler(t *testing.T) { // 1 share = 512 bytes = 5120 gas BlobSizes: []uint32{uint32(share.AvailableBytesFromSparseShares(10))}, }, - txGas: 1000000, + txGas: func(_ uint32) uint32 { + return 1000000 + }, gasConsumed: 10000, + versions: []uint64{v2.Version, appconsts.LatestVersion}, wantErr: false, }, } for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - anteHandler := ante.NewMinGasPFBDecorator(mockBlobKeeper{}) - ctx := sdk.Context{}.WithGasMeter(sdk.NewGasMeter(tc.txGas)).WithIsCheckTx(true) - ctx.GasMeter().ConsumeGas(tc.gasConsumed, "test") - txBuilder := txConfig.NewTxBuilder() - require.NoError(t, txBuilder.SetMsgs(tc.pfb)) - tx := txBuilder.GetTx() - _, err := anteHandler.AnteHandle(ctx, tx, false, func(ctx sdk.Context, _ sdk.Tx, _ bool) (sdk.Context, error) { return ctx, nil }) - if tc.wantErr { - require.Error(t, err) - } else { - require.NoError(t, err) - } - }) + for _, currentVersion := range tc.versions { + t.Run(fmt.Sprintf("%s v%d", tc.name, currentVersion), func(t *testing.T) { + anteHandler := ante.NewMinGasPFBDecorator(mockBlobKeeper{}) + var gasPerBlobByte uint32 + if currentVersion == v2.Version { + gasPerBlobByte = testGasPerBlobByte + } else { + gasPerBlobByte = appconsts.GasPerBlobByte(currentVersion) + } + + ctx := sdk.NewContext(nil, tmproto.Header{ + Version: version.Consensus{ + App: currentVersion, + }, + }, true, nil).WithGasMeter(sdk.NewGasMeter(uint64(tc.txGas(gasPerBlobByte)))).WithIsCheckTx(true) + + ctx.GasMeter().ConsumeGas(tc.gasConsumed, "test") + txBuilder := txConfig.NewTxBuilder() + require.NoError(t, txBuilder.SetMsgs(tc.pfb)) + tx := txBuilder.GetTx() + _, err := anteHandler.AnteHandle(ctx, tx, false, func(ctx sdk.Context, _ sdk.Tx, _ bool) (sdk.Context, error) { return ctx, nil }) + if tc.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } } } diff --git a/x/blob/keeper/gas_test.go b/x/blob/keeper/gas_test.go index 8c7bd534ff..a1e17a2bdd 100644 --- a/x/blob/keeper/gas_test.go +++ b/x/blob/keeper/gas_test.go @@ -3,6 +3,7 @@ package keeper_test import ( "testing" + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" "github.com/celestiaorg/celestia-app/v3/x/blob/types" "github.com/celestiaorg/go-square/v2/share" sdk "github.com/cosmos/cosmos-sdk/types" @@ -23,33 +24,33 @@ func TestPayForBlobGas(t *testing.T) { { name: "1 byte blob", // occupies 1 share msg: types.MsgPayForBlobs{BlobSizes: []uint32{1}}, - wantGasConsumed: uint64(1*share.ShareSize*types.DefaultGasPerBlobByte + paramLookUpCost), // 1 share * 512 bytes per share * 8 gas per byte + 1060 gas for fetching param = 5156 gas + wantGasConsumed: uint64(1*share.ShareSize*appconsts.GasPerBlobByte(appconsts.LatestVersion) + paramLookUpCost), // 1 share * 512 bytes per share * 8 gas per byte + 1060 gas for fetching param = 5156 gas }, { name: "100 byte blob", // occupies 1 share msg: types.MsgPayForBlobs{BlobSizes: []uint32{100}}, - wantGasConsumed: uint64(1*share.ShareSize*types.DefaultGasPerBlobByte + paramLookUpCost), + wantGasConsumed: uint64(1*share.ShareSize*appconsts.GasPerBlobByte(appconsts.LatestVersion) + paramLookUpCost), }, { name: "1024 byte blob", // occupies 3 shares because share prefix (e.g. namespace, info byte) msg: types.MsgPayForBlobs{BlobSizes: []uint32{1024}}, - wantGasConsumed: uint64(3*share.ShareSize*types.DefaultGasPerBlobByte + paramLookUpCost), // 3 shares * 512 bytes per share * 8 gas per byte + 1060 gas for fetching param = 13348 gas + wantGasConsumed: uint64(3*share.ShareSize*appconsts.GasPerBlobByte(appconsts.LatestVersion) + paramLookUpCost), // 3 shares * 512 bytes per share * 8 gas per byte + 1060 gas for fetching param = 13348 gas }, { name: "3 blobs, 1 share each", msg: types.MsgPayForBlobs{BlobSizes: []uint32{1, 1, 1}}, - wantGasConsumed: uint64(3*share.ShareSize*types.DefaultGasPerBlobByte + paramLookUpCost), // 3 shares * 512 bytes per share * 8 gas per byte + 1060 gas for fetching param = 13348 gas + wantGasConsumed: uint64(3*share.ShareSize*appconsts.GasPerBlobByte(appconsts.LatestVersion) + paramLookUpCost), // 3 shares * 512 bytes per share * 8 gas per byte + 1060 gas for fetching param = 13348 gas }, { name: "3 blobs, 6 shares total", msg: types.MsgPayForBlobs{BlobSizes: []uint32{1024, 800, 100}}, - wantGasConsumed: uint64(6*share.ShareSize*types.DefaultGasPerBlobByte + paramLookUpCost), // 6 shares * 512 bytes per share * 8 gas per byte + 1060 gas for fetching param = 25636 gas + wantGasConsumed: uint64(6*share.ShareSize*appconsts.GasPerBlobByte(appconsts.LatestVersion) + paramLookUpCost), // 6 shares * 512 bytes per share * 8 gas per byte + 1060 gas for fetching param = 25636 gas }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - k, stateStore, _ := CreateKeeper(t) + k, stateStore, _ := CreateKeeper(t, appconsts.LatestVersion) ctx := sdk.NewContext(stateStore, tmproto.Header{}, false, nil) _, err := k.PayForBlobs(sdk.WrapSDKContext(ctx), &tc.msg) require.NoError(t, err) @@ -62,7 +63,7 @@ func TestPayForBlobGas(t *testing.T) { func TestChangingGasParam(t *testing.T) { msg := types.MsgPayForBlobs{BlobSizes: []uint32{1024}} - k, stateStore, _ := CreateKeeper(t) + k, stateStore, _ := CreateKeeper(t, appconsts.LatestVersion) tempCtx := sdk.NewContext(stateStore, tmproto.Header{}, false, nil) ctx1 := sdk.NewContext(stateStore, tmproto.Header{}, false, nil) diff --git a/x/blob/keeper/genesis_test.go b/x/blob/keeper/genesis_test.go index d291deebf8..120f9cb21e 100644 --- a/x/blob/keeper/genesis_test.go +++ b/x/blob/keeper/genesis_test.go @@ -3,6 +3,7 @@ package keeper_test import ( "testing" + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" "github.com/celestiaorg/celestia-app/v3/x/blob" "github.com/celestiaorg/celestia-app/v3/x/blob/types" "github.com/stretchr/testify/require" @@ -13,7 +14,7 @@ func TestGenesis(t *testing.T) { Params: types.DefaultParams(), } - k, _, ctx := CreateKeeper(t) + k, _, ctx := CreateKeeper(t, appconsts.LatestVersion) blob.InitGenesis(ctx, *k, genesisState) got := blob.ExportGenesis(ctx, *k) require.NotNil(t, got) diff --git a/x/blob/keeper/grpc_query_params_test.go b/x/blob/keeper/grpc_query_params_test.go index 70f60860ee..e8c367cd35 100644 --- a/x/blob/keeper/grpc_query_params_test.go +++ b/x/blob/keeper/grpc_query_params_test.go @@ -3,13 +3,14 @@ package keeper_test import ( "testing" + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" "github.com/celestiaorg/celestia-app/v3/x/blob/types" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/stretchr/testify/require" ) func TestParamsQuery(t *testing.T) { - keeper, _, ctx := CreateKeeper(t) + keeper, _, ctx := CreateKeeper(t, appconsts.LatestVersion) wctx := sdk.WrapSDKContext(ctx) params := types.DefaultParams() keeper.SetParams(ctx, params) diff --git a/x/blob/keeper/keeper.go b/x/blob/keeper/keeper.go index 72a3fb7605..a7e0f3cc71 100644 --- a/x/blob/keeper/keeper.go +++ b/x/blob/keeper/keeper.go @@ -4,6 +4,8 @@ import ( "context" "fmt" + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" + v2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" "github.com/celestiaorg/celestia-app/v3/x/blob/types" "github.com/cosmos/cosmos-sdk/codec" sdk "github.com/cosmos/cosmos-sdk/types" @@ -43,7 +45,14 @@ func (k Keeper) Logger(ctx sdk.Context) log.Logger { func (k Keeper) PayForBlobs(goCtx context.Context, msg *types.MsgPayForBlobs) (*types.MsgPayForBlobsResponse, error) { ctx := sdk.UnwrapSDKContext(goCtx) - gasToConsume := types.GasToConsume(msg.BlobSizes, k.GasPerBlobByte(ctx)) + // GasPerBlobByte is a versioned param from version 3 onwards. + var gasToConsume uint64 + if ctx.BlockHeader().Version.App <= v2.Version { + gasToConsume = types.GasToConsume(msg.BlobSizes, k.GasPerBlobByte(ctx)) + } else { + gasToConsume = types.GasToConsume(msg.BlobSizes, appconsts.GasPerBlobByte(ctx.BlockHeader().Version.App)) + } + ctx.GasMeter().ConsumeGas(gasToConsume, payForBlobGasDescriptor) err := ctx.EventManager().EmitTypedEvent( diff --git a/x/blob/keeper/keeper_test.go b/x/blob/keeper/keeper_test.go index e3da5cb12a..8fee7f81a8 100644 --- a/x/blob/keeper/keeper_test.go +++ b/x/blob/keeper/keeper_test.go @@ -27,7 +27,7 @@ import ( // TestPayForBlobs verifies the attributes on the emitted event. func TestPayForBlobs(t *testing.T) { - k, _, ctx := CreateKeeper(t) + k, _, ctx := CreateKeeper(t, appconsts.LatestVersion) signer := "celestia15drmhzw5kwgenvemy30rqqqgq52axf5wwrruf7" namespace := share.MustNewV0Namespace(bytes.Repeat([]byte{1}, share.NamespaceVersionZeroIDSize)) namespaces := [][]byte{namespace.Bytes()} @@ -72,7 +72,7 @@ func createMsgPayForBlob(t *testing.T, signer string, namespace share.Namespace, return msg } -func CreateKeeper(t *testing.T) (*keeper.Keeper, store.CommitMultiStore, sdk.Context) { +func CreateKeeper(t *testing.T, version uint64) (*keeper.Keeper, store.CommitMultiStore, sdk.Context) { storeKey := sdk.NewKVStoreKey(paramtypes.StoreKey) tStoreKey := storetypes.NewTransientStoreKey(paramtypes.TStoreKey) @@ -87,7 +87,7 @@ func CreateKeeper(t *testing.T) (*keeper.Keeper, store.CommitMultiStore, sdk.Con ctx := sdk.NewContext(stateStore, tmproto.Header{ Version: tmversion.Consensus{ Block: 1, - App: 1, + App: version, }, }, false, nil) diff --git a/x/blob/keeper/params_test.go b/x/blob/keeper/params_test.go index 9431ef3828..53f5dbca20 100644 --- a/x/blob/keeper/params_test.go +++ b/x/blob/keeper/params_test.go @@ -3,12 +3,13 @@ package keeper_test import ( "testing" + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" "github.com/celestiaorg/celestia-app/v3/x/blob/types" "github.com/stretchr/testify/require" ) func TestGetParams(t *testing.T) { - k, _, ctx := CreateKeeper(t) + k, _, ctx := CreateKeeper(t, appconsts.LatestVersion) params := types.DefaultParams() k.SetParams(ctx, params) diff --git a/x/blob/types/payforblob.go b/x/blob/types/payforblob.go index 92eca0a4d7..b49eb05394 100644 --- a/x/blob/types/payforblob.go +++ b/x/blob/types/payforblob.go @@ -10,7 +10,6 @@ import ( "github.com/celestiaorg/go-square/v2/share" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/cosmos/cosmos-sdk/x/auth/migrations/legacytx" - auth "github.com/cosmos/cosmos-sdk/x/auth/types" "github.com/tendermint/tendermint/crypto/merkle" "golang.org/x/exp/slices" ) @@ -162,10 +161,9 @@ func EstimateGas(blobSizes []uint32, gasPerByte uint32, txSizeCost uint64) uint6 return GasToConsume(blobSizes, gasPerByte) + (txSizeCost * BytesPerBlobInfo * uint64(len(blobSizes))) + PFBGasFixedCost } -// DefaultEstimateGas runs EstimateGas with the system defaults. The network may change these values -// through governance, thus this function should predominantly be used in testing. +// DefaultEstimateGas runs EstimateGas with the system defaults. func DefaultEstimateGas(blobSizes []uint32) uint64 { - return EstimateGas(blobSizes, appconsts.DefaultGasPerBlobByte, auth.DefaultTxSizeCostPerByte) + return EstimateGas(blobSizes, appconsts.DefaultGasPerBlobByte, appconsts.DefaultTxSizeCostPerByte) } // ValidateBlobNamespace returns an error if the provided namespace is an diff --git a/x/minfee/grpc_query_test.go b/x/minfee/grpc_query_test.go index a7ab0fdfef..9e797ee828 100644 --- a/x/minfee/grpc_query_test.go +++ b/x/minfee/grpc_query_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/celestiaorg/celestia-app/v3/app" - v2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" testutil "github.com/celestiaorg/celestia-app/v3/test/util" "github.com/celestiaorg/celestia-app/v3/x/minfee" sdk "github.com/cosmos/cosmos-sdk/types" @@ -24,5 +24,5 @@ func TestQueryNetworkMinGasPrice(t *testing.T) { require.NoError(t, err) // Check the response - require.Equal(t, v2.NetworkMinGasPrice, resp.NetworkMinGasPrice.MustFloat64()) + require.Equal(t, appconsts.DefaultNetworkMinGasPrice, resp.NetworkMinGasPrice.MustFloat64()) } diff --git a/x/minfee/params.go b/x/minfee/params.go index ef17044d7c..2f1edc3ee8 100644 --- a/x/minfee/params.go +++ b/x/minfee/params.go @@ -3,7 +3,7 @@ package minfee import ( "fmt" - v2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" sdk "github.com/cosmos/cosmos-sdk/types" paramtypes "github.com/cosmos/cosmos-sdk/x/params/types" ) @@ -18,7 +18,7 @@ var ( ) func init() { - DefaultNetworkMinGasPriceDec, err := sdk.NewDecFromStr(fmt.Sprintf("%f", v2.NetworkMinGasPrice)) + DefaultNetworkMinGasPriceDec, err := sdk.NewDecFromStr(fmt.Sprintf("%f", appconsts.DefaultNetworkMinGasPrice)) if err != nil { panic(err) } From d3c6a31acd055dee058028e51f7287f27cc260c6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Sep 2024 10:28:44 -0400 Subject: [PATCH 35/85] chore(deps): Bump k8s.io/apimachinery from 0.30.2 to 0.31.1 (#3884) Bumps [k8s.io/apimachinery](https://github.com/kubernetes/apimachinery) from 0.30.2 to 0.31.1.
Commits
  • a8f449e Falls back to SPDY for gorilla/websocket https proxy error
  • 62791ec Merge pull request #125571 from liggitt/filter-auth-02-sar
  • cc2ba35 add field and label selectors to authorization attributes
  • ce76a8f generate
  • 35052c5 add subjectaccessreview field and label selectors
  • ab06869 Merge pull request #126105 from benluddy/cbor-framer
  • 429f4e4 Implement runtime.Framer for CBOR Sequences.
  • d7e1c53 Merge pull request #126018 from aroradaman/bump-k8s-utils
  • 07cb122 Merge pull request #125748 from benluddy/cbor-custom-marshalers
  • dd17456 bump k8s.io/utils
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=k8s.io/apimachinery&package-manager=go_modules&previous-version=0.30.2&new-version=0.31.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 14 ++++++++------ go.sum | 48 +++++++++++++++++++++++++++--------------------- 2 files changed, 35 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index bf9aaa9178..110ecaa091 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( google.golang.org/grpc v1.66.2 google.golang.org/protobuf v1.34.2 gopkg.in/yaml.v2 v2.4.0 - k8s.io/apimachinery v0.30.2 + k8s.io/apimachinery v0.31.1 ) require ( @@ -91,16 +91,17 @@ require ( github.com/ethereum/c-kzg-4844 v1.0.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-ini/ini v1.67.0 // indirect github.com/go-kit/kit v0.12.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.22.3 // indirect + github.com/go-openapi/swag v0.22.4 // indirect github.com/go-playground/validator/v10 v10.11.2 // indirect github.com/goccy/go-json v0.10.3 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect @@ -163,7 +164,7 @@ require ( github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect - github.com/moby/spdystream v0.2.0 // indirect + github.com/moby/spdystream v0.4.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mtibben/percent v0.2.1 // indirect @@ -199,6 +200,7 @@ require ( github.com/tklauser/numcpus v0.6.1 // indirect github.com/ugorji/go/codec v1.2.9 // indirect github.com/ulikunitz/xz v0.5.10 // indirect + github.com/x448/float16 v0.8.4 // indirect github.com/zondax/hid v0.9.2 // indirect github.com/zondax/ledger-go v0.14.3 // indirect go.etcd.io/bbolt v1.3.6 // indirect @@ -228,9 +230,9 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/api v0.30.2 // indirect k8s.io/client-go v0.30.2 // indirect - k8s.io/klog/v2 v2.120.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect - k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect + k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect nhooyr.io/websocket v1.8.6 // indirect rsc.io/tmplfunc v0.0.3 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/go.sum b/go.sum index b0c0f27e3d..f756fcb1b5 100644 --- a/go.sum +++ b/go.sum @@ -544,6 +544,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= @@ -578,8 +580,8 @@ github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= @@ -592,8 +594,9 @@ github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaL github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= +github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= @@ -608,9 +611,10 @@ github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5Nq github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= @@ -730,8 +734,8 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 h1:E/LAvt58di64hlYjx7AsNS6C/ysHWYo+2qPCZKTQhRo= -github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af h1:kmjWCqn2qkEml422C2Rrd27c3VGxi6a/6HNq8QmHRKM= +github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= @@ -1023,8 +1027,8 @@ github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8oh github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= -github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= -github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/spdystream v0.4.0 h1:Vy79D6mHeJJjiPdFEL2yku1kl0chZpJfZcPpb16BRl8= +github.com/moby/spdystream v0.4.0/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -1070,14 +1074,14 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY= -github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM= +github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= +github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.31.0 h1:54UJxxj6cPInHS3a35wm6BK/F9nHYueZ1NVujHDrnXE= -github.com/onsi/gomega v1.31.0/go.mod h1:DW9aCi7U6Yi40wNVAvT6kzFnEVEI5n3DloYBiKiT6zk= +github.com/onsi/gomega v1.20.0 h1:8W0cWlwFkflGPLltQvLRB7ZVD5HuP6ng320w2IS245Q= +github.com/onsi/gomega v1.20.0/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= @@ -1177,8 +1181,8 @@ github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRr github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.8.3 h1:O+qNyWn7Z+F9M0ILBHgMVPuB1xTOucVd5gtaYyXBpRo= github.com/rs/cors v1.8.3/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= @@ -1320,6 +1324,8 @@ github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+ github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= @@ -2076,16 +2082,16 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= k8s.io/api v0.30.2 h1:+ZhRj+28QT4UOH+BKznu4CBgPWgkXO7XAvMcMl0qKvI= k8s.io/api v0.30.2/go.mod h1:ULg5g9JvOev2dG0u2hig4Z7tQ2hHIuS+m8MNZ+X6EmI= -k8s.io/apimachinery v0.30.2 h1:fEMcnBj6qkzzPGSVsAZtQThU62SmQ4ZymlXRC5yFSCg= -k8s.io/apimachinery v0.30.2/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/apimachinery v0.31.1 h1:mhcUBbj7KUjaVhyXILglcVjuS4nYXiwC+KKFBgIVy7U= +k8s.io/apimachinery v0.31.1/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= k8s.io/client-go v0.30.2 h1:sBIVJdojUNPDU/jObC+18tXWcTJVcwyqS9diGdWHk50= k8s.io/client-go v0.30.2/go.mod h1:JglKSWULm9xlJLx4KCkfLLQ7XwtlbflV6uFFSHTMgVs= -k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= -k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= nhooyr.io/websocket v1.8.6 h1:s+C3xAMLwGmlI31Nyn/eAehUlZPwfYZu2JXM621Q5/k= nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw= From 983385a1abe2361fb7f4a78fdaf86bda8ae9af18 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Sep 2024 10:28:59 -0400 Subject: [PATCH 36/85] chore(deps): Bump github.com/celestiaorg/go-square from 1.0.0 to 1.1.0 (#3883) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/celestiaorg/go-square](https://github.com/celestiaorg/go-square) from 1.0.0 to 1.1.0.
Release notes

Sourced from github.com/celestiaorg/go-square's releases.

v1.1.0

What's Changed

New Contributors

Full Changelog: https://github.com/celestiaorg/go-square/compare/v1.0.1...v1.1.0

v1.0.1

What's Changed

Full Changelog: https://github.com/celestiaorg/go-square/compare/v1.0.0...v1.0.1

Commits
  • be3c280 fix(blob): use proper share.MaxShareVersion const (#64)
  • c6540fc perf(namespace): speed up Namespace comparisons (#63)
  • e4e8a8b chore(deps): bump google.golang.org/protobuf from 1.33.0 to 1.34.1 (#57)
  • 4063a54 refactor: rename to ShareReservedBytes (#61)
  • ae56f63 chore(deps): bump github.com/celestiaorg/nmt from 0.20.0 to 0.21.0 (#62)
  • 838b0a2 chore(deps): bump golangci/golangci-lint-action from 5.1.0 to 6.0.1 (#59)
  • dea967e chore(deps): bump golangci/golangci-lint-action from 4.0.0 to 5.1.0 (#56)
  • 6c6cdf5 chore(deps): bump celestiaorg/.github from 0.3.3 to 0.4.1 (#54)
  • ef245e5 chore(deps): bump celestiaorg/.github from 0.3.1 to 0.3.3 (#53)
  • 17df7a2 chore(deps): bump google.golang.org/protobuf from 1.32.0 to 1.33.0 (#51)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/celestiaorg/go-square&package-manager=go_modules&previous-version=1.0.0&new-version=1.1.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 110ecaa091..5802cd499f 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( cosmossdk.io/errors v1.0.1 cosmossdk.io/math v1.3.0 github.com/celestiaorg/blobstream-contracts/v3 v3.1.0 - github.com/celestiaorg/go-square v1.0.0 + github.com/celestiaorg/go-square v1.1.0 github.com/celestiaorg/go-square/v2 v2.0.0 github.com/celestiaorg/knuu v0.15.2 github.com/celestiaorg/nmt v0.22.1 diff --git a/go.sum b/go.sum index f756fcb1b5..46a0053db2 100644 --- a/go.sum +++ b/go.sum @@ -321,8 +321,8 @@ github.com/celestiaorg/celestia-core v1.41.0-tm-v0.34.29 h1:hRdTxe+Dz6kiqifRZCC9 github.com/celestiaorg/celestia-core v1.41.0-tm-v0.34.29/go.mod h1:H6vjzdoqTt4qmbf11z1Lnc9YLUp/B8ITEQLhU92ghqQ= github.com/celestiaorg/cosmos-sdk v1.24.1-sdk-v0.46.16 h1:SeQ7Y/CyOcUMKo7mQiexaj/pZ/xIgyuZFIwYZwpSkWE= github.com/celestiaorg/cosmos-sdk v1.24.1-sdk-v0.46.16/go.mod h1:Bpl1LSWiDpQumgOhhMTZBMopqa0j7fRasIhvTZB44P0= -github.com/celestiaorg/go-square v1.0.0 h1:pb1leaUi2WWSpri6ubNJOCUWdAoPr6AYGlT19F/Y/4k= -github.com/celestiaorg/go-square v1.0.0/go.mod h1:XMv5SGCeGSkynW2OOsedugaW/rQlvzxGzWGxTKsyYOU= +github.com/celestiaorg/go-square v1.1.0 h1:K4tBL5PCJwDtpBfyDxxZ3N962aC9VYb5/bw3LjagEtY= +github.com/celestiaorg/go-square v1.1.0/go.mod h1:1EXMErhDrWJM8B8V9hN7dqJ2kUTClfwdqMOmF9yQUa0= github.com/celestiaorg/go-square/v2 v2.0.0 h1:U5QV8/de5lc7glosfgyHhcxbFwNuwU4+6aYZ2RgjM04= github.com/celestiaorg/go-square/v2 v2.0.0/go.mod h1:y0BolG0tRM7UN1sAQyDDUkT+aMJPwFIjviVvnCB62C0= github.com/celestiaorg/knuu v0.15.2 h1:l6MrFTfDfrNtbvOLSjD/YzRpaaBFIIvlWIYXRNNL1/s= From f3e99673581092a610b955022ae58ee300754340 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Sep 2024 11:16:47 -0400 Subject: [PATCH 37/85] chore(deps): Bump google.golang.org/grpc from 1.66.2 to 1.67.0 (#3891) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.66.2 to 1.67.0.
Release notes

Sourced from google.golang.org/grpc's releases.

Release 1.67.0

Bug Fixes

  • ringhash: when used with multiple EDS priorities, fix bug that could prevent a higher priority from recovering from transient failure. (#7364)

Behavior Changes

  • In accordance with RFC 7540, clients and servers will now reject TLS connections that don't support ALPN. This can be disabled by setting the environment variable GRPC_ENFORCE_ALPN_ENABLED to false (case insensitive). Please file a bug if you encounter any issues with this behavior. The environment variable to revert this behavior will be removed in an upcoming release. (#7535)
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=google.golang.org/grpc&package-manager=go_modules&previous-version=1.66.2&new-version=1.67.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 14 +++++++------- go.sum | 28 ++++++++++++++-------------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index 5802cd499f..91a2cb0eb5 100644 --- a/go.mod +++ b/go.mod @@ -31,8 +31,8 @@ require ( github.com/tendermint/tendermint v0.34.29 github.com/tendermint/tm-db v0.6.7 golang.org/x/exp v0.0.0-20240213143201-ec583247a57a - google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 - google.golang.org/grpc v1.66.2 + google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 + google.golang.org/grpc v1.67.0 google.golang.org/protobuf v1.34.2 gopkg.in/yaml.v2 v2.4.0 k8s.io/apimachinery v0.31.1 @@ -40,7 +40,7 @@ require ( require ( cloud.google.com/go v0.112.1 // indirect - cloud.google.com/go/compute/metadata v0.3.0 // indirect + cloud.google.com/go/compute/metadata v0.5.0 // indirect cloud.google.com/go/iam v1.1.6 // indirect cloud.google.com/go/storage v1.38.0 // indirect filippo.io/edwards25519 v1.0.0-rc.1 // indirect @@ -106,7 +106,7 @@ require ( github.com/goccy/go-json v0.10.3 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect github.com/gogo/gateway v1.1.0 // indirect - github.com/golang/glog v1.2.1 // indirect + github.com/golang/glog v1.2.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect github.com/google/btree v1.1.2 // indirect @@ -216,15 +216,15 @@ require ( go.uber.org/zap v1.27.0 // indirect golang.org/x/crypto v0.26.0 // indirect golang.org/x/net v0.28.0 // indirect - golang.org/x/oauth2 v0.21.0 // indirect + golang.org/x/oauth2 v0.22.0 // indirect golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.23.0 // indirect + golang.org/x/sys v0.24.0 // indirect golang.org/x/term v0.23.0 // indirect golang.org/x/text v0.17.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/api v0.169.0 // indirect google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 46a0053db2..78dc3649d2 100644 --- a/go.sum +++ b/go.sum @@ -71,8 +71,8 @@ cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= -cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= -cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY= +cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY= cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= @@ -638,8 +638,8 @@ github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.2.1 h1:OptwRhECazUx5ix5TTWC3EZhsZEHWcYWY4FQHTIubm4= -github.com/golang/glog v1.2.1/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= +github.com/golang/glog v1.2.2 h1:1+mZ9upx1Dh6FmUTFR1naJ77miKiXgALjWOZ3NVFPmY= +github.com/golang/glog v1.2.2/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -1556,8 +1556,8 @@ golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= -golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= -golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= +golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1686,8 +1686,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1965,10 +1965,10 @@ google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= -google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 h1:+rdxYoE3E5htTEWIe15GlN6IfvbURM//Jt0mmkmm6ZU= -google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117/go.mod h1:OimBR/bc1wPO9iV4NC2bpyjy3VnAwZh5EBPQdtaE5oo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 h1:wKguEg1hsxI2/L3hUYrpo1RVi48K+uTyzKqprwLXsb8= +google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142/go.mod h1:d6be+8HhtEtucleCbxpPW9PA9XwISACu8nvpPqF0BVo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 h1:e7S5W7MGGLaSu8j3YjdezkZ+m1/Nm0uRVRMEMGk26Xs= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -2011,8 +2011,8 @@ google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.66.2 h1:3QdXkuq3Bkh7w+ywLdLvM56cmGvQHUMZpiCzt6Rqaoo= -google.golang.org/grpc v1.66.2/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/grpc v1.67.0 h1:IdH9y6PF5MPSdAntIcpjQ+tXO41pcQsfZV2RxtQgVcw= +google.golang.org/grpc v1.67.0/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= From 45518aa6e136901ec4897e0cf2e26c7676e77fa2 Mon Sep 17 00:00:00 2001 From: CHAMI Rachid Date: Mon, 23 Sep 2024 12:17:38 +0200 Subject: [PATCH 38/85] fix: force no bbr when starting local devnet (#3892) ## Overview adds the `--force-no-bbr` flag when starting the local devnet so that it works. --- local_devnet/scripts/start_core0.sh | 3 ++- local_devnet/scripts/start_node_and_create_validator.sh | 9 +++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/local_devnet/scripts/start_core0.sh b/local_devnet/scripts/start_core0.sh index 1218ee1386..ee01080ac2 100644 --- a/local_devnet/scripts/start_core0.sh +++ b/local_devnet/scripts/start_core0.sh @@ -17,4 +17,5 @@ fi /bin/celestia-appd start \ --moniker core0 \ --rpc.laddr tcp://0.0.0.0:26657 \ - --home /opt + --home /opt \ + --force-no-bbr diff --git a/local_devnet/scripts/start_node_and_create_validator.sh b/local_devnet/scripts/start_node_and_create_validator.sh index c9ac25f3bb..2b29953666 100644 --- a/local_devnet/scripts/start_node_and_create_validator.sh +++ b/local_devnet/scripts/start_node_and_create_validator.sh @@ -66,7 +66,8 @@ fi # start node celestia-appd start \ ---home="${CELESTIA_HOME}" \ ---moniker="${MONIKER}" \ ---p2p.persistent_peers=e3c592c0c2ad4b05cef3791456b0d6dd4da72ed2@core0:26656 \ ---rpc.laddr=tcp://0.0.0.0:26657 + --home="${CELESTIA_HOME}" \ + --moniker="${MONIKER}" \ + --p2p.persistent_peers=e3c592c0c2ad4b05cef3791456b0d6dd4da72ed2@core0:26656 \ + --rpc.laddr=tcp://0.0.0.0:26657 \ + --force-no-bbr From 0e73077dbcc838ec6a90aca17e7104667eb153ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nina=20/=20=E1=83=9C=E1=83=98=E1=83=9C=E1=83=90?= Date: Mon, 23 Sep 2024 22:10:32 +0200 Subject: [PATCH 39/85] test: consistent app hash on v2 (#3879) ## Overview Fixes #3624 --- app/test/consistent_apphash_test.go | 224 +++++++++++++++++++--------- 1 file changed, 153 insertions(+), 71 deletions(-) diff --git a/app/test/consistent_apphash_test.go b/app/test/consistent_apphash_test.go index 972f42da58..9a108e4dd1 100644 --- a/app/test/consistent_apphash_test.go +++ b/app/test/consistent_apphash_test.go @@ -8,19 +8,25 @@ import ( "github.com/celestiaorg/celestia-app/v3/app" "github.com/celestiaorg/celestia-app/v3/app/encoding" "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" + v1 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v1" + v2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" "github.com/celestiaorg/celestia-app/v3/pkg/user" testutil "github.com/celestiaorg/celestia-app/v3/test/util" "github.com/celestiaorg/celestia-app/v3/test/util/blobfactory" "github.com/celestiaorg/celestia-app/v3/test/util/testfactory" blobtypes "github.com/celestiaorg/celestia-app/v3/x/blob/types" + blobstreamtypes "github.com/celestiaorg/celestia-app/v3/x/blobstream/types" + signal "github.com/celestiaorg/celestia-app/v3/x/signal/types" "github.com/celestiaorg/go-square/v2/share" "github.com/celestiaorg/go-square/v2/tx" + "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/codec" "github.com/cosmos/cosmos-sdk/crypto/hd" "github.com/cosmos/cosmos-sdk/crypto/keyring" "github.com/cosmos/cosmos-sdk/crypto/keys/ed25519" "github.com/cosmos/cosmos-sdk/crypto/types" sdk "github.com/cosmos/cosmos-sdk/types" + vestingtypes "github.com/cosmos/cosmos-sdk/x/auth/vesting/types" "github.com/cosmos/cosmos-sdk/x/authz" banktypes "github.com/cosmos/cosmos-sdk/x/bank/types" crisisTypes "github.com/cosmos/cosmos-sdk/x/crisis/types" @@ -29,72 +35,125 @@ import ( govtypes "github.com/cosmos/cosmos-sdk/x/gov/types/v1" slashingtypes "github.com/cosmos/cosmos-sdk/x/slashing/types" stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types" + gethcommon "github.com/ethereum/go-ethereum/common" "github.com/stretchr/testify/require" abci "github.com/tendermint/tendermint/abci/types" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" "github.com/tendermint/tendermint/proto/tendermint/version" ) -type BlobTx struct { +type blobTx struct { author string blobs []*share.Blob txOptions []user.TxOption } -// TestConsistentAppHash executes all state machine messages, generates an app hash, +type ( + encodedSdkMessages func(*testing.T, []sdk.AccAddress, []stakingtypes.Validator, *app.App, *user.Signer, *user.Signer) ([][]byte, [][]byte, [][]byte) + encodedBlobTxs func(*testing.T, *user.Signer, []sdk.AccAddress) []byte +) + +type appHashTest struct { + name string + version uint64 + encodedSdkMessages encodedSdkMessages + encodedBlobTxs encodedBlobTxs + expectedDataRoot []byte + expectedAppHash []byte +} + +// TestConsistentAppHash executes all state machine messages on all app versions, generates an app hash, // and compares it against a previously generated hash from the same set of transactions. // App hashes across different commits should be consistent. func TestConsistentAppHash(t *testing.T) { - // Expected app hash produced by v1.x - https://github.com/celestiaorg/celestia-app/blob/v1.x/app/consistent_apphash_test.go - expectedAppHash := []byte{84, 216, 210, 48, 113, 204, 234, 21, 150, 236, 97, 87, 242, 184, 45, 248, 116, 127, 49, 88, 134, 197, 202, 125, 44, 210, 67, 144, 107, 51, 145, 65} - expectedDataRoot := []byte{100, 59, 112, 241, 238, 49, 50, 64, 105, 90, 209, 211, 49, 254, 211, 83, 133, 88, 5, 89, 221, 116, 141, 72, 33, 110, 16, 78, 5, 48, 118, 72} - - // Initialize testApp - testApp := testutil.NewTestApp() - enc := encoding.MakeConfig(app.ModuleEncodingRegisters...) - - // Create deterministic keys - kr, pubKeys := deterministicKeyRing(enc.Codec) - - // Apply genesis state to the app. - valKeyRing, _, err := testutil.SetupDeterministicGenesisState(testApp, pubKeys, 20_000_000_000, app.DefaultInitialConsensusParams()) - require.NoError(t, err) + tc := []appHashTest{ + { + name: "execute sdk messages and blob tx on v1", + version: v1.Version, + encodedSdkMessages: encodedSdkMessagesV1, + encodedBlobTxs: createEncodedBlobTx, + expectedDataRoot: []byte{30, 142, 46, 120, 191, 30, 242, 150, 164, 242, 166, 245, 89, 183, 181, 41, 88, 197, 11, 19, 243, 46, 69, 97, 3, 51, 27, 133, 68, 95, 95, 121}, + // Expected app hash produced by v1.x - https://github.com/celestiaorg/celestia-app/blob/v1.x/app/consistent_apphash_test.go + expectedAppHash: []byte{57, 128, 107, 57, 6, 131, 221, 188, 181, 181, 135, 58, 37, 240, 135, 66, 199, 107, 80, 154, 240, 176, 57, 36, 238, 69, 25, 188, 86, 203, 145, 145}, + }, + { + name: "execute sdk messages and blob tx on v2", + version: v2.Version, + encodedSdkMessages: func(t *testing.T, accountAddresses []sdk.AccAddress, genValidators []stakingtypes.Validator, testApp *app.App, signer *user.Signer, valSigner *user.Signer) ([][]byte, [][]byte, [][]byte) { + firstBlockEncodedTxs, secondBlockEncodedTxs, thirdBlockEncodedTxs := encodedSdkMessagesV1(t, accountAddresses, genValidators, testApp, signer, valSigner) + encodedMessagesV2 := encodedSdkMessagesV2(t, genValidators, valSigner) + thirdBlockEncodedTxs = append(thirdBlockEncodedTxs, encodedMessagesV2...) + + return firstBlockEncodedTxs, secondBlockEncodedTxs, thirdBlockEncodedTxs + }, + encodedBlobTxs: createEncodedBlobTx, + expectedDataRoot: []byte{200, 61, 245, 166, 119, 211, 170, 2, 73, 239, 253, 97, 243, 112, 116, 196, 70, 41, 201, 172, 123, 28, 15, 182, 52, 222, 122, 243, 95, 97, 66, 233}, + // Expected app hash produced on v2.x - https://github.com/celestiaorg/celestia-app/blob/v2.x/app/test/consistent_apphash_test.go + expectedAppHash: []byte{14, 115, 34, 28, 33, 70, 118, 3, 111, 250, 161, 185, 187, 151, 54, 78, 86, 37, 44, 252, 8, 26, 164, 251, 36, 20, 151, 170, 181, 84, 32, 136}, + }, + } - // ------------ Genesis User Accounts ------------ + for _, tt := range tc { + t.Run(tt.name, func(t *testing.T) { + testApp := testutil.NewTestApp() + enc := encoding.MakeConfig(app.ModuleEncodingRegisters...) + // Create deterministic keys + kr, pubKeys := deterministicKeyRing(enc.Codec) + consensusParams := app.DefaultConsensusParams() + consensusParams.Version.AppVersion = tt.version + // Apply genesis state to the app. + valKeyRing, _, err := testutil.SetupDeterministicGenesisState(testApp, pubKeys, 20_000_000_000, consensusParams) + require.NoError(t, err) + + // Get account names and addresses from the keyring and create signer + signer, accountAddresses := getAccountsAndCreateSigner(t, kr, enc.TxConfig, testutil.ChainID, tt.version, testApp) + // Validators from genesis state + genValidators := testApp.StakingKeeper.GetAllValidators(testApp.NewContext(false, tmproto.Header{})) + valSigner, _ := getAccountsAndCreateSigner(t, valKeyRing, enc.TxConfig, testutil.ChainID, tt.version, testApp) + + // Convert validators to ABCI validators + abciValidators, err := convertToABCIValidators(genValidators) + require.NoError(t, err) + + firstBlockTxs, secondBlockTxs, thirdBlockTxs := tt.encodedSdkMessages(t, accountAddresses, genValidators, testApp, signer, valSigner) + encodedBlobTx := tt.encodedBlobTxs(t, signer, accountAddresses) + + // Execute the first block + _, firstBlockAppHash, err := executeTxs(testApp, []byte{}, firstBlockTxs, abciValidators, testApp.LastCommitID().Hash) + require.NoError(t, err) + // Execute the second block + _, secondBlockAppHash, err := executeTxs(testApp, encodedBlobTx, secondBlockTxs, abciValidators, firstBlockAppHash) + require.NoError(t, err) + // Execute the final block and get the data root alongside the final app hash + finalDataRoot, finalAppHash, err := executeTxs(testApp, []byte{}, thirdBlockTxs, abciValidators, secondBlockAppHash) + require.NoError(t, err) + + // Require that the app hash is equal to the app hash produced on a different commit + require.Equal(t, tt.expectedAppHash, finalAppHash) + // Require that the data root is equal to the data root produced on a different commit + require.Equal(t, tt.expectedDataRoot, finalDataRoot) + }) + } +} +// getAccountsAndCreateSigner returns a signer with accounts +func getAccountsAndCreateSigner(t *testing.T, kr keyring.Keyring, enc client.TxConfig, chainID string, appVersion uint64, testApp *app.App) (*user.Signer, []sdk.AccAddress) { // Get account names and addresses from the keyring accountNames := testfactory.GetAccountNames(kr) accountAddresses := testfactory.GetAddresses(kr) - // Query keyring account infos accountInfos := queryAccountInfo(testApp, accountNames, kr) - // Create accounts for the signer accounts := createAccounts(accountInfos, accountNames) - // Create a signer with accounts - signer, err := user.NewSigner(kr, enc.TxConfig, testutil.ChainID, app.DefaultInitialVersion, accounts...) - require.NoError(t, err) - - // ------------ Genesis Validator Accounts ------------ - - // Validators from genesis state - genValidators := testApp.StakingKeeper.GetAllValidators(testApp.NewContext(false, tmproto.Header{})) - - // Get validator account names from the validator keyring - valAccountNames := testfactory.GetAccountNames(valKeyRing) - - // Query validator account infos - valAccountInfos := queryAccountInfo(testApp, valAccountNames, valKeyRing) - - // Create accounts for the validators' signer - valAccounts := createAccounts(valAccountInfos, valAccountNames) - - // Create a signer with validator accounts - valSigner, err := user.NewSigner(valKeyRing, enc.TxConfig, testutil.ChainID, app.DefaultInitialVersion, valAccounts...) + signer, err := user.NewSigner(kr, enc, chainID, appVersion, accounts...) require.NoError(t, err) + return signer, accountAddresses +} - // ----------- Create SDK Messages ------------ +// encodedSdkMessagesV1 returns encoded SDK messages for v1 +func encodedSdkMessagesV1(t *testing.T, accountAddresses []sdk.AccAddress, genValidators []stakingtypes.Validator, testApp *app.App, signer *user.Signer, valSigner *user.Signer) ([][]byte, [][]byte, [][]byte) { + // ----------- Create v1 SDK Messages ------------ amount := sdk.NewCoins(sdk.NewCoin(app.BondDenom, sdk.NewIntFromUint64(1_000))) // Minimum deposit required for a gov proposal to become active @@ -242,6 +301,27 @@ func TestConsistentAppHash(t *testing.T) { msgWithdrawDelegatorReward := distribution.NewMsgWithdrawDelegatorReward(accountAddresses[0], genValidators[0].GetOperator()) secondBlockSdkMsgs = append(secondBlockSdkMsgs, msgWithdrawDelegatorReward) + // NewMsgCreatePeriodicVestingAccount - creates a periodic vesting account + newAddress := sdk.AccAddress(ed25519.GenPrivKeyFromSecret([]byte("anotherAddress")).PubKey().Address()) + vestingPeriod := []vestingtypes.Period{ + { + Length: 3600, + Amount: amount, + }, + } + msgCreatePeriodicVestingAccount := vestingtypes.NewMsgCreatePeriodicVestingAccount(accountAddresses[3], newAddress, 2, vestingPeriod) + secondBlockSdkMsgs = append(secondBlockSdkMsgs, msgCreatePeriodicVestingAccount) + + // NewMsgCreatePermanentLockedAccount - creates a permanent locked account + newAddress = sdk.AccAddress(ed25519.GenPrivKeyFromSecret([]byte("anotherAddress2")).PubKey().Address()) + msgCreatePermamentLockedAccount := vestingtypes.NewMsgCreatePermanentLockedAccount(accountAddresses[3], newAddress, amount) + secondBlockSdkMsgs = append(secondBlockSdkMsgs, msgCreatePermamentLockedAccount) + + // NewMsgCreateVestingAccount - creates a vesting account + newAddress = sdk.AccAddress(ed25519.GenPrivKeyFromSecret([]byte("anotherAddress3")).PubKey().Address()) + msgCreateVestingAccount := vestingtypes.NewMsgCreateVestingAccount(accountAddresses[3], newAddress, amount, 1, 2, false) + secondBlockSdkMsgs = append(secondBlockSdkMsgs, msgCreateVestingAccount) + // ------------ Third Block ------------ // Txs within the third block are signed by the validator's signer @@ -255,51 +335,53 @@ func TestConsistentAppHash(t *testing.T) { msgUnjail := slashingtypes.NewMsgUnjail(genValidators[3].GetOperator()) thirdBlockSdkMsgs = append(thirdBlockSdkMsgs, msgUnjail) - // ------------ Construct Txs ------------ + // NewMsgRegisterEVMAddress - registers an EVM address + // This message was removed in v2 + if testApp.AppVersion() == v1.Version { + msgRegisterEVMAddress := blobstreamtypes.NewMsgRegisterEVMAddress(genValidators[1].GetOperator(), gethcommon.HexToAddress("hi")) + thirdBlockSdkMsgs = append(thirdBlockSdkMsgs, msgRegisterEVMAddress) + } - // Create SDK transactions from the list of messages - // and separate them into 3 different blocks - firstBlockEncodedTxs, err := processSdkMessages(signer, firstBlockSdkMsgs) + firstBlockTxs, err := processSdkMessages(signer, firstBlockSdkMsgs) require.NoError(t, err) - - secondBlockEncodedTxs, err := processSdkMessages(signer, secondBlockSdkMsgs) + secondBlockTxs, err := processSdkMessages(signer, secondBlockSdkMsgs) + require.NoError(t, err) + thirdBlockTxs, err := processSdkMessages(valSigner, thirdBlockSdkMsgs) require.NoError(t, err) - thirdBlockEncodedTxs, err := processSdkMessages(valSigner, thirdBlockSdkMsgs) + return firstBlockTxs, secondBlockTxs, thirdBlockTxs +} + +// encodedSdkMessagesV2 returns encoded SDK messages introduced in v2 +func encodedSdkMessagesV2(t *testing.T, genValidators []stakingtypes.Validator, valSigner *user.Signer) [][]byte { + var v2Messages []sdk.Msg + msgTryUpgrade := signal.NewMsgTryUpgrade(sdk.AccAddress(genValidators[0].GetOperator())) + v2Messages = append(v2Messages, msgTryUpgrade) + + msgSignalVersion := signal.NewMsgSignalVersion(genValidators[0].GetOperator(), 2) + v2Messages = append(v2Messages, msgSignalVersion) + + encodedTxs, err := processSdkMessages(valSigner, v2Messages) require.NoError(t, err) + return encodedTxs +} + +// createEncodedBlobTx creates, signs and returns an encoded blob transaction +func createEncodedBlobTx(t *testing.T, signer *user.Signer, accountAddresses []sdk.AccAddress) []byte { + senderAcc := signer.AccountByAddress(accountAddresses[1]) blob, err := share.NewBlob(fixedNamespace(), []byte{1}, appconsts.DefaultShareVersion, nil) require.NoError(t, err) // Create a Blob Tx - blobTx := BlobTx{ - author: accountNames[1], + blobTx := blobTx{ + author: senderAcc.Name(), blobs: []*share.Blob{blob}, txOptions: blobfactory.DefaultTxOpts(), } encodedBlobTx, _, err := signer.CreatePayForBlobs(blobTx.author, blobTx.blobs, blobTx.txOptions...) require.NoError(t, err) - - // Convert validators to ABCI validators - abciValidators, err := convertToABCIValidators(genValidators) - require.NoError(t, err) - - // Execute the first block - _, firstBlockAppHash, err := executeTxs(testApp, []byte{}, firstBlockEncodedTxs, abciValidators, testApp.LastCommitID().Hash) - require.NoError(t, err) - - // Execute the second block - _, secondBlockAppHash, err := executeTxs(testApp, encodedBlobTx, secondBlockEncodedTxs, abciValidators, firstBlockAppHash) - require.NoError(t, err) - - // Execute the final block and get the data root alongside the final app hash - finalDataRoot, finalAppHash, err := executeTxs(testApp, []byte{}, thirdBlockEncodedTxs, abciValidators, secondBlockAppHash) - require.NoError(t, err) - - // Require that the app hash is equal to the app hash produced on a different commit - require.Equal(t, expectedAppHash, finalAppHash) - // Require that the data root is equal to the data root produced on a different commit - require.Equal(t, expectedDataRoot, finalDataRoot) + return encodedBlobTx } // fixedNamespace returns a hardcoded namespace @@ -388,7 +470,7 @@ func executeTxs(testApp *app.App, encodedBlobTx []byte, encodedSdkTxs [][]byte, dataHash := resPrepareProposal.BlockData.Hash header := tmproto.Header{ - Version: version.Consensus{App: 1}, + Version: version.Consensus{App: testApp.AppVersion()}, DataHash: resPrepareProposal.BlockData.Hash, ChainID: chainID, Time: genesisTime.Add(time.Duration(height) * time.Minute), From 27dc26fc08aaec0ccd838c168de6ca62e7450df2 Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Tue, 24 Sep 2024 11:00:11 +0200 Subject: [PATCH 40/85] feat!: add chainbuilder tool (#3886) Adds a tool for quickly generating an entire chain (based on a single validator) --- go.mod | 13 +- go.sum | 24 +- pkg/user/account.go | 4 + test/util/genesis/document.go | 4 +- test/util/genesis/files.go | 34 +- test/util/genesis/genesis.go | 10 + test/util/testnode/network.go | 4 +- tools/chainbuilder/README.md | 27 ++ tools/chainbuilder/benchmark_test.go | 25 ++ tools/chainbuilder/integration_test.go | 103 +++++ tools/chainbuilder/main.go | 547 +++++++++++++++++++++++++ 11 files changed, 766 insertions(+), 29 deletions(-) create mode 100644 tools/chainbuilder/README.md create mode 100644 tools/chainbuilder/benchmark_test.go create mode 100644 tools/chainbuilder/integration_test.go create mode 100644 tools/chainbuilder/main.go diff --git a/go.mod b/go.mod index 91a2cb0eb5..42b518927a 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/celestiaorg/knuu v0.15.2 github.com/celestiaorg/nmt v0.22.1 github.com/celestiaorg/rsmt2d v0.14.0 + github.com/cometbft/cometbft-db v0.7.0 github.com/cosmos/cosmos-proto v1.0.0-beta.5 github.com/cosmos/cosmos-sdk v0.46.16 github.com/cosmos/gogoproto v1.7.0 @@ -59,17 +60,17 @@ require ( github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect github.com/celestiaorg/bittwister v0.0.0-20231213180407-65cdbaf5b8c7 // indirect github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 // indirect - github.com/cenkalti/backoff/v4 v4.1.3 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chzyer/readline v1.5.1 // indirect github.com/cilium/ebpf v0.12.3 // indirect github.com/cockroachdb/apd/v2 v2.0.2 // indirect github.com/coinbase/rosetta-sdk-go v0.7.9 // indirect - github.com/cometbft/cometbft-db v0.7.0 // indirect github.com/confio/ics23/go v0.9.1 // indirect github.com/consensys/bavard v0.1.13 // indirect github.com/consensys/gnark-crypto v0.12.1 // indirect + github.com/containerd/continuity v0.4.2 // indirect github.com/cosmos/btcutil v1.0.5 // indirect github.com/cosmos/go-bip39 v1.0.0 // indirect github.com/cosmos/gorocksdb v1.2.0 // indirect @@ -85,6 +86,10 @@ require ( github.com/dgraph-io/badger/v2 v2.2007.4 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/docker v26.1.5+incompatible // indirect + github.com/docker/go-connections v0.4.1-0.20210727194412-58542c764a11 // indirect + github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/dvsekhvalnov/jose2go v1.6.0 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect @@ -171,6 +176,8 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/onsi/ginkgo v1.16.5 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 // indirect github.com/pkg/errors v0.9.1 // indirect @@ -203,7 +210,7 @@ require ( github.com/x448/float16 v0.8.4 // indirect github.com/zondax/hid v0.9.2 // indirect github.com/zondax/ledger-go v0.14.3 // indirect - go.etcd.io/bbolt v1.3.6 // indirect + go.etcd.io/bbolt v1.3.10 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect diff --git a/go.sum b/go.sum index 78dc3649d2..d196f87450 100644 --- a/go.sum +++ b/go.sum @@ -213,6 +213,7 @@ github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3 github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo= github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= @@ -336,8 +337,8 @@ github.com/celestiaorg/rsmt2d v0.14.0/go.mod h1:4kxqiTdFev49sGiKXTDjohbWYOG5GlcI github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= -github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= @@ -403,8 +404,10 @@ github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1 github.com/consensys/gnark-crypto v0.5.3/go.mod h1:hOdPlWQV1gDLp7faZVeg8Y0iEPFaOUnCc4XeCCk96p0= github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= -github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= -github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= +github.com/containerd/continuity v0.4.2 h1:v3y/4Yz5jwnvqPKJJ+7Wf93fyWoCB3F5EclWG023MDM= +github.com/containerd/continuity v0.4.2/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -477,10 +480,13 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUn github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v26.1.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.1-0.20210727194412-58542c764a11 h1:IPrmumsT9t5BS7XcPhgsCTlkWbYg80SEXUzDpReaU6Y= github.com/docker/go-connections v0.4.1-0.20210727194412-58542c764a11/go.mod h1:a6bNUGTbQBsY6VRHTr4h/rkOXjl244DyRD0tx3fgq4Q= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -1085,8 +1091,8 @@ github.com/onsi/gomega v1.20.0/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeR github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b h1:YWuSjZCQAPM8UUBLkYUk1e+rZcvWHJmFb6i6rM44Xs8= -github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/opencontainers/runc v1.1.3 h1:vIXrkId+0/J2Ymu2m7VjGvbSlAId9XNRPhn2p4b+d8w= github.com/opencontainers/runc v1.1.3/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= @@ -1209,6 +1215,7 @@ github.com/shirou/gopsutil v3.21.6+incompatible h1:mmZtAlWSd8U2HeRTjswbnDLPxqsEo github.com/shirou/gopsutil v3.21.6+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= @@ -1348,8 +1355,8 @@ gitlab.com/NebulousLabs/errors v0.0.0-20200929122200-06c536cf6975/go.mod h1:ZkMZ gitlab.com/NebulousLabs/fastrand v0.0.0-20181126182046-603482d69e40 h1:dizWJqTWjwyD8KGcMOwgrkqu1JIkofYgKkmDeNE7oAs= gitlab.com/NebulousLabs/fastrand v0.0.0-20181126182046-603482d69e40/go.mod h1:rOnSnoRyxMI3fe/7KIbVcsHRGxe30OONv8dEgo+vCfA= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= +go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= @@ -1629,7 +1636,6 @@ golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/pkg/user/account.go b/pkg/user/account.go index 8ab1d031f4..9d66dc0540 100644 --- a/pkg/user/account.go +++ b/pkg/user/account.go @@ -40,6 +40,10 @@ func (a Account) PubKey() cryptotypes.PubKey { return a.pubKey } +func (a Account) AccountNumber() uint64 { + return a.accountNumber +} + // Sequence returns the sequence number of the account. // This is locally tracked func (a Account) Sequence() uint64 { diff --git a/test/util/genesis/document.go b/test/util/genesis/document.go index 3cc61f08b9..f45921431b 100644 --- a/test/util/genesis/document.go +++ b/test/util/genesis/document.go @@ -24,6 +24,7 @@ func Document( chainID string, gentxs []json.RawMessage, accounts []Account, + genesisTime time.Time, mods ...Modifier, ) (*coretypes.GenesisDoc, error) { genutilGenState := genutiltypes.DefaultGenesisState() @@ -73,7 +74,7 @@ func Document( // Create the genesis doc genesisDoc := &coretypes.GenesisDoc{ ChainID: chainID, - GenesisTime: time.Now(), + GenesisTime: genesisTime, ConsensusParams: params, AppState: stateBz, } @@ -101,7 +102,6 @@ func accountsToSDKTypes(accounts []Account) ([]banktypes.Balance, []authtypes.Ge ) genBals[i] = banktypes.Balance{Address: addr.String(), Coins: balances.Sort()} - genAccs[i] = authtypes.NewBaseAccount(addr, account.PubKey, uint64(i), 0) } return genBals, genAccs, nil diff --git a/test/util/genesis/files.go b/test/util/genesis/files.go index ff702d3c15..6d2452187b 100644 --- a/test/util/genesis/files.go +++ b/test/util/genesis/files.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" + srvconfig "github.com/cosmos/cosmos-sdk/server/config" "github.com/tendermint/tendermint/config" tmos "github.com/tendermint/tendermint/libs/os" "github.com/tendermint/tendermint/p2p" @@ -17,53 +18,58 @@ import ( func InitFiles( rootDir string, tmConfig *config.Config, + appCfg *srvconfig.Config, genesis *Genesis, validatorIndex int, -) (basePath string, err error) { +) error { val, has := genesis.Validator(validatorIndex) if !has { - return "", fmt.Errorf("validator %d not found", validatorIndex) + return fmt.Errorf("validator %d not found", validatorIndex) } - basePath = filepath.Join(rootDir, ".celestia-app") - tmConfig.SetRoot(basePath) + tmConfig.SetRoot(rootDir) // save the genesis file - configPath := filepath.Join(basePath, "config") - err = os.MkdirAll(configPath, os.ModePerm) + configPath := filepath.Join(rootDir, "config") + err := os.MkdirAll(configPath, os.ModePerm) if err != nil { - return "", err + return err } genesisDoc, err := genesis.Export() if err != nil { - return "", fmt.Errorf("exporting genesis: %w", err) + return fmt.Errorf("exporting genesis: %w", err) } err = genesisDoc.SaveAs(tmConfig.GenesisFile()) if err != nil { - return "", err + return err } pvStateFile := tmConfig.PrivValidatorStateFile() if err := tmos.EnsureDir(filepath.Dir(pvStateFile), 0o777); err != nil { - return "", err + return err } pvKeyFile := tmConfig.PrivValidatorKeyFile() if err := tmos.EnsureDir(filepath.Dir(pvKeyFile), 0o777); err != nil { - return "", err + return err } filePV := privval.NewFilePV(val.ConsensusKey, pvKeyFile, pvStateFile) filePV.Save() nodeKeyFile := tmConfig.NodeKeyFile() if err := tmos.EnsureDir(filepath.Dir(nodeKeyFile), 0o777); err != nil { - return "", err + return err } nodeKey := &p2p.NodeKey{ PrivKey: val.NetworkKey, } if err := nodeKey.SaveAs(nodeKeyFile); err != nil { - return "", err + return err } - return basePath, nil + appConfigFilePath := filepath.Join(rootDir, "config", "app.toml") + srvconfig.WriteConfigFile(appConfigFilePath, appCfg) + + config.WriteConfigFile(filepath.Join(rootDir, "config", "config.toml"), tmConfig) + + return nil } diff --git a/test/util/genesis/genesis.go b/test/util/genesis/genesis.go index 5a52fdbb29..83eab54250 100644 --- a/test/util/genesis/genesis.go +++ b/test/util/genesis/genesis.go @@ -120,6 +120,11 @@ func (g *Genesis) WithKeyringAccounts(accs ...KeyringAccount) *Genesis { return g } +func (g *Genesis) WithKeyring(kr keyring.Keyring) *Genesis { + g.kr = kr + return g +} + // AddAccount adds an existing account to the genesis. func (g *Genesis) AddAccount(account Account) error { if err := account.ValidateBasic(); err != nil { @@ -208,6 +213,7 @@ func (g *Genesis) Export() (*coretypes.GenesisDoc, error) { g.ChainID, gentxs, g.accounts, + g.GenesisTime, g.genOps..., ) } @@ -220,3 +226,7 @@ func (g *Genesis) Validator(i int) (Validator, bool) { } return Validator{}, false } + +func (g *Genesis) EncodingConfig() encoding.Config { + return g.ecfg +} diff --git a/test/util/testnode/network.go b/test/util/testnode/network.go index 79a1dc7d69..4c0421c895 100644 --- a/test/util/testnode/network.go +++ b/test/util/testnode/network.go @@ -2,6 +2,7 @@ package testnode import ( "context" + "path/filepath" "testing" "github.com/celestiaorg/celestia-app/v3/test/util/genesis" @@ -19,7 +20,8 @@ func NewNetwork(t testing.TB, config *Config) (cctx Context, rpcAddr, grpcAddr s t.Helper() // initialize the genesis file and validator files for the first validator. - baseDir, err := genesis.InitFiles(t.TempDir(), config.TmConfig, config.Genesis, 0) + baseDir := filepath.Join(t.TempDir(), "testnode") + err := genesis.InitFiles(baseDir, config.TmConfig, config.AppConfig, config.Genesis, 0) require.NoError(t, err) tmNode, app, err := NewCometNode(baseDir, &config.UniversalTestingConfig) diff --git a/tools/chainbuilder/README.md b/tools/chainbuilder/README.md new file mode 100644 index 0000000000..99911468ac --- /dev/null +++ b/tools/chainbuilder/README.md @@ -0,0 +1,27 @@ +# Chainbuilder + +`chainbuilder` is a tool for building a Celestia chain for testing and development purposes. + +## Usage + +Use `go` to run the binary as follows: + +``` +go run ./tools/chainbuilder +``` + +This will create a directory with the name `testnode-{chainID}`. All files will be populated and blocks generated based on specified input. You can run a validator on the file system afterwards by calling: + +``` +celestia-appd start --home /path/to/testnode-{chainID} +``` + +The following are the set of options when generating a chain: + +- `num-blocks` the number of blocks to be generated (default: 100) +- `block-size` the size of the blocks to be generated (default <2MB). This will be a single PFB transaction +- `square-size` the size of the max square (default: 128) +- `existing-dir` point this to a directory if you want to extend an existing chain rather than create a new one +- `namespace` allows you to pick a custom v0 namespace. By default "test" will be chosen. + +This tool takes roughly 60-70ms per 2MB block. diff --git a/tools/chainbuilder/benchmark_test.go b/tools/chainbuilder/benchmark_test.go new file mode 100644 index 0000000000..b72d9297e5 --- /dev/null +++ b/tools/chainbuilder/benchmark_test.go @@ -0,0 +1,25 @@ +package main + +import ( + "context" + "testing" + "time" + + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" +) + +func BenchmarkRun(b *testing.B) { + cfg := BuilderConfig{ + NumBlocks: 100, + BlockSize: appconsts.DefaultMaxBytes, + BlockInterval: time.Second, + } + + dir := b.TempDir() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := Run(context.Background(), cfg, dir); err != nil { + b.Fatal(err) + } + } +} diff --git a/tools/chainbuilder/integration_test.go b/tools/chainbuilder/integration_test.go new file mode 100644 index 0000000000..eb671cd1f0 --- /dev/null +++ b/tools/chainbuilder/integration_test.go @@ -0,0 +1,103 @@ +package main + +import ( + "context" + "fmt" + "path/filepath" + "testing" + "time" + + "github.com/celestiaorg/celestia-app/v3/app" + "github.com/celestiaorg/celestia-app/v3/app/encoding" + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" + "github.com/celestiaorg/celestia-app/v3/test/util" + "github.com/celestiaorg/celestia-app/v3/test/util/testnode" + "github.com/cosmos/cosmos-sdk/baseapp" + "github.com/tendermint/tendermint/libs/log" + tmrand "github.com/tendermint/tendermint/libs/rand" + "github.com/tendermint/tendermint/node" + "github.com/tendermint/tendermint/p2p" + "github.com/tendermint/tendermint/privval" + "github.com/tendermint/tendermint/proxy" + "github.com/tendermint/tendermint/rpc/client/local" + tmdbm "github.com/tendermint/tm-db" + + "github.com/stretchr/testify/require" +) + +func TestRun(t *testing.T) { + if testing.Short() { + t.Skip("skipping chainbuilder tool test") + } + + numBlocks := 10 + + cfg := BuilderConfig{ + NumBlocks: numBlocks, + BlockSize: appconsts.DefaultMaxBytes, + BlockInterval: time.Second, + ChainID: tmrand.Str(6), + Namespace: defaultNamespace, + } + + dir := t.TempDir() + + // First run + err := Run(context.Background(), cfg, dir) + require.NoError(t, err) + + // Second run with existing directory + cfg.ExistingDir = filepath.Join(dir, fmt.Sprintf("testnode-%s", cfg.ChainID)) + err = Run(context.Background(), cfg, dir) + require.NoError(t, err) + + tmCfg := testnode.DefaultTendermintConfig() + tmCfg.SetRoot(cfg.ExistingDir) + + appDB, err := tmdbm.NewDB("application", tmdbm.GoLevelDBBackend, tmCfg.DBDir()) + require.NoError(t, err) + + encCfg := encoding.MakeConfig(app.ModuleBasics) + + app := app.New( + log.NewNopLogger(), + appDB, + nil, + 0, + encCfg, + 0, + util.EmptyAppOptions{}, + baseapp.SetMinGasPrices(fmt.Sprintf("%f%s", appconsts.DefaultMinGasPrice, appconsts.BondDenom)), + ) + + nodeKey, err := p2p.LoadNodeKey(tmCfg.NodeKeyFile()) + require.NoError(t, err) + + cometNode, err := node.NewNode( + tmCfg, + privval.LoadOrGenFilePV(tmCfg.PrivValidatorKeyFile(), tmCfg.PrivValidatorStateFile()), + nodeKey, + proxy.NewLocalClientCreator(app), + node.DefaultGenesisDocProviderFunc(tmCfg), + node.DefaultDBProvider, + node.DefaultMetricsProvider(tmCfg.Instrumentation), + log.NewNopLogger(), + ) + require.NoError(t, err) + + require.NoError(t, cometNode.Start()) + defer func() { _ = cometNode.Stop() }() + + client := local.New(cometNode) + status, err := client.Status(context.Background()) + require.NoError(t, err) + require.NotNil(t, status) + // assert that the new node eventually makes progress in the chain + require.Eventually(t, func() bool { + status, err := client.Status(context.Background()) + require.NoError(t, err) + return status.SyncInfo.LatestBlockHeight >= int64(numBlocks*2) + }, time.Second*10, time.Millisecond*100) + require.NoError(t, cometNode.Stop()) + cometNode.Wait() +} diff --git a/tools/chainbuilder/main.go b/tools/chainbuilder/main.go new file mode 100644 index 0000000000..fe385084b3 --- /dev/null +++ b/tools/chainbuilder/main.go @@ -0,0 +1,547 @@ +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/celestiaorg/go-square/v2" + "github.com/celestiaorg/go-square/v2/share" + dbm "github.com/cometbft/cometbft-db" + "github.com/cosmos/cosmos-sdk/baseapp" + "github.com/cosmos/cosmos-sdk/crypto/keyring" + "github.com/spf13/cobra" + abci "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/crypto/merkle" + "github.com/tendermint/tendermint/libs/log" + tmrand "github.com/tendermint/tendermint/libs/rand" + "github.com/tendermint/tendermint/privval" + smproto "github.com/tendermint/tendermint/proto/tendermint/state" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" + sm "github.com/tendermint/tendermint/state" + "github.com/tendermint/tendermint/store" + "github.com/tendermint/tendermint/types" + tmdbm "github.com/tendermint/tm-db" + + "github.com/celestiaorg/celestia-app/v3/app" + "github.com/celestiaorg/celestia-app/v3/app/encoding" + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" + "github.com/celestiaorg/celestia-app/v3/pkg/da" + "github.com/celestiaorg/celestia-app/v3/pkg/user" + "github.com/celestiaorg/celestia-app/v3/test/util" + "github.com/celestiaorg/celestia-app/v3/test/util/genesis" + "github.com/celestiaorg/celestia-app/v3/test/util/testnode" + blobtypes "github.com/celestiaorg/celestia-app/v3/x/blob/types" +) + +var defaultNamespace share.Namespace + +const ( + defaultNamespaceStr = "test" + maxSquareSize = 512 +) + +func init() { + defaultNamespace = share.MustNewV0Namespace([]byte(defaultNamespaceStr)) +} + +func main() { + rootCmd := &cobra.Command{ + Use: "chainbuilder", + Short: "Build a Celestia chain", + RunE: func(cmd *cobra.Command, _ []string) error { + numBlocks, _ := cmd.Flags().GetInt("num-blocks") + blockSize, _ := cmd.Flags().GetInt("block-size") + blockInterval, _ := cmd.Flags().GetDuration("block-interval") + existingDir, _ := cmd.Flags().GetString("existing-dir") + namespaceStr, _ := cmd.Flags().GetString("namespace") + upToTime, _ := cmd.Flags().GetBool("up-to-now") + appVersion, _ := cmd.Flags().GetUint64("app-version") + chainID, _ := cmd.Flags().GetString("chain-id") + var namespace share.Namespace + if namespaceStr == "" { + namespace = defaultNamespace + } else { + var err error + namespace, err = share.NewV0Namespace([]byte(namespaceStr)) + if err != nil { + return fmt.Errorf("invalid namespace: %w", err) + } + } + + cfg := BuilderConfig{ + NumBlocks: numBlocks, + BlockSize: blockSize, + BlockInterval: blockInterval, + ExistingDir: existingDir, + Namespace: namespace, + ChainID: tmrand.Str(6), + UpToTime: upToTime, + AppVersion: appVersion, + } + + if chainID != "" { + cfg.ChainID = chainID + } + + dir, err := os.Getwd() + if err != nil { + return fmt.Errorf("failed to get current working directory: %w", err) + } + + return Run(cmd.Context(), cfg, dir) + }, + } + + rootCmd.Flags().Int("num-blocks", 100, "Number of blocks to generate") + rootCmd.Flags().Int("block-size", appconsts.DefaultMaxBytes, "Size of each block in bytes") + rootCmd.Flags().Duration("block-interval", time.Second, "Interval between blocks") + rootCmd.Flags().String("existing-dir", "", "Existing directory to load chain from") + rootCmd.Flags().String("namespace", "", "Custom namespace for the chain") + rootCmd.Flags().Bool("up-to-now", false, "Tool will terminate if the block time reaches the current time") + rootCmd.Flags().Uint64("app-version", appconsts.LatestVersion, "App version to use for the chain") + rootCmd.Flags().String("chain-id", "", "Chain ID to use for the chain. Defaults to a random 6 character string") + rootCmd.SilenceUsage = true + rootCmd.SilenceErrors = true + if err := rootCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +type BuilderConfig struct { + NumBlocks int + BlockSize int + BlockInterval time.Duration + ExistingDir string + Namespace share.Namespace + ChainID string + AppVersion uint64 + UpToTime bool +} + +func Run(ctx context.Context, cfg BuilderConfig, dir string) error { + startTime := time.Now().Add(-1 * cfg.BlockInterval * time.Duration(cfg.NumBlocks)).UTC() + currentTime := startTime + + encCfg := encoding.MakeConfig(app.ModuleBasics) + tmCfg := app.DefaultConsensusConfig() + var ( + gen *genesis.Genesis + kr keyring.Keyring + err error + ) + if cfg.ExistingDir == "" { + dir = filepath.Join(dir, fmt.Sprintf("testnode-%s", cfg.ChainID)) + kr, err = keyring.New(app.Name, keyring.BackendTest, dir, nil, encCfg.Codec) + if err != nil { + return fmt.Errorf("failed to create keyring: %w", err) + } + + validator := genesis.NewDefaultValidator(testnode.DefaultValidatorAccountName) + appCfg := app.DefaultAppConfig() + appCfg.Pruning = "everything" // we just want the last two states + appCfg.StateSync.SnapshotInterval = 0 + cp := app.DefaultConsensusParams() + + cp.Version.AppVersion = cfg.AppVersion // set the app version + gen = genesis.NewDefaultGenesis(). + WithConsensusParams(cp). + WithKeyring(kr). + WithChainID(cfg.ChainID). + WithGenesisTime(startTime). + WithValidators(validator) + + if err := genesis.InitFiles(dir, tmCfg, appCfg, gen, 0); err != nil { + return fmt.Errorf("failed to initialize genesis files: %w", err) + } + fmt.Println("Creating chain from scratch with Chain ID:", gen.ChainID) + } else { + cfgPath := filepath.Join(cfg.ExistingDir, "config/config.toml") + if _, err := os.Stat(cfgPath); os.IsNotExist(err) { + return fmt.Errorf("config file for existing chain not found at %s", cfgPath) + } + fmt.Println("Loading chain from existing directory:", cfg.ExistingDir) + tmCfg.SetRoot(cfg.ExistingDir) + kr, err = keyring.New(app.Name, keyring.BackendTest, cfg.ExistingDir, nil, encCfg.Codec) + if err != nil { + return fmt.Errorf("failed to load keyring: %w", err) + } + } + + validatorKey := privval.LoadFilePV(tmCfg.PrivValidatorKeyFile(), tmCfg.PrivValidatorStateFile()) + validatorAddr := validatorKey.Key.Address + + blockDB, err := dbm.NewDB("blockstore", dbm.GoLevelDBBackend, tmCfg.DBDir()) + if err != nil { + return fmt.Errorf("failed to create block database: %w", err) + } + + blockStore := store.NewBlockStore(blockDB) + + stateDB, err := dbm.NewDB("state", dbm.GoLevelDBBackend, tmCfg.DBDir()) + if err != nil { + return fmt.Errorf("failed to create state database: %w", err) + } + + stateStore := sm.NewStore(stateDB, sm.StoreOptions{ + DiscardABCIResponses: true, + }) + + appDB, err := tmdbm.NewDB("application", tmdbm.GoLevelDBBackend, tmCfg.DBDir()) + if err != nil { + return fmt.Errorf("failed to create application database: %w", err) + } + + simApp := app.New( + log.NewNopLogger(), + appDB, + nil, + 0, + encCfg, + 0, + util.EmptyAppOptions{}, + baseapp.SetMinGasPrices(fmt.Sprintf("%f%s", appconsts.DefaultMinGasPrice, appconsts.BondDenom)), + ) + + infoResp := simApp.Info(abci.RequestInfo{}) + + lastHeight := blockStore.Height() + if infoResp.LastBlockHeight != lastHeight { + return fmt.Errorf("last application height is %d, but the block store height is %d", infoResp.LastBlockHeight, lastHeight) + } + + if lastHeight == 0 { + if gen == nil { + return fmt.Errorf("non empty directory but no blocks found") + } + + genDoc, err := gen.Export() + if err != nil { + return fmt.Errorf("failed to export genesis document: %w", err) + } + + state, err := stateStore.LoadFromDBOrGenesisDoc(genDoc) + if err != nil { + return fmt.Errorf("failed to load state from database or genesis document: %w", err) + } + + validators := make([]*types.Validator, len(genDoc.Validators)) + for i, val := range genDoc.Validators { + validators[i] = types.NewValidator(val.PubKey, val.Power) + } + validatorSet := types.NewValidatorSet(validators) + nextVals := types.TM2PB.ValidatorUpdates(validatorSet) + csParams := types.TM2PB.ConsensusParams(genDoc.ConsensusParams) + res := simApp.InitChain(abci.RequestInitChain{ + ChainId: genDoc.ChainID, + Time: genDoc.GenesisTime, + ConsensusParams: csParams, + Validators: nextVals, + AppStateBytes: genDoc.AppState, + InitialHeight: genDoc.InitialHeight, + }) + + vals, err := types.PB2TM.ValidatorUpdates(res.Validators) + if err != nil { + return fmt.Errorf("failed to convert validator updates: %w", err) + } + state.Validators = types.NewValidatorSet(vals) + state.NextValidators = types.NewValidatorSet(vals).CopyIncrementProposerPriority(1) + state.AppHash = res.AppHash + state.LastResultsHash = merkle.HashFromByteSlices(nil) + if err := stateStore.Save(state); err != nil { + return fmt.Errorf("failed to save initial state: %w", err) + } + currentTime = currentTime.Add(cfg.BlockInterval) + } else { + fmt.Println("Starting from height", lastHeight) + } + state, err := stateStore.Load() + if err != nil { + return fmt.Errorf("failed to load state: %w", err) + } + if cfg.ExistingDir != "" { + // if this is extending an existing chain, we want to start + // the time to be where the existing chain left off + currentTime = state.LastBlockTime.Add(cfg.BlockInterval) + } + + if state.ConsensusParams.Version.AppVersion != cfg.AppVersion { + return fmt.Errorf("app version mismatch: state has %d, but cfg has %d", state.ConsensusParams.Version.AppVersion, cfg.AppVersion) + } + + if state.LastBlockHeight != lastHeight { + return fmt.Errorf("last block height mismatch: state has %d, but block store has %d", state.LastBlockHeight, lastHeight) + } + + validatorPower := state.Validators.Validators[0].VotingPower + + signer, err := user.NewSigner( + kr, + encCfg.TxConfig, + state.ChainID, + state.ConsensusParams.Version.AppVersion, + user.NewAccount(testnode.DefaultValidatorAccountName, 0, uint64(lastHeight)+1), + ) + if err != nil { + return fmt.Errorf("failed to create new signer: %w", err) + } + + var ( + errCh = make(chan error, 2) + dataCh = make(chan *tmproto.Data, 100) + persistCh = make(chan persistData, 100) + commit = types.NewCommit(0, 0, types.BlockID{}, nil) + ) + if lastHeight > 0 { + commit = blockStore.LoadSeenCommit(lastHeight) + } + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + go func() { + errCh <- generateSquareRoutine(ctx, signer, cfg, dataCh) + }() + + go func() { + errCh <- persistDataRoutine(ctx, stateStore, blockStore, persistCh) + }() + + lastBlock := blockStore.LoadBlock(blockStore.Height()) + + for height := lastHeight + 1; height <= int64(cfg.NumBlocks)+lastHeight; height++ { + if cfg.UpToTime && lastBlock != nil && lastBlock.Time.Add(cfg.BlockInterval).After(time.Now().UTC()) { + fmt.Printf("blocks cannot be generated into the future, stopping at height %d\n", lastBlock.Height) + break + } + + select { + case <-ctx.Done(): + return ctx.Err() + case dataPB := <-dataCh: + data, err := types.DataFromProto(dataPB) + if err != nil { + return fmt.Errorf("failed to convert data from protobuf: %w", err) + } + block, blockParts := state.MakeBlock(height, data, commit, nil, validatorAddr) + blockID := types.BlockID{ + Hash: block.Hash(), + PartSetHeader: blockParts.Header(), + } + + precommitVote := &tmproto.Vote{ + Height: height, + Round: 0, + Type: tmproto.PrecommitType, + BlockID: blockID.ToProto(), + ValidatorAddress: validatorAddr, + Timestamp: currentTime, + Signature: nil, + } + + if err := validatorKey.SignVote(state.ChainID, precommitVote); err != nil { + return fmt.Errorf("failed to sign precommit vote (%s): %w", precommitVote.String(), err) + } + + commitSig := types.CommitSig{ + BlockIDFlag: types.BlockIDFlagCommit, + ValidatorAddress: validatorAddr, + Timestamp: currentTime, + Signature: precommitVote.Signature, + } + commit = types.NewCommit(height, 0, blockID, []types.CommitSig{commitSig}) + + var lastCommitInfo abci.LastCommitInfo + if height > 1 { + lastCommitInfo = abci.LastCommitInfo{ + Round: 0, + Votes: []abci.VoteInfo{ + { + Validator: abci.Validator{ + Address: validatorAddr, + Power: validatorPower, + }, + SignedLastBlock: true, + }, + }, + } + } + + beginBlockResp := simApp.BeginBlock(abci.RequestBeginBlock{ + Hash: block.Hash(), + Header: *block.Header.ToProto(), + LastCommitInfo: lastCommitInfo, + }) + + deliverTxResponses := make([]*abci.ResponseDeliverTx, len(block.Data.Txs)) + + for idx, tx := range block.Data.Txs { + blobTx, isBlobTx := types.UnmarshalBlobTx(tx) + if isBlobTx { + tx = blobTx.Tx + } + deliverTxResponse := simApp.DeliverTx(abci.RequestDeliverTx{ + Tx: tx, + }) + if deliverTxResponse.Code != abci.CodeTypeOK { + return fmt.Errorf("failed to deliver tx: %s", deliverTxResponse.Log) + } + deliverTxResponses[idx] = &deliverTxResponse + } + + endBlockResp := simApp.EndBlock(abci.RequestEndBlock{ + Height: block.Height, + }) + + commitResp := simApp.Commit() + state.LastBlockHeight = height + state.LastBlockID = blockID + state.LastBlockTime = block.Time + state.LastValidators = state.Validators + state.Validators = state.NextValidators + state.NextValidators = state.NextValidators.CopyIncrementProposerPriority(1) + state.AppHash = commitResp.Data + state.LastResultsHash = sm.ABCIResponsesResultsHash(&smproto.ABCIResponses{ + DeliverTxs: deliverTxResponses, + BeginBlock: &beginBlockResp, + EndBlock: &endBlockResp, + }) + currentTime = currentTime.Add(cfg.BlockInterval) + persistCh <- persistData{ + state: state.Copy(), + block: block, + seenCommit: &types.Commit{ + Height: commit.Height, + Round: commit.Round, + BlockID: commit.BlockID, + Signatures: []types.CommitSig{commitSig}, + }, + } + } + } + + close(dataCh) + close(persistCh) + + var firstErr error + for i := 0; i < cap(errCh); i++ { + err := <-errCh + if err != nil && firstErr == nil { + firstErr = err + } + } + + if err := blockDB.Close(); err != nil { + return fmt.Errorf("failed to close block database: %w", err) + } + if err := stateDB.Close(); err != nil { + return fmt.Errorf("failed to close state database: %w", err) + } + if err := appDB.Close(); err != nil { + return fmt.Errorf("failed to close application database: %w", err) + } + + fmt.Println("Chain built successfully", state.LastBlockHeight) + + return firstErr +} + +func generateSquareRoutine( + ctx context.Context, + signer *user.Signer, + cfg BuilderConfig, + dataCh chan<- *tmproto.Data, +) error { + for i := 0; i < cfg.NumBlocks; i++ { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + account := signer.Accounts()[0] + + blob, err := share.NewV0Blob(cfg.Namespace, crypto.CRandBytes(cfg.BlockSize)) + if err != nil { + return err + } + + blobGas := blobtypes.DefaultEstimateGas([]uint32{uint32(cfg.BlockSize)}) + fee := float64(blobGas) * appconsts.DefaultMinGasPrice * 2 + tx, _, err := signer.CreatePayForBlobs(account.Name(), []*share.Blob{blob}, user.SetGasLimit(blobGas), user.SetFee(uint64(fee))) + if err != nil { + return err + } + if err := signer.IncrementSequence(account.Name()); err != nil { + return err + } + + dataSquare, txs, err := square.Build( + [][]byte{tx}, + maxSquareSize, + appconsts.SubtreeRootThreshold(1), + ) + if err != nil { + return err + } + + eds, err := da.ExtendShares(share.ToBytes(dataSquare)) + if err != nil { + return err + } + + dah, err := da.NewDataAvailabilityHeader(eds) + if err != nil { + return err + } + + select { + case dataCh <- &tmproto.Data{ + Txs: txs, + Hash: dah.Hash(), + SquareSize: uint64(dataSquare.Size()), + }: + case <-ctx.Done(): + return ctx.Err() + } + } + return nil +} + +type persistData struct { + state sm.State + block *types.Block + seenCommit *types.Commit +} + +func persistDataRoutine( + ctx context.Context, + stateStore sm.Store, + blockStore *store.BlockStore, + dataCh <-chan persistData, +) error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + case data, ok := <-dataCh: + if !ok { + return nil + } + blockParts := data.block.MakePartSet(types.BlockPartSizeBytes) + blockStore.SaveBlock(data.block, blockParts, data.seenCommit) + if blockStore.Height()%100 == 0 { + fmt.Println("Reached height", blockStore.Height()) + } + + if err := stateStore.Save(data.state); err != nil { + return err + } + } + } +} From 418ddbfe4bcde2836e3460caba9add61f97a5709 Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Tue, 24 Sep 2024 11:37:50 +0200 Subject: [PATCH 41/85] feat: add upgrade sequence to tx sim (#3890) Adds the ability to use txsim to upgrade a network --- test/txsim/run_test.go | 52 +++++++++++++++++++++++++++ test/txsim/upgrade.go | 82 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 test/txsim/upgrade.go diff --git a/test/txsim/run_test.go b/test/txsim/run_test.go index 46e4523710..4a922f15a1 100644 --- a/test/txsim/run_test.go +++ b/test/txsim/run_test.go @@ -12,12 +12,17 @@ import ( "github.com/celestiaorg/celestia-app/v3/app" "github.com/celestiaorg/celestia-app/v3/app/encoding" + v2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" + v3 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v3" "github.com/celestiaorg/celestia-app/v3/test/txsim" "github.com/celestiaorg/celestia-app/v3/test/util/testnode" "github.com/cosmos/cosmos-sdk/crypto/keyring" sdk "github.com/cosmos/cosmos-sdk/types" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" blob "github.com/celestiaorg/celestia-app/v3/x/blob/types" + signaltypes "github.com/celestiaorg/celestia-app/v3/x/signal/types" bank "github.com/cosmos/cosmos-sdk/x/bank/types" distribution "github.com/cosmos/cosmos-sdk/x/distribution/types" staking "github.com/cosmos/cosmos-sdk/x/staking/types" @@ -152,3 +157,50 @@ func Setup(t testing.TB) (keyring.Keyring, string, string) { return cctx.Keyring, rpcAddr, grpcAddr } + +func TestTxSimUpgrade(t *testing.T) { + if testing.Short() { + t.Skip("skipping TestTxSimUpgrade in short mode.") + } + cp := app.DefaultConsensusParams() + cp.Version.AppVersion = v2.Version + cfg := testnode.DefaultConfig(). + WithTimeoutCommit(300 * time.Millisecond). + WithConsensusParams(cp). + WithFundedAccounts("txsim-master") + cctx, _, grpcAddr := testnode.NewNetwork(t, cfg) + + // updrade to v3 at height 20 + sequences := []txsim.Sequence{ + txsim.NewUpgradeSequence(v3.Version, 20), + } + + opts := txsim.DefaultOptions(). + // SuppressLogs(). + WithPollTime(time.Millisecond * 100) + + err := txsim.Run( + cctx.GoContext(), + grpcAddr, + cctx.Keyring, + encoding.MakeConfig(app.ModuleEncodingRegisters...), + opts, + sequences..., + ) + require.NoError(t, err) + + conn, err := grpc.NewClient(grpcAddr, grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + defer conn.Close() + + querier := signaltypes.NewQueryClient(conn) + + // We can't check that the upgrade was successful because the upgrade height is thousands of blocks away + // and even at 300 millisecond block times, it would take too long. Instead we just want to assert + // that the upgrade is ready to be performed + require.Eventually(t, func() bool { + upgradePlan, err := querier.GetUpgrade(cctx.GoContext(), &signaltypes.QueryGetUpgradeRequest{}) + require.NoError(t, err) + return upgradePlan.Upgrade != nil && upgradePlan.Upgrade.AppVersion == v3.Version + }, time.Second*20, time.Millisecond*100) +} diff --git a/test/txsim/upgrade.go b/test/txsim/upgrade.go new file mode 100644 index 0000000000..0811f32232 --- /dev/null +++ b/test/txsim/upgrade.go @@ -0,0 +1,82 @@ +package txsim + +import ( + "context" + "errors" + "math/rand" + + signaltypes "github.com/celestiaorg/celestia-app/v3/x/signal/types" + "github.com/cosmos/cosmos-sdk/types" + stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types" + "github.com/gogo/protobuf/grpc" +) + +var _ Sequence = &UpgradeSequence{} + +const fundsForUpgrade = 100_000 + +// UpgradeSequence simulates an upgrade proposal and voting process +type UpgradeSequence struct { + voted map[string]bool + height int64 + version uint64 + account types.AccAddress + hasUpgraded bool +} + +func NewUpgradeSequence(version uint64, height int64) *UpgradeSequence { + return &UpgradeSequence{version: version, height: height, voted: make(map[string]bool)} +} + +func (s *UpgradeSequence) Clone(_ int) []Sequence { + panic("cloning not supported for upgrade sequence. Only a single sequence is needed") +} + +// this is a no-op for the upgrade sequence +func (s *UpgradeSequence) Init(_ context.Context, _ grpc.ClientConn, allocateAccounts AccountAllocator, _ *rand.Rand, _ bool) { + s.account = allocateAccounts(1, fundsForUpgrade)[0] +} + +func (s *UpgradeSequence) Next(ctx context.Context, querier grpc.ClientConn, _ *rand.Rand) (Operation, error) { + if s.hasUpgraded { + return Operation{}, ErrEndOfSequence + } + + stakingQuerier := stakingtypes.NewQueryClient(querier) + validatorsResp, err := stakingQuerier.Validators(ctx, &stakingtypes.QueryValidatorsRequest{}) + if err != nil { + return Operation{}, err + } + + if len(validatorsResp.Validators) == 0 { + return Operation{}, errors.New("no validators found") + } + + // Choose a random validator to be the authority + var msg types.Msg + for _, validator := range validatorsResp.Validators { + if !s.voted[validator.OperatorAddress] { + msg = &signaltypes.MsgSignalVersion{ + ValidatorAddress: validator.OperatorAddress, + Version: s.version, + } + s.voted[validator.OperatorAddress] = true + } + } + // if all validators have voted, we can now try to upgrade. + if msg == nil { + msg = signaltypes.NewMsgTryUpgrade(s.account) + s.hasUpgraded = true + } + + delay := uint64(0) + // apply a delay to the first sequence only + if len(s.voted) == 0 { + delay = uint64(s.height) + } + + return Operation{ + Msgs: []types.Msg{msg}, + Delay: delay, + }, nil +} From 051b268b9928ae6aa361fffb0f24e699c1882d07 Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Tue, 24 Sep 2024 12:16:06 +0200 Subject: [PATCH 42/85] feat!: add custom build tags to override certain global variables (#3897) Closes: https://github.com/celestiaorg/celestia-app/issues/3603 Technically this uses LD flags not build flags --- Dockerfile | 8 ++++++++ Makefile | 8 +++++++- app/app.go | 3 ++- app/module/configurator_test.go | 2 +- app/test/upgrade_test.go | 7 +++---- pkg/appconsts/global_consts.go | 19 +++++++++++++++++++ pkg/appconsts/overrides.go | 10 ++++++++++ pkg/appconsts/versioned_consts.go | 9 +++++++++ x/signal/integration_test.go | 4 ++-- x/signal/keeper.go | 19 ++++++++++--------- x/signal/keeper_test.go | 9 +++++---- 11 files changed, 76 insertions(+), 22 deletions(-) create mode 100644 pkg/appconsts/overrides.go diff --git a/Dockerfile b/Dockerfile index 050d45a306..2b504642db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,12 @@ ARG BUILDER_IMAGE=docker.io/golang:1.22.6-alpine3.19 ARG RUNTIME_IMAGE=docker.io/alpine:3.19 ARG TARGETOS ARG TARGETARCH +# Use build args to override the maxuimum square size of the docker image i.e. +# docker build --build-arg MAX_SQUARE_SIZE=64 -t celestia-app:latest . +ARG MAX_SQUARE_SIZE +# Use build args to override the upgrade height delay of the docker image i.e. +# docker build --build-arg UPGRADE_HEIGHT_DELAY=1000 -t celestia-app:latest . +ARG UPGRADE_HEIGHT_DELAY # Stage 1: Build the celestia-appd binary inside a builder image that will be discarded later. # Ignore hadolint rule because hadolint can't parse the variable. @@ -28,6 +34,8 @@ COPY . /celestia-app WORKDIR /celestia-app RUN uname -a &&\ CGO_ENABLED=${CGO_ENABLED} GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + OVERRIDE_MAX_SQUARE_SIZE=${MAX_SQUARE_SIZE} \ + OVERRIDE_UPGRADE_HEIGHT_DELAY=${UPGRADE_HEIGHT_DELAY} \ make build # Stage 2: Create a minimal image to run the celestia-appd binary diff --git a/Makefile b/Makefile index d291c6d2b3..afe8520e2f 100644 --- a/Makefile +++ b/Makefile @@ -8,12 +8,18 @@ PROJECTNAME=$(shell basename "$(PWD)") HTTPS_GIT := https://github.com/celestiaorg/celestia-app.git PACKAGE_NAME := github.com/celestiaorg/celestia-app/v3 GOLANG_CROSS_VERSION ?= v1.22.6 +# Set this to override the max square size of the binary +OVERRIDE_MAX_SQUARE_SIZE ?= +# Set this to override the upgrade height delay of the binary +OVERRIDE_UPGRADE_HEIGHT_DELAY ?= # process linker flags ldflags = -X github.com/cosmos/cosmos-sdk/version.Name=celestia-app \ -X github.com/cosmos/cosmos-sdk/version.AppName=celestia-appd \ -X github.com/cosmos/cosmos-sdk/version.Version=$(VERSION) \ -X github.com/cosmos/cosmos-sdk/version.Commit=$(COMMIT) \ + -X github.com/celestiaorg/celestia-app/v3/pkg/appconsts.OverrideSquareSizeUpperBoundStr=$(OVERRIDE_MAX_SQUARE_SIZE) \ + -X github.com/celestiaorg/celestia-app/v3/pkg/appconsts.OverrideUpgradeHeightDelayStr=$(OVERRIDE_UPGRADE_HEIGHT_DELAY) BUILD_FLAGS := -tags "ledger" -ldflags '$(ldflags)' @@ -31,7 +37,7 @@ build: mod .PHONY: build ## install: Build and install the celestia-appd binary into the $GOPATH/bin directory. -install: go.sum check-bbr +install: check-bbr @echo "--> Installing celestia-appd" @go install $(BUILD_FLAGS) ./cmd/celestia-appd .PHONY: install diff --git a/app/app.go b/app/app.go index d31d33fcbf..ce44df0b2f 100644 --- a/app/app.go +++ b/app/app.go @@ -10,6 +10,7 @@ import ( celestiatx "github.com/celestiaorg/celestia-app/v3/app/grpc/tx" "github.com/celestiaorg/celestia-app/v3/app/module" "github.com/celestiaorg/celestia-app/v3/app/posthandler" + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" appv1 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v1" appv2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" appv3 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v3" @@ -279,7 +280,7 @@ func New( ), ) - app.SignalKeeper = signal.NewKeeper(appCodec, keys[signaltypes.StoreKey], app.StakingKeeper) + app.SignalKeeper = signal.NewKeeper(appCodec, keys[signaltypes.StoreKey], app.StakingKeeper, appconsts.UpgradeHeightDelay()) app.IBCKeeper = ibckeeper.NewKeeper( appCodec, diff --git a/app/module/configurator_test.go b/app/module/configurator_test.go index a93713938a..ed504c9ac3 100644 --- a/app/module/configurator_test.go +++ b/app/module/configurator_test.go @@ -37,7 +37,7 @@ func TestConfigurator(t *testing.T) { stateStore.MountStoreWithDB(storeKey, storetypes.StoreTypeIAVL, db) require.NoError(t, stateStore.LoadLatestVersion()) - keeper := signal.NewKeeper(config.Codec, storeKey, nil) + keeper := signal.NewKeeper(config.Codec, storeKey, nil, 0) require.NotNil(t, keeper) upgradeModule := signal.NewAppModule(keeper) manager, err := module.NewManager([]module.VersionedModule{ diff --git a/app/test/upgrade_test.go b/app/test/upgrade_test.go index 459a33aefc..ab188e01ac 100644 --- a/app/test/upgrade_test.go +++ b/app/test/upgrade_test.go @@ -17,7 +17,6 @@ import ( "github.com/celestiaorg/celestia-app/v3/test/util/testnode" blobstreamtypes "github.com/celestiaorg/celestia-app/v3/x/blobstream/types" "github.com/celestiaorg/celestia-app/v3/x/minfee" - "github.com/celestiaorg/celestia-app/v3/x/signal" signaltypes "github.com/celestiaorg/celestia-app/v3/x/signal/types" "github.com/celestiaorg/go-square/v2/share" "github.com/celestiaorg/go-square/v2/tx" @@ -99,7 +98,7 @@ func TestAppUpgradeV3(t *testing.T) { // brace yourselfs, this part may take a while initialHeight := int64(4) - for height := initialHeight; height < initialHeight+signal.DefaultUpgradeHeightDelay; height++ { + for height := initialHeight; height < initialHeight+appconsts.DefaultUpgradeHeightDelay; height++ { _ = testApp.BeginBlock(abci.RequestBeginBlock{ Header: tmproto.Header{ Height: height, @@ -108,7 +107,7 @@ func TestAppUpgradeV3(t *testing.T) { }) endBlockResp = testApp.EndBlock(abci.RequestEndBlock{ - Height: 3 + signal.DefaultUpgradeHeightDelay, + Height: 3 + appconsts.DefaultUpgradeHeightDelay, }) _ = testApp.Commit() @@ -130,7 +129,7 @@ func TestAppUpgradeV3(t *testing.T) { _ = testApp.BeginBlock(abci.RequestBeginBlock{ Header: tmproto.Header{ ChainID: genesis.ChainID, - Height: initialHeight + signal.DefaultUpgradeHeightDelay, + Height: initialHeight + appconsts.DefaultUpgradeHeightDelay, Version: tmversion.Consensus{App: 3}, }, }) diff --git a/pkg/appconsts/global_consts.go b/pkg/appconsts/global_consts.go index 340f651840..2903da0414 100644 --- a/pkg/appconsts/global_consts.go +++ b/pkg/appconsts/global_consts.go @@ -1,6 +1,8 @@ package appconsts import ( + "strconv" + "github.com/celestiaorg/go-square/v2/share" "github.com/celestiaorg/rsmt2d" "github.com/tendermint/tendermint/pkg/consts" @@ -24,6 +26,11 @@ const ( // BondDenom defines the native staking denomination BondDenom = "utia" + + // DefaultUpgradeHeightDelay is the number of blocks after a quorum has been + // reached that the chain should upgrade to the new version. Assuming a block + // interval of 12 seconds, this is 7 days. + DefaultUpgradeHeightDelay = int64(7 * 24 * 60 * 60 / 12) // 7 days * 24 hours * 60 minutes * 60 seconds / 12 seconds per block = 50,400 blocks. ) var ( @@ -44,6 +51,18 @@ var ( SupportedShareVersions = share.SupportedShareVersions ) +// returns the delay in blocks after a quorum has been reached that the chain should upgrade to the new version. +func UpgradeHeightDelay() int64 { + if OverrideUpgradeHeightDelayStr != "" { + parsedValue, err := strconv.ParseInt(OverrideUpgradeHeightDelayStr, 10, 64) + if err != nil { + panic("Invalid OverrideUpgradeHeightDelayStr value") + } + return parsedValue + } + return DefaultUpgradeHeightDelay +} + // HashLength returns the length of a hash in bytes. func HashLength() int { return hashLength diff --git a/pkg/appconsts/overrides.go b/pkg/appconsts/overrides.go new file mode 100644 index 0000000000..39c5f4c6d0 --- /dev/null +++ b/pkg/appconsts/overrides.go @@ -0,0 +1,10 @@ +package appconsts + +// Set of values that can be overridden at compile time to modify the behavior of the app. +// WARNING: This should only be modified for testing purposes. All nodes in a network +// must have the same values for these constants. +// Look at the Makefile to see how these are set. +var ( + OverrideSquareSizeUpperBoundStr string + OverrideUpgradeHeightDelayStr string +) diff --git a/pkg/appconsts/versioned_consts.go b/pkg/appconsts/versioned_consts.go index 7bb1088a8b..c5ffb952a6 100644 --- a/pkg/appconsts/versioned_consts.go +++ b/pkg/appconsts/versioned_consts.go @@ -1,6 +1,8 @@ package appconsts import ( + "strconv" + v3 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v3" ) @@ -22,6 +24,13 @@ func SubtreeRootThreshold(_ uint64) int { // SquareSizeUpperBound imposes an upper bound on the max effective square size. func SquareSizeUpperBound(_ uint64) int { + if OverrideSquareSizeUpperBoundStr != "" { + parsedValue, err := strconv.Atoi(OverrideSquareSizeUpperBoundStr) + if err != nil { + panic("Invalid OverrideSquareSizeUpperBoundStr value") + } + return parsedValue + } return v3.SquareSizeUpperBound } diff --git a/x/signal/integration_test.go b/x/signal/integration_test.go index fb016e0813..23b8158dd2 100644 --- a/x/signal/integration_test.go +++ b/x/signal/integration_test.go @@ -4,9 +4,9 @@ import ( "testing" "github.com/celestiaorg/celestia-app/v3/app" + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" v2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" testutil "github.com/celestiaorg/celestia-app/v3/test/util" - "github.com/celestiaorg/celestia-app/v3/x/signal" "github.com/celestiaorg/celestia-app/v3/x/signal/types" "github.com/stretchr/testify/require" @@ -77,7 +77,7 @@ func TestUpgradeIntegration(t *testing.T) { require.False(t, shouldUpgrade) require.EqualValues(t, 0, version) - ctx = ctx.WithBlockHeight(ctx.BlockHeight() + signal.DefaultUpgradeHeightDelay) + ctx = ctx.WithBlockHeight(ctx.BlockHeight() + appconsts.DefaultUpgradeHeightDelay) shouldUpgrade, version = app.SignalKeeper.ShouldUpgrade(ctx) require.True(t, shouldUpgrade) diff --git a/x/signal/keeper.go b/x/signal/keeper.go index ed280c67f4..0a23d93119 100644 --- a/x/signal/keeper.go +++ b/x/signal/keeper.go @@ -12,11 +12,6 @@ import ( stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types" ) -// DefaultUpgradeHeightDelay is the number of blocks after a quorum has been -// reached that the chain should upgrade to the new version. Assuming a block -// interval of 12 seconds, this is 7 days. -const DefaultUpgradeHeightDelay = int64(7 * 24 * 60 * 60 / 12) // 7 days * 24 hours * 60 minutes * 60 seconds / 12 seconds per block = 50,400 blocks. - // Keeper implements the MsgServer and QueryServer interfaces var ( _ types.MsgServer = &Keeper{} @@ -46,6 +41,10 @@ type Keeper struct { // stakingKeeper is used to fetch validators to calculate the total power // signalled to a version. stakingKeeper StakingKeeper + + // upgradeHeightDelayBlocks is the number of blocks after a quorum has been + // reached that the chain should upgrade to the new version + upgradeHeightDelayBlocks int64 } // NewKeeper returns a signal keeper. @@ -53,11 +52,13 @@ func NewKeeper( binaryCodec codec.BinaryCodec, storeKey storetypes.StoreKey, stakingKeeper StakingKeeper, + upgradeHeightDelayBlocks int64, ) Keeper { return Keeper{ - binaryCodec: binaryCodec, - storeKey: storeKey, - stakingKeeper: stakingKeeper, + binaryCodec: binaryCodec, + storeKey: storeKey, + stakingKeeper: stakingKeeper, + upgradeHeightDelayBlocks: upgradeHeightDelayBlocks, } } @@ -108,7 +109,7 @@ func (k *Keeper) TryUpgrade(ctx context.Context, _ *types.MsgTryUpgrade) (*types } upgrade := types.Upgrade{ AppVersion: version, - UpgradeHeight: sdkCtx.BlockHeader().Height + DefaultUpgradeHeightDelay, + UpgradeHeight: sdkCtx.BlockHeader().Height + k.upgradeHeightDelayBlocks, } k.setUpgrade(sdkCtx, upgrade) } diff --git a/x/signal/keeper_test.go b/x/signal/keeper_test.go index a476562696..71627a8417 100644 --- a/x/signal/keeper_test.go +++ b/x/signal/keeper_test.go @@ -12,6 +12,7 @@ import ( "github.com/celestiaorg/celestia-app/v3/app" "github.com/celestiaorg/celestia-app/v3/app/encoding" + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" v1 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v1" v2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" "github.com/celestiaorg/celestia-app/v3/x/signal" @@ -68,7 +69,7 @@ func TestGetVotingPowerThreshold(t *testing.T) { t.Run(tc.name, func(t *testing.T) { config := encoding.MakeConfig(app.ModuleEncodingRegisters...) stakingKeeper := newMockStakingKeeper(tc.validators) - k := signal.NewKeeper(config.Codec, nil, stakingKeeper) + k := signal.NewKeeper(config.Codec, nil, stakingKeeper, appconsts.DefaultUpgradeHeightDelay) got := k.GetVotingPowerThreshold(sdk.Context{}) assert.Equal(t, tc.want, got, fmt.Sprintf("want %v, got %v", tc.want.String(), got.String())) }) @@ -182,7 +183,7 @@ func TestTallyingLogic(t *testing.T) { require.False(t, shouldUpgrade) // should be false because upgrade height hasn't been reached. require.Equal(t, uint64(0), version) - ctx = ctx.WithBlockHeight(ctx.BlockHeight() + signal.DefaultUpgradeHeightDelay) + ctx = ctx.WithBlockHeight(ctx.BlockHeight() + appconsts.DefaultUpgradeHeightDelay) shouldUpgrade, version = upgradeKeeper.ShouldUpgrade(ctx) require.True(t, shouldUpgrade) // should be true because upgrade height has been reached. @@ -425,7 +426,7 @@ func TestGetUpgrade(t *testing.T) { got, err := upgradeKeeper.GetUpgrade(ctx, &types.QueryGetUpgradeRequest{}) require.NoError(t, err) assert.Equal(t, v2.Version, got.Upgrade.AppVersion) - assert.Equal(t, signal.DefaultUpgradeHeightDelay, got.Upgrade.UpgradeHeight) + assert.Equal(t, appconsts.DefaultUpgradeHeightDelay, got.Upgrade.UpgradeHeight) }) } @@ -451,7 +452,7 @@ func setup(t *testing.T) (signal.Keeper, sdk.Context, *mockStakingKeeper) { ) config := encoding.MakeConfig(app.ModuleEncodingRegisters...) - upgradeKeeper := signal.NewKeeper(config.Codec, signalStore, mockStakingKeeper) + upgradeKeeper := signal.NewKeeper(config.Codec, signalStore, mockStakingKeeper, appconsts.DefaultUpgradeHeightDelay) return upgradeKeeper, mockCtx, mockStakingKeeper } From af38e411496b323df68b316715f06172ab45c0ad Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Tue, 24 Sep 2024 18:39:38 +0200 Subject: [PATCH 43/85] feat: add cli command for using upgrade sequences with txsim (#3903) We need this so our e2e tests can schedule an upgrade to a new network version --- test/cmd/txsim/cli.go | 35 +++++++++++++++++++++++++++++++++++ test/cmd/txsim/cli_test.go | 1 + 2 files changed, 36 insertions(+) diff --git a/test/cmd/txsim/cli.go b/test/cmd/txsim/cli.go index 62988c3eae..bf334437c4 100644 --- a/test/cmd/txsim/cli.go +++ b/test/cmd/txsim/cli.go @@ -40,6 +40,7 @@ var ( send, sendIterations, sendAmount int stake, stakeValue, blob int useFeegrant, suppressLogs bool + upgradeSchedule string ) func main() { @@ -130,6 +131,15 @@ well funded account that can act as the master account. The command runs until a sequences = append(sequences, txsim.NewBlobSequence(sizes, blobsPerPFB).Clone(blob)...) } + upgradeScheduleMap, err := parseUpgradeSchedule(upgradeSchedule) + if err != nil { + return fmt.Errorf("invalid upgrade schedule: %w", err) + } + + for height, version := range upgradeScheduleMap { + sequences = append(sequences, txsim.NewUpgradeSequence(version, height)) + } + if seed == 0 { if os.Getenv(TxsimSeed) != "" { seed, err = strconv.ParseInt(os.Getenv(TxsimSeed), 10, 64) @@ -195,6 +205,7 @@ func flags() *flag.FlagSet { flags.IntVar(&stake, "stake", 0, "number of stake sequences to run") flags.IntVar(&stakeValue, "stake-value", 1000, "amount of initial stake per sequence") flags.IntVar(&blob, "blob", 0, "number of blob sequences to run") + flags.StringVar(&upgradeSchedule, "upgrade-schedule", "", "upgrade schedule for the network in format height:version i.e. 100:3,200:4") flags.StringVar(&blobSizes, "blob-sizes", "100-1000", "range of blob sizes to send") flags.StringVar(&blobAmounts, "blob-amounts", "1", "range of blobs per PFB specified as a single value or a min-max range (e.g., 10 or 5-10). A single value indicates the exact number of blobs to be created.") flags.BoolVar(&useFeegrant, "feegrant", false, "use the feegrant module to pay for fees") @@ -224,3 +235,27 @@ func readRange(r string) (txsim.Range, error) { return txsim.NewRange(n, m), nil } + +func parseUpgradeSchedule(schedule string) (map[int64]uint64, error) { + scheduleMap := make(map[int64]uint64) + if schedule == "" { + return nil, nil + } + scheduleParts := strings.Split(schedule, ",") + for _, part := range scheduleParts { + parts := strings.Split(part, ":") + if len(parts) != 2 { + return nil, fmt.Errorf("invalid upgrade schedule format: %s", part) + } + height, err := strconv.ParseInt(parts[0], 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid height in upgrade schedule: %s", parts[0]) + } + version, err := strconv.ParseUint(parts[1], 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid version in upgrade schedule: %s", parts[1]) + } + scheduleMap[height] = version + } + return scheduleMap, nil +} diff --git a/test/cmd/txsim/cli_test.go b/test/cmd/txsim/cli_test.go index 3520221af0..ad6d1c2268 100644 --- a/test/cmd/txsim/cli_test.go +++ b/test/cmd/txsim/cli_test.go @@ -29,6 +29,7 @@ func TestTxsimCommandFlags(t *testing.T) { "--grpc-endpoint", grpcAddr, "--blob", "5", "--seed", "1234", + "--upgrade-schedule", "10:3", }) err := cmd.ExecuteContext(ctx) require.NoError(t, err) From 9624691c4b6a73f6ad5e6def13cfc6f7359025ff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Sep 2024 09:32:33 -0400 Subject: [PATCH 44/85] chore(deps): Bump celestiaorg/.github from 0.4.3 to 0.4.4 (#3906) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [celestiaorg/.github](https://github.com/celestiaorg/.github) from 0.4.3 to 0.4.4.
Release notes

Sourced from celestiaorg/.github's releases.

v0.4.4

What's Changed

Full Changelog: https://github.com/celestiaorg/.github/compare/v0.4.3...v0.4.4

Commits
  • 0dffae8 feat: add checkout ref to support building from a specific ref manually (#112)
  • b1ea695 chore(deps): bump actions/add-to-project from 0.5.0 to 1.0.2 (#111)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=celestiaorg/.github&package-manager=github_actions&previous-version=0.4.3&new-version=0.4.4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/create_release_tracking_epic.yml | 2 +- .github/workflows/docker-build-publish.yml | 4 ++-- .github/workflows/lint.yml | 4 ++-- .github/workflows/pr-review-requester.yml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/create_release_tracking_epic.yml b/.github/workflows/create_release_tracking_epic.yml index 2fa90be14a..9d2fe78162 100644 --- a/.github/workflows/create_release_tracking_epic.yml +++ b/.github/workflows/create_release_tracking_epic.yml @@ -7,7 +7,7 @@ on: types: [released] jobs: trigger_issue: - uses: celestiaorg/.github/.github/workflows/reusable_create_release_tracking_epic.yml@v0.4.3 + uses: celestiaorg/.github/.github/workflows/reusable_create_release_tracking_epic.yml@v0.4.4 secrets: inherit with: release-repo: ${{ github.repository }} diff --git a/.github/workflows/docker-build-publish.yml b/.github/workflows/docker-build-publish.yml index f805f343dd..44a0f3a034 100644 --- a/.github/workflows/docker-build-publish.yml +++ b/.github/workflows/docker-build-publish.yml @@ -20,7 +20,7 @@ jobs: permissions: contents: write packages: write - uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_pipeline.yml@v0.4.3 + uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_pipeline.yml@v0.4.4 with: dockerfile: Dockerfile secrets: inherit @@ -29,7 +29,7 @@ jobs: permissions: contents: write packages: write - uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_pipeline.yml@v0.4.3 + uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_pipeline.yml@v0.4.4 with: dockerfile: docker/Dockerfile_txsim packageName: txsim diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4716adabf1..443bb3c707 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -30,10 +30,10 @@ jobs: # hadolint lints the Dockerfile hadolint: - uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_lint.yml@v0.4.3 + uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_lint.yml@v0.4.4 yamllint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: celestiaorg/.github/.github/actions/yamllint@v0.4.3 + - uses: celestiaorg/.github/.github/actions/yamllint@v0.4.4 diff --git a/.github/workflows/pr-review-requester.yml b/.github/workflows/pr-review-requester.yml index a46df08794..d6afa96637 100644 --- a/.github/workflows/pr-review-requester.yml +++ b/.github/workflows/pr-review-requester.yml @@ -11,7 +11,7 @@ on: jobs: auto-request-review: name: Auto request reviews - uses: celestiaorg/.github/.github/workflows/reusable_housekeeping.yml@v0.4.3 + uses: celestiaorg/.github/.github/workflows/reusable_housekeeping.yml@v0.4.4 secrets: inherit # write access for issues and pull requests is needed because the called # workflow requires write access to issues and pull requests and the From 00805f0ece68d20f2fa3d927d26cc56d7f9f400a Mon Sep 17 00:00:00 2001 From: smuu <18609909+smuu@users.noreply.github.com> Date: Wed, 25 Sep 2024 15:35:39 +0200 Subject: [PATCH 45/85] feat: build on network tags and manual dispatch (#3905) This PR add two features: 1. Allow docker to trigger on network specific tags. 2. This PR shows how to use the new input for the docker ci workflow added in celestiaorg/.github#112 This would allow triggering the docker workflow on a tag that (such as the new arabica and mocha tags) that didn't trigger the workflow originally. Ref: - https://github.com/celestiaorg/celestia-node/pull/3765 - https://github.com/celestiaorg/celestia-node/pull/3766 --- .github/workflows/docker-build-publish.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/docker-build-publish.yml b/.github/workflows/docker-build-publish.yml index 44a0f3a034..9d43669287 100644 --- a/.github/workflows/docker-build-publish.yml +++ b/.github/workflows/docker-build-publish.yml @@ -13,7 +13,15 @@ on: - "v[0-9]+.[0-9]+.[0-9]+-alpha.[0-9]+" - "v[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+" - "v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+" + - "v[0-9]+.[0-9]+.[0-9]+-arabica" + - "v[0-9]+.[0-9]+.[0-9]+-mocha" pull_request: + workflow_dispatch: + inputs: + ref: + description: "The checkout reference (ie tag, branch, sha)" + required: true + type: string jobs: docker-security-build: From e24c0e8f72ea571bec637625a232b858c23a92e8 Mon Sep 17 00:00:00 2001 From: Moji Date: Wed, 25 Sep 2024 17:07:43 +0330 Subject: [PATCH 46/85] chore: add a knuu getter (#3900) While attempting to use testnet pkg in the node repo. There is a need to use `knuu` object that is initialized by the `testnet.New()` to create new instances and interact with `knuu`. Since it is not exported, this PR proposed to have a getter func for it, so users can interact with knuu easily. --- test/e2e/testnet/testnet.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/e2e/testnet/testnet.go b/test/e2e/testnet/testnet.go index 6b4ff136ed..c04c723b73 100644 --- a/test/e2e/testnet/testnet.go +++ b/test/e2e/testnet/testnet.go @@ -57,6 +57,10 @@ func New(ctx context.Context, name string, seed int64, grafana *GrafanaInfo, cha }, nil } +func (t *Testnet) Knuu() *knuu.Knuu { + return t.knuu +} + func (t *Testnet) NewPreloader() (*preloader.Preloader, error) { if t.knuu == nil { return nil, errors.New("knuu is not initialized") From e9278ed5a457d5b1052e733a6074dba990732101 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nina=20/=20=E1=83=9C=E1=83=98=E1=83=9C=E1=83=90?= Date: Wed, 25 Sep 2024 21:24:36 +0200 Subject: [PATCH 47/85] refactor: index submitted txs in tx client and improve nonce management (#3830) ## Overview Fixes #3899 --------- Co-authored-by: Callum Waters Co-authored-by: Rootul P --- Makefile | 2 +- pkg/user/pruning_test.go | 50 ++++++++++ pkg/user/tx_client.go | 198 +++++++++++++++++++------------------ pkg/user/tx_client_test.go | 113 ++++++++++++++++----- 4 files changed, 242 insertions(+), 121 deletions(-) create mode 100644 pkg/user/pruning_test.go diff --git a/Makefile b/Makefile index afe8520e2f..67d69d5c24 100644 --- a/Makefile +++ b/Makefile @@ -149,7 +149,7 @@ test-race: # TODO: Remove the -skip flag once the following tests no longer contain data races. # https://github.com/celestiaorg/celestia-app/issues/1369 @echo "--> Running tests in race mode" - @go test ./... -v -race -skip "TestPrepareProposalConsistency|TestIntegrationTestSuite|TestBlobstreamRPCQueries|TestSquareSizeIntegrationTest|TestStandardSDKIntegrationTestSuite|TestTxsimCommandFlags|TestTxsimCommandEnvVar|TestMintIntegrationTestSuite|TestBlobstreamCLI|TestUpgrade|TestMaliciousTestNode|TestBigBlobSuite|TestQGBIntegrationSuite|TestSignerTestSuite|TestPriorityTestSuite|TestTimeInPrepareProposalContext|TestBlobstream|TestCLITestSuite|TestLegacyUpgrade|TestSignerTwins|TestConcurrentTxSubmission|TestTxClientTestSuite|Test_testnode" + @go test ./... -v -race -skip "TestPrepareProposalConsistency|TestIntegrationTestSuite|TestBlobstreamRPCQueries|TestSquareSizeIntegrationTest|TestStandardSDKIntegrationTestSuite|TestTxsimCommandFlags|TestTxsimCommandEnvVar|TestMintIntegrationTestSuite|TestBlobstreamCLI|TestUpgrade|TestMaliciousTestNode|TestBigBlobSuite|TestQGBIntegrationSuite|TestSignerTestSuite|TestPriorityTestSuite|TestTimeInPrepareProposalContext|TestBlobstream|TestCLITestSuite|TestLegacyUpgrade|TestSignerTwins|TestConcurrentTxSubmission|TestTxClientTestSuite|Test_testnode|TestEvictions" .PHONY: test-race ## test-bench: Run unit tests in bench mode. diff --git a/pkg/user/pruning_test.go b/pkg/user/pruning_test.go new file mode 100644 index 0000000000..6019325b19 --- /dev/null +++ b/pkg/user/pruning_test.go @@ -0,0 +1,50 @@ +package user + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestPruningInTxTracker(t *testing.T) { + txClient := &TxClient{ + txTracker: make(map[string]txInfo), + } + numTransactions := 10 + + // Add 10 transactions to the tracker that are 10 and 5 minutes old + var txsToBePruned int + var txsNotReadyToBePruned int + for i := 0; i < numTransactions; i++ { + // 5 transactions will be pruned + if i%2 == 0 { + txClient.txTracker["tx"+fmt.Sprint(i)] = txInfo{ + signer: "signer" + fmt.Sprint(i), + sequence: uint64(i), + timestamp: time.Now(). + Add(-10 * time.Minute), + } + txsToBePruned++ + } else { + txClient.txTracker["tx"+fmt.Sprint(i)] = txInfo{ + signer: "signer" + fmt.Sprint(i), + sequence: uint64(i), + timestamp: time.Now(). + Add(-5 * time.Minute), + } + txsNotReadyToBePruned++ + } + } + + txTrackerBeforePruning := len(txClient.txTracker) + + // All transactions were indexed + require.Equal(t, numTransactions, len(txClient.txTracker)) + txClient.pruneTxTracker() + // Prunes the transactions that are 10 minutes old + // 5 transactions will be pruned + require.Equal(t, txsToBePruned, txTrackerBeforePruning-txsToBePruned) + require.Equal(t, len(txClient.txTracker), txsNotReadyToBePruned) +} diff --git a/pkg/user/tx_client.go b/pkg/user/tx_client.go index a403a91991..ce2f7b8933 100644 --- a/pkg/user/tx_client.go +++ b/pkg/user/tx_client.go @@ -12,7 +12,6 @@ import ( "time" "github.com/celestiaorg/go-square/v2/share" - blobtx "github.com/celestiaorg/go-square/v2/tx" "github.com/cosmos/cosmos-sdk/client" nodeservice "github.com/cosmos/cosmos-sdk/client/grpc/node" "github.com/cosmos/cosmos-sdk/client/grpc/tmservice" @@ -27,7 +26,6 @@ import ( "github.com/celestiaorg/celestia-app/v3/app" "github.com/celestiaorg/celestia-app/v3/app/encoding" - apperrors "github.com/celestiaorg/celestia-app/v3/app/errors" "github.com/celestiaorg/celestia-app/v3/app/grpc/tx" "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" "github.com/celestiaorg/celestia-app/v3/x/blob/types" @@ -35,12 +33,21 @@ import ( ) const ( - DefaultPollTime = 3 * time.Second - DefaultGasMultiplier float64 = 1.1 + DefaultPollTime = 3 * time.Second + DefaultGasMultiplier float64 = 1.1 + txTrackerPruningInterval = 10 * time.Minute ) type Option func(client *TxClient) +// txInfo is a struct that holds the sequence and the signer of a transaction +// in the local tx pool. +type txInfo struct { + sequence uint64 + signer string + timestamp time.Time +} + // TxResponse is a response from the chain after // a transaction has been submitted. type TxResponse struct { @@ -137,6 +144,9 @@ type TxClient struct { defaultGasPrice float64 defaultAccount string defaultAddress sdktypes.AccAddress + // txTracker maps the tx hash to the Sequence and signer of the transaction + // that was submitted to the chain + txTracker map[string]txInfo } // NewTxClient returns a new signer using the provided keyring @@ -169,6 +179,7 @@ func NewTxClient( defaultGasPrice: appconsts.DefaultMinGasPrice, defaultAccount: records[0].Name, defaultAddress: addr, + txTracker: make(map[string]txInfo), } for _, opt := range options { @@ -302,6 +313,12 @@ func (client *TxClient) SubmitTx(ctx context.Context, msgs []sdktypes.Msg, opts func (client *TxClient) BroadcastTx(ctx context.Context, msgs []sdktypes.Msg, opts ...TxOption) (*sdktypes.TxResponse, error) { client.mtx.Lock() defer client.mtx.Unlock() + + // prune transactions that are older than 10 minutes + // pruning has to be done in broadcast, since users + // might not always call ConfirmTx(). + client.pruneTxTracker() + account, err := client.getAccountNameFromMsgs(msgs) if err != nil { return nil, err @@ -368,23 +385,20 @@ func (client *TxClient) broadcastTx(ctx context.Context, txBytes []byte, signer return nil, err } if resp.TxResponse.Code != abci.CodeTypeOK { - if apperrors.IsNonceMismatchCode(resp.TxResponse.Code) { - // query the account to update the sequence number on-chain for the account - _, seqNum, err := QueryAccount(ctx, client.grpc, client.registry, client.signer.accounts[signer].address) - if err != nil { - return nil, fmt.Errorf("querying account for new sequence number: %w\noriginal tx response: %s", err, resp.TxResponse.RawLog) - } - if err := client.signer.SetSequence(signer, seqNum); err != nil { - return nil, fmt.Errorf("setting sequence: %w", err) - } - return client.retryBroadcastingTx(ctx, txBytes) - } broadcastTxErr := &BroadcastTxError{ TxHash: resp.TxResponse.TxHash, Code: resp.TxResponse.Code, ErrorLog: resp.TxResponse.RawLog, } - return resp.TxResponse, broadcastTxErr + return nil, broadcastTxErr + } + + // save the sequence and signer of the transaction in the local txTracker + // before the sequence is incremented + client.txTracker[resp.TxResponse.TxHash] = txInfo{ + sequence: client.signer.accounts[signer].Sequence(), + signer: signer, + timestamp: time.Now(), } // after the transaction has been submitted, we can increment the @@ -395,62 +409,13 @@ func (client *TxClient) broadcastTx(ctx context.Context, txBytes []byte, signer return resp.TxResponse, nil } -// retryBroadcastingTx creates a new transaction by copying over an existing transaction but creates a new signature with the -// new sequence number. It then calls `broadcastTx` and attempts to submit the transaction -func (client *TxClient) retryBroadcastingTx(ctx context.Context, txBytes []byte) (*sdktypes.TxResponse, error) { - blobTx, isBlobTx, err := blobtx.UnmarshalBlobTx(txBytes) - if isBlobTx { - // only check the error if the bytes are supposed to be of type blob tx - if err != nil { - return nil, err +// pruneTxTracker removes transactions from the local tx tracker that are older than 10 minutes +func (client *TxClient) pruneTxTracker() { + for hash, txInfo := range client.txTracker { + if time.Since(txInfo.timestamp) >= txTrackerPruningInterval { + delete(client.txTracker, hash) } - txBytes = blobTx.Tx - } - tx, err := client.signer.DecodeTx(txBytes) - if err != nil { - return nil, err } - - opts := make([]TxOption, 0) - if granter := tx.FeeGranter(); granter != nil { - opts = append(opts, SetFeeGranter(granter)) - } - if payer := tx.FeePayer(); payer != nil { - opts = append(opts, SetFeePayer(payer)) - } - if memo := tx.GetMemo(); memo != "" { - opts = append(opts, SetMemo(memo)) - } - if fee := tx.GetFee(); fee != nil { - opts = append(opts, SetFee(fee.AmountOf(appconsts.BondDenom).Uint64())) - } - if gas := tx.GetGas(); gas > 0 { - opts = append(opts, SetGasLimit(gas)) - } - - txBuilder, err := client.signer.txBuilder(tx.GetMsgs(), opts...) - if err != nil { - return nil, err - } - signer, _, err := client.signer.signTransaction(txBuilder) - if err != nil { - return nil, fmt.Errorf("resigning transaction: %w", err) - } - - newTxBytes, err := client.signer.EncodeTx(txBuilder.GetTx()) - if err != nil { - return nil, err - } - - // rewrap the blob tx if it was originally a blob tx - if isBlobTx { - newTxBytes, err = blobtx.MarshalBlobTx(newTxBytes, blobTx.Blobs...) - if err != nil { - return nil, err - } - } - - return client.broadcastTx(ctx, newTxBytes, signer) } // ConfirmTx periodically pings the provided node for the commitment of a transaction by its @@ -468,40 +433,68 @@ func (client *TxClient) ConfirmTx(ctx context.Context, txHash string) (*TxRespon return nil, err } - if resp != nil { - switch resp.Status { - case core.TxStatusPending: - // Continue polling if the transaction is still pending - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-pollTicker.C: - continue - } - case core.TxStatusCommitted: - txResponse := &TxResponse{ - Height: resp.Height, - TxHash: txHash, - Code: resp.ExecutionCode, - } - if resp.ExecutionCode != abci.CodeTypeOK { - executionErr := &ExecutionError{ - TxHash: txHash, - Code: resp.ExecutionCode, - ErrorLog: resp.Error, - } - return nil, executionErr + switch resp.Status { + case core.TxStatusPending: + // Continue polling if the transaction is still pending + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-pollTicker.C: + continue + } + case core.TxStatusCommitted: + txResponse := &TxResponse{ + Height: resp.Height, + TxHash: txHash, + Code: resp.ExecutionCode, + } + if resp.ExecutionCode != abci.CodeTypeOK { + executionErr := &ExecutionError{ + TxHash: txHash, + Code: resp.ExecutionCode, + ErrorLog: resp.Error, } - return txResponse, nil - case core.TxStatusEvicted: - return nil, fmt.Errorf("tx was evicted from the mempool") - default: - return nil, fmt.Errorf("unknown tx: %s", txHash) + client.deleteFromTxTracker(txHash) + return nil, executionErr } + client.deleteFromTxTracker(txHash) + return txResponse, nil + case core.TxStatusEvicted: + return nil, client.handleEvictions(txHash) + default: + client.deleteFromTxTracker(txHash) + return nil, fmt.Errorf("transaction with hash %s not found; it was likely rejected", txHash) } } } +// handleEvictions handles the scenario where a transaction is evicted from the mempool. +// It removes the evicted transaction from the local tx tracker without incrementing +// the signer's sequence. +func (client *TxClient) handleEvictions(txHash string) error { + client.mtx.Lock() + defer client.mtx.Unlock() + // Get transaction from the local tx tracker + txInfo, exists := client.txTracker[txHash] + if !exists { + return fmt.Errorf("tx: %s not found in tx client txTracker; likely failed during broadcast", txHash) + } + // The sequence should be rolled back to the sequence of the transaction that was evicted to be + // ready for resubmission. All transactions with a later nonce will be kicked by the nodes tx pool. + if err := client.signer.SetSequence(txInfo.signer, txInfo.sequence); err != nil { + return fmt.Errorf("setting sequence: %w", err) + } + delete(client.txTracker, txHash) + return fmt.Errorf("tx was evicted from the mempool") +} + +// deleteFromTxTracker safely deletes a transaction from the local tx tracker. +func (client *TxClient) deleteFromTxTracker(txHash string) { + client.mtx.Lock() + defer client.mtx.Unlock() + delete(client.txTracker, txHash) +} + // EstimateGas simulates the transaction, calculating the amount of gas that was consumed during execution. The final // result will be multiplied by gasMultiplier(that is set in TxClient) func (client *TxClient) EstimateGas(ctx context.Context, msgs []sdktypes.Msg, opts ...TxOption) (uint64, error) { @@ -576,6 +569,7 @@ func (client *TxClient) checkAccountLoaded(ctx context.Context, account string) if err != nil { return fmt.Errorf("retrieving address from keyring: %w", err) } + // FIXME: have a less trusting way of getting the account number and sequence accNum, sequence, err := QueryAccount(ctx, client.grpc, client.registry, addr) if err != nil { return fmt.Errorf("querying account %s: %w", account, err) @@ -604,6 +598,14 @@ func (client *TxClient) getAccountNameFromMsgs(msgs []sdktypes.Msg) (string, err return record.Name, nil } +// GetTxFromTxTracker gets transaction info from the tx client's local tx tracker by its hash +func (client *TxClient) GetTxFromTxTracker(hash string) (sequence uint64, signer string, exists bool) { + client.mtx.Lock() + defer client.mtx.Unlock() + txInfo, exists := client.txTracker[hash] + return txInfo.sequence, txInfo.signer, exists +} + // Signer exposes the tx clients underlying signer func (client *TxClient) Signer() *Signer { return client.signer diff --git a/pkg/user/tx_client_test.go b/pkg/user/tx_client_test.go index 1588c3a060..6ae0c2efa2 100644 --- a/pkg/user/tx_client_test.go +++ b/pkg/user/tx_client_test.go @@ -5,21 +5,21 @@ import ( "testing" "time" + "github.com/celestiaorg/celestia-app/v3/app" + "github.com/celestiaorg/celestia-app/v3/app/encoding" + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" + "github.com/celestiaorg/celestia-app/v3/pkg/user" + "github.com/celestiaorg/celestia-app/v3/test/util/blobfactory" + "github.com/celestiaorg/celestia-app/v3/test/util/testnode" sdk "github.com/cosmos/cosmos-sdk/types" sdktx "github.com/cosmos/cosmos-sdk/types/tx" + "github.com/cosmos/cosmos-sdk/x/authz" bank "github.com/cosmos/cosmos-sdk/x/bank/types" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" abci "github.com/tendermint/tendermint/abci/types" "github.com/tendermint/tendermint/libs/rand" - - "github.com/celestiaorg/celestia-app/v3/app" - "github.com/celestiaorg/celestia-app/v3/app/encoding" - "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" - "github.com/celestiaorg/celestia-app/v3/pkg/user" - "github.com/celestiaorg/celestia-app/v3/test/util/blobfactory" - "github.com/celestiaorg/celestia-app/v3/test/util/testnode" ) func TestTxClientTestSuite(t *testing.T) { @@ -39,15 +39,7 @@ type TxClientTestSuite struct { } func (suite *TxClientTestSuite) SetupSuite() { - suite.encCfg = encoding.MakeConfig(app.ModuleEncodingRegisters...) - config := testnode.DefaultConfig(). - WithFundedAccounts("a", "b", "c"). - WithAppCreator(testnode.CustomAppCreator("0utia")) - suite.ctx, _, _ = testnode.NewNetwork(suite.T(), config) - _, err := suite.ctx.WaitForHeight(1) - suite.Require().NoError(err) - suite.txClient, err = user.SetupTxClient(suite.ctx.GoContext(), suite.ctx.Keyring, suite.ctx.GRPCClient, suite.encCfg, user.WithGasMultiplier(1.2)) - suite.Require().NoError(err) + suite.encCfg, suite.txClient, suite.ctx = setupTxClient(suite.T(), testnode.DefaultTendermintConfig().Mempool.TTLDuration) suite.serviceClient = sdktx.NewServiceClient(suite.ctx.GRPCClient) } @@ -162,9 +154,11 @@ func (suite *TxClientTestSuite) TestConfirmTx() { ctx, cancel := context.WithTimeout(suite.ctx.GoContext(), time.Second) defer cancel() + seqBeforeBroadcast := suite.txClient.Signer().Account(suite.txClient.DefaultAccountName()).Sequence() msg := bank.NewMsgSend(suite.txClient.DefaultAddress(), testnode.RandomAddress().(sdk.AccAddress), sdk.NewCoins(sdk.NewInt64Coin(app.BondDenom, 10))) resp, err := suite.txClient.BroadcastTx(ctx, []sdk.Msg{msg}) require.NoError(t, err) + assertTxInTxTracker(t, suite.txClient, resp.TxHash, suite.txClient.DefaultAccountName(), seqBeforeBroadcast) _, err = suite.txClient.ConfirmTx(ctx, resp.TxHash) require.Error(t, err) @@ -174,48 +168,90 @@ func (suite *TxClientTestSuite) TestConfirmTx() { t.Run("should error when tx is not found", func(t *testing.T) { ctx, cancel := context.WithTimeout(suite.ctx.GoContext(), 5*time.Second) defer cancel() - _, err := suite.txClient.ConfirmTx(ctx, "E32BD15CAF57AF15D17B0D63CF4E63A9835DD1CEBB059C335C79586BC3013728") - require.Contains(t, err.Error(), "unknown tx: E32BD15CAF57AF15D17B0D63CF4E63A9835DD1CEBB059C335C79586BC3013728") + resp, err := suite.txClient.ConfirmTx(ctx, "E32BD15CAF57AF15D17B0D63CF4E63A9835DD1CEBB059C335C79586BC3013728") + require.Contains(t, err.Error(), "transaction with hash E32BD15CAF57AF15D17B0D63CF4E63A9835DD1CEBB059C335C79586BC3013728 not found; it was likely rejected") + require.Nil(t, resp) }) t.Run("should return error log when execution fails", func(t *testing.T) { + seqBeforeBroadcast := suite.txClient.Signer().Account(suite.txClient.DefaultAccountName()).Sequence() innerMsg := bank.NewMsgSend(testnode.RandomAddress().(sdk.AccAddress), testnode.RandomAddress().(sdk.AccAddress), sdk.NewCoins(sdk.NewInt64Coin(app.BondDenom, 10))) msg := authz.NewMsgExec(suite.txClient.DefaultAddress(), []sdk.Msg{innerMsg}) resp, err := suite.txClient.BroadcastTx(suite.ctx.GoContext(), []sdk.Msg{&msg}, fee, gas) require.NoError(t, err) - _, err = suite.txClient.ConfirmTx(suite.ctx.GoContext(), resp.TxHash) + assertTxInTxTracker(t, suite.txClient, resp.TxHash, suite.txClient.DefaultAccountName(), seqBeforeBroadcast) + + confirmTxResp, err := suite.txClient.ConfirmTx(suite.ctx.GoContext(), resp.TxHash) require.Error(t, err) require.Contains(t, err.Error(), "authorization not found") + require.Nil(t, confirmTxResp) + require.True(t, wasRemovedFromTxTracker(resp.TxHash, suite.txClient)) }) t.Run("should success when tx is found immediately", func(t *testing.T) { addr := suite.txClient.DefaultAddress() + seqBeforeBroadcast := suite.txClient.Signer().Account(suite.txClient.DefaultAccountName()).Sequence() msg := bank.NewMsgSend(addr, testnode.RandomAddress().(sdk.AccAddress), sdk.NewCoins(sdk.NewInt64Coin(app.BondDenom, 10))) resp, err := suite.txClient.BroadcastTx(suite.ctx.GoContext(), []sdk.Msg{msg}, fee, gas) require.NoError(t, err) - require.NotNil(t, resp) + require.Equal(t, resp.Code, abci.CodeTypeOK) + assertTxInTxTracker(t, suite.txClient, resp.TxHash, suite.txClient.DefaultAccountName(), seqBeforeBroadcast) + ctx, cancel := context.WithTimeout(suite.ctx.GoContext(), 30*time.Second) defer cancel() confirmTxResp, err := suite.txClient.ConfirmTx(ctx, resp.TxHash) require.NoError(t, err) require.Equal(t, abci.CodeTypeOK, confirmTxResp.Code) + require.True(t, wasRemovedFromTxTracker(resp.TxHash, suite.txClient)) }) t.Run("should error when tx is found with a non-zero error code", func(t *testing.T) { balance := suite.queryCurrentBalance(t) addr := suite.txClient.DefaultAddress() + seqBeforeBroadcast := suite.txClient.Signer().Account(suite.txClient.DefaultAccountName()).Sequence() // Create a msg send with out of balance, ensure this tx fails msg := bank.NewMsgSend(addr, testnode.RandomAddress().(sdk.AccAddress), sdk.NewCoins(sdk.NewInt64Coin(app.BondDenom, 1+balance))) resp, err := suite.txClient.BroadcastTx(suite.ctx.GoContext(), []sdk.Msg{msg}, fee, gas) require.NoError(t, err) - require.NotNil(t, resp) - _, err = suite.txClient.ConfirmTx(suite.ctx.GoContext(), resp.TxHash) + require.Equal(t, resp.Code, abci.CodeTypeOK) + assertTxInTxTracker(t, suite.txClient, resp.TxHash, suite.txClient.DefaultAccountName(), seqBeforeBroadcast) + + confirmTxResp, err := suite.txClient.ConfirmTx(suite.ctx.GoContext(), resp.TxHash) require.Error(t, err) + require.Nil(t, confirmTxResp) code := err.(*user.ExecutionError).Code require.NotEqual(t, abci.CodeTypeOK, code) + require.True(t, wasRemovedFromTxTracker(resp.TxHash, suite.txClient)) }) } +func TestEvictions(t *testing.T) { + _, txClient, ctx := setupTxClient(t, 1*time.Nanosecond) + + fee := user.SetFee(1e6) + gas := user.SetGasLimit(1e6) + + // Keep submitting the transaction until we get the eviction error + sender := txClient.Signer().Account(txClient.DefaultAccountName()) + msg := bank.NewMsgSend(sender.Address(), testnode.RandomAddress().(sdk.AccAddress), sdk.NewCoins(sdk.NewInt64Coin(app.BondDenom, 10))) + var seqBeforeEviction uint64 + // Loop five times until the tx is evicted + for i := 0; i < 5; i++ { + seqBeforeEviction = sender.Sequence() + resp, err := txClient.BroadcastTx(ctx.GoContext(), []sdk.Msg{msg}, fee, gas) + require.NoError(t, err) + _, err = txClient.ConfirmTx(ctx.GoContext(), resp.TxHash) + if err != nil { + if err.Error() == "tx was evicted from the mempool" { + break + } + } + } + + seqAfterEviction := sender.Sequence() + require.Equal(t, seqBeforeEviction, seqAfterEviction) +} + func (suite *TxClientTestSuite) TestGasEstimation() { addr := suite.txClient.DefaultAddress() msg := bank.NewMsgSend(addr, testnode.RandomAddress().(sdk.AccAddress), sdk.NewCoins(sdk.NewInt64Coin(app.BondDenom, 10))) @@ -281,3 +317,36 @@ func (suite *TxClientTestSuite) queryCurrentBalance(t *testing.T) int64 { require.NoError(t, err) return balanceResp.Balances.AmountOf(app.BondDenom).Int64() } + +func wasRemovedFromTxTracker(txHash string, txClient *user.TxClient) bool { + seq, signer, exists := txClient.GetTxFromTxTracker(txHash) + return !exists && seq == 0 && signer == "" +} + +// asserts that a tx was indexed in the tx tracker and that the sequence does not increase +func assertTxInTxTracker(t *testing.T, txClient *user.TxClient, txHash string, expectedSigner string, seqBeforeBroadcast uint64) { + seqFromTxTracker, signer, exists := txClient.GetTxFromTxTracker(txHash) + require.True(t, exists) + require.Equal(t, expectedSigner, signer) + seqAfterBroadcast := txClient.Signer().Account(expectedSigner).Sequence() + // TxInfo is indexed before the nonce is increased + require.Equal(t, seqBeforeBroadcast, seqFromTxTracker) + // Successfully broadcast transaction increases the sequence + require.Equal(t, seqAfterBroadcast, seqBeforeBroadcast+1) +} + +func setupTxClient(t *testing.T, ttlDuration time.Duration) (encoding.Config, *user.TxClient, testnode.Context) { + encCfg := encoding.MakeConfig(app.ModuleEncodingRegisters...) + defaultTmConfig := testnode.DefaultTendermintConfig() + defaultTmConfig.Mempool.TTLDuration = ttlDuration + testnodeConfig := testnode.DefaultConfig(). + WithTendermintConfig(defaultTmConfig). + WithFundedAccounts("a", "b", "c"). + WithAppCreator(testnode.CustomAppCreator("0utia")) + ctx, _, _ := testnode.NewNetwork(t, testnodeConfig) + _, err := ctx.WaitForHeight(1) + require.NoError(t, err) + txClient, err := user.SetupTxClient(ctx.GoContext(), ctx.Keyring, ctx.GRPCClient, encCfg, user.WithGasMultiplier(1.2)) + require.NoError(t, err) + return encCfg, txClient, ctx +} From 10711c527ca7c2d3b4a8da089feab4ea3771bb44 Mon Sep 17 00:00:00 2001 From: Matthew Sevey <15232757+MSevey@users.noreply.github.com> Date: Wed, 25 Sep 2024 15:26:48 -0400 Subject: [PATCH 48/85] chore: ci regex matching clean up and docker workflow update (#3908) ## Overview This PR adds the required input for the using the docker workflow when manually triggered. This PR also updates the regex matching to a more general wildcard regex for more flexibility in the future to avoid issues like images not being built due to adding `arabica` to the tag. --- .github/workflows/ci-release.yml | 7 ++----- .github/workflows/codeql.yml | 10 +++++----- .github/workflows/docker-build-publish.yml | 13 ++++--------- 3 files changed, 11 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci-release.yml b/.github/workflows/ci-release.yml index f4bbe29713..5fe828d636 100644 --- a/.github/workflows/ci-release.yml +++ b/.github/workflows/ci-release.yml @@ -6,12 +6,9 @@ on: push: branches: - main - - "v[0-9]+.x" + - "v*" tags: - - "v[0-9]+.[0-9]+.[0-9]+" - - "v[0-9]+.[0-9]+.[0-9]+-alpha.[0-9]+" - - "v[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+" - - "v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+" + - "v*" pull_request: jobs: diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 84f229e2e6..6ef0382d21 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -13,9 +13,9 @@ name: "CodeQL" on: push: - branches: ["main", "v[0-9].[0-9].x", "v[0-9].[0-9][0-9].x", "v[0-9].x"] + branches: ["main", "v*"] schedule: - - cron: '24 20 * * 4' + - cron: "24 20 * * 4" jobs: analyze: @@ -25,7 +25,7 @@ jobs: # - https://gh.io/supported-runners-and-hardware-resources # - https://gh.io/using-larger-runners # Consider using larger runners for possible analysis time improvements. - runs-on: 'ubuntu-latest' + runs-on: "ubuntu-latest" timeout-minutes: 360 permissions: actions: read @@ -35,7 +35,7 @@ jobs: strategy: fail-fast: false matrix: - language: ['go'] + language: ["go"] # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support steps: @@ -56,7 +56,7 @@ jobs: - uses: actions/setup-go@v5 with: - go-version-file: 'go.mod' + go-version-file: "go.mod" - name: Build binary run: | diff --git a/.github/workflows/docker-build-publish.yml b/.github/workflows/docker-build-publish.yml index 9d43669287..320f7dff10 100644 --- a/.github/workflows/docker-build-publish.yml +++ b/.github/workflows/docker-build-publish.yml @@ -5,16 +5,9 @@ on: push: branches: - "main" - - "v[0-9].[0-9].x" - - "v[0-9].[0-9][0-9].x" - - "v[0-9].x" + - "v*" tags: - - "v[0-9]+.[0-9]+.[0-9]+" - - "v[0-9]+.[0-9]+.[0-9]+-alpha.[0-9]+" - - "v[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+" - - "v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+" - - "v[0-9]+.[0-9]+.[0-9]+-arabica" - - "v[0-9]+.[0-9]+.[0-9]+-mocha" + - "v*" pull_request: workflow_dispatch: inputs: @@ -31,6 +24,7 @@ jobs: uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_pipeline.yml@v0.4.4 with: dockerfile: Dockerfile + checkout_ref: ${{ github.event.inputs.ref }} secrets: inherit docker-txsim-build: @@ -41,4 +35,5 @@ jobs: with: dockerfile: docker/Dockerfile_txsim packageName: txsim + checkout_ref: ${{ github.event.inputs.ref }} secrets: inherit From 32fc6903478ea08eba728ac9cd4ffedf9ef72d98 Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Wed, 25 Sep 2024 22:00:54 +0200 Subject: [PATCH 49/85] chore!: clean up tx sim docker usage (#3902) This PR tidies up our docker usage. Namely, it puts our two Dockerfiles in the `docker` directory, and the txsim one separately in the `docker/txsim` folder. It also cleans up the indirection with the flags so our e2e tests can pass them directly. --- .github/workflows/docker-build-publish.yml | 4 +- .github/workflows/lint.yml | 2 + Makefile | 9 +- Dockerfile => docker/Dockerfile | 0 docker/txsim.sh | 93 ------------------- docker/{Dockerfile_txsim => txsim/Dockerfile} | 15 +-- docker/{ => txsim}/README.md | 0 docker/txsim/entrypoint.sh | 7 ++ test/e2e/testnet/txsimNode.go | 13 ++- 9 files changed, 27 insertions(+), 116 deletions(-) rename Dockerfile => docker/Dockerfile (100%) delete mode 100644 docker/txsim.sh rename docker/{Dockerfile_txsim => txsim/Dockerfile} (75%) rename docker/{ => txsim}/README.md (100%) create mode 100644 docker/txsim/entrypoint.sh diff --git a/.github/workflows/docker-build-publish.yml b/.github/workflows/docker-build-publish.yml index 320f7dff10..a4976c0109 100644 --- a/.github/workflows/docker-build-publish.yml +++ b/.github/workflows/docker-build-publish.yml @@ -23,7 +23,7 @@ jobs: packages: write uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_pipeline.yml@v0.4.4 with: - dockerfile: Dockerfile + dockerfile: docker/Dockerfile checkout_ref: ${{ github.event.inputs.ref }} secrets: inherit @@ -33,7 +33,7 @@ jobs: packages: write uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_pipeline.yml@v0.4.4 with: - dockerfile: docker/Dockerfile_txsim + dockerfile: docker/txsim/Dockerfile packageName: txsim checkout_ref: ${{ github.event.inputs.ref }} secrets: inherit diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 443bb3c707..cf7074f605 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -31,6 +31,8 @@ jobs: # hadolint lints the Dockerfile hadolint: uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_lint.yml@v0.4.4 + with: + dockerfile: "docker/Dockerfile" yamllint: runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 67d69d5c24..f9d1f279de 100644 --- a/Makefile +++ b/Makefile @@ -83,13 +83,13 @@ proto-format: ## build-docker: Build the celestia-appd docker image from the current branch. Requires docker. build-docker: @echo "--> Building Docker image" - $(DOCKER) build -t celestiaorg/celestia-app -f Dockerfile . + $(DOCKER) build -t celestiaorg/celestia-app -f docker/Dockerfile . .PHONY: build-docker ## build-ghcr-docker: Build the celestia-appd docker image from the last commit. Requires docker. build-ghcr-docker: @echo "--> Building Docker image" - $(DOCKER) build -t ghcr.io/celestiaorg/celestia-app:$(COMMIT) -f Dockerfile . + $(DOCKER) build -t ghcr.io/celestiaorg/celestia-app:$(COMMIT) -f docker/Dockerfile . .PHONY: build-ghcr-docker ## publish-ghcr-docker: Publish the celestia-appd docker image. Requires docker. @@ -106,7 +106,8 @@ lint: @echo "--> Running markdownlint" @markdownlint --config .markdownlint.yaml '**/*.md' @echo "--> Running hadolint" - @hadolint Dockerfile + @hadolint docker/Dockerfile + @hadolint docker/txsim/Dockerfile @echo "--> Running yamllint" @yamllint --no-warnings . -c .yamllint.yml .PHONY: lint @@ -186,7 +187,7 @@ txsim-build: ## txsim-build-docker: Build the tx simulator Docker image. Requires Docker. txsim-build-docker: - docker build -t ghcr.io/celestiaorg/txsim -f docker/Dockerfile_txsim . + docker build -t ghcr.io/celestiaorg/txsim -f docker/txsim/Dockerfile . .PHONY: txsim-build-docker ## adr-gen: Download the ADR template from the celestiaorg/.github repo. diff --git a/Dockerfile b/docker/Dockerfile similarity index 100% rename from Dockerfile rename to docker/Dockerfile diff --git a/docker/txsim.sh b/docker/txsim.sh deleted file mode 100644 index c27cedb3ea..0000000000 --- a/docker/txsim.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/bash - -CREATE_KEY=0 -KEY_PATH="/home/celestia" -GRPC_ENDPOINT="" -POLL_TIME="" -BLOB=0 -BLOB_AMOUNTS="1" -BLOB_SIZES="100-1000" -KEY_MNEMONIC="" -SEED=0 -SEND=0 -SEND_AMOUNT=1000 -SEND_ITERATIONS=1000 -STAKE=0 -STAKE_VALUE=1000 - -while getopts "k:p:r:g:t:b:a:s:m:d:e:i:v:u:w:" opt; do - case ${opt} in - k ) - CREATE_KEY=$OPTARG - ;; - p ) - KEY_PATH=$OPTARG - ;; - g ) - GRPC_ENDPOINT=$OPTARG - ;; - t ) - POLL_TIME=$OPTARG - ;; - b ) - BLOB=$OPTARG - ;; - a ) - BLOB_AMOUNTS=$OPTARG - ;; - s ) - BLOB_SIZES=$OPTARG - ;; - m ) - KEY_MNEMONIC=$OPTARG - ;; - d ) - SEED=$OPTARG - ;; - e ) - SEND=$OPTARG - ;; - i ) - SEND_AMOUNT=$OPTARG - ;; - v ) - SEND_ITERATIONS=$OPTARG - ;; - u ) - STAKE=$OPTARG - ;; - w ) - STAKE_VALUE=$OPTARG - ;; - \? ) - echo "Invalid option: $OPTARG" 1>&2 - exit 1 - ;; - : ) - echo "Invalid option: $OPTARG requires an argument" 1>&2 - exit 1 - ;; - esac -done -shift $((OPTIND -1)) - -if [ "$CREATE_KEY" -eq 1 ]; then - echo "Creating a new keyring-test for the txsim" - /bin/celestia-appd keys add sim --keyring-backend test --home $KEY_PATH - sleep 5 -fi - -# Running a tx simulator -txsim --key-path $KEY_PATH \ - --grpc-endpoint $GRPC_ENDPOINT \ - --poll-time $POLL_TIME \ - --blob $BLOB \ - --blob-amounts $BLOB_AMOUNTS \ - --blob-sizes $BLOB_SIZES \ - --key-mnemonic "$KEY_MNEMONIC" \ - --seed $SEED \ - --send $SEND \ - --send-amount $SEND_AMOUNT \ - --send-iterations $SEND_ITERATIONS \ - --stake $STAKE \ - --stake-value $STAKE_VALUE diff --git a/docker/Dockerfile_txsim b/docker/txsim/Dockerfile similarity index 75% rename from docker/Dockerfile_txsim rename to docker/txsim/Dockerfile index 471db9c76a..9ffa797ac7 100644 --- a/docker/Dockerfile_txsim +++ b/docker/txsim/Dockerfile @@ -1,4 +1,4 @@ -# Stage 1: generate celestia-appd binary +# Stage 1: generate txsim binary FROM --platform=$BUILDPLATFORM docker.io/golang:1.22.6-alpine3.19 as builder ARG TARGETOS @@ -16,11 +16,10 @@ RUN apk update && apk add --no-cache \ musl-dev COPY . /celestia-app WORKDIR /celestia-app -# we need the celestia-appd build as we might want to create an account -# internally for txsimulation + RUN uname -a &&\ CGO_ENABLED=${CGO_ENABLED} GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ - make build && make txsim-build + make txsim-build # Stage 2: create a minimal image with the binary FROM docker.io/alpine:3.20 @@ -45,18 +44,14 @@ RUN apk update && apk add --no-cache \ -s /sbin/nologin \ -u ${UID} -# Copy in the celestia-appd binary -COPY --from=builder /celestia-app/build/celestia-appd /bin/celestia-appd +# Copy in the txsim binary COPY --from=builder /celestia-app/build/txsim /bin/txsim -COPY --chown=${USER_NAME}:${USER_NAME} docker/txsim.sh /opt/entrypoint.sh +COPY --chown=${USER_NAME}:${USER_NAME} docker/txsim/entrypoint.sh /opt/entrypoint.sh USER ${USER_NAME} # Set the working directory to the home directory. WORKDIR ${CELESTIA_HOME} -# grpc, rpc, api ports -EXPOSE 26657 1317 9090 - ENTRYPOINT [ "/bin/bash", "/opt/entrypoint.sh" ] diff --git a/docker/README.md b/docker/txsim/README.md similarity index 100% rename from docker/README.md rename to docker/txsim/README.md diff --git a/docker/txsim/entrypoint.sh b/docker/txsim/entrypoint.sh new file mode 100644 index 0000000000..f919a4d33d --- /dev/null +++ b/docker/txsim/entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +echo "Starting txsim with command:" +echo "/bin/txsim $@" +echo "" + +exec /bin/txsim $@ diff --git a/test/e2e/testnet/txsimNode.go b/test/e2e/testnet/txsimNode.go index 66bd005f3e..25a17f6d15 100644 --- a/test/e2e/testnet/txsimNode.go +++ b/test/e2e/testnet/txsimNode.go @@ -66,13 +66,12 @@ func CreateTxClient( return nil, err } args := []string{ - fmt.Sprintf("-k %d", 0), - fmt.Sprintf("-g %s", endpoint), - fmt.Sprintf("-t %ds", pollTime), - fmt.Sprintf("-b %d ", sequences), - fmt.Sprintf("-d %d ", seed), - fmt.Sprintf("-a %d ", blobsPerSeq), - fmt.Sprintf("-s %s ", blobRange), + fmt.Sprintf("-grpc-endpoint %s", endpoint), + fmt.Sprintf("-poll-time %ds", pollTime), + fmt.Sprintf("-seed %d ", seed), + fmt.Sprintf("-blob %d ", sequences), + fmt.Sprintf("-blob-amounts %d ", blobsPerSeq), + fmt.Sprintf("-blob-sizes %s ", blobRange), } if err := txIns.Build().SetArgs(args...); err != nil { From 66e55116d7141b3d44743f868f58f7b741d9165a Mon Sep 17 00:00:00 2001 From: Matthew Sevey <15232757+MSevey@users.noreply.github.com> Date: Thu, 26 Sep 2024 15:00:39 -0400 Subject: [PATCH 50/85] chore(deps): bump docker workflow version (#3915) ## Overview Brings in a patch fix for the docker workflow --- .github/workflows/docker-build-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build-publish.yml b/.github/workflows/docker-build-publish.yml index a4976c0109..c25e21aa2e 100644 --- a/.github/workflows/docker-build-publish.yml +++ b/.github/workflows/docker-build-publish.yml @@ -21,7 +21,7 @@ jobs: permissions: contents: write packages: write - uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_pipeline.yml@v0.4.4 + uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_pipeline.yml@v0.4.5 with: dockerfile: docker/Dockerfile checkout_ref: ${{ github.event.inputs.ref }} @@ -31,7 +31,7 @@ jobs: permissions: contents: write packages: write - uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_pipeline.yml@v0.4.4 + uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_pipeline.yml@v0.4.5 with: dockerfile: docker/txsim/Dockerfile packageName: txsim From 21e78b960aff7c5d505c6fbe902108b57aa9ad04 Mon Sep 17 00:00:00 2001 From: kobakaku <82507211+kobakaku@users.noreply.github.com> Date: Mon, 30 Sep 2024 23:16:26 +0900 Subject: [PATCH 51/85] chore(ci): forward port setting in the mergify.yml (#3922) Closes https://github.com/celestiaorg/celestia-app/issues/3917 ## Overview It modifies mergify.yml. By adding the `forwardport:main` tag to the v1.x or v2.x branch, it can forward-port it to the main branch. --- .github/mergify.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/mergify.yml b/.github/mergify.yml index 1e60d88bb2..f32c3fafcf 100644 --- a/.github/mergify.yml +++ b/.github/mergify.yml @@ -15,3 +15,19 @@ pull_request_rules: backport: branches: - v2.x + - name: forward-port patches to main branch (v1.x) + conditions: + - base=v1.x + - label=forwardport:main + actions: + backport: + branches: + - main + - name: forward-port patches to main branch (v2.x) + conditions: + - base=v2.x + - label=forwardport:main + actions: + backport: + branches: + - main From 363a8c5a81bddcd9a4c88a7ee8c9f2516e1c36a6 Mon Sep 17 00:00:00 2001 From: kobakaku <82507211+kobakaku@users.noreply.github.com> Date: Tue, 1 Oct 2024 01:02:22 +0900 Subject: [PATCH 52/85] fix(lint): golang-client lint errors (#3924) Closes https://github.com/celestiaorg/celestia-app/issues/3919 ## Overview Fix the lint errors of golangci-lint. Merge this [PR](https://github.com/celestiaorg/celestia-app/pull/3923) to this branch. --- .golangci.yml | 2 +- app/test/priority_test.go | 1 - pkg/da/data_availability_header_test.go | 3 --- x/blob/client/testutil/integration_test.go | 1 - x/blobstream/keeper/keeper_data_commitment.go | 4 ++-- x/blobstream/keeper/keeper_valset_test.go | 1 - x/mint/client/testutil/suite_test.go | 6 ------ x/mint/simulation/decoder_test.go | 1 - 8 files changed, 3 insertions(+), 16 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index d14fe1963a..59be9b021d 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -4,7 +4,7 @@ run: linters: enable: - - exportloopref + - copyloopvar - gofumpt - misspell - nakedret diff --git a/app/test/priority_test.go b/app/test/priority_test.go index 72586e5718..da7936dd0d 100644 --- a/app/test/priority_test.go +++ b/app/test/priority_test.go @@ -80,7 +80,6 @@ func (s *PriorityTestSuite) TestPriorityByGasPrice() { wg := &sync.WaitGroup{} for _, accName := range s.accountNames { wg.Add(1) - accName := accName // new variable per iteration go func() { defer wg.Done() // ensure that it is greater than the min gas price diff --git a/pkg/da/data_availability_header_test.go b/pkg/da/data_availability_header_test.go index fa73aeaa5e..7649750d09 100644 --- a/pkg/da/data_availability_header_test.go +++ b/pkg/da/data_availability_header_test.go @@ -88,7 +88,6 @@ func TestExtendShares(t *testing.T) { } for _, tt := range tests { - tt := tt _, err := ExtendShares(tt.shares) if tt.expectedErr { require.NotNil(t, err) @@ -122,7 +121,6 @@ func TestDataAvailabilityHeaderProtoConversion(t *testing.T) { } for _, tt := range tests { - tt := tt pdah, err := tt.dah.ToProto() require.NoError(t, err) resDah, err := DataAvailabilityHeaderFromProto(pdah) @@ -203,7 +201,6 @@ func Test_DAHValidateBasic(t *testing.T) { } for _, tt := range tests { - tt := tt err := tt.dah.ValidateBasic() if tt.expectErr { require.True(t, strings.Contains(err.Error(), tt.errStr), tt.name) diff --git a/x/blob/client/testutil/integration_test.go b/x/blob/client/testutil/integration_test.go index d29de7f658..b5f6c24db2 100644 --- a/x/blob/client/testutil/integration_test.go +++ b/x/blob/client/testutil/integration_test.go @@ -143,7 +143,6 @@ func (s *IntegrationTestSuite) TestSubmitPayForBlob() { } for _, tc := range testCases { - tc := tc require.NoError(s.ctx.WaitForNextBlock()) s.Run(tc.name, func() { cmd := paycli.CmdPayForBlob() diff --git a/x/blobstream/keeper/keeper_data_commitment.go b/x/blobstream/keeper/keeper_data_commitment.go index 0b0d656992..19bfc6d01c 100644 --- a/x/blobstream/keeper/keeper_data_commitment.go +++ b/x/blobstream/keeper/keeper_data_commitment.go @@ -109,7 +109,7 @@ func (k Keeper) GetLatestDataCommitment(ctx sdk.Context) (types.DataCommitment, return types.DataCommitment{}, err } if !found { - return types.DataCommitment{}, errors.Wrapf(types.ErrAttestationNotFound, fmt.Sprintf("nonce %d", i)) + return types.DataCommitment{}, errors.Wrapf(types.ErrAttestationNotFound, "nonce %d", i) } dcc, ok := att.(*types.DataCommitment) if !ok { @@ -136,7 +136,7 @@ func (k Keeper) HasDataCommitmentInStore(ctx sdk.Context) (bool, error) { return false, err } if !found { - return false, errors.Wrapf(types.ErrAttestationNotFound, fmt.Sprintf("nonce %d", i)) + return false, errors.Wrapf(types.ErrAttestationNotFound, "nonce %d", i) } _, ok := att.(*types.DataCommitment) if !ok { diff --git a/x/blobstream/keeper/keeper_valset_test.go b/x/blobstream/keeper/keeper_valset_test.go index d4b72406bf..6a73da03eb 100644 --- a/x/blobstream/keeper/keeper_valset_test.go +++ b/x/blobstream/keeper/keeper_valset_test.go @@ -57,7 +57,6 @@ func TestCurrentValsetNormalization(t *testing.T) { }, } for msg, spec := range specs { - spec := spec t.Run(msg, func(t *testing.T) { input, ctx := testutil.SetupTestChain(t, spec.srcPowers) r, err := input.BlobstreamKeeper.GetCurrentValset(ctx) diff --git a/x/mint/client/testutil/suite_test.go b/x/mint/client/testutil/suite_test.go index 9be2f45fc8..4910fcc99d 100644 --- a/x/mint/client/testutil/suite_test.go +++ b/x/mint/client/testutil/suite_test.go @@ -68,8 +68,6 @@ func (s *IntegrationTestSuite) TestGetCmdQueryInflationRate() { } for _, tc := range testCases { - tc := tc - s.Run(tc.name, func() { cmd := cli.GetCmdQueryInflationRate() @@ -103,8 +101,6 @@ func (s *IntegrationTestSuite) TestGetCmdQueryAnnualProvisions() { expectedAnnualProvision := mint.InitialInflationRateAsDec().MulInt(sdk.NewInt(testnode.DefaultInitialBalance)) for _, tc := range testCases { - tc := tc - s.Run(tc.name, func() { cmd := cli.GetCmdQueryAnnualProvisions() out, err := clitestutil.ExecTestCLICmd(s.cctx.Context, cmd, tc.args) @@ -134,8 +130,6 @@ func (s *IntegrationTestSuite) TestGetCmdQueryGenesisTime() { } for _, tc := range testCases { - tc := tc - s.Run(tc.name, func() { cmd := cli.GetCmdQueryGenesisTime() out, err := clitestutil.ExecTestCLICmd(s.cctx.Context, cmd, tc.args) diff --git a/x/mint/simulation/decoder_test.go b/x/mint/simulation/decoder_test.go index 74a634db87..c4122a316a 100644 --- a/x/mint/simulation/decoder_test.go +++ b/x/mint/simulation/decoder_test.go @@ -51,7 +51,6 @@ func TestDecodeStore(t *testing.T) { } for i, tt := range tests { - i, tt := i, tt t.Run(tt.name, func(t *testing.T) { if tt.expectPanic { require.Panics(t, func() { decoder(kvPairs.Pairs[i], kvPairs.Pairs[i]) }, tt.name) From 36257e8baac366b43a05b527226f0b6b02e0ca16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nina=20/=20=E1=83=9C=E1=83=98=E1=83=9C=E1=83=90?= Date: Mon, 30 Sep 2024 19:29:38 +0200 Subject: [PATCH 53/85] docs: remaining docs from versioned gas scheduler variables (#3914) ## Overview - Fixes #3762 - Adds ante handler v3 with updated links --- specs/src/SUMMARY.md | 2 ++ specs/src/ante_handler.md | 1 + specs/src/ante_handler_v3.md | 25 +++++++++++++++++++++++++ specs/src/parameters.md | 1 + 4 files changed, 29 insertions(+) create mode 100644 specs/src/ante_handler_v3.md diff --git a/specs/src/SUMMARY.md b/specs/src/SUMMARY.md index f5aa23cec1..e6c4633574 100644 --- a/specs/src/SUMMARY.md +++ b/specs/src/SUMMARY.md @@ -12,6 +12,7 @@ Celestia App Specifications - [AnteHandler](./ante_handler.md) - [AnteHandler v1](./ante_handler_v1.md) - [AnteHandler v2](./ante_handler_v2.md) + - [AnteHandler v3](./ante_handler_v3.md) - [Fraud Proofs](./fraud_proofs.md) - [Networking](./networking.md) - [Public-Key Cryptography](./public_key_cryptography.md) @@ -24,3 +25,4 @@ Celestia App Specifications - [Parameters](./parameters.md) - [Parameters v1](./parameters_v1.md) - [Parameters v2](./parameters_v2.md) + - [Parameters v3](./parameters_v3.md) diff --git a/specs/src/ante_handler.md b/specs/src/ante_handler.md index 2ebe759a79..d002c1e7eb 100644 --- a/specs/src/ante_handler.md +++ b/specs/src/ante_handler.md @@ -11,3 +11,4 @@ The AnteHandler is defined in `app/ante/ante.go`. The app version impacts AnteHa - [AnteHandler v1](./ante_handler_v1.md) - [AnteHandler v2](./ante_handler_v2.md) +- [AnteHandler v3](./ante_handler_v3.md) diff --git a/specs/src/ante_handler_v3.md b/specs/src/ante_handler_v3.md new file mode 100644 index 0000000000..59f32d45d2 --- /dev/null +++ b/specs/src/ante_handler_v3.md @@ -0,0 +1,25 @@ +# AnteHandler v3 + +The AnteHandler chains together several decorators to ensure the following criteria are met for app version 3: + +- The tx does not contain any messages that are unsupported by the current app version. See `MsgVersioningGateKeeper`. +- The tx does not contain any [extension options](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L119-L122). +- The tx passes `ValidateBasic()`. +- The tx's [timeout_height](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L115-L117) has not been reached if one is specified. +- The tx's [memo](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L110-L113) is <= the max memo characters where [`MaxMemoCharacters = 256`](). +- The tx's [gas_limit](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L211-L213) is > the gas consumed based on the tx's size where [`TxSizeCostPerByte = 10`](https://github.com/celestiaorg/celestia-app/blob/32fc6903478ea08eba728ac9cd4ffedf9ef72d98/pkg/appconsts/v3/app_consts.go#L8). +- The tx's feepayer has enough funds to pay fees for the tx. The tx's feepayer is the feegranter (if specified) or the tx's first signer. Note the [feegrant](https://github.com/cosmos/cosmos-sdk/blob/v0.46.15/x/feegrant/README.md) module is enabled. +- The tx's gas price is >= the network minimum gas price where [`NetworkMinGasPrice = 0.000001` utia](https://github.com/celestiaorg/celestia-app/blob/32fc6903478ea08eba728ac9cd4ffedf9ef72d98/pkg/appconsts/initial_consts.go#L33). +- The tx's count of signatures <= the max number of signatures. The max number of signatures is [`TxSigLimit = 7`](https://github.com/cosmos/cosmos-sdk/blob/a429238fc267da88a8548bfebe0ba7fb28b82a13/x/auth/README.md?plain=1#L231). +- The tx's [gas_limit](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L211-L213) is > the gas consumed based on the tx's signatures. +- The tx's [signatures](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/types/tx/signing/signature.go#L10-L26) are valid. For each signature, ensure that the signature's sequence number (a.k.a nonce) matches the account sequence number of the signer. +- The tx's [gas_limit](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L211-L213) is > the gas consumed based on the blob size(s). Since blobs are charged based on the number of shares they occupy, the gas consumed is calculated as follows: `gasToConsume = sharesNeeded(blob) * bytesPerShare * gasPerBlobByte`. Where `bytesPerShare` is a global constant (an alias for [`ShareSize = 512`](https://github.com/celestiaorg/celestia-app/blob/c90e61d5a2d0c0bd0e123df4ab416f6f0d141b7f/pkg/appconsts/global_consts.go#L27-L28)) and `gasPerBlobByte` is a versioned constant that can be modified through hard forks (the [`DefaultGasPerBlobByte = 8`](https://github.com/celestiaorg/celestia-app/blob/32fc6903478ea08eba728ac9cd4ffedf9ef72d98/pkg/appconsts/v3/app_consts.go#L8)). +- The tx's total blob share count is <= the max blob share count. The max blob share count is derived from the maximum valid square size. The max valid square size is the minimum of: `GovMaxSquareSize` and `SquareSizeUpperBound`. +- The tx does not contain a message of type [MsgSubmitProposal](https://github.com/cosmos/cosmos-sdk/blob/d6d929843bbd331b885467475bcb3050788e30ca/proto/cosmos/gov/v1/tx.proto#L33-L43) with zero proposal messages. +- The tx is not an IBC packet or update message that has already been processed. + +In addition to the above criteria, the AnteHandler also has a number of side-effects: + +- Tx fees are deducted from the tx's feepayer and added to the fee collector module account. +- Tx priority is calculated based on the smallest denomination of gas price in the tx and set in context. +- The nonce of all tx signers is incremented by 1. diff --git a/specs/src/parameters.md b/specs/src/parameters.md index 4dcecce93d..2e32514c9d 100644 --- a/specs/src/parameters.md +++ b/specs/src/parameters.md @@ -4,3 +4,4 @@ The parameters in the application depend on the app version: - [Parameters v1](./parameters_v1.md) - [Parameters v2](./parameters_v2.md) +- [Parameters v3](./parameters_v3.md) From 18037f7b901e0eba7a2d995f01ea138d39db5f08 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Oct 2024 10:31:42 -0400 Subject: [PATCH 54/85] chore(deps): Bump celestiaorg/.github from 0.4.4 to 0.4.5 (#3920) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [celestiaorg/.github](https://github.com/celestiaorg/.github) from 0.4.4 to 0.4.5.
Release notes

Sourced from celestiaorg/.github's releases.

v0.4.5

What's Changed

Full Changelog: https://github.com/celestiaorg/.github/compare/v0.4.4...v0.4.5

Commits
  • bacacab fix: add type=raw tag for workflow dispatch events to metadata step (#114)
  • fda2184 chore(ci): update workflow test file:
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=celestiaorg/.github&package-manager=github_actions&previous-version=0.4.4&new-version=0.4.5)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/create_release_tracking_epic.yml | 2 +- .github/workflows/lint.yml | 4 ++-- .github/workflows/pr-review-requester.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/create_release_tracking_epic.yml b/.github/workflows/create_release_tracking_epic.yml index 9d2fe78162..b14fb0406a 100644 --- a/.github/workflows/create_release_tracking_epic.yml +++ b/.github/workflows/create_release_tracking_epic.yml @@ -7,7 +7,7 @@ on: types: [released] jobs: trigger_issue: - uses: celestiaorg/.github/.github/workflows/reusable_create_release_tracking_epic.yml@v0.4.4 + uses: celestiaorg/.github/.github/workflows/reusable_create_release_tracking_epic.yml@v0.4.5 secrets: inherit with: release-repo: ${{ github.repository }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index cf7074f605..a148d3f6f0 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -30,7 +30,7 @@ jobs: # hadolint lints the Dockerfile hadolint: - uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_lint.yml@v0.4.4 + uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_lint.yml@v0.4.5 with: dockerfile: "docker/Dockerfile" @@ -38,4 +38,4 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: celestiaorg/.github/.github/actions/yamllint@v0.4.4 + - uses: celestiaorg/.github/.github/actions/yamllint@v0.4.5 diff --git a/.github/workflows/pr-review-requester.yml b/.github/workflows/pr-review-requester.yml index d6afa96637..8396723699 100644 --- a/.github/workflows/pr-review-requester.yml +++ b/.github/workflows/pr-review-requester.yml @@ -11,7 +11,7 @@ on: jobs: auto-request-review: name: Auto request reviews - uses: celestiaorg/.github/.github/workflows/reusable_housekeeping.yml@v0.4.4 + uses: celestiaorg/.github/.github/workflows/reusable_housekeeping.yml@v0.4.5 secrets: inherit # write access for issues and pull requests is needed because the called # workflow requires write access to issues and pull requests and the From 9f0ba4e2578efe1f84d83d761ff3ec76e67b2a5a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Oct 2024 09:17:27 -0400 Subject: [PATCH 55/85] chore(deps): Bump codecov/codecov-action from 4.5.0 to 4.6.0 (#3932) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 4.5.0 to 4.6.0.
Release notes

Sourced from codecov/codecov-action's releases.

v4.6.0

What's Changed

... (truncated)

Commits
  • b9fd7d1 chore(release):4.6.0 (#1587)
  • 6f7612c fix: bump eslint parser deps (#1586)
  • 26c7e28 build(deps): bump actions/checkout from 4.1.7 to 4.2.0 (#1583)
  • 6f744f7 build(deps): bump github/codeql-action from 3.26.8 to 3.26.9 (#1584)
  • 543c3d4 chore: fix typo of OSS (#1578)
  • e379426 build(deps-dev): bump @​vercel/ncc from 0.38.1 to 0.38.2 (#1577)
  • 42656e4 build(deps): bump github/codeql-action from 3.26.7 to 3.26.8 (#1575)
  • 2296b6b build(deps-dev): bump eslint from 8.57.0 to 8.57.1 (#1571)
  • bd77bc3 build(deps): bump github/codeql-action from 3.26.6 to 3.26.7 (#1569)
  • 180b964 build(deps-dev): bump @​types/jest from 29.5.12 to 29.5.13 (#1567)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=codecov/codecov-action&package-manager=github_actions&previous-version=4.5.0&new-version=4.6.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 21185f78c8..b135a91611 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,7 +41,7 @@ jobs: run: make test-coverage - name: Upload coverage.txt - uses: codecov/codecov-action@v4.5.0 + uses: codecov/codecov-action@v4.6.0 with: file: ./coverage.txt From f0e7fba53ea0c3741d97c318e5c3f42a1ff215ff Mon Sep 17 00:00:00 2001 From: Rootul P Date: Wed, 2 Oct 2024 09:33:53 -0400 Subject: [PATCH 56/85] chore: upgrade to Go 1.23.1 (#3848) ## Description Upgrade to Go 1.23.1 motivated by https://github.com/celestiaorg/celestia-app/pull/3847. I had to modify GoReleaser config because https://github.com/goreleaser/goreleaser/issues/5127. Maintainers need to update their local golangic-lint binary to v1.61.0. See [golangci-lint install](https://golangci-lint.run/welcome/install/). --------- Co-authored-by: CHAMI Rachid --- .github/workflows/lint.yml | 3 +-- .goreleaser.yaml | 2 +- Makefile | 10 ++++++---- README.md | 4 ++-- docker/Dockerfile | 2 +- docker/txsim/Dockerfile | 2 +- go.mod | 2 +- go.work.example | 2 +- test/interchain/go.mod | 2 +- tools/blocketa/README.md | 2 +- 10 files changed, 16 insertions(+), 15 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a148d3f6f0..999eaf166b 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -22,10 +22,9 @@ jobs: go.sum - uses: golangci/golangci-lint-action@v6.1.0 with: - version: v1.59.1 + version: v1.61.0 args: --timeout 10m github-token: ${{ secrets.github_token }} - skip-pkg-cache: true if: env.GIT_DIFF # hadolint lints the Dockerfile diff --git a/.goreleaser.yaml b/.goreleaser.yaml index f36e8e0435..120d8762dc 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -108,7 +108,7 @@ archives: checksum: name_template: "checksums.txt" snapshot: - name_template: "{{ incpatch .Version }}-next" + version_template: "{{ incpatch .Version }}-next" changelog: sort: asc filters: diff --git a/Makefile b/Makefile index f9d1f279de..83c8fe3a03 100644 --- a/Makefile +++ b/Makefile @@ -7,11 +7,13 @@ DOCKER_PROTO_BUILDER := docker run -v $(shell pwd):/workspace --workdir /workspa PROJECTNAME=$(shell basename "$(PWD)") HTTPS_GIT := https://github.com/celestiaorg/celestia-app.git PACKAGE_NAME := github.com/celestiaorg/celestia-app/v3 -GOLANG_CROSS_VERSION ?= v1.22.6 +# Before upgrading the GOLANG_CROSS_VERSION, please verify that a Docker image exists with the new tag. +# See https://github.com/goreleaser/goreleaser-cross/pkgs/container/goreleaser-cross +GOLANG_CROSS_VERSION ?= v1.23.1 # Set this to override the max square size of the binary -OVERRIDE_MAX_SQUARE_SIZE ?= +OVERRIDE_MAX_SQUARE_SIZE ?= # Set this to override the upgrade height delay of the binary -OVERRIDE_UPGRADE_HEIGHT_DELAY ?= +OVERRIDE_UPGRADE_HEIGHT_DELAY ?= # process linker flags ldflags = -X github.com/cosmos/cosmos-sdk/version.Name=celestia-app \ @@ -253,4 +255,4 @@ enable-bbr: else \ echo "BBR is already enabled."; \ fi -.PHONY: enable-bbr \ No newline at end of file +.PHONY: enable-bbr diff --git a/README.md b/README.md index f1f94636c7..8d94c06884 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ node | | | | ### Source -1. [Install Go](https://go.dev/doc/install) 1.22.6 +1. [Install Go](https://go.dev/doc/install) 1.23.1 1. Clone this repo 1. Install the celestia-app CLI @@ -134,7 +134,7 @@ This repo contains multiple go modules. When using it, rename `go.work.example` ### Tools -1. Install [golangci-lint](https://golangci-lint.run/welcome/install) 1.59.1 +1. Install [golangci-lint](https://golangci-lint.run/welcome/install) 1.61.0 1. Install [markdownlint](https://github.com/DavidAnson/markdownlint) 0.39.0 1. Install [hadolint](https://github.com/hadolint/hadolint) 1. Install [yamllint](https://yamllint.readthedocs.io/en/stable/quickstart.html) diff --git a/docker/Dockerfile b/docker/Dockerfile index 2b504642db..0e3c4bf51d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -4,7 +4,7 @@ # # Separating the builder and runtime image allows the runtime image to be # considerably smaller because it doesn't need to have Golang installed. -ARG BUILDER_IMAGE=docker.io/golang:1.22.6-alpine3.19 +ARG BUILDER_IMAGE=docker.io/golang:1.23.1-alpine3.20 ARG RUNTIME_IMAGE=docker.io/alpine:3.19 ARG TARGETOS ARG TARGETARCH diff --git a/docker/txsim/Dockerfile b/docker/txsim/Dockerfile index 9ffa797ac7..563a06ab09 100644 --- a/docker/txsim/Dockerfile +++ b/docker/txsim/Dockerfile @@ -1,5 +1,5 @@ # Stage 1: generate txsim binary -FROM --platform=$BUILDPLATFORM docker.io/golang:1.22.6-alpine3.19 as builder +FROM --platform=$BUILDPLATFORM docker.io/golang:1.23.1-alpine3.20 as builder ARG TARGETOS ARG TARGETARCH diff --git a/go.mod b/go.mod index 42b518927a..19279ba25d 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/celestiaorg/celestia-app/v3 -go 1.22.6 +go 1.23.1 require ( cosmossdk.io/errors v1.0.1 diff --git a/go.work.example b/go.work.example index 85b25f4976..64b0c2f5d2 100644 --- a/go.work.example +++ b/go.work.example @@ -1,4 +1,4 @@ -go 1.22.6 +go 1.23.1 use ( . diff --git a/test/interchain/go.mod b/test/interchain/go.mod index bbcd6956b1..2b6b52b585 100644 --- a/test/interchain/go.mod +++ b/test/interchain/go.mod @@ -1,6 +1,6 @@ module github.com/celestiaorg/celestia-app/test/interchain -go 1.22.6 +go 1.23.1 require ( cosmossdk.io/math v1.3.0 diff --git a/tools/blocketa/README.md b/tools/blocketa/README.md index f7de7539b5..1f8c2b4d1b 100644 --- a/tools/blocketa/README.md +++ b/tools/blocketa/README.md @@ -15,4 +15,4 @@ arrivalTime: 2024-08-28 17:24:23.483542677 +0000 UTC ``` > [!NOTE] -> The block time is currently hard-coded. If you're running this for a network with a different block time, you'll need to update the `blockTime` constant in the main.go file. You can use [https://www.mintscan.io/celestia/block](https://www.mintscan.io/celestia/block/) or the blocktime tool. +> The block time is currently hard-coded. If you're running this for a network with a different block time, you'll need to update the `blockTime` constant in the main.go file. You can use [https://www.mintscan.io/celestia/block](https://www.mintscan.io/celestia/block/) or the blocktime tool. \ No newline at end of file From 0f446000fdd5321f55bb577f00329ea2c33c9777 Mon Sep 17 00:00:00 2001 From: Moji Date: Wed, 2 Oct 2024 17:10:31 +0330 Subject: [PATCH 57/85] chore: update e2e tests to knuu 0.16.1 (#3911) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR proposes updating the e2e tests code to use the latest version of `knuu` ([v0.16.1](https://github.com/celestiaorg/knuu/releases/tag/v0.16.1)) The main changes of this release are: 1. Use unique static names for instances: now `knuu` does not add a random string to the names of the instances to make them predictable; therefore, it paves the path to the long running tests. 2. CTRL+C to run timeout handler with no sleep: optimizes the termination request by the user 3. Ability to fetch logs from instances 4. Adjust stopped status actions for the update image: From now on, user can `Stop` the instance, then update the image, arguments, start command and or env vars, then commit and start the instance. This is handy for upgrade test. BTW: the e2e tests are passing locally. - test-e2e2024/09/25 10:42:37 --- ✅ PASS: MinorVersionCompatibility - test-e2e2024/09/25 10:44:47 --- ✅ PASS: MajorUpgradeToV2 - test-e2e2024/09/25 10:46:34 --- ✅ PASS: E2ESimple --- go.mod | 21 ++++------ go.sum | 56 ++++++++----------------- test/e2e/major_upgrade_v2.go | 9 ++-- test/e2e/minor_version_compatibility.go | 5 ++- test/e2e/simple.go | 4 +- test/e2e/testnet/node.go | 14 ++++++- test/e2e/testnet/testnet.go | 11 ++++- 7 files changed, 57 insertions(+), 63 deletions(-) diff --git a/go.mod b/go.mod index 19279ba25d..f446db194d 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/celestiaorg/blobstream-contracts/v3 v3.1.0 github.com/celestiaorg/go-square v1.1.0 github.com/celestiaorg/go-square/v2 v2.0.0 - github.com/celestiaorg/knuu v0.15.2 + github.com/celestiaorg/knuu v0.16.1 github.com/celestiaorg/nmt v0.22.1 github.com/celestiaorg/rsmt2d v0.14.0 github.com/cometbft/cometbft-db v0.7.0 @@ -86,10 +86,6 @@ require ( github.com/dgraph-io/badger/v2 v2.2007.4 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect - github.com/distribution/reference v0.6.0 // indirect - github.com/docker/docker v26.1.5+incompatible // indirect - github.com/docker/go-connections v0.4.1-0.20210727194412-58542c764a11 // indirect - github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/dvsekhvalnov/jose2go v1.6.0 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect @@ -107,7 +103,6 @@ require ( github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.4 // indirect - github.com/go-playground/validator/v10 v10.11.2 // indirect github.com/goccy/go-json v0.10.3 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect github.com/gogo/gateway v1.1.0 // indirect @@ -176,7 +171,6 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/onsi/ginkgo v1.16.5 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 // indirect @@ -205,7 +199,6 @@ require ( github.com/tidwall/btree v1.5.0 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect - github.com/ugorji/go/codec v1.2.9 // indirect github.com/ulikunitz/xz v0.5.10 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/zondax/hid v0.9.2 // indirect @@ -221,13 +214,13 @@ require ( go.opentelemetry.io/otel/trace v1.26.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.26.0 // indirect - golang.org/x/net v0.28.0 // indirect + golang.org/x/crypto v0.27.0 // indirect + golang.org/x/net v0.29.0 // indirect golang.org/x/oauth2 v0.22.0 // indirect golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.24.0 // indirect - golang.org/x/term v0.23.0 // indirect - golang.org/x/text v0.17.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/term v0.24.0 // indirect + golang.org/x/text v0.18.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/api v0.169.0 // indirect google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect @@ -240,7 +233,7 @@ require ( k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect - nhooyr.io/websocket v1.8.6 // indirect + nhooyr.io/websocket v1.8.17 // indirect rsc.io/tmplfunc v0.0.3 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect diff --git a/go.sum b/go.sum index d196f87450..652f175022 100644 --- a/go.sum +++ b/go.sum @@ -213,7 +213,6 @@ github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3 github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo= github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= @@ -326,8 +325,8 @@ github.com/celestiaorg/go-square v1.1.0 h1:K4tBL5PCJwDtpBfyDxxZ3N962aC9VYb5/bw3L github.com/celestiaorg/go-square v1.1.0/go.mod h1:1EXMErhDrWJM8B8V9hN7dqJ2kUTClfwdqMOmF9yQUa0= github.com/celestiaorg/go-square/v2 v2.0.0 h1:U5QV8/de5lc7glosfgyHhcxbFwNuwU4+6aYZ2RgjM04= github.com/celestiaorg/go-square/v2 v2.0.0/go.mod h1:y0BolG0tRM7UN1sAQyDDUkT+aMJPwFIjviVvnCB62C0= -github.com/celestiaorg/knuu v0.15.2 h1:l6MrFTfDfrNtbvOLSjD/YzRpaaBFIIvlWIYXRNNL1/s= -github.com/celestiaorg/knuu v0.15.2/go.mod h1:hTKu4uGH/Dx7eaYgT26coKx/vbALEQdLdYOwNxkAIG0= +github.com/celestiaorg/knuu v0.16.1 h1:EOR/c9kvc0jZet/mma2qwAdlvEbl94bW9cC8FItkyBE= +github.com/celestiaorg/knuu v0.16.1/go.mod h1:y20nUmVWVgbzxBKHqmbwp3C0ZJ9J9ovCg1ylHo85hdQ= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4/go.mod h1:fzuHnhzj1pUygGz+1ZkB3uQbEUL4htqCGJ4Qs2LwMZA= github.com/celestiaorg/nmt v0.22.1 h1:t7fqoP5MJ8mBns5DB2XjfcPxQpS3CKMkY+v+BEkDxYc= @@ -406,8 +405,6 @@ github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJ github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= github.com/containerd/continuity v0.4.2 h1:v3y/4Yz5jwnvqPKJJ+7Wf93fyWoCB3F5EclWG023MDM= github.com/containerd/continuity v0.4.2/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= -github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= -github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -480,13 +477,10 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUn github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= -github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v26.1.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.1-0.20210727194412-58542c764a11 h1:IPrmumsT9t5BS7XcPhgsCTlkWbYg80SEXUzDpReaU6Y= github.com/docker/go-connections v0.4.1-0.20210727194412-58542c764a11/go.mod h1:a6bNUGTbQBsY6VRHTr4h/rkOXjl244DyRD0tx3fgq4Q= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -559,11 +553,8 @@ github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= -github.com/gin-gonic/gin v1.7.0 h1:jGB9xAJQ12AIGNB4HguylppmDK1Am9ppF7XnGXXJuoU= -github.com/gin-gonic/gin v1.7.0/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= @@ -605,14 +596,8 @@ github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogB github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= -github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= -github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= -github.com/go-playground/validator/v10 v10.11.2 h1:q3SHpufmypg+erIExEKUmsgmhDTyhcJ38oeKGACXohU= -github.com/go-playground/validator/v10 v10.11.2/go.mod h1:NieE624vt4SCTJtD87arVLvdmjPAeV8BQlHtMnw9D7s= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= @@ -621,11 +606,8 @@ github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8Wd github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= -github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= @@ -956,8 +938,6 @@ github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= -github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= @@ -1196,8 +1176,8 @@ github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= -github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday v1.6.0 h1:KqfZb0pUVN2lYqZUYRddxF4OR8ZMURnJIG5Y3VRLtww= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -1215,7 +1195,6 @@ github.com/shirou/gopsutil v3.21.6+incompatible h1:mmZtAlWSd8U2HeRTjswbnDLPxqsEo github.com/shirou/gopsutil v3.21.6+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= @@ -1315,8 +1294,6 @@ github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3C github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/ugorji/go/codec v1.2.9 h1:rmenucSohSTiyL09Y+l2OCk+FrMxGMzho2+tjr5ticU= -github.com/ugorji/go/codec v1.2.9/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -1427,8 +1404,8 @@ golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= -golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1536,8 +1513,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1692,14 +1669,14 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= -golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= -golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1710,8 +1687,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1784,8 +1761,8 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= -golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= +golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= +golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2098,8 +2075,9 @@ k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7F k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -nhooyr.io/websocket v1.8.6 h1:s+C3xAMLwGmlI31Nyn/eAehUlZPwfYZu2JXM621Q5/k= nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= +nhooyr.io/websocket v1.8.17 h1:KEVeLJkUywCKVsnLIDlD/5gtayKp8VoCkksHCGGfT9Y= +nhooyr.io/websocket v1.8.17/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c= pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw= pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/test/e2e/major_upgrade_v2.go b/test/e2e/major_upgrade_v2.go index db21d65306..f2d1a43b21 100644 --- a/test/e2e/major_upgrade_v2.go +++ b/test/e2e/major_upgrade_v2.go @@ -16,13 +16,16 @@ import ( ) func MajorUpgradeToV2(logger *log.Logger) error { - numNodes := 4 - upgradeHeight := int64(10) + var ( + numNodes = 4 + upgradeHeight = int64(10) + ) + ctx, cancel := context.WithCancel(context.Background()) defer cancel() logger.Println("Creating testnet") - testNet, err := testnet.New(ctx, "runMajorUpgradeToV2", seed, nil, "test") + testNet, err := testnet.New(ctx, "MajorUpgradeToV2", seed, nil, "test") testnet.NoError("failed to create testnet", err) defer testNet.Cleanup(ctx) diff --git a/test/e2e/minor_version_compatibility.go b/test/e2e/minor_version_compatibility.go index c13d93f23c..afa8158d0b 100644 --- a/test/e2e/minor_version_compatibility.go +++ b/test/e2e/minor_version_compatibility.go @@ -18,6 +18,8 @@ import ( ) func MinorVersionCompatibility(logger *log.Logger) error { + const numNodes = 4 + versionStr, err := getAllVersions() testnet.NoError("failed to get versions", err) versions1 := testnet.ParseVersions(versionStr).FilterMajor(v1.Version).FilterOutReleaseCandidates() @@ -27,14 +29,13 @@ func MinorVersionCompatibility(logger *log.Logger) error { if len(versions) == 0 { logger.Fatal("no versions to test") } - numNodes := 4 r := rand.New(rand.NewSource(seed)) logger.Println("Running minor version compatibility test", "versions", versions) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - testNet, err := testnet.New(ctx, "runMinorVersionCompatibility", seed, nil, "test") + testNet, err := testnet.New(ctx, "MinorVersionCompatibility", seed, nil, "test") testnet.NoError("failed to create testnet", err) defer testNet.Cleanup(ctx) diff --git a/test/e2e/simple.go b/test/e2e/simple.go index 7999c726a0..e065467f1c 100644 --- a/test/e2e/simple.go +++ b/test/e2e/simple.go @@ -15,7 +15,9 @@ import ( // and MsgSends over 30 seconds and then asserts that at least 10 transactions were // committed. func E2ESimple(logger *log.Logger) error { - ctx := context.Background() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + testNet, err := testnet.New(ctx, "E2ESimple", seed, nil, "test") testnet.NoError("failed to create testnet", err) diff --git a/test/e2e/testnet/node.go b/test/e2e/testnet/node.go index 14293da5fc..71a82d3a75 100644 --- a/test/e2e/testnet/node.go +++ b/test/e2e/testnet/node.go @@ -395,11 +395,21 @@ func (n *Node) GenesisValidator() genesis.Validator { } func (n *Node) Upgrade(ctx context.Context, version string) error { - if err := n.Instance.Execution().UpgradeImage(ctx, DockerImageName(version)); err != nil { + if err := n.Instance.Execution().Stop(ctx); err != nil { return err } - return n.Instance.Execution().WaitInstanceIsRunning(ctx) + if err := n.Instance.Execution().SetImage(ctx, DockerImageName(version)); err != nil { + return err + } + + // New set of args can be set here + // Or/and the start command can also be set here + + if err := n.Instance.Build().Commit(ctx); err != nil { + return err + } + return n.Instance.Execution().Start(ctx) } func DockerImageName(version string) string { diff --git a/test/e2e/testnet/testnet.go b/test/e2e/testnet/testnet.go index c04c723b73..fc1ed6f67e 100644 --- a/test/e2e/testnet/testnet.go +++ b/test/e2e/testnet/testnet.go @@ -45,7 +45,12 @@ func New(ctx context.Context, name string, seed int64, grafana *GrafanaInfo, cha return nil, err } - log.Info().Str("scope", kn.Scope).Msg("Knuu initialized") + log.Info(). + Str("scope", kn.Scope). + Str("TestName", name). + Msg("Knuu initialized") + + kn.HandleStopSignal(ctx) return &Testnet{ seed: seed, @@ -65,7 +70,9 @@ func (t *Testnet) NewPreloader() (*preloader.Preloader, error) { if t.knuu == nil { return nil, errors.New("knuu is not initialized") } - return preloader.New(t.knuu.SystemDependencies) + // Since there is one dedicated knuu object for the testnet, each one has its own namespace, and + // there is one preloader per testnet, can use the same preloader name for all nodes + return preloader.New("preloader", t.knuu.SystemDependencies) } func (t *Testnet) SetConsensusParams(params *tmproto.ConsensusParams) { From c713401b0f69b2c138006195310eec2a9525d829 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Oct 2024 09:46:16 -0400 Subject: [PATCH 58/85] chore(deps): Bump golangci/golangci-lint-action from 6.1.0 to 6.1.1 (#3940) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 6.1.0 to 6.1.1.
Release notes

Sourced from golangci/golangci-lint-action's releases.

v6.1.1

What's Changed

Changes

Documentation

Dependencies

New Contributors

Full Changelog: https://github.com/golangci/golangci-lint-action/compare/v6.1.0...v6.1.1

Commits
  • 971e284 build(deps-dev): bump the dev-dependencies group with 3 updates (#1108)
  • bbe7eb5 build(deps): bump @​types/node from 22.5.5 to 22.7.4 in the dependencies group...
  • ebae5ce build(deps-dev): bump the dev-dependencies group with 3 updates (#1105)
  • 06c3f3a build(deps): bump @​types/node from 22.5.4 to 22.5.5 in the dependencies group...
  • 56689d8 build(deps-dev): bump the dev-dependencies group with 3 updates (#1103)
  • c7bab6f fix: clean go install output (#1102)
  • 33f56cc build(deps-dev): bump the dev-dependencies group with 3 updates (#1099)
  • e954224 build(deps): bump @​types/node from 22.5.2 to 22.5.4 in the dependencies group...
  • 68de804 build(deps): bump @​types/node from 22.5.1 to 22.5.2 in the dependencies group...
  • 22a3756 build(deps-dev): bump the dev-dependencies group with 2 updates (#1097)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golangci/golangci-lint-action&package-manager=github_actions&previous-version=6.1.0&new-version=6.1.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 999eaf166b..87d07b6ae3 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -20,7 +20,7 @@ jobs: **/**.go go.mod go.sum - - uses: golangci/golangci-lint-action@v6.1.0 + - uses: golangci/golangci-lint-action@v6.1.1 with: version: v1.61.0 args: --timeout 10m From c7f26d6d791f62f9aca8ceba0ab1e72598ab0e60 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Fri, 4 Oct 2024 13:36:18 +0000 Subject: [PATCH 59/85] feat(statesync): extract app version from snapshot (backport #3871) (#3936) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes https://github.com/celestiaorg/celestia-app/issues/3818 ~~Blocked on https://github.com/celestiaorg/celestia-core/pull/1477~~ I had to bump the celestia-core depedency to get a feature that I wanted. That implied upgrading Go and cosmos-sdk versions. ## Testing I verified that logs added in this PR show up when state syncing on Mocha. The logs show that the app version provided in the `OfferSnapshot` ABCI method was plumbed through correctly. ``` ./scripts/mocha.sh ... 2:44PM INF Offering snapshot to ABCI app format=2 hash="$�ڢ�\x1b� �zΊ\x15���`��왹\b^G:\x03+ߢޯ" height=2782000 module=statesync 2:44PM INF offering snapshot app_version=2 height=2782000 2:44PM INF mounting keys for snapshot app_version=2 ```
This is an automatic backport of pull request #3871 done by [Mergify](https://mergify.com). --------- Co-authored-by: Rootul P --- app/app.go | 27 ++++++-- app/app_test.go | 92 +++++++++++++++++-------- go.mod | 54 +++++++-------- go.sum | 130 +++++++++++++++++------------------ scripts/single-node.sh | 4 +- scripts/state-sync.sh | 108 +++++++++++++++++++++++++++++ tools/blocketa/README.md | 2 +- tools/chainbuilder/README.md | 4 +- 8 files changed, 292 insertions(+), 129 deletions(-) create mode 100755 scripts/state-sync.sh diff --git a/app/app.go b/app/app.go index ce44df0b2f..ecba661020 100644 --- a/app/app.go +++ b/app/app.go @@ -580,7 +580,7 @@ func setDefaultAppVersion(req abci.RequestInitChain) abci.RequestInitChain { // mountKeysAndInit mounts the keys for the provided app version and then // invokes baseapp.Init(). func (app *App) mountKeysAndInit(appVersion uint64) { - app.BaseApp.Logger().Info(fmt.Sprintf("mounting KV stores for app version %v", appVersion)) + app.Logger().Info(fmt.Sprintf("mounting KV stores for app version %v", appVersion)) app.MountKVStores(app.versionedKeys(appVersion)) // Invoke load latest version for its side-effect of invoking baseapp.Init() @@ -805,19 +805,38 @@ func (app *App) OfferSnapshot(req abci.RequestOfferSnapshot) abci.ResponseOfferS return app.BaseApp.OfferSnapshot(req) } + app.Logger().Info("offering snapshot", "height", req.Snapshot.Height, "app_version", req.AppVersion) + if req.AppVersion != 0 { + if !isSupportedAppVersion(req.AppVersion) { + app.Logger().Info("rejecting snapshot because unsupported app version", "app_version", req.AppVersion) + return abci.ResponseOfferSnapshot{ + Result: abci.ResponseOfferSnapshot_REJECT, + } + } + + app.Logger().Info("mounting keys for snapshot", "app_version", req.AppVersion) + app.mountKeysAndInit(req.AppVersion) + return app.BaseApp.OfferSnapshot(req) + } + + // If the app version is not set in the snapshot, this falls back to inferring the app version based on the upgrade height. if app.upgradeHeightV2 == 0 { - app.Logger().Debug("v2 upgrade height not set, assuming app version 2") + app.Logger().Info("v2 upgrade height not set, assuming app version 2") app.mountKeysAndInit(v2) return app.BaseApp.OfferSnapshot(req) } if req.Snapshot.Height >= uint64(app.upgradeHeightV2) { - app.Logger().Debug("snapshot height is greater than or equal to upgrade height, assuming app version 2") + app.Logger().Info("snapshot height is greater than or equal to upgrade height, assuming app version 2") app.mountKeysAndInit(v2) return app.BaseApp.OfferSnapshot(req) } - app.Logger().Debug("snapshot height is less than upgrade height, assuming app version 1") + app.Logger().Info("snapshot height is less than upgrade height, assuming app version 1") app.mountKeysAndInit(v1) return app.BaseApp.OfferSnapshot(req) } + +func isSupportedAppVersion(appVersion uint64) bool { + return appVersion == v1 || appVersion == v2 || appVersion == v3 +} diff --git a/app/app_test.go b/app/app_test.go index 16bca43ec7..a745ca2f8e 100644 --- a/app/app_test.go +++ b/app/app_test.go @@ -2,6 +2,7 @@ package app_test import ( "encoding/json" + "path/filepath" "testing" "github.com/celestiaorg/celestia-app/v3/app" @@ -112,42 +113,79 @@ func TestInitChain(t *testing.T) { } func TestOfferSnapshot(t *testing.T) { - logger := log.NewNopLogger() - db := tmdb.NewMemDB() - traceStore := &NoopWriter{} - invCheckPeriod := uint(1) - encodingConfig := encoding.MakeConfig(app.ModuleEncodingRegisters...) - upgradeHeight := int64(0) - appOptions := NoopAppOptions{} - snapshotOption := getSnapshotOption(t) - app := app.New(logger, db, traceStore, invCheckPeriod, encodingConfig, upgradeHeight, appOptions, snapshotOption) - - t.Run("should return ACCEPT", func(t *testing.T) { - request := abci.RequestOfferSnapshot{ - Snapshot: &abci.Snapshot{ - Height: 0x1b07ec, - Format: 0x2, - Chunks: 0x1, - Hash: []uint8{0xaf, 0xa5, 0xe, 0x16, 0x45, 0x4, 0x2e, 0x45, 0xd3, 0x49, 0xdf, 0x83, 0x2a, 0x57, 0x9d, 0x64, 0xc8, 0xad, 0xa5, 0xb, 0x65, 0x1b, 0x46, 0xd6, 0xc3, 0x85, 0x6, 0x51, 0xd7, 0x45, 0x8e, 0xb8}, - Metadata: []uint8{0xa, 0x20, 0xaf, 0xa5, 0xe, 0x16, 0x45, 0x4, 0x2e, 0x45, 0xd3, 0x49, 0xdf, 0x83, 0x2a, 0x57, 0x9d, 0x64, 0xc8, 0xad, 0xa5, 0xb, 0x65, 0x1b, 0x46, 0xd6, 0xc3, 0x85, 0x6, 0x51, 0xd7, 0x45, 0x8e, 0xb8}, - }, - AppHash: []byte("apphash"), - } + t.Run("should ACCEPT a snapshot with app version 0", func(t *testing.T) { + // Snapshots taken before the app version field was introduced to RequestOfferSnapshot should still be accepted. + app := createTestApp(t) + request := createRequest() + want := abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_ACCEPT} + got := app.OfferSnapshot(request) + assert.Equal(t, want, got) + }) + t.Run("should ACCEPT a snapshot with app version 1", func(t *testing.T) { + app := createTestApp(t) + request := createRequest() + request.AppVersion = 1 + want := abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_ACCEPT} + got := app.OfferSnapshot(request) + assert.Equal(t, want, got) + }) + t.Run("should ACCEPT a snapshot with app version 2", func(t *testing.T) { + app := createTestApp(t) + request := createRequest() + request.AppVersion = 2 + want := abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_ACCEPT} + got := app.OfferSnapshot(request) + assert.Equal(t, want, got) + }) + t.Run("should ACCEPT a snapshot with app version 3", func(t *testing.T) { + app := createTestApp(t) + request := createRequest() + request.AppVersion = 3 want := abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_ACCEPT} got := app.OfferSnapshot(request) assert.Equal(t, want, got) }) + t.Run("should REJECT a snapshot with unsupported app version", func(t *testing.T) { + app := createTestApp(t) + request := createRequest() + request.AppVersion = 4 // unsupported app version + want := abci.ResponseOfferSnapshot{Result: abci.ResponseOfferSnapshot_REJECT} + got := app.OfferSnapshot(request) + assert.Equal(t, want, got) + }) } -func getSnapshotOption(t *testing.T) func(*baseapp.BaseApp) { - snapshotDir := t.TempDir() - snapshotDB, err := tmdb.NewDB("metadata", tmdb.GoLevelDBBackend, t.TempDir()) +func createTestApp(t *testing.T) *app.App { + db := tmdb.NewMemDB() + config := encoding.MakeConfig(app.ModuleEncodingRegisters...) + upgradeHeight := int64(3) + snapshotDir := filepath.Join(t.TempDir(), "data", "snapshots") + snapshotDB, err := tmdb.NewDB("metadata", tmdb.GoLevelDBBackend, snapshotDir) require.NoError(t, err) snapshotStore, err := snapshots.NewStore(snapshotDB, snapshotDir) require.NoError(t, err) - interval := uint64(10) - keepRecent := uint32(10) - return baseapp.SetSnapshot(snapshotStore, snapshottypes.NewSnapshotOptions(interval, keepRecent)) + baseAppOption := baseapp.SetSnapshot(snapshotStore, snapshottypes.NewSnapshotOptions(10, 10)) + testApp := app.New(log.NewNopLogger(), db, nil, 0, config, upgradeHeight, util.EmptyAppOptions{}, baseAppOption) + require.NoError(t, err) + response := testApp.Info(abci.RequestInfo{}) + require.Equal(t, uint64(0), response.AppVersion) + return testApp +} + +func createRequest() abci.RequestOfferSnapshot { + return abci.RequestOfferSnapshot{ + // Snapshot was created by logging the contents of OfferSnapshot on a + // node that was syncing via state sync. + Snapshot: &abci.Snapshot{ + Height: 0x1b07ec, + Format: 0x2, + Chunks: 0x1, + Hash: []uint8{0xaf, 0xa5, 0xe, 0x16, 0x45, 0x4, 0x2e, 0x45, 0xd3, 0x49, 0xdf, 0x83, 0x2a, 0x57, 0x9d, 0x64, 0xc8, 0xad, 0xa5, 0xb, 0x65, 0x1b, 0x46, 0xd6, 0xc3, 0x85, 0x6, 0x51, 0xd7, 0x45, 0x8e, 0xb8}, + Metadata: []uint8{0xa, 0x20, 0xaf, 0xa5, 0xe, 0x16, 0x45, 0x4, 0x2e, 0x45, 0xd3, 0x49, 0xdf, 0x83, 0x2a, 0x57, 0x9d, 0x64, 0xc8, 0xad, 0xa5, 0xb, 0x65, 0x1b, 0x46, 0xd6, 0xc3, 0x85, 0x6, 0x51, 0xd7, 0x45, 0x8e, 0xb8}, + }, + AppHash: []byte("apphash"), + AppVersion: 0, // unit tests will override this + } } // NoopWriter is a no-op implementation of a writer. diff --git a/go.mod b/go.mod index f446db194d..25de005f38 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/cosmos/gogoproto v1.7.0 github.com/cosmos/ibc-apps/middleware/packet-forward-middleware/v6 v6.1.2 github.com/cosmos/ibc-go/v6 v6.2.2 - github.com/ethereum/go-ethereum v1.14.7 + github.com/ethereum/go-ethereum v1.14.11 github.com/gogo/protobuf v1.3.3 github.com/golang/mock v1.6.0 github.com/golang/protobuf v1.5.4 @@ -26,14 +26,14 @@ require ( github.com/rakyll/statik v0.1.7 github.com/rs/zerolog v1.33.0 github.com/spf13/cast v1.6.0 - github.com/spf13/cobra v1.8.0 + github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 github.com/tendermint/tendermint v0.34.29 github.com/tendermint/tm-db v0.6.7 - golang.org/x/exp v0.0.0-20240213143201-ec583247a57a - google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 - google.golang.org/grpc v1.67.0 + golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e + google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 + google.golang.org/grpc v1.66.0 google.golang.org/protobuf v1.34.2 gopkg.in/yaml.v2 v2.4.0 k8s.io/apimachinery v0.31.1 @@ -56,8 +56,9 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect github.com/bgentry/speakeasy v0.1.0 // indirect - github.com/bits-and-blooms/bitset v1.10.0 // indirect - github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect + github.com/bits-and-blooms/bitset v1.13.0 // indirect + github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect + github.com/bufbuild/protocompile v0.14.1 // indirect github.com/celestiaorg/bittwister v0.0.0-20231213180407-65cdbaf5b8c7 // indirect github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect @@ -76,6 +77,7 @@ require ( github.com/cosmos/gorocksdb v1.2.0 // indirect github.com/cosmos/iavl v0.19.6 // indirect github.com/cosmos/ledger-cosmos-go v0.13.2 // indirect + github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c // indirect github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect github.com/creachadair/taskgroup v0.3.2 // indirect github.com/danieljoos/wincred v1.1.2 // indirect @@ -90,6 +92,7 @@ require ( github.com/dvsekhvalnov/jose2go v1.6.0 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/ethereum/c-kzg-4844 v1.0.0 // indirect + github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect @@ -119,7 +122,7 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.2 // indirect github.com/gorilla/handlers v1.5.2 // indirect - github.com/gorilla/websocket v1.5.0 // indirect + github.com/gorilla/websocket v1.5.3 // indirect github.com/grafana/otel-profiling-go v0.5.1 // indirect github.com/grafana/pyroscope-go v1.1.2 // indirect github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect @@ -131,12 +134,11 @@ require ( github.com/hashicorp/go-getter v1.7.4 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/go-safetemp v1.0.0 // indirect - github.com/hashicorp/go-uuid v1.0.2 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hdevalence/ed25519consensus v0.0.0-20220222234857-c00d1f31bab3 // indirect - github.com/holiman/uint256 v1.3.0 // indirect + github.com/holiman/uint256 v1.3.1 // indirect github.com/iancoleman/orderedmap v0.2.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/improbable-eng/grpc-web v0.15.0 // indirect @@ -148,16 +150,15 @@ require ( github.com/klauspost/compress v1.17.9 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/klauspost/reedsolomon v1.12.1 // indirect - github.com/lib/pq v1.10.7 // indirect + github.com/lib/pq v1.10.9 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/manifoldco/promptui v0.9.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-runewidth v0.0.14 // indirect github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0 // indirect - github.com/minio/highwayhash v1.0.2 // indirect + github.com/minio/highwayhash v1.0.3 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/minio/minio-go/v7 v7.0.74 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect @@ -172,17 +173,16 @@ require ( github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect github.com/onsi/ginkgo v1.16.5 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect - github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.19.1 // indirect - github.com/prometheus/client_model v0.6.0 // indirect - github.com/prometheus/common v0.53.0 // indirect - github.com/prometheus/procfs v0.12.0 // indirect + github.com/prometheus/client_golang v1.20.3 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/regen-network/cosmos-proto v0.3.1 // indirect - github.com/rivo/uniseg v0.4.4 // indirect github.com/rs/cors v1.8.3 // indirect github.com/rs/xid v1.5.0 // indirect github.com/sasha-s/go-deadlock v0.3.1 // indirect @@ -192,7 +192,7 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/viper v1.15.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect - github.com/supranational/blst v0.3.11 // indirect + github.com/supranational/blst v0.3.13 // indirect github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect github.com/tendermint/go-amino v0.16.0 // indirect @@ -207,11 +207,11 @@ require ( go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect - go.opentelemetry.io/otel v1.26.0 // indirect + go.opentelemetry.io/otel v1.30.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.18.0 // indirect - go.opentelemetry.io/otel/metric v1.26.0 // indirect - go.opentelemetry.io/otel/sdk v1.26.0 // indirect - go.opentelemetry.io/otel/trace v1.26.0 // indirect + go.opentelemetry.io/otel/metric v1.30.0 // indirect + go.opentelemetry.io/otel/sdk v1.30.0 // indirect + go.opentelemetry.io/otel/trace v1.30.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/crypto v0.27.0 // indirect @@ -224,7 +224,7 @@ require ( golang.org/x/time v0.5.0 // indirect google.golang.org/api v0.169.0 // indirect google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect @@ -241,11 +241,11 @@ require ( ) replace ( - github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.24.1-sdk-v0.46.16 + github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.25.0-sdk-v0.46.16 // Pin to ledger-cosmos-go v0.12.4 to avoid a breaking change introduced in v0.13.0 // The following replace statement can be removed when we upgrade to cosmos-sdk >= v0.50.0 github.com/cosmos/ledger-cosmos-go => github.com/cosmos/ledger-cosmos-go v0.12.4 github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 - github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.41.0-tm-v0.34.29 + github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.42.0-tm-v0.34.35 ) diff --git a/go.sum b/go.sum index 652f175022..253139838d 100644 --- a/go.sum +++ b/go.sum @@ -279,8 +279,8 @@ github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1U github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bits-and-blooms/bitset v1.10.0 h1:ePXTeiPEazB5+opbv5fr8umg2R/1NlzgDsyepwsSr88= -github.com/bits-and-blooms/bitset v1.10.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= +github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/btcsuite/btcd v0.0.0-20190315201642-aa6e0f35703c/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= @@ -289,8 +289,8 @@ github.com/btcsuite/btcd v0.21.0-beta.0.20201114000516-e9c7a5ac6401/go.mod h1:Sv github.com/btcsuite/btcd v0.22.1 h1:CnwP9LM/M9xuRrGSCGeMVs9iv09uMqwsVX7EeIpgV2c= github.com/btcsuite/btcd v0.22.1/go.mod h1:wqgTSL29+50LRkmOVknEdmt8ZojIzhuWvgu/iptuN7Y= github.com/btcsuite/btcd/btcec/v2 v2.1.2/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE= -github.com/btcsuite/btcd/btcec/v2 v2.3.2 h1:5n0X6hX0Zk+6omWcihdYvdAlGf2DfasC0GMf7DClJ3U= -github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= +github.com/btcsuite/btcd/btcec/v2 v2.3.4 h1:3EJjcN70HCu/mwqlUsGK8GcNVyLVxFDlWurTXGPFfiQ= +github.com/btcsuite/btcd/btcec/v2 v2.3.4/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= github.com/btcsuite/btcd/btcutil v1.1.3 h1:xfbtw8lwpp0G6NwSHb+UE67ryTFHJAiNuipusjXSohQ= github.com/btcsuite/btcd/btcutil v1.1.3/go.mod h1:UR7dsSJzJUfMmFiiLlIrMq1lS9jh9EdCV7FStZSnpi0= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= @@ -308,8 +308,8 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= -github.com/bufbuild/protocompile v0.5.1 h1:mixz5lJX4Hiz4FpqFREJHIXLfaLBntfaJv1h+/jS+Qg= -github.com/bufbuild/protocompile v0.5.1/go.mod h1:G5iLmavmF4NsYtpZFvE3B/zFch2GIY8+wjsYLR/lc40= +github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw= +github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= @@ -317,10 +317,10 @@ github.com/celestiaorg/bittwister v0.0.0-20231213180407-65cdbaf5b8c7 h1:nxplQi8w github.com/celestiaorg/bittwister v0.0.0-20231213180407-65cdbaf5b8c7/go.mod h1:1EF5MfOxVf0WC51Gb7pJ6bcZxnXKNAf9pqWtjgPBAYc= github.com/celestiaorg/blobstream-contracts/v3 v3.1.0 h1:h1Y4V3EMQ2mFmNtWt2sIhZIuyASInj1a9ExI8xOsTOw= github.com/celestiaorg/blobstream-contracts/v3 v3.1.0/go.mod h1:x4DKyfKOSv1ZJM9NwV+Pw01kH2CD7N5zTFclXIVJ6GQ= -github.com/celestiaorg/celestia-core v1.41.0-tm-v0.34.29 h1:hRdTxe+Dz6kiqifRZCC9qYQiTJME7CzAZodrTHlhhnk= -github.com/celestiaorg/celestia-core v1.41.0-tm-v0.34.29/go.mod h1:H6vjzdoqTt4qmbf11z1Lnc9YLUp/B8ITEQLhU92ghqQ= -github.com/celestiaorg/cosmos-sdk v1.24.1-sdk-v0.46.16 h1:SeQ7Y/CyOcUMKo7mQiexaj/pZ/xIgyuZFIwYZwpSkWE= -github.com/celestiaorg/cosmos-sdk v1.24.1-sdk-v0.46.16/go.mod h1:Bpl1LSWiDpQumgOhhMTZBMopqa0j7fRasIhvTZB44P0= +github.com/celestiaorg/celestia-core v1.42.0-tm-v0.34.35 h1:bWy5XOgeuuSLe0Lc/htL9/QLEURBjA1JTvEko6bEBhg= +github.com/celestiaorg/celestia-core v1.42.0-tm-v0.34.35/go.mod h1:/fK0n3ps09t5uErBQe1QZbrE81L81MNUzWpFyWQLDT0= +github.com/celestiaorg/cosmos-sdk v1.25.0-sdk-v0.46.16 h1:f+fTe7GGk0/qgdzyqB8kk8EcDf9d6MC22khBTQiDXsU= +github.com/celestiaorg/cosmos-sdk v1.25.0-sdk-v0.46.16/go.mod h1:07Z8HJqS8Rw4XlZ+ok3D3NM/X/in8mvcGLvl0Zb5wrA= github.com/celestiaorg/go-square v1.1.0 h1:K4tBL5PCJwDtpBfyDxxZ3N962aC9VYb5/bw3LjagEtY= github.com/celestiaorg/go-square v1.1.0/go.mod h1:1EXMErhDrWJM8B8V9hN7dqJ2kUTClfwdqMOmF9yQUa0= github.com/celestiaorg/go-square/v2 v2.0.0 h1:U5QV8/de5lc7glosfgyHhcxbFwNuwU4+6aYZ2RgjM04= @@ -381,8 +381,8 @@ github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/e github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v1.1.1 h1:XnKU22oiCLy2Xn8vp1re67cXg4SAasg/WDt1NtcRFaw= -github.com/cockroachdb/pebble v1.1.1/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU= +github.com/cockroachdb/pebble v1.1.2 h1:CUh2IPtR4swHlEj48Rhfzw6l/d0qA31fItcIszQVIsA= +github.com/cockroachdb/pebble v1.1.2/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= @@ -433,8 +433,8 @@ github.com/cosmos/ledger-cosmos-go v0.12.4/go.mod h1:fjfVWRf++Xkygt9wzCsjEBdjcf7 github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= -github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c h1:uQYC5Z1mdLRPrZhHjHxufI8+2UG/i25QG92j0Er9p6I= github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs= github.com/crate-crypto/go-kzg-4844 v1.0.0 h1:TsSgHwrkTKecKJ4kadtHi4b3xHW5dCFUDFnUp1TsawI= @@ -514,10 +514,10 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/ethereum/c-kzg-4844 v1.0.0 h1:0X1LBXxaEtYD9xsyj9B9ctQEZIpnvVDeoBx8aHEwTNA= github.com/ethereum/c-kzg-4844 v1.0.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/ethereum/go-ethereum v1.10.17/go.mod h1:Lt5WzjM07XlXc95YzrhosmR4J9Ahd6X2wyEV2SvGhk0= -github.com/ethereum/go-ethereum v1.14.7 h1:EHpv3dE8evQmpVEQ/Ne2ahB06n2mQptdwqaMNhAT29g= -github.com/ethereum/go-ethereum v1.14.7/go.mod h1:Mq0biU2jbdmKSZoqOj29017ygFrMnB5/Rifwp980W4o= -github.com/ethereum/go-verkle v0.1.1-0.20240306133620-7d920df305f0 h1:KrE8I4reeVvf7C1tm8elRjj4BdscTYzz/WAbYyf/JI4= -github.com/ethereum/go-verkle v0.1.1-0.20240306133620-7d920df305f0/go.mod h1:D9AJLVXSyZQXJQVk8oh1EwjISE+sJTn2duYIZC0dy3w= +github.com/ethereum/go-ethereum v1.14.11 h1:8nFDCUUE67rPc6AKxFj7JKaOa2W/W1Rse3oS6LvvxEY= +github.com/ethereum/go-ethereum v1.14.11/go.mod h1:+l/fr42Mma+xBnhefL/+z11/hcmJ2egl+ScIVPjhc7E= +github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9 h1:8NfxH2iXvJ60YRB8ChToFTUzl8awsc3cJ8CbLjGIl/A= +github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= @@ -531,8 +531,6 @@ github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYF github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= -github.com/fjl/memsize v0.0.2 h1:27txuSD9or+NZlnOWdKUxeBzTAUkWCVh+4Gf2dWFOzA= -github.com/fjl/memsize v0.0.2/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= @@ -764,8 +762,8 @@ github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWS github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= -github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grafana/otel-profiling-go v0.5.1 h1:stVPKAFZSa7eGiqbYuG25VcqYksR6iWvF3YH66t4qL8= github.com/grafana/otel-profiling-go v0.5.1/go.mod h1:ftN/t5A/4gQI19/8MoWurBEtC6gFw8Dns1sJZ9W4Tls= github.com/grafana/pyroscope-go v1.1.2 h1:7vCfdORYQMCxIzI3NlYAs3FcBP760+gWuYWOyiVyYx8= @@ -812,9 +810,8 @@ github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoD github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= -github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= @@ -836,8 +833,8 @@ github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4/go.mod h1:5GuXa7vkL8 github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= -github.com/holiman/uint256 v1.3.0 h1:4wdcm/tnd0xXdu7iS3ruNvxkWwrb4aeBQv19ayYn8F4= -github.com/holiman/uint256 v1.3.0/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= +github.com/holiman/uint256 v1.3.1 h1:JfTzmih28bittyHM8z360dCjIA9dbPIBlcTI6lmctQs= +github.com/holiman/uint256 v1.3.1/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/huin/goupnp v1.0.3-0.20220313090229-ca81a64b4204/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= @@ -939,8 +936,8 @@ github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7 github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= -github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= @@ -978,8 +975,8 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= -github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -987,8 +984,8 @@ github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3N github.com/mimoo/StrobeGo v0.0.0-20181016162300-f8f6d4d2b643/go.mod h1:43+3pMjjKimDBf5Kr4ZFNGbLql1zKkbImw+fZbw3geM= github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0 h1:QRUSJEgZn2Snx0EmT/QLXibWjSUDjKWvXIT19NBVp94= github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0/go.mod h1:43+3pMjjKimDBf5Kr4ZFNGbLql1zKkbImw+fZbw3geM= -github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= -github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= +github.com/minio/highwayhash v1.0.3 h1:kbnuUMoHYyVl7szWjSxJnxw11k2U709jqFPPmIUyD6Q= +github.com/minio/highwayhash v1.0.3/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.74 h1:fTo/XlPBTSpo3BAMshlwKL5RspXRv9us5UeHEGYCFe0= @@ -1066,8 +1063,8 @@ github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.20.0 h1:8W0cWlwFkflGPLltQvLRB7ZVD5HuP6ng320w2IS245Q= -github.com/onsi/gomega v1.20.0/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= +github.com/onsi/gomega v1.34.2 h1:pNCwDkzrsv7MS9kpaQvVb1aVLahQXyJ/Tv5oAZMI3i8= +github.com/onsi/gomega v1.34.2/go.mod h1:v1xfxRgk0KIsG+QOdm7p8UosrOzPYRo60fd3B/1Dukc= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= @@ -1093,8 +1090,8 @@ github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144T github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= -github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= @@ -1121,16 +1118,16 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= -github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_golang v1.20.3 h1:oPksm4K8B+Vt35tUhw6GbSNSgVlVSBH0qELP/7u83l4= +github.com/prometheus/client_golang v1.20.3/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= -github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= @@ -1139,16 +1136,16 @@ github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt2 github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= -github.com/prometheus/common v0.53.0 h1:U2pL9w9nmJwJDa4qqLQ3ZaePJ6ZTwt7cMD3AG3+aLCE= -github.com/prometheus/common v0.53.0/go.mod h1:BrxBKv3FWBIGXw89Mg1AeBq7FSyRzXWI3l3e7W3RN5U= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.3.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= -github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ= github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc= @@ -1160,9 +1157,8 @@ github.com/regen-network/cosmos-proto v0.3.1/go.mod h1:jO0sVX6a1B36nmE8C9xBFXpNw github.com/regen-network/protobuf v1.3.3-alpha.regen.1 h1:OHEc+q5iIAXpqiqFKeLpu5NwTIkVXUs48vFMwzqpqY4= github.com/regen-network/protobuf v1.3.3-alpha.regen.1/go.mod h1:2DjTFR1HhMQhiWC5sZ4OhQ3+NtdbZ6oBDKQwq5Ou+FI= github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= -github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -1215,8 +1211,8 @@ github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= -github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= @@ -1254,8 +1250,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= -github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/supranational/blst v0.3.13 h1:AYeSxdOMacwu7FBmpfloBz5pbFXDmJL33RuwnKtmTjk= +github.com/supranational/blst v0.3.13/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok= @@ -1351,19 +1347,19 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.4 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= -go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= -go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= +go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= +go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.18.0 h1:hSWWvDjXHVLq9DkmB+77fl8v7+t+yYiS+eNkiplDK54= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.18.0/go.mod h1:zG7KQql1WjZCaUJd+L/ReSYx4bjbYJxg5ws9ws+mYes= go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= -go.opentelemetry.io/otel/metric v1.26.0 h1:7S39CLuY5Jgg9CrnA9HHiEjGMF/X2VHvoXGgSllRz30= -go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= +go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= +go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= -go.opentelemetry.io/otel/sdk v1.26.0 h1:Y7bumHf5tAiDlRYFmGqetNcLaVUZmh4iYfmGxtmz7F8= -go.opentelemetry.io/otel/sdk v1.26.0/go.mod h1:0p8MXpqLeJ0pzcszQQN4F0S5FVjBLgypeGSngLsmirs= +go.opentelemetry.io/otel/sdk v1.30.0 h1:cHdik6irO49R5IysVhdn8oaiR9m8XluDaJAs4DfOrYE= +go.opentelemetry.io/otel/sdk v1.30.0/go.mod h1:p14X4Ok8S+sygzblytT1nqG98QG2KYKv++HE0LY/mhg= go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= -go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= -go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= +go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= +go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -1420,8 +1416,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= -golang.org/x/exp v0.0.0-20240213143201-ec583247a57a h1:HinSgX1tJRX3KsL//Gxynpw5CTOAIPhgL4W8PNiIpVE= -golang.org/x/exp v0.0.0-20240213143201-ec583247a57a/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e h1:I88y4caeGeuDQxgdoFPUq097j7kNfw6uvuiNxUBfcBk= +golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -1567,7 +1563,6 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1669,6 +1664,7 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -1948,10 +1944,10 @@ google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= -google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 h1:wKguEg1hsxI2/L3hUYrpo1RVi48K+uTyzKqprwLXsb8= -google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142/go.mod h1:d6be+8HhtEtucleCbxpPW9PA9XwISACu8nvpPqF0BVo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 h1:e7S5W7MGGLaSu8j3YjdezkZ+m1/Nm0uRVRMEMGk26Xs= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 h1:+rdxYoE3E5htTEWIe15GlN6IfvbURM//Jt0mmkmm6ZU= +google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117/go.mod h1:OimBR/bc1wPO9iV4NC2bpyjy3VnAwZh5EBPQdtaE5oo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -1994,8 +1990,8 @@ google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.67.0 h1:IdH9y6PF5MPSdAntIcpjQ+tXO41pcQsfZV2RxtQgVcw= -google.golang.org/grpc v1.67.0/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= diff --git a/scripts/single-node.sh b/scripts/single-node.sh index d7d19b4f07..b6a6e6486a 100755 --- a/scripts/single-node.sh +++ b/scripts/single-node.sh @@ -1,5 +1,7 @@ #!/bin/sh +# This script starts a single node testnet. + # Stop script execution if an error is encountered set -o errexit # Stop script execution if an undefined variable is used @@ -60,7 +62,7 @@ createGenesis() { # If you encounter: `sed: -I or -i may not be used with stdin` on MacOS you can mitigate by installing gnu-sed # https://gist.github.com/andre3k1/e3a1a7133fded5de5a9ee99c87c6fa0d?permalink_comment_id=3082272#gistcomment-3082272 - # Override the default RPC servier listening address + # Override the default RPC server listening address sed -i'.bak' 's#"tcp://127.0.0.1:26657"#"tcp://0.0.0.0:26657"#g' "${CELESTIA_APP_HOME}"/config/config.toml # Enable transaction indexing diff --git a/scripts/state-sync.sh b/scripts/state-sync.sh new file mode 100755 index 0000000000..9d7a8417b0 --- /dev/null +++ b/scripts/state-sync.sh @@ -0,0 +1,108 @@ +#!/bin/sh + +# This script starts a single node and attempts to state sync with a node +# started via ./single-node.sh + +# Stop script execution if an error is encountered +set -o errexit +# Stop script execution if an undefined variable is used +set -o nounset + +if ! [ -x "$(command -v celestia-appd)" ] +then + echo "celestia-appd could not be found. Please install the celestia-appd binary using 'make install' and make sure the PATH contains the directory where the binary exists. By default, go will install the binary under '~/go/bin'" + exit 1 +fi + +CHAIN_ID="test" +KEY_NAME="validator" +KEYRING_BACKEND="test" +COINS="1000000000000000utia" +DELEGATION_AMOUNT="5000000000utia" +SINGLE_NODE_HOME="${HOME}/.celestia-app" +CELESTIA_APP_HOME="${HOME}/.celestia-app-state-sync" +CELESTIA_APP_VERSION=$(celestia-appd version 2>&1) +GENESIS_FILE="${CELESTIA_APP_HOME}/config/genesis.json" +FEES="500utia" +RPC="0.0.0.0:26657" + +echo "celestia-app home: ${CELESTIA_APP_HOME}" +echo "celestia-app version: ${CELESTIA_APP_VERSION}" +echo "" + +BLOCK_HEIGHT=$(curl -s $RPC/block | jq -r .result.block.header.height); +TRUST_HASH=$(curl -s "$RPC/block?height=$BLOCK_HEIGHT" | jq -r .result.block_id.hash) + +echo "Block height: $BLOCK_HEIGHT" +echo "Trust hash: $TRUST_HASH" +echo "Enabling state sync in config.toml..." +sed -i.bak -E "s|^(enable[[:space:]]+=[[:space:]]+).*$|\1true| ; \ +s|^(rpc_servers[[:space:]]+=[[:space:]]+).*$|\1\"$RPC,$RPC\"| ; \ +s|^(trust_height[[:space:]]+=[[:space:]]+).*$|\1$BLOCK_HEIGHT| ; \ +s|^(trust_hash[[:space:]]+=[[:space:]]+).*$|\1\"$TRUST_HASH\"|" $CELESTIA_APP_HOME/config/config.toml + +PEER=$(curl -s http://${RPC}/status | jq -r '.result.node_info.id + "@127.0.0.1:26656"') +echo "Setting persistent peer to ${PEER}" + +createGenesis() { + echo "Initializing validator and node config files..." + celestia-appd init ${CHAIN_ID} \ + --chain-id ${CHAIN_ID} \ + --home "${CELESTIA_APP_HOME}" \ + > /dev/null 2>&1 # Hide output to reduce terminal noise + + echo "Adding a new key to the keyring..." + celestia-appd keys add ${KEY_NAME} \ + --keyring-backend=${KEYRING_BACKEND} \ + --home "${CELESTIA_APP_HOME}" \ + > /dev/null 2>&1 # Hide output to reduce terminal noise + + echo "Copying genesis.json from the node started via ./single-node.sh..." + cp ${SINGLE_NODE_HOME}/config/genesis.json ${CELESTIA_APP_HOME}/config/genesis.json + + # If you encounter: `sed: -I or -i may not be used with stdin` on MacOS you can mitigate by installing gnu-sed + # https://gist.github.com/andre3k1/e3a1a7133fded5de5a9ee99c87c6fa0d?permalink_comment_id=3082272#gistcomment-3082272 + + # Override the default RPC server listening address to not conflict with the node started via ./single-node.sh + sed -i'.bak' 's#"tcp://127.0.0.1:26657"#"tcp://0.0.0.0:26000"#g' "${CELESTIA_APP_HOME}"/config/config.toml + + # Override the p2p address to not conflict with the node started via ./single-node.sh + sed -i'.bak' 's#laddr = "tcp://0.0.0.0:26656"#laddr = "tcp://0.0.0.0:36656"#g' "${CELESTIA_APP_HOME}"/config/config.toml + + # Enable transaction indexing + sed -i'.bak' 's#"null"#"kv"#g' "${CELESTIA_APP_HOME}"/config/config.toml + + # Persist ABCI responses + sed -i'.bak' 's#discard_abci_responses = true#discard_abci_responses = false#g' "${CELESTIA_APP_HOME}"/config/config.toml + + # Override the log level to debug + # sed -i'.bak' 's#log_level = "info"#log_level = "debug"#g' "${CELESTIA_APP_HOME}"/config/config.toml +} + +deleteCelestiaAppHome() { + echo "Deleting $CELESTIA_APP_HOME..." + rm -r "$CELESTIA_APP_HOME" +} + +startCelestiaApp() { + echo "Starting celestia-app..." + celestia-appd start \ + --home "${CELESTIA_APP_HOME}" \ + --grpc.enable \ + --grpc.address="0.0.0.0:9999" \ + --p2p.persistent_peers=${PEER} \ + --fast_sync false \ + --v2-upgrade-height 3 +} + +if [ -f $GENESIS_FILE ]; then + echo "Do you want to delete existing ${CELESTIA_APP_HOME}? [y/n]" + read -r response + if [ "$response" = "y" ]; then + deleteCelestiaAppHome + createGenesis + fi +else + createGenesis +fi +startCelestiaApp diff --git a/tools/blocketa/README.md b/tools/blocketa/README.md index 1f8c2b4d1b..f7de7539b5 100644 --- a/tools/blocketa/README.md +++ b/tools/blocketa/README.md @@ -15,4 +15,4 @@ arrivalTime: 2024-08-28 17:24:23.483542677 +0000 UTC ``` > [!NOTE] -> The block time is currently hard-coded. If you're running this for a network with a different block time, you'll need to update the `blockTime` constant in the main.go file. You can use [https://www.mintscan.io/celestia/block](https://www.mintscan.io/celestia/block/) or the blocktime tool. \ No newline at end of file +> The block time is currently hard-coded. If you're running this for a network with a different block time, you'll need to update the `blockTime` constant in the main.go file. You can use [https://www.mintscan.io/celestia/block](https://www.mintscan.io/celestia/block/) or the blocktime tool. diff --git a/tools/chainbuilder/README.md b/tools/chainbuilder/README.md index 99911468ac..50ceb3c06d 100644 --- a/tools/chainbuilder/README.md +++ b/tools/chainbuilder/README.md @@ -6,13 +6,13 @@ Use `go` to run the binary as follows: -``` +```shell go run ./tools/chainbuilder ``` This will create a directory with the name `testnode-{chainID}`. All files will be populated and blocks generated based on specified input. You can run a validator on the file system afterwards by calling: -``` +```shell celestia-appd start --home /path/to/testnode-{chainID} ``` From d9877a5e8b0485aa8f75cf3e2bf52d15eb5c163e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Oct 2024 21:35:30 +0000 Subject: [PATCH 60/85] chore(deps): Bump google.golang.org/grpc from 1.67.0 to 1.67.1 (#3927) Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.67.0 to 1.67.1.
Release notes

Sourced from google.golang.org/grpc's releases.

Release 1.67.1

Bug Fixes

  • transport: Fix a bug causing stream failures due to miscalculation of the flow control window in both clients and servers. (#7667)
  • xds/server: Fix xDS Server memory leak. (#7681)
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=google.golang.org/grpc&package-manager=go_modules&previous-version=1.67.0&new-version=1.67.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 25de005f38..8eb572023a 100644 --- a/go.mod +++ b/go.mod @@ -32,8 +32,8 @@ require ( github.com/tendermint/tendermint v0.34.29 github.com/tendermint/tm-db v0.6.7 golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e - google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 - google.golang.org/grpc v1.66.0 + google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 + google.golang.org/grpc v1.67.1 google.golang.org/protobuf v1.34.2 gopkg.in/yaml.v2 v2.4.0 k8s.io/apimachinery v0.31.1 diff --git a/go.sum b/go.sum index 253139838d..93305cf50a 100644 --- a/go.sum +++ b/go.sum @@ -1944,8 +1944,8 @@ google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= -google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117 h1:+rdxYoE3E5htTEWIe15GlN6IfvbURM//Jt0mmkmm6ZU= -google.golang.org/genproto/googleapis/api v0.0.0-20240604185151-ef581f913117/go.mod h1:OimBR/bc1wPO9iV4NC2bpyjy3VnAwZh5EBPQdtaE5oo= +google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 h1:wKguEg1hsxI2/L3hUYrpo1RVi48K+uTyzKqprwLXsb8= +google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142/go.mod h1:d6be+8HhtEtucleCbxpPW9PA9XwISACu8nvpPqF0BVo= google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -1990,8 +1990,8 @@ google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= -google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= +google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= From 95ca5c73d8fcbb837367248ba41e936f07d67987 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Oct 2024 08:41:43 -0400 Subject: [PATCH 61/85] chore(deps): Bump google.golang.org/protobuf from 1.34.2 to 1.35.1 (#3949) Bumps google.golang.org/protobuf from 1.34.2 to 1.35.1. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=google.golang.org/protobuf&package-manager=go_modules&previous-version=1.34.2&new-version=1.35.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8eb572023a..4cfec4a781 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 google.golang.org/grpc v1.67.1 - google.golang.org/protobuf v1.34.2 + google.golang.org/protobuf v1.35.1 gopkg.in/yaml.v2 v2.4.0 k8s.io/apimachinery v0.31.1 ) diff --git a/go.sum b/go.sum index 93305cf50a..dcb63dcc5e 100644 --- a/go.sum +++ b/go.sum @@ -2008,8 +2008,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From b99ef0c1d1a8a5fdc5d6a641b522f3b6420401b0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Oct 2024 10:36:03 -0400 Subject: [PATCH 62/85] chore(deps): Bump github.com/celestiaorg/nmt from 0.22.1 to 0.22.2 (#3951) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/celestiaorg/nmt](https://github.com/celestiaorg/nmt) from 0.22.1 to 0.22.2.
Release notes

Sourced from github.com/celestiaorg/nmt's releases.

v0.22.2

What's Changed

Full Changelog: https://github.com/celestiaorg/nmt/compare/v0.22.1...v0.22.2

Commits
  • c1a9cfc chore(deps): bump github.com/tidwall/gjson from 1.17.3 to 1.18.0 (#277)
  • b2a0187 fix(ci): update the Buf CI to push tags to the Buf Schema Registry (#280)
  • d6a3436 ci: support releasing proto definitions to BSR (#274)
  • 29e9433 docs: add comment to leaf case in verify subtree roots inclusion (#272)
  • cce31e8 chore(deps): bump github.com/tidwall/gjson from 1.17.1 to 1.17.3 (#271)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/celestiaorg/nmt&package-manager=go_modules&previous-version=0.22.1&new-version=0.22.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 4cfec4a781..1745a22218 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/celestiaorg/go-square v1.1.0 github.com/celestiaorg/go-square/v2 v2.0.0 github.com/celestiaorg/knuu v0.16.1 - github.com/celestiaorg/nmt v0.22.1 + github.com/celestiaorg/nmt v0.22.2 github.com/celestiaorg/rsmt2d v0.14.0 github.com/cometbft/cometbft-db v0.7.0 github.com/cosmos/cosmos-proto v1.0.0-beta.5 diff --git a/go.sum b/go.sum index dcb63dcc5e..0d528ba941 100644 --- a/go.sum +++ b/go.sum @@ -329,8 +329,8 @@ github.com/celestiaorg/knuu v0.16.1 h1:EOR/c9kvc0jZet/mma2qwAdlvEbl94bW9cC8FItky github.com/celestiaorg/knuu v0.16.1/go.mod h1:y20nUmVWVgbzxBKHqmbwp3C0ZJ9J9ovCg1ylHo85hdQ= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4/go.mod h1:fzuHnhzj1pUygGz+1ZkB3uQbEUL4htqCGJ4Qs2LwMZA= -github.com/celestiaorg/nmt v0.22.1 h1:t7fqoP5MJ8mBns5DB2XjfcPxQpS3CKMkY+v+BEkDxYc= -github.com/celestiaorg/nmt v0.22.1/go.mod h1:ia/EpCk0enD5yO5frcxoNoFToz2Ghtk2i+blmCRjIY8= +github.com/celestiaorg/nmt v0.22.2 h1:JmOMtZL9zWAed1hiwb9DDs+ELcKp/ZQZ3rPverge/V8= +github.com/celestiaorg/nmt v0.22.2/go.mod h1:/7huDiSRL/d2EGhoiKctgSzmLOJoWG8yEfbFtY1+Mow= github.com/celestiaorg/rsmt2d v0.14.0 h1:L7XJ3tRJDY8sQcvCjzHq0L7JmsmaSD+VItymIYFLqYc= github.com/celestiaorg/rsmt2d v0.14.0/go.mod h1:4kxqiTdFev49sGiKXTDjohbWYOG5GlcIfftTgaBJnpc= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= @@ -1264,8 +1264,8 @@ github.com/tidwall/btree v1.5.0 h1:iV0yVY/frd7r6qGBXfEYs7DH0gTDgrKTrDjS7xt/IyQ= github.com/tidwall/btree v1.5.0/go.mod h1:LGm8L/DZjPLmeWGjv5kFrY8dL4uVhMmzmmLYmsObdKE= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= -github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= From 72f85492ff972030c079e20408cd807a0ee33675 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Oct 2024 16:37:03 +0000 Subject: [PATCH 63/85] chore(deps): Bump github.com/celestiaorg/go-square from 1.1.0 to 1.1.1 (#3950) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/celestiaorg/go-square](https://github.com/celestiaorg/go-square) from 1.1.0 to 1.1.1.
Release notes

Sourced from github.com/celestiaorg/go-square's releases.

v1.1.1

What's Changed

New Contributors

Full Changelog: https://github.com/celestiaorg/go-square/compare/v1.1.0...v1.1.1

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/celestiaorg/go-square&package-manager=go_modules&previous-version=1.1.0&new-version=1.1.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 1745a22218..5c0895b401 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( cosmossdk.io/errors v1.0.1 cosmossdk.io/math v1.3.0 github.com/celestiaorg/blobstream-contracts/v3 v3.1.0 - github.com/celestiaorg/go-square v1.1.0 + github.com/celestiaorg/go-square v1.1.1 github.com/celestiaorg/go-square/v2 v2.0.0 github.com/celestiaorg/knuu v0.16.1 github.com/celestiaorg/nmt v0.22.2 diff --git a/go.sum b/go.sum index 0d528ba941..c5b22440e2 100644 --- a/go.sum +++ b/go.sum @@ -321,8 +321,8 @@ github.com/celestiaorg/celestia-core v1.42.0-tm-v0.34.35 h1:bWy5XOgeuuSLe0Lc/htL github.com/celestiaorg/celestia-core v1.42.0-tm-v0.34.35/go.mod h1:/fK0n3ps09t5uErBQe1QZbrE81L81MNUzWpFyWQLDT0= github.com/celestiaorg/cosmos-sdk v1.25.0-sdk-v0.46.16 h1:f+fTe7GGk0/qgdzyqB8kk8EcDf9d6MC22khBTQiDXsU= github.com/celestiaorg/cosmos-sdk v1.25.0-sdk-v0.46.16/go.mod h1:07Z8HJqS8Rw4XlZ+ok3D3NM/X/in8mvcGLvl0Zb5wrA= -github.com/celestiaorg/go-square v1.1.0 h1:K4tBL5PCJwDtpBfyDxxZ3N962aC9VYb5/bw3LjagEtY= -github.com/celestiaorg/go-square v1.1.0/go.mod h1:1EXMErhDrWJM8B8V9hN7dqJ2kUTClfwdqMOmF9yQUa0= +github.com/celestiaorg/go-square v1.1.1 h1:Cy3p8WVspVcyOqHM8BWFuuYPwMitO1pYGe+ImILFZRA= +github.com/celestiaorg/go-square v1.1.1/go.mod h1:1EXMErhDrWJM8B8V9hN7dqJ2kUTClfwdqMOmF9yQUa0= github.com/celestiaorg/go-square/v2 v2.0.0 h1:U5QV8/de5lc7glosfgyHhcxbFwNuwU4+6aYZ2RgjM04= github.com/celestiaorg/go-square/v2 v2.0.0/go.mod h1:y0BolG0tRM7UN1sAQyDDUkT+aMJPwFIjviVvnCB62C0= github.com/celestiaorg/knuu v0.16.1 h1:EOR/c9kvc0jZet/mma2qwAdlvEbl94bW9cC8FItkyBE= From 206b96c313f33755cc723d0ef496010220a4a7c2 Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Wed, 9 Oct 2024 19:31:41 +0200 Subject: [PATCH 64/85] test: v3 e2e upgrade (#3910) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes https://github.com/celestiaorg/celestia-app/issues/3772 Opens https://github.com/celestiaorg/celestia-app/issues/3947 ## Testing ``` make test-e2e MajorUpgradeToV3 test-e2e2024/10/08 22:36:00 --- ✅ PASS: MajorUpgradeToV3 ``` --------- Co-authored-by: Rootul P --- test/cmd/txsim/cli.go | 13 ++- test/e2e/benchmark/benchmark.go | 9 +- test/e2e/main.go | 1 + test/e2e/major_upgrade_v2.go | 3 +- test/e2e/major_upgrade_v3.go | 95 ++++++++++++++++++++ test/e2e/minor_version_compatibility.go | 3 +- test/e2e/simple.go | 4 +- test/e2e/testnet/key_generator.go | 37 ++++++++ test/e2e/testnet/node.go | 21 ++--- test/e2e/testnet/testnet.go | 112 ++++++++++++++---------- test/e2e/testnet/txsimNode.go | 55 ++++++++---- test/e2e/testnet/util.go | 32 ------- test/txsim/upgrade.go | 50 ++++++----- 13 files changed, 303 insertions(+), 132 deletions(-) create mode 100644 test/e2e/major_upgrade_v3.go create mode 100644 test/e2e/testnet/key_generator.go diff --git a/test/cmd/txsim/cli.go b/test/cmd/txsim/cli.go index bf334437c4..55cac6d4fd 100644 --- a/test/cmd/txsim/cli.go +++ b/test/cmd/txsim/cli.go @@ -41,6 +41,7 @@ var ( stake, stakeValue, blob int useFeegrant, suppressLogs bool upgradeSchedule string + blobShareVersion int ) func main() { @@ -102,8 +103,8 @@ well funded account that can act as the master account. The command runs until a masterAccName = os.Getenv(TxsimMasterAccName) } - if stake == 0 && send == 0 && blob == 0 { - return errors.New("no sequences specified. Use --stake, --send or --blob") + if stake == 0 && send == 0 && blob == 0 && upgradeSchedule == "" { + return errors.New("no sequences specified. Use --stake, --send, --upgrade-schedule or --blob") } // setup the sequences @@ -128,7 +129,12 @@ well funded account that can act as the master account. The command runs until a return fmt.Errorf("invalid blob amounts: %w", err) } - sequences = append(sequences, txsim.NewBlobSequence(sizes, blobsPerPFB).Clone(blob)...) + sequence := txsim.NewBlobSequence(sizes, blobsPerPFB) + if blobShareVersion >= 0 { + sequence.WithShareVersion(uint8(blobShareVersion)) + } + + sequences = append(sequences, sequence.Clone(blob)...) } upgradeScheduleMap, err := parseUpgradeSchedule(upgradeSchedule) @@ -210,6 +216,7 @@ func flags() *flag.FlagSet { flags.StringVar(&blobAmounts, "blob-amounts", "1", "range of blobs per PFB specified as a single value or a min-max range (e.g., 10 or 5-10). A single value indicates the exact number of blobs to be created.") flags.BoolVar(&useFeegrant, "feegrant", false, "use the feegrant module to pay for fees") flags.BoolVar(&suppressLogs, "suppressLogs", false, "disable logging") + flags.IntVar(&blobShareVersion, "blob-share-version", -1, "optionally specify a share version to use for the blob sequences") return flags } diff --git a/test/e2e/benchmark/benchmark.go b/test/e2e/benchmark/benchmark.go index 81b2748cb1..83843549ea 100644 --- a/test/e2e/benchmark/benchmark.go +++ b/test/e2e/benchmark/benchmark.go @@ -56,11 +56,16 @@ func (b *BenchmarkTest) SetupNodes() error { // create tx clients and point them to the validators log.Println("Creating tx clients") - err = b.CreateTxClients(ctx, b.manifest.TxClientVersion, + err = b.CreateTxClients( + ctx, + b.manifest.TxClientVersion, b.manifest.BlobSequences, b.manifest.BlobSizes, b.manifest.BlobsPerSeq, - b.manifest.TxClientsResource, gRPCEndpoints) + b.manifest.TxClientsResource, + gRPCEndpoints, + map[int64]uint64{}, // upgrade schedule + ) testnet.NoError("failed to create tx clients", err) log.Println("Setting up testnet") diff --git a/test/e2e/main.go b/test/e2e/main.go index 0928779dbb..f58cd67526 100644 --- a/test/e2e/main.go +++ b/test/e2e/main.go @@ -23,6 +23,7 @@ func main() { tests := []Test{ {"MinorVersionCompatibility", MinorVersionCompatibility}, {"MajorUpgradeToV2", MajorUpgradeToV2}, + {"MajorUpgradeToV3", MajorUpgradeToV3}, {"E2ESimple", E2ESimple}, } diff --git a/test/e2e/major_upgrade_v2.go b/test/e2e/major_upgrade_v2.go index f2d1a43b21..de4fd79709 100644 --- a/test/e2e/major_upgrade_v2.go +++ b/test/e2e/major_upgrade_v2.go @@ -52,7 +52,8 @@ func MajorUpgradeToV2(logger *log.Logger) error { logger.Println("Creating txsim") endpoints, err := testNet.RemoteGRPCEndpoints(ctx) testnet.NoError("failed to get remote gRPC endpoints", err) - err = testNet.CreateTxClient(ctx, "txsim", testnet.TxsimVersion, 1, "100-2000", 100, testnet.DefaultResources, endpoints[0]) + upgradeSchedule := map[int64]uint64{} + err = testNet.CreateTxClient(ctx, "txsim", testnet.TxsimVersion, 1, "100-2000", 100, testnet.DefaultResources, endpoints[0], upgradeSchedule) testnet.NoError("failed to create tx client", err) logger.Println("Setting up testnet") diff --git a/test/e2e/major_upgrade_v3.go b/test/e2e/major_upgrade_v3.go new file mode 100644 index 0000000000..22723125f2 --- /dev/null +++ b/test/e2e/major_upgrade_v3.go @@ -0,0 +1,95 @@ +package main + +import ( + "context" + "fmt" + "log" + "time" + + "github.com/celestiaorg/celestia-app/v3/app" + v2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" + v3 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v3" + "github.com/celestiaorg/celestia-app/v3/test/e2e/testnet" +) + +func MajorUpgradeToV3(logger *log.Logger) error { + numNodes := 4 + upgradeHeightV3 := int64(20) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + logger.Println("Creating testnet") + testNet, err := testnet.New(ctx, "MajorUpgradeToV3", seed, nil, "test") + testnet.NoError("failed to create testnet", err) + + defer testNet.Cleanup(ctx) + + // HACKHACK: use a version of celestia-app built from a commit on this PR. + // This can be removed after the PR is merged to main and we override the + // upgrade height delay to one block in a new Docker image. + version := "1a20c01" + + logger.Println("Running major upgrade to v3 test", "version", version) + + consensusParams := app.DefaultConsensusParams() + consensusParams.Version.AppVersion = v2.Version // Start the test on v2 + testNet.SetConsensusParams(consensusParams) + + preloader, err := testNet.NewPreloader() + testnet.NoError("failed to create preloader", err) + + err = preloader.AddImage(ctx, testnet.DockerImageName(version)) + testnet.NoError("failed to add image", err) + defer func() { _ = preloader.EmptyImages(ctx) }() + + logger.Println("Creating genesis nodes") + for i := 0; i < numNodes; i++ { + err := testNet.CreateGenesisNode(ctx, version, 10000000, 0, testnet.DefaultResources, true) + testnet.NoError("failed to create genesis node", err) + } + + logger.Println("Creating txsim") + endpoints, err := testNet.RemoteGRPCEndpoints(ctx) + testnet.NoError("failed to get remote gRPC endpoints", err) + upgradeSchedule := map[int64]uint64{ + upgradeHeightV3: v3.Version, + } + + err = testNet.CreateTxClient(ctx, "txsim", version, 1, "100-2000", 100, testnet.DefaultResources, endpoints[0], upgradeSchedule) + testnet.NoError("failed to create tx client", err) + + logger.Println("Setting up testnet") + testnet.NoError("Failed to setup testnet", testNet.Setup(ctx)) + logger.Println("Starting testnet") + testnet.NoError("Failed to start testnet", testNet.Start(ctx)) + + timer := time.NewTimer(10 * time.Minute) + defer timer.Stop() + ticker := time.NewTicker(3 * time.Second) + defer ticker.Stop() + + logger.Println("waiting for upgrade") + for _, node := range testNet.Nodes() { + client, err := node.Client() + testnet.NoError("failed to get client", err) + + upgradeComplete := false + lastHeight := int64(0) + for !upgradeComplete { + select { + case <-timer.C: + return fmt.Errorf("failed to upgrade to v3, last height: %d", lastHeight) + case <-ticker.C: + resp, err := client.Header(ctx, nil) + testnet.NoError("failed to get header", err) + if resp.Header.Version.App == v3.Version { + upgradeComplete = true + } + logger.Printf("height %v", resp.Header.Height) + lastHeight = resp.Header.Height + } + } + } + + return nil +} diff --git a/test/e2e/minor_version_compatibility.go b/test/e2e/minor_version_compatibility.go index afa8158d0b..fbd7fb971f 100644 --- a/test/e2e/minor_version_compatibility.go +++ b/test/e2e/minor_version_compatibility.go @@ -63,7 +63,8 @@ func MinorVersionCompatibility(logger *log.Logger) error { logger.Println("Creating txsim") endpoints, err := testNet.RemoteGRPCEndpoints(ctx) testnet.NoError("failed to get remote gRPC endpoints", err) - err = testNet.CreateTxClient(ctx, "txsim", testnet.TxsimVersion, 1, "100-2000", 100, testnet.DefaultResources, endpoints[0]) + upgradeSchedule := map[int64]uint64{} + err = testNet.CreateTxClient(ctx, "txsim", testnet.TxsimVersion, 1, "100-2000", 100, testnet.DefaultResources, endpoints[0], upgradeSchedule) testnet.NoError("failed to create tx client", err) // start the testnet diff --git a/test/e2e/simple.go b/test/e2e/simple.go index e065467f1c..242c4ecde2 100644 --- a/test/e2e/simple.go +++ b/test/e2e/simple.go @@ -35,8 +35,8 @@ func E2ESimple(logger *log.Logger) error { logger.Println("Creating txsim") endpoints, err := testNet.RemoteGRPCEndpoints(ctx) testnet.NoError("failed to get remote gRPC endpoints", err) - err = testNet.CreateTxClient(ctx, "txsim", testnet.TxsimVersion, 10, - "100-2000", 100, testnet.DefaultResources, endpoints[0]) + upgradeSchedule := map[int64]uint64{} + err = testNet.CreateTxClient(ctx, "txsim", testnet.TxsimVersion, 10, "100-2000", 100, testnet.DefaultResources, endpoints[0], upgradeSchedule) testnet.NoError("failed to create tx client", err) logger.Println("Setting up testnets") diff --git a/test/e2e/testnet/key_generator.go b/test/e2e/testnet/key_generator.go new file mode 100644 index 0000000000..cfaa39e866 --- /dev/null +++ b/test/e2e/testnet/key_generator.go @@ -0,0 +1,37 @@ +package testnet + +import ( + "io" + "math/rand" + + "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/crypto/ed25519" + "github.com/tendermint/tendermint/crypto/secp256k1" +) + +type keyGenerator struct { + random *rand.Rand +} + +func newKeyGenerator(seed int64) *keyGenerator { + return &keyGenerator{ + random: rand.New(rand.NewSource(seed)), //nolint:gosec + } +} + +func (g *keyGenerator) Generate(keyType string) crypto.PrivKey { + seed := make([]byte, ed25519.SeedSize) + + _, err := io.ReadFull(g.random, seed) + if err != nil { + panic(err) // this shouldn't happen + } + switch keyType { + case "secp256k1": + return secp256k1.GenPrivKeySecp256k1(seed) + case "", "ed25519": + return ed25519.GenPrivKeyFromSecret(seed) + default: + panic("KeyType not supported") // should not make it this far + } +} diff --git a/test/e2e/testnet/node.go b/test/e2e/testnet/node.go index 71a82d3a75..cf96ee905d 100644 --- a/test/e2e/testnet/node.go +++ b/test/e2e/testnet/node.go @@ -59,8 +59,7 @@ type Node struct { // PullRoundStateTraces retrieves the round state traces from a node. // It will save them to the provided path. -func (n *Node) PullRoundStateTraces(path string) ([]trace.Event[schema.RoundState], error, -) { +func (n *Node) PullRoundStateTraces(path string) ([]trace.Event[schema.RoundState], error) { addr := n.AddressTracing() log.Info().Str("Address", addr).Msg("Pulling round state traces") @@ -73,8 +72,7 @@ func (n *Node) PullRoundStateTraces(path string) ([]trace.Event[schema.RoundStat // PullBlockSummaryTraces retrieves the block summary traces from a node. // It will save them to the provided path. -func (n *Node) PullBlockSummaryTraces(path string) ([]trace.Event[schema.BlockSummary], error, -) { +func (n *Node) PullBlockSummaryTraces(path string) ([]trace.Event[schema.BlockSummary], error) { addr := n.AddressTracing() log.Info().Str("Address", addr).Msg("Pulling block summary traces") @@ -99,11 +97,14 @@ type Resources struct { func NewNode( ctx context.Context, - name, version string, - startHeight, selfDelegation int64, + name string, + version string, + startHeight int64, + selfDelegation int64, peers []string, - signerKey, networkKey crypto.PrivKey, - upgradeHeight int64, + signerKey crypto.PrivKey, + networkKey crypto.PrivKey, + upgradeHeightV2 int64, resources Resources, grafana *GrafanaInfo, kn *knuu.Knuu, @@ -159,8 +160,8 @@ func NewNode( if disableBBR { args = append(args, "--force-no-bbr") } - if upgradeHeight != 0 { - args = append(args, fmt.Sprintf("--v2-upgrade-height=%d", upgradeHeight)) + if upgradeHeightV2 != 0 { + args = append(args, fmt.Sprintf("--v2-upgrade-height=%d", upgradeHeightV2)) } if err := knInstance.Build().SetArgs(args...); err != nil { diff --git a/test/e2e/testnet/testnet.go b/test/e2e/testnet/testnet.go index fc1ed6f67e..159d59ab96 100644 --- a/test/e2e/testnet/testnet.go +++ b/test/e2e/testnet/testnet.go @@ -30,10 +30,7 @@ type Testnet struct { knuu *knuu.Knuu } -func New(ctx context.Context, name string, seed int64, grafana *GrafanaInfo, chainID string, - genesisModifiers ...genesis.Modifier) ( - *Testnet, error, -) { +func New(ctx context.Context, name string, seed int64, grafana *GrafanaInfo, chainID string, genesisModifiers ...genesis.Modifier) (*Testnet, error) { identifier := fmt.Sprintf("%s_%s", name, time.Now().Format("20060102_150405")) kn, err := knuu.New(ctx, knuu.Options{ Scope: identifier, @@ -83,14 +80,10 @@ func (t *Testnet) SetConsensusMaxBlockSize(size int64) { t.genesis.ConsensusParams.Block.MaxBytes = size } -func (t *Testnet) CreateGenesisNode(ctx context.Context, version string, selfDelegation, upgradeHeight int64, resources Resources, disableBBR bool) error { +func (t *Testnet) CreateGenesisNode(ctx context.Context, version string, selfDelegation, upgradeHeightV2 int64, resources Resources, disableBBR bool) error { signerKey := t.keygen.Generate(ed25519Type) networkKey := t.keygen.Generate(ed25519Type) - node, err := NewNode(ctx, - fmt.Sprintf("val%d", len(t.nodes)), version, 0, - selfDelegation, nil, signerKey, networkKey, - upgradeHeight, resources, t.grafana, t.knuu, disableBBR, - ) + node, err := NewNode(ctx, fmt.Sprintf("val%d", len(t.nodes)), version, 0, selfDelegation, nil, signerKey, networkKey, upgradeHeightV2, resources, t.grafana, t.knuu, disableBBR) if err != nil { return err } @@ -101,9 +94,9 @@ func (t *Testnet) CreateGenesisNode(ctx context.Context, version string, selfDel return nil } -func (t *Testnet) CreateGenesisNodes(ctx context.Context, num int, version string, selfDelegation, upgradeHeight int64, resources Resources, disableBBR bool) error { +func (t *Testnet) CreateGenesisNodes(ctx context.Context, num int, version string, selfDelegation, upgradeHeightV2 int64, resources Resources, disableBBR bool) error { for i := 0; i < num; i++ { - if err := t.CreateGenesisNode(ctx, version, selfDelegation, upgradeHeight, resources, disableBBR); err != nil { + if err := t.CreateGenesisNode(ctx, version, selfDelegation, upgradeHeightV2, resources, disableBBR); err != nil { return err } } @@ -117,11 +110,11 @@ func (t *Testnet) CreateTxClients(ctx context.Context, blobPerSequence int, resources Resources, grpcEndpoints []string, + upgradeSchedule map[int64]uint64, ) error { for i, grpcEndpoint := range grpcEndpoints { name := fmt.Sprintf("txsim%d", i) - err := t.CreateTxClient(ctx, name, version, sequences, - blobRange, blobPerSequence, resources, grpcEndpoint) + err := t.CreateTxClient(ctx, name, version, sequences, blobRange, blobPerSequence, resources, grpcEndpoint, upgradeSchedule) if err != nil { log.Err(err).Str("name", name). Str("grpc endpoint", grpcEndpoint). @@ -136,46 +129,76 @@ func (t *Testnet) CreateTxClients(ctx context.Context, return nil } -// CreateTxClient creates a txsim node and sets it up -// name: name of the txsim knuu instance -// version: version of the txsim docker image to be pulled from the registry -// specified by txsimDockerSrcURL -// seed: seed for the txsim -// sequences: number of sequences to be run by the txsim -// blobRange: range of blob sizes to be used by the txsim in bytes -// pollTime: time in seconds between each sequence -// resources: resources to be allocated to the txsim -// grpcEndpoint: grpc endpoint of the node to which the txsim will connect and send transactions +// CreateTxClient creates a txsim node and sets it up. +// +// Parameters: +// ctx: Context for managing the lifecycle. +// name: Name of the txsim knuu instance. +// version: Version of the txsim Docker image to pull. +// blobSequences: Number of blob sequences to run by the txsim. +// blobRange: Range of blob sizes in bytes used by the txsim. +// blobPerSequence: Number of blobs per sequence. +// resources: Resources allocated to the txsim. +// grpcEndpoint: gRPC endpoint of the node for transaction submission. +// upgradeSchedule: Map from height to version for scheduled upgrades (v3 and onwards). func (t *Testnet) CreateTxClient( ctx context.Context, - name, version string, - sequences int, + name string, + version string, + blobSequences int, blobRange string, blobPerSequence int, resources Resources, grpcEndpoint string, + upgradeSchedule map[int64]uint64, ) error { - // create an account, and store it in a temp directory and add the account as genesis account to - // the testnet txsimKeyringDir := filepath.Join(os.TempDir(), name) - log.Info(). - Str("name", name). - Str("directory", txsimKeyringDir). - Msg("txsim keyring directory created") - _, err := t.CreateAccount(name, 1e16, txsimKeyringDir) + defer os.RemoveAll(txsimKeyringDir) + + config := encoding.MakeConfig(app.ModuleEncodingRegisters...).Codec + txsimKeyring, err := keyring.New(app.Name, keyring.BackendTest, txsimKeyringDir, nil, config) if err != nil { - return err + return fmt.Errorf("failed to create keyring: %w", err) } - // Create a txsim node using the key stored in the txsimKeyringDir - txsim, err := CreateTxClient(ctx, name, version, grpcEndpoint, t.seed, - sequences, blobRange, blobPerSequence, 1, resources, txsimRootDir, t.knuu) + key, _, err := txsimKeyring.NewMnemonic(name, keyring.English, "", "", hd.Secp256k1) + if err != nil { + return fmt.Errorf("failed to create mnemonic: %w", err) + } + pk, err := key.GetPubKey() + if err != nil { + return fmt.Errorf("failed to get public key: %w", err) + } + err = t.genesis.AddAccount(genesis.Account{ + PubKey: pk, + Balance: 1e16, + Name: name, + }) + if err != nil { + return fmt.Errorf("failed to add account to genesis: %w", err) + } + + // Copy the keys from the genesis keyring to the txsim keyring so that txsim + // can submit MsgSignalVersion on behalf of the validators. + for _, node := range t.Nodes() { + armor, err := t.Genesis().Keyring().ExportPrivKeyArmor(node.Name, "") + if err != nil { + return fmt.Errorf("failed to export key: %w", err) + } + err = txsimKeyring.ImportPrivKey(node.Name, armor, "") + if err != nil { + return fmt.Errorf("failed to import key: %w", err) + } + } + + txsim, err := CreateTxClient(ctx, name, version, grpcEndpoint, t.seed, blobSequences, blobRange, blobPerSequence, 1, resources, txsimKeyringDir, t.knuu, upgradeSchedule) if err != nil { log.Err(err). Str("name", name). Msg("error creating txsim") return err } + err = txsim.Instance.Build().Commit(ctx) if err != nil { log.Err(err). @@ -185,7 +208,7 @@ func (t *Testnet) CreateTxClient( } // copy over the keyring directory to the txsim instance - err = txsim.Instance.Storage().AddFolder(txsimKeyringDir, txsimRootDir, "10001:10001") + err = txsim.Instance.Storage().AddFolder(txsimKeyringDir, txsimKeyringDir, "10001:10001") if err != nil { log.Err(err). Str("directory", txsimKeyringDir). @@ -234,8 +257,7 @@ func (t *Testnet) CreateAccount(name string, tokens int64, txsimKeyringDir strin if txsimKeyringDir == "" { kr = keyring.NewInMemory(cdc) } else { // create a keyring with the specified directory - kr, err = keyring.New(app.Name, keyring.BackendTest, - txsimKeyringDir, nil, cdc) + kr, err = keyring.New(app.Name, keyring.BackendTest, txsimKeyringDir, nil, cdc) if err != nil { return nil, err } @@ -268,11 +290,7 @@ func (t *Testnet) CreateAccount(name string, tokens int64, txsimKeyringDir strin func (t *Testnet) CreateNode(ctx context.Context, version string, startHeight, upgradeHeight int64, resources Resources, disableBBR bool) error { signerKey := t.keygen.Generate(ed25519Type) networkKey := t.keygen.Generate(ed25519Type) - node, err := NewNode(ctx, - fmt.Sprintf("val%d", len(t.nodes)), version, - startHeight, 0, nil, signerKey, networkKey, - upgradeHeight, resources, t.grafana, t.knuu, disableBBR, - ) + node, err := NewNode(ctx, fmt.Sprintf("val%d", len(t.nodes)), version, startHeight, 0, nil, signerKey, networkKey, upgradeHeight, resources, t.grafana, t.knuu, disableBBR) if err != nil { return err } @@ -457,3 +475,7 @@ func (t *Testnet) Node(i int) *Node { func (t *Testnet) Nodes() []*Node { return t.nodes } + +func (t *Testnet) Genesis() *genesis.Genesis { + return t.genesis +} diff --git a/test/e2e/testnet/txsimNode.go b/test/e2e/testnet/txsimNode.go index 25a17f6d15..bfa21da875 100644 --- a/test/e2e/testnet/txsimNode.go +++ b/test/e2e/testnet/txsimNode.go @@ -4,7 +4,9 @@ package testnet import ( "context" "fmt" + "strings" + "github.com/celestiaorg/go-square/v2/share" "github.com/celestiaorg/knuu/pkg/instance" "github.com/celestiaorg/knuu/pkg/knuu" "github.com/rs/zerolog/log" @@ -23,20 +25,23 @@ type TxSim struct { Instance *instance.Instance } +// CreateTxClient returns a new TxSim instance. func CreateTxClient( ctx context.Context, - name, version string, + name string, + version string, endpoint string, seed int64, - sequences int, + blobSequences int, blobRange string, blobsPerSeq int, pollTime int, resources Resources, volumePath string, knuu *knuu.Knuu, + upgradeSchedule map[int64]uint64, ) (*TxSim, error) { - txIns, err := knuu.NewInstance(name) + instance, err := knuu.NewInstance(name) if err != nil { return nil, err } @@ -45,7 +50,7 @@ func CreateTxClient( Str("name", name). Str("image", image). Msg("setting image for tx client") - err = txIns.Build().SetImage(ctx, image) + err = instance.Build().SetImage(ctx, image) if err != nil { log.Err(err). Str("name", name). @@ -53,33 +58,53 @@ func CreateTxClient( Msg("failed to set image for tx client") return nil, err } - err = txIns.Resources().SetMemory(resources.MemoryRequest, resources.MemoryLimit) + err = instance.Resources().SetMemory(resources.MemoryRequest, resources.MemoryLimit) if err != nil { return nil, err } - err = txIns.Resources().SetCPU(resources.CPU) + err = instance.Resources().SetCPU(resources.CPU) if err != nil { return nil, err } - err = txIns.Storage().AddVolumeWithOwner(volumePath, resources.Volume, 10001) + err = instance.Storage().AddVolumeWithOwner(volumePath, resources.Volume, 10001) if err != nil { return nil, err } args := []string{ - fmt.Sprintf("-grpc-endpoint %s", endpoint), - fmt.Sprintf("-poll-time %ds", pollTime), - fmt.Sprintf("-seed %d ", seed), - fmt.Sprintf("-blob %d ", sequences), - fmt.Sprintf("-blob-amounts %d ", blobsPerSeq), - fmt.Sprintf("-blob-sizes %s ", blobRange), + fmt.Sprintf("--key-path %s", volumePath), + fmt.Sprintf("--grpc-endpoint %s", endpoint), + fmt.Sprintf("--poll-time %ds", pollTime), + fmt.Sprintf("--seed %d", seed), + fmt.Sprintf("--blob %d", blobSequences), + fmt.Sprintf("--blob-amounts %d", blobsPerSeq), + fmt.Sprintf("--blob-sizes %s", blobRange), + fmt.Sprintf("--upgrade-schedule %s", stringifyUpgradeSchedule(upgradeSchedule)), + fmt.Sprintf("--blob-share-version %d", share.ShareVersionZero), } - if err := txIns.Build().SetArgs(args...); err != nil { + if err := instance.Build().SetArgs(args...); err != nil { return nil, err } + log.Info(). + Str("name", name). + Str("image", image). + Str("args", strings.Join(args, " ")). + Msg("created tx client") + return &TxSim{ Name: name, - Instance: txIns, + Instance: instance, }, nil } + +func stringifyUpgradeSchedule(schedule map[int64]uint64) string { + if schedule == nil { + return "" + } + scheduleParts := make([]string, 0, len(schedule)) + for height, version := range schedule { + scheduleParts = append(scheduleParts, fmt.Sprintf("%d:%d", height, version)) + } + return strings.Join(scheduleParts, ",") +} diff --git a/test/e2e/testnet/util.go b/test/e2e/testnet/util.go index 0d89494858..edf89aceda 100644 --- a/test/e2e/testnet/util.go +++ b/test/e2e/testnet/util.go @@ -1,43 +1,11 @@ package testnet import ( - "io" - "math/rand" "os" "github.com/rs/zerolog/log" - "github.com/tendermint/tendermint/crypto" - "github.com/tendermint/tendermint/crypto/ed25519" - "github.com/tendermint/tendermint/crypto/secp256k1" ) -type keyGenerator struct { - random *rand.Rand -} - -func newKeyGenerator(seed int64) *keyGenerator { - return &keyGenerator{ - random: rand.New(rand.NewSource(seed)), //nolint:gosec - } -} - -func (g *keyGenerator) Generate(keyType string) crypto.PrivKey { - seed := make([]byte, ed25519.SeedSize) - - _, err := io.ReadFull(g.random, seed) - if err != nil { - panic(err) // this shouldn't happen - } - switch keyType { - case "secp256k1": - return secp256k1.GenPrivKeySecp256k1(seed) - case "", "ed25519": - return ed25519.GenPrivKeyFromSecret(seed) - default: - panic("KeyType not supported") // should not make it this far - } -} - type GrafanaInfo struct { Endpoint string Username string diff --git a/test/txsim/upgrade.go b/test/txsim/upgrade.go index 0811f32232..e2015b98d1 100644 --- a/test/txsim/upgrade.go +++ b/test/txsim/upgrade.go @@ -15,17 +15,24 @@ var _ Sequence = &UpgradeSequence{} const fundsForUpgrade = 100_000 -// UpgradeSequence simulates an upgrade proposal and voting process +// UpgradeSequence simulates a sequence of validators submitting +// MsgSignalVersions for a particular version and then eventually a +// MsgTryUpgrade. type UpgradeSequence struct { - voted map[string]bool - height int64 - version uint64 - account types.AccAddress + // signalled is a map from validator address to a boolean indicating if they have signalled. + signalled map[string]bool + // height is the first height at which the upgrade sequence is run. + height int64 + // version is the version that validators are signalling for. + version uint64 + // account is the address of the account that submits the MsgTryUpgrade. + account types.AccAddress + // hasUpgraded is true if the MsgTryUpgrade has been submitted. hasUpgraded bool } func NewUpgradeSequence(version uint64, height int64) *UpgradeSequence { - return &UpgradeSequence{version: version, height: height, voted: make(map[string]bool)} + return &UpgradeSequence{version: version, height: height, signalled: make(map[string]bool)} } func (s *UpgradeSequence) Clone(_ int) []Sequence { @@ -52,29 +59,30 @@ func (s *UpgradeSequence) Next(ctx context.Context, querier grpc.ClientConn, _ * return Operation{}, errors.New("no validators found") } + delay := uint64(0) + // apply a delay to the first signal only + if len(s.signalled) == 0 { + delay = uint64(s.height) + } + // Choose a random validator to be the authority - var msg types.Msg for _, validator := range validatorsResp.Validators { - if !s.voted[validator.OperatorAddress] { - msg = &signaltypes.MsgSignalVersion{ + if !s.signalled[validator.OperatorAddress] { + s.signalled[validator.OperatorAddress] = true + msg := &signaltypes.MsgSignalVersion{ ValidatorAddress: validator.OperatorAddress, Version: s.version, } - s.voted[validator.OperatorAddress] = true + return Operation{ + Msgs: []types.Msg{msg}, + Delay: delay, + }, nil } } - // if all validators have voted, we can now try to upgrade. - if msg == nil { - msg = signaltypes.NewMsgTryUpgrade(s.account) - s.hasUpgraded = true - } - - delay := uint64(0) - // apply a delay to the first sequence only - if len(s.voted) == 0 { - delay = uint64(s.height) - } + // if all validators have voted, we can now try to upgrade. + s.hasUpgraded = true + msg := signaltypes.NewMsgTryUpgrade(s.account) return Operation{ Msgs: []types.Msg{msg}, Delay: delay, From 8c356c810d3becd66c97ac2d20a08caebc2656a0 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Thu, 10 Oct 2024 11:53:50 -0400 Subject: [PATCH 65/85] docs: add share version 1 to specs (#3955) Closes https://github.com/celestiaorg/celestia-app/issues/3867 --- specs/src/consensus.md | 1 + specs/src/figures/first_share_with_signer.dot | 25 +++++++++++++++ specs/src/figures/first_share_with_signer.svg | 31 +++++++++++++++++++ specs/src/shares.md | 18 +++++++++-- 4 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 specs/src/figures/first_share_with_signer.dot create mode 100644 specs/src/figures/first_share_with_signer.svg diff --git a/specs/src/consensus.md b/specs/src/consensus.md index 69dc17f136..fa708ce95d 100644 --- a/specs/src/consensus.md +++ b/specs/src/consensus.md @@ -32,6 +32,7 @@ | `SHARE_INFO_BYTES` | `uint64` | `1` | `byte` | The number of bytes used for [share](data_structures.md#share) information | | `SHARE_RESERVED_BYTES` | `uint64` | `4` | `byte` | The number of bytes used to store the index of the first transaction in a transaction share. Must be able to represent any integer up to and including `SHARE_SIZE - 1`. | | `SHARE_SIZE` | `uint64` | `512` | `byte` | Size of transaction and blob [shares](data_structures.md#share), in bytes. | +| `SignerSize` | `int` | `20` | `byte` | The number of bytes used to store the signer in a [share](data_structures.md#share). | | `STATE_SUBTREE_RESERVED_BYTES` | `uint64` | `1` | `byte` | Number of bytes reserved to identify state subtrees. | | `UNBONDING_DURATION` | `uint32` | | `block` | Duration, in blocks, for unbonding a validator or delegation. | | `v1.Version` | `uint64` | `1` | | First version of the application. Breaking changes (hard forks) must update this parameter. | diff --git a/specs/src/figures/first_share_with_signer.dot b/specs/src/figures/first_share_with_signer.dot new file mode 100644 index 0000000000..d22e7eff1e --- /dev/null +++ b/specs/src/figures/first_share_with_signer.dot @@ -0,0 +1,25 @@ +digraph G { + node [shape = record, penwidth = 0]; + + share [label=< + + + + + + + + + + + + + + + + + + +
0129303454512
namespace versionnamespace idinfo bytesequence lengthsignerblob1
+ >]; +} diff --git a/specs/src/figures/first_share_with_signer.svg b/specs/src/figures/first_share_with_signer.svg new file mode 100644 index 0000000000..71dfe3d8e3 --- /dev/null +++ b/specs/src/figures/first_share_with_signer.svg @@ -0,0 +1,31 @@ + + + +G + + + +share + +0 +1 +29 +30 +34 +54 +512 + +namespace version + +namespace id + +info byte + +sequence length + +signer + +blob1 + + + \ No newline at end of file diff --git a/specs/src/shares.md b/specs/src/shares.md index ce84a8950a..59ed91c9f3 100644 --- a/specs/src/shares.md +++ b/specs/src/shares.md @@ -23,6 +23,12 @@ User submitted transactions are split into shares (see [share splitting](#share- ## Share Format +### Share Version + +The share version is a 7-bit big-endian unsigned integer that is used to indicate the version of the [share format](#share-format). A new share version MUST be introduced if the share format changes in a way that is not backwards compatible. There are two share versions [share version 0](#share-version-0) and [share version 1](#share-version-1). + +### Share Version 0 + Every share has a fixed size [`SHARE_SIZE`](./consensus.md#constants). The share format below is consistent for all shares: - The first [`NAMESPACE_VERSION_SIZE`](./consensus.md#constants) bytes of a share's raw data is the namespace version of that share (denoted by "namespace version" in the figure below). @@ -44,9 +50,17 @@ Continuation share in a sequence: Since raw data that exceeds [`SHARE_SIZE`](./consensus.md#constants)`-`[`NAMESPACE_SIZE`](./consensus.md#constants)`-`[`SHARE_INFO_BYTES`](./consensus.md#constants) `-` [`SEQUENCE_BYTES`](./consensus.md#constants) bytes will span more than one share, developers MAY choose to encode additional metadata in their raw blob data prior to inclusion in a Celestia block. For example, Celestia transaction shares encode additional metadata in the form of "reserved bytes". -### Share Version +### Share Version 1 + +Share version 1 is similar to share version 0 with the addition of a `signer` field. The signer is located after the sequence length in the first share. The signer is [`SIGNER_SIZE`](./consensus.md#constants) bytes. + +First share in a sequence with signer: + +![figure 3: first share with signer](./figures/first_share_with_signer.svg) + +Continuation share in a sequence: -The share version is a 7-bit big-endian unsigned integer that is used to indicate the version of the [share format](#share-format). The only supported share version is `0`. A new share version MUST be introduced if the share format changes in a way that is not backwards compatible. +![figure 4: share continuation](./figures/share_continuation.svg) ## Transaction Shares From a7476c6c7e7d8bbb82f9570999130d188be20c2f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 10 Oct 2024 13:30:04 -0400 Subject: [PATCH 66/85] chore(deps): Bump github.com/cometbft/cometbft-db from 0.7.0 to 1.0.1 (#3901) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/cometbft/cometbft-db](https://github.com/cometbft/cometbft-db) from 0.7.0 to 1.0.1.
Release notes

Sourced from github.com/cometbft/cometbft-db's releases.

v1.0.1

https://github.com/cometbft/cometbft-db/blob/v1.0.1/CHANGELOG.md#v101

v1.0.0

CHANGELOG

v0.15.0

CHANGELOG

v0.14.0

📖 CHANGELOG

v0.13.0

📖 CHANGELOG

v0.12.0

:book: CHANGELOG

v0.11.0

:book: CHANGELOG

v0.10.0

:book: CHANGELOG

v0.9.5

CHANGELOG

v0.9.4

CHANGELOG

v0.9.3

📖 CHANGELOG

v0.9.0

:book: CHANGELOG

v0.8.0

:book: CHANGELOG

Changelog

Sourced from github.com/cometbft/cometbft-db's changelog.

v1.0.1

September 23, 2024

This release reverts the addition of the goleveldb flag, which was deemed as too disruptive to users.

v1.0.0

September 20, 2024

This release swaps the "default" DB from goleveldb to pebbledb. There's now a goleveldb build flag that must be used when using goleveldb. If you're using pebbledb, you don't need a build flag anymore.

BREAKING

  • Add goleveldb build flag. (#202)

v0.15.0

September 9, 2024

This release bumps the Go version to 1.23.

BREAKING CHANGES

  • [go/runtime] Bump minimum Go version to v1.23 (#4039)

v0.14.0

Aug 9, 2024

This release reinstates boltdb and cleveldb as deprecated backend types. Please note that we discourage the use of them, as we plan to discontinue support in a future release.

DEPENDENCIES

  • reinstate BoltDB and ClevelDB as backend DBs (#177)

v0.13.0

Aug 2, 2024

This release:

  • changes the contract of the Iterator Key() and Value() APIs. Namely, the caller is now responsible for creating a copy of their returned value if they want to modify it.
  • removes support for boltDB and clevelDB, which were marked as deprecated in release v0.12.0.

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/cometbft/cometbft-db&package-manager=go_modules&previous-version=0.7.0&new-version=1.0.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Rootul P --- go.mod | 25 +++++++++++++++++++------ go.sum | 39 ++++++++++++++++++++++++--------------- 2 files changed, 43 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index 5c0895b401..5c9de33eb4 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/celestiaorg/knuu v0.16.1 github.com/celestiaorg/nmt v0.22.2 github.com/celestiaorg/rsmt2d v0.14.0 - github.com/cometbft/cometbft-db v0.7.0 + github.com/cometbft/cometbft-db v1.0.1 github.com/cosmos/cosmos-proto v1.0.0-beta.5 github.com/cosmos/cosmos-sdk v0.46.16 github.com/cosmos/gogoproto v1.7.0 @@ -48,6 +48,7 @@ require ( github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/99designs/keyring v1.2.2 // indirect github.com/ChainSafe/go-schnorrkel v1.0.0 // indirect + github.com/DataDog/zstd v1.5.0 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/StackExchange/wmi v1.2.1 // indirect github.com/Workiva/go-datastructures v1.0.53 // indirect @@ -67,6 +68,12 @@ require ( github.com/chzyer/readline v1.5.1 // indirect github.com/cilium/ebpf v0.12.3 // indirect github.com/cockroachdb/apd/v2 v2.0.2 // indirect + github.com/cockroachdb/errors v1.11.3 // indirect + github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect + github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect + github.com/cockroachdb/pebble v1.1.2 // indirect + github.com/cockroachdb/redact v1.1.5 // indirect + github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect github.com/coinbase/rosetta-sdk-go v0.7.9 // indirect github.com/confio/ics23/go v0.9.1 // indirect github.com/consensys/bavard v0.1.13 // indirect @@ -86,7 +93,8 @@ require ( github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect github.com/dgraph-io/badger/v2 v2.2007.4 // indirect - github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dgraph-io/badger/v4 v4.3.0 // indirect + github.com/dgraph-io/ristretto v0.1.2-0.20240116140435-c67e07994f91 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/dvsekhvalnov/jose2go v1.6.0 // indirect @@ -96,6 +104,7 @@ require ( github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/getsentry/sentry-go v0.27.0 // indirect github.com/go-ini/ini v1.67.0 // indirect github.com/go-kit/kit v0.12.0 // indirect github.com/go-kit/log v0.2.1 // indirect @@ -109,10 +118,10 @@ require ( github.com/goccy/go-json v0.10.3 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect github.com/gogo/gateway v1.1.0 // indirect - github.com/golang/glog v1.2.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect - github.com/google/btree v1.1.2 // indirect + github.com/google/btree v1.1.3 // indirect + github.com/google/flatbuffers v1.12.1 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect @@ -150,8 +159,11 @@ require ( github.com/klauspost/compress v1.17.9 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/klauspost/reedsolomon v1.12.1 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect github.com/lib/pq v1.10.9 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect + github.com/linxGnu/grocksdb v1.9.3 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/manifoldco/promptui v0.9.0 // indirect @@ -183,6 +195,8 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/regen-network/cosmos-proto v0.3.1 // indirect + github.com/rivo/uniseg v0.4.4 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/rs/cors v1.8.3 // indirect github.com/rs/xid v1.5.0 // indirect github.com/sasha-s/go-deadlock v0.3.1 // indirect @@ -194,7 +208,6 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect github.com/supranational/blst v0.3.13 // indirect github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect - github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect github.com/tendermint/go-amino v0.16.0 // indirect github.com/tidwall/btree v1.5.0 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect @@ -203,7 +216,7 @@ require ( github.com/x448/float16 v0.8.4 // indirect github.com/zondax/hid v0.9.2 // indirect github.com/zondax/ledger-go v0.14.3 // indirect - go.etcd.io/bbolt v1.3.10 // indirect + go.etcd.io/bbolt v1.3.11 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect diff --git a/go.sum b/go.sum index c5b22440e2..f5df8c2707 100644 --- a/go.sum +++ b/go.sum @@ -375,6 +375,8 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH github.com/cockroachdb/apd/v2 v2.0.2 h1:weh8u7Cneje73dDh+2tEVLUvyBc89iwepWCD8b8034E= github.com/cockroachdb/apd/v2 v2.0.2/go.mod h1:DDxRlzC2lo3/vSlmSoS7JkqbbrARPuFOGr0B9pvN3Gw= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I= github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8= github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4= @@ -391,8 +393,8 @@ github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE github.com/coinbase/kryptology v1.8.0/go.mod h1:RYXOAPdzOGUe3qlSFkMGn58i3xUA8hmxYHksuq+8ciI= github.com/coinbase/rosetta-sdk-go v0.7.9 h1:lqllBjMnazTjIqYrOGv8h8jxjg9+hJazIGZr9ZvoCcA= github.com/coinbase/rosetta-sdk-go v0.7.9/go.mod h1:0/knutI7XGVqXmmH4OQD8OckFrbQ8yMsUZTG7FXCR2M= -github.com/cometbft/cometbft-db v0.7.0 h1:uBjbrBx4QzU0zOEnU8KxoDl18dMNgDh+zZRUE0ucsbo= -github.com/cometbft/cometbft-db v0.7.0/go.mod h1:yiKJIm2WKrt6x8Cyxtq9YTEcIMPcEe4XPxhgX59Fzf0= +github.com/cometbft/cometbft-db v1.0.1 h1:SylKuLseMLQKw3+i8y8KozZyJcQSL98qEe2CGMCGTYE= +github.com/cometbft/cometbft-db v1.0.1/go.mod h1:EBrFs1GDRiTqrWXYi4v90Awf/gcdD5ExzdPbg4X8+mk= github.com/confio/ics23/go v0.9.1 h1:3MV46eeWwO3xCauKyAtuAdJYMyPnnchW4iLr2bTw6/U= github.com/confio/ics23/go v0.9.1/go.mod h1:4LPZ2NYqnYIVRklaozjNR1FScgDJ2s5Xrp+e/mYVRak= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= @@ -467,10 +469,12 @@ github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f h1:U5y3Y5UE0w7amNe github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE= github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= +github.com/dgraph-io/badger/v4 v4.3.0 h1:lcsCE1/1qrRhqP+zYx6xDZb8n7U+QlwNicpc676Ub40= +github.com/dgraph-io/badger/v4 v4.3.0/go.mod h1:Sc0T595g8zqAQRDf44n+z3wG4BOqLwceaFntt8KPxUM= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= -github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgraph-io/ristretto v0.1.2-0.20240116140435-c67e07994f91 h1:Pux6+xANi0I7RRo5E1gflI4EZ2yx3BGZ75JkAIvGEOA= +github.com/dgraph-io/ristretto v0.1.2-0.20240116140435-c67e07994f91/go.mod h1:swkazRqnUf1N62d0Nutz7KIj2UKqsm/H8tD0nBJAXqM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= @@ -556,6 +560,8 @@ github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwv github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -624,8 +630,6 @@ github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.2.2 h1:1+mZ9upx1Dh6FmUTFR1naJ77miKiXgALjWOZ3NVFPmY= -github.com/golang/glog v1.2.2/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -670,9 +674,11 @@ github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= -github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -942,6 +948,8 @@ github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6 github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= +github.com/linxGnu/grocksdb v1.9.3 h1:s1cbPcOd0cU2SKXRG1nEqCOWYAELQjdqg3RVI2MH9ik= +github.com/linxGnu/grocksdb v1.9.3/go.mod h1:QYiYypR2d4v63Wj1adOOfzglnoII0gLj3PNh4fZkcFA= github.com/lucasjones/reggen v0.0.0-20180717132126-cdb49ff09d77/go.mod h1:5ELEyG+X8f+meRWHuqUOewBOhvHkl7M76pdGEansxW4= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -1102,6 +1110,9 @@ github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -1157,12 +1168,13 @@ github.com/regen-network/cosmos-proto v0.3.1/go.mod h1:jO0sVX6a1B36nmE8C9xBFXpNw github.com/regen-network/protobuf v1.3.3-alpha.regen.1 h1:OHEc+q5iIAXpqiqFKeLpu5NwTIkVXUs48vFMwzqpqY4= github.com/regen-network/protobuf v1.3.3-alpha.regen.1/go.mod h1:2DjTFR1HhMQhiWC5sZ4OhQ3+NtdbZ6oBDKQwq5Ou+FI= github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= -github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= -github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= +github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= @@ -1254,8 +1266,6 @@ github.com/supranational/blst v0.3.13 h1:AYeSxdOMacwu7FBmpfloBz5pbFXDmJL33RuwnKt github.com/supranational/blst v0.3.13/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= -github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok= -github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8= github.com/tendermint/go-amino v0.16.0 h1:GyhmgQKvqF82e2oZeuMSp9JTN0N09emoSZlb2lyGa2E= github.com/tendermint/go-amino v0.16.0/go.mod h1:TQU0M1i/ImAo+tYpZi73AU3V/dKeCoMC9Sphe2ZwGME= github.com/tendermint/tm-db v0.6.7 h1:fE00Cbl0jayAoqlExN6oyQJ7fR/ZtoVOmvPJ//+shu8= @@ -1328,8 +1338,8 @@ gitlab.com/NebulousLabs/errors v0.0.0-20200929122200-06c536cf6975/go.mod h1:ZkMZ gitlab.com/NebulousLabs/fastrand v0.0.0-20181126182046-603482d69e40 h1:dizWJqTWjwyD8KGcMOwgrkqu1JIkofYgKkmDeNE7oAs= gitlab.com/NebulousLabs/fastrand v0.0.0-20181126182046-603482d69e40/go.mod h1:rOnSnoRyxMI3fe/7KIbVcsHRGxe30OONv8dEgo+vCfA= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= -go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= +go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= +go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= @@ -1656,7 +1666,6 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From ca222a8615849634930d8947603063921771c7c6 Mon Sep 17 00:00:00 2001 From: Evan Forbes <42654277+evan-forbes@users.noreply.github.com> Date: Thu, 10 Oct 2024 18:18:31 -0500 Subject: [PATCH 67/85] chore: optimize checkTx (#3954) ## Overview this PR makes a simple optimization for checkTx --------- Co-authored-by: Rootul P Co-authored-by: CHAMI Rachid --- app/process_proposal.go | 3 + app/test/prepare_proposal_test.go | 32 ++++ app/test/process_proposal_test.go | 28 ++++ app/validate_txs.go | 2 + x/blob/ante/blob_share_decorator.go | 13 +- x/blob/ante/blob_share_decorator_test.go | 189 +++++++++++------------ 6 files changed, 162 insertions(+), 105 deletions(-) diff --git a/app/process_proposal.go b/app/process_proposal.go index fc10bb88e2..2d091b6e81 100644 --- a/app/process_proposal.go +++ b/app/process_proposal.go @@ -67,6 +67,9 @@ func (app *App) ProcessProposal(req abci.RequestProcessProposal) (resp abci.Resp tx = blobTx.Tx } + // todo: uncomment once we're sure this isn't consensus breaking + // sdkCtx = sdkCtx.WithTxBytes(tx) + sdkTx, err := app.txConfig.TxDecoder()(tx) if err != nil { if req.Header.Version.App == v1 { diff --git a/app/test/prepare_proposal_test.go b/app/test/prepare_proposal_test.go index 8bfcc281f8..73bc5dfe5c 100644 --- a/app/test/prepare_proposal_test.go +++ b/app/test/prepare_proposal_test.go @@ -137,6 +137,22 @@ func TestPrepareProposalFiltering(t *testing.T) { require.NoError(t, err) noAccountTx := []byte(testutil.SendTxWithManualSequence(t, encConf.TxConfig, kr, nilAccount, accounts[0], 1000, "", 0, 6)) + // create a tx that can't be included in a 64 x 64 when accounting for the + // pfb along with the shares + tooManyShareBtx := blobfactory.ManyMultiBlobTx( + t, + encConf.TxConfig, + kr, + testutil.ChainID, + accounts[3:4], + infos[3:4], + blobfactory.NestedBlobs( + t, + testfactory.RandomBlobNamespaces(tmrand.NewRand(), 4000), + [][]int{repeat(4000, 1)}, + ), + )[0] + type test struct { name string txs func() [][]byte @@ -181,6 +197,13 @@ func TestPrepareProposalFiltering(t *testing.T) { }, prunedTxs: [][]byte{noAccountTx}, }, + { + name: "blob tx with too many shares", + txs: func() [][]byte { + return [][]byte{tooManyShareBtx} + }, + prunedTxs: [][]byte{tooManyShareBtx}, + }, } for _, tt := range tests { @@ -216,3 +239,12 @@ func queryAccountInfo(capp *app.App, accs []string, kr keyring.Keyring) []blobfa } return infos } + +// repeat returns a slice of length n with each element set to val. +func repeat[T any](n int, val T) []T { + result := make([]T, n) + for i := range result { + result[i] = val + } + return result +} diff --git a/app/test/process_proposal_test.go b/app/test/process_proposal_test.go index df390dcdfa..f890cfe0f1 100644 --- a/app/test/process_proposal_test.go +++ b/app/test/process_proposal_test.go @@ -20,6 +20,7 @@ import ( "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" v1 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v1" v2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" + v3 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v3" "github.com/celestiaorg/celestia-app/v3/pkg/da" "github.com/celestiaorg/celestia-app/v3/pkg/user" testutil "github.com/celestiaorg/celestia-app/v3/test/util" @@ -80,6 +81,20 @@ func TestProcessProposal(t *testing.T) { ns1 := share.MustNewV0Namespace(bytes.Repeat([]byte{1}, share.NamespaceVersionZeroIDSize)) data := bytes.Repeat([]byte{1}, 13) + tooManyShareBtx := blobfactory.ManyMultiBlobTx( + t, + enc, + kr, + testutil.ChainID, + accounts[3:4], + infos[3:4], + blobfactory.NestedBlobs( + t, + testfactory.RandomBlobNamespaces(tmrand.NewRand(), 4000), + [][]int{repeat(4000, 1)}, + ), + )[0] + type test struct { name string input *tmproto.Data @@ -299,6 +314,19 @@ func TestProcessProposal(t *testing.T) { appVersion: appconsts.LatestVersion, expectedResult: abci.ResponseProcessProposal_REJECT, }, + { + name: "blob tx that takes up too many shares", + input: &tmproto.Data{ + Txs: [][]byte{}, + }, + mutator: func(d *tmproto.Data) { + // this tx will get filtered out by prepare proposal before this + // so we add it here + d.Txs = append(d.Txs, tooManyShareBtx) + }, + appVersion: v3.Version, + expectedResult: abci.ResponseProcessProposal_REJECT, + }, } for _, tt := range tests { diff --git a/app/validate_txs.go b/app/validate_txs.go index 3538e221a0..c524fe25e6 100644 --- a/app/validate_txs.go +++ b/app/validate_txs.go @@ -45,6 +45,7 @@ func FilterTxs(logger log.Logger, ctx sdk.Context, handler sdk.AnteHandler, txCo func filterStdTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handler sdk.AnteHandler, txs [][]byte) ([][]byte, sdk.Context) { n := 0 for _, tx := range txs { + ctx = ctx.WithTxBytes(tx) sdkTx, err := dec(tx) if err != nil { logger.Error("decoding already checked transaction", "tx", tmbytes.HexBytes(coretypes.Tx(tx).Hash()), "error", err) @@ -78,6 +79,7 @@ func filterStdTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handler func filterBlobTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handler sdk.AnteHandler, txs []*tx.BlobTx) ([]*tx.BlobTx, sdk.Context) { n := 0 for _, tx := range txs { + ctx = ctx.WithTxBytes(tx.Tx) sdkTx, err := dec(tx.Tx) if err != nil { logger.Error("decoding already checked blob transaction", "tx", tmbytes.HexBytes(coretypes.Tx(tx.Tx).Hash()), "error", err) diff --git a/x/blob/ante/blob_share_decorator.go b/x/blob/ante/blob_share_decorator.go index 3a23f45588..90eb7780c6 100644 --- a/x/blob/ante/blob_share_decorator.go +++ b/x/blob/ante/blob_share_decorator.go @@ -36,7 +36,7 @@ func (d BlobShareDecorator) AnteHandle(ctx sdk.Context, tx sdk.Tx, simulate bool maxBlobShares := d.getMaxBlobShares(ctx) for _, m := range tx.GetMsgs() { if pfb, ok := m.(*blobtypes.MsgPayForBlobs); ok { - if sharesNeeded := getSharesNeeded(pfb.BlobSizes); sharesNeeded > maxBlobShares { + if sharesNeeded := getSharesNeeded(uint32(len(ctx.TxBytes())), pfb.BlobSizes); sharesNeeded > maxBlobShares { return ctx, errors.Wrapf(blobtypes.ErrBlobsTooLarge, "the number of shares occupied by blobs in this MsgPayForBlobs %d exceeds the max number of shares available for blob data %d", sharesNeeded, maxBlobShares) } } @@ -49,10 +49,8 @@ func (d BlobShareDecorator) AnteHandle(ctx sdk.Context, tx sdk.Tx, simulate bool func (d BlobShareDecorator) getMaxBlobShares(ctx sdk.Context) int { squareSize := d.getMaxSquareSize(ctx) totalShares := squareSize * squareSize - // The PFB tx share must occupy at least one share so the number of blob shares - // is at most one less than totalShares. - blobShares := totalShares - 1 - return blobShares + // the shares used up by the tx are calculated in `getSharesNeeded` + return totalShares } // getMaxSquareSize returns the maximum square size based on the current values @@ -74,8 +72,9 @@ func (d BlobShareDecorator) getMaxSquareSize(ctx sdk.Context) int { } // getSharesNeeded returns the total number of shares needed to represent all of -// the blobs described by blobSizes. -func getSharesNeeded(blobSizes []uint32) (sum int) { +// the blobs described by blobSizes along with the shares used by the tx +func getSharesNeeded(txSize uint32, blobSizes []uint32) (sum int) { + sum = share.CompactSharesNeeded(txSize) for _, blobSize := range blobSizes { sum += share.SparseSharesNeeded(blobSize) } diff --git a/x/blob/ante/blob_share_decorator_test.go b/x/blob/ante/blob_share_decorator_test.go index 6c788e1935..e65dca6c3f 100644 --- a/x/blob/ante/blob_share_decorator_test.go +++ b/x/blob/ante/blob_share_decorator_test.go @@ -7,12 +7,18 @@ import ( "github.com/celestiaorg/celestia-app/v3/app/encoding" v1 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v1" v2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" + "github.com/celestiaorg/celestia-app/v3/pkg/user" + "github.com/celestiaorg/celestia-app/v3/test/util/blobfactory" + "github.com/celestiaorg/celestia-app/v3/test/util/testfactory" + "github.com/celestiaorg/celestia-app/v3/test/util/testnode" ante "github.com/celestiaorg/celestia-app/v3/x/blob/ante" blob "github.com/celestiaorg/celestia-app/v3/x/blob/types" "github.com/celestiaorg/go-square/v2/share" + blobtx "github.com/celestiaorg/go-square/v2/tx" sdk "github.com/cosmos/cosmos-sdk/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + tmrand "github.com/tendermint/tendermint/libs/rand" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" version "github.com/tendermint/tendermint/proto/tendermint/version" ) @@ -24,141 +30,137 @@ const ( func TestBlobShareDecorator(t *testing.T) { type testCase struct { - name string - pfb *blob.MsgPayForBlobs - appVersion uint64 - wantErr error + name string + blobsPerPFB, blobSize int + appVersion uint64 + wantErr error } + rand := tmrand.NewRand() + testCases := []testCase{ { - name: "want no error if appVersion v1 and 8 MiB blob", - pfb: &blob.MsgPayForBlobs{ - BlobSizes: []uint32{8 * mebibyte}, - }, - appVersion: v1.Version, + name: "want no error if appVersion v1 and 8 MiB blob", + blobsPerPFB: 1, + blobSize: 8 * mebibyte, + appVersion: v1.Version, }, { - name: "PFB with 1 blob that is 1 byte", - pfb: &blob.MsgPayForBlobs{ - BlobSizes: []uint32{1}, - }, - appVersion: v2.Version, + name: "PFB with 1 blob that is 1 byte", + blobsPerPFB: 1, + blobSize: 1, + appVersion: v2.Version, }, { - name: "PFB with 1 blob that is 1 MiB", - pfb: &blob.MsgPayForBlobs{ - BlobSizes: []uint32{mebibyte}, - }, - appVersion: v2.Version, + name: "PFB with 1 blob that is 1 MiB", + blobsPerPFB: 1, + blobSize: 1 * mebibyte, + appVersion: v2.Version, }, { - name: "PFB with 1 blob that is 2 MiB", - pfb: &blob.MsgPayForBlobs{ - BlobSizes: []uint32{2 * mebibyte}, - }, - appVersion: v2.Version, + name: "PFB with 1 blob that is 2 MiB", + blobsPerPFB: 1, + blobSize: 2 * mebibyte, + appVersion: v2.Version, // This test case should return an error because a square size of 64 // has exactly 2 MiB of total capacity so the total blob capacity // will be slightly smaller than 2 MiB. wantErr: blob.ErrBlobsTooLarge, }, { - name: "PFB with 2 blobs that are 1 byte each", - pfb: &blob.MsgPayForBlobs{ - BlobSizes: []uint32{1, 1}, - }, - appVersion: v2.Version, + name: "PFB with 2 blobs that are 1 byte each", + blobsPerPFB: 2, + blobSize: 1, + appVersion: v2.Version, }, { - name: "PFB with 2 blobs that are 1 MiB each", - pfb: &blob.MsgPayForBlobs{ - BlobSizes: []uint32{mebibyte, mebibyte}, - }, - appVersion: v2.Version, + name: "PFB with 2 blobs that are 1 MiB each", + blobsPerPFB: 2, + blobSize: 1 * mebibyte, + appVersion: v2.Version, // This test case should return an error for the same reason a // single blob that is 2 MiB returns an error. wantErr: blob.ErrBlobsTooLarge, }, { - name: "PFB with many single byte blobs should fit", - pfb: &blob.MsgPayForBlobs{ - // 4095 blobs each of size 1 byte should occupy 4095 shares. - // When square size is 64, there are 4095 shares available to - // blob shares so we don't expect an error for this test case. - BlobSizes: repeat(4095, 1), - }, - appVersion: v2.Version, + name: "PFB with many single byte blobs should fit", + blobsPerPFB: 3000, + blobSize: 1, + appVersion: v2.Version, }, { - name: "PFB with too many single byte blobs should not fit", - pfb: &blob.MsgPayForBlobs{ - // 4096 blobs each of size 1 byte should occupy 4096 shares. - // When square size is 64, there are 4095 shares available to - // blob shares so we expect an error for this test case. - BlobSizes: repeat(4096, 1), - }, - appVersion: v2.Version, - wantErr: blob.ErrBlobsTooLarge, + name: "PFB with too many single byte blobs should not fit", + blobsPerPFB: 4000, + blobSize: 1, + appVersion: v2.Version, + wantErr: blob.ErrBlobsTooLarge, }, { - name: "PFB with 1 blob that is 1 share", - pfb: &blob.MsgPayForBlobs{ - BlobSizes: []uint32{uint32(share.AvailableBytesFromSparseShares(1))}, - }, - appVersion: v2.Version, + name: "PFB with 1 blob that is 1 share", + blobsPerPFB: 1, + blobSize: 100, + appVersion: v2.Version, }, { - name: "PFB with 1 blob that occupies total square - 1", - pfb: &blob.MsgPayForBlobs{ - BlobSizes: []uint32{uint32(share.AvailableBytesFromSparseShares((squareSize * squareSize) - 1))}, - }, - appVersion: v2.Version, + name: "PFB with 1 blob that occupies total square - 1", + blobsPerPFB: 1, + blobSize: share.AvailableBytesFromSparseShares(squareSize*squareSize - 1), + appVersion: v2.Version, }, { - name: "PFB with 1 blob that occupies total square", - pfb: &blob.MsgPayForBlobs{ - BlobSizes: []uint32{uint32(share.AvailableBytesFromSparseShares(squareSize * squareSize))}, - }, - appVersion: v2.Version, + name: "PFB with 1 blob that occupies total square", + blobsPerPFB: 1, + blobSize: share.AvailableBytesFromSparseShares(squareSize * squareSize), + appVersion: v2.Version, // This test case should return an error because if the blob // occupies the total square, there is no space for the PFB tx // share. wantErr: blob.ErrBlobsTooLarge, }, { - name: "PFB with 2 blobs that are 1 share each", - pfb: &blob.MsgPayForBlobs{ - BlobSizes: []uint32{ - uint32(share.AvailableBytesFromSparseShares(1)), - uint32(share.AvailableBytesFromSparseShares(1)), - }, - }, - appVersion: v2.Version, + name: "PFB with 2 blobs that are 1 share each", + blobsPerPFB: 2, + blobSize: 100, + appVersion: v2.Version, }, { - name: "PFB with 2 blobs that occupy half the square each", - pfb: &blob.MsgPayForBlobs{ - BlobSizes: []uint32{ - uint32(share.AvailableBytesFromSparseShares(squareSize * squareSize / 2)), - uint32(share.AvailableBytesFromSparseShares(squareSize * squareSize / 2)), - }, - }, - appVersion: v2.Version, - wantErr: blob.ErrBlobsTooLarge, + name: "PFB with 2 blobs that occupy half the square each", + blobsPerPFB: 2, + blobSize: share.AvailableBytesFromSparseShares(squareSize * squareSize / 2), + appVersion: v2.Version, + wantErr: blob.ErrBlobsTooLarge, }, } - txConfig := encoding.MakeConfig(app.ModuleEncodingRegisters...).TxConfig + ecfg := encoding.MakeConfig(app.ModuleEncodingRegisters...) + for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - txBuilder := txConfig.NewTxBuilder() - require.NoError(t, txBuilder.SetMsgs(tc.pfb)) - tx := txBuilder.GetTx() + kr, _ := testnode.NewKeyring(testfactory.TestAccName) + signer, err := user.NewSigner( + kr, + ecfg.TxConfig, + testfactory.ChainID, + tc.appVersion, + user.NewAccount(testfactory.TestAccName, 1, 0), + ) + require.NoError(t, err) + + blobTx := blobfactory.RandBlobTxs(signer, rand, 1, tc.blobsPerPFB, tc.blobSize) + + btx, isBlob, err := blobtx.UnmarshalBlobTx([]byte(blobTx[0])) + require.NoError(t, err) + require.True(t, isBlob) + + sdkTx, err := ecfg.TxConfig.TxDecoder()(btx.Tx) + require.NoError(t, err) decorator := ante.NewBlobShareDecorator(mockBlobKeeper{}) - ctx := sdk.Context{}.WithIsCheckTx(true).WithBlockHeader(tmproto.Header{Version: version.Consensus{App: tc.appVersion}}) - _, err := decorator.AnteHandle(ctx, tx, false, mockNext) + ctx := sdk.Context{}. + WithIsCheckTx(true). + WithBlockHeader(tmproto.Header{Version: version.Consensus{App: tc.appVersion}}). + WithTxBytes(btx.Tx) + _, err = decorator.AnteHandle(ctx, sdkTx, false, mockNext) assert.ErrorIs(t, tc.wantErr, err) }) } @@ -167,12 +169,3 @@ func TestBlobShareDecorator(t *testing.T) { func mockNext(ctx sdk.Context, _ sdk.Tx, _ bool) (sdk.Context, error) { return ctx, nil } - -// repeat returns a slice of length n with each element set to val. -func repeat(n int, val uint32) []uint32 { - result := make([]uint32, n) - for i := range result { - result[i] = val - } - return result -} From d4ccdfee693ae42f0629200a764db6ba2066a4b1 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Fri, 11 Oct 2024 09:41:55 -0400 Subject: [PATCH 68/85] fix: TestOfferSnapshot flake (#3957) Closes https://github.com/celestiaorg/celestia-app/issues/3956 --- app/app_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/app/app_test.go b/app/app_test.go index a745ca2f8e..ae1c6089f5 100644 --- a/app/app_test.go +++ b/app/app_test.go @@ -2,6 +2,7 @@ package app_test import ( "encoding/json" + "os" "path/filepath" "testing" @@ -160,7 +161,15 @@ func createTestApp(t *testing.T) *app.App { config := encoding.MakeConfig(app.ModuleEncodingRegisters...) upgradeHeight := int64(3) snapshotDir := filepath.Join(t.TempDir(), "data", "snapshots") + t.Cleanup(func() { + err := os.RemoveAll(snapshotDir) + require.NoError(t, err) + }) snapshotDB, err := tmdb.NewDB("metadata", tmdb.GoLevelDBBackend, snapshotDir) + t.Cleanup(func() { + err := snapshotDB.Close() + require.NoError(t, err) + }) require.NoError(t, err) snapshotStore, err := snapshots.NewStore(snapshotDB, snapshotDir) require.NoError(t, err) From d61a68612227bdb0f19d72f2d1a994c64714a67c Mon Sep 17 00:00:00 2001 From: Rootul P Date: Fri, 11 Oct 2024 10:20:57 -0400 Subject: [PATCH 69/85] docs(specs): update app version for params v3 (#3962) I missed this when reviewing https://github.com/celestiaorg/celestia-app/pull/3735 --- specs/src/parameters_v3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/src/parameters_v3.md b/specs/src/parameters_v3.md index 491e4834c1..860cdb286f 100644 --- a/specs/src/parameters_v3.md +++ b/specs/src/parameters_v3.md @@ -32,7 +32,7 @@ hardcoded in the application or they are blocked by the `x/paramfilter` module. | consensus.evidence.MaxAgeNumBlocks | 120960 | The maximum number of blocks before evidence is considered invalid. This value will stop CometBFT from pruning block data. | True | | consensus.evidence.MaxBytes | 1MiB | Maximum size in bytes used by evidence in a given block. | True | | consensus.validator.PubKeyTypes | Ed25519 | The type of public key used by validators. | False | -| consensus.Version.AppVersion | 2 | Determines protocol rules used for a given height. Incremented by the application upon an upgrade. | True | +| consensus.Version.AppVersion | 3 | Determines protocol rules used for a given height. Incremented by the application upon an upgrade. | True | | distribution.BaseProposerReward | 0 | Reward in the mint denomination for proposing a block. | True | | distribution.BonusProposerReward | 0 | Extra reward in the mint denomination for proposers based on the voting power included in the commit. | True | | distribution.CommunityTax | 0.02 (2%) | Percentage of the inflation sent to the community pool. | True | From dcea3132d85eac25c0422a956dd99d17a3f22d9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nina=20/=20=E1=83=9C=E1=83=98=E1=83=9C=E1=83=90?= Date: Fri, 11 Oct 2024 13:00:16 -0400 Subject: [PATCH 70/85] feat!: limit the max tx size to 2 MiB (#3909) ## Overview Fixes [#3686](https://github.com/celestiaorg/celestia-app/issues/3686) --------- Co-authored-by: Rootul P --- app/ante/ante.go | 2 + app/ante/max_tx_size.go | 33 ++++++++ app/ante/max_tx_size_test.go | 78 +++++++++++++++++++ app/ante/{tx_size.go => tx_size_gas.go} | 0 .../{tx_size_test.go => tx_size_gas_test.go} | 0 app/process_proposal.go | 5 ++ app/test/prepare_proposal_test.go | 33 ++++++++ app/test/process_proposal_test.go | 41 ++++++++++ app/validate_txs.go | 8 ++ pkg/appconsts/v3/app_consts.go | 1 + pkg/appconsts/versioned_consts.go | 4 + pkg/appconsts/versioned_consts_test.go | 6 ++ specs/src/ante_handler_v3.md | 1 + test/util/blobfactory/payforblob_factory.go | 2 +- 14 files changed, 213 insertions(+), 1 deletion(-) create mode 100644 app/ante/max_tx_size.go create mode 100644 app/ante/max_tx_size_test.go rename app/ante/{tx_size.go => tx_size_gas.go} (100%) rename app/ante/{tx_size_test.go => tx_size_gas_test.go} (100%) diff --git a/app/ante/ante.go b/app/ante/ante.go index 89b6c1d050..c82313b65e 100644 --- a/app/ante/ante.go +++ b/app/ante/ante.go @@ -32,6 +32,8 @@ func NewAnteHandler( // Set up the context with a gas meter. // Must be called before gas consumption occurs in any other decorator. ante.NewSetUpContextDecorator(), + // Ensure the tx is not larger than the configured threshold. + NewMaxTxSizeDecorator(), // Ensure the tx does not contain any extension options. ante.NewExtensionOptionsDecorator(nil), // Ensure the tx passes ValidateBasic. diff --git a/app/ante/max_tx_size.go b/app/ante/max_tx_size.go new file mode 100644 index 0000000000..2936081aca --- /dev/null +++ b/app/ante/max_tx_size.go @@ -0,0 +1,33 @@ +package ante + +import ( + "fmt" + + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" + v3 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v3" + sdk "github.com/cosmos/cosmos-sdk/types" +) + +// MaxTxSizeDecorator ensures that a tx can not be larger than +// application's configured versioned constant. +type MaxTxSizeDecorator struct{} + +func NewMaxTxSizeDecorator() MaxTxSizeDecorator { + return MaxTxSizeDecorator{} +} + +// AnteHandle implements the AnteHandler interface. It ensures that tx size is under application's configured threshold. +func (d MaxTxSizeDecorator) AnteHandle(ctx sdk.Context, tx sdk.Tx, simulate bool, next sdk.AnteHandler) (sdk.Context, error) { + // Tx size rule applies to app versions v3 and onwards. + if ctx.BlockHeader().Version.App < v3.Version { + return next(ctx, tx, simulate) + } + + currentTxSize := len(ctx.TxBytes()) + maxTxBytes := appconsts.MaxTxBytes(ctx.BlockHeader().Version.App) + if currentTxSize > maxTxBytes { + bytesOverLimit := currentTxSize - maxTxBytes + return ctx, fmt.Errorf("tx size %d bytes is larger than the application's configured threshold of %d bytes. Please reduce the size by %d bytes", currentTxSize, maxTxBytes, bytesOverLimit) + } + return next(ctx, tx, simulate) +} diff --git a/app/ante/max_tx_size_test.go b/app/ante/max_tx_size_test.go new file mode 100644 index 0000000000..3847a0b6a1 --- /dev/null +++ b/app/ante/max_tx_size_test.go @@ -0,0 +1,78 @@ +package ante_test + +import ( + "testing" + + "github.com/celestiaorg/celestia-app/v3/app/ante" + v2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" + v3 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v3" + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/stretchr/testify/require" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" + version "github.com/tendermint/tendermint/proto/tendermint/version" +) + +func TestMaxTxSizeDecorator(t *testing.T) { + decorator := ante.NewMaxTxSizeDecorator() + anteHandler := sdk.ChainAnteDecorators(decorator) + + testCases := []struct { + name string + txSize int + expectError bool + appVersion uint64 + isCheckTx []bool + }{ + { + name: "good tx; under max tx bytes threshold", + txSize: v3.MaxTxBytes - 1, + appVersion: v3.Version, + expectError: false, + isCheckTx: []bool{true, false}, + }, + { + name: "bad tx; over max tx bytes threshold", + txSize: v3.MaxTxBytes + 1, + appVersion: v3.Version, + expectError: true, + isCheckTx: []bool{true, false}, + }, + { + name: "good tx; equal to max tx bytes threshold", + txSize: v3.MaxTxBytes, + appVersion: v3.Version, + expectError: false, + isCheckTx: []bool{true, false}, + }, + { + name: "good tx; limit only applies to v3 and above", + txSize: v3.MaxTxBytes + 10, + appVersion: v2.Version, + expectError: false, + isCheckTx: []bool{true, false}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + for _, isCheckTx := range tc.isCheckTx { + + ctx := sdk.NewContext(nil, tmproto.Header{ + Version: version.Consensus{ + App: tc.appVersion, + }, + }, isCheckTx, nil) + + txBytes := make([]byte, tc.txSize) + + ctx = ctx.WithTxBytes(txBytes) + _, err := anteHandler(ctx, nil, false) + if tc.expectError { + require.Error(t, err) + } else { + require.NoError(t, err) + } + } + }) + } +} diff --git a/app/ante/tx_size.go b/app/ante/tx_size_gas.go similarity index 100% rename from app/ante/tx_size.go rename to app/ante/tx_size_gas.go diff --git a/app/ante/tx_size_test.go b/app/ante/tx_size_gas_test.go similarity index 100% rename from app/ante/tx_size_test.go rename to app/ante/tx_size_gas_test.go diff --git a/app/process_proposal.go b/app/process_proposal.go index 2d091b6e81..68e574f67d 100644 --- a/app/process_proposal.go +++ b/app/process_proposal.go @@ -71,6 +71,11 @@ func (app *App) ProcessProposal(req abci.RequestProcessProposal) (resp abci.Resp // sdkCtx = sdkCtx.WithTxBytes(tx) sdkTx, err := app.txConfig.TxDecoder()(tx) + // Set the tx bytes in the context for app version v3 and greater + if sdkCtx.BlockHeader().Version.App >= 3 { + sdkCtx = sdkCtx.WithTxBytes(tx) + } + if err != nil { if req.Header.Version.App == v1 { // For appVersion 1, there was no block validity rule that all diff --git a/app/test/prepare_proposal_test.go b/app/test/prepare_proposal_test.go index 73bc5dfe5c..58ebc0daae 100644 --- a/app/test/prepare_proposal_test.go +++ b/app/test/prepare_proposal_test.go @@ -1,6 +1,7 @@ package app_test import ( + "strings" "testing" "time" @@ -16,6 +17,7 @@ import ( "github.com/celestiaorg/celestia-app/v3/app" "github.com/celestiaorg/celestia-app/v3/app/encoding" "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" + "github.com/celestiaorg/celestia-app/v3/pkg/user" testutil "github.com/celestiaorg/celestia-app/v3/test/util" "github.com/celestiaorg/celestia-app/v3/test/util/blobfactory" "github.com/celestiaorg/celestia-app/v3/test/util/testfactory" @@ -39,6 +41,7 @@ func TestPrepareProposalPutsPFBsAtEnd(t *testing.T) { accnts[:numBlobTxs], infos[:numBlobTxs], testfactory.Repeat([]*share.Blob{protoBlob}, numBlobTxs), + blobfactory.DefaultTxOpts()..., ) normalTxs := testutil.SendTxsWithAccounts( @@ -96,6 +99,7 @@ func TestPrepareProposalFiltering(t *testing.T) { testfactory.RandomBlobNamespaces(tmrand.NewRand(), 3), [][]int{{100}, {1000}, {420}}, ), + blobfactory.DefaultTxOpts()..., ) // create 3 MsgSend transactions that are signed with valid account numbers @@ -153,6 +157,28 @@ func TestPrepareProposalFiltering(t *testing.T) { ), )[0] + // memo is 2 MiB resulting in the transaction being over limit + largeString := strings.Repeat("a", 2*1024*1024) + + // 3 transactions over MaxTxBytes limit + largeTxs := coretypes.Txs(testutil.SendTxsWithAccounts(t, testApp, encConf.TxConfig, kr, 1000, accounts[0], accounts[:3], testutil.ChainID, user.SetMemo(largeString))).ToSliceOfBytes() + + // 3 blobTxs over MaxTxBytes limit + largeBlobTxs := blobfactory.ManyMultiBlobTx( + t, + encConf.TxConfig, + kr, + testutil.ChainID, + accounts[:3], + infos[:3], + blobfactory.NestedBlobs( + t, + testfactory.RandomBlobNamespaces(tmrand.NewRand(), 3), + [][]int{{100}, {1000}, {420}}, + ), + user.SetMemo(largeString), + ) + type test struct { name string txs func() [][]byte @@ -204,6 +230,13 @@ func TestPrepareProposalFiltering(t *testing.T) { }, prunedTxs: [][]byte{tooManyShareBtx}, }, + { + name: "blobTxs and sendTxs that exceed MaxTxBytes limit", + txs: func() [][]byte { + return append(largeTxs, largeBlobTxs...) // All txs are over MaxTxBytes limit + }, + prunedTxs: append(largeTxs, largeBlobTxs...), + }, } for _, tt := range tests { diff --git a/app/test/process_proposal_test.go b/app/test/process_proposal_test.go index f890cfe0f1..3ec0756b67 100644 --- a/app/test/process_proposal_test.go +++ b/app/test/process_proposal_test.go @@ -3,6 +3,7 @@ package app_test import ( "bytes" "fmt" + "strings" "testing" "time" @@ -50,6 +51,26 @@ func TestProcessProposal(t *testing.T) { testfactory.RandomBlobNamespaces(tmrand.NewRand(), 4), [][]int{{100}, {1000}, {420}, {300}}, ), + blobfactory.DefaultTxOpts()..., + ) + + largeMemo := strings.Repeat("a", appconsts.MaxTxBytes(appconsts.LatestVersion)) + + // create 2 single blobTxs that include a large memo making the transaction + // larger than the configured max tx bytes + largeBlobTxs := blobfactory.ManyMultiBlobTx( + t, enc, kr, testutil.ChainID, accounts[3:], infos[3:], + blobfactory.NestedBlobs( + t, + testfactory.RandomBlobNamespaces(tmrand.NewRand(), 4), + [][]int{{100}, {1000}, {420}, {300}}, + ), + user.SetMemo(largeMemo)) + + // create 1 large sendTx that includes a large memo making the + // transaction over the configured max tx bytes limit + largeSendTx := testutil.SendTxsWithAccounts( + t, testApp, enc, kr, 1000, accounts[0], accounts[1:2], testutil.ChainID, user.SetMemo(largeMemo), ) // create 3 MsgSend transactions that are signed with valid account numbers @@ -327,6 +348,26 @@ func TestProcessProposal(t *testing.T) { appVersion: v3.Version, expectedResult: abci.ResponseProcessProposal_REJECT, }, + { + name: "blob txs larger than configured max tx bytes", + input: validData(), + mutator: func(d *tmproto.Data) { + d.Txs = append(d.Txs, largeBlobTxs...) + d.Hash = calculateNewDataHash(t, d.Txs) + }, + appVersion: appconsts.LatestVersion, + expectedResult: abci.ResponseProcessProposal_REJECT, + }, + { + name: "send tx larger than configured max tx bytes", + input: validData(), + mutator: func(d *tmproto.Data) { + d.Txs = append(coretypes.Txs(largeSendTx).ToSliceOfBytes(), d.Txs...) + d.Hash = calculateNewDataHash(t, d.Txs) + }, + appVersion: appconsts.LatestVersion, + expectedResult: abci.ResponseProcessProposal_REJECT, + }, } for _, tt := range tests { diff --git a/app/validate_txs.go b/app/validate_txs.go index c524fe25e6..d8f9c76611 100644 --- a/app/validate_txs.go +++ b/app/validate_txs.go @@ -51,6 +51,10 @@ func filterStdTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handler logger.Error("decoding already checked transaction", "tx", tmbytes.HexBytes(coretypes.Tx(tx).Hash()), "error", err) continue } + + // Set the tx size on the context before calling the AnteHandler + ctx = ctx.WithTxBytes(tx) + ctx, err = handler(ctx, sdkTx, false) // either the transaction is invalid (ie incorrect nonce) and we // simply want to remove this tx, or we're catching a panic from one @@ -85,6 +89,10 @@ func filterBlobTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handle logger.Error("decoding already checked blob transaction", "tx", tmbytes.HexBytes(coretypes.Tx(tx.Tx).Hash()), "error", err) continue } + + // Set the tx size on the context before calling the AnteHandler + ctx = ctx.WithTxBytes(tx.Tx) + ctx, err = handler(ctx, sdkTx, false) // either the transaction is invalid (ie incorrect nonce) and we // simply want to remove this tx, or we're catching a panic from one diff --git a/pkg/appconsts/v3/app_consts.go b/pkg/appconsts/v3/app_consts.go index 3fa92f49a0..2d16d59993 100644 --- a/pkg/appconsts/v3/app_consts.go +++ b/pkg/appconsts/v3/app_consts.go @@ -6,4 +6,5 @@ const ( SubtreeRootThreshold int = 64 TxSizeCostPerByte uint64 = 10 GasPerBlobByte uint32 = 8 + MaxTxBytes int = 2097152 // 2 MiB in bytes ) diff --git a/pkg/appconsts/versioned_consts.go b/pkg/appconsts/versioned_consts.go index c5ffb952a6..6a186afe71 100644 --- a/pkg/appconsts/versioned_consts.go +++ b/pkg/appconsts/versioned_consts.go @@ -42,6 +42,10 @@ func GasPerBlobByte(_ uint64) uint32 { return v3.GasPerBlobByte } +func MaxTxBytes(_ uint64) int { + return v3.MaxTxBytes +} + var ( DefaultSubtreeRootThreshold = SubtreeRootThreshold(LatestVersion) DefaultSquareSizeUpperBound = SquareSizeUpperBound(LatestVersion) diff --git a/pkg/appconsts/versioned_consts_test.go b/pkg/appconsts/versioned_consts_test.go index da29d71241..90a681b627 100644 --- a/pkg/appconsts/versioned_consts_test.go +++ b/pkg/appconsts/versioned_consts_test.go @@ -66,6 +66,12 @@ func TestVersionedConsts(t *testing.T) { expectedConstant: v3.GasPerBlobByte, got: appconsts.GasPerBlobByte(v3.Version), }, + { + name: "MaxTxBytes v3", + version: v3.Version, + expectedConstant: v3.MaxTxBytes, + got: appconsts.MaxTxBytes(v3.Version), + }, } for _, tc := range testCases { diff --git a/specs/src/ante_handler_v3.md b/specs/src/ante_handler_v3.md index 59f32d45d2..f379ef2d00 100644 --- a/specs/src/ante_handler_v3.md +++ b/specs/src/ante_handler_v3.md @@ -3,6 +3,7 @@ The AnteHandler chains together several decorators to ensure the following criteria are met for app version 3: - The tx does not contain any messages that are unsupported by the current app version. See `MsgVersioningGateKeeper`. +- The tx size is not larger than the application's configured versioned constant MaxTxBytes. - The tx does not contain any [extension options](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L119-L122). - The tx passes `ValidateBasic()`. - The tx's [timeout_height](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L115-L117) has not been reached if one is specified. diff --git a/test/util/blobfactory/payforblob_factory.go b/test/util/blobfactory/payforblob_factory.go index 1a4aa6d10c..f9b05359bc 100644 --- a/test/util/blobfactory/payforblob_factory.go +++ b/test/util/blobfactory/payforblob_factory.go @@ -245,10 +245,10 @@ func ManyMultiBlobTx( accounts []string, accInfos []AccountInfo, blobs [][]*share.Blob, + opts ...user.TxOption, ) [][]byte { t.Helper() txs := make([][]byte, len(accounts)) - opts := DefaultTxOpts() for i, acc := range accounts { signer, err := user.NewSigner(kr, enc, chainid, appconsts.LatestVersion, user.NewAccount(acc, accInfos[i].AccountNum, accInfos[i].Sequence)) require.NoError(t, err) From 8ba82c1b872b7f5686d9bb91b93a0442223d7bb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nina=20/=20=E1=83=9C=E1=83=98=E1=83=9C=E1=83=90?= Date: Fri, 11 Oct 2024 14:03:25 -0400 Subject: [PATCH 71/85] refactor: cap tx size flups (#3967) ## Overview - `MaxTxBytes` to `MaxTxSize` in order to better distinguish between the mempool and the application param - Removed the logic setting `ctx.WithTxBytes` for the transaction twice in one function due to two prs working on similar changes --- app/ante/max_tx_size.go | 8 ++++---- app/ante/max_tx_size_test.go | 14 +++++++------- app/test/prepare_proposal_test.go | 8 ++++---- app/test/process_proposal_test.go | 10 +++++----- app/validate_txs.go | 2 -- pkg/appconsts/v3/app_consts.go | 2 +- pkg/appconsts/versioned_consts.go | 4 ++-- pkg/appconsts/versioned_consts_test.go | 6 +++--- specs/src/ante_handler_v3.md | 2 +- 9 files changed, 27 insertions(+), 29 deletions(-) diff --git a/app/ante/max_tx_size.go b/app/ante/max_tx_size.go index 2936081aca..a9525777e2 100644 --- a/app/ante/max_tx_size.go +++ b/app/ante/max_tx_size.go @@ -24,10 +24,10 @@ func (d MaxTxSizeDecorator) AnteHandle(ctx sdk.Context, tx sdk.Tx, simulate bool } currentTxSize := len(ctx.TxBytes()) - maxTxBytes := appconsts.MaxTxBytes(ctx.BlockHeader().Version.App) - if currentTxSize > maxTxBytes { - bytesOverLimit := currentTxSize - maxTxBytes - return ctx, fmt.Errorf("tx size %d bytes is larger than the application's configured threshold of %d bytes. Please reduce the size by %d bytes", currentTxSize, maxTxBytes, bytesOverLimit) + maxTxSize := appconsts.MaxTxSize(ctx.BlockHeader().Version.App) + if currentTxSize > maxTxSize { + bytesOverLimit := currentTxSize - maxTxSize + return ctx, fmt.Errorf("tx size %d bytes is larger than the application's configured threshold of %d bytes. Please reduce the size by %d bytes", currentTxSize, maxTxSize, bytesOverLimit) } return next(ctx, tx, simulate) } diff --git a/app/ante/max_tx_size_test.go b/app/ante/max_tx_size_test.go index 3847a0b6a1..b52ceeb084 100644 --- a/app/ante/max_tx_size_test.go +++ b/app/ante/max_tx_size_test.go @@ -24,29 +24,29 @@ func TestMaxTxSizeDecorator(t *testing.T) { isCheckTx []bool }{ { - name: "good tx; under max tx bytes threshold", - txSize: v3.MaxTxBytes - 1, + name: "good tx; under max tx size threshold", + txSize: v3.MaxTxSize - 1, appVersion: v3.Version, expectError: false, isCheckTx: []bool{true, false}, }, { - name: "bad tx; over max tx bytes threshold", - txSize: v3.MaxTxBytes + 1, + name: "bad tx; over max tx size threshold", + txSize: v3.MaxTxSize + 1, appVersion: v3.Version, expectError: true, isCheckTx: []bool{true, false}, }, { - name: "good tx; equal to max tx bytes threshold", - txSize: v3.MaxTxBytes, + name: "good tx; equal to max tx size threshold", + txSize: v3.MaxTxSize, appVersion: v3.Version, expectError: false, isCheckTx: []bool{true, false}, }, { name: "good tx; limit only applies to v3 and above", - txSize: v3.MaxTxBytes + 10, + txSize: v3.MaxTxSize + 10, appVersion: v2.Version, expectError: false, isCheckTx: []bool{true, false}, diff --git a/app/test/prepare_proposal_test.go b/app/test/prepare_proposal_test.go index 58ebc0daae..64e105178d 100644 --- a/app/test/prepare_proposal_test.go +++ b/app/test/prepare_proposal_test.go @@ -160,10 +160,10 @@ func TestPrepareProposalFiltering(t *testing.T) { // memo is 2 MiB resulting in the transaction being over limit largeString := strings.Repeat("a", 2*1024*1024) - // 3 transactions over MaxTxBytes limit + // 3 transactions over MaxTxSize limit largeTxs := coretypes.Txs(testutil.SendTxsWithAccounts(t, testApp, encConf.TxConfig, kr, 1000, accounts[0], accounts[:3], testutil.ChainID, user.SetMemo(largeString))).ToSliceOfBytes() - // 3 blobTxs over MaxTxBytes limit + // 3 blobTxs over MaxTxSize limit largeBlobTxs := blobfactory.ManyMultiBlobTx( t, encConf.TxConfig, @@ -231,9 +231,9 @@ func TestPrepareProposalFiltering(t *testing.T) { prunedTxs: [][]byte{tooManyShareBtx}, }, { - name: "blobTxs and sendTxs that exceed MaxTxBytes limit", + name: "blobTxs and sendTxs that exceed MaxTxSize limit", txs: func() [][]byte { - return append(largeTxs, largeBlobTxs...) // All txs are over MaxTxBytes limit + return append(largeTxs, largeBlobTxs...) // All txs are over MaxTxSize limit }, prunedTxs: append(largeTxs, largeBlobTxs...), }, diff --git a/app/test/process_proposal_test.go b/app/test/process_proposal_test.go index 3ec0756b67..69f18b8bd4 100644 --- a/app/test/process_proposal_test.go +++ b/app/test/process_proposal_test.go @@ -54,10 +54,10 @@ func TestProcessProposal(t *testing.T) { blobfactory.DefaultTxOpts()..., ) - largeMemo := strings.Repeat("a", appconsts.MaxTxBytes(appconsts.LatestVersion)) + largeMemo := strings.Repeat("a", appconsts.MaxTxSize(appconsts.LatestVersion)) // create 2 single blobTxs that include a large memo making the transaction - // larger than the configured max tx bytes + // larger than the configured max tx size largeBlobTxs := blobfactory.ManyMultiBlobTx( t, enc, kr, testutil.ChainID, accounts[3:], infos[3:], blobfactory.NestedBlobs( @@ -68,7 +68,7 @@ func TestProcessProposal(t *testing.T) { user.SetMemo(largeMemo)) // create 1 large sendTx that includes a large memo making the - // transaction over the configured max tx bytes limit + // transaction over the configured max tx size limit largeSendTx := testutil.SendTxsWithAccounts( t, testApp, enc, kr, 1000, accounts[0], accounts[1:2], testutil.ChainID, user.SetMemo(largeMemo), ) @@ -349,7 +349,7 @@ func TestProcessProposal(t *testing.T) { expectedResult: abci.ResponseProcessProposal_REJECT, }, { - name: "blob txs larger than configured max tx bytes", + name: "blob txs larger than configured max tx size", input: validData(), mutator: func(d *tmproto.Data) { d.Txs = append(d.Txs, largeBlobTxs...) @@ -359,7 +359,7 @@ func TestProcessProposal(t *testing.T) { expectedResult: abci.ResponseProcessProposal_REJECT, }, { - name: "send tx larger than configured max tx bytes", + name: "send tx larger than configured max tx size", input: validData(), mutator: func(d *tmproto.Data) { d.Txs = append(coretypes.Txs(largeSendTx).ToSliceOfBytes(), d.Txs...) diff --git a/app/validate_txs.go b/app/validate_txs.go index d8f9c76611..d41cc872b2 100644 --- a/app/validate_txs.go +++ b/app/validate_txs.go @@ -45,7 +45,6 @@ func FilterTxs(logger log.Logger, ctx sdk.Context, handler sdk.AnteHandler, txCo func filterStdTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handler sdk.AnteHandler, txs [][]byte) ([][]byte, sdk.Context) { n := 0 for _, tx := range txs { - ctx = ctx.WithTxBytes(tx) sdkTx, err := dec(tx) if err != nil { logger.Error("decoding already checked transaction", "tx", tmbytes.HexBytes(coretypes.Tx(tx).Hash()), "error", err) @@ -83,7 +82,6 @@ func filterStdTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handler func filterBlobTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handler sdk.AnteHandler, txs []*tx.BlobTx) ([]*tx.BlobTx, sdk.Context) { n := 0 for _, tx := range txs { - ctx = ctx.WithTxBytes(tx.Tx) sdkTx, err := dec(tx.Tx) if err != nil { logger.Error("decoding already checked blob transaction", "tx", tmbytes.HexBytes(coretypes.Tx(tx.Tx).Hash()), "error", err) diff --git a/pkg/appconsts/v3/app_consts.go b/pkg/appconsts/v3/app_consts.go index 2d16d59993..4b65941de1 100644 --- a/pkg/appconsts/v3/app_consts.go +++ b/pkg/appconsts/v3/app_consts.go @@ -6,5 +6,5 @@ const ( SubtreeRootThreshold int = 64 TxSizeCostPerByte uint64 = 10 GasPerBlobByte uint32 = 8 - MaxTxBytes int = 2097152 // 2 MiB in bytes + MaxTxSize int = 2097152 // 2 MiB in bytes ) diff --git a/pkg/appconsts/versioned_consts.go b/pkg/appconsts/versioned_consts.go index 6a186afe71..1eb40deb83 100644 --- a/pkg/appconsts/versioned_consts.go +++ b/pkg/appconsts/versioned_consts.go @@ -42,8 +42,8 @@ func GasPerBlobByte(_ uint64) uint32 { return v3.GasPerBlobByte } -func MaxTxBytes(_ uint64) int { - return v3.MaxTxBytes +func MaxTxSize(_ uint64) int { + return v3.MaxTxSize } var ( diff --git a/pkg/appconsts/versioned_consts_test.go b/pkg/appconsts/versioned_consts_test.go index 90a681b627..f621c0199e 100644 --- a/pkg/appconsts/versioned_consts_test.go +++ b/pkg/appconsts/versioned_consts_test.go @@ -67,10 +67,10 @@ func TestVersionedConsts(t *testing.T) { got: appconsts.GasPerBlobByte(v3.Version), }, { - name: "MaxTxBytes v3", + name: "MaxTxSize v3", version: v3.Version, - expectedConstant: v3.MaxTxBytes, - got: appconsts.MaxTxBytes(v3.Version), + expectedConstant: v3.MaxTxSize, + got: appconsts.MaxTxSize(v3.Version), }, } diff --git a/specs/src/ante_handler_v3.md b/specs/src/ante_handler_v3.md index f379ef2d00..f2f6087d71 100644 --- a/specs/src/ante_handler_v3.md +++ b/specs/src/ante_handler_v3.md @@ -3,7 +3,7 @@ The AnteHandler chains together several decorators to ensure the following criteria are met for app version 3: - The tx does not contain any messages that are unsupported by the current app version. See `MsgVersioningGateKeeper`. -- The tx size is not larger than the application's configured versioned constant MaxTxBytes. +- The tx size is not larger than the application's configured versioned constant MaxTxSize. - The tx does not contain any [extension options](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L119-L122). - The tx passes `ValidateBasic()`. - The tx's [timeout_height](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L115-L117) has not been reached if one is specified. From cdb4856c654b6571d958466419b8e1b6a6777984 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?nina=20/=20=E1=83=9C=E1=83=98=E1=83=9C=E1=83=90?= Date: Fri, 11 Oct 2024 15:31:38 -0400 Subject: [PATCH 72/85] chore: add link to MaxTxSize in ante handler v3 (#3968) --- specs/src/ante_handler_v3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/src/ante_handler_v3.md b/specs/src/ante_handler_v3.md index f2f6087d71..cff67cd117 100644 --- a/specs/src/ante_handler_v3.md +++ b/specs/src/ante_handler_v3.md @@ -3,7 +3,7 @@ The AnteHandler chains together several decorators to ensure the following criteria are met for app version 3: - The tx does not contain any messages that are unsupported by the current app version. See `MsgVersioningGateKeeper`. -- The tx size is not larger than the application's configured versioned constant MaxTxSize. +- The tx size is not larger than the application's configured versioned constant [MaxTxSize](https://github.com/celestiaorg/celestia-app/blob/8ba82c1b872b7f5686d9bb91b93a0442223d7bb2/pkg/appconsts/v3/app_consts.go#L9). - The tx does not contain any [extension options](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L119-L122). - The tx passes `ValidateBasic()`. - The tx's [timeout_height](https://github.com/cosmos/cosmos-sdk/blob/22c28366466e64ebf0df1ce5bec8b1130523552c/proto/cosmos/tx/v1beta1/tx.proto#L115-L117) has not been reached if one is specified. From 5e6e7ebccdeadffe1d7e9c89e744a90b52fe9763 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Fri, 11 Oct 2024 17:27:02 -0400 Subject: [PATCH 73/85] docs: guide to manually publish docker images (#3941) Motivated by https://github.com/celestiaorg/celestia-app/issues/3934 --- docs/maintainers/docker.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 docs/maintainers/docker.md diff --git a/docs/maintainers/docker.md b/docs/maintainers/docker.md new file mode 100644 index 0000000000..37dfff2d9e --- /dev/null +++ b/docs/maintainers/docker.md @@ -0,0 +1,29 @@ +# Docker + +## Context + +Github Actions should automatically build and publish a Docker image for each release. If Github Actions failed, you can manually build and publish a Docker image using this guide. + +## Prerequisites + +1. Navigate to and generate a new token with the `write:packages` scope. + +## Steps + +1. Verify that a Docker image with the correct tag doesn't already exist for the release you're trying to create publish on [GHCR](https://github.com/celestiaorg/celestia-app/pkgs/container/celestia-app/versions) + +1. In a new terminal + + ```shell + # Set the CR_PAT environment variable to your token + export CR_PAT=YOUR_TOKEN + # Login to the GitHub Container Registry + echo $CR_PAT | docker login ghcr.io -u USERNAME --password-stdin + + # Tell docker to use buildx for the multiple platform support + docker buildx create --use + # Build the image, in this example the v2.2.0-mocha image + docker buildx build --platform linux/amd64,linux/arm64 -t ghcr.io/celestiaorg/celestia-app:v2.2.0-mocha --push . + ``` + +1. Verify that a Docker image with the correct tag was published on [GHCR](https://github.com/celestiaorg/celestia-app/pkgs/container/celestia-app/versions). From 6936685611cc0141484b88947d91cfa03167a108 Mon Sep 17 00:00:00 2001 From: Moji Date: Mon, 14 Oct 2024 17:30:32 +0330 Subject: [PATCH 74/85] chore: pass knuu as an argument to the testnet.New (#3952) As pointed out before in [one of the comments](https://github.com/celestiaorg/celestia-app/pull/3900#pullrequestreview-2325619673), if we pass knuu as an argument to the testnet constructor, user will have more flexibility in playing with knuu options and can optimize the resource consumption based on the test requirements. This PR attempts to achieve that goal. --- test/e2e/benchmark/benchmark.go | 27 +++++++++++-- test/e2e/benchmark/throughput.go | 4 -- test/e2e/main.go | 2 +- test/e2e/major_upgrade_v2.go | 22 +++++++--- test/e2e/major_upgrade_v3.go | 17 +++++++- test/e2e/minor_version_compatibility.go | 21 ++++++++-- test/e2e/simple.go | 17 ++++++-- test/e2e/testnet/testnet.go | 54 ++++++++++++------------- 8 files changed, 114 insertions(+), 50 deletions(-) diff --git a/test/e2e/benchmark/benchmark.go b/test/e2e/benchmark/benchmark.go index 83843549ea..bc4503d61d 100644 --- a/test/e2e/benchmark/benchmark.go +++ b/test/e2e/benchmark/benchmark.go @@ -12,22 +12,41 @@ import ( "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" "github.com/celestiaorg/celestia-app/v3/test/e2e/testnet" "github.com/celestiaorg/celestia-app/v3/test/util/testnode" + "github.com/celestiaorg/knuu/pkg/knuu" ) +const timeFormat = "20060102_150405" + type BenchmarkTest struct { *testnet.Testnet manifest *Manifest } func NewBenchmarkTest(name string, manifest *Manifest) (*BenchmarkTest, error) { - // create a new testnet - testNet, err := testnet.New(context.Background(), name, seed, - testnet.GetGrafanaInfoFromEnvVar(), manifest.ChainID, - manifest.GetGenesisModifiers()...) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + scope := fmt.Sprintf("%s_%s", name, time.Now().Format(timeFormat)) + kn, err := knuu.New(ctx, knuu.Options{ + Scope: scope, + ProxyEnabled: true, + }) if err != nil { return nil, err } + // context.Background() is used to allow the stopSignal to be functional even after this function returns + kn.HandleStopSignal(context.Background()) + + log.Printf("Knuu initialized with scope %s", kn.Scope) + + testNet, err := testnet.New(kn, testnet.Options{ + Grafana: testnet.GetGrafanaInfoFromEnvVar(), + ChainID: manifest.ChainID, + GenesisModifiers: manifest.GetGenesisModifiers(), + }) + testnet.NoError("failed to create testnet", err) + testNet.SetConsensusParams(manifest.GetConsensusParams()) return &BenchmarkTest{Testnet: testNet, manifest: manifest}, nil } diff --git a/test/e2e/benchmark/throughput.go b/test/e2e/benchmark/throughput.go index 337d700525..75bf1bdf6c 100644 --- a/test/e2e/benchmark/throughput.go +++ b/test/e2e/benchmark/throughput.go @@ -10,10 +10,6 @@ import ( "k8s.io/apimachinery/pkg/api/resource" ) -const ( - seed = 42 -) - var bigBlockManifest = Manifest{ ChainID: "test", Validators: 2, diff --git a/test/e2e/main.go b/test/e2e/main.go index f58cd67526..768907e2cf 100644 --- a/test/e2e/main.go +++ b/test/e2e/main.go @@ -7,7 +7,7 @@ import ( ) const ( - seed = 42 + timeFormat = "20060102_150405" ) type TestFunc func(logger *log.Logger) error diff --git a/test/e2e/major_upgrade_v2.go b/test/e2e/major_upgrade_v2.go index de4fd79709..3e44eb0bc3 100644 --- a/test/e2e/major_upgrade_v2.go +++ b/test/e2e/major_upgrade_v2.go @@ -12,20 +12,30 @@ import ( v1 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v1" v2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" "github.com/celestiaorg/celestia-app/v3/test/e2e/testnet" + "github.com/celestiaorg/knuu/pkg/knuu" "github.com/tendermint/tendermint/rpc/client/http" ) func MajorUpgradeToV2(logger *log.Logger) error { - var ( - numNodes = 4 - upgradeHeight = int64(10) - ) + testName := "MajorUpgradeToV2" + numNodes := 4 + upgradeHeight := int64(10) ctx, cancel := context.WithCancel(context.Background()) defer cancel() + scope := fmt.Sprintf("%s_%s", testName, time.Now().Format(timeFormat)) + kn, err := knuu.New(ctx, knuu.Options{ + Scope: scope, + ProxyEnabled: true, + }) + testnet.NoError("failed to initialize Knuu", err) + + kn.HandleStopSignal(ctx) + logger.Printf("Knuu initialized with scope %s", kn.Scope) + logger.Println("Creating testnet") - testNet, err := testnet.New(ctx, "MajorUpgradeToV2", seed, nil, "test") + testNet, err := testnet.New(kn, testnet.Options{}) testnet.NoError("failed to create testnet", err) defer testNet.Cleanup(ctx) @@ -33,7 +43,7 @@ func MajorUpgradeToV2(logger *log.Logger) error { latestVersion, err := testnet.GetLatestVersion() testnet.NoError("failed to get latest version", err) - logger.Println("Running major upgrade to v2 test", "version", latestVersion) + logger.Printf("Running %s test with version %s", testName, latestVersion) testNet.SetConsensusParams(app.DefaultInitialConsensusParams()) diff --git a/test/e2e/major_upgrade_v3.go b/test/e2e/major_upgrade_v3.go index 22723125f2..f82fae2804 100644 --- a/test/e2e/major_upgrade_v3.go +++ b/test/e2e/major_upgrade_v3.go @@ -10,16 +10,29 @@ import ( v2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" v3 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v3" "github.com/celestiaorg/celestia-app/v3/test/e2e/testnet" + "github.com/celestiaorg/knuu/pkg/knuu" ) func MajorUpgradeToV3(logger *log.Logger) error { + testName := "MajorUpgradeToV3" numNodes := 4 - upgradeHeightV3 := int64(20) + upgradeHeightV3 := int64(10) + ctx, cancel := context.WithCancel(context.Background()) defer cancel() + scope := fmt.Sprintf("%s_%s", testName, time.Now().Format(timeFormat)) + kn, err := knuu.New(ctx, knuu.Options{ + Scope: scope, + ProxyEnabled: true, + }) + testnet.NoError("failed to initialize Knuu", err) + + kn.HandleStopSignal(ctx) + logger.Printf("Knuu initialized with scope %s", kn.Scope) + logger.Println("Creating testnet") - testNet, err := testnet.New(ctx, "MajorUpgradeToV3", seed, nil, "test") + testNet, err := testnet.New(kn, testnet.Options{}) testnet.NoError("failed to create testnet", err) defer testNet.Cleanup(ctx) diff --git a/test/e2e/minor_version_compatibility.go b/test/e2e/minor_version_compatibility.go index fbd7fb971f..bb81d5e021 100644 --- a/test/e2e/minor_version_compatibility.go +++ b/test/e2e/minor_version_compatibility.go @@ -15,10 +15,14 @@ import ( v1 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v1" v2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" "github.com/celestiaorg/celestia-app/v3/test/e2e/testnet" + "github.com/celestiaorg/knuu/pkg/knuu" ) func MinorVersionCompatibility(logger *log.Logger) error { - const numNodes = 4 + const ( + testName = "MinorVersionCompatibility" + numNodes = 4 + ) versionStr, err := getAllVersions() testnet.NoError("failed to get versions", err) @@ -29,13 +33,24 @@ func MinorVersionCompatibility(logger *log.Logger) error { if len(versions) == 0 { logger.Fatal("no versions to test") } + seed := testnet.DefaultSeed r := rand.New(rand.NewSource(seed)) - logger.Println("Running minor version compatibility test", "versions", versions) + logger.Printf("Running %s test with versions %s", testName, versions) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - testNet, err := testnet.New(ctx, "MinorVersionCompatibility", seed, nil, "test") + identifier := fmt.Sprintf("%s_%s", testName, time.Now().Format(timeFormat)) + kn, err := knuu.New(ctx, knuu.Options{ + Scope: identifier, + ProxyEnabled: true, + }) + testnet.NoError("failed to initialize Knuu", err) + + kn.HandleStopSignal(ctx) + logger.Printf("Knuu initialized with scope %s", kn.Scope) + + testNet, err := testnet.New(kn, testnet.Options{Seed: seed}) testnet.NoError("failed to create testnet", err) defer testNet.Cleanup(ctx) diff --git a/test/e2e/simple.go b/test/e2e/simple.go index 242c4ecde2..2a25c46e2b 100644 --- a/test/e2e/simple.go +++ b/test/e2e/simple.go @@ -9,24 +9,35 @@ import ( "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" "github.com/celestiaorg/celestia-app/v3/test/e2e/testnet" "github.com/celestiaorg/celestia-app/v3/test/util/testnode" + "github.com/celestiaorg/knuu/pkg/knuu" ) // This test runs a simple testnet with 4 validators. It submits both MsgPayForBlobs // and MsgSends over 30 seconds and then asserts that at least 10 transactions were // committed. func E2ESimple(logger *log.Logger) error { + const testName = "E2ESimple" + ctx, cancel := context.WithCancel(context.Background()) defer cancel() - testNet, err := testnet.New(ctx, "E2ESimple", seed, nil, "test") + identifier := fmt.Sprintf("%s_%s", testName, time.Now().Format(timeFormat)) + kn, err := knuu.New(ctx, knuu.Options{ + Scope: identifier, + ProxyEnabled: true, + }) + testnet.NoError("failed to initialize Knuu", err) + kn.HandleStopSignal(ctx) + logger.Printf("Knuu initialized with scope %s", kn.Scope) + + testNet, err := testnet.New(kn, testnet.Options{}) testnet.NoError("failed to create testnet", err) defer testNet.Cleanup(ctx) latestVersion, err := testnet.GetLatestVersion() testnet.NoError("failed to get latest version", err) - - logger.Println("Running E2ESimple test", "version", latestVersion) + logger.Printf("Running %s test with version %s", testName, latestVersion) logger.Println("Creating testnet validators") testnet.NoError("failed to create genesis nodes", diff --git a/test/e2e/testnet/testnet.go b/test/e2e/testnet/testnet.go index 159d59ab96..5b73daf596 100644 --- a/test/e2e/testnet/testnet.go +++ b/test/e2e/testnet/testnet.go @@ -20,6 +20,11 @@ import ( tmproto "github.com/tendermint/tendermint/proto/tendermint/types" ) +const ( + DefaultSeed int64 = 42 + DefaultChainID = "test-chain-id" +) + type Testnet struct { seed int64 nodes []*Node @@ -30,39 +35,25 @@ type Testnet struct { knuu *knuu.Knuu } -func New(ctx context.Context, name string, seed int64, grafana *GrafanaInfo, chainID string, genesisModifiers ...genesis.Modifier) (*Testnet, error) { - identifier := fmt.Sprintf("%s_%s", name, time.Now().Format("20060102_150405")) - kn, err := knuu.New(ctx, knuu.Options{ - Scope: identifier, - ProxyEnabled: true, - // if the tests timeout, pass the timeout option - // Timeout: 120 * time.Minute, - }) - if err != nil { - return nil, err - } - - log.Info(). - Str("scope", kn.Scope). - Str("TestName", name). - Msg("Knuu initialized") - - kn.HandleStopSignal(ctx) +type Options struct { + Seed int64 + Grafana *GrafanaInfo + ChainID string + GenesisModifiers []genesis.Modifier +} +func New(knuu *knuu.Knuu, opts Options) (*Testnet, error) { + opts.setDefaults() return &Testnet{ - seed: seed, + seed: opts.Seed, nodes: make([]*Node, 0), - genesis: genesis.NewDefaultGenesis().WithChainID(chainID).WithModifiers(genesisModifiers...), - keygen: newKeyGenerator(seed), - grafana: grafana, - knuu: kn, + genesis: genesis.NewDefaultGenesis().WithChainID(opts.ChainID).WithModifiers(opts.GenesisModifiers...), + keygen: newKeyGenerator(opts.Seed), + grafana: opts.Grafana, + knuu: knuu, }, nil } -func (t *Testnet) Knuu() *knuu.Knuu { - return t.knuu -} - func (t *Testnet) NewPreloader() (*preloader.Preloader, error) { if t.knuu == nil { return nil, errors.New("knuu is not initialized") @@ -479,3 +470,12 @@ func (t *Testnet) Nodes() []*Node { func (t *Testnet) Genesis() *genesis.Genesis { return t.genesis } + +func (o *Options) setDefaults() { + if o.ChainID == "" { + o.ChainID = DefaultChainID + } + if o.Seed == 0 { + o.Seed = DefaultSeed + } +} From 919a26f2d4f6d990117259f855027b5b9691abf8 Mon Sep 17 00:00:00 2001 From: CHAMI Rachid Date: Mon, 14 Oct 2024 20:04:27 +0200 Subject: [PATCH 75/85] chore: appconsts godoc cosmetics (#3969) Just polishing some go docs --- pkg/appconsts/global_consts.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/appconsts/global_consts.go b/pkg/appconsts/global_consts.go index 2903da0414..424d4bbe3f 100644 --- a/pkg/appconsts/global_consts.go +++ b/pkg/appconsts/global_consts.go @@ -11,7 +11,7 @@ import ( // These constants were originally sourced from: // https://github.com/celestiaorg/celestia-specs/blob/master/src/specs/consensus.md#constants // -// They can not change throughout the lifetime of a network. +// They cannot change throughout the lifetime of a network. const ( // DefaultShareVersion is the defacto share version. Use this if you are // unsure of which version to use. @@ -20,7 +20,7 @@ const ( // MinSquareSize is the smallest original square width. MinSquareSize = 1 - // MinshareCount is the minimum number of shares allowed in the original + // MinShareCount is the minimum number of shares allowed in the original // data square. MinShareCount = MinSquareSize * MinSquareSize @@ -51,7 +51,7 @@ var ( SupportedShareVersions = share.SupportedShareVersions ) -// returns the delay in blocks after a quorum has been reached that the chain should upgrade to the new version. +// UpgradeHeightDelay returns the delay in blocks after a quorum has been reached that the chain should upgrade to the new version. func UpgradeHeightDelay() int64 { if OverrideUpgradeHeightDelayStr != "" { parsedValue, err := strconv.ParseInt(OverrideUpgradeHeightDelayStr, 10, 64) From a742bd3245dde76898a4514ac116792025fb9f57 Mon Sep 17 00:00:00 2001 From: CHAMI Rachid Date: Tue, 15 Oct 2024 03:17:56 +0200 Subject: [PATCH 76/85] feat: limit the number of messages in prepare proposal (#3942) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR limits the number of transactions in prepare proposal. --------- Co-authored-by: Rootul P Co-authored-by: nina / ნინა --- Makefile | 2 +- app/test/prepare_proposal_test.go | 154 +++++++++++++++++++++++++++++- app/validate_txs.go | 18 +++- pkg/appconsts/global_consts.go | 9 ++ test/util/test_app.go | 35 ++++++- 5 files changed, 212 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 83c8fe3a03..708b2e286d 100644 --- a/Makefile +++ b/Makefile @@ -152,7 +152,7 @@ test-race: # TODO: Remove the -skip flag once the following tests no longer contain data races. # https://github.com/celestiaorg/celestia-app/issues/1369 @echo "--> Running tests in race mode" - @go test ./... -v -race -skip "TestPrepareProposalConsistency|TestIntegrationTestSuite|TestBlobstreamRPCQueries|TestSquareSizeIntegrationTest|TestStandardSDKIntegrationTestSuite|TestTxsimCommandFlags|TestTxsimCommandEnvVar|TestMintIntegrationTestSuite|TestBlobstreamCLI|TestUpgrade|TestMaliciousTestNode|TestBigBlobSuite|TestQGBIntegrationSuite|TestSignerTestSuite|TestPriorityTestSuite|TestTimeInPrepareProposalContext|TestBlobstream|TestCLITestSuite|TestLegacyUpgrade|TestSignerTwins|TestConcurrentTxSubmission|TestTxClientTestSuite|Test_testnode|TestEvictions" + @go test -timeout 15m ./... -v -race -skip "TestPrepareProposalConsistency|TestIntegrationTestSuite|TestBlobstreamRPCQueries|TestSquareSizeIntegrationTest|TestStandardSDKIntegrationTestSuite|TestTxsimCommandFlags|TestTxsimCommandEnvVar|TestMintIntegrationTestSuite|TestBlobstreamCLI|TestUpgrade|TestMaliciousTestNode|TestBigBlobSuite|TestQGBIntegrationSuite|TestSignerTestSuite|TestPriorityTestSuite|TestTimeInPrepareProposalContext|TestBlobstream|TestCLITestSuite|TestLegacyUpgrade|TestSignerTwins|TestConcurrentTxSubmission|TestTxClientTestSuite|Test_testnode|TestEvictions" .PHONY: test-race ## test-bench: Run unit tests in bench mode. diff --git a/app/test/prepare_proposal_test.go b/app/test/prepare_proposal_test.go index 64e105178d..e0c963de8f 100644 --- a/app/test/prepare_proposal_test.go +++ b/app/test/prepare_proposal_test.go @@ -1,10 +1,21 @@ package app_test import ( + "crypto/rand" "strings" "testing" "time" + blobtypes "github.com/celestiaorg/celestia-app/v3/x/blob/types" + blobtx "github.com/celestiaorg/go-square/v2/tx" + + "github.com/celestiaorg/celestia-app/v3/pkg/user" + "github.com/celestiaorg/celestia-app/v3/test/util/testnode" + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/cosmos/cosmos-sdk/x/auth/types" + banktypes "github.com/cosmos/cosmos-sdk/x/bank/types" + "github.com/stretchr/testify/assert" + tmrand "github.com/tendermint/tendermint/libs/rand" "github.com/cosmos/cosmos-sdk/crypto/hd" @@ -17,7 +28,6 @@ import ( "github.com/celestiaorg/celestia-app/v3/app" "github.com/celestiaorg/celestia-app/v3/app/encoding" "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" - "github.com/celestiaorg/celestia-app/v3/pkg/user" testutil "github.com/celestiaorg/celestia-app/v3/test/util" "github.com/celestiaorg/celestia-app/v3/test/util/blobfactory" "github.com/celestiaorg/celestia-app/v3/test/util/testfactory" @@ -260,6 +270,148 @@ func TestPrepareProposalFiltering(t *testing.T) { } } +func TestPrepareProposalCappingNumberOfMessages(t *testing.T) { + if testing.Short() { + t.Skip("skipping prepare proposal capping number of transactions test in short mode.") + } + // creating a big number of accounts so that every account + // only creates a single transaction. This is for transactions + // to be skipped without worrying about the sequence number being + // sequential. + numberOfAccounts := 8000 + accounts := testnode.GenerateAccounts(numberOfAccounts) + consensusParams := app.DefaultConsensusParams() + testApp, kr := testutil.SetupTestAppWithGenesisValSetAndMaxSquareSize(consensusParams, 128, accounts...) + enc := encoding.MakeConfig(app.ModuleEncodingRegisters...) + + addrs := make([]sdk.AccAddress, 0, numberOfAccounts) + accs := make([]types.AccountI, 0, numberOfAccounts) + signers := make([]*user.Signer, 0, numberOfAccounts) + for index, account := range accounts { + addr := testfactory.GetAddress(kr, account) + addrs = append(addrs, addr) + acc := testutil.DirectQueryAccount(testApp, addrs[index]) + accs = append(accs, acc) + signer, err := user.NewSigner(kr, enc.TxConfig, testutil.ChainID, appconsts.LatestVersion, user.NewAccount(account, acc.GetAccountNumber(), acc.GetSequence())) + require.NoError(t, err) + signers = append(signers, signer) + } + + numberOfPFBs := appconsts.PFBTransactionCap + 500 + pfbTxs := make([][]byte, 0, numberOfPFBs) + randomBytes := make([]byte, 2000) + _, err := rand.Read(randomBytes) + require.NoError(t, err) + accountIndex := 0 + for i := 0; i < numberOfPFBs; i++ { + blob, err := share.NewBlob(share.RandomNamespace(), randomBytes, 1, accs[accountIndex].GetAddress().Bytes()) + require.NoError(t, err) + tx, _, err := signers[accountIndex].CreatePayForBlobs(accounts[accountIndex], []*share.Blob{blob}, user.SetGasLimit(2549760000), user.SetFee(10000)) + require.NoError(t, err) + pfbTxs = append(pfbTxs, tx) + accountIndex++ + } + + multiPFBsPerTxs := make([][]byte, 0, numberOfPFBs) + numberOfMsgsPerTx := 10 + for i := 0; i < numberOfPFBs; i++ { + msgs := make([]sdk.Msg, 0) + blobs := make([]*share.Blob, 0) + for j := 0; j < numberOfMsgsPerTx; j++ { + blob, err := share.NewBlob(share.RandomNamespace(), randomBytes, 1, accs[accountIndex].GetAddress().Bytes()) + require.NoError(t, err) + msg, err := blobtypes.NewMsgPayForBlobs(addrs[accountIndex].String(), appconsts.LatestVersion, blob) + require.NoError(t, err) + msgs = append(msgs, msg) + blobs = append(blobs, blob) + } + txBytes, err := signers[accountIndex].CreateTx(msgs, user.SetGasLimit(2549760000), user.SetFee(10000)) + require.NoError(t, err) + blobTx, err := blobtx.MarshalBlobTx(txBytes, blobs...) + require.NoError(t, err) + multiPFBsPerTxs = append(multiPFBsPerTxs, blobTx) + accountIndex++ + } + + numberOfMsgSends := appconsts.NonPFBTransactionCap + 500 + msgSendTxs := make([][]byte, 0, numberOfMsgSends) + for i := 0; i < numberOfMsgSends; i++ { + msg := banktypes.NewMsgSend( + addrs[accountIndex], + testnode.RandomAddress().(sdk.AccAddress), + sdk.NewCoins(sdk.NewInt64Coin(appconsts.BondDenom, 10)), + ) + rawTx, err := signers[accountIndex].CreateTx([]sdk.Msg{msg}, user.SetGasLimit(1000000), user.SetFee(10)) + require.NoError(t, err) + msgSendTxs = append(msgSendTxs, rawTx) + accountIndex++ + } + + testCases := []struct { + name string + inputTransactions [][]byte + expectedTransactions [][]byte + }{ + { + name: "capping only PFB transactions", + inputTransactions: pfbTxs[:appconsts.PFBTransactionCap+50], + expectedTransactions: pfbTxs[:appconsts.PFBTransactionCap], + }, + { + name: "capping only PFB transactions with multiple messages", + inputTransactions: multiPFBsPerTxs[:appconsts.PFBTransactionCap], + expectedTransactions: multiPFBsPerTxs[:appconsts.PFBTransactionCap/numberOfMsgsPerTx], + }, + { + name: "capping only msg send transactions", + inputTransactions: msgSendTxs[:appconsts.NonPFBTransactionCap+50], + expectedTransactions: msgSendTxs[:appconsts.NonPFBTransactionCap], + }, + { + name: "capping msg send after pfb transactions", + inputTransactions: func() [][]byte { + input := make([][]byte, 0, len(msgSendTxs)+100) + input = append(input, pfbTxs[:100]...) + input = append(input, msgSendTxs...) + return input + }(), + expectedTransactions: func() [][]byte { + expected := make([][]byte, 0, appconsts.NonPFBTransactionCap+100) + expected = append(expected, msgSendTxs[:appconsts.NonPFBTransactionCap]...) + expected = append(expected, pfbTxs[:100]...) + return expected + }(), + }, + { + name: "capping pfb after msg send transactions", + inputTransactions: func() [][]byte { + input := make([][]byte, 0, len(pfbTxs)+100) + input = append(input, msgSendTxs[:100]...) + input = append(input, pfbTxs...) + return input + }(), + expectedTransactions: func() [][]byte { + expected := make([][]byte, 0, appconsts.PFBTransactionCap+100) + expected = append(expected, msgSendTxs[:100]...) + expected = append(expected, pfbTxs[:appconsts.PFBTransactionCap]...) + return expected + }(), + }, + } + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + resp := testApp.PrepareProposal(abci.RequestPrepareProposal{ + BlockData: &tmproto.Data{ + Txs: testCase.inputTransactions, + }, + ChainId: testApp.GetChainID(), + Height: 10, + }) + assert.Equal(t, testCase.expectedTransactions, resp.BlockData.Txs) + }) + } +} + func queryAccountInfo(capp *app.App, accs []string, kr keyring.Keyring) []blobfactory.AccountInfo { infos := make([]blobfactory.AccountInfo, len(accs)) for i, acc := range accs { diff --git a/app/validate_txs.go b/app/validate_txs.go index d41cc872b2..7ff37a8fbb 100644 --- a/app/validate_txs.go +++ b/app/validate_txs.go @@ -1,6 +1,7 @@ package app import ( + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" "github.com/celestiaorg/go-square/v2/tx" "github.com/cosmos/cosmos-sdk/client" "github.com/cosmos/cosmos-sdk/telemetry" @@ -44,6 +45,7 @@ func FilterTxs(logger log.Logger, ctx sdk.Context, handler sdk.AnteHandler, txCo // function used to apply the ante handler. func filterStdTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handler sdk.AnteHandler, txs [][]byte) ([][]byte, sdk.Context) { n := 0 + nonPFBTransactionsCount := 0 for _, tx := range txs { sdkTx, err := dec(tx) if err != nil { @@ -54,6 +56,13 @@ func filterStdTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handler // Set the tx size on the context before calling the AnteHandler ctx = ctx.WithTxBytes(tx) + msgTypes := msgTypes(sdkTx) + if nonPFBTransactionsCount+len(sdkTx.GetMsgs()) > appconsts.NonPFBTransactionCap { + logger.Debug("skipping tx because the sdk message cap was reached", "tx", tmbytes.HexBytes(coretypes.Tx(tx).Hash())) + continue + } + nonPFBTransactionsCount += len(sdkTx.GetMsgs()) + ctx, err = handler(ctx, sdkTx, false) // either the transaction is invalid (ie incorrect nonce) and we // simply want to remove this tx, or we're catching a panic from one @@ -63,7 +72,7 @@ func filterStdTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handler "filtering already checked transaction", "tx", tmbytes.HexBytes(coretypes.Tx(tx).Hash()), "error", err, - "msgs", msgTypes(sdkTx), + "msgs", msgTypes, ) telemetry.IncrCounter(1, "prepare_proposal", "invalid_std_txs") continue @@ -81,6 +90,7 @@ func filterStdTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handler // function used to apply the ante handler. func filterBlobTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handler sdk.AnteHandler, txs []*tx.BlobTx) ([]*tx.BlobTx, sdk.Context) { n := 0 + pfbTransactionCount := 0 for _, tx := range txs { sdkTx, err := dec(tx.Tx) if err != nil { @@ -91,6 +101,12 @@ func filterBlobTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handle // Set the tx size on the context before calling the AnteHandler ctx = ctx.WithTxBytes(tx.Tx) + if pfbTransactionCount+len(sdkTx.GetMsgs()) > appconsts.PFBTransactionCap { + logger.Debug("skipping tx because the pfb transaction cap was reached", "tx", tmbytes.HexBytes(coretypes.Tx(tx.Tx).Hash())) + continue + } + pfbTransactionCount += len(sdkTx.GetMsgs()) + ctx, err = handler(ctx, sdkTx, false) // either the transaction is invalid (ie incorrect nonce) and we // simply want to remove this tx, or we're catching a panic from one diff --git a/pkg/appconsts/global_consts.go b/pkg/appconsts/global_consts.go index 424d4bbe3f..ca73aa8fee 100644 --- a/pkg/appconsts/global_consts.go +++ b/pkg/appconsts/global_consts.go @@ -67,3 +67,12 @@ func UpgradeHeightDelay() int64 { func HashLength() int { return hashLength } + +// The following consts are not consensus breaking and will be applied straight after this binary is started. +const ( + // NonPFBTransactionCap is the maximum number of SDK messages, aside from PFBs, that a block can contain. + NonPFBTransactionCap = 200 + + // PFBTransactionCap is the maximum number of PFB messages a block can contain. + PFBTransactionCap = 600 +) diff --git a/test/util/test_app.go b/test/util/test_app.go index 8b4a433e9c..300d020e9d 100644 --- a/test/util/test_app.go +++ b/test/util/test_app.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "os" + "strings" "testing" "time" @@ -63,7 +64,17 @@ func (ao EmptyAppOptions) Get(_ string) interface{} { // of the app from first genesis account. A no-op logger is set in app. func SetupTestAppWithGenesisValSet(cparams *tmproto.ConsensusParams, genAccounts ...string) (*app.App, keyring.Keyring) { testApp, valSet, kr := NewTestAppWithGenesisSet(cparams, genAccounts...) + initialiseTestApp(testApp, valSet, cparams) + return testApp, kr +} + +func SetupTestAppWithGenesisValSetAndMaxSquareSize(cparams *tmproto.ConsensusParams, maxSquareSize int, genAccounts ...string) (*app.App, keyring.Keyring) { + testApp, valSet, kr := NewTestAppWithGenesisSetAndMaxSquareSize(cparams, maxSquareSize, genAccounts...) + initialiseTestApp(testApp, valSet, cparams) + return testApp, kr +} +func initialiseTestApp(testApp *app.App, valSet *tmtypes.ValidatorSet, cparams *tmproto.ConsensusParams) { // commit genesis changes testApp.Commit() testApp.BeginBlock(abci.RequestBeginBlock{Header: tmproto.Header{ @@ -76,8 +87,6 @@ func SetupTestAppWithGenesisValSet(cparams *tmproto.ConsensusParams, genAccounts App: cparams.Version.AppVersion, }, }}) - - return testApp, kr } // NewTestApp creates a new app instance with an empty memDB and a no-op logger. @@ -178,7 +187,27 @@ func SetupDeterministicGenesisState(testApp *app.App, pubKeys []cryptotypes.PubK func NewTestAppWithGenesisSet(cparams *tmproto.ConsensusParams, genAccounts ...string) (*app.App, *tmtypes.ValidatorSet, keyring.Keyring) { testApp := NewTestApp() genesisState, valSet, kr := GenesisStateWithSingleValidator(testApp, genAccounts...) + testApp = InitialiseTestAppWithGenesis(testApp, cparams, genesisState) + return testApp, valSet, kr +} +// NewTestAppWithGenesisSetAndMaxSquareSize initializes a new app with genesis accounts and a specific max square size +// and returns the testApp, validator set and keyring. +func NewTestAppWithGenesisSetAndMaxSquareSize(cparams *tmproto.ConsensusParams, maxSquareSize int, genAccounts ...string) (*app.App, *tmtypes.ValidatorSet, keyring.Keyring) { + testApp := NewTestApp() + genesisState, valSet, kr := GenesisStateWithSingleValidator(testApp, genAccounts...) + + // hacky way of changing the gov max square size without changing the consts + blobJSON := string(genesisState["blob"]) + replace := strings.Replace(blobJSON, fmt.Sprintf("%d", appconsts.DefaultGovMaxSquareSize), fmt.Sprintf("%d", maxSquareSize), 1) + genesisState["blob"] = json.RawMessage(replace) + + testApp = InitialiseTestAppWithGenesis(testApp, cparams, genesisState) + return testApp, valSet, kr +} + +// InitialiseTestAppWithGenesis initializes the provided app with the provided genesis. +func InitialiseTestAppWithGenesis(testApp *app.App, cparams *tmproto.ConsensusParams, genesisState app.GenesisState) *app.App { stateBytes, err := json.MarshalIndent(genesisState, "", " ") if err != nil { panic(err) @@ -208,7 +237,7 @@ func NewTestAppWithGenesisSet(cparams *tmproto.ConsensusParams, genAccounts ...s ChainId: ChainID, }, ) - return testApp, valSet, kr + return testApp } // AddDeterministicValidatorToGenesis adds a set of five validators to the genesis. From b1f43f5ed19d2fcf2cfb199ebca8988c6a272476 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Tue, 15 Oct 2024 10:02:28 -0400 Subject: [PATCH 77/85] feat: increase default mempool `TTLNumBlocks` to 12 (#3961) Motivation in the **Mempool TTL duration** section of https://github.com/celestiaorg/celestia-app/issues/3959#issuecomment-2405769487 ## Description Modify the default consensus node config for mempools so that: - TTLNumBlocks from 5 to 12 because 12 six second blocks = 72 seconds which is comparable to the existing 75 second TTLDuration. - ~~MaxTxBytes from approx 7.5 MiB to 8 MiB because it's easier to understand and communicate~~ - ~~MaxTxsBytes from approx 37.6 MiB to 40 MiB because it's easier to understand and communicate~~ ## Testing `make install && ./scripts/single-node.sh` then cat the config ```toml version = "v1" ttl-duration = "1m15s" ttl-num-blocks = 12 ``` --- app/default_overrides.go | 15 ++++----------- app/default_overrides_test.go | 4 ++-- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/app/default_overrides.go b/app/default_overrides.go index 0ceedaeee1..fbb53f7441 100644 --- a/app/default_overrides.go +++ b/app/default_overrides.go @@ -8,7 +8,6 @@ import ( "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" "github.com/celestiaorg/celestia-app/v3/x/mint" minttypes "github.com/celestiaorg/celestia-app/v3/x/mint/types" - "github.com/celestiaorg/go-square/v2/share" "github.com/cosmos/cosmos-sdk/codec" serverconfig "github.com/cosmos/cosmos-sdk/server/config" sdk "github.com/cosmos/cosmos-sdk/types" @@ -259,19 +258,13 @@ func DefaultEvidenceParams() tmproto.EvidenceParams { func DefaultConsensusConfig() *tmcfg.Config { cfg := tmcfg.DefaultConfig() // Set broadcast timeout to be 50 seconds in order to avoid timeouts for long block times - // TODO: make TimeoutBroadcastTx configurable per https://github.com/celestiaorg/celestia-app/issues/1034 cfg.RPC.TimeoutBroadcastTxCommit = 50 * time.Second cfg.RPC.MaxBodyBytes = int64(8388608) // 8 MiB - cfg.Mempool.TTLNumBlocks = 5 - cfg.Mempool.TTLDuration = time.Duration(cfg.Mempool.TTLNumBlocks) * appconsts.GoalBlockTime - // Given that there is a stateful transaction size check in CheckTx, - // We set a loose upper bound on what we expect the transaction to - // be based on the upper bound size of the entire block for the given - // version. This acts as a first line of DoS protection - upperBoundBytes := appconsts.DefaultSquareSizeUpperBound * appconsts.DefaultSquareSizeUpperBound * share.ContinuationSparseShareContentSize - cfg.Mempool.MaxTxBytes = upperBoundBytes - cfg.Mempool.MaxTxsBytes = int64(upperBoundBytes) * cfg.Mempool.TTLNumBlocks + cfg.Mempool.TTLNumBlocks = 12 + cfg.Mempool.TTLDuration = 75 * time.Second + cfg.Mempool.MaxTxBytes = 7_897_088 + cfg.Mempool.MaxTxsBytes = 39_485_440 cfg.Mempool.Version = "v1" // prioritized mempool cfg.Consensus.TimeoutPropose = appconsts.TimeoutPropose diff --git a/app/default_overrides_test.go b/app/default_overrides_test.go index 674cd13d9f..c78195717a 100644 --- a/app/default_overrides_test.go +++ b/app/default_overrides_test.go @@ -80,11 +80,11 @@ func TestDefaultConsensusConfig(t *testing.T) { Size: tmcfg.DefaultMempoolConfig().Size, WalPath: tmcfg.DefaultMempoolConfig().WalPath, - // overrides + // Overrides MaxTxBytes: 7_897_088, MaxTxsBytes: 39_485_440, TTLDuration: 75 * time.Second, - TTLNumBlocks: 5, + TTLNumBlocks: 12, Version: "v1", } assert.Equal(t, want, *got.Mempool) From 3d49b605bb1709835442b7676e8e4f23e351a2ee Mon Sep 17 00:00:00 2001 From: Rootul P Date: Wed, 16 Oct 2024 10:29:08 -0400 Subject: [PATCH 78/85] fix: version for tags with `-arabica`, `-mocha`, `-rc` suffixes (#3977) Closes https://github.com/celestiaorg/celestia-app/issues/3933 ## Testing ``` $ git checkout v2.2.0-mocha $ make debug-version GIT_TAG: v2.2.0-mocha VERSION: 2.2.0-mocha $ git tag --delete v2.2.0-mocha Deleted tag 'v2.2.0-mocha' (was d3f8b2997) $ make debug-version GIT_TAG: v2.2.0-arabica VERSION: 2.2.0-arabica $ git tag --delete v2.2.0-arabica Deleted tag 'v2.2.0-arabica' (was d3f8b2997) $ make debug-version GIT_TAG: v2.2.0-rc0 VERSION: 2.2.0-rc0 $ git tag --delete v2.2.0-rc0 Deleted tag 'v2.2.0-rc0' (was d3f8b2997) $ make debug-version GIT_TAG: v2.1.2-4-gd3f8b2997 VERSION: 2.1.2-4-gd3f8b2997 ``` --- Makefile | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 708b2e286d..e095b08bb5 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,20 @@ -VERSION := $(shell echo $(shell git describe --tags 2>/dev/null || git log -1 --format='%h') | sed 's/^v//') +# GIT_TAG is an environment variable that is set to the latest git tag on the +# current commit with the following example priority: v2.2.0, v2.2.0-mocha, +# v2.2.0-arabica, v2.2.0-rc0, v2.2.0-beta, v2.2.0-alpha. If no tag points to the +# current commit, git describe is used. The priority in this command is +# necessary because `git tag --sort=-creatordate` only works for annotated tags +# with metadata. Git tags created via GitHub releases are not annotated and do +# not have metadata like creatordate. Therefore, this command is a hacky attempt +# to get the most recent tag on the current commit according to Celestia's +# testnet versioning scheme + SemVer. +GIT_TAG := $(shell git tag --points-at HEAD --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$$' \ + || git tag --points-at HEAD --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+-mocha$$' \ + || git tag --points-at HEAD --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+-arabica$$' \ + || git tag --points-at HEAD --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]*$$' \ + || git tag --points-at HEAD --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+-(beta)$$' \ + || git tag --points-at HEAD --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+-(alpha)$$' \ + || git describe --tags) +VERSION := $(shell echo $(GIT_TAG) | sed 's/^v//') COMMIT := $(shell git rev-parse --short HEAD) DOCKER := $(shell which docker) DOCKER_BUF := $(DOCKER) run --rm -v $(CURDIR):/workspace --workdir /workspace bufbuild/buf @@ -256,3 +272,9 @@ enable-bbr: echo "BBR is already enabled."; \ fi .PHONY: enable-bbr + +## debug-version: Print the git tag and version. +debug-version: + @echo "GIT_TAG: $(GIT_TAG)" + @echo "VERSION: $(VERSION)" +.PHONY: debug-version From c2d14d912d4b57ce85e3e8d84bc8d8ba03652438 Mon Sep 17 00:00:00 2001 From: Pippo Date: Wed, 16 Oct 2024 19:44:13 +0100 Subject: [PATCH 79/85] fix(ci): classify `-arabica` and `-mocha` releases as prereleases (#3929) Update suffix to match -mocha and -arabica. Fixes #3926 --------- Co-authored-by: Rootul P --- .goreleaser.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 120d8762dc..4f08a63b64 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -118,4 +118,4 @@ changelog: release: prerelease: auto git: - prerelease_suffix: "-rc" + prerelease_suffix: "-" From 2507aafc20b6f405ede3def2f7439580ed092731 Mon Sep 17 00:00:00 2001 From: Sanaz Taheri <35961250+staheri14@users.noreply.github.com> Date: Wed, 16 Oct 2024 13:08:17 -0700 Subject: [PATCH 80/85] feat!: versioned timeouts (#3882) Closes https://github.com/celestiaorg/celestia-app/issues/3859 manually tested in https://github.com/celestiaorg/celestia-app/pull/3882#issuecomment-2415280773 also tested in knuu --------- Co-authored-by: evan-forbes Co-authored-by: Evan Forbes <42654277+evan-forbes@users.noreply.github.com> Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Co-authored-by: Rootul P --- app/app.go | 14 ++- app/default_overrides.go | 4 +- app/module/configurator_test.go | 2 +- app/test/upgrade_test.go | 36 +++++-- go.mod | 2 +- go.sum | 4 +- local_devnet/docker-compose.yml | 4 + pkg/appconsts/consensus_consts.go | 2 - pkg/appconsts/global_consts.go | 19 ---- pkg/appconsts/v1/app_consts.go | 8 ++ pkg/appconsts/v2/app_consts.go | 8 ++ pkg/appconsts/v3/app_consts.go | 8 ++ pkg/appconsts/versioned_consts.go | 45 +++++++++ test/e2e/benchmark/benchmark.go | 4 +- test/e2e/benchmark/manifest.go | 4 + test/e2e/major_upgrade_v3.go | 106 ++++++++++++++++++--- test/e2e/simple.go | 2 +- test/e2e/testnet/testnet.go | 7 +- test/e2e/testnet/txsimNode.go | 5 +- test/txsim/run_test.go | 2 + test/util/genesis/genesis.go | 6 +- test/util/testnode/app_wrapper.go | 21 ++++ test/util/testnode/config.go | 8 +- test/util/testnode/network.go | 1 + test/util/testnode/node_interaction_api.go | 2 + x/blobstream/integration_test.go | 2 + x/signal/integration_test.go | 2 +- x/signal/keeper.go | 15 +-- x/signal/keeper_test.go | 8 +- 29 files changed, 276 insertions(+), 75 deletions(-) create mode 100644 test/util/testnode/app_wrapper.go diff --git a/app/app.go b/app/app.go index ecba661020..b13e8f7f3f 100644 --- a/app/app.go +++ b/app/app.go @@ -280,7 +280,7 @@ func New( ), ) - app.SignalKeeper = signal.NewKeeper(appCodec, keys[signaltypes.StoreKey], app.StakingKeeper, appconsts.UpgradeHeightDelay()) + app.SignalKeeper = signal.NewKeeper(appCodec, keys[signaltypes.StoreKey], app.StakingKeeper) app.IBCKeeper = ibckeeper.NewKeeper( appCodec, @@ -458,7 +458,7 @@ func (app *App) BeginBlocker(ctx sdk.Context, req abci.RequestBeginBlock) abci.R func (app *App) EndBlocker(ctx sdk.Context, req abci.RequestEndBlock) abci.ResponseEndBlock { res := app.manager.EndBlock(ctx, req) currentVersion := app.AppVersion() - // For v1 only we upgrade using a agreed upon height known ahead of time + // For v1 only we upgrade using an agreed upon height known ahead of time if currentVersion == v1 { // check that we are at the height before the upgrade if req.Height == app.upgradeHeightV2-1 { @@ -480,6 +480,8 @@ func (app *App) EndBlocker(ctx sdk.Context, req abci.RequestEndBlock) abci.Respo app.SignalKeeper.ResetTally(ctx) } } + res.Timeouts.TimeoutCommit = appconsts.GetTimeoutCommit(currentVersion) + res.Timeouts.TimeoutPropose = appconsts.GetTimeoutPropose(currentVersion) return res } @@ -535,11 +537,15 @@ func (app *App) Info(req abci.RequestInfo) abci.ResponseInfo { if resp.AppVersion > 0 && !app.IsSealed() { app.mountKeysAndInit(resp.AppVersion) } + + resp.Timeouts.TimeoutPropose = appconsts.GetTimeoutPropose(resp.AppVersion) + resp.Timeouts.TimeoutCommit = appconsts.GetTimeoutCommit(resp.AppVersion) + return resp } // InitChain implements the ABCI interface. This method is a wrapper around -// baseapp's InitChain so we can take the app version and setup the multicommit +// baseapp's InitChain so that we can take the app version and setup the multicommit // store. // // Side-effect: calls baseapp.Init() @@ -558,6 +564,8 @@ func (app *App) InitChain(req abci.RequestInitChain) (res abci.ResponseInitChain app.SetInitialAppVersionInConsensusParams(ctx, appVersion) app.SetAppVersion(ctx, appVersion) } + res.Timeouts.TimeoutCommit = appconsts.GetTimeoutCommit(appVersion) + res.Timeouts.TimeoutPropose = appconsts.GetTimeoutPropose(appVersion) return res } diff --git a/app/default_overrides.go b/app/default_overrides.go index fbb53f7441..9e0b5dd382 100644 --- a/app/default_overrides.go +++ b/app/default_overrides.go @@ -267,8 +267,8 @@ func DefaultConsensusConfig() *tmcfg.Config { cfg.Mempool.MaxTxsBytes = 39_485_440 cfg.Mempool.Version = "v1" // prioritized mempool - cfg.Consensus.TimeoutPropose = appconsts.TimeoutPropose - cfg.Consensus.TimeoutCommit = appconsts.TimeoutCommit + cfg.Consensus.TimeoutPropose = appconsts.GetTimeoutPropose(appconsts.LatestVersion) + cfg.Consensus.TimeoutCommit = appconsts.GetTimeoutCommit(appconsts.LatestVersion) cfg.Consensus.SkipTimeoutCommit = false cfg.TxIndex.Indexer = "null" diff --git a/app/module/configurator_test.go b/app/module/configurator_test.go index ed504c9ac3..a93713938a 100644 --- a/app/module/configurator_test.go +++ b/app/module/configurator_test.go @@ -37,7 +37,7 @@ func TestConfigurator(t *testing.T) { stateStore.MountStoreWithDB(storeKey, storetypes.StoreTypeIAVL, db) require.NoError(t, stateStore.LoadLatestVersion()) - keeper := signal.NewKeeper(config.Codec, storeKey, nil, 0) + keeper := signal.NewKeeper(config.Codec, storeKey, nil) require.NotNil(t, keeper) upgradeModule := signal.NewAppModule(keeper) manager, err := module.NewManager([]module.VersionedModule{ diff --git a/app/test/upgrade_test.go b/app/test/upgrade_test.go index ab188e01ac..4c2677c8e0 100644 --- a/app/test/upgrade_test.go +++ b/app/test/upgrade_test.go @@ -37,6 +37,10 @@ func TestAppUpgradeV3(t *testing.T) { if testing.Short() { t.Skip("skipping TestAppUpgradeV3 in short mode") } + + appconsts.OverrideUpgradeHeightDelayStr = "1" + defer func() { appconsts.OverrideUpgradeHeightDelayStr = "" }() + testApp, genesis := SetupTestAppWithUpgradeHeight(t, 3) upgradeFromV1ToV2(t, testApp) @@ -88,6 +92,10 @@ func TestAppUpgradeV3(t *testing.T) { Height: 3, }) require.Equal(t, v2.Version, endBlockResp.ConsensusParamUpdates.Version.AppVersion) + require.Equal(t, appconsts.GetTimeoutCommit(v2.Version), + endBlockResp.Timeouts.TimeoutCommit) + require.Equal(t, appconsts.GetTimeoutPropose(v2.Version), + endBlockResp.Timeouts.TimeoutPropose) testApp.Commit() require.NoError(t, signer.IncrementSequence(testnode.DefaultValidatorAccountName)) @@ -98,18 +106,22 @@ func TestAppUpgradeV3(t *testing.T) { // brace yourselfs, this part may take a while initialHeight := int64(4) - for height := initialHeight; height < initialHeight+appconsts.DefaultUpgradeHeightDelay; height++ { + for height := initialHeight; height < initialHeight+appconsts.UpgradeHeightDelay(v2.Version); height++ { + appVersion := v2.Version _ = testApp.BeginBlock(abci.RequestBeginBlock{ Header: tmproto.Header{ Height: height, - Version: tmversion.Consensus{App: 2}, + Version: tmversion.Consensus{App: appVersion}, }, }) endBlockResp = testApp.EndBlock(abci.RequestEndBlock{ - Height: 3 + appconsts.DefaultUpgradeHeightDelay, + Height: 3 + appconsts.UpgradeHeightDelay(v2.Version), }) + require.Equal(t, appconsts.GetTimeoutCommit(appVersion), endBlockResp.Timeouts.TimeoutCommit) + require.Equal(t, appconsts.GetTimeoutPropose(appVersion), endBlockResp.Timeouts.TimeoutPropose) + _ = testApp.Commit() } require.Equal(t, v3.Version, endBlockResp.ConsensusParamUpdates.Version.AppVersion) @@ -129,7 +141,7 @@ func TestAppUpgradeV3(t *testing.T) { _ = testApp.BeginBlock(abci.RequestBeginBlock{ Header: tmproto.Header{ ChainID: genesis.ChainID, - Height: initialHeight + appconsts.DefaultUpgradeHeightDelay, + Height: initialHeight + appconsts.UpgradeHeightDelay(v3.Version), Version: tmversion.Consensus{App: 3}, }, }) @@ -139,7 +151,10 @@ func TestAppUpgradeV3(t *testing.T) { }) require.Equal(t, abci.CodeTypeOK, deliverTxResp.Code, deliverTxResp.Log) - _ = testApp.EndBlock(abci.RequestEndBlock{}) + respEndBlock := testApp.EndBlock(abci. + RequestEndBlock{Height: initialHeight + appconsts.UpgradeHeightDelay(v3.Version)}) + require.Equal(t, appconsts.GetTimeoutCommit(v3.Version), respEndBlock.Timeouts.TimeoutCommit) + require.Equal(t, appconsts.GetTimeoutPropose(v3.Version), respEndBlock.Timeouts.TimeoutPropose) } // TestAppUpgradeV2 verifies that the all module's params are overridden during an @@ -271,7 +286,10 @@ func SetupTestAppWithUpgradeHeight(t *testing.T, upgradeHeight int64) (*app.App, // assert that the chain starts with version provided in genesis infoResp := testApp.Info(abci.RequestInfo{}) - require.EqualValues(t, app.DefaultInitialConsensusParams().Version.AppVersion, infoResp.AppVersion) + appVersion := app.DefaultInitialConsensusParams().Version.AppVersion + require.EqualValues(t, appVersion, infoResp.AppVersion) + require.EqualValues(t, appconsts.GetTimeoutCommit(appVersion), infoResp.Timeouts.TimeoutCommit) + require.EqualValues(t, appconsts.GetTimeoutPropose(appVersion), infoResp.Timeouts.TimeoutPropose) supportedVersions := []uint64{v1.Version, v2.Version, v3.Version} require.Equal(t, supportedVersions, testApp.SupportedVersions()) @@ -286,7 +304,11 @@ func upgradeFromV1ToV2(t *testing.T, testApp *app.App) { Height: 2, Version: tmversion.Consensus{App: 1}, }}) - testApp.EndBlock(abci.RequestEndBlock{Height: 2}) + endBlockResp := testApp.EndBlock(abci.RequestEndBlock{Height: 2}) + require.Equal(t, appconsts.GetTimeoutCommit(v1.Version), + endBlockResp.Timeouts.TimeoutCommit) + require.Equal(t, appconsts.GetTimeoutPropose(v1.Version), + endBlockResp.Timeouts.TimeoutPropose) testApp.Commit() require.EqualValues(t, 2, testApp.AppVersion()) } diff --git a/go.mod b/go.mod index 5c9de33eb4..7ad4054fc1 100644 --- a/go.mod +++ b/go.mod @@ -260,5 +260,5 @@ replace ( github.com/cosmos/ledger-cosmos-go => github.com/cosmos/ledger-cosmos-go v0.12.4 github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 - github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.42.0-tm-v0.34.35 + github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.43.0-tm-v0.34.35 ) diff --git a/go.sum b/go.sum index f5df8c2707..22fd80bf43 100644 --- a/go.sum +++ b/go.sum @@ -317,8 +317,8 @@ github.com/celestiaorg/bittwister v0.0.0-20231213180407-65cdbaf5b8c7 h1:nxplQi8w github.com/celestiaorg/bittwister v0.0.0-20231213180407-65cdbaf5b8c7/go.mod h1:1EF5MfOxVf0WC51Gb7pJ6bcZxnXKNAf9pqWtjgPBAYc= github.com/celestiaorg/blobstream-contracts/v3 v3.1.0 h1:h1Y4V3EMQ2mFmNtWt2sIhZIuyASInj1a9ExI8xOsTOw= github.com/celestiaorg/blobstream-contracts/v3 v3.1.0/go.mod h1:x4DKyfKOSv1ZJM9NwV+Pw01kH2CD7N5zTFclXIVJ6GQ= -github.com/celestiaorg/celestia-core v1.42.0-tm-v0.34.35 h1:bWy5XOgeuuSLe0Lc/htL9/QLEURBjA1JTvEko6bEBhg= -github.com/celestiaorg/celestia-core v1.42.0-tm-v0.34.35/go.mod h1:/fK0n3ps09t5uErBQe1QZbrE81L81MNUzWpFyWQLDT0= +github.com/celestiaorg/celestia-core v1.43.0-tm-v0.34.35 h1:L4GTm+JUXhB0a/nGPMq6jEqqe6THuYSQ8m2kUCtZYqw= +github.com/celestiaorg/celestia-core v1.43.0-tm-v0.34.35/go.mod h1:bFr0lAGwaJ0mOHSBmib5/ca5pbBf1yKWGPs93Td0HPw= github.com/celestiaorg/cosmos-sdk v1.25.0-sdk-v0.46.16 h1:f+fTe7GGk0/qgdzyqB8kk8EcDf9d6MC22khBTQiDXsU= github.com/celestiaorg/cosmos-sdk v1.25.0-sdk-v0.46.16/go.mod h1:07Z8HJqS8Rw4XlZ+ok3D3NM/X/in8mvcGLvl0Zb5wrA= github.com/celestiaorg/go-square v1.1.1 h1:Cy3p8WVspVcyOqHM8BWFuuYPwMitO1pYGe+ImILFZRA= diff --git a/local_devnet/docker-compose.yml b/local_devnet/docker-compose.yml index 15c0a38704..1cf768d53d 100644 --- a/local_devnet/docker-compose.yml +++ b/local_devnet/docker-compose.yml @@ -6,6 +6,7 @@ services: container_name: core0 build: context: .. + dockerfile: ./docker/Dockerfile expose: - "26660" # for prometheus ports: @@ -31,6 +32,7 @@ services: container_name: core1 build: context: .. + dockerfile: ./docker/Dockerfile expose: - "26660" # for prometheus depends_on: @@ -59,6 +61,7 @@ services: container_name: core2 build: context: .. + dockerfile: ./docker/Dockerfile expose: - "26660" # for prometheus depends_on: @@ -87,6 +90,7 @@ services: container_name: core3 build: context: .. + dockerfile: ./docker/Dockerfile expose: - "26660" # for prometheus depends_on: diff --git a/pkg/appconsts/consensus_consts.go b/pkg/appconsts/consensus_consts.go index c26e4078eb..43aa335f93 100644 --- a/pkg/appconsts/consensus_consts.go +++ b/pkg/appconsts/consensus_consts.go @@ -3,8 +3,6 @@ package appconsts import "time" const ( - TimeoutPropose = time.Second * 10 - TimeoutCommit = time.Second * 11 // GoalBlockTime is the target time interval between blocks. Since the block // interval isn't enforced at consensus, the real block interval isn't // guaranteed to exactly match GoalBlockTime. GoalBlockTime is currently targeted diff --git a/pkg/appconsts/global_consts.go b/pkg/appconsts/global_consts.go index ca73aa8fee..ce6f7d71ee 100644 --- a/pkg/appconsts/global_consts.go +++ b/pkg/appconsts/global_consts.go @@ -1,8 +1,6 @@ package appconsts import ( - "strconv" - "github.com/celestiaorg/go-square/v2/share" "github.com/celestiaorg/rsmt2d" "github.com/tendermint/tendermint/pkg/consts" @@ -26,11 +24,6 @@ const ( // BondDenom defines the native staking denomination BondDenom = "utia" - - // DefaultUpgradeHeightDelay is the number of blocks after a quorum has been - // reached that the chain should upgrade to the new version. Assuming a block - // interval of 12 seconds, this is 7 days. - DefaultUpgradeHeightDelay = int64(7 * 24 * 60 * 60 / 12) // 7 days * 24 hours * 60 minutes * 60 seconds / 12 seconds per block = 50,400 blocks. ) var ( @@ -51,18 +44,6 @@ var ( SupportedShareVersions = share.SupportedShareVersions ) -// UpgradeHeightDelay returns the delay in blocks after a quorum has been reached that the chain should upgrade to the new version. -func UpgradeHeightDelay() int64 { - if OverrideUpgradeHeightDelayStr != "" { - parsedValue, err := strconv.ParseInt(OverrideUpgradeHeightDelayStr, 10, 64) - if err != nil { - panic("Invalid OverrideUpgradeHeightDelayStr value") - } - return parsedValue - } - return DefaultUpgradeHeightDelay -} - // HashLength returns the length of a hash in bytes. func HashLength() int { return hashLength diff --git a/pkg/appconsts/v1/app_consts.go b/pkg/appconsts/v1/app_consts.go index 72b040f819..873d3ec18a 100644 --- a/pkg/appconsts/v1/app_consts.go +++ b/pkg/appconsts/v1/app_consts.go @@ -1,7 +1,15 @@ package v1 +import "time" + const ( Version uint64 = 1 SquareSizeUpperBound int = 128 SubtreeRootThreshold int = 64 + TimeoutPropose = time.Second * 10 + TimeoutCommit = time.Second * 11 + // UpgradeHeightDelay is the number of blocks after a quorum has been + // reached that the chain should upgrade to the new version. Assuming a block + // interval of 12 seconds, this is 7 days. + UpgradeHeightDelay = int64(7 * 24 * 60 * 60 / 12) // 7 days * 24 hours * 60 minutes * 60 seconds / 12 seconds per block = 50,400 blocks. ) diff --git a/pkg/appconsts/v2/app_consts.go b/pkg/appconsts/v2/app_consts.go index d5829e84c5..d02a97079b 100644 --- a/pkg/appconsts/v2/app_consts.go +++ b/pkg/appconsts/v2/app_consts.go @@ -1,7 +1,15 @@ package v2 +import "time" + const ( Version uint64 = 2 SquareSizeUpperBound int = 128 SubtreeRootThreshold int = 64 + TimeoutPropose = time.Second * 10 + TimeoutCommit = time.Second * 11 + // UpgradeHeightDelay is the number of blocks after a quorum has been + // reached that the chain should upgrade to the new version. Assuming a block + // interval of 12 seconds, this is 7 days. + UpgradeHeightDelay = int64(7 * 24 * 60 * 60 / 12) // 7 days * 24 hours * 60 minutes * 60 seconds / 12 seconds per block = 50,400 blocks. ) diff --git a/pkg/appconsts/v3/app_consts.go b/pkg/appconsts/v3/app_consts.go index 4b65941de1..3f9279518d 100644 --- a/pkg/appconsts/v3/app_consts.go +++ b/pkg/appconsts/v3/app_consts.go @@ -1,5 +1,7 @@ package v3 +import "time" + const ( Version uint64 = 3 SquareSizeUpperBound int = 128 @@ -7,4 +9,10 @@ const ( TxSizeCostPerByte uint64 = 10 GasPerBlobByte uint32 = 8 MaxTxSize int = 2097152 // 2 MiB in bytes + TimeoutPropose = time.Millisecond * 3500 + TimeoutCommit = time.Millisecond * 4200 + // UpgradeHeightDelay is the number of blocks after a quorum has been + // reached that the chain should upgrade to the new version. Assuming a block + // interval of 12 seconds, this is 7 days. + UpgradeHeightDelay = int64(7 * 24 * 60 * 60 / 6) // 7 days * 24 hours * 60 minutes * 60 seconds / 6 seconds per block = 100,800 blocks. ) diff --git a/pkg/appconsts/versioned_consts.go b/pkg/appconsts/versioned_consts.go index 1eb40deb83..2455e87791 100644 --- a/pkg/appconsts/versioned_consts.go +++ b/pkg/appconsts/versioned_consts.go @@ -2,7 +2,10 @@ package appconsts import ( "strconv" + "time" + v1 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v1" + v2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" v3 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v3" ) @@ -52,3 +55,45 @@ var ( DefaultTxSizeCostPerByte = TxSizeCostPerByte(LatestVersion) DefaultGasPerBlobByte = GasPerBlobByte(LatestVersion) ) + +func GetTimeoutPropose(v uint64) time.Duration { + switch v { + case v1.Version: + return v1.TimeoutPropose + case v2.Version: + return v2.TimeoutPropose + default: + return v3.TimeoutPropose + } +} + +func GetTimeoutCommit(v uint64) time.Duration { + switch v { + case v1.Version: + return v1.TimeoutCommit + case v2.Version: + return v2.TimeoutCommit + default: + return v3.TimeoutCommit + } +} + +// UpgradeHeightDelay returns the delay in blocks after a quorum has been reached that the chain should upgrade to the new version. +func UpgradeHeightDelay(v uint64) int64 { + if OverrideUpgradeHeightDelayStr != "" { + parsedValue, err := strconv.ParseInt(OverrideUpgradeHeightDelayStr, 10, 64) + if err != nil { + panic("Invalid OverrideUpgradeHeightDelayStr value") + } + return parsedValue + } + switch v { + case v1.Version: + return v1.UpgradeHeightDelay + case v2.Version: + return v2.UpgradeHeightDelay + default: + return v3.UpgradeHeightDelay + + } +} diff --git a/test/e2e/benchmark/benchmark.go b/test/e2e/benchmark/benchmark.go index bc4503d61d..5af7b8539e 100644 --- a/test/e2e/benchmark/benchmark.go +++ b/test/e2e/benchmark/benchmark.go @@ -22,6 +22,8 @@ type BenchmarkTest struct { manifest *Manifest } +// NewBenchmarkTest wraps around testnet.New to create a new benchmark test. +// It may modify genesis consensus parameters based on manifest. func NewBenchmarkTest(name string, manifest *Manifest) (*BenchmarkTest, error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -52,7 +54,7 @@ func NewBenchmarkTest(name string, manifest *Manifest) (*BenchmarkTest, error) { } // SetupNodes creates genesis nodes and tx clients based on the manifest. -// There will be manifest.Validators validators and manifest.TxClients tx clients. +// There will be manifest.Validators many validators and manifest.TxClients many tx clients. // Each tx client connects to one validator. If TxClients are fewer than Validators, some validators will not have a tx client. func (b *BenchmarkTest) SetupNodes() error { ctx := context.Background() diff --git a/test/e2e/benchmark/manifest.go b/test/e2e/benchmark/manifest.go index 64b319fcc8..b212ff85c4 100644 --- a/test/e2e/benchmark/manifest.go +++ b/test/e2e/benchmark/manifest.go @@ -80,6 +80,8 @@ type Manifest struct { GovMaxSquareSize int64 DisableBBR bool + + GenesisAppVersion uint64 } func (m *Manifest) GetGenesisModifiers() []genesis.Modifier { @@ -88,6 +90,7 @@ func (m *Manifest) GetGenesisModifiers() []genesis.Modifier { blobParams := blobtypes.DefaultParams() blobParams.GovMaxSquareSize = uint64(m.GovMaxSquareSize) + modifiers = append(modifiers, genesis.SetBlobParams(ecfg.Codec, blobParams)) return modifiers @@ -96,6 +99,7 @@ func (m *Manifest) GetGenesisModifiers() []genesis.Modifier { func (m *Manifest) GetConsensusParams() *tmproto.ConsensusParams { cparams := app.DefaultConsensusParams() cparams.Block.MaxBytes = m.MaxBlockBytes + cparams.Version.AppVersion = m.GenesisAppVersion return cparams } diff --git a/test/e2e/major_upgrade_v3.go b/test/e2e/major_upgrade_v3.go index f82fae2804..35cb79b045 100644 --- a/test/e2e/major_upgrade_v3.go +++ b/test/e2e/major_upgrade_v3.go @@ -7,16 +7,18 @@ import ( "time" "github.com/celestiaorg/celestia-app/v3/app" + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" v2 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v2" v3 "github.com/celestiaorg/celestia-app/v3/pkg/appconsts/v3" "github.com/celestiaorg/celestia-app/v3/test/e2e/testnet" "github.com/celestiaorg/knuu/pkg/knuu" + tmtypes "github.com/tendermint/tendermint/types" ) func MajorUpgradeToV3(logger *log.Logger) error { testName := "MajorUpgradeToV3" numNodes := 4 - upgradeHeightV3 := int64(10) + upgradeHeightV3 := int64(40) ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -37,12 +39,8 @@ func MajorUpgradeToV3(logger *log.Logger) error { defer testNet.Cleanup(ctx) - // HACKHACK: use a version of celestia-app built from a commit on this PR. - // This can be removed after the PR is merged to main and we override the - // upgrade height delay to one block in a new Docker image. - version := "1a20c01" - - logger.Println("Running major upgrade to v3 test", "version", version) + latestVersion, err := testnet.GetLatestVersion() + testnet.NoError("failed to get latest version", err) consensusParams := app.DefaultConsensusParams() consensusParams.Version.AppVersion = v2.Version // Start the test on v2 @@ -51,13 +49,13 @@ func MajorUpgradeToV3(logger *log.Logger) error { preloader, err := testNet.NewPreloader() testnet.NoError("failed to create preloader", err) - err = preloader.AddImage(ctx, testnet.DockerImageName(version)) + err = preloader.AddImage(ctx, testnet.DockerImageName(latestVersion)) testnet.NoError("failed to add image", err) defer func() { _ = preloader.EmptyImages(ctx) }() logger.Println("Creating genesis nodes") for i := 0; i < numNodes; i++ { - err := testNet.CreateGenesisNode(ctx, version, 10000000, 0, testnet.DefaultResources, true) + err := testNet.CreateGenesisNode(ctx, latestVersion, 10000000, 0, testnet.DefaultResources, true) testnet.NoError("failed to create genesis node", err) } @@ -68,7 +66,7 @@ func MajorUpgradeToV3(logger *log.Logger) error { upgradeHeightV3: v3.Version, } - err = testNet.CreateTxClient(ctx, "txsim", version, 1, "100-2000", 100, testnet.DefaultResources, endpoints[0], upgradeSchedule) + err = testNet.CreateTxClient(ctx, "txsim", latestVersion, 1, "100-2000", 100, testnet.DefaultResources, endpoints[0], upgradeSchedule) testnet.NoError("failed to create tx client", err) logger.Println("Setting up testnet") @@ -76,16 +74,18 @@ func MajorUpgradeToV3(logger *log.Logger) error { logger.Println("Starting testnet") testnet.NoError("Failed to start testnet", testNet.Start(ctx)) - timer := time.NewTimer(10 * time.Minute) + timer := time.NewTimer(20 * time.Minute) defer timer.Stop() ticker := time.NewTicker(3 * time.Second) defer ticker.Stop() logger.Println("waiting for upgrade") + + // wait for the upgrade to complete + var upgradedHeight int64 for _, node := range testNet.Nodes() { client, err := node.Client() testnet.NoError("failed to get client", err) - upgradeComplete := false lastHeight := int64(0) for !upgradeComplete { @@ -97,6 +97,9 @@ func MajorUpgradeToV3(logger *log.Logger) error { testnet.NoError("failed to get header", err) if resp.Header.Version.App == v3.Version { upgradeComplete = true + if upgradedHeight == 0 { + upgradedHeight = resp.Header.Height + } } logger.Printf("height %v", resp.Header.Height) lastHeight = resp.Header.Height @@ -104,5 +107,84 @@ func MajorUpgradeToV3(logger *log.Logger) error { } } + // check if the timeouts are set correctly + rpcNode := testNet.Nodes()[0] + client, err := rpcNode.Client() + testnet.NoError("failed to get client", err) + + startHeight := upgradeHeightV3 - 5 + endHeight := upgradedHeight + 5 + + type versionDuration struct { + dur time.Duration + block *tmtypes.Block + } + + blockSummaries := make([]versionDuration, 0, endHeight-startHeight) + var prevBlockTime time.Time + + for h := startHeight; h < endHeight; h++ { + resp, err := client.Block(ctx, &h) + testnet.NoError("failed to get header", err) + blockTime := resp.Block.Time + + if h == startHeight { + if resp.Block.Version.App != v2.Version { + return fmt.Errorf("expected start height %v was app version 2", startHeight) + } + prevBlockTime = blockTime + continue + } + + blockDur := blockTime.Sub(prevBlockTime) + prevBlockTime = blockTime + blockSummaries = append(blockSummaries, versionDuration{dur: blockDur, block: resp.Block}) + } + + preciseUpgradeHeight := 0 + multipleRounds := 0 + for _, b := range blockSummaries { + + // check for the precise upgrade height and skip, as the block time + // won't match due to the off by 1 nature of the block time. + if b.block.Version.App == v3.Version && preciseUpgradeHeight == 0 { + preciseUpgradeHeight = int(b.block.Height) + continue + } + + // don't test heights with multiple rounds as the times are off and fail + // later if there are too many + if b.block.LastCommit.Round > 0 { + multipleRounds++ + continue + } + + if b.dur < appconsts.GetTimeoutCommit(b.block.Version.App) { + return fmt.Errorf( + "block was too fast for corresponding version: version %v duration %v upgrade height %v height %v", + b.block.Version.App, + b.dur, + preciseUpgradeHeight, + b.block.Height, + ) + } + + // check if the time decreased for v3 + if b.block.Version.App == v3.Version && b.dur > appconsts.GetTimeoutCommit(b.block.Version.App)+5 { + return fmt.Errorf( + "block was too slow for corresponding version: version %v duration %v upgrade height %v height %v", + b.block.Version.App, + b.dur, + preciseUpgradeHeight, + b.block.Height, + ) + } + + } + + if multipleRounds > 2 { + return fmt.Errorf("too many multiple rounds for test to be reliable: %d", multipleRounds) + } + return nil } diff --git a/test/e2e/simple.go b/test/e2e/simple.go index 2a25c46e2b..8dab0278eb 100644 --- a/test/e2e/simple.go +++ b/test/e2e/simple.go @@ -12,7 +12,7 @@ import ( "github.com/celestiaorg/knuu/pkg/knuu" ) -// This test runs a simple testnet with 4 validators. It submits both MsgPayForBlobs +// E2ESimple runs a simple testnet with 4 validators. It submits both MsgPayForBlobs // and MsgSends over 30 seconds and then asserts that at least 10 transactions were // committed. func E2ESimple(logger *log.Logger) error { diff --git a/test/e2e/testnet/testnet.go b/test/e2e/testnet/testnet.go index 5b73daf596..7cc74fb011 100644 --- a/test/e2e/testnet/testnet.go +++ b/test/e2e/testnet/testnet.go @@ -376,6 +376,7 @@ func (t *Testnet) WaitToSync(ctx context.Context) error { genesisNodes = append(genesisNodes, node) } } + for _, node := range genesisNodes { log.Info().Str("name", node.Name).Msg( "waiting for node to sync") @@ -410,18 +411,18 @@ func (t *Testnet) WaitToSync(ctx context.Context) error { // For that, use WaitToSync. func (t *Testnet) StartNodes(ctx context.Context) error { genesisNodes := make([]*Node, 0) + // identify genesis nodes for _, node := range t.nodes { if node.StartHeight == 0 { genesisNodes = append(genesisNodes, node) } - } - // start genesis nodes asynchronously - for _, node := range genesisNodes { + err := node.StartAsync(ctx) if err != nil { return fmt.Errorf("node %s failed to start: %w", node.Name, err) } } + log.Info().Msg("create endpoint proxies for genesis nodes") // wait for instances to be running for _, node := range genesisNodes { diff --git a/test/e2e/testnet/txsimNode.go b/test/e2e/testnet/txsimNode.go index bfa21da875..c63062598b 100644 --- a/test/e2e/testnet/txsimNode.go +++ b/test/e2e/testnet/txsimNode.go @@ -78,10 +78,13 @@ func CreateTxClient( fmt.Sprintf("--blob %d", blobSequences), fmt.Sprintf("--blob-amounts %d", blobsPerSeq), fmt.Sprintf("--blob-sizes %s", blobRange), - fmt.Sprintf("--upgrade-schedule %s", stringifyUpgradeSchedule(upgradeSchedule)), fmt.Sprintf("--blob-share-version %d", share.ShareVersionZero), } + if len(upgradeSchedule) > 0 { + args = append(args, fmt.Sprintf("--upgrade-schedule %s", stringifyUpgradeSchedule(upgradeSchedule))) + } + if err := instance.Build().SetArgs(args...); err != nil { return nil, err } diff --git a/test/txsim/run_test.go b/test/txsim/run_test.go index 4a922f15a1..d4cbdf22ba 100644 --- a/test/txsim/run_test.go +++ b/test/txsim/run_test.go @@ -170,6 +170,8 @@ func TestTxSimUpgrade(t *testing.T) { WithFundedAccounts("txsim-master") cctx, _, grpcAddr := testnode.NewNetwork(t, cfg) + require.NoError(t, cctx.WaitForNextBlock()) + // updrade to v3 at height 20 sequences := []txsim.Sequence{ txsim.NewUpgradeSequence(v3.Version, 20), diff --git a/test/util/genesis/genesis.go b/test/util/genesis/genesis.go index 83eab54250..f0fadf0b59 100644 --- a/test/util/genesis/genesis.go +++ b/test/util/genesis/genesis.go @@ -73,7 +73,7 @@ func NewDefaultGenesis() *Genesis { return g } -// WithModifier adds a genesis modifier to the genesis. +// WithModifiers adds a genesis modifier to the genesis. func (g *Genesis) WithModifiers(ops ...Modifier) *Genesis { g.genOps = append(g.genOps, ops...) return g @@ -97,7 +97,7 @@ func (g *Genesis) WithGenesisTime(genesisTime time.Time) *Genesis { return g } -// WithAccounts adds the given validators to the genesis. +// WithValidators adds the given validators to the genesis. func (g *Genesis) WithValidators(vals ...Validator) *Genesis { for _, val := range vals { err := g.NewValidator(val) @@ -180,7 +180,7 @@ func (g *Genesis) AddValidator(val Validator) error { return nil } -// Creates a new validator account and adds it to the genesis. +// NewValidator creates a new validator account and adds it to the genesis. func (g *Genesis) NewValidator(val Validator) error { // Add the validator's genesis account if err := g.NewAccount(val.KeyringAccount); err != nil { diff --git a/test/util/testnode/app_wrapper.go b/test/util/testnode/app_wrapper.go new file mode 100644 index 0000000000..e885f4c4ed --- /dev/null +++ b/test/util/testnode/app_wrapper.go @@ -0,0 +1,21 @@ +package testnode + +import ( + "time" + + "github.com/celestiaorg/celestia-app/v3/app" + sdk "github.com/cosmos/cosmos-sdk/types" + abci "github.com/tendermint/tendermint/abci/types" +) + +// wrapEndBlocker overrides the app's endblocker to set the timeout commit to a +// different value for testnode. +func wrapEndBlocker(app *app.App, timeoutCommit time.Duration) func(ctx sdk.Context, req abci.RequestEndBlock) abci.ResponseEndBlock { + endBlocker := func(ctx sdk.Context, req abci.RequestEndBlock) abci.ResponseEndBlock { + resp := app.EndBlocker(ctx, req) + resp.Timeouts.TimeoutCommit = timeoutCommit + return resp + } + + return endBlocker +} diff --git a/test/util/testnode/config.go b/test/util/testnode/config.go index 7062735a4c..0ec6e7920e 100644 --- a/test/util/testnode/config.go +++ b/test/util/testnode/config.go @@ -174,7 +174,7 @@ func DefaultTendermintConfig() *tmconfig.Config { func DefaultAppCreator() srvtypes.AppCreator { return func(_ log.Logger, _ tmdb.DB, _ io.Writer, _ srvtypes.AppOptions) srvtypes.Application { encodingConfig := encoding.MakeConfig(app.ModuleEncodingRegisters...) - return app.New( + app := app.New( log.NewNopLogger(), tmdb.NewMemDB(), nil, // trace store @@ -184,13 +184,15 @@ func DefaultAppCreator() srvtypes.AppCreator { simapp.EmptyAppOptions{}, baseapp.SetMinGasPrices(fmt.Sprintf("%v%v", appconsts.DefaultMinGasPrice, app.BondDenom)), ) + app.SetEndBlocker(wrapEndBlocker(app, time.Millisecond*30)) + return app } } func CustomAppCreator(minGasPrice string) srvtypes.AppCreator { return func(_ log.Logger, _ tmdb.DB, _ io.Writer, _ srvtypes.AppOptions) srvtypes.Application { encodingConfig := encoding.MakeConfig(app.ModuleEncodingRegisters...) - return app.New( + app := app.New( log.NewNopLogger(), tmdb.NewMemDB(), nil, // trace store @@ -200,6 +202,8 @@ func CustomAppCreator(minGasPrice string) srvtypes.AppCreator { simapp.EmptyAppOptions{}, baseapp.SetMinGasPrices(minGasPrice), ) + app.SetEndBlocker(wrapEndBlocker(app, time.Millisecond*0)) + return app } } diff --git a/test/util/testnode/network.go b/test/util/testnode/network.go index 4c0421c895..eab7384c91 100644 --- a/test/util/testnode/network.go +++ b/test/util/testnode/network.go @@ -33,6 +33,7 @@ func NewNetwork(t testing.TB, config *Config) (cctx Context, rpcAddr, grpcAddr s }) cctx = NewContext(ctx, config.Genesis.Keyring(), config.TmConfig, config.Genesis.ChainID, config.AppConfig.API.Address) + cctx.tmNode = tmNode cctx, stopNode, err := StartNode(tmNode, cctx) require.NoError(t, err) diff --git a/test/util/testnode/node_interaction_api.go b/test/util/testnode/node_interaction_api.go index 162840c819..a5c175e86d 100644 --- a/test/util/testnode/node_interaction_api.go +++ b/test/util/testnode/node_interaction_api.go @@ -22,6 +22,7 @@ import ( abci "github.com/tendermint/tendermint/abci/types" tmconfig "github.com/tendermint/tendermint/config" tmrand "github.com/tendermint/tendermint/libs/rand" + "github.com/tendermint/tendermint/node" rpctypes "github.com/tendermint/tendermint/rpc/core/types" ) @@ -33,6 +34,7 @@ type Context struct { goContext context.Context client.Context apiAddress string + tmNode *node.Node } func NewContext(goContext context.Context, keyring keyring.Keyring, tmConfig *tmconfig.Config, chainID, apiAddress string) Context { diff --git a/x/blobstream/integration_test.go b/x/blobstream/integration_test.go index a8f19bf4b6..7bb03a49ee 100644 --- a/x/blobstream/integration_test.go +++ b/x/blobstream/integration_test.go @@ -46,6 +46,8 @@ func (s *BlobstreamIntegrationSuite) SetupSuite() { cctx, _, _ := testnode.NewNetwork(t, cfg) s.ecfg = encoding.MakeConfig(app.ModuleEncodingRegisters...) s.cctx = cctx + + require.NoError(t, s.cctx.WaitForBlocks(10)) } func (s *BlobstreamIntegrationSuite) TestBlobstream() { diff --git a/x/signal/integration_test.go b/x/signal/integration_test.go index 23b8158dd2..c3dd2419dd 100644 --- a/x/signal/integration_test.go +++ b/x/signal/integration_test.go @@ -77,7 +77,7 @@ func TestUpgradeIntegration(t *testing.T) { require.False(t, shouldUpgrade) require.EqualValues(t, 0, version) - ctx = ctx.WithBlockHeight(ctx.BlockHeight() + appconsts.DefaultUpgradeHeightDelay) + ctx = ctx.WithBlockHeight(ctx.BlockHeight() + appconsts.UpgradeHeightDelay(version)) shouldUpgrade, version = app.SignalKeeper.ShouldUpgrade(ctx) require.True(t, shouldUpgrade) diff --git a/x/signal/keeper.go b/x/signal/keeper.go index 0a23d93119..3ee13a9708 100644 --- a/x/signal/keeper.go +++ b/x/signal/keeper.go @@ -5,6 +5,7 @@ import ( "encoding/binary" sdkmath "cosmossdk.io/math" + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" "github.com/celestiaorg/celestia-app/v3/x/signal/types" "github.com/cosmos/cosmos-sdk/codec" storetypes "github.com/cosmos/cosmos-sdk/store/types" @@ -41,10 +42,6 @@ type Keeper struct { // stakingKeeper is used to fetch validators to calculate the total power // signalled to a version. stakingKeeper StakingKeeper - - // upgradeHeightDelayBlocks is the number of blocks after a quorum has been - // reached that the chain should upgrade to the new version - upgradeHeightDelayBlocks int64 } // NewKeeper returns a signal keeper. @@ -52,13 +49,11 @@ func NewKeeper( binaryCodec codec.BinaryCodec, storeKey storetypes.StoreKey, stakingKeeper StakingKeeper, - upgradeHeightDelayBlocks int64, ) Keeper { return Keeper{ - binaryCodec: binaryCodec, - storeKey: storeKey, - stakingKeeper: stakingKeeper, - upgradeHeightDelayBlocks: upgradeHeightDelayBlocks, + binaryCodec: binaryCodec, + storeKey: storeKey, + stakingKeeper: stakingKeeper, } } @@ -109,7 +104,7 @@ func (k *Keeper) TryUpgrade(ctx context.Context, _ *types.MsgTryUpgrade) (*types } upgrade := types.Upgrade{ AppVersion: version, - UpgradeHeight: sdkCtx.BlockHeader().Height + k.upgradeHeightDelayBlocks, + UpgradeHeight: sdkCtx.BlockHeader().Height + appconsts.UpgradeHeightDelay(version), } k.setUpgrade(sdkCtx, upgrade) } diff --git a/x/signal/keeper_test.go b/x/signal/keeper_test.go index 71627a8417..f79d1c3884 100644 --- a/x/signal/keeper_test.go +++ b/x/signal/keeper_test.go @@ -69,7 +69,7 @@ func TestGetVotingPowerThreshold(t *testing.T) { t.Run(tc.name, func(t *testing.T) { config := encoding.MakeConfig(app.ModuleEncodingRegisters...) stakingKeeper := newMockStakingKeeper(tc.validators) - k := signal.NewKeeper(config.Codec, nil, stakingKeeper, appconsts.DefaultUpgradeHeightDelay) + k := signal.NewKeeper(config.Codec, nil, stakingKeeper) got := k.GetVotingPowerThreshold(sdk.Context{}) assert.Equal(t, tc.want, got, fmt.Sprintf("want %v, got %v", tc.want.String(), got.String())) }) @@ -183,7 +183,7 @@ func TestTallyingLogic(t *testing.T) { require.False(t, shouldUpgrade) // should be false because upgrade height hasn't been reached. require.Equal(t, uint64(0), version) - ctx = ctx.WithBlockHeight(ctx.BlockHeight() + appconsts.DefaultUpgradeHeightDelay) + ctx = ctx.WithBlockHeight(ctx.BlockHeight() + appconsts.UpgradeHeightDelay(version)) shouldUpgrade, version = upgradeKeeper.ShouldUpgrade(ctx) require.True(t, shouldUpgrade) // should be true because upgrade height has been reached. @@ -426,7 +426,7 @@ func TestGetUpgrade(t *testing.T) { got, err := upgradeKeeper.GetUpgrade(ctx, &types.QueryGetUpgradeRequest{}) require.NoError(t, err) assert.Equal(t, v2.Version, got.Upgrade.AppVersion) - assert.Equal(t, appconsts.DefaultUpgradeHeightDelay, got.Upgrade.UpgradeHeight) + assert.Equal(t, appconsts.UpgradeHeightDelay(v2.Version), got.Upgrade.UpgradeHeight) }) } @@ -452,7 +452,7 @@ func setup(t *testing.T) (signal.Keeper, sdk.Context, *mockStakingKeeper) { ) config := encoding.MakeConfig(app.ModuleEncodingRegisters...) - upgradeKeeper := signal.NewKeeper(config.Codec, signalStore, mockStakingKeeper, appconsts.DefaultUpgradeHeightDelay) + upgradeKeeper := signal.NewKeeper(config.Codec, signalStore, mockStakingKeeper) return upgradeKeeper, mockCtx, mockStakingKeeper } From f217064d3d90acccf8a3d3f91f444beb8dd84ee7 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Fri, 18 Oct 2024 15:59:18 -0400 Subject: [PATCH 81/85] refactor: rename limits from tx to msg (#3984) Motivation: https://github.com/celestiaorg/CIPs/pull/220#discussion_r1803281819 IMO we should either include this in v3.0.0-rc0 or close it b/c not worth it to break these constant names after that. --- app/test/prepare_proposal_test.go | 24 ++++++++++++------------ app/validate_txs.go | 16 ++++++++-------- pkg/appconsts/global_consts.go | 12 +++++++----- 3 files changed, 27 insertions(+), 25 deletions(-) diff --git a/app/test/prepare_proposal_test.go b/app/test/prepare_proposal_test.go index e0c963de8f..f274d9da6a 100644 --- a/app/test/prepare_proposal_test.go +++ b/app/test/prepare_proposal_test.go @@ -297,7 +297,7 @@ func TestPrepareProposalCappingNumberOfMessages(t *testing.T) { signers = append(signers, signer) } - numberOfPFBs := appconsts.PFBTransactionCap + 500 + numberOfPFBs := appconsts.MaxPFBMessages + 500 pfbTxs := make([][]byte, 0, numberOfPFBs) randomBytes := make([]byte, 2000) _, err := rand.Read(randomBytes) @@ -333,7 +333,7 @@ func TestPrepareProposalCappingNumberOfMessages(t *testing.T) { accountIndex++ } - numberOfMsgSends := appconsts.NonPFBTransactionCap + 500 + numberOfMsgSends := appconsts.MaxNonPFBMessages + 500 msgSendTxs := make([][]byte, 0, numberOfMsgSends) for i := 0; i < numberOfMsgSends; i++ { msg := banktypes.NewMsgSend( @@ -354,18 +354,18 @@ func TestPrepareProposalCappingNumberOfMessages(t *testing.T) { }{ { name: "capping only PFB transactions", - inputTransactions: pfbTxs[:appconsts.PFBTransactionCap+50], - expectedTransactions: pfbTxs[:appconsts.PFBTransactionCap], + inputTransactions: pfbTxs[:appconsts.MaxPFBMessages+50], + expectedTransactions: pfbTxs[:appconsts.MaxPFBMessages], }, { name: "capping only PFB transactions with multiple messages", - inputTransactions: multiPFBsPerTxs[:appconsts.PFBTransactionCap], - expectedTransactions: multiPFBsPerTxs[:appconsts.PFBTransactionCap/numberOfMsgsPerTx], + inputTransactions: multiPFBsPerTxs[:appconsts.MaxPFBMessages], + expectedTransactions: multiPFBsPerTxs[:appconsts.MaxPFBMessages/numberOfMsgsPerTx], }, { name: "capping only msg send transactions", - inputTransactions: msgSendTxs[:appconsts.NonPFBTransactionCap+50], - expectedTransactions: msgSendTxs[:appconsts.NonPFBTransactionCap], + inputTransactions: msgSendTxs[:appconsts.MaxNonPFBMessages+50], + expectedTransactions: msgSendTxs[:appconsts.MaxNonPFBMessages], }, { name: "capping msg send after pfb transactions", @@ -376,8 +376,8 @@ func TestPrepareProposalCappingNumberOfMessages(t *testing.T) { return input }(), expectedTransactions: func() [][]byte { - expected := make([][]byte, 0, appconsts.NonPFBTransactionCap+100) - expected = append(expected, msgSendTxs[:appconsts.NonPFBTransactionCap]...) + expected := make([][]byte, 0, appconsts.MaxNonPFBMessages+100) + expected = append(expected, msgSendTxs[:appconsts.MaxNonPFBMessages]...) expected = append(expected, pfbTxs[:100]...) return expected }(), @@ -391,9 +391,9 @@ func TestPrepareProposalCappingNumberOfMessages(t *testing.T) { return input }(), expectedTransactions: func() [][]byte { - expected := make([][]byte, 0, appconsts.PFBTransactionCap+100) + expected := make([][]byte, 0, appconsts.MaxPFBMessages+100) expected = append(expected, msgSendTxs[:100]...) - expected = append(expected, pfbTxs[:appconsts.PFBTransactionCap]...) + expected = append(expected, pfbTxs[:appconsts.MaxPFBMessages]...) return expected }(), }, diff --git a/app/validate_txs.go b/app/validate_txs.go index 7ff37a8fbb..ee3edfbb6f 100644 --- a/app/validate_txs.go +++ b/app/validate_txs.go @@ -45,7 +45,7 @@ func FilterTxs(logger log.Logger, ctx sdk.Context, handler sdk.AnteHandler, txCo // function used to apply the ante handler. func filterStdTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handler sdk.AnteHandler, txs [][]byte) ([][]byte, sdk.Context) { n := 0 - nonPFBTransactionsCount := 0 + nonPFBMessageCount := 0 for _, tx := range txs { sdkTx, err := dec(tx) if err != nil { @@ -57,11 +57,11 @@ func filterStdTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handler ctx = ctx.WithTxBytes(tx) msgTypes := msgTypes(sdkTx) - if nonPFBTransactionsCount+len(sdkTx.GetMsgs()) > appconsts.NonPFBTransactionCap { - logger.Debug("skipping tx because the sdk message cap was reached", "tx", tmbytes.HexBytes(coretypes.Tx(tx).Hash())) + if nonPFBMessageCount+len(sdkTx.GetMsgs()) > appconsts.MaxNonPFBMessages { + logger.Debug("skipping tx because the max non PFB message count was reached", "tx", tmbytes.HexBytes(coretypes.Tx(tx).Hash())) continue } - nonPFBTransactionsCount += len(sdkTx.GetMsgs()) + nonPFBMessageCount += len(sdkTx.GetMsgs()) ctx, err = handler(ctx, sdkTx, false) // either the transaction is invalid (ie incorrect nonce) and we @@ -90,7 +90,7 @@ func filterStdTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handler // function used to apply the ante handler. func filterBlobTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handler sdk.AnteHandler, txs []*tx.BlobTx) ([]*tx.BlobTx, sdk.Context) { n := 0 - pfbTransactionCount := 0 + pfbMessageCount := 0 for _, tx := range txs { sdkTx, err := dec(tx.Tx) if err != nil { @@ -101,11 +101,11 @@ func filterBlobTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handle // Set the tx size on the context before calling the AnteHandler ctx = ctx.WithTxBytes(tx.Tx) - if pfbTransactionCount+len(sdkTx.GetMsgs()) > appconsts.PFBTransactionCap { - logger.Debug("skipping tx because the pfb transaction cap was reached", "tx", tmbytes.HexBytes(coretypes.Tx(tx.Tx).Hash())) + if pfbMessageCount+len(sdkTx.GetMsgs()) > appconsts.MaxPFBMessages { + logger.Debug("skipping tx because the max pfb message count was reached", "tx", tmbytes.HexBytes(coretypes.Tx(tx.Tx).Hash())) continue } - pfbTransactionCount += len(sdkTx.GetMsgs()) + pfbMessageCount += len(sdkTx.GetMsgs()) ctx, err = handler(ctx, sdkTx, false) // either the transaction is invalid (ie incorrect nonce) and we diff --git a/pkg/appconsts/global_consts.go b/pkg/appconsts/global_consts.go index ce6f7d71ee..d8cc1bf87c 100644 --- a/pkg/appconsts/global_consts.go +++ b/pkg/appconsts/global_consts.go @@ -49,11 +49,13 @@ func HashLength() int { return hashLength } -// The following consts are not consensus breaking and will be applied straight after this binary is started. +// The following consts are not consensus breaking and will be applied straight +// after this binary is started. const ( - // NonPFBTransactionCap is the maximum number of SDK messages, aside from PFBs, that a block can contain. - NonPFBTransactionCap = 200 + // MaxNonPFBMessages is the maximum number of SDK messages, aside from + // PFBs, that a block can contain. + MaxNonPFBMessages = 200 - // PFBTransactionCap is the maximum number of PFB messages a block can contain. - PFBTransactionCap = 600 + // MaxPFBMessages is the maximum number of PFB messages a block can contain. + MaxPFBMessages = 600 ) From d4eb75e9936357eca0ff3870bf79891970aa56cf Mon Sep 17 00:00:00 2001 From: CHAMI Rachid Date: Mon, 21 Oct 2024 08:22:49 +0200 Subject: [PATCH 82/85] chore: ABCI methods benchmarks (#3904) Works on this: https://github.com/celestiaorg/celestia-app/issues/3898 To run the benchmarks, run the following in the root directory: ```shell go test -bench= app/benchmark_test.go ``` with `` is: - `BenchmarkCheckTx_MsgSend_1`: for a benchmark of a `MsgSend` transaction in `CheckTx` - `BenchmarkCheckTx_MsgSend_8MB`: for a benchmark of 8mb block worth of `MsgSend` transactions in `CheckTx` - `BenchmarkDeliverTx_MsgSend_1`: for a benchmark of a `MsgSend` transaction in `DeliverTx` - `BenchmarkDeliverTx_MsgSend_8MB`: for a benchmark of 8mb block worth of `MsgSend` transactions in `DeliverTx` - `BenchmarkPrepareProposal_MsgSend_1`: for a benchmark of a block containing a single`MsgSend` transaction in `PrepareProposal` - `BenchmarkPrepareProposal_MsgSend_8MB`: for a benchmark of an 8mb block containing `MsgSend` transactions in `PrepareProposal` - `BenchmarkProcessProposal_MsgSend_1`: for a benchmark of a block containing a single`MsgSend` transaction in `ProcessProposal` - `BenchmarkProcessProposal_MsgSend_8MB`: for a benchmark of an 8mb block containing`MsgSend` transactions in `ProcessProposal` - ... Note: keeping this as a draft because it doesn't necessarily need to be merged Benchmark run on: Macbook pro M3 max 48GB RAM --------- Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> Co-authored-by: Rootul P --- app/benchmarks/README.md | 27 + .../benchmark_ibc_update_client_test.go | 518 +++++++++++ app/benchmarks/benchmark_msg_send_test.go | 333 +++++++ app/benchmarks/benchmark_pfb_test.go | 381 ++++++++ app/benchmarks/results.md | 839 ++++++++++++++++++ pkg/appconsts/global_consts.go | 11 - pkg/appconsts/prepare_proposal_consts.go | 15 + .../prepare_proposal_consts_bench.go | 14 + test/util/test_app.go | 8 +- 9 files changed, 2133 insertions(+), 13 deletions(-) create mode 100644 app/benchmarks/README.md create mode 100644 app/benchmarks/benchmark_ibc_update_client_test.go create mode 100644 app/benchmarks/benchmark_msg_send_test.go create mode 100644 app/benchmarks/benchmark_pfb_test.go create mode 100644 app/benchmarks/results.md create mode 100644 pkg/appconsts/prepare_proposal_consts.go create mode 100644 pkg/appconsts/prepare_proposal_consts_bench.go diff --git a/app/benchmarks/README.md b/app/benchmarks/README.md new file mode 100644 index 0000000000..f7f720741b --- /dev/null +++ b/app/benchmarks/README.md @@ -0,0 +1,27 @@ +# Benchmarks + +This package contains benchmarks for the ABCI methods with the following transaction types: + +- Message send +- IBC update client +- PayForBlobs + +## How to run + +To run the benchmarks, run the following in the root directory: + +```shell +go test -tags=bench_abci_methods -bench= app/benchmarks/benchmark_* +``` + +## Results + +The results are outlined in the [results](results.md) document. + +## Key takeaways + +We decided to softly limit the number of messages contained in a block, via introducing the `MaxPFBMessages` and `MaxNonPFBMessages`, and checking against them in prepare proposal. + +This way, the default block construction mechanism will only propose blocks that respect these limitations. And if a block that doesn't respect them reached consensus, it will still be accepted since this rule is not consensus breaking. + +As specified in [results](results.md) document, those results were generated on 16 core 48GB RAM machine, and gave us certain thresholds. However, when we run the same experiments on the recommended validator setup, 4 cores 16GB RAM, the numbers were lower. These low numbers are what we used in the limits. diff --git a/app/benchmarks/benchmark_ibc_update_client_test.go b/app/benchmarks/benchmark_ibc_update_client_test.go new file mode 100644 index 0000000000..873c560a95 --- /dev/null +++ b/app/benchmarks/benchmark_ibc_update_client_test.go @@ -0,0 +1,518 @@ +//go:build bench_abci_methods + +package benchmarks_test + +import ( + "fmt" + "math" + "testing" + "time" + + "github.com/celestiaorg/celestia-app/v3/app" + "github.com/celestiaorg/celestia-app/v3/app/encoding" + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" + "github.com/celestiaorg/celestia-app/v3/pkg/user" + testutil "github.com/celestiaorg/celestia-app/v3/test/util" + "github.com/celestiaorg/celestia-app/v3/test/util/testfactory" + dbm "github.com/cometbft/cometbft-db" + sdk "github.com/cosmos/cosmos-sdk/types" + types3 "github.com/cosmos/ibc-go/v6/modules/core/02-client/types" + types2 "github.com/cosmos/ibc-go/v6/modules/core/23-commitment/types" + types4 "github.com/cosmos/ibc-go/v6/modules/light-clients/07-tendermint/types" + "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/abci/types" + "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/crypto/tmhash" + crypto2 "github.com/tendermint/tendermint/proto/tendermint/crypto" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" + tmprotoversion "github.com/tendermint/tendermint/proto/tendermint/version" + "github.com/tendermint/tendermint/version" + + "github.com/tendermint/tendermint/crypto/ed25519" + sm "github.com/tendermint/tendermint/state" + types0 "github.com/tendermint/tendermint/types" +) + +func BenchmarkIBC_CheckTx_Update_Client_Multi(b *testing.B) { + testCases := []struct { + numberOfValidators int + }{ + {numberOfValidators: 2}, + {numberOfValidators: 10}, + {numberOfValidators: 25}, + {numberOfValidators: 50}, + {numberOfValidators: 75}, + {numberOfValidators: 100}, + {numberOfValidators: 125}, + {numberOfValidators: 150}, + {numberOfValidators: 175}, + {numberOfValidators: 200}, + {numberOfValidators: 225}, + {numberOfValidators: 250}, + {numberOfValidators: 300}, + {numberOfValidators: 400}, + {numberOfValidators: 500}, + } + for _, testCase := range testCases { + b.Run(fmt.Sprintf("number of validators: %d", testCase.numberOfValidators), func(b *testing.B) { + benchmarkIBCCheckTxUpdateClient(b, testCase.numberOfValidators) + }) + } +} + +func benchmarkIBCCheckTxUpdateClient(b *testing.B, numberOfValidators int) { + testApp, rawTxs := generateIBCUpdateClientTransaction(b, numberOfValidators, 1, 1) + testApp.Commit() + + checkTxRequest := types.RequestCheckTx{ + Type: types.CheckTxType_New, + Tx: rawTxs[0], + } + + b.ResetTimer() + resp := testApp.CheckTx(checkTxRequest) + b.StopTimer() + require.Equal(b, uint32(0), resp.Code) + require.Equal(b, "", resp.Codespace) + b.ReportMetric(float64(resp.GasUsed), "gas_used") + b.ReportMetric(float64(len(rawTxs[0])), "transaction_size(byte)") + b.ReportMetric(float64(numberOfValidators), "number_of_validators") + b.ReportMetric(float64(2*numberOfValidators/3), "number_of_verified_signatures") +} + +func BenchmarkIBC_DeliverTx_Update_Client_Multi(b *testing.B) { + testCases := []struct { + numberOfValidators int + }{ + {numberOfValidators: 2}, + {numberOfValidators: 10}, + {numberOfValidators: 25}, + {numberOfValidators: 50}, + {numberOfValidators: 75}, + {numberOfValidators: 100}, + {numberOfValidators: 125}, + {numberOfValidators: 150}, + {numberOfValidators: 175}, + {numberOfValidators: 200}, + {numberOfValidators: 225}, + {numberOfValidators: 250}, + {numberOfValidators: 300}, + {numberOfValidators: 400}, + {numberOfValidators: 500}, + } + for _, testCase := range testCases { + b.Run(fmt.Sprintf("number of validators: %d", testCase.numberOfValidators), func(b *testing.B) { + benchmarkIBCDeliverTxUpdateClient(b, testCase.numberOfValidators) + }) + } +} + +func benchmarkIBCDeliverTxUpdateClient(b *testing.B, numberOfValidators int) { + testApp, rawTxs := generateIBCUpdateClientTransaction(b, numberOfValidators, 1, 1) + + deliverTxRequest := types.RequestDeliverTx{ + Tx: rawTxs[0], + } + + b.ResetTimer() + resp := testApp.DeliverTx(deliverTxRequest) + b.StopTimer() + require.Equal(b, uint32(0), resp.Code) + require.Equal(b, "", resp.Codespace) + b.ReportMetric(float64(resp.GasUsed), "gas_used") + b.ReportMetric(float64(len(rawTxs[0])), "transaction_size(byte)") + b.ReportMetric(float64(numberOfValidators), "number_of_validators") + b.ReportMetric(float64(2*numberOfValidators/3), "number_of_verified_signatures") +} + +func BenchmarkIBC_PrepareProposal_Update_Client_Multi(b *testing.B) { + testCases := []struct { + numberOfTransactions, numberOfValidators int + }{ + {numberOfTransactions: 6_000, numberOfValidators: 2}, + {numberOfTransactions: 3_000, numberOfValidators: 10}, + {numberOfTransactions: 2_000, numberOfValidators: 25}, + {numberOfTransactions: 1_000, numberOfValidators: 50}, + {numberOfTransactions: 500, numberOfValidators: 75}, + {numberOfTransactions: 500, numberOfValidators: 100}, + {numberOfTransactions: 500, numberOfValidators: 125}, + {numberOfTransactions: 500, numberOfValidators: 150}, + {numberOfTransactions: 500, numberOfValidators: 175}, + {numberOfTransactions: 500, numberOfValidators: 200}, + {numberOfTransactions: 500, numberOfValidators: 225}, + {numberOfTransactions: 500, numberOfValidators: 250}, + {numberOfTransactions: 500, numberOfValidators: 300}, + {numberOfTransactions: 500, numberOfValidators: 400}, + {numberOfTransactions: 500, numberOfValidators: 500}, + } + for _, testCase := range testCases { + b.Run(fmt.Sprintf("number of validators: %d", testCase.numberOfValidators), func(b *testing.B) { + benchmarkIBCPrepareProposalUpdateClient(b, testCase.numberOfValidators, testCase.numberOfTransactions) + }) + } +} + +func benchmarkIBCPrepareProposalUpdateClient(b *testing.B, numberOfValidators, count int) { + testApp, rawTxs := generateIBCUpdateClientTransaction(b, numberOfValidators, count, 0) + + blockData := &tmproto.Data{ + Txs: rawTxs, + } + prepareProposalRequest := types.RequestPrepareProposal{ + BlockData: blockData, + ChainId: testApp.GetChainID(), + Height: 10, + } + + b.ResetTimer() + prepareProposalResponse := testApp.PrepareProposal(prepareProposalRequest) + b.StopTimer() + require.GreaterOrEqual(b, len(prepareProposalResponse.BlockData.Txs), 1) + b.ReportMetric(float64(b.Elapsed().Nanoseconds()), "prepare_proposal_time(ns)") + b.ReportMetric(float64(len(prepareProposalResponse.BlockData.Txs)), "number_of_transactions") + b.ReportMetric(float64(len(rawTxs[0])), "transactions_size(byte)") + b.ReportMetric(calculateBlockSizeInMb(prepareProposalResponse.BlockData.Txs), "block_size(mb)") + b.ReportMetric(float64(calculateTotalGasUsed(testApp, prepareProposalResponse.BlockData.Txs)), "total_gas_used") + b.ReportMetric(float64(numberOfValidators), "number_of_validators") + b.ReportMetric(float64(2*numberOfValidators/3), "number_of_verified_signatures") +} + +func BenchmarkIBC_ProcessProposal_Update_Client_Multi(b *testing.B) { + testCases := []struct { + numberOfTransactions, numberOfValidators int + }{ + {numberOfTransactions: 6_000, numberOfValidators: 2}, + {numberOfTransactions: 3_000, numberOfValidators: 10}, + {numberOfTransactions: 2_000, numberOfValidators: 25}, + {numberOfTransactions: 1_000, numberOfValidators: 50}, + {numberOfTransactions: 500, numberOfValidators: 75}, + {numberOfTransactions: 500, numberOfValidators: 100}, + {numberOfTransactions: 500, numberOfValidators: 125}, + {numberOfTransactions: 500, numberOfValidators: 150}, + {numberOfTransactions: 500, numberOfValidators: 175}, + {numberOfTransactions: 500, numberOfValidators: 200}, + {numberOfTransactions: 500, numberOfValidators: 225}, + {numberOfTransactions: 500, numberOfValidators: 250}, + {numberOfTransactions: 500, numberOfValidators: 300}, + {numberOfTransactions: 500, numberOfValidators: 400}, + {numberOfTransactions: 500, numberOfValidators: 500}, + } + for _, testCase := range testCases { + b.Run(fmt.Sprintf("number of validators: %d", testCase.numberOfValidators), func(b *testing.B) { + benchmarkIBCProcessProposalUpdateClient(b, testCase.numberOfValidators, testCase.numberOfTransactions) + }) + } +} + +func benchmarkIBCProcessProposalUpdateClient(b *testing.B, numberOfValidators, count int) { + testApp, rawTxs := generateIBCUpdateClientTransaction(b, numberOfValidators, count, 0) + + blockData := &tmproto.Data{ + Txs: rawTxs, + } + prepareProposalRequest := types.RequestPrepareProposal{ + BlockData: blockData, + ChainId: testApp.GetChainID(), + Height: 10, + } + + prepareProposalResponse := testApp.PrepareProposal(prepareProposalRequest) + require.GreaterOrEqual(b, len(prepareProposalResponse.BlockData.Txs), 1) + + processProposalRequest := types.RequestProcessProposal{ + BlockData: prepareProposalResponse.BlockData, + Header: tmproto.Header{ + Height: 10, + DataHash: prepareProposalResponse.BlockData.Hash, + ChainID: testutil.ChainID, + Version: tmprotoversion.Consensus{ + App: testApp.AppVersion(), + }, + }, + } + + b.ResetTimer() + resp := testApp.ProcessProposal(processProposalRequest) + b.StopTimer() + require.Equal(b, types.ResponseProcessProposal_ACCEPT, resp.Result) + + b.ReportMetric(float64(b.Elapsed().Nanoseconds()), "process_proposal_time(ns)") + b.ReportMetric(float64(len(prepareProposalResponse.BlockData.Txs)), "number_of_transactions") + b.ReportMetric(float64(len(rawTxs[0])), "transactions_size(byte)") + b.ReportMetric(calculateBlockSizeInMb(prepareProposalResponse.BlockData.Txs), "block_size(mb)") + b.ReportMetric(float64(calculateTotalGasUsed(testApp, prepareProposalResponse.BlockData.Txs)), "total_gas_used") + b.ReportMetric(float64(numberOfValidators), "number_of_validators") + b.ReportMetric(float64(2*numberOfValidators/3), "number_of_verified_signatures") +} + +// generateIBCUpdateClientTransaction creates a test app then generates an IBC +// update client transaction with the specified number of validators. +// Note: the number of the verified signatures is: 2 * numberOfValidators / 3 +// the offset is just a hack for transactions to be processed by the needed +// ABCI method. +func generateIBCUpdateClientTransaction(b *testing.B, numberOfValidators int, numberOfMessages int, offsetAccountSequence int) (*app.App, [][]byte) { + account := "test" + testApp, kr := testutil.SetupTestAppWithGenesisValSetAndMaxSquareSize(app.DefaultConsensusParams(), 128, account) + addr := testfactory.GetAddress(kr, account) + enc := encoding.MakeConfig(app.ModuleEncodingRegisters...) + acc := testutil.DirectQueryAccount(testApp, addr) + signer, err := user.NewSigner(kr, enc.TxConfig, testutil.ChainID, appconsts.LatestVersion, user.NewAccount(account, acc.GetAccountNumber(), acc.GetSequence())) + require.NoError(b, err) + + msgs := generateUpdateClientTransaction( + b, + testApp, + *signer, + acc.GetAddress().String(), + account, + numberOfValidators, + numberOfMessages, + ) + + accountSequence := testutil.DirectQueryAccount(testApp, addr).GetSequence() + err = signer.SetSequence(account, accountSequence+uint64(offsetAccountSequence)) + require.NoError(b, err) + rawTxs := make([][]byte, 0, numberOfMessages) + for i := 0; i < numberOfMessages; i++ { + rawTx, err := signer.CreateTx([]sdk.Msg{msgs[i]}, user.SetGasLimit(25497600000), user.SetFee(100000)) + require.NoError(b, err) + rawTxs = append(rawTxs, rawTx) + accountSequence++ + err = signer.SetSequence(account, accountSequence) + require.NoError(b, err) + } + + return testApp, rawTxs +} + +func generateUpdateClientTransaction(b *testing.B, app *app.App, signer user.Signer, signerAddr string, signerName string, numberOfValidators int, numberOfMsgs int) []*types3.MsgUpdateClient { + state, _, privVals := makeState(numberOfValidators, 5) + wBefore := time.Now() + time.Sleep(time.Second) + w := time.Now() + lastResultHash := crypto.CRandBytes(tmhash.Size) + lastCommitHash := crypto.CRandBytes(tmhash.Size) + lastBlockHash := crypto.CRandBytes(tmhash.Size) + lastBlockID := makeBlockID(lastBlockHash, 1000, []byte("hash")) + header := tmproto.Header{ + Version: tmprotoversion.Consensus{Block: version.BlockProtocol, App: 1}, + ChainID: state.ChainID, + Height: 5, + Time: w, + LastCommitHash: lastCommitHash, + DataHash: crypto.CRandBytes(tmhash.Size), + ValidatorsHash: state.Validators.Hash(), + NextValidatorsHash: state.Validators.Hash(), + ConsensusHash: crypto.CRandBytes(tmhash.Size), + AppHash: crypto.CRandBytes(tmhash.Size), + LastResultsHash: lastResultHash, + EvidenceHash: crypto.CRandBytes(tmhash.Size), + ProposerAddress: crypto.CRandBytes(crypto.AddressSize), + LastBlockId: lastBlockID.ToProto(), + } + t := types0.Header{ + Version: tmprotoversion.Consensus{Block: version.BlockProtocol, App: 1}, + ChainID: state.ChainID, + Height: 5, + Time: w, + LastCommitHash: header.LastCommitHash, + DataHash: header.DataHash, + ValidatorsHash: header.ValidatorsHash, + NextValidatorsHash: header.NextValidatorsHash, + ConsensusHash: header.ConsensusHash, + AppHash: header.AppHash, + LastResultsHash: header.LastResultsHash, + EvidenceHash: header.EvidenceHash, + ProposerAddress: header.ProposerAddress, + LastBlockID: lastBlockID, + } + header0Hash := t.Hash() + blockID := makeBlockID(header0Hash, 1000, []byte("partshash")) + commit, err := makeValidCommit(5, blockID, state.Validators, privVals) + require.NoError(b, err) + signatures := make([]tmproto.CommitSig, numberOfValidators) + validators := make([]*tmproto.Validator, numberOfValidators) + for i := 0; i < numberOfValidators; i++ { + signatures[i] = tmproto.CommitSig{ + BlockIdFlag: tmproto.BlockIDFlag(commit.Signatures[i].BlockIDFlag), + ValidatorAddress: commit.Signatures[i].ValidatorAddress, + Timestamp: commit.Signatures[i].Timestamp, + Signature: commit.Signatures[i].Signature, + } + validators[i] = &tmproto.Validator{ + Address: state.Validators.Validators[i].Address, + PubKey: crypto2.PublicKey{Sum: &crypto2.PublicKey_Ed25519{Ed25519: state.Validators.Validators[i].PubKey.Bytes()}}, + VotingPower: state.Validators.Validators[i].VotingPower, + ProposerPriority: state.Validators.Validators[i].ProposerPriority, + } + } + sh := tmproto.SignedHeader{ + Header: &header, + Commit: &tmproto.Commit{ + Height: commit.Height, + Round: commit.Round, + BlockID: tmproto.BlockID{ + Hash: header0Hash, + PartSetHeader: tmproto.PartSetHeader{ + Total: commit.BlockID.PartSetHeader.Total, + Hash: commit.BlockID.PartSetHeader.Hash, + }, + }, + Signatures: signatures, + }, + } + clientState := types4.ClientState{ + ChainId: chainID, + TrustLevel: types4.Fraction{Numerator: 1, Denominator: 3}, + TrustingPeriod: time.Hour * 24 * 21 * 100, // we want to always accept the upgrade + UnbondingPeriod: time.Hour * 24 * 21 * 101, + MaxClockDrift: math.MaxInt64 - 1, + FrozenHeight: types3.Height{}, + LatestHeight: types3.Height{ + RevisionNumber: 0, + RevisionHeight: 4, + }, + ProofSpecs: types2.GetSDKSpecs(), + AllowUpdateAfterExpiry: true, + AllowUpdateAfterMisbehaviour: true, + } + consensusState := types4.ConsensusState{ + Timestamp: wBefore, + Root: types2.MerkleRoot{Hash: lastBlockHash}, + NextValidatorsHash: state.Validators.Hash(), + } + + msgs := make([]*types3.MsgUpdateClient, numberOfMsgs) + for index := 0; index < numberOfMsgs; index++ { + createClientMsg, err := types3.NewMsgCreateClient(&clientState, &consensusState, signerAddr) + require.NoError(b, err) + rawTx, err := signer.CreateTx([]sdk.Msg{createClientMsg}, user.SetGasLimit(2549760000), user.SetFee(10000)) + require.NoError(b, err) + resp := app.DeliverTx(types.RequestDeliverTx{Tx: rawTx}) + var clientName string + for _, event := range resp.Events { + if event.Type == types3.EventTypeCreateClient { + for _, attribute := range event.Attributes { + if string(attribute.Key) == types3.AttributeKeyClientID { + clientName = string(attribute.Value) + } + } + } + } + require.NotEmpty(b, clientName) + + msg, err := types3.NewMsgUpdateClient( + clientName, + &types4.Header{ + SignedHeader: &sh, + ValidatorSet: &tmproto.ValidatorSet{ + Validators: validators, + Proposer: &tmproto.Validator{ + Address: state.Validators.Proposer.Address, + PubKey: crypto2.PublicKey{Sum: &crypto2.PublicKey_Ed25519{Ed25519: state.Validators.Proposer.PubKey.Bytes()}}, + VotingPower: state.Validators.Proposer.VotingPower, + ProposerPriority: state.Validators.Proposer.ProposerPriority, + }, + TotalVotingPower: state.Validators.TotalVotingPower(), + }, + TrustedHeight: types3.Height{ + RevisionNumber: 0, + RevisionHeight: 4, + }, + TrustedValidators: &tmproto.ValidatorSet{ + Validators: validators, + Proposer: &tmproto.Validator{ + Address: state.Validators.Proposer.Address, + PubKey: crypto2.PublicKey{Sum: &crypto2.PublicKey_Ed25519{Ed25519: state.Validators.Proposer.PubKey.Bytes()}}, + VotingPower: state.Validators.Proposer.VotingPower, + ProposerPriority: state.Validators.Proposer.ProposerPriority, + }, + TotalVotingPower: state.Validators.TotalVotingPower(), + }, + }, + signerAddr, + ) + require.NoError(b, err) + msgs[index] = msg + err = signer.IncrementSequence(signerName) + require.NoError(b, err) + } + + return msgs +} + +var chainID = "test" + +func makeState(nVals, height int) (sm.State, dbm.DB, map[string]types0.PrivValidator) { + vals := make([]types0.GenesisValidator, nVals) + privVals := make(map[string]types0.PrivValidator, nVals) + for i := 0; i < nVals; i++ { + secret := []byte(fmt.Sprintf("test%d", i)) + pk := ed25519.GenPrivKeyFromSecret(secret) + valAddr := pk.PubKey().Address() + vals[i] = types0.GenesisValidator{ + Address: valAddr, + PubKey: pk.PubKey(), + Power: 1000, + Name: fmt.Sprintf("test%d", i), + } + privVals[valAddr.String()] = types0.NewMockPVWithParams(pk, false, false) + } + s, _ := sm.MakeGenesisState(&types0.GenesisDoc{ + ChainID: chainID, + Validators: vals, + AppHash: nil, + }) + + stateDB := dbm.NewMemDB() + stateStore := sm.NewStore(stateDB, sm.StoreOptions{ + DiscardABCIResponses: false, + }) + if err := stateStore.Save(s); err != nil { + panic(err) + } + + for i := 1; i < height; i++ { + s.LastBlockHeight++ + s.LastValidators = s.Validators.Copy() + if err := stateStore.Save(s); err != nil { + panic(err) + } + } + + return s, stateDB, privVals +} + +func makeValidCommit( + height int64, + blockID types0.BlockID, + vals *types0.ValidatorSet, + privVals map[string]types0.PrivValidator, +) (*types0.Commit, error) { + sigs := make([]types0.CommitSig, 0) + for i := 0; i < vals.Size(); i++ { + _, val := vals.GetByIndex(int32(i)) + vote, err := types0.MakeVote(height, blockID, vals, privVals[val.Address.String()], chainID, time.Now()) + if err != nil { + return nil, err + } + sigs = append(sigs, vote.CommitSig()) + } + return types0.NewCommit(height, 0, blockID, sigs), nil +} + +func makeBlockID(hash []byte, partSetSize uint32, partSetHash []byte) types0.BlockID { + var ( + h = make([]byte, tmhash.Size) + psH = make([]byte, tmhash.Size) + ) + copy(h, hash) + copy(psH, partSetHash) + return types0.BlockID{ + Hash: h, + PartSetHeader: types0.PartSetHeader{ + Total: partSetSize, + Hash: psH, + }, + } +} diff --git a/app/benchmarks/benchmark_msg_send_test.go b/app/benchmarks/benchmark_msg_send_test.go new file mode 100644 index 0000000000..91557e39f9 --- /dev/null +++ b/app/benchmarks/benchmark_msg_send_test.go @@ -0,0 +1,333 @@ +//go:build bench_abci_methods + +package benchmarks_test + +import ( + "fmt" + "testing" + "time" + + "github.com/celestiaorg/celestia-app/v3/app" + "github.com/celestiaorg/celestia-app/v3/app/encoding" + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" + "github.com/celestiaorg/celestia-app/v3/pkg/user" + testutil "github.com/celestiaorg/celestia-app/v3/test/util" + "github.com/celestiaorg/celestia-app/v3/test/util/testfactory" + "github.com/celestiaorg/celestia-app/v3/test/util/testnode" + sdk "github.com/cosmos/cosmos-sdk/types" + banktypes "github.com/cosmos/cosmos-sdk/x/bank/types" + "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/abci/types" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" + "github.com/tendermint/tendermint/proto/tendermint/version" +) + +func BenchmarkCheckTx_MsgSend_1(b *testing.B) { + testApp, rawTxs := generateMsgSendTransactions(b, 1) + testApp.Commit() + + checkTxRequest := types.RequestCheckTx{ + Tx: rawTxs[0], + Type: types.CheckTxType_New, + } + + b.ResetTimer() + resp := testApp.CheckTx(checkTxRequest) + b.StopTimer() + require.Equal(b, uint32(0), resp.Code) + require.Equal(b, "", resp.Codespace) + b.ReportMetric(float64(resp.GasUsed), "gas_used") +} + +func BenchmarkCheckTx_MsgSend_8MB(b *testing.B) { + testApp, rawTxs := generateMsgSendTransactions(b, 31645) + testApp.Commit() + + var totalGas int64 + b.ResetTimer() + for _, tx := range rawTxs { + checkTxRequest := types.RequestCheckTx{ + Tx: tx, + Type: types.CheckTxType_New, + } + b.StartTimer() + resp := testApp.CheckTx(checkTxRequest) + b.StopTimer() + require.Equal(b, uint32(0), resp.Code) + require.Equal(b, "", resp.Codespace) + totalGas += resp.GasUsed + } + + b.StopTimer() + b.ReportMetric(float64(totalGas), "total_gas_used") +} + +func BenchmarkDeliverTx_MsgSend_1(b *testing.B) { + testApp, rawTxs := generateMsgSendTransactions(b, 1) + + deliverTxRequest := types.RequestDeliverTx{ + Tx: rawTxs[0], + } + + b.ResetTimer() + resp := testApp.DeliverTx(deliverTxRequest) + b.StopTimer() + require.Equal(b, uint32(0), resp.Code) + require.Equal(b, "", resp.Codespace) + b.ReportMetric(float64(resp.GasUsed), "gas_used") +} + +func BenchmarkDeliverTx_MsgSend_8MB(b *testing.B) { + testApp, rawTxs := generateMsgSendTransactions(b, 31645) + + var totalGas int64 + b.ResetTimer() + for _, tx := range rawTxs { + deliverTxRequest := types.RequestDeliverTx{ + Tx: tx, + } + b.StartTimer() + resp := testApp.DeliverTx(deliverTxRequest) + b.StopTimer() + require.Equal(b, uint32(0), resp.Code) + require.Equal(b, "", resp.Codespace) + totalGas += resp.GasUsed + } + b.StopTimer() + b.ReportMetric(float64(totalGas), "total_gas_used") +} + +func BenchmarkPrepareProposal_MsgSend_1(b *testing.B) { + testApp, rawTxs := generateMsgSendTransactions(b, 1) + + prepareProposalRequest := types.RequestPrepareProposal{ + BlockData: &tmproto.Data{ + Txs: rawTxs, + }, + ChainId: testApp.GetChainID(), + Height: 10, + } + + b.ResetTimer() + resp := testApp.PrepareProposal(prepareProposalRequest) + b.StopTimer() + require.GreaterOrEqual(b, len(resp.BlockData.Txs), 1) + b.ReportMetric(float64(calculateTotalGasUsed(testApp, resp.BlockData.Txs)), "total_gas_used") +} + +func BenchmarkPrepareProposal_MsgSend_8MB(b *testing.B) { + // a full 8mb block equals to around 31645 msg send transactions. + // using 31645 to let prepare proposal choose the maximum + testApp, rawTxs := generateMsgSendTransactions(b, 31645) + + blockData := &tmproto.Data{ + Txs: rawTxs, + } + prepareProposalRequest := types.RequestPrepareProposal{ + BlockData: blockData, + ChainId: testApp.GetChainID(), + Height: 10, + } + + b.ResetTimer() + resp := testApp.PrepareProposal(prepareProposalRequest) + b.StopTimer() + require.GreaterOrEqual(b, len(resp.BlockData.Txs), 1) + b.ReportMetric(float64(len(resp.BlockData.Txs)), "number_of_transactions") + b.ReportMetric(calculateBlockSizeInMb(resp.BlockData.Txs), "block_size(mb)") + b.ReportMetric(float64(calculateTotalGasUsed(testApp, resp.BlockData.Txs)), "total_gas_used") +} + +func BenchmarkProcessProposal_MsgSend_1(b *testing.B) { + testApp, rawTxs := generateMsgSendTransactions(b, 1) + + blockData := &tmproto.Data{ + Txs: rawTxs, + } + prepareProposalRequest := types.RequestPrepareProposal{ + BlockData: blockData, + ChainId: testApp.GetChainID(), + Height: 10, + } + prepareProposalResponse := testApp.PrepareProposal(prepareProposalRequest) + require.GreaterOrEqual(b, len(prepareProposalResponse.BlockData.Txs), 1) + + processProposalRequest := types.RequestProcessProposal{ + BlockData: prepareProposalResponse.BlockData, + Header: tmproto.Header{ + Height: 1, + DataHash: prepareProposalResponse.BlockData.Hash, + ChainID: testutil.ChainID, + Version: version.Consensus{ + App: testApp.AppVersion(), + }, + }, + } + + b.ResetTimer() + resp := testApp.ProcessProposal(processProposalRequest) + b.StopTimer() + require.Equal(b, types.ResponseProcessProposal_ACCEPT, resp.Result) + + b.ReportMetric(float64(calculateTotalGasUsed(testApp, prepareProposalResponse.BlockData.Txs)), "total_gas_used") +} + +func BenchmarkProcessProposal_MsgSend_8MB(b *testing.B) { + // a full 8mb block equals to around 31645 msg send transactions. + // using 31645 to let prepare proposal choose the maximum + testApp, rawTxs := generateMsgSendTransactions(b, 31645) + + blockData := &tmproto.Data{ + Txs: rawTxs, + } + prepareProposalRequest := types.RequestPrepareProposal{ + BlockData: blockData, + ChainId: testApp.GetChainID(), + Height: 10, + } + prepareProposalResponse := testApp.PrepareProposal(prepareProposalRequest) + require.GreaterOrEqual(b, len(prepareProposalResponse.BlockData.Txs), 1) + + b.ReportMetric(float64(len(prepareProposalResponse.BlockData.Txs)), "number_of_transactions") + b.ReportMetric(calculateBlockSizeInMb(prepareProposalResponse.BlockData.Txs), "block_size_(mb)") + b.ReportMetric(float64(calculateTotalGasUsed(testApp, prepareProposalResponse.BlockData.Txs)), "total_gas_used") + + processProposalRequest := types.RequestProcessProposal{ + BlockData: prepareProposalResponse.BlockData, + Header: tmproto.Header{ + Height: 10, + DataHash: prepareProposalResponse.BlockData.Hash, + ChainID: testutil.ChainID, + Version: version.Consensus{ + App: testApp.AppVersion(), + }, + }, + } + + b.ResetTimer() + resp := testApp.ProcessProposal(processProposalRequest) + b.StopTimer() + require.Equal(b, types.ResponseProcessProposal_ACCEPT, resp.Result) + + b.ReportMetric(float64(calculateTotalGasUsed(testApp, prepareProposalResponse.BlockData.Txs)), "total_gas_used") +} + +func BenchmarkProcessProposal_MsgSend_8MB_Find_Half_Sec(b *testing.B) { + targetTimeLowerBound := 0.499 + targetTimeUpperBound := 0.511 + numberOfTransaction := 5500 + testApp, rawTxs := generateMsgSendTransactions(b, numberOfTransaction) + start := 0 + end := numberOfTransaction + segment := end - start + for { + if segment == 1 { + break + } + + prepareProposalRequest := types.RequestPrepareProposal{ + BlockData: &tmproto.Data{ + Txs: rawTxs[start:end], + }, + ChainId: testApp.GetChainID(), + Height: 10, + } + prepareProposalResponse := testApp.PrepareProposal(prepareProposalRequest) + require.GreaterOrEqual(b, len(prepareProposalResponse.BlockData.Txs), 1) + + processProposalRequest := types.RequestProcessProposal{ + BlockData: prepareProposalResponse.BlockData, + Header: tmproto.Header{ + Height: 10, + DataHash: prepareProposalResponse.BlockData.Hash, + ChainID: testutil.ChainID, + Version: version.Consensus{ + App: testApp.AppVersion(), + }, + }, + } + + startTime := time.Now() + resp := testApp.ProcessProposal(processProposalRequest) + endTime := time.Now() + require.Equal(b, types.ResponseProcessProposal_ACCEPT, resp.Result) + + timeElapsed := float64(endTime.Sub(startTime).Nanoseconds()) / 1e9 + + switch { + case timeElapsed < targetTimeLowerBound: + newEnd := end + segment/2 + if newEnd > len(rawTxs) { + newEnd = len(rawTxs) + } + end = newEnd + segment = end - start + if segment <= 1 { + break + } + continue + case timeElapsed > targetTimeUpperBound: + newEnd := end / 2 + if newEnd <= start { + break + } + end = newEnd + segment = end - start + continue + default: + b.ReportMetric(timeElapsed, fmt.Sprintf("elapsedTime(s)_%d", end-start)) + } + break + } +} + +// generateMsgSendTransactions creates a test app then generates a number +// of valid msg send transactions. +func generateMsgSendTransactions(b *testing.B, count int) (*app.App, [][]byte) { + account := "test" + testApp, kr := testutil.SetupTestAppWithGenesisValSetAndMaxSquareSize(app.DefaultConsensusParams(), 128, account) + addr := testfactory.GetAddress(kr, account) + enc := encoding.MakeConfig(app.ModuleEncodingRegisters...) + acc := testutil.DirectQueryAccount(testApp, addr) + signer, err := user.NewSigner(kr, enc.TxConfig, testutil.ChainID, appconsts.LatestVersion, user.NewAccount(account, acc.GetAccountNumber(), acc.GetSequence())) + require.NoError(b, err) + rawTxs := make([][]byte, 0, count) + for i := 0; i < count; i++ { + msg := banktypes.NewMsgSend( + addr, + testnode.RandomAddress().(sdk.AccAddress), + sdk.NewCoins(sdk.NewInt64Coin(appconsts.BondDenom, 10)), + ) + rawTx, err := signer.CreateTx([]sdk.Msg{msg}, user.SetGasLimit(1000000), user.SetFee(10)) + require.NoError(b, err) + rawTxs = append(rawTxs, rawTx) + err = signer.IncrementSequence(account) + require.NoError(b, err) + } + return testApp, rawTxs +} + +// megabyte the number of bytes in a megabyte +const megabyte = 1048576 + +// calculateBlockSizeInMb returns the block size in mb given a set +// of raw transactions. +func calculateBlockSizeInMb(txs [][]byte) float64 { + numberOfBytes := 0 + for _, tx := range txs { + numberOfBytes += len(tx) + } + mb := float64(numberOfBytes) / megabyte + return mb +} + +// calculateTotalGasUsed simulates the provided transactions and returns the +// total gas used by all of them +func calculateTotalGasUsed(testApp *app.App, txs [][]byte) uint64 { + var totalGas uint64 + for _, tx := range txs { + gasInfo, _, _ := testApp.Simulate(tx) + totalGas += gasInfo.GasUsed + } + return totalGas +} diff --git a/app/benchmarks/benchmark_pfb_test.go b/app/benchmarks/benchmark_pfb_test.go new file mode 100644 index 0000000000..274dbf8353 --- /dev/null +++ b/app/benchmarks/benchmark_pfb_test.go @@ -0,0 +1,381 @@ +//go:build bench_abci_methods + +package benchmarks_test + +import ( + "fmt" + "testing" + "time" + + "github.com/tendermint/tendermint/crypto" + "github.com/tendermint/tendermint/libs/log" + + "github.com/celestiaorg/celestia-app/v3/app" + "github.com/celestiaorg/celestia-app/v3/app/encoding" + "github.com/celestiaorg/celestia-app/v3/pkg/appconsts" + "github.com/celestiaorg/celestia-app/v3/pkg/user" + testutil "github.com/celestiaorg/celestia-app/v3/test/util" + "github.com/celestiaorg/celestia-app/v3/test/util/testfactory" + "github.com/celestiaorg/go-square/v2/share" + blobtx "github.com/celestiaorg/go-square/v2/tx" + "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/abci/types" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" + "github.com/tendermint/tendermint/proto/tendermint/version" +) + +func init() { + testutil.TestAppLogger = log.NewNopLogger() +} + +func BenchmarkCheckTx_PFB_Multi(b *testing.B) { + testCases := []struct { + blobSize int + }{ + {blobSize: 300}, + {blobSize: 500}, + {blobSize: 1000}, + {blobSize: 5000}, + {blobSize: 10_000}, + {blobSize: 50_000}, + {blobSize: 100_000}, + {blobSize: 200_000}, + {blobSize: 300_000}, + {blobSize: 400_000}, + {blobSize: 500_000}, + {blobSize: 1_000_000}, + {blobSize: 2_000_000}, + {blobSize: 3_000_000}, + {blobSize: 4_000_000}, + {blobSize: 5_000_000}, + {blobSize: 6_000_000}, + } + for _, testCase := range testCases { + b.Run(fmt.Sprintf("%d bytes", testCase.blobSize), func(b *testing.B) { + benchmarkCheckTxPFB(b, testCase.blobSize) + }) + } +} + +func benchmarkCheckTxPFB(b *testing.B, size int) { + testApp, rawTxs := generatePayForBlobTransactions(b, 1, size) + testApp.Commit() + + checkTxRequest := types.RequestCheckTx{ + Tx: rawTxs[0], + Type: types.CheckTxType_New, + } + + b.ResetTimer() + resp := testApp.CheckTx(checkTxRequest) + b.StopTimer() + require.Equal(b, uint32(0), resp.Code) + require.Equal(b, "", resp.Codespace) + b.ReportMetric(float64(resp.GasUsed), "gas_used") + b.ReportMetric(float64(len(rawTxs[0])), "transaction_size(byte)") +} + +func BenchmarkDeliverTx_PFB_Multi(b *testing.B) { + testCases := []struct { + blobSize int + }{ + {blobSize: 300}, + {blobSize: 500}, + {blobSize: 1000}, + {blobSize: 5000}, + {blobSize: 10_000}, + {blobSize: 50_000}, + {blobSize: 100_000}, + {blobSize: 200_000}, + {blobSize: 300_000}, + {blobSize: 400_000}, + {blobSize: 500_000}, + {blobSize: 1_000_000}, + {blobSize: 2_000_000}, + {blobSize: 3_000_000}, + {blobSize: 4_000_000}, + {blobSize: 5_000_000}, + {blobSize: 6_000_000}, + } + for _, testCase := range testCases { + b.Run(fmt.Sprintf("%d bytes", testCase.blobSize), func(b *testing.B) { + benchmarkDeliverTxPFB(b, testCase.blobSize) + }) + } +} + +func benchmarkDeliverTxPFB(b *testing.B, size int) { + testApp, rawTxs := generatePayForBlobTransactions(b, 1, size) + + blobTx, ok, err := blobtx.UnmarshalBlobTx(rawTxs[0]) + require.NoError(b, err) + require.True(b, ok) + + deliverTxRequest := types.RequestDeliverTx{ + Tx: blobTx.Tx, + } + + b.ResetTimer() + resp := testApp.DeliverTx(deliverTxRequest) + b.StopTimer() + require.Equal(b, uint32(0), resp.Code) + require.Equal(b, "", resp.Codespace) + b.ReportMetric(float64(resp.GasUsed), "gas_used") + b.ReportMetric(float64(len(rawTxs[0])), "transaction_size(byte)") +} + +func BenchmarkPrepareProposal_PFB_Multi(b *testing.B) { + testCases := []struct { + numberOfTransactions, blobSize int + }{ + {numberOfTransactions: 15_000, blobSize: 300}, + {numberOfTransactions: 10_000, blobSize: 500}, + {numberOfTransactions: 6_000, blobSize: 1000}, + {numberOfTransactions: 3_000, blobSize: 5000}, + {numberOfTransactions: 1_000, blobSize: 10_000}, + {numberOfTransactions: 500, blobSize: 50_000}, + {numberOfTransactions: 100, blobSize: 100_000}, + {numberOfTransactions: 100, blobSize: 200_000}, + {numberOfTransactions: 50, blobSize: 300_000}, + {numberOfTransactions: 50, blobSize: 400_000}, + {numberOfTransactions: 30, blobSize: 500_000}, + {numberOfTransactions: 10, blobSize: 1_000_000}, + {numberOfTransactions: 5, blobSize: 2_000_000}, + {numberOfTransactions: 3, blobSize: 3_000_000}, + {numberOfTransactions: 3, blobSize: 4_000_000}, + {numberOfTransactions: 2, blobSize: 5_000_000}, + {numberOfTransactions: 2, blobSize: 6_000_000}, + } + for _, testCase := range testCases { + b.Run(fmt.Sprintf("%d transactions of %d bytes", testCase.numberOfTransactions, testCase.blobSize), func(b *testing.B) { + benchmarkPrepareProposalPFB(b, testCase.numberOfTransactions, testCase.blobSize) + }) + } +} + +func benchmarkPrepareProposalPFB(b *testing.B, count, size int) { + testApp, rawTxs := generatePayForBlobTransactions(b, count, size) + + blockData := &tmproto.Data{ + Txs: rawTxs, + } + prepareProposalRequest := types.RequestPrepareProposal{ + BlockData: blockData, + ChainId: testApp.GetChainID(), + Height: 10, + } + + b.ResetTimer() + prepareProposalResponse := testApp.PrepareProposal(prepareProposalRequest) + b.StopTimer() + require.GreaterOrEqual(b, len(prepareProposalResponse.BlockData.Txs), 1) + b.ReportMetric(float64(b.Elapsed().Nanoseconds()), "prepare_proposal_time(ns)") + b.ReportMetric(float64(len(prepareProposalResponse.BlockData.Txs)), "number_of_transactions") + b.ReportMetric(float64(len(rawTxs[0])), "transactions_size(byte)") + b.ReportMetric(calculateBlockSizeInMb(prepareProposalResponse.BlockData.Txs), "block_size(mb)") + b.ReportMetric(float64(calculateTotalGasUsed(testApp, rawTxs)), "total_gas_used") +} + +func BenchmarkProcessProposal_PFB_Multi(b *testing.B) { + testCases := []struct { + numberOfTransactions, blobSize int + }{ + {numberOfTransactions: 15_000, blobSize: 300}, + {numberOfTransactions: 10_000, blobSize: 500}, + {numberOfTransactions: 6_000, blobSize: 1000}, + {numberOfTransactions: 3_000, blobSize: 5000}, + {numberOfTransactions: 1_000, blobSize: 10_000}, + {numberOfTransactions: 500, blobSize: 50_000}, + {numberOfTransactions: 100, blobSize: 100_000}, + {numberOfTransactions: 100, blobSize: 200_000}, + {numberOfTransactions: 50, blobSize: 300_000}, + {numberOfTransactions: 50, blobSize: 400_000}, + {numberOfTransactions: 30, blobSize: 500_000}, + {numberOfTransactions: 10, blobSize: 1_000_000}, + {numberOfTransactions: 5, blobSize: 2_000_000}, + {numberOfTransactions: 3, blobSize: 3_000_000}, + {numberOfTransactions: 3, blobSize: 4_000_000}, + {numberOfTransactions: 2, blobSize: 5_000_000}, + {numberOfTransactions: 2, blobSize: 6_000_000}, + } + for _, testCase := range testCases { + b.Run(fmt.Sprintf("%d transactions of %d bytes", testCase.numberOfTransactions, testCase.blobSize), func(b *testing.B) { + benchmarkProcessProposalPFB(b, testCase.numberOfTransactions, testCase.blobSize) + }) + } +} + +func benchmarkProcessProposalPFB(b *testing.B, count, size int) { + testApp, rawTxs := generatePayForBlobTransactions(b, count, size) + + blockData := &tmproto.Data{ + Txs: rawTxs, + } + prepareProposalRequest := types.RequestPrepareProposal{ + BlockData: blockData, + ChainId: testApp.GetChainID(), + Height: 10, + } + + prepareProposalResponse := testApp.PrepareProposal(prepareProposalRequest) + require.GreaterOrEqual(b, len(prepareProposalResponse.BlockData.Txs), 1) + + processProposalRequest := types.RequestProcessProposal{ + BlockData: prepareProposalResponse.BlockData, + Header: tmproto.Header{ + Height: 10, + DataHash: prepareProposalResponse.BlockData.Hash, + ChainID: testutil.ChainID, + Version: version.Consensus{ + App: testApp.AppVersion(), + }, + }, + } + + b.ResetTimer() + resp := testApp.ProcessProposal(processProposalRequest) + b.StopTimer() + require.Equal(b, types.ResponseProcessProposal_ACCEPT, resp.Result) + + b.ReportMetric(float64(b.Elapsed().Nanoseconds()), "process_proposal_time(ns)") + b.ReportMetric(float64(len(prepareProposalResponse.BlockData.Txs)), "number_of_transactions") + b.ReportMetric(float64(len(rawTxs[0])), "transactions_size(byte)") + b.ReportMetric(calculateBlockSizeInMb(prepareProposalResponse.BlockData.Txs), "block_size(mb)") + b.ReportMetric(float64(calculateTotalGasUsed(testApp, rawTxs)), "total_gas_used") +} + +func BenchmarkProcessProposal_PFB_Half_Second(b *testing.B) { + testCases := []struct { + numberOfTransactions, blobSize int + }{ + {numberOfTransactions: 11_000, blobSize: 50}, + {numberOfTransactions: 11_000, blobSize: 100}, + {numberOfTransactions: 11_000, blobSize: 200}, + {numberOfTransactions: 11_000, blobSize: 300}, + {numberOfTransactions: 11_000, blobSize: 400}, + {numberOfTransactions: 7000, blobSize: 500}, + {numberOfTransactions: 7000, blobSize: 600}, + {numberOfTransactions: 5000, blobSize: 1_000}, + {numberOfTransactions: 5000, blobSize: 1200}, + {numberOfTransactions: 5000, blobSize: 1500}, + {numberOfTransactions: 5000, blobSize: 1800}, + {numberOfTransactions: 5000, blobSize: 2000}, + } + for _, testCase := range testCases { + b.Run(fmt.Sprintf("%d transactions of %d bytes", testCase.numberOfTransactions, testCase.blobSize), func(b *testing.B) { + benchmarkProcessProposalPFBHalfSecond(b, testCase.numberOfTransactions, testCase.blobSize) + }) + } +} + +func benchmarkProcessProposalPFBHalfSecond(b *testing.B, count, size int) { + testApp, rawTxs := generatePayForBlobTransactions(b, count, size) + + targetTimeLowerBound := 0.499 + targetTimeUpperBound := 0.511 + + start := 0 + end := count + segment := end - start + maxIterations := 100000 + iterations := 0 + for { + iterations++ + if iterations >= maxIterations { + b.Errorf("Maximum iterations reached without achieving target processing time") + break + } + if segment == 1 { + break + } + + prepareProposalRequest := types.RequestPrepareProposal{ + BlockData: &tmproto.Data{ + Txs: rawTxs[start:end], + }, + ChainId: testApp.GetChainID(), + Height: 10, + } + prepareProposalResponse := testApp.PrepareProposal(prepareProposalRequest) + require.GreaterOrEqual(b, len(prepareProposalResponse.BlockData.Txs), 1) + + processProposalRequest := types.RequestProcessProposal{ + BlockData: prepareProposalResponse.BlockData, + Header: tmproto.Header{ + Height: 10, + DataHash: prepareProposalResponse.BlockData.Hash, + ChainID: testutil.ChainID, + Version: version.Consensus{ + App: testApp.AppVersion(), + }, + }, + } + + startTime := time.Now() + resp := testApp.ProcessProposal(processProposalRequest) + endTime := time.Now() + require.Equal(b, types.ResponseProcessProposal_ACCEPT, resp.Result) + + timeElapsed := float64(endTime.Sub(startTime).Nanoseconds()) / 1e9 + + switch { + case timeElapsed < targetTimeLowerBound: + newEnd := end + segment/2 + if newEnd > len(rawTxs) { + newEnd = len(rawTxs) + } + end = newEnd + segment = end - start + if segment <= 1 { + break + } + continue + case timeElapsed > targetTimeUpperBound: + newEnd := end / 2 + if newEnd <= start { + break + } + end = newEnd + segment = end - start + continue + default: + b.ReportMetric( + timeElapsed, + fmt.Sprintf( + "processProposalTime(s)_%d_%d_%f", + end-start, + size, + calculateBlockSizeInMb(prepareProposalResponse.BlockData.Txs[start:end]), + ), + ) + } + break + } +} + +// generatePayForBlobTransactions creates a test app then generates a number +// of valid PFB transactions. +func generatePayForBlobTransactions(b *testing.B, count int, size int) (*app.App, [][]byte) { + account := "test" + testApp, kr := testutil.SetupTestAppWithGenesisValSetAndMaxSquareSize(app.DefaultConsensusParams(), 128, account) + addr := testfactory.GetAddress(kr, account) + enc := encoding.MakeConfig(app.ModuleEncodingRegisters...) + acc := testutil.DirectQueryAccount(testApp, addr) + accountSequence := acc.GetSequence() + signer, err := user.NewSigner(kr, enc.TxConfig, testutil.ChainID, appconsts.LatestVersion, user.NewAccount(account, acc.GetAccountNumber(), acc.GetSequence())) + require.NoError(b, err) + + rawTxs := make([][]byte, 0, count) + randomBytes := crypto.CRandBytes(size) + blob, err := share.NewBlob(share.RandomNamespace(), randomBytes, 1, acc.GetAddress().Bytes()) + require.NoError(b, err) + for i := 0; i < count; i++ { + tx, _, err := signer.CreatePayForBlobs(account, []*share.Blob{blob}, user.SetGasLimit(2549760000), user.SetFee(10000)) + require.NoError(b, err) + rawTxs = append(rawTxs, tx) + accountSequence++ + err = signer.SetSequence(account, accountSequence) + require.NoError(b, err) + } + return testApp, rawTxs +} diff --git a/app/benchmarks/results.md b/app/benchmarks/results.md new file mode 100644 index 0000000000..9562a6a340 --- /dev/null +++ b/app/benchmarks/results.md @@ -0,0 +1,839 @@ + +# Benchmark results + +This document contains the results of the benchmarks defined under `app/benchmarks`. + +The benchmarks were run on a Macbook Pro M3 MAX with 16 cores 48GB RAM. + +The benchmarks will be run using an in memory DB, then a local db, goleveldb. + +## In memory DB benchmarks + +### `sendMsg` benchmarks + +#### CheckTx + +A single `checkTx` of a `sendMsg` message takes 0.0003585 **ns** to execute. And it uses 74374 gas. + +The transactions in an `8mb` block containing 31645 `sendMsg` messages take 6,29 s (6293858682 ns) to run `checkTx` on all of them. The total gas used is 1884371034 gas. + +#### DeliverTx + +A single `deliverTx` of a `sendMsg` message takes 0.0002890 **ns** to execute. And it uses 103251 gas. + +The transactions in an `8mb` block containing 31645 `sendMsg` messages take 7,56 s (7564111078 ns) to run `deliverTx` on all of them. The total gas used is 2801272121 gas. + +#### PrepareProposal + +A single `prepareProposal` of a `sendMsg` message takes 0.0002801 **ns** to execute. And it uses 101110 gas. + +An `8mb` block containing 31645 `sendMsg` messages takes 5,04 s (5049140917 ns) to execute. The total gas used 1843040790 gas. + +#### ProcessProposal + +A single `processProposal` of a `sendMsg` message takes 0.0002313 **ns** to execute. And it uses 101110 gas. + +An `8mb` block containing 31645 `sendMsg` messages takes 5,17 s (5179850250 ns) to execute. The total gas used 1,843,040,790 gas. + +For the processing time of a block full of `sendMsg`, we benchmarked how much time they take depending on the number of transactions, and we have the following results: + +| Number of Transactions | ElapsedTime(s) | Number of Transactions | ElapsedTime(s) | +|------------------------|----------------|------------------------|----------------| +| 1650 | 0.2494 | 1670 | 0.2594 | +| 1690 | 0.2628 | 1739 | 0.2723 | +| 1761 | 0.2732 | 1782 | 0.2770 | +| 1856 | 0.2878 | 1878 | 0.2976 | +| 1901 | 0.2990 | 1956 | 0.3023 | +| 1980 | 0.3076 | 2004 | 0.3232 | +| 2062 | 0.3252 | 2088 | 0.3257 | +| 2112 | 0.3326 | 2138 | 0.3417 | +| 2200 | 0.3398 | 2227 | 0.3495 | +| 2254 | 0.3545 | 2319 | 0.3688 | +| 2349 | 0.3684 | 2376 | 0.3771 | +| 2475 | 0.3972 | 2505 | 0.3928 | +| 2535 | 0.4080 | 2608 | 0.4098 | +| 2641 | 0.4123 | 2673 | 0.4135 | +| 2750 | 0.4614 | 2784 | 0.4333 | +| 2817 | 0.4537 | 2851 | 0.4530 | +| 2934 | 0.4633 | 2970 | 0.4623 | +| 3006 | 0.4863 | 3093 | 0.4821 | +| 3132 | 0.4888 | 3168 | 0.4962 | +| 3207 | 0.5058 | 3300 | 0.5119 | +| 3340 | 0.5275 | 3381 | 0.5280 | +| 3478 | 0.5441 | 3523 | 0.5473 | +| 3564 | 0.5546 | 3712 | 0.5743 | +| 3757 | 0.6081 | 3802 | 0.5970 | +| 3912 | 0.6093 | 3961 | 0.6125 | +| 4009 | 0.6329 | 4125 | 0.6663 | +| 4176 | 0.6395 | 4225 | 0.6615 | +| 4276 | 0.6844 | 4401 | 0.7190 | +| 4455 | 0.6943 | 4509 | 0.7006 | +| 4639 | 0.7219 | 4698 | 0.7365 | +| 4752 | 0.7340 | 5500 | 0.8489 | + +### `PFB` benchmarks + +#### CheckTx: `BenchmarkCheckTx_PFB_Multi` + +Benchmarks of `CheckTx` for a single PFB with different sizes: + +| Benchmark Name | Time (ns/op) | Gas Used | Transaction Size (Bytes) | Transaction Size (MB) | +|---------------------------------------------|--------------|----------|--------------------------|-----------------------| +| BenchmarkCheckTx_PFB_Multi/300_bytes-16 | 0.0003121 ns | 74,664 | 703 | 0.000703 MB | +| BenchmarkCheckTx_PFB_Multi/500_bytes-16 | 0.0003392 ns | 74,664 | 903 | 0.000903 MB | +| BenchmarkCheckTx_PFB_Multi/1000_bytes-16 | 0.0002797 ns | 74,664 | 1,403 | 0.001403 MB | +| BenchmarkCheckTx_PFB_Multi/5000_bytes-16 | 0.0002818 ns | 74,664 | 5,403 | 0.005403 MB | +| BenchmarkCheckTx_PFB_Multi/10000_bytes-16 | 0.0003094 ns | 74,664 | 10,403 | 0.010403 MB | +| BenchmarkCheckTx_PFB_Multi/50000_bytes-16 | 0.0004127 ns | 74,674 | 50,406 | 0.050406 MB | +| BenchmarkCheckTx_PFB_Multi/100000_bytes-16 | 0.0004789 ns | 74,674 | 100,406 | 0.100406 MB | +| BenchmarkCheckTx_PFB_Multi/200000_bytes-16 | 0.0006958 ns | 74,674 | 200,406 | 0.200406 MB | +| BenchmarkCheckTx_PFB_Multi/300000_bytes-16 | 0.0008678 ns | 74,674 | 300,406 | 0.300406 MB | +| BenchmarkCheckTx_PFB_Multi/400000_bytes-16 | 0.001076 ns | 74,674 | 400,406 | 0.400406 MB | +| BenchmarkCheckTx_PFB_Multi/500000_bytes-16 | 0.001307 ns | 74,674 | 500,406 | 0.500406 MB | +| BenchmarkCheckTx_PFB_Multi/1000000_bytes-16 | 0.002291 ns | 74,674 | 1,000,406 | 1.000406 MB | +| BenchmarkCheckTx_PFB_Multi/2000000_bytes-16 | 0.005049 ns | 74,674 | 2,000,406 | 2.000406 MB | +| BenchmarkCheckTx_PFB_Multi/3000000_bytes-16 | 0.006911 ns | 74,684 | 3,000,409 | 3.000409 MB | +| BenchmarkCheckTx_PFB_Multi/4000000_bytes-16 | 0.008246 ns | 74,684 | 4,000,409 | 4.000409 MB | +| BenchmarkCheckTx_PFB_Multi/5000000_bytes-16 | 0.01127 ns | 74,684 | 5,000,409 | 5.000409 MB | +| BenchmarkCheckTx_PFB_Multi/6000000_bytes-16 | 0.01316 ns | 74,684 | 6,000,409 | 6.000409 MB | + +#### DeliverTx: `BenchmarkDeliverTx_PFB_Multi` + +Benchmarks of `DeliverTx` for a single PFB with different sizes: + +| Benchmark Name | Time (ns/op) | Gas Used | Transaction Size (Bytes) | Transaction Size (MB) | +|-----------------------------------------------|--------------|------------|--------------------------|-----------------------| +| BenchmarkDeliverTx_PFB_Multi/300_bytes-16 | 0.0002718 ns | 77,682 | 703 | 0.000703 MB | +| BenchmarkDeliverTx_PFB_Multi/500_bytes-16 | 0.0002574 ns | 81,778 | 903 | 0.000903 MB | +| BenchmarkDeliverTx_PFB_Multi/1000_bytes-16 | 0.0002509 ns | 85,874 | 1,403 | 0.001403 MB | +| BenchmarkDeliverTx_PFB_Multi/5000_bytes-16 | 0.0002755 ns | 118,642 | 5,403 | 0.005403 MB | +| BenchmarkDeliverTx_PFB_Multi/10000_bytes-16 | 0.0002726 ns | 159,602 | 10,403 | 0.010403 MB | +| BenchmarkDeliverTx_PFB_Multi/50000_bytes-16 | 0.0002795 ns | 499,580 | 50,406 | 0.050406 MB | +| BenchmarkDeliverTx_PFB_Multi/100000_bytes-16 | 0.0002488 ns | 925,564 | 100,406 | 0.100406 MB | +| BenchmarkDeliverTx_PFB_Multi/200000_bytes-16 | 0.0002487 ns | 1,773,436 | 200,406 | 0.200406 MB | +| BenchmarkDeliverTx_PFB_Multi/300000_bytes-16 | 0.0002887 ns | 2,625,404 | 300,406 | 0.300406 MB | +| BenchmarkDeliverTx_PFB_Multi/400000_bytes-16 | 0.0002810 ns | 3,473,276 | 400,406 | 0.400406 MB | +| BenchmarkDeliverTx_PFB_Multi/500000_bytes-16 | 0.0002616 ns | 4,325,244 | 500,406 | 0.500406 MB | +| BenchmarkDeliverTx_PFB_Multi/1000000_bytes-16 | 0.0003983 ns | 8,572,796 | 1,000,406 | 1.000406 MB | +| BenchmarkDeliverTx_PFB_Multi/2000000_bytes-16 | 0.0003368 ns | 17,071,996 | 2,000,406 | 2.000406 MB | +| BenchmarkDeliverTx_PFB_Multi/3000000_bytes-16 | 0.0005770 ns | 25,571,206 | 3,000,409 | 3.000409 MB | +| BenchmarkDeliverTx_PFB_Multi/4000000_bytes-16 | 0.0003752 ns | 34,066,310 | 4,000,409 | 4.000409 MB | +| BenchmarkDeliverTx_PFB_Multi/5000000_bytes-16 | 0.0003788 ns | 42,565,510 | 5,000,409 | 5.000409 MB | +| BenchmarkDeliverTx_PFB_Multi/6000000_bytes-16 | 0.0003975 ns | 51,064,710 | 6,000,409 | 6.000409 MB | + +#### PrepareProposal: `BenchmarkPrepareProposal_PFB_Multi` + +The benchmarks for `PrepareProposal` for 8mb blocks containing PFBs of different sizes: + +| Benchmark Name | Block Size (MB) | Number of Transactions | Prepare Proposal Time (s) | Total Gas Used | Transaction Size (Bytes) | Transaction Size (MB) | +|------------------------------------------------------------------------|-----------------|------------------------|---------------------------|-----------------|--------------------------|-----------------------| +| BenchmarkPrepareProposal_PFB_Multi/15000_transactions_of_300_bytes-16 | 6.239 | 10,318 | 2.411 s | 988,490,895,000 | 703 | 0.000703 MB | +| BenchmarkPrepareProposal_PFB_Multi/10000_transactions_of_500_bytes-16 | 5.035 | 6,331 | 1.710 s | 439,343,930,000 | 903 | 0.000903 MB | +| BenchmarkPrepareProposal_PFB_Multi/6000_transactions_of_1000_bytes-16 | 5.809 | 4,566 | 1.033 s | 158,174,358,000 | 1,403 | 0.001403 MB | +| BenchmarkPrepareProposal_PFB_Multi/3000_transactions_of_5000_bytes-16 | 7.188 | 1,413 | 0.547 s | 39,550,179,000 | 5,403 | 0.005403 MB | +| BenchmarkPrepareProposal_PFB_Multi/1000_transactions_of_10000_bytes-16 | 7.470 | 758 | 0.210 s | 4,397,393,000 | 10,403 | 0.010403 MB | +| BenchmarkPrepareProposal_PFB_Multi/500_transactions_of_50000_bytes-16 | 7.441 | 155 | 0.127 s | 1,100,446,500 | 50,406 | 0.050406 MB | +| BenchmarkPrepareProposal_PFB_Multi/100_transactions_of_100000_bytes-16 | 7.368 | 77 | 0.045 s | 44,369,300 | 100,406 | 0.100406 MB | +| BenchmarkPrepareProposal_PFB_Multi/100_transactions_of_200000_bytes-16 | 7.260 | 38 | 0.059 s | 44,369,300 | 200,406 | 0.200406 MB | +| BenchmarkPrepareProposal_PFB_Multi/50_transactions_of_300000_bytes-16 | 7.161 | 25 | 0.056 s | 11,202,150 | 300,406 | 0.300406 MB | +| BenchmarkPrepareProposal_PFB_Multi/50_transactions_of_400000_bytes-16 | 7.254 | 19 | 0.054 s | 11,202,150 | 400,406 | 0.400406 MB | +| BenchmarkPrepareProposal_PFB_Multi/30_transactions_of_500000_bytes-16 | 7.157 | 15 | 0.041 s | 4,085,490 | 500,406 | 0.500406 MB | +| BenchmarkPrepareProposal_PFB_Multi/10_transactions_of_1000000_bytes-16 | 6.678 | 7 | 0.031 s | 483,230 | 1,000,406 | 1.000406 MB | +| BenchmarkPrepareProposal_PFB_Multi/5_transactions_of_2000000_bytes-16 | 5.723 | 3 | 0.032 s | 131,790 | 2,000,406 | 2.000406 MB | +| BenchmarkPrepareProposal_PFB_Multi/3_transactions_of_3000000_bytes-16 | 5.723 | 2 | 0.042 s | 52,716 | 3,000,409 | 3.000409 MB | +| BenchmarkPrepareProposal_PFB_Multi/3_transactions_of_4000000_bytes-16 | 3.815 | 1 | 0.040 s | 52,716 | 4,000,409 | 4.000409 MB | +| BenchmarkPrepareProposal_PFB_Multi/2_transactions_of_5000000_bytes-16 | 4.769 | 1 | 0.039 s | 26,358 | 5,000,409 | 5.000409 MB | +| BenchmarkPrepareProposal_PFB_Multi/2_transactions_of_6000000_bytes-16 | 5.722 | 1 | 0.032 s | 26,358 | 6,000,409 | 6.000409 MB | + +#### ProcessProposal: `BenchmarkProcessProposal_PFB_Multi` + +The benchmarks for `ProcessProposal` for 8mb blocks containing PFBs of different sizes: + +| Benchmark Name | Block Size (MB) | Number of Transactions | Process Proposal Time (s) | Total Gas Used | Transaction Size (Bytes) | Transaction Size (MB) | +|------------------------------------------------------------------------|-----------------|------------------------|---------------------------|-----------------|--------------------------|-----------------------| +| BenchmarkProcessProposal_PFB_Multi/15000_transactions_of_300_bytes-16 | 6.239 | 10,318 | 1.767 s | 988,490,895,000 | 703 | 0.000703 MB | +| BenchmarkProcessProposal_PFB_Multi/10000_transactions_of_500_bytes-16 | 5.035 | 6,331 | 1.101 s | 439,343,930,000 | 903 | 0.000903 MB | +| BenchmarkProcessProposal_PFB_Multi/6000_transactions_of_1000_bytes-16 | 5.809 | 4,566 | 0.820 s | 158,174,358,000 | 1,403 | 0.001403 MB | +| BenchmarkProcessProposal_PFB_Multi/3000_transactions_of_5000_bytes-16 | 7.188 | 1,413 | 0.300 s | 39,550,179,000 | 5,403 | 0.005403 MB | +| BenchmarkProcessProposal_PFB_Multi/1000_transactions_of_10000_bytes-16 | 7.470 | 758 | 0.185 s | 4,397,393,000 | 10,403 | 0.010403 MB | +| BenchmarkProcessProposal_PFB_Multi/500_transactions_of_50000_bytes-16 | 7.441 | 155 | 0.092 s | 1,100,446,500 | 50,406 | 0.050406 MB | +| BenchmarkProcessProposal_PFB_Multi/100_transactions_of_100000_bytes-16 | 7.368 | 77 | 0.089 s | 44,369,300 | 100,406 | 0.100406 MB | +| BenchmarkProcessProposal_PFB_Multi/100_transactions_of_200000_bytes-16 | 7.260 | 38 | 0.060 s | 44,369,300 | 200,406 | 0.200406 MB | +| BenchmarkProcessProposal_PFB_Multi/50_transactions_of_300000_bytes-16 | 7.161 | 25 | 0.048 s | 11,202,150 | 300,406 | 0.300406 MB | +| BenchmarkProcessProposal_PFB_Multi/50_transactions_of_400000_bytes-16 | 7.254 | 19 | 0.051 s | 11,202,150 | 400,406 | 0.400406 MB | +| BenchmarkProcessProposal_PFB_Multi/30_transactions_of_500000_bytes-16 | 7.157 | 15 | 0.062 s | 4,085,490 | 500,406 | 0.500406 MB | +| BenchmarkProcessProposal_PFB_Multi/10_transactions_of_1000000_bytes-16 | 6.678 | 7 | 0.047 s | 483,230 | 1,000,406 | 1.000406 MB | +| BenchmarkProcessProposal_PFB_Multi/5_transactions_of_2000000_bytes-16 | 5.723 | 3 | 0.043 s | 131,790 | 2,000,406 | 2.000406 MB | +| BenchmarkProcessProposal_PFB_Multi/3_transactions_of_3000000_bytes-16 | 5.723 | 2 | 0.053 s | 52,716 | 3,000,409 | 3.000409 MB | +| BenchmarkProcessProposal_PFB_Multi/3_transactions_of_4000000_bytes-16 | 3.815 | 1 | 0.047 s | 52,716 | 4,000,409 | 4.000409 MB | +| BenchmarkProcessProposal_PFB_Multi/2_transactions_of_5000000_bytes-16 | 4.769 | 1 | 0.068 s | 26,358 | 5,000,409 | 5.000409 MB | +| BenchmarkProcessProposal_PFB_Multi/2_transactions_of_6000000_bytes-16 | 5.722 | 1 | 0.047 s | 26,358 | 6,000,409 | 6.000409 MB | + +### IBC `UpdateClient` benchmarks + +#### CheckTx: `BenchmarkIBC_CheckTx_Update_Client_Multi` + +The benchmarks of executing `checkTx` on a single transaction containing an IBC `updateClient` with different numbers of required signatures: + +| Benchmark Name | Time (ns/op) | Gas Used | Number of Validators | Number of Verified Signatures | Transaction Size (Bytes) | Transaction Size (MB) | +|-----------------------------------------------------------------------|--------------|-----------|----------------------|-------------------------------|--------------------------|-----------------------| +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_2-16 | 0.0007940 ns | 108,598 | 2.0 | 1.0 | 1,396 | 0.001396 MB | +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_10-16 | 0.002127 ns | 127,710 | 10.0 | 6.0 | 3,303 | 0.003303 MB | +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_25-16 | 0.003694 ns | 163,430 | 25.0 | 16.0 | 6,875 | 0.006875 MB | +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_50-16 | 0.004701 ns | 222,930 | 50.0 | 33.0 | 12,825 | 0.012825 MB | +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_75-16 | 0.004095 ns | 282,480 | 75.0 | 50.0 | 18,780 | 0.018780 MB | +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_100-16 | 0.004112 ns | 340,928 | 100.0 | 66.0 | 24,629 | 0.024629 MB | +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_125-16 | 0.007009 ns | 400,178 | 125.0 | 83.0 | 30,554 | 0.030554 MB | +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_150-16 | 0.004906 ns | 460,980 | 150.0 | 100.0 | 36,630 | 0.036630 MB | +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_175-16 | 0.01056 ns | 520,500 | 175.0 | 116.0 | 42,582 | 0.042582 MB | +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_200-16 | 0.01181 ns | 580,000 | 200.0 | 133.0 | 48,532 | 0.048532 MB | +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_225-16 | 0.01339 ns | 637,198 | 225.0 | 150.0 | 54,256 | 0.054256 MB | +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_250-16 | 0.01411 ns | 699,020 | 250.0 | 166.0 | 60,434 | 0.060434 MB | +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_300-16 | 0.01931 ns | 818,020 | 300.0 | 200.0 | 72,334 | 0.072334 MB | +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_400-16 | 0.02312 ns | 1,056,020 | 400.0 | 266.0 | 96,134 | 0.096134 MB | +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_500-16 | 0.01675 ns | 1,288,968 | 500.0 | 333.0 | 119,433 | 0.119433 MB | + +#### DeliverTx: `BenchmarkIBC_DeliverTx_Update_Client_Multi` + +The benchmarks of executing `deliverTx` on a single transaction containing an IBC `updateClient` with different numbers of required signatures: + +| Benchmark Name | Time (ns/op) | Gas Used | Number of Validators | Number of Verified Signatures | Transaction Size (Bytes) | Transaction Size (MB) | +|-------------------------------------------------------------------------|--------------|-----------|----------------------|-------------------------------|--------------------------|-----------------------| +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_2-16 | 0.0006931 ns | 107,520 | 2.0 | 1.0 | 1,396 | 0.001396 MB | +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_10-16 | 0.004647 ns | 126,480 | 10.0 | 6.0 | 3,292 | 0.003292 MB | +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_25-16 | 0.005861 ns | 162,352 | 25.0 | 16.0 | 6,875 | 0.006875 MB | +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_50-16 | 0.009248 ns | 221,852 | 50.0 | 33.0 | 12,825 | 0.012825 MB | +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_75-16 | 0.01252 ns | 281,402 | 75.0 | 50.0 | 18,780 | 0.018780 MB | +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_100-16 | 0.01239 ns | 339,850 | 100.0 | 66.0 | 24,629 | 0.024629 MB | +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_125-16 | 0.01300 ns | 400,402 | 125.0 | 83.0 | 30,680 | 0.030680 MB | +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_150-16 | 0.01691 ns | 459,902 | 150.0 | 100.0 | 36,630 | 0.036630 MB | +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_175-16 | 0.01560 ns | 517,620 | 175.0 | 116.0 | 42,406 | 0.042406 MB | +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_200-16 | 0.01894 ns | 578,922 | 200.0 | 133.0 | 48,532 | 0.048532 MB | +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_225-16 | 0.01714 ns | 638,422 | 225.0 | 150.0 | 54,482 | 0.054482 MB | +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_250-16 | 0.01736 ns | 697,942 | 250.0 | 166.0 | 60,434 | 0.060434 MB | +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_300-16 | 0.02008 ns | 816,942 | 300.0 | 200.0 | 72,334 | 0.072334 MB | +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_400-16 | 0.02320 ns | 1,054,942 | 400.0 | 266.0 | 96,134 | 0.096134 MB | +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_500-16 | 0.02724 ns | 1,288,522 | 500.0 | 333.0 | 119,492 | 0.119492 MB | + +#### PrepareProposal: `BenchmarkIBC_PrepareProposal_Update_Client_Multi` + +Benchmarks of an `8mb` containing the maximum number of IBC `UpdateClient` with different number of signatures: + +| Benchmark Name | Block Size (MB) | Number of Transactions | Number of Validators | Number of Verified Signatures | Prepare Proposal Time (s) | Total Gas Used | Transaction Size (Bytes) | Transaction Size (MB) | +|-------------------------------------------------------------------------------|------------------|-------------------------|----------------------|-------------------------------|-----------------------------|------------------|----------------------------|------------------------| +| BenchmarkIBC_PrepareProposal_Update_Client_Multi/number_of_validators:_10-16 | 7.464 | 2,367 | 10.0 | 6.0 | 0.571 s | 266,926,655 | 3,373 | 0.003373 MB | +| BenchmarkIBC_PrepareProposal_Update_Client_Multi/number_of_validators:_25-16 | 7.465 | 1,138 | 25.0 | 16.0 | 0.436 s | 249,391,655 | 6,945 | 0.006945 MB | +| BenchmarkIBC_PrepareProposal_Update_Client_Multi/number_of_validators:_50-16 | 7.462 | 610.0 | 50.0 | 33.0 | 0.271 s | 184,196,655 | 12,895 | 0.012895 MB | +| BenchmarkIBC_PrepareProposal_Update_Client_Multi/number_of_validators:_75-16 | 7.452 | 416.0 | 75.0 | 50.0 | 0.181 s | 121,879,155 | 18,850 | 0.018850 MB | +| BenchmarkIBC_PrepareProposal_Update_Client_Multi/number_of_validators:_100-16 | 7.453 | 316.0 | 100.0 | 66.0 | 0.180 s | 151,629,155 | 24,800 | 0.024800 MB | +| BenchmarkIBC_PrepareProposal_Update_Client_Multi/number_of_validators:_125-16 | 7.462 | 255.0 | 125.0 | 83.0 | 0.197 s | 181,379,155 | 30,750 | 0.030750 MB | +| BenchmarkIBC_PrepareProposal_Update_Client_Multi/number_of_validators:_150-16 | 7.441 | 213.0 | 150.0 | 100.0 | 0.207 s | 211,129,155 | 36,700 | 0.036700 MB | +| BenchmarkIBC_PrepareProposal_Update_Client_Multi/number_of_validators:_175-16 | 7.432 | 183.0 | 175.0 | 116.0 | 0.215 s | 240,889,155 | 42,652 | 0.042652 MB | +| BenchmarkIBC_PrepareProposal_Update_Client_Multi/number_of_validators:_200-16 | 7.467 | 162.0 | 200.0 | 133.0 | 0.227 s | 269,634,155 | 48,401 | 0.048401 MB | +| BenchmarkIBC_PrepareProposal_Update_Client_Multi/number_of_validators:_225-16 | 7.451 | 144.0 | 225.0 | 150.0 | 0.235 s | 299,259,155 | 54,326 | 0.054326 MB | +| BenchmarkIBC_PrepareProposal_Update_Client_Multi/number_of_validators:_250-16 | 7.462 | 130.0 | 250.0 | 166.0 | 0.242 s | 328,894,155 | 60,253 | 0.060253 MB | +| BenchmarkIBC_PrepareProposal_Update_Client_Multi/number_of_validators:_300-16 | 7.450 | 108.0 | 300.0 | 200.0 | 0.270 s | 389,649,155 | 72,404 | 0.072404 MB | +| BenchmarkIBC_PrepareProposal_Update_Client_Multi/number_of_validators:_400-16 | 7.426 | 81.0 | 400.0 | 266.0 | 0.304 s | 508,649,155 | 96,204 | 0.096204 MB | +| BenchmarkIBC_PrepareProposal_Update_Client_Multi/number_of_validators:_500-16 | 7.404 | 65.0 | 500.0 | 333.0 | 0.361 s | 625,144,155 | 119,503 | 0.119503 MB | + +#### ProcessProposal: `BenchmarkIBC_ProcessProposal_Update_Client_Multi` + +Benchmarks of an `8mb` containing the maximum number of IBC `UpdateClient` with different number of signatures: + +| Benchmark Name | Block Size (MB) | Number of Transactions | Number of Validators | Number of Verified Signatures | Process Proposal Time (s) | Total Gas Used | Transaction Size (Bytes) | Transaction Size (MB) | +|-------------------------------------------------------------------------------|-----------------|------------------------|----------------------|-------------------------------|---------------------------|----------------|--------------------------|-----------------------| +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_2-16 | 7.457 | 5,574 | 2.0 | 1.0 | 1.022 s | 419,611,655 | 1,469 | 0.001469 MB | +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_10-16 | 7.464 | 2,367 | 10.0 | 6.0 | 0.455 s | 266,926,655 | 3,373 | 0.003373 MB | +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_25-16 | 7.465 | 1,138 | 25.0 | 16.0 | 0.270 s | 249,391,655 | 6,945 | 0.006945 MB | +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_50-16 | 7.462 | 610.0 | 50.0 | 33.0 | 0.181 s | 184,196,655 | 12,895 | 0.012895 MB | +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_75-16 | 7.452 | 416.0 | 75.0 | 50.0 | 0.150 s | 121,879,155 | 18,850 | 0.018850 MB | +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_100-16 | 7.453 | 316.0 | 100.0 | 66.0 | 0.132 s | 151,629,155 | 24,800 | 0.024800 MB | +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_125-16 | 7.462 | 255.0 | 125.0 | 83.0 | 0.122 s | 181,379,155 | 30,750 | 0.030750 MB | +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_150-16 | 7.441 | 213.0 | 150.0 | 100.0 | 0.107 s | 211,129,155 | 36,700 | 0.036700 MB | +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_175-16 | 7.442 | 184.0 | 175.0 | 116.0 | 0.092 s | 240,009,155 | 42,476 | 0.042476 MB | +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_200-16 | 7.452 | 161.0 | 200.0 | 133.0 | 0.098 s | 270,639,155 | 48,602 | 0.048602 MB | +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_225-16 | 7.430 | 143.0 | 225.0 | 150.0 | 0.089 s | 300,389,155 | 54,552 | 0.054552 MB | +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_250-16 | 7.435 | 129.0 | 250.0 | 166.0 | 0.081 s | 330,149,155 | 60,504 | 0.060504 MB | +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_300-16 | 7.450 | 108.0 | 300.0 | 200.0 | 0.078 s | 389,649,155 | 72,404 | 0.072404 MB | +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_400-16 | 7.426 | 81.0 | 400.0 | 266.0 | 0.077 s | 508,649,155 | 96,204 | 0.096204 MB | +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_500-16 | 7.435 | 65.0 | 500.0 | 333.0 | 0.092 s | 627,649,155 | 120,004 | 0.120004 MB | + +#### Process proposal time with different number of transactions per block + +**50 bytes blobs**: + +| Number of Transactions | Block Size (bytes) | Elapsed Time (s) | +|------------------------|--------------------|------------------| +| 1467 | 0.532979 | 0.2508 | +| 1546 | 0.561684 | 0.2766 | +| 1566 | 0.568951 | 0.2511 | +| 1650 | 0.599472 | 0.2711 | +| 1739 | 0.631810 | 0.3007 | +| 1761 | 0.639804 | 0.2832 | +| 1856 | 0.674322 | 0.3017 | +| 1956 | 0.710657 | 0.3203 | +| 1980 | 0.719378 | 0.3291 | +| 2062 | 0.749172 | 0.3670 | +| 2088 | 0.758619 | 0.3426 | +| 2200 | 0.799314 | 0.3610 | +| 2319 | 0.842553 | 0.3980 | +| 2349 | 0.853454 | 0.3794 | +| 2475 | 0.899236 | 0.4086 | +| 2608 | 0.947561 | 0.4555 | +| 2641 | 0.959552 | 0.4561 | +| 2750 | 0.999157 | 0.4920 | +| 2784 | 1.011511 | 0.4782 | +| 2934 | 1.066013 | 0.5209 | +| 2970 | 1.079094 | 0.5069 | +| 3093 | 1.123786 | 0.5816 | +| 3132 | 1.137957 | 0.5360 | +| 3300 | 1.198999 | 0.5766 | +| 3478 | 1.263676 | 0.6072 | +| 3523 | 1.280026 | 0.6028 | +| 3712 | 1.348700 | 0.6394 | +| 3912 | 1.421370 | 0.6928 | +| 3961 | 1.439174 | 0.6559 | +| 4125 | 1.498763 | 0.7463 | +| 4176 | 1.517294 | 0.6967 | +| 5500 | 1.998369 | 0.9183 | +| 11000 | 3.753713 | 1.732 | + +**100 bytes blobs**: + +| Number of Transactions | Block Size (bytes) | Elapsed Time (s) | +|------------------------|--------------------|------------------| +| 1546 | 0.636877 | 0.2726 | +| 1739 | 0.716391 | 0.2762 | +| 1956 | 0.805792 | 0.3207 | +| 2062 | 0.849463 | 0.3361 | +| 2319 | 0.955343 | 0.3774 | +| 2608 | 1.074408 | 0.4387 | +| 2750 | 1.132910 | 0.4873 | +| 2934 | 1.208715 | 0.5015 | +| 3093 | 1.274221 | 0.5202 | +| 3478 | 1.432837 | 0.5797 | +| 3912 | 1.611639 | 0.6520 | +| 4125 | 1.699392 | 0.6758 | +| 5500 | 2.265875 | 0.9318 | +| 11000 | 4.256186 | 1.685 | + +**200 bytes blobs**: + +| Number of Transactions | Block Size (bytes) | Elapsed Time (s) | +|------------------------|--------------------|------------------| +| 1546 | 0.787264 | 0.2472 | +| 1739 | 0.885551 | 0.3009 | +| 1956 | 0.996061 | 0.3188 | +| 2062 | 1.050043 | 0.3400 | +| 2319 | 1.180923 | 0.3781 | +| 2608 | 1.328100 | 0.4439 | +| 2750 | 1.400415 | 0.4720 | +| 2934 | 1.494120 | 0.5049 | +| 3093 | 1.575092 | 0.5384 | +| 3478 | 1.771158 | 0.5913 | +| 3912 | 1.992178 | 0.6459 | +| 4125 | 2.100651 | 0.6927 | +| 5500 | 2.800886 | 0.8970 | +| 11000 | 5.254511 | 1.691 | + +**300 bytes blobs**: + +| Number of Transactions | Block Size (bytes) | Elapsed Time (s) | +|------------------------|--------------------|------------------| +| 1546 | 0.934702 | 0.2506 | +| 1739 | 1.051395 | 0.2910 | +| 1956 | 1.182600 | 0.3316 | +| 2062 | 1.246691 | 0.3439 | +| 2319 | 1.402081 | 0.3830 | +| 2608 | 1.576818 | 0.4674 | +| 2750 | 1.662676 | 0.4803 | +| 2934 | 1.773928 | 0.5110 | +| 3093 | 1.870064 | 0.5431 | +| 3478 | 2.102846 | 0.6002 | +| 3912 | 2.365255 | 0.6659 | +| 4125 | 2.494041 | 0.7052 | +| 5500 | 3.325407 | 0.9117 | +| 11000 | 6.238512 | 1.688 | + +**400 bytes blobs**: + +| Number of Transactions | Block Size (bytes) | Elapsed Time (s) | +|------------------------|--------------------|------------------| +| 1375 | 0.962440 | 0.2425 | +| 1467 | 1.026840 | 0.2564 | +| 1546 | 1.082140 | 0.2583 | +| 1650 | 1.154940 | 0.2713 | +| 1739 | 1.217239 | 0.2854 | +| 1856 | 1.299139 | 0.3204 | +| 1956 | 1.369139 | 0.3205 | +| 2062 | 1.443338 | 0.3535 | +| 2200 | 1.539938 | 0.3674 | +| 2319 | 1.623238 | 0.3873 | +| 2475 | 1.732437 | 0.4184 | +| 2608 | 1.825537 | 0.4635 | +| 2750 | 1.924936 | 0.5227 | +| 2784 | 1.948736 | 0.5029 | +| 2934 | 2.053736 | 0.5193 | +| 3093 | 2.165035 | 0.5505 | +| 3300 | 2.309935 | 0.6121 | +| 3478 | 2.434534 | 0.6077 | +| 3712 | 2.598333 | 0.6534 | +| 3912 | 2.738333 | 0.6625 | +| 5500 | 3.849928 | 0.9410 | +| 11000 | 7.222513 | 1.782 | + +**500 bytes blobs**: + +| Number of Transactions | Block Size (bytes) | Elapsed Time (s) | +|------------------------|--------------------|------------------| +| 1476 | 1.173903 | 0.2640 | +| 1660 | 1.320250 | 0.3192 | +| 1750 | 1.391832 | 0.3249 | +| 1867 | 1.484890 | 0.3494 | +| 1968 | 1.565222 | 0.3664 | +| 2214 | 1.760881 | 0.4322 | +| 2490 | 1.980402 | 0.4667 | +| 2625 | 2.087776 | 0.4795 | +| 2800 | 2.226965 | 0.5033 | +| 2952 | 2.347860 | 0.5529 | +| 3321 | 2.641350 | 0.6263 | +| 3500 | 2.783720 | 0.6101 | +| 3735 | 2.970631 | 0.6629 | +| 3937 | 3.131294 | 0.7341 | +| 7000 | 5.035397 | 1.127 | + +**600 bytes blobs**: + +| Number of Transactions | Block Size (bytes) | Elapsed Time (s) | +|------------------------|--------------------|------------------| +| 1400 | 1.246969 | 0.2492 | +| 1417 | 1.262112 | 0.2554 | +| 1432 | 1.275473 | 0.2465 | +| 1476 | 1.314665 | 0.2575 | +| 1494 | 1.330698 | 0.2716 | +| 1510 | 1.344950 | 0.2729 | +| 1575 | 1.402847 | 0.2777 | +| 1593 | 1.418880 | 0.3210 | +| 1611 | 1.434914 | 0.3269 | +| 1660 | 1.478559 | 0.3331 | +| 1680 | 1.496374 | 0.3202 | +| 1698 | 1.512407 | 0.3387 | +| 1750 | 1.558725 | 0.3430 | +| 1771 | 1.577431 | 0.3476 | +| 1791 | 1.595245 | 0.3550 | +| 1812 | 1.613951 | 0.3526 | +| 1867 | 1.662941 | 0.3702 | +| 1890 | 1.683428 | 0.3592 | +| 1910 | 1.701242 | 0.3728 | +| 1968 | 1.752905 | 0.3790 | +| 1992 | 1.774282 | 0.3636 | +| 2014 | 1.793879 | 0.3740 | +| 2100 | 1.870481 | 0.4125 | +| 2125 | 1.892750 | 0.3915 | +| 2148 | 1.913237 | 0.4158 | +| 2214 | 1.972025 | 0.4057 | +| 2241 | 1.996075 | 0.4231 | +| 2265 | 2.017452 | 0.4210 | +| 2362 | 2.103853 | 0.4392 | +| 2389 | 2.127903 | 0.4406 | +| 2416 | 2.151953 | 0.4700 | +| 2490 | 2.217867 | 0.4615 | +| 2520 | 2.244589 | 0.4727 | +| 2547 | 2.268639 | 0.4743 | +| 2625 | 2.338116 | 0.4812 | +| 2656 | 2.365728 | 0.4923 | +| 2686 | 2.392450 | 0.4905 | +| 2718 | 2.420954 | 0.5042 | +| 2800 | 2.493994 | 0.5309 | +| 2835 | 2.525169 | 0.5166 | +| 2865 | 2.551891 | 0.5340 | +| 2952 | 2.629385 | 0.5378 | +| 2988 | 2.661451 | 0.5504 | +| 3021 | 2.690845 | 0.5532 | +| 3150 | 2.805750 | 0.5948 | +| 3187 | 2.838707 | 0.5747 | +| 3222 | 2.869883 | 0.5986 | +| 3321 | 2.958065 | 0.6170 | +| 3361 | 2.993694 | 0.6092 | +| 3397 | 3.025761 | 0.6193 | +| 3500 | 3.117506 | 0.6357 | +| 3543 | 3.155807 | 0.6425 | +| 3583 | 3.191437 | 0.6764 | +| 3624 | 3.227957 | 0.6628 | +| 3735 | 3.326828 | 0.6819 | +| 3780 | 3.366911 | 0.6935 | +| 3820 | 3.402540 | 0.7127 | +| 3937 | 3.506756 | 0.7093 | +| 3984 | 3.548620 | 0.7404 | +| 4029 | 3.588703 | 0.7535 | +| 7000 | 5.639168 | 1.133 | + +**1000 bytes blobs**: + +| Number of Transactions | Block Size (bytes) | Elapsed Time (s) | +|------------------------|--------------------|------------------| +| 1333 | 1.695789 | 0.2682 | +| 1348 | 1.714872 | 0.2605 | +| 1406 | 1.788660 | 0.2858 | +| 1422 | 1.809015 | 0.2827 | +| 1437 | 1.828098 | 0.2881 | +| 1499 | 1.906975 | 0.2945 | +| 1516 | 1.928602 | 0.2985 | +| 1581 | 2.011295 | 0.3039 | +| 1599 | 2.034195 | 0.3111 | +| 1616 | 2.055822 | 0.3185 | +| 1686 | 2.144876 | 0.3450 | +| 1705 | 2.169048 | 0.3501 | +| 1778 | 2.261919 | 0.3496 | +| 1798 | 2.287363 | 0.3554 | +| 1818 | 2.312807 | 0.3507 | +| 1875 | 2.385323 | 0.3849 | +| 1896 | 2.412039 | 0.3877 | +| 1917 | 2.438755 | 0.3746 | +| 1999 | 2.543076 | 0.3815 | +| 2022 | 2.572336 | 0.4042 | +| 2109 | 2.683018 | 0.4223 | +| 2133 | 2.713551 | 0.4126 | +| 2155 | 2.741539 | 0.4115 | +| 2248 | 2.859854 | 0.4183 | +| 2274 | 2.892931 | 0.4343 | +| 2371 | 3.016335 | 0.4642 | +| 2398 | 3.050684 | 0.4631 | +| 2424 | 3.083761 | 0.4575 | +| 2500 | 3.180449 | 0.4825 | +| 2529 | 3.217342 | 0.4757 | +| 2557 | 3.252964 | 0.4812 | +| 2667 | 3.392906 | 0.5144 | +| 2697 | 3.431072 | 0.5141 | +| 2727 | 3.469238 | 0.5071 | +| 2812 | 3.577375 | 0.5250 | +| 2844 | 3.618086 | 0.5359 | +| 2875 | 3.657524 | 0.5506 | +| 2998 | 3.814005 | 0.5659 | +| 3033 | 3.858532 | 0.5797 | +| 3163 | 4.023918 | 0.5964 | +| 3199 | 4.069717 | 0.6023 | +| 3232 | 4.111700 | 0.6142 | +| 3372 | 4.289808 | 0.6249 | +| 3411 | 4.339424 | 0.6465 | +| 3556 | 4.523893 | 0.6488 | +| 3597 | 4.576054 | 0.6829 | +| 3636 | 4.625669 | 0.6699 | +| 3750 | 4.770700 | 0.6820 | +| 3793 | 4.825405 | 0.6983 | +| 3835 | 4.878838 | 0.6991 | +| 5000 | 5.808817 | 0.8490 | + +**1200 bytes blobs**: + +| Number of Transactions | Block Size (bytes) | Elapsed Time (s) | +|------------------------|--------------------|------------------| +| 1406 | 2.056833 | 0.2758 | +| 1500 | 2.194349 | 0.3071 | +| 1581 | 2.312847 | 0.3054 | +| 1687 | 2.467918 | 0.3332 | +| 1778 | 2.601046 | 0.3569 | +| 1875 | 2.742950 | 0.3688 | +| 2000 | 2.925817 | 0.3793 | +| 2109 | 3.085278 | 0.4087 | +| 2250 | 3.291552 | 0.4359 | +| 2371 | 3.468567 | 0.4462 | +| 2500 | 3.657286 | 0.4789 | +| 2530 | 3.701174 | 0.4999 | +| 2667 | 3.901596 | 0.4836 | +| 2812 | 4.113722 | 0.5371 | +| 3000 | 4.388754 | 0.5768 | +| 3163 | 4.627213 | 0.5897 | +| 3375 | 4.937355 | 0.6156 | +| 3556 | 5.202147 | 0.6549 | +| 3750 | 5.485956 | 0.6933 | +| 4000 | 5.851690 | 0.7415 | +| 5000 | 6.679712 | 0.8498 | + +**1500 bytes blobs**: + +| Number of Transactions | Block Size (bytes) | Elapsed Time (s) | +|------------------------|--------------------|------------------| +| 1406 | 2.459093 | 0.2941 | +| 1581 | 2.765175 | 0.3109 | +| 1778 | 3.109735 | 0.3373 | +| 1875 | 3.279392 | 0.3706 | +| 2109 | 3.688667 | 0.4100 | +| 2371 | 4.146915 | 0.4601 | +| 2500 | 4.372541 | 0.4735 | +| 2667 | 4.664631 | 0.5013 | +| 2812 | 4.918242 | 0.5260 | +| 3163 | 5.532154 | 0.5946 | +| 3556 | 6.219526 | 0.6634 | +| 3750 | 6.245762 | 0.6879 | +| 5000 | 6.245762 | 0.6781 | + +**1800 bytes blobs**: + +| Number of Transactions | Block Size (bytes) | Elapsed Time (s) | +|------------------------|--------------------|------------------| +| 1333 | 2.712788 | 0.2643 | +| 1406 | 2.861353 | 0.2840 | +| 1422 | 2.893915 | 0.2843 | +| 1499 | 3.050621 | 0.2956 | +| 1581 | 3.217503 | 0.3094 | +| 1599 | 3.254135 | 0.3302 | +| 1686 | 3.431192 | 0.3396 | +| 1778 | 3.618425 | 0.3407 | +| 1798 | 3.659128 | 0.3397 | +| 1875 | 3.815834 | 0.3777 | +| 1896 | 3.858572 | 0.3813 | +| 1999 | 4.068192 | 0.3647 | +| 2109 | 4.292057 | 0.4191 | +| 2133 | 4.340900 | 0.4057 | +| 2248 | 4.574942 | 0.4349 | +| 2371 | 4.825264 | 0.4446 | +| 2398 | 4.880213 | 0.4481 | +| 2500 | 5.087797 | 0.4676 | +| 2529 | 5.146816 | 0.4740 | +| 2667 | 5.427666 | 0.5127 | +| 2697 | 5.488720 | 0.5039 | +| 2812 | 5.722761 | 0.5547 | +| 2844 | 5.787886 | 0.5411 | +| 2998 | 6.101297 | 0.5710 | +| 3163 | 6.437096 | 0.5896 | +| 3199 | 6.510361 | 0.5965 | +| 3372 | 6.862440 | 0.6149 | +| 3556 | 7.236906 | 0.6572 | +| 3597 | 7.267433 | 0.6716 | +| 5000 | 7.267433 | 0.6742 | + +**2000 bytes blobs**: + +| Number of Transactions | Block Size (bytes) | Elapsed Time (s) | +|------------------------|--------------------|------------------| +| 1406 | 3.129526 | 0.2732 | +| 1581 | 3.519054 | 0.3078 | +| 1778 | 3.957552 | 0.3477 | +| 1875 | 4.173462 | 0.3764 | +| 2109 | 4.694317 | 0.4059 | +| 2371 | 5.277496 | 0.4412 | +| 2500 | 5.564634 | 0.4664 | +| 2667 | 5.936356 | 0.5006 | +| 2812 | 6.259108 | 0.5262 | +| 3163 | 6.526213 | 0.5574 | +| 3556 | 6.526213 | 0.5667 | +| 3750 | 6.526213 | 0.5509 | +| 5000 | 6.526213 | 0.5556 | + +## GoLevelDB benchmarks + +### `sendMsg` benchmarks + +#### CheckTx + +A single `checkTx` of a `sendMsg` message takes 0.0003071 **ns** to execute. And it uses 74374 gas. + +The transactions in an `8mb` block containing 31645 `sendMsg` messages take 6,45 s (6455816060 ns) to run `checkTx` on all of them. The total gas used is 1884371034 gas. + +#### DeliverTx + +A single `deliverTx` of a `sendMsg` message takes 0.0003948 **ns** to execute. And it uses 103251 gas. + +The transactions in an `8mb` block containing 31645 `sendMsg` messages take 7,50 s (7506830940 ns) to run `deliverTx` on all of them. The total gas used is 2801272121 gas. + +#### PrepareProposal + +A single `prepareProposal` of a `sendMsg` message takes 0.0003943 **ns** to execute. And it uses 101110 gas. + +An `8mb` block containing 31645 `sendMsg` messages takes 5,2 s (5242159792 ns) to execute. The total gas used 1843040790 gas. + +#### ProcessProposal + +A single `processProposal` of a `sendMsg` message takes 0.0003010 **ns** to execute. And it uses 101110 gas. + +An `8mb` block containing 31645 `sendMsg` messages takes 5,21 s (5214205041 ns) to execute. The total gas used 1843040790 gas. + +### `PFB` benchmarks + +#### CheckTx: `BenchmarkCheckTx_PFB_Multi` + +Benchmarks of `CheckTx` for a single PFB with different sizes: + +| Benchmark Name | Time (ns/op) | Gas Used | Transaction Size (Bytes) | Transaction Size (MB) | +|---------------------------------------------|--------------|----------|--------------------------|-----------------------| +| BenchmarkCheckTx_PFB_Multi/300_bytes-16 | 0.0005847 ns | 74,664 | 703 | 0.000703 MB | +| BenchmarkCheckTx_PFB_Multi/500_bytes-16 | 0.0005136 ns | 74,664 | 903 | 0.000903 MB | +| BenchmarkCheckTx_PFB_Multi/1000_bytes-16 | 0.0005754 ns | 74,664 | 1,403 | 0.001403 MB | +| BenchmarkCheckTx_PFB_Multi/5000_bytes-16 | 0.0005706 ns | 74,664 | 5,403 | 0.005403 MB | +| BenchmarkCheckTx_PFB_Multi/10000_bytes-16 | 0.0006885 ns | 74,664 | 10,403 | 0.010403 MB | +| BenchmarkCheckTx_PFB_Multi/50000_bytes-16 | 0.0006683 ns | 74,674 | 50,406 | 0.050406 MB | +| BenchmarkCheckTx_PFB_Multi/100000_bytes-16 | 0.0008378 ns | 74,674 | 100,406 | 0.100406 MB | +| BenchmarkCheckTx_PFB_Multi/200000_bytes-16 | 0.001130 ns | 74,674 | 200,406 | 0.200406 MB | +| BenchmarkCheckTx_PFB_Multi/300000_bytes-16 | 0.001164 ns | 74,674 | 300,406 | 0.300406 MB | +| BenchmarkCheckTx_PFB_Multi/400000_bytes-16 | 0.001550 ns | 74,674 | 400,406 | 0.400406 MB | +| BenchmarkCheckTx_PFB_Multi/500000_bytes-16 | 0.001829 ns | 74,674 | 500,406 | 0.500406 MB | +| BenchmarkCheckTx_PFB_Multi/1000000_bytes-16 | 0.002452 ns | 74,674 | 1,000,406 | 1.000406 MB | +| BenchmarkCheckTx_PFB_Multi/2000000_bytes-16 | 0.004647 ns | 74,674 | 2,000,406 | 2.000406 MB | +| BenchmarkCheckTx_PFB_Multi/3000000_bytes-16 | 0.006415 ns | 74,684 | 3,000,409 | 3.000409 MB | +| BenchmarkCheckTx_PFB_Multi/4000000_bytes-16 | 0.007709 ns | 74,684 | 4,000,409 | 4.000409 MB | +| BenchmarkCheckTx_PFB_Multi/5000000_bytes-16 | 0.01014 ns | 74,684 | 5,000,409 | 5.000409 MB | +| BenchmarkCheckTx_PFB_Multi/6000000_bytes-16 | 0.01153 ns | 74,684 | 6,000,409 | 6.000409 MB | + +#### DeliverTx: `BenchmarkDeliverTx_PFB_Multi` + +Benchmarks of `DeliverTx` for a single PFB with different sizes: + +| Benchmark Name | Time (ns/op) | Gas Used | Transaction Size (Bytes) | Transaction Size (MB) | +|-----------------------------------------------|--------------|------------|--------------------------|-----------------------| +| BenchmarkDeliverTx_PFB_Multi/300_bytes-16 | 0.0005010 ns | 77,682 | 703 | 0.000703 MB | +| BenchmarkDeliverTx_PFB_Multi/500_bytes-16 | 0.0004297 ns | 81,778 | 903 | 0.000903 MB | +| BenchmarkDeliverTx_PFB_Multi/1000_bytes-16 | 0.0005227 ns | 85,874 | 1,403 | 0.001403 MB | +| BenchmarkDeliverTx_PFB_Multi/5000_bytes-16 | 0.0005552 ns | 118,642 | 5,403 | 0.005403 MB | +| BenchmarkDeliverTx_PFB_Multi/10000_bytes-16 | 0.0004537 ns | 159,602 | 10,403 | 0.010403 MB | +| BenchmarkDeliverTx_PFB_Multi/50000_bytes-16 | 0.0004896 ns | 499,580 | 50,406 | 0.050406 MB | +| BenchmarkDeliverTx_PFB_Multi/100000_bytes-16 | 0.0005505 ns | 925,564 | 100,406 | 0.100406 MB | +| BenchmarkDeliverTx_PFB_Multi/200000_bytes-16 | 0.0003661 ns | 1,773,436 | 200,406 | 0.200406 MB | +| BenchmarkDeliverTx_PFB_Multi/300000_bytes-16 | 0.0004681 ns | 2,625,404 | 300,406 | 0.300406 MB | +| BenchmarkDeliverTx_PFB_Multi/400000_bytes-16 | 0.0003012 ns | 3,473,276 | 400,406 | 0.400406 MB | +| BenchmarkDeliverTx_PFB_Multi/500000_bytes-16 | 0.0003164 ns | 4,325,244 | 500,406 | 0.500406 MB | +| BenchmarkDeliverTx_PFB_Multi/1000000_bytes-16 | 0.0004873 ns | 8,572,796 | 1,000,406 | 1.000406 MB | +| BenchmarkDeliverTx_PFB_Multi/2000000_bytes-16 | 0.0004004 ns | 17,071,996 | 2,000,406 | 2.000406 MB | +| BenchmarkDeliverTx_PFB_Multi/3000000_bytes-16 | 0.0003486 ns | 25,571,206 | 3,000,409 | 3.000409 MB | +| BenchmarkDeliverTx_PFB_Multi/4000000_bytes-16 | 0.0004354 ns | 34,066,310 | 4,000,409 | 4.000409 MB | +| BenchmarkDeliverTx_PFB_Multi/5000000_bytes-16 | 0.0003734 ns | 42,565,510 | 5,000,409 | 5.000409 MB | +| BenchmarkDeliverTx_PFB_Multi/6000000_bytes-16 | 0.0003595 ns | 51,064,710 | 6,000,409 | 6.000409 MB | + +#### PrepareProposal: `BenchmarkPrepareProposal_PFB_Multi` + +The benchmarks for `PrepareProposal` for 8mb blocks containing PFBs of different sizes: + +| Benchmark Name | Block Size (MB) | Number of Transactions | Prepare Proposal Time (s) | Total Gas Used | Transaction Size (Bytes) | Transaction Size (MB) | +|------------------------------------------------------------------------|-----------------|------------------------|---------------------------|-----------------|--------------------------|-----------------------| +| BenchmarkPrepareProposal_PFB_Multi/15000_transactions_of_300_bytes-16 | 6.239 | 10,318 | 2.452 s | 988,490,895,000 | 703 | 0.000703 MB | +| BenchmarkPrepareProposal_PFB_Multi/10000_transactions_of_500_bytes-16 | 5.035 | 6,331 | 1.721 s | 439,343,930,000 | 903 | 0.000903 MB | +| BenchmarkPrepareProposal_PFB_Multi/6000_transactions_of_1000_bytes-16 | 5.809 | 4,566 | 1.063 s | 158,174,358,000 | 1,403 | 0.001403 MB | +| BenchmarkPrepareProposal_PFB_Multi/3000_transactions_of_5000_bytes-16 | 7.188 | 1,413 | 0.527 s | 39,550,179,000 | 5,403 | 0.005403 MB | +| BenchmarkPrepareProposal_PFB_Multi/1000_transactions_of_10000_bytes-16 | 7.470 | 758 | 0.210 s | 4,397,393,000 | 10,403 | 0.010403 MB | +| BenchmarkPrepareProposal_PFB_Multi/500_transactions_of_50000_bytes-16 | 7.441 | 155 | 0.125 s | 1,100,446,500 | 50,406 | 0.050406 MB | +| BenchmarkPrepareProposal_PFB_Multi/100_transactions_of_100000_bytes-16 | 7.368 | 77 | 0.061 s | 44,369,300 | 100,406 | 0.100406 MB | +| BenchmarkPrepareProposal_PFB_Multi/100_transactions_of_200000_bytes-16 | 7.260 | 38 | 0.058 s | 44,369,300 | 200,406 | 0.200406 MB | +| BenchmarkPrepareProposal_PFB_Multi/50_transactions_of_300000_bytes-16 | 7.161 | 25 | 0.042 s | 11,202,150 | 300,406 | 0.300406 MB | +| BenchmarkPrepareProposal_PFB_Multi/50_transactions_of_400000_bytes-16 | 7.254 | 19 | 0.038 s | 11,202,150 | 400,406 | 0.400406 MB | +| BenchmarkPrepareProposal_PFB_Multi/30_transactions_of_500000_bytes-16 | 7.157 | 15 | 0.031 s | 4,085,490 | 500,406 | 0.500406 MB | +| BenchmarkPrepareProposal_PFB_Multi/10_transactions_of_1000000_bytes-16 | 6.678 | 7 | 0.026 s | 483,230 | 1,000,406 | 1.000406 MB | +| BenchmarkPrepareProposal_PFB_Multi/5_transactions_of_2000000_bytes-16 | 5.723 | 3 | 0.027 s | 131,790 | 2,000,406 | 2.000406 MB | +| BenchmarkPrepareProposal_PFB_Multi/3_transactions_of_3000000_bytes-16 | 5.723 | 2 | 0.030 s | 52,716 | 3,000,409 | 3.000409 MB | +| BenchmarkPrepareProposal_PFB_Multi/3_transactions_of_4000000_bytes-16 | 3.815 | 1 | 0.026 s | 52,716 | 4,000,409 | 4.000409 MB | +| BenchmarkPrepareProposal_PFB_Multi/2_transactions_of_5000000_bytes-16 | 4.769 | 1 | 0.031 s | 26,358 | 5,000,409 | 5.000409 MB | +| BenchmarkPrepareProposal_PFB_Multi/2_transactions_of_6000000_bytes-16 | 5.722 | 1 | 0.028 s | 26,358 | 6,000,409 | 6.000409 MB | + +#### ProcessProposal: `BenchmarkProcessProposal_PFB_Multi` + +The benchmarks for `ProcessProposal` for 8mb blocks containing PFBs of different sizes: + +| Benchmark Name | Block Size (MB) | Number of Transactions | Process Proposal Time (s) | Total Gas Used | Transaction Size (Bytes) | Transaction Size (MB) | +|------------------------------------------------------------------------|-----------------|------------------------|---------------------------|-----------------|--------------------------|-----------------------| +| BenchmarkProcessProposal_PFB_Multi/15000_transactions_of_300_bytes-16 | 6.239 | 10,318 | 1.813 s | 988,490,895,000 | 703 | 0.000703 MB | +| BenchmarkProcessProposal_PFB_Multi/10000_transactions_of_500_bytes-16 | 5.035 | 6,331 | 1.120 s | 439,343,930,000 | 903 | 0.000903 MB | +| BenchmarkProcessProposal_PFB_Multi/6000_transactions_of_1000_bytes-16 | 5.809 | 4,566 | 0.829 s | 158,174,358,000 | 1,403 | 0.001403 MB | +| BenchmarkProcessProposal_PFB_Multi/3000_transactions_of_5000_bytes-16 | 7.188 | 1,413 | 0.290 s | 39,550,179,000 | 5,403 | 0.005403 MB | +| BenchmarkProcessProposal_PFB_Multi/1000_transactions_of_10000_bytes-16 | 7.470 | 758 | 0.188 s | 4,397,393,000 | 10,403 | 0.010403 MB | +| BenchmarkProcessProposal_PFB_Multi/500_transactions_of_50000_bytes-16 | 7.441 | 155 | 0.076 s | 1,100,446,500 | 50,406 | 0.050406 MB | +| BenchmarkProcessProposal_PFB_Multi/100_transactions_of_100000_bytes-16 | 7.368 | 77 | 0.056 s | 44,369,300 | 100,406 | 0.100406 MB | +| BenchmarkProcessProposal_PFB_Multi/100_transactions_of_200000_bytes-16 | 7.260 | 38 | 0.050 s | 44,369,300 | 200,406 | 0.200406 MB | +| BenchmarkProcessProposal_PFB_Multi/50_transactions_of_300000_bytes-16 | 7.161 | 25 | 0.048 s | 11,202,150 | 300,406 | 0.300406 MB | +| BenchmarkProcessProposal_PFB_Multi/50_transactions_of_400000_bytes-16 | 7.254 | 19 | 0.048 s | 11,202,150 | 400,406 | 0.400406 MB | +| BenchmarkProcessProposal_PFB_Multi/30_transactions_of_500000_bytes-16 | 7.157 | 15 | 0.043 s | 4,085,490 | 500,406 | 0.500406 MB | +| BenchmarkProcessProposal_PFB_Multi/10_transactions_of_1000000_bytes-16 | 6.678 | 7 | 0.041 s | 483,230 | 1,000,406 | 1.000406 MB | +| BenchmarkProcessProposal_PFB_Multi/5_transactions_of_2000000_bytes-16 | 5.723 | 3 | 0.053 s | 131,790 | 2,000,406 | 2.000406 MB | +| BenchmarkProcessProposal_PFB_Multi/3_transactions_of_3000000_bytes-16 | 5.723 | 2 | 0.037 s | 52,716 | 3,000,409 | 3.000409 MB | +| BenchmarkProcessProposal_PFB_Multi/3_transactions_of_4000000_bytes-16 | 3.815 | 1 | 0.071 s | 52,716 | 4,000,409 | 4.000409 MB | +| BenchmarkProcessProposal_PFB_Multi/2_transactions_of_5000000_bytes-16 | 4.769 | 1 | 0.034 s | 26,358 | 5,000,409 | 5.000409 MB | +| BenchmarkProcessProposal_PFB_Multi/2_transactions_of_6000000_bytes-16 | 5.722 | 1 | 0.062 s | 26,358 | 6,000,409 | 6.000409 MB | + +### IBC `UpdateClient` benchmarks + +#### CheckTx: `BenchmarkIBC_CheckTx_Update_Client_Multi` + +The benchmarks of executing `checkTx` on a single transaction containing an IBC `updateClient` with different numbers of required signatures: + +| Benchmark Name | Time (ns/op) | Total Gas Used | Number of Validators | Number of Verified Signatures | Transaction Size (Bytes) | Transaction Size (MB) | +|-----------------------------------------------------------------------|--------------|----------------|----------------------|-------------------------------|--------------------------|-----------------------| +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_2-16 | 1,370 | 108,670 | 2 | 1 | 1,399 | 0.001399 MB | +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_10-16 | 3,577 | 127,710 | 10 | 6 | 3,303 | 0.003303 MB | +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_25-16 | 7,432 | 163,430 | 25 | 16 | 6,875 | 0.006875 MB | +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_50-16 | 9,879 | 222,930 | 50 | 33 | 12,825 | 0.012825 MB | +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_75-16 | 12,060 | 282,480 | 75 | 50 | 18,780 | 0.018780 MB | +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_100-16 | 13,080 | 341,980 | 100 | 66 | 24,730 | 0.024730 MB | +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_125-16 | 14,390 | 401,480 | 125 | 83 | 30,680 | 0.030680 MB | +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_150-16 | 16,440 | 459,428 | 150 | 100 | 36,479 | 0.036479 MB | +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_175-16 | 17,370 | 520,500 | 175 | 116 | 42,582 | 0.042582 MB | +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_200-16 | 18,840 | 580,000 | 200 | 133 | 48,532 | 0.048532 MB | +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_225-16 | 21,760 | 637,198 | 225 | 150 | 54,256 | 0.054256 MB | +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_250-16 | 19,680 | 699,020 | 250 | 166 | 60,434 | 0.060434 MB | +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_300-16 | 22,580 | 818,020 | 300 | 200 | 72,334 | 0.072334 MB | +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_400-16 | 25,990 | 1,056,020 | 400 | 266 | 96,134 | 0.096134 MB | +| BenchmarkIBC_CheckTx_Update_Client_Multi/number_of_validators:_500-16 | 27,100 | 1,288,968 | 500 | 333 | 119,433 | 0.119433 MB | + +#### DeliverTx: `BenchmarkIBC_DeliverTx_Update_Client_Multi` + +The benchmarks of executing `deliverTx` on a single transaction containing an IBC `updateClient` with different numbers of required signatures: + +| Benchmark Name | Time (ns/op) | Gas Used | Number of Validators | Number of Verified Signatures | Transaction Size (Bytes) | Transaction Size (MB) | +|-------------------------------------------------------------------------|--------------|-----------|----------------------|-------------------------------|--------------------------|-----------------------| +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_2-16 | 1,575 | 107,592 | 2 | 1 | 1,399 | 0.001399 MB | +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_10-16 | 1,240 | 126,632 | 10 | 6 | 3,303 | 0.003303 MB | +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_25-16 | 1,142 | 162,352 | 25 | 16 | 6,875 | 0.006875 MB | +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_50-16 | 16,260 | 221,852 | 50 | 33 | 12,825 | 0.012825 MB | +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_75-16 | 13,120 | 281,402 | 75 | 50 | 18,780 | 0.018780 MB | +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_100-16 | 7,336 | 340,902 | 100 | 66 | 24,730 | 0.024730 MB | +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_125-16 | 7,668 | 399,100 | 125 | 83 | 30,554 | 0.030554 MB | +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_150-16 | 5,603 | 459,902 | 150 | 100 | 36,630 | 0.036630 MB | +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_175-16 | 11,050 | 519,422 | 175 | 116 | 42,582 | 0.042582 MB | +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_200-16 | 9,553 | 578,922 | 200 | 133 | 48,532 | 0.048532 MB | +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_225-16 | 13,170 | 638,422 | 225 | 150 | 54,482 | 0.054482 MB | +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_250-16 | 8,286 | 695,390 | 250 | 166 | 60,183 | 0.060183 MB | +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_300-16 | 15,820 | 816,942 | 300 | 200 | 72,334 | 0.072334 MB | +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_400-16 | 19,650 | 1,050,890 | 400 | 266 | 95,733 | 0.095733 MB | +| BenchmarkIBC_DeliverTx_Update_Client_Multi/number_of_validators:_500-16 | 22,900 | 1,292,942 | 500 | 333 | 119,934 | 0.119934 MB | + +#### PrepareProposal: `BenchmarkIBC_PrepareProposal_Update_Client_Multi` + +Benchmarks of an `8mb` containing the maximum number of IBC `UpdateClient` with different number of signatures: + +| Benchmark Name | Block Size (MB) | Number of Transactions | Number of Validators | Number of Verified Signatures | Prepare Proposal Time (s) | Total Gas Used | Transaction Size (Bytes) | Transaction Size (MB) | +|-------------------------------------------------------------------------------|-----------------|------------------------|----------------------|-------------------------------|---------------------------|----------------|--------------------------|-----------------------| +| BenchmarkIBC_PrepareProposal_Update_Client_Multi/number_of_validators:_2-16 | 7.457 | 5,574 | 2 | 1 | 1.0729 | 389,819,345 | 1,469 | 0.001469 | +| BenchmarkIBC_PrepareProposal_Update_Client_Multi/number_of_validators:_10-16 | 7.464 | 2,367 | 10 | 6 | 0.5564 | 210,605,480 | 3,373 | 0.003373 | +| BenchmarkIBC_PrepareProposal_Update_Client_Multi/number_of_validators:_25-16 | 7.462 | 1,142 | 25 | 16 | 0.4047 | 142,106,425 | 6,919 | 0.006919 | +| BenchmarkIBC_PrepareProposal_Update_Client_Multi/number_of_validators:_50-16 | 7.462 | 610 | 50 | 33 | 0.2432 | 112,364,505 | 12,895 | 0.012895 | +| BenchmarkIBC_PrepareProposal_Update_Client_Multi/number_of_validators:_75-16 | 7.452 | 416 | 75 | 50 | 0.1357 | 101,405,415 | 18,850 | 0.018850 | +| BenchmarkIBC_PrepareProposal_Update_Client_Multi/number_of_validators:_100-16 | 7.453 | 316 | 100 | 66 | 0.1573 | 95,833,915 | 24,800 | 0.024800 | +| BenchmarkIBC_PrepareProposal_Update_Client_Multi/number_of_validators:_125-16 | 7.460 | 256 | 125 | 83 | 0.1653 | 92,549,255 | 30,624 | 0.030624 | +| BenchmarkIBC_PrepareProposal_Update_Client_Multi/number_of_validators:_150-16 | 7.445 | 214 | 150 | 100 | 0.1804 | 90,046,805 | 36,549 | 0.036549 | +| BenchmarkIBC_PrepareProposal_Update_Client_Multi/number_of_validators:_175-16 | 7.432 | 183 | 175 | 116 | 0.1916 | 88,172,820 | 42,652 | 0.042652 | +| BenchmarkIBC_PrepareProposal_Update_Client_Multi/number_of_validators:_200-16 | 7.452 | 161 | 200 | 133 | 0.2167 | 87,153,710 | 48,602 | 0.048602 | +| BenchmarkIBC_PrepareProposal_Update_Client_Multi/number_of_validators:_225-16 | 7.430 | 143 | 225 | 150 | 0.2065 | 85,919,620 | 54,552 | 0.054552 | +| BenchmarkIBC_PrepareProposal_Update_Client_Multi/number_of_validators:_250-16 | 7.435 | 129 | 250 | 166 | 0.2292 | 85,187,130 | 60,504 | 0.060504 | +| BenchmarkIBC_PrepareProposal_Update_Client_Multi/number_of_validators:_300-16 | 7.450 | 108 | 300 | 200 | 0.2440 | 84,173,555 | 72,404 | 0.072404 | +| BenchmarkIBC_PrepareProposal_Update_Client_Multi/number_of_validators:_400-16 | 7.426 | 81 | 400 | 266 | 0.2959 | 82,411,590 | 96,204 | 0.096204 | +| BenchmarkIBC_PrepareProposal_Update_Client_Multi/number_of_validators:_500-16 | 7.435 | 65 | 500 | 333 | 0.3309 | 81,605,510 | 120,004 | 0.120004 | + +#### ProcessProposal: `BenchmarkIBC_ProcessProposal_Update_Client_Multi` + +Benchmarks of an `8mb` containing the maximum number of IBC `UpdateClient` with different number of signatures: + +| Benchmark Name | Block Size (MB) | Number of Transactions | Number of Validators | Number of Verified Signatures | Process Proposal Time (s) | Total Gas Used | Transaction Size (Bytes) | Transaction Size (MB) | +|-------------------------------------------------------------------------------|-----------------|------------------------|----------------------|-------------------------------|---------------------------|----------------|--------------------------|-----------------------| +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_2-16 | 7.457 | 5,586 | 2 | 1 | 1.0388 | 390,490,985 | 1,466 | 0.001466 | +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_10-16 | 7.464 | 2,367 | 10 | 6 | 0.4714 | 210,605,480 | 3,373 | 0.003373 | +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_25-16 | 7.465 | 1,138 | 25 | 16 | 0.2771 | 141,904,565 | 6,945 | 0.006945 | +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_50-16 | 7.462 | 610 | 50 | 33 | 0.1598 | 112,364,505 | 12,895 | 0.012895 | +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_75-16 | 7.452 | 416 | 75 | 50 | 0.1227 | 101,405,415 | 18,850 | 0.018850 | +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_100-16 | 7.453 | 316 | 100 | 66 | 0.1112 | 95,833,915 | 24,800 | 0.024800 | +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_125-16 | 7.462 | 255 | 125 | 83 | 0.1012 | 92,509,080 | 30,750 | 0.030750 | +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_150-16 | 7.441 | 213 | 150 | 100 | 0.1035 | 89,947,710 | 36,700 | 0.036700 | +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_175-16 | 7.432 | 183 | 175 | 116 | 0.0878 | 88,172,820 | 42,652 | 0.042652 | +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_200-16 | 7.467 | 162 | 200 | 133 | 0.0974 | 87,369,345 | 48,401 | 0.048401 | +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_225-16 | 7.451 | 144 | 225 | 150 | 0.0789 | 86,194,935 | 54,326 | 0.054326 | +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_250-16 | 7.428 | 129 | 250 | 166 | 0.0775 | 85,109,730 | 60,444 | 0.060444 | +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_300-16 | 7.450 | 108 | 300 | 200 | 0.0879 | 84,173,555 | 72,404 | 0.072404 | +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_400-16 | 7.426 | 81 | 400 | 266 | 0.0616 | 82,411,590 | 96,204 | 0.096204 | +| BenchmarkIBC_ProcessProposal_Update_Client_Multi/number_of_validators:_500-16 | 7.435 | 65 | 500 | 333 | 0.0596 | 81,605,510 | 120,004 | 0.120004 | + diff --git a/pkg/appconsts/global_consts.go b/pkg/appconsts/global_consts.go index d8cc1bf87c..1bcdb7ac6c 100644 --- a/pkg/appconsts/global_consts.go +++ b/pkg/appconsts/global_consts.go @@ -48,14 +48,3 @@ var ( func HashLength() int { return hashLength } - -// The following consts are not consensus breaking and will be applied straight -// after this binary is started. -const ( - // MaxNonPFBMessages is the maximum number of SDK messages, aside from - // PFBs, that a block can contain. - MaxNonPFBMessages = 200 - - // MaxPFBMessages is the maximum number of PFB messages a block can contain. - MaxPFBMessages = 600 -) diff --git a/pkg/appconsts/prepare_proposal_consts.go b/pkg/appconsts/prepare_proposal_consts.go new file mode 100644 index 0000000000..c933aabd68 --- /dev/null +++ b/pkg/appconsts/prepare_proposal_consts.go @@ -0,0 +1,15 @@ +//go:build !bench_abci_methods + +package appconsts + +// The following consts are not consensus breaking and will be applied straight +// after this binary is started. +// These numbers softly constrain the processing time of blocks to 0.25sec. +// The benchmarks used to find these limits can be found in `app/benchmarks`. +const ( + // MaxPFBMessages is the maximum number of SDK messages, aside from PFBs, that a block can contain. + MaxPFBMessages = 200 + + // MaxNonPFBMessages is the maximum number of PFB messages a block can contain. + MaxNonPFBMessages = 600 +) diff --git a/pkg/appconsts/prepare_proposal_consts_bench.go b/pkg/appconsts/prepare_proposal_consts_bench.go new file mode 100644 index 0000000000..cc8141f791 --- /dev/null +++ b/pkg/appconsts/prepare_proposal_consts_bench.go @@ -0,0 +1,14 @@ +//go:build bench_abci_methods + +package appconsts + +// Note: these constants are set to these values only when running `bench_abci_methods` benchmarks. +// For the production values, check prepare_proposal_consts.go file. + +const ( + // MaxPFBMessages arbitrary high numbers for running benchmarks. + MaxPFBMessages = 999999999999 + + // MaxNonPFBMessages arbitrary high numbers for running benchmarks. + MaxNonPFBMessages = 999999999999 +) diff --git a/test/util/test_app.go b/test/util/test_app.go index 300d020e9d..f9f40ef91c 100644 --- a/test/util/test_app.go +++ b/test/util/test_app.go @@ -44,7 +44,10 @@ import ( const ChainID = testfactory.ChainID -var GenesisTime = time.Date(2023, 1, 1, 1, 1, 1, 1, time.UTC).UTC() +var ( + GenesisTime = time.Date(2023, 1, 1, 1, 1, 1, 1, time.UTC).UTC() + TestAppLogger = log.NewTMLogger(os.Stdout) +) // Get flags every time the simulator is run func init() { @@ -78,6 +81,7 @@ func initialiseTestApp(testApp *app.App, valSet *tmtypes.ValidatorSet, cparams * // commit genesis changes testApp.Commit() testApp.BeginBlock(abci.RequestBeginBlock{Header: tmproto.Header{ + Time: time.Now(), ChainID: ChainID, Height: testApp.LastBlockHeight() + 1, AppHash: testApp.LastCommitID().Hash, @@ -99,7 +103,7 @@ func NewTestApp() *app.App { encCfg := encoding.MakeConfig(app.ModuleEncodingRegisters...) return app.New( - log.NewTMLogger(os.Stdout), db, nil, + TestAppLogger, db, nil, cast.ToUint(emptyOpts.Get(server.FlagInvCheckPeriod)), encCfg, 0, From 0b3c5165f45d897091c745f4cd8a3e9f9143d703 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Mon, 21 Oct 2024 05:00:00 -0400 Subject: [PATCH 83/85] docs: release-guide link to release notes (#3989) Closes https://github.com/celestiaorg/celestia-app/issues/3913 --- docs/maintainers/release-guide.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/maintainers/release-guide.md b/docs/maintainers/release-guide.md index 33d181aaab..698799235e 100644 --- a/docs/maintainers/release-guide.md +++ b/docs/maintainers/release-guide.md @@ -30,7 +30,9 @@ The target audience for this guide is maintainers of this repo. In general, the Follow the [creating a release candidate](#creating-a-release-candidate) section with the following considerations: - The version tag should not include the `-rc` suffix. +- If the release targets a testnet, suffix the release with `-arabica` or `-mocha`. - The release notes should contain an **Upgrade Notice** section with notable changes for node operators or library consumers. +- The release notes section should contain a link to https://github.com/celestiaorg/celestia-app/blob/main/docs/release-notes/release-notes.md where we capture breaking changes After creating the release: From 0ad4d72417c011a3b175f03f47c16d93c632499e Mon Sep 17 00:00:00 2001 From: CHAMI Rachid Date: Mon, 21 Oct 2024 14:47:20 +0200 Subject: [PATCH 84/85] chore: rename megabyte to mebibyte (#3994) Handles the renaming feedback in https://github.com/celestiaorg/celestia-app/pull/3904#discussion_r1808009238 --- app/benchmarks/benchmark_msg_send_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/benchmarks/benchmark_msg_send_test.go b/app/benchmarks/benchmark_msg_send_test.go index 91557e39f9..7e79db9a67 100644 --- a/app/benchmarks/benchmark_msg_send_test.go +++ b/app/benchmarks/benchmark_msg_send_test.go @@ -307,8 +307,8 @@ func generateMsgSendTransactions(b *testing.B, count int) (*app.App, [][]byte) { return testApp, rawTxs } -// megabyte the number of bytes in a megabyte -const megabyte = 1048576 +// mebibyte the number of bytes in a mebibyte +const mebibyte = 1048576 // calculateBlockSizeInMb returns the block size in mb given a set // of raw transactions. @@ -317,7 +317,7 @@ func calculateBlockSizeInMb(txs [][]byte) float64 { for _, tx := range txs { numberOfBytes += len(tx) } - mb := float64(numberOfBytes) / megabyte + mb := float64(numberOfBytes) / mebibyte return mb } From 9322d295c6325290cdf80b38f785bd418b9dd928 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Mon, 21 Oct 2024 08:55:33 -0400 Subject: [PATCH 85/85] docs: authored blobs audit report (#3993) Closes https://github.com/celestiaorg/celestia-app/issues/3992 --- README.md | 11 ++++++----- docs/audit/informal-systems-authored-blobs.pdf | Bin 0 -> 21393 bytes 2 files changed, 6 insertions(+), 5 deletions(-) create mode 100644 docs/audit/informal-systems-authored-blobs.pdf diff --git a/README.md b/README.md index 8d94c06884..7854890901 100644 --- a/README.md +++ b/README.md @@ -169,8 +169,9 @@ Package-specific READMEs aim to explain implementation details for developers th ## Audits -| Date | Auditor | Version | Report | -|------------|-----------------------------------------------|-------------------------------------------------------------------------------------|---------------------------------------------------------------| -| 2023/9/15 | [Informal Systems](https://informal.systems/) | [v1.0.0-rc6](https://github.com/celestiaorg/celestia-app/releases/tag/v1.0.0-rc6) | [informal-systems.pdf](docs/audit/informal-systems.pdf) | -| 2023/10/17 | [Binary Builders](https://binary.builders/) | [v1.0.0-rc10](https://github.com/celestiaorg/celestia-app/releases/tag/v1.0.0-rc10) | [binary-builders.pdf](docs/audit/binary-builders.pdf) | -| 2024/7/1 | [Informal Systems](https://informal.systems/) | [v2.0.0-rc1](https://github.com/celestiaorg/celestia-app/releases/tag/v2.0.0-rc1) | [informal-systems-v2.pdf](docs/audit/informal-systems-v2.pdf) | +| Date | Auditor | Version | Report | +|------------|-----------------------------------------------|--------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------| +| 2023/9/15 | [Informal Systems](https://informal.systems/) | [v1.0.0-rc6](https://github.com/celestiaorg/celestia-app/releases/tag/v1.0.0-rc6) | [informal-systems.pdf](docs/audit/informal-systems.pdf) | +| 2023/10/17 | [Binary Builders](https://binary.builders/) | [v1.0.0-rc10](https://github.com/celestiaorg/celestia-app/releases/tag/v1.0.0-rc10) | [binary-builders.pdf](docs/audit/binary-builders.pdf) | +| 2024/7/1 | [Informal Systems](https://informal.systems/) | [v2.0.0-rc1](https://github.com/celestiaorg/celestia-app/releases/tag/v2.0.0-rc1) | [informal-systems-v2.pdf](docs/audit/informal-systems-v2.pdf) | +| 2024/9/20 | [Informal Systems](https://informal.systems/) | [306c587](https://github.com/celestiaorg/celestia-app/commit/306c58745d135d31c3777a1af2f58d50adbd32c8) | [informal-systems-authored-blobs.pdf](docs/audit/informal-systems-authored-blobs.pdf) | diff --git a/docs/audit/informal-systems-authored-blobs.pdf b/docs/audit/informal-systems-authored-blobs.pdf new file mode 100644 index 0000000000000000000000000000000000000000..b2ac02776a28cf5fb47d5d81bfe2cd573aa40bfd GIT binary patch literal 21393 zcma&NQ>-vhv!=Oi+qP}nwr$(CZQJ|Xwr$(C?fyG6ojH^4ob*kllDey8<$da1Rip|c zVzi8OtWczfM{`S1j0^+}1olQ&P&_>JVwN^8rcU%?Hij;yBBsXnCZ_Zd&Mr=-hPF^1IkoCrPFrjUE5dp2s2S7EvSua`Aa{Pwo9(jYnVrb8i*ifg5g?=_ z-Qxfh;y>|xulij8^-1)Iy76Od>BF~qyxxEgwmn)}b^$*8VpGdrJPL#V9hoqZzTne|+BY|JW>W z=rJxfZ}zkuFYp`qp11h$vbJzD?_%%3_*)UP|9Un@Cu#Wm1!D#|M(rvBI}u-t zo6p zT$gV&ZL~E}C(xPW?|&BO)R>Y1d~AmsFK_G|GOmiC+#Q6c}dCQy8g6vK38bpj1p{#ft;$4@L-X z52pC&G!Y6?C#mA8Z$siy47Hj-5Y;5*%O}l_B8Q3O3Wu;`lVMOvy{A?H1SFWKvsNS9 zgs>t07eJlnT&WSY-~?<)img?*LE_`Leo$;lMIc263)pRfiO@fQAFkdkgLnfOto^awyWh(6!lA+Z08Fo z&dAou;MLMglCagTn?bkO2+2U&_(Q8FJs=gID1{&*^0M8s=L^Fo6}2r z3hOnmMCxFA_8Hc!jy=i_0JdpmyE-fX44@W8YPSM+Oxnquo@Y9<6R&ASgqL`(eNuGG5h;T9uQs&TC z7svMpu7!-h20^SXgW=EwM>L6ew9vQ_R>N}Pl2;V)uw4rfUZl$rh#VLsZgNm%u{O)2 zT%bPmX)c54Yr6gj;Xk!m|1y$zB}uwgGD%lrzkg}Wm=h^FI<@&E#}^~sI`Xa9rtaLO zG>4Mwmh3R$zvWL)Q(o)1?1~$Dx#?E*pM`Xrw~ybAyjT2q(Q>)=uQ#XN`#iL%Qc+Si z)D4P4k=;~JL}pIPWJW`6l(9!n&VJ{cg{;1E&pxfXEU`|;=x(trQ}}vHoTTk*_>NeL z!UJa?#D<3D_Q&yl;l8Jv!i2O)#`6B_YeQYX3NAff<(i-8NIpp>=JcGocj#FU5*Sj9 zOY}t7&$#cP*NKr++E)qOewxR~1}zt9vu_jY7hm_026XOy$H;_e?;s`PWqCOl$arp_ zxcTjK<|79N+eyQgluQ$%qR5J*%`zn^ptP+v)QwWZqAit3YG!Ba#;t;q*GO;klnxPI z;95==Uq<#kK-J=O3aC=f=^h(0^&!5!;N=Kd=5&q&=%trmmGTxzIXxxPPNZ`>rzcJ6 zbL(eYoF~a2KMu|cKi94)lIer%A3OJw+@!|l^iA7NJ|lzuexOQ3Kyi#-E&M)yRQ!7_1_SO1o{+d7kK9??03-ZkarA-??U#&U4Efrl0p{WJi-!ohtDEXB;n$fuLyIcSa-KC(-cU4s_xTOnHp( zcF|I%f{C_;<>XxiPP?LXqd-)oVND*9OcC_b*bTun5o3kO6taP30gy7|McqPiigM^z zN}pSzC_wY)gX*}TEY>)`_Jo_dMMd*zOS*g7`C0byS9&fXD0uyMQ)xyK)@duz6?*3& znA|4M=Bn(%Zldp!9@U)vR)w?6N9roSTrS9(S~bopfk=s^q@o(T6V1D^YNel8)*fVL zt${N_DXdLEz*Cq3_j#A$p`X>rn0ZmMGNS9dQ8BwPTlHB;6#OAYh9k~2rHeUE2OS(l@`t(a|42N_bz$tAUOxD=KxbwRLSj zE|zE!I8BvUCMsTEfGeQ4h5*V&bNo?oFVPy>Yr)FC$0b6{#oUx~$ z!cAp#41fW--OdB%3s8x~a)_jWX4*-Yx(t+%j?l<5ggXJiQCr?&xM0gv@+<+bQCGfB z2&tVw=9F=?u)8o=7)a_T6C7X}lr0_`HS868;2SI6 zGjKKV!}y~7BEvj5Mv?+Mfx#DFTaS$5iw=gkFdc3WREQA0I2}$OS!&2=t0{1i2)l>E zeLQ&9;MP>kO6XwRc`w_*JQq4AJJQk^uvUC>Fl6YWe&@0b$x3bD=VaX9I##yawQS6u z2Exn)-QixP9Q~RKf))+O=ZfO9+uUEt2pg_f(ta477XK7RWU6=5hCV`kByo~rjY{b zMiU1$XYfW~iNy`ZI0cNeiasBT*Ds4TtNBxe3p|HY=D(BjMVuz#R(&7GqRr&tpdaAE zyJ=1{PN+`8r`g zahx@A1X2%zQ>(B(e7gV)a0!GP7&b_&Uv8{*GZRZM{<&hx3eZya@qxjSqal+@+}_py z>seg6YinflD%EPzg$Lp$3K^=563`?CNSP7o#DGT=q9I4Wh2El#YTq2nk)pZ0Ifiys^C73|cre(VrR)Saay}=9t zx+_0EdNo?~@SyyB6Hl)cApIFlNsWr8a2}nq^Ai~%hck`SZb$)7=rk0$$!neVCAc;$ zO(>~oOo2eAw)|D}O87a`+yPILKdo-ui|s8}^yq0;un%SNNuu}YU0({~bBZnF@UePS^#2V6B z;sfI5^g%q~M4E&%Lw0@1G!DGn>ElhCk-oY^K)xU%rB9hLCp`b>T z&#s~}1|(j19K`3uq#CcdVofnJ=E6e*#H(dR{mT%rypvBMvpo`8J39)r<@Ei_AkjMJ z;qbq8H9QBekfVWCbRlq!s7~j{*q9?4TxuK&RwiSt5D~)GQ%nPW1s!E92d)BepMGd4 zb4$*0c_5 zs`~NwO~BFhtr9C0GZ{Z0Hj)-Ubx*@POQOe`wbRu1t3wSz=dKob`I{BM_kua3BaiDC zZkHR$YyjmDuNt}mWA>xb`Q~>Cm43V>{4QHXDw>^wlgoeV>c4@?eG~`kUSMnEp?Aig zn&#`SHxEv2<2>WE)_HlwnGJC3y**^!u1ek+9;FZ|LoJ$1>vT zV0N|WJt$=F{E%44Kh`~-!7#V0&*=LD)t&fJ@c;CMBWtIR43^+rE2-LC{969L;qzAX z;{SK#X8AA3{r^}%j0EhAEFAxhMa0U$$njtJ`$of4M@99^f41XoHsA|J`;=|y+@Q1) z1~}48E6dVejH<4(!or9MiX=LSGKc^p8;EK@DQ!Q@^wNMTr69?&=n#ph`@)*4u7ki~ z?mN%foTd6L|Lt#Yd+sf-E9WW4+&_<706+j1I8f;BVnF$f(+1lcIG+Xx;gUZSnbkQ# zJsC_O2ejZJht=51;OTnO0w7%{=ZF^?znMbqWdBFA`;Lp6gXgi^N{3*VegAWllW?gV zuC4b9c>Wg%z!QOo)hTaY*8$|*0qE=sAqW@uLFU>RI;RU&Bq`|aj?f&5V_(%eA{6!? zu3Nt$MY!@kemk_A z+J+UV0>wqbqF6~YveZ7sH_+Mvo)XpvyOdmKx(hC&CTHE_$I!!`Knx%@6Tj1wX`+U1lx9gzrKHO z#6U#Hec!KC0g7J${sBbF0MhaTWy%$c${;2Ov@{{ytRQMmIJ5&|njnu(kbWT+oiMCU zAczJW`##xrkXJjbI{{*nLfhLuA5Q?k0Vq$1nh!A7darUlx*KT}pDn^QTdZhA1gsm5 zh>eJixQzJqzas=A@nvt$|yz=yt$1 zgN|LW_oM;4J7DWD`*kgVtp!{)YyyN;!mWUBq5UHk>6^>0MX~q^J$zfuX$rr~WT$rgKkws}(D%)EM z^dbx#nIwKhbh*M3P?ke;N+jCSwfHlcO~K(AaKdif`Nc{ z&njlHewBXikw*w7B}?PWiNl5_o8!o9;mfQar68szk~t7EdFpsw8H>g+8G z!K30k+D6kH`x0uO{$?@pRpt1<*){~ld$DD}T$w!-FZYfsR+V$+Z|R;jyaHn_KoHgM5biu-Sb9+vGhisU#6Dui5-D zsEq=63%MGw4};$Unrjn7TVpm(j%uVJvS@wTvsSy&(!X53VxY5^VqNpAF7)wUL7&C7 z8`8(_o21#w5>0$W&RkK7Zp-^}pw8U26UtY|2c|6T)yr7s>dED8@#{rp zzhbo-&R=S=2AXM~BT88*IVkk=25tekg6T}Wgu&z3*D%!fzE?h~uw4rJ?yppP*geU5 zDX)d}LAwQs(>pTHMvT&Bn{tO2CAH#?qa-ZZke&`lepY0=yJn{5=)vmh$mx*?Jr-7e z=`7Pv`iQMYv7r#?{#D{@v1NUuOS{o)U*TrQr8xE+ZC^axbE8kZh0ZLpwY$B!rxFQ&^c+iT*sE&Ii}y9Mu0Zr?qlWv+qUi%o_J4 zk%b^hX7(I!7U7k9FAQn4e2)29y@%9FhH2$WD#V#5_rtO50Bzs%=)htHgBvp<^or4t zsHmb&2(pyO=m|~hj>XuvB*nm>-?G0Jqi%EdS}a2I+B=cGf|7Sd=?TiU(9N5}7KV?ikL-&r2cXZYz-`IfI2OK3HcI{Y-36ZkX~s#_cWwXH90a*?Uy#%Ss^ z(;2sQaL13DO7s;sW9JCQZ*^d(6n%cc8L+hl+W~29o@8=`!0*s&3~*kb7w+NZ^2}sS zXfRbdO@`5B7a1unmF{>N3GXny-rL}M3xG`sOM+1*9D5cTgWXl`_Hj)GJ^BP}o`*da z%s?_aw4ruQX!{i$WnYCtFM7v*m8Ls+;AV&Wv?$Wlhp%~vY}gbFb*88T~N6(dwBZDenTx#xBqj` zzaxfQUG?qTSIFNDb$-=vpYLY$vXvV`uRNTZ8WK)QvQkpu4pgh2FK}2u^;CG+`54s^ z#!7c1z??o8-eCZ{_Pe;Ig1O(pK=3v&8;=}@me)kwT39*SGYVpq z-ZG5Fl_#Th`H|Ciq_+NXS^r%;+OyeQNyzEh6BQ0~pO7E6gifrvoi=*QQrkD}Exp}) z>^F8tH#Q$*%^L`;6D&_kK07_R5GvnF5}%88aLQ9nn)6;$dmTWDn}!bX{e|SH6OoMf zx^TrN8^3R}aj;{NR)|3=Nbe{Bz11xP3k%#)c4AiGJ>}~Hz`Ac;9$ZTNr|$Yr%^1C^msiaB& zoIFul7&4M@|KaTg14b+@6vc{^8u_$&yqH_k{CS}X!}@uc%8}J%u{^KLxp6%pFG@2V3yIO^a6ULq*f|@Fqt$CQ zQ(Q4`CwEIUYDIE)gBQ9oZ^8@Lly@vEvF#?dd~!td>Y>jS?bY^Cf&1p&rHDf9z2tmr+}l@cG$16Frghg#ai5Wb=VmXxl4kEdzXiW z#`#oyZ3oWQ%>K`Ntj)RCn^|aB@_>~P%)LywVVl=`01NI4$NEt`s$H^5QL0~cP>t%W zCgq=VrfHrgroN=Xl>GYsejBbY$^vsE;}rvajGw59nphl24y_SCkW6_tPASJb8Q2`z zU~M^I6*X>^E<6T7Jq1zcGWetl*If*1>`2#CejicwzT>`jl7Q9fu(Q=U4w2Lbjsy>+ z`-n%Tzk$*ERgOFm0Da^EwcHcV+ZvpbIx~B85srh^F<-tOmAxxIcrIVY=R+}~bb+D# z`)ws9FFY?++(LG)c%Zy!N%3M8@*sM!Qn>pg&EM8no^WM^y!Z{|>2jze)dR{e_3lx! zZ4exmDqhWw75fxqt=9MOoXE^>6#RA@X%<=f7IAabi%WBJlW-yX1i!^TQE^M2z_+1z z^7h`gEIgc=W}{wn#6OSMR;|!+K%XAf5J@o#qb)B&*d@~0Yprc}*w?1@$|RTR^3z>U z!)km=Ae&;e(6X><;y+^!h(*LB$iMM4ZJf9>UZi+I#Fo>mnbASz^S2_}H?#DNFdkT% zoU{4@6|vFJ!&P!RS94r_)Fu9bPA`@+dDN0JQMNq^?r$K>hU6c8k};RVzMFAG{Fy|@3l5Vp-^Z_TW%$a68BeWcxHDL@ zAa{3bs3x)mA@!iX+B;X*b5Z()#V|fGb{$L7ExK1`c%9*vI>cQ(3|t}|--copa!21P zq~_pWHeL1e{LrKxK~zr5zVih$IT`bA?-?O$?@tDiE| zgN@s^^Tp$WJ)ORi6BslCH+71>UmA0}+)j#yC!?jyue;@jk96SC_YwcE*@Wyim)COZ zJ5QD8AeHek`C{ofid!UW)YoiXTDDs5K}z{QfP-7+$vNJwf)8<%JsN6A$9@|XpW`r6 zdcUc4$84m>$^|JsO{8s$5+jm}rE3KrwqPsexs4XdrbIhyJx&`p#dX)>jr!LO?(s(P zjQ19Ial=%vT$mzz8Lm5Uk2KBzYmb%(CO6+z*WcRR^-UpdcDnhG>(4a!E4Q0IwQiHm znE8!i)31s>TIu+iHi?Ns!Dz|!H}MyI+*|a%zxgWv8C>?~+yUTGkyzQimxr_f$${S- zd2wrATw>Vcjz9|!It39o#VEizRIJACqCxM3+gL+kw2`z_NnlOmDjwJQ0yWK9P3rSc zm7;euA}76AzWF$JCLYRyK5plgLaNYPXSuugIE}bm-!-M@i^C(+W)Zwc4t)nJsW*7{bcpR7sH`cvd0d*{@9sybiLw~GFj?r7&jK?(!rXl&jN_rR0n{i@N?Cz6a4 z^#xrL&0a!%vg(cUWr3ad-nwhwLWs52mfBJgP@j_n`ql6{M-`sr`#;n4TRic{KU{as=YQ*&~to3&xy zFZWoiW_Ybw4_B$J-}r)_;K%Z;5qA*Wa;eHM;gDxElnBZQP7i#rg+)Ay0$#&>4(#cqh&geAz$(nW1Lq58 z4T2V>QG;XOY7dO>VW)HAmoStGSgxwAq3rgudJm+{C{%_pw?LQ!-t)NPVg@)cs1AZf z3Lk#JwK0LHb!C4tVMb*ek;Z*tj=Bby@+Y$7G_i0TEMO zD4D{&gNh+Wc$^Ue2^1^AdZc~U1ITV79B_4P-MkC zf&~!U3EFWK|8U6hJ(v}S3EP!eYnGE2JcpsPH-=4+#GKug0rkEC89rL0xVZT*PY!01p zX7B5C#w>&KiKuIYWJIF`K8W~4VHII=?YhE#Y9LRk(d z@ywGC+2j5{BbAd4a@&z(@P8a~k>w?6N%1VgQt5FLYbv=az$(V74OOmH^{Qv8u$Ok1 ziI*C#ah*y(HZ8nEqqAOX0>4oTZHCvOk+e44Vo-bix^88jf`YXkr?_w_s-|OiyND3 zQ974g$9#E9?DtHgN7V<_U@ZNcsY6Y>JkTgFpc}qKo-Bs2*JcQ8X9jFb$Hlhlvh2OS zc=?-0(kAPlYtkRp$E3--1KMVP=-feBw?K$7lKw)?!@(gTRps^vYvW~C&bw9-OsZYG zrsjXn>h-B-JsQROi}4MZ*zV{4tK1naofFFS`YM}T+>d+hbBDTX8$;SYd#kEUORtwj zD_dlI-f5ErPe0(cadwX6{-)8V{QCG+85N`m44+C-S5RX>TYPM>( zF%YEr@RrBAGq=2tl9g>MI-5TWs`gK#t>$NaXLhHrck97dp5FkIJ2bs}F7Iz$>#=M6 z%#WFpetn6Q5JMu9HK=mt-BTwK(5r(q2Vw>Aq7~CsSu}E8x#amv-fKDjA5ZnZY#-5^ z=-Y88c0bXLrv|@00PYk`&zgI$D_8ngy~k~xO8D~pv~t&WFx$<+ivaKs`_xf-#c@-J zm_-{bU5K@jSa59Kvgq0r-8KJQ61Q$&qJ(9=Qs*Z8zm6JnQV8)g6T@?#lf`5OVVGfmM zYRho>#xGR0fhI;P7Q;=9$+JDaOuBL#F9?b@z6nD@!=?~>H7!RE=IZySUixl5PX~X8 z{4M=ILP@F98vebV>Oa=;_ik4&*HY+Ak@52KA9Oq}!-n&3aMhFiL~2UL(^Y7+oBkd0 za%PTLishUh==BOnmOuqr#xZ1{k_%@vL@GCN;$nSGL?YK^Cc@9<{wTSsrzcgwFMm0i z&*F5^x@gn-H|14^p|gwn$;plF2&5Qd$zDUTN{EsmGm4h){RMc&=Vu<-el2XucX@={ zzw*09CJfvj7+#;cIwk)zf_wD|^dv`ywQu1G(jKvo*$#~{F*shH-CP{Oq?T%Ak`C3; zP|Lw=`B%b{yLxxzdIx;|TDF~yn{w|S10C?-qPK>} z@T~b7Y!5#AHJfD&tfsUk6NBM7KC4o)8?l#CG(1uuWEf(HU>ZF| z@`Mt20zrL{u!fT(2yM|WBv7~sJwa#aiN2^jOlZ)i)DvjU36k-Jfnra_fjrh>ZH&^x z#<(rhu_ylF9t>}5cjb;yFFrU!9q^aG z_{HidZ()@Orh zD;>epMWuCN{&PF&dOIdMp;*Gp0sp;>4YNW({ORbUm`s!S@bcgr^SqufMSRCoUHjNE zOPy{pzfoy>O}dVY7oS+7mwB$V(BmVn84es9_QtY<4e|9Hv%bf!rHe1Y?WXG)?B|{y z5asc?dV{!VXMT=?+ztbPZ3A?^m&MLCH-Ws7;cT8-xQ-Y{Io$Jw{bQ}z-WGI|SP1B> zvws1FtbeUfNsY5RH!na)r2(=NNJ!B+DM7+)52R4Mk*=8hILu$gbaAu}a4&8rbbhP?SdxKeK~cLL=re$KybK%#mb7NfAdV!yC_4 z;WAE9UcF%bSV$cU^eQ8qfn>EHPf{oxUOlbzotc|<&z?U28u;~BIp1!3UN^1(R{73= z00D@f!Jq~zl}F|5Tb+mF-zRtgD&34ub0GhCR-G9Eq`@7uv-tdMw3a%eOaju@8ss&& z{N{Q36~Oiv5bwB4O8ec4`hyZy1o)=cE2al76oyPyGif&s>7D2<)Vq;W}+12&?5 zK$D`~-t*gQxLYwyU5FoC_<5!17wc=5 zFU@T2Shcq_d~4mT16B-Uo5i%7YPxLPy2#a4MO6V=os+oV6^pSg0i-= zI#@Yb!>!4dW2!E+boF!9TjH*VKOny!=#e7D$_Dp-xIs$qDRIJ>IDn!V#K|3W%L{!^ao5Uf~+)vK<|maLDmY;F89qfpr#nw)(*h7L$nnj z+6qH60>rxhW9S5GMyAQ*PPCMFKnSySas=22xe->5+o9(Zcqaspv)F~J3QB!%k;&=H#_P?cZ_!V1Ga@;V`Q92!A7p?-|K{J5fnj>8_q z5nUGIbwtyMI;0Ca#vaf?A+C(r%1E>0YQ(u=AjPj$n0M-nzZZ+U8lY}QbN$1*QRlix zDoeiX$7;6Y`iExgjH?m93e9-*GG(DnY~M0rt=PvR_Rmyb73`0XGWndYDR1IMO9WT7ewQ zL0GnB@PRE^mI={_C@GdBQueaKrQFo;+q^fv`8`f52f2#ENm97MM&)$rS`2wcQOROb zDT}B%z4})wEl=ShG=gM!nRHQj-AW}v8+G{jSj#%wYsNsPW>LZ97(NOg zVXnG~3LU>s3{U;i%e-}>dtWdQF2?7#7`2O!i1#Qj`X;f_(c}ZX+j)Vx>@i~Bw?1Rg zXk>fI;Wn>*Tx}k$*gxcR12~q5o};jQLVwzxrSRY2kj7kgGoPX&zm@GD%-(d*z99+idfd`UAhB~n4_hn&7jH|Y9 zTP^BBvAfyeIWK@bZ@q%NctrfVa-!fRJrvP)2(yT<_wh{R@GLejlRHkgluhy|JtX`T zzvvGt5=*w5O@Jn7iS$o(ws}G>bI987`3kymtAMvkAa0I^l@=cjoBhGkm_fq7Lz~$E z0IpYaKeD&WPbyWtY_J37_>sG@QJy|>{#b_3`V`^5yJ;&-$u)&%AXx^30$GJ9By*i3Yp)6ZJndn z&4|e|q8P?7cyqUy2HHHcy43J38MEMdIfHf%s2sUO&~hJs*szSw$2TQygrTDR-SJ;C zCquJ^novt^xIKc_4Pz)^aV^79{DEwSa8;EiIwToxO*U(IOyZ|Y!eTU$dn=ldMj<}M$l@+>(pYOHSfVy% zhi$zNs=OS$&kcD$?v@)s6XNj-^UKOx~wMel9j=>bJgvcEqc#!F;P`mVy;Paa-;j$`Jl4ChBI)5rh!g$?f?whW zQ&Y3)JYG*FG}!HBm2rE0Opo|{GQkiw>di*WyRf!4E7a++e|YCzsrRzeML{EG?>s64 zHDhh1BS((vTJ7fD&fQV!|M|>YKG5+B@?yPTV&+Vxmzz8}^Gr4rcI1lByq=`ytrdkm zZsT2jl$PgNDP0fjm6;Jk0p+K2wPTaLm?c8**+vrQjYQQCEdF*x$(zN^`2^c~ZkoQn zxjFI@1LtUQf-1&0H+k6Bm+Gq-!y$viV0!%Szw`W9T;d!~mX(T>(KftnJTfvL(^PuD{8t!Hed>kz{VmXzt8eRZfV+P1augP&ax$>aZ7w=4JcosTRl$Cuq zF%?pudCB`c0_tM_3*h+Zlf?hkydczzdxb_4yOPvDF+bsv-0!jwzoaxv^1vbX$}#Fj zG=_Eg+~2^8lPftQiGjD`ZS7RgFEAALz2?aeNnSfI6WfcBs+T4qVTyd2-(9cFH;NqfzYwGlQzj2ZcC|Jh<+qF?pRp%qoO#OdgRu7N}?k z?djx>LB~xHPb)Hd@Ixb!*d~Z48?n!iv`H(B@hpid(!)?hX7-POh5VkYu zw~#=}4=NT6<<|(>y&+q04Fpqw$Z>|DV-M1tf+mpAj0Uj-RcJ#X(j>|d3o3{*C45-O zUmNsl1N-R>xOidA5Sq0jOM_N+;96G*_cc4hN;lAJq{An&U~REAl>F@LfNreeY(eP{ zd*jf9HYy;*oq#6Hz}j!u?8b>fW+;c&NF-renm~mUsAkJ(6dINBD{{&>oJw4hG`AE*N zyAKQ*W+O3>GlucRh~HBf|4kHQq?$nA762qnSyu{yv@Sg#O*?uelp)wPfOI5?VGI-Q zQHgiDh?_vG8*r9caWvzguhR(LZDj$iEWuim+=L*RMIdzvx>SmBPA_U;XH_833--Z+ z&U8V7K~xK=LqPCMQt_+dc|LA@E5!5P=*iXoMJk z_0)S_pMM`YOwbT9^rER>=vb)fS)o;Jc~dWNW7+8qkX3Puyea%e*Z9?YiYIn{1IW?? zDf@#e*x8F0;-=gdtyAm^?D^B%`k&gRU`QQ8tz3hj=q;md?r-?&RbO*Ee>2QCtrwC% zIRCN)cI}H}i-=9~#ABbu$uS;Y)PoD;j#OV-#Bp`Y?4pEaUb8S)_zl_BG*}ZQ4g8jw z+UwuzTejcrkgi5-p9A=jy+;oX?{GCZ?7$iB2fQbQ?au#UQe*qyCp8u(MyCH}Qe$Fg z|F2hHH|mxS+AFAhZ@eQ>N3@Nt60OH?&&^X>`DoN>rrK#XoDXChqj5Lu>yuJQMiLT( zN(N;(Tw<}%deP@2?5CG$afEf2DY_V}`yveIcOVqYHQKzL zm)jFfS^-5r?{Xfvaws;z{73J;IN58qo1MkC3M^0x-3V(M>~^!UR(#5904bjiHk&M+ z#yPM|9D3Wqch2K5Owl!ieT-(3=qW*wA=7= z1Jn$F{=$-)r|is0DceYON=YdDMSp@wnOwn*X@D|40pBpFJT1VU+)fw-_%q;-W!;8G z>_4W5oskWZXJlbyx0x0>_j&C%!{d<$?)u(u|Bk5zkUo@-rxNNxQh?-%`+~k9r~P-s z;SUb+(d>fm`sem3yqs9K!}NY2{GzOTLErl2JRsZ#+79R`4v5=9^Mde)i0=WU{s?^l z;RlEAk>68N`ez();s&zs(RiTZ2lx*hIe^@U^bfr6L2`oG_l-FM(td!8d-44M&JND* zv3$^WcMh&@Zf?v*$z}6wM_kr7S>ICfD z`qkQ@e0RZe|6$sN^E!ZEhx!JjhGoVZTb?&uYET3SQ2?_BB&pv^We2H`r|J=;Re(TCp#qDE)lwx?$BNWand*pOLWBpYp)1b$YJRyCbfv}9$+~A^gA>#9EH7>D zx=Gf#TNKcbT~oHLc6{fWHWtN|(BEhaxzWUi^w=N&Y3!+>{~$f(WPIj0GQ9!CcLq6% z!B{jq!6V9)R;)w}If!US3BxBlxZ2o6z+!ZNf|a-N+AxG{_1v1xnqybT*W1^VUCds6 z_3+WD3}`DEyFTbNtx!eW29)1{ZHro)byI=fYk)*xL`tlAW6(eRD}=tGFE~|4B$h^n zB&)umn$l?RsLgS{2y+Dt&x>~RI=Y%((dpt}tvSuTBs3l`J26TRSyJz^Z)>;q#0s)&9*^Ahoag2QKosiMQh*5IoE(pP zWths!p8EqTH$?1R8n{k9q@jxa&vCdUFJjBm9+6Zcss?-mWU0t3RCK4X4B-@7C3>lq za_A)yW>*5NOB@_+Iy$`pbQF3Sct+2M>a1LN@f0Jc<=i}Z$J@7A5z8>ld-@-Waozw& zg9#F=7o!qJc0~9{`?#$mT5*XQgbL=2jS1O8p%R<%)eftSRIJ~?n*HJAPa;Q~RkP$Q zmy@9KbnP7#`}*>1Op~#1&##xbuIq`AE^B>bg>{r`Rg!G*|ES}Ub$$Q(-1oW8=SYU_ktQe>(V%SU%<3UNYWl&M*(NU26dI#Mh1hDp!wakv zQi9#=zX(e4igrxRZHB(13ziHeSIE5geHoeXYT$}xq54SF*WBKq(ZPm0IjXcV`u7UG zo(rQg*(0mIXWTw!2%U8|B<+W1XtZ5wFw^>@yI|W^+iLyJJH;Y@4C7WWM?THC`X_y& zy=}m4B&xU=f8sE&)x9bmx!3nVX<$?DSd2Qj-%SRdgkChRzit$ta*TN56!7|o#=(Uf znFSdxndOfvuP)djTmRrXjhi3(edxOzgH$B&a?aR5jk6PRkpAe?Hp-vc1Dx9s_^q{gB_Zj5xv-o`a5qc z+aAk*fBx2vg*u{3O`>ZeEUs9ncakkiOYNoc^S4voI7==NCyErb@rq+IPlkvPnc%w zvpV{=Tb3bh^|sE}bfCvA4we55j3EafxbPS$$I4098G>Gx^=R3+m5fD#632d<9i|yL ziK4@fGONtfrCppArnF{2hjC(@mcZjp^}&S|AxztAIhf+}2&b&U~_mH$Al2MHXhE9TLlJnIhKdo++9gA9W3J}U% z_i2c}X|dr%UWMw~)+BLM=bc0==`RDnH`(rd2CZ$zoPtLXnn`euso9kDkbwqqwi7>l<0WZCW zzQ?lJ)wpee2DPCk$Br?pJU6cKisT|#dKqJRX>n%FAX46GFpZDbK2>nU{5PIvH>ak%Eur(oH9l>`xiT7nXnM1@llixm*P zD4jUSGKJ9fkjKk8^*_jShYvgkUfePm8hyoM^yH^0^=UrO#Bslw?wk$QxhxX4NI$SM1^cj zGlCW=kkw54jTNTI>mv3e@*=D~dYK5Y+E6&N z&HoA&_FgV7i#$^ZiSdtipg`Qg0L-Z)2`H_pQzN2FMQ)bQRodmlW(1FQ!%Mp2%>Quxcm zfcR_Azf?i`1P3*WB}ha~4k`temO?_Mq~KC;DC`ahtV)pt`4GHCOzgc0Zf*dIZU~@G z@+5du)ItC-Ko;?bj-#A!PYGZx zVr&hf6e5X`q*P-q0zps`um*_(OmN~toa5uOw5~Jb}P9G2E5!#S= zGqhrRU)(-bH;g;YxUi>_MirD(&yevzn}DY8_-G-_WKy9dWHKAp55XxMYLx{=`)w4k z-<8!#Qnsaw9v)vbOx~P*={BeRb42<^=GV`A0-NUM z_)gBN`c`YV74%|mmmd*x->c6~tEb$37YY2`yWzYE!M*;&<=1zhvl&Z^er%#K*`XgLSr<~H- zY)9YOfWCdH3N@>*A$KUsV>kW1CTjN%8N?5qK6ll~@04ny>BWkY6H`$DG{(09u0#ZUtsjtzCXqSjol4+lwyMHr#Z8*6+2 zax!Qvw;Z=NCObk!=SEf^>hB_hTvGXKj5>%#y%Q9BiNIdA75nzd7m3^)?~x*))rwlt zHAR&L&>GYkbd2eu*m;cO1*>5WJzC-};{ zng)7v#Uyxg%=tOh@!o zjLB$^8~WqCe5WIEQ(9+N;?e2lqNnmqeq{1_@sNsOAx*HVXcKQk$)gdstC8|!E&JNGy?86 z7Gr6X=K1u=wbwhdlgW)^P8@n2qQb$tyj8_b>LD`g{tA(o^7WrR{c(Knt*Hm5Wuu47 zq^^6fX_o~Rsvd;+**ge&ijUcOnm`fU=*8Nw)>??l{G6U6fupaO1nKN3uCzXg`BIZS z(1ORuc!7Yk!KvU5#ZjkA=Gi$n%?JJcY$o61b{9*vI-=&fPn>A7xyswUP%Z8Qj)@c2 z{tZQK7Dgex@fX-S$c@F`jCRQ%=i5$Q$gKwBK+Fw1Dd3^u-K(F!vkg%jj?6A^QT0K8 z2=NFmw>+x9edBKOfKk{*36=dZ5k3T+g31u8Q11=sQb!V1k#z5FpN4EO**0vkocS!^ z^iS5r)`b+`^?FVNP5(UJrlX4OZ;3I*;k!QItu@BvfnoE~p8Ng^n%x zGU}um@ztlGdZHf9Z^Vvygtxq&V&+P_mx<&;HSKLHK7oHvPktVi0-!s;9 zbVPLWlS17q7uC7#jSy;>nfz8NCMZ);H*m({S`B@(y8>*lcOq5?cp)yyit*@s*5n=z%i&P$fcwIys>gW(rHS;<~|6;8WDvn!+!7^9SM@BNK@ydL+{ z`cJ%1pGTA7$*=F4C6h~evs`WZEMgNKhzaolkH z-3NE7yyQ2wRZHZww<6&{O+lzQxv^vZ1Gnilr4~PS`L3*cNwffwIjaXR5QkqJCEov@ z>Z$yl;tM~*7{1&YR1ZkHxy!BqY^^KYM$Yw$9LUpFZu?L5Ht)B6ead8MxWQ=cX2=;M zH}@qaY<=q8%4?Y;!My5r1e4cFvWz{XaP6bqGsizt{g4*t8K1<27eoI0C(YFU=STiL zw!`f`_@yNlRaea`c#SuG*OyuD=pKiiZA+DuI{jHkSVOhxW?OQ{d@! zIwXlRh(=o|bLoVqJzmOrpXahW9dVPL3Hg?WzW?hvUQRZU!1vKY{=6C1=h=+KSCB@V zUR=8O?NzwZ<003=-J)b8Dwu#t9^3G1 z?g^2!!SE}GyG@c-KcPE2`}I3{r$;f`(A4om_H$6oRpW2SZAg-v7B}V)>RA;u5C!HB;t!#k^BOv4`05R@3+pTR~~gZh|M5pNvl|pLx>R zdF2TEg#SU5(Eosrwud7D@Hgoh|F7S!e?yOjG0qWeZUq5A0Z<#Twub|bWF-Yaz$ou( z04sSo1STVead5Dg#=>wI1P*~i%HX6BP%KtPN>)}DijsGP;UG9HR0<}IlyZQ`z#WkG z5TuNwy$nnS<6w`2qHMs%1cDFXZ`3vZH(f#CaIgtJ07r@R52_LxdC%01W{kqzQrd`& zTyRyPr~k~ldwH-&#I1`ap@%x_cT32jkAr2SxImxm@&E_6U-d%Kh(%<{l9EAhUHg^# zCAy2Ag>*t9su!!!5pvS6B7_fFDo$RI=SIUM_X?P)IJ@ckq{2XI&!bt^p5GU$u`5${ zCb&ueaG&kEn`VAMIIe4tSXe-mdPc`Dbbjbx5lx4LE_brH{`Wl-eK6iWBnoUZKp|2P N2!oK2x}gTc{{bvryKw*j literal 0 HcmV?d00001