feat!: support authored blobs (#3765)

This PR uses the go-square to support authored blobs (using the v1 share
version).

In this PR is the new block validation logic which ensures that all
"authored" blobs have a signer that matches the signer of the PFB

TxSim now will randomly use blobs of share version 0 and 1

app/default_overrides.go

@@ -8,6 +8,7 @@ import (
 	"github.com/celestiaorg/celestia-app/v3/pkg/appconsts"
 	"github.com/celestiaorg/celestia-app/v3/x/mint"
 	minttypes "github.com/celestiaorg/celestia-app/v3/x/mint/types"
+	"github.com/celestiaorg/go-square/v2/share"
 	"github.com/cosmos/cosmos-sdk/codec"
 	serverconfig "github.com/cosmos/cosmos-sdk/server/config"
 	sdk "github.com/cosmos/cosmos-sdk/types"
@@ -268,7 +269,7 @@ func DefaultConsensusConfig() *tmcfg.Config {
 	// We set a loose upper bound on what we expect the transaction to
 	// be based on the upper bound size of the entire block for the given
 	// version. This acts as a first line of DoS protection
-	upperBoundBytes := appconsts.DefaultSquareSizeUpperBound * appconsts.DefaultSquareSizeUpperBound * appconsts.ContinuationSparseShareContentSize
+	upperBoundBytes := appconsts.DefaultSquareSizeUpperBound * appconsts.DefaultSquareSizeUpperBound * share.ContinuationSparseShareContentSize
 	cfg.Mempool.MaxTxBytes = upperBoundBytes
 	cfg.Mempool.MaxTxsBytes = int64(upperBoundBytes) * cfg.Mempool.TTLNumBlocks
 	cfg.Mempool.Version = "v1" // prioritized mempool

app/errors/insufficient_gas_price_test.go

@@ -40,7 +40,7 @@ func TestInsufficientMinGasPriceIntegration(t *testing.T) {
 	signer, err := user.NewSigner(kr, enc.TxConfig, testutil.ChainID, appconsts.LatestVersion, user.NewAccount(account, acc.GetAccountNumber(), acc.GetSequence()))
 	require.NoError(t, err)
 
-	b, err := blob.NewBlob(share.RandomNamespace(), []byte("hello world"), 0)
+	b, err := blob.NewV0Blob(share.RandomNamespace(), []byte("hello world"))
 	require.NoError(t, err)
 
 	msg, err := blob.NewMsgPayForBlobs(signer.Account(account).Address().String(), appconsts.LatestVersion, b)

app/errors/nonce_mismatch_test.go

@@ -35,7 +35,7 @@ func TestNonceMismatchIntegration(t *testing.T) {
 	signer, err := user.NewSigner(kr, enc.TxConfig, testutil.ChainID, appconsts.LatestVersion, user.NewAccount(account, acc.GetAccountNumber(), acc.GetSequence()+1))
 	require.NoError(t, err)
 
-	b, err := blob.NewBlob(share.RandomNamespace(), []byte("hello world"), 0)
+	b, err := blob.NewV0Blob(share.RandomNamespace(), []byte("hello world"))
 	require.NoError(t, err)
 
 	msg, err := blob.NewMsgPayForBlobs(signer.Account(account).Address().String(), appconsts.LatestVersion, b)

app/test/check_tx_test.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/cosmos/cosmos-sdk/client"
 	"github.com/cosmos/cosmos-sdk/crypto/keyring"
+	sdk "github.com/cosmos/cosmos-sdk/types"
 	tmrand "github.com/tendermint/tendermint/libs/rand"
 
 	"github.com/celestiaorg/celestia-app/v3/app"
@@ -14,6 +15,7 @@ import (
 	"github.com/celestiaorg/celestia-app/v3/pkg/user"
 	testutil "github.com/celestiaorg/celestia-app/v3/test/util"
 	"github.com/celestiaorg/celestia-app/v3/test/util/blobfactory"
+	"github.com/celestiaorg/celestia-app/v3/test/util/testnode"
 	blobtypes "github.com/celestiaorg/celestia-app/v3/x/blob/types"
 	"github.com/celestiaorg/go-square/v2/share"
 	"github.com/celestiaorg/go-square/v2/tx"
@@ -30,7 +32,7 @@ func TestCheckTx(t *testing.T) {
 	ns1, err := share.NewV0Namespace(bytes.Repeat([]byte{1}, share.NamespaceVersionZeroIDSize))
 	require.NoError(t, err)
 
-	accs := []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}
+	accs := []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k"}
 
 	testApp, kr := testutil.SetupTestAppWithGenesisValSet(app.DefaultConsensusParams(), accs...)
 	testApp.Commit()
@@ -182,6 +184,32 @@ func TestCheckTx(t *testing.T) {
 			},
 			expectedABCICode: blobtypes.ErrBlobsTooLarge.ABCICode(),
 		},
+		{
+			name:      "v1 blob with invalid signer",
+			checkType: abci.CheckTxType_New,
+			getTx: func() []byte {
+				signer := createSigner(t, kr, accs[10], encCfg.TxConfig, 11)
+				blob, err := share.NewV1Blob(share.RandomBlobNamespace(), []byte("data"), testnode.RandomAddress().(sdk.AccAddress))
+				require.NoError(t, err)
+				blobTx, _, err := signer.CreatePayForBlobs(accs[10], []*share.Blob{blob}, opts...)
+				require.NoError(t, err)
+				return blobTx
+			},
+			expectedABCICode: blobtypes.ErrInvalidBlobSigner.ABCICode(),
+		},
+		{
+			name:      "v1 blob with valid signer",
+			checkType: abci.CheckTxType_New,
+			getTx: func() []byte {
+				signer := createSigner(t, kr, accs[10], encCfg.TxConfig, 11)
+				blob, err := share.NewV1Blob(share.RandomBlobNamespace(), []byte("data"), signer.Account(accs[10]).Address())
+				require.NoError(t, err)
+				blobTx, _, err := signer.CreatePayForBlobs(accs[10], []*share.Blob{blob}, opts...)
+				require.NoError(t, err)
+				return blobTx
+			},
+			expectedABCICode: abci.CodeTypeOK,
+		},
 	}
 
 	for _, tt := range tests {

app/test/consistent_apphash_test.go

@@ -381,6 +381,9 @@ func executeTxs(testApp *app.App, encodedBlobTx []byte, encodedSdkTxs [][]byte,
 		// Dynamically increase time so the validator can be unjailed (1m duration)
 		Time: genesisTime.Add(time.Duration(height) * time.Minute),
 	})
+	if len(resPrepareProposal.BlockData.Txs) != len(encodedSdkTxs) {
+		return nil, nil, fmt.Errorf("PrepareProposal removed transactions. Was %d, now %d", len(encodedSdkTxs), len(resPrepareProposal.BlockData.Txs))
+	}
 
 	dataHash := resPrepareProposal.BlockData.Hash
 

app/test/fuzz_abci_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/celestiaorg/celestia-app/v3/pkg/appconsts"
 	"github.com/celestiaorg/celestia-app/v3/pkg/user"
 	testutil "github.com/celestiaorg/celestia-app/v3/test/util"
+	"github.com/celestiaorg/go-square/v2/share"
 	"github.com/stretchr/testify/require"
 	abci "github.com/tendermint/tendermint/abci/types"
 	tmrand "github.com/tendermint/tendermint/libs/rand"
@@ -62,17 +63,17 @@ func TestPrepareProposalConsistency(t *testing.T) {
 		},
 		{
 			"max",
-			maxShareCount * appconsts.ContinuationSparseShareContentSize,
+			maxShareCount * share.ContinuationSparseShareContentSize,
 			appconsts.DefaultSquareSizeUpperBound,
 		},
 		{
 			"larger MaxBytes than SquareSize",
-			maxShareCount * appconsts.ContinuationSparseShareContentSize,
+			maxShareCount * share.ContinuationSparseShareContentSize,
 			appconsts.DefaultGovMaxSquareSize,
 		},
 		{
 			"smaller MaxBytes than SquareSize",
-			32 * 32 * appconsts.ContinuationSparseShareContentSize,
+			32 * 32 * share.ContinuationSparseShareContentSize,
 			appconsts.DefaultGovMaxSquareSize,
 		},
 	}

app/test/integration_test.go

@@ -289,7 +289,7 @@ func (s *IntegrationTestSuite) TestEmptyBlock() {
 func newBlobWithSize(size int) *share.Blob {
 	ns := share.MustNewV0Namespace(bytes.Repeat([]byte{1}, share.NamespaceVersionZeroIDSize))
 	data := tmrand.Bytes(size)
-	blob, err := share.NewBlob(ns, data, appconsts.ShareVersionZero, nil)
+	blob, err := share.NewBlob(ns, data, share.ShareVersionZero, nil)
 	if err != nil {
 		panic(err)
 	}

app/test/process_proposal_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 	"time"
 
+	sdk "github.com/cosmos/cosmos-sdk/types"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	abci "github.com/tendermint/tendermint/abci/types"
@@ -24,6 +25,8 @@ import (
 	testutil "github.com/celestiaorg/celestia-app/v3/test/util"
 	"github.com/celestiaorg/celestia-app/v3/test/util/blobfactory"
 	"github.com/celestiaorg/celestia-app/v3/test/util/testfactory"
+	"github.com/celestiaorg/celestia-app/v3/test/util/testnode"
+	blobtypes "github.com/celestiaorg/celestia-app/v3/x/blob/types"
 	"github.com/celestiaorg/go-square/v2"
 	"github.com/celestiaorg/go-square/v2/share"
 	"github.com/celestiaorg/go-square/v2/tx"
@@ -117,7 +120,7 @@ func TestProcessProposal(t *testing.T) {
 			mutator: func(d *tmproto.Data) {
 				blobTx, _, err := tx.UnmarshalBlobTx(blobTxs[0])
 				require.NoError(t, err)
-				newBlob, err := share.NewBlob(ns1, data, appconsts.ShareVersionZero, nil)
+				newBlob, err := share.NewBlob(ns1, data, share.ShareVersionZero, nil)
 				require.NoError(t, err)
 				blobTx.Blobs[0] = newBlob
 				blobTxBytes, _ := tx.MarshalBlobTx(blobTx.Tx, blobTx.Blobs...)
@@ -132,7 +135,7 @@ func TestProcessProposal(t *testing.T) {
 			mutator: func(d *tmproto.Data) {
 				blobTx, _, err := tx.UnmarshalBlobTx(blobTxs[0])
 				require.NoError(t, err)
-				newBlob, err := share.NewBlob(share.TxNamespace, data, appconsts.ShareVersionZero, nil)
+				newBlob, err := share.NewBlob(share.TxNamespace, data, share.ShareVersionZero, nil)
 				require.NoError(t, err)
 				blobTx.Blobs[0] = newBlob
 				blobTxBytes, _ := tx.MarshalBlobTx(blobTx.Tx, blobTx.Blobs...)
@@ -259,6 +262,43 @@ func TestProcessProposal(t *testing.T) {
 			appVersion:     appconsts.LatestVersion,
 			expectedResult: abci.ResponseProcessProposal_REJECT,
 		},
+		{
+			name:  "valid v1 authored blob",
+			input: validData(),
+			mutator: func(d *tmproto.Data) {
+				addr := signer.Account(accounts[0]).Address()
+				blob, err := share.NewV1Blob(ns1, data, addr)
+				require.NoError(t, err)
+				rawTx, _, err := signer.CreatePayForBlobs(accounts[0], []*share.Blob{blob}, user.SetGasLimit(100000), user.SetFee(100000))
+				require.NoError(t, err)
+				d.Txs[0] = rawTx
+				d.Hash = calculateNewDataHash(t, d.Txs)
+			},
+			appVersion:     appconsts.LatestVersion,
+			expectedResult: abci.ResponseProcessProposal_ACCEPT,
+		},
+		{
+			name:  "v1 authored blob with invalid signer",
+			input: validData(),
+			mutator: func(d *tmproto.Data) {
+				addr := signer.Account(accounts[0]).Address()
+				falseAddr := testnode.RandomAddress().(sdk.AccAddress)
+				blob, err := share.NewV1Blob(ns1, data, falseAddr)
+				require.NoError(t, err)
+				msg, err := blobtypes.NewMsgPayForBlobs(addr.String(), appconsts.LatestVersion, blob)
+				require.NoError(t, err)
+
+				rawTx, err := signer.CreateTx([]sdk.Msg{msg}, user.SetGasLimit(100000), user.SetFee(100000))
+				require.NoError(t, err)
+
+				blobTxBytes, err := tx.MarshalBlobTx(rawTx, blob)
+				require.NoError(t, err)
+				d.Txs[0] = blobTxBytes
+				d.Hash = calculateNewDataHash(t, d.Txs)
+			},
+			appVersion:     appconsts.LatestVersion,
+			expectedResult: abci.ResponseProcessProposal_REJECT,
+		},
 	}
 
 	for _, tt := range tests {

app/test/square_size_test.go

@@ -86,7 +86,7 @@ func (s *SquareSizeIntegrationTest) TestSquareSizeUpperBound() {
 		{
 			name:                  "gov square size == hardcoded max",
 			govMaxSquareSize:      appconsts.DefaultSquareSizeUpperBound,
-			maxBytes:              appconsts.DefaultSquareSizeUpperBound * appconsts.DefaultSquareSizeUpperBound * appconsts.ContinuationSparseShareContentSize,
+			maxBytes:              appconsts.DefaultUpperBoundMaxBytes,
 			expectedMaxSquareSize: appconsts.DefaultSquareSizeUpperBound,
 		},
 	}

app/test/std_sdk_test.go

@@ -365,7 +365,7 @@ func (s *StandardSDKIntegrationTestSuite) TestGRPCQueries() {
 
 		txSubmitter, err := user.SetupTxClient(s.cctx.GoContext(), s.cctx.Keyring, s.cctx.GRPCClient, s.ecfg)
 		require.NoError(t, err)
-		blobs := blobfactory.RandBlobsWithNamespace([]share.Namespace{share.RandomNamespace()}, []int{1000})
+		blobs := blobfactory.RandV0BlobsWithNamespace([]share.Namespace{share.RandomNamespace()}, []int{1000})
 		res, err := txSubmitter.SubmitPayForBlob(s.cctx.GoContext(), blobs, blobfactory.DefaultTxOpts()...)
 		require.NoError(t, err)
 

pkg/appconsts/global_consts.go

@@ -1,9 +1,7 @@
 package appconsts
 
 import (
-	"math"
-
-	ns "github.com/celestiaorg/go-square/v2/share"
+	"github.com/celestiaorg/go-square/v2/share"
 	"github.com/celestiaorg/rsmt2d"
 	"github.com/tendermint/tendermint/pkg/consts"
 )
@@ -13,55 +11,9 @@ import (
 //
 // They can not change throughout the lifetime of a network.
 const (
-	// NamespaceVersionSize is the size of a namespace version in bytes.
-	NamespaceVersionSize = ns.NamespaceVersionSize
-	// NamespaceVersionMaxValue is the maximum value a namespace version can be.
-	// This const must be updated if NamespaceVersionSize is changed.
-	NamespaceVersionMaxValue = math.MaxUint8
-
-	// NamespaceIDSize is the size of a namespace ID in bytes.
-	NamespaceIDSize = ns.NamespaceIDSize
-
-	// NamespaceSize is the size of a namespace (version + ID) in bytes.
-	NamespaceSize = ns.NamespaceSize
-
-	// ShareSize is the size of a share in bytes.
-	ShareSize = 512
-
-	// ShareInfoBytes is the number of bytes reserved for information. The info
-	// byte contains the share version and a sequence start indicator.
-	ShareInfoBytes = 1
-
-	// SequenceLenBytes is the number of bytes reserved for the sequence length
-	// that is present in the first share of a sequence.
-	SequenceLenBytes = 4
-
-	// ShareVersionZero is the first share version format.
-	ShareVersionZero = uint8(0)
-
 	// DefaultShareVersion is the defacto share version. Use this if you are
 	// unsure of which version to use.
-	DefaultShareVersion = ShareVersionZero
-
-	// CompactShareReservedBytes is the number of bytes reserved for the location of
-	// the first unit (transaction, ISR) in a compact share.
-	CompactShareReservedBytes = 4
-
-	// FirstCompactShareContentSize is the number of bytes usable for data in
-	// the first compact share of a sequence.
-	FirstCompactShareContentSize = ShareSize - NamespaceSize - ShareInfoBytes - SequenceLenBytes - CompactShareReservedBytes
-
-	// ContinuationCompactShareContentSize is the number of bytes usable for
-	// data in a continuation compact share of a sequence.
-	ContinuationCompactShareContentSize = ShareSize - NamespaceSize - ShareInfoBytes - CompactShareReservedBytes
-
-	// FirstSparseShareContentSize is the number of bytes usable for data in the
-	// first sparse share of a sequence.
-	FirstSparseShareContentSize = ShareSize - NamespaceSize - ShareInfoBytes - SequenceLenBytes
-
-	// ContinuationSparseShareContentSize is the number of bytes usable for data
-	// in a continuation sparse share of a sequence.
-	ContinuationSparseShareContentSize = ShareSize - NamespaceSize - ShareInfoBytes
+	DefaultShareVersion = share.ShareVersionZero
 
 	// MinSquareSize is the smallest original square width.
 	MinSquareSize = 1
@@ -70,9 +22,6 @@ const (
 	// data square.
 	MinShareCount = MinSquareSize * MinSquareSize
 
-	// MaxShareVersion is the maximum value a share version can be.
-	MaxShareVersion = 127
-
 	// BondDenom defines the native staking denomination
 	BondDenom = "utia"
 )
@@ -92,7 +41,7 @@ var (
 	DefaultCodec = rsmt2d.NewLeoRSCodec
 
 	// SupportedShareVersions is a list of supported share versions.
-	SupportedShareVersions = []uint8{ShareVersionZero}
+	SupportedShareVersions = share.SupportedShareVersions
 )
 
 // HashLength returns the length of a hash in bytes.

pkg/appconsts/initial_consts.go

@@ -1,6 +1,10 @@
 package appconsts
 
-import "time"
+import (
+	"time"
+
+	"github.com/celestiaorg/go-square/v2/share"
+)
 
 // The following defaults correspond to initial parameters of the network that can be changed, not via app versions
 // but other means such as on-chain governance, or the nodes local config
@@ -11,7 +15,7 @@ const (
 
 	// DefaultMaxBytes is the default value for the governance modifiable
 	// maximum number of bytes allowed in a valid block.
-	DefaultMaxBytes = DefaultGovMaxSquareSize * DefaultGovMaxSquareSize * ContinuationSparseShareContentSize
+	DefaultMaxBytes = DefaultGovMaxSquareSize * DefaultGovMaxSquareSize * share.ContinuationSparseShareContentSize
 
 	// DefaultGasPerBlobByte is the default gas cost deducted per byte of blob
 	// included in a PayForBlobs txn
@@ -27,3 +31,5 @@ const (
 	// time can be subject to slashing under conditions of misbehavior.
 	DefaultUnbondingTime = 3 * 7 * 24 * time.Hour
 )
+
+var DefaultUpperBoundMaxBytes = DefaultSquareSizeUpperBound * DefaultSquareSizeUpperBound * share.ContinuationSparseShareContentSize

pkg/da/data_availability_header_test.go

@@ -7,7 +7,7 @@ import (
 	"testing"
 
 	"github.com/celestiaorg/celestia-app/v3/pkg/appconsts"
-	"github.com/celestiaorg/go-square/v2/share"
+	sh "github.com/celestiaorg/go-square/v2/share"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -245,7 +245,7 @@ func TestSquareSize(t *testing.T) {
 // generateShares generates count number of shares with a constant namespace and
 // share contents.
 func generateShares(count int) (shares [][]byte) {
-	ns1 := share.MustNewV0Namespace(bytes.Repeat([]byte{1}, share.NamespaceVersionZeroIDSize))
+	ns1 := sh.MustNewV0Namespace(bytes.Repeat([]byte{1}, sh.NamespaceVersionZeroIDSize))
 
 	for i := 0; i < count; i++ {
 		share := generateShare(ns1.Bytes())
@@ -256,7 +256,7 @@ func generateShares(count int) (shares [][]byte) {
 }
 
 func generateShare(namespace []byte) (share []byte) {
-	remainder := bytes.Repeat([]byte{0xFF}, appconsts.ShareSize-len(namespace))
+	remainder := bytes.Repeat([]byte{0xFF}, sh.ShareSize-len(namespace))
 	share = append(share, namespace...)
 	share = append(share, remainder...)
 	return share

pkg/inclusion/nmt_caching_test.go

@@ -190,7 +190,7 @@ func chunkSlice(slice [][]byte, chunkSize int) [][][]byte {
 // namespace.
 func generateRandNamespacedRawData(count int) (result [][]byte) {
 	for i := 0; i < count; i++ {
-		rawData := tmrand.Bytes(appconsts.ShareSize)
+		rawData := tmrand.Bytes(share.ShareSize)
 		namespace := share.RandomBlobNamespace().Bytes()
 		copy(rawData, namespace)
 		result = append(result, rawData)

pkg/user/signer.go

@@ -92,6 +92,10 @@ func (s *Signer) CreatePayForBlobs(accountName string, blobs []*share.Blob, opts
 		return nil, 0, fmt.Errorf("account %s not found", accountName)
 	}
 
+	if err := blobtypes.ValidateBlobs(blobs...); err != nil {
+		return nil, 0, err
+	}
+
 	msg, err := blobtypes.NewMsgPayForBlobs(acc.address.String(), s.appVersion, blobs...)
 	if err != nil {
 		return nil, 0, err