add conversion from eds to shares

go.mod

@@ -9,61 +9,32 @@ require (
 	github.com/celestiaorg/celestia-core v0.0.2-0.20210924001615-488ac31b4b3c
 	github.com/celestiaorg/nmt v0.7.0
 	github.com/celestiaorg/rsmt2d v0.3.0
-	github.com/google/uuid v1.3.0 // indirect
-	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/huin/goupnp v1.0.2 // indirect
 	github.com/ipfs/go-bitswap v0.3.4
 	github.com/ipfs/go-block-format v0.0.3
-	github.com/ipfs/go-blockservice v0.0.3
+	github.com/ipfs/go-blockservice v0.1.7
 	github.com/ipfs/go-cid v0.0.7
 	github.com/ipfs/go-datastore v0.4.6
 	github.com/ipfs/go-ds-badger2 v0.1.1
 	github.com/ipfs/go-ipfs-blockstore v0.1.6
 	github.com/ipfs/go-ipfs-exchange-interface v0.0.1
 	github.com/ipfs/go-ipld-cbor v0.0.5 // indirect
 	github.com/ipfs/go-ipld-format v0.2.0
-	github.com/ipfs/go-log v1.0.5 // indirect
 	github.com/ipfs/go-log/v2 v2.3.0
-	github.com/ipfs/go-merkledag v0.0.3
+	github.com/ipfs/go-merkledag v0.3.2
 	github.com/ipfs/go-peertaskqueue v0.4.0 // indirect
-	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
-	github.com/koron/go-ssdp v0.0.2 // indirect
-	github.com/libp2p/go-addr-util v0.1.0 // indirect
-	github.com/libp2p/go-libp2p v0.14.2
+	github.com/ipld/go-ipld-prime v0.11.0 // indirect
+	github.com/libp2p/go-libp2p v0.15.0
 	github.com/libp2p/go-libp2p-connmgr v0.2.4
 	github.com/libp2p/go-libp2p-core v0.9.0
-	github.com/libp2p/go-libp2p-discovery v0.5.1 // indirect
-	github.com/libp2p/go-libp2p-kad-dht v0.11.1
-	github.com/libp2p/go-libp2p-noise v0.2.2 // indirect
-	github.com/libp2p/go-libp2p-peerstore v0.2.7
-	github.com/libp2p/go-libp2p-pubsub v0.5.0
-	github.com/libp2p/go-libp2p-quic-transport v0.11.2 // indirect
+	github.com/libp2p/go-libp2p-kad-dht v0.13.1
+	github.com/libp2p/go-libp2p-peerstore v0.2.8
+	github.com/libp2p/go-libp2p-pubsub v0.5.4
 	github.com/libp2p/go-libp2p-routing-helpers v0.2.3
-	github.com/libp2p/go-libp2p-tls v0.2.0 // indirect
-	github.com/libp2p/go-libp2p-transport-upgrader v0.4.6 // indirect
-	github.com/libp2p/go-reuseport-transport v0.0.5 // indirect
-	github.com/libp2p/go-ws-transport v0.5.0 // indirect
-	github.com/miekg/dns v1.1.43 // indirect
-	github.com/minio/sha256-simd v1.0.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/multiformats/go-base32 v0.0.4
 	github.com/multiformats/go-multiaddr v0.4.0
-	github.com/multiformats/go-multihash v0.0.14
-	github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e // indirect
-	github.com/prometheus/common v0.30.0 // indirect
-	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/multiformats/go-multihash v0.0.15
 	github.com/stretchr/testify v1.7.1-0.20210427113832-6241f9ab9942
-	github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a // indirect
-	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/fx v1.14.2
-	go.uber.org/multierr v1.7.0 // indirect
 	go.uber.org/zap v1.19.0
-	golang.org/x/crypto v0.0.0-20210813211128-0a44fdfbc16e // indirect
-	golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d // indirect
-	golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 // indirect
-	golang.org/x/text v0.3.7 // indirect
-	google.golang.org/grpc v1.40.0 // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
 )
-
-replace github.com/ipfs/go-verifcid => github.com/lazyledger/go-verifcid v0.0.1-lazypatch

ipld/plugin/nmt.go

@@ -6,14 +6,16 @@ import (
 	"crypto/sha256"
 	"errors"
 	"fmt"
+	"hash"
 	"io"
 
 	blocks "github.com/ipfs/go-block-format"
 	cid "github.com/ipfs/go-cid"
 	ipld "github.com/ipfs/go-ipld-format"
+	mh "github.com/multiformats/go-multihash"
+	mhcore "github.com/multiformats/go-multihash/core"
 
 	"github.com/celestiaorg/nmt"
-	mh "github.com/multiformats/go-multihash"
 )
 
 const (
@@ -47,7 +49,11 @@ func init() {
 		Sha256Namespace8Flagged,
 		"sha2-256-namespace8-flagged",
 		nmtHashSize,
-		sumSha256Namespace8Flagged,
+		func() hash.Hash {
+			return &namespaceHasher{
+				nmt.NewNmtHasher(sha256.New(), nmt.DefaultNamespaceIDLen, true),
+			}
+		},
 	)
 	// this should already happen when the plugin is injected but it doesn't for some CI tests
 	ipld.DefaultBlockDecoder.Register(NmtCodec, NmtNodeParser)
@@ -60,7 +66,7 @@ func mustRegisterNamespacedCodec(
 	codec uint64,
 	name string,
 	defaultLength int,
-	hashFunc mh.HashFunc,
+	hashFunc func() hash.Hash,
 ) {
 	if _, ok := mh.Codes[codec]; !ok {
 		// make sure that the Codec wasn't registered from somewhere different than this plugin already:
@@ -72,20 +78,20 @@ func mustRegisterNamespacedCodec(
 		mh.Names[name] = codec
 		mh.DefaultLengths[codec] = defaultLength
 
-		if err := mh.RegisterHashFunc(codec, hashFunc); err != nil {
-			panic(fmt.Sprintf("could not register hash function: %v", mh.Codes[codec]))
-		}
+		mhcore.Register(codec, hashFunc)
 	}
 }
 
-// sumSha256Namespace8Flagged is the mh.HashFunc used to hash leaf and inner nodes.
-// It is registered as a mh.HashFunc in the go-multihash module.
-func sumSha256Namespace8Flagged(data []byte, _length int) ([]byte, error) {
+type namespaceHasher struct {
+	hash.Hash
+}
+
+func (n namespaceHasher) Sum(data []byte) []byte {
 	isLeafData := data[0] == nmt.LeafPrefix
 	if isLeafData {
-		return nmt.Sha256Namespace8FlaggedLeaf(data[1:]), nil
+		return nmt.Sha256Namespace8FlaggedLeaf(data[1:])
 	}
-	return nmt.Sha256Namespace8FlaggedInner(data[1:]), nil
+	return nmt.Sha256Namespace8FlaggedInner(data[1:])
 }
 
 // DataSquareRowOrColumnRawInputParser reads the raw shares and extract the IPLD nodes from the NMT tree.
@@ -382,7 +388,7 @@ func CidFromNamespacedSha256(namespacedHash []byte) (cid.Cid, error) {
 	if err != nil {
 		return cid.Undef, err
 	}
-	return cid.NewCidV1(NmtCodec, buf), nil
+	return cid.NewCidV1(NmtCodec, mh.Multihash(buf)), nil
 }
 
 // MustCidFromNamespacedSha256 is a wrapper around cidFromNamespacedSha256 that panics

ipld/read.go

@@ -4,13 +4,13 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"github.com/celestiaorg/celestia-core/pkg/da"
 	"math/rand"
 
 	"github.com/celestiaorg/rsmt2d"
 	"github.com/ipfs/go-cid"
 	ipld "github.com/ipfs/go-ipld-format"
 
+	"github.com/celestiaorg/celestia-core/pkg/da"
 	"github.com/celestiaorg/celestia-core/pkg/wrapper"
 	"github.com/celestiaorg/celestia-node/ipld/plugin"
 )
@@ -198,7 +198,7 @@ func (sc *shareCounter) flatten() [][]byte {
 }
 
 // GetLeafData fetches and returns the data for leaf leafIndex of root rootCid.
-// It stops and returns an error if the provided context is cancelled before
+// It stops and returns an error if the provided context is canceled before
 // finishing
 func GetLeafData(
 	ctx context.Context,

ipld/read_test.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"context"
 	"crypto/sha256"
-	"fmt"
 	"math"
 	"math/rand"
 	"testing"
@@ -125,11 +124,12 @@ func TestRetrieveBlockData(t *testing.T) {
 
 		tc := tc
 		t.Run(tc.name, func(t *testing.T) {
-			// generate EDS
+			// // generate EDS
 			eds := generateRandEDS(t, tc.squareSize)
-			fmt.Println(eds.Width())
 
-			in, err := PutData(ctx, eds, dag)
+			shares := convertEDStoShares(eds)
+
+			in, err := PutData(ctx, shares, dag)
 			require.NoError(t, err)
 
 			// limit with deadline, specifically retrieval
@@ -146,6 +146,22 @@ func TestRetrieveBlockData(t *testing.T) {
 	}
 }
 
+func Test_ConvertEDStoShares(t *testing.T) {
+	squareWidth := 16
+	origShares := RandNamespacedShares(t, squareWidth*squareWidth)
+	rawshares := origShares.Raw()
+
+	// create the nmt wrapper to generate row and col commitments
+	tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(squareWidth))
+
+	// compute extended square
+	eds, err := rsmt2d.ComputeExtendedDataSquare(rawshares, rsmt2d.NewRSGF8Codec(), tree.Constructor)
+	require.NoError(t, err)
+
+	resshares := convertEDStoShares(eds)
+	require.Equal(t, rawshares, resshares)
+}
+
 func generateRandEDS(t *testing.T, originalSquareWidth int) *rsmt2d.ExtendedDataSquare {
 	shareCount := originalSquareWidth * originalSquareWidth
 

ipld/test_helpers.go

@@ -17,7 +17,7 @@ import (
 )
 
 // TODO(Wondertan): Move to rsmt2d
-// TODO(Wondertan): Propose use of int by default instead of uint for the sake convenience and Golang practises
+// TODO(Wondertan): Propose use of int by default instead of uint for the sake convenience and Golang practices
 func EqualEDS(a *rsmt2d.ExtendedDataSquare, b *rsmt2d.ExtendedDataSquare) bool {
 	if a.Width() != b.Width() {
 		return false

ipld/write.go

@@ -3,6 +3,8 @@ package ipld
 import (
 	"context"
 	"fmt"
+	"math"
+
 	"github.com/celestiaorg/nmt"
 	"github.com/celestiaorg/rsmt2d"
 	ipld "github.com/ipfs/go-ipld-format"
@@ -11,22 +13,35 @@ import (
 )
 
 // PutData posts erasured block data to IPFS using the provided ipld.NodeAdder.
-func PutData(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, adder ipld.NodeAdder) (*rsmt2d.ExtendedDataSquare, error) {
+func PutData(ctx context.Context, shares [][]byte, adder ipld.NodeAdder) (*rsmt2d.ExtendedDataSquare, error) {
+	if len(shares) == 0 {
+		return nil, fmt.Errorf("empty data") // empty block is not an empty Data
+	}
 	// create nmt adder wrapping batch adder
 	batchAdder := NewNmtNodeAdder(ctx, ipld.NewBatch(ctx, adder))
 	// create the nmt wrapper to generate row and col commitments
-	tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(eds.Width()), nmt.NodeVisitor(batchAdder.Visit))
-	// recompute the eds // TODO @renaynay: is this right?
-	shares := make([][]byte, 0)
-	for i := 0; i < int(eds.Width()/2); i++ {
-		shares = append(shares, eds.Row(uint(i))...)
-	}
-	recomputed, err := rsmt2d.ComputeExtendedDataSquare(shares, rsmt2d.NewRSGF8Codec(), tree.Constructor)
+	squareSize := uint32(math.Sqrt(float64(len(shares))))
+	fmt.Println("square size when put data", squareSize)
+	tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(squareSize), nmt.NodeVisitor(batchAdder.Visit))
+	// recompute the eds
+	eds, err := rsmt2d.ComputeExtendedDataSquare(shares, rsmt2d.NewRSGF8Codec(), tree.Constructor)
 	if err != nil {
 		return nil, fmt.Errorf("failure to recompute the extended data square: %w", err)
 	}
 	// compute roots
-	recomputed.ColRoots()
+	eds.RowRoots()
 	// commit the batch to ipfs
 	return eds, batchAdder.Commit()
 }
+
+func convertEDStoShares(eds *rsmt2d.ExtendedDataSquare) [][]byte {
+	origWidth := eds.Width() / 2
+	origShares := make([][]byte, origWidth*origWidth)
+	for i := uint(0); i < origWidth; i++ {
+		row := eds.Row(i)
+		for j := uint(0); j < origWidth; j++ {
+			origShares[(i*origWidth)+j] = row[j]
+		}
+	}
+	return origShares
+}