ipld: adding IPLD package to provide helpers for dealing with IPLD-based data storage

go.mod

@@ -2,19 +2,27 @@ module github.com/celestiaorg/celestia-node
 
 go 1.16
 
+replace github.com/ipfs/go-verifcid => github.com/celestiaorg/go-verifcid v0.0.1-lazypatch
+
 require (
 	github.com/BurntSushi/toml v0.4.1
 	github.com/celestiaorg/celestia-core v0.0.2-0.20210924001615-488ac31b4b3c
+	github.com/celestiaorg/nmt v0.7.0
 	github.com/celestiaorg/rsmt2d v0.3.0
-	github.com/ipfs/go-bitswap v0.4.0
+	github.com/ipfs/go-bitswap v0.3.4
+	github.com/ipfs/go-block-format v0.0.3
 	github.com/ipfs/go-blockservice v0.1.7
+	github.com/ipfs/go-cid v0.0.7
 	github.com/ipfs/go-datastore v0.4.6
 	github.com/ipfs/go-ds-badger2 v0.1.1
 	github.com/ipfs/go-ipfs-blockstore v0.1.6
 	github.com/ipfs/go-ipfs-exchange-interface v0.0.1
+	github.com/ipfs/go-ipld-cbor v0.0.5 // indirect
 	github.com/ipfs/go-ipld-format v0.2.0
 	github.com/ipfs/go-log/v2 v2.3.0
-	github.com/ipfs/go-merkledag v0.4.0
+	github.com/ipfs/go-merkledag v0.3.2
+	github.com/ipfs/go-peertaskqueue v0.4.0 // indirect
+	github.com/ipld/go-ipld-prime v0.11.0 // indirect
 	github.com/libp2p/go-libp2p v0.15.0
 	github.com/libp2p/go-libp2p-connmgr v0.2.4
 	github.com/libp2p/go-libp2p-core v0.9.0
@@ -25,6 +33,7 @@ require (
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/multiformats/go-base32 v0.0.4
 	github.com/multiformats/go-multiaddr v0.4.0
+	github.com/multiformats/go-multihash v0.0.15
 	github.com/stretchr/testify v1.7.1-0.20210427113832-6241f9ab9942
 	go.uber.org/fx v1.14.2
 	go.uber.org/zap v1.19.0

ipld/nmt_adder.go

@@ -0,0 +1,64 @@
+package ipld
+
+import (
+	"context"
+
+	"github.com/ipfs/go-cid"
+	ipld "github.com/ipfs/go-ipld-format"
+
+	"github.com/celestiaorg/celestia-node/ipld/plugin"
+)
+
+// NmtNodeAdder adds ipld.Nodes to the underlying ipld.Batch if it is inserted
+// into an nmt tree
+type NmtNodeAdder struct {
+	ctx    context.Context
+	batch  *ipld.Batch
+	leaves *cid.Set
+	err    error
+}
+
+// NewNmtNodeAdder returns a new NmtNodeAdder with the provided context and
+// batch. Note that the context provided should have a timeout
+// It is not thread-safe.
+func NewNmtNodeAdder(ctx context.Context, batch *ipld.Batch) *NmtNodeAdder {
+	return &NmtNodeAdder{
+		batch:  batch,
+		ctx:    ctx,
+		leaves: cid.NewSet(),
+	}
+}
+
+// Visit is a NodeVisitor that can be used during the creation of a new NMT to
+// create and add ipld.Nodes to the Batch while computing the root of the NMT.
+func (n *NmtNodeAdder) Visit(hash []byte, children ...[]byte) {
+	if n.err != nil {
+		return // protect from further visits if there is an error
+	}
+
+	id := plugin.MustCidFromNamespacedSha256(hash)
+	switch len(children) {
+	case 1:
+		if n.leaves.Visit(id) {
+			n.err = n.batch.Add(n.ctx, plugin.NewNMTLeafNode(id, children[0]))
+		}
+	case 2:
+		n.err = n.batch.Add(n.ctx, plugin.NewNMTNode(id, children[0], children[1]))
+	default:
+		panic("expected a binary tree")
+	}
+}
+
+// Batch return the ipld.Batch originally provided to the NmtNodeAdder
+func (n *NmtNodeAdder) Batch() *ipld.Batch {
+	return n.batch
+}
+
+// Commit checks for errors happened during Visit and if absent commits data to inner Batch.
+func (n *NmtNodeAdder) Commit() error {
+	if n.err != nil {
+		return n.err
+	}
+
+	return n.batch.Commit()
+}