Skip to content

Commit

Permalink
add file cache for ipldv2
Browse files Browse the repository at this point in the history
  • Loading branch information
walldiss committed Oct 17, 2023
1 parent 2baea7d commit 46067e7
Show file tree
Hide file tree
Showing 9 changed files with 465 additions and 32 deletions.
6 changes: 3 additions & 3 deletions share/eds/byzantine/share_proof.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ type ShareWithProof struct {
// Share is a full data including namespace
share.Share
// Proof is a Merkle Proof of current share
Proof *nmt.Proof
Proof nmt.Proof
}

// NewShareWithProof takes the given leaf and its path, starting from the tree root,
Expand All @@ -38,7 +38,7 @@ func NewShareWithProof(index int, share share.Share, pathToLeaf []cid.Cid) *Shar
proof := nmt.NewInclusionProof(index, index+1, rangeProofs, true)
return &ShareWithProof{
share,
&proof,
proof,
}
}

Expand Down Expand Up @@ -119,7 +119,7 @@ func ProtoToShare(protoShares []*pb.Share) []*ShareWithProof {
continue
}
proof := ProtoToProof(share.Proof)
shares[i] = &ShareWithProof{share.Data, &proof}
shares[i] = &ShareWithProof{share.Data, proof}
}
return shares
}
Expand Down
146 changes: 146 additions & 0 deletions share/eds/cache_file.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
package eds

import (
"context"
"errors"
"fmt"

"github.com/ipfs/boxo/blockservice"
blocks "github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"

"github.com/celestiaorg/celestia-app/pkg/wrapper"
"github.com/celestiaorg/nmt"
"github.com/celestiaorg/rsmt2d"

"github.com/celestiaorg/celestia-node/share"
"github.com/celestiaorg/celestia-node/share/eds/byzantine"
"github.com/celestiaorg/celestia-node/share/ipld"
)

type CacheFile struct {
File

// TODO(@walldiss): add columns support
rowCache map[int]inMemoryAxis
// disableCache disables caching of rows for testing purposes
disableCache bool
}

type inMemoryAxis struct {
shares []share.Share
proofs blockservice.BlockGetter
}

func NewCacheFile(f File) *CacheFile {
return &CacheFile{
File: f,
rowCache: make(map[int]inMemoryAxis),
}
}

func (f *CacheFile) ShareWithProof(
ctx context.Context,
idx int,
axis rsmt2d.Axis,
axisRoot []byte,
) (*byzantine.ShareWithProof, error) {
sqrLn := f.Size()
axsIdx, shrIdx := idx/sqrLn, idx%sqrLn
if axis == rsmt2d.Col {
axsIdx, shrIdx = shrIdx, axsIdx
}

row := f.rowCache[axsIdx]
if row.proofs == nil {
shrs, err := f.Axis(axsIdx, axis)
if err != nil {
return nil, err
}

// calculate proofs
adder := ipld.NewProofsAdder(sqrLn*2, ipld.CollectShares)
tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(sqrLn/2), uint(axsIdx),
nmt.NodeVisitor(adder.VisitFn()))
for _, shr := range shrs {
err = tree.Push(shr)
if err != nil {
return nil, err
}
}

if _, err := tree.Root(); err != nil {
return nil, err
}

row = inMemoryAxis{
shares: shrs,
proofs: newRowProofsGetter(adder.Proofs()),
}

if !f.disableCache {
f.rowCache[axsIdx] = row
}
}

// TODO(@walldiss): find prealloc size for proofs
proof := make([]cid.Cid, 0, 8)
rootCid := ipld.MustCidFromNamespacedSha256(axisRoot)
proofs, err := ipld.GetProof(ctx, row.proofs, rootCid, proof, shrIdx, sqrLn)
if err != nil {
return nil, fmt.Errorf("bulding proof from cache: %w", err)
}

return byzantine.NewShareWithProof(shrIdx, row.shares[shrIdx], proofs), nil
}

func (f *CacheFile) Axis(idx int, axis rsmt2d.Axis) ([]share.Share, error) {
row, ok := f.rowCache[idx]
if ok {
return row.shares, nil
}

shrs, err := f.File.Axis(idx, axis)
if err != nil {
return nil, err
}

// cache row shares
if !f.disableCache {
f.rowCache[idx] = inMemoryAxis{
shares: shrs,
}
}
return shrs, nil
}

// TODO(@walldiss): needs to be implemented
func (f *CacheFile) EDS() (*rsmt2d.ExtendedDataSquare, error) {
return f.File.EDS()
}

// rowProofsGetter implements blockservice.BlockGetter interface
type rowProofsGetter struct {
proofs map[cid.Cid]blocks.Block
}

func newRowProofsGetter(rawProofs map[cid.Cid][]byte) *rowProofsGetter {
proofs := make(map[cid.Cid]blocks.Block, len(rawProofs))
for k, v := range rawProofs {
proofs[k] = blocks.NewBlock(v)
}
return &rowProofsGetter{
proofs: proofs,
}
}

func (r rowProofsGetter) GetBlock(_ context.Context, c cid.Cid) (blocks.Block, error) {
if b, ok := r.proofs[c]; ok {
return b, nil
}
return nil, errors.New("block not found")
}

func (r rowProofsGetter) GetBlocks(_ context.Context, _ []cid.Cid) <-chan blocks.Block {
panic("not implemented")
}
30 changes: 23 additions & 7 deletions share/eds/file.go
Original file line number Diff line number Diff line change
@@ -1,23 +1,31 @@
package eds

import (
"context"
"fmt"
"io"
"os"

"golang.org/x/exp/mmap"

"github.com/celestiaorg/celestia-app/pkg/wrapper"
"github.com/celestiaorg/nmt"
"github.com/celestiaorg/rsmt2d"

"github.com/celestiaorg/celestia-node/share"
"github.com/celestiaorg/celestia-node/share/eds/byzantine"
)

// TODO(@walldiss): Add context to all operations, so it don't backfire later
type File interface {
io.Closer
Size() int
ShareWithProof(idx int, axis rsmt2d.Axis) (share.Share, nmt.Proof, error)
ShareWithProof(
ctx context.Context,
idx int,
axis rsmt2d.Axis,
axisRoot []byte,
// TODO(@walldiss): move ShareWithProof to share pkg
) (*byzantine.ShareWithProof, error)
Axis(idx int, axis rsmt2d.Axis) ([]share.Share, error)
AxisHalf(idx int, axis rsmt2d.Axis) ([]share.Share, error)
EDS() (*rsmt2d.ExtendedDataSquare, error)
Expand Down Expand Up @@ -179,7 +187,12 @@ func (f *LazyFile) AxisHalf(idx int, axis rsmt2d.Axis) ([]share.Share, error) {
return fullAxis[:len(fullAxis)/2], nil
}

func (f *LazyFile) ShareWithProof(idx int, axis rsmt2d.Axis) (share.Share, nmt.Proof, error) {
func (f *LazyFile) ShareWithProof(
_ context.Context,
idx int,
axis rsmt2d.Axis,
_ []byte,
) (*byzantine.ShareWithProof, error) {
// TODO: Cache the axis as well as computed tree
sqrLn := int(f.hdr.squareSize)
axsIdx, shrIdx := idx/sqrLn, idx%sqrLn
Expand All @@ -189,23 +202,26 @@ func (f *LazyFile) ShareWithProof(idx int, axis rsmt2d.Axis) (share.Share, nmt.P

shrs, err := f.Axis(axsIdx, axis)
if err != nil {
return nil, nmt.Proof{}, err
return nil, err
}

tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(sqrLn/2), uint(axsIdx))
for _, shr := range shrs {
err = tree.Push(shr)
if err != nil {
return nil, nmt.Proof{}, err
return nil, err
}
}

proof, err := tree.ProveRange(shrIdx, shrIdx+1)
if err != nil {
return nil, nmt.Proof{}, err
return nil, err
}

return shrs[shrIdx], proof, nil
return &byzantine.ShareWithProof{
Share: shrs[shrIdx],
Proof: proof,
}, nil
}

func (f *LazyFile) EDS() (*rsmt2d.ExtendedDataSquare, error) {
Expand Down
Loading

0 comments on commit 46067e7

Please sign in to comment.