Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(storageincentives): extensive rewrite [WIP] #4375

Closed
wants to merge 1 commit into from
Closed

refactor(storageincentives): extensive rewrite [WIP] #4375

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions pkg/api/api.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
"github.com/ethersphere/bee/pkg/steward"
storage "github.com/ethersphere/bee/pkg/storage"
"github.com/ethersphere/bee/pkg/storageincentives"
"github.com/ethersphere/bee/pkg/storageincentives/sampler"
"github.com/ethersphere/bee/pkg/storageincentives/staking"
storer "github.com/ethersphere/bee/pkg/storer"
"github.com/ethersphere/bee/pkg/swarm"
Expand Down Expand Up @@ -153,6 +154,7 @@
probe *Probe
metricsRegistry *prometheus.Registry
stakingContract staking.Contract
sampler *sampler.Sampler
Options

http.Handler
Expand Down Expand Up @@ -181,6 +183,7 @@
batchStore postage.Storer
stamperStore storage.Store
syncStatus func() (bool, error)
sample *sampler.Sampler

Check failure on line 186 in pkg/api/api.go

View workflow job for this annotation

GitHub Actions / Lint 

field `sample` is unused (unused)

swap swap.Interface
transaction transaction.Service
Expand Down Expand Up @@ -245,6 +248,7 @@
Post postage.Service
PostageContract postagecontract.Interface
Staking staking.Contract
Sampler *sampler.Sampler
Steward steward.Interface
SyncStatus func() (bool, error)
NodeStatus *status.Service
Expand Down Expand Up @@ -323,6 +327,7 @@
s.postageContract = e.PostageContract
s.steward = e.Steward
s.stakingContract = e.Staking
s.sampler = e.Sampler

s.pingpong = e.Pingpong
s.topologyDriver = e.TopologyDriver
Expand Down
38 changes: 23 additions & 15 deletions pkg/api/api_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,10 @@
testingc "github.com/ethersphere/bee/pkg/storage/testing"
"github.com/ethersphere/bee/pkg/storageincentives"
"github.com/ethersphere/bee/pkg/storageincentives/redistribution"
"github.com/ethersphere/bee/pkg/storageincentives/sampler"
"github.com/ethersphere/bee/pkg/storageincentives/staking"
mock2 "github.com/ethersphere/bee/pkg/storageincentives/staking/mock"
storer "github.com/ethersphere/bee/pkg/storer"
mockstorer "github.com/ethersphere/bee/pkg/storer/mock"
"github.com/ethersphere/bee/pkg/swarm"
"github.com/ethersphere/bee/pkg/topology/lightnode"
Expand Down Expand Up @@ -213,7 +215,7 @@
s.SetP2P(o.P2P)

if o.RedistributionAgent == nil {
o.RedistributionAgent, _ = createRedistributionAgentService(t, o.Overlay, o.StateStorer, erc20, transaction, backend, o.BatchStore)
o.RedistributionAgent, _ = createRedistributionAgentService(t, o.Overlay, backend, o.BatchStore, o.StateStorer, erc20, transaction)
s.SetRedistributionAgent(o.RedistributionAgent)
}
testutil.CleanupCloser(t, o.RedistributionAgent)
Expand Down Expand Up @@ -689,16 +691,16 @@
func createRedistributionAgentService(
t *testing.T,
addr swarm.Address,
storer storage.StateStorer,
backend postage.ChainBackend,
batchstore sampler.Batchstore,
stateStore storage.StateStorer,
erc20Service erc20.Service,
tranService transaction.Service,
backend storageincentives.ChainBackend,
chainStateGetter postage.ChainStateGetter,
) (*storageincentives.Agent, error) {
t.Helper()

const blocksPerRound uint64 = 12
const blocksPerPhase uint64 = 4

Check failure on line 703 in pkg/api/api_test.go

View workflow job for this annotation

GitHub Actions / Lint 

const `blocksPerPhase` is unused (unused)
postageContract := contractMock.New(contractMock.WithExpiresBatchesFunc(func(context.Context) error {
return nil
}),
Expand All @@ -707,28 +709,34 @@
return true, nil
}))
contract := &mockContract{}

fullySyncedFunc := func() bool { return true }
healthyFunc := func() bool { return true }
return storageincentives.New(
log.Noop,
addr,
common.Address{},
time.Millisecond*10,
backend,
contract,
postageContract,
stakingContract,
mockstorer.NewReserve(),
func() bool { return true },
time.Millisecond*10,
blocksPerRound,
blocksPerPhase,
storer,
chainStateGetter,
sampler.New(backend, batchstore, &mockSampler{}, fullySyncedFunc, healthyFunc),
stateStore,
erc20Service,
tranService,
&mockHealth{},
log.Noop,
)
}

type mockSampler struct{}

func (*mockSampler) StorageRadius() uint8 {
return 0
}

func (*mockSampler) Iterate(uint8, func(storer.Chunk) (bool, error)) error {
return nil
}

type contractCall int

func (c contractCall) String() string {
Expand Down Expand Up @@ -776,7 +784,7 @@
return false, nil
}

func (m *mockContract) Claim(context.Context, redistribution.ChunkInclusionProofs) (common.Hash, error) {
func (m *mockContract) Claim(context.Context, []redistribution.Proof) (common.Hash, error) {
m.mtx.Lock()
defer m.mtx.Unlock()
m.callsList = append(m.callsList, claimCall)
Expand Down
97 changes: 48 additions & 49 deletions pkg/api/rchash.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,38 +16,32 @@ import (
"github.com/gorilla/mux"
)

type RCHashResponse struct {
Hash swarm.Address `json:"hash"`
Proofs ChunkInclusionProofs `json:"proofs"`
Duration time.Duration `json:"duration"`
type SampleWithProofs struct {
Hash swarm.Address `json:"hash"`
Proofs []Proof `json:"proofs"`
Duration time.Duration `json:"duration"`
}

type ChunkInclusionProofs struct {
A ChunkInclusionProof `json:"proof1"`
B ChunkInclusionProof `json:"proof2"`
C ChunkInclusionProof `json:"proofLast"`
}

// ChunkInclusionProof structure must exactly match
// Proof structure must exactly match
// corresponding structure (of the same name) in Redistribution.sol smart contract.
// github.com/ethersphere/storage-incentives/blob/ph_f2/src/Redistribution.sol
// github.com/ethersphere/storage-incentives/blob/master/src/Redistribution.sol (when merged to master)
type ChunkInclusionProof struct {
ProofSegments []string `json:"proofSegments"`
ProveSegment string `json:"proveSegment"`
ProofSegments2 []string `json:"proofSegments2"`
ProveSegment2 string `json:"proveSegment2"`
ChunkSpan uint64 `json:"chunkSpan"`
ProofSegments3 []string `json:"proofSegments3"`
PostageProof PostageProof `json:"postageProof"`
SocProof []SOCProof `json:"socProof"`
type Proof struct {
Sisters []string `json:"sisters"`
Data string `json:"data"`
Sisters2 []string `json:"sisters2"`
Data2 string `json:"data2"`
Sisters3 []string `json:"sisters3"`
ChunkSpan uint64 `json:"chunkSpan"`
PostageProof PostageProof `json:"postageProof"`
SocProof []SOCProof `json:"socProof"`
}

// SOCProof structure must exactly match
// corresponding structure (of the same name) in Redistribution.sol smart contract.
type PostageProof struct {
Signature string `json:"signature"`
PostageId string `json:"postageId"`
BatchID string `json:"BatchID"`
Index string `json:"index"`
TimeStamp string `json:"timeStamp"`
}
Expand All @@ -61,50 +55,54 @@ type SOCProof struct {
ChunkAddr string `json:"chunkAddr"`
}

func renderChunkInclusionProofs(proofs redistribution.ChunkInclusionProofs) ChunkInclusionProofs {
return ChunkInclusionProofs{
A: renderChunkInclusionProof(proofs.A),
B: renderChunkInclusionProof(proofs.B),
C: renderChunkInclusionProof(proofs.C),
func renderProofs(proofs []redistribution.Proof) []Proof {
out := make([]Proof, 3)
for i, p := range proofs {
out[i] = renderProof(p)
}
return out
}

func renderChunkInclusionProof(proof redistribution.ChunkInclusionProof) ChunkInclusionProof {
func renderProof(proof redistribution.Proof) Proof {
var socProof []SOCProof
if len(proof.SocProof) == 1 {
socProof = []SOCProof{{
Signer: hex.EncodeToString(proof.SocProof[0].Signer.Bytes()),
Signature: hex.EncodeToString(proof.SocProof[0].Signature[:]),
Identifier: hex.EncodeToString(proof.SocProof[0].Identifier.Bytes()),
ChunkAddr: hex.EncodeToString(proof.SocProof[0].ChunkAddr.Bytes()),
}}
socProof = []SOCProof{
{
Signer: toHex(proof.SocProof[0].Signer),
Signature: toHex(proof.SocProof[0].Signature),
Identifier: toHex(proof.SocProof[0].Identifier[:]),
ChunkAddr: toHex(proof.SocProof[0].ChunkAddr[:]),
},
}
}

return ChunkInclusionProof{
ProveSegment: hex.EncodeToString(proof.ProveSegment.Bytes()),
ProofSegments: renderCommonHash(proof.ProofSegments),
ProveSegment2: hex.EncodeToString(proof.ProveSegment2.Bytes()),
ProofSegments2: renderCommonHash(proof.ProofSegments2),
ProofSegments3: renderCommonHash(proof.ProofSegments3),
ChunkSpan: proof.ChunkSpan,
return Proof{
Data: toHex(proof.Data[:]),
Sisters: renderHash(proof.Sisters...),
Data2: toHex(proof.Data2[:]),
Sisters2: renderHash(proof.Sisters2...),
Sisters3: renderHash(proof.Sisters3...),
ChunkSpan: proof.ChunkSpan,
PostageProof: PostageProof{
Signature: hex.EncodeToString(proof.PostageProof.Signature[:]),
PostageId: hex.EncodeToString(proof.PostageProof.PostageId[:]),
Signature: toHex(proof.PostageProof.Signature),
BatchID: toHex(proof.PostageProof.BatchId[:]),
Index: strconv.FormatUint(proof.PostageProof.Index, 16),
TimeStamp: strconv.FormatUint(proof.PostageProof.TimeStamp, 16),
},
SocProof: socProof,
}
}

func renderCommonHash(proofSegments []common.Hash) []string {
output := make([]string, len(proofSegments))
for i, s := range proofSegments {
output[i] = hex.EncodeToString(s.Bytes())
func renderHash(hs ...common.Hash) []string {
output := make([]string, len(hs))
for i, h := range hs {
output[i] = hex.EncodeToString(h.Bytes())
}
return output
}

var toHex func([]byte) string = hex.EncodeToString

// This API is kept for testing the sampler. As a result, no documentation or tests are added here.
func (s *Service) rchash(w http.ResponseWriter, r *http.Request) {
logger := s.logger.WithName("get_rchash").Build()
Expand All @@ -123,17 +121,18 @@ func (s *Service) rchash(w http.ResponseWriter, r *http.Request) {

anchor2 := []byte(paths.Anchor2)

swp, err := s.redistributionAgent.SampleWithProofs(r.Context(), anchor1, anchor2, paths.Depth)
var round uint64
swp, err := s.sampler.ReserveSampleWithProofs(r.Context(), anchor1, anchor2, paths.Depth, round)
if err != nil {
logger.Error(err, "failed making sample with proofs")
jsonhttp.InternalServerError(w, "failed making sample with proofs")
return
}

resp := RCHashResponse{
resp := SampleWithProofs{
Hash: swp.Hash,
Duration: swp.Duration,
Proofs: renderChunkInclusionProofs(swp.Proofs),
Proofs: renderProofs(swp.Proofs),
}

jsonhttp.OK(w, resp)
Expand Down
15 changes: 8 additions & 7 deletions pkg/bmt/proof.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,10 @@

// Proof represents a Merkle proof of segment
type Proof struct {
ProveSegment []byte
ProofSegments [][]byte
Span []byte
Index int
Data []byte
Sisters [][]byte
Span []byte
Index int
}

// Hash overrides base hash function of Hasher to fill buffer with zeros until chunk length
Expand All @@ -28,6 +28,7 @@
return p.Hasher.Hash(b)
}


Check failure on line 31 in pkg/bmt/proof.go

View workflow job for this annotation

GitHub Actions / Lint 

File is not `gofmt`-ed with `-s` (gofmt)
// Proof returns the inclusion proof of the i-th data segment
func (p Prover) Proof(i int) Proof {
index := i
Expand Down Expand Up @@ -60,9 +61,9 @@
func (p Prover) Verify(i int, proof Proof) (root []byte, err error) {
var section []byte
if i%2 == 0 {
section = append(append(section, proof.ProveSegment...), proof.ProofSegments[0]...)
section = append(append(section, proof.Data...), proof.Sisters[0]...)
} else {
section = append(append(section, proof.ProofSegments[0]...), proof.ProveSegment...)
section = append(append(section, proof.Sisters[0]...), proof.Data...)
}
i = i / 2
n := p.bmt.leaves[i]
Expand All @@ -74,7 +75,7 @@
}
n = n.parent

for _, sister := range proof.ProofSegments[1:] {
for _, sister := range proof.Sisters[1:] {
if isLeft {
root, err = doHash(hasher, root, sister)
} else {
Expand Down
18 changes: 9 additions & 9 deletions pkg/bmt/proof_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,9 +79,9 @@ func TestProofCorrectness(t *testing.T) {
"887c22bd8750d34016ac3c66b5ff102dacdd73f6b014e710b51e8022af9a1968",
}

verifySegments(t, expSegmentStrings, proof.ProofSegments)
verifySegments(t, expSegmentStrings, proof.Sisters)

if !bytes.Equal(proof.ProveSegment, testDataPadded[:hh.Size()]) {
if !bytes.Equal(proof.Data, testDataPadded[:hh.Size()]) {
t.Fatal("section incorrect")
}

Expand All @@ -105,9 +105,9 @@ func TestProofCorrectness(t *testing.T) {
"745bae095b6ff5416b4a351a167f731db6d6f5924f30cd88d48e74261795d27b",
}

verifySegments(t, expSegmentStrings, proof.ProofSegments)
verifySegments(t, expSegmentStrings, proof.Sisters)

if !bytes.Equal(proof.ProveSegment, testDataPadded[127*hh.Size():]) {
if !bytes.Equal(proof.Data, testDataPadded[127*hh.Size():]) {
t.Fatal("section incorrect")
}

Expand All @@ -131,9 +131,9 @@ func TestProofCorrectness(t *testing.T) {
"745bae095b6ff5416b4a351a167f731db6d6f5924f30cd88d48e74261795d27b",
}

verifySegments(t, expSegmentStrings, proof.ProofSegments)
verifySegments(t, expSegmentStrings, proof.Sisters)

if !bytes.Equal(proof.ProveSegment, testDataPadded[64*hh.Size():65*hh.Size()]) {
if !bytes.Equal(proof.Data, testDataPadded[64*hh.Size():65*hh.Size()]) {
t.Fatal("section incorrect")
}

Expand Down Expand Up @@ -167,9 +167,9 @@ func TestProofCorrectness(t *testing.T) {
segment := testDataPadded[64*hh.Size() : 65*hh.Size()]

rootHash, err := pr.Verify(64, bmt.Proof{
ProveSegment: segment,
ProofSegments: segments,
Span: bmt.LengthToSpan(4096),
Data: segment,
Sisters: segments,
Span: bmt.LengthToSpan(4096),
})
if err != nil {
t.Fatal(err)
Expand Down
Loading
Loading