From 19e029cb2a62b8fe801d48e5eb4b13fae133f26d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tr=C3=B3n?= Date: Thu, 28 Sep 2023 13:35:41 +0200 Subject: [PATCH 01/24] feat(storageincentives): storage incentives phase 4 (#4345) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Viktor Levente Tóth Co-authored-by: nugaon <50576770+nugaon@users.noreply.github.com> Co-authored-by: istae <14264581+istae@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 +- pkg/api/api.go | 5 + pkg/api/api_test.go | 5 +- pkg/api/rchash.go | 119 +++++++++- pkg/api/router.go | 12 +- pkg/bmt/bmt.go | 13 ++ pkg/bmt/proof.go | 35 ++- pkg/bmt/proof_test.go | 30 ++- pkg/bmt/trhasher.go | 25 --- pkg/config/chain.go | 6 +- pkg/storageincentives/agent.go | 48 +++- pkg/storageincentives/agent_test.go | 4 +- pkg/storageincentives/export_test.go | 5 +- pkg/storageincentives/proof.go | 206 ++++++++++++++++-- pkg/storageincentives/proof_test.go | 143 +++++++++++- .../redistribution/inclusionproof.go | 105 +++++++++ .../redistribution/redistribution.go | 10 +- .../redistribution/redistribution_test.go | 50 ++++- pkg/storageincentives/redistributionstate.go | 7 +- pkg/storageincentives/soc_mine_test.go | 201 +++++++++++++++++ .../testdata/inclusion-proofs.json | 126 +++++++++++ pkg/storer/sample.go | 73 +++++-- pkg/swarm/hasher.go | 10 +- pkg/swarm/hasher_test.go | 2 +- 25 files changed, 1103 insertions(+), 143 deletions(-) delete mode 100644 pkg/bmt/trhasher.go create mode 100644 pkg/storageincentives/redistribution/inclusionproof.go create mode 100644 pkg/storageincentives/soc_mine_test.go create mode 100644 pkg/storageincentives/testdata/inclusion-proofs.json diff --git a/go.mod b/go.mod index 95539b3cbd6..9ca865cf074 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/coreos/go-semver v0.3.0 github.com/ethereum/go-ethereum v1.12.2 github.com/ethersphere/go-price-oracle-abi v0.1.0 - github.com/ethersphere/go-storage-incentives-abi v0.5.0 + github.com/ethersphere/go-storage-incentives-abi v0.6.0-rc3 github.com/ethersphere/go-sw3-abi v0.4.0 github.com/ethersphere/langos v1.0.0 github.com/go-playground/validator/v10 v10.11.1 diff --git a/go.sum b/go.sum index 0668711533d..8b98c44d2ed 100644 --- a/go.sum +++ b/go.sum @@ -223,8 +223,8 @@ github.com/ethereum/go-ethereum v1.12.2 h1:eGHJ4ij7oyVqUQn48LBz3B7pvQ8sV0wGJiIE6 github.com/ethereum/go-ethereum v1.12.2/go.mod h1:1cRAEV+rp/xX0zraSCBnu9Py3HQ+geRMj3HdR+k0wfI= github.com/ethersphere/go-price-oracle-abi v0.1.0 h1:yg/hK8nETNvk+GEBASlbakMFv/CVp7HXiycrHw1pRV8= github.com/ethersphere/go-price-oracle-abi v0.1.0/go.mod h1:sI/Qj4/zJ23/b1enzwMMv0/hLTpPNVNacEwCWjo6yBk= -github.com/ethersphere/go-storage-incentives-abi v0.5.0 h1:dd01OZmPraCjOIiSX5FsCfFFwUR2b9PuTO/LDcYxS+s= -github.com/ethersphere/go-storage-incentives-abi v0.5.0/go.mod h1:SXvJVtM4sEsaSKD0jc1ClpDLw8ErPoROZDme4Wrc/Nc= +github.com/ethersphere/go-storage-incentives-abi v0.6.0-rc3 h1:tXux2FnhuU6DbrY+Z4nVQMGp63JkJPq7pKb5Xi2Sjxo= +github.com/ethersphere/go-storage-incentives-abi v0.6.0-rc3/go.mod h1:SXvJVtM4sEsaSKD0jc1ClpDLw8ErPoROZDme4Wrc/Nc= github.com/ethersphere/go-sw3-abi v0.4.0 h1:T3ANY+ktWrPAwe2U0tZi+DILpkHzto5ym/XwV/Bbz8g= github.com/ethersphere/go-sw3-abi v0.4.0/go.mod h1:BmpsvJ8idQZdYEtWnvxA8POYQ8Rl/NhyCdF0zLMOOJU= github.com/ethersphere/langos v1.0.0 h1:NBtNKzXTTRSue95uOlzPN4py7Aofs0xWPzyj4AI1Vcc= diff --git a/pkg/api/api.go b/pkg/api/api.go index 3ecea7ad000..49331689a0f 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -10,6 +10,7 @@ import ( "context" "crypto/ecdsa" "encoding/base64" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -286,6 +287,10 @@ func New( buf, err := base64.URLEncoding.DecodeString(v) return string(buf), err }, + "decHex": func(v string) (string, error) { + buf, err := hex.DecodeString(v) + return string(buf), err + }, } s.validate = validator.New() s.validate.RegisterTagNameFunc(func(fld reflect.StructField) string { diff --git a/pkg/api/api_test.go b/pkg/api/api_test.go index 8655853474f..21e535f10ac 100644 --- a/pkg/api/api_test.go +++ b/pkg/api/api_test.go @@ -57,6 +57,7 @@ import ( "github.com/ethersphere/bee/pkg/storage/inmemstore" testingc "github.com/ethersphere/bee/pkg/storage/testing" "github.com/ethersphere/bee/pkg/storageincentives" + "github.com/ethersphere/bee/pkg/storageincentives/redistribution" "github.com/ethersphere/bee/pkg/storageincentives/staking" mock2 "github.com/ethersphere/bee/pkg/storageincentives/staking/mock" mockstorer "github.com/ethersphere/bee/pkg/storer/mock" @@ -775,14 +776,14 @@ func (m *mockContract) IsWinner(context.Context) (bool, error) { return false, nil } -func (m *mockContract) Claim(context.Context) (common.Hash, error) { +func (m *mockContract) Claim(context.Context, redistribution.ChunkInclusionProofs) (common.Hash, error) { m.mtx.Lock() defer m.mtx.Unlock() m.callsList = append(m.callsList, claimCall) return common.Hash{}, nil } -func (m *mockContract) Commit(context.Context, []byte, *big.Int) (common.Hash, error) { +func (m *mockContract) Commit(context.Context, []byte, uint32) (common.Hash, error) { m.mtx.Lock() defer m.mtx.Unlock() m.callsList = append(m.callsList, commitCall) diff --git a/pkg/api/rchash.go b/pkg/api/rchash.go index c37f642da2b..ad7a25f9992 100644 --- a/pkg/api/rchash.go +++ b/pkg/api/rchash.go @@ -6,40 +6,135 @@ package api import ( "encoding/hex" "net/http" + "strconv" + "time" + "github.com/ethereum/go-ethereum/common" "github.com/ethersphere/bee/pkg/jsonhttp" - "github.com/ethersphere/bee/pkg/storageincentives" + "github.com/ethersphere/bee/pkg/storageincentives/redistribution" + "github.com/ethersphere/bee/pkg/swarm" "github.com/gorilla/mux" ) -type RCHashResponse storageincentives.SampleWithProofs +type RCHashResponse struct { + Hash swarm.Address `json:"hash"` + Proofs ChunkInclusionProofs `json:"proofs"` + Duration time.Duration `json:"duration"` +} + +type ChunkInclusionProofs struct { + A ChunkInclusionProof `json:"proof1"` + B ChunkInclusionProof `json:"proof2"` + C ChunkInclusionProof `json:"proofLast"` +} + +// ChunkInclusionProof structure must exactly match +// corresponding structure (of the same name) in Redistribution.sol smart contract. +// github.com/ethersphere/storage-incentives/blob/ph_f2/src/Redistribution.sol +// github.com/ethersphere/storage-incentives/blob/master/src/Redistribution.sol (when merged to master) +type ChunkInclusionProof struct { + ProofSegments []string `json:"proofSegments"` + ProveSegment string `json:"proveSegment"` + ProofSegments2 []string `json:"proofSegments2"` + ProveSegment2 string `json:"proveSegment2"` + ChunkSpan uint64 `json:"chunkSpan"` + ProofSegments3 []string `json:"proofSegments3"` + PostageProof PostageProof `json:"postageProof"` + SocProof []SOCProof `json:"socProof"` +} + +// SOCProof structure must exactly match +// corresponding structure (of the same name) in Redistribution.sol smart contract. +type PostageProof struct { + Signature string `json:"signature"` + PostageId string `json:"postageId"` + Index string `json:"index"` + TimeStamp string `json:"timeStamp"` +} + +// SOCProof structure must exactly match +// corresponding structure (of the same name) in Redistribution.sol smart contract. +type SOCProof struct { + Signer string `json:"signer"` + Signature string `json:"signature"` + Identifier string `json:"identifier"` + ChunkAddr string `json:"chunkAddr"` +} + +func renderChunkInclusionProofs(proofs redistribution.ChunkInclusionProofs) ChunkInclusionProofs { + return ChunkInclusionProofs{ + A: renderChunkInclusionProof(proofs.A), + B: renderChunkInclusionProof(proofs.B), + C: renderChunkInclusionProof(proofs.C), + } +} + +func renderChunkInclusionProof(proof redistribution.ChunkInclusionProof) ChunkInclusionProof { + var socProof []SOCProof + if len(proof.SocProof) == 1 { + socProof = []SOCProof{{ + Signer: hex.EncodeToString(proof.SocProof[0].Signer.Bytes()), + Signature: hex.EncodeToString(proof.SocProof[0].Signature[:]), + Identifier: hex.EncodeToString(proof.SocProof[0].Identifier.Bytes()), + ChunkAddr: hex.EncodeToString(proof.SocProof[0].ChunkAddr.Bytes()), + }} + } + + return ChunkInclusionProof{ + ProveSegment: hex.EncodeToString(proof.ProveSegment.Bytes()), + ProofSegments: renderCommonHash(proof.ProofSegments), + ProveSegment2: hex.EncodeToString(proof.ProveSegment2.Bytes()), + ProofSegments2: renderCommonHash(proof.ProofSegments2), + ProofSegments3: renderCommonHash(proof.ProofSegments3), + ChunkSpan: proof.ChunkSpan, + PostageProof: PostageProof{ + Signature: hex.EncodeToString(proof.PostageProof.Signature[:]), + PostageId: hex.EncodeToString(proof.PostageProof.PostageId[:]), + Index: strconv.FormatUint(proof.PostageProof.Index, 16), + TimeStamp: strconv.FormatUint(proof.PostageProof.TimeStamp, 16), + }, + SocProof: socProof, + } +} + +func renderCommonHash(proofSegments []common.Hash) []string { + output := make([]string, len(proofSegments)) + for i, s := range proofSegments { + output[i] = hex.EncodeToString(s.Bytes()) + } + return output +} // This API is kept for testing the sampler. As a result, no documentation or tests are added here. func (s *Service) rchash(w http.ResponseWriter, r *http.Request) { logger := s.logger.WithName("get_rchash").Build() paths := struct { - Depth *uint8 `map:"depth" validate:"required"` - Anchor1 string `map:"anchor1" validate:"required"` + Depth uint8 `map:"depth"` + Anchor1 string `map:"anchor1,decHex" validate:"required"` + Anchor2 string `map:"anchor2,decHex" validate:"required"` }{} if response := s.mapStructure(mux.Vars(r), &paths); response != nil { response("invalid path params", logger, w) return } - anchor1, err := hex.DecodeString(paths.Anchor1) - if err != nil { - logger.Error(err, "invalid hex params") - jsonhttp.InternalServerError(w, "invalid hex params") - return - } + anchor1 := []byte(paths.Anchor1) - resp, err := s.redistributionAgent.SampleWithProofs(r.Context(), anchor1, *paths.Depth) + anchor2 := []byte(paths.Anchor2) + + swp, err := s.redistributionAgent.SampleWithProofs(r.Context(), anchor1, anchor2, paths.Depth) if err != nil { logger.Error(err, "failed making sample with proofs") jsonhttp.InternalServerError(w, "failed making sample with proofs") return } - jsonhttp.OK(w, RCHashResponse(resp)) + resp := RCHashResponse{ + Hash: swp.Hash, + Duration: swp.Duration, + Proofs: renderChunkInclusionProofs(swp.Proofs), + } + + jsonhttp.OK(w, resp) } diff --git a/pkg/api/router.go b/pkg/api/router.go index b7f83a6623c..6408fd7b9af 100644 --- a/pkg/api/router.go +++ b/pkg/api/router.go @@ -339,12 +339,6 @@ func (s *Service) mountAPI() { web.FinalHandlerFunc(s.healthHandler), )) - handle("/rchash/{depth}/{anchor1}", web.ChainHandlers( - web.FinalHandler(jsonhttp.MethodHandler{ - "GET": http.HandlerFunc(s.rchash), - }), - )) - if s.Restricted { handle("/auth", jsonhttp.MethodHandler{ "POST": web.ChainHandlers( @@ -601,4 +595,10 @@ func (s *Service) mountBusinessDebug(restricted bool) { web.FinalHandlerFunc(s.statusGetPeersHandler), ), }) + + handle("/rchash/{depth}/{anchor1}/{anchor2}", web.ChainHandlers( + web.FinalHandler(jsonhttp.MethodHandler{ + "GET": http.HandlerFunc(s.rchash), + }), + )) } diff --git a/pkg/bmt/bmt.go b/pkg/bmt/bmt.go index e13aa3ec1cb..f314e012776 100644 --- a/pkg/bmt/bmt.go +++ b/pkg/bmt/bmt.go @@ -40,6 +40,19 @@ type Hasher struct { span []byte // The span of the data subsumed under the chunk } +// NewHasher gives back an instance of a Hasher struct +func NewHasher(hasherFact func() hash.Hash) *Hasher { + conf := NewConf(hasherFact, swarm.BmtBranches, 32) + + return &Hasher{ + Conf: conf, + result: make(chan []byte), + errc: make(chan error, 1), + span: make([]byte, SpanSize), + bmt: newTree(conf.segmentSize, conf.maxSize, conf.depth, conf.hasher), + } +} + // Capacity returns the maximum amount of bytes that will be processed by this hasher implementation. // since BMT assumes a balanced binary tree, capacity it is always a power of 2 func (h *Hasher) Capacity() int { diff --git a/pkg/bmt/proof.go b/pkg/bmt/proof.go index b9a958db9ab..cc59fd33766 100644 --- a/pkg/bmt/proof.go +++ b/pkg/bmt/proof.go @@ -17,6 +17,17 @@ type Proof struct { Index int } +// Hash overrides base hash function of Hasher to fill buffer with zeros until chunk length +func (p Prover) Hash(b []byte) ([]byte, error) { + for i := p.size; i < p.maxSize; i += len(zerosection) { + _, err := p.Hasher.Write(zerosection) + if err != nil { + return nil, err + } + } + return p.Hasher.Hash(b) +} + // Proof returns the inclusion proof of the i-th data segment func (p Prover) Proof(i int) Proof { index := i @@ -36,26 +47,38 @@ func (p Prover) Proof(i int) Proof { secsize := 2 * p.segmentSize offset := i * secsize section := p.bmt.buffer[offset : offset+secsize] - return Proof{section, sisters, p.span, index} + segment, firstSegmentSister := section[:p.segmentSize], section[p.segmentSize:] + if index%2 != 0 { + segment, firstSegmentSister = firstSegmentSister, segment + } + sisters = append([][]byte{firstSegmentSister}, sisters...) + return Proof{segment, sisters, p.span, index} } // Verify returns the bmt hash obtained from the proof which can then be checked against // the BMT hash of the chunk func (p Prover) Verify(i int, proof Proof) (root []byte, err error) { + var section []byte + if i%2 == 0 { + section = append(append(section, proof.ProveSegment...), proof.ProofSegments[0]...) + } else { + section = append(append(section, proof.ProofSegments[0]...), proof.ProveSegment...) + } i = i / 2 n := p.bmt.leaves[i] + hasher := p.hasher() isLeft := n.isLeft - root, err = doHash(n.hasher, proof.ProveSegment) + root, err = doHash(hasher, section) if err != nil { return nil, err } n = n.parent - for _, sister := range proof.ProofSegments { + for _, sister := range proof.ProofSegments[1:] { if isLeft { - root, err = doHash(n.hasher, root, sister) + root, err = doHash(hasher, root, sister) } else { - root, err = doHash(n.hasher, sister, root) + root, err = doHash(hasher, sister, root) } if err != nil { return nil, err @@ -63,7 +86,7 @@ func (p Prover) Verify(i int, proof Proof) (root []byte, err error) { isLeft = n.isLeft n = n.parent } - return sha3hash(proof.Span, root) + return doHash(hasher, proof.Span, root) } func (n *node) getSister(isLeft bool) []byte { diff --git a/pkg/bmt/proof_test.go b/pkg/bmt/proof_test.go index 337b1bf3420..1b7f6d3b3dd 100644 --- a/pkg/bmt/proof_test.go +++ b/pkg/bmt/proof_test.go @@ -20,7 +20,8 @@ func TestProofCorrectness(t *testing.T) { t.Parallel() testData := []byte("hello world") - testData = append(testData, make([]byte, 4096-len(testData))...) + testDataPadded := make([]byte, swarm.ChunkSize) + copy(testDataPadded, testData) verifySegments := func(t *testing.T, exp []string, found [][]byte) { t.Helper() @@ -57,8 +58,8 @@ func TestProofCorrectness(t *testing.T) { if err != nil { t.Fatal(err) } - - rh, err := hh.Hash(nil) + pr := bmt.Prover{hh} + rh, err := pr.Hash(nil) if err != nil { t.Fatal(err) } @@ -66,9 +67,10 @@ func TestProofCorrectness(t *testing.T) { t.Run("proof for left most", func(t *testing.T) { t.Parallel() - proof := bmt.Prover{hh}.Proof(0) + proof := pr.Proof(0) expSegmentStrings := []string{ + "0000000000000000000000000000000000000000000000000000000000000000", "ad3228b676f7d3cd4284a5443f17f1962b36e491b30a40b2405849e597ba5fb5", "b4c11951957c6f8f642c4af61cd6b24640fec6dc7fc607ee8206a99e92410d30", "21ddb9a356815c3fac1026b6dec5df3124afbadb485c9ba5a3e3398a04b7ba85", @@ -79,7 +81,7 @@ func TestProofCorrectness(t *testing.T) { verifySegments(t, expSegmentStrings, proof.ProofSegments) - if !bytes.Equal(proof.ProveSegment, testData[:2*hh.Size()]) { + if !bytes.Equal(proof.ProveSegment, testDataPadded[:hh.Size()]) { t.Fatal("section incorrect") } @@ -91,9 +93,10 @@ func TestProofCorrectness(t *testing.T) { t.Run("proof for right most", func(t *testing.T) { t.Parallel() - proof := bmt.Prover{hh}.Proof(127) + proof := pr.Proof(127) expSegmentStrings := []string{ + "0000000000000000000000000000000000000000000000000000000000000000", "ad3228b676f7d3cd4284a5443f17f1962b36e491b30a40b2405849e597ba5fb5", "b4c11951957c6f8f642c4af61cd6b24640fec6dc7fc607ee8206a99e92410d30", "21ddb9a356815c3fac1026b6dec5df3124afbadb485c9ba5a3e3398a04b7ba85", @@ -104,7 +107,7 @@ func TestProofCorrectness(t *testing.T) { verifySegments(t, expSegmentStrings, proof.ProofSegments) - if !bytes.Equal(proof.ProveSegment, testData[126*hh.Size():]) { + if !bytes.Equal(proof.ProveSegment, testDataPadded[127*hh.Size():]) { t.Fatal("section incorrect") } @@ -116,9 +119,10 @@ func TestProofCorrectness(t *testing.T) { t.Run("proof for middle", func(t *testing.T) { t.Parallel() - proof := bmt.Prover{hh}.Proof(64) + proof := pr.Proof(64) expSegmentStrings := []string{ + "0000000000000000000000000000000000000000000000000000000000000000", "ad3228b676f7d3cd4284a5443f17f1962b36e491b30a40b2405849e597ba5fb5", "b4c11951957c6f8f642c4af61cd6b24640fec6dc7fc607ee8206a99e92410d30", "21ddb9a356815c3fac1026b6dec5df3124afbadb485c9ba5a3e3398a04b7ba85", @@ -129,7 +133,7 @@ func TestProofCorrectness(t *testing.T) { verifySegments(t, expSegmentStrings, proof.ProofSegments) - if !bytes.Equal(proof.ProveSegment, testData[64*hh.Size():66*hh.Size()]) { + if !bytes.Equal(proof.ProveSegment, testDataPadded[64*hh.Size():65*hh.Size()]) { t.Fatal("section incorrect") } @@ -142,6 +146,7 @@ func TestProofCorrectness(t *testing.T) { t.Parallel() segmentStrings := []string{ + "0000000000000000000000000000000000000000000000000000000000000000", "ad3228b676f7d3cd4284a5443f17f1962b36e491b30a40b2405849e597ba5fb5", "b4c11951957c6f8f642c4af61cd6b24640fec6dc7fc607ee8206a99e92410d30", "21ddb9a356815c3fac1026b6dec5df3124afbadb485c9ba5a3e3398a04b7ba85", @@ -159,9 +164,9 @@ func TestProofCorrectness(t *testing.T) { segments = append(segments, decoded) } - segment := testData[64*hh.Size() : 66*hh.Size()] + segment := testDataPadded[64*hh.Size() : 65*hh.Size()] - rootHash, err := bmt.Prover{hh}.Verify(64, bmt.Proof{ + rootHash, err := pr.Verify(64, bmt.Proof{ ProveSegment: segment, ProofSegments: segments, Span: bmt.LengthToSpan(4096), @@ -200,6 +205,7 @@ func TestProof(t *testing.T) { } rh, err := hh.Hash(nil) + pr := bmt.Prover{hh} if err != nil { t.Fatal(err) } @@ -209,7 +215,7 @@ func TestProof(t *testing.T) { t.Run(fmt.Sprintf("segmentIndex %d", i), func(t *testing.T) { t.Parallel() - proof := bmt.Prover{hh}.Proof(i) + proof := pr.Proof(i) h := pool.Get() defer pool.Put(h) diff --git a/pkg/bmt/trhasher.go b/pkg/bmt/trhasher.go deleted file mode 100644 index 00df6664b85..00000000000 --- a/pkg/bmt/trhasher.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2023 The Swarm Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package bmt - -import ( - "hash" - - "github.com/ethersphere/bee/pkg/swarm" -) - -func NewTrHasher(prefix []byte) *Hasher { - capacity := 32 - hasherFact := func() hash.Hash { return swarm.NewTrHasher(prefix) } - conf := NewConf(hasherFact, swarm.BmtBranches, capacity) - - return &Hasher{ - Conf: conf, - result: make(chan []byte), - errc: make(chan error, 1), - span: make([]byte, SpanSize), - bmt: newTree(conf.segmentSize, conf.maxSize, conf.depth, conf.hasher), - } -} diff --git a/pkg/config/chain.go b/pkg/config/chain.go index 2fd2e6f8fe7..4d7fd19e419 100644 --- a/pkg/config/chain.go +++ b/pkg/config/chain.go @@ -42,7 +42,7 @@ var ( SwarmTokenSymbol: "gBZZ", StakingAddress: common.HexToAddress(abi.TestnetStakingAddress), - PostageStampAddress: common.HexToAddress(abi.TestnetPostageStampStampAddress), + PostageStampAddress: common.HexToAddress(abi.TestnetPostageStampAddress), RedistributionAddress: common.HexToAddress(abi.TestnetRedistributionAddress), SwapPriceOracleAddress: common.HexToAddress("0x0c9de531dcb38b758fe8a2c163444a5e54ee0db2"), CurrentFactoryAddress: common.HexToAddress("0x73c412512E1cA0be3b89b77aB3466dA6A1B9d273"), @@ -51,7 +51,7 @@ var ( }, StakingABI: abi.TestnetStakingABI, - PostageStampABI: abi.TestnetPostageStampStampABI, + PostageStampABI: abi.TestnetPostageStampABI, RedistributionABI: abi.TestnetRedistributionABI, } @@ -84,7 +84,7 @@ func GetByChainID(chainID int64) (ChainConfig, bool) { NativeTokenSymbol: Testnet.NativeTokenSymbol, SwarmTokenSymbol: Testnet.SwarmTokenSymbol, StakingABI: abi.TestnetStakingABI, - PostageStampABI: abi.TestnetPostageStampStampABI, + PostageStampABI: abi.TestnetPostageStampABI, RedistributionABI: abi.TestnetRedistributionABI, }, false } diff --git a/pkg/storageincentives/agent.go b/pkg/storageincentives/agent.go index 3a6ba36074a..b856625025e 100644 --- a/pkg/storageincentives/agent.go +++ b/pkg/storageincentives/agent.go @@ -353,7 +353,22 @@ func (a *Agent) handleClaim(ctx context.Context, round uint64) error { a.logger.Info("could not set balance", "err", err) } - txHash, err := a.contract.Claim(ctx) + sampleData, exists := a.state.SampleData(round - 1) + if !exists { + return fmt.Errorf("sample not found") + } + + anchor2, err := a.contract.ReserveSalt(ctx) + if err != nil { + a.logger.Info("failed getting anchor after second reveal", "err", err) + } + + proofs, err := makeInclusionProofs(sampleData.ReserveSampleItems, sampleData.Anchor1, anchor2) + if err != nil { + return fmt.Errorf("making inclusion proofs: %w", err) + } + + txHash, err := a.contract.Claim(ctx, proofs) if err != nil { a.metrics.ErrClaim.Inc() return fmt.Errorf("claiming win: %w", err) @@ -418,10 +433,12 @@ func (a *Agent) handleSample(ctx context.Context, round uint64) (bool, error) { if err != nil { return false, err } + dur := time.Since(now) + a.metrics.SampleDuration.Set(dur.Seconds()) a.logger.Info("produced sample", "hash", sample.ReserveSampleHash, "radius", sample.StorageRadius, "round", round) - a.state.SetSampleData(round, sample, time.Since(now)) + a.state.SetSampleData(round, sample, dur) return true, nil } @@ -437,12 +454,10 @@ func (a *Agent) makeSample(ctx context.Context, storageRadius uint8) (SampleData return SampleData{}, err } - t := time.Now() rSample, err := a.store.ReserveSample(ctx, salt, storageRadius, uint64(timeLimiter), a.minBatchBalance()) if err != nil { return SampleData{}, err } - a.metrics.SampleDuration.Set(time.Since(t).Seconds()) sampleHash, err := sampleHash(rSample.Items) if err != nil { @@ -450,8 +465,10 @@ func (a *Agent) makeSample(ctx context.Context, storageRadius uint8) (SampleData } sample := SampleData{ - ReserveSampleHash: sampleHash, - StorageRadius: storageRadius, + Anchor1: salt, + ReserveSampleItems: rSample.Items, + ReserveSampleHash: sampleHash, + StorageRadius: storageRadius, } return sample, nil @@ -493,7 +510,7 @@ func (a *Agent) commit(ctx context.Context, sample SampleData, round uint64) err return err } - txHash, err := a.contract.Commit(ctx, obfuscatedHash, big.NewInt(int64(round))) + txHash, err := a.contract.Commit(ctx, obfuscatedHash, uint32(round)) if err != nil { a.metrics.ErrCommit.Inc() return err @@ -538,14 +555,16 @@ func (a *Agent) Status() (*Status, error) { } type SampleWithProofs struct { - Items []storer.SampleItem - Hash swarm.Address - Duration time.Duration + Hash swarm.Address `json:"hash"` + Proofs redistribution.ChunkInclusionProofs `json:"proofs"` + Duration time.Duration `json:"duration"` } +// SampleWithProofs is called only by rchash API func (a *Agent) SampleWithProofs( ctx context.Context, anchor1 []byte, + anchor2 []byte, storageRadius uint8, ) (SampleWithProofs, error) { sampleStartTime := time.Now() @@ -562,12 +581,17 @@ func (a *Agent) SampleWithProofs( hash, err := sampleHash(rSample.Items) if err != nil { - return SampleWithProofs{}, fmt.Errorf("sample hash: %w:", err) + return SampleWithProofs{}, fmt.Errorf("sample hash: %w", err) + } + + proofs, err := makeInclusionProofs(rSample.Items, anchor1, anchor2) + if err != nil { + return SampleWithProofs{}, fmt.Errorf("make proofs: %w", err) } return SampleWithProofs{ - Items: rSample.Items, Hash: hash, + Proofs: proofs, Duration: time.Since(sampleStartTime), }, nil } diff --git a/pkg/storageincentives/agent_test.go b/pkg/storageincentives/agent_test.go index 5d7ca5ffe90..70ec3193975 100644 --- a/pkg/storageincentives/agent_test.go +++ b/pkg/storageincentives/agent_test.go @@ -276,14 +276,14 @@ func (m *mockContract) IsWinner(context.Context) (bool, error) { return false, nil } -func (m *mockContract) Claim(context.Context) (common.Hash, error) { +func (m *mockContract) Claim(context.Context, redistribution.ChunkInclusionProofs) (common.Hash, error) { m.mtx.Lock() defer m.mtx.Unlock() m.callsList = append(m.callsList, claimCall) return common.Hash{}, nil } -func (m *mockContract) Commit(context.Context, []byte, *big.Int) (common.Hash, error) { +func (m *mockContract) Commit(context.Context, []byte, uint32) (common.Hash, error) { m.mtx.Lock() defer m.mtx.Unlock() m.callsList = append(m.callsList, commitCall) diff --git a/pkg/storageincentives/export_test.go b/pkg/storageincentives/export_test.go index 8c26fc1761c..f617501ba29 100644 --- a/pkg/storageincentives/export_test.go +++ b/pkg/storageincentives/export_test.go @@ -5,6 +5,7 @@ package storageincentives var ( - NewEvents = newEvents - SampleChunk = sampleChunk + NewEvents = newEvents + SampleChunk = sampleChunk + MakeInclusionProofs = makeInclusionProofs ) diff --git a/pkg/storageincentives/proof.go b/pkg/storageincentives/proof.go index 6415dd51af6..ccc04cd935b 100644 --- a/pkg/storageincentives/proof.go +++ b/pkg/storageincentives/proof.go @@ -5,12 +5,196 @@ package storageincentives import ( + "errors" + "fmt" + "hash" + "math/big" + + "github.com/ethersphere/bee/pkg/bmt" "github.com/ethersphere/bee/pkg/bmtpool" "github.com/ethersphere/bee/pkg/cac" + "github.com/ethersphere/bee/pkg/soc" + "github.com/ethersphere/bee/pkg/storageincentives/redistribution" storer "github.com/ethersphere/bee/pkg/storer" "github.com/ethersphere/bee/pkg/swarm" ) +var errProofCreation = errors.New("reserve commitment hasher: failure in proof creation") + +// spanOffset returns the byte index of chunkdata where the spansize starts +func spanOffset(sampleItem storer.SampleItem) uint8 { + ch := swarm.NewChunk(sampleItem.ChunkAddress, sampleItem.ChunkData) + if soc.Valid(ch) { + return swarm.HashSize + swarm.SocSignatureSize + } + + return 0 +} + +// makeInclusionProofs creates transaction data for claim method. +// In the document this logic, result data, is also called Proof of entitlement (POE). +func makeInclusionProofs( + reserveSampleItems []storer.SampleItem, + anchor1 []byte, + anchor2 []byte, +) (redistribution.ChunkInclusionProofs, error) { + if len(reserveSampleItems) != storer.SampleSize { + return redistribution.ChunkInclusionProofs{}, fmt.Errorf("reserve sample items should have %d elements", storer.SampleSize) + } + if len(anchor1) == 0 { + return redistribution.ChunkInclusionProofs{}, errors.New("anchor1 is not set") + } + if len(anchor2) == 0 { + return redistribution.ChunkInclusionProofs{}, errors.New("anchor2 is not set") + } + + require3 := storer.SampleSize - 1 + require1 := new(big.Int).Mod(new(big.Int).SetBytes(anchor2), big.NewInt(int64(require3))).Uint64() + require2 := new(big.Int).Mod(new(big.Int).SetBytes(anchor2), big.NewInt(int64(require3-1))).Uint64() + if require2 >= require1 { + require2++ + } + + prefixHasherFactory := func() hash.Hash { + return swarm.NewPrefixHasher(anchor1) + } + prefixHasherPool := bmt.NewPool(bmt.NewConf(prefixHasherFactory, swarm.BmtBranches, 8)) + + // Sample chunk proofs + rccontent := bmt.Prover{Hasher: bmtpool.Get()} + rccontent.SetHeaderInt64(swarm.HashSize * storer.SampleSize * 2) + rsc, err := sampleChunk(reserveSampleItems) + if err != nil { + return redistribution.ChunkInclusionProofs{}, errProofCreation + } + rscData := rsc.Data() + _, err = rccontent.Write(rscData[swarm.SpanSize:]) + if err != nil { + return redistribution.ChunkInclusionProofs{}, errProofCreation + } + _, err = rccontent.Hash(nil) + if err != nil { + return redistribution.ChunkInclusionProofs{}, errProofCreation + } + proof1p1 := rccontent.Proof(int(require1) * 2) + proof2p1 := rccontent.Proof(int(require2) * 2) + proofLastp1 := rccontent.Proof(require3 * 2) + bmtpool.Put(rccontent.Hasher) + + // Witness1 proofs + segmentIndex := int(new(big.Int).Mod(new(big.Int).SetBytes(anchor2), big.NewInt(int64(128))).Uint64()) + // OG chunk proof + chunk1Content := bmt.Prover{Hasher: bmtpool.Get()} + chunk1Offset := spanOffset(reserveSampleItems[require1]) + chunk1Content.SetHeader(reserveSampleItems[require1].ChunkData[chunk1Offset : chunk1Offset+swarm.SpanSize]) + chunk1ContentPayload := reserveSampleItems[require1].ChunkData[chunk1Offset+swarm.SpanSize:] + _, err = chunk1Content.Write(chunk1ContentPayload) + if err != nil { + return redistribution.ChunkInclusionProofs{}, errProofCreation + } + _, err = chunk1Content.Hash(nil) + if err != nil { + return redistribution.ChunkInclusionProofs{}, errProofCreation + } + proof1p2 := chunk1Content.Proof(segmentIndex) + // TR chunk proof + chunk1TrContent := bmt.Prover{Hasher: prefixHasherPool.Get()} + chunk1TrContent.SetHeader(reserveSampleItems[require1].ChunkData[chunk1Offset : chunk1Offset+swarm.SpanSize]) + _, err = chunk1TrContent.Write(chunk1ContentPayload) + if err != nil { + return redistribution.ChunkInclusionProofs{}, errProofCreation + } + _, err = chunk1TrContent.Hash(nil) + if err != nil { + return redistribution.ChunkInclusionProofs{}, errProofCreation + } + proof1p3 := chunk1TrContent.Proof(segmentIndex) + // cleanup + bmtpool.Put(chunk1Content.Hasher) + prefixHasherPool.Put(chunk1TrContent.Hasher) + + // Witness2 proofs + // OG Chunk proof + chunk2Offset := spanOffset(reserveSampleItems[require2]) + chunk2Content := bmt.Prover{Hasher: bmtpool.Get()} + chunk2ContentPayload := reserveSampleItems[require2].ChunkData[chunk2Offset+swarm.SpanSize:] + chunk2Content.SetHeader(reserveSampleItems[require2].ChunkData[chunk2Offset : chunk2Offset+swarm.SpanSize]) + _, err = chunk2Content.Write(chunk2ContentPayload) + if err != nil { + return redistribution.ChunkInclusionProofs{}, errProofCreation + } + _, err = chunk2Content.Hash(nil) + if err != nil { + return redistribution.ChunkInclusionProofs{}, errProofCreation + } + proof2p2 := chunk2Content.Proof(segmentIndex) + // TR Chunk proof + chunk2TrContent := bmt.Prover{Hasher: prefixHasherPool.Get()} + chunk2TrContent.SetHeader(reserveSampleItems[require2].ChunkData[chunk2Offset : chunk2Offset+swarm.SpanSize]) + _, err = chunk2TrContent.Write(chunk2ContentPayload) + if err != nil { + return redistribution.ChunkInclusionProofs{}, errProofCreation + } + _, err = chunk2TrContent.Hash(nil) + if err != nil { + return redistribution.ChunkInclusionProofs{}, errProofCreation + } + proof2p3 := chunk2TrContent.Proof(segmentIndex) + // cleanup + bmtpool.Put(chunk2Content.Hasher) + prefixHasherPool.Put(chunk2TrContent.Hasher) + + // Witness3 proofs + // OG Chunk proof + chunkLastOffset := spanOffset(reserveSampleItems[require3]) + chunkLastContent := bmt.Prover{Hasher: bmtpool.Get()} + chunkLastContent.SetHeader(reserveSampleItems[require3].ChunkData[chunkLastOffset : chunkLastOffset+swarm.SpanSize]) + chunkLastContentPayload := reserveSampleItems[require3].ChunkData[chunkLastOffset+swarm.SpanSize:] + _, err = chunkLastContent.Write(chunkLastContentPayload) + if err != nil { + return redistribution.ChunkInclusionProofs{}, errProofCreation + } + _, err = chunkLastContent.Hash(nil) + if err != nil { + return redistribution.ChunkInclusionProofs{}, errProofCreation + } + proofLastp2 := chunkLastContent.Proof(segmentIndex) + // TR Chunk Proof + chunkLastTrContent := bmt.Prover{Hasher: prefixHasherPool.Get()} + chunkLastTrContent.SetHeader(reserveSampleItems[require3].ChunkData[chunkLastOffset : chunkLastOffset+swarm.SpanSize]) + _, err = chunkLastTrContent.Write(chunkLastContentPayload) + if err != nil { + return redistribution.ChunkInclusionProofs{}, errProofCreation + } + _, err = chunkLastTrContent.Hash(nil) + if err != nil { + return redistribution.ChunkInclusionProofs{}, errProofCreation + } + proofLastp3 := chunkLastTrContent.Proof(segmentIndex) + // cleanup + bmtpool.Put(chunkLastContent.Hasher) + prefixHasherPool.Put(chunkLastTrContent.Hasher) + + // map to output and add SOC related data if it is necessary + A, err := redistribution.NewChunkInclusionProof(proof1p1, proof1p2, proof1p3, reserveSampleItems[require1]) + if err != nil { + return redistribution.ChunkInclusionProofs{}, err + } + B, err := redistribution.NewChunkInclusionProof(proof2p1, proof2p2, proof2p3, reserveSampleItems[require2]) + if err != nil { + return redistribution.ChunkInclusionProofs{}, err + } + C, err := redistribution.NewChunkInclusionProof(proofLastp1, proofLastp2, proofLastp3, reserveSampleItems[require3]) + if err != nil { + return redistribution.ChunkInclusionProofs{}, err + } + return redistribution.ChunkInclusionProofs{ + A: A, + B: B, + C: C, + }, nil +} + func sampleChunk(items []storer.SampleItem) (swarm.Chunk, error) { contentSize := len(items) * 2 * swarm.HashSize @@ -27,23 +211,9 @@ func sampleChunk(items []storer.SampleItem) (swarm.Chunk, error) { } func sampleHash(items []storer.SampleItem) (swarm.Address, error) { - hasher := bmtpool.Get() - defer bmtpool.Put(hasher) - - for _, s := range items { - _, err := hasher.Write(s.TransformedAddress.Bytes()) - if err != nil { - return swarm.ZeroAddress, err - } + ch, err := sampleChunk(items) + if err != nil { + return swarm.ZeroAddress, err } - hash := hasher.Sum(nil) - - return swarm.NewAddress(hash), nil - - // PH4_Logic: - // ch, err := sampleChunk(items) - // if err != nil { - // return swarm.ZeroAddress, err - // } - // return ch.Address(), nil + return ch.Address(), nil } diff --git a/pkg/storageincentives/proof_test.go b/pkg/storageincentives/proof_test.go index 36350c2af2c..9ee3745e1af 100644 --- a/pkg/storageincentives/proof_test.go +++ b/pkg/storageincentives/proof_test.go @@ -6,20 +6,157 @@ package storageincentives_test import ( "bytes" + _ "embed" + "encoding/json" + "fmt" + "math/big" "testing" + "github.com/ethersphere/bee/pkg/cac" + "github.com/ethersphere/bee/pkg/crypto" + "github.com/ethersphere/bee/pkg/postage" + postagetesting "github.com/ethersphere/bee/pkg/postage/testing" + "github.com/ethersphere/bee/pkg/soc" "github.com/ethersphere/bee/pkg/storageincentives" + "github.com/ethersphere/bee/pkg/storageincentives/redistribution" storer "github.com/ethersphere/bee/pkg/storer" "github.com/ethersphere/bee/pkg/swarm" + "github.com/ethersphere/bee/pkg/util/testutil" + "github.com/google/go-cmp/cmp" ) +// Test asserts valid case for MakeInclusionProofs. +func TestMakeInclusionProofs(t *testing.T) { + t.Parallel() + + anchor := testutil.RandBytes(t, 1) + sample := storer.RandSample(t, anchor) + + _, err := storageincentives.MakeInclusionProofs(sample.Items, anchor, anchor) + if err != nil { + t.Fatal(err) + } +} + +//go:embed testdata/inclusion-proofs.json +var testData []byte + +// Test asserts that MakeInclusionProofs will generate the same +// output for given sample. +func TestMakeInclusionProofsRegression(t *testing.T) { + t.Parallel() + + const sampleSize = 16 + + keyRaw := `00000000000000000000000000000000` + privKey, err := crypto.DecodeSecp256k1PrivateKey([]byte(keyRaw)) + if err != nil { + t.Fatal(err) + } + signer := crypto.NewDefaultSigner(privKey) + + stampID, _ := crypto.LegacyKeccak256([]byte("The Inverted Jenny")) + index := []byte{0, 0, 0, 0, 0, 8, 3, 3} + timestamp := []byte{0, 0, 0, 0, 0, 3, 3, 8} + stamper := func(addr swarm.Address) *postage.Stamp { + sig := postagetesting.MustNewValidSignature(signer, addr, stampID, index, timestamp) + return postage.NewStamp(stampID, index, timestamp, sig) + } + + anchor1 := big.NewInt(100).Bytes() + anchor2 := big.NewInt(30).Bytes() // this anchor will pick chunks 3, 6, 15 + + // generate chunks that will be used as sample + sampleChunks := make([]swarm.Chunk, 0, sampleSize) + for i := 0; i < sampleSize; i++ { + ch, err := cac.New([]byte(fmt.Sprintf("Unstoppable data! Chunk #%d", i+1))) + if err != nil { + t.Fatal(err) + } + + if i%2 == 0 { + id, err := crypto.LegacyKeccak256([]byte(fmt.Sprintf("ID #%d", i+1))) + if err != nil { + t.Fatal(err) + } + + socCh, err := soc.New(id, ch).Sign(signer) + if err != nil { + t.Fatal(err) + } + + ch = socCh + } + + ch = ch.WithStamp(stamper(ch.Address())) + + sampleChunks = append(sampleChunks, ch) + } + + // make sample from chunks + sample, err := storer.MakeSampleUsingChunks(sampleChunks, anchor1) + if err != nil { + t.Fatal(err) + } + + // assert that sample chunk hash/address does not change + sch, err := storageincentives.SampleChunk(sample.Items) + if err != nil { + t.Fatal(err) + } + if want := swarm.MustParseHexAddress("193bbea3dd0656d813c2c1e27b821f141286bbe6ab0dbf8e26fc7dd491e8f921"); !sch.Address().Equal(want) { + t.Fatalf("expecting sample chunk address %v, got %v", want, sch.Address()) + } + + // assert that inclusion proofs values does not change + proofs, err := storageincentives.MakeInclusionProofs(sample.Items, anchor1, anchor2) + if err != nil { + t.Fatal(err) + } + + var expectedProofs redistribution.ChunkInclusionProofs + + err = json.Unmarshal(testData, &expectedProofs) + if err != nil { + t.Fatal(err) + } + + if diff := cmp.Diff(proofs, expectedProofs); diff != "" { + t.Fatalf("unexpected inclusion proofs (-want +have):\n%s", diff) + } +} + +// Test asserts cases when MakeInclusionProofs should return error. +func TestMakeInclusionProofsExpectedError(t *testing.T) { + t.Parallel() + + t.Run("invalid sample length", func(t *testing.T) { + anchor := testutil.RandBytes(t, 8) + sample := storer.RandSample(t, anchor) + + _, err := storageincentives.MakeInclusionProofs(sample.Items[:1], anchor, anchor) + if err == nil { + t.Fatal("expecting error") + } + }) + + t.Run("empty anchor", func(t *testing.T) { + sample := storer.RandSample(t, []byte{}) + + _, err := storageincentives.MakeInclusionProofs(sample.Items[:1], []byte{}, []byte{}) + if err == nil { + t.Fatal("expecting error") + } + }) +} + // Tests asserts that creating sample chunk is valid for all lengths [1-MaxSampleSize] func TestSampleChunk(t *testing.T) { t.Parallel() sample := storer.RandSample(t, nil) - for i := 1; i < len(sample.Items); i++ { + for i := 0; i < len(sample.Items); i++ { items := sample.Items[:i] chunk, err := storageincentives.SampleChunk(items) @@ -40,6 +177,10 @@ func TestSampleChunk(t *testing.T) { } pos += swarm.HashSize } + + if !chunk.Address().IsValidNonEmpty() { + t.Error("address shouldn't be empty") + } } } diff --git a/pkg/storageincentives/redistribution/inclusionproof.go b/pkg/storageincentives/redistribution/inclusionproof.go new file mode 100644 index 00000000000..b786d1d3001 --- /dev/null +++ b/pkg/storageincentives/redistribution/inclusionproof.go @@ -0,0 +1,105 @@ +// Copyright 2023 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package redistribution + +import ( + "encoding/binary" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethersphere/bee/pkg/bmt" + "github.com/ethersphere/bee/pkg/soc" + "github.com/ethersphere/bee/pkg/storer" + "github.com/ethersphere/bee/pkg/swarm" +) + +type ChunkInclusionProofs struct { + A ChunkInclusionProof `json:"proof1"` + B ChunkInclusionProof `json:"proof2"` + C ChunkInclusionProof `json:"proofLast"` +} + +// ChunkInclusionProof structure must exactly match +// corresponding structure (of the same name) in Redistribution.sol smart contract. +// github.com/ethersphere/storage-incentives/blob/ph_f2/src/Redistribution.sol +// github.com/ethersphere/storage-incentives/blob/master/src/Redistribution.sol (when merged to master) +type ChunkInclusionProof struct { + ProofSegments []common.Hash `json:"proofSegments"` + ProveSegment common.Hash `json:"proveSegment"` + ProofSegments2 []common.Hash `json:"proofSegments2"` + ProveSegment2 common.Hash `json:"proveSegment2"` + ChunkSpan uint64 `json:"chunkSpan"` + ProofSegments3 []common.Hash `json:"proofSegments3"` + PostageProof PostageProof `json:"postageProof"` + SocProof []SOCProof `json:"socProof"` +} + +// SOCProof structure must exactly match +// corresponding structure (of the same name) in Redistribution.sol smart contract. +type PostageProof struct { + Signature []byte `json:"signature"` + PostageId common.Hash `json:"postageId"` + Index uint64 `json:"index"` + TimeStamp uint64 `json:"timeStamp"` +} + +// SOCProof structure must exactly match +// corresponding structure (of the same name) in Redistribution.sol smart contract. +type SOCProof struct { + Signer common.Address `json:"signer"` + Signature []byte `json:"signature"` + Identifier common.Hash `json:"identifier"` + ChunkAddr common.Hash `json:"chunkAddr"` +} + +// NewChunkInclusionProof transforms arguments to ChunkInclusionProof object +func NewChunkInclusionProof(proofp1, proofp2, proofp3 bmt.Proof, sampleItem storer.SampleItem) (ChunkInclusionProof, error) { + socProof, err := makeSOCProof(sampleItem) + if err != nil { + return ChunkInclusionProof{}, err + } + + return ChunkInclusionProof{ + ProofSegments: toCommonHash(proofp1.ProofSegments), + ProveSegment: common.BytesToHash(proofp1.ProveSegment), + ProofSegments2: toCommonHash(proofp2.ProofSegments), + ProveSegment2: common.BytesToHash(proofp2.ProveSegment), + ChunkSpan: binary.LittleEndian.Uint64(proofp2.Span[:swarm.SpanSize]), // should be uint64 on the other size; copied from pkg/api/bytes.go + ProofSegments3: toCommonHash(proofp3.ProofSegments), + PostageProof: PostageProof{ + Signature: sampleItem.Stamp.Sig(), + PostageId: common.BytesToHash(sampleItem.Stamp.BatchID()), + Index: binary.BigEndian.Uint64(sampleItem.Stamp.Index()), + TimeStamp: binary.BigEndian.Uint64(sampleItem.Stamp.Timestamp()), + }, + SocProof: socProof, + }, nil +} + +func toCommonHash(hashes [][]byte) []common.Hash { + output := make([]common.Hash, len(hashes)) + for i, s := range hashes { + output[i] = common.BytesToHash(s) + } + return output +} + +func makeSOCProof(sampleItem storer.SampleItem) ([]SOCProof, error) { + ch := swarm.NewChunk(sampleItem.ChunkAddress, sampleItem.ChunkData) + if !soc.Valid(ch) { + return []SOCProof{}, nil + } + + socCh, err := soc.FromChunk(ch) + if err != nil { + return []SOCProof{}, err + } + + return []SOCProof{{ + Signer: common.BytesToAddress(socCh.OwnerAddress()), + Signature: socCh.Signature(), + Identifier: common.BytesToHash(socCh.ID()), + ChunkAddr: common.BytesToHash(socCh.WrappedChunk().Address().Bytes()), + }}, nil +} diff --git a/pkg/storageincentives/redistribution/redistribution.go b/pkg/storageincentives/redistribution/redistribution.go index d7187726b42..2c163f5a76c 100644 --- a/pkg/storageincentives/redistribution/redistribution.go +++ b/pkg/storageincentives/redistribution/redistribution.go @@ -23,8 +23,8 @@ type Contract interface { ReserveSalt(context.Context) ([]byte, error) IsPlaying(context.Context, uint8) (bool, error) IsWinner(context.Context) (bool, error) - Claim(context.Context) (common.Hash, error) - Commit(context.Context, []byte, *big.Int) (common.Hash, error) + Claim(context.Context, ChunkInclusionProofs) (common.Hash, error) + Commit(context.Context, []byte, uint32) (common.Hash, error) Reveal(context.Context, uint8, []byte, []byte) (common.Hash, error) } @@ -92,8 +92,8 @@ func (c *contract) IsWinner(ctx context.Context) (isWinner bool, err error) { } // Claim sends a transaction to blockchain if a win is claimed. -func (c *contract) Claim(ctx context.Context) (common.Hash, error) { - callData, err := c.incentivesContractABI.Pack("claim") +func (c *contract) Claim(ctx context.Context, proofs ChunkInclusionProofs) (common.Hash, error) { + callData, err := c.incentivesContractABI.Pack("claim", proofs.A, proofs.B, proofs.C) if err != nil { return common.Hash{}, err } @@ -115,7 +115,7 @@ func (c *contract) Claim(ctx context.Context) (common.Hash, error) { } // Commit submits the obfusHash hash by sending a transaction to the blockchain. -func (c *contract) Commit(ctx context.Context, obfusHash []byte, round *big.Int) (common.Hash, error) { +func (c *contract) Commit(ctx context.Context, obfusHash []byte, round uint32) (common.Hash, error) { callData, err := c.incentivesContractABI.Pack("commit", common.BytesToHash(obfusHash), common.BytesToHash(c.overlay.Bytes()), round) if err != nil { return common.Hash{}, err diff --git a/pkg/storageincentives/redistribution/redistribution_test.go b/pkg/storageincentives/redistribution/redistribution_test.go index d47790d3d4b..943d3013ba4 100644 --- a/pkg/storageincentives/redistribution/redistribution_test.go +++ b/pkg/storageincentives/redistribution/redistribution_test.go @@ -7,6 +7,7 @@ package redistribution_test import ( "bytes" "context" + "encoding/binary" "errors" "fmt" "math/big" @@ -27,6 +28,36 @@ import ( var redistributionContractABI = abiutil.MustParseABI(chaincfg.Testnet.RedistributionABI) +func randChunkInclusionProof(t *testing.T) redistribution.ChunkInclusionProof { + t.Helper() + + return redistribution.ChunkInclusionProof{ + ProofSegments: []common.Hash{common.BytesToHash(testutil.RandBytes(t, 32))}, + ProveSegment: common.BytesToHash(testutil.RandBytes(t, 32)), + ProofSegments2: []common.Hash{common.BytesToHash(testutil.RandBytes(t, 32))}, + ProveSegment2: common.BytesToHash(testutil.RandBytes(t, 32)), + ProofSegments3: []common.Hash{common.BytesToHash(testutil.RandBytes(t, 32))}, + PostageProof: redistribution.PostageProof{ + Signature: testutil.RandBytes(t, 65), + PostageId: common.BytesToHash(testutil.RandBytes(t, 32)), + Index: binary.BigEndian.Uint64(testutil.RandBytes(t, 8)), + TimeStamp: binary.BigEndian.Uint64(testutil.RandBytes(t, 8)), + }, + ChunkSpan: 1, + SocProof: []redistribution.SOCProof{}, + } +} + +func randChunkInclusionProofs(t *testing.T) redistribution.ChunkInclusionProofs { + t.Helper() + + return redistribution.ChunkInclusionProofs{ + A: randChunkInclusionProof(t), + B: randChunkInclusionProof(t), + C: randChunkInclusionProof(t), + } +} + func TestRedistribution(t *testing.T) { t.Parallel() @@ -153,7 +184,9 @@ func TestRedistribution(t *testing.T) { t.Run("Claim", func(t *testing.T) { t.Parallel() - expectedCallData, err := redistributionContractABI.Pack("claim") + proofs := randChunkInclusionProofs(t) + + expectedCallData, err := redistributionContractABI.Pack("claim", proofs.A, proofs.B, proofs.C) if err != nil { t.Fatal(err) } @@ -183,7 +216,7 @@ func TestRedistribution(t *testing.T) { redistributionContractABI, ) - _, err = contract.Claim(ctx) + _, err = contract.Claim(ctx, proofs) if err != nil { t.Fatal(err) } @@ -192,7 +225,8 @@ func TestRedistribution(t *testing.T) { t.Run("Claim with tx reverted", func(t *testing.T) { t.Parallel() - expectedCallData, err := redistributionContractABI.Pack("claim") + proofs := randChunkInclusionProofs(t) + expectedCallData, err := redistributionContractABI.Pack("claim", proofs.A, proofs.B, proofs.C) if err != nil { t.Fatal(err) } @@ -222,7 +256,7 @@ func TestRedistribution(t *testing.T) { redistributionContractABI, ) - _, err = contract.Claim(ctx) + _, err = contract.Claim(ctx, proofs) if !errors.Is(err, transaction.ErrTransactionReverted) { t.Fatal(err) } @@ -233,7 +267,7 @@ func TestRedistribution(t *testing.T) { var obfus [32]byte testobfus := common.Hex2Bytes("hash") copy(obfus[:], testobfus) - expectedCallData, err := redistributionContractABI.Pack("commit", obfus, common.BytesToHash(owner.Bytes()), big.NewInt(0)) + expectedCallData, err := redistributionContractABI.Pack("commit", obfus, common.BytesToHash(owner.Bytes()), uint32(0)) if err != nil { t.Fatal(err) } @@ -263,7 +297,7 @@ func TestRedistribution(t *testing.T) { redistributionContractABI, ) - _, err = contract.Commit(ctx, testobfus, big.NewInt(0)) + _, err = contract.Commit(ctx, testobfus, uint32(0)) if err != nil { t.Fatal(err) } @@ -368,7 +402,7 @@ func TestRedistribution(t *testing.T) { t.Run("invalid call data", func(t *testing.T) { t.Parallel() - expectedCallData, err := redistributionContractABI.Pack("commit", common.BytesToHash(common.Hex2Bytes("some hash")), common.BytesToHash(common.Hex2Bytes("some address")), big.NewInt(0)) + expectedCallData, err := redistributionContractABI.Pack("commit", common.BytesToHash(common.Hex2Bytes("some hash")), common.BytesToHash(common.Hex2Bytes("some address")), uint32(0)) if err != nil { t.Fatal(err) } @@ -390,7 +424,7 @@ func TestRedistribution(t *testing.T) { redistributionContractABI, ) - _, err = contract.Commit(ctx, common.Hex2Bytes("hash"), big.NewInt(0)) + _, err = contract.Commit(ctx, common.Hex2Bytes("hash"), uint32(0)) if err == nil { t.Fatal("expected error") } diff --git a/pkg/storageincentives/redistributionstate.go b/pkg/storageincentives/redistributionstate.go index b61c7a5cbd2..b1f4b60dd03 100644 --- a/pkg/storageincentives/redistributionstate.go +++ b/pkg/storageincentives/redistributionstate.go @@ -15,6 +15,7 @@ import ( "github.com/ethersphere/bee/pkg/log" "github.com/ethersphere/bee/pkg/settlement/swap/erc20" "github.com/ethersphere/bee/pkg/storage" + storer "github.com/ethersphere/bee/pkg/storer" "github.com/ethersphere/bee/pkg/swarm" "github.com/ethersphere/bee/pkg/transaction" ) @@ -63,8 +64,10 @@ type RoundData struct { } type SampleData struct { - ReserveSampleHash swarm.Address - StorageRadius uint8 + Anchor1 []byte + ReserveSampleItems []storer.SampleItem + ReserveSampleHash swarm.Address + StorageRadius uint8 } func NewStatus() *Status { diff --git a/pkg/storageincentives/soc_mine_test.go b/pkg/storageincentives/soc_mine_test.go new file mode 100644 index 00000000000..d5b93241b21 --- /dev/null +++ b/pkg/storageincentives/soc_mine_test.go @@ -0,0 +1,201 @@ +// Copyright 2023 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package storageincentives_test + +import ( + "context" + "encoding/binary" + "encoding/hex" + "fmt" + "hash" + "math/big" + "os" + "sync" + "testing" + + "github.com/ethersphere/bee/pkg/bmt" + "github.com/ethersphere/bee/pkg/cac" + "github.com/ethersphere/bee/pkg/crypto" + "github.com/ethersphere/bee/pkg/soc" + "github.com/ethersphere/bee/pkg/swarm" + "golang.org/x/sync/errgroup" +) + +// TestSocMine dumps a sample made out SOCs to upload for storage incestives + +// dump chunks + +// go test -v ./pkg/storageincentives/ -run TestSocMine -count 1 > socs.txt + +// to generate uploads using the input +// cat socs.txt | tail 19 | head 16 | perl -pne 's/([a-f0-9]+)\t([a-f0-9]+)\t([a-f0-9]+)\t([a-f0-9]+)/echo -n $4 | xxd -r -p | curl -X POST \"http:\/\/localhost:1633\/soc\/$1\/$2?sig=$3\" -H \"accept: application\/json, text\/plain, \/\" -H \"content-type: application\/octet-stream\" -H \"swarm-postage-batch-id: 14b26beca257e763609143c6b04c2c487f01a051798c535c2f542ce75a97c05f\" --data-binary \@-/' +func TestSocMine(t *testing.T) { + t.Parallel() + // the anchor used in neighbourhood selection and reserve salt for sampling + prefix, err := hex.DecodeString("3617319a054d772f909f7c479a2cebe5066e836a939412e32403c99029b92eff") + if err != nil { + t.Fatal(err) + } + // the transformed address hasher factory function + prefixhasher := func() hash.Hash { return swarm.NewPrefixHasher(prefix) } + trHasher := func() hash.Hash { return bmt.NewHasher(prefixhasher) } + // the bignum cast of the maximum sample value (upper bound on transformed addresses as a 256-bit article) + // this constant is for a minimum reserve size of 2 million chunks with sample size of 16 + // = 1.284401 * 10^71 = 1284401 + 66 0-s + mstring := "1284401" + for i := 0; i < 66; i++ { + mstring = mstring + "0" + } + n, ok := new(big.Int).SetString(mstring, 10) + if !ok { + t.Fatalf("SetString: error setting to '%s'", mstring) + } + // the filter function on the SOC address + // meant to make sure we pass check for proof of retrievability for + // a node of overlay 0x65xxx with a reserve depth of 1, i.e., + // SOC address must start with zero bit + filterSOCAddr := func(a swarm.Address) bool { + return a.Bytes()[0]&0x80 != 0x00 + } + // the filter function on the transformed address using the density estimation constant + filterTrAddr := func(a swarm.Address) (bool, error) { + m := new(big.Int).SetBytes(a.Bytes()) + return m.Cmp(n) < 0, nil + } + // setup the signer with a private key from a fixture + data, err := hex.DecodeString("634fb5a872396d9693e5c9f9d7233cfa93f395c093371017ff44aa9ae6564cdd") + if err != nil { + t.Fatal(err) + } + privKey, err := crypto.DecodeSecp256k1PrivateKey(data) + if err != nil { + t.Fatal(err) + } + signer := crypto.NewDefaultSigner(privKey) + + sampleSize := 16 + // for sanity check: given a filterSOCAddr requiring a 0 leading bit (chance of 1/2) + // we expect an overall rough 4 million chunks to be mined to create this sample + // for 8 workers that is half a million round on average per worker + err = makeChunks(t, signer, sampleSize, filterSOCAddr, filterTrAddr, trHasher) + if err != nil { + t.Fatal(err) + } +} + +func makeChunks(t *testing.T, signer crypto.Signer, sampleSize int, filterSOCAddr func(swarm.Address) bool, filterTrAddr func(swarm.Address) (bool, error), trHasher func() hash.Hash) error { + t.Helper() + + // set owner address from signer + ethAddress, err := signer.EthereumAddress() + if err != nil { + return err + } + ownerAddressBytes := ethAddress.Bytes() + + // use the same wrapped chunk for all mined SOCs + payload := []byte("foo") + ch, err := cac.New(payload) + if err != nil { + return err + } + + var done bool // to signal sampleSize number of chunks found + sampleC := make(chan *soc.SOC, 1) // channel to push resuls on + sample := make([]*soc.SOC, sampleSize) // to collect the sample + ctx, cancel := context.WithCancel(context.Background()) + eg, ectx := errgroup.WithContext(ctx) + // the main loop terminating after sampleSize SOCs have been generated + eg.Go(func() error { + defer cancel() + for i := 0; i < sampleSize; i++ { + select { + case sample[i] = <-sampleC: + case <-ectx.Done(): + return ectx.Err() + } + } + done = true + return nil + }) + + // loop to start mining workers + count := 8 // number of parallel workers + wg := sync.WaitGroup{} + for i := 0; i < count; i++ { + i := i + wg.Add(1) + eg.Go(func() (err error) { + offset := i * 4 + found := 0 + for seed := uint32(1); ; seed++ { + select { + case <-ectx.Done(): + defer wg.Done() + t.Logf("LOG quit worker: %d, rounds: %d, found: %d\n", i, seed, found) + return ectx.Err() + default: + } + id := make([]byte, 32) + binary.BigEndian.PutUint32(id[offset:], seed) + s := soc.New(id, ch) + addr, err := soc.CreateAddress(id, ownerAddressBytes) + if err != nil { + return err + } + // continue if mined SOC addr is not good + if !filterSOCAddr(addr) { + continue + } + hasher := trHasher() + data := s.WrappedChunk().Data() + hasher.(*bmt.Hasher).SetHeader(data[:8]) + _, err = hasher.Write(data[8:]) + if err != nil { + return err + } + trAddr := hasher.Sum(nil) + // hashing the transformed wrapped chunk address with the SOC address + // to arrive at a unique transformed SOC address despite identical payloads + trSocAddr, err := soc.CreateAddress(addr.Bytes(), trAddr) + if err != nil { + return err + } + ok, err := filterTrAddr(trSocAddr) + if err != nil { + return err + } + if ok { + select { + case sampleC <- s: + found++ + t.Logf("LOG worker: %d, rounds: %d, found: %d, id:%x\n", i, seed, found, id) + case <-ectx.Done(): + defer wg.Done() + t.Logf("LOG quit worker: %d, rounds: %d, found: %d\n", i, seed, found) + return ectx.Err() + } + } + } + }) + } + if err := eg.Wait(); !done && err != nil { + return err + } + wg.Wait() + for _, s := range sample { + + // signs the chunk + sch, err := s.Sign(signer) + if err != nil { + return err + } + data := sch.Data() + id, sig, payload := data[:32], data[32:97], data[97:] + fmt.Fprintf(os.Stdout, "%x\t%x\t%x\t%x\n", ownerAddressBytes, id, sig, payload) + + } + return nil +} diff --git a/pkg/storageincentives/testdata/inclusion-proofs.json b/pkg/storageincentives/testdata/inclusion-proofs.json new file mode 100644 index 00000000000..fe79666096e --- /dev/null +++ b/pkg/storageincentives/testdata/inclusion-proofs.json @@ -0,0 +1,126 @@ +{ + "proof1": { + "proofSegments": [ + "0x0875605dea48e812c9685ffba220a2b848bdbafdb95e02d087ba4a32925ea34f", + "0xf873df729270d5f4064286f3f018385a07cb4228734d8aca794299fee6e3e3e5", + "0x1fa8767fe303fe7487f5d58e4d72e5e170cf135f58a91b4fe19e4b19e5b67b5a", + "0x0f64ed713e25291e2c5a0561f584fa78c55a399e31919903d215dd622bcfd0ec", + "0x34dac0c73538614801c1ad16e272ef57f0b96a972073d15418f38daf9eb401c0", + "0x0eb01ebfc9ed27500cd4dfc979272d1f0913cc9f66540d7e8005811109e1cf2d", + "0x887c22bd8750d34016ac3c66b5ff102dacdd73f6b014e710b51e8022af9a1968" + ], + "proveSegment": "0x7133885ac59dca7b97773acb740e978d41a4af45bd563067c8a3d863578488f1", + "proofSegments2": [ + "0x0000000000000000000000000000000000000000000000000000000000000000", + "0xad3228b676f7d3cd4284a5443f17f1962b36e491b30a40b2405849e597ba5fb5", + "0xb4c11951957c6f8f642c4af61cd6b24640fec6dc7fc607ee8206a99e92410d30", + "0x21ddb9a356815c3fac1026b6dec5df3124afbadb485c9ba5a3e3398a04b7ba85", + "0x2047b070a295f8d517121d9ac9b3d5f9a944bac6cfab72dd5a7c625ab4558b0a", + "0x0eb01ebfc9ed27500cd4dfc979272d1f0913cc9f66540d7e8005811109e1cf2d", + "0x887c22bd8750d34016ac3c66b5ff102dacdd73f6b014e710b51e8022af9a1968" + ], + "proveSegment2": "0x0000000000000000000000000000000000000000000000000000000000000000", + "chunkSpan": 26, + "proofSegments3": [ + "0x0000000000000000000000000000000000000000000000000000000000000000", + "0xa7f526447b68535121d36909a7585c9610d4fe6d4115540464c70499b0d7136d", + "0x066dd7ce6f4f1c97e78ff1c271916db25cb06128c92f8c8520807a0fa2ba93ff", + "0xdf43c86b00db2156e769e8a8df1f08dc89ab5661c6fbaa9563f96fb9c051fc63", + "0x7327aecc9178bab420bb6fe482e07b65af69775b55666ec1ac8ab3da5bcec6dc", + "0xb68323ecaad1185a5e078f41c94c59d0b6dda5d57e109866e64d44acb8702846", + "0x478adfa93a7bb904d0aa86ff0d559f43aa915ee7865592e717b72a24452181cb" + ], + "postageProof": { + "signature": "p8jRioJ504AxaevPTlp9vdTf/vpZHqrY0c6qY2p5Otl15/exCHvOpBdlJbAALt3grL/aINvS37vnd8qziWj9xhs=", + "postageId": "0x4c8efc14c8e3cee608174f995d7afe155897bf643a31226e4f1363bc97686aef", + "index": 525059, + "timeStamp": 197384 + }, + "socProof": [ + { + "signer": "0x827b44d53df2854057713b25cdd653eb70fe36c4", + "signature": "TpV2lJM45MI/RwO/gTZyVquFmzKTT+9Nsu5Gp2v2vjVOlqxii4eEst4Lvq5ZdUaXgxktbRcFSF/Krdje3ebiqhs=", + "identifier": "0x6223cfdd75a40440ccd32d0b11b24f08562ec63b1ea3b8cb1a59dfc3e3c33595", + "chunkAddr": "0xf32442586d93d8c002372ed41fa2ea1f281f38311c161d535c3665de5d9bfd92" + } + ] + }, + "proof2": { + "proofSegments": [ + "0x463aeb4ca5f000064c082e56eba387004265d2f47bf1226ef2d86cb163bcca3a", + "0x829af58b2a2f1c6c156baa196f03be4df510a96419f2dd54c456d3da30166312", + "0xdee4815ec42efa507b79cf4eb1f272e07be1b526cbd48137a287d9e5b2b2808a", + "0x0f64ed713e25291e2c5a0561f584fa78c55a399e31919903d215dd622bcfd0ec", + "0x34dac0c73538614801c1ad16e272ef57f0b96a972073d15418f38daf9eb401c0", + "0x0eb01ebfc9ed27500cd4dfc979272d1f0913cc9f66540d7e8005811109e1cf2d", + "0x887c22bd8750d34016ac3c66b5ff102dacdd73f6b014e710b51e8022af9a1968" + ], + "proveSegment": "0x535e6df58a122a8f5e6c851c19b3e042f4cd1b5c5a8c499581c9f6d4e3509182", + "proofSegments2": [ + "0x0000000000000000000000000000000000000000000000000000000000000000", + "0xad3228b676f7d3cd4284a5443f17f1962b36e491b30a40b2405849e597ba5fb5", + "0xb4c11951957c6f8f642c4af61cd6b24640fec6dc7fc607ee8206a99e92410d30", + "0x21ddb9a356815c3fac1026b6dec5df3124afbadb485c9ba5a3e3398a04b7ba85", + "0x46f43b515833749217540ac60c79e0c6a54c73f3500850b5869b31d5c89d101f", + "0x0eb01ebfc9ed27500cd4dfc979272d1f0913cc9f66540d7e8005811109e1cf2d", + "0x887c22bd8750d34016ac3c66b5ff102dacdd73f6b014e710b51e8022af9a1968" + ], + "proveSegment2": "0x0000000000000000000000000000000000000000000000000000000000000000", + "chunkSpan": 26, + "proofSegments3": [ + "0x0000000000000000000000000000000000000000000000000000000000000000", + "0xa7f526447b68535121d36909a7585c9610d4fe6d4115540464c70499b0d7136d", + "0x066dd7ce6f4f1c97e78ff1c271916db25cb06128c92f8c8520807a0fa2ba93ff", + "0xdf43c86b00db2156e769e8a8df1f08dc89ab5661c6fbaa9563f96fb9c051fc63", + "0x4284c510d7d64c9e052c73bddadb1fca522fd26caf2ebf007faad50a9a0f09fa", + "0xb68323ecaad1185a5e078f41c94c59d0b6dda5d57e109866e64d44acb8702846", + "0x478adfa93a7bb904d0aa86ff0d559f43aa915ee7865592e717b72a24452181cb" + ], + "postageProof": { + "signature": "sCdPzaWeiq/+6AMCGXGnZKAXziwPQcjOtu796oBwVvYhqY/qtevzO7YGXknAUPQT7IhAsAj8Ik2ILOUkTOPgFxw=", + "postageId": "0x4c8efc14c8e3cee608174f995d7afe155897bf643a31226e4f1363bc97686aef", + "index": 525059, + "timeStamp": 197384 + }, + "socProof": [] + }, + "proofLast": { + "proofSegments": [ + "0xfee18543782df46a86f85456e62dc973a4c84369b6b1cd4f93e57fe247f9730e", + "0x23a0858ee2b8b4cb0ba66d3533f468d6b583a6b77df0cc78fc6df64dc735a917", + "0xb6bffa54dec44ad57349f9aef6cb65a1f8807f15447462ec519751220e5a5bc3", + "0x553aae9948fc13c33d8b353cf5694ecadc7c40c8316ce09cbd4d864dbb94f026", + "0xaf7db874a9b5addf602b3e899194480a32afec6d6cd4ec0fadf9e065db739dd5", + "0x0eb01ebfc9ed27500cd4dfc979272d1f0913cc9f66540d7e8005811109e1cf2d", + "0x887c22bd8750d34016ac3c66b5ff102dacdd73f6b014e710b51e8022af9a1968" + ], + "proveSegment": "0x5ba2c8b912fad4aeb4a11a960946d07b9f66bc40ac54d87224914d75f5aeea5f", + "proofSegments2": [ + "0x0000000000000000000000000000000000000000000000000000000000000000", + "0xad3228b676f7d3cd4284a5443f17f1962b36e491b30a40b2405849e597ba5fb5", + "0xb4c11951957c6f8f642c4af61cd6b24640fec6dc7fc607ee8206a99e92410d30", + "0x21ddb9a356815c3fac1026b6dec5df3124afbadb485c9ba5a3e3398a04b7ba85", + "0x7f575db255ef42dcaeb7658df9f33fe5a1aad5d41af51a72a381acea29d98a12", + "0x0eb01ebfc9ed27500cd4dfc979272d1f0913cc9f66540d7e8005811109e1cf2d", + "0x887c22bd8750d34016ac3c66b5ff102dacdd73f6b014e710b51e8022af9a1968" + ], + "proveSegment2": "0x0000000000000000000000000000000000000000000000000000000000000000", + "chunkSpan": 27, + "proofSegments3": [ + "0x0000000000000000000000000000000000000000000000000000000000000000", + "0xa7f526447b68535121d36909a7585c9610d4fe6d4115540464c70499b0d7136d", + "0x066dd7ce6f4f1c97e78ff1c271916db25cb06128c92f8c8520807a0fa2ba93ff", + "0xdf43c86b00db2156e769e8a8df1f08dc89ab5661c6fbaa9563f96fb9c051fc63", + "0x7683427ba0ef1fbebf97f2fc36859df88ead8123369fe38d7b767b7a7eda5294", + "0xb68323ecaad1185a5e078f41c94c59d0b6dda5d57e109866e64d44acb8702846", + "0x478adfa93a7bb904d0aa86ff0d559f43aa915ee7865592e717b72a24452181cb" + ], + "postageProof": { + "signature": "Z0fFjOhhNIbGlvW7c5PJxZCUNxlpw6Ur+vdRksYF9K18cMbnH90yDiDQBeQulO4yECwjTrRl9PX9nbYPytA1axw=", + "postageId": "0x4c8efc14c8e3cee608174f995d7afe155897bf643a31226e4f1363bc97686aef", + "index": 525059, + "timeStamp": 197384 + }, + "socProof": [] + } +} \ No newline at end of file diff --git a/pkg/storer/sample.go b/pkg/storer/sample.go index 690e06affc3..7a37fb62e99 100644 --- a/pkg/storer/sample.go +++ b/pkg/storer/sample.go @@ -7,9 +7,9 @@ package storer import ( "bytes" "context" - "crypto/hmac" "encoding/binary" "fmt" + "hash" "math/big" "sort" "sync" @@ -26,7 +26,7 @@ import ( "golang.org/x/sync/errgroup" ) -const SampleSize = 8 +const SampleSize = 16 type SampleItem struct { TransformedAddress swarm.Address @@ -40,18 +40,37 @@ type Sample struct { Items []SampleItem } +// RandSample returns Sample with random values. func RandSample(t *testing.T, anchor []byte) Sample { t.Helper() - hasher := bmt.NewTrHasher(anchor) - - items := make([]SampleItem, SampleSize) + chunks := make([]swarm.Chunk, SampleSize) for i := 0; i < SampleSize; i++ { ch := chunk.GenerateTestRandomChunk() + if i%3 == 0 { + ch = chunk.GenerateTestRandomSoChunk(t, ch) + } + chunks[i] = ch + } + + sample, err := MakeSampleUsingChunks(chunks, anchor) + if err != nil { + t.Fatal(err) + } - tr, err := transformedAddress(hasher, ch, swarm.ChunkTypeContentAddressed) + return sample +} + +// MakeSampleUsingChunks returns Sample constructed using supplied chunks. +func MakeSampleUsingChunks(chunks []swarm.Chunk, anchor []byte) (Sample, error) { + prefixHasherFactory := func() hash.Hash { + return swarm.NewPrefixHasher(anchor) + } + items := make([]SampleItem, len(chunks)) + for i, ch := range chunks { + tr, err := transformedAddress(bmt.NewHasher(prefixHasherFactory), ch, swarm.ChunkTypeContentAddressed) if err != nil { - t.Fatal(err) + return Sample{}, err } items[i] = SampleItem{ @@ -66,7 +85,7 @@ func RandSample(t *testing.T, anchor []byte) Sample { return items[i].TransformedAddress.Compare(items[j].TransformedAddress) == -1 }) - return Sample{Items: items} + return Sample{Items: items}, nil } func newStamp(s swarm.Stamp) *postage.Stamp { @@ -135,19 +154,25 @@ func (db *DB) ReserveSample( // Phase 2: Get the chunk data and calculate transformed hash sampleItemChan := make(chan SampleItem, 64) + prefixHasherFactory := func() hash.Hash { + return swarm.NewPrefixHasher(anchor) + } + const workers = 6 + for i := 0; i < workers; i++ { g.Go(func() error { wstat := SampleStats{} + hasher := bmt.NewHasher(prefixHasherFactory) defer func() { addStats(wstat) }() - hmacr := hmac.New(swarm.NewHasher, anchor) for chItem := range chunkC { // exclude chunks who's batches balance are below minimum if _, found := excludedBatchIDs[string(chItem.BatchID)]; found { wstat.BelowBalanceIgnored++ + continue } @@ -169,16 +194,12 @@ func (db *DB) ReserveSample( wstat.ChunkLoadDuration += time.Since(chunkLoadStart) - hmacrStart := time.Now() - - hmacr.Reset() - _, err = hmacr.Write(chunk.Data()) + taddrStart := time.Now() + taddr, err := transformedAddress(hasher, chunk, chItem.Type) if err != nil { return err } - taddr := swarm.NewAddress(hmacr.Sum(nil)) - - wstat.HmacrDuration += time.Since(hmacrStart) + wstat.TaddrDuration += time.Since(taddrStart) select { case sampleItemChan <- SampleItem{ @@ -222,6 +243,15 @@ func (db *DB) ReserveSample( } } + contains := func(addr swarm.Address) int { + for index, item := range sampleItems { + if item.ChunkAddress.Compare(addr) == 0 { + return index + } + } + return -1 + } + // Phase 3: Assemble the sample. Here we need to assemble only the first SampleSize // no of items from the results of the 2nd phase. // In this step stamps are loaded and validated only if chunk will be added to sample. @@ -259,6 +289,13 @@ func (db *DB) ReserveSample( stats.ValidStampDuration += time.Since(start) item.Stamp = stamp + + // ensuring to pass the check order function of redistribution contract + if index := contains(item.TransformedAddress); index != -1 { + // replace the chunk at index + sampleItems[index] = item + continue + } insert(item) stats.SampleInserts++ } @@ -359,7 +396,7 @@ type SampleStats struct { NewIgnored int64 InvalidStamp int64 BelowBalanceIgnored int64 - HmacrDuration time.Duration + TaddrDuration time.Duration ValidStampDuration time.Duration BatchesBelowValueDuration time.Duration RogueChunk int64 @@ -376,7 +413,7 @@ func (s *SampleStats) add(other SampleStats) { s.NewIgnored += other.NewIgnored s.InvalidStamp += other.InvalidStamp s.BelowBalanceIgnored += other.BelowBalanceIgnored - s.HmacrDuration += other.HmacrDuration + s.TaddrDuration += other.TaddrDuration s.ValidStampDuration += other.ValidStampDuration s.BatchesBelowValueDuration += other.BatchesBelowValueDuration s.RogueChunk += other.RogueChunk diff --git a/pkg/swarm/hasher.go b/pkg/swarm/hasher.go index 485b61ab398..b9823bb50a1 100644 --- a/pkg/swarm/hasher.go +++ b/pkg/swarm/hasher.go @@ -15,15 +15,15 @@ func NewHasher() hash.Hash { return sha3.NewLegacyKeccak256() } -type trHasher struct { +type PrefixHasher struct { hash.Hash prefix []byte } -// NewTrHasher returns new hasher which is Keccak-256 hasher +// NewPrefixHasher returns new hasher which is Keccak-256 hasher // with prefix value added as initial data. -func NewTrHasher(prefix []byte) hash.Hash { - h := &trHasher{ +func NewPrefixHasher(prefix []byte) hash.Hash { + h := &PrefixHasher{ Hash: NewHasher(), prefix: prefix, } @@ -32,7 +32,7 @@ func NewTrHasher(prefix []byte) hash.Hash { return h } -func (h *trHasher) Reset() { +func (h *PrefixHasher) Reset() { h.Hash.Reset() _, _ = h.Write(h.prefix) } diff --git a/pkg/swarm/hasher_test.go b/pkg/swarm/hasher_test.go index bfee0a78e98..3811e09605a 100644 --- a/pkg/swarm/hasher_test.go +++ b/pkg/swarm/hasher_test.go @@ -66,7 +66,7 @@ func TestNewTrHasher(t *testing.T) { // Run tests cases against TrHasher for _, tc := range tests { - h := swarm.NewTrHasher(tc.prefix) + h := swarm.NewPrefixHasher(tc.prefix) _, err := h.Write(tc.plaintext) if err != nil { From d92945e5e7782b1cd9049728a377de3347e100ee Mon Sep 17 00:00:00 2001 From: mrekucci Date: Fri, 29 Sep 2023 17:58:44 +0400 Subject: [PATCH 02/24] fix: include the reason when the redistribution contract call fails (#4358) --- pkg/postage/postagecontract/contract.go | 15 ++++++--------- pkg/storageincentives/agent.go | 2 +- .../redistribution/redistribution.go | 6 ++++-- pkg/transaction/mock/transaction.go | 4 ++++ pkg/transaction/transaction.go | 18 ++++++++++++++++++ 5 files changed, 33 insertions(+), 12 deletions(-) diff --git a/pkg/postage/postagecontract/contract.go b/pkg/postage/postagecontract/contract.go index 94c3dc99061..666037c39d6 100644 --- a/pkg/postage/postagecontract/contract.go +++ b/pkg/postage/postagecontract/contract.go @@ -178,7 +178,7 @@ func (c *postageContract) sendApproveTransaction(ctx context.Context, amount *bi return receipt, nil } -func (c *postageContract) sendTransaction(ctx context.Context, callData []byte, desc string) (*types.Receipt, error) { +func (c *postageContract) sendTransaction(ctx context.Context, callData []byte, desc string) (receipt *types.Receipt, err error) { request := &transaction.TxRequest{ To: &c.postageStampContractAddress, Data: callData, @@ -187,25 +187,22 @@ func (c *postageContract) sendTransaction(ctx context.Context, callData []byte, Value: big.NewInt(0), Description: desc, } + defer func() { + err = c.transactionService.UnwrapRevertReason(ctx, request, err) + }() txHash, err := c.transactionService.Send(ctx, request, transaction.DefaultTipBoostPercent) if err != nil { return nil, err } - receipt, err := c.transactionService.WaitForReceipt(ctx, txHash) + receipt, err = c.transactionService.WaitForReceipt(ctx, txHash) if err != nil { return nil, err } if receipt.Status == 0 { - err := transaction.ErrTransactionReverted - if res, cErr := c.transactionService.Call(ctx, request); cErr == nil { - if reason, uErr := abi.UnpackRevert(res); uErr == nil { - err = fmt.Errorf("%w: reason: %s", err, reason) - } - } - return nil, err + return nil, transaction.ErrTransactionReverted } return receipt, nil diff --git a/pkg/storageincentives/agent.go b/pkg/storageincentives/agent.go index b856625025e..16e7770c859 100644 --- a/pkg/storageincentives/agent.go +++ b/pkg/storageincentives/agent.go @@ -24,7 +24,7 @@ import ( "github.com/ethersphere/bee/pkg/storage" "github.com/ethersphere/bee/pkg/storageincentives/redistribution" "github.com/ethersphere/bee/pkg/storageincentives/staking" - storer "github.com/ethersphere/bee/pkg/storer" + "github.com/ethersphere/bee/pkg/storer" "github.com/ethersphere/bee/pkg/swarm" "github.com/ethersphere/bee/pkg/transaction" ) diff --git a/pkg/storageincentives/redistribution/redistribution.go b/pkg/storageincentives/redistribution/redistribution.go index 2c163f5a76c..ce0692da943 100644 --- a/pkg/storageincentives/redistribution/redistribution.go +++ b/pkg/storageincentives/redistribution/redistribution.go @@ -180,8 +180,10 @@ func (c *contract) ReserveSalt(ctx context.Context) ([]byte, error) { return salt[:], nil } -func (c *contract) sendAndWait(ctx context.Context, request *transaction.TxRequest, boostPercent int) (common.Hash, error) { - txHash, err := c.txService.Send(ctx, request, boostPercent) +func (c *contract) sendAndWait(ctx context.Context, request *transaction.TxRequest, boostPercent int) (txHash common.Hash, err error) { + defer func() { err = c.txService.UnwrapRevertReason(ctx, request, err) }() + + txHash, err = c.txService.Send(ctx, request, boostPercent) if err != nil { return txHash, err } diff --git a/pkg/transaction/mock/transaction.go b/pkg/transaction/mock/transaction.go index f9a557f00be..6110dfeea1d 100644 --- a/pkg/transaction/mock/transaction.go +++ b/pkg/transaction/mock/transaction.go @@ -97,6 +97,10 @@ func (m *transactionServiceMock) TransactionFee(ctx context.Context, txHash comm return big.NewInt(0), nil } +func (m *transactionServiceMock) UnwrapRevertReason(_ context.Context, _ *transaction.TxRequest, err error) error { + return err +} + // Option is the option passed to the mock Chequebook service type Option interface { apply(*transactionServiceMock) diff --git a/pkg/transaction/transaction.go b/pkg/transaction/transaction.go index d8cb6d797d3..c4246631023 100644 --- a/pkg/transaction/transaction.go +++ b/pkg/transaction/transaction.go @@ -14,6 +14,7 @@ import ( "time" "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethersphere/bee/pkg/crypto" @@ -94,6 +95,9 @@ type Service interface { CancelTransaction(ctx context.Context, originalTxHash common.Hash) (common.Hash, error) // TransactionFee retrieves the transaction fee TransactionFee(ctx context.Context, txHash common.Hash) (*big.Int, error) + // UnwrapRevertReason tries to unwrap the revert reason if the given error is not nil. + // The original error is wrapped in case the revert reason exists. + UnwrapRevertReason(ctx context.Context, req *TxRequest, err error) error } type transactionService struct { @@ -584,3 +588,17 @@ func (t *transactionService) TransactionFee(ctx context.Context, txHash common.H } return trx.Cost(), nil } + +func (t *transactionService) UnwrapRevertReason(ctx context.Context, req *TxRequest, err error) error { + if err == nil { + return nil + } + + if res, cErr := t.Call(ctx, req); cErr == nil { + if reason, uErr := abi.UnpackRevert(res); uErr == nil { + err = fmt.Errorf("%w: reason: %s", err, reason) + } + } + + return err +} From 9b0da30f263984e9efc101fe866d838e1ed6e007 Mon Sep 17 00:00:00 2001 From: istae <14264581+istae@users.noreply.github.com> Date: Mon, 2 Oct 2023 18:01:02 +0300 Subject: [PATCH 03/24] fix(transaction): swallow estimate gas error (#4366) --- pkg/storageincentives/agent.go | 10 +-- pkg/transaction/transaction.go | 3 +- pkg/transaction/transaction_test.go | 121 ++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+), 6 deletions(-) diff --git a/pkg/storageincentives/agent.go b/pkg/storageincentives/agent.go index 16e7770c859..51a62393a8c 100644 --- a/pkg/storageincentives/agent.go +++ b/pkg/storageincentives/agent.go @@ -131,7 +131,7 @@ func (a *Agent) start(blockTime time.Duration, blocksPerRound, blocksPerPhase ui phaseEvents := newEvents() defer phaseEvents.Close() - logPhaseResult := func(phase PhaseType, round uint64, err error) { + logErr := func(phase PhaseType, round uint64, err error) { if err != nil { a.logger.Error(err, "phase failed", "phase", phase, "round", round) } @@ -142,13 +142,13 @@ func (a *Agent) start(blockTime time.Duration, blocksPerRound, blocksPerPhase ui round, _ := a.state.currentRoundAndPhase() err := a.handleCommit(ctx, round) - logPhaseResult(commit, round, err) + logErr(commit, round, err) }) phaseEvents.On(reveal, func(ctx context.Context) { phaseEvents.Cancel(commit, sample) round, _ := a.state.currentRoundAndPhase() - logPhaseResult(reveal, round, a.handleReveal(ctx, round)) + logErr(reveal, round, a.handleReveal(ctx, round)) }) phaseEvents.On(claim, func(ctx context.Context) { @@ -156,13 +156,13 @@ func (a *Agent) start(blockTime time.Duration, blocksPerRound, blocksPerPhase ui phaseEvents.Publish(sample) round, _ := a.state.currentRoundAndPhase() - logPhaseResult(claim, round, a.handleClaim(ctx, round)) + logErr(claim, round, a.handleClaim(ctx, round)) }) phaseEvents.On(sample, func(ctx context.Context) { round, _ := a.state.currentRoundAndPhase() isPhasePlayed, err := a.handleSample(ctx, round) - logPhaseResult(sample, round, err) + logErr(sample, round, err) // Sample handled could potentially take long time, therefore it could overlap with commit // phase of next round. When that case happens commit event needs to be triggered once more diff --git a/pkg/transaction/transaction.go b/pkg/transaction/transaction.go index c4246631023..05a07fbf461 100644 --- a/pkg/transaction/transaction.go +++ b/pkg/transaction/transaction.go @@ -282,7 +282,8 @@ func (t *transactionService) prepareTransaction(ctx context.Context, request *Tx Data: request.Data, }) if err != nil { - return nil, err + t.logger.Debug("estimage gas failed", "error", err) + gasLimit = request.MinEstimatedGasLimit } gasLimit += gasLimit / 4 // add 25% on top diff --git a/pkg/transaction/transaction_test.go b/pkg/transaction/transaction_test.go index 60e0989a8a0..8794fd33281 100644 --- a/pkg/transaction/transaction_test.go +++ b/pkg/transaction/transaction_test.go @@ -7,6 +7,7 @@ package transaction_test import ( "bytes" "context" + "errors" "fmt" "math/big" "testing" @@ -213,6 +214,126 @@ func TestTransactionSend(t *testing.T) { } }) + t.Run("send with estimate error", func(t *testing.T) { + t.Parallel() + + signedTx := types.NewTx(&types.DynamicFeeTx{ + ChainID: chainID, + Nonce: nonce, + To: &recipient, + Value: value, + Gas: estimatedGasLimit, + GasFeeCap: defaultGasFee, + GasTipCap: suggestedGasTip, + Data: txData, + }) + request := &transaction.TxRequest{ + To: &recipient, + Data: txData, + Value: value, + MinEstimatedGasLimit: estimatedGasLimit, + } + store := storemock.NewStateStore() + err := store.Put(nonceKey(sender), nonce) + if err != nil { + t.Fatal(err) + } + + transactionService, err := transaction.NewService(logger, + backendmock.New( + backendmock.WithSendTransactionFunc(func(ctx context.Context, tx *types.Transaction) error { + if tx != signedTx { + t.Fatal("not sending signed transaction") + } + return nil + }), + backendmock.WithEstimateGasFunc(func(ctx context.Context, call ethereum.CallMsg) (gas uint64, err error) { + return 0, errors.New("estimate failure") + }), + backendmock.WithSuggestGasPriceFunc(func(ctx context.Context) (*big.Int, error) { + return suggestedGasPrice, nil + }), + backendmock.WithPendingNonceAtFunc(func(ctx context.Context, account common.Address) (uint64, error) { + return nonce - 1, nil + }), + backendmock.WithSuggestGasTipCapFunc(func(ctx context.Context) (*big.Int, error) { + return suggestedGasTip, nil + }), + ), + signerMockForTransaction(t, signedTx, sender, chainID), + store, + chainID, + monitormock.New( + monitormock.WithWatchTransactionFunc(func(txHash common.Hash, nonce uint64) (<-chan types.Receipt, <-chan error, error) { + return nil, nil, nil + }), + ), + ) + if err != nil { + t.Fatal(err) + } + testutil.CleanupCloser(t, transactionService) + + txHash, err := transactionService.Send(context.Background(), request, 0) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(txHash.Bytes(), signedTx.Hash().Bytes()) { + t.Fatal("returning wrong transaction hash") + } + + var storedNonce uint64 + err = store.Get(nonceKey(sender), &storedNonce) + if err != nil { + t.Fatal(err) + } + if storedNonce != nonce+1 { + t.Fatalf("nonce not stored correctly: want %d, got %d", nonce+1, storedNonce) + } + + storedTransaction, err := transactionService.StoredTransaction(txHash) + if err != nil { + t.Fatal(err) + } + + if storedTransaction.To == nil || *storedTransaction.To != recipient { + t.Fatalf("got wrong recipient in stored transaction. wanted %x, got %x", recipient, storedTransaction.To) + } + + if !bytes.Equal(storedTransaction.Data, request.Data) { + t.Fatalf("got wrong data in stored transaction. wanted %x, got %x", request.Data, storedTransaction.Data) + } + + if storedTransaction.Description != request.Description { + t.Fatalf("got wrong description in stored transaction. wanted %x, got %x", request.Description, storedTransaction.Description) + } + + if storedTransaction.GasLimit != estimatedGasLimit { + t.Fatalf("got wrong gas limit in stored transaction. wanted %d, got %d", estimatedGasLimit, storedTransaction.GasLimit) + } + + if defaultGasFee.Cmp(storedTransaction.GasPrice) != 0 { + t.Fatalf("got wrong gas price in stored transaction. wanted %d, got %d", defaultGasFee, storedTransaction.GasPrice) + } + + if storedTransaction.Nonce != nonce { + t.Fatalf("got wrong nonce in stored transaction. wanted %d, got %d", nonce, storedTransaction.Nonce) + } + + pending, err := transactionService.PendingTransactions() + if err != nil { + t.Fatal(err) + } + if len(pending) != 1 { + t.Fatalf("expected one pending transaction, got %d", len(pending)) + } + + if pending[0] != txHash { + t.Fatalf("got wrong pending transaction. wanted %x, got %x", txHash, pending[0]) + } + }) + t.Run("sendWithBoost", func(t *testing.T) { t.Parallel() From b041fb2acd8e330072658d89fdb298293c62629d Mon Sep 17 00:00:00 2001 From: istae <14264581+istae@users.noreply.github.com> Date: Tue, 3 Oct 2023 11:56:32 +0300 Subject: [PATCH 04/24] fix(postage): added postage stamps json marshaling (#4369) --- pkg/postage/stamp.go | 30 ++++++++++++++++++++++++++++++ pkg/postage/stamp_test.go | 19 +++++++++++++++++++ pkg/storer/sample.go | 4 ++-- 3 files changed, 51 insertions(+), 2 deletions(-) diff --git a/pkg/postage/stamp.go b/pkg/postage/stamp.go index 7d65f4eee5b..4fff65797ad 100644 --- a/pkg/postage/stamp.go +++ b/pkg/postage/stamp.go @@ -6,6 +6,7 @@ package postage import ( "bytes" + "encoding/json" "errors" "fmt" @@ -118,6 +119,35 @@ func (s *Stamp) UnmarshalBinary(buf []byte) error { return nil } +type stampJson struct { + BatchID []byte `json:"batchID"` + Index []byte `json:"index"` + Timestamp []byte `json:"timestamp"` + Sig []byte `json:"sig"` +} + +func (s *Stamp) MarshalJSON() ([]byte, error) { + return json.Marshal(&stampJson{ + s.batchID, + s.index, + s.timestamp, + s.sig, + }) +} + +func (a *Stamp) UnmarshalJSON(b []byte) error { + v := &stampJson{} + err := json.Unmarshal(b, v) + if err != nil { + return err + } + a.batchID = v.BatchID + a.index = v.Index + a.timestamp = v.Timestamp + a.sig = v.Sig + return nil +} + // ToSignDigest creates a digest to represent the stamp which is to be signed by the owner. func ToSignDigest(addr, batchId, index, timestamp []byte) ([]byte, error) { h := swarm.NewHasher() diff --git a/pkg/postage/stamp_test.go b/pkg/postage/stamp_test.go index f791d8e7fe6..ab98a197349 100644 --- a/pkg/postage/stamp_test.go +++ b/pkg/postage/stamp_test.go @@ -6,6 +6,7 @@ package postage_test import ( "bytes" + "encoding/json" "math/big" "testing" @@ -31,6 +32,24 @@ func TestStampMarshalling(t *testing.T) { compareStamps(t, sExp, s) } +// TestStampMarshalling tests the idempotence of binary marshal/unmarshals for Stamps. +func TestStampJsonMarshalling(t *testing.T) { + sExp := postagetesting.MustNewStamp() + + b, err := json.Marshal(sExp) + if err != nil { + t.Fatal(err) + } + + s := postage.NewStamp(nil, nil, nil, nil) + err = json.Unmarshal(b, s) + if err != nil { + t.Fatal(err) + } + + compareStamps(t, sExp, s) +} + func compareStamps(t *testing.T, s1, s2 *postage.Stamp) { t.Helper() diff --git a/pkg/storer/sample.go b/pkg/storer/sample.go index 7a37fb62e99..196bb4d26a9 100644 --- a/pkg/storer/sample.go +++ b/pkg/storer/sample.go @@ -32,7 +32,7 @@ type SampleItem struct { TransformedAddress swarm.Address ChunkAddress swarm.Address ChunkData []byte - Stamp swarm.Stamp + Stamp *postage.Stamp } type Sample struct { @@ -288,7 +288,7 @@ func (db *DB) ReserveSample( stats.ValidStampDuration += time.Since(start) - item.Stamp = stamp + item.Stamp = postage.NewStamp(stamp.BatchID(), stamp.Index(), stamp.Timestamp(), stamp.Sig()) // ensuring to pass the check order function of redistribution contract if index := contains(item.TransformedAddress); index != -1 { From 3f388c5df2a81897173462d7f1d56d00d373cf87 Mon Sep 17 00:00:00 2001 From: istae <14264581+istae@users.noreply.github.com> Date: Tue, 3 Oct 2023 12:11:19 +0300 Subject: [PATCH 05/24] chore: update si (#4370) --- go.mod | 2 +- go.sum | 4 ++-- pkg/pusher/pusher.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 9ca865cf074..d9b4860f937 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/coreos/go-semver v0.3.0 github.com/ethereum/go-ethereum v1.12.2 github.com/ethersphere/go-price-oracle-abi v0.1.0 - github.com/ethersphere/go-storage-incentives-abi v0.6.0-rc3 + github.com/ethersphere/go-storage-incentives-abi v0.6.0-rc5 github.com/ethersphere/go-sw3-abi v0.4.0 github.com/ethersphere/langos v1.0.0 github.com/go-playground/validator/v10 v10.11.1 diff --git a/go.sum b/go.sum index 8b98c44d2ed..40d8731ff09 100644 --- a/go.sum +++ b/go.sum @@ -223,8 +223,8 @@ github.com/ethereum/go-ethereum v1.12.2 h1:eGHJ4ij7oyVqUQn48LBz3B7pvQ8sV0wGJiIE6 github.com/ethereum/go-ethereum v1.12.2/go.mod h1:1cRAEV+rp/xX0zraSCBnu9Py3HQ+geRMj3HdR+k0wfI= github.com/ethersphere/go-price-oracle-abi v0.1.0 h1:yg/hK8nETNvk+GEBASlbakMFv/CVp7HXiycrHw1pRV8= github.com/ethersphere/go-price-oracle-abi v0.1.0/go.mod h1:sI/Qj4/zJ23/b1enzwMMv0/hLTpPNVNacEwCWjo6yBk= -github.com/ethersphere/go-storage-incentives-abi v0.6.0-rc3 h1:tXux2FnhuU6DbrY+Z4nVQMGp63JkJPq7pKb5Xi2Sjxo= -github.com/ethersphere/go-storage-incentives-abi v0.6.0-rc3/go.mod h1:SXvJVtM4sEsaSKD0jc1ClpDLw8ErPoROZDme4Wrc/Nc= +github.com/ethersphere/go-storage-incentives-abi v0.6.0-rc5 h1:JaiGMFoycByFkaCv6KFg79tfRlBJ5zY01dLWPnf48v0= +github.com/ethersphere/go-storage-incentives-abi v0.6.0-rc5/go.mod h1:SXvJVtM4sEsaSKD0jc1ClpDLw8ErPoROZDme4Wrc/Nc= github.com/ethersphere/go-sw3-abi v0.4.0 h1:T3ANY+ktWrPAwe2U0tZi+DILpkHzto5ym/XwV/Bbz8g= github.com/ethersphere/go-sw3-abi v0.4.0/go.mod h1:BmpsvJ8idQZdYEtWnvxA8POYQ8Rl/NhyCdF0zLMOOJU= github.com/ethersphere/langos v1.0.0 h1:NBtNKzXTTRSue95uOlzPN4py7Aofs0xWPzyj4AI1Vcc= diff --git a/pkg/pusher/pusher.go b/pkg/pusher/pusher.go index 2fff5739af2..f7abbe66d2c 100644 --- a/pkg/pusher/pusher.go +++ b/pkg/pusher/pusher.go @@ -287,7 +287,7 @@ func (s *Service) pushDeferred(ctx context.Context, logger log.Logger, op *Op) ( } case err == nil: if err := s.checkReceipt(receipt, loggerV1); err != nil { - loggerV1.Error(err, "pusher: failed checking receipt") + loggerV1.Error(err, "pusher: failed checking receipt", "chunk_address", op.Chunk.Address()) return true, err } if err := s.storer.Report(ctx, op.Chunk, storage.ChunkSynced); err != nil { @@ -341,7 +341,7 @@ func (s *Service) pushDirect(ctx context.Context, logger log.Logger, op *Op) err case err == nil: err = s.checkReceipt(receipt, loggerV1) if err != nil { - loggerV1.Error(err, "pusher: failed checking receipt") + loggerV1.Error(err, "pusher: failed checking receipt", "chunk_address", op.Chunk.Address()) } default: loggerV1.Error(err, "pusher: failed PushChunkToClosest") From d6b34ce32982e581e14c612c2ce2dd35e73e8a54 Mon Sep 17 00:00:00 2001 From: mrekucci Date: Tue, 3 Oct 2023 13:13:26 +0400 Subject: [PATCH 06/24] feat: unwrap smart contract custom abi errors (#4367) --- pkg/node/node.go | 20 +++--------- pkg/postage/postagecontract/contract.go | 27 +++++++++++++--- .../redistribution/redistribution.go | 9 +++++- pkg/storageincentives/staking/contract.go | 32 +++++++++++++++---- pkg/transaction/mock/transaction.go | 2 +- pkg/transaction/transaction.go | 25 +++++++++++---- 6 files changed, 79 insertions(+), 36 deletions(-) diff --git a/pkg/node/node.go b/pkg/node/node.go index 36fea876a8b..ddcdad37771 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -19,12 +19,10 @@ import ( "net/http" "path/filepath" "runtime" - "strings" "sync" "sync/atomic" "time" - "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethersphere/bee/pkg/accounting" "github.com/ethersphere/bee/pkg/addressbook" @@ -71,6 +69,7 @@ import ( "github.com/ethersphere/bee/pkg/topology/lightnode" "github.com/ethersphere/bee/pkg/tracing" "github.com/ethersphere/bee/pkg/transaction" + "github.com/ethersphere/bee/pkg/util/abiutil" "github.com/ethersphere/bee/pkg/util/ioutil" "github.com/ethersphere/bee/pkg/util/nbhdutil" "github.com/ethersphere/bee/pkg/util/syncutil" @@ -681,10 +680,7 @@ func NewBee( return nil, errors.New("no known postage stamp addresses for this network") } - postageStampContractABI, err := abi.JSON(strings.NewReader(chainCfg.PostageStampABI)) - if err != nil { - return nil, fmt.Errorf("unable to parse postage stamp ABI: %w", err) - } + postageStampContractABI := abiutil.MustParseABI(chainCfg.PostageStampABI) bzzTokenAddress, err := postagecontract.LookupERC20Address(ctx, transactionService, postageStampContractAddress, postageStampContractABI, chainEnabled) if err != nil { @@ -1032,11 +1028,7 @@ func NewBee( stakingContractAddress = common.HexToAddress(o.StakingContractAddress) } - stakingContractABI, err := abi.JSON(strings.NewReader(chainCfg.StakingABI)) - if err != nil { - return nil, fmt.Errorf("unable to parse staking ABI: %w", err) - } - stakingContract := staking.New(swarmAddress, overlayEthAddress, stakingContractAddress, stakingContractABI, bzzTokenAddress, transactionService, common.BytesToHash(nonce)) + stakingContract := staking.New(swarmAddress, overlayEthAddress, stakingContractAddress, abiutil.MustParseABI(chainCfg.StakingABI), bzzTokenAddress, transactionService, common.BytesToHash(nonce)) var ( pullerService *puller.Puller @@ -1059,16 +1051,12 @@ func NewBee( } redistributionContractAddress = common.HexToAddress(o.RedistributionContractAddress) } - redistributionContractABI, err := abi.JSON(strings.NewReader(chainCfg.RedistributionABI)) - if err != nil { - return nil, fmt.Errorf("unable to parse redistribution ABI: %w", err) - } isFullySynced := func() bool { return localStore.ReserveSize() >= reserveTreshold && pullerService.SyncRate() == 0 } - redistributionContract := redistribution.New(swarmAddress, logger, transactionService, redistributionContractAddress, redistributionContractABI) + redistributionContract := redistribution.New(swarmAddress, logger, transactionService, redistributionContractAddress, abiutil.MustParseABI(chainCfg.RedistributionABI)) agent, err = storageincentives.New( swarmAddress, overlayEthAddress, diff --git a/pkg/postage/postagecontract/contract.go b/pkg/postage/postagecontract/contract.go index 666037c39d6..234cc4afdd8 100644 --- a/pkg/postage/postagecontract/contract.go +++ b/pkg/postage/postagecontract/contract.go @@ -148,25 +148,36 @@ func (c *postageContract) expireLimitedBatches(ctx context.Context, count *big.I return nil } -func (c *postageContract) sendApproveTransaction(ctx context.Context, amount *big.Int) (*types.Receipt, error) { +func (c *postageContract) sendApproveTransaction(ctx context.Context, amount *big.Int) (receipt *types.Receipt, err error) { callData, err := erc20ABI.Pack("approve", c.postageStampContractAddress, amount) if err != nil { return nil, err } - txHash, err := c.transactionService.Send(ctx, &transaction.TxRequest{ + request := &transaction.TxRequest{ To: &c.bzzTokenAddress, Data: callData, GasPrice: sctx.GetGasPrice(ctx), GasLimit: 65000, Value: big.NewInt(0), Description: approveDescription, - }, transaction.DefaultTipBoostPercent) + } + + defer func() { + err = c.transactionService.UnwrapABIError( + ctx, + request, + err, + c.postageStampContractABI.Errors, + ) + }() + + txHash, err := c.transactionService.Send(ctx, request, transaction.DefaultTipBoostPercent) if err != nil { return nil, err } - receipt, err := c.transactionService.WaitForReceipt(ctx, txHash) + receipt, err = c.transactionService.WaitForReceipt(ctx, txHash) if err != nil { return nil, err } @@ -187,8 +198,14 @@ func (c *postageContract) sendTransaction(ctx context.Context, callData []byte, Value: big.NewInt(0), Description: desc, } + defer func() { - err = c.transactionService.UnwrapRevertReason(ctx, request, err) + err = c.transactionService.UnwrapABIError( + ctx, + request, + err, + c.postageStampContractABI.Errors, + ) }() txHash, err := c.transactionService.Send(ctx, request, transaction.DefaultTipBoostPercent) diff --git a/pkg/storageincentives/redistribution/redistribution.go b/pkg/storageincentives/redistribution/redistribution.go index ce0692da943..8b83537cf90 100644 --- a/pkg/storageincentives/redistribution/redistribution.go +++ b/pkg/storageincentives/redistribution/redistribution.go @@ -181,7 +181,14 @@ func (c *contract) ReserveSalt(ctx context.Context) ([]byte, error) { } func (c *contract) sendAndWait(ctx context.Context, request *transaction.TxRequest, boostPercent int) (txHash common.Hash, err error) { - defer func() { err = c.txService.UnwrapRevertReason(ctx, request, err) }() + defer func() { + err = c.txService.UnwrapABIError( + ctx, + request, + err, + c.incentivesContractABI.Errors, + ) + }() txHash, err = c.txService.Send(ctx, request, boostPercent) if err != nil { diff --git a/pkg/storageincentives/staking/contract.go b/pkg/storageincentives/staking/contract.go index aa05267c197..014b1a67adc 100644 --- a/pkg/storageincentives/staking/contract.go +++ b/pkg/storageincentives/staking/contract.go @@ -77,25 +77,36 @@ func New( } } -func (c *contract) sendApproveTransaction(ctx context.Context, amount *big.Int) (*types.Receipt, error) { +func (c *contract) sendApproveTransaction(ctx context.Context, amount *big.Int) (receipt *types.Receipt, err error) { callData, err := erc20ABI.Pack("approve", c.stakingContractAddress, amount) if err != nil { return nil, err } - txHash, err := c.transactionService.Send(ctx, &transaction.TxRequest{ + request := &transaction.TxRequest{ To: &c.bzzTokenAddress, Data: callData, GasPrice: sctx.GetGasPrice(ctx), GasLimit: 65000, Value: big.NewInt(0), Description: approveDescription, - }, 0) + } + + defer func() { + err = c.transactionService.UnwrapABIError( + ctx, + request, + err, + c.stakingContractABI.Errors, + ) + }() + + txHash, err := c.transactionService.Send(ctx, request, 0) if err != nil { return nil, err } - receipt, err := c.transactionService.WaitForReceipt(ctx, txHash) + receipt, err = c.transactionService.WaitForReceipt(ctx, txHash) if err != nil { return nil, err } @@ -107,7 +118,7 @@ func (c *contract) sendApproveTransaction(ctx context.Context, amount *big.Int) return receipt, nil } -func (c *contract) sendTransaction(ctx context.Context, callData []byte, desc string) (*types.Receipt, error) { +func (c *contract) sendTransaction(ctx context.Context, callData []byte, desc string) (receipt *types.Receipt, err error) { request := &transaction.TxRequest{ To: &c.stakingContractAddress, Data: callData, @@ -117,12 +128,21 @@ func (c *contract) sendTransaction(ctx context.Context, callData []byte, desc st Description: desc, } + defer func() { + err = c.transactionService.UnwrapABIError( + ctx, + request, + err, + c.stakingContractABI.Errors, + ) + }() + txHash, err := c.transactionService.Send(ctx, request, transaction.DefaultTipBoostPercent) if err != nil { return nil, err } - receipt, err := c.transactionService.WaitForReceipt(ctx, txHash) + receipt, err = c.transactionService.WaitForReceipt(ctx, txHash) if err != nil { return nil, err } diff --git a/pkg/transaction/mock/transaction.go b/pkg/transaction/mock/transaction.go index 6110dfeea1d..7478daa63e6 100644 --- a/pkg/transaction/mock/transaction.go +++ b/pkg/transaction/mock/transaction.go @@ -97,7 +97,7 @@ func (m *transactionServiceMock) TransactionFee(ctx context.Context, txHash comm return big.NewInt(0), nil } -func (m *transactionServiceMock) UnwrapRevertReason(_ context.Context, _ *transaction.TxRequest, err error) error { +func (m *transactionServiceMock) UnwrapABIError(_ context.Context, _ *transaction.TxRequest, err error, _ map[string]abi.Error) error { return err } diff --git a/pkg/transaction/transaction.go b/pkg/transaction/transaction.go index 05a07fbf461..28d3912c78d 100644 --- a/pkg/transaction/transaction.go +++ b/pkg/transaction/transaction.go @@ -5,6 +5,7 @@ package transaction import ( + "bytes" "errors" "fmt" "io" @@ -95,9 +96,9 @@ type Service interface { CancelTransaction(ctx context.Context, originalTxHash common.Hash) (common.Hash, error) // TransactionFee retrieves the transaction fee TransactionFee(ctx context.Context, txHash common.Hash) (*big.Int, error) - // UnwrapRevertReason tries to unwrap the revert reason if the given error is not nil. - // The original error is wrapped in case the revert reason exists. - UnwrapRevertReason(ctx context.Context, req *TxRequest, err error) error + // UnwrapABIError tries to unwrap the ABI error if the given error is not nil. + // The original error is wrapped together with the ABI error if it exists. + UnwrapABIError(ctx context.Context, req *TxRequest, err error, abiErrors map[string]abi.Error) error } type transactionService struct { @@ -590,14 +591,24 @@ func (t *transactionService) TransactionFee(ctx context.Context, txHash common.H return trx.Cost(), nil } -func (t *transactionService) UnwrapRevertReason(ctx context.Context, req *TxRequest, err error) error { +func (t *transactionService) UnwrapABIError(ctx context.Context, req *TxRequest, err error, abiErrors map[string]abi.Error) error { if err == nil { return nil } - if res, cErr := t.Call(ctx, req); cErr == nil { - if reason, uErr := abi.UnpackRevert(res); uErr == nil { - err = fmt.Errorf("%w: reason: %s", err, reason) + res, cErr := t.Call(ctx, req) + if cErr != nil { + return err + } + + if reason, uErr := abi.UnpackRevert(res); uErr == nil { + return fmt.Errorf("%w: %s", err, reason) + } + + for _, abiError := range abiErrors { + if bytes.Equal(res[:4], abiError.ID[:4]) { + //abiError.Unpack(res[4:]) + return fmt.Errorf("%w: %s", err, abiError) } } From 6e3acab58c25f56deb8323a81c3b676dc7672f05 Mon Sep 17 00:00:00 2001 From: istae <14264581+istae@users.noreply.github.com> Date: Tue, 3 Oct 2023 17:23:38 +0300 Subject: [PATCH 07/24] chore: increase max commit length --- commitlint.config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commitlint.config.js b/commitlint.config.js index f84cd119c5f..2dc4be1ada7 100644 --- a/commitlint.config.js +++ b/commitlint.config.js @@ -4,7 +4,7 @@ module.exports = { 'body-case': [0, 'never'], 'footer-leading-blank': [1, 'always'], 'footer-max-line-length': [2, 'always', 72], - 'header-max-length': [2, 'always', 72], + 'header-max-length': [2, 'always', 100], 'subject-case': [ 2, 'never', From 2d16af2c6a749e677c5b30239b25b553d8dd7fd8 Mon Sep 17 00:00:00 2001 From: zelig Date: Wed, 4 Oct 2023 04:14:11 +0200 Subject: [PATCH 08/24] refactor(storageincentives): extensive rewrite --- pkg/api/api.go | 5 + pkg/api/api_test.go | 38 +- pkg/api/rchash.go | 97 ++-- pkg/bmt/proof.go | 15 +- pkg/bmt/proof_test.go | 18 +- pkg/node/node.go | 21 +- pkg/postage/batchstore/mock/store.go | 7 + pkg/postage/batchstore/store.go | 52 +- pkg/postage/interface.go | 12 + pkg/postage/noop.go | 2 + pkg/storageincentives/agent.go | 462 +++++++---------- pkg/storageincentives/agent_test.go | 110 ++-- pkg/storageincentives/events.go | 51 +- pkg/storageincentives/events_test.go | 59 +-- pkg/storageincentives/export_test.go | 11 - pkg/storageincentives/proof.go | 219 -------- pkg/storageincentives/proof_test.go | 203 -------- .../redistribution/abi_types.go | 87 ++++ .../redistribution/abi_types_test.go | 3 + .../redistribution/inclusionproof.go | 105 ---- .../redistribution/redistribution.go | 6 +- .../redistribution/redistribution_test.go | 53 +- pkg/storageincentives/redistributionstate.go | 25 +- .../redistributionstate_test.go | 41 +- pkg/storageincentives/sampler/debug.go | 35 ++ pkg/storageincentives/sampler/export_test.go | 6 + pkg/storageincentives/sampler/proof.go | 53 ++ pkg/storageincentives/sampler/proof_test.go | 7 + pkg/storageincentives/sampler/sampler.go | 470 ++++++++++++++++++ pkg/storageincentives/sampler/sampler_test.go | 58 +++ .../testdata/inclusion-proofs.json | 131 ++++- pkg/storer/mock/mockreserve.go | 13 - pkg/storer/sample_test.go | 195 -------- pkg/storer/sampler.go | 50 ++ pkg/storer/sampler_test.go | 193 +++++++ pkg/storer/storer.go | 2 - 36 files changed, 1577 insertions(+), 1338 deletions(-) delete mode 100644 pkg/storageincentives/export_test.go delete mode 100644 pkg/storageincentives/proof.go delete mode 100644 pkg/storageincentives/proof_test.go create mode 100644 pkg/storageincentives/redistribution/abi_types.go create mode 100644 pkg/storageincentives/redistribution/abi_types_test.go delete mode 100644 pkg/storageincentives/redistribution/inclusionproof.go create mode 100644 pkg/storageincentives/sampler/debug.go create mode 100644 pkg/storageincentives/sampler/export_test.go create mode 100644 pkg/storageincentives/sampler/proof.go create mode 100644 pkg/storageincentives/sampler/proof_test.go create mode 100644 pkg/storageincentives/sampler/sampler.go create mode 100644 pkg/storageincentives/sampler/sampler_test.go delete mode 100644 pkg/storer/sample_test.go create mode 100644 pkg/storer/sampler.go create mode 100644 pkg/storer/sampler_test.go diff --git a/pkg/api/api.go b/pkg/api/api.go index 49331689a0f..d4ea6baf63a 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -51,6 +51,7 @@ import ( "github.com/ethersphere/bee/pkg/steward" storage "github.com/ethersphere/bee/pkg/storage" "github.com/ethersphere/bee/pkg/storageincentives" + "github.com/ethersphere/bee/pkg/storageincentives/sampler" "github.com/ethersphere/bee/pkg/storageincentives/staking" storer "github.com/ethersphere/bee/pkg/storer" "github.com/ethersphere/bee/pkg/swarm" @@ -153,6 +154,7 @@ type Service struct { probe *Probe metricsRegistry *prometheus.Registry stakingContract staking.Contract + sampler *sampler.Sampler Options http.Handler @@ -181,6 +183,7 @@ type Service struct { batchStore postage.Storer stamperStore storage.Store syncStatus func() (bool, error) + sample *sampler.Sampler swap swap.Interface transaction transaction.Service @@ -245,6 +248,7 @@ type ExtraOptions struct { Post postage.Service PostageContract postagecontract.Interface Staking staking.Contract + Sampler *sampler.Sampler Steward steward.Interface SyncStatus func() (bool, error) NodeStatus *status.Service @@ -323,6 +327,7 @@ func (s *Service) Configure(signer crypto.Signer, auth auth.Authenticator, trace s.postageContract = e.PostageContract s.steward = e.Steward s.stakingContract = e.Staking + s.sampler = e.Sampler s.pingpong = e.Pingpong s.topologyDriver = e.TopologyDriver diff --git a/pkg/api/api_test.go b/pkg/api/api_test.go index 21e535f10ac..53309f1d63e 100644 --- a/pkg/api/api_test.go +++ b/pkg/api/api_test.go @@ -58,8 +58,10 @@ import ( testingc "github.com/ethersphere/bee/pkg/storage/testing" "github.com/ethersphere/bee/pkg/storageincentives" "github.com/ethersphere/bee/pkg/storageincentives/redistribution" + "github.com/ethersphere/bee/pkg/storageincentives/sampler" "github.com/ethersphere/bee/pkg/storageincentives/staking" mock2 "github.com/ethersphere/bee/pkg/storageincentives/staking/mock" + storer "github.com/ethersphere/bee/pkg/storer" mockstorer "github.com/ethersphere/bee/pkg/storer/mock" "github.com/ethersphere/bee/pkg/swarm" "github.com/ethersphere/bee/pkg/topology/lightnode" @@ -213,7 +215,7 @@ func newTestServer(t *testing.T, o testServerOptions) (*http.Client, *websocket. s.SetP2P(o.P2P) if o.RedistributionAgent == nil { - o.RedistributionAgent, _ = createRedistributionAgentService(t, o.Overlay, o.StateStorer, erc20, transaction, backend, o.BatchStore) + o.RedistributionAgent, _ = createRedistributionAgentService(t, o.Overlay, backend, o.BatchStore, o.StateStorer, erc20, transaction) s.SetRedistributionAgent(o.RedistributionAgent) } testutil.CleanupCloser(t, o.RedistributionAgent) @@ -689,11 +691,11 @@ func (c *chanStorer) Has(addr swarm.Address) bool { func createRedistributionAgentService( t *testing.T, addr swarm.Address, - storer storage.StateStorer, + backend postage.ChainBackend, + batchstore sampler.Batchstore, + stateStore storage.StateStorer, erc20Service erc20.Service, tranService transaction.Service, - backend storageincentives.ChainBackend, - chainStateGetter postage.ChainStateGetter, ) (*storageincentives.Agent, error) { t.Helper() @@ -707,28 +709,34 @@ func createRedistributionAgentService( return true, nil })) contract := &mockContract{} - + fullySyncedFunc := func() bool { return true } + healthyFunc := func() bool { return true } return storageincentives.New( + log.Noop, addr, common.Address{}, + time.Millisecond*10, backend, contract, postageContract, stakingContract, - mockstorer.NewReserve(), - func() bool { return true }, - time.Millisecond*10, - blocksPerRound, - blocksPerPhase, - storer, - chainStateGetter, + sampler.New(backend, batchstore, &mockSampler{}, fullySyncedFunc, healthyFunc), + stateStore, erc20Service, tranService, - &mockHealth{}, - log.Noop, ) } +type mockSampler struct{} + +func (*mockSampler) StorageRadius() uint8 { + return 0 +} + +func (*mockSampler) Iterate(uint8, func(storer.Chunk) (bool, error)) error { + return nil +} + type contractCall int func (c contractCall) String() string { @@ -776,7 +784,7 @@ func (m *mockContract) IsWinner(context.Context) (bool, error) { return false, nil } -func (m *mockContract) Claim(context.Context, redistribution.ChunkInclusionProofs) (common.Hash, error) { +func (m *mockContract) Claim(context.Context, []redistribution.Proof) (common.Hash, error) { m.mtx.Lock() defer m.mtx.Unlock() m.callsList = append(m.callsList, claimCall) diff --git a/pkg/api/rchash.go b/pkg/api/rchash.go index ad7a25f9992..736293d61e1 100644 --- a/pkg/api/rchash.go +++ b/pkg/api/rchash.go @@ -16,38 +16,32 @@ import ( "github.com/gorilla/mux" ) -type RCHashResponse struct { - Hash swarm.Address `json:"hash"` - Proofs ChunkInclusionProofs `json:"proofs"` - Duration time.Duration `json:"duration"` +type SampleWithProofs struct { + Hash swarm.Address `json:"hash"` + Proofs []Proof `json:"proofs"` + Duration time.Duration `json:"duration"` } -type ChunkInclusionProofs struct { - A ChunkInclusionProof `json:"proof1"` - B ChunkInclusionProof `json:"proof2"` - C ChunkInclusionProof `json:"proofLast"` -} - -// ChunkInclusionProof structure must exactly match +// Proof structure must exactly match // corresponding structure (of the same name) in Redistribution.sol smart contract. // github.com/ethersphere/storage-incentives/blob/ph_f2/src/Redistribution.sol // github.com/ethersphere/storage-incentives/blob/master/src/Redistribution.sol (when merged to master) -type ChunkInclusionProof struct { - ProofSegments []string `json:"proofSegments"` - ProveSegment string `json:"proveSegment"` - ProofSegments2 []string `json:"proofSegments2"` - ProveSegment2 string `json:"proveSegment2"` - ChunkSpan uint64 `json:"chunkSpan"` - ProofSegments3 []string `json:"proofSegments3"` - PostageProof PostageProof `json:"postageProof"` - SocProof []SOCProof `json:"socProof"` +type Proof struct { + Sisters []string `json:"sisters"` + Data string `json:"data"` + Sisters2 []string `json:"sisters2"` + Data2 string `json:"data2"` + Sisters3 []string `json:"sisters3"` + ChunkSpan uint64 `json:"chunkSpan"` + PostageProof PostageProof `json:"postageProof"` + SocProof []SOCProof `json:"socProof"` } // SOCProof structure must exactly match // corresponding structure (of the same name) in Redistribution.sol smart contract. type PostageProof struct { Signature string `json:"signature"` - PostageId string `json:"postageId"` + BatchID string `json:"BatchID"` Index string `json:"index"` TimeStamp string `json:"timeStamp"` } @@ -61,35 +55,37 @@ type SOCProof struct { ChunkAddr string `json:"chunkAddr"` } -func renderChunkInclusionProofs(proofs redistribution.ChunkInclusionProofs) ChunkInclusionProofs { - return ChunkInclusionProofs{ - A: renderChunkInclusionProof(proofs.A), - B: renderChunkInclusionProof(proofs.B), - C: renderChunkInclusionProof(proofs.C), +func renderProofs(proofs []redistribution.Proof) []Proof { + out := make([]Proof, 3) + for i, p := range proofs { + out[i] = renderProof(p) } + return out } -func renderChunkInclusionProof(proof redistribution.ChunkInclusionProof) ChunkInclusionProof { +func renderProof(proof redistribution.Proof) Proof { var socProof []SOCProof if len(proof.SocProof) == 1 { - socProof = []SOCProof{{ - Signer: hex.EncodeToString(proof.SocProof[0].Signer.Bytes()), - Signature: hex.EncodeToString(proof.SocProof[0].Signature[:]), - Identifier: hex.EncodeToString(proof.SocProof[0].Identifier.Bytes()), - ChunkAddr: hex.EncodeToString(proof.SocProof[0].ChunkAddr.Bytes()), - }} + socProof = []SOCProof{ + { + Signer: toHex(proof.SocProof[0].Signer), + Signature: toHex(proof.SocProof[0].Signature), + Identifier: toHex(proof.SocProof[0].Identifier[:]), + ChunkAddr: toHex(proof.SocProof[0].ChunkAddr[:]), + }, + } } - return ChunkInclusionProof{ - ProveSegment: hex.EncodeToString(proof.ProveSegment.Bytes()), - ProofSegments: renderCommonHash(proof.ProofSegments), - ProveSegment2: hex.EncodeToString(proof.ProveSegment2.Bytes()), - ProofSegments2: renderCommonHash(proof.ProofSegments2), - ProofSegments3: renderCommonHash(proof.ProofSegments3), - ChunkSpan: proof.ChunkSpan, + return Proof{ + Data: toHex(proof.Data[:]), + Sisters: renderHash(proof.Sisters...), + Data2: toHex(proof.Data2[:]), + Sisters2: renderHash(proof.Sisters2...), + Sisters3: renderHash(proof.Sisters3...), + ChunkSpan: proof.ChunkSpan, PostageProof: PostageProof{ - Signature: hex.EncodeToString(proof.PostageProof.Signature[:]), - PostageId: hex.EncodeToString(proof.PostageProof.PostageId[:]), + Signature: toHex(proof.PostageProof.Signature), + BatchID: toHex(proof.PostageProof.BatchId[:]), Index: strconv.FormatUint(proof.PostageProof.Index, 16), TimeStamp: strconv.FormatUint(proof.PostageProof.TimeStamp, 16), }, @@ -97,14 +93,16 @@ func renderChunkInclusionProof(proof redistribution.ChunkInclusionProof) ChunkIn } } -func renderCommonHash(proofSegments []common.Hash) []string { - output := make([]string, len(proofSegments)) - for i, s := range proofSegments { - output[i] = hex.EncodeToString(s.Bytes()) +func renderHash(hs ...common.Hash) []string { + output := make([]string, len(hs)) + for i, h := range hs { + output[i] = hex.EncodeToString(h.Bytes()) } return output } +var toHex func([]byte) string = hex.EncodeToString + // This API is kept for testing the sampler. As a result, no documentation or tests are added here. func (s *Service) rchash(w http.ResponseWriter, r *http.Request) { logger := s.logger.WithName("get_rchash").Build() @@ -123,17 +121,18 @@ func (s *Service) rchash(w http.ResponseWriter, r *http.Request) { anchor2 := []byte(paths.Anchor2) - swp, err := s.redistributionAgent.SampleWithProofs(r.Context(), anchor1, anchor2, paths.Depth) + var round uint64 + swp, err := s.sampler.ReserveSampleWithProofs(r.Context(), anchor1, anchor2, paths.Depth, round) if err != nil { logger.Error(err, "failed making sample with proofs") jsonhttp.InternalServerError(w, "failed making sample with proofs") return } - resp := RCHashResponse{ + resp := SampleWithProofs{ Hash: swp.Hash, Duration: swp.Duration, - Proofs: renderChunkInclusionProofs(swp.Proofs), + Proofs: renderProofs(swp.Proofs), } jsonhttp.OK(w, resp) diff --git a/pkg/bmt/proof.go b/pkg/bmt/proof.go index cc59fd33766..d753b6556d4 100644 --- a/pkg/bmt/proof.go +++ b/pkg/bmt/proof.go @@ -11,10 +11,10 @@ type Prover struct { // Proof represents a Merkle proof of segment type Proof struct { - ProveSegment []byte - ProofSegments [][]byte - Span []byte - Index int + Data []byte + Sisters [][]byte + Span []byte + Index int } // Hash overrides base hash function of Hasher to fill buffer with zeros until chunk length @@ -28,6 +28,7 @@ func (p Prover) Hash(b []byte) ([]byte, error) { return p.Hasher.Hash(b) } + // Proof returns the inclusion proof of the i-th data segment func (p Prover) Proof(i int) Proof { index := i @@ -60,9 +61,9 @@ func (p Prover) Proof(i int) Proof { func (p Prover) Verify(i int, proof Proof) (root []byte, err error) { var section []byte if i%2 == 0 { - section = append(append(section, proof.ProveSegment...), proof.ProofSegments[0]...) + section = append(append(section, proof.Data...), proof.Sisters[0]...) } else { - section = append(append(section, proof.ProofSegments[0]...), proof.ProveSegment...) + section = append(append(section, proof.Sisters[0]...), proof.Data...) } i = i / 2 n := p.bmt.leaves[i] @@ -74,7 +75,7 @@ func (p Prover) Verify(i int, proof Proof) (root []byte, err error) { } n = n.parent - for _, sister := range proof.ProofSegments[1:] { + for _, sister := range proof.Sisters[1:] { if isLeft { root, err = doHash(hasher, root, sister) } else { diff --git a/pkg/bmt/proof_test.go b/pkg/bmt/proof_test.go index 1b7f6d3b3dd..8609f7f3746 100644 --- a/pkg/bmt/proof_test.go +++ b/pkg/bmt/proof_test.go @@ -79,9 +79,9 @@ func TestProofCorrectness(t *testing.T) { "887c22bd8750d34016ac3c66b5ff102dacdd73f6b014e710b51e8022af9a1968", } - verifySegments(t, expSegmentStrings, proof.ProofSegments) + verifySegments(t, expSegmentStrings, proof.Sisters) - if !bytes.Equal(proof.ProveSegment, testDataPadded[:hh.Size()]) { + if !bytes.Equal(proof.Data, testDataPadded[:hh.Size()]) { t.Fatal("section incorrect") } @@ -105,9 +105,9 @@ func TestProofCorrectness(t *testing.T) { "745bae095b6ff5416b4a351a167f731db6d6f5924f30cd88d48e74261795d27b", } - verifySegments(t, expSegmentStrings, proof.ProofSegments) + verifySegments(t, expSegmentStrings, proof.Sisters) - if !bytes.Equal(proof.ProveSegment, testDataPadded[127*hh.Size():]) { + if !bytes.Equal(proof.Data, testDataPadded[127*hh.Size():]) { t.Fatal("section incorrect") } @@ -131,9 +131,9 @@ func TestProofCorrectness(t *testing.T) { "745bae095b6ff5416b4a351a167f731db6d6f5924f30cd88d48e74261795d27b", } - verifySegments(t, expSegmentStrings, proof.ProofSegments) + verifySegments(t, expSegmentStrings, proof.Sisters) - if !bytes.Equal(proof.ProveSegment, testDataPadded[64*hh.Size():65*hh.Size()]) { + if !bytes.Equal(proof.Data, testDataPadded[64*hh.Size():65*hh.Size()]) { t.Fatal("section incorrect") } @@ -167,9 +167,9 @@ func TestProofCorrectness(t *testing.T) { segment := testDataPadded[64*hh.Size() : 65*hh.Size()] rootHash, err := pr.Verify(64, bmt.Proof{ - ProveSegment: segment, - ProofSegments: segments, - Span: bmt.LengthToSpan(4096), + Data: segment, + Sisters: segments, + Span: bmt.LengthToSpan(4096), }) if err != nil { t.Fatal(err) diff --git a/pkg/node/node.go b/pkg/node/node.go index ddcdad37771..5288bce7d86 100644 --- a/pkg/node/node.go +++ b/pkg/node/node.go @@ -61,6 +61,7 @@ import ( "github.com/ethersphere/bee/pkg/steward" "github.com/ethersphere/bee/pkg/storageincentives" "github.com/ethersphere/bee/pkg/storageincentives/redistribution" + "github.com/ethersphere/bee/pkg/storageincentives/sampler" "github.com/ethersphere/bee/pkg/storageincentives/staking" storer "github.com/ethersphere/bee/pkg/storer" "github.com/ethersphere/bee/pkg/swarm" @@ -681,7 +682,6 @@ func NewBee( } postageStampContractABI := abiutil.MustParseABI(chainCfg.PostageStampABI) - bzzTokenAddress, err := postagecontract.LookupERC20Address(ctx, transactionService, postageStampContractAddress, postageStampContractABI, chainEnabled) if err != nil { return nil, err @@ -1051,30 +1051,31 @@ func NewBee( } redistributionContractAddress = common.HexToAddress(o.RedistributionContractAddress) } - isFullySynced := func() bool { return localStore.ReserveSize() >= reserveTreshold && pullerService.SyncRate() == 0 } redistributionContract := redistribution.New(swarmAddress, logger, transactionService, redistributionContractAddress, abiutil.MustParseABI(chainCfg.RedistributionABI)) + s := sampler.New( + chainBackend, + batchStore, + localStore, + isFullySynced, + saludService.IsHealthy, + ) agent, err = storageincentives.New( + logger, swarmAddress, overlayEthAddress, + o.BlockTime, chainBackend, redistributionContract, postageStampContractService, stakingContract, - localStore, - isFullySynced, - o.BlockTime, - storageincentives.DefaultBlocksPerRound, - storageincentives.DefaultBlocksPerPhase, + s, stateStore, - batchStore, erc20Service, transactionService, - saludService, - logger, ) if err != nil { return nil, fmt.Errorf("storage incentives agent: %w", err) diff --git a/pkg/postage/batchstore/mock/store.go b/pkg/postage/batchstore/mock/store.go index f3c10bf236d..9e06f7d8e7a 100644 --- a/pkg/postage/batchstore/mock/store.go +++ b/pkg/postage/batchstore/mock/store.go @@ -144,6 +144,13 @@ func (bs *BatchStore) Iterate(f func(*postage.Batch) (bool, error)) error { return err } +// IterateByValue mocks the IterateByvalue method from the BatchStore +func (bs *BatchStore) IterateByValue(f func([]byte, *big.Int) (bool, error)) error { + return bs.Iterate(func(b *postage.Batch) (bool, error) { + return f(b.ID, b.Value) + }) +} + // Save mocks the Save method from the BatchStore. func (bs *BatchStore) Save(batch *postage.Batch) error { bs.mtx.Lock() diff --git a/pkg/postage/batchstore/store.go b/pkg/postage/batchstore/store.go index 99069292ea3..466f3ca1d3e 100644 --- a/pkg/postage/batchstore/store.go +++ b/pkg/postage/batchstore/store.go @@ -126,6 +126,13 @@ func (s *store) Iterate(cb func(*postage.Batch) (bool, error)) error { }) } +// IterateByValue iterates on batches by value ascending order +func (s *store) IterateByValue(cb func(id []byte, val *big.Int) (bool, error)) error { + return s.store.Iterate(valueKeyPrefix, func(key, _ []byte) (bool, error) { + return cb(valueKeyToID(key), valueKeyToValue(key)) + }) +} + // Save is implementation of postage.Storer interface Save method. // This method has side effects; it also updates the radius of the node if successful. func (s *store) Save(batch *postage.Batch) error { @@ -269,44 +276,38 @@ func (s *store) saveBatch(b *postage.Batch) error { // cleanup evicts and removes expired batch. // Must be called under lock. func (s *store) cleanup() error { + var ids [][]byte + var keys []string + totalAmount := s.cs.Load().TotalAmount - var evictions []*postage.Batch - - err := s.store.Iterate(valueKeyPrefix, func(key, value []byte) (stop bool, err error) { - - b, err := s.Get(valueKeyToID(key)) - if err != nil { - return false, err - } - + err := s.IterateByValue(func(id []byte, val *big.Int) (bool, error) { // batches whose balance is below the total cumulative payout - if b.Value.Cmp(s.cs.Load().TotalAmount) <= 0 { - evictions = append(evictions, b) - } else { - return true, nil // stop early as an optimization at first value above the total cumulative payout + if val.Cmp(totalAmount) <= 0 { + ids = append(ids, id) + keys = append(keys, valueKey(val, id)) + return false, nil } - - return false, nil + return true, nil // stop early as an optimization at first value above the total cumulative payout }) if err != nil { return err } - for _, b := range evictions { - err := s.store.Delete(valueKey(b.Value, b.ID)) + for i, id := range ids { + err := s.store.Delete(keys[i]) if err != nil { - return fmt.Errorf("delete value key for batch %x: %w", b.ID, err) + return fmt.Errorf("delete value key for batch %x: %w", id, err) } - err = s.store.Delete(batchKey(b.ID)) + err = s.store.Delete(batchKey(id)) if err != nil { - return fmt.Errorf("delete batch %x: %w", b.ID, err) + return fmt.Errorf("delete batch %x: %w", id, err) } - err = s.evictFn(b.ID) + err = s.evictFn(id) if err != nil { - return fmt.Errorf("evict batch %x: %w", b.ID, err) + return fmt.Errorf("evict batch %x: %w", id, err) } if s.batchExpiry != nil { - s.batchExpiry.HandleStampExpiry(b.ID) + s.batchExpiry.HandleStampExpiry(id) } } @@ -373,6 +374,11 @@ func valueKeyToID(key []byte) []byte { return key[l-32 : l] } +// valueKeyToValue extracts the value from a value key - used in value-based iteration. +func valueKeyToValue(key []byte) *big.Int { + return new(big.Int).SetBytes(key[:32]) +} + func (s *store) SetBatchExpiryHandler(be postage.BatchExpiryHandler) { s.batchExpiry = be } diff --git a/pkg/postage/interface.go b/pkg/postage/interface.go index 38660c13b48..9612a5f1086 100644 --- a/pkg/postage/interface.go +++ b/pkg/postage/interface.go @@ -37,12 +37,19 @@ type ChainSnapshot struct { FirstBlockNumber uint64 `json:"firstBlockNumber"` Timestamp int64 `json:"timestamp"` } +type ChainBackend interface { + BlockNumber(context.Context) (uint64, error) + HeaderByNumber(context.Context, *big.Int) (*types.Header, error) + BalanceAt(ctx context.Context, address common.Address, block *big.Int) (*big.Int, error) + SuggestGasPrice(ctx context.Context) (*big.Int, error) +} // Storer represents the persistence layer for batches // on the current (highest available) block. type Storer interface { ChainStateGetter CommitmentGetter + ValueIterator Radius() uint8 @@ -84,6 +91,11 @@ type ChainStateGetter interface { GetChainState() *ChainState } +type ValueIterator interface { + // IterateByValue iterates on batches by value ascending order + IterateByValue(cb func(id []byte, val *big.Int) (bool, error)) error +} + // Listener provides a blockchain event iterator. type Listener interface { io.Closer diff --git a/pkg/postage/noop.go b/pkg/postage/noop.go index ea8387deee8..1b3c331f8ba 100644 --- a/pkg/postage/noop.go +++ b/pkg/postage/noop.go @@ -26,6 +26,8 @@ func (b *NoOpBatchStore) Exists([]byte) (bool, error) { return false, nil } func (b *NoOpBatchStore) Iterate(func(*Batch) (bool, error)) error { return nil } +func (b *NoOpBatchStore) IterateByValue(func([]byte, *big.Int) (bool, error)) error { return nil } + func (b *NoOpBatchStore) Save(*Batch) error { return nil } func (b *NoOpBatchStore) Update(*Batch, *big.Int, uint8) error { return nil } diff --git a/pkg/storageincentives/agent.go b/pkg/storageincentives/agent.go index 51a62393a8c..a0669e97875 100644 --- a/pkg/storageincentives/agent.go +++ b/pkg/storageincentives/agent.go @@ -11,11 +11,9 @@ import ( "fmt" "io" "math/big" - "sync" "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" "github.com/ethersphere/bee/pkg/crypto" "github.com/ethersphere/bee/pkg/log" "github.com/ethersphere/bee/pkg/postage" @@ -23,8 +21,8 @@ import ( "github.com/ethersphere/bee/pkg/settlement/swap/erc20" "github.com/ethersphere/bee/pkg/storage" "github.com/ethersphere/bee/pkg/storageincentives/redistribution" + "github.com/ethersphere/bee/pkg/storageincentives/sampler" "github.com/ethersphere/bee/pkg/storageincentives/staking" - "github.com/ethersphere/bee/pkg/storer" "github.com/ethersphere/bee/pkg/swarm" "github.com/ethersphere/bee/pkg/transaction" ) @@ -32,9 +30,8 @@ import ( const loggerName = "storageincentives" const ( - DefaultBlocksPerRound = 152 - DefaultBlocksPerPhase = DefaultBlocksPerRound / 4 - + blocksPerRound = 152 + blocksPerPhase = blocksPerRound / 4 // min # of transactions our wallet should be able to cover minTxCountToCover = 15 @@ -42,70 +39,62 @@ const ( avgTxGas = 250_000 ) -type ChainBackend interface { - BlockNumber(context.Context) (uint64, error) - HeaderByNumber(context.Context, *big.Int) (*types.Header, error) - BalanceAt(ctx context.Context, address common.Address, block *big.Int) (*big.Int, error) - SuggestGasPrice(ctx context.Context) (*big.Int, error) -} - -type Health interface { - IsHealthy() bool -} - type Agent struct { logger log.Logger metrics metrics - backend ChainBackend - blocksPerRound uint64 + overlay swarm.Address + handlers []func(context.Context, uint64) error + backend postage.ChainBackend contract redistribution.Contract batchExpirer postagecontract.PostageBatchExpirer redistributionStatuser staking.RedistributionStatuser - store storer.Reserve - fullSyncedFunc func() bool - overlay swarm.Address + sampler *sampler.Sampler quit chan struct{} - wg sync.WaitGroup + stopped chan struct{} state *RedistributionState - chainStateGetter postage.ChainStateGetter - commitLock sync.Mutex - health Health } -func New(overlay swarm.Address, +func handlers(a *Agent) []func(context.Context, uint64) error { + return []func(context.Context, uint64) error{ + a.handleCommit, + a.handleReveal, + a.handleClaim, + a.handleSample, + } +} + +// handlersFunc is made available in tests +var handlersFunc func(a *Agent) []func(context.Context, uint64) error = handlers + +func New( + logger log.Logger, + overlay swarm.Address, ethAddress common.Address, - backend ChainBackend, + blockTime time.Duration, + blockbackend postage.ChainBackend, contract redistribution.Contract, batchExpirer postagecontract.PostageBatchExpirer, redistributionStatuser staking.RedistributionStatuser, - store storer.Reserve, - fullSyncedFunc func() bool, - blockTime time.Duration, - blocksPerRound, - blocksPerPhase uint64, + s *sampler.Sampler, stateStore storage.StateStorer, - chainStateGetter postage.ChainStateGetter, erc20Service erc20.Service, tranService transaction.Service, - health Health, - logger log.Logger, ) (*Agent, error) { a := &Agent{ - overlay: overlay, - metrics: newMetrics(), - backend: backend, logger: logger.WithName(loggerName).Register(), + metrics: newMetrics(), + overlay: overlay, + backend: blockbackend, + sampler: s, contract: contract, batchExpirer: batchExpirer, - store: store, - fullSyncedFunc: fullSyncedFunc, - blocksPerRound: blocksPerRound, - quit: make(chan struct{}), redistributionStatuser: redistributionStatuser, - health: health, - chainStateGetter: chainStateGetter, + quit: make(chan struct{}), + stopped: make(chan struct{}), } + a.handlers = handlersFunc(a) + state, err := NewRedistributionState(logger, ethAddress, stateStore, erc20Service, tranService) if err != nil { return nil, err @@ -113,155 +102,142 @@ func New(overlay swarm.Address, a.state = state - a.wg.Add(1) - go a.start(blockTime, a.blocksPerRound, blocksPerPhase) + go a.start(blockTime) return a, nil } +type phaseAndBlock struct { + phase PhaseType + block uint64 +} + // start polls the current block number, calculates, and publishes only once the current phase. // Each round is blocksPerRound long and is divided into three blocksPerPhase long phases: commit, reveal, claim. // The sample phase is triggered upon entering the claim phase and may run until the end of the commit phase. // If our neighborhood is selected to participate, a sample is created during the sample phase. In the commit phase, // the sample is submitted, and in the reveal phase, the obfuscation key from the commit phase is submitted. // Next, in the claim phase, we check if we've won, and the cycle repeats. The cycle must occur in the length of one round. -func (a *Agent) start(blockTime time.Duration, blocksPerRound, blocksPerPhase uint64) { - defer a.wg.Done() - - phaseEvents := newEvents() +func (a *Agent) start(blockTime time.Duration) { + defer close(a.stopped) + phaseEvents := NewEvents() defer phaseEvents.Close() - logErr := func(phase PhaseType, round uint64, err error) { - if err != nil { - a.logger.Error(err, "phase failed", "phase", phase, "round", round) - } - } - - phaseEvents.On(commit, func(ctx context.Context) { + phaseEvents.On(commit, func(quit chan struct{}, blockNumber uint64) { phaseEvents.Cancel(claim) - - round, _ := a.state.currentRoundAndPhase() - err := a.handleCommit(ctx, round) - logErr(commit, round, err) + a.handlePhase(commit, quit, blockNumber) }) - phaseEvents.On(reveal, func(ctx context.Context) { + phaseEvents.On(reveal, func(quit chan struct{}, blockNumber uint64) { phaseEvents.Cancel(commit, sample) - round, _ := a.state.currentRoundAndPhase() - logErr(reveal, round, a.handleReveal(ctx, round)) + a.handlePhase(reveal, quit, blockNumber) }) - phaseEvents.On(claim, func(ctx context.Context) { + phaseEvents.On(claim, func(quit chan struct{}, blockNumber uint64) { phaseEvents.Cancel(reveal) - phaseEvents.Publish(sample) - - round, _ := a.state.currentRoundAndPhase() - logErr(claim, round, a.handleClaim(ctx, round)) + phaseEvents.Publish(sample, blockNumber) + a.handlePhase(claim, quit, blockNumber) }) - phaseEvents.On(sample, func(ctx context.Context) { - round, _ := a.state.currentRoundAndPhase() - isPhasePlayed, err := a.handleSample(ctx, round) - logErr(sample, round, err) - - // Sample handled could potentially take long time, therefore it could overlap with commit - // phase of next round. When that case happens commit event needs to be triggered once more - // in order to handle commit phase with delay. - currentRound, currentPhase := a.state.currentRoundAndPhase() - if isPhasePlayed && - currentPhase == commit && - currentRound-1 == round { - phaseEvents.Publish(commit) - } + phaseEvents.On(sample, func(quit chan struct{}, blockNumber uint64) { + a.handlePhase(sample, quit, blockNumber) }) - - var ( - prevPhase PhaseType = -1 - currentPhase PhaseType - ) - - phaseCheck := func(ctx context.Context) { - ctx, cancel := context.WithTimeout(ctx, blockTime*time.Duration(blocksPerRound)) - defer cancel() - - a.metrics.BackendCalls.Inc() - block, err := a.backend.BlockNumber(ctx) - if err != nil { - a.metrics.BackendErrors.Inc() - a.logger.Error(err, "getting block number") - return - } - - a.state.SetCurrentBlock(block) - - round := block / blocksPerRound - - a.metrics.Round.Set(float64(round)) - - p := block % blocksPerRound - if p < blocksPerPhase { - currentPhase = commit // [0, 37] - } else if p >= blocksPerPhase && p < 2*blocksPerPhase { // [38, 75] - currentPhase = reveal - } else if p >= 2*blocksPerPhase { - currentPhase = claim // [76, 151] - } - - // write the current phase only once - if currentPhase == prevPhase { + phaseC := make(chan phaseAndBlock) + phaseCheckInterval := blockTime + // optimization, we do not need to check the phase change at every new block + if blocksPerPhase > 10 { + phaseCheckInterval = blockTime * 5 + } + ticker := time.NewTicker(phaseCheckInterval) + defer ticker.Stop() + var phase PhaseType + var block uint64 + ctx, cancel := context.WithCancel(context.Background()) + for { + select { + case pr := <-phaseC: + phase, block = pr.phase, pr.block + phaseEvents.Publish(phase, block) + round := block / blocksPerRound + + go a.metrics.CurrentPhase.Set(float64(phase)) + a.metrics.Round.Set(float64(round)) + + a.logger.Info("entered new phase", "phase", phase.String(), "round", round, "block", block) + a.state.SetCurrentBlock(block) + a.state.SetCurrentEvent(phase, round) + // a.state.SetFullySynced(a.fullSyncedFunc()) + // a.state.SetHealthy(a.healthyFunc()) + go a.state.purgeStaleRoundData() + + case <-ticker.C: + cancel() + ctx, cancel = context.WithCancel(context.Background()) + go func() { + defer cancel() + a.getPhase(ctx, phase, phaseC) + }() + + case <-a.quit: return } + } +} - prevPhase = currentPhase - a.metrics.CurrentPhase.Set(float64(currentPhase)) - - a.logger.Info("entered new phase", "phase", currentPhase.String(), "round", round, "block", block) - - a.state.SetCurrentEvent(currentPhase, round) - a.state.SetFullySynced(a.fullSyncedFunc()) - a.state.SetHealthy(a.health.IsHealthy()) - go a.state.purgeStaleRoundData() +func currentPhase(ctx context.Context, phase PhaseType, block uint64) (changed bool, newphase PhaseType) { + rem := block % blocksPerRound + newphase = phase + switch { + case rem < blocksPerPhase: // [0, 37] + newphase = commit + case rem >= blocksPerPhase && rem < 2*blocksPerPhase: // [38, 75] + newphase = reveal + default: + newphase = claim // [76, 151] + } + return newphase == phase, newphase +} - isFrozen, err := a.redistributionStatuser.IsOverlayFrozen(ctx, block) - if err != nil { - a.logger.Error(err, "error checking if stake is frozen") - } else { - a.state.SetFrozen(isFrozen, round) - } +// getPhase feeds the phase to the channel if changed +func (a *Agent) getPhase(ctx context.Context, phase PhaseType, phaseC chan phaseAndBlock) { + a.metrics.BackendCalls.Inc() + block, err := a.backend.BlockNumber(ctx) + if err != nil { + a.metrics.BackendErrors.Inc() + a.logger.Error(err, "getting block number from chain backend") + return + } - phaseEvents.Publish(currentPhase) + changed, phase := currentPhase(ctx, phase, block) + if !changed { + return + } + select { + case phaseC <- phaseAndBlock{phase, block}: + case <-ctx.Done(): } +} + +func (a *Agent) handlePhase(phase PhaseType, quit chan struct{}, blockNumber uint64) { + round := blockNumber / blocksPerRound + handler := a.handlers[phase] ctx, cancel := context.WithCancel(context.Background()) + defer cancel() go func() { - <-a.quit - cancel() - }() - - // manually invoke phaseCheck initially in order to set initial data asap - phaseCheck(ctx) - - phaseCheckInterval := blockTime - // optimization, we do not need to check the phase change at every new block - if blocksPerPhase > 10 { - phaseCheckInterval = blockTime * 5 - } - - for { select { + case <-quit: + cancel() case <-ctx.Done(): - return - case <-time.After(phaseCheckInterval): - phaseCheck(ctx) } + }() + err := handler(ctx, round) + if err != nil { + a.logger.Error(err, "phase failed", "phase", phase, "round", round) } } func (a *Agent) handleCommit(ctx context.Context, round uint64) error { - // commit event handler has to be guarded with lock to avoid - // race conditions when handler is triggered again from sample phase - a.commitLock.Lock() - defer a.commitLock.Unlock() if _, exists := a.state.CommitKey(round); exists { // already committed on this round, phase is skipped @@ -269,7 +245,7 @@ func (a *Agent) handleCommit(ctx context.Context, round uint64) error { } // the sample has to come from previous round to be able to commit it - sample, exists := a.state.SampleData(round - 1) + sample, exists := a.state.Data(round - 1) if !exists { // In absence of sample, phase is skipped return nil @@ -294,7 +270,7 @@ func (a *Agent) handleReveal(ctx context.Context, round uint64) error { } // reveal requires sample from previous round - sample, exists := a.state.SampleData(round - 1) + sample, exists := a.state.Data(round - 1) if !exists { // Sample must have been saved so far return fmt.Errorf("sample not found in reveal phase") @@ -302,8 +278,8 @@ func (a *Agent) handleReveal(ctx context.Context, round uint64) error { a.metrics.RevealPhase.Inc() - rsh := sample.ReserveSampleHash.Bytes() - txHash, err := a.contract.Reveal(ctx, sample.StorageRadius, rsh, commitKey) + rsh := sample.Hash.Bytes() + txHash, err := a.contract.Reveal(ctx, sample.Depth, rsh, commitKey) if err != nil { a.metrics.ErrReveal.Inc() return err @@ -353,17 +329,18 @@ func (a *Agent) handleClaim(ctx context.Context, round uint64) error { a.logger.Info("could not set balance", "err", err) } - sampleData, exists := a.state.SampleData(round - 1) + sampleData, exists := a.state.Data(round - 1) if !exists { return fmt.Errorf("sample not found") } - anchor2, err := a.contract.ReserveSalt(ctx) + anchor, err := a.contract.ReserveSalt(ctx) if err != nil { a.logger.Info("failed getting anchor after second reveal", "err", err) } - proofs, err := makeInclusionProofs(sampleData.ReserveSampleItems, sampleData.Anchor1, anchor2) + proofs, err := sampler.MakeProofs(sampleData, anchor) + if err != nil { return fmt.Errorf("making inclusion proofs: %w", err) } @@ -388,22 +365,28 @@ func (a *Agent) handleClaim(ctx context.Context, round uint64) error { return nil } -func (a *Agent) handleSample(ctx context.Context, round uint64) (bool, error) { - storageRadius := a.store.StorageRadius() +func (a *Agent) isPlaying(ctx context.Context, round uint64) (bool, uint8, error) { + isFrozen, err := a.redistributionStatuser.IsOverlayFrozen(ctx, round*blocksPerRound) + if err != nil { + a.logger.Error(err, "error checking if stake is frozen") + } else { + a.state.SetFrozen(isFrozen, round) + } if a.state.IsFrozen() { a.logger.Info("skipping round because node is frozen") - return false, nil + return false, 0, nil } - isPlaying, err := a.contract.IsPlaying(ctx, storageRadius) + depth := a.sampler.StorageRadius() + isPlaying, err := a.contract.IsPlaying(ctx, depth) if err != nil { a.metrics.ErrCheckIsPlaying.Inc() - return false, err + return false, 0, err } if !isPlaying { a.logger.Info("not playing in this round") - return false, nil + return false, 0, nil } a.state.SetLastSelectedRound(round + 1) a.metrics.NeighborhoodSelected.Inc() @@ -411,92 +394,50 @@ func (a *Agent) handleSample(ctx context.Context, round uint64) (bool, error) { if !a.state.IsFullySynced() { a.logger.Info("skipping round because node is not fully synced") - return false, nil + return false, 0, nil } if !a.state.IsHealthy() { a.logger.Info("skipping round because node is unhealhy", "round", round) - return false, nil + return false, 0, nil } _, hasFunds, err := a.HasEnoughFundsToPlay(ctx) if err != nil { - return false, fmt.Errorf("has enough funds to play: %w", err) - } else if !hasFunds { + return false, 0, fmt.Errorf("has enough funds to play: %w", err) + } + if !hasFunds { a.logger.Info("insufficient funds to play in next round", "round", round) a.metrics.InsufficientFundsToPlay.Inc() - return false, nil + return false, 0, nil } - - now := time.Now() - sample, err := a.makeSample(ctx, storageRadius) - if err != nil { - return false, err - } - dur := time.Since(now) - a.metrics.SampleDuration.Set(dur.Seconds()) - - a.logger.Info("produced sample", "hash", sample.ReserveSampleHash, "radius", sample.StorageRadius, "round", round) - - a.state.SetSampleData(round, sample, dur) - - return true, nil + return true, depth, nil } -func (a *Agent) makeSample(ctx context.Context, storageRadius uint8) (SampleData, error) { - salt, err := a.contract.ReserveSalt(ctx) - if err != nil { - return SampleData{}, err - } - - timeLimiter, err := a.getPreviousRoundTime(ctx) - if err != nil { - return SampleData{}, err +func (a *Agent) handleSample(ctx context.Context, round uint64) error { + isPlaying, depth, err := a.isPlaying(ctx, round) + if !isPlaying { + return nil } - rSample, err := a.store.ReserveSample(ctx, salt, storageRadius, uint64(timeLimiter), a.minBatchBalance()) + salt, err := a.contract.ReserveSalt(ctx) if err != nil { - return SampleData{}, err + return err } - sampleHash, err := sampleHash(rSample.Items) + sample, err := a.sampler.MakeSample(ctx, salt, depth, round) if err != nil { - return SampleData{}, err + return err } - sample := SampleData{ - Anchor1: salt, - ReserveSampleItems: rSample.Items, - ReserveSampleHash: sampleHash, - StorageRadius: storageRadius, - } + a.logger.Info("produced sample", "hash", sample.Hash, "depth", sample.Depth, "round", round) - return sample, nil -} + a.state.SetData(round, sample) -func (a *Agent) minBatchBalance() *big.Int { - cs := a.chainStateGetter.GetChainState() - nextRoundBlockNumber := ((a.state.currentBlock() / a.blocksPerRound) + 2) * a.blocksPerRound - difference := nextRoundBlockNumber - cs.Block - minBalance := new(big.Int).Add(cs.TotalAmount, new(big.Int).Mul(cs.CurrentPrice, big.NewInt(int64(difference)))) - - return minBalance -} - -func (a *Agent) getPreviousRoundTime(ctx context.Context) (time.Duration, error) { - previousRoundBlockNumber := ((a.state.currentBlock() / a.blocksPerRound) - 1) * a.blocksPerRound - - a.metrics.BackendCalls.Inc() - timeLimiterBlock, err := a.backend.HeaderByNumber(ctx, new(big.Int).SetUint64(previousRoundBlockNumber)) - if err != nil { - a.metrics.BackendErrors.Inc() - return 0, err - } - - return time.Duration(timeLimiterBlock.Time) * time.Second / time.Nanosecond, nil + return nil } -func (a *Agent) commit(ctx context.Context, sample SampleData, round uint64) error { +func (a *Agent) commit(ctx context.Context, sample sampler.Data, round uint64) error { a.metrics.CommitPhase.Inc() key := make([]byte, swarm.HashSize) @@ -504,8 +445,7 @@ func (a *Agent) commit(ctx context.Context, sample SampleData, round uint64) err return err } - rsh := sample.ReserveSampleHash.Bytes() - obfuscatedHash, err := a.wrapCommit(sample.StorageRadius, rsh, key) + obfuscatedHash, err := a.wrapCommit(sample.Depth, sample.Hash.Bytes(), key) if err != nil { return err } @@ -525,24 +465,17 @@ func (a *Agent) commit(ctx context.Context, sample SampleData, round uint64) err func (a *Agent) Close() error { close(a.quit) - stopped := make(chan struct{}) - go func() { - a.wg.Wait() - close(stopped) - }() - select { - case <-stopped: + case <-a.stopped: return nil case <-time.After(5 * time.Second): return errors.New("stopping incentives with ongoing worker goroutine") } } -func (a *Agent) wrapCommit(storageRadius uint8, sample []byte, key []byte) ([]byte, error) { - storageRadiusByte := []byte{storageRadius} - - data := append(a.overlay.Bytes(), storageRadiusByte...) +func (a *Agent) wrapCommit(depth uint8, sample []byte, key []byte) ([]byte, error) { + data := append([]byte{}, a.overlay.Bytes()...) + data = append(data, depth) data = append(data, sample...) data = append(data, key...) @@ -554,48 +487,6 @@ func (a *Agent) Status() (*Status, error) { return a.state.Status() } -type SampleWithProofs struct { - Hash swarm.Address `json:"hash"` - Proofs redistribution.ChunkInclusionProofs `json:"proofs"` - Duration time.Duration `json:"duration"` -} - -// SampleWithProofs is called only by rchash API -func (a *Agent) SampleWithProofs( - ctx context.Context, - anchor1 []byte, - anchor2 []byte, - storageRadius uint8, -) (SampleWithProofs, error) { - sampleStartTime := time.Now() - - timeLimiter, err := a.getPreviousRoundTime(ctx) - if err != nil { - return SampleWithProofs{}, err - } - - rSample, err := a.store.ReserveSample(ctx, anchor1, storageRadius, uint64(timeLimiter), a.minBatchBalance()) - if err != nil { - return SampleWithProofs{}, err - } - - hash, err := sampleHash(rSample.Items) - if err != nil { - return SampleWithProofs{}, fmt.Errorf("sample hash: %w", err) - } - - proofs, err := makeInclusionProofs(rSample.Items, anchor1, anchor2) - if err != nil { - return SampleWithProofs{}, fmt.Errorf("make proofs: %w", err) - } - - return SampleWithProofs{ - Hash: hash, - Proofs: proofs, - Duration: time.Since(sampleStartTime), - }, nil -} - func (a *Agent) HasEnoughFundsToPlay(ctx context.Context) (*big.Int, bool, error) { balance, err := a.backend.BalanceAt(ctx, a.state.ethAddress, nil) if err != nil { @@ -609,6 +500,5 @@ func (a *Agent) HasEnoughFundsToPlay(ctx context.Context) (*big.Int, bool, error avgTxFee := new(big.Int).Mul(big.NewInt(avgTxGas), price) minBalance := new(big.Int).Mul(avgTxFee, big.NewInt(minTxCountToCover)) - return minBalance, balance.Cmp(minBalance) >= 1, nil } diff --git a/pkg/storageincentives/agent_test.go b/pkg/storageincentives/agent_test.go index 70ec3193975..d0951479789 100644 --- a/pkg/storageincentives/agent_test.go +++ b/pkg/storageincentives/agent_test.go @@ -21,9 +21,8 @@ import ( statestore "github.com/ethersphere/bee/pkg/statestore/mock" "github.com/ethersphere/bee/pkg/storageincentives" "github.com/ethersphere/bee/pkg/storageincentives/redistribution" + "github.com/ethersphere/bee/pkg/storageincentives/sampler" "github.com/ethersphere/bee/pkg/storageincentives/staking/mock" - "github.com/ethersphere/bee/pkg/storer" - resMock "github.com/ethersphere/bee/pkg/storer/mock" "github.com/ethersphere/bee/pkg/swarm" transactionmock "github.com/ethersphere/bee/pkg/transaction/mock" "github.com/ethersphere/bee/pkg/util/testutil" @@ -34,55 +33,43 @@ func TestAgent(t *testing.T) { bigBalance := big.NewInt(4_000_000_000) tests := []struct { - name string - blocksPerRound uint64 - blocksPerPhase uint64 - incrementBy uint64 - limit uint64 - expectedCalls bool - balance *big.Int + name string + incrementBy uint64 + limit uint64 + expectedCalls bool + balance *big.Int }{{ - name: "3 blocks per phase, same block number returns twice", - blocksPerRound: 9, - blocksPerPhase: 3, - incrementBy: 1, - expectedCalls: true, - limit: 108, // computed with blocksPerRound * (exptectedCalls + 2) - balance: bigBalance, + name: "3 blocks per phase, same block number returns twice", + incrementBy: 1, + expectedCalls: true, + limit: 108, // computed with blocksPerRound * (exptectedCalls + 2) + balance: bigBalance, }, { - name: "3 blocks per phase, block number returns every block", - blocksPerRound: 9, - blocksPerPhase: 3, - incrementBy: 1, - expectedCalls: true, - limit: 108, - balance: bigBalance, + name: "3 blocks per phase, block number returns every block", + incrementBy: 1, + expectedCalls: true, + limit: 108, + balance: bigBalance, }, { - name: "no expected calls - block number returns late after each phase", - blocksPerRound: 9, - blocksPerPhase: 3, - incrementBy: 6, - expectedCalls: false, - limit: 108, - balance: bigBalance, + name: "no expected calls - block number returns late after each phase", + incrementBy: 6, + expectedCalls: false, + limit: 108, + balance: bigBalance, }, { - name: "4 blocks per phase, block number returns every other block", - blocksPerRound: 12, - blocksPerPhase: 4, - incrementBy: 2, - expectedCalls: true, - limit: 144, - balance: bigBalance, + name: "4 blocks per phase, block number returns every other block", + incrementBy: 2, + expectedCalls: true, + limit: 144, + balance: bigBalance, }, { // This test case is based on previous, but this time agent will not have enough // balance to participate in the game so no calls are going to be made. - name: "no expected calls - insufficient balance", - blocksPerRound: 12, - blocksPerPhase: 4, - incrementBy: 2, - expectedCalls: false, - limit: 144, - balance: big.NewInt(0), + name: "no expected calls - insufficient balance", + incrementBy: 2, + expectedCalls: false, + limit: 144, + balance: big.NewInt(0), }, } @@ -103,12 +90,11 @@ func TestAgent(t *testing.T) { } }, incrementBy: tc.incrementBy, - block: tc.blocksPerRound, balance: tc.balance, } contract := &mockContract{} - service, _ := createService(t, addr, backend, contract, tc.blocksPerRound, tc.blocksPerPhase) + service, _ := createService(t, addr, backend, contract) testutil.CleanupCloser(t, service) <-wait @@ -153,10 +139,9 @@ func TestAgent(t *testing.T) { func createService( t *testing.T, addr swarm.Address, - backend storageincentives.ChainBackend, + backend postage.ChainBackend, contract redistribution.Contract, - blocksPerRound uint64, - blocksPerPhase uint64) (*storageincentives.Agent, error) { +) (*storageincentives.Agent, error) { t.Helper() postageContract := contractMock.New(contractMock.WithExpiresBatchesFunc(func(context.Context) error { @@ -167,29 +152,22 @@ func createService( return false, nil })) - reserve := resMock.NewReserve( - resMock.WithRadius(0), - resMock.WithSample(storer.RandSample(t, nil)), - ) - + var sampler *sampler.Sampler return storageincentives.New( - addr, common.Address{}, + log.Noop, + addr, + common.Address{}, + time.Millisecond*100, backend, contract, postageContract, stakingContract, - reserve, - func() bool { return true }, - time.Millisecond*100, - blocksPerRound, - blocksPerPhase, + sampler, statestore.NewStateStore(), - &postage.NoOpBatchStore{}, erc20mock.New(), transactionmock.New(), - &mockHealth{}, - log.Noop, ) + } type mockchainBackend struct { @@ -276,7 +254,7 @@ func (m *mockContract) IsWinner(context.Context) (bool, error) { return false, nil } -func (m *mockContract) Claim(context.Context, redistribution.ChunkInclusionProofs) (common.Hash, error) { +func (m *mockContract) Claim(context.Context, []redistribution.Proof) (common.Hash, error) { m.mtx.Lock() defer m.mtx.Unlock() m.callsList = append(m.callsList, claimCall) @@ -296,7 +274,3 @@ func (m *mockContract) Reveal(context.Context, uint8, []byte, []byte) (common.Ha m.callsList = append(m.callsList, revealCall) return common.Hash{}, nil } - -type mockHealth struct{} - -func (m *mockHealth) IsHealthy() bool { return true } diff --git a/pkg/storageincentives/events.go b/pkg/storageincentives/events.go index f8640405abc..bc5ede4676d 100644 --- a/pkg/storageincentives/events.go +++ b/pkg/storageincentives/events.go @@ -5,7 +5,6 @@ package storageincentives import ( - "context" "sync" ) @@ -16,6 +15,7 @@ const ( reveal claim sample + phaseCount ) func (p PhaseType) String() string { @@ -34,42 +34,34 @@ func (p PhaseType) String() string { } type events struct { - mtx sync.Mutex - ev map[PhaseType]*event + mtx sync.Mutex + events [phaseCount]event + quit [phaseCount]chan struct{} } -type event struct { - funcs []func(context.Context) - ctx context.Context - cancel context.CancelFunc -} +type event []func(quit chan struct{}, blockNumber uint64) -func newEvents() *events { - return &events{ - ev: make(map[PhaseType]*event), - } +func NewEvents() *events { + return &events{} } -func (e *events) On(phase PhaseType, f func(context.Context)) { +func (e *events) On(phase PhaseType, f func(quit chan struct{}, blockNumber uint64)) { e.mtx.Lock() defer e.mtx.Unlock() - if _, ok := e.ev[phase]; !ok { - ctx, cancel := context.WithCancel(context.Background()) - e.ev[phase] = &event{ctx: ctx, cancel: cancel} + fs := e.events[phase] + if len(fs) == 0 { + e.quit[phase] = make(chan struct{}) } - - e.ev[phase].funcs = append(e.ev[phase].funcs, f) + e.events[phase] = append(fs, f) } -func (e *events) Publish(phase PhaseType) { +func (e *events) Publish(phase PhaseType, blockNumber uint64) { e.mtx.Lock() defer e.mtx.Unlock() - if ev, ok := e.ev[phase]; ok { - for _, v := range ev.funcs { - go v(ev.ctx) - } + for _, v := range e.events[phase] { + go v(e.quit[phase], blockNumber) } } @@ -78,12 +70,8 @@ func (e *events) Cancel(phases ...PhaseType) { defer e.mtx.Unlock() for _, phase := range phases { - if ev, ok := e.ev[phase]; ok { - ev.cancel() - ctx, cancel := context.WithCancel(context.Background()) - ev.ctx = ctx - ev.cancel = cancel - } + close(e.quit[phase]) + e.quit[phase] = make(chan struct{}) } } @@ -91,8 +79,7 @@ func (e *events) Close() { e.mtx.Lock() defer e.mtx.Unlock() - for k, ev := range e.ev { - ev.cancel() - delete(e.ev, k) + for _, ch := range e.quit { + close(ch) } } diff --git a/pkg/storageincentives/events_test.go b/pkg/storageincentives/events_test.go index 6c40fe8fc2e..3c0ec8f3f11 100644 --- a/pkg/storageincentives/events_test.go +++ b/pkg/storageincentives/events_test.go @@ -5,7 +5,6 @@ package storageincentives_test import ( - "context" "testing" "time" @@ -21,33 +20,31 @@ func TestClose(t *testing.T) { done2 := make(chan struct{}) done3 := make(chan struct{}) - ev.On(1, func(ctx context.Context) { - <-ctx.Done() + ev.On(1, func(quit chan struct{}, blockNumber uint64) { + <-quit close(done1) }) - ev.On(1, func(ctx context.Context) { - <-ctx.Done() + ev.On(1, func(quit chan struct{}, blockNumber uint64) { + <-quit close(done2) }) - ev.On(2, func(ctx context.Context) { - <-ctx.Done() + ev.On(2, func(quit chan struct{}, blockNumber uint64) { + <-quit close(done3) }) - ev.Publish(1) - ev.Publish(2) + ev.Publish(1, 0) + ev.Publish(2, 0) ev.Close() - for i := 0; i < 3; i++ { + for i, c := range []chan struct{}{done1, done2, done3} { select { - case <-done1: - case <-done2: - case <-done3: + case <-c: case <-time.After(time.Second): - t.Fatal("timeout") + t.Fatalf("timeout waiting for process %d to quit", i) } } } @@ -59,35 +56,41 @@ func TestPhaseCancel(t *testing.T) { done1 := make(chan struct{}) done2 := make(chan struct{}) + done3 := make(chan struct{}) defer ev.Close() // ensure no panics occur on an empty publish - ev.Publish(0) + ev.Publish(0, 0) - ev.On(1, func(ctx context.Context) { - <-ctx.Done() + ev.On(1, func(quit chan struct{}, blockNumber uint64) { + <-quit close(done1) }) - ev.On(2, func(ctx context.Context) { - <-ctx.Done() + ev.On(2, func(quit chan struct{}, blockNumber uint64) { + <-quit close(done2) }) - ev.On(3, func(ctx context.Context) { - ev.Cancel(1, 2) + ev.On(3, func(quit chan struct{}, blockNumber uint64) { + }) - ev.Publish(1) - ev.Publish(2) - ev.Publish(3) + ev.Publish(1, 0) + ev.Publish(2, 0) + ev.Publish(3, 0) + ev.Cancel(1, 2) - for i := 0; i < 2; i++ { + for i, c := range []chan struct{}{done1, done2} { select { - case <-done1: - case <-done2: + case <-c: case <-time.After(time.Second): - t.Fatal("timeout") + t.Fatalf("timeout waiting for process %d to quit", i) } } + select { + case <-done3: + t.Fatalf("process 3 quit unexpectedly") + case <-time.After(50 * time.Millisecond): + } } diff --git a/pkg/storageincentives/export_test.go b/pkg/storageincentives/export_test.go deleted file mode 100644 index f617501ba29..00000000000 --- a/pkg/storageincentives/export_test.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2022 The Swarm Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package storageincentives - -var ( - NewEvents = newEvents - SampleChunk = sampleChunk - MakeInclusionProofs = makeInclusionProofs -) diff --git a/pkg/storageincentives/proof.go b/pkg/storageincentives/proof.go deleted file mode 100644 index ccc04cd935b..00000000000 --- a/pkg/storageincentives/proof.go +++ /dev/null @@ -1,219 +0,0 @@ -// Copyright 2023 The Swarm Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package storageincentives - -import ( - "errors" - "fmt" - "hash" - "math/big" - - "github.com/ethersphere/bee/pkg/bmt" - "github.com/ethersphere/bee/pkg/bmtpool" - "github.com/ethersphere/bee/pkg/cac" - "github.com/ethersphere/bee/pkg/soc" - "github.com/ethersphere/bee/pkg/storageincentives/redistribution" - storer "github.com/ethersphere/bee/pkg/storer" - "github.com/ethersphere/bee/pkg/swarm" -) - -var errProofCreation = errors.New("reserve commitment hasher: failure in proof creation") - -// spanOffset returns the byte index of chunkdata where the spansize starts -func spanOffset(sampleItem storer.SampleItem) uint8 { - ch := swarm.NewChunk(sampleItem.ChunkAddress, sampleItem.ChunkData) - if soc.Valid(ch) { - return swarm.HashSize + swarm.SocSignatureSize - } - - return 0 -} - -// makeInclusionProofs creates transaction data for claim method. -// In the document this logic, result data, is also called Proof of entitlement (POE). -func makeInclusionProofs( - reserveSampleItems []storer.SampleItem, - anchor1 []byte, - anchor2 []byte, -) (redistribution.ChunkInclusionProofs, error) { - if len(reserveSampleItems) != storer.SampleSize { - return redistribution.ChunkInclusionProofs{}, fmt.Errorf("reserve sample items should have %d elements", storer.SampleSize) - } - if len(anchor1) == 0 { - return redistribution.ChunkInclusionProofs{}, errors.New("anchor1 is not set") - } - if len(anchor2) == 0 { - return redistribution.ChunkInclusionProofs{}, errors.New("anchor2 is not set") - } - - require3 := storer.SampleSize - 1 - require1 := new(big.Int).Mod(new(big.Int).SetBytes(anchor2), big.NewInt(int64(require3))).Uint64() - require2 := new(big.Int).Mod(new(big.Int).SetBytes(anchor2), big.NewInt(int64(require3-1))).Uint64() - if require2 >= require1 { - require2++ - } - - prefixHasherFactory := func() hash.Hash { - return swarm.NewPrefixHasher(anchor1) - } - prefixHasherPool := bmt.NewPool(bmt.NewConf(prefixHasherFactory, swarm.BmtBranches, 8)) - - // Sample chunk proofs - rccontent := bmt.Prover{Hasher: bmtpool.Get()} - rccontent.SetHeaderInt64(swarm.HashSize * storer.SampleSize * 2) - rsc, err := sampleChunk(reserveSampleItems) - if err != nil { - return redistribution.ChunkInclusionProofs{}, errProofCreation - } - rscData := rsc.Data() - _, err = rccontent.Write(rscData[swarm.SpanSize:]) - if err != nil { - return redistribution.ChunkInclusionProofs{}, errProofCreation - } - _, err = rccontent.Hash(nil) - if err != nil { - return redistribution.ChunkInclusionProofs{}, errProofCreation - } - proof1p1 := rccontent.Proof(int(require1) * 2) - proof2p1 := rccontent.Proof(int(require2) * 2) - proofLastp1 := rccontent.Proof(require3 * 2) - bmtpool.Put(rccontent.Hasher) - - // Witness1 proofs - segmentIndex := int(new(big.Int).Mod(new(big.Int).SetBytes(anchor2), big.NewInt(int64(128))).Uint64()) - // OG chunk proof - chunk1Content := bmt.Prover{Hasher: bmtpool.Get()} - chunk1Offset := spanOffset(reserveSampleItems[require1]) - chunk1Content.SetHeader(reserveSampleItems[require1].ChunkData[chunk1Offset : chunk1Offset+swarm.SpanSize]) - chunk1ContentPayload := reserveSampleItems[require1].ChunkData[chunk1Offset+swarm.SpanSize:] - _, err = chunk1Content.Write(chunk1ContentPayload) - if err != nil { - return redistribution.ChunkInclusionProofs{}, errProofCreation - } - _, err = chunk1Content.Hash(nil) - if err != nil { - return redistribution.ChunkInclusionProofs{}, errProofCreation - } - proof1p2 := chunk1Content.Proof(segmentIndex) - // TR chunk proof - chunk1TrContent := bmt.Prover{Hasher: prefixHasherPool.Get()} - chunk1TrContent.SetHeader(reserveSampleItems[require1].ChunkData[chunk1Offset : chunk1Offset+swarm.SpanSize]) - _, err = chunk1TrContent.Write(chunk1ContentPayload) - if err != nil { - return redistribution.ChunkInclusionProofs{}, errProofCreation - } - _, err = chunk1TrContent.Hash(nil) - if err != nil { - return redistribution.ChunkInclusionProofs{}, errProofCreation - } - proof1p3 := chunk1TrContent.Proof(segmentIndex) - // cleanup - bmtpool.Put(chunk1Content.Hasher) - prefixHasherPool.Put(chunk1TrContent.Hasher) - - // Witness2 proofs - // OG Chunk proof - chunk2Offset := spanOffset(reserveSampleItems[require2]) - chunk2Content := bmt.Prover{Hasher: bmtpool.Get()} - chunk2ContentPayload := reserveSampleItems[require2].ChunkData[chunk2Offset+swarm.SpanSize:] - chunk2Content.SetHeader(reserveSampleItems[require2].ChunkData[chunk2Offset : chunk2Offset+swarm.SpanSize]) - _, err = chunk2Content.Write(chunk2ContentPayload) - if err != nil { - return redistribution.ChunkInclusionProofs{}, errProofCreation - } - _, err = chunk2Content.Hash(nil) - if err != nil { - return redistribution.ChunkInclusionProofs{}, errProofCreation - } - proof2p2 := chunk2Content.Proof(segmentIndex) - // TR Chunk proof - chunk2TrContent := bmt.Prover{Hasher: prefixHasherPool.Get()} - chunk2TrContent.SetHeader(reserveSampleItems[require2].ChunkData[chunk2Offset : chunk2Offset+swarm.SpanSize]) - _, err = chunk2TrContent.Write(chunk2ContentPayload) - if err != nil { - return redistribution.ChunkInclusionProofs{}, errProofCreation - } - _, err = chunk2TrContent.Hash(nil) - if err != nil { - return redistribution.ChunkInclusionProofs{}, errProofCreation - } - proof2p3 := chunk2TrContent.Proof(segmentIndex) - // cleanup - bmtpool.Put(chunk2Content.Hasher) - prefixHasherPool.Put(chunk2TrContent.Hasher) - - // Witness3 proofs - // OG Chunk proof - chunkLastOffset := spanOffset(reserveSampleItems[require3]) - chunkLastContent := bmt.Prover{Hasher: bmtpool.Get()} - chunkLastContent.SetHeader(reserveSampleItems[require3].ChunkData[chunkLastOffset : chunkLastOffset+swarm.SpanSize]) - chunkLastContentPayload := reserveSampleItems[require3].ChunkData[chunkLastOffset+swarm.SpanSize:] - _, err = chunkLastContent.Write(chunkLastContentPayload) - if err != nil { - return redistribution.ChunkInclusionProofs{}, errProofCreation - } - _, err = chunkLastContent.Hash(nil) - if err != nil { - return redistribution.ChunkInclusionProofs{}, errProofCreation - } - proofLastp2 := chunkLastContent.Proof(segmentIndex) - // TR Chunk Proof - chunkLastTrContent := bmt.Prover{Hasher: prefixHasherPool.Get()} - chunkLastTrContent.SetHeader(reserveSampleItems[require3].ChunkData[chunkLastOffset : chunkLastOffset+swarm.SpanSize]) - _, err = chunkLastTrContent.Write(chunkLastContentPayload) - if err != nil { - return redistribution.ChunkInclusionProofs{}, errProofCreation - } - _, err = chunkLastTrContent.Hash(nil) - if err != nil { - return redistribution.ChunkInclusionProofs{}, errProofCreation - } - proofLastp3 := chunkLastTrContent.Proof(segmentIndex) - // cleanup - bmtpool.Put(chunkLastContent.Hasher) - prefixHasherPool.Put(chunkLastTrContent.Hasher) - - // map to output and add SOC related data if it is necessary - A, err := redistribution.NewChunkInclusionProof(proof1p1, proof1p2, proof1p3, reserveSampleItems[require1]) - if err != nil { - return redistribution.ChunkInclusionProofs{}, err - } - B, err := redistribution.NewChunkInclusionProof(proof2p1, proof2p2, proof2p3, reserveSampleItems[require2]) - if err != nil { - return redistribution.ChunkInclusionProofs{}, err - } - C, err := redistribution.NewChunkInclusionProof(proofLastp1, proofLastp2, proofLastp3, reserveSampleItems[require3]) - if err != nil { - return redistribution.ChunkInclusionProofs{}, err - } - return redistribution.ChunkInclusionProofs{ - A: A, - B: B, - C: C, - }, nil -} - -func sampleChunk(items []storer.SampleItem) (swarm.Chunk, error) { - contentSize := len(items) * 2 * swarm.HashSize - - pos := 0 - content := make([]byte, contentSize) - for _, s := range items { - copy(content[pos:], s.ChunkAddress.Bytes()) - pos += swarm.HashSize - copy(content[pos:], s.TransformedAddress.Bytes()) - pos += swarm.HashSize - } - - return cac.New(content) -} - -func sampleHash(items []storer.SampleItem) (swarm.Address, error) { - ch, err := sampleChunk(items) - if err != nil { - return swarm.ZeroAddress, err - } - return ch.Address(), nil -} diff --git a/pkg/storageincentives/proof_test.go b/pkg/storageincentives/proof_test.go deleted file mode 100644 index 9ee3745e1af..00000000000 --- a/pkg/storageincentives/proof_test.go +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright 2023 The Swarm Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package storageincentives_test - -import ( - "bytes" - _ "embed" - "encoding/json" - "fmt" - "math/big" - "testing" - - "github.com/ethersphere/bee/pkg/cac" - "github.com/ethersphere/bee/pkg/crypto" - "github.com/ethersphere/bee/pkg/postage" - postagetesting "github.com/ethersphere/bee/pkg/postage/testing" - "github.com/ethersphere/bee/pkg/soc" - "github.com/ethersphere/bee/pkg/storageincentives" - "github.com/ethersphere/bee/pkg/storageincentives/redistribution" - storer "github.com/ethersphere/bee/pkg/storer" - "github.com/ethersphere/bee/pkg/swarm" - "github.com/ethersphere/bee/pkg/util/testutil" - "github.com/google/go-cmp/cmp" -) - -// Test asserts valid case for MakeInclusionProofs. -func TestMakeInclusionProofs(t *testing.T) { - t.Parallel() - - anchor := testutil.RandBytes(t, 1) - sample := storer.RandSample(t, anchor) - - _, err := storageincentives.MakeInclusionProofs(sample.Items, anchor, anchor) - if err != nil { - t.Fatal(err) - } -} - -//go:embed testdata/inclusion-proofs.json -var testData []byte - -// Test asserts that MakeInclusionProofs will generate the same -// output for given sample. -func TestMakeInclusionProofsRegression(t *testing.T) { - t.Parallel() - - const sampleSize = 16 - - keyRaw := `00000000000000000000000000000000` - privKey, err := crypto.DecodeSecp256k1PrivateKey([]byte(keyRaw)) - if err != nil { - t.Fatal(err) - } - signer := crypto.NewDefaultSigner(privKey) - - stampID, _ := crypto.LegacyKeccak256([]byte("The Inverted Jenny")) - index := []byte{0, 0, 0, 0, 0, 8, 3, 3} - timestamp := []byte{0, 0, 0, 0, 0, 3, 3, 8} - stamper := func(addr swarm.Address) *postage.Stamp { - sig := postagetesting.MustNewValidSignature(signer, addr, stampID, index, timestamp) - return postage.NewStamp(stampID, index, timestamp, sig) - } - - anchor1 := big.NewInt(100).Bytes() - anchor2 := big.NewInt(30).Bytes() // this anchor will pick chunks 3, 6, 15 - - // generate chunks that will be used as sample - sampleChunks := make([]swarm.Chunk, 0, sampleSize) - for i := 0; i < sampleSize; i++ { - ch, err := cac.New([]byte(fmt.Sprintf("Unstoppable data! Chunk #%d", i+1))) - if err != nil { - t.Fatal(err) - } - - if i%2 == 0 { - id, err := crypto.LegacyKeccak256([]byte(fmt.Sprintf("ID #%d", i+1))) - if err != nil { - t.Fatal(err) - } - - socCh, err := soc.New(id, ch).Sign(signer) - if err != nil { - t.Fatal(err) - } - - ch = socCh - } - - ch = ch.WithStamp(stamper(ch.Address())) - - sampleChunks = append(sampleChunks, ch) - } - - // make sample from chunks - sample, err := storer.MakeSampleUsingChunks(sampleChunks, anchor1) - if err != nil { - t.Fatal(err) - } - - // assert that sample chunk hash/address does not change - sch, err := storageincentives.SampleChunk(sample.Items) - if err != nil { - t.Fatal(err) - } - if want := swarm.MustParseHexAddress("193bbea3dd0656d813c2c1e27b821f141286bbe6ab0dbf8e26fc7dd491e8f921"); !sch.Address().Equal(want) { - t.Fatalf("expecting sample chunk address %v, got %v", want, sch.Address()) - } - - // assert that inclusion proofs values does not change - proofs, err := storageincentives.MakeInclusionProofs(sample.Items, anchor1, anchor2) - if err != nil { - t.Fatal(err) - } - - var expectedProofs redistribution.ChunkInclusionProofs - - err = json.Unmarshal(testData, &expectedProofs) - if err != nil { - t.Fatal(err) - } - - if diff := cmp.Diff(proofs, expectedProofs); diff != "" { - t.Fatalf("unexpected inclusion proofs (-want +have):\n%s", diff) - } -} - -// Test asserts cases when MakeInclusionProofs should return error. -func TestMakeInclusionProofsExpectedError(t *testing.T) { - t.Parallel() - - t.Run("invalid sample length", func(t *testing.T) { - anchor := testutil.RandBytes(t, 8) - sample := storer.RandSample(t, anchor) - - _, err := storageincentives.MakeInclusionProofs(sample.Items[:1], anchor, anchor) - if err == nil { - t.Fatal("expecting error") - } - }) - - t.Run("empty anchor", func(t *testing.T) { - sample := storer.RandSample(t, []byte{}) - - _, err := storageincentives.MakeInclusionProofs(sample.Items[:1], []byte{}, []byte{}) - if err == nil { - t.Fatal("expecting error") - } - }) -} - -// Tests asserts that creating sample chunk is valid for all lengths [1-MaxSampleSize] -func TestSampleChunk(t *testing.T) { - t.Parallel() - - sample := storer.RandSample(t, nil) - - for i := 0; i < len(sample.Items); i++ { - items := sample.Items[:i] - - chunk, err := storageincentives.SampleChunk(items) - if err != nil { - t.Fatal(err) - } - - data := chunk.Data()[swarm.SpanSize:] - pos := 0 - for _, item := range items { - if !bytes.Equal(data[pos:pos+swarm.HashSize], item.ChunkAddress.Bytes()) { - t.Error("expected chunk address") - } - pos += swarm.HashSize - - if !bytes.Equal(data[pos:pos+swarm.HashSize], item.TransformedAddress.Bytes()) { - t.Error("expected transformed address") - } - pos += swarm.HashSize - } - - if !chunk.Address().IsValidNonEmpty() { - t.Error("address shouldn't be empty") - } - } -} - -// Tests asserts that creating sample chunk should fail because it will exceed -// capacity of chunk data. -func TestSampleChunkExpectedError(t *testing.T) { - t.Parallel() - - sampleItem := storer.RandSample(t, nil).Items[0] - - items := make([]storer.SampleItem, 65) - for i := range items { - items[i] = sampleItem - } - - _, err := storageincentives.SampleChunk(items) - if err == nil { - t.Fatal("expecting error") - } -} diff --git a/pkg/storageincentives/redistribution/abi_types.go b/pkg/storageincentives/redistribution/abi_types.go new file mode 100644 index 00000000000..1b3cfb2d8b9 --- /dev/null +++ b/pkg/storageincentives/redistribution/abi_types.go @@ -0,0 +1,87 @@ +// Copyright 2023 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Used for inclusion proof utilities + +package redistribution + +import ( + "encoding/binary" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethersphere/bee/pkg/bmt" + "github.com/ethersphere/bee/pkg/soc" + "github.com/ethersphere/bee/pkg/swarm" +) + +// Proof structure must exactly match +// corresponding structure (of the same name) in Redistribution.sol smart contract. +// github.com/ethersphere/storage-incentives/blob/master/src/Redistribution.sol +type Proof struct { + Sisters []common.Hash + Data common.Hash + Sisters2 []common.Hash + Data2 common.Hash + Sisters3 []common.Hash + ChunkSpan uint64 + PostageProof PostageProof + SocProof []SOCProof +} + +// SOCProof structure must exactly match +// corresponding structure (of the same name) in Redistribution.sol smart contract. +type PostageProof struct { + Signature []byte + BatchId common.Hash + Index uint64 + TimeStamp uint64 +} + +// SOCProof structure must exactly match +// corresponding structure (of the same name) in Redistribution.sol smart contract. +type SOCProof struct { + Signer []byte + Signature []byte + Identifier common.Hash + ChunkAddr common.Hash +} + +func bytes32(bs ...[]byte) []common.Hash { + bbs := make([]common.Hash, len(bs)) + for i, b := range bs { + var bb [32]byte + copy(bb[:], b) + bbs[i] = common.Hash(bb) + } + return bbs +} + +// NewProof transforms arguments to abi-compatible Proof object +func NewProof(wp1, wp2, wp3 bmt.Proof, stamp swarm.Stamp, sch *soc.SOC) Proof { + var socProof []SOCProof + if sch == nil { + socProof = []SOCProof{{ + Signer: sch.OwnerAddress(), + Signature: sch.Signature(), + Identifier: bytes32(sch.ID())[0], + ChunkAddr: bytes32(sch.WrappedChunk().Address().Bytes())[0], + }} + } + + return Proof{ + Sisters: bytes32(wp1.Sisters...), + Data: bytes32(wp1.Data)[0], + Sisters2: bytes32(wp2.Sisters...), + Data2: bytes32(wp2.Data)[0], + Sisters3: bytes32(wp3.Sisters...), + ChunkSpan: binary.LittleEndian.Uint64(wp2.Span[:swarm.SpanSize]), // should be uint64 on the other size; copied from pkg/api/bytes.go + PostageProof: PostageProof{ + Signature: stamp.Sig(), + BatchId: bytes32(stamp.BatchID())[0], + Index: binary.BigEndian.Uint64(stamp.Index()), + TimeStamp: binary.BigEndian.Uint64(stamp.Timestamp()), + }, + SocProof: socProof, + } +} diff --git a/pkg/storageincentives/redistribution/abi_types_test.go b/pkg/storageincentives/redistribution/abi_types_test.go new file mode 100644 index 00000000000..0ad3e6146c5 --- /dev/null +++ b/pkg/storageincentives/redistribution/abi_types_test.go @@ -0,0 +1,3 @@ +package redistribution_test + +// must include tests for just the contract calldata abi packaging \ No newline at end of file diff --git a/pkg/storageincentives/redistribution/inclusionproof.go b/pkg/storageincentives/redistribution/inclusionproof.go deleted file mode 100644 index b786d1d3001..00000000000 --- a/pkg/storageincentives/redistribution/inclusionproof.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright 2023 The Swarm Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package redistribution - -import ( - "encoding/binary" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethersphere/bee/pkg/bmt" - "github.com/ethersphere/bee/pkg/soc" - "github.com/ethersphere/bee/pkg/storer" - "github.com/ethersphere/bee/pkg/swarm" -) - -type ChunkInclusionProofs struct { - A ChunkInclusionProof `json:"proof1"` - B ChunkInclusionProof `json:"proof2"` - C ChunkInclusionProof `json:"proofLast"` -} - -// ChunkInclusionProof structure must exactly match -// corresponding structure (of the same name) in Redistribution.sol smart contract. -// github.com/ethersphere/storage-incentives/blob/ph_f2/src/Redistribution.sol -// github.com/ethersphere/storage-incentives/blob/master/src/Redistribution.sol (when merged to master) -type ChunkInclusionProof struct { - ProofSegments []common.Hash `json:"proofSegments"` - ProveSegment common.Hash `json:"proveSegment"` - ProofSegments2 []common.Hash `json:"proofSegments2"` - ProveSegment2 common.Hash `json:"proveSegment2"` - ChunkSpan uint64 `json:"chunkSpan"` - ProofSegments3 []common.Hash `json:"proofSegments3"` - PostageProof PostageProof `json:"postageProof"` - SocProof []SOCProof `json:"socProof"` -} - -// SOCProof structure must exactly match -// corresponding structure (of the same name) in Redistribution.sol smart contract. -type PostageProof struct { - Signature []byte `json:"signature"` - PostageId common.Hash `json:"postageId"` - Index uint64 `json:"index"` - TimeStamp uint64 `json:"timeStamp"` -} - -// SOCProof structure must exactly match -// corresponding structure (of the same name) in Redistribution.sol smart contract. -type SOCProof struct { - Signer common.Address `json:"signer"` - Signature []byte `json:"signature"` - Identifier common.Hash `json:"identifier"` - ChunkAddr common.Hash `json:"chunkAddr"` -} - -// NewChunkInclusionProof transforms arguments to ChunkInclusionProof object -func NewChunkInclusionProof(proofp1, proofp2, proofp3 bmt.Proof, sampleItem storer.SampleItem) (ChunkInclusionProof, error) { - socProof, err := makeSOCProof(sampleItem) - if err != nil { - return ChunkInclusionProof{}, err - } - - return ChunkInclusionProof{ - ProofSegments: toCommonHash(proofp1.ProofSegments), - ProveSegment: common.BytesToHash(proofp1.ProveSegment), - ProofSegments2: toCommonHash(proofp2.ProofSegments), - ProveSegment2: common.BytesToHash(proofp2.ProveSegment), - ChunkSpan: binary.LittleEndian.Uint64(proofp2.Span[:swarm.SpanSize]), // should be uint64 on the other size; copied from pkg/api/bytes.go - ProofSegments3: toCommonHash(proofp3.ProofSegments), - PostageProof: PostageProof{ - Signature: sampleItem.Stamp.Sig(), - PostageId: common.BytesToHash(sampleItem.Stamp.BatchID()), - Index: binary.BigEndian.Uint64(sampleItem.Stamp.Index()), - TimeStamp: binary.BigEndian.Uint64(sampleItem.Stamp.Timestamp()), - }, - SocProof: socProof, - }, nil -} - -func toCommonHash(hashes [][]byte) []common.Hash { - output := make([]common.Hash, len(hashes)) - for i, s := range hashes { - output[i] = common.BytesToHash(s) - } - return output -} - -func makeSOCProof(sampleItem storer.SampleItem) ([]SOCProof, error) { - ch := swarm.NewChunk(sampleItem.ChunkAddress, sampleItem.ChunkData) - if !soc.Valid(ch) { - return []SOCProof{}, nil - } - - socCh, err := soc.FromChunk(ch) - if err != nil { - return []SOCProof{}, err - } - - return []SOCProof{{ - Signer: common.BytesToAddress(socCh.OwnerAddress()), - Signature: socCh.Signature(), - Identifier: common.BytesToHash(socCh.ID()), - ChunkAddr: common.BytesToHash(socCh.WrappedChunk().Address().Bytes()), - }}, nil -} diff --git a/pkg/storageincentives/redistribution/redistribution.go b/pkg/storageincentives/redistribution/redistribution.go index 8b83537cf90..d63273434d4 100644 --- a/pkg/storageincentives/redistribution/redistribution.go +++ b/pkg/storageincentives/redistribution/redistribution.go @@ -23,7 +23,7 @@ type Contract interface { ReserveSalt(context.Context) ([]byte, error) IsPlaying(context.Context, uint8) (bool, error) IsWinner(context.Context) (bool, error) - Claim(context.Context, ChunkInclusionProofs) (common.Hash, error) + Claim(context.Context, []Proof) (common.Hash, error) Commit(context.Context, []byte, uint32) (common.Hash, error) Reveal(context.Context, uint8, []byte, []byte) (common.Hash, error) } @@ -92,8 +92,8 @@ func (c *contract) IsWinner(ctx context.Context) (isWinner bool, err error) { } // Claim sends a transaction to blockchain if a win is claimed. -func (c *contract) Claim(ctx context.Context, proofs ChunkInclusionProofs) (common.Hash, error) { - callData, err := c.incentivesContractABI.Pack("claim", proofs.A, proofs.B, proofs.C) +func (c *contract) Claim(ctx context.Context, proofs []Proof) (common.Hash, error) { + callData, err := c.incentivesContractABI.Pack("claim", proofs) if err != nil { return common.Hash{}, err } diff --git a/pkg/storageincentives/redistribution/redistribution_test.go b/pkg/storageincentives/redistribution/redistribution_test.go index 943d3013ba4..688c52c0689 100644 --- a/pkg/storageincentives/redistribution/redistribution_test.go +++ b/pkg/storageincentives/redistribution/redistribution_test.go @@ -7,15 +7,14 @@ package redistribution_test import ( "bytes" "context" - "encoding/binary" "errors" "fmt" "math/big" + "math/rand" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" - chaincfg "github.com/ethersphere/bee/pkg/config" "github.com/ethersphere/bee/pkg/log" "github.com/ethersphere/bee/pkg/sctx" "github.com/ethersphere/bee/pkg/storageincentives/redistribution" @@ -26,35 +25,45 @@ import ( "github.com/ethersphere/bee/pkg/util/testutil" ) -var redistributionContractABI = abiutil.MustParseABI(chaincfg.Testnet.RedistributionABI) +// var redistributionContractABI = abiutil.MustParseABI(chaincfg.Testnet.RedistributionABI) +var redistributionContractABI = abiutil.MustParseABI(redistribution.TestnetRedistributionABI) -func randChunkInclusionProof(t *testing.T) redistribution.ChunkInclusionProof { +func randomHashes(t testing.TB, n int) []common.Hash { + rhs := make([]common.Hash, n) + for _, rh := range rhs { + copy(rh[:], testutil.RandBytes(t, 32)) + } + return rhs +} + +// TODO uncomment when ABI is updated +func randProof(t *testing.T) redistribution.Proof { t.Helper() - return redistribution.ChunkInclusionProof{ - ProofSegments: []common.Hash{common.BytesToHash(testutil.RandBytes(t, 32))}, - ProveSegment: common.BytesToHash(testutil.RandBytes(t, 32)), - ProofSegments2: []common.Hash{common.BytesToHash(testutil.RandBytes(t, 32))}, - ProveSegment2: common.BytesToHash(testutil.RandBytes(t, 32)), - ProofSegments3: []common.Hash{common.BytesToHash(testutil.RandBytes(t, 32))}, + return redistribution.Proof{ + Sisters: randomHashes(t, 7), + Data: randomHashes(t, 1)[0], + Sisters2: randomHashes(t, 7), + Data2: randomHashes(t, 1)[0], + Sisters3: randomHashes(t, 7), PostageProof: redistribution.PostageProof{ Signature: testutil.RandBytes(t, 65), - PostageId: common.BytesToHash(testutil.RandBytes(t, 32)), - Index: binary.BigEndian.Uint64(testutil.RandBytes(t, 8)), - TimeStamp: binary.BigEndian.Uint64(testutil.RandBytes(t, 8)), + BatchId: randomHashes(t, 1)[0], + Index: uint64(rand.Int63()), + TimeStamp: uint64(rand.Int63()), }, ChunkSpan: 1, SocProof: []redistribution.SOCProof{}, } } -func randChunkInclusionProofs(t *testing.T) redistribution.ChunkInclusionProofs { +func randProofs(t *testing.T) []redistribution.Proof { t.Helper() - return redistribution.ChunkInclusionProofs{ - A: randChunkInclusionProof(t), - B: randChunkInclusionProof(t), - C: randChunkInclusionProof(t), + return []redistribution.Proof{ + randProof(t), + randProof(t), + randProof(t), } } @@ -184,9 +193,9 @@ func TestRedistribution(t *testing.T) { t.Run("Claim", func(t *testing.T) { t.Parallel() - proofs := randChunkInclusionProofs(t) + proofs := randProofs(t) - expectedCallData, err := redistributionContractABI.Pack("claim", proofs.A, proofs.B, proofs.C) + expectedCallData, err := redistributionContractABI.Pack("claim", proofs) if err != nil { t.Fatal(err) } @@ -225,8 +234,8 @@ func TestRedistribution(t *testing.T) { t.Run("Claim with tx reverted", func(t *testing.T) { t.Parallel() - proofs := randChunkInclusionProofs(t) - expectedCallData, err := redistributionContractABI.Pack("claim", proofs.A, proofs.B, proofs.C) + proofs := randProofs(t) + expectedCallData, err := redistributionContractABI.Pack("claim", proofs) if err != nil { t.Fatal(err) } diff --git a/pkg/storageincentives/redistributionstate.go b/pkg/storageincentives/redistributionstate.go index b1f4b60dd03..6cc201c7db1 100644 --- a/pkg/storageincentives/redistributionstate.go +++ b/pkg/storageincentives/redistributionstate.go @@ -15,8 +15,7 @@ import ( "github.com/ethersphere/bee/pkg/log" "github.com/ethersphere/bee/pkg/settlement/swap/erc20" "github.com/ethersphere/bee/pkg/storage" - storer "github.com/ethersphere/bee/pkg/storer" - "github.com/ethersphere/bee/pkg/swarm" + "github.com/ethersphere/bee/pkg/storageincentives/sampler" "github.com/ethersphere/bee/pkg/transaction" ) @@ -59,17 +58,10 @@ type Status struct { type RoundData struct { CommitKey []byte - SampleData *SampleData + Data *sampler.Data HasRevealed bool } -type SampleData struct { - Anchor1 []byte - ReserveSampleItems []storer.SampleItem - ReserveSampleHash swarm.Address - StorageRadius uint8 -} - func NewStatus() *Status { return &Status{ Reward: big.NewInt(0), @@ -232,26 +224,25 @@ func (r *RedistributionState) SetBalance(ctx context.Context) error { return nil } -func (r *RedistributionState) SampleData(round uint64) (SampleData, bool) { +func (r *RedistributionState) Data(round uint64) (sampler.Data, bool) { r.mtx.Lock() defer r.mtx.Unlock() rd, ok := r.status.RoundData[round] - if !ok || rd.SampleData == nil { - return SampleData{}, false + if !ok || rd.Data == nil { + return sampler.Data{}, false } - return *rd.SampleData, true + return *rd.Data, true } -func (r *RedistributionState) SetSampleData(round uint64, sd SampleData, dur time.Duration) { +func (r *RedistributionState) SetData(round uint64, sd sampler.Data) { r.mtx.Lock() defer r.mtx.Unlock() rd := r.status.RoundData[round] - rd.SampleData = &sd + rd.Data = &sd r.status.RoundData[round] = rd - r.status.SampleDuration = dur r.save() } diff --git a/pkg/storageincentives/redistributionstate_test.go b/pkg/storageincentives/redistributionstate_test.go index 9c7f750d64c..b71910d9338 100644 --- a/pkg/storageincentives/redistributionstate_test.go +++ b/pkg/storageincentives/redistributionstate_test.go @@ -13,6 +13,7 @@ import ( "github.com/ethereum/go-ethereum/common" erc20mock "github.com/ethersphere/bee/pkg/settlement/swap/erc20/mock" "github.com/ethersphere/bee/pkg/statestore/mock" + "github.com/ethersphere/bee/pkg/storageincentives/sampler" "github.com/ethersphere/bee/pkg/swarm" transactionmock "github.com/ethersphere/bee/pkg/transaction/mock" "github.com/ethersphere/bee/pkg/util/testutil" @@ -93,23 +94,23 @@ func TestState(t *testing.T) { func TestStateRoundData(t *testing.T) { t.Parallel() - t.Run("sample data", func(t *testing.T) { + t.Run("sample Data", func(t *testing.T) { t.Parallel() state := createRedistribution(t, nil, nil) - _, exists := state.SampleData(1) + _, exists := state.Data(1) if exists { t.Error("should not exists") } - savedSample := SampleData{ - ReserveSampleHash: swarm.RandAddress(t), - StorageRadius: 3, + savedSample := sampler.Data{ + Hash: swarm.RandAddress(t), + Depth: 3, } - state.SetSampleData(1, savedSample, 0) + state.SetData(1, savedSample) - sample, exists := state.SampleData(1) + sample, exists := state.Data(1) if !exists { t.Error("should exist") } @@ -163,30 +164,30 @@ func TestPurgeRoundData(t *testing.T) { state := createRedistribution(t, nil, nil) - // helper function which populates data at specified round + // helper function which populates sampler.Data at specified round populateDataAtRound := func(round uint64) { - savedSample := SampleData{ - ReserveSampleHash: swarm.RandAddress(t), - StorageRadius: 3, + savedSample := sampler.Data{ + Hash: swarm.RandAddress(t), + Depth: 3, } commitKey := testutil.RandBytes(t, swarm.HashSize) - state.SetSampleData(round, savedSample, 0) + state.SetData(round, savedSample) state.SetCommitKey(round, commitKey) state.SetHasRevealed(round) } - // asserts if there is, or there isn't, data at specified round + // asserts if there is, or there isn't, sampler.Data at specified round assertHasDataAtRound := func(round uint64, shouldHaveData bool) { check := func(exists bool) { if shouldHaveData && !exists { - t.Error("should have data") + t.Error("should have sampler.Data") } else if !shouldHaveData && exists { - t.Error("should not have data") + t.Error("should not have sampler.Data") } } - _, exists1 := state.SampleData(round) + _, exists1 := state.Data(round) _, exists2 := state.CommitKey(round) exists3 := state.HasRevealed(round) @@ -198,7 +199,7 @@ func TestPurgeRoundData(t *testing.T) { const roundsCount = 100 hasRoundData := make([]bool, roundsCount) - // Populate data at random rounds + // Populate sampler.Data at random rounds for i := uint64(0); i < roundsCount; i++ { v := rand.Int()%2 == 0 hasRoundData[i] = v @@ -208,8 +209,8 @@ func TestPurgeRoundData(t *testing.T) { assertHasDataAtRound(i, v) } - // Run purge successively and assert that all data is purged up to - // currentRound - purgeDataOlderThenXRounds + // Run purge successively and assert that all sampler.Data is purged up to + // currentRound - purgesampler.DataOlderThenXRounds for i := uint64(0); i < roundsCount; i++ { state.SetCurrentEvent(0, i) state.purgeStaleRoundData() @@ -223,7 +224,7 @@ func TestPurgeRoundData(t *testing.T) { } } - // Purge remaining data in single go + // Purge remaining sampler.Data in single go round := uint64(roundsCount + purgeStaleDataThreshold) state.SetCurrentEvent(0, round) state.purgeStaleRoundData() diff --git a/pkg/storageincentives/sampler/debug.go b/pkg/storageincentives/sampler/debug.go new file mode 100644 index 00000000000..4abbfc38a68 --- /dev/null +++ b/pkg/storageincentives/sampler/debug.go @@ -0,0 +1,35 @@ +// this file together with rchash endpoint and api support should be removed + +package sampler + +import ( + "context" + "fmt" + "time" + + "github.com/ethersphere/bee/pkg/storageincentives/redistribution" + "github.com/ethersphere/bee/pkg/swarm" +) + +type SampleWithProofs struct { + Hash swarm.Address `json:"hash"` + Proofs []redistribution.Proof `json:"proofs"` + Duration time.Duration `json:"duration"` +} + +// ReserveWithProofs is only called by rchash API +func (s *Sampler) ReserveSampleWithProofs(ctx context.Context, anchor1, anchor2 []byte, depth uint8, round uint64) (swp SampleWithProofs, err error) { + t := time.Now() + + sample, err := s.MakeSample(ctx, anchor1, depth, round) + if err != nil { + return swp, fmt.Errorf("mock reserve sampling: %w", err) + } + + proofs, err := MakeProofs(sample, anchor2) + if err != nil { + return swp, fmt.Errorf("make proofs: %w", err) + } + + return SampleWithProofs{sample.Hash, proofs, time.Since(t)}, nil +} diff --git a/pkg/storageincentives/sampler/export_test.go b/pkg/storageincentives/sampler/export_test.go new file mode 100644 index 00000000000..d61b1f72ca8 --- /dev/null +++ b/pkg/storageincentives/sampler/export_test.go @@ -0,0 +1,6 @@ +package sampler + +var ( + TransformedAddressCAC = transformedAddressCAC + NewStamp = newStamp +) diff --git a/pkg/storageincentives/sampler/proof.go b/pkg/storageincentives/sampler/proof.go new file mode 100644 index 00000000000..0337e749b9a --- /dev/null +++ b/pkg/storageincentives/sampler/proof.go @@ -0,0 +1,53 @@ +// Copyright 2023 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sampler + +import ( + "math/big" + + "github.com/ethersphere/bee/pkg/bmt" + "github.com/ethersphere/bee/pkg/storageincentives/redistribution" +) + +// MakeProofs creates transaction data for the claim method aka Proof of entitlement (POE). +func MakeProofs( + sample Data, + anchor2 []byte, +) ([]redistribution.Proof, error) { + // witness indexes selecting the index-th item of the sample + index3 := uint64(Size) - 1 + rand := new(big.Int).SetBytes(anchor2) + index1 := new(big.Int).Mod(rand, new(big.Int).SetUint64(index3)).Uint64() + index2 := new(big.Int).Mod(rand, new(big.Int).SetUint64(index3-1)).Uint64() + if index2 >= index1 { + index2++ + } + + // reserve sample inclusion proofs for witness chunks + indexes := []uint64{index1, index2, index3} + witnessProofs := make([]*bmt.Proof, len(indexes)) + for i, idx := range indexes { + proof := sample.Prover.Proof(int(idx) * 2) + witnessProofs[i] = &proof + } + + // data retention proofs for the witness chunks + segmentIndex := int(new(big.Int).Mod(new(big.Int).SetBytes(anchor2), big.NewInt(int64(128))).Uint64()) + retentionProofs := make([]*bmt.Proof, len(indexes)) + transformProofs := make([]*bmt.Proof, len(indexes)) + for i, idx := range indexes { + item := sample.Items[idx] + proof := item.Prover.Proof(segmentIndex) + retentionProofs[i] = &proof + transProof := sample.Items[idx].TransProver.Proof(segmentIndex) + transformProofs[i] = &transProof + } + + proofs := make([]redistribution.Proof, len(indexes)) + for i := range proofs { + proofs[i] = redistribution.NewProof(*witnessProofs[i], *retentionProofs[i], *transformProofs[i], sample.Items[i].Stamp, sample.Items[i].SOC) + } + return proofs, nil +} diff --git a/pkg/storageincentives/sampler/proof_test.go b/pkg/storageincentives/sampler/proof_test.go new file mode 100644 index 00000000000..12323aa1942 --- /dev/null +++ b/pkg/storageincentives/sampler/proof_test.go @@ -0,0 +1,7 @@ +// Copyright 2023 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sampler_test + +// must create Data from fixtures and run MakeProofs on it and compare to fixtures diff --git a/pkg/storageincentives/sampler/sampler.go b/pkg/storageincentives/sampler/sampler.go new file mode 100644 index 00000000000..c75312681bd --- /dev/null +++ b/pkg/storageincentives/sampler/sampler.go @@ -0,0 +1,470 @@ +// Copyright 2023 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package sampler + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + "hash" + "math/big" + "sync" + "time" + + "github.com/ethersphere/bee/pkg/bmt" + "github.com/ethersphere/bee/pkg/bmtpool" + "github.com/ethersphere/bee/pkg/postage" + "github.com/ethersphere/bee/pkg/soc" + storer "github.com/ethersphere/bee/pkg/storer" + + "github.com/ethersphere/bee/pkg/swarm" + "golang.org/x/sync/errgroup" +) + +const Size = 16 + +type Batchstore interface { + postage.ChainStateGetter + postage.ValueIterator +} + +type Sampler struct { + backend postage.ChainBackend + batchstore Batchstore + sampler storer.Sampler + fullSyncedFunc func() bool + healthyFunc func() bool +} + +func New( + backend postage.ChainBackend, + batchstore Batchstore, + sampler storer.Sampler, + fullSyncedFunc, + healthyFunc func() bool, +) *Sampler { + return &Sampler{ + batchstore: batchstore, + sampler: sampler, + fullSyncedFunc: fullSyncedFunc, + healthyFunc: healthyFunc, + } +} + +type Sample struct { + Stats Stats + Data Data +} +type Item struct { + Address swarm.Address + TransAddress swarm.Address + Prover bmt.Prover + TransProver bmt.Prover + LoadStamp func() (*postage.Stamp, error) + Stamp *postage.Stamp + SOC *soc.SOC +} + +func newStamp(s swarm.Stamp) *postage.Stamp { + return postage.NewStamp(s.BatchID(), s.Index(), s.Timestamp(), s.Sig()) +} + +type Data struct { + Depth uint8 + Hash swarm.Address + Prover bmt.Prover + Items []Item +} + +func Chunk(depth uint8, items []Item) (*Data, error) { + content := make([]byte, len(items)*2*swarm.HashSize) + for i, s := range items { + copy(content[i*2*swarm.HashSize:], s.Address.Bytes()) + copy(content[(i*2+1)*swarm.HashSize:], s.TransAddress.Bytes()) + } + hasher := bmtpool.Get() + prover := bmt.Prover{Hasher: hasher} + prover.SetHeaderInt64(int64(len(content))) + _, err := prover.Write(content) + if err != nil { + return nil, err + } + hash, err := prover.Hash(nil) + if err != nil { + return nil, err + } + return &Data{ + Depth: depth, + Hash: swarm.NewAddress(hash), + Prover: prover, + Items: items, + }, nil +} + +func (s *Sampler) MakeSample(ctx context.Context, salt []byte, depth uint8, round uint64) (Data, error) { + + maxTimeStamp, err := s.maxTimeStamp(ctx, round) + if err != nil { + return Data{}, err + } + + filterFunc, err := s.getBatchFilterFunc(round) + if err != nil { + return Data{}, fmt.Errorf("get batches with balance below value: %w", err) + } + + sample, err := s.reserveSample(ctx, salt, depth, maxTimeStamp, filterFunc) + return sample.Data, err +} + +// Reserve generates the sample of reserve storage of a node required for the +// storage incentives agent to participate in the lottery round. In order to generate +// this we need to iterate through all the chunks in the node's reserve and +// calculate the transformed hashes of all the chunks using the anchor as the salt. +// In order to generate the transformed hashes, we use bmt hash with a prefixed basehash +// keccak256 with anchor as the prefix. Nodes need to calculate the +// in the most optimal way and there are time restrictions. The lottery round is a +// time based round, so nodes participating in the round need to perform this +// calculation within the round limits. +// In order to optimize this we use a simple pipeline pattern: +// Iterate chunk addresses -> Get the chunk data and calculate transformed hash -> Assemble the +func (s *Sampler) reserveSample( + ctx context.Context, + anchor []byte, + depth uint8, + maxTimeStamp uint64, + filterFunc func(batchID []byte) bool, +) (Sample, error) { + g, ctx := errgroup.WithContext(ctx) + chunkC := make(chan storer.Chunk, 64) + // all performance stats should be removed, this is a benchmarking exercise, any interal data is not really meaningful + // as it is presented to the user + allStats := &Stats{} + statsLock := sync.Mutex{} + addStats := func(stats Stats) { + statsLock.Lock() + allStats.add(stats) + statsLock.Unlock() + } + start := time.Now() + + // Phase 1: Iterate chunk addresses + g.Go(func() error { + start := time.Now() + stats := Stats{} + defer func() { + stats.IterationDuration = time.Since(start) + close(chunkC) + addStats(stats) + }() + + err := s.sampler.Iterate(depth, func(c storer.Chunk) (bool, error) { + select { + case chunkC <- c: + stats.TotalIterated++ + return false, nil + case <-ctx.Done(): + return false, ctx.Err() + } + }) + return err + }) + + // Phase 2: Get the chunk data and calculate transformed hash + ItemChan := make(chan Item, 64) + + prefixHasherFactory := func() hash.Hash { + return swarm.NewPrefixHasher(anchor) + } + transHasherPool := bmt.NewPool(bmt.NewConf(prefixHasherFactory, swarm.BmtBranches, bmtpool.Capacity)) + + const workers = 6 + + for i := 0; i < workers; i++ { + g.Go(func() error { + wstat := Stats{} + defer func() { + addStats(wstat) + }() + + transProver := bmt.Prover{Hasher: transHasherPool.Get()} + for c := range chunkC { + // exclude chunks whose batches balance are below minimum + if filterFunc(c.BatchID) { + wstat.BelowBalanceIgnored++ + continue + } + + chunkLoadStart := time.Now() + + chunk, err := c.Chunk(ctx) + if err != nil { + wstat.ChunkLoadFailed++ + return fmt.Errorf("failed loading chunk at address=%x: %w", c.Address, err) + } + + wstat.ChunkLoadDuration += time.Since(chunkLoadStart) + + var sch *soc.SOC + if c.Type == swarm.ChunkTypeSingleOwner { + sch, err = soc.FromChunk(chunk) + if err != nil { + return err + } + } + + taddrStart := time.Now() + taddr, err := transformedAddress(transProver, chunk, c.Type) + if err != nil { + return err + } + wstat.TaddrDuration += time.Since(taddrStart) + + select { + case ItemChan <- Item{ + TransAddress: taddr, + Address: c.Address, + Stamp: postage.NewStamp(c.BatchID, nil, nil, nil), + LoadStamp: c.Stamp, + TransProver: transProver, + SOC: sch, + }: + case <-ctx.Done(): + return ctx.Err() + } + } + + return nil + }) + } + + go func() { + _ = g.Wait() + close(ItemChan) + }() + + Items := make([]Item, 0, Size) + // insert function will insert the new item in its correct place. If the + // size goes beyond what we need we omit the last item. + insert := func(item Item) { + added := false + for i, sItem := range Items { + if le(item.TransAddress, sItem.TransAddress) { + Items = append(Items[:i+1], Items[i:]...) + Items[i] = item + added = true + break + } + } + if len(Items) > Size { + Items = Items[:Size] + } + if len(Items) < Size && !added { + Items = append(Items, item) + } + } + + // Phase 3: Assemble the . Here we need to assemble only the first Size + // no of items from the results of the 2nd phase. + // In this step stamps are loaded and validated only if chunk will be added to . + stats := Stats{} + for item := range ItemChan { + currentMaxAddr := swarm.EmptyAddress + if len(Items) > 0 { + currentMaxAddr = Items[len(Items)-1].TransAddress + } + + if len(Items) >= Size && le(currentMaxAddr, item.TransAddress) { + continue + } + + stamp, err := item.LoadStamp() + if err != nil { + stats.StampLoadFailed++ + // db.logger.Debug("failed loading stamp", "chunk_address", item.ChunkAddress, "error", err) + continue + } + + // if _, err := db.validStamp(ch); err != nil { + // stats.InvalidStamp++ + // db.logger.Debug("invalid stamp for chunk", "chunk_address", ch.Address(), "error", err) + // continue + // } + + if binary.BigEndian.Uint64(stamp.Timestamp()) > maxTimeStamp { + stats.NewIgnored++ + continue + } + item.Stamp = stamp + + insert(item) + stats.Inserts++ + } + addStats(stats) + + allStats.TotalDuration = time.Since(start) + + if err := g.Wait(); err != nil { + // db.logger.Info("reserve r finished with error", "err", err, "duration", time.Since(t), "storage_radius", storageRadius, "stats", fmt.Sprintf("%+v", allStats)) + + return Sample{}, fmt.Errorf("r: failed creating : %w", err) + } + + // db.logger.Info("reserve r finished", "duration", time.Since(t), "storage_radius", storageRadius, "consensus_time_ns", consensusTime, "stats", fmt.Sprintf("%+v", allStats)) + + data, err := s.sampleChunk(depth, Items) + if err != nil { + return Sample{}, err + } + return Sample{Stats: *allStats, Data: *data}, nil +} +func (s *Sampler) sampleChunk(depth uint8, items []Item) (*Data, error) { + size := len(items) * 2 * swarm.HashSize + content := make([]byte, size) + for i, s := range items { + copy(content[i*swarm.HashSize:], s.Address.Bytes()) + copy(content[(i+1)*2*swarm.HashSize:], s.TransAddress.Bytes()) + } + prover := bmt.Prover{Hasher: bmtpool.Get()} + prover.SetHeaderInt64(int64(size)) + _, err := prover.Write(content) + if err != nil { + return &Data{}, err + } + hash, err := prover.Hash(nil) + if err != nil { + return &Data{}, err + } + return &Data{depth, swarm.NewAddress(hash), prover, items}, nil +} + +// less function uses the byte compare to check for lexicographic ordering +func le(a, b swarm.Address) bool { + return bytes.Compare(a.Bytes(), b.Bytes()) == -1 +} + +func transformedAddress(hasher bmt.Prover, chunk swarm.Chunk, chType swarm.ChunkType) (swarm.Address, error) { + switch chType { + case swarm.ChunkTypeContentAddressed: + return transformedAddressCAC(hasher, chunk) + case swarm.ChunkTypeSingleOwner: + return transformedAddressSOC(hasher, chunk) + default: + return swarm.ZeroAddress, fmt.Errorf("chunk type [%v] is is not valid", chType) + } +} + +func transformedAddressCAC(hasher bmt.Prover, chunk swarm.Chunk) (swarm.Address, error) { + hasher.Reset() + hasher.SetHeader(chunk.Data()[:bmt.SpanSize]) + + _, err := hasher.Write(chunk.Data()[bmt.SpanSize:]) + if err != nil { + return swarm.ZeroAddress, err + } + + taddr, err := hasher.Hash(nil) + if err != nil { + return swarm.ZeroAddress, err + } + + return swarm.NewAddress(taddr), nil +} + +func transformedAddressSOC(hasher bmt.Prover, chunk swarm.Chunk) (swarm.Address, error) { + // Calculate transformed address from wrapped chunk + sChunk, err := soc.FromChunk(chunk) + if err != nil { + return swarm.ZeroAddress, err + } + taddrCac, err := transformedAddressCAC(hasher, sChunk.WrappedChunk()) + if err != nil { + return swarm.ZeroAddress, err + } + + // Hash address and transformed address to make transformed address for this SOC + sHasher := swarm.NewHasher() + if _, err := sHasher.Write(chunk.Address().Bytes()); err != nil { + return swarm.ZeroAddress, err + } + if _, err := sHasher.Write(taddrCac.Bytes()); err != nil { + return swarm.ZeroAddress, err + } + + return swarm.NewAddress(sHasher.Sum(nil)), nil +} + +type Stats struct { + TotalDuration time.Duration + TotalIterated int64 + IterationDuration time.Duration + Inserts int64 + NewIgnored int64 + InvalidStamp int64 + BelowBalanceIgnored int64 + TaddrDuration time.Duration + ValidStampDuration time.Duration + BatchesBelowValueDuration time.Duration + RogueChunk int64 + ChunkLoadDuration time.Duration + ChunkLoadFailed int64 + StampLoadFailed int64 +} + +func (s *Stats) add(other Stats) { + s.TotalDuration += other.TotalDuration + s.TotalIterated += other.TotalIterated + s.IterationDuration += other.IterationDuration + s.Inserts += other.Inserts + s.NewIgnored += other.NewIgnored + s.InvalidStamp += other.InvalidStamp + s.BelowBalanceIgnored += other.BelowBalanceIgnored + s.TaddrDuration += other.TaddrDuration + s.ValidStampDuration += other.ValidStampDuration + s.BatchesBelowValueDuration += other.BatchesBelowValueDuration + s.RogueChunk += other.RogueChunk + s.ChunkLoadDuration += other.ChunkLoadDuration + s.ChunkLoadFailed += other.ChunkLoadFailed + s.StampLoadFailed += other.StampLoadFailed +} + +func (s *Sampler) StorageRadius() uint8 { + return s.sampler.StorageRadius() +} + +func (s *Sampler) getBatchFilterFunc(round uint64) (func(batchID []byte) bool, error) { + cs := s.batchstore.GetChainState() + blocksToLive := (round+2)*152 - cs.Block + // blocksToLive := (round+2)*blocksPerRound - cs.Block + minBalance := new(big.Int).Add(cs.TotalAmount, new(big.Int).Mul(cs.CurrentPrice, big.NewInt(int64(blocksToLive)))) + + excluded := make(map[string]struct{}) + err := s.batchstore.IterateByValue(func(id []byte, val *big.Int) (bool, error) { + if val.Cmp(minBalance) > 0 { + return true, nil + } + excluded[string(id)] = struct{}{} + return false, nil + }) + return func(id []byte) bool { + _, found := excluded[string(id)] + return found + }, err +} + +func (s *Sampler) maxTimeStamp(ctx context.Context, round uint64) (uint64, error) { + previousRoundBlockNumber := new(big.Int).SetUint64((round - 1) * 152) + // previousRoundBlockNumber := new(b/ig.Int).SetUint64((round - 1) * blocksPerRound) + + // s.metrics.BackendCalls.Inc() + lastBlock, err := s.backend.HeaderByNumber(ctx, previousRoundBlockNumber) + if err != nil { + // s.metrics.BackendErrors.Inc() + return 0, err + } + + return lastBlock.Time, nil +} diff --git a/pkg/storageincentives/sampler/sampler_test.go b/pkg/storageincentives/sampler/sampler_test.go new file mode 100644 index 00000000000..c4d6314dc53 --- /dev/null +++ b/pkg/storageincentives/sampler/sampler_test.go @@ -0,0 +1,58 @@ +package sampler_test + +import ( + "hash" + "sort" + + "github.com/ethersphere/bee/pkg/bmt" + "github.com/ethersphere/bee/pkg/bmtpool" + chunk "github.com/ethersphere/bee/pkg/storage/testing" + "github.com/ethersphere/bee/pkg/storageincentives/sampler" + "github.com/ethersphere/bee/pkg/swarm" +) + +// RandSample returns Sample with random values. +func RandSample(anchor []byte) (sampler.Data, error) { + chunks := make([]swarm.Chunk, sampler.Size) + for i := 0; i < sampler.Size; i++ { + ch := chunk.GenerateTestRandomChunk() + // if i%3 == 0 { + // ch = chunk.GenerateTestRandomSoChunk(t, ch) + // } + chunks[i] = ch + } + + return MakeSampleUsingChunks(chunks, anchor) +} + +// MakeSampleUsingChunks returns Sample constructed using supplied chunks. +func MakeSampleUsingChunks(chunks []swarm.Chunk, anchor []byte) (sampler.Data, error) { + prefixHasherFactory := func() hash.Hash { + return swarm.NewPrefixHasher(anchor) + } + items := make([]sampler.Item, len(chunks)) + for i, ch := range chunks { + // these provers need to be used for proofs later and assumed are in hashed state + // also the hasher taken from bmtpool musst not be put back + prover := bmt.Prover{Hasher: bmtpool.Get()} + transProver := bmt.Prover{Hasher: bmt.NewHasher(prefixHasherFactory)} + tr, err := sampler.TransformedAddressCAC(transProver, ch) + if err != nil { + return sampler.Data{}, err + } + + items[i] = sampler.Item{ + Address: ch.Address(), + TransAddress: tr, + Prover: prover, + TransProver: transProver, + Stamp: sampler.NewStamp(ch.Stamp()), + } + } + + sort.Slice(items, func(i, j int) bool { + return items[i].TransAddress.Compare(items[j].TransAddress) == -1 + }) + + return sampler.Data{Items: items}, nil +} diff --git a/pkg/storageincentives/testdata/inclusion-proofs.json b/pkg/storageincentives/testdata/inclusion-proofs.json index fe79666096e..7688022650e 100644 --- a/pkg/storageincentives/testdata/inclusion-proofs.json +++ b/pkg/storageincentives/testdata/inclusion-proofs.json @@ -1,3 +1,131 @@ +<<<<<<< HEAD +[ + { + "ChunkSpan" : 26, + "Data" : "0x7133885ac59dca7b97773acb740e978d41a4af45bd563067c8a3d863578488f1", + "Data2" : "0x0000000000000000000000000000000000000000000000000000000000000000", + "PostageProof" : { + "BatchId" : "0x4c8efc14c8e3cee608174f995d7afe155897bf643a31226e4f1363bc97686aef", + "Index" : 525059, + "Signature" : "p8jRioJ504AxaevPTlp9vdTf/vpZHqrY0c6qY2p5Otl15/exCHvOpBdlJbAALt3grL/aINvS37vnd8qziWj9xhs=", + "TimeStamp" : 197384 + }, + "Sisters" : [ + "0x0875605dea48e812c9685ffba220a2b848bdbafdb95e02d087ba4a32925ea34f", + "0xf873df729270d5f4064286f3f018385a07cb4228734d8aca794299fee6e3e3e5", + "0x1fa8767fe303fe7487f5d58e4d72e5e170cf135f58a91b4fe19e4b19e5b67b5a", + "0x0f64ed713e25291e2c5a0561f584fa78c55a399e31919903d215dd622bcfd0ec", + "0x34dac0c73538614801c1ad16e272ef57f0b96a972073d15418f38daf9eb401c0", + "0x0eb01ebfc9ed27500cd4dfc979272d1f0913cc9f66540d7e8005811109e1cf2d", + "0x887c22bd8750d34016ac3c66b5ff102dacdd73f6b014e710b51e8022af9a1968" + ], + "Sisters2" : [ + "0x0000000000000000000000000000000000000000000000000000000000000000", + "0xad3228b676f7d3cd4284a5443f17f1962b36e491b30a40b2405849e597ba5fb5", + "0xb4c11951957c6f8f642c4af61cd6b24640fec6dc7fc607ee8206a99e92410d30", + "0x21ddb9a356815c3fac1026b6dec5df3124afbadb485c9ba5a3e3398a04b7ba85", + "0x2047b070a295f8d517121d9ac9b3d5f9a944bac6cfab72dd5a7c625ab4558b0a", + "0x0eb01ebfc9ed27500cd4dfc979272d1f0913cc9f66540d7e8005811109e1cf2d", + "0x887c22bd8750d34016ac3c66b5ff102dacdd73f6b014e710b51e8022af9a1968" + ], + "Sisters3" : [ + "0x0000000000000000000000000000000000000000000000000000000000000000", + "0xa7f526447b68535121d36909a7585c9610d4fe6d4115540464c70499b0d7136d", + "0x066dd7ce6f4f1c97e78ff1c271916db25cb06128c92f8c8520807a0fa2ba93ff", + "0xdf43c86b00db2156e769e8a8df1f08dc89ab5661c6fbaa9563f96fb9c051fc63", + "0x7327aecc9178bab420bb6fe482e07b65af69775b55666ec1ac8ab3da5bcec6dc", + "0xb68323ecaad1185a5e078f41c94c59d0b6dda5d57e109866e64d44acb8702846", + "0x478adfa93a7bb904d0aa86ff0d559f43aa915ee7865592e717b72a24452181cb" + ], + "SocProof" : [ + { + "ChunkAddr" : "0xf32442586d93d8c002372ed41fa2ea1f281f38311c161d535c3665de5d9bfd92", + "Identifier" : "0x6223cfdd75a40440ccd32d0b11b24f08562ec63b1ea3b8cb1a59dfc3e3c33595", + "Signature" : "TpV2lJM45MI/RwO/gTZyVquFmzKTT+9Nsu5Gp2v2vjVOlqxii4eEst4Lvq5ZdUaXgxktbRcFSF/Krdje3ebiqhs=", + "Signer" : "gntE1T3yhUBXcTslzdZT63D+NsQ=" + } + ] + }, + { + "ChunkSpan" : 26, + "Data" : "0x535e6df58a122a8f5e6c851c19b3e042f4cd1b5c5a8c499581c9f6d4e3509182", + "Data2" : "0x0000000000000000000000000000000000000000000000000000000000000000", + "PostageProof" : { + "BatchId" : "0x4c8efc14c8e3cee608174f995d7afe155897bf643a31226e4f1363bc97686aef", + "Index" : 525059, + "Signature" : "sCdPzaWeiq/+6AMCGXGnZKAXziwPQcjOtu796oBwVvYhqY/qtevzO7YGXknAUPQT7IhAsAj8Ik2ILOUkTOPgFxw=", + "TimeStamp" : 197384 + }, + "Sisters" : [ + "0x463aeb4ca5f000064c082e56eba387004265d2f47bf1226ef2d86cb163bcca3a", + "0x829af58b2a2f1c6c156baa196f03be4df510a96419f2dd54c456d3da30166312", + "0xdee4815ec42efa507b79cf4eb1f272e07be1b526cbd48137a287d9e5b2b2808a", + "0x0f64ed713e25291e2c5a0561f584fa78c55a399e31919903d215dd622bcfd0ec", + "0x34dac0c73538614801c1ad16e272ef57f0b96a972073d15418f38daf9eb401c0", + "0x0eb01ebfc9ed27500cd4dfc979272d1f0913cc9f66540d7e8005811109e1cf2d", + "0x887c22bd8750d34016ac3c66b5ff102dacdd73f6b014e710b51e8022af9a1968" + ], + "Sisters2" : [ + "0x0000000000000000000000000000000000000000000000000000000000000000", + "0xad3228b676f7d3cd4284a5443f17f1962b36e491b30a40b2405849e597ba5fb5", + "0xb4c11951957c6f8f642c4af61cd6b24640fec6dc7fc607ee8206a99e92410d30", + "0x21ddb9a356815c3fac1026b6dec5df3124afbadb485c9ba5a3e3398a04b7ba85", + "0x46f43b515833749217540ac60c79e0c6a54c73f3500850b5869b31d5c89d101f", + "0x0eb01ebfc9ed27500cd4dfc979272d1f0913cc9f66540d7e8005811109e1cf2d", + "0x887c22bd8750d34016ac3c66b5ff102dacdd73f6b014e710b51e8022af9a1968" + ], + "Sisters3" : [ + "0x0000000000000000000000000000000000000000000000000000000000000000", + "0xa7f526447b68535121d36909a7585c9610d4fe6d4115540464c70499b0d7136d", + "0x066dd7ce6f4f1c97e78ff1c271916db25cb06128c92f8c8520807a0fa2ba93ff", + "0xdf43c86b00db2156e769e8a8df1f08dc89ab5661c6fbaa9563f96fb9c051fc63", + "0x4284c510d7d64c9e052c73bddadb1fca522fd26caf2ebf007faad50a9a0f09fa", + "0xb68323ecaad1185a5e078f41c94c59d0b6dda5d57e109866e64d44acb8702846", + "0x478adfa93a7bb904d0aa86ff0d559f43aa915ee7865592e717b72a24452181cb" + ], + "SocProof" : [] + }, + { + "ChunkSpan" : 27, + "Data" : "0x5ba2c8b912fad4aeb4a11a960946d07b9f66bc40ac54d87224914d75f5aeea5f", + "Data2" : "0x0000000000000000000000000000000000000000000000000000000000000000", + "PostageProof" : { + "BatchId" : "0x4c8efc14c8e3cee608174f995d7afe155897bf643a31226e4f1363bc97686aef", + "Index" : 525059, + "Signature" : "Z0fFjOhhNIbGlvW7c5PJxZCUNxlpw6Ur+vdRksYF9K18cMbnH90yDiDQBeQulO4yECwjTrRl9PX9nbYPytA1axw=", + "TimeStamp" : 197384 + }, + "Sisters" : [ + "0xfee18543782df46a86f85456e62dc973a4c84369b6b1cd4f93e57fe247f9730e", + "0x23a0858ee2b8b4cb0ba66d3533f468d6b583a6b77df0cc78fc6df64dc735a917", + "0xb6bffa54dec44ad57349f9aef6cb65a1f8807f15447462ec519751220e5a5bc3", + "0x553aae9948fc13c33d8b353cf5694ecadc7c40c8316ce09cbd4d864dbb94f026", + "0xaf7db874a9b5addf602b3e899194480a32afec6d6cd4ec0fadf9e065db739dd5", + "0x0eb01ebfc9ed27500cd4dfc979272d1f0913cc9f66540d7e8005811109e1cf2d", + "0x887c22bd8750d34016ac3c66b5ff102dacdd73f6b014e710b51e8022af9a1968" + ], + "Sisters2" : [ + "0x0000000000000000000000000000000000000000000000000000000000000000", + "0xad3228b676f7d3cd4284a5443f17f1962b36e491b30a40b2405849e597ba5fb5", + "0xb4c11951957c6f8f642c4af61cd6b24640fec6dc7fc607ee8206a99e92410d30", + "0x21ddb9a356815c3fac1026b6dec5df3124afbadb485c9ba5a3e3398a04b7ba85", + "0x7f575db255ef42dcaeb7658df9f33fe5a1aad5d41af51a72a381acea29d98a12", + "0x0eb01ebfc9ed27500cd4dfc979272d1f0913cc9f66540d7e8005811109e1cf2d", + "0x887c22bd8750d34016ac3c66b5ff102dacdd73f6b014e710b51e8022af9a1968" + ], + "Sisters3" : [ + "0x0000000000000000000000000000000000000000000000000000000000000000", + "0xa7f526447b68535121d36909a7585c9610d4fe6d4115540464c70499b0d7136d", + "0x066dd7ce6f4f1c97e78ff1c271916db25cb06128c92f8c8520807a0fa2ba93ff", + "0xdf43c86b00db2156e769e8a8df1f08dc89ab5661c6fbaa9563f96fb9c051fc63", + "0x7683427ba0ef1fbebf97f2fc36859df88ead8123369fe38d7b767b7a7eda5294", + "0xb68323ecaad1185a5e078f41c94c59d0b6dda5d57e109866e64d44acb8702846", + "0x478adfa93a7bb904d0aa86ff0d559f43aa915ee7865592e717b72a24452181cb" + ], + "SocProof" : [] + } +] +======= { "proof1": { "proofSegments": [ @@ -123,4 +251,5 @@ }, "socProof": [] } -} \ No newline at end of file +} +>>>>>>> origin/master diff --git a/pkg/storer/mock/mockreserve.go b/pkg/storer/mock/mockreserve.go index d6d70390242..241652e08f5 100644 --- a/pkg/storer/mock/mockreserve.go +++ b/pkg/storer/mock/mockreserve.go @@ -6,7 +6,6 @@ package mockstorer import ( "context" - "math/big" "sync" storage "github.com/ethersphere/bee/pkg/storage" @@ -83,12 +82,6 @@ func WithPutHook(f func(swarm.Chunk) error) Option { }) } -func WithSample(s storer.Sample) Option { - return optionFunc(func(p *ReserveStore) { - p.sample = s - }) -} - var _ storer.ReserveStore = (*ReserveStore)(nil) type ReserveStore struct { @@ -110,8 +103,6 @@ type ReserveStore struct { subResponses []chunksResponse putHook func(swarm.Chunk) error - - sample storer.Sample } // NewReserve returns a new Reserve mock. @@ -238,10 +229,6 @@ func (s *ReserveStore) ReserveHas(addr swarm.Address, batchID []byte) (bool, err return true, nil } -func (s *ReserveStore) ReserveSample(context.Context, []byte, uint8, uint64, *big.Int) (storer.Sample, error) { - return s.sample, nil -} - type Option interface { apply(*ReserveStore) } diff --git a/pkg/storer/sample_test.go b/pkg/storer/sample_test.go deleted file mode 100644 index 215b190fc1a..00000000000 --- a/pkg/storer/sample_test.go +++ /dev/null @@ -1,195 +0,0 @@ -// Copyright 2023 The Swarm Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package storer_test - -import ( - "context" - "github.com/ethersphere/bee/pkg/postage" - "math/rand" - "testing" - "time" - - postagetesting "github.com/ethersphere/bee/pkg/postage/testing" - chunk "github.com/ethersphere/bee/pkg/storage/testing" - "github.com/ethersphere/bee/pkg/storer" - "github.com/ethersphere/bee/pkg/swarm" - "github.com/google/go-cmp/cmp" -) - -func TestReserveSampler(t *testing.T) { - const chunkCountPerPO = 10 - const maxPO = 10 - - randChunks := func(baseAddr swarm.Address, timeVar uint64) []swarm.Chunk { - var chs []swarm.Chunk - for po := 0; po < maxPO; po++ { - for i := 0; i < chunkCountPerPO; i++ { - ch := chunk.GenerateValidRandomChunkAt(baseAddr, po).WithBatch(0, 3, 2, false) - if rand.Intn(2) == 0 { // 50% chance to wrap CAC into SOC - ch = chunk.GenerateTestRandomSoChunk(t, ch) - } - - // override stamp timestamp to be before the consensus timestamp - ch = ch.WithStamp(postagetesting.MustNewStampWithTimestamp(timeVar)) - chs = append(chs, ch) - } - } - return chs - } - - testF := func(t *testing.T, baseAddr swarm.Address, st *storer.DB) { - t.Helper() - - timeVar := uint64(time.Now().UnixNano()) - chs := randChunks(baseAddr, timeVar-1) - - putter := st.ReservePutter() - for _, ch := range chs { - err := putter.Put(context.Background(), ch) - if err != nil { - t.Fatal(err) - } - } - - t.Run("reserve size", reserveSizeTest(st.Reserve(), chunkCountPerPO*maxPO)) - - var sample1 storer.Sample - - t.Run("reserve sample 1", func(t *testing.T) { - sample, err := st.ReserveSample(context.TODO(), []byte("anchor"), 5, timeVar, nil) - if err != nil { - t.Fatal(err) - } - - assertValidSample(t, sample) - assertSampleNoErrors(t, sample) - - if sample.Stats.NewIgnored != 0 { - t.Fatalf("sample should not have ignored chunks") - } - - sample1 = sample - }) - - // We generate another 100 chunks. With these new chunks in the reserve, statistically - // some of them should definitely make it to the sample based on lex ordering. - chs = randChunks(baseAddr, timeVar+1) - putter = st.ReservePutter() - for _, ch := range chs { - err := putter.Put(context.Background(), ch) - if err != nil { - t.Fatal(err) - } - } - - time.Sleep(time.Second) - - t.Run("reserve size", reserveSizeTest(st.Reserve(), 2*chunkCountPerPO*maxPO)) - - // Now we generate another sample with the older timestamp. This should give us - // the exact same sample, ensuring that none of the later chunks were considered. - t.Run("reserve sample 2", func(t *testing.T) { - sample, err := st.ReserveSample(context.TODO(), []byte("anchor"), 5, timeVar, nil) - if err != nil { - t.Fatal(err) - } - - if diff := cmp.Diff(sample.Items, sample1.Items, cmp.AllowUnexported(postage.Stamp{})); diff != "" { - t.Fatalf("samples different (-want +have):\n%s", diff) - } - - if sample.Stats.NewIgnored == 0 { - t.Fatalf("sample should have some ignored chunks") - } - - assertSampleNoErrors(t, sample) - }) - - } - - t.Run("disk", func(t *testing.T) { - t.Parallel() - baseAddr := swarm.RandAddress(t) - opts := dbTestOps(baseAddr, 1000, nil, nil, time.Second) - opts.ValidStamp = func(ch swarm.Chunk) (swarm.Chunk, error) { return ch, nil } - - storer, err := diskStorer(t, opts)() - if err != nil { - t.Fatal(err) - } - testF(t, baseAddr, storer) - }) - t.Run("mem", func(t *testing.T) { - t.Parallel() - baseAddr := swarm.RandAddress(t) - opts := dbTestOps(baseAddr, 1000, nil, nil, time.Second) - opts.ValidStamp = func(ch swarm.Chunk) (swarm.Chunk, error) { return ch, nil } - - storer, err := memStorer(t, opts)() - if err != nil { - t.Fatal(err) - } - testF(t, baseAddr, storer) - }) -} - -func TestRandSample(t *testing.T) { - t.Parallel() - - sample := storer.RandSample(t, nil) - assertValidSample(t, sample) -} - -func assertValidSample(t *testing.T, sample storer.Sample) { - t.Helper() - - // Assert that sample size is exactly storer.SampleSize - if len(sample.Items) != storer.SampleSize { - t.Fatalf("incorrect no of sample items, exp %d found %d", storer.SampleSize, len(sample.Items)) - } - - // Assert that sample item has all fields set - assertSampleItem := func(item storer.SampleItem, i int) { - if !item.TransformedAddress.IsValidNonEmpty() { - t.Fatalf("sample item [%d]: transformed address should be set", i) - } - if !item.ChunkAddress.IsValidNonEmpty() { - t.Fatalf("sample item [%d]: chunk address should be set", i) - } - if item.ChunkData == nil { - t.Fatalf("sample item [%d]: chunk data should be set", i) - } - if item.Stamp == nil { - t.Fatalf("sample item [%d]: stamp should be set", i) - } - } - for i, item := range sample.Items { - assertSampleItem(item, i) - } - - // Assert that transformed addresses are in ascending order - for i := 0; i < len(sample.Items)-1; i++ { - if sample.Items[i].TransformedAddress.Compare(sample.Items[i+1].TransformedAddress) != -1 { - t.Fatalf("incorrect order of samples") - } - } -} - -func assertSampleNoErrors(t *testing.T, sample storer.Sample) { - t.Helper() - - if sample.Stats.ChunkLoadFailed != 0 { - t.Fatalf("got unexpected failed chunk loads") - } - if sample.Stats.RogueChunk != 0 { - t.Fatalf("got unexpected rouge chunks") - } - if sample.Stats.StampLoadFailed != 0 { - t.Fatalf("got unexpected failed stamp loads") - } - if sample.Stats.InvalidStamp != 0 { - t.Fatalf("got unexpected invalid stamps") - } -} diff --git a/pkg/storer/sampler.go b/pkg/storer/sampler.go new file mode 100644 index 00000000000..b9a1592ce84 --- /dev/null +++ b/pkg/storer/sampler.go @@ -0,0 +1,50 @@ +// Copyright 2023 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package storer + +import ( + "context" + + "github.com/ethersphere/bee/pkg/postage" + "github.com/ethersphere/bee/pkg/storer/internal/chunkstamp" + "github.com/ethersphere/bee/pkg/storer/internal/reserve" + "github.com/ethersphere/bee/pkg/swarm" +) + +// Sampler interface providing the iterator to iterate through the reserve +type Sampler interface { + Iterate(depth uint8, f func(Chunk) (bool, error)) error + StorageRadius() uint8 +} + +// Chunk serves to reify the partial info about a chunk coming from indexstore +// the Chunk and Stamp functions allow lazy loading of chunk data and/or postage stamp +type Chunk struct { + Address swarm.Address + BatchID []byte + Type swarm.ChunkType + db *DB +} + +// Chunk returns the swarm.Chunk +func (c *Chunk) Chunk(ctx context.Context) (swarm.Chunk, error) { + return c.db.ChunkStore().Get(ctx, c.Address) +} + +// Stamp returns the postage stamp for the chunk +func (c *Chunk) Stamp() (*postage.Stamp, error) { + s, err := chunkstamp.LoadWithBatchID(c.db.repo.IndexStore(), "reserve", c.Address, c.BatchID) + if err != nil { + return nil, err + } + return postage.NewStamp(c.BatchID, s.Index(), s.Timestamp(), s.Sig()), nil +} + +// Iterate iterates through the reserve and applies f to all chunks at and above the PO depth +func (db *DB) Iterate(depth uint8, f func(Chunk) (bool, error)) error { + return db.reserve.IterateChunksItems(db.repo, depth, func(chi reserve.ChunkItem) (bool, error) { + return f(Chunk{chi.ChunkAddress, chi.BatchID, chi.Type, db}) + }) +} diff --git a/pkg/storer/sampler_test.go b/pkg/storer/sampler_test.go new file mode 100644 index 00000000000..5b6b6952970 --- /dev/null +++ b/pkg/storer/sampler_test.go @@ -0,0 +1,193 @@ +// Copyright 2023 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package storer_test + +import ( + "context" + "math/rand" + "testing" + "time" + + postagetesting "github.com/ethersphere/bee/pkg/postage/testing" + chunk "github.com/ethersphere/bee/pkg/storage/testing" + "github.com/ethersphere/bee/pkg/storer" + "github.com/ethersphere/bee/pkg/swarm" +) + +func TestReserveSampler(t *testing.T) { + const chunkCountPerPO = 10 + const maxPO = 10 + + randChunks := func(baseAddr swarm.Address, timeVar uint64) []swarm.Chunk { + var chs []swarm.Chunk + for po := 0; po < maxPO; po++ { + for i := 0; i < chunkCountPerPO; i++ { + ch := chunk.GenerateValidRandomChunkAt(baseAddr, po).WithBatch(0, 3, 2, false) + if rand.Intn(2) == 0 { // 50% chance to wrap CAC into SOC + ch = chunk.GenerateTestRandomSoChunk(t, ch) + } + + // override stamp timestamp to be before the consensus timestamp + ch = ch.WithStamp(postagetesting.MustNewStampWithTimestamp(timeVar)) + chs = append(chs, ch) + } + } + return chs + } + + testF := func(t *testing.T, baseAddr swarm.Address, st *storer.DB) { + t.Helper() + + timeVar := uint64(time.Now().UnixNano()) + chs := randChunks(baseAddr, timeVar-1) + + putter := st.ReservePutter() + for _, ch := range chs { + err := putter.Put(context.Background(), ch) + if err != nil { + t.Fatal(err) + } + } + + t.Run("reserve size", reserveSizeTest(st.Reserve(), chunkCountPerPO*maxPO)) + + // var sample1 storer.Sample + + // t.Run("reserve sample 1", func(t *testing.T) { + // sample, err := st.ReserveSample(context.TODO(), []byte("anchor"), 5, timeVar, nil) + // if err != nil { + // t.Fatal(err) + // } + + // assertValidSample(t, sample) + // assertSampleNoErrors(t, sample) + + // if sample.Stats.NewIgnored != 0 { + // t.Fatalf("sample should not have ignored chunks") + // } + + // sample1 = sample + // }) + + // // We generate another 100 chunks. With these new chunks in the reserve, statistically + // // some of them should definitely make it to the sample based on lex ordering. + // chs = randChunks(baseAddr, timeVar+1) + // putter = st.ReservePutter() + // for _, ch := range chs { + // err := putter.Put(context.Background(), ch) + // if err != nil { + // t.Fatal(err) + // } + // } + + // time.Sleep(time.Second) + + // t.Run("reserve size", reserveSizeTest(st.Reserve(), 2*chunkCountPerPO*maxPO)) + + // // Now we generate another sample with the older timestamp. This should give us + // // the exact same sample, ensuring that none of the later chunks were considered. + // t.Run("reserve sample 2", func(t *testing.T) { + // sample, err := st.ReserveSample(context.TODO(), []byte("anchor"), 5, timeVar, nil) + // if err != nil { + // t.Fatal(err) + // } + + // if diff := cmp.Diff(sample.Items, sample1.Items, cmp.AllowUnexported(postage.Stamp{})); diff != "" { + // t.Fatalf("samples different (-want +have):\n%s", diff) + // } + + // if sample.Stats.NewIgnored == 0 { + // t.Fatalf("sample should have some ignored chunks") + // } + + // assertSampleNoErrors(t, sample) + // }) + + } + + t.Run("disk", func(t *testing.T) { + t.Parallel() + baseAddr := swarm.RandAddress(t) + opts := dbTestOps(baseAddr, 1000, nil, nil, time.Second) + opts.ValidStamp = func(ch swarm.Chunk) (swarm.Chunk, error) { return ch, nil } + + storer, err := diskStorer(t, opts)() + if err != nil { + t.Fatal(err) + } + testF(t, baseAddr, storer) + }) + t.Run("mem", func(t *testing.T) { + t.Parallel() + baseAddr := swarm.RandAddress(t) + opts := dbTestOps(baseAddr, 1000, nil, nil, time.Second) + opts.ValidStamp = func(ch swarm.Chunk) (swarm.Chunk, error) { return ch, nil } + + storer, err := memStorer(t, opts)() + if err != nil { + t.Fatal(err) + } + testF(t, baseAddr, storer) + }) +} + +// func TestRandSample(t *testing.T) { +// t.Parallel() + +// sample := storer.RandSample(t, nil) +// assertValidSample(t, sample) +// } + +// func assertValidSample(t *testing.T, sample storer.Sample) { +// t.Helper() + +// // Assert that sample size is exactly storer.SampleSize +// if len(sample.Items) != storer.SampleSize { +// t.Fatalf("incorrect no of sample items, exp %d found %d", storer.SampleSize, len(sample.Items)) +// } + +// // Assert that sample item has all fields set +// assertSampleItem := func(item storer.SampleItem, i int) { +// if !item.TransformedAddress.IsValidNonEmpty() { +// t.Fatalf("sample item [%d]: transformed address should be set", i) +// } +// if !item.ChunkAddress.IsValidNonEmpty() { +// t.Fatalf("sample item [%d]: chunk address should be set", i) +// } +// if item.ChunkData == nil { +// t.Fatalf("sample item [%d]: chunk data should be set", i) +// } +// if item.Stamp == nil { +// t.Fatalf("sample item [%d]: stamp should be set", i) +// } +// } +// for i, item := range sample.Items { +// assertSampleItem(item, i) +// } + +// // Assert that transformed addresses are in ascending order +// for i := 0; i < len(sample.Items)-1; i++ { +// if sample.Items[i].TransformedAddress.Compare(sample.Items[i+1].TransformedAddress) != -1 { +// t.Fatalf("incorrect order of samples") +// } +// } +// } + +// func assertSampleNoErrors(t *testing.T, sample storer.Sample) { +// t.Helper() + +// if sample.Stats.ChunkLoadFailed != 0 { +// t.Fatalf("got unexpected failed chunk loads") +// } +// if sample.Stats.RogueChunk != 0 { +// t.Fatalf("got unexpected rouge chunks") +// } +// if sample.Stats.StampLoadFailed != 0 { +// t.Fatalf("got unexpected failed stamp loads") +// } +// if sample.Stats.InvalidStamp != 0 { +// t.Fatalf("got unexpected invalid stamps") +// } +// } diff --git a/pkg/storer/storer.go b/pkg/storer/storer.go index 771889a9689..c74c4e177fa 100644 --- a/pkg/storer/storer.go +++ b/pkg/storer/storer.go @@ -10,7 +10,6 @@ import ( "fmt" "io" "io/fs" - "math/big" "os" "path" "path/filepath" @@ -139,7 +138,6 @@ var _ Reserve = (*DB)(nil) type Reserve interface { ReserveStore EvictBatch(ctx context.Context, batchID []byte) error - ReserveSample(context.Context, []byte, uint8, uint64, *big.Int) (Sample, error) ReserveSize() int } From c6038b92bed570da0a9925596d649030e84ee358 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Thu, 5 Oct 2023 11:45:20 +0200 Subject: [PATCH 09/24] test: abi path --- pkg/storageincentives/redistribution/redistribution_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/storageincentives/redistribution/redistribution_test.go b/pkg/storageincentives/redistribution/redistribution_test.go index 688c52c0689..595caff9a34 100644 --- a/pkg/storageincentives/redistribution/redistribution_test.go +++ b/pkg/storageincentives/redistribution/redistribution_test.go @@ -15,6 +15,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethersphere/bee/pkg/config" "github.com/ethersphere/bee/pkg/log" "github.com/ethersphere/bee/pkg/sctx" "github.com/ethersphere/bee/pkg/storageincentives/redistribution" @@ -25,8 +26,7 @@ import ( "github.com/ethersphere/bee/pkg/util/testutil" ) -// var redistributionContractABI = abiutil.MustParseABI(chaincfg.Testnet.RedistributionABI) -var redistributionContractABI = abiutil.MustParseABI(redistribution.TestnetRedistributionABI) +var redistributionContractABI = abiutil.MustParseABI(config.Testnet.RedistributionABI) func randomHashes(t testing.TB, n int) []common.Hash { rhs := make([]common.Hash, n) From 02806ee42f1e8cfac6e867acaa60be2806a542c3 Mon Sep 17 00:00:00 2001 From: istae <14264581+istae@users.noreply.github.com> Date: Thu, 5 Oct 2023 13:35:12 +0300 Subject: [PATCH 10/24] fix: updated contract abi version and docker setup versions --- .github/workflows/beekeeper.yml | 2 +- go.mod | 2 +- go.sum | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/beekeeper.yml b/.github/workflows/beekeeper.yml index 01c83d3e233..c4dcef2884d 100644 --- a/.github/workflows/beekeeper.yml +++ b/.github/workflows/beekeeper.yml @@ -11,7 +11,7 @@ env: K3S_VERSION: "v1.22.17+k3s1" REPLICA: 3 RUN_TYPE: "PR RUN" - SETUP_CONTRACT_IMAGE_TAG: "0.9.9" + SETUP_CONTRACT_IMAGE_TAG: "1.0.3" BEELOCAL_BRANCH: "main" BEEKEEPER_BRANCH: "master" BEEKEEPER_METRICS_ENABLED: false diff --git a/go.mod b/go.mod index d9b4860f937..a95616bfed9 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/coreos/go-semver v0.3.0 github.com/ethereum/go-ethereum v1.12.2 github.com/ethersphere/go-price-oracle-abi v0.1.0 - github.com/ethersphere/go-storage-incentives-abi v0.6.0-rc5 + github.com/ethersphere/go-storage-incentives-abi v0.6.0-rc7 github.com/ethersphere/go-sw3-abi v0.4.0 github.com/ethersphere/langos v1.0.0 github.com/go-playground/validator/v10 v10.11.1 diff --git a/go.sum b/go.sum index 40d8731ff09..9689d88f2f7 100644 --- a/go.sum +++ b/go.sum @@ -225,6 +225,8 @@ github.com/ethersphere/go-price-oracle-abi v0.1.0 h1:yg/hK8nETNvk+GEBASlbakMFv/C github.com/ethersphere/go-price-oracle-abi v0.1.0/go.mod h1:sI/Qj4/zJ23/b1enzwMMv0/hLTpPNVNacEwCWjo6yBk= github.com/ethersphere/go-storage-incentives-abi v0.6.0-rc5 h1:JaiGMFoycByFkaCv6KFg79tfRlBJ5zY01dLWPnf48v0= github.com/ethersphere/go-storage-incentives-abi v0.6.0-rc5/go.mod h1:SXvJVtM4sEsaSKD0jc1ClpDLw8ErPoROZDme4Wrc/Nc= +github.com/ethersphere/go-storage-incentives-abi v0.6.0-rc7 h1:SPg0WY5rCWt7Y72hAj0o1juLtNi9lqx7b/6fNaNa4H0= +github.com/ethersphere/go-storage-incentives-abi v0.6.0-rc7/go.mod h1:SXvJVtM4sEsaSKD0jc1ClpDLw8ErPoROZDme4Wrc/Nc= github.com/ethersphere/go-sw3-abi v0.4.0 h1:T3ANY+ktWrPAwe2U0tZi+DILpkHzto5ym/XwV/Bbz8g= github.com/ethersphere/go-sw3-abi v0.4.0/go.mod h1:BmpsvJ8idQZdYEtWnvxA8POYQ8Rl/NhyCdF0zLMOOJU= github.com/ethersphere/langos v1.0.0 h1:NBtNKzXTTRSue95uOlzPN4py7Aofs0xWPzyj4AI1Vcc= From 8b58d9f94045d954453d33f8b381adb9a8f66ae7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Thu, 5 Oct 2023 12:37:49 +0200 Subject: [PATCH 11/24] fix: claim proof params --- pkg/storageincentives/redistribution/redistribution_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/storageincentives/redistribution/redistribution_test.go b/pkg/storageincentives/redistribution/redistribution_test.go index 595caff9a34..31ebaf9d74f 100644 --- a/pkg/storageincentives/redistribution/redistribution_test.go +++ b/pkg/storageincentives/redistribution/redistribution_test.go @@ -195,7 +195,7 @@ func TestRedistribution(t *testing.T) { proofs := randProofs(t) - expectedCallData, err := redistributionContractABI.Pack("claim", proofs) + expectedCallData, err := redistributionContractABI.Pack("claim", proofs[0], proofs[1], proofs[2]) if err != nil { t.Fatal(err) } @@ -235,7 +235,7 @@ func TestRedistribution(t *testing.T) { t.Parallel() proofs := randProofs(t) - expectedCallData, err := redistributionContractABI.Pack("claim", proofs) + expectedCallData, err := redistributionContractABI.Pack("claim", proofs[0], proofs[1], proofs[2]) if err != nil { t.Fatal(err) } From 7d1d6aa4ba1ab0a1a3c1b199b9a0294b0df2ff36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Thu, 5 Oct 2023 12:41:09 +0200 Subject: [PATCH 12/24] chore: add license header --- pkg/storageincentives/redistribution/abi_types_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/storageincentives/redistribution/abi_types_test.go b/pkg/storageincentives/redistribution/abi_types_test.go index 0ad3e6146c5..0e5e3a23a76 100644 --- a/pkg/storageincentives/redistribution/abi_types_test.go +++ b/pkg/storageincentives/redistribution/abi_types_test.go @@ -1,3 +1,7 @@ +// Copyright 2023 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package redistribution_test -// must include tests for just the contract calldata abi packaging \ No newline at end of file +// must include tests for just the contract calldata abi packaging From 84798a3ad6dc886acf0f2b020acc7ec7ad7b028f Mon Sep 17 00:00:00 2001 From: mrekucci Date: Thu, 5 Oct 2023 18:37:29 +0400 Subject: [PATCH 13/24] feat: include abi custom error params in the returned error (#4376) --- go.mod | 32 +++++---- go.sum | 88 +++++++++++++------------ pkg/transaction/backendmock/backend.go | 12 +++- pkg/transaction/transaction.go | 48 ++++++++++++-- pkg/transaction/transaction_test.go | 89 ++++++++++++++++++++++++++ 5 files changed, 210 insertions(+), 59 deletions(-) diff --git a/go.mod b/go.mod index a95616bfed9..8a633f57ce8 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/btcsuite/btcd v0.22.3 github.com/casbin/casbin/v2 v2.35.0 github.com/coreos/go-semver v0.3.0 - github.com/ethereum/go-ethereum v1.12.2 + github.com/ethereum/go-ethereum v1.13.2 github.com/ethersphere/go-price-oracle-abi v0.1.0 github.com/ethersphere/go-storage-incentives-abi v0.6.0-rc7 github.com/ethersphere/go-sw3-abi v0.4.0 @@ -32,7 +32,7 @@ require ( github.com/opentracing/opentracing-go v1.2.0 github.com/prometheus/client_golang v1.14.0 github.com/spf13/afero v1.6.0 - github.com/spf13/cobra v1.0.0 + github.com/spf13/cobra v1.5.0 github.com/spf13/viper v1.7.0 github.com/stretchr/testify v1.8.1 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 @@ -42,12 +42,12 @@ require ( gitlab.com/nolash/go-mockbytes v0.0.7 go.uber.org/atomic v1.10.0 go.uber.org/goleak v1.1.12 - golang.org/x/crypto v0.9.0 + golang.org/x/crypto v0.12.0 golang.org/x/exp v0.0.0-20230810033253-352e893a4cad golang.org/x/net v0.10.0 golang.org/x/sync v0.3.0 - golang.org/x/sys v0.9.0 - golang.org/x/term v0.8.0 + golang.org/x/sys v0.11.0 + golang.org/x/term v0.11.0 golang.org/x/time v0.3.0 gopkg.in/yaml.v2 v2.4.0 resenje.org/multex v0.1.0 @@ -56,14 +56,23 @@ require ( ) require ( + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/bits-and-blooms/bitset v1.5.0 // indirect + github.com/consensys/bavard v0.1.13 // indirect + github.com/consensys/gnark-crypto v0.10.0 // indirect + github.com/crate-crypto/go-kzg-4844 v0.3.0 // indirect github.com/deckarep/golang-set/v2 v2.1.0 // indirect + github.com/ethereum/c-kzg-4844 v0.3.1 // indirect github.com/holiman/uint256 v1.2.3 // indirect + github.com/mmcloughlin/addchain v0.4.0 // indirect + github.com/supranational/blst v0.3.11 // indirect + rsc.io/tmplfunc v0.0.3 // indirect ) require ( github.com/BurntSushi/toml v1.1.0 // indirect github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible // indirect - github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 // indirect + github.com/StackExchange/wmi v1.2.1 // indirect github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect @@ -97,7 +106,7 @@ require ( github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.4 github.com/hashicorp/hcl v1.0.0 // indirect - github.com/huin/goupnp v1.0.3 // indirect + github.com/huin/goupnp v1.3.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/ipfs/go-log/v2 v2.5.1 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect @@ -154,10 +163,10 @@ require ( github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/cast v1.3.0 // indirect github.com/spf13/jwalterweatherman v1.0.0 // indirect - github.com/spf13/pflag v1.0.3 // indirect + github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.2.0 // indirect - github.com/tklauser/go-sysconf v0.3.6 // indirect - github.com/tklauser/numcpus v0.2.2 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect github.com/uber/jaeger-lib v2.2.0+incompatible // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect github.com/wealdtech/go-multicodec v1.4.0 // indirect @@ -167,11 +176,10 @@ require ( go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.24.0 // indirect golang.org/x/mod v0.11.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/text v0.12.0 // indirect golang.org/x/tools v0.9.1 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/ini.v1 v1.57.0 // indirect - gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.1.7 // indirect nhooyr.io/websocket v1.8.7 // indirect diff --git a/go.sum b/go.sum index 9689d88f2f7..a52e3ef0c9a 100644 --- a/go.sum +++ b/go.sum @@ -63,15 +63,18 @@ github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= -github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= +github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/HdrHistogram/hdrhistogram-go v0.0.0-20200919145931-8dac23c8dac1 h1:nEjGZtKHMK92888VT6XkzKwyiW14v5FFRGeWq2uV7N0= github.com/HdrHistogram/hdrhistogram-go v0.0.0-20200919145931-8dac23c8dac1/go.mod h1:nxrse8/Tzg2tg3DZcZjm6qEclQKK70g0KxO61gFFZD4= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 h1:5sXbqlSomvdjlRbWyNqkPsJ3Fg+tQZCbgeX1VGljbQY= github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= +github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o= github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= @@ -89,7 +92,6 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= @@ -111,7 +113,8 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bits-and-blooms/bitset v1.7.0 h1:YjAGVd3XmtK9ktAbX8Zg2g2PwLIMjGREZJHlV4j7NEo= +github.com/bits-and-blooms/bitset v1.5.0 h1:NpE8frKRLGHIcEzkR+gZhiioW1+WbYV6fKwD6ZIpQT8= +github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= @@ -155,21 +158,22 @@ github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8= -github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= -github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 h1:ytcWPaNPhNoGMWEhDvS3zToKcDpRsLuRolQJBVGdozk= -github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= +github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQu+Y= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= +github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06 h1:T+Np/xtzIjYM/P5NAw0e2Rf1FGvzDau1h54MKvx8G7w= +github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw= +github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= +github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= github.com/consensys/gnark-crypto v0.10.0 h1:zRh22SR7o4K35SoNqouS9J/TKHTyU2QWaj5ldehyXtA= +github.com/consensys/gnark-crypto v0.10.0/go.mod h1:Iq/P3HHl0ElSjsg2E1gsMwhAyxnxoKK5nVyZKd+/KhU= github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.0.4 h1:jN/mbWBEaz+T1pi5OFtnkQ+8qnmEbAr1Oo1FRm5B0dA= github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -181,7 +185,9 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crate-crypto/go-kzg-4844 v0.3.0 h1:UBlWE0CgyFqqzTI+IFyCzA7A3Zw4iip6uzRv5NIXG0A= +github.com/crate-crypto/go-kzg-4844 v0.3.0/go.mod h1:SBP7ikXEgDnUPONgm33HtuDZEDtWa3L4QtN1ocJSEQ4= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -218,9 +224,10 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ethereum/c-kzg-4844 v0.3.1 h1:sR65+68+WdnMKxseNWxSJuAv2tsUrihTpVBTfM/U5Zg= +github.com/ethereum/c-kzg-4844 v0.3.1/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/ethereum/go-ethereum v1.10.4/go.mod h1:nEE0TP5MtxGzOMd7egIrbPJMQBnhVU3ELNxhBglIzhg= -github.com/ethereum/go-ethereum v1.12.2 h1:eGHJ4ij7oyVqUQn48LBz3B7pvQ8sV0wGJiIE6gDq/6Y= -github.com/ethereum/go-ethereum v1.12.2/go.mod h1:1cRAEV+rp/xX0zraSCBnu9Py3HQ+geRMj3HdR+k0wfI= +github.com/ethereum/go-ethereum v1.13.2 h1:g9mCpfPWqCA1OL4e6C98PeVttb0HadfBRuKTGvMnOvw= +github.com/ethereum/go-ethereum v1.13.2/go.mod h1:gkQ5Ygi64ZBh9M/4iXY1R8WqoNCx1Ey0CkYn2BD4/fw= github.com/ethersphere/go-price-oracle-abi v0.1.0 h1:yg/hK8nETNvk+GEBASlbakMFv/CVp7HXiycrHw1pRV8= github.com/ethersphere/go-price-oracle-abi v0.1.0/go.mod h1:sI/Qj4/zJ23/b1enzwMMv0/hLTpPNVNacEwCWjo6yBk= github.com/ethersphere/go-storage-incentives-abi v0.6.0-rc5 h1:JaiGMFoycByFkaCv6KFg79tfRlBJ5zY01dLWPnf48v0= @@ -247,7 +254,6 @@ github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbS github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 h1:f6D9Hr8xV8uYKlyuj8XIruxlh9WjVjdh1gIicAS7ays= github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= -github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= @@ -389,6 +395,7 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20230207041349-798e818bf904 h1:4/hN5RUoecvl+RmJRE2YxKWtnnQls6rQjjW5oV7qg2U= github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -405,7 +412,6 @@ github.com/gorilla/handlers v1.4.2 h1:0QniY0USkHQ1RGCLfKxeNHK9bkDHGRYGNDFBCS+YAR github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= @@ -454,8 +460,8 @@ github.com/holiman/uint256 v1.2.3/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZm github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goupnp v1.0.1-0.20210310174557-0ca763054c88/go.mod h1:nNs7wvRfN1eKaMknBydLNQU6146XQim8t4h+q90biWo= -github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= -github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= +github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= +github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= @@ -535,8 +541,8 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -544,6 +550,7 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= @@ -575,7 +582,6 @@ github.com/libp2p/go-yamux/v4 v4.0.0 h1:+Y80dV2Yx/kv7Y7JKu0LECyVdMXm1VUoko+VQ9rB github.com/libp2p/go-yamux/v4 v4.0.0/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5wwmtQP1YB4= github.com/lucas-clemente/quic-go v0.15.2/go.mod h1:qxmO5Y4ZMhdNkunGfxuZnZXnJwYpW9vjQkyrZ7BsgUI= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -625,7 +631,6 @@ github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= @@ -635,6 +640,8 @@ github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxd github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= +github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= +github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -807,14 +814,15 @@ github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= @@ -867,13 +875,13 @@ github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU= +github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= @@ -898,6 +906,7 @@ github.com/stvp/go-udp-testing v0.0.0-20201019212854-469649b16807/go.mod h1:7jxm github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= +github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= @@ -907,10 +916,12 @@ github.com/tdewolff/parse/v2 v2.4.2/go.mod h1:WzaJpRSbwq++EIQHYIRTpbYKNA3gn9it1I github.com/tdewolff/test v1.0.6/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= -github.com/tklauser/go-sysconf v0.3.6 h1:oc1sJWvKkmvIxhDHeKWvZS4f6AW+YcoguSfRF2/Hmo4= github.com/tklauser/go-sysconf v0.3.6/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= -github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= @@ -919,7 +930,6 @@ github.com/uber/jaeger-client-go v2.24.0+incompatible h1:CGchgJcHsDd2jWnaL4XngBy github.com/uber/jaeger-client-go v2.24.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw= github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= @@ -927,7 +937,7 @@ github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLY github.com/urfave/cli v1.22.2 h1:gsqYFH8bb9ekPA12kRo0hfjngWQjkJPlN9R0N78BoUo= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= -github.com/urfave/cli/v2 v2.24.1 h1:/QYYr7g0EhwXEML8jO+8OYt5trPnLHS0p3mrgExJ5NU= +github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/vmihailenco/msgpack/v5 v5.3.4 h1:qMKAwOV+meBw2Y8k9cVwAy7qErtYCwBzZ2ellBfvnqc= @@ -943,7 +953,6 @@ github.com/wealdtech/go-string2eth v1.1.0/go.mod h1:RUzsLjJtbZaJ/3UKn9kY19a/vCCU github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1008,8 +1017,8 @@ golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1070,7 +1079,6 @@ golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1211,13 +1219,14 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1226,8 +1235,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1369,7 +1378,6 @@ google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9M google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= @@ -1411,7 +1419,6 @@ gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mail.v2 v2.3.1/go.mod h1:htwXN1Qh09vZJ1NVKxQqHPBaCBbzKhp5GzuJEA4VJWw= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= -gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= @@ -1465,5 +1472,6 @@ rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= +rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/pkg/transaction/backendmock/backend.go b/pkg/transaction/backendmock/backend.go index 8ce38614b6e..7f787fe3d60 100644 --- a/pkg/transaction/backendmock/backend.go +++ b/pkg/transaction/backendmock/backend.go @@ -17,6 +17,7 @@ import ( type backendMock struct { codeAt func(ctx context.Context, contract common.Address, blockNumber *big.Int) ([]byte, error) + callContract func(ctx context.Context, call ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) sendTransaction func(ctx context.Context, tx *types.Transaction) error suggestGasPrice func(ctx context.Context) (*big.Int, error) suggestGasTipCap func(ctx context.Context) (*big.Int, error) @@ -38,7 +39,10 @@ func (m *backendMock) CodeAt(ctx context.Context, contract common.Address, block return nil, errors.New("not implemented") } -func (*backendMock) CallContract(ctx context.Context, call ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) { +func (m *backendMock) CallContract(ctx context.Context, call ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) { + if m.callContract != nil { + return m.callContract(ctx, call, blockNumber) + } return nil, errors.New("not implemented") } @@ -161,6 +165,12 @@ type optionFunc func(*backendMock) func (f optionFunc) apply(r *backendMock) { f(r) } +func WithCallContractFunc(f func(ctx context.Context, call ethereum.CallMsg, blockNumber *big.Int) ([]byte, error)) Option { + return optionFunc(func(s *backendMock) { + s.callContract = f + }) +} + func WithCodeAtFunc(f func(ctx context.Context, contract common.Address, blockNumber *big.Int) ([]byte, error)) Option { return optionFunc(func(s *backendMock) { s.codeAt = f diff --git a/pkg/transaction/transaction.go b/pkg/transaction/transaction.go index 28d3912c78d..d16494ba7b6 100644 --- a/pkg/transaction/transaction.go +++ b/pkg/transaction/transaction.go @@ -18,6 +18,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rpc" "github.com/ethersphere/bee/pkg/crypto" "github.com/ethersphere/bee/pkg/log" "github.com/ethersphere/bee/pkg/sctx" @@ -596,20 +597,55 @@ func (t *transactionService) UnwrapABIError(ctx context.Context, req *TxRequest, return nil } - res, cErr := t.Call(ctx, req) - if cErr != nil { + _, cErr := t.Call(ctx, req) + if cErr == nil { return err } + err = fmt.Errorf("%w: %s", err, cErr) - if reason, uErr := abi.UnpackRevert(res); uErr == nil { + var derr rpc.DataError + if !errors.As(cErr, &derr) { + return err + } + + res, ok := derr.ErrorData().(string) + if !ok { + return err + } + buf := common.FromHex(res) + + if reason, uErr := abi.UnpackRevert(buf); uErr == nil { return fmt.Errorf("%w: %s", err, reason) } for _, abiError := range abiErrors { - if bytes.Equal(res[:4], abiError.ID[:4]) { - //abiError.Unpack(res[4:]) - return fmt.Errorf("%w: %s", err, abiError) + if !bytes.Equal(buf[:4], abiError.ID[:4]) { + continue + } + + data, uErr := abiError.Unpack(buf) + if uErr != nil { + continue } + + values, ok := data.([]interface{}) + if !ok { + values = make([]interface{}, len(abiError.Inputs)) + for i := range values { + values[i] = "?" + } + } + + params := make([]string, len(abiError.Inputs)) + for i, input := range abiError.Inputs { + if input.Name == "" { + input.Name = fmt.Sprintf("arg%d", i) + } + params[i] = fmt.Sprintf("%s=%v", input.Name, values[i]) + + } + + return fmt.Errorf("%w: %s(%s)", err, abiError.Name, strings.Join(params, ",")) } return err diff --git a/pkg/transaction/transaction_test.go b/pkg/transaction/transaction_test.go index 8794fd33281..e00e061cd62 100644 --- a/pkg/transaction/transaction_test.go +++ b/pkg/transaction/transaction_test.go @@ -10,11 +10,14 @@ import ( "errors" "fmt" "math/big" + "strings" "testing" + "time" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rpc" "github.com/ethersphere/bee/pkg/crypto" signermock "github.com/ethersphere/bee/pkg/crypto/mock" "github.com/ethersphere/bee/pkg/log" @@ -23,6 +26,7 @@ import ( "github.com/ethersphere/bee/pkg/transaction" "github.com/ethersphere/bee/pkg/transaction/backendmock" "github.com/ethersphere/bee/pkg/transaction/monitormock" + "github.com/ethersphere/bee/pkg/util/abiutil" "github.com/ethersphere/bee/pkg/util/testutil" ) @@ -898,3 +902,88 @@ func TestTransactionCancel(t *testing.T) { } }) } + +// rpcAPIError is a copy of engine.EngineAPIError from go-ethereum pkg. +type rpcAPIError struct { + code int + msg string + err string +} + +func (e *rpcAPIError) ErrorCode() int { return e.code } +func (e *rpcAPIError) Error() string { return e.msg } +func (e *rpcAPIError) ErrorData() interface{} { return e.err } + +var _ rpc.DataError = (*rpcAPIError)(nil) + +func TestTransactionService_UnwrapABIError(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + var ( + recipient = common.HexToAddress("0xbbbddd") + chainID = big.NewInt(5) + nonce = uint64(10) + gasTip = big.NewInt(100) + gasFee = big.NewInt(1100) + txData = common.Hex2Bytes("0xabcdee") + value = big.NewInt(1) + + // This is the ABI of the following contract: https://goerli.etherscan.io/address/0xd29d9e385f19d888557cd609006bb1934cb5d1e2#code + contractABI = abiutil.MustParseABI(`[{"inputs":[{"internalType":"uint256","name":"available","type":"uint256"},{"internalType":"uint256","name":"required","type":"uint256"}],"name":"InsufficientBalance","type":"error"},{"inputs":[{"internalType":"address","name":"to","type":"address"},{"internalType":"uint256","name":"amount","type":"uint256"}],"name":"transfer","outputs":[],"stateMutability":"nonpayable","type":"function"}]`) + rpcAPIErr = &rpcAPIError{ + code: 3, + msg: "execution reverted", + err: "0xcf4791810000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000006f", // This is the ABI encoded error form the following failed transaction: https://goerli.etherscan.io/tx/0x74a2577db1c325c41e38977aa1eb32ab03dfa17cc1fa0649e84f3d8c0f0882ee + } + ) + + gasTipCap := new(big.Int).Div(new(big.Int).Mul(big.NewInt(int64(10)+100), gasTip), big.NewInt(100)) + gasFeeCap := new(big.Int).Add(gasFee, gasTipCap) + + signedTx := types.NewTx(&types.DynamicFeeTx{ + ChainID: chainID, + Nonce: nonce, + To: &recipient, + Value: value, + Gas: 21000, + GasTipCap: gasTipCap, + GasFeeCap: gasFeeCap, + Data: txData, + }) + request := &transaction.TxRequest{ + To: &recipient, + Data: txData, + Value: value, + } + + transactionService, err := transaction.NewService(log.Noop, + backendmock.New( + backendmock.WithCallContractFunc(func(ctx context.Context, call ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) { + return nil, rpcAPIErr + }), + ), + signerMockForTransaction(t, signedTx, recipient, chainID), + storemock.NewStateStore(), + chainID, + monitormock.New(), + ) + if err != nil { + t.Fatal(err) + } + testutil.CleanupCloser(t, transactionService) + + originErr := errors.New("origin error") + wrappedErr := transactionService.UnwrapABIError(ctx, request, originErr, contractABI.Errors) + if !errors.Is(wrappedErr, originErr) { + t.Fatal("origin error not wrapped") + } + if !strings.Contains(wrappedErr.Error(), rpcAPIErr.Error()) { + t.Fatal("wrapped error without rpc api main error") + } + if !strings.Contains(wrappedErr.Error(), "InsufficientBalance(available=0,required=111)") { + t.Fatal("wrapped error without rpc api error data") + } +} From f8e7ea98aceb5e9e6080e86cc1c8e777a1a919e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Thu, 5 Oct 2023 17:10:08 +0200 Subject: [PATCH 14/24] feat: json definitions --- .../redistribution/abi_types.go | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/pkg/storageincentives/redistribution/abi_types.go b/pkg/storageincentives/redistribution/abi_types.go index 1b3cfb2d8b9..6ed30ffbd31 100644 --- a/pkg/storageincentives/redistribution/abi_types.go +++ b/pkg/storageincentives/redistribution/abi_types.go @@ -19,32 +19,32 @@ import ( // corresponding structure (of the same name) in Redistribution.sol smart contract. // github.com/ethersphere/storage-incentives/blob/master/src/Redistribution.sol type Proof struct { - Sisters []common.Hash - Data common.Hash - Sisters2 []common.Hash - Data2 common.Hash - Sisters3 []common.Hash - ChunkSpan uint64 - PostageProof PostageProof - SocProof []SOCProof + Sisters []common.Hash `json:"proofSegments"` + Data common.Hash `json:"proveSegment"` + Sisters2 []common.Hash `json:"proofSegments2"` + Data2 common.Hash `json:"proveSegment2"` + Sisters3 []common.Hash `json:"proofSegments3"` + ChunkSpan uint64 `json:"chunkSpan"` + PostageProof PostageProof `json:"postageProof"` + SocProof []SOCProof `json:"socProof"` } // SOCProof structure must exactly match // corresponding structure (of the same name) in Redistribution.sol smart contract. type PostageProof struct { - Signature []byte - BatchId common.Hash - Index uint64 - TimeStamp uint64 + Signature []byte `json:"signature"` + BatchId common.Hash `json:"batchId"` + Index uint64 `json:"index"` + TimeStamp uint64 `json:"timeStamp"` } // SOCProof structure must exactly match // corresponding structure (of the same name) in Redistribution.sol smart contract. type SOCProof struct { - Signer []byte - Signature []byte - Identifier common.Hash - ChunkAddr common.Hash + Signer []byte `json:"signer"` + Signature []byte `json:"signature"` + Identifier common.Hash `json:"identifier"` + ChunkAddr common.Hash `json:"chunkAddr"` } func bytes32(bs ...[]byte) []common.Hash { From 31d5ec5af1a04b5641ad3f3f17464a20be32268b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Thu, 5 Oct 2023 17:10:37 +0200 Subject: [PATCH 15/24] test: correct data types --- pkg/storageincentives/redistribution/redistribution_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/storageincentives/redistribution/redistribution_test.go b/pkg/storageincentives/redistribution/redistribution_test.go index 31ebaf9d74f..57d531cc034 100644 --- a/pkg/storageincentives/redistribution/redistribution_test.go +++ b/pkg/storageincentives/redistribution/redistribution_test.go @@ -276,7 +276,7 @@ func TestRedistribution(t *testing.T) { var obfus [32]byte testobfus := common.Hex2Bytes("hash") copy(obfus[:], testobfus) - expectedCallData, err := redistributionContractABI.Pack("commit", obfus, common.BytesToHash(owner.Bytes()), uint32(0)) + expectedCallData, err := redistributionContractABI.Pack("commit", obfus, common.BytesToHash(owner.Bytes()), uint64(0)) if err != nil { t.Fatal(err) } @@ -411,7 +411,7 @@ func TestRedistribution(t *testing.T) { t.Run("invalid call data", func(t *testing.T) { t.Parallel() - expectedCallData, err := redistributionContractABI.Pack("commit", common.BytesToHash(common.Hex2Bytes("some hash")), common.BytesToHash(common.Hex2Bytes("some address")), uint32(0)) + expectedCallData, err := redistributionContractABI.Pack("commit", common.BytesToHash(common.Hex2Bytes("some hash")), common.BytesToHash(common.Hex2Bytes("some address")), uint64(0)) if err != nil { t.Fatal(err) } From ed8d7f692708115940719d4ce0756c03ab8f634e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Thu, 5 Oct 2023 17:41:58 +0200 Subject: [PATCH 16/24] fix: uint32 to uint64 on Commit --- pkg/api/api_test.go | 2 +- pkg/storageincentives/agent.go | 2 +- pkg/storageincentives/agent_test.go | 2 +- pkg/storageincentives/redistribution/redistribution.go | 4 ++-- pkg/storageincentives/redistribution/redistribution_test.go | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/api/api_test.go b/pkg/api/api_test.go index 53309f1d63e..a2318ef2fae 100644 --- a/pkg/api/api_test.go +++ b/pkg/api/api_test.go @@ -791,7 +791,7 @@ func (m *mockContract) Claim(context.Context, []redistribution.Proof) (common.Ha return common.Hash{}, nil } -func (m *mockContract) Commit(context.Context, []byte, uint32) (common.Hash, error) { +func (m *mockContract) Commit(context.Context, []byte, uint64) (common.Hash, error) { m.mtx.Lock() defer m.mtx.Unlock() m.callsList = append(m.callsList, commitCall) diff --git a/pkg/storageincentives/agent.go b/pkg/storageincentives/agent.go index a0669e97875..850f16c994e 100644 --- a/pkg/storageincentives/agent.go +++ b/pkg/storageincentives/agent.go @@ -450,7 +450,7 @@ func (a *Agent) commit(ctx context.Context, sample sampler.Data, round uint64) e return err } - txHash, err := a.contract.Commit(ctx, obfuscatedHash, uint32(round)) + txHash, err := a.contract.Commit(ctx, obfuscatedHash, uint64(round)) if err != nil { a.metrics.ErrCommit.Inc() return err diff --git a/pkg/storageincentives/agent_test.go b/pkg/storageincentives/agent_test.go index d0951479789..e8fafcef67a 100644 --- a/pkg/storageincentives/agent_test.go +++ b/pkg/storageincentives/agent_test.go @@ -261,7 +261,7 @@ func (m *mockContract) Claim(context.Context, []redistribution.Proof) (common.Ha return common.Hash{}, nil } -func (m *mockContract) Commit(context.Context, []byte, uint32) (common.Hash, error) { +func (m *mockContract) Commit(context.Context, []byte, uint64) (common.Hash, error) { m.mtx.Lock() defer m.mtx.Unlock() m.callsList = append(m.callsList, commitCall) diff --git a/pkg/storageincentives/redistribution/redistribution.go b/pkg/storageincentives/redistribution/redistribution.go index d63273434d4..91dfa9ba83c 100644 --- a/pkg/storageincentives/redistribution/redistribution.go +++ b/pkg/storageincentives/redistribution/redistribution.go @@ -24,7 +24,7 @@ type Contract interface { IsPlaying(context.Context, uint8) (bool, error) IsWinner(context.Context) (bool, error) Claim(context.Context, []Proof) (common.Hash, error) - Commit(context.Context, []byte, uint32) (common.Hash, error) + Commit(context.Context, []byte, uint64) (common.Hash, error) Reveal(context.Context, uint8, []byte, []byte) (common.Hash, error) } @@ -115,7 +115,7 @@ func (c *contract) Claim(ctx context.Context, proofs []Proof) (common.Hash, erro } // Commit submits the obfusHash hash by sending a transaction to the blockchain. -func (c *contract) Commit(ctx context.Context, obfusHash []byte, round uint32) (common.Hash, error) { +func (c *contract) Commit(ctx context.Context, obfusHash []byte, round uint64) (common.Hash, error) { callData, err := c.incentivesContractABI.Pack("commit", common.BytesToHash(obfusHash), common.BytesToHash(c.overlay.Bytes()), round) if err != nil { return common.Hash{}, err diff --git a/pkg/storageincentives/redistribution/redistribution_test.go b/pkg/storageincentives/redistribution/redistribution_test.go index 57d531cc034..4775d822349 100644 --- a/pkg/storageincentives/redistribution/redistribution_test.go +++ b/pkg/storageincentives/redistribution/redistribution_test.go @@ -306,7 +306,7 @@ func TestRedistribution(t *testing.T) { redistributionContractABI, ) - _, err = contract.Commit(ctx, testobfus, uint32(0)) + _, err = contract.Commit(ctx, testobfus, uint64(0)) if err != nil { t.Fatal(err) } @@ -433,7 +433,7 @@ func TestRedistribution(t *testing.T) { redistributionContractABI, ) - _, err = contract.Commit(ctx, common.Hex2Bytes("hash"), uint32(0)) + _, err = contract.Commit(ctx, common.Hex2Bytes("hash"), uint64(0)) if err == nil { t.Fatal("expected error") } From e9b4def2f02d2bbc246df1b89bb2cff70fff5097 Mon Sep 17 00:00:00 2001 From: mrekucci Date: Thu, 5 Oct 2023 19:44:37 +0400 Subject: [PATCH 17/24] chore: bump storage-incentives abi to v0.6.0-rc8 (#4381) --- go.mod | 2 +- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 8a633f57ce8..60b6367e2c3 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/coreos/go-semver v0.3.0 github.com/ethereum/go-ethereum v1.13.2 github.com/ethersphere/go-price-oracle-abi v0.1.0 - github.com/ethersphere/go-storage-incentives-abi v0.6.0-rc7 + github.com/ethersphere/go-storage-incentives-abi v0.6.0-rc8 github.com/ethersphere/go-sw3-abi v0.4.0 github.com/ethersphere/langos v1.0.0 github.com/go-playground/validator/v10 v10.11.1 diff --git a/go.sum b/go.sum index a52e3ef0c9a..8ba393a5fb4 100644 --- a/go.sum +++ b/go.sum @@ -230,10 +230,8 @@ github.com/ethereum/go-ethereum v1.13.2 h1:g9mCpfPWqCA1OL4e6C98PeVttb0HadfBRuKTG github.com/ethereum/go-ethereum v1.13.2/go.mod h1:gkQ5Ygi64ZBh9M/4iXY1R8WqoNCx1Ey0CkYn2BD4/fw= github.com/ethersphere/go-price-oracle-abi v0.1.0 h1:yg/hK8nETNvk+GEBASlbakMFv/CVp7HXiycrHw1pRV8= github.com/ethersphere/go-price-oracle-abi v0.1.0/go.mod h1:sI/Qj4/zJ23/b1enzwMMv0/hLTpPNVNacEwCWjo6yBk= -github.com/ethersphere/go-storage-incentives-abi v0.6.0-rc5 h1:JaiGMFoycByFkaCv6KFg79tfRlBJ5zY01dLWPnf48v0= -github.com/ethersphere/go-storage-incentives-abi v0.6.0-rc5/go.mod h1:SXvJVtM4sEsaSKD0jc1ClpDLw8ErPoROZDme4Wrc/Nc= -github.com/ethersphere/go-storage-incentives-abi v0.6.0-rc7 h1:SPg0WY5rCWt7Y72hAj0o1juLtNi9lqx7b/6fNaNa4H0= -github.com/ethersphere/go-storage-incentives-abi v0.6.0-rc7/go.mod h1:SXvJVtM4sEsaSKD0jc1ClpDLw8ErPoROZDme4Wrc/Nc= +github.com/ethersphere/go-storage-incentives-abi v0.6.0-rc8 h1:+7xFfUGYWhDcXEQyanH6W7CRsNBo9fr/x5hfsmGQ9UY= +github.com/ethersphere/go-storage-incentives-abi v0.6.0-rc8/go.mod h1:SXvJVtM4sEsaSKD0jc1ClpDLw8ErPoROZDme4Wrc/Nc= github.com/ethersphere/go-sw3-abi v0.4.0 h1:T3ANY+ktWrPAwe2U0tZi+DILpkHzto5ym/XwV/Bbz8g= github.com/ethersphere/go-sw3-abi v0.4.0/go.mod h1:BmpsvJ8idQZdYEtWnvxA8POYQ8Rl/NhyCdF0zLMOOJU= github.com/ethersphere/langos v1.0.0 h1:NBtNKzXTTRSue95uOlzPN4py7Aofs0xWPzyj4AI1Vcc= From 4afe1b49e019c3af8f4b3ac1b9850b40506b1f22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Thu, 5 Oct 2023 17:52:54 +0200 Subject: [PATCH 18/24] refactor: proofSegments and proveSegment --- pkg/api/rchash.go | 10 +++---- .../redistribution/abi_types.go | 28 +++++++++---------- .../redistribution/redistribution_test.go | 10 +++---- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/pkg/api/rchash.go b/pkg/api/rchash.go index 736293d61e1..a7b20847ca3 100644 --- a/pkg/api/rchash.go +++ b/pkg/api/rchash.go @@ -77,11 +77,11 @@ func renderProof(proof redistribution.Proof) Proof { } return Proof{ - Data: toHex(proof.Data[:]), - Sisters: renderHash(proof.Sisters...), - Data2: toHex(proof.Data2[:]), - Sisters2: renderHash(proof.Sisters2...), - Sisters3: renderHash(proof.Sisters3...), + Data: toHex(proof.ProveSegment[:]), + Sisters: renderHash(proof.ProofSegments...), + Data2: toHex(proof.ProveSegment2[:]), + Sisters2: renderHash(proof.ProofSegments2...), + Sisters3: renderHash(proof.ProofSegments3...), ChunkSpan: proof.ChunkSpan, PostageProof: PostageProof{ Signature: toHex(proof.PostageProof.Signature), diff --git a/pkg/storageincentives/redistribution/abi_types.go b/pkg/storageincentives/redistribution/abi_types.go index 6ed30ffbd31..277789d9bd3 100644 --- a/pkg/storageincentives/redistribution/abi_types.go +++ b/pkg/storageincentives/redistribution/abi_types.go @@ -19,14 +19,14 @@ import ( // corresponding structure (of the same name) in Redistribution.sol smart contract. // github.com/ethersphere/storage-incentives/blob/master/src/Redistribution.sol type Proof struct { - Sisters []common.Hash `json:"proofSegments"` - Data common.Hash `json:"proveSegment"` - Sisters2 []common.Hash `json:"proofSegments2"` - Data2 common.Hash `json:"proveSegment2"` - Sisters3 []common.Hash `json:"proofSegments3"` - ChunkSpan uint64 `json:"chunkSpan"` - PostageProof PostageProof `json:"postageProof"` - SocProof []SOCProof `json:"socProof"` + ProofSegments []common.Hash `json:"proofSegments"` + ProveSegment common.Hash `json:"proveSegment"` + ProofSegments2 []common.Hash `json:"proofSegments2"` + ProveSegment2 common.Hash `json:"proveSegment2"` + ChunkSpan uint64 `json:"chunkSpan"` + ProofSegments3 []common.Hash `json:"proofSegments3"` + PostageProof PostageProof `json:"postageProof"` + SocProof []SOCProof `json:"socProof"` } // SOCProof structure must exactly match @@ -70,12 +70,12 @@ func NewProof(wp1, wp2, wp3 bmt.Proof, stamp swarm.Stamp, sch *soc.SOC) Proof { } return Proof{ - Sisters: bytes32(wp1.Sisters...), - Data: bytes32(wp1.Data)[0], - Sisters2: bytes32(wp2.Sisters...), - Data2: bytes32(wp2.Data)[0], - Sisters3: bytes32(wp3.Sisters...), - ChunkSpan: binary.LittleEndian.Uint64(wp2.Span[:swarm.SpanSize]), // should be uint64 on the other size; copied from pkg/api/bytes.go + ProofSegments: bytes32(wp1.Sisters...), + ProveSegment: bytes32(wp1.Data)[0], + ProofSegments2: bytes32(wp2.Sisters...), + ProveSegment2: bytes32(wp2.Data)[0], + ProofSegments3: bytes32(wp3.Sisters...), + ChunkSpan: binary.LittleEndian.Uint64(wp2.Span[:swarm.SpanSize]), // should be uint64 on the other size; copied from pkg/api/bytes.go PostageProof: PostageProof{ Signature: stamp.Sig(), BatchId: bytes32(stamp.BatchID())[0], diff --git a/pkg/storageincentives/redistribution/redistribution_test.go b/pkg/storageincentives/redistribution/redistribution_test.go index 4775d822349..84c60000cd4 100644 --- a/pkg/storageincentives/redistribution/redistribution_test.go +++ b/pkg/storageincentives/redistribution/redistribution_test.go @@ -41,11 +41,11 @@ func randProof(t *testing.T) redistribution.Proof { t.Helper() return redistribution.Proof{ - Sisters: randomHashes(t, 7), - Data: randomHashes(t, 1)[0], - Sisters2: randomHashes(t, 7), - Data2: randomHashes(t, 1)[0], - Sisters3: randomHashes(t, 7), + ProofSegments: randomHashes(t, 7), + ProveSegment: randomHashes(t, 1)[0], + ProofSegments2: randomHashes(t, 7), + ProveSegment2: randomHashes(t, 1)[0], + ProofSegments3: randomHashes(t, 7), PostageProof: redistribution.PostageProof{ Signature: testutil.RandBytes(t, 65), BatchId: randomHashes(t, 1)[0], From 92fe19a1d0ae4a97445414a8532db4b183a18e41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Thu, 5 Oct 2023 18:18:09 +0200 Subject: [PATCH 19/24] refactor: batch id to postage id --- pkg/api/rchash.go | 2 +- pkg/storageincentives/redistribution/abi_types.go | 4 ++-- pkg/storageincentives/redistribution/redistribution_test.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/api/rchash.go b/pkg/api/rchash.go index a7b20847ca3..4af59ea246b 100644 --- a/pkg/api/rchash.go +++ b/pkg/api/rchash.go @@ -85,7 +85,7 @@ func renderProof(proof redistribution.Proof) Proof { ChunkSpan: proof.ChunkSpan, PostageProof: PostageProof{ Signature: toHex(proof.PostageProof.Signature), - BatchID: toHex(proof.PostageProof.BatchId[:]), + BatchID: toHex(proof.PostageProof.PostageId[:]), Index: strconv.FormatUint(proof.PostageProof.Index, 16), TimeStamp: strconv.FormatUint(proof.PostageProof.TimeStamp, 16), }, diff --git a/pkg/storageincentives/redistribution/abi_types.go b/pkg/storageincentives/redistribution/abi_types.go index 277789d9bd3..32f3434e82e 100644 --- a/pkg/storageincentives/redistribution/abi_types.go +++ b/pkg/storageincentives/redistribution/abi_types.go @@ -33,7 +33,7 @@ type Proof struct { // corresponding structure (of the same name) in Redistribution.sol smart contract. type PostageProof struct { Signature []byte `json:"signature"` - BatchId common.Hash `json:"batchId"` + PostageId common.Hash `json:"batchId"` Index uint64 `json:"index"` TimeStamp uint64 `json:"timeStamp"` } @@ -78,7 +78,7 @@ func NewProof(wp1, wp2, wp3 bmt.Proof, stamp swarm.Stamp, sch *soc.SOC) Proof { ChunkSpan: binary.LittleEndian.Uint64(wp2.Span[:swarm.SpanSize]), // should be uint64 on the other size; copied from pkg/api/bytes.go PostageProof: PostageProof{ Signature: stamp.Sig(), - BatchId: bytes32(stamp.BatchID())[0], + PostageId: bytes32(stamp.BatchID())[0], Index: binary.BigEndian.Uint64(stamp.Index()), TimeStamp: binary.BigEndian.Uint64(stamp.Timestamp()), }, diff --git a/pkg/storageincentives/redistribution/redistribution_test.go b/pkg/storageincentives/redistribution/redistribution_test.go index 84c60000cd4..5b0669efe1c 100644 --- a/pkg/storageincentives/redistribution/redistribution_test.go +++ b/pkg/storageincentives/redistribution/redistribution_test.go @@ -48,7 +48,7 @@ func randProof(t *testing.T) redistribution.Proof { ProofSegments3: randomHashes(t, 7), PostageProof: redistribution.PostageProof{ Signature: testutil.RandBytes(t, 65), - BatchId: randomHashes(t, 1)[0], + PostageId: randomHashes(t, 1)[0], Index: uint64(rand.Int63()), TimeStamp: uint64(rand.Int63()), }, From cab809e1570d59a2a546ab68d58be5e9a5163e47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Thu, 5 Oct 2023 19:05:48 +0200 Subject: [PATCH 20/24] fix: batchstore valueKeyToValue --- pkg/postage/batchstore/store.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/postage/batchstore/store.go b/pkg/postage/batchstore/store.go index 466f3ca1d3e..cce2deeae7d 100644 --- a/pkg/postage/batchstore/store.go +++ b/pkg/postage/batchstore/store.go @@ -376,7 +376,8 @@ func valueKeyToID(key []byte) []byte { // valueKeyToValue extracts the value from a value key - used in value-based iteration. func valueKeyToValue(key []byte) *big.Int { - return new(big.Int).SetBytes(key[:32]) + offset := len(valueKeyPrefix) + return new(big.Int).SetBytes(key[offset : offset+32]) } func (s *store) SetBatchExpiryHandler(be postage.BatchExpiryHandler) { From 3c50960b2f4f50b2653d1ffce2e1b9cef36e0c57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Thu, 5 Oct 2023 19:11:27 +0200 Subject: [PATCH 21/24] refactor: remove json tags --- .../redistribution/abi_types.go | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pkg/storageincentives/redistribution/abi_types.go b/pkg/storageincentives/redistribution/abi_types.go index 32f3434e82e..f6d9a6d5727 100644 --- a/pkg/storageincentives/redistribution/abi_types.go +++ b/pkg/storageincentives/redistribution/abi_types.go @@ -19,23 +19,23 @@ import ( // corresponding structure (of the same name) in Redistribution.sol smart contract. // github.com/ethersphere/storage-incentives/blob/master/src/Redistribution.sol type Proof struct { - ProofSegments []common.Hash `json:"proofSegments"` - ProveSegment common.Hash `json:"proveSegment"` - ProofSegments2 []common.Hash `json:"proofSegments2"` - ProveSegment2 common.Hash `json:"proveSegment2"` - ChunkSpan uint64 `json:"chunkSpan"` - ProofSegments3 []common.Hash `json:"proofSegments3"` - PostageProof PostageProof `json:"postageProof"` - SocProof []SOCProof `json:"socProof"` + ProofSegments []common.Hash + ProveSegment common.Hash + ProofSegments2 []common.Hash + ProveSegment2 common.Hash + ChunkSpan uint64 + ProofSegments3 []common.Hash + PostageProof PostageProof + SocProof []SOCProof } // SOCProof structure must exactly match // corresponding structure (of the same name) in Redistribution.sol smart contract. type PostageProof struct { - Signature []byte `json:"signature"` - PostageId common.Hash `json:"batchId"` - Index uint64 `json:"index"` - TimeStamp uint64 `json:"timeStamp"` + Signature []byte + PostageId common.Hash + Index uint64 + TimeStamp uint64 } // SOCProof structure must exactly match From 68e23ba19c6b71eced414f4d21954c4e84e49043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Thu, 5 Oct 2023 19:19:34 +0200 Subject: [PATCH 22/24] refactor: uint64 to uint32 --- pkg/api/api_test.go | 2 +- pkg/storageincentives/agent.go | 2 +- pkg/storageincentives/agent_test.go | 2 +- pkg/storageincentives/redistribution/redistribution.go | 6 +++--- .../redistribution/redistribution_test.go | 8 ++++---- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/api/api_test.go b/pkg/api/api_test.go index a2318ef2fae..53309f1d63e 100644 --- a/pkg/api/api_test.go +++ b/pkg/api/api_test.go @@ -791,7 +791,7 @@ func (m *mockContract) Claim(context.Context, []redistribution.Proof) (common.Ha return common.Hash{}, nil } -func (m *mockContract) Commit(context.Context, []byte, uint64) (common.Hash, error) { +func (m *mockContract) Commit(context.Context, []byte, uint32) (common.Hash, error) { m.mtx.Lock() defer m.mtx.Unlock() m.callsList = append(m.callsList, commitCall) diff --git a/pkg/storageincentives/agent.go b/pkg/storageincentives/agent.go index 850f16c994e..a0669e97875 100644 --- a/pkg/storageincentives/agent.go +++ b/pkg/storageincentives/agent.go @@ -450,7 +450,7 @@ func (a *Agent) commit(ctx context.Context, sample sampler.Data, round uint64) e return err } - txHash, err := a.contract.Commit(ctx, obfuscatedHash, uint64(round)) + txHash, err := a.contract.Commit(ctx, obfuscatedHash, uint32(round)) if err != nil { a.metrics.ErrCommit.Inc() return err diff --git a/pkg/storageincentives/agent_test.go b/pkg/storageincentives/agent_test.go index e8fafcef67a..d0951479789 100644 --- a/pkg/storageincentives/agent_test.go +++ b/pkg/storageincentives/agent_test.go @@ -261,7 +261,7 @@ func (m *mockContract) Claim(context.Context, []redistribution.Proof) (common.Ha return common.Hash{}, nil } -func (m *mockContract) Commit(context.Context, []byte, uint64) (common.Hash, error) { +func (m *mockContract) Commit(context.Context, []byte, uint32) (common.Hash, error) { m.mtx.Lock() defer m.mtx.Unlock() m.callsList = append(m.callsList, commitCall) diff --git a/pkg/storageincentives/redistribution/redistribution.go b/pkg/storageincentives/redistribution/redistribution.go index 91dfa9ba83c..f803704cffe 100644 --- a/pkg/storageincentives/redistribution/redistribution.go +++ b/pkg/storageincentives/redistribution/redistribution.go @@ -24,7 +24,7 @@ type Contract interface { IsPlaying(context.Context, uint8) (bool, error) IsWinner(context.Context) (bool, error) Claim(context.Context, []Proof) (common.Hash, error) - Commit(context.Context, []byte, uint64) (common.Hash, error) + Commit(context.Context, []byte, uint32) (common.Hash, error) Reveal(context.Context, uint8, []byte, []byte) (common.Hash, error) } @@ -93,7 +93,7 @@ func (c *contract) IsWinner(ctx context.Context) (isWinner bool, err error) { // Claim sends a transaction to blockchain if a win is claimed. func (c *contract) Claim(ctx context.Context, proofs []Proof) (common.Hash, error) { - callData, err := c.incentivesContractABI.Pack("claim", proofs) + callData, err := c.incentivesContractABI.Pack("claim", proofs[0], proofs[1], proofs[2]) if err != nil { return common.Hash{}, err } @@ -115,7 +115,7 @@ func (c *contract) Claim(ctx context.Context, proofs []Proof) (common.Hash, erro } // Commit submits the obfusHash hash by sending a transaction to the blockchain. -func (c *contract) Commit(ctx context.Context, obfusHash []byte, round uint64) (common.Hash, error) { +func (c *contract) Commit(ctx context.Context, obfusHash []byte, round uint32) (common.Hash, error) { callData, err := c.incentivesContractABI.Pack("commit", common.BytesToHash(obfusHash), common.BytesToHash(c.overlay.Bytes()), round) if err != nil { return common.Hash{}, err diff --git a/pkg/storageincentives/redistribution/redistribution_test.go b/pkg/storageincentives/redistribution/redistribution_test.go index 5b0669efe1c..319cc2fa66f 100644 --- a/pkg/storageincentives/redistribution/redistribution_test.go +++ b/pkg/storageincentives/redistribution/redistribution_test.go @@ -276,7 +276,7 @@ func TestRedistribution(t *testing.T) { var obfus [32]byte testobfus := common.Hex2Bytes("hash") copy(obfus[:], testobfus) - expectedCallData, err := redistributionContractABI.Pack("commit", obfus, common.BytesToHash(owner.Bytes()), uint64(0)) + expectedCallData, err := redistributionContractABI.Pack("commit", obfus, common.BytesToHash(owner.Bytes()), uint32(0)) if err != nil { t.Fatal(err) } @@ -306,7 +306,7 @@ func TestRedistribution(t *testing.T) { redistributionContractABI, ) - _, err = contract.Commit(ctx, testobfus, uint64(0)) + _, err = contract.Commit(ctx, testobfus, uint32(0)) if err != nil { t.Fatal(err) } @@ -411,7 +411,7 @@ func TestRedistribution(t *testing.T) { t.Run("invalid call data", func(t *testing.T) { t.Parallel() - expectedCallData, err := redistributionContractABI.Pack("commit", common.BytesToHash(common.Hex2Bytes("some hash")), common.BytesToHash(common.Hex2Bytes("some address")), uint64(0)) + expectedCallData, err := redistributionContractABI.Pack("commit", common.BytesToHash(common.Hex2Bytes("some hash")), common.BytesToHash(common.Hex2Bytes("some address")), uint32(0)) if err != nil { t.Fatal(err) } @@ -433,7 +433,7 @@ func TestRedistribution(t *testing.T) { redistributionContractABI, ) - _, err = contract.Commit(ctx, common.Hex2Bytes("hash"), uint64(0)) + _, err = contract.Commit(ctx, common.Hex2Bytes("hash"), uint32(0)) if err == nil { t.Fatal("expected error") } From bcbd4e603679b930c24960f810c5b033aa90bd1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Levente=20T=C3=B3th?= Date: Thu, 5 Oct 2023 19:34:24 +0200 Subject: [PATCH 23/24] fix: channel close if not nil --- pkg/storageincentives/events.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/storageincentives/events.go b/pkg/storageincentives/events.go index bc5ede4676d..ec2db98a682 100644 --- a/pkg/storageincentives/events.go +++ b/pkg/storageincentives/events.go @@ -80,6 +80,8 @@ func (e *events) Close() { defer e.mtx.Unlock() for _, ch := range e.quit { - close(ch) + if ch != nil { + close(ch) + } } } From 990391591861fd2e17bb6092658d5244db35edf4 Mon Sep 17 00:00:00 2001 From: mrekucci Date: Mon, 9 Oct 2023 13:45:10 +0400 Subject: [PATCH 24/24] fix: redistribution contract abi type change (#4382) --- .github/workflows/beekeeper.yml | 2 +- pkg/api/api_test.go | 2 +- pkg/storageincentives/agent.go | 2 +- pkg/storageincentives/agent_test.go | 2 +- pkg/storageincentives/redistribution/redistribution.go | 4 ++-- .../redistribution/redistribution_test.go | 8 ++++---- pkg/transaction/transaction.go | 2 +- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/beekeeper.yml b/.github/workflows/beekeeper.yml index c4dcef2884d..147d93648ff 100644 --- a/.github/workflows/beekeeper.yml +++ b/.github/workflows/beekeeper.yml @@ -11,7 +11,7 @@ env: K3S_VERSION: "v1.22.17+k3s1" REPLICA: 3 RUN_TYPE: "PR RUN" - SETUP_CONTRACT_IMAGE_TAG: "1.0.3" + SETUP_CONTRACT_IMAGE_TAG: "1.0.4" BEELOCAL_BRANCH: "main" BEEKEEPER_BRANCH: "master" BEEKEEPER_METRICS_ENABLED: false diff --git a/pkg/api/api_test.go b/pkg/api/api_test.go index 21e535f10ac..a31bb87ff5d 100644 --- a/pkg/api/api_test.go +++ b/pkg/api/api_test.go @@ -783,7 +783,7 @@ func (m *mockContract) Claim(context.Context, redistribution.ChunkInclusionProof return common.Hash{}, nil } -func (m *mockContract) Commit(context.Context, []byte, uint32) (common.Hash, error) { +func (m *mockContract) Commit(context.Context, []byte, uint64) (common.Hash, error) { m.mtx.Lock() defer m.mtx.Unlock() m.callsList = append(m.callsList, commitCall) diff --git a/pkg/storageincentives/agent.go b/pkg/storageincentives/agent.go index 51a62393a8c..2bd0bf0ffb4 100644 --- a/pkg/storageincentives/agent.go +++ b/pkg/storageincentives/agent.go @@ -510,7 +510,7 @@ func (a *Agent) commit(ctx context.Context, sample SampleData, round uint64) err return err } - txHash, err := a.contract.Commit(ctx, obfuscatedHash, uint32(round)) + txHash, err := a.contract.Commit(ctx, obfuscatedHash, round) if err != nil { a.metrics.ErrCommit.Inc() return err diff --git a/pkg/storageincentives/agent_test.go b/pkg/storageincentives/agent_test.go index 70ec3193975..fe5bc6a937f 100644 --- a/pkg/storageincentives/agent_test.go +++ b/pkg/storageincentives/agent_test.go @@ -283,7 +283,7 @@ func (m *mockContract) Claim(context.Context, redistribution.ChunkInclusionProof return common.Hash{}, nil } -func (m *mockContract) Commit(context.Context, []byte, uint32) (common.Hash, error) { +func (m *mockContract) Commit(context.Context, []byte, uint64) (common.Hash, error) { m.mtx.Lock() defer m.mtx.Unlock() m.callsList = append(m.callsList, commitCall) diff --git a/pkg/storageincentives/redistribution/redistribution.go b/pkg/storageincentives/redistribution/redistribution.go index 8b83537cf90..01ddb9195b0 100644 --- a/pkg/storageincentives/redistribution/redistribution.go +++ b/pkg/storageincentives/redistribution/redistribution.go @@ -24,7 +24,7 @@ type Contract interface { IsPlaying(context.Context, uint8) (bool, error) IsWinner(context.Context) (bool, error) Claim(context.Context, ChunkInclusionProofs) (common.Hash, error) - Commit(context.Context, []byte, uint32) (common.Hash, error) + Commit(context.Context, []byte, uint64) (common.Hash, error) Reveal(context.Context, uint8, []byte, []byte) (common.Hash, error) } @@ -115,7 +115,7 @@ func (c *contract) Claim(ctx context.Context, proofs ChunkInclusionProofs) (comm } // Commit submits the obfusHash hash by sending a transaction to the blockchain. -func (c *contract) Commit(ctx context.Context, obfusHash []byte, round uint32) (common.Hash, error) { +func (c *contract) Commit(ctx context.Context, obfusHash []byte, round uint64) (common.Hash, error) { callData, err := c.incentivesContractABI.Pack("commit", common.BytesToHash(obfusHash), common.BytesToHash(c.overlay.Bytes()), round) if err != nil { return common.Hash{}, err diff --git a/pkg/storageincentives/redistribution/redistribution_test.go b/pkg/storageincentives/redistribution/redistribution_test.go index 943d3013ba4..4a8e84b421b 100644 --- a/pkg/storageincentives/redistribution/redistribution_test.go +++ b/pkg/storageincentives/redistribution/redistribution_test.go @@ -267,7 +267,7 @@ func TestRedistribution(t *testing.T) { var obfus [32]byte testobfus := common.Hex2Bytes("hash") copy(obfus[:], testobfus) - expectedCallData, err := redistributionContractABI.Pack("commit", obfus, common.BytesToHash(owner.Bytes()), uint32(0)) + expectedCallData, err := redistributionContractABI.Pack("commit", obfus, common.BytesToHash(owner.Bytes()), uint64(0)) if err != nil { t.Fatal(err) } @@ -297,7 +297,7 @@ func TestRedistribution(t *testing.T) { redistributionContractABI, ) - _, err = contract.Commit(ctx, testobfus, uint32(0)) + _, err = contract.Commit(ctx, testobfus, 0) if err != nil { t.Fatal(err) } @@ -402,7 +402,7 @@ func TestRedistribution(t *testing.T) { t.Run("invalid call data", func(t *testing.T) { t.Parallel() - expectedCallData, err := redistributionContractABI.Pack("commit", common.BytesToHash(common.Hex2Bytes("some hash")), common.BytesToHash(common.Hex2Bytes("some address")), uint32(0)) + expectedCallData, err := redistributionContractABI.Pack("commit", common.BytesToHash(common.Hex2Bytes("some hash")), common.BytesToHash(common.Hex2Bytes("some address")), uint64(0)) if err != nil { t.Fatal(err) } @@ -424,7 +424,7 @@ func TestRedistribution(t *testing.T) { redistributionContractABI, ) - _, err = contract.Commit(ctx, common.Hex2Bytes("hash"), uint32(0)) + _, err = contract.Commit(ctx, common.Hex2Bytes("hash"), 0) if err == nil { t.Fatal("expected error") } diff --git a/pkg/transaction/transaction.go b/pkg/transaction/transaction.go index d16494ba7b6..e969a785c3d 100644 --- a/pkg/transaction/transaction.go +++ b/pkg/transaction/transaction.go @@ -601,7 +601,7 @@ func (t *transactionService) UnwrapABIError(ctx context.Context, req *TxRequest, if cErr == nil { return err } - err = fmt.Errorf("%w: %s", err, cErr) + err = fmt.Errorf("%w: %s", err, cErr) //nolint:errorlint var derr rpc.DataError if !errors.As(cErr, &derr) {