Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(ph4): proof of entitlement #4040

Closed
wants to merge 110 commits into from
Closed
Show file tree
Hide file tree
Changes from 83 commits
Commits
Show all changes
110 commits
Select commit Hold shift + click to select a range
29805aa
feat(status): add chainstate total amount (#4010)
istae Apr 19, 2023
a627e04
feat(kademlia): allow light nodes to be picked in oversaturated bins …
istae Apr 19, 2023
3142369
fix(depthmonitor): change threshold back to 50 percent (#4013)
istae Apr 20, 2023
5355ce0
feat(pushync): forwarder multiplexer (#4008)
istae Apr 20, 2023
043621e
feat: add chunk type information to reserve
vladopajic Apr 20, 2023
ff61d17
fix: lint
vladopajic Apr 20, 2023
74baca1
chore: add string
vladopajic Apr 22, 2023
a8cd6b6
feat: ph4 sampling
vladopajic Apr 22, 2023
05beed8
fix: add soc tr
vladopajic Apr 22, 2023
c66d24f
chore: improvements
vladopajic Apr 22, 2023
34ae9c4
fix: test
vladopajic Apr 22, 2023
41d6eef
fix: blocklist endpoint updates (#4015)
acha-bill Apr 24, 2023
be325fa
chore: cosmetics
vladopajic Apr 24, 2023
1a650d2
chore: cosmetics
vladopajic Apr 24, 2023
bede351
chore: cosmetics
vladopajic Apr 24, 2023
ae5fa86
chore: zero addr
vladopajic Apr 25, 2023
617209c
fix(status): include batch total amount (#4031)
istae Apr 25, 2023
2d5f13e
fix: negative work group counter on shutdown (#4033)
Apr 25, 2023
3417a11
fix(pushsync): best attempt to find next peer (#4034)
istae Apr 26, 2023
8789cae
fix(blocklist): duration and full node fix (#4035)
istae Apr 26, 2023
3db3dab
feat: log error on put chunk failure (#4037)
Apr 26, 2023
ffd41a3
feat: add log messages (#4038)
Apr 26, 2023
b4ebfc9
feat: initial work for proof of entitlement
vladopajic Apr 26, 2023
9f80fb6
chore: some lint fixes
vladopajic Apr 27, 2023
806c670
chore: improvements
vladopajic Apr 28, 2023
5548dd2
fix: removed pushsync replication (#4039)
istae Apr 28, 2023
335d5ac
fix: version bump
istae May 1, 2023
f5a96ae
fix: return when peer is gone (#4043)
istae May 1, 2023
00989d0
feat: localstore v2 interfaces
Aug 12, 2022
539144e
refactor: improve storage interfaces (#3188)
Aug 13, 2022
36572f1
feat: in-mem implementation and testsuite (#3190)
aloknerurkar Aug 16, 2022
82f2f94
feat: inmem chunkstorer and testsuite (#3210)
aloknerurkar Aug 23, 2022
66f53b7
feat: leveldb store (#3203)
notanatol Aug 25, 2022
1f94768
refactor: cleanups (#3239)
Sep 1, 2022
eadba3d
feat: improve testsuite for Item serialization (#3252)
Sep 6, 2022
502a983
feat: localstore benchmarks (#3237)
notanatol Sep 7, 2022
23712f0
feat: tests for benchmarking code (#3270)
notanatol Sep 12, 2022
aaad65f
feat: add pinning store (#3236)
aloknerurkar Sep 15, 2022
5835727
feat: localstore batching (#3279)
notanatol Sep 21, 2022
7173668
feat: add upload store (#3250)
Sep 23, 2022
7d5a93b
refactor: batchedstore tests and bench cleanup (#3330)
Sep 27, 2022
5572c14
feat(localstorev2): add chunkstore (#3315)
aloknerurkar Sep 28, 2022
39ca1d4
fix: move upload store (#3357)
aloknerurkar Oct 3, 2022
aa76991
feat: migration initiation (#3424)
umerm-work Oct 25, 2022
a977849
feat(migration): store migration helpers (#3421)
vladopajic Oct 26, 2022
8381891
feat(localstorev2): add cache store (#3469)
aloknerurkar Nov 29, 2022
d3c417a
feat: lock-base transaction implementation (#3198)
Dec 1, 2022
d6d8385
feat(storagev2): simplify ChunkStore Put API (#3672)
aloknerurkar Dec 22, 2022
17c95e2
refactor: upload store with tags (#3690)
aloknerurkar Jan 12, 2023
92db2ef
feat: add stamp index (#3701)
Jan 21, 2023
5b7d939
fix: transaction put logic (#3708)
Jan 23, 2023
910fcad
chore: multiple stamps per chunk (#3707)
vladopajic Jan 23, 2023
f923937
fix(localstorev2): broken tests (#3719)
aloknerurkar Jan 24, 2023
59455d6
refactor: improve transaction Done call API (#3720)
Jan 24, 2023
fc4b237
feat: tx chunkstore and multiple stamp support (#3702)
aloknerurkar Jan 24, 2023
f3ca829
fix: tx chunk and index stores common tx state (#3723)
Jan 25, 2023
b7abe47
feat: add iteration by prefix and skip first item (#3725)
Jan 26, 2023
25fe118
fix: check for existing stamp index on upload (#3727)
Jan 27, 2023
67324ed
feat(localstorev2): localstore container implementation (#3741)
aloknerurkar Jan 31, 2023
6273079
feat: iterate to start at a prefix and continue until the end of the …
istae Jan 31, 2023
d4136bc
feat: localstore container cache interfaces (#3755)
aloknerurkar Feb 3, 2023
bf925ac
feat: check for existing index on stamp call (#3744)
Feb 6, 2023
41826ff
feat(localstorev2): netstore interfaces (#3766)
aloknerurkar Feb 9, 2023
be79bbe
chore: restructure localstorev2 pkg (#3777)
aloknerurkar Feb 10, 2023
41d75cc
refactor: cleanup tx revert operation tracking (#3793)
Feb 16, 2023
f115530
feat(localstorev2): migration (#3791)
vladopajic Feb 16, 2023
b9c41bd
feat(localstorev2): sharky recovery (#3794)
vladopajic Feb 21, 2023
98408c7
fix: upload store iterator (#3826)
notanatol Feb 23, 2023
185e541
refactor: move out chunk stamp index from chunk store (#3820)
Feb 24, 2023
491908b
fix: branch due to merge conflicts
Mar 1, 2023
8b70d0a
fix: slash separator character (#3855)
Mar 2, 2023
dc8057c
feat: localstorev2 reserve (#3775)
istae Mar 2, 2023
d7f37c2
fix: tx chunk store put and delete operation correctness (#3854)
Mar 3, 2023
4607990
feat: push subscribe (#3857)
notanatol Mar 13, 2023
ff700bb
feat: add metrics for the new storage (#3877)
Mar 13, 2023
0ba91e7
feat(reserve): ph4 sampling (#4028)
vladopajic May 3, 2023
c230905
Merge branch 'localstorev2.0' into ph4-proofs
vladopajic May 3, 2023
f59ca61
fix: merge
vladopajic May 3, 2023
36cb5d9
fix: merge fix
vladopajic May 3, 2023
66a5bd4
chore: added RecoverBatchOwner
vladopajic May 3, 2023
dd65957
chore: more effort
vladopajic May 3, 2023
4d8c9b9
chore: more effort
vladopajic May 3, 2023
4a1352b
chore: cosmetics
vladopajic May 3, 2023
6b76fa4
feat: use new storagev2 interfaces (#3798)
aloknerurkar Mar 27, 2023
9fd8c50
fix: conficts
istae Mar 27, 2023
7185b29
fix: test fixed
istae Mar 27, 2023
f186435
fix: metrics
Mar 29, 2023
19a6bbb
perf: validate chunk stamps directly (#3951)
istae Mar 31, 2023
19f5d48
feat(storer): add debug information for storer (#3952)
aloknerurkar Apr 1, 2023
b7a4aff
fix: pullsync ignore overwrite error (#3950)
istae Apr 3, 2023
c0588f1
fix: set reserve capacity correctly
Apr 17, 2023
ea87013
fix: status and remaining conflicts
Apr 18, 2023
0037099
fix: do not use sync writes in leveldb
Apr 18, 2023
7f8c9cc
fix: lint
Apr 18, 2023
312aa08
fix: test race
Apr 18, 2023
1fdff56
fix: merge (#4014)
istae Apr 18, 2023
af6b504
fix: misc fixes from rewrite testing (#4020)
aloknerurkar Apr 20, 2023
847b7b7
fix: reserve config
Apr 20, 2023
250b324
fix(pullsync, reserve): misc fixes and memory improvements (#4024)
aloknerurkar May 3, 2023
b1e3b4d
fix(localstorev2): metrics fix and cleanups (#4027)
istae May 3, 2023
f9a680a
feat: add chunk type information to reserve (#4021)
vladopajic May 3, 2023
e6a82ac
feat(reserve): ph4 sampling (#4028)
vladopajic May 3, 2023
7c769ff
fix: store on direct push
May 4, 2023
040f2eb
fix: tag handling on restart
May 4, 2023
a9149c2
chore: improvements
vladopajic May 4, 2023
c8b6fa0
Merge branch 'localstorev2.0' into ph4-proofs
vladopajic May 4, 2023
ed7ac87
chore: merge fix
vladopajic May 4, 2023
bc86bfc
chore: cmp
vladopajic May 4, 2023
2235e99
Merge branch 'ph4-sampling' into ph4-proofs
vladopajic May 5, 2023
e3f6cc3
chore: abi wip
vladopajic May 9, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/beekeeper.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,11 @@ env:
RUN_TYPE: "PR RUN"
SETUP_CONTRACT_IMAGE_TAG: "0.9.9"
BEELOCAL_BRANCH: "main"
BEEKEEPER_BRANCH: "master"
BEEKEEPER_BRANCH: "delay-download"
BEEKEEPER_METRICS_ENABLED: false
REACHABILITY_OVERRIDE_PUBLIC: true
BATCHFACTOR_OVERRIDE_PUBLIC: 2
TIMEOUT: 10m
TIMEOUT: 30m
BUCKET_NAME: swarm-beekeeper-artifacts
AWS_ACCESS_KEY_ID: ${{ secrets.DO_AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.DO_AWS_SECRET_ACCESS_KEY }}
Expand Down
2 changes: 1 addition & 1 deletion openapi/Swarm.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -943,7 +943,7 @@ paths:
content:
application/json:
schema:
$ref: "SwarmCommon.yaml#/components/schemas/Peers"
$ref: "SwarmCommon.yaml#/components/schemas/BlockListedPeers"
"500":
$ref: "SwarmCommon.yaml#/components/responses/500"
default:
Expand Down
18 changes: 17 additions & 1 deletion openapi/SwarmCommon.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
openapi: 3.0.3
info:
version: 3.1.0
version: 3.2.0
title: Common Data Types
description: |
\*****bzzz*****
Expand Down Expand Up @@ -331,6 +331,20 @@ components:
items:
$ref: "#/components/schemas/Address"

BlockListedPeers:
type: array
items:
type: object
properties:
address:
$ref: "#/components/schemas/Address"
fullNode:
type: boolean
reason:
type: string
duration:
type: integer

PssRecipient:
type: string

Expand Down Expand Up @@ -841,6 +855,8 @@ components:
requestFailed:
nullable: true
type: boolean
batchTotalAmount:
type: string

StatusResponse:
type: object
Expand Down
1 change: 1 addition & 0 deletions pkg/api/export_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ type (
PingpongResponse = pingpongResponse
PeerConnectResponse = peerConnectResponse
PeersResponse = peersResponse
BlockedListedPeersResponse = blockListedPeersResponse
AddressesResponse = addressesResponse
WelcomeMessageRequest = welcomeMessageRequest
WelcomeMessageResponse = welcomeMessageResponse
Expand Down
29 changes: 27 additions & 2 deletions pkg/api/peer.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,10 +83,20 @@ type Peer struct {
FullNode bool `json:"fullNode"`
}

type BlockListedPeer struct {
Peer
Reason string `json:"reason"`
Duration int `json:"duration"`
}

type peersResponse struct {
Peers []Peer `json:"peers"`
}

type blockListedPeersResponse struct {
Peers []BlockListedPeer `json:"peers"`
}

func (s *Service) peersHandler(w http.ResponseWriter, _ *http.Request) {
jsonhttp.OK(w, peersResponse{
Peers: mapPeers(s.p2p.Peers()),
Expand All @@ -103,8 +113,8 @@ func (s *Service) blocklistedPeersHandler(w http.ResponseWriter, _ *http.Request
return
}

jsonhttp.OK(w, peersResponse{
Peers: mapPeers(peers),
jsonhttp.OK(w, blockListedPeersResponse{
Peers: mapBlockListedPeers(peers),
})
}

Expand All @@ -118,3 +128,18 @@ func mapPeers(peers []p2p.Peer) (out []Peer) {
}
return
}

func mapBlockListedPeers(peers []p2p.BlockListedPeer) []BlockListedPeer {
out := make([]BlockListedPeer, 0, len(peers))
for _, peer := range peers {
out = append(out, BlockListedPeer{
Peer: Peer{
Address: peer.Address,
FullNode: peer.FullNode,
},
Reason: peer.Reason,
Duration: int(peer.Duration.Seconds()),
})
}
return out
}
12 changes: 6 additions & 6 deletions pkg/api/peer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -189,14 +189,14 @@ func TestBlocklistedPeers(t *testing.T) {
overlay := swarm.MustParseHexAddress("ca1e9f3938cc1425c6061b96ad9eb93e134dfe8734ad490164ef20af9d1cf59c")
testServer, _, _, _ := newTestServer(t, testServerOptions{
DebugAPI: true,
P2P: mock.New(mock.WithBlocklistedPeersFunc(func() ([]p2p.Peer, error) {
return []p2p.Peer{{Address: overlay}}, nil
P2P: mock.New(mock.WithBlocklistedPeersFunc(func() ([]p2p.BlockListedPeer, error) {
return []p2p.BlockListedPeer{{Peer: p2p.Peer{Address: overlay}}}, nil
})),
})

jsonhttptest.Request(t, testServer, http.MethodGet, "/blocklist", http.StatusOK,
jsonhttptest.WithExpectedJSONResponse(api.PeersResponse{
Peers: []api.Peer{{Address: overlay}},
jsonhttptest.WithExpectedJSONResponse(api.BlockedListedPeersResponse{
Peers: []api.BlockListedPeer{{Peer: api.Peer{Address: overlay}, Duration: 0}},
}),
)
}
Expand All @@ -207,8 +207,8 @@ func TestBlocklistedPeersErr(t *testing.T) {
overlay := swarm.MustParseHexAddress("ca1e9f3938cc1425c6061b96ad9eb93e134dfe8734ad490164ef20af9d1cf59c")
testServer, _, _, _ := newTestServer(t, testServerOptions{
DebugAPI: true,
P2P: mock.New(mock.WithBlocklistedPeersFunc(func() ([]p2p.Peer, error) {
return []p2p.Peer{{Address: overlay}}, errors.New("some error")
P2P: mock.New(mock.WithBlocklistedPeersFunc(func() ([]p2p.BlockListedPeer, error) {
return []p2p.BlockListedPeer{{Peer: p2p.Peer{Address: overlay}}}, errors.New("some error")
})),
})

Expand Down
5 changes: 4 additions & 1 deletion pkg/api/status.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,15 @@ import (

type statusSnapshotResponse struct {
Peer string `json:"peer"`
BeeMode string `json:"beeMode"`
Proximity uint8 `json:"proximity"`
BeeMode string `json:"beeMode"`
ReserveSize uint64 `json:"reserveSize"`
PullsyncRate float64 `json:"pullsyncRate"`
StorageRadius uint8 `json:"storageRadius"`
ConnectedPeers uint64 `json:"connectedPeers"`
NeighborhoodSize uint64 `json:"neighborhoodSize"`
RequestFailed bool `json:"requestFailed,omitempty"`
BatchTotalAmount string `json:"batchTotalAmount"`
}

type statusResponse struct {
Expand Down Expand Up @@ -75,6 +76,7 @@ func (s *Service) statusGetHandler(w http.ResponseWriter, _ *http.Request) {
StorageRadius: uint8(ss.StorageRadius),
ConnectedPeers: ss.ConnectedPeers,
NeighborhoodSize: ss.NeighborhoodSize,
BatchTotalAmount: ss.BatchTotalAmount,
})
}

Expand Down Expand Up @@ -118,6 +120,7 @@ func (s *Service) statusGetPeersHandler(w http.ResponseWriter, r *http.Request)
snapshot.StorageRadius = uint8(ss.StorageRadius)
snapshot.ConnectedPeers = ss.ConnectedPeers
snapshot.NeighborhoodSize = ss.NeighborhoodSize
snapshot.BatchTotalAmount = ss.BatchTotalAmount
}

mu.Lock()
Expand Down
37 changes: 23 additions & 14 deletions pkg/api/status_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,15 @@
package api_test

import (
"math/big"
"net/http"
"testing"

"github.com/ethersphere/bee/pkg/api"
"github.com/ethersphere/bee/pkg/jsonhttp"
"github.com/ethersphere/bee/pkg/jsonhttp/jsonhttptest"
"github.com/ethersphere/bee/pkg/log"
"github.com/ethersphere/bee/pkg/postage"
"github.com/ethersphere/bee/pkg/status"
"github.com/ethersphere/bee/pkg/topology"
)
Expand All @@ -32,25 +34,31 @@ func TestGetStatus(t *testing.T) {
StorageRadius: 8,
ConnectedPeers: 0,
NeighborhoodSize: 0,
BatchTotalAmount: "1",
}

ssMock := &statusSnapshotMock{
syncRate: ssr.PullsyncRate,
reserveSize: int(ssr.ReserveSize),
storageRadius: ssr.StorageRadius,
chainstate: &postage.ChainState{TotalAmount: big.NewInt(1)},
}

statusSvc := status.NewService(
log.Noop,
nil,
new(topologyPeersIterNoopMock),
mode.String(),
ssMock,
)

statusSvc.SetStorage(ssMock)
statusSvc.SetSync(ssMock)

client, _, _, _ := newTestServer(t, testServerOptions{
BeeMode: mode,
DebugAPI: true,
NodeStatus: status.NewService(
log.Noop,
nil,
new(topologyPeersIterNoopMock),
mode.String(),
ssMock,
ssMock,
),
BeeMode: mode,
DebugAPI: true,
NodeStatus: statusSvc,
})

jsonhttptest.Request(t, client, http.MethodGet, url, http.StatusOK,
Expand All @@ -70,7 +78,6 @@ func TestGetStatus(t *testing.T) {
new(topologyPeersIterNoopMock),
"",
nil,
nil,
),
})

Expand Down Expand Up @@ -101,8 +108,10 @@ type statusSnapshotMock struct {
syncRate float64
reserveSize int
storageRadius uint8
chainstate *postage.ChainState
}

func (m *statusSnapshotMock) SyncRate() float64 { return m.syncRate }
func (m *statusSnapshotMock) ReserveSize() int { return m.reserveSize }
func (m *statusSnapshotMock) StorageRadius() uint8 { return m.storageRadius }
func (m *statusSnapshotMock) SyncRate() float64 { return m.syncRate }
func (m *statusSnapshotMock) ReserveSize() int { return m.reserveSize }
func (m *statusSnapshotMock) StorageRadius() uint8 { return m.storageRadius }
func (m *statusSnapshotMock) GetChainState() *postage.ChainState { return m.chainstate }
13 changes: 11 additions & 2 deletions pkg/node/node.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ import (
"github.com/ethersphere/bee/pkg/settlement/swap/erc20"
"github.com/ethersphere/bee/pkg/settlement/swap/priceoracle"
"github.com/ethersphere/bee/pkg/shed"
"github.com/ethersphere/bee/pkg/status"
"github.com/ethersphere/bee/pkg/steward"
"github.com/ethersphere/bee/pkg/storageincentives"
"github.com/ethersphere/bee/pkg/storageincentives/redistribution"
Expand Down Expand Up @@ -931,7 +932,7 @@ func NewBee(

validStamp := postage.ValidStamp(batchStore)

pushSyncProtocol := pushsync.New(swarmAddress, nonce, p2ps, localStore, kad, o.FullNodeMode, pssService.TryUnwrap, validStamp, logger, acc, pricer, signer, tracer, warmupTime)
pushSyncProtocol := pushsync.New(swarmAddress, nonce, p2ps, localStore, kad, o.FullNodeMode, pssService.TryUnwrap, validStamp, logger, acc, pricer, signer, tracer)

// set the pushSyncer in the PSS
pssService.SetPushSyncer(pushSyncProtocol)
Expand Down Expand Up @@ -974,6 +975,11 @@ func NewBee(
return nil, fmt.Errorf("pullsync protocol: %w", err)
}

nodeStatus := status.NewService(logger, p2ps, kad, beeNodeMode.String(), batchStore)
if err = p2ps.AddProtocol(nodeStatus.Protocol()); err != nil {
return nil, fmt.Errorf("status service: %w", err)
}

stakingContractAddress := chainCfg.StakingAddress
if o.StakingContractAddress != "" {
if !common.IsHexAddress(o.StakingContractAddress) {
Expand All @@ -1000,6 +1006,9 @@ func NewBee(

localStore.StartReserveWorker(pullerService)

nodeStatus.SetStorage(localStore)
nodeStatus.SetSync(pullerService)

if o.EnableStorageIncentives {

redistributionContractAddress := chainCfg.RedistributionAddress
Expand Down Expand Up @@ -1270,7 +1279,7 @@ func (b *Bee) Shutdown() error {
}

var wg sync.WaitGroup
wg.Add(7)
wg.Add(9)
go func() {
defer wg.Done()
tryClose(b.chainSyncerCloser, "chain syncer")
Expand Down
39 changes: 39 additions & 0 deletions pkg/p2p/libp2p/connections_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"errors"
"io"
"math/rand"
"reflect"
"strings"
"sync"
"testing"
Expand Down Expand Up @@ -598,6 +599,44 @@ func TestBlocklisting(t *testing.T) {
expectPeers(t, s2)
}

func TestBlocklistedPeers(t *testing.T) {
t.Parallel()
s1, overlay1 := newService(t, 1, libp2pServiceOpts{libp2pOpts: libp2p.Options{
FullNode: true,
}})
s2, _ := newService(t, 1, libp2pServiceOpts{})
addr1 := serviceUnderlayAddress(t, s1)
_, err := s2.Connect(context.Background(), addr1)
if err != nil {
t.Fatal(err)
}

if err = s2.Blocklist(overlay1, 0, testBlocklistMsg); err != nil {
t.Fatal(err)
}
blocklistedPeers, err := s2.BlocklistedPeers()
if err != nil {
t.Fatal("could not get blocklisted peers", err)
}

want := []p2p.BlockListedPeer{
{
Peer: p2p.Peer{
Address: overlay1,
// we can't mock peerRegistry. So we can't know if the peer is a full node or not.
// TODO: consider injecting peerRegistry in libp2pService
FullNode: true,
},
Reason: testBlocklistMsg,
Duration: 0,
},
}

if !reflect.DeepEqual(want, blocklistedPeers) {
t.Fatal("want", want, "got", blocklistedPeers)
}
}

func TestTopologyNotifier(t *testing.T) {
t.Parallel()

Expand Down
Loading