From cf63d66903cd5b9638a20a3a120bb5d94790bffe Mon Sep 17 00:00:00 2001 From: zelig Date: Wed, 20 Dec 2023 22:51:57 +0100 Subject: [PATCH 01/52] fix: copyright header year --- pkg/replicas/replicas.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/replicas/replicas.go b/pkg/replicas/replicas.go index 18b21d4b8b8..43b1d8d9b18 100644 --- a/pkg/replicas/replicas.go +++ b/pkg/replicas/replicas.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Swarm Authors. All rights reserved. +// Copyright 2023 The Swarm Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. From 9bb8112f2d3de7ab1a1250e9de783ce7381d1592 Mon Sep 17 00:00:00 2001 From: zelig Date: Wed, 20 Dec 2023 22:58:37 +0100 Subject: [PATCH 02/52] fix: openapi param name --- pkg/api/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/api/api.go b/pkg/api/api.go index 26d11213e5a..6d55aae9454 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -83,7 +83,7 @@ const ( SwarmRedundancyLevelHeader = "Swarm-Redundancy-Level" SwarmRedundancyStrategyHeader = "Swarm-Redundancy-Strategy" SwarmRedundancyFallbackModeHeader = "Swarm-Redundancy-Fallback-Mode" - SwarmChunkRetrievalTimeoutHeader = "Swarm-Chunk-Retrieval-Timeout-Level" + SwarmChunkRetrievalTimeoutHeader = "Swarm-Chunk-Retrieval-Timeout" ImmutableHeader = "Immutable" GasPriceHeader = "Gas-Price" From d628dc40d20a91fe380f8a7433b1a11ca167e60c Mon Sep 17 00:00:00 2001 From: zelig Date: Wed, 20 Dec 2023 23:27:15 +0100 Subject: [PATCH 03/52] refactor(getter): add delayed and forgetting store mocks --- pkg/file/redundancy/getter/getter_test.go | 31 +--------- pkg/storer/mock/forgetting.go | 70 +++++++++++++++++++++++ 2 files changed, 73 insertions(+), 28 deletions(-) create mode 100644 pkg/storer/mock/forgetting.go diff --git a/pkg/file/redundancy/getter/getter_test.go b/pkg/file/redundancy/getter/getter_test.go index bb00f7ddd02..2b85d11c589 100644 --- a/pkg/file/redundancy/getter/getter_test.go +++ b/pkg/file/redundancy/getter/getter_test.go @@ -21,37 +21,12 @@ import ( "github.com/ethersphere/bee/pkg/file/redundancy/getter" "github.com/ethersphere/bee/pkg/storage" inmem "github.com/ethersphere/bee/pkg/storage/inmemchunkstore" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" "github.com/ethersphere/bee/pkg/swarm" "github.com/klauspost/reedsolomon" "golang.org/x/sync/errgroup" ) -type delayed struct { - storage.ChunkStore - cache map[string]time.Duration - mu sync.Mutex -} - -func (d *delayed) delay(addr swarm.Address, delay time.Duration) { - d.mu.Lock() - defer d.mu.Unlock() - d.cache[addr.String()] = delay -} - -func (d *delayed) Get(ctx context.Context, addr swarm.Address) (ch swarm.Chunk, err error) { - d.mu.Lock() - defer d.mu.Unlock() - if delay, ok := d.cache[addr.String()]; ok && delay > 0 { - select { - case <-time.After(delay): - delete(d.cache, addr.String()) - case <-ctx.Done(): - return nil, ctx.Err() - } - } - return d.ChunkStore.Get(ctx, addr) -} - // TestGetter tests the retrieval of chunks with missing data shards // using the RACE strategy for a number of erasure code parameters func TestGetterRACE(t *testing.T) { @@ -181,7 +156,7 @@ func testDecodingFallback(t *testing.T, s getter.Strategy, strict bool) { bufSize := 12 shardCnt := 6 - store := &delayed{ChunkStore: inmem.New(), cache: make(map[string]time.Duration)} + store := mockstorer.NewDelayedStore(inmem.New()) buf := make([][]byte, bufSize) addrs := initData(t, buf, shardCnt, store) @@ -205,7 +180,7 @@ func testDecodingFallback(t *testing.T, s getter.Strategy, strict bool) { if s == getter.NONE { delay += getter.StrategyTimeout } - store.delay(addrs[delayed], delay) + store.Delay(addrs[delayed], delay) // create getter start := time.Now() g := getter.New(addrs, shardCnt, store, store, s, strict, getter.StrategyTimeout/2, func() {}) diff --git a/pkg/storer/mock/forgetting.go b/pkg/storer/mock/forgetting.go new file mode 100644 index 00000000000..4401c4b5e11 --- /dev/null +++ b/pkg/storer/mock/forgetting.go @@ -0,0 +1,70 @@ +// Copyright 2023 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package mockstorer + +import ( + "context" + "sync" + "sync/atomic" + "time" + + storage "github.com/ethersphere/bee/pkg/storage" + "github.com/ethersphere/bee/pkg/swarm" +) + +type DelayedStore struct { + storage.ChunkStore + cache map[string]time.Duration + mu sync.Mutex +} + +func NewDelayedStore(s storage.ChunkStore) *DelayedStore { + return &DelayedStore{ + ChunkStore: s, + cache: make(map[string]time.Duration), + } +} + +func (d *DelayedStore) Delay(addr swarm.Address, delay time.Duration) { + d.mu.Lock() + defer d.mu.Unlock() + d.cache[addr.String()] = delay +} + +func (d *DelayedStore) Get(ctx context.Context, addr swarm.Address) (ch swarm.Chunk, err error) { + d.mu.Lock() + defer d.mu.Unlock() + if delay, ok := d.cache[addr.String()]; ok && delay > 0 { + select { + case <-time.After(delay): + delete(d.cache, addr.String()) + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return d.ChunkStore.Get(ctx, addr) +} + +type ForgettingStore struct { + *DelayedStore + oneIn int + delay time.Duration + n atomic.Uint32 +} + +func NewForgettingStore(s storage.ChunkStore, oneIn int, delay time.Duration) *ForgettingStore { + return &ForgettingStore{ + DelayedStore: NewDelayedStore(s), + oneIn: oneIn, + delay: delay, + } +} + +func (f *ForgettingStore) Put(ctx context.Context, ch swarm.Chunk) (err error) { + if f.oneIn > 0 && int(f.n.Add(1))%f.oneIn == 0 { + f.DelayedStore.Delay(ch.Address(), f.delay) + } + return f.DelayedStore.Put(ctx, ch) +} From a202707bedbe5520efb2c0ef19ba24446428e3d1 Mon Sep 17 00:00:00 2001 From: zelig Date: Wed, 20 Dec 2023 23:30:58 +0100 Subject: [PATCH 04/52] fix(getter): timeout header parsing for time duration --- pkg/api/bzz.go | 5 ++++- pkg/file/joiner/joiner.go | 5 ++++- pkg/file/joiner/joiner_test.go | 2 +- pkg/file/redundancy/getter/strategies.go | 26 ++++++++++++++++++------ 4 files changed, 29 insertions(+), 9 deletions(-) diff --git a/pkg/api/bzz.go b/pkg/api/bzz.go index 3f8cfbac1f6..da4b2d603d3 100644 --- a/pkg/api/bzz.go +++ b/pkg/api/bzz.go @@ -274,6 +274,7 @@ func (s *Service) serveReference(logger log.Logger, address swarm.Address, pathV headers := struct { Cache *bool `map:"Swarm-Cache"` }{} + if response := s.mapStructure(r.Header, &headers); response != nil { response("invalid header params", logger, w) return @@ -459,8 +460,9 @@ func (s *Service) downloadHandler(logger log.Logger, w http.ResponseWriter, r *h Cache *bool `map:"Swarm-Cache"` Strategy getter.Strategy `map:"Swarm-Redundancy-Strategy"` FallbackMode bool `map:"Swarm-Redundancy-Fallback-Mode"` - ChunkRetrievalTimeout time.Duration `map:"Swarm-Chunk-Retrieval-Timeout"` + ChunkRetrievalTimeout string `map:"Swarm-Chunk-Retrieval-Timeout"` }{} + if response := s.mapStructure(r.Header, &headers); response != nil { response("invalid header params", logger, w) return @@ -471,6 +473,7 @@ func (s *Service) downloadHandler(logger log.Logger, w http.ResponseWriter, r *h } ctx := r.Context() + fmt.Println("downloadHandler", "Strategy", headers.Strategy, "FallbackMode", headers.FallbackMode, "ChunkRetrievalTimeout", headers.ChunkRetrievalTimeout) ctx = getter.SetStrategy(ctx, headers.Strategy) ctx = getter.SetStrict(ctx, headers.FallbackMode) ctx = getter.SetFetchTimeout(ctx, headers.ChunkRetrievalTimeout) diff --git a/pkg/file/joiner/joiner.go b/pkg/file/joiner/joiner.go index c1ea6f5b4ea..020e74711f2 100644 --- a/pkg/file/joiner/joiner.go +++ b/pkg/file/joiner/joiner.go @@ -125,7 +125,10 @@ func New(ctx context.Context, g storage.Getter, putter storage.Putter, address s if rLevel != redundancy.NONE { _, parities := file.ReferenceCount(uint64(span), rLevel, encryption) rootParity = parities - strategy, strict, fetcherTimeout = getter.GetParamsFromContext(ctx) + strategy, strict, fetcherTimeout, err = getter.GetParamsFromContext(ctx) + if err != nil { + return nil, 0, err + } spanFn = chunkToSpan if encryption { maxBranching = rLevel.GetMaxEncShards() diff --git a/pkg/file/joiner/joiner_test.go b/pkg/file/joiner/joiner_test.go index 7a97080171b..8c65cab20ff 100644 --- a/pkg/file/joiner/joiner_test.go +++ b/pkg/file/joiner/joiner_test.go @@ -1115,7 +1115,7 @@ func TestJoinerRedundancy(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 15*getter.StrategyTimeout) defer cancel() - ctx = getter.SetFetchTimeout(ctx, getter.StrategyTimeout) + ctx = getter.SetFetchTimeout(ctx, getter.StrategyTimeout.String()) joinReader, rootSpan, err := joiner.New(ctx, store, store, swarmAddr) if err != nil { t.Fatal(err) diff --git a/pkg/file/redundancy/getter/strategies.go b/pkg/file/redundancy/getter/strategies.go index 8bf944e8ae7..03557caee43 100644 --- a/pkg/file/redundancy/getter/strategies.go +++ b/pkg/file/redundancy/getter/strategies.go @@ -30,15 +30,29 @@ const ( ) // GetParamsFromContext extracts the strategy and strict mode from the context -func GetParamsFromContext(ctx context.Context) (s Strategy, strict bool, fetcherTimeout time.Duration) { - s, _ = ctx.Value(strategyKey{}).(Strategy) - strict, _ = ctx.Value(modeKey{}).(bool) - fetcherTimeout, _ = ctx.Value(fetcherTimeoutKey{}).(time.Duration) - return s, strict, fetcherTimeout +func GetParamsFromContext(ctx context.Context) (s Strategy, strict bool, fetcherTimeout time.Duration, err error) { + var ok bool + s, ok = ctx.Value(strategyKey{}).(Strategy) + if !ok { + return s, strict, fetcherTimeout, fmt.Errorf("error setting strategy from context") + } + strict, ok = ctx.Value(modeKey{}).(bool) + if !ok { + return s, strict, fetcherTimeout, fmt.Errorf("error setting fallback mode from context") + } + fetcherTimeoutVal, ok := ctx.Value(fetcherTimeoutKey{}).(string) + if !ok { + return s, strict, fetcherTimeout, fmt.Errorf("error setting fetcher timeout from context") + } + fetcherTimeout, err = time.ParseDuration(fetcherTimeoutVal) + if err != nil { + return s, strict, fetcherTimeout, fmt.Errorf("error parsing fetcher timeout from context") + } + return s, strict, fetcherTimeout, nil } // SetFetchTimeout sets the timeout for each fetch -func SetFetchTimeout(ctx context.Context, timeout time.Duration) context.Context { +func SetFetchTimeout(ctx context.Context, timeout string) context.Context { return context.WithValue(ctx, fetcherTimeoutKey{}, timeout) } From df9a82be4bebddbe93c26cfbf2953e535af0e5d1 Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 21 Dec 2023 03:44:48 +0100 Subject: [PATCH 05/52] feat(api): extensive api tests for redundancy --- pkg/api/bzz_test.go | 101 +++++++++++++++ pkg/util/ioutil/pseudorand/reader.go | 144 ++++++++++++++++++++++ pkg/util/ioutil/pseudorand/reader_test.go | 71 +++++++++++ 3 files changed, 316 insertions(+) create mode 100644 pkg/util/ioutil/pseudorand/reader.go create mode 100644 pkg/util/ioutil/pseudorand/reader_test.go diff --git a/pkg/api/bzz_test.go b/pkg/api/bzz_test.go index d2ad44e507c..254f3412106 100644 --- a/pkg/api/bzz_test.go +++ b/pkg/api/bzz_test.go @@ -16,6 +16,7 @@ import ( "strconv" "strings" "testing" + "time" "github.com/ethersphere/bee/pkg/api" "github.com/ethersphere/bee/pkg/file/loadsave" @@ -25,11 +26,111 @@ import ( "github.com/ethersphere/bee/pkg/manifest" mockbatchstore "github.com/ethersphere/bee/pkg/postage/batchstore/mock" mockpost "github.com/ethersphere/bee/pkg/postage/mock" + "github.com/ethersphere/bee/pkg/storage/inmemchunkstore" mockstorer "github.com/ethersphere/bee/pkg/storer/mock" "github.com/ethersphere/bee/pkg/swarm" + "github.com/ethersphere/bee/pkg/util/ioutil/pseudorand" ) // nolint:paralleltest,tparallel +func TestBzzUploadWithRedundancy(t *testing.T) { + fileUploadResource := "/bzz" + fileDownloadResource := func(addr string) string { return "/bzz/" + addr } + fetchTimeout := 200 * time.Millisecond + storerMock := mockstorer.NewWithChunkStore(mockstorer.NewForgettingStore(inmemchunkstore.New(), 2, fetchTimeout)) + client, _, _, _ := newTestServer(t, testServerOptions{ + Storer: storerMock, + Logger: log.Noop, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + + type testCase struct { + name string + redundancy string + size int + encrypt string + } + var tcs []testCase + for _, encrypt := range []string{"true", "false"} { + for _, level := range []string{"0", "1", "2", "3", "4"} { + for _, size := range []int{1, 42, 420, 4095, 4096, 4200, 128*4096 + 4095} { + tcs = append(tcs, testCase{ + name: fmt.Sprintf("level-%s-encrypt-%s-size-%d", level, encrypt, size), + redundancy: level, + size: size, + encrypt: encrypt, + }) + } + } + } + + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + seed, err := pseudorand.NewSeed() + if err != nil { + t.Fatal(err) + } + reader := pseudorand.NewReader(seed, tc.size) + + var refResponse api.BzzUploadResponse + jsonhttptest.Request(t, client, http.MethodPost, fileUploadResource, + http.StatusCreated, + jsonhttptest.WithRequestHeader(api.SwarmDeferredUploadHeader, "True"), + jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, batchOkStr), + jsonhttptest.WithRequestBody(reader), + jsonhttptest.WithRequestHeader(api.SwarmEncryptHeader, tc.encrypt), + jsonhttptest.WithRequestHeader(api.SwarmRedundancyLevelHeader, tc.redundancy), + jsonhttptest.WithRequestHeader(api.ContentTypeHeader, "image/jpeg; charset=utf-8"), + jsonhttptest.WithUnmarshalJSONResponse(&refResponse), + ) + + t.Run("download without redundancy should fail", func(t *testing.T) { + req, err := http.NewRequest("GET", fileDownloadResource(refResponse.Reference.String()), nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set(api.SwarmRedundancyStrategyHeader, "0") + req.Header.Set(api.SwarmRedundancyFallbackModeHeader, "false") + req.Header.Set(api.SwarmChunkRetrievalTimeoutHeader, fetchTimeout.String()) + + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("expected status %d; got %d", http.StatusNotFound, resp.StatusCode) + } + }) + + t.Run("download with redundancy should fail", func(t *testing.T) { + req, err := http.NewRequest("GET", fileDownloadResource(refResponse.Reference.String()), nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set(api.SwarmRedundancyStrategyHeader, "0") + req.Header.Set(api.SwarmRedundancyFallbackModeHeader, "true") + req.Header.Set(api.SwarmChunkRetrievalTimeoutHeader, fetchTimeout.String()) + + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected status %d; got %d", http.StatusOK, resp.StatusCode) + } + + if !reader.Equal(resp.Body) { + t.Fatalf("content mismatch") + } + }) + }) + } +} + func TestBzzFiles(t *testing.T) { t.Parallel() diff --git a/pkg/util/ioutil/pseudorand/reader.go b/pkg/util/ioutil/pseudorand/reader.go new file mode 100644 index 00000000000..d1d14314845 --- /dev/null +++ b/pkg/util/ioutil/pseudorand/reader.go @@ -0,0 +1,144 @@ +// Copyright 2023 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// this is a pseudorandom reader that generates a deterministic +// sequence of bytes based on the seed. It is used in tests to +// enable large volumes of pseudorandom data to be generated +// and compared without having to store the data in memory. +package pseudorand + +import ( + "bytes" + "crypto/rand" + "encoding/binary" + "errors" + "io" + + "github.com/ethersphere/bee/pkg/swarm" +) + +const bufSize = 4096 + +// Reader is a pseudorandom reader that generates a deterministic +// sequence of bytes based on the seed. +type Reader struct { + cur int + len int + seg [40]byte + buf [bufSize]byte +} + +// NewSeed creates a new seed. +func NewSeed() ([]byte, error) { + seed := make([]byte, 32) + _, err := io.ReadFull(rand.Reader, seed) + return seed, err +} + +// New creates a new pseudorandom reader seeded with the given seed. +func NewReader(seed []byte, len int) *Reader { + r := &Reader{len: len} + _ = copy(r.buf[8:], seed) + r.fill() + return r +} + +// Read reads len(buf) bytes into buf. It returns the number of bytes read (0 <= n <= len(buf)) and any error encountered. Even if Read returns n < len(buf), it may use all of buf as scratch space during the call. If some data is available but not len(buf) bytes, Read conventionally returns what is available instead of waiting for more. +func (r *Reader) Read(buf []byte) (n int, err error) { + toRead := r.len - r.cur + if toRead < len(buf) { + buf = buf[:toRead] + } + n = copy(buf, r.buf[r.cur%bufSize:]) + r.cur += n + if r.cur == r.len { + return n, io.EOF + } + if r.cur%bufSize == 0 { + r.fill() + } + return n, nil +} + +// Reset resets the reader to the beginning of the stream. +func (r *Reader) Reset() { + r.Seek(0, 0) + r.fill() +} + +// Equal compares the contents of the reader with the contents of +// the given reader. It returns true if the contents are equal upto n bytes +func (r1 *Reader) Equal(r2 io.Reader) bool { + r1.Reset() + buf1 := make([]byte, bufSize) + buf2 := make([]byte, bufSize) + read := func(r io.Reader, buf []byte) (n int, err error) { + for n < len(buf) && err == nil { + i, e := r.Read(buf[n:]) + if e == nil && i == 0 { + return n, nil + } + err = e + n += i + } + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + err = nil + } + return n, err + } + for { + n1, err := read(r1, buf1) + if err != nil { + return false + } + n2, err := read(r2, buf2) + if err != nil { + return false + } + if n1 < bufSize || n2 < bufSize { + return n1 == n2 && bytes.Equal(buf1[:n1], buf2[:n2]) + } + if !bytes.Equal(buf1, buf2) { + return false + } + } +} + +// Seek sets the offset for the next Read to offset, interpreted +// according to whence: 0 means relative to the start of the file, +// 1 means relative to the current offset, and 2 means relative to +// the end. It returns the new offset and an error, if any. +func (r *Reader) Seek(offset int64, whence int) (int64, error) { + switch whence { + case 0: + r.cur = int(offset) + case 1: + r.cur += int(offset) + case 2: + r.cur = 4096 + int(offset) + } + return int64(r.cur), nil +} + +// Offset returns the current offset of the reader. +func (r *Reader) Offset() int64 { + return int64(r.cur) +} + +// ReadAt reads len(buf) bytes into buf starting at offset off. +func (r *Reader) ReadAt(buf []byte, off int64) (n int, err error) { + r.cur = int(off) + return r.Read(buf) +} + +// fill fills the buffer with the hash of the current segment. +func (r *Reader) fill() { + h := swarm.NewHasher() + for i := 0; i < len(r.buf); i += 32 { + binary.BigEndian.PutUint64(r.seg[:], uint64((r.cur+i)/32)) + h.Reset() + _, _ = h.Write(r.seg[:]) + copy(r.buf[i:i+32], h.Sum(nil)) + } +} diff --git a/pkg/util/ioutil/pseudorand/reader_test.go b/pkg/util/ioutil/pseudorand/reader_test.go new file mode 100644 index 00000000000..4c786adf215 --- /dev/null +++ b/pkg/util/ioutil/pseudorand/reader_test.go @@ -0,0 +1,71 @@ +// Copyright 2023 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pseudorand_test + +import ( + "bytes" + "io" + "math/rand" + "testing" + + "github.com/ethersphere/bee/pkg/util/ioutil/pseudorand" +) + +func TestReader(t *testing.T) { + size := 42000 + seed := make([]byte, 32) + r := pseudorand.NewReader(seed, size) + content, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + t.Run("deterministicity", func(t *testing.T) { + r2 := pseudorand.NewReader(seed, size) + content2, err := io.ReadAll(r2) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(content, content2) { + t.Fatal("content mismatch") + } + }) + t.Run("re-readability", func(t *testing.T) { + r.Reset() + var read []byte + buf := make([]byte, 8200) + total := 0 + for { + s := rand.Intn(820) + n, err := r.Read(buf[:s]) + total += n + read = append(read, buf[:n]...) + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + } + read = read[:total] + if len(read) != len(content) { + t.Fatal("content length mismatch. expected", len(content), "got", len(read)) + } + if !bytes.Equal(content, read) { + t.Fatal("content mismatch") + } + }) + t.Run("comparison", func(t *testing.T) { + if !r.Equal(bytes.NewBuffer(content)) { + t.Fatal("content mismatch") + } + if r.Equal(bytes.NewBuffer(content[:len(content)-1])) { + t.Fatal("content should not match") + } + content[0] = content[0] + 1 + if r.Equal(bytes.NewBuffer(content)) { + t.Fatal("content should not match") + } + }) +} From b54e8320089f4f253f28885fc58b6a855166ea9f Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 21 Dec 2023 08:02:13 +0100 Subject: [PATCH 06/52] fix(api): redundancy param strict = !fallbackmode (value of header) --- pkg/api/bzz.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/api/bzz.go b/pkg/api/bzz.go index da4b2d603d3..aff97d0fe3d 100644 --- a/pkg/api/bzz.go +++ b/pkg/api/bzz.go @@ -464,6 +464,7 @@ func (s *Service) downloadHandler(logger log.Logger, w http.ResponseWriter, r *h }{} if response := s.mapStructure(r.Header, &headers); response != nil { + fmt.Println("downloadHandler", "invalid header params") response("invalid header params", logger, w) return } @@ -475,7 +476,7 @@ func (s *Service) downloadHandler(logger log.Logger, w http.ResponseWriter, r *h ctx := r.Context() fmt.Println("downloadHandler", "Strategy", headers.Strategy, "FallbackMode", headers.FallbackMode, "ChunkRetrievalTimeout", headers.ChunkRetrievalTimeout) ctx = getter.SetStrategy(ctx, headers.Strategy) - ctx = getter.SetStrict(ctx, headers.FallbackMode) + ctx = getter.SetStrict(ctx, !headers.FallbackMode) ctx = getter.SetFetchTimeout(ctx, headers.ChunkRetrievalTimeout) reader, l, err := joiner.New(ctx, s.storer.Download(cache), s.storer.Cache(), reference) if err != nil { From 0738117265d9ec805e4c7865defb49669e374d8f Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 21 Dec 2023 11:14:25 +0100 Subject: [PATCH 07/52] test: tmp, trying to figure out context --- pkg/api/bzz.go | 18 ++++++++++++----- pkg/api/bzz_test.go | 11 +++++++---- pkg/file/joiner/joiner.go | 19 +++++++++++------- pkg/file/redundancy/getter/strategies.go | 25 ++++++++++++++++++++---- 4 files changed, 53 insertions(+), 20 deletions(-) diff --git a/pkg/api/bzz.go b/pkg/api/bzz.go index aff97d0fe3d..af02a1a1e37 100644 --- a/pkg/api/bzz.go +++ b/pkg/api/bzz.go @@ -134,6 +134,7 @@ func (s *Service) fileUploadHandler( response("invalid query params", logger, w) return } + fmt.Println("fileUploadHandler", "rlevel", rLevel) p := requestPipelineFn(putter, encrypt, rLevel) ctx := r.Context() @@ -252,6 +253,7 @@ func (s *Service) fileUploadHandler( func (s *Service) bzzDownloadHandler(w http.ResponseWriter, r *http.Request) { logger := tracing.NewLoggerWithTraceID(r.Context(), s.logger.WithName("get_bzz_by_path").Build()) + fmt.Println("bzzDownloadHandler", "r.URL.Path", r.URL.Path) paths := struct { Address swarm.Address `map:"address,resolve" validate:"required"` Path string `map:"path"` @@ -272,10 +274,14 @@ func (s *Service) serveReference(logger log.Logger, address swarm.Address, pathV loggerV1 := logger.V(1).Build() headers := struct { - Cache *bool `map:"Swarm-Cache"` + Cache *bool `map:"Swarm-Cache"` + Strategy getter.Strategy `map:"Swarm-Redundancy-Strategy"` + FallbackMode bool `map:"Swarm-Redundancy-Fallback-Mode"` + ChunkRetrievalTimeout string `map:"Swarm-Chunk-Retrieval-Timeout"` }{} if response := s.mapStructure(r.Header, &headers); response != nil { + fmt.Println("downloadHandler", "invalid header params") response("invalid header params", logger, w) return } @@ -283,10 +289,14 @@ func (s *Service) serveReference(logger log.Logger, address swarm.Address, pathV if headers.Cache != nil { cache = *headers.Cache } + + fmt.Println("serveReference", "address", address, "pathVar", pathVar, "cache", cache) ls := loadsave.NewReadonly(s.storer.Download(cache)) feedDereferenced := false ctx := r.Context() + fmt.Println("serveReference", "Strategy", headers.Strategy, "FallbackMode", headers.FallbackMode, "ChunkRetrievalTimeout", headers.ChunkRetrievalTimeout) + getter.SetParamsInContext(ctx, headers.Strategy, headers.FallbackMode, headers.ChunkRetrievalTimeout) FETCH: // read manifest entry @@ -367,7 +377,7 @@ FETCH: jsonhttp.NotFound(w, "address not found or incorrect") return } - + fmt.Println("bzz download: path", pathVar) me, err := m.Lookup(ctx, pathVar) if err != nil { loggerV1.Debug("bzz download: invalid path", "address", address, "path", pathVar, "error", err) @@ -475,9 +485,7 @@ func (s *Service) downloadHandler(logger log.Logger, w http.ResponseWriter, r *h ctx := r.Context() fmt.Println("downloadHandler", "Strategy", headers.Strategy, "FallbackMode", headers.FallbackMode, "ChunkRetrievalTimeout", headers.ChunkRetrievalTimeout) - ctx = getter.SetStrategy(ctx, headers.Strategy) - ctx = getter.SetStrict(ctx, !headers.FallbackMode) - ctx = getter.SetFetchTimeout(ctx, headers.ChunkRetrievalTimeout) + getter.SetParamsInContext(ctx, headers.Strategy, headers.FallbackMode, headers.ChunkRetrievalTimeout) reader, l, err := joiner.New(ctx, s.storer.Download(cache), s.storer.Cache(), reference) if err != nil { if errors.Is(err, storage.ErrNotFound) || errors.Is(err, topology.ErrNotFound) { diff --git a/pkg/api/bzz_test.go b/pkg/api/bzz_test.go index 254f3412106..8a5e1fb2590 100644 --- a/pkg/api/bzz_test.go +++ b/pkg/api/bzz_test.go @@ -40,8 +40,9 @@ func TestBzzUploadWithRedundancy(t *testing.T) { storerMock := mockstorer.NewWithChunkStore(mockstorer.NewForgettingStore(inmemchunkstore.New(), 2, fetchTimeout)) client, _, _, _ := newTestServer(t, testServerOptions{ Storer: storerMock, - Logger: log.Noop, - Post: mockpost.New(mockpost.WithAcceptAll()), + Logger: log.NewLogger("stdout", log.WithVerbosity(log.VerbosityDebug), log.WithCallerFunc()), + // Logger: log.Noop , + Post: mockpost.New(mockpost.WithAcceptAll()), }) type testCase struct { @@ -51,7 +52,7 @@ func TestBzzUploadWithRedundancy(t *testing.T) { encrypt string } var tcs []testCase - for _, encrypt := range []string{"true", "false"} { + for _, encrypt := range []string{"false", "true"} { for _, level := range []string{"0", "1", "2", "3", "4"} { for _, size := range []int{1, 42, 420, 4095, 4096, 4200, 128*4096 + 4095} { tcs = append(tcs, testCase{ @@ -85,6 +86,7 @@ func TestBzzUploadWithRedundancy(t *testing.T) { ) t.Run("download without redundancy should fail", func(t *testing.T) { + t.Skip("no") req, err := http.NewRequest("GET", fileDownloadResource(refResponse.Reference.String()), nil) if err != nil { t.Fatal(err) @@ -104,7 +106,8 @@ func TestBzzUploadWithRedundancy(t *testing.T) { } }) - t.Run("download with redundancy should fail", func(t *testing.T) { + t.Run("download with redundancy should succeed", func(t *testing.T) { + // t.Skip("no") req, err := http.NewRequest("GET", fileDownloadResource(refResponse.Reference.String()), nil) if err != nil { t.Fatal(err) diff --git a/pkg/file/joiner/joiner.go b/pkg/file/joiner/joiner.go index 020e74711f2..92c3f9e8f86 100644 --- a/pkg/file/joiner/joiner.go +++ b/pkg/file/joiner/joiner.go @@ -8,6 +8,7 @@ package joiner import ( "context" "errors" + "fmt" "io" "sync" "sync/atomic" @@ -52,6 +53,7 @@ type decoderCache struct { // NewDecoderCache creates a new decoder cache func NewDecoderCache(g storage.Getter, p storage.Putter, strategy getter.Strategy, strict bool, fetcherTimeout time.Duration) *decoderCache { + fmt.Println("NewDecoderCache", strategy, strict, fetcherTimeout) return &decoderCache{ fetcher: g, putter: p, @@ -118,23 +120,26 @@ func New(ctx context.Context, g storage.Getter, putter storage.Putter, address s spanFn := func(data []byte) (redundancy.Level, int64) { return 0, int64(bmt.LengthFromSpan(data[:swarm.SpanSize])) } - var strategy getter.Strategy - var strict bool - var fetcherTimeout time.Duration + strategy, strict, fetcherTimeout, err := getter.GetParamsFromContext(ctx) + if err != nil { + fmt.Printf("get params from context: %v\n", err) + return nil, 0, err + } // override stuff if root chunk has redundancy if rLevel != redundancy.NONE { _, parities := file.ReferenceCount(uint64(span), rLevel, encryption) rootParity = parities - strategy, strict, fetcherTimeout, err = getter.GetParamsFromContext(ctx) - if err != nil { - return nil, 0, err - } + spanFn = chunkToSpan if encryption { maxBranching = rLevel.GetMaxEncShards() } else { maxBranching = rLevel.GetMaxShards() } + } else { + // if root chunk has no redundancy, strategy is ignored and set to NONE and strict is set to true + strategy = getter.NONE + strict = true } j := &joiner{ diff --git a/pkg/file/redundancy/getter/strategies.go b/pkg/file/redundancy/getter/strategies.go index 03557caee43..6639f09b0b5 100644 --- a/pkg/file/redundancy/getter/strategies.go +++ b/pkg/file/redundancy/getter/strategies.go @@ -7,6 +7,7 @@ package getter import ( "context" "fmt" + "strconv" "time" ) @@ -31,15 +32,25 @@ const ( // GetParamsFromContext extracts the strategy and strict mode from the context func GetParamsFromContext(ctx context.Context) (s Strategy, strict bool, fetcherTimeout time.Duration, err error) { - var ok bool - s, ok = ctx.Value(strategyKey{}).(Strategy) + // var ok bool + sStr, ok := ctx.Value(strategyKey{}).(string) if !ok { return s, strict, fetcherTimeout, fmt.Errorf("error setting strategy from context") } - strict, ok = ctx.Value(modeKey{}).(bool) + sNum, err := strconv.ParseUint(sStr, 10, 2) + if err != nil { + return s, strict, fetcherTimeout, fmt.Errorf("error parsing strategy from context") + } + s = Strategy(sNum) + + strictStr, ok := ctx.Value(modeKey{}).(string) if !ok { return s, strict, fetcherTimeout, fmt.Errorf("error setting fallback mode from context") } + strict, err = strconv.ParseBool(strictStr) + if err != nil { + return s, strict, fetcherTimeout, fmt.Errorf("error parsing fallback mode from context") + } fetcherTimeoutVal, ok := ctx.Value(fetcherTimeoutKey{}).(string) if !ok { return s, strict, fetcherTimeout, fmt.Errorf("error setting fetcher timeout from context") @@ -58,7 +69,7 @@ func SetFetchTimeout(ctx context.Context, timeout string) context.Context { // SetStrategy sets the strategy for the retrieval func SetStrategy(ctx context.Context, s Strategy) context.Context { - return context.WithValue(ctx, strategyKey{}, s) + return context.WithValue(ctx, strategyKey{}, int(s)) } // SetStrict sets the strict mode for the retrieval @@ -66,6 +77,12 @@ func SetStrict(ctx context.Context, strict bool) context.Context { return context.WithValue(ctx, modeKey{}, strict) } +func SetParamsInContext(ctx context.Context, s Strategy, fallbackmode bool, fetchTimeout string) context.Context { + ctx = SetStrategy(ctx, s) + ctx = SetStrict(ctx, !fallbackmode) + ctx = SetFetchTimeout(ctx, fetchTimeout) + return ctx +} func (g *decoder) prefetch(ctx context.Context, strategy int, strict bool, strategyTimeout, fetchTimeout time.Duration) { if strict && strategy == NONE { return From 14c8627156b7f76f0251e920122f70bdd483b834 Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 21 Dec 2023 13:13:40 +0100 Subject: [PATCH 08/52] fix(api): context passing --- pkg/api/bzz.go | 4 ++-- pkg/api/bzz_test.go | 14 ++++++------ pkg/file/redundancy/getter/strategies.go | 28 ++++++++++++++---------- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/pkg/api/bzz.go b/pkg/api/bzz.go index af02a1a1e37..9d1ce02e1dd 100644 --- a/pkg/api/bzz.go +++ b/pkg/api/bzz.go @@ -296,7 +296,7 @@ func (s *Service) serveReference(logger log.Logger, address swarm.Address, pathV ctx := r.Context() fmt.Println("serveReference", "Strategy", headers.Strategy, "FallbackMode", headers.FallbackMode, "ChunkRetrievalTimeout", headers.ChunkRetrievalTimeout) - getter.SetParamsInContext(ctx, headers.Strategy, headers.FallbackMode, headers.ChunkRetrievalTimeout) + ctx = getter.SetParamsInContext(ctx, headers.Strategy, headers.FallbackMode, headers.ChunkRetrievalTimeout) FETCH: // read manifest entry @@ -485,7 +485,7 @@ func (s *Service) downloadHandler(logger log.Logger, w http.ResponseWriter, r *h ctx := r.Context() fmt.Println("downloadHandler", "Strategy", headers.Strategy, "FallbackMode", headers.FallbackMode, "ChunkRetrievalTimeout", headers.ChunkRetrievalTimeout) - getter.SetParamsInContext(ctx, headers.Strategy, headers.FallbackMode, headers.ChunkRetrievalTimeout) + ctx = getter.SetParamsInContext(ctx, headers.Strategy, headers.FallbackMode, headers.ChunkRetrievalTimeout) reader, l, err := joiner.New(ctx, s.storer.Download(cache), s.storer.Cache(), reference) if err != nil { if errors.Is(err, storage.ErrNotFound) || errors.Is(err, topology.ErrNotFound) { diff --git a/pkg/api/bzz_test.go b/pkg/api/bzz_test.go index 8a5e1fb2590..f1d5684ae3f 100644 --- a/pkg/api/bzz_test.go +++ b/pkg/api/bzz_test.go @@ -37,7 +37,7 @@ func TestBzzUploadWithRedundancy(t *testing.T) { fileUploadResource := "/bzz" fileDownloadResource := func(addr string) string { return "/bzz/" + addr } fetchTimeout := 200 * time.Millisecond - storerMock := mockstorer.NewWithChunkStore(mockstorer.NewForgettingStore(inmemchunkstore.New(), 2, fetchTimeout)) + storerMock := mockstorer.NewWithChunkStore(mockstorer.NewForgettingStore(inmemchunkstore.New(), 2, 8*fetchTimeout)) client, _, _, _ := newTestServer(t, testServerOptions{ Storer: storerMock, Logger: log.NewLogger("stdout", log.WithVerbosity(log.VerbosityDebug), log.WithCallerFunc()), @@ -54,7 +54,8 @@ func TestBzzUploadWithRedundancy(t *testing.T) { var tcs []testCase for _, encrypt := range []string{"false", "true"} { for _, level := range []string{"0", "1", "2", "3", "4"} { - for _, size := range []int{1, 42, 420, 4095, 4096, 4200, 128*4096 + 4095} { + // for _, size := range []int{4095, 4096, 42 * 420} { + for _, size := range []int{31 * 4096, 128 * 4096, 128*4096 + 4095} { tcs = append(tcs, testCase{ name: fmt.Sprintf("level-%s-encrypt-%s-size-%d", level, encrypt, size), redundancy: level, @@ -86,7 +87,7 @@ func TestBzzUploadWithRedundancy(t *testing.T) { ) t.Run("download without redundancy should fail", func(t *testing.T) { - t.Skip("no") + // t.Skip("no") req, err := http.NewRequest("GET", fileDownloadResource(refResponse.Reference.String()), nil) if err != nil { t.Fatal(err) @@ -100,7 +101,9 @@ func TestBzzUploadWithRedundancy(t *testing.T) { t.Fatal(err) } defer resp.Body.Close() - + if resp.StatusCode != http.StatusOK && !reader.Equal(resp.Body) { + t.Fatalf("content mismatch") + } if resp.StatusCode != http.StatusNotFound { t.Fatalf("expected status %d; got %d", http.StatusNotFound, resp.StatusCode) } @@ -126,9 +129,6 @@ func TestBzzUploadWithRedundancy(t *testing.T) { t.Fatalf("expected status %d; got %d", http.StatusOK, resp.StatusCode) } - if !reader.Equal(resp.Body) { - t.Fatalf("content mismatch") - } }) }) } diff --git a/pkg/file/redundancy/getter/strategies.go b/pkg/file/redundancy/getter/strategies.go index 6639f09b0b5..075bd55da2d 100644 --- a/pkg/file/redundancy/getter/strategies.go +++ b/pkg/file/redundancy/getter/strategies.go @@ -7,7 +7,6 @@ package getter import ( "context" "fmt" - "strconv" "time" ) @@ -32,22 +31,29 @@ const ( // GetParamsFromContext extracts the strategy and strict mode from the context func GetParamsFromContext(ctx context.Context) (s Strategy, strict bool, fetcherTimeout time.Duration, err error) { - // var ok bool - sStr, ok := ctx.Value(strategyKey{}).(string) + // // var ok bool + // sStr, ok := ctx.Value(strategyKey{}).(string) + // if !ok { + // return s, strict, fetcherTimeout, fmt.Errorf("error setting strategy from context") + // } + // sNum, err := strconv.ParseUint(sStr, 10, 2) + // if err != nil { + // return s, strict, fetcherTimeout, fmt.Errorf("error parsing strategy from context") + // } + // s = Strategy(sNum) + s, ok := ctx.Value(strategyKey{}).(Strategy) if !ok { return s, strict, fetcherTimeout, fmt.Errorf("error setting strategy from context") } - sNum, err := strconv.ParseUint(sStr, 10, 2) - if err != nil { - return s, strict, fetcherTimeout, fmt.Errorf("error parsing strategy from context") - } - s = Strategy(sNum) - - strictStr, ok := ctx.Value(modeKey{}).(string) + // strictStr, ok := ctx.Value(modeKey{}).(string) + // if !ok { + // return s, strict, fetcherTimeout, fmt.Errorf("error setting fallback mode from context") + // } + // strict, err = strconv.ParseBool(strictStr) + strict, ok = ctx.Value(modeKey{}).(bool) if !ok { return s, strict, fetcherTimeout, fmt.Errorf("error setting fallback mode from context") } - strict, err = strconv.ParseBool(strictStr) if err != nil { return s, strict, fetcherTimeout, fmt.Errorf("error parsing fallback mode from context") } From fad106259bdf84abd038aeafc9a61e090ddd4f92 Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 21 Dec 2023 13:57:06 +0100 Subject: [PATCH 09/52] fix(api): amend value passing through context --- pkg/api/bzz_test.go | 44 +++++++++++++++++++---------------- pkg/storer/mock/forgetting.go | 1 + 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/pkg/api/bzz_test.go b/pkg/api/bzz_test.go index f1d5684ae3f..0e2d58ff76a 100644 --- a/pkg/api/bzz_test.go +++ b/pkg/api/bzz_test.go @@ -36,42 +36,44 @@ import ( func TestBzzUploadWithRedundancy(t *testing.T) { fileUploadResource := "/bzz" fileDownloadResource := func(addr string) string { return "/bzz/" + addr } - fetchTimeout := 200 * time.Millisecond - storerMock := mockstorer.NewWithChunkStore(mockstorer.NewForgettingStore(inmemchunkstore.New(), 2, 8*fetchTimeout)) - client, _, _, _ := newTestServer(t, testServerOptions{ - Storer: storerMock, - Logger: log.NewLogger("stdout", log.WithVerbosity(log.VerbosityDebug), log.WithCallerFunc()), - // Logger: log.Noop , - Post: mockpost.New(mockpost.WithAcceptAll()), - }) type testCase struct { - name string - redundancy string - size int - encrypt string + name string + level int + size int + encrypt string } var tcs []testCase for _, encrypt := range []string{"false", "true"} { - for _, level := range []string{"0", "1", "2", "3", "4"} { + for _, level := range []int{0, 1, 2, 3, 4} { // for _, size := range []int{4095, 4096, 42 * 420} { for _, size := range []int{31 * 4096, 128 * 4096, 128*4096 + 4095} { tcs = append(tcs, testCase{ - name: fmt.Sprintf("level-%s-encrypt-%s-size-%d", level, encrypt, size), - redundancy: level, - size: size, - encrypt: encrypt, + name: fmt.Sprintf("level-%d-encrypt-%s-size-%d", level, encrypt, size), + level: level, + size: size, + encrypt: encrypt, }) } } } + oneIns := []int{0, 100, 20, 10, 2} for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { seed, err := pseudorand.NewSeed() if err != nil { t.Fatal(err) } + fetchTimeout := 200 * time.Millisecond + storerMock := mockstorer.NewWithChunkStore(mockstorer.NewForgettingStore(inmemchunkstore.New(), oneIns[tc.level], fetchTimeout)) + client, _, _, _ := newTestServer(t, testServerOptions{ + Storer: storerMock, + Logger: log.NewLogger("stdout", log.WithVerbosity(log.VerbosityDebug), log.WithCallerFunc()), + // Logger: log.Noop , + Post: mockpost.New(mockpost.WithAcceptAll()), + }) + reader := pseudorand.NewReader(seed, tc.size) var refResponse api.BzzUploadResponse @@ -81,13 +83,15 @@ func TestBzzUploadWithRedundancy(t *testing.T) { jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, batchOkStr), jsonhttptest.WithRequestBody(reader), jsonhttptest.WithRequestHeader(api.SwarmEncryptHeader, tc.encrypt), - jsonhttptest.WithRequestHeader(api.SwarmRedundancyLevelHeader, tc.redundancy), + jsonhttptest.WithRequestHeader(api.SwarmRedundancyLevelHeader, fmt.Sprintf("%d", tc.level)), jsonhttptest.WithRequestHeader(api.ContentTypeHeader, "image/jpeg; charset=utf-8"), jsonhttptest.WithUnmarshalJSONResponse(&refResponse), ) t.Run("download without redundancy should fail", func(t *testing.T) { - // t.Skip("no") + if tc.level == 0 { + t.Skip("NA") + } req, err := http.NewRequest("GET", fileDownloadResource(refResponse.Reference.String()), nil) if err != nil { t.Fatal(err) @@ -101,7 +105,7 @@ func TestBzzUploadWithRedundancy(t *testing.T) { t.Fatal(err) } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK && !reader.Equal(resp.Body) { + if resp.StatusCode == http.StatusOK && !reader.Equal(resp.Body) { t.Fatalf("content mismatch") } if resp.StatusCode != http.StatusNotFound { diff --git a/pkg/storer/mock/forgetting.go b/pkg/storer/mock/forgetting.go index 4401c4b5e11..e7794975eb2 100644 --- a/pkg/storer/mock/forgetting.go +++ b/pkg/storer/mock/forgetting.go @@ -40,6 +40,7 @@ func (d *DelayedStore) Get(ctx context.Context, addr swarm.Address) (ch swarm.Ch select { case <-time.After(delay): delete(d.cache, addr.String()) + _ = d.ChunkStore.Delete(ctx, addr) case <-ctx.Done(): return nil, ctx.Err() } From 37e4b5fec3aa8d5d06523ee06f7c06cb48bf9513 Mon Sep 17 00:00:00 2001 From: zelig Date: Mon, 1 Jan 2024 07:04:35 +0100 Subject: [PATCH 10/52] fix(pipeline): capitalisation --- pkg/file/pipeline/hashtrie/hashtrie_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/file/pipeline/hashtrie/hashtrie_test.go b/pkg/file/pipeline/hashtrie/hashtrie_test.go index c3729f78901..35733665c59 100644 --- a/pkg/file/pipeline/hashtrie/hashtrie_test.go +++ b/pkg/file/pipeline/hashtrie/hashtrie_test.go @@ -46,7 +46,7 @@ func init() { binary.LittleEndian.PutUint64(span, 1) } -// NewErasureHashTrieWriter returns back an redundancy param and a HastTrieWriter pipeline +// newErasureHashTrieWriter returns back an redundancy param and a HastTrieWriter pipeline // which are using simple BMT and StoreWriter pipelines for chunk writes func newErasureHashTrieWriter( ctx context.Context, From 44b52615a476fdbb56a000375fbeb103a438a0f7 Mon Sep 17 00:00:00 2001 From: zelig Date: Mon, 1 Jan 2024 07:19:24 +0100 Subject: [PATCH 11/52] refactor(pseudorand): fix and add match --- pkg/util/ioutil/pseudorand/reader.go | 75 ++++++++++++++++------- pkg/util/ioutil/pseudorand/reader_test.go | 39 +++++++++--- 2 files changed, 84 insertions(+), 30 deletions(-) diff --git a/pkg/util/ioutil/pseudorand/reader.go b/pkg/util/ioutil/pseudorand/reader.go index d1d14314845..95e4a69c79d 100644 --- a/pkg/util/ioutil/pseudorand/reader.go +++ b/pkg/util/ioutil/pseudorand/reader.go @@ -46,11 +46,12 @@ func NewReader(seed []byte, len int) *Reader { // Read reads len(buf) bytes into buf. It returns the number of bytes read (0 <= n <= len(buf)) and any error encountered. Even if Read returns n < len(buf), it may use all of buf as scratch space during the call. If some data is available but not len(buf) bytes, Read conventionally returns what is available instead of waiting for more. func (r *Reader) Read(buf []byte) (n int, err error) { - toRead := r.len - r.cur + cur := r.cur % bufSize + toRead := min(bufSize-cur, r.len-r.cur) if toRead < len(buf) { buf = buf[:toRead] } - n = copy(buf, r.buf[r.cur%bufSize:]) + n = copy(buf, r.buf[cur:]) r.cur += n if r.cur == r.len { return n, io.EOF @@ -61,18 +62,27 @@ func (r *Reader) Read(buf []byte) (n int, err error) { return n, nil } -// Reset resets the reader to the beginning of the stream. -func (r *Reader) Reset() { - r.Seek(0, 0) - r.fill() +// Equal compares the contents of the reader with the contents of +// the given reader. It returns true if the contents are equal upto n bytes +func (r1 *Reader) Equal(r2 io.Reader) (bool, error) { + ok, err := r1.Match(r2, r1.len) + if err != nil { + return false, err + } + if !ok { + return false, nil + } + n, err := io.ReadFull(r2, make([]byte, 1)) + if err == io.EOF || err == io.ErrUnexpectedEOF { + return n == 0, nil + } + return false, err } -// Equal compares the contents of the reader with the contents of +// Match compares the contents of the reader with the contents of // the given reader. It returns true if the contents are equal upto n bytes -func (r1 *Reader) Equal(r2 io.Reader) bool { - r1.Reset() - buf1 := make([]byte, bufSize) - buf2 := make([]byte, bufSize) +func (r1 *Reader) Match(r2 io.Reader, l int) (bool, error) { + read := func(r io.Reader, buf []byte) (n int, err error) { for n < len(buf) && err == nil { i, e := r.Read(buf[n:]) @@ -87,22 +97,32 @@ func (r1 *Reader) Equal(r2 io.Reader) bool { } return n, err } - for { + + buf1 := make([]byte, bufSize) + buf2 := make([]byte, bufSize) + for l > 0 { + if l <= len(buf1) { + buf1 = buf1[:l] + buf2 = buf2[:l] + } + n1, err := read(r1, buf1) if err != nil { - return false + return false, err } n2, err := read(r2, buf2) if err != nil { - return false + return false, err } - if n1 < bufSize || n2 < bufSize { - return n1 == n2 && bytes.Equal(buf1[:n1], buf2[:n2]) + if n1 != n2 { + return false, nil } - if !bytes.Equal(buf1, buf2) { - return false + if !bytes.Equal(buf1[:n1], buf2[:n2]) { + return false, nil } + l -= n1 } + return true, nil } // Seek sets the offset for the next Read to offset, interpreted @@ -116,8 +136,9 @@ func (r *Reader) Seek(offset int64, whence int) (int64, error) { case 1: r.cur += int(offset) case 2: - r.cur = 4096 + int(offset) + r.cur = r.len - int(offset) } + r.fill() return int64(r.cur), nil } @@ -128,17 +149,25 @@ func (r *Reader) Offset() int64 { // ReadAt reads len(buf) bytes into buf starting at offset off. func (r *Reader) ReadAt(buf []byte, off int64) (n int, err error) { - r.cur = int(off) + if _, err := r.Seek(off, io.SeekStart); err != nil { + return 0, err + } return r.Read(buf) } // fill fills the buffer with the hash of the current segment. func (r *Reader) fill() { + if r.cur >= r.len { + return + } + bufSegments := bufSize / 32 + start := r.cur / bufSegments + rem := (r.cur % bufSize) / 32 h := swarm.NewHasher() - for i := 0; i < len(r.buf); i += 32 { - binary.BigEndian.PutUint64(r.seg[:], uint64((r.cur+i)/32)) + for i := 32 * rem; i < len(r.buf); i += 32 { + binary.BigEndian.PutUint64(r.seg[:], uint64((start+i)/32)) h.Reset() _, _ = h.Write(r.seg[:]) - copy(r.buf[i:i+32], h.Sum(nil)) + copy(r.buf[i:], h.Sum(nil)) } } diff --git a/pkg/util/ioutil/pseudorand/reader_test.go b/pkg/util/ioutil/pseudorand/reader_test.go index 4c786adf215..631d51a1509 100644 --- a/pkg/util/ioutil/pseudorand/reader_test.go +++ b/pkg/util/ioutil/pseudorand/reader_test.go @@ -6,6 +6,7 @@ package pseudorand_test import ( "bytes" + "fmt" "io" "math/rand" "testing" @@ -31,8 +32,14 @@ func TestReader(t *testing.T) { t.Fatal("content mismatch") } }) + t.Run("randomness", func(t *testing.T) { + bufSize:= 4096 + if bytes.Equal(content[:bufSize], content[bufSize:2*bufSize]) { + t.Fatal("buffers should not match") + } + }) t.Run("re-readability", func(t *testing.T) { - r.Reset() + r.Seek(0, io.SeekStart) var read []byte buf := make([]byte, 8200) total := 0 @@ -57,15 +64,33 @@ func TestReader(t *testing.T) { } }) t.Run("comparison", func(t *testing.T) { - if !r.Equal(bytes.NewBuffer(content)) { + r.Seek(0, io.SeekStart) + if eq, err := r.Equal(bytes.NewBuffer(content)); err != nil { + t.Fatal(err) + } else if !eq { t.Fatal("content mismatch") } - if r.Equal(bytes.NewBuffer(content[:len(content)-1])) { - t.Fatal("content should not match") + r.Seek(0, io.SeekStart) + if eq, err := r.Equal(bytes.NewBuffer(content[:len(content)-1])); err != nil { + t.Fatal(err) + } else if eq { + t.Fatal("content should match") } - content[0] = content[0] + 1 - if r.Equal(bytes.NewBuffer(content)) { - t.Fatal("content should not match") + }) + t.Run("seek and match", func(t *testing.T) { + for i := 0; i < 20; i++ { + off := rand.Intn(size) + n := rand.Intn(size - off) + t.Run(fmt.Sprintf("off=%d n=%d", off, n), func(t *testing.T) { + r.Seek(int64(off), io.SeekStart) + ok, err := r.Match(bytes.NewBuffer(content[off:off+n]), n) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("content mismatch") + } + }) } }) } From bd4acad72a3ba8f01d0c76b448e5d506a6ea9ccf Mon Sep 17 00:00:00 2001 From: zelig Date: Mon, 1 Jan 2024 07:24:17 +0100 Subject: [PATCH 12/52] refactor(storer/mock): redesign forgettingstore --- pkg/storer/mock/forgetting.go | 83 +++++++++++++++++++++++++++++------ 1 file changed, 70 insertions(+), 13 deletions(-) diff --git a/pkg/storer/mock/forgetting.go b/pkg/storer/mock/forgetting.go index e7794975eb2..5588b5c263e 100644 --- a/pkg/storer/mock/forgetting.go +++ b/pkg/storer/mock/forgetting.go @@ -40,7 +40,6 @@ func (d *DelayedStore) Get(ctx context.Context, addr swarm.Address) (ch swarm.Ch select { case <-time.After(delay): delete(d.cache, addr.String()) - _ = d.ChunkStore.Delete(ctx, addr) case <-ctx.Done(): return nil, ctx.Err() } @@ -49,23 +48,81 @@ func (d *DelayedStore) Get(ctx context.Context, addr swarm.Address) (ch swarm.Ch } type ForgettingStore struct { - *DelayedStore - oneIn int - delay time.Duration - n atomic.Uint32 + storage.ChunkStore + record atomic.Bool + mu sync.Mutex + n atomic.Int64 + missed map[string]struct{} +} + +func NewForgettingStore(s storage.ChunkStore) *ForgettingStore { + return &ForgettingStore{ChunkStore: s, missed: make(map[string]struct{})} +} + +func (f *ForgettingStore) Stored() int64 { + return f.n.Load() +} + +func (f *ForgettingStore) Record() { + f.record.Store(true) +} + +func (f *ForgettingStore) Unrecord() { + f.record.Store(false) +} + +func (f *ForgettingStore) Miss(addr swarm.Address) { + f.mu.Lock() + defer f.mu.Unlock() + f.miss(addr) +} + +func (f *ForgettingStore) Unmiss(addr swarm.Address) { + f.mu.Lock() + defer f.mu.Unlock() + f.unmiss(addr) +} + +func (f *ForgettingStore) miss(addr swarm.Address) { + f.missed[addr.String()] = struct{}{} +} + +func (f *ForgettingStore) unmiss(addr swarm.Address) { + delete(f.missed, addr.String()) +} + +func (f *ForgettingStore) isMiss(addr swarm.Address) bool { + _, ok := f.missed[addr.String()] + return ok +} + +func (f *ForgettingStore) Reset() { + f.missed = make(map[string]struct{}) +} + +func (f *ForgettingStore) Missed() int { + return len(f.missed) } -func NewForgettingStore(s storage.ChunkStore, oneIn int, delay time.Duration) *ForgettingStore { - return &ForgettingStore{ - DelayedStore: NewDelayedStore(s), - oneIn: oneIn, - delay: delay, +// Get implements the ChunkStore interface. +// if in recording phase, record the chunk address as miss and returns Get on the embedded store +// if in forgetting phase, returns ErrNotFound if the chunk address is recorded as miss +func (f *ForgettingStore) Get(ctx context.Context, addr swarm.Address) (ch swarm.Chunk, err error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.record.Load() { + f.miss(addr) + } else if f.isMiss(addr) { + return nil, storage.ErrNotFound } + return f.ChunkStore.Get(ctx, addr) } +// Put implements the ChunkStore interface. func (f *ForgettingStore) Put(ctx context.Context, ch swarm.Chunk) (err error) { - if f.oneIn > 0 && int(f.n.Add(1))%f.oneIn == 0 { - f.DelayedStore.Delay(ch.Address(), f.delay) + f.n.Add(1) + if !f.record.Load() { + f.Unmiss(ch.Address()) } - return f.DelayedStore.Put(ctx, ch) + return f.ChunkStore.Put(ctx, ch) } From 817816dbe83d340dfb5dc297e1fcffed3219eb23 Mon Sep 17 00:00:00 2001 From: zelig Date: Mon, 1 Jan 2024 12:26:45 +0100 Subject: [PATCH 13/52] feat(joiner): introduce fetchtimeout on decoder get calls --- pkg/file/joiner/joiner.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/file/joiner/joiner.go b/pkg/file/joiner/joiner.go index 92c3f9e8f86..14ca618718a 100644 --- a/pkg/file/joiner/joiner.go +++ b/pkg/file/joiner/joiner.go @@ -8,7 +8,6 @@ package joiner import ( "context" "errors" - "fmt" "io" "sync" "sync/atomic" @@ -53,7 +52,6 @@ type decoderCache struct { // NewDecoderCache creates a new decoder cache func NewDecoderCache(g storage.Getter, p storage.Putter, strategy getter.Strategy, strict bool, fetcherTimeout time.Duration) *decoderCache { - fmt.Println("NewDecoderCache", strategy, strict, fetcherTimeout) return &decoderCache{ fetcher: g, putter: p, @@ -122,7 +120,6 @@ func New(ctx context.Context, g storage.Getter, putter storage.Putter, address s } strategy, strict, fetcherTimeout, err := getter.GetParamsFromContext(ctx) if err != nil { - fmt.Printf("get params from context: %v\n", err) return nil, 0, err } // override stuff if root chunk has redundancy @@ -254,7 +251,9 @@ func (j *joiner) readAtOffset( func(address swarm.Address, b []byte, cur, subTrieSize, off, bufferOffset, bytesToRead, subtrieSpanLimit int64) { eg.Go(func() error { - ch, err := g.Get(j.ctx, addr) + ctx, cancel := context.WithTimeout(j.ctx, j.decoders.fetcherTimeout) + defer cancel() + ch, err := g.Get(ctx, addr) if err != nil { return err } From c01ceb0dc0413b11e7656546a9cd17bf1c6a186c Mon Sep 17 00:00:00 2001 From: zelig Date: Mon, 1 Jan 2024 12:35:49 +0100 Subject: [PATCH 14/52] fix(redundancy/getter): fill incomplete shard, recover from cancelled ctx --- pkg/file/redundancy/getter/getter.go | 42 +++++++++++++++++++++--- pkg/file/redundancy/getter/strategies.go | 18 ---------- 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/pkg/file/redundancy/getter/getter.go b/pkg/file/redundancy/getter/getter.go index e23f2a3e6af..8f08dcad633 100644 --- a/pkg/file/redundancy/getter/getter.go +++ b/pkg/file/redundancy/getter/getter.go @@ -28,6 +28,8 @@ type decoder struct { waits []chan struct{} // wait channels for each chunk rsbuf [][]byte // RS buffer of data + parity shards for erasure decoding ready chan struct{} // signal channel for successful retrieval of shardCnt chunks + lastIdx int // index of the last data chunk in the RS buffer + lastLen int // length of the last data chunk in the RS buffer shardCnt int // number of data shards parityCnt int // number of parity shards wg sync.WaitGroup // wait group to wait for all goroutines to finish @@ -42,7 +44,7 @@ type Getter interface { io.Closer } -// New returns a decoder object used tos retrieve children of an intermediate chunk +// New returns a decoder object used to retrieve children of an intermediate chunk func New(addrs []swarm.Address, shardCnt int, g storage.Getter, p storage.Putter, strategy Strategy, strict bool, fetchTimeout time.Duration, remove func()) Getter { ctx, cancel := context.WithCancel(context.Background()) size := len(addrs) @@ -97,10 +99,40 @@ func (g *decoder) Get(ctx context.Context, addr swarm.Address) (swarm.Chunk, err } select { case <-g.waits[i]: - return swarm.NewChunk(addr, g.rsbuf[i]), nil case <-ctx.Done(): - return nil, ctx.Err() + select { + case <-g.ready: + select { + case <-g.waits[i]: + case <-time.After(1 * time.Second): + return nil, ctx.Err() + } + default: + return nil, ctx.Err() + } + } + return swarm.NewChunk(addr, g.getData(i)), nil +} + +// setData sets the data shard in the RS buffer +func (g *decoder) setData(i int, chdata []byte) { + data := chdata + // pad the chunk with zeros if it is smaller than swarm.ChunkSize + if len(data) < swarm.ChunkWithSpanSize { + g.lastIdx = i + g.lastLen = len(data) + data = make([]byte, swarm.ChunkWithSpanSize) + copy(data, chdata) } + g.rsbuf[i] = data +} + +// getData returns the data shard from the RS buffer +func (g *decoder) getData(i int) []byte { + if i > 0 && i == g.lastIdx { // zero value of g.lastIdx is impossible + return g.rsbuf[i][:g.lastLen] // cut padding + } + return g.rsbuf[i] } // fly commits to retrieve the chunk (fly and land) @@ -139,8 +171,8 @@ func (g *decoder) fetch(ctx context.Context, i int) { } // write chunk to rsbuf and signal waiters - g.rsbuf[i] = ch.Data() // save the chunk in the RS buffer - if i < len(g.waits) { + g.setData(i, ch.Data()) // save the chunk in the RS buffer + if i < len(g.waits) { // if the chunk is a data shard close(g.waits[i]) // signal that the chunk is retrieved } diff --git a/pkg/file/redundancy/getter/strategies.go b/pkg/file/redundancy/getter/strategies.go index 075bd55da2d..8694449ee55 100644 --- a/pkg/file/redundancy/getter/strategies.go +++ b/pkg/file/redundancy/getter/strategies.go @@ -31,32 +31,14 @@ const ( // GetParamsFromContext extracts the strategy and strict mode from the context func GetParamsFromContext(ctx context.Context) (s Strategy, strict bool, fetcherTimeout time.Duration, err error) { - // // var ok bool - // sStr, ok := ctx.Value(strategyKey{}).(string) - // if !ok { - // return s, strict, fetcherTimeout, fmt.Errorf("error setting strategy from context") - // } - // sNum, err := strconv.ParseUint(sStr, 10, 2) - // if err != nil { - // return s, strict, fetcherTimeout, fmt.Errorf("error parsing strategy from context") - // } - // s = Strategy(sNum) s, ok := ctx.Value(strategyKey{}).(Strategy) if !ok { return s, strict, fetcherTimeout, fmt.Errorf("error setting strategy from context") } - // strictStr, ok := ctx.Value(modeKey{}).(string) - // if !ok { - // return s, strict, fetcherTimeout, fmt.Errorf("error setting fallback mode from context") - // } - // strict, err = strconv.ParseBool(strictStr) strict, ok = ctx.Value(modeKey{}).(bool) if !ok { return s, strict, fetcherTimeout, fmt.Errorf("error setting fallback mode from context") } - if err != nil { - return s, strict, fetcherTimeout, fmt.Errorf("error parsing fallback mode from context") - } fetcherTimeoutVal, ok := ctx.Value(fetcherTimeoutKey{}).(string) if !ok { return s, strict, fetcherTimeout, fmt.Errorf("error setting fetcher timeout from context") From 808ba4c51b902dc2ff563ef3e258490d6772d5bd Mon Sep 17 00:00:00 2001 From: zelig Date: Mon, 1 Jan 2024 12:37:21 +0100 Subject: [PATCH 15/52] test: add extensive multilevel joiner tests for redundancy --- pkg/file/joiner/joiner_test.go | 142 +++++++++++++++++++++++++++++++-- 1 file changed, 137 insertions(+), 5 deletions(-) diff --git a/pkg/file/joiner/joiner_test.go b/pkg/file/joiner/joiner_test.go index 8c65cab20ff..205825841e0 100644 --- a/pkg/file/joiner/joiner_test.go +++ b/pkg/file/joiner/joiner_test.go @@ -24,10 +24,13 @@ import ( "github.com/ethersphere/bee/pkg/file/redundancy/getter" "github.com/ethersphere/bee/pkg/file/splitter" filetest "github.com/ethersphere/bee/pkg/file/testing" + "github.com/ethersphere/bee/pkg/replicas" storage "github.com/ethersphere/bee/pkg/storage" "github.com/ethersphere/bee/pkg/storage/inmemchunkstore" testingc "github.com/ethersphere/bee/pkg/storage/testing" + mockstorer "github.com/ethersphere/bee/pkg/storer/mock" "github.com/ethersphere/bee/pkg/swarm" + "github.com/ethersphere/bee/pkg/util/ioutil/pseudorand" "github.com/ethersphere/bee/pkg/util/testutil" "gitlab.com/nolash/go-mockbytes" "golang.org/x/sync/errgroup" @@ -1063,10 +1066,8 @@ func TestJoinerRedundancy(t *testing.T) { } { tc := tc t.Run(fmt.Sprintf("redundancy=%d encryption=%t", tc.rLevel, tc.encryptChunk), func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) defer cancel() - shardCnt := tc.rLevel.GetMaxShards() parityCnt := tc.rLevel.GetParities(shardCnt) if tc.encryptChunk { @@ -1111,11 +1112,9 @@ func TestJoinerRedundancy(t *testing.T) { } // all data can be read back readCheck := func(t *testing.T, expErr error) { - t.Helper() - ctx, cancel := context.WithTimeout(context.Background(), 15*getter.StrategyTimeout) defer cancel() - ctx = getter.SetFetchTimeout(ctx, getter.StrategyTimeout.String()) + ctx = getter.SetParamsInContext(ctx, getter.RACE, true, (10 * getter.StrategyTimeout).String()) joinReader, rootSpan, err := joiner.New(ctx, store, store, swarmAddr) if err != nil { t.Fatal(err) @@ -1153,6 +1152,7 @@ func TestJoinerRedundancy(t *testing.T) { }) } err = eg.Wait() + if !errors.Is(err, expErr) { t.Fatalf("unexpected error reading chunkdata at chunk position %d: expected %v. got %v", i, expErr, err) } @@ -1191,3 +1191,135 @@ func TestJoinerRedundancy(t *testing.T) { }) } } + +func TestJoinerRedundancyMultilevel(t *testing.T) { + + strategyTimeout := getter.StrategyTimeout + defer func() { getter.StrategyTimeout = strategyTimeout }() + getter.StrategyTimeout = 100 * time.Millisecond + + test := func(t *testing.T, rLevel redundancy.Level, encrypt bool, size int) { + store := mockstorer.NewForgettingStore(inmemchunkstore.New()) + testutil.CleanupCloser(t, store) + seed, err := pseudorand.NewSeed() + if err != nil { + t.Fatal(err) + } + dataReader := pseudorand.NewReader(seed, size*swarm.ChunkSize) + ctx := context.Background() + ctx = replicas.SetLevel(ctx, redundancy.NONE) + pipe := builder.NewPipelineBuilder(ctx, store, encrypt, rLevel) + addr, err := builder.FeedPipeline(ctx, pipe, dataReader) + if err != nil { + t.Fatal(err) + } + expRead := swarm.ChunkSize + buf := make([]byte, expRead) + offset := 0 + // mrand.Intn(tc.size) * expRead + canReadRange := func(t *testing.T, s getter.Strategy, fallback bool, canRead bool) { + ctx := context.Background() + ctx = getter.SetParamsInContext(ctx, s, fallback, (8 * getter.StrategyTimeout).String()) + ctx, cancel := context.WithTimeout(ctx, 4*getter.StrategyTimeout) + defer cancel() + j, _, err := joiner.New(ctx, store, store, addr) + if err != nil { + t.Fatal(err) + } + n, err := j.ReadAt(buf, int64(offset)) + if !canRead { + if !errors.Is(err, storage.ErrNotFound) && !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("expected error %v or %v. got %v", storage.ErrNotFound, context.DeadlineExceeded, err) + } + return + } + if err != nil { + t.Fatal(err) + } + if n != expRead { + t.Errorf("read %d bytes out of %d", n, expRead) + } + dataReader.Seek(int64(offset), io.SeekStart) + ok, err := dataReader.Match(bytes.NewBuffer(buf), expRead) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Error("content mismatch") + } + } + + // first sanity check and and recover a range + t.Run("NONE w/o fallback CAN retrieve", func(t *testing.T) { + store.Record() + defer store.Unrecord() + canReadRange(t, getter.NONE, false, true) + }) + + // do not forget the root chunk + store.Unmiss(swarm.NewAddress(addr.Bytes()[:swarm.HashSize])) + // after we forget the chunks on the way to the range, we should not be able to retrieve + t.Run("NONE w/o fallback CANNOT retrieve", func(t *testing.T) { + canReadRange(t, getter.NONE, false, false) + }) + + // we lost a data chunk, we cannot recover using DATA only strategy with no fallback + t.Run("DATA w/o fallback CANNOT retrieve", func(t *testing.T) { + canReadRange(t, getter.DATA, false, false) + }) + + if rLevel == 0 { + // allowing fallback mode will not help if no redundancy used for upload + t.Run("DATA w fallback CANNOT retrieve", func(t *testing.T) { + canReadRange(t, getter.DATA, true, false) + }) + return + } + // allowing fallback mode will make the range retrievable using erasure decoding + t.Run("DATA w fallback CAN retrieve", func(t *testing.T) { + canReadRange(t, getter.RACE, true, true) + }) + // after the reconstructed data is stored, we can retrieve the range using DATA only mode + t.Run("after recovery, NONE w/o fallback CAN retrieve", func(t *testing.T) { + canReadRange(t, getter.NONE, false, true) + }) + } + _ = test + for _, rLevel := range []redundancy.Level{0, 1, 2, 3, 4} { + rLevel := rLevel + t.Run(fmt.Sprintf("rLevel=%v", rLevel), func(t *testing.T) { + for _, encrypt := range []bool{false, true} { + encrypt := encrypt + shardCnt := rLevel.GetMaxShards() + if encrypt { + shardCnt = rLevel.GetMaxEncShards() + } + for _, levels := range []int{1, 2, 3} { + chunkCnt := 1 + switch levels { + case 1: + chunkCnt = 2 + case 2: + chunkCnt = shardCnt + 1 + case 3: + chunkCnt = shardCnt*shardCnt + 1 + } + t.Run(fmt.Sprintf("encrypt=%v levels=%d chunks=%d incomplete", encrypt, levels, chunkCnt), func(t *testing.T) { + test(t, rLevel, encrypt, chunkCnt) + }) + switch levels { + case 1: + chunkCnt = shardCnt + case 2: + chunkCnt = shardCnt * shardCnt + case 3: + continue + } + t.Run(fmt.Sprintf("encrypt=%v levels=%d chunks=%d full", encrypt, levels, chunkCnt), func(t *testing.T) { + test(t, rLevel, encrypt, chunkCnt) + }) + } + } + }) + } +} From 2de20079eedb6f78b37318e254890f5bde6d234b Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 4 Jan 2024 09:23:08 +0100 Subject: [PATCH 16/52] fix: check seek error and add Size func --- pkg/util/ioutil/pseudorand/reader.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkg/util/ioutil/pseudorand/reader.go b/pkg/util/ioutil/pseudorand/reader.go index 95e4a69c79d..9fcc780dd3a 100644 --- a/pkg/util/ioutil/pseudorand/reader.go +++ b/pkg/util/ioutil/pseudorand/reader.go @@ -13,6 +13,7 @@ import ( "crypto/rand" "encoding/binary" "errors" + "fmt" "io" "github.com/ethersphere/bee/pkg/swarm" @@ -44,6 +45,11 @@ func NewReader(seed []byte, len int) *Reader { return r } +// Size returns the size of the reader. +func (r *Reader) Size() int { + return r.len +} + // Read reads len(buf) bytes into buf. It returns the number of bytes read (0 <= n <= len(buf)) and any error encountered. Even if Read returns n < len(buf), it may use all of buf as scratch space during the call. If some data is available but not len(buf) bytes, Read conventionally returns what is available instead of waiting for more. func (r *Reader) Read(buf []byte) (n int, err error) { cur := r.cur % bufSize @@ -138,6 +144,9 @@ func (r *Reader) Seek(offset int64, whence int) (int64, error) { case 2: r.cur = r.len - int(offset) } + if r.cur < 0 || r.cur > r.len { + return 0, fmt.Errorf("seek: invalid offset %d", r.cur) + } r.fill() return int64(r.cur), nil } From fdb02ec04de2b532aa3448b20e5aac428df60a35 Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 4 Jan 2024 09:24:51 +0100 Subject: [PATCH 17/52] fix: remove spurious selector in import --- pkg/api/debugstorage_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/api/debugstorage_test.go b/pkg/api/debugstorage_test.go index b83ddb5af99..72d8800b882 100644 --- a/pkg/api/debugstorage_test.go +++ b/pkg/api/debugstorage_test.go @@ -9,7 +9,7 @@ import ( "testing" "github.com/ethersphere/bee/pkg/jsonhttp/jsonhttptest" - storer "github.com/ethersphere/bee/pkg/storer" + "github.com/ethersphere/bee/pkg/storer" mockstorer "github.com/ethersphere/bee/pkg/storer/mock" ) From 2dcaef49ad8a36da28dd0751225aab3c673379c7 Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 4 Jan 2024 09:26:33 +0100 Subject: [PATCH 18/52] fix: improve GetParamsFromContext and remove debug lines --- pkg/file/redundancy/getter/getter.go | 3 +++ pkg/file/redundancy/getter/strategies.go | 8 +++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pkg/file/redundancy/getter/getter.go b/pkg/file/redundancy/getter/getter.go index 8f08dcad633..e283498934f 100644 --- a/pkg/file/redundancy/getter/getter.go +++ b/pkg/file/redundancy/getter/getter.go @@ -250,6 +250,9 @@ func (g *decoder) save(ctx context.Context, missing []int) error { return nil } +// Close terminates the prefetch loop, waits for all goroutines to finish and +// removes the decoder from the cache +// it implements the io.Closer interface func (g *decoder) Close() error { g.cancel() g.wg.Wait() diff --git a/pkg/file/redundancy/getter/strategies.go b/pkg/file/redundancy/getter/strategies.go index 8694449ee55..911440bf4ca 100644 --- a/pkg/file/redundancy/getter/strategies.go +++ b/pkg/file/redundancy/getter/strategies.go @@ -43,9 +43,12 @@ func GetParamsFromContext(ctx context.Context) (s Strategy, strict bool, fetcher if !ok { return s, strict, fetcherTimeout, fmt.Errorf("error setting fetcher timeout from context") } + if fetcherTimeoutVal == "" { + fetcherTimeoutVal = "30s" + } fetcherTimeout, err = time.ParseDuration(fetcherTimeoutVal) if err != nil { - return s, strict, fetcherTimeout, fmt.Errorf("error parsing fetcher timeout from context") + return s, strict, fetcherTimeout, fmt.Errorf("error parsing fetcher timeout from context: %w", err) } return s, strict, fetcherTimeout, nil } @@ -87,6 +90,7 @@ func (g *decoder) prefetch(ctx context.Context, strategy int, strict bool, strat if s == PROX { // NOT IMPLEMENTED return fmt.Errorf("strategy %d not implemented", s) } + fmt.Printf("prefetching starts with strategy %d", s) var stop <-chan time.Time if s < RACE { @@ -101,8 +105,10 @@ func (g *decoder) prefetch(ctx context.Context, strategy int, strict bool, strat select { // successfully retrieved shardCnt number of chunks case <-g.ready: + fmt.Printf("prefetching with strategy %d provided enough shards for decoding", s) cancelAll() case <-stop: + fmt.Printf("prefetching with strategy %d timed out", s) return fmt.Errorf("prefetching with strategy %d timed out", s) case <-ctx.Done(): return nil From 65e3f235be13cca141691e10082bc3e81eb44b00 Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 4 Jan 2024 09:35:45 +0100 Subject: [PATCH 19/52] fix: add comments and improve test --- pkg/file/joiner/joiner_test.go | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/pkg/file/joiner/joiner_test.go b/pkg/file/joiner/joiner_test.go index 205825841e0..e8ee6d40a5c 100644 --- a/pkg/file/joiner/joiner_test.go +++ b/pkg/file/joiner/joiner_test.go @@ -1192,6 +1192,29 @@ func TestJoinerRedundancy(t *testing.T) { } } +// TestJoinerRedundancyMultilevel tests the joiner with all combinations of +// redundancy level, encryption and size (levels, i.e., the height of the swarm hash tree). +// +// The test cases have the following structure: +// +// 1. upload a file with a given redundancy level and encryption +// +// 2. [positive test] download the file by the reference returned by the upload API response +// This uses range queries to target specific (number of) chunks of the file structure +// During path traversal in the swarm hash tree, the underlying mocksore (forgetting) +// is in 'recording' mode, flagging all the retrieved chunks as chunks to forget. +// This is to simulate the scenario where some of the chunks are not available/lost +// +// 3. [negative test] attempt at downloading the file using once again the same root hash +// and a no-redundancy strategy to find the file inaccessible after forgetting. +// 3a. [negative test] download file using NONE without fallback and fail +// 3b. [negative test] download file using DATA without fallback and fail +// +// 4. [positive test] download file using DATA with fallback to allow for +// reconstruction via erasure coding and succeed. +// +// 5. [positive test] after recovery chunks are saved, so fotgetting no longer +// repeat 3a/3b but this time succeed func TestJoinerRedundancyMultilevel(t *testing.T) { strategyTimeout := getter.StrategyTimeout @@ -1219,8 +1242,8 @@ func TestJoinerRedundancyMultilevel(t *testing.T) { // mrand.Intn(tc.size) * expRead canReadRange := func(t *testing.T, s getter.Strategy, fallback bool, canRead bool) { ctx := context.Background() - ctx = getter.SetParamsInContext(ctx, s, fallback, (8 * getter.StrategyTimeout).String()) - ctx, cancel := context.WithTimeout(ctx, 4*getter.StrategyTimeout) + ctx = getter.SetParamsInContext(ctx, s, fallback, (24 * getter.StrategyTimeout).String()) + ctx, cancel := context.WithTimeout(ctx, 32*getter.StrategyTimeout) defer cancel() j, _, err := joiner.New(ctx, store, store, addr) if err != nil { @@ -1277,7 +1300,7 @@ func TestJoinerRedundancyMultilevel(t *testing.T) { } // allowing fallback mode will make the range retrievable using erasure decoding t.Run("DATA w fallback CAN retrieve", func(t *testing.T) { - canReadRange(t, getter.RACE, true, true) + canReadRange(t, getter.DATA, true, true) }) // after the reconstructed data is stored, we can retrieve the range using DATA only mode t.Run("after recovery, NONE w/o fallback CAN retrieve", func(t *testing.T) { From 5a59b652f114117972e589ea4856c3fa7c9c45f4 Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 4 Jan 2024 09:49:48 +0100 Subject: [PATCH 20/52] fix: disable langos and lookahead buffer on top of joiner --- pkg/api/bzz.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pkg/api/bzz.go b/pkg/api/bzz.go index 9d1ce02e1dd..0d3a20de3f1 100644 --- a/pkg/api/bzz.go +++ b/pkg/api/bzz.go @@ -31,7 +31,6 @@ import ( "github.com/ethersphere/bee/pkg/swarm" "github.com/ethersphere/bee/pkg/topology" "github.com/ethersphere/bee/pkg/tracing" - "github.com/ethersphere/langos" "github.com/gorilla/mux" ) @@ -377,7 +376,6 @@ FETCH: jsonhttp.NotFound(w, "address not found or incorrect") return } - fmt.Println("bzz download: path", pathVar) me, err := m.Lookup(ctx, pathVar) if err != nil { loggerV1.Debug("bzz download: invalid path", "address", address, "path", pathVar, "error", err) @@ -474,7 +472,6 @@ func (s *Service) downloadHandler(logger log.Logger, w http.ResponseWriter, r *h }{} if response := s.mapStructure(r.Header, &headers); response != nil { - fmt.Println("downloadHandler", "invalid header params") response("invalid header params", logger, w) return } @@ -484,7 +481,6 @@ func (s *Service) downloadHandler(logger log.Logger, w http.ResponseWriter, r *h } ctx := r.Context() - fmt.Println("downloadHandler", "Strategy", headers.Strategy, "FallbackMode", headers.FallbackMode, "ChunkRetrievalTimeout", headers.ChunkRetrievalTimeout) ctx = getter.SetParamsInContext(ctx, headers.Strategy, headers.FallbackMode, headers.ChunkRetrievalTimeout) reader, l, err := joiner.New(ctx, s.storer.Download(cache), s.storer.Cache(), reference) if err != nil { @@ -509,7 +505,9 @@ func (s *Service) downloadHandler(logger log.Logger, w http.ResponseWriter, r *h } w.Header().Set(ContentLengthHeader, strconv.FormatInt(l, 10)) w.Header().Set("Access-Control-Expose-Headers", ContentDispositionHeader) - http.ServeContent(w, r, "", time.Now(), langos.NewBufferedLangos(reader, lookaheadBufferSize(l))) + http.ServeContent(w, r, "", time.Now(), reader) + // NOTE: temporary workaround for testing, watch this... + // http.ServeContent(w, r, "", time.Now(), langos.NewBufferedLangos(reader, lookaheadBufferSize(l))) } // manifestMetadataLoad returns the value for a key stored in the metadata of From 2ac6937c72487136bd3df19645c96775d1bacdce Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 4 Jan 2024 09:52:27 +0100 Subject: [PATCH 21/52] fix: bzz upload/download tests with redundancy and improve range query parsing --- pkg/api/bzz_test.go | 335 +++++++++++++++++++++++++++++--------------- 1 file changed, 224 insertions(+), 111 deletions(-) diff --git a/pkg/api/bzz_test.go b/pkg/api/bzz_test.go index 0e2d58ff76a..3efbd2c8620 100644 --- a/pkg/api/bzz_test.go +++ b/pkg/api/bzz_test.go @@ -20,6 +20,7 @@ import ( "github.com/ethersphere/bee/pkg/api" "github.com/ethersphere/bee/pkg/file/loadsave" + "github.com/ethersphere/bee/pkg/file/redundancy" "github.com/ethersphere/bee/pkg/jsonhttp" "github.com/ethersphere/bee/pkg/jsonhttp/jsonhttptest" "github.com/ethersphere/bee/pkg/log" @@ -33,109 +34,198 @@ import ( ) // nolint:paralleltest,tparallel -func TestBzzUploadWithRedundancy(t *testing.T) { + +// TestBzzUploadDownloadWithRedundancy tests the API for upload and download files +// with all combinations of redundancy level, encryption and size (levels, i.e., the +// +// height of the swarm hash tree). +// +// This is a variation on the same play as TestJoinerRedundancy +// but here the tested scenario is simplified since we are not testing the intricacies of +// download strategies, but only correct parameter passing and correct recovery functionality +// +// The test cases have the following structure: +// +// 1. upload a file with a given redundancy level and encryption +// +// 2. [positive test] download the file by the reference returned by the upload API response +// This uses range queries to target specific (number of) chunks of the file structure +// During path traversal in the swarm hash tree, the underlying mocksore (forgetting) +// is in 'recording' mode, flagging all the retrieved chunks as chunks to forget. +// This is to simulate the scenario where some of the chunks are not available/lost +// NOTE: For this to work one needs to switch off lookaheadbuffer functionality +// (see langos pkg) +// +// 3. [negative test] attempt at downloading the file using once again the same root hash +// and the same redundancy strategy to find the file inaccessible after forgetting. +// +// 4. [positive test] attempt at downloading the file using a strategy that allows for +// using redundancy to reconstruct the file and find the file recoverable. +func TestBzzUploadDownloadWithRedundancy(t *testing.T) { fileUploadResource := "/bzz" - fileDownloadResource := func(addr string) string { return "/bzz/" + addr } + fileDownloadResource := func(addr string) string { return "/bzz/" + addr + "/" } - type testCase struct { - name string - level int - size int - encrypt string - } - var tcs []testCase - for _, encrypt := range []string{"false", "true"} { - for _, level := range []int{0, 1, 2, 3, 4} { - // for _, size := range []int{4095, 4096, 42 * 420} { - for _, size := range []int{31 * 4096, 128 * 4096, 128*4096 + 4095} { - tcs = append(tcs, testCase{ - name: fmt.Sprintf("level-%d-encrypt-%s-size-%d", level, encrypt, size), - level: level, - size: size, - encrypt: encrypt, - }) - } + testRedundancy := func(t *testing.T, rLevel redundancy.Level, encrypt bool, levels int, chunkCnt int, shardCnt int, parityCnt int) { + seed, err := pseudorand.NewSeed() + if err != nil { + t.Fatal(err) } - } + fetchTimeout := 200 * time.Millisecond + store := mockstorer.NewForgettingStore(inmemchunkstore.New()) + storerMock := mockstorer.NewWithChunkStore(store) + client, _, _, _ := newTestServer(t, testServerOptions{ + Storer: storerMock, + Logger: log.Noop, + Post: mockpost.New(mockpost.WithAcceptAll()), + }) - oneIns := []int{0, 100, 20, 10, 2} - for _, tc := range tcs { - t.Run(tc.name, func(t *testing.T) { - seed, err := pseudorand.NewSeed() - if err != nil { - t.Fatal(err) - } - fetchTimeout := 200 * time.Millisecond - storerMock := mockstorer.NewWithChunkStore(mockstorer.NewForgettingStore(inmemchunkstore.New(), oneIns[tc.level], fetchTimeout)) - client, _, _, _ := newTestServer(t, testServerOptions{ - Storer: storerMock, - Logger: log.NewLogger("stdout", log.WithVerbosity(log.VerbosityDebug), log.WithCallerFunc()), - // Logger: log.Noop , - Post: mockpost.New(mockpost.WithAcceptAll()), - }) + dataReader := pseudorand.NewReader(seed, chunkCnt*swarm.ChunkSize) - reader := pseudorand.NewReader(seed, tc.size) + var refResponse api.BzzUploadResponse + jsonhttptest.Request(t, client, http.MethodPost, fileUploadResource, + http.StatusCreated, + jsonhttptest.WithRequestHeader(api.SwarmDeferredUploadHeader, "True"), + jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, batchOkStr), + jsonhttptest.WithRequestBody(dataReader), + jsonhttptest.WithRequestHeader(api.SwarmEncryptHeader, fmt.Sprintf("%t", encrypt)), + jsonhttptest.WithRequestHeader(api.SwarmRedundancyLevelHeader, fmt.Sprintf("%d", rLevel)), + jsonhttptest.WithRequestHeader(api.ContentTypeHeader, "image/jpeg; charset=utf-8"), + jsonhttptest.WithUnmarshalJSONResponse(&refResponse), + ) - var refResponse api.BzzUploadResponse - jsonhttptest.Request(t, client, http.MethodPost, fileUploadResource, - http.StatusCreated, - jsonhttptest.WithRequestHeader(api.SwarmDeferredUploadHeader, "True"), - jsonhttptest.WithRequestHeader(api.SwarmPostageBatchIdHeader, batchOkStr), - jsonhttptest.WithRequestBody(reader), - jsonhttptest.WithRequestHeader(api.SwarmEncryptHeader, tc.encrypt), - jsonhttptest.WithRequestHeader(api.SwarmRedundancyLevelHeader, fmt.Sprintf("%d", tc.level)), - jsonhttptest.WithRequestHeader(api.ContentTypeHeader, "image/jpeg; charset=utf-8"), - jsonhttptest.WithUnmarshalJSONResponse(&refResponse), + t.Run("download multiple ranges without redundancy should succeed", func(t *testing.T) { + // the underlying chunk store is in recording mode, so all chunks retrieved + // in this test will be forgotten in the subsequent ones. + store.Record() + defer store.Unrecord() + // we intend to forget as many chunks as possible for the given redundancy level + forget := parityCnt + if parityCnt > shardCnt { + forget = shardCnt + } + if levels == 1 { + forget = 2 + } + start, end := 420, 450 + gap := swarm.ChunkSize + for j := 2; j < levels; j++ { + gap *= shardCnt + } + ranges := make([][2]int, forget) + for i := 0; i < forget; i++ { + pre := i * gap + ranges[i] = [2]int{pre + start, pre + end} + } + fmt.Println("size", chunkCnt*swarm.ChunkSize, "ranges", ranges) + rangeHeader, want := createRangeHeader(dataReader, ranges) + + var body []byte + respHeaders := jsonhttptest.Request(t, client, http.MethodGet, + fileDownloadResource(refResponse.Reference.String()), + http.StatusPartialContent, + jsonhttptest.WithRequestHeader(api.RangeHeader, rangeHeader), + // set for the replicas so that no replica gets deleted + jsonhttptest.WithRequestHeader(api.SwarmRedundancyLevelHeader, "0"), + jsonhttptest.WithRequestHeader(api.SwarmRedundancyStrategyHeader, "0"), + jsonhttptest.WithRequestHeader(api.SwarmRedundancyFallbackModeHeader, "false"), + jsonhttptest.WithRequestHeader(api.SwarmChunkRetrievalTimeoutHeader, fetchTimeout.String()), + jsonhttptest.WithPutResponseBody(&body), ) - t.Run("download without redundancy should fail", func(t *testing.T) { - if tc.level == 0 { - t.Skip("NA") - } - req, err := http.NewRequest("GET", fileDownloadResource(refResponse.Reference.String()), nil) - if err != nil { - t.Fatal(err) - } - req.Header.Set(api.SwarmRedundancyStrategyHeader, "0") - req.Header.Set(api.SwarmRedundancyFallbackModeHeader, "false") - req.Header.Set(api.SwarmChunkRetrievalTimeoutHeader, fetchTimeout.String()) + got := parseRangeParts(t, respHeaders.Get(api.ContentTypeHeader), body) - resp, err := client.Do(req) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - if resp.StatusCode == http.StatusOK && !reader.Equal(resp.Body) { - t.Fatalf("content mismatch") - } - if resp.StatusCode != http.StatusNotFound { - t.Fatalf("expected status %d; got %d", http.StatusNotFound, resp.StatusCode) + if len(got) != len(want) { + t.Fatalf("got %v parts, want %v parts", len(got), len(want)) + } + for i := 0; i < len(want); i++ { + if !bytes.Equal(got[i], want[i]) { + t.Errorf("part %v: got %q, want %q", i, string(got[i]), string(want[i])) } - }) + } + }) - t.Run("download with redundancy should succeed", func(t *testing.T) { - // t.Skip("no") - req, err := http.NewRequest("GET", fileDownloadResource(refResponse.Reference.String()), nil) - if err != nil { - t.Fatal(err) - } - req.Header.Set(api.SwarmRedundancyStrategyHeader, "0") - req.Header.Set(api.SwarmRedundancyFallbackModeHeader, "true") - req.Header.Set(api.SwarmChunkRetrievalTimeoutHeader, fetchTimeout.String()) + t.Run("download without redundancy should NOT succeed", func(t *testing.T) { + if rLevel == 0 { + t.Skip("NA") + } + req, err := http.NewRequest("GET", fileDownloadResource(refResponse.Reference.String()), nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set(api.SwarmRedundancyStrategyHeader, "0") + req.Header.Set(api.SwarmRedundancyFallbackModeHeader, "false") + req.Header.Set(api.SwarmChunkRetrievalTimeoutHeader, fetchTimeout.String()) - resp, err := client.Do(req) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected status %d; got %d", http.StatusOK, resp.StatusCode) + } + _, err = io.ReadAll(resp.Body) + if err != io.ErrUnexpectedEOF { + t.Fatalf("expected error %v; got %v", io.ErrUnexpectedEOF, err) + } + }) - if resp.StatusCode != http.StatusOK { - t.Fatalf("expected status %d; got %d", http.StatusOK, resp.StatusCode) - } + t.Run("download with redundancy should succeed", func(t *testing.T) { + req, err := http.NewRequest("GET", fileDownloadResource(refResponse.Reference.String()), nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set(api.SwarmRedundancyStrategyHeader, "0") + req.Header.Set(api.SwarmRedundancyFallbackModeHeader, "true") + req.Header.Set(api.SwarmChunkRetrievalTimeoutHeader, fetchTimeout.String()) - }) + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected status %d; got %d", http.StatusOK, resp.StatusCode) + } + dataReader.Seek(0, io.SeekStart) + ok, err := dataReader.Equal(resp.Body) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatalf("content mismatch") + } + }) + } + for _, rLevel := range []redundancy.Level{1, 2, 3, 4} { + t.Run(fmt.Sprintf("level=%d", rLevel), func(t *testing.T) { + for _, encrypt := range []bool{false, true} { + encrypt := encrypt + shardCnt := rLevel.GetMaxShards() + parityCnt := rLevel.GetParities(shardCnt) + if encrypt { + shardCnt = rLevel.GetMaxEncShards() + parityCnt = rLevel.GetEncParities(shardCnt) + } + for _, levels := range []int{1, 2, 3} { + chunkCnt := 1 + switch levels { + case 1: + chunkCnt = 2 + case 2: + chunkCnt = shardCnt + 1 + case 3: + chunkCnt = shardCnt*shardCnt + 1 + } + t.Run(fmt.Sprintf("encrypt=%v levels=%d chunks=%d", encrypt, levels, chunkCnt), func(t *testing.T) { + testRedundancy(t, rLevel, encrypt, levels, chunkCnt, shardCnt, parityCnt) + }) + } + } }) } + } func TestBzzFiles(t *testing.T) { @@ -567,35 +657,58 @@ func TestBzzFilesRangeRequests(t *testing.T) { } } -func createRangeHeader(data []byte, ranges [][2]int) (header string, parts [][]byte) { - header = "bytes=" - for i, r := range ranges { - if i > 0 { - header += ", " +func createRangeHeader(data interface{}, ranges [][2]int) (header string, parts [][]byte) { + getLen := func() int { + switch data.(type) { + case []byte: + return len(data.([]byte)) + case interface{ Size() int }: + return data.(interface{ Size() int }).Size() + default: + panic("unknown data type") } - if r[0] >= 0 && r[1] >= 0 { - parts = append(parts, data[r[0]:r[1]]) - // Range: =-, end is inclusive - header += fmt.Sprintf("%v-%v", r[0], r[1]-1) - } else { - if r[0] >= 0 { - header += strconv.Itoa(r[0]) // Range: =- - parts = append(parts, data[r[0]:]) + } + getRange := func(start, end int) []byte { + switch data.(type) { + case []byte: + return data.([]byte)[start:end] + case io.ReadSeeker: + buf := make([]byte, end-start) + r := data.(io.ReadSeeker) + _, err := r.Seek(int64(start), io.SeekStart) + if err != nil { + panic(err) } - header += "-" - if r[1] >= 0 { - if r[0] >= 0 { - // Range: =-, end is inclusive - header += strconv.Itoa(r[1] - 1) - } else { - // Range: =-, the parameter is length - header += strconv.Itoa(r[1]) - } - parts = append(parts, data[:r[1]]) + _, err = io.ReadFull(r, buf) + if err != nil { + panic(err) } + return buf + default: + panic("unknown data type") + } + } + + rangeStrs := make([]string, len(ranges)) + for i, r := range ranges { + start, end := r[0], r[1] + switch { + case start < 0: + // Range: =-, the parameter is length + rangeStrs[i] = "-" + strconv.Itoa(end) + start = 0 + case r[1] < 0: + // Range: =- + rangeStrs[i] = strconv.Itoa(start) + "-" + end = getLen() + default: + // Range: =-, end is inclusive + rangeStrs[i] = fmt.Sprintf("%v-%v", start, end-1) } + parts = append(parts, getRange(start, end)) } - return + header = "bytes=" + strings.Join(rangeStrs, ", ") // nolint:staticcheck + return header, parts } func parseRangeParts(t *testing.T, contentType string, body []byte) (parts [][]byte) { From b7b1a768427483f2b66f011259d5161d248d4f03 Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 4 Jan 2024 22:11:05 +0100 Subject: [PATCH 22/52] fix(getter): fix default assignment to fetcher timeout --- pkg/file/redundancy/getter/strategies.go | 29 ++++++++++++++---------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/pkg/file/redundancy/getter/strategies.go b/pkg/file/redundancy/getter/strategies.go index 911440bf4ca..5536ae4b549 100644 --- a/pkg/file/redundancy/getter/strategies.go +++ b/pkg/file/redundancy/getter/strategies.go @@ -31,17 +31,25 @@ const ( // GetParamsFromContext extracts the strategy and strict mode from the context func GetParamsFromContext(ctx context.Context) (s Strategy, strict bool, fetcherTimeout time.Duration, err error) { - s, ok := ctx.Value(strategyKey{}).(Strategy) - if !ok { - return s, strict, fetcherTimeout, fmt.Errorf("error setting strategy from context") + var ok bool + s, strict, fetcherTimeoutVal := NONE, true, "30s" + if val := ctx.Value(strategyKey{}); val != nil { + s, ok = val.(Strategy) + if !ok { + return s, strict, fetcherTimeout, fmt.Errorf("error setting strategy from context") + } } - strict, ok = ctx.Value(modeKey{}).(bool) - if !ok { - return s, strict, fetcherTimeout, fmt.Errorf("error setting fallback mode from context") + if val := ctx.Value(modeKey{}); val != nil { + strict, ok = val.(bool) + if !ok { + return s, strict, fetcherTimeout, fmt.Errorf("error setting fallback mode from context") + } } - fetcherTimeoutVal, ok := ctx.Value(fetcherTimeoutKey{}).(string) - if !ok { - return s, strict, fetcherTimeout, fmt.Errorf("error setting fetcher timeout from context") + if val := ctx.Value(fetcherTimeoutKey{}); val != nil { + fetcherTimeoutVal, ok = val.(string) + if !ok { + return s, strict, fetcherTimeout, fmt.Errorf("error setting fetcher timeout from context") + } } if fetcherTimeoutVal == "" { fetcherTimeoutVal = "30s" @@ -90,7 +98,6 @@ func (g *decoder) prefetch(ctx context.Context, strategy int, strict bool, strat if s == PROX { // NOT IMPLEMENTED return fmt.Errorf("strategy %d not implemented", s) } - fmt.Printf("prefetching starts with strategy %d", s) var stop <-chan time.Time if s < RACE { @@ -105,10 +112,8 @@ func (g *decoder) prefetch(ctx context.Context, strategy int, strict bool, strat select { // successfully retrieved shardCnt number of chunks case <-g.ready: - fmt.Printf("prefetching with strategy %d provided enough shards for decoding", s) cancelAll() case <-stop: - fmt.Printf("prefetching with strategy %d timed out", s) return fmt.Errorf("prefetching with strategy %d timed out", s) case <-ctx.Done(): return nil From 7846bcdea733e256a9dc14840ad12ecf8c08215f Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 4 Jan 2024 22:13:43 +0100 Subject: [PATCH 23/52] fix: langos back, debug print lines removed --- pkg/api/bzz.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/pkg/api/bzz.go b/pkg/api/bzz.go index 0d3a20de3f1..c2afa43b7f0 100644 --- a/pkg/api/bzz.go +++ b/pkg/api/bzz.go @@ -31,6 +31,7 @@ import ( "github.com/ethersphere/bee/pkg/swarm" "github.com/ethersphere/bee/pkg/topology" "github.com/ethersphere/bee/pkg/tracing" + "github.com/ethersphere/langos" "github.com/gorilla/mux" ) @@ -133,7 +134,6 @@ func (s *Service) fileUploadHandler( response("invalid query params", logger, w) return } - fmt.Println("fileUploadHandler", "rlevel", rLevel) p := requestPipelineFn(putter, encrypt, rLevel) ctx := r.Context() @@ -252,7 +252,6 @@ func (s *Service) fileUploadHandler( func (s *Service) bzzDownloadHandler(w http.ResponseWriter, r *http.Request) { logger := tracing.NewLoggerWithTraceID(r.Context(), s.logger.WithName("get_bzz_by_path").Build()) - fmt.Println("bzzDownloadHandler", "r.URL.Path", r.URL.Path) paths := struct { Address swarm.Address `map:"address,resolve" validate:"required"` Path string `map:"path"` @@ -280,7 +279,6 @@ func (s *Service) serveReference(logger log.Logger, address swarm.Address, pathV }{} if response := s.mapStructure(r.Header, &headers); response != nil { - fmt.Println("downloadHandler", "invalid header params") response("invalid header params", logger, w) return } @@ -289,13 +287,11 @@ func (s *Service) serveReference(logger log.Logger, address swarm.Address, pathV cache = *headers.Cache } - fmt.Println("serveReference", "address", address, "pathVar", pathVar, "cache", cache) ls := loadsave.NewReadonly(s.storer.Download(cache)) feedDereferenced := false ctx := r.Context() - fmt.Println("serveReference", "Strategy", headers.Strategy, "FallbackMode", headers.FallbackMode, "ChunkRetrievalTimeout", headers.ChunkRetrievalTimeout) - ctx = getter.SetParamsInContext(ctx, headers.Strategy, headers.FallbackMode, headers.ChunkRetrievalTimeout) +f ctx = getter.SetParamsInContext(ctx, headers.Strategy, headers.FallbackMode, headers.ChunkRetrievalTimeout) FETCH: // read manifest entry @@ -505,9 +501,9 @@ func (s *Service) downloadHandler(logger log.Logger, w http.ResponseWriter, r *h } w.Header().Set(ContentLengthHeader, strconv.FormatInt(l, 10)) w.Header().Set("Access-Control-Expose-Headers", ContentDispositionHeader) - http.ServeContent(w, r, "", time.Now(), reader) + // http.ServeContent(w, r, "", time.Now(), reader) // NOTE: temporary workaround for testing, watch this... - // http.ServeContent(w, r, "", time.Now(), langos.NewBufferedLangos(reader, lookaheadBufferSize(l))) + http.ServeContent(w, r, "", time.Now(), langos.NewBufferedLangos(reader, lookaheadBufferSize(l))) } // manifestMetadataLoad returns the value for a key stored in the metadata of From 89a756c49bb2e9e7260631a99959d85dae9a3fca Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 4 Jan 2024 22:15:27 +0100 Subject: [PATCH 24/52] test: temporary debug lines in redundancy tests --- pkg/api/bzz_test.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/pkg/api/bzz_test.go b/pkg/api/bzz_test.go index 3efbd2c8620..33398269f12 100644 --- a/pkg/api/bzz_test.go +++ b/pkg/api/bzz_test.go @@ -62,6 +62,7 @@ import ( // 4. [positive test] attempt at downloading the file using a strategy that allows for // using redundancy to reconstruct the file and find the file recoverable. func TestBzzUploadDownloadWithRedundancy(t *testing.T) { + t.Skip("skipping until we sort out langos lookaheadbuffer functionality") fileUploadResource := "/bzz" fileDownloadResource := func(addr string) string { return "/bzz/" + addr + "/" } @@ -175,7 +176,7 @@ func TestBzzUploadDownloadWithRedundancy(t *testing.T) { if err != nil { t.Fatal(err) } - req.Header.Set(api.SwarmRedundancyStrategyHeader, "0") + req.Header.Set(api.SwarmRedundancyStrategyHeader, "3") req.Header.Set(api.SwarmRedundancyFallbackModeHeader, "true") req.Header.Set(api.SwarmChunkRetrievalTimeoutHeader, fetchTimeout.String()) @@ -189,13 +190,25 @@ func TestBzzUploadDownloadWithRedundancy(t *testing.T) { t.Fatalf("expected status %d; got %d", http.StatusOK, resp.StatusCode) } dataReader.Seek(0, io.SeekStart) - ok, err := dataReader.Equal(resp.Body) + data, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + expdata, err := io.ReadAll(dataReader) if err != nil { t.Fatal(err) } - if !ok { + if !bytes.Equal(data, expdata) { + fmt.Printf("exp %x\ngot %x\n", expdata, data) t.Fatalf("content mismatch") } + // ok, err := dataReader.Equal(resp.Body) + // if err != nil { + // t.Fatal(err) + // } + // if !ok { + // t.Fatalf("content mismatch") + // } }) } for _, rLevel := range []redundancy.Level{1, 2, 3, 4} { From 57f430e1a8e1ae54f0766de6b6ffcf916fe2f40a Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 4 Jan 2024 22:16:13 +0100 Subject: [PATCH 25/52] refactor(replicas,redundancy): move level in context accessors --- pkg/file/joiner/joiner.go | 4 +++- pkg/file/joiner/joiner_test.go | 3 +-- pkg/file/pipeline/hashtrie/hashtrie_test.go | 3 +-- pkg/file/redundancy/level.go | 17 +++++++++++++++++ pkg/replicas/putter.go | 3 ++- pkg/replicas/putter_test.go | 4 ++-- pkg/replicas/replicas.go | 19 ------------------- pkg/steward/steward_test.go | 3 +-- 8 files changed, 27 insertions(+), 29 deletions(-) diff --git a/pkg/file/joiner/joiner.go b/pkg/file/joiner/joiner.go index 14ca618718a..d5a9b6bcaba 100644 --- a/pkg/file/joiner/joiner.go +++ b/pkg/file/joiner/joiner.go @@ -8,6 +8,7 @@ package joiner import ( "context" "errors" + "fmt" "io" "sync" "sync/atomic" @@ -98,7 +99,7 @@ func (g *decoderCache) GetOrCreate(addrs []swarm.Address, shardCnt int) storage. // New creates a new Joiner. A Joiner provides Read, Seek and Size functionalities. func New(ctx context.Context, g storage.Getter, putter storage.Putter, address swarm.Address) (file.Joiner, int64, error) { // retrieve the root chunk to read the total data length the be retrieved - rLevel := replicas.GetLevelFromContext(ctx) + rLevel := redundancy.GetLevelFromContext(ctx) rootChunkGetter := store.New(g) if rLevel != redundancy.NONE { rootChunkGetter = store.New(replicas.NewGetter(g, rLevel)) @@ -120,6 +121,7 @@ func New(ctx context.Context, g storage.Getter, putter storage.Putter, address s } strategy, strict, fetcherTimeout, err := getter.GetParamsFromContext(ctx) if err != nil { + fmt.Println("error getting params from context", err) return nil, 0, err } // override stuff if root chunk has redundancy diff --git a/pkg/file/joiner/joiner_test.go b/pkg/file/joiner/joiner_test.go index e8ee6d40a5c..ee9a7d0a30b 100644 --- a/pkg/file/joiner/joiner_test.go +++ b/pkg/file/joiner/joiner_test.go @@ -24,7 +24,6 @@ import ( "github.com/ethersphere/bee/pkg/file/redundancy/getter" "github.com/ethersphere/bee/pkg/file/splitter" filetest "github.com/ethersphere/bee/pkg/file/testing" - "github.com/ethersphere/bee/pkg/replicas" storage "github.com/ethersphere/bee/pkg/storage" "github.com/ethersphere/bee/pkg/storage/inmemchunkstore" testingc "github.com/ethersphere/bee/pkg/storage/testing" @@ -1230,7 +1229,7 @@ func TestJoinerRedundancyMultilevel(t *testing.T) { } dataReader := pseudorand.NewReader(seed, size*swarm.ChunkSize) ctx := context.Background() - ctx = replicas.SetLevel(ctx, redundancy.NONE) + ctx = redundancy.SetLevelInContext(ctx, redundancy.NONE) pipe := builder.NewPipelineBuilder(ctx, store, encrypt, rLevel) addr, err := builder.FeedPipeline(ctx, pipe, dataReader) if err != nil { diff --git a/pkg/file/pipeline/hashtrie/hashtrie_test.go b/pkg/file/pipeline/hashtrie/hashtrie_test.go index 35733665c59..a6c866a8900 100644 --- a/pkg/file/pipeline/hashtrie/hashtrie_test.go +++ b/pkg/file/pipeline/hashtrie/hashtrie_test.go @@ -24,7 +24,6 @@ import ( "github.com/ethersphere/bee/pkg/file/pipeline/mock" "github.com/ethersphere/bee/pkg/file/pipeline/store" "github.com/ethersphere/bee/pkg/file/redundancy" - "github.com/ethersphere/bee/pkg/replicas" "github.com/ethersphere/bee/pkg/storage" "github.com/ethersphere/bee/pkg/storage/inmemchunkstore" "github.com/ethersphere/bee/pkg/swarm" @@ -316,7 +315,7 @@ func TestRedundancy(t *testing.T) { tc := tc t.Run(tc.desc, func(t *testing.T) { t.Parallel() - subCtx := replicas.SetLevel(ctx, tc.level) + subCtx := redundancy.SetLevelInContext(ctx, tc.level) s := inmemchunkstore.New() intermediateChunkCounter := mock.NewChainWriter() diff --git a/pkg/file/redundancy/level.go b/pkg/file/redundancy/level.go index 3dfba1cd084..caaca3742c0 100644 --- a/pkg/file/redundancy/level.go +++ b/pkg/file/redundancy/level.go @@ -5,6 +5,7 @@ package redundancy import ( + "context" "errors" "fmt" @@ -167,3 +168,19 @@ func GetReplicaCounts() [5]int { // for the five levels of redundancy are 0, 2, 4, 5, 19 // we use an approximation as the successive powers of 2 var replicaCounts = [5]int{0, 2, 4, 8, 16} + +type levelKey struct{} + +// SetLevelInContext sets the redundancy level in the context +func SetLevelInContext(ctx context.Context, level Level) context.Context { + return context.WithValue(ctx, levelKey{}, level) +} + +// GetLevelFromContext is a helper function to extract the redundancy level from the context +func GetLevelFromContext(ctx context.Context) Level { + rlevel := NONE + if val := ctx.Value(levelKey{}); val != nil { + rlevel = val.(Level) + } + return rlevel +} diff --git a/pkg/replicas/putter.go b/pkg/replicas/putter.go index f2334a994b8..4aa55b638f0 100644 --- a/pkg/replicas/putter.go +++ b/pkg/replicas/putter.go @@ -11,6 +11,7 @@ import ( "errors" "sync" + "github.com/ethersphere/bee/pkg/file/redundancy" "github.com/ethersphere/bee/pkg/soc" "github.com/ethersphere/bee/pkg/storage" "github.com/ethersphere/bee/pkg/swarm" @@ -29,7 +30,7 @@ func NewPutter(p storage.Putter) storage.Putter { // Put makes the getter satisfy the storage.Getter interface func (p *putter) Put(ctx context.Context, ch swarm.Chunk) (err error) { - rlevel := GetLevelFromContext(ctx) + rlevel := redundancy.GetLevelFromContext(ctx) errs := []error{p.putter.Put(ctx, ch)} if rlevel == 0 { return errs[0] diff --git a/pkg/replicas/putter_test.go b/pkg/replicas/putter_test.go index 7a90b5ec308..7d05624ebca 100644 --- a/pkg/replicas/putter_test.go +++ b/pkg/replicas/putter_test.go @@ -75,7 +75,7 @@ func TestPutter(t *testing.T) { t.Fatal(err) } ctx := context.Background() - ctx = replicas.SetLevel(ctx, tc.level) + ctx = redundancy.SetLevelInContext(ctx, tc.level) ch, err := cac.New(buf) if err != nil { @@ -174,7 +174,7 @@ func TestPutter(t *testing.T) { ctx := context.Background() ctx, cancel := context.WithTimeout(ctx, 50*time.Millisecond) defer cancel() - ctx = replicas.SetLevel(ctx, tc.level) + ctx = redundancy.SetLevelInContext(ctx, tc.level) ch, err := cac.New(buf) if err != nil { t.Fatal(err) diff --git a/pkg/replicas/replicas.go b/pkg/replicas/replicas.go index 43b1d8d9b18..fd45b28a3b1 100644 --- a/pkg/replicas/replicas.go +++ b/pkg/replicas/replicas.go @@ -11,7 +11,6 @@ package replicas import ( - "context" "time" "github.com/ethersphere/bee/pkg/crypto" @@ -19,31 +18,13 @@ import ( "github.com/ethersphere/bee/pkg/swarm" ) -type redundancyLevelType struct{} - var ( - // redundancyLevel is the context key for the redundancy level - redundancyLevel redundancyLevelType // RetryInterval is the duration between successive additional requests RetryInterval = 300 * time.Millisecond privKey, _ = crypto.DecodeSecp256k1PrivateKey(append([]byte{1}, make([]byte, 31)...)) signer = crypto.NewDefaultSigner(privKey) ) -// SetLevel sets the redundancy level in the context -func SetLevel(ctx context.Context, level redundancy.Level) context.Context { - return context.WithValue(ctx, redundancyLevel, level) -} - -// GetLevelFromContext is a helper function to extract the redundancy level from the context -func GetLevelFromContext(ctx context.Context) redundancy.Level { - rlevel := redundancy.PARANOID - if val := ctx.Value(redundancyLevel); val != nil { - rlevel = val.(redundancy.Level) - } - return rlevel -} - // replicator running the find for replicas type replicator struct { addr []byte // chunk address diff --git a/pkg/steward/steward_test.go b/pkg/steward/steward_test.go index 8e2083abc80..fe3ccd05ba9 100644 --- a/pkg/steward/steward_test.go +++ b/pkg/steward/steward_test.go @@ -17,7 +17,6 @@ import ( "github.com/ethersphere/bee/pkg/file/pipeline/builder" "github.com/ethersphere/bee/pkg/file/redundancy" postagetesting "github.com/ethersphere/bee/pkg/postage/mock" - "github.com/ethersphere/bee/pkg/replicas" "github.com/ethersphere/bee/pkg/steward" storage "github.com/ethersphere/bee/pkg/storage" "github.com/ethersphere/bee/pkg/storage/inmemchunkstore" @@ -49,7 +48,7 @@ func TestSteward(t *testing.T) { s = steward.New(store, localRetrieval, inmem) stamper = postagetesting.NewStamper() ) - ctx = replicas.SetLevel(ctx, redundancy.NONE) + ctx = redundancy.SetLevelInContext(ctx, redundancy.NONE) n, err := rand.Read(data) if n != cap(data) { From 867e1cd86e7b5f084ba95b6de2de7585aae5dfd8 Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 4 Jan 2024 22:18:18 +0100 Subject: [PATCH 26/52] fix(api): typo --- pkg/api/bzz.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/api/bzz.go b/pkg/api/bzz.go index c2afa43b7f0..b1888073b20 100644 --- a/pkg/api/bzz.go +++ b/pkg/api/bzz.go @@ -291,7 +291,7 @@ func (s *Service) serveReference(logger log.Logger, address swarm.Address, pathV feedDereferenced := false ctx := r.Context() -f ctx = getter.SetParamsInContext(ctx, headers.Strategy, headers.FallbackMode, headers.ChunkRetrievalTimeout) + ctx = getter.SetParamsInContext(ctx, headers.Strategy, headers.FallbackMode, headers.ChunkRetrievalTimeout) FETCH: // read manifest entry From 484310a60bf63f6720dc69ccb831bf059001675f Mon Sep 17 00:00:00 2001 From: zelig Date: Fri, 5 Jan 2024 07:06:14 +0100 Subject: [PATCH 27/52] fix: set NONE as default replicas getter --- pkg/file/redundancy/level.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/file/redundancy/level.go b/pkg/file/redundancy/level.go index caaca3742c0..4aa8c75bfe8 100644 --- a/pkg/file/redundancy/level.go +++ b/pkg/file/redundancy/level.go @@ -178,7 +178,7 @@ func SetLevelInContext(ctx context.Context, level Level) context.Context { // GetLevelFromContext is a helper function to extract the redundancy level from the context func GetLevelFromContext(ctx context.Context) Level { - rlevel := NONE + rlevel := PARANOID if val := ctx.Value(levelKey{}); val != nil { rlevel = val.(Level) } From f208167e9d1a8d1e2a1829c72d11df04b3ab78bd Mon Sep 17 00:00:00 2001 From: zelig Date: Fri, 5 Jan 2024 07:07:57 +0100 Subject: [PATCH 28/52] test: remove t.Parallel from inclusion proof test --- pkg/storageincentives/proof_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/storageincentives/proof_test.go b/pkg/storageincentives/proof_test.go index 9ee3745e1af..dcd7002f913 100644 --- a/pkg/storageincentives/proof_test.go +++ b/pkg/storageincentives/proof_test.go @@ -44,8 +44,6 @@ var testData []byte // Test asserts that MakeInclusionProofs will generate the same // output for given sample. func TestMakeInclusionProofsRegression(t *testing.T) { - t.Parallel() - const sampleSize = 16 keyRaw := `00000000000000000000000000000000` From eb8fb7168d3e1839a9ad3a8dba910c793e848275 Mon Sep 17 00:00:00 2001 From: zelig Date: Fri, 5 Jan 2024 18:53:40 +0100 Subject: [PATCH 29/52] fix: heed linter complaints --- pkg/api/bzz.go | 5 ++- pkg/api/bzz_test.go | 42 +++++++++-------------- pkg/file/joiner/joiner.go | 2 -- pkg/file/joiner/joiner_test.go | 13 +++++-- pkg/file/redundancy/getter/getter_test.go | 2 +- pkg/file/redundancy/getter/strategies.go | 2 +- pkg/util/ioutil/pseudorand/reader.go | 6 ++-- pkg/util/ioutil/pseudorand/reader_test.go | 37 ++++++++++++++++---- 8 files changed, 64 insertions(+), 45 deletions(-) diff --git a/pkg/api/bzz.go b/pkg/api/bzz.go index b1888073b20..b787aa5b132 100644 --- a/pkg/api/bzz.go +++ b/pkg/api/bzz.go @@ -31,7 +31,6 @@ import ( "github.com/ethersphere/bee/pkg/swarm" "github.com/ethersphere/bee/pkg/topology" "github.com/ethersphere/bee/pkg/tracing" - "github.com/ethersphere/langos" "github.com/gorilla/mux" ) @@ -501,9 +500,9 @@ func (s *Service) downloadHandler(logger log.Logger, w http.ResponseWriter, r *h } w.Header().Set(ContentLengthHeader, strconv.FormatInt(l, 10)) w.Header().Set("Access-Control-Expose-Headers", ContentDispositionHeader) - // http.ServeContent(w, r, "", time.Now(), reader) + http.ServeContent(w, r, "", time.Now(), reader) // NOTE: temporary workaround for testing, watch this... - http.ServeContent(w, r, "", time.Now(), langos.NewBufferedLangos(reader, lookaheadBufferSize(l))) + // http.ServeContent(w, r, "", time.Now(), langos.NewBufferedLangos(reader, lookaheadBufferSize(l))) } // manifestMetadataLoad returns the value for a key stored in the metadata of diff --git a/pkg/api/bzz_test.go b/pkg/api/bzz_test.go index 33398269f12..999a9ee3400 100644 --- a/pkg/api/bzz_test.go +++ b/pkg/api/bzz_test.go @@ -33,7 +33,7 @@ import ( "github.com/ethersphere/bee/pkg/util/ioutil/pseudorand" ) -// nolint:paralleltest,tparallel +// nolint:paralleltest,tparallel,thelper // TestBzzUploadDownloadWithRedundancy tests the API for upload and download files // with all combinations of redundancy level, encryption and size (levels, i.e., the @@ -61,8 +61,9 @@ import ( // // 4. [positive test] attempt at downloading the file using a strategy that allows for // using redundancy to reconstruct the file and find the file recoverable. +// +// nolint:thelper func TestBzzUploadDownloadWithRedundancy(t *testing.T) { - t.Skip("skipping until we sort out langos lookaheadbuffer functionality") fileUploadResource := "/bzz" fileDownloadResource := func(addr string) string { return "/bzz/" + addr + "/" } @@ -117,7 +118,6 @@ func TestBzzUploadDownloadWithRedundancy(t *testing.T) { pre := i * gap ranges[i] = [2]int{pre + start, pre + end} } - fmt.Println("size", chunkCnt*swarm.ChunkSize, "ranges", ranges) rangeHeader, want := createRangeHeader(dataReader, ranges) var body []byte @@ -149,7 +149,7 @@ func TestBzzUploadDownloadWithRedundancy(t *testing.T) { if rLevel == 0 { t.Skip("NA") } - req, err := http.NewRequest("GET", fileDownloadResource(refResponse.Reference.String()), nil) + req, err := http.NewRequestWithContext(context.Background(), "GET", fileDownloadResource(refResponse.Reference.String()), nil) if err != nil { t.Fatal(err) } @@ -166,13 +166,13 @@ func TestBzzUploadDownloadWithRedundancy(t *testing.T) { t.Fatalf("expected status %d; got %d", http.StatusOK, resp.StatusCode) } _, err = io.ReadAll(resp.Body) - if err != io.ErrUnexpectedEOF { + if !errors.Is(err, io.ErrUnexpectedEOF) { t.Fatalf("expected error %v; got %v", io.ErrUnexpectedEOF, err) } }) t.Run("download with redundancy should succeed", func(t *testing.T) { - req, err := http.NewRequest("GET", fileDownloadResource(refResponse.Reference.String()), nil) + req, err := http.NewRequestWithContext(context.TODO(), "GET", fileDownloadResource(refResponse.Reference.String()), nil) if err != nil { t.Fatal(err) } @@ -189,26 +189,17 @@ func TestBzzUploadDownloadWithRedundancy(t *testing.T) { if resp.StatusCode != http.StatusOK { t.Fatalf("expected status %d; got %d", http.StatusOK, resp.StatusCode) } - dataReader.Seek(0, io.SeekStart) - data, err := io.ReadAll(resp.Body) + _, err = dataReader.Seek(0, io.SeekStart) if err != nil { t.Fatal(err) } - expdata, err := io.ReadAll(dataReader) + ok, err := dataReader.Equal(resp.Body) if err != nil { t.Fatal(err) } - if !bytes.Equal(data, expdata) { - fmt.Printf("exp %x\ngot %x\n", expdata, data) + if !ok { t.Fatalf("content mismatch") } - // ok, err := dataReader.Equal(resp.Body) - // if err != nil { - // t.Fatal(err) - // } - // if !ok { - // t.Fatalf("content mismatch") - // } }) } for _, rLevel := range []redundancy.Level{1, 2, 3, 4} { @@ -672,27 +663,26 @@ func TestBzzFilesRangeRequests(t *testing.T) { func createRangeHeader(data interface{}, ranges [][2]int) (header string, parts [][]byte) { getLen := func() int { - switch data.(type) { + switch data := data.(type) { case []byte: - return len(data.([]byte)) + return len(data) case interface{ Size() int }: - return data.(interface{ Size() int }).Size() + return data.Size() default: panic("unknown data type") } } getRange := func(start, end int) []byte { - switch data.(type) { + switch data := data.(type) { case []byte: - return data.([]byte)[start:end] + return data[start:end] case io.ReadSeeker: buf := make([]byte, end-start) - r := data.(io.ReadSeeker) - _, err := r.Seek(int64(start), io.SeekStart) + _, err := data.Seek(int64(start), io.SeekStart) if err != nil { panic(err) } - _, err = io.ReadFull(r, buf) + _, err = io.ReadFull(data, buf) if err != nil { panic(err) } diff --git a/pkg/file/joiner/joiner.go b/pkg/file/joiner/joiner.go index d5a9b6bcaba..c59c2146669 100644 --- a/pkg/file/joiner/joiner.go +++ b/pkg/file/joiner/joiner.go @@ -8,7 +8,6 @@ package joiner import ( "context" "errors" - "fmt" "io" "sync" "sync/atomic" @@ -121,7 +120,6 @@ func New(ctx context.Context, g storage.Getter, putter storage.Putter, address s } strategy, strict, fetcherTimeout, err := getter.GetParamsFromContext(ctx) if err != nil { - fmt.Println("error getting params from context", err) return nil, 0, err } // override stuff if root chunk has redundancy diff --git a/pkg/file/joiner/joiner_test.go b/pkg/file/joiner/joiner_test.go index ee9a7d0a30b..3b5ecc93a30 100644 --- a/pkg/file/joiner/joiner_test.go +++ b/pkg/file/joiner/joiner_test.go @@ -35,6 +35,8 @@ import ( "golang.org/x/sync/errgroup" ) +// nolint:paralleltest,tparallel,thelper + func TestJoiner_ErrReferenceLength(t *testing.T) { t.Parallel() @@ -1020,6 +1022,7 @@ func (m *mockPutter) store(cnt int) error { return nil } +// nolint:thelper func TestJoinerRedundancy(t *testing.T) { strategyTimeout := getter.StrategyTimeout @@ -1214,6 +1217,8 @@ func TestJoinerRedundancy(t *testing.T) { // // 5. [positive test] after recovery chunks are saved, so fotgetting no longer // repeat 3a/3b but this time succeed +// +// nolint:thelper func TestJoinerRedundancyMultilevel(t *testing.T) { strategyTimeout := getter.StrategyTimeout @@ -1237,8 +1242,7 @@ func TestJoinerRedundancyMultilevel(t *testing.T) { } expRead := swarm.ChunkSize buf := make([]byte, expRead) - offset := 0 - // mrand.Intn(tc.size) * expRead + offset := mrand.Intn(size) * expRead canReadRange := func(t *testing.T, s getter.Strategy, fallback bool, canRead bool) { ctx := context.Background() ctx = getter.SetParamsInContext(ctx, s, fallback, (24 * getter.StrategyTimeout).String()) @@ -1261,7 +1265,10 @@ func TestJoinerRedundancyMultilevel(t *testing.T) { if n != expRead { t.Errorf("read %d bytes out of %d", n, expRead) } - dataReader.Seek(int64(offset), io.SeekStart) + _, err = dataReader.Seek(int64(offset), io.SeekStart) + if err != nil { + t.Fatal(err) + } ok, err := dataReader.Match(bytes.NewBuffer(buf), expRead) if err != nil { t.Fatal(err) diff --git a/pkg/file/redundancy/getter/getter_test.go b/pkg/file/redundancy/getter/getter_test.go index 2b85d11c589..03fda3b8425 100644 --- a/pkg/file/redundancy/getter/getter_test.go +++ b/pkg/file/redundancy/getter/getter_test.go @@ -156,7 +156,7 @@ func testDecodingFallback(t *testing.T, s getter.Strategy, strict bool) { bufSize := 12 shardCnt := 6 - store := mockstorer.NewDelayedStore(inmem.New()) + store := mockstorer.NewDelayedStore(inmem.New()) buf := make([][]byte, bufSize) addrs := initData(t, buf, shardCnt, store) diff --git a/pkg/file/redundancy/getter/strategies.go b/pkg/file/redundancy/getter/strategies.go index 5536ae4b549..9b0dcfb7935 100644 --- a/pkg/file/redundancy/getter/strategies.go +++ b/pkg/file/redundancy/getter/strategies.go @@ -68,7 +68,7 @@ func SetFetchTimeout(ctx context.Context, timeout string) context.Context { // SetStrategy sets the strategy for the retrieval func SetStrategy(ctx context.Context, s Strategy) context.Context { - return context.WithValue(ctx, strategyKey{}, int(s)) + return context.WithValue(ctx, strategyKey{}, s) } // SetStrict sets the strict mode for the retrieval diff --git a/pkg/util/ioutil/pseudorand/reader.go b/pkg/util/ioutil/pseudorand/reader.go index 9fcc780dd3a..d7fcf0c3646 100644 --- a/pkg/util/ioutil/pseudorand/reader.go +++ b/pkg/util/ioutil/pseudorand/reader.go @@ -38,8 +38,8 @@ func NewSeed() ([]byte, error) { } // New creates a new pseudorandom reader seeded with the given seed. -func NewReader(seed []byte, len int) *Reader { - r := &Reader{len: len} +func NewReader(seed []byte, l int) *Reader { + r := &Reader{len: l} _ = copy(r.buf[8:], seed) r.fill() return r @@ -79,7 +79,7 @@ func (r1 *Reader) Equal(r2 io.Reader) (bool, error) { return false, nil } n, err := io.ReadFull(r2, make([]byte, 1)) - if err == io.EOF || err == io.ErrUnexpectedEOF { + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { return n == 0, nil } return false, err diff --git a/pkg/util/ioutil/pseudorand/reader_test.go b/pkg/util/ioutil/pseudorand/reader_test.go index 631d51a1509..ef6d492553b 100644 --- a/pkg/util/ioutil/pseudorand/reader_test.go +++ b/pkg/util/ioutil/pseudorand/reader_test.go @@ -6,6 +6,7 @@ package pseudorand_test import ( "bytes" + "errors" "fmt" "io" "math/rand" @@ -33,13 +34,19 @@ func TestReader(t *testing.T) { } }) t.Run("randomness", func(t *testing.T) { - bufSize:= 4096 + bufSize := 4096 if bytes.Equal(content[:bufSize], content[bufSize:2*bufSize]) { t.Fatal("buffers should not match") } }) t.Run("re-readability", func(t *testing.T) { - r.Seek(0, io.SeekStart) + ns, err := r.Seek(0, io.SeekStart) + if err != nil { + t.Fatal(err) + } + if ns != 0 { + t.Fatal("seek mismatch") + } var read []byte buf := make([]byte, 8200) total := 0 @@ -48,7 +55,7 @@ func TestReader(t *testing.T) { n, err := r.Read(buf[:s]) total += n read = append(read, buf[:n]...) - if err == io.EOF { + if errors.Is(err, io.EOF) { break } if err != nil { @@ -64,13 +71,25 @@ func TestReader(t *testing.T) { } }) t.Run("comparison", func(t *testing.T) { - r.Seek(0, io.SeekStart) + ns, err := r.Seek(0, io.SeekStart) + if err != nil { + t.Fatal(err) + } + if ns != 0 { + t.Fatal("seek mismatch") + } if eq, err := r.Equal(bytes.NewBuffer(content)); err != nil { t.Fatal(err) } else if !eq { t.Fatal("content mismatch") } - r.Seek(0, io.SeekStart) + ns, err = r.Seek(0, io.SeekStart) + if err != nil { + t.Fatal(err) + } + if ns != 0 { + t.Fatal("seek mismatch") + } if eq, err := r.Equal(bytes.NewBuffer(content[:len(content)-1])); err != nil { t.Fatal(err) } else if eq { @@ -82,7 +101,13 @@ func TestReader(t *testing.T) { off := rand.Intn(size) n := rand.Intn(size - off) t.Run(fmt.Sprintf("off=%d n=%d", off, n), func(t *testing.T) { - r.Seek(int64(off), io.SeekStart) + ns, err := r.Seek(int64(off), io.SeekStart) + if err != nil { + t.Fatal(err) + } + if ns != int64(off) { + t.Fatal("seek mismatch") + } ok, err := r.Match(bytes.NewBuffer(content[off:off+n]), n) if err != nil { t.Fatal(err) From 56ba9727f526e21a0e4a27ff4fae5f9593d78d8f Mon Sep 17 00:00:00 2001 From: zelig Date: Sat, 6 Jan 2024 04:39:53 +0100 Subject: [PATCH 30/52] chore: remove langos from deps --- go.mod | 1 - go.sum | 2 -- 2 files changed, 3 deletions(-) diff --git a/go.mod b/go.mod index 5f38b82e566..3e7bf2f4e3d 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,6 @@ require ( github.com/ethersphere/go-price-oracle-abi v0.1.0 github.com/ethersphere/go-storage-incentives-abi v0.6.0 github.com/ethersphere/go-sw3-abi v0.4.0 - github.com/ethersphere/langos v1.0.0 github.com/go-playground/validator/v10 v10.11.1 github.com/gogo/protobuf v1.3.2 github.com/google/go-cmp v0.5.9 diff --git a/go.sum b/go.sum index 0afd81c3918..c08d1ef71bb 100644 --- a/go.sum +++ b/go.sum @@ -244,8 +244,6 @@ github.com/ethersphere/go-storage-incentives-abi v0.6.0 h1:lfGViU/wJg/CyXlntNvTQ github.com/ethersphere/go-storage-incentives-abi v0.6.0/go.mod h1:SXvJVtM4sEsaSKD0jc1ClpDLw8ErPoROZDme4Wrc/Nc= github.com/ethersphere/go-sw3-abi v0.4.0 h1:T3ANY+ktWrPAwe2U0tZi+DILpkHzto5ym/XwV/Bbz8g= github.com/ethersphere/go-sw3-abi v0.4.0/go.mod h1:BmpsvJ8idQZdYEtWnvxA8POYQ8Rl/NhyCdF0zLMOOJU= -github.com/ethersphere/langos v1.0.0 h1:NBtNKzXTTRSue95uOlzPN4py7Aofs0xWPzyj4AI1Vcc= -github.com/ethersphere/langos v1.0.0/go.mod h1:dlcN2j4O8sQ+BlCaxeBu43bgr4RQ+inJ+pHwLeZg5Tw= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= From a2194d3dd4680f1db247f4de46e6e6337ee0f477 Mon Sep 17 00:00:00 2001 From: zelig Date: Sat, 6 Jan 2024 04:40:44 +0100 Subject: [PATCH 31/52] test: speed up --- pkg/file/joiner/joiner_test.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/file/joiner/joiner_test.go b/pkg/file/joiner/joiner_test.go index 3b5ecc93a30..69ce791612a 100644 --- a/pkg/file/joiner/joiner_test.go +++ b/pkg/file/joiner/joiner_test.go @@ -1245,8 +1245,8 @@ func TestJoinerRedundancyMultilevel(t *testing.T) { offset := mrand.Intn(size) * expRead canReadRange := func(t *testing.T, s getter.Strategy, fallback bool, canRead bool) { ctx := context.Background() - ctx = getter.SetParamsInContext(ctx, s, fallback, (24 * getter.StrategyTimeout).String()) - ctx, cancel := context.WithTimeout(ctx, 32*getter.StrategyTimeout) + ctx = getter.SetParamsInContext(ctx, s, fallback, (6 * getter.StrategyTimeout).String()) + ctx, cancel := context.WithTimeout(ctx, 10*getter.StrategyTimeout) defer cancel() j, _, err := joiner.New(ctx, store, store, addr) if err != nil { @@ -1313,9 +1313,11 @@ func TestJoinerRedundancyMultilevel(t *testing.T) { canReadRange(t, getter.NONE, false, true) }) } - _ = test + r2level := []int{2, 1, 2, 3, 2} + encryptChunk := []bool{false, false, true, true, true} for _, rLevel := range []redundancy.Level{0, 1, 2, 3, 4} { rLevel := rLevel + // speeding up tests by skipping some of them t.Run(fmt.Sprintf("rLevel=%v", rLevel), func(t *testing.T) { for _, encrypt := range []bool{false, true} { encrypt := encrypt @@ -1334,6 +1336,9 @@ func TestJoinerRedundancyMultilevel(t *testing.T) { chunkCnt = shardCnt*shardCnt + 1 } t.Run(fmt.Sprintf("encrypt=%v levels=%d chunks=%d incomplete", encrypt, levels, chunkCnt), func(t *testing.T) { + if r2level[rLevel] != levels || encrypt != encryptChunk[rLevel] { + t.Skip("skipping to save time") + } test(t, rLevel, encrypt, chunkCnt) }) switch levels { From 050a3e8b8cf1053c37b33bac121b9e7d030ad443 Mon Sep 17 00:00:00 2001 From: zelig Date: Sat, 6 Jan 2024 04:56:48 +0100 Subject: [PATCH 32/52] fix: remove unused consts --- pkg/api/api.go | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/pkg/api/api.go b/pkg/api/api.go index 6d55aae9454..19b0e718e70 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -99,18 +99,6 @@ const ( OriginHeader = "Origin" ) -// The size of buffer used for prefetching content with Langos. -// Warning: This value influences the number of chunk requests and chunker join goroutines -// per file request. -// Recommended value is 8 or 16 times the io.Copy default buffer value which is 32kB, depending -// on the file size. Use lookaheadBufferSize() to get the correct buffer size for the request. -const ( - smallFileBufferSize = 8 * 32 * 1024 - largeFileBufferSize = 16 * 32 * 1024 - - largeBufferFilesizeThreshold = 10 * 1000000 // ten megs -) - const ( multiPartFormData = "multipart/form-data" contentTypeTar = "application/x-tar" @@ -615,13 +603,6 @@ func (s *Service) gasConfigMiddleware(handlerName string) func(h http.Handler) h } } -func lookaheadBufferSize(size int64) int { - if size <= largeBufferFilesizeThreshold { - return smallFileBufferSize - } - return largeFileBufferSize -} - // corsHandler sets CORS headers to HTTP response if allowed origins are configured. func (s *Service) corsHandler(h http.Handler) http.Handler { allowedHeaders := []string{ From 2f52ef35fa4ffac8f2bbcab17b84678053807ee5 Mon Sep 17 00:00:00 2001 From: zelig Date: Sat, 6 Jan 2024 06:08:57 +0100 Subject: [PATCH 33/52] test: speed up --- pkg/api/bzz_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/api/bzz_test.go b/pkg/api/bzz_test.go index 999a9ee3400..6544f02174b 100644 --- a/pkg/api/bzz_test.go +++ b/pkg/api/bzz_test.go @@ -223,6 +223,9 @@ func TestBzzUploadDownloadWithRedundancy(t *testing.T) { chunkCnt = shardCnt*shardCnt + 1 } t.Run(fmt.Sprintf("encrypt=%v levels=%d chunks=%d", encrypt, levels, chunkCnt), func(t *testing.T) { + if levels > 2 && (encrypt == (rLevel%2 == 1)) { + t.Skip("skipping to save time") + } testRedundancy(t, rLevel, encrypt, levels, chunkCnt, shardCnt, parityCnt) }) } From e563c82120ee9681177859ca0cf3e5c7af806014 Mon Sep 17 00:00:00 2001 From: zelig Date: Sat, 6 Jan 2024 07:54:41 +0100 Subject: [PATCH 34/52] fix(getter): lastIdx unnecessary --- pkg/file/redundancy/getter/getter.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/file/redundancy/getter/getter.go b/pkg/file/redundancy/getter/getter.go index e283498934f..787f251c438 100644 --- a/pkg/file/redundancy/getter/getter.go +++ b/pkg/file/redundancy/getter/getter.go @@ -28,7 +28,6 @@ type decoder struct { waits []chan struct{} // wait channels for each chunk rsbuf [][]byte // RS buffer of data + parity shards for erasure decoding ready chan struct{} // signal channel for successful retrieval of shardCnt chunks - lastIdx int // index of the last data chunk in the RS buffer lastLen int // length of the last data chunk in the RS buffer shardCnt int // number of data shards parityCnt int // number of parity shards @@ -119,7 +118,6 @@ func (g *decoder) setData(i int, chdata []byte) { data := chdata // pad the chunk with zeros if it is smaller than swarm.ChunkSize if len(data) < swarm.ChunkWithSpanSize { - g.lastIdx = i g.lastLen = len(data) data = make([]byte, swarm.ChunkWithSpanSize) copy(data, chdata) @@ -129,7 +127,7 @@ func (g *decoder) setData(i int, chdata []byte) { // getData returns the data shard from the RS buffer func (g *decoder) getData(i int) []byte { - if i > 0 && i == g.lastIdx { // zero value of g.lastIdx is impossible + if i == g.shardCnt-1 && g.lastLen > 0 { return g.rsbuf[i][:g.lastLen] // cut padding } return g.rsbuf[i] From a0b2dc9a7d5afb63b0923c8a123daa4d3dc9a6fd Mon Sep 17 00:00:00 2001 From: zelig Date: Sat, 6 Jan 2024 08:31:00 +0100 Subject: [PATCH 35/52] test: increase timeout --- pkg/file/joiner/joiner_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/file/joiner/joiner_test.go b/pkg/file/joiner/joiner_test.go index 69ce791612a..aceef3234c9 100644 --- a/pkg/file/joiner/joiner_test.go +++ b/pkg/file/joiner/joiner_test.go @@ -1245,7 +1245,7 @@ func TestJoinerRedundancyMultilevel(t *testing.T) { offset := mrand.Intn(size) * expRead canReadRange := func(t *testing.T, s getter.Strategy, fallback bool, canRead bool) { ctx := context.Background() - ctx = getter.SetParamsInContext(ctx, s, fallback, (6 * getter.StrategyTimeout).String()) + ctx = getter.SetParamsInContext(ctx, s, fallback, (10 * getter.StrategyTimeout).String()) ctx, cancel := context.WithTimeout(ctx, 10*getter.StrategyTimeout) defer cancel() j, _, err := joiner.New(ctx, store, store, addr) From 5f8862aa9cbd40658513806585036346c7e9ea6d Mon Sep 17 00:00:00 2001 From: zelig Date: Wed, 10 Jan 2024 07:46:17 +0100 Subject: [PATCH 36/52] fix: typo --- openapi/SwarmCommon.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openapi/SwarmCommon.yaml b/openapi/SwarmCommon.yaml index 4a7faadc154..07119cc6894 100644 --- a/openapi/SwarmCommon.yaml +++ b/openapi/SwarmCommon.yaml @@ -954,7 +954,7 @@ components: required: false description: > Specify the retrieve strategy on redundant data. - The mumbers stand for NONE, DATA, PROX and RACE, respectively. + The numbers stand for NONE, DATA, PROX and RACE, respectively. Strategy NONE means no prefetching takes place. Strategy DATA means only data chunks are prefetched. Strategy PROX means only chunks that are close to the node are prefetched. From 0624ea47e3e30eab8de8ee5ab72596fc137011d0 Mon Sep 17 00:00:00 2001 From: zelig Date: Wed, 10 Jan 2024 07:47:08 +0100 Subject: [PATCH 37/52] fix: incorrectly placed context timeouts --- pkg/api/bzz_test.go | 18 +++---- pkg/file/joiner/joiner.go | 4 +- pkg/file/redundancy/getter/getter.go | 64 ++++++++++++----------- pkg/file/redundancy/getter/getter_test.go | 10 ++-- pkg/file/redundancy/getter/strategies.go | 4 +- 5 files changed, 50 insertions(+), 50 deletions(-) diff --git a/pkg/api/bzz_test.go b/pkg/api/bzz_test.go index 6544f02174b..5027197ba3f 100644 --- a/pkg/api/bzz_test.go +++ b/pkg/api/bzz_test.go @@ -72,7 +72,7 @@ func TestBzzUploadDownloadWithRedundancy(t *testing.T) { if err != nil { t.Fatal(err) } - fetchTimeout := 200 * time.Millisecond + fetchTimeout := 100 * time.Millisecond store := mockstorer.NewForgettingStore(inmemchunkstore.New()) storerMock := mockstorer.NewWithChunkStore(store) client, _, _, _ := newTestServer(t, testServerOptions{ @@ -149,7 +149,9 @@ func TestBzzUploadDownloadWithRedundancy(t *testing.T) { if rLevel == 0 { t.Skip("NA") } - req, err := http.NewRequestWithContext(context.Background(), "GET", fileDownloadResource(refResponse.Reference.String()), nil) + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(ctx, "GET", fileDownloadResource(refResponse.Reference.String()), nil) if err != nil { t.Fatal(err) } @@ -157,16 +159,8 @@ func TestBzzUploadDownloadWithRedundancy(t *testing.T) { req.Header.Set(api.SwarmRedundancyFallbackModeHeader, "false") req.Header.Set(api.SwarmChunkRetrievalTimeoutHeader, fetchTimeout.String()) - resp, err := client.Do(req) - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - t.Fatalf("expected status %d; got %d", http.StatusOK, resp.StatusCode) - } - _, err = io.ReadAll(resp.Body) - if !errors.Is(err, io.ErrUnexpectedEOF) { + _, err = client.Do(req) + if !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("expected error %v; got %v", io.ErrUnexpectedEOF, err) } }) diff --git a/pkg/file/joiner/joiner.go b/pkg/file/joiner/joiner.go index c59c2146669..4b50ab47977 100644 --- a/pkg/file/joiner/joiner.go +++ b/pkg/file/joiner/joiner.go @@ -251,9 +251,7 @@ func (j *joiner) readAtOffset( func(address swarm.Address, b []byte, cur, subTrieSize, off, bufferOffset, bytesToRead, subtrieSpanLimit int64) { eg.Go(func() error { - ctx, cancel := context.WithTimeout(j.ctx, j.decoders.fetcherTimeout) - defer cancel() - ch, err := g.Get(ctx, addr) + ch, err := g.Get(j.ctx, addr) if err != nil { return err } diff --git a/pkg/file/redundancy/getter/getter.go b/pkg/file/redundancy/getter/getter.go index 787f251c438..196de329523 100644 --- a/pkg/file/redundancy/getter/getter.go +++ b/pkg/file/redundancy/getter/getter.go @@ -20,22 +20,23 @@ import ( // if retrieves children of an intermediate chunk potentially using erasure decoding // it caches sibling chunks if erasure decoding started already type decoder struct { - fetcher storage.Getter // network retrieval interface to fetch chunks - putter storage.Putter // interface to local storage to save reconstructed chunks - addrs []swarm.Address // all addresses of the intermediate chunk - inflight []atomic.Bool // locks to protect wait channels and RS buffer - cache map[string]int // map from chunk address shard position index - waits []chan struct{} // wait channels for each chunk - rsbuf [][]byte // RS buffer of data + parity shards for erasure decoding - ready chan struct{} // signal channel for successful retrieval of shardCnt chunks - lastLen int // length of the last data chunk in the RS buffer - shardCnt int // number of data shards - parityCnt int // number of parity shards - wg sync.WaitGroup // wait group to wait for all goroutines to finish - mu sync.Mutex // mutex to protect buffer - fetchedCnt atomic.Int32 // count successful retrievals - cancel func() // cancel function for RS decoding - remove func() // callback to remove decoder from decoders cache + fetcher storage.Getter // network retrieval interface to fetch chunks + putter storage.Putter // interface to local storage to save reconstructed chunks + addrs []swarm.Address // all addresses of the intermediate chunk + inflight []atomic.Bool // locks to protect wait channels and RS buffer + cache map[string]int // map from chunk address shard position index + waits []chan struct{} // wait channels for each chunk + rsbuf [][]byte // RS buffer of data + parity shards for erasure decoding + ready chan struct{} // signal channel for successful retrieval of shardCnt chunks + lastLen int // length of the last data chunk in the RS buffer + shardCnt int // number of data shards + parityCnt int // number of parity shards + wg sync.WaitGroup // wait group to wait for all goroutines to finish + mu sync.Mutex // mutex to protect buffer + fetchedCnt atomic.Int32 // count successful retrievals + fetchTimeout time.Duration // timeout for fetching a chunk + cancel func() // cancel function for RS decoding + remove func() // callback to remove decoder from decoders cache } type Getter interface { @@ -53,18 +54,19 @@ func New(addrs []swarm.Address, shardCnt int, g storage.Getter, p storage.Putter strategyTimeout := StrategyTimeout rsg := &decoder{ - fetcher: g, - putter: p, - addrs: addrs, - inflight: make([]atomic.Bool, size), - cache: make(map[string]int, size), - waits: make([]chan struct{}, shardCnt), - rsbuf: make([][]byte, size), - ready: make(chan struct{}, 1), - cancel: cancel, - remove: remove, - shardCnt: shardCnt, - parityCnt: size - shardCnt, + fetcher: g, + putter: p, + addrs: addrs, + inflight: make([]atomic.Bool, size), + cache: make(map[string]int, size), + waits: make([]chan struct{}, shardCnt), + rsbuf: make([][]byte, size), + ready: make(chan struct{}, 1), + cancel: cancel, + remove: remove, + shardCnt: shardCnt, + parityCnt: size - shardCnt, + fetchTimeout: fetchTimeout, } // after init, cache and wait channels are immutable, need no locking @@ -76,7 +78,7 @@ func New(addrs []swarm.Address, shardCnt int, g storage.Getter, p storage.Putter // prefetch chunks according to strategy rsg.wg.Add(1) go func() { - rsg.prefetch(ctx, strategy, strict, strategyTimeout, fetchTimeout) + rsg.prefetch(ctx, strategy, strict, strategyTimeout) rsg.wg.Done() }() return rsg @@ -145,7 +147,9 @@ func (g *decoder) fly(i int, up bool) (success bool) { // it races with erasure recovery which takes precedence even if it started later // due to the fact that erasure recovery could only implement global locking on all shards func (g *decoder) fetch(ctx context.Context, i int) { - ch, err := g.fetcher.Get(ctx, g.addrs[i]) + fctx, cancel := context.WithTimeout(ctx, g.fetchTimeout) + defer cancel() + ch, err := g.fetcher.Get(fctx, g.addrs[i]) if err != nil { _ = g.fly(i, false) // unset inflight return diff --git a/pkg/file/redundancy/getter/getter_test.go b/pkg/file/redundancy/getter/getter_test.go index 03fda3b8425..e0f59d9bd74 100644 --- a/pkg/file/redundancy/getter/getter_test.go +++ b/pkg/file/redundancy/getter/getter_test.go @@ -176,7 +176,7 @@ func testDecodingFallback(t *testing.T, s getter.Strategy, strict bool) { waitDelayed, waitErased := make(chan error, 1), make(chan error, 1) // complete retrieval of delayed chunk by putting it into the store after a while - delay := +getter.StrategyTimeout / 4 + delay := getter.StrategyTimeout / 4 if s == getter.NONE { delay += getter.StrategyTimeout } @@ -194,11 +194,15 @@ func testDecodingFallback(t *testing.T, s getter.Strategy, strict bool) { // delayed and erased chunk retrieval completes go func() { defer wg.Done() + ctx, cancel := context.WithTimeout(ctx, getter.StrategyTimeout*time.Duration(5-s)) + defer cancel() _, err := g.Get(ctx, addrs[delayed]) waitDelayed <- err }() go func() { defer wg.Done() + ctx, cancel := context.WithTimeout(ctx, getter.StrategyTimeout*time.Duration(5-s)) + defer cancel() _, err := g.Get(ctx, addrs[erased]) waitErased <- err }() @@ -240,10 +244,10 @@ func testDecodingFallback(t *testing.T, s getter.Strategy, strict bool) { case strict: t.Fatalf("unexpected completion of erased chunk retrieval. got round %d", round) case s == getter.NONE: - if round < 2 { + if round < 3 { t.Fatalf("unexpected early completion of erased chunk retrieval. got round %d", round) } - if round > 2 { + if round > 3 { t.Fatalf("unexpected late completion of erased chunk retrieval. got round %d", round) } case s == getter.DATA: diff --git a/pkg/file/redundancy/getter/strategies.go b/pkg/file/redundancy/getter/strategies.go index 9b0dcfb7935..b60f307c19c 100644 --- a/pkg/file/redundancy/getter/strategies.go +++ b/pkg/file/redundancy/getter/strategies.go @@ -82,7 +82,7 @@ func SetParamsInContext(ctx context.Context, s Strategy, fallbackmode bool, fetc ctx = SetFetchTimeout(ctx, fetchTimeout) return ctx } -func (g *decoder) prefetch(ctx context.Context, strategy int, strict bool, strategyTimeout, fetchTimeout time.Duration) { +func (g *decoder) prefetch(ctx context.Context, strategy int, strict bool, strategyTimeout time.Duration) { if strict && strategy == NONE { return } @@ -105,7 +105,7 @@ func (g *decoder) prefetch(ctx context.Context, strategy int, strict bool, strat defer timer.Stop() stop = timer.C } - lctx, cancel := context.WithTimeout(ctx, fetchTimeout) + lctx, cancel := context.WithCancel(ctx) cancels = append(cancels, cancel) prefetch(lctx, g, s) From 2e974676a118bb3f788817d073edebf4de01c4fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Tr=C3=B3n?= Date: Fri, 19 Jan 2024 04:50:24 +0100 Subject: [PATCH 38/52] fix: comment Co-authored-by: nugaon <50576770+nugaon@users.noreply.github.com> --- pkg/api/bzz_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/api/bzz_test.go b/pkg/api/bzz_test.go index 5027197ba3f..ccb3d1e5544 100644 --- a/pkg/api/bzz_test.go +++ b/pkg/api/bzz_test.go @@ -49,7 +49,7 @@ import ( // 1. upload a file with a given redundancy level and encryption // // 2. [positive test] download the file by the reference returned by the upload API response -// This uses range queries to target specific (number of) chunks of the file structure +// This uses range queries to target specific (number of) chunks of the file structure. // During path traversal in the swarm hash tree, the underlying mocksore (forgetting) // is in 'recording' mode, flagging all the retrieved chunks as chunks to forget. // This is to simulate the scenario where some of the chunks are not available/lost From 02ddf615d30d0c314f6cb6be134e81979d2b1ce9 Mon Sep 17 00:00:00 2001 From: Anatol <87016465+notanatol@users.noreply.github.com> Date: Fri, 19 Jan 2024 20:41:35 +0200 Subject: [PATCH 39/52] fix: slow redundancy test (#4537) --- pkg/api/bzz_test.go | 5 ++++- pkg/file/joiner/joiner_test.go | 5 ++++- pkg/replicas/getter.go | 21 ++++----------------- pkg/replicas/getter_test.go | 14 +++----------- 4 files changed, 15 insertions(+), 30 deletions(-) diff --git a/pkg/api/bzz_test.go b/pkg/api/bzz_test.go index ccb3d1e5544..1a7c577dc60 100644 --- a/pkg/api/bzz_test.go +++ b/pkg/api/bzz_test.go @@ -68,6 +68,7 @@ func TestBzzUploadDownloadWithRedundancy(t *testing.T) { fileDownloadResource := func(addr string) string { return "/bzz/" + addr + "/" } testRedundancy := func(t *testing.T, rLevel redundancy.Level, encrypt bool, levels int, chunkCnt int, shardCnt int, parityCnt int) { + t.Helper() seed, err := pseudorand.NewSeed() if err != nil { t.Fatal(err) @@ -197,6 +198,7 @@ func TestBzzUploadDownloadWithRedundancy(t *testing.T) { }) } for _, rLevel := range []redundancy.Level{1, 2, 3, 4} { + rLevel := rLevel t.Run(fmt.Sprintf("level=%d", rLevel), func(t *testing.T) { for _, encrypt := range []bool{false, true} { encrypt := encrypt @@ -216,17 +218,18 @@ func TestBzzUploadDownloadWithRedundancy(t *testing.T) { case 3: chunkCnt = shardCnt*shardCnt + 1 } + levels := levels t.Run(fmt.Sprintf("encrypt=%v levels=%d chunks=%d", encrypt, levels, chunkCnt), func(t *testing.T) { if levels > 2 && (encrypt == (rLevel%2 == 1)) { t.Skip("skipping to save time") } + t.Parallel() testRedundancy(t, rLevel, encrypt, levels, chunkCnt, shardCnt, parityCnt) }) } } }) } - } func TestBzzFiles(t *testing.T) { diff --git a/pkg/file/joiner/joiner_test.go b/pkg/file/joiner/joiner_test.go index aceef3234c9..13b9968be5b 100644 --- a/pkg/file/joiner/joiner_test.go +++ b/pkg/file/joiner/joiner_test.go @@ -1128,10 +1128,11 @@ func TestJoinerRedundancy(t *testing.T) { } i := 0 eg, ectx := errgroup.WithContext(ctx) + scnt: for ; i < shardCnt; i++ { select { case <-ectx.Done(): - break + break scnt default: } i := i @@ -1226,6 +1227,7 @@ func TestJoinerRedundancyMultilevel(t *testing.T) { getter.StrategyTimeout = 100 * time.Millisecond test := func(t *testing.T, rLevel redundancy.Level, encrypt bool, size int) { + t.Helper() store := mockstorer.NewForgettingStore(inmemchunkstore.New()) testutil.CleanupCloser(t, store) seed, err := pseudorand.NewSeed() @@ -1319,6 +1321,7 @@ func TestJoinerRedundancyMultilevel(t *testing.T) { rLevel := rLevel // speeding up tests by skipping some of them t.Run(fmt.Sprintf("rLevel=%v", rLevel), func(t *testing.T) { + t.Parallel() for _, encrypt := range []bool{false, true} { encrypt := encrypt shardCnt := rLevel.GetMaxShards() diff --git a/pkg/replicas/getter.go b/pkg/replicas/getter.go index 496bc5d658a..26345919958 100644 --- a/pkg/replicas/getter.go +++ b/pkg/replicas/getter.go @@ -24,20 +24,7 @@ import ( // then the probability of Swarmageddon is less than 0.000001 // assuming the error rate of chunk retrievals stays below the level expressed // as depth by the publisher. -type ErrSwarmageddon struct { - error -} - -func (err *ErrSwarmageddon) Unwrap() []error { - if err == nil || err.error == nil { - return nil - } - var uwe interface{ Unwrap() []error } - if !errors.As(err.error, &uwe) { - return nil - } - return uwe.Unwrap() -} +var ErrSwarmageddon = errors.New("swarmageddon has begun") // getter is the private implementation of storage.Getter, an interface for // retrieving chunks. This getter embeds the original simple chunk getter and extends it @@ -69,7 +56,7 @@ func (g *getter) Get(ctx context.Context, addr swarm.Address) (ch swarm.Chunk, e resultC := make(chan swarm.Chunk) // errc collects the errors errc := make(chan error, 17) - var errs []error + var errs error errcnt := 0 // concurrently call to retrieve chunk using original CAC address @@ -108,10 +95,10 @@ func (g *getter) Get(ctx context.Context, addr swarm.Address) (ch swarm.Chunk, e return chunk, nil case err = <-errc: - errs = append(errs, err) + errs = errors.Join(errs, err) errcnt++ if errcnt > total { - return nil, &ErrSwarmageddon{errors.Join(errs...)} + return nil, errors.Join(ErrSwarmageddon, errs) } // ticker switches on the address channel diff --git a/pkg/replicas/getter_test.go b/pkg/replicas/getter_test.go index 3b11ad26d94..7435b0574fd 100644 --- a/pkg/replicas/getter_test.go +++ b/pkg/replicas/getter_test.go @@ -195,18 +195,11 @@ func TestGetter(t *testing.T) { } t.Run("returns correct error", func(t *testing.T) { - var esg *replicas.ErrSwarmageddon - if !errors.As(err, &esg) { + if !errors.Is(err, replicas.ErrSwarmageddon) { t.Fatalf("incorrect error. want Swarmageddon. got %v", err) } - errs := esg.Unwrap() - for _, err := range errs { - if !errors.Is(err, tc.failure.err) { - t.Fatalf("incorrect error. want it to wrap %v. got %v", tc.failure.err, err) - } - } - if len(errs) != tc.count+1 { - t.Fatalf("incorrect error. want %d. got %d", tc.count+1, len(errs)) + if !errors.Is(err, tc.failure.err) { + t.Fatalf("incorrect error. want it to wrap %v. got %v", tc.failure.err, err) } }) } @@ -265,7 +258,6 @@ func TestGetter(t *testing.T) { } } }) - }) } } From 218fe7e7ebf99a6f3e97cc624c09d7b35807ebd0 Mon Sep 17 00:00:00 2001 From: dysordys Date: Sat, 20 Jan 2024 02:49:30 +0100 Subject: [PATCH 40/52] fix(redunadncy): amend parity numbers (#4547) Co-authored-by: Gyorgy Barabas --- pkg/file/pipeline/hashtrie/hashtrie_test.go | 4 +-- pkg/file/redundancy/level.go | 40 ++++++++++----------- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/pkg/file/pipeline/hashtrie/hashtrie_test.go b/pkg/file/pipeline/hashtrie/hashtrie_test.go index a6c866a8900..4a966e265f3 100644 --- a/pkg/file/pipeline/hashtrie/hashtrie_test.go +++ b/pkg/file/pipeline/hashtrie/hashtrie_test.go @@ -302,14 +302,14 @@ func TestRedundancy(t *testing.T) { level: redundancy.INSANE, encryption: false, writes: 98, // 97 chunk references fit into one chunk + 1 carrier - parities: 38, // 31 (full ch) + 7 (2 ref) + parities: 37, // 31 (full ch) + 6 (2 ref) }, { desc: "redundancy write for encrypted data", level: redundancy.PARANOID, encryption: true, writes: 21, // 21 encrypted chunk references fit into one chunk + 1 carrier - parities: 118, // // 88 (full ch) + 30 (2 ref) + parities: 116, // // 87 (full ch) + 29 (2 ref) }, } { tc := tc diff --git a/pkg/file/redundancy/level.go b/pkg/file/redundancy/level.go index 4aa8c75bfe8..f7b4f5c19f1 100644 --- a/pkg/file/redundancy/level.go +++ b/pkg/file/redundancy/level.go @@ -107,44 +107,40 @@ func (l Level) Decrement() Level { // TABLE INITS var mediumEt = newErasureTable( - []int{94, 68, 46, 28, 14, 5, 1}, - []int{9, 8, 7, 6, 5, 4, 3}, + []int{95, 69, 47, 29, 15, 6, 2, 1}, + []int{9, 8, 7, 6, 5, 4, 3, 2}, ) var encMediumEt = newErasureTable( - []int{47, 34, 23, 14, 7, 2}, - []int{9, 8, 7, 6, 5, 4}, + []int{47, 34, 23, 14, 7, 3, 1}, + []int{9, 8, 7, 6, 5, 4, 3}, ) var strongEt = newErasureTable( - []int{104, 95, 86, 77, 69, 61, 53, 46, 39, 32, 26, 20, 15, 10, 6, 3, 1}, - []int{21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5}, + []int{105, 96, 87, 78, 70, 62, 54, 47, 40, 33, 27, 21, 16, 11, 7, 4, 2, 1}, + []int{21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4}, ) var encStrongEt = newErasureTable( - []int{52, 47, 43, 38, 34, 30, 26, 23, 19, 16, 13, 10, 7, 5, 3, 1}, - []int{21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6}, + []int{52, 48, 43, 39, 35, 31, 27, 23, 20, 16, 13, 10, 8, 5, 3, 2, 1}, + []int{21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5}, ) var insaneEt = newErasureTable( - []int{92, 87, 82, 77, 73, 68, 63, 59, 54, 50, 45, 41, 37, 33, 29, 26, 22, 19, 16, 13, 10, 8, 5, 3, 2, 1}, - []int{31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6}, + []int{93, 88, 83, 78, 74, 69, 64, 60, 55, 51, 46, 42, 38, 34, 30, 27, 23, 20, 17, 14, 11, 9, 6, 4, 3, 2, 1}, + []int{31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5}, ) var encInsaneEt = newErasureTable( - []int{46, 43, 41, 38, 36, 34, 31, 29, 27, 25, 22, 20, 18, 16, 14, 13, 11, 9, 8, 6, 5, 4, 2, 1}, - []int{31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 7}, + []int{46, 44, 41, 39, 37, 34, 32, 30, 27, 25, 23, 21, 19, 17, 15, 13, 11, 10, 8, 7, 5, 4, 3, 2, 1}, + []int{31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 6}, ) var paranoidEt = newErasureTable( []int{ - 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, - 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, - 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, - 7, 6, 5, 4, 3, 2, 1, + 37, 36, 35, 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, }, []int{ - 90, 88, 87, 85, 84, 82, 81, 79, 77, 76, - 74, 72, 71, 69, 67, 66, 64, 62, 60, 59, - 57, 55, 53, 51, 49, 48, 46, 44, 41, 39, - 37, 35, 32, 30, 27, 24, 20, + 89, 87, 86, 84, 83, 81, 80, 78, 76, 75, 73, 71, 70, 68, 66, 65, 63, 61, 59, 58, + 56, 54, 52, 50, 48, 47, 45, 43, 40, 38, 36, 34, 31, 29, 26, 23, 19, }, ) var encParanoidEt = newErasureTable( @@ -153,8 +149,8 @@ var encParanoidEt = newErasureTable( 8, 7, 6, 5, 4, 3, 2, 1, }, []int{ - 88, 85, 82, 79, 76, 72, 69, 66, 62, 59, - 55, 51, 48, 44, 39, 35, 30, 24, + 87, 84, 81, 78, 75, 71, 68, 65, 61, 58, + 54, 50, 47, 43, 38, 34, 29, 23, }, ) From 6139ab0a91c46519311e988bed2a2768aa4726d3 Mon Sep 17 00:00:00 2001 From: zelig Date: Sat, 20 Jan 2024 03:30:39 +0100 Subject: [PATCH 41/52] test(getter): decrease flakiness --- pkg/file/redundancy/getter/getter_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/file/redundancy/getter/getter_test.go b/pkg/file/redundancy/getter/getter_test.go index e0f59d9bd74..5b7d946ac4a 100644 --- a/pkg/file/redundancy/getter/getter_test.go +++ b/pkg/file/redundancy/getter/getter_test.go @@ -152,7 +152,7 @@ func testDecodingFallback(t *testing.T, s getter.Strategy, strict bool) { strategyTimeout := getter.StrategyTimeout defer func() { getter.StrategyTimeout = strategyTimeout }() - getter.StrategyTimeout = 100 * time.Millisecond + getter.StrategyTimeout = 150 * time.Millisecond bufSize := 12 shardCnt := 6 From 74bd8487617d97597aeb47e248fae22c0da0a255 Mon Sep 17 00:00:00 2001 From: zelig Date: Sat, 20 Jan 2024 03:32:42 +0100 Subject: [PATCH 42/52] chore(pseudorand): move from ioutil to testutil --- pkg/api/bzz_test.go | 2 +- pkg/file/joiner/joiner_test.go | 2 +- pkg/util/{ioutil => testutil}/pseudorand/reader.go | 0 pkg/util/{ioutil => testutil}/pseudorand/reader_test.go | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename pkg/util/{ioutil => testutil}/pseudorand/reader.go (100%) rename pkg/util/{ioutil => testutil}/pseudorand/reader_test.go (97%) diff --git a/pkg/api/bzz_test.go b/pkg/api/bzz_test.go index 1a7c577dc60..edf90a78c48 100644 --- a/pkg/api/bzz_test.go +++ b/pkg/api/bzz_test.go @@ -30,7 +30,7 @@ import ( "github.com/ethersphere/bee/pkg/storage/inmemchunkstore" mockstorer "github.com/ethersphere/bee/pkg/storer/mock" "github.com/ethersphere/bee/pkg/swarm" - "github.com/ethersphere/bee/pkg/util/ioutil/pseudorand" + "github.com/ethersphere/bee/pkg/util/testutil/pseudorand" ) // nolint:paralleltest,tparallel,thelper diff --git a/pkg/file/joiner/joiner_test.go b/pkg/file/joiner/joiner_test.go index 13b9968be5b..3044ef0a994 100644 --- a/pkg/file/joiner/joiner_test.go +++ b/pkg/file/joiner/joiner_test.go @@ -29,8 +29,8 @@ import ( testingc "github.com/ethersphere/bee/pkg/storage/testing" mockstorer "github.com/ethersphere/bee/pkg/storer/mock" "github.com/ethersphere/bee/pkg/swarm" - "github.com/ethersphere/bee/pkg/util/ioutil/pseudorand" "github.com/ethersphere/bee/pkg/util/testutil" + "github.com/ethersphere/bee/pkg/util/testutil/pseudorand" "gitlab.com/nolash/go-mockbytes" "golang.org/x/sync/errgroup" ) diff --git a/pkg/util/ioutil/pseudorand/reader.go b/pkg/util/testutil/pseudorand/reader.go similarity index 100% rename from pkg/util/ioutil/pseudorand/reader.go rename to pkg/util/testutil/pseudorand/reader.go diff --git a/pkg/util/ioutil/pseudorand/reader_test.go b/pkg/util/testutil/pseudorand/reader_test.go similarity index 97% rename from pkg/util/ioutil/pseudorand/reader_test.go rename to pkg/util/testutil/pseudorand/reader_test.go index ef6d492553b..4ec85a90d73 100644 --- a/pkg/util/ioutil/pseudorand/reader_test.go +++ b/pkg/util/testutil/pseudorand/reader_test.go @@ -12,7 +12,7 @@ import ( "math/rand" "testing" - "github.com/ethersphere/bee/pkg/util/ioutil/pseudorand" + "github.com/ethersphere/bee/pkg/util/testutil/pseudorand" ) func TestReader(t *testing.T) { From 68c3483de2c78d32c7a923cde06f67e182fbb6c4 Mon Sep 17 00:00:00 2001 From: zelig Date: Tue, 23 Jan 2024 05:25:46 +0100 Subject: [PATCH 43/52] refactor(redundancy): introduce getter Config and improve default params --- pkg/api/bzz.go | 4 +- pkg/file/joiner/joiner.go | 35 +++---- pkg/file/joiner/joiner_test.go | 43 ++++---- pkg/file/redundancy/getter/getter.go | 81 ++++++++------- pkg/file/redundancy/getter/getter_test.go | 43 ++++---- pkg/file/redundancy/getter/strategies.go | 115 +++++++++++++++------- 6 files changed, 182 insertions(+), 139 deletions(-) diff --git a/pkg/api/bzz.go b/pkg/api/bzz.go index b787aa5b132..fc5b5ec56be 100644 --- a/pkg/api/bzz.go +++ b/pkg/api/bzz.go @@ -290,7 +290,7 @@ func (s *Service) serveReference(logger log.Logger, address swarm.Address, pathV feedDereferenced := false ctx := r.Context() - ctx = getter.SetParamsInContext(ctx, headers.Strategy, headers.FallbackMode, headers.ChunkRetrievalTimeout) + ctx = getter.SetConfigInContext(ctx, headers.Strategy, headers.FallbackMode, headers.ChunkRetrievalTimeout, getter.DefaultStrategyTimeout.String()) FETCH: // read manifest entry @@ -476,7 +476,7 @@ func (s *Service) downloadHandler(logger log.Logger, w http.ResponseWriter, r *h } ctx := r.Context() - ctx = getter.SetParamsInContext(ctx, headers.Strategy, headers.FallbackMode, headers.ChunkRetrievalTimeout) + ctx = getter.SetConfigInContext(ctx, headers.Strategy, headers.FallbackMode, headers.ChunkRetrievalTimeout, getter.DefaultStrategyTimeout.String()) reader, l, err := joiner.New(ctx, s.storer.Download(cache), s.storer.Cache(), reference) if err != nil { if errors.Is(err, storage.ErrNotFound) || errors.Is(err, topology.ErrNotFound) { diff --git a/pkg/file/joiner/joiner.go b/pkg/file/joiner/joiner.go index 4b50ab47977..dc9c850084c 100644 --- a/pkg/file/joiner/joiner.go +++ b/pkg/file/joiner/joiner.go @@ -11,7 +11,6 @@ import ( "io" "sync" "sync/atomic" - "time" "github.com/ethersphere/bee/pkg/bmt" "github.com/ethersphere/bee/pkg/encryption" @@ -41,24 +40,20 @@ type joiner struct { // decoderCache is cache of decoders for intermediate chunks type decoderCache struct { - fetcher storage.Getter // network retrieval interface to fetch chunks - putter storage.Putter // interface to local storage to save reconstructed chunks - mu sync.Mutex // mutex to protect cache - cache map[string]storage.Getter // map from chunk address to RS getter - strategy getter.Strategy // strategy to use for retrieval - strict bool // strict mode - fetcherTimeout time.Duration // timeout for each fetch + fetcher storage.Getter // network retrieval interface to fetch chunks + putter storage.Putter // interface to local storage to save reconstructed chunks + mu sync.Mutex // mutex to protect cache + cache map[string]storage.Getter // map from chunk address to RS getter + config getter.Config // getter configuration } // NewDecoderCache creates a new decoder cache -func NewDecoderCache(g storage.Getter, p storage.Putter, strategy getter.Strategy, strict bool, fetcherTimeout time.Duration) *decoderCache { +func NewDecoderCache(g storage.Getter, p storage.Putter, conf getter.Config) *decoderCache { return &decoderCache{ - fetcher: g, - putter: p, - cache: make(map[string]storage.Getter), - strategy: strategy, - strict: strict, - fetcherTimeout: fetcherTimeout, + fetcher: g, + putter: p, + cache: make(map[string]storage.Getter), + config: conf, } } @@ -90,7 +85,7 @@ func (g *decoderCache) GetOrCreate(addrs []swarm.Address, shardCnt int) storage. defer g.mu.Unlock() g.cache[key] = nil } - d = getter.New(addrs, shardCnt, g.fetcher, g.putter, g.strategy, g.strict, g.fetcherTimeout, remove) + d = getter.New(addrs, shardCnt, g.fetcher, g.putter, remove, g.config) g.cache[key] = d return d } @@ -118,7 +113,7 @@ func New(ctx context.Context, g storage.Getter, putter storage.Putter, address s spanFn := func(data []byte) (redundancy.Level, int64) { return 0, int64(bmt.LengthFromSpan(data[:swarm.SpanSize])) } - strategy, strict, fetcherTimeout, err := getter.GetParamsFromContext(ctx) + conf, err := getter.NewConfigFromContext(ctx, getter.DefaultConfig) if err != nil { return nil, 0, err } @@ -135,15 +130,15 @@ func New(ctx context.Context, g storage.Getter, putter storage.Putter, address s } } else { // if root chunk has no redundancy, strategy is ignored and set to NONE and strict is set to true - strategy = getter.NONE - strict = true + conf.Strategy = getter.DATA + conf.Strict = true } j := &joiner{ addr: rootChunk.Address(), refLength: refLength, ctx: ctx, - decoders: NewDecoderCache(g, putter, strategy, strict, fetcherTimeout), + decoders: NewDecoderCache(g, putter, conf), span: span, rootData: rootData, rootParity: rootParity, diff --git a/pkg/file/joiner/joiner_test.go b/pkg/file/joiner/joiner_test.go index 3044ef0a994..154a5978dd0 100644 --- a/pkg/file/joiner/joiner_test.go +++ b/pkg/file/joiner/joiner_test.go @@ -1024,11 +1024,7 @@ func (m *mockPutter) store(cnt int) error { // nolint:thelper func TestJoinerRedundancy(t *testing.T) { - - strategyTimeout := getter.StrategyTimeout - defer func() { getter.StrategyTimeout = strategyTimeout }() - getter.StrategyTimeout = 100 * time.Millisecond - + t.Parallel() for _, tc := range []struct { rLevel redundancy.Level encryptChunk bool @@ -1112,11 +1108,12 @@ func TestJoinerRedundancy(t *testing.T) { if err != nil { t.Fatal(err) } + strategyTimeout := 100 * time.Millisecond // all data can be read back readCheck := func(t *testing.T, expErr error) { - ctx, cancel := context.WithTimeout(context.Background(), 15*getter.StrategyTimeout) + ctx, cancel := context.WithTimeout(context.Background(), 15*strategyTimeout) defer cancel() - ctx = getter.SetParamsInContext(ctx, getter.RACE, true, (10 * getter.StrategyTimeout).String()) + ctx = getter.SetConfigInContext(ctx, getter.RACE, true, (10 * strategyTimeout).String(), strategyTimeout.String()) joinReader, rootSpan, err := joiner.New(ctx, store, store, swarmAddr) if err != nil { t.Fatal(err) @@ -1221,12 +1218,8 @@ func TestJoinerRedundancy(t *testing.T) { // // nolint:thelper func TestJoinerRedundancyMultilevel(t *testing.T) { - - strategyTimeout := getter.StrategyTimeout - defer func() { getter.StrategyTimeout = strategyTimeout }() - getter.StrategyTimeout = 100 * time.Millisecond - - test := func(t *testing.T, rLevel redundancy.Level, encrypt bool, size int) { + t.Parallel() + test := func(t *testing.T, rLevel redundancy.Level, encrypt bool, levels, size int) { t.Helper() store := mockstorer.NewForgettingStore(inmemchunkstore.New()) testutil.CleanupCloser(t, store) @@ -1236,6 +1229,7 @@ func TestJoinerRedundancyMultilevel(t *testing.T) { } dataReader := pseudorand.NewReader(seed, size*swarm.ChunkSize) ctx := context.Background() + // ctx = redundancy.SetLevelInContext(ctx, rLevel) ctx = redundancy.SetLevelInContext(ctx, redundancy.NONE) pipe := builder.NewPipelineBuilder(ctx, store, encrypt, rLevel) addr, err := builder.FeedPipeline(ctx, pipe, dataReader) @@ -1245,10 +1239,11 @@ func TestJoinerRedundancyMultilevel(t *testing.T) { expRead := swarm.ChunkSize buf := make([]byte, expRead) offset := mrand.Intn(size) * expRead - canReadRange := func(t *testing.T, s getter.Strategy, fallback bool, canRead bool) { + canReadRange := func(t *testing.T, s getter.Strategy, fallback bool, levels int, canRead bool) { ctx := context.Background() - ctx = getter.SetParamsInContext(ctx, s, fallback, (10 * getter.StrategyTimeout).String()) - ctx, cancel := context.WithTimeout(ctx, 10*getter.StrategyTimeout) + strategyTimeout := 100 * time.Millisecond + ctx = getter.SetConfigInContext(ctx, s, fallback, (2 * strategyTimeout).String(), strategyTimeout.String()) + ctx, cancel := context.WithTimeout(ctx, time.Duration(levels*3+1)*strategyTimeout) defer cancel() j, _, err := joiner.New(ctx, store, store, addr) if err != nil { @@ -1284,35 +1279,35 @@ func TestJoinerRedundancyMultilevel(t *testing.T) { t.Run("NONE w/o fallback CAN retrieve", func(t *testing.T) { store.Record() defer store.Unrecord() - canReadRange(t, getter.NONE, false, true) + canReadRange(t, getter.NONE, false, levels, true) }) // do not forget the root chunk store.Unmiss(swarm.NewAddress(addr.Bytes()[:swarm.HashSize])) // after we forget the chunks on the way to the range, we should not be able to retrieve t.Run("NONE w/o fallback CANNOT retrieve", func(t *testing.T) { - canReadRange(t, getter.NONE, false, false) + canReadRange(t, getter.NONE, false, levels, false) }) // we lost a data chunk, we cannot recover using DATA only strategy with no fallback t.Run("DATA w/o fallback CANNOT retrieve", func(t *testing.T) { - canReadRange(t, getter.DATA, false, false) + canReadRange(t, getter.DATA, false, levels, false) }) if rLevel == 0 { // allowing fallback mode will not help if no redundancy used for upload t.Run("DATA w fallback CANNOT retrieve", func(t *testing.T) { - canReadRange(t, getter.DATA, true, false) + canReadRange(t, getter.DATA, true, levels, false) }) return } // allowing fallback mode will make the range retrievable using erasure decoding t.Run("DATA w fallback CAN retrieve", func(t *testing.T) { - canReadRange(t, getter.DATA, true, true) + canReadRange(t, getter.DATA, true, levels, true) }) // after the reconstructed data is stored, we can retrieve the range using DATA only mode t.Run("after recovery, NONE w/o fallback CAN retrieve", func(t *testing.T) { - canReadRange(t, getter.NONE, false, true) + canReadRange(t, getter.NONE, false, levels, true) }) } r2level := []int{2, 1, 2, 3, 2} @@ -1342,7 +1337,7 @@ func TestJoinerRedundancyMultilevel(t *testing.T) { if r2level[rLevel] != levels || encrypt != encryptChunk[rLevel] { t.Skip("skipping to save time") } - test(t, rLevel, encrypt, chunkCnt) + test(t, rLevel, encrypt, levels, chunkCnt) }) switch levels { case 1: @@ -1353,7 +1348,7 @@ func TestJoinerRedundancyMultilevel(t *testing.T) { continue } t.Run(fmt.Sprintf("encrypt=%v levels=%d chunks=%d full", encrypt, levels, chunkCnt), func(t *testing.T) { - test(t, rLevel, encrypt, chunkCnt) + test(t, rLevel, encrypt, levels, chunkCnt) }) } } diff --git a/pkg/file/redundancy/getter/getter.go b/pkg/file/redundancy/getter/getter.go index 196de329523..9f64674ed28 100644 --- a/pkg/file/redundancy/getter/getter.go +++ b/pkg/file/redundancy/getter/getter.go @@ -20,23 +20,24 @@ import ( // if retrieves children of an intermediate chunk potentially using erasure decoding // it caches sibling chunks if erasure decoding started already type decoder struct { - fetcher storage.Getter // network retrieval interface to fetch chunks - putter storage.Putter // interface to local storage to save reconstructed chunks - addrs []swarm.Address // all addresses of the intermediate chunk - inflight []atomic.Bool // locks to protect wait channels and RS buffer - cache map[string]int // map from chunk address shard position index - waits []chan struct{} // wait channels for each chunk - rsbuf [][]byte // RS buffer of data + parity shards for erasure decoding - ready chan struct{} // signal channel for successful retrieval of shardCnt chunks - lastLen int // length of the last data chunk in the RS buffer - shardCnt int // number of data shards - parityCnt int // number of parity shards - wg sync.WaitGroup // wait group to wait for all goroutines to finish - mu sync.Mutex // mutex to protect buffer - fetchedCnt atomic.Int32 // count successful retrievals - fetchTimeout time.Duration // timeout for fetching a chunk - cancel func() // cancel function for RS decoding - remove func() // callback to remove decoder from decoders cache + fetcher storage.Getter // network retrieval interface to fetch chunks + putter storage.Putter // interface to local storage to save reconstructed chunks + addrs []swarm.Address // all addresses of the intermediate chunk + inflight []atomic.Bool // locks to protect wait channels and RS buffer + cache map[string]int // map from chunk address shard position index + waits []chan struct{} // wait channels for each chunk + rsbuf [][]byte // RS buffer of data + parity shards for erasure decoding + ready chan struct{} // signal channel for successful retrieval of shardCnt chunks + lastLen int // length of the last data chunk in the RS buffer + shardCnt int // number of data shards + parityCnt int // number of parity shards + wg sync.WaitGroup // wait group to wait for all goroutines to finish + mu sync.Mutex // mutex to protect buffer + err error // error of the last erasure decoding + fetchedCnt atomic.Int32 // count successful retrievals + cancel func() // cancel function for RS decoding + remove func() // callback to remove decoder from decoders cache + config Config // configuration } type Getter interface { @@ -45,28 +46,24 @@ type Getter interface { } // New returns a decoder object used to retrieve children of an intermediate chunk -func New(addrs []swarm.Address, shardCnt int, g storage.Getter, p storage.Putter, strategy Strategy, strict bool, fetchTimeout time.Duration, remove func()) Getter { +func New(addrs []swarm.Address, shardCnt int, g storage.Getter, p storage.Putter, remove func(), conf Config) Getter { ctx, cancel := context.WithCancel(context.Background()) size := len(addrs) - if fetchTimeout == 0 { - fetchTimeout = 30 * time.Second - } - strategyTimeout := StrategyTimeout rsg := &decoder{ - fetcher: g, - putter: p, - addrs: addrs, - inflight: make([]atomic.Bool, size), - cache: make(map[string]int, size), - waits: make([]chan struct{}, shardCnt), - rsbuf: make([][]byte, size), - ready: make(chan struct{}, 1), - cancel: cancel, - remove: remove, - shardCnt: shardCnt, - parityCnt: size - shardCnt, - fetchTimeout: fetchTimeout, + fetcher: g, + putter: p, + addrs: addrs, + inflight: make([]atomic.Bool, size), + cache: make(map[string]int, size), + waits: make([]chan struct{}, shardCnt), + rsbuf: make([][]byte, size), + ready: make(chan struct{}, 1), + cancel: cancel, + remove: remove, + shardCnt: shardCnt, + parityCnt: size - shardCnt, + config: conf, } // after init, cache and wait channels are immutable, need no locking @@ -76,11 +73,13 @@ func New(addrs []swarm.Address, shardCnt int, g storage.Getter, p storage.Putter } // prefetch chunks according to strategy - rsg.wg.Add(1) - go func() { - rsg.prefetch(ctx, strategy, strict, strategyTimeout) - rsg.wg.Done() - }() + if !conf.Strict || conf.Strategy != NONE { + rsg.wg.Add(1) + go func() { + rsg.prefetch(ctx) + rsg.wg.Done() + }() + } return rsg } @@ -147,7 +146,7 @@ func (g *decoder) fly(i int, up bool) (success bool) { // it races with erasure recovery which takes precedence even if it started later // due to the fact that erasure recovery could only implement global locking on all shards func (g *decoder) fetch(ctx context.Context, i int) { - fctx, cancel := context.WithTimeout(ctx, g.fetchTimeout) + fctx, cancel := context.WithTimeout(ctx, g.config.FetchTimeout) defer cancel() ch, err := g.fetcher.Get(fctx, g.addrs[i]) if err != nil { diff --git a/pkg/file/redundancy/getter/getter_test.go b/pkg/file/redundancy/getter/getter_test.go index 5b7d946ac4a..ce0378fc68a 100644 --- a/pkg/file/redundancy/getter/getter_test.go +++ b/pkg/file/redundancy/getter/getter_test.go @@ -93,11 +93,7 @@ func TestGetterFallback(t *testing.T) { func testDecodingRACE(t *testing.T, bufSize, shardCnt, erasureCnt int) { t.Helper() - - strategyTimeout := getter.StrategyTimeout - defer func() { getter.StrategyTimeout = strategyTimeout }() - getter.StrategyTimeout = 100 * time.Millisecond - + strategyTimeout := 100 * time.Millisecond store := inmem.New() buf := make([][]byte, bufSize) addrs := initData(t, buf, shardCnt, store) @@ -115,7 +111,12 @@ func testDecodingRACE(t *testing.T, bufSize, shardCnt, erasureCnt int) { } ctx, cancel := context.WithCancel(context.TODO()) defer cancel() - g := getter.New(addrs, shardCnt, store, store, getter.RACE, false, 2*getter.StrategyTimeout, func() {}) + conf := getter.Config{ + Strategy: getter.RACE, + FetchTimeout: 2 * strategyTimeout, + StrategyTimeout: strategyTimeout, + } + g := getter.New(addrs, shardCnt, store, store, func() {}, conf) defer g.Close() parityCnt := len(buf) - shardCnt q := make(chan error, 1) @@ -126,7 +127,7 @@ func testDecodingRACE(t *testing.T, bufSize, shardCnt, erasureCnt int) { err := context.DeadlineExceeded select { case err = <-q: - case <-time.After(getter.StrategyTimeout * 10): + case <-time.After(strategyTimeout * 2): } switch { case erasureCnt > parityCnt: @@ -150,9 +151,7 @@ func testDecodingRACE(t *testing.T, bufSize, shardCnt, erasureCnt int) { func testDecodingFallback(t *testing.T, s getter.Strategy, strict bool) { t.Helper() - strategyTimeout := getter.StrategyTimeout - defer func() { getter.StrategyTimeout = strategyTimeout }() - getter.StrategyTimeout = 150 * time.Millisecond + strategyTimeout := 150 * time.Millisecond bufSize := 12 shardCnt := 6 @@ -176,14 +175,20 @@ func testDecodingFallback(t *testing.T, s getter.Strategy, strict bool) { waitDelayed, waitErased := make(chan error, 1), make(chan error, 1) // complete retrieval of delayed chunk by putting it into the store after a while - delay := getter.StrategyTimeout / 4 + delay := strategyTimeout / 4 if s == getter.NONE { - delay += getter.StrategyTimeout + delay += strategyTimeout } store.Delay(addrs[delayed], delay) // create getter start := time.Now() - g := getter.New(addrs, shardCnt, store, store, s, strict, getter.StrategyTimeout/2, func() {}) + conf := getter.Config{ + Strategy: s, + Strict: strict, + FetchTimeout: strategyTimeout / 2, + StrategyTimeout: strategyTimeout, + } + g := getter.New(addrs, shardCnt, store, store, func() {}, conf) defer g.Close() // launch delayed and erased chunk retrieval @@ -194,14 +199,14 @@ func testDecodingFallback(t *testing.T, s getter.Strategy, strict bool) { // delayed and erased chunk retrieval completes go func() { defer wg.Done() - ctx, cancel := context.WithTimeout(ctx, getter.StrategyTimeout*time.Duration(5-s)) + ctx, cancel := context.WithTimeout(ctx, strategyTimeout*time.Duration(5-s)) defer cancel() _, err := g.Get(ctx, addrs[delayed]) waitDelayed <- err }() go func() { defer wg.Done() - ctx, cancel := context.WithTimeout(ctx, getter.StrategyTimeout*time.Duration(5-s)) + ctx, cancel := context.WithTimeout(ctx, strategyTimeout*time.Duration(5-s)) defer cancel() _, err := g.Get(ctx, addrs[erased]) waitErased <- err @@ -213,7 +218,7 @@ func testDecodingFallback(t *testing.T, s getter.Strategy, strict bool) { if err != nil { t.Fatal("unexpected error", err) } - round := time.Since(start) / getter.StrategyTimeout + round := time.Since(start) / strategyTimeout switch { case strict && s == getter.NONE: if round < 1 { @@ -239,7 +244,7 @@ func testDecodingFallback(t *testing.T, s getter.Strategy, strict bool) { if err != nil { t.Fatal("unexpected error", err) } - round = time.Since(start) / getter.StrategyTimeout + round = time.Since(start) / strategyTimeout switch { case strict: t.Fatalf("unexpected completion of erased chunk retrieval. got round %d", round) @@ -260,12 +265,12 @@ func testDecodingFallback(t *testing.T, s getter.Strategy, strict bool) { } checkShardsAvailable(t, store, addrs[:erased], buf[:erased]) - case <-time.After(getter.StrategyTimeout * 2): + case <-time.After(strategyTimeout * 2): if !strict { t.Fatal("unexpected timeout using strategy", s, "with strict", strict) } } - case <-time.After(getter.StrategyTimeout * 3): + case <-time.After(strategyTimeout * 3): if !strict || s != getter.NONE { t.Fatal("unexpected timeout using strategy", s, "with strict", strict) } diff --git a/pkg/file/redundancy/getter/strategies.go b/pkg/file/redundancy/getter/strategies.go index b60f307c19c..3b800d85011 100644 --- a/pkg/file/redundancy/getter/strategies.go +++ b/pkg/file/redundancy/getter/strategies.go @@ -6,21 +6,34 @@ package getter import ( "context" + "errors" "fmt" "time" ) -var ( - StrategyTimeout = 500 * time.Millisecond // timeout for each strategy +const ( + DefaultStrategy = NONE // default prefetching strategy + DefaultStrict = true // default fallback modes + DefaultFetchTimeout = 30 * time.Second // timeout for each chunk retrieval + DefaultStrategyTimeout = 300 * time.Millisecond // timeout for each strategy ) type ( - strategyKey struct{} - modeKey struct{} - fetcherTimeoutKey struct{} - Strategy = int + strategyKey struct{} + modeKey struct{} + fetchTimeoutKey struct{} + strategyTimeoutKey struct{} + Strategy = int ) +// Config is the configuration for the getter - public +type Config struct { + Strategy Strategy + Strict bool + FetchTimeout time.Duration + StrategyTimeout time.Duration +} + const ( NONE Strategy = iota // no prefetching and no decoding DATA // just retrieve data shards no decoding @@ -29,41 +42,57 @@ const ( strategyCnt ) -// GetParamsFromContext extracts the strategy and strict mode from the context -func GetParamsFromContext(ctx context.Context) (s Strategy, strict bool, fetcherTimeout time.Duration, err error) { +// DefaultConfig is the default configuration for the getter +var DefaultConfig = Config{ + Strategy: DefaultStrategy, + Strict: DefaultStrict, + FetchTimeout: DefaultFetchTimeout, + StrategyTimeout: DefaultStrategyTimeout, +} + +// NewConfigFromContext returns a new Config based on the context +func NewConfigFromContext(ctx context.Context, def Config) (conf Config, err error) { var ok bool - s, strict, fetcherTimeoutVal := NONE, true, "30s" + conf = def + e := func(s string, errs ...error) error { + if len(errs) > 0 { + return fmt.Errorf("error setting %s from context: %w", s, errors.Join(errs...)) + } + return fmt.Errorf("error setting %s from context", s) + } if val := ctx.Value(strategyKey{}); val != nil { - s, ok = val.(Strategy) + conf.Strategy, ok = val.(Strategy) if !ok { - return s, strict, fetcherTimeout, fmt.Errorf("error setting strategy from context") + return conf, e("strategy") } } if val := ctx.Value(modeKey{}); val != nil { - strict, ok = val.(bool) + conf.Strict, ok = val.(bool) if !ok { - return s, strict, fetcherTimeout, fmt.Errorf("error setting fallback mode from context") + return conf, e("fallback mode") } } - if val := ctx.Value(fetcherTimeoutKey{}); val != nil { - fetcherTimeoutVal, ok = val.(string) + if val := ctx.Value(fetchTimeoutKey{}); val != nil { + fetchTimeoutVal, ok := val.(string) if !ok { - return s, strict, fetcherTimeout, fmt.Errorf("error setting fetcher timeout from context") + return conf, e("fetcher timeout") + } + conf.FetchTimeout, err = time.ParseDuration(fetchTimeoutVal) + if err != nil { + return conf, e("fetcher timeout", err) } } - if fetcherTimeoutVal == "" { - fetcherTimeoutVal = "30s" - } - fetcherTimeout, err = time.ParseDuration(fetcherTimeoutVal) - if err != nil { - return s, strict, fetcherTimeout, fmt.Errorf("error parsing fetcher timeout from context: %w", err) + if val := ctx.Value(strategyTimeoutKey{}); val != nil { + strategyTimeoutVal, ok := val.(string) + if !ok { + return conf, e("fetcher timeout") + } + conf.StrategyTimeout, err = time.ParseDuration(strategyTimeoutVal) + if err != nil { + return conf, e("fetcher timeout", err) + } } - return s, strict, fetcherTimeout, nil -} - -// SetFetchTimeout sets the timeout for each fetch -func SetFetchTimeout(ctx context.Context, timeout string) context.Context { - return context.WithValue(ctx, fetcherTimeoutKey{}, timeout) + return conf, nil } // SetStrategy sets the strategy for the retrieval @@ -76,14 +105,27 @@ func SetStrict(ctx context.Context, strict bool) context.Context { return context.WithValue(ctx, modeKey{}, strict) } -func SetParamsInContext(ctx context.Context, s Strategy, fallbackmode bool, fetchTimeout string) context.Context { +// SetFetchTimeout sets the timeout for each fetch +func SetFetchTimeout(ctx context.Context, timeout string) context.Context { + return context.WithValue(ctx, fetchTimeoutKey{}, timeout) +} + +// SetStrategyTimeout sets the timeout for each strategy +func SetStrategyTimeout(ctx context.Context, timeout string) context.Context { + return context.WithValue(ctx, fetchTimeoutKey{}, timeout) +} + +// SetConfigInContext sets the config params in the context +func SetConfigInContext(ctx context.Context, s Strategy, fallbackmode bool, fetchTimeout, strategyTimeout string) context.Context { ctx = SetStrategy(ctx, s) ctx = SetStrict(ctx, !fallbackmode) ctx = SetFetchTimeout(ctx, fetchTimeout) + ctx = SetStrategyTimeout(ctx, strategyTimeout) return ctx } -func (g *decoder) prefetch(ctx context.Context, strategy int, strict bool, strategyTimeout time.Duration) { - if strict && strategy == NONE { + +func (g *decoder) prefetch(ctx context.Context) { + if g.config.Strict && g.config.Strategy == NONE { return } defer g.remove() @@ -101,7 +143,7 @@ func (g *decoder) prefetch(ctx context.Context, strategy int, strict bool, strat var stop <-chan time.Time if s < RACE { - timer := time.NewTimer(strategyTimeout) + timer := time.NewTimer(g.config.StrategyTimeout) defer timer.Stop() stop = timer.C } @@ -123,8 +165,15 @@ func (g *decoder) prefetch(ctx context.Context, strategy int, strict bool, strat return g.recover(ctx) // context to cancel when shardCnt chunks are retrieved } var err error - for s := strategy; s == strategy || (err != nil && !strict && s < strategyCnt); s++ { + for s := g.config.Strategy; s < strategyCnt; s++ { err = run(s) + if g.config.Strict || err == nil { + break + } + } + + if err != nil { + g.err = err } } From 2821ff57ce44e1ab527b3b4ba17fd3f6b98dabcb Mon Sep 17 00:00:00 2001 From: zelig Date: Tue, 23 Jan 2024 05:32:56 +0100 Subject: [PATCH 44/52] fix: simplify error --- pkg/file/redundancy/getter/strategies.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/file/redundancy/getter/strategies.go b/pkg/file/redundancy/getter/strategies.go index 3b800d85011..dc16be174ac 100644 --- a/pkg/file/redundancy/getter/strategies.go +++ b/pkg/file/redundancy/getter/strategies.go @@ -138,7 +138,7 @@ func (g *decoder) prefetch(ctx context.Context) { defer cancelAll() run := func(s Strategy) error { if s == PROX { // NOT IMPLEMENTED - return fmt.Errorf("strategy %d not implemented", s) + return errors.New("strategy not implemented") } var stop <-chan time.Time @@ -172,9 +172,7 @@ func (g *decoder) prefetch(ctx context.Context) { } } - if err != nil { - g.err = err - } + g.err = err } // prefetch launches the retrieval of chunks based on the strategy From 606f2a879782835b33b83cd05a8c7f27a816d22e Mon Sep 17 00:00:00 2001 From: zelig Date: Tue, 23 Jan 2024 17:04:26 +0100 Subject: [PATCH 45/52] test: add race detection detection --- pkg/util/testutil/racedetection/off.go | 8 ++++++++ pkg/util/testutil/racedetection/on.go | 6 ++++++ pkg/util/testutil/racedetection/race.go | 5 +++++ 3 files changed, 19 insertions(+) create mode 100644 pkg/util/testutil/racedetection/off.go create mode 100644 pkg/util/testutil/racedetection/on.go create mode 100644 pkg/util/testutil/racedetection/race.go diff --git a/pkg/util/testutil/racedetection/off.go b/pkg/util/testutil/racedetection/off.go new file mode 100644 index 00000000000..73620f67f3f --- /dev/null +++ b/pkg/util/testutil/racedetection/off.go @@ -0,0 +1,8 @@ +//go:build !race +// +build !race + +package racedetection + +const On = false + + diff --git a/pkg/util/testutil/racedetection/on.go b/pkg/util/testutil/racedetection/on.go new file mode 100644 index 00000000000..4a304ad0adf --- /dev/null +++ b/pkg/util/testutil/racedetection/on.go @@ -0,0 +1,6 @@ +//go:build race +// +build race + +package racedetection + +const On = true diff --git a/pkg/util/testutil/racedetection/race.go b/pkg/util/testutil/racedetection/race.go new file mode 100644 index 00000000000..14baac2349f --- /dev/null +++ b/pkg/util/testutil/racedetection/race.go @@ -0,0 +1,5 @@ +package racedetection + +func IsOn() bool { + return On +} From 6d4f2cbc7bbd2b4c3b8b8ad8438e43eefb146a4a Mon Sep 17 00:00:00 2001 From: zelig Date: Tue, 23 Jan 2024 17:05:01 +0100 Subject: [PATCH 46/52] fix: contexts --- pkg/file/joiner/joiner_test.go | 4 ++++ pkg/file/redundancy/getter/getter.go | 14 ++------------ pkg/file/redundancy/getter/strategies.go | 6 +++--- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/pkg/file/joiner/joiner_test.go b/pkg/file/joiner/joiner_test.go index 154a5978dd0..21a674f10b5 100644 --- a/pkg/file/joiner/joiner_test.go +++ b/pkg/file/joiner/joiner_test.go @@ -31,6 +31,7 @@ import ( "github.com/ethersphere/bee/pkg/swarm" "github.com/ethersphere/bee/pkg/util/testutil" "github.com/ethersphere/bee/pkg/util/testutil/pseudorand" + "github.com/ethersphere/bee/pkg/util/testutil/racedetection" "gitlab.com/nolash/go-mockbytes" "golang.org/x/sync/errgroup" ) @@ -1242,6 +1243,9 @@ func TestJoinerRedundancyMultilevel(t *testing.T) { canReadRange := func(t *testing.T, s getter.Strategy, fallback bool, levels int, canRead bool) { ctx := context.Background() strategyTimeout := 100 * time.Millisecond + if racedetection.IsOn() { + strategyTimeout *= 2 + } ctx = getter.SetConfigInContext(ctx, s, fallback, (2 * strategyTimeout).String(), strategyTimeout.String()) ctx, cancel := context.WithTimeout(ctx, time.Duration(levels*3+1)*strategyTimeout) defer cancel() diff --git a/pkg/file/redundancy/getter/getter.go b/pkg/file/redundancy/getter/getter.go index 9f64674ed28..4e8da1b6390 100644 --- a/pkg/file/redundancy/getter/getter.go +++ b/pkg/file/redundancy/getter/getter.go @@ -9,7 +9,6 @@ import ( "io" "sync" "sync/atomic" - "time" "github.com/ethersphere/bee/pkg/storage" "github.com/ethersphere/bee/pkg/swarm" @@ -76,7 +75,7 @@ func New(addrs []swarm.Address, shardCnt int, g storage.Getter, p storage.Putter if !conf.Strict || conf.Strategy != NONE { rsg.wg.Add(1) go func() { - rsg.prefetch(ctx) + rsg.err = rsg.prefetch(ctx) rsg.wg.Done() }() } @@ -100,16 +99,7 @@ func (g *decoder) Get(ctx context.Context, addr swarm.Address) (swarm.Chunk, err select { case <-g.waits[i]: case <-ctx.Done(): - select { - case <-g.ready: - select { - case <-g.waits[i]: - case <-time.After(1 * time.Second): - return nil, ctx.Err() - } - default: - return nil, ctx.Err() - } + return nil, ctx.Err() } return swarm.NewChunk(addr, g.getData(i)), nil } diff --git a/pkg/file/redundancy/getter/strategies.go b/pkg/file/redundancy/getter/strategies.go index dc16be174ac..bb5188e9cc2 100644 --- a/pkg/file/redundancy/getter/strategies.go +++ b/pkg/file/redundancy/getter/strategies.go @@ -124,9 +124,9 @@ func SetConfigInContext(ctx context.Context, s Strategy, fallbackmode bool, fetc return ctx } -func (g *decoder) prefetch(ctx context.Context) { +func (g *decoder) prefetch(ctx context.Context) error { if g.config.Strict && g.config.Strategy == NONE { - return + return nil } defer g.remove() var cancels []func() @@ -172,7 +172,7 @@ func (g *decoder) prefetch(ctx context.Context) { } } - g.err = err + return err } // prefetch launches the retrieval of chunks based on the strategy From 3dc5df91aaf40fa761281b281cc5e590a2e05f38 Mon Sep 17 00:00:00 2001 From: zelig Date: Wed, 24 Jan 2024 12:01:02 +0100 Subject: [PATCH 47/52] test: fix timeout --- pkg/file/joiner/joiner_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/file/joiner/joiner_test.go b/pkg/file/joiner/joiner_test.go index 21a674f10b5..c76406495a5 100644 --- a/pkg/file/joiner/joiner_test.go +++ b/pkg/file/joiner/joiner_test.go @@ -1247,7 +1247,7 @@ func TestJoinerRedundancyMultilevel(t *testing.T) { strategyTimeout *= 2 } ctx = getter.SetConfigInContext(ctx, s, fallback, (2 * strategyTimeout).String(), strategyTimeout.String()) - ctx, cancel := context.WithTimeout(ctx, time.Duration(levels*3+1)*strategyTimeout) + ctx, cancel := context.WithTimeout(ctx, time.Duration(levels*3+2)*strategyTimeout) defer cancel() j, _, err := joiner.New(ctx, store, store, addr) if err != nil { From 5dc180253e85770b4f4fb94d76b226e035895544 Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 25 Jan 2024 07:16:56 +0100 Subject: [PATCH 48/52] test: increase timeout --- pkg/file/redundancy/getter/getter_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/file/redundancy/getter/getter_test.go b/pkg/file/redundancy/getter/getter_test.go index ce0378fc68a..d24164ea8ed 100644 --- a/pkg/file/redundancy/getter/getter_test.go +++ b/pkg/file/redundancy/getter/getter_test.go @@ -23,6 +23,7 @@ import ( inmem "github.com/ethersphere/bee/pkg/storage/inmemchunkstore" mockstorer "github.com/ethersphere/bee/pkg/storer/mock" "github.com/ethersphere/bee/pkg/swarm" + "github.com/ethersphere/bee/pkg/util/testutil/racedetection" "github.com/klauspost/reedsolomon" "golang.org/x/sync/errgroup" ) @@ -125,9 +126,13 @@ func testDecodingRACE(t *testing.T, bufSize, shardCnt, erasureCnt int) { q <- err }() err := context.DeadlineExceeded + wait := strategyTimeout * 2 + if racedetection.On { + wait = strategyTimeout * 3 + } select { case err = <-q: - case <-time.After(strategyTimeout * 2): + case <-time.After(wait): } switch { case erasureCnt > parityCnt: From a1eb8938cc9579ed1f2a97a5c08b7a0d8dac6a9c Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 25 Jan 2024 07:17:33 +0100 Subject: [PATCH 49/52] fix: file header added to new files --- pkg/util/testutil/racedetection/off.go | 6 ++++-- pkg/util/testutil/racedetection/on.go | 4 ++++ pkg/util/testutil/racedetection/race.go | 4 ++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/pkg/util/testutil/racedetection/off.go b/pkg/util/testutil/racedetection/off.go index 73620f67f3f..d57125bfd03 100644 --- a/pkg/util/testutil/racedetection/off.go +++ b/pkg/util/testutil/racedetection/off.go @@ -1,8 +1,10 @@ +// Copyright 2020 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + //go:build !race // +build !race package racedetection const On = false - - diff --git a/pkg/util/testutil/racedetection/on.go b/pkg/util/testutil/racedetection/on.go index 4a304ad0adf..92438ecfdc3 100644 --- a/pkg/util/testutil/racedetection/on.go +++ b/pkg/util/testutil/racedetection/on.go @@ -1,3 +1,7 @@ +// Copyright 2020 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + //go:build race // +build race diff --git a/pkg/util/testutil/racedetection/race.go b/pkg/util/testutil/racedetection/race.go index 14baac2349f..411295e87f6 100644 --- a/pkg/util/testutil/racedetection/race.go +++ b/pkg/util/testutil/racedetection/race.go @@ -1,3 +1,7 @@ +// Copyright 2020 The Swarm Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package racedetection func IsOn() bool { From 3a47dc6b884e56dcc7b4229a77978cbd196a2cca Mon Sep 17 00:00:00 2001 From: zelig Date: Thu, 25 Jan 2024 13:36:19 +0100 Subject: [PATCH 50/52] fix(joiner): reintroduces langos restrict no change in behaviour, lookahead via header --- go.mod | 1 + go.sum | 2 ++ pkg/api/api.go | 1 + pkg/api/bzz.go | 35 +++++++++++++++++++++++++++++++++-- pkg/api/bzz_test.go | 1 + 5 files changed, 38 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 3e7bf2f4e3d..5f38b82e566 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/ethersphere/go-price-oracle-abi v0.1.0 github.com/ethersphere/go-storage-incentives-abi v0.6.0 github.com/ethersphere/go-sw3-abi v0.4.0 + github.com/ethersphere/langos v1.0.0 github.com/go-playground/validator/v10 v10.11.1 github.com/gogo/protobuf v1.3.2 github.com/google/go-cmp v0.5.9 diff --git a/go.sum b/go.sum index c08d1ef71bb..0afd81c3918 100644 --- a/go.sum +++ b/go.sum @@ -244,6 +244,8 @@ github.com/ethersphere/go-storage-incentives-abi v0.6.0 h1:lfGViU/wJg/CyXlntNvTQ github.com/ethersphere/go-storage-incentives-abi v0.6.0/go.mod h1:SXvJVtM4sEsaSKD0jc1ClpDLw8ErPoROZDme4Wrc/Nc= github.com/ethersphere/go-sw3-abi v0.4.0 h1:T3ANY+ktWrPAwe2U0tZi+DILpkHzto5ym/XwV/Bbz8g= github.com/ethersphere/go-sw3-abi v0.4.0/go.mod h1:BmpsvJ8idQZdYEtWnvxA8POYQ8Rl/NhyCdF0zLMOOJU= +github.com/ethersphere/langos v1.0.0 h1:NBtNKzXTTRSue95uOlzPN4py7Aofs0xWPzyj4AI1Vcc= +github.com/ethersphere/langos v1.0.0/go.mod h1:dlcN2j4O8sQ+BlCaxeBu43bgr4RQ+inJ+pHwLeZg5Tw= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= diff --git a/pkg/api/api.go b/pkg/api/api.go index 19b0e718e70..470174c68d4 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -84,6 +84,7 @@ const ( SwarmRedundancyStrategyHeader = "Swarm-Redundancy-Strategy" SwarmRedundancyFallbackModeHeader = "Swarm-Redundancy-Fallback-Mode" SwarmChunkRetrievalTimeoutHeader = "Swarm-Chunk-Retrieval-Timeout" + SwarmLookAheadBufferSizeHeader = "Swarm-Lookahead-Buffer-Size" ImmutableHeader = "Immutable" GasPriceHeader = "Gas-Price" diff --git a/pkg/api/bzz.go b/pkg/api/bzz.go index fc5b5ec56be..f94de54ae4b 100644 --- a/pkg/api/bzz.go +++ b/pkg/api/bzz.go @@ -31,9 +31,29 @@ import ( "github.com/ethersphere/bee/pkg/swarm" "github.com/ethersphere/bee/pkg/topology" "github.com/ethersphere/bee/pkg/tracing" + "github.com/ethersphere/langos" "github.com/gorilla/mux" ) +// The size of buffer used for prefetching content with Langos when not using erasure coding +// Warning: This value influences the number of chunk requests and chunker join goroutines +// per file request. +// Recommended value is 8 or 16 times the io.Copy default buffer value which is 32kB, depending +// on the file size. Use lookaheadBufferSize() to get the correct buffer size for the request. +const ( + smallFileBufferSize = 8 * 32 * 1024 + largeFileBufferSize = 16 * 32 * 1024 + + largeBufferFilesizeThreshold = 10 * 1000000 // ten megs +) + +func lookaheadBufferSize(size int64) int { + if size <= largeBufferFilesizeThreshold { + return smallFileBufferSize + } + return largeFileBufferSize +} + func (s *Service) bzzUploadHandler(w http.ResponseWriter, r *http.Request) { logger := tracing.NewLoggerWithTraceID(r.Context(), s.logger.WithName("post_bzz").Build()) @@ -275,6 +295,7 @@ func (s *Service) serveReference(logger log.Logger, address swarm.Address, pathV Strategy getter.Strategy `map:"Swarm-Redundancy-Strategy"` FallbackMode bool `map:"Swarm-Redundancy-Fallback-Mode"` ChunkRetrievalTimeout string `map:"Swarm-Chunk-Retrieval-Timeout"` + LookaheadBufferSize string `map:"Swarm-Lookahead-Buffer-Size"` }{} if response := s.mapStructure(r.Header, &headers); response != nil { @@ -464,6 +485,7 @@ func (s *Service) downloadHandler(logger log.Logger, w http.ResponseWriter, r *h Strategy getter.Strategy `map:"Swarm-Redundancy-Strategy"` FallbackMode bool `map:"Swarm-Redundancy-Fallback-Mode"` ChunkRetrievalTimeout string `map:"Swarm-Chunk-Retrieval-Timeout"` + LookaheadBufferSize *string `map:"Swarm-Lookahead-Buffer-Size"` }{} if response := s.mapStructure(r.Header, &headers); response != nil { @@ -500,9 +522,18 @@ func (s *Service) downloadHandler(logger log.Logger, w http.ResponseWriter, r *h } w.Header().Set(ContentLengthHeader, strconv.FormatInt(l, 10)) w.Header().Set("Access-Control-Expose-Headers", ContentDispositionHeader) + bufSize := int64(lookaheadBufferSize(l)) + if headers.LookaheadBufferSize != nil { + bufSize, err = strconv.ParseInt(*headers.LookaheadBufferSize, 10, 64) + if err != nil { + logger.Debug("parsing lookahead buffer size", "error", err) + bufSize = 0 + } + } + if bufSize > 0 { + http.ServeContent(w, r, "", time.Now(), langos.NewBufferedLangos(reader, int(bufSize))) + } http.ServeContent(w, r, "", time.Now(), reader) - // NOTE: temporary workaround for testing, watch this... - // http.ServeContent(w, r, "", time.Now(), langos.NewBufferedLangos(reader, lookaheadBufferSize(l))) } // manifestMetadataLoad returns the value for a key stored in the metadata of diff --git a/pkg/api/bzz_test.go b/pkg/api/bzz_test.go index edf90a78c48..634d4eb0166 100644 --- a/pkg/api/bzz_test.go +++ b/pkg/api/bzz_test.go @@ -126,6 +126,7 @@ func TestBzzUploadDownloadWithRedundancy(t *testing.T) { fileDownloadResource(refResponse.Reference.String()), http.StatusPartialContent, jsonhttptest.WithRequestHeader(api.RangeHeader, rangeHeader), + jsonhttptest.WithRequestHeader(api.SwarmLookAheadBufferSizeHeader, "0"), // set for the replicas so that no replica gets deleted jsonhttptest.WithRequestHeader(api.SwarmRedundancyLevelHeader, "0"), jsonhttptest.WithRequestHeader(api.SwarmRedundancyStrategyHeader, "0"), From ec66cc47d5996c4fbdec60c2544e7ccb209b5d3a Mon Sep 17 00:00:00 2001 From: zelig Date: Sat, 3 Feb 2024 16:58:07 +0100 Subject: [PATCH 51/52] fix(api): return after Serve --- pkg/api/bzz.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/api/bzz.go b/pkg/api/bzz.go index f94de54ae4b..67a80a58918 100644 --- a/pkg/api/bzz.go +++ b/pkg/api/bzz.go @@ -295,7 +295,7 @@ func (s *Service) serveReference(logger log.Logger, address swarm.Address, pathV Strategy getter.Strategy `map:"Swarm-Redundancy-Strategy"` FallbackMode bool `map:"Swarm-Redundancy-Fallback-Mode"` ChunkRetrievalTimeout string `map:"Swarm-Chunk-Retrieval-Timeout"` - LookaheadBufferSize string `map:"Swarm-Lookahead-Buffer-Size"` + LookaheadBufferSize *string `map:"Swarm-Lookahead-Buffer-Size"` }{} if response := s.mapStructure(r.Header, &headers); response != nil { @@ -532,6 +532,7 @@ func (s *Service) downloadHandler(logger log.Logger, w http.ResponseWriter, r *h } if bufSize > 0 { http.ServeContent(w, r, "", time.Now(), langos.NewBufferedLangos(reader, int(bufSize))) + return } http.ServeContent(w, r, "", time.Now(), reader) } From 969a1343eb75dc7682d8112c8bc308da1baec716 Mon Sep 17 00:00:00 2001 From: zelig Date: Sat, 3 Feb 2024 22:45:23 +0100 Subject: [PATCH 52/52] test(file): fix timeouts --- pkg/file/joiner/joiner_test.go | 5 +++-- pkg/file/redundancy/getter/getter_test.go | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/file/joiner/joiner_test.go b/pkg/file/joiner/joiner_test.go index c76406495a5..15d46bf220b 100644 --- a/pkg/file/joiner/joiner_test.go +++ b/pkg/file/joiner/joiner_test.go @@ -1243,11 +1243,12 @@ func TestJoinerRedundancyMultilevel(t *testing.T) { canReadRange := func(t *testing.T, s getter.Strategy, fallback bool, levels int, canRead bool) { ctx := context.Background() strategyTimeout := 100 * time.Millisecond + decodingTimeout := 600 * time.Millisecond if racedetection.IsOn() { - strategyTimeout *= 2 + decodingTimeout *= 2 } ctx = getter.SetConfigInContext(ctx, s, fallback, (2 * strategyTimeout).String(), strategyTimeout.String()) - ctx, cancel := context.WithTimeout(ctx, time.Duration(levels*3+2)*strategyTimeout) + ctx, cancel := context.WithTimeout(ctx, time.Duration(levels)*(3*strategyTimeout+decodingTimeout)) defer cancel() j, _, err := joiner.New(ctx, store, store, addr) if err != nil { diff --git a/pkg/file/redundancy/getter/getter_test.go b/pkg/file/redundancy/getter/getter_test.go index d24164ea8ed..b18caa55c12 100644 --- a/pkg/file/redundancy/getter/getter_test.go +++ b/pkg/file/redundancy/getter/getter_test.go @@ -95,6 +95,9 @@ func TestGetterFallback(t *testing.T) { func testDecodingRACE(t *testing.T, bufSize, shardCnt, erasureCnt int) { t.Helper() strategyTimeout := 100 * time.Millisecond + if racedetection.On { + strategyTimeout *= 2 + } store := inmem.New() buf := make([][]byte, bufSize) addrs := initData(t, buf, shardCnt, store) @@ -128,7 +131,7 @@ func testDecodingRACE(t *testing.T, bufSize, shardCnt, erasureCnt int) { err := context.DeadlineExceeded wait := strategyTimeout * 2 if racedetection.On { - wait = strategyTimeout * 3 + wait *= 2 } select { case err = <-q: