bump versions, wevert chunked download on wasm

cmd/dfs/cmd/dev.go

@@ -2,6 +2,8 @@ package cmd
 
 import (
 	"context"
+	"crypto/rand"
+	"encoding/hex"
 	"fmt"
 	"os"
 	"os/signal"
@@ -72,6 +74,8 @@ func startDevServer() {
 	}()
 	done := make(chan os.Signal, 1)
 	signal.Notify(done, syscall.SIGINT, syscall.SIGTERM)
-
+	batchOk := make([]byte, 32)
+	_, _ = rand.Read(batchOk)
+	fmt.Println(hex.EncodeToString(batchOk))
 	<-done
 }

go.mod

@@ -1,13 +1,15 @@
 module github.com/fairdatasociety/fairOS-dfs
 
-go 1.21
+go 1.22
+
+toolchain go1.22.0
 
 require (
 	github.com/btcsuite/btcd/btcec/v2 v2.3.3
 	github.com/c-bata/go-prompt v0.2.6
 	github.com/dustin/go-humanize v1.0.1
-	github.com/ethereum/go-ethereum v1.14.3
-	github.com/ethersphere/bee/v2 v2.0.1
+	github.com/ethereum/go-ethereum v1.14.7
+	github.com/ethersphere/bee/v2 v2.1.1-0.20240611213428-4b04c086d8db
 	github.com/ethersphere/bmt v0.1.4
 	github.com/golang-jwt/jwt/v5 v5.2.1
 	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb
@@ -31,33 +33,31 @@ require (
 	github.com/tyler-smith/go-bip39 v1.1.0
 	github.com/wealdtech/go-ens/v3 v3.6.0
 	go.uber.org/goleak v1.3.0
-	golang.org/x/crypto v0.23.0
-	golang.org/x/term v0.20.0
+	golang.org/x/crypto v0.25.0
+	golang.org/x/term v0.22.0
 	gopkg.in/yaml.v2 v2.4.0
 	resenje.org/jsonhttp v0.2.3
 )
 
 require (
-	github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible // indirect
 	github.com/KyleBanks/depth v1.2.1 // indirect
-	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/armon/go-radix v1.0.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bits-and-blooms/bitset v1.10.0 // indirect
 	github.com/btcsuite/btcd v0.22.3 // indirect
 	github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 // indirect
 	github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce // indirect
-	github.com/casbin/casbin/v2 v2.35.0 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/consensys/bavard v0.1.13 // indirect
 	github.com/consensys/gnark-crypto v0.12.1 // indirect
 	github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/deckarep/golang-set/v2 v2.1.0 // indirect
+	github.com/deckarep/golang-set/v2 v2.6.0 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
 	github.com/ethereum/c-kzg-4844 v1.0.0 // indirect
 	github.com/ethersphere/go-price-oracle-abi v0.2.0 // indirect
-	github.com/ethersphere/go-storage-incentives-abi v0.6.2 // indirect
+	github.com/ethersphere/go-storage-incentives-abi v0.8.6 // indirect
 	github.com/ethersphere/go-sw3-abi v0.6.5 // indirect
 	github.com/ethersphere/langos v1.0.0 // indirect
 	github.com/felixge/httpsnoop v1.0.3 // indirect
@@ -76,7 +76,7 @@ require (
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/holiman/uint256 v1.2.4 // indirect
+	github.com/holiman/uint256 v1.3.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/ipfs/go-cid v0.4.1 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
@@ -143,11 +143,11 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/exp v0.0.0-20240213143201-ec583247a57a // indirect
 	golang.org/x/mod v0.17.0 // indirect
-	golang.org/x/net v0.24.0 // indirect
+	golang.org/x/net v0.25.0 // indirect
 	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/sys v0.20.0 // indirect
-	golang.org/x/text v0.15.0 // indirect
-	golang.org/x/tools v0.20.0 // indirect
+	golang.org/x/sys v0.22.0 // indirect
+	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
@@ -158,4 +158,6 @@ require (
 	rsc.io/tmplfunc v0.0.3 // indirect
 )
 
+//replace github.com/ethersphere/bee/v2 => ../../ethersphere/bee
+
 replace github.com/codahale/hdrhistogram => github.com/HdrHistogram/hdrhistogram-go v0.0.0-20200919145931-8dac23c8dac1

pkg/blockstore/bee/client.go

@@ -49,6 +49,7 @@ const (
 	bzzUrl                    = "/bzz"
 	tagsUrl                   = "/tags"
 	pinsUrl                   = "/pins/"
+	feedsUrl                  = "/feeds/"
 	swarmPinHeader            = "Swarm-Pin"
 	swarmEncryptHeader        = "Swarm-Encrypt"
 	swarmPostageBatchId       = "Swarm-Postage-Batch-Id"
@@ -114,6 +115,7 @@ func NewBeeClient(apiUrl, postageBlockId string, shouldPin bool, redundancyLevel
 		postageBlockId:     postageBlockId,
 		logger:             logger,
 		shouldPin:          shouldPin,
+		redundancyLevel:    redundancyLevel,
 	}
 }
 
@@ -189,7 +191,7 @@ func (s *Client) UploadSOC(owner, id, signature string, data []byte) (address []
 	// the postage block id to store the SOC chunk
 	req.Header.Set(swarmPostageBatchId, s.postageBlockId)
 	req.Header.Set(contentTypeHeader, "application/octet-stream")
-	req.Header.Set(swarmDeferredUploadHeader, "true")
+	req.Header.Set(swarmDeferredUploadHeader, "false")
 
 	// TODO change this in the future when we have some alternative to pin SOC
 	// This is a temporary fix to force soc pinning
@@ -661,6 +663,102 @@ func (s *Client) CreateTag(address []byte) (uint32, error) {
 	return resp.UID, nil
 }
 
+func (s *Client) CreateFeedManifest(owner, topic string) (swarm.Address, error) {
+	to := time.Now()
+
+	fullUrl := s.url + feedsUrl + owner + "/" + topic
+	fmt.Println("fullUrl: ", fullUrl)
+	req, err := http.NewRequest(http.MethodPost, fullUrl, nil)
+	if err != nil {
+		return swarm.ZeroAddress, err
+	}
+	req.Close = true
+
+	req.Header.Set(swarmPostageBatchId, s.postageBlockId)
+
+	response, err := s.Do(req)
+	if err != nil {
+		return swarm.ZeroAddress, err
+	}
+	// skipcq: GO-S2307
+	defer response.Body.Close()
+
+	respData, err := io.ReadAll(response.Body)
+	if err != nil {
+		return swarm.ZeroAddress, errors.New("error create feed manifest")
+	}
+
+	if response.StatusCode != http.StatusOK && response.StatusCode != http.StatusCreated {
+		var beeErr *beeError
+		err = json.Unmarshal(respData, &beeErr)
+		if err != nil {
+			return swarm.ZeroAddress, errors.New(string(respData))
+		}
+		return swarm.ZeroAddress, errors.New(beeErr.Message)
+	}
+
+	var resp bytesPostResponse
+	err = json.Unmarshal(respData, &resp)
+	if err != nil {
+		return swarm.ZeroAddress, fmt.Errorf("error unmarshalling response")
+	}
+	fields := logrus.Fields{
+		"owner":    owner,
+		"topic":    topic,
+		"duration": time.Since(to).String(),
+	}
+	s.logger.WithFields(fields).Log(logrus.DebugLevel, "create feed manifest: ")
+	return resp.Reference, nil
+}
+
+func (s *Client) GetLatestFeedManifest(owner, topic string) ([]byte, string, string, error) {
+	to := time.Now()
+
+	fullUrl := s.url + feedsUrl + owner + "/" + topic
+	fmt.Println("get ullUrl: ", fullUrl)
+
+	req, err := http.NewRequest(http.MethodGet, fullUrl, nil)
+	if err != nil {
+		return nil, "", "", err
+	}
+	req.Close = true
+
+	response, err := s.Do(req)
+	if err != nil {
+		return nil, "", "", err
+	}
+	// skipcq: GO-S2307
+	defer response.Body.Close()
+
+	respData, err := io.ReadAll(response.Body)
+	if err != nil {
+		return nil, "", "", errors.New("error getting latest feed manifest")
+	}
+
+	if response.StatusCode != http.StatusOK && response.StatusCode != http.StatusCreated {
+		var beeErr *beeError
+		err = json.Unmarshal(respData, &beeErr)
+		if err != nil {
+			return nil, "", "", errors.New(string(respData))
+		}
+		return nil, "", "", errors.New(beeErr.Message)
+	}
+
+	var resp bytesPostResponse
+	err = json.Unmarshal(respData, &resp)
+	if err != nil {
+		return nil, "", "", fmt.Errorf("error unmarshalling response")
+	}
+	fields := logrus.Fields{
+		"owner":    owner,
+		"topic":    topic,
+		"duration": time.Since(to).String(),
+	}
+	s.logger.WithFields(fields).Log(logrus.DebugLevel, "get latest feed manifest: ")
+
+	return resp.Reference.Bytes(), response.Header.Get("swarm-feed-index"), response.Header.Get("swarm-feed-index-next"), nil
+}
+
 // GetTag gets sync status of a given tag
 func (s *Client) GetTag(tag uint32) (int64, int64, int64, error) {
 	// gateway proxy does not have tags api exposed

pkg/blockstore/bee/mock/client.go

@@ -17,8 +17,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	accountingmock "github.com/ethersphere/bee/v2/pkg/accounting/mock"
 	"github.com/ethersphere/bee/v2/pkg/api"
-	"github.com/ethersphere/bee/v2/pkg/auth"
-	mockauth "github.com/ethersphere/bee/v2/pkg/auth/mock"
+
 	"github.com/ethersphere/bee/v2/pkg/crypto"
 	"github.com/ethersphere/bee/v2/pkg/feeds"
 	"github.com/ethersphere/bee/v2/pkg/log"
@@ -84,9 +83,6 @@ type TestServerOptions struct {
 	Post               postage.Service
 	Steward            steward.Interface
 	WsHeaders          http.Header
-	Authenticator      auth.Authenticator
-	DebugAPI           bool
-	Restricted         bool
 	DirectUpload       bool
 	Probe              *api.Probe
 
@@ -136,13 +132,6 @@ func NewTestBeeServer(t *testing.T, o TestServerOptions) string {
 	if o.SyncStatus == nil {
 		o.SyncStatus = func() (bool, error) { return true, nil }
 	}
-	if o.Authenticator == nil {
-		o.Authenticator = &mockauth.Auth{
-			EnforceFunc: func(_, _, _ string) (bool, error) {
-				return true, nil
-			},
-		}
-	}
 
 	topologyDriver := topologymock.NewTopologyDriver(o.TopologyOpts...)
 	acc := accountingmock.NewAccounting(o.AccountingOpts...)
@@ -206,18 +195,14 @@ func NewTestBeeServer(t *testing.T, o TestServerOptions) string {
 	})
 	testutil.CleanupCloser(t, tracerCloser)
 
-	s.Configure(signer, o.Authenticator, noOpTracer, api.Options{
+	s.Configure(signer, noOpTracer, api.Options{
 		CORSAllowedOrigins: o.CORSAllowedOrigins,
 		WsPingPeriod:       o.WsPingPeriod,
-		Restricted:         o.Restricted,
 	}, extraOpts, 1, erc20)
 
-	if o.DebugAPI {
-		s.MountTechnicalDebug()
-		s.MountDebug(false)
-	} else {
-		s.MountAPI()
-	}
+	s.MountTechnicalDebug()
+	s.MountDebug()
+	s.MountAPI()
 
 	ts := httptest.NewServer(s)
 	t.Cleanup(ts.Close)

pkg/blockstore/client.go

@@ -35,4 +35,6 @@ type Client interface {
 	DeleteReference(address []byte) error
 	CreateTag(address []byte) (uint32, error)
 	GetTag(tag uint32) (int64, int64, int64, error)
+	CreateFeedManifest(owner, topic string) (address swarm.Address, err error)
+	GetLatestFeedManifest(owner, topic string) (address []byte, index, nextIndex string, err error)
 }

pkg/blockstore/putergetter/bzzUpdate.go

@@ -0,0 +1 @@
+package putergetter