Merge branch 'master' of https://github.com/ethersphere/beekeeper int…

…o golang-1.21

.github/workflows/release.yaml

@@ -11,19 +11,18 @@ jobs:
   goreleaser:
     runs-on: ubuntu-latest
     steps:
-      - name: Setup Go
-        uses: actions/setup-go@v1
-        with:
-          go-version: 1.21
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
+      - name: Setup Go
+        uses: actions/setup-go@v3
+        with:
+          go-version-file: go.mod
       - name: Run GoReleaser
-        uses: goreleaser/goreleaser-action@v1
+        uses: goreleaser/goreleaser-action@v4
         with:
           version: latest
-          args: release --clean
-          key: ${{ secrets.YOUR_PRIVATE_KEY }}
+          args: release --clean --timeout 1h
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

README.md

@@ -308,20 +308,20 @@ Command **node-funder** uses https://github.com/ethersphere/node-funder tool to
 
 It has following flags:
 ```
---addresses strings            Comma-separated list of Bee node addresses (must start with 0x). Overrides namespace and cluster name.
---chain-node-endpoint string   Endpoint to chain node. Required.
---cluster-name string          Cluster name. Ignored if addresses or namespace are set.
---help                         help for node-funder
---min-native float             Minimum amount of chain native coins (xDAI) nodes should have.
---min-swarm float              Minimum amount of swarm tokens (xBZZ) nodes should have.
---namespace string             Kubernetes namespace. Overrides cluster name if set.
---timeout duration             Timeout. (default 5m0s)
---wallet-key string            Hex-encoded private key for the Bee node wallet. Required.
+--addresses strings       Comma-separated list of Bee node addresses (must start with 0x). Overrides namespace and cluster name.
+--geth-url string         Endpoint to chain node. Required.
+--cluster-name string     Cluster name. Ignored if addresses or namespace are set.
+--help                    help for node-funder
+--min-native float        Minimum amount of chain native coins (xDAI) nodes should have.
+--min-swarm float         Minimum amount of swarm tokens (xBZZ) nodes should have.
+--namespace string        Kubernetes namespace. Overrides cluster name if set.
+--timeout duration        Timeout. (default 5m0s)
+--wallet-key string       Hex-encoded private key for the Bee node wallet. Required.
 ```
 
 example:
 ```
-beekeeper node-funder --chain-node-endpoint="http://geth-swap.default.testnet.internal" --wallet-key="4663c222787e30c1994b59044aa5045377a6e79193a8ead88293926b535c722d" --namespace=default --min-swarm=180 --min-native=2.2 --log-verbosity=3
+beekeeper node-funder --geth-url="http://geth-swap.default.testnet.internal" --wallet-key="4663c222787e30c1994b59044aa5045377a6e79193a8ead88293926b535c722d" --namespace=default --min-swarm=180 --min-native=2.2 --log-verbosity=3
 ```
 
 # Global flags

cmd/beekeeper/cmd/node_funder.go

@@ -18,7 +18,7 @@ func (c *command) initNodeFunderCmd() (err error) {
 		optionNameAddresses         = "addresses"
 		optionNameNamespace         = "namespace"
 		optionClusterName           = "cluster-name"
-		optionNameChainNodeEndpoint = "chain-node-endpoint"
+		optionNameChainNodeEndpoint = "geth-url"
 		optionNameWalletKey         = "wallet-key"
 		optionNameMinNative         = "min-native"
 		optionNameMinSwarm          = "min-swarm"
@@ -55,7 +55,7 @@ func (c *command) initNodeFunderCmd() (err error) {
 
 			// chain node endpoint check
 			if cfg.ChainNodeEndpoint = c.globalConfig.GetString(optionNameChainNodeEndpoint); cfg.ChainNodeEndpoint == "" {
-				return errors.New("chain node endpoint not provided")
+				return errors.New("chain node endpoint (geth-url) not provided")
 			}
 
 			// wallet key check

pkg/bee/client.go

@@ -418,10 +418,11 @@ func (c *Client) GetOrCreateBatch(ctx context.Context, amount int64, depth uint6
 		if !b.Exists {
 			continue
 		}
-		max := 1 << (b.Depth - b.BucketDepth)
-		hasFreeSlots := b.Utilization < uint32(max)
+		if b.ImmutableFlag { // skip immutable batches
+			continue
+		}
 
-		if b.Usable && (b.BatchTTL == -1 || b.BatchTTL > 0) && hasFreeSlots {
+		if b.Usable && (b.BatchTTL == -1 || b.BatchTTL > 0) {
 			return b.BatchID, nil
 		}
 	}

pkg/check/smoke/metrics.go

@@ -6,17 +6,36 @@ import (
 )
 
 type metrics struct {
-	UploadErrors     prometheus.Counter
-	UploadAttempts   prometheus.Counter
-	DownloadErrors   prometheus.Counter
-	DownloadMismatch prometheus.Counter
-	DownloadAttempts prometheus.Counter
-	UploadDuration   prometheus.Histogram
-	DownloadDuration prometheus.Histogram
+	BatchCreateErrors   prometheus.Counter
+	BatchCreateAttempts prometheus.Counter
+	UploadErrors        prometheus.Counter
+	UploadAttempts      prometheus.Counter
+	DownloadErrors      prometheus.Counter
+	DownloadMismatch    prometheus.Counter
+	DownloadAttempts    prometheus.Counter
+	UploadDuration      prometheus.Histogram
+	DownloadDuration    prometheus.Histogram
+	UploadSize          prometheus.Gauge
 }
 
 func newMetrics(subsystem string) metrics {
 	return metrics{
+		BatchCreateAttempts: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Namespace: m.Namespace,
+				Subsystem: subsystem,
+				Name:      "batch_create_attempts",
+				Help:      "Number of batch create attempts.",
+			},
+		),
+		BatchCreateErrors: prometheus.NewCounter(
+			prometheus.CounterOpts{
+				Namespace: m.Namespace,
+				Subsystem: subsystem,
+				Name:      "batch_create_errors",
+				Help:      "Total errors encountered while creating batches.",
+			},
+		),
 		UploadAttempts: prometheus.NewCounter(
 			prometheus.CounterOpts{
 				Namespace: m.Namespace,
@@ -73,6 +92,14 @@ func newMetrics(subsystem string) metrics {
 				Help:      "Data download duration through the /bytes endpoint.",
 			},
 		),
+		UploadSize: prometheus.NewGauge(
+			prometheus.GaugeOpts{
+				Namespace: m.Namespace,
+				Subsystem: subsystem,
+				Name:      "upload_size",
+				Help:      "Amount of data uploaded per upload.",
+			},
+		),
 	}
 }
 

pkg/check/smoke/smoke.go

@@ -33,7 +33,6 @@ type Options struct {
 	UploadGroups    []string
 	DownloaderCount int
 	DownloadGroups  []string
-	GasPrice        string
 	MaxUseBatch     time.Duration
 }
 
@@ -42,15 +41,14 @@ func NewDefaultOptions() Options {
 	return Options{
 		ContentSize:     5000000,
 		RndSeed:         time.Now().UnixNano(),
-		PostageAmount:   1000000,
-		PostageDepth:    20,
+		PostageAmount:   50_000_000,
+		PostageDepth:    24,
 		TxOnErrWait:     10 * time.Second,
 		RxOnErrWait:     10 * time.Second,
-		NodesSyncWait:   time.Second * 30,
+		NodesSyncWait:   time.Minute,
 		Duration:        12 * time.Hour,
-		UploadTimeout:   5 * time.Minute,
-		DownloadTimeout: 5 * time.Minute,
-		GasPrice:        "100000000000",
+		UploadTimeout:   60 * time.Minute,
+		DownloadTimeout: 60 * time.Minute,
 		MaxUseBatch:     12 * time.Hour,
 	}
 }
@@ -91,10 +89,10 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int
 
 	time.Sleep(5 * time.Second) // Wait for the nodes to warmup.
 
-	batches := NewStore(o.MaxUseBatch)
-
 	test := &test{clients: clients, logger: c.logger}
 
+	c.metrics.UploadSize.Set(float64(o.ContentSize))
+
 	for i := 0; true; i++ {
 		select {
 		case <-ctx.Done():
@@ -125,6 +123,8 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int
 			txData     []byte
 			rxData     []byte
 			address    swarm.Address
+			batchID    string
+			uploaded   bool
 		)
 
 		txData = make([]byte, o.ContentSize)
@@ -140,39 +140,39 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int
 		for retries := 0; retries < 3; retries++ {
 			txCancel()
 
+			uploaded = false
+
 			select {
 			case <-ctx.Done():
 				return nil
 			case <-time.After(o.TxOnErrWait):
 			}
 
-			c.metrics.UploadAttempts.Inc()
-
 			txCtx, txCancel = context.WithTimeout(ctx, o.UploadTimeout)
 
-			batchID := batches.Get(txName)
-			if batchID == "" {
-				batchID, err = clients[txName].CreatePostageBatch(txCtx, o.PostageAmount, o.PostageDepth, "load-test", true)
-				if err != nil {
-					c.logger.Errorf("create new batch: %v", err)
-					continue
-				}
-				batches.Store(txName, batchID)
+			c.metrics.BatchCreateAttempts.Inc()
+
+			batchID, err = clients[txName].GetOrCreateBatch(txCtx, o.PostageAmount, o.PostageDepth, "load-test")
+			if err != nil {
+				c.logger.Errorf("create new batch: %v", err)
+				c.metrics.BatchCreateErrors.Inc()
+				continue
 			}
 
+			c.metrics.UploadAttempts.Inc()
 			address, txDuration, err = test.upload(txCtx, txName, txData, batchID)
 			if err != nil {
 				c.metrics.UploadErrors.Inc()
 				c.logger.Infof("upload failed: %v", err)
 				c.logger.Infof("retrying in: %v", o.TxOnErrWait)
 			} else {
+				uploaded = true
 				break
 			}
 		}
 		txCancel()
-
-		if err != nil {
-			continue // skip
+		if !uploaded {
+			continue
 		}
 
 		c.metrics.UploadDuration.Observe(txDuration.Seconds())

pkg/config/check.go

@@ -362,7 +362,6 @@ var Checks = map[string]CheckType{
 				RxOnErrWait   *time.Duration `yaml:"rx-on-err-wait"`
 				NodesSyncWait *time.Duration `yaml:"nodes-sync-wait"`
 				Duration      *time.Duration `yaml:"duration"`
-				MaxUseBatch   *time.Duration `yaml:"max-use-batch"`
 			})
 			if err := check.Options.Decode(checkOpts); err != nil {
 				return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err)