From 8d70cb3a8b8aefa49d84dee930f47da7a6c930da Mon Sep 17 00:00:00 2001 From: istae <14264581+istae@users.noreply.github.com> Date: Tue, 3 Oct 2023 11:57:24 +0300 Subject: [PATCH 1/6] fix: create batch failures should not count toward upload error (#358) --- pkg/check/smoke/metrics.go | 32 +++++++++++++++++++++++++------- pkg/check/smoke/smoke.go | 5 +++-- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/pkg/check/smoke/metrics.go b/pkg/check/smoke/metrics.go index 554c07409..f46cdb0ac 100644 --- a/pkg/check/smoke/metrics.go +++ b/pkg/check/smoke/metrics.go @@ -6,17 +6,35 @@ import ( ) type metrics struct { - UploadErrors prometheus.Counter - UploadAttempts prometheus.Counter - DownloadErrors prometheus.Counter - DownloadMismatch prometheus.Counter - DownloadAttempts prometheus.Counter - UploadDuration prometheus.Histogram - DownloadDuration prometheus.Histogram + BatchCreateErrors prometheus.Counter + BatchCreateAttempts prometheus.Counter + UploadErrors prometheus.Counter + UploadAttempts prometheus.Counter + DownloadErrors prometheus.Counter + DownloadMismatch prometheus.Counter + DownloadAttempts prometheus.Counter + UploadDuration prometheus.Histogram + DownloadDuration prometheus.Histogram } func newMetrics(subsystem string) metrics { return metrics{ + BatchCreateAttempts: prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "batch_create_attempts", + Help: "Number of batch create attempts.", + }, + ), + BatchCreateErrors: prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "batch_create_errors", + Help: "Total errors encountered while creating batches.", + }, + ), UploadAttempts: prometheus.NewCounter( prometheus.CounterOpts{ Namespace: m.Namespace, diff --git a/pkg/check/smoke/smoke.go b/pkg/check/smoke/smoke.go index 0a586321c..bf8185948 100644 --- a/pkg/check/smoke/smoke.go +++ b/pkg/check/smoke/smoke.go @@ -146,20 +146,21 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int case <-time.After(o.TxOnErrWait): } - c.metrics.UploadAttempts.Inc() - txCtx, txCancel = context.WithTimeout(ctx, o.UploadTimeout) + c.metrics.BatchCreateAttempts.Inc() batchID := batches.Get(txName) if batchID == "" { batchID, err = clients[txName].CreatePostageBatch(txCtx, o.PostageAmount, o.PostageDepth, "load-test", true) if err != nil { c.logger.Errorf("create new batch: %v", err) + c.metrics.BatchCreateErrors.Inc() continue } batches.Store(txName, batchID) } + c.metrics.UploadAttempts.Inc() address, txDuration, err = test.upload(txCtx, txName, txData, batchID) if err != nil { c.metrics.UploadErrors.Inc() From 655c050fe63e72ebc5878d2c771fbc0a4784b56a Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic <35105035+gacevicljubisa@users.noreply.github.com> Date: Wed, 4 Oct 2023 09:56:24 +0200 Subject: [PATCH 2/6] chore: update release.yaml (#357) * chore: update release.yaml to use --clean instead of --rm-dist * ci: bump actions * chore(node-funder): rename chain-node-endpoint flag to geth-url use the same value from config * fix(node-funder): improve erorr msg for not provided geth-url --------- Co-authored-by: Ivan Vandot --- .github/workflows/release.yaml | 15 +++++++-------- README.md | 20 ++++++++++---------- cmd/beekeeper/cmd/node_funder.go | 4 ++-- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 73cf26bcf..ecb1259fc 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -11,19 +11,18 @@ jobs: goreleaser: runs-on: ubuntu-latest steps: - - name: Setup Go - uses: actions/setup-go@v1 - with: - go-version: 1.18 - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version-file: go.mod - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v1 + uses: goreleaser/goreleaser-action@v4 with: version: latest - args: release --rm-dist - key: ${{ secrets.YOUR_PRIVATE_KEY }} + args: release --clean --timeout 1h env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/README.md b/README.md index 107c22a65..544c9ff3e 100644 --- a/README.md +++ b/README.md @@ -308,20 +308,20 @@ Command **node-funder** uses https://github.com/ethersphere/node-funder tool to It has following flags: ``` ---addresses strings Comma-separated list of Bee node addresses (must start with 0x). Overrides namespace and cluster name. ---chain-node-endpoint string Endpoint to chain node. Required. ---cluster-name string Cluster name. Ignored if addresses or namespace are set. ---help help for node-funder ---min-native float Minimum amount of chain native coins (xDAI) nodes should have. ---min-swarm float Minimum amount of swarm tokens (xBZZ) nodes should have. ---namespace string Kubernetes namespace. Overrides cluster name if set. ---timeout duration Timeout. (default 5m0s) ---wallet-key string Hex-encoded private key for the Bee node wallet. Required. +--addresses strings Comma-separated list of Bee node addresses (must start with 0x). Overrides namespace and cluster name. +--geth-url string Endpoint to chain node. Required. +--cluster-name string Cluster name. Ignored if addresses or namespace are set. +--help help for node-funder +--min-native float Minimum amount of chain native coins (xDAI) nodes should have. +--min-swarm float Minimum amount of swarm tokens (xBZZ) nodes should have. +--namespace string Kubernetes namespace. Overrides cluster name if set. +--timeout duration Timeout. (default 5m0s) +--wallet-key string Hex-encoded private key for the Bee node wallet. Required. ``` example: ``` -beekeeper node-funder --chain-node-endpoint="http://geth-swap.default.testnet.internal" --wallet-key="4663c222787e30c1994b59044aa5045377a6e79193a8ead88293926b535c722d" --namespace=default --min-swarm=180 --min-native=2.2 --log-verbosity=3 +beekeeper node-funder --geth-url="http://geth-swap.default.testnet.internal" --wallet-key="4663c222787e30c1994b59044aa5045377a6e79193a8ead88293926b535c722d" --namespace=default --min-swarm=180 --min-native=2.2 --log-verbosity=3 ``` # Global flags diff --git a/cmd/beekeeper/cmd/node_funder.go b/cmd/beekeeper/cmd/node_funder.go index d914d8794..b7989f0ad 100644 --- a/cmd/beekeeper/cmd/node_funder.go +++ b/cmd/beekeeper/cmd/node_funder.go @@ -18,7 +18,7 @@ func (c *command) initNodeFunderCmd() (err error) { optionNameAddresses = "addresses" optionNameNamespace = "namespace" optionClusterName = "cluster-name" - optionNameChainNodeEndpoint = "chain-node-endpoint" + optionNameChainNodeEndpoint = "geth-url" optionNameWalletKey = "wallet-key" optionNameMinNative = "min-native" optionNameMinSwarm = "min-swarm" @@ -55,7 +55,7 @@ func (c *command) initNodeFunderCmd() (err error) { // chain node endpoint check if cfg.ChainNodeEndpoint = c.globalConfig.GetString(optionNameChainNodeEndpoint); cfg.ChainNodeEndpoint == "" { - return errors.New("chain node endpoint not provided") + return errors.New("chain node endpoint (geth-url) not provided") } // wallet key check From 5eead4586b17358579d2220eb4d961495ed35e71 Mon Sep 17 00:00:00 2001 From: istae <14264581+istae@users.noreply.github.com> Date: Tue, 10 Oct 2023 19:49:43 +0300 Subject: [PATCH 3/6] fix: use existing mutable batch (#360) --- pkg/bee/client.go | 7 ++++--- pkg/check/smoke/smoke.go | 23 ++++++++--------------- pkg/config/check.go | 1 - 3 files changed, 12 insertions(+), 19 deletions(-) diff --git a/pkg/bee/client.go b/pkg/bee/client.go index 85b697e7a..f0de2350e 100644 --- a/pkg/bee/client.go +++ b/pkg/bee/client.go @@ -418,10 +418,11 @@ func (c *Client) GetOrCreateBatch(ctx context.Context, amount int64, depth uint6 if !b.Exists { continue } - max := 1 << (b.Depth - b.BucketDepth) - hasFreeSlots := b.Utilization < uint32(max) + if b.ImmutableFlag { // skip immutable batches + continue + } - if b.Usable && (b.BatchTTL == -1 || b.BatchTTL > 0) && hasFreeSlots { + if b.Usable && (b.BatchTTL == -1 || b.BatchTTL > 0) { return b.BatchID, nil } } diff --git a/pkg/check/smoke/smoke.go b/pkg/check/smoke/smoke.go index bf8185948..09423731d 100644 --- a/pkg/check/smoke/smoke.go +++ b/pkg/check/smoke/smoke.go @@ -33,7 +33,6 @@ type Options struct { UploadGroups []string DownloaderCount int DownloadGroups []string - GasPrice string MaxUseBatch time.Duration } @@ -42,15 +41,14 @@ func NewDefaultOptions() Options { return Options{ ContentSize: 5000000, RndSeed: time.Now().UnixNano(), - PostageAmount: 1000000, - PostageDepth: 20, + PostageAmount: 50_000_000, + PostageDepth: 24, TxOnErrWait: 10 * time.Second, RxOnErrWait: 10 * time.Second, NodesSyncWait: time.Second * 30, Duration: 12 * time.Hour, UploadTimeout: 5 * time.Minute, DownloadTimeout: 5 * time.Minute, - GasPrice: "100000000000", MaxUseBatch: 12 * time.Hour, } } @@ -91,8 +89,6 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int time.Sleep(5 * time.Second) // Wait for the nodes to warmup. - batches := NewStore(o.MaxUseBatch) - test := &test{clients: clients, logger: c.logger} for i := 0; true; i++ { @@ -149,15 +145,12 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int txCtx, txCancel = context.WithTimeout(ctx, o.UploadTimeout) c.metrics.BatchCreateAttempts.Inc() - batchID := batches.Get(txName) - if batchID == "" { - batchID, err = clients[txName].CreatePostageBatch(txCtx, o.PostageAmount, o.PostageDepth, "load-test", true) - if err != nil { - c.logger.Errorf("create new batch: %v", err) - c.metrics.BatchCreateErrors.Inc() - continue - } - batches.Store(txName, batchID) + + batchID, err := clients[txName].GetOrCreateBatch(txCtx, o.PostageAmount, o.PostageDepth, "load-test") + if err != nil { + c.logger.Errorf("create new batch: %v", err) + c.metrics.BatchCreateErrors.Inc() + continue } c.metrics.UploadAttempts.Inc() diff --git a/pkg/config/check.go b/pkg/config/check.go index 8060126a8..490b2c0b2 100644 --- a/pkg/config/check.go +++ b/pkg/config/check.go @@ -362,7 +362,6 @@ var Checks = map[string]CheckType{ RxOnErrWait *time.Duration `yaml:"rx-on-err-wait"` NodesSyncWait *time.Duration `yaml:"nodes-sync-wait"` Duration *time.Duration `yaml:"duration"` - MaxUseBatch *time.Duration `yaml:"max-use-batch"` }) if err := check.Options.Decode(checkOpts); err != nil { return nil, fmt.Errorf("decoding check %s options: %w", check.Type, err) From 573453e10fd17712048edbd2e1c98180545a08ed Mon Sep 17 00:00:00 2001 From: istae <14264581+istae@users.noreply.github.com> Date: Wed, 11 Oct 2023 00:04:47 +0300 Subject: [PATCH 4/6] fix: smoke test --- pkg/check/smoke/smoke.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/check/smoke/smoke.go b/pkg/check/smoke/smoke.go index 09423731d..42332812a 100644 --- a/pkg/check/smoke/smoke.go +++ b/pkg/check/smoke/smoke.go @@ -121,6 +121,8 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int txData []byte rxData []byte address swarm.Address + batchID string + uploaded bool ) txData = make([]byte, o.ContentSize) @@ -136,6 +138,8 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int for retries := 0; retries < 3; retries++ { txCancel() + uploaded = false + select { case <-ctx.Done(): return nil @@ -146,7 +150,7 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int c.metrics.BatchCreateAttempts.Inc() - batchID, err := clients[txName].GetOrCreateBatch(txCtx, o.PostageAmount, o.PostageDepth, "load-test") + batchID, err = clients[txName].GetOrCreateBatch(txCtx, o.PostageAmount, o.PostageDepth, "load-test") if err != nil { c.logger.Errorf("create new batch: %v", err) c.metrics.BatchCreateErrors.Inc() @@ -160,13 +164,13 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int c.logger.Infof("upload failed: %v", err) c.logger.Infof("retrying in: %v", o.TxOnErrWait) } else { + uploaded = true break } } txCancel() - - if err != nil { - continue // skip + if !uploaded { + continue } c.metrics.UploadDuration.Observe(txDuration.Seconds()) From 40d37ca1fbeb102476bc9def900c264b0ecb1985 Mon Sep 17 00:00:00 2001 From: istae <14264581+istae@users.noreply.github.com> Date: Wed, 11 Oct 2023 13:54:13 +0300 Subject: [PATCH 5/6] fix: adjusted smoke timeouts --- pkg/check/smoke/smoke.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/check/smoke/smoke.go b/pkg/check/smoke/smoke.go index 42332812a..c92d00aa0 100644 --- a/pkg/check/smoke/smoke.go +++ b/pkg/check/smoke/smoke.go @@ -45,10 +45,10 @@ func NewDefaultOptions() Options { PostageDepth: 24, TxOnErrWait: 10 * time.Second, RxOnErrWait: 10 * time.Second, - NodesSyncWait: time.Second * 30, + NodesSyncWait: time.Minute, Duration: 12 * time.Hour, - UploadTimeout: 5 * time.Minute, - DownloadTimeout: 5 * time.Minute, + UploadTimeout: 60 * time.Minute, + DownloadTimeout: 60 * time.Minute, MaxUseBatch: 12 * time.Hour, } } From b8afde526a9b1c5950bdee992e60f9698fb22e01 Mon Sep 17 00:00:00 2001 From: istae <14264581+istae@users.noreply.github.com> Date: Wed, 11 Oct 2023 14:19:17 +0300 Subject: [PATCH 6/6] chore: added metric for upload size --- pkg/check/smoke/metrics.go | 9 +++++++++ pkg/check/smoke/smoke.go | 2 ++ 2 files changed, 11 insertions(+) diff --git a/pkg/check/smoke/metrics.go b/pkg/check/smoke/metrics.go index f46cdb0ac..4759650ed 100644 --- a/pkg/check/smoke/metrics.go +++ b/pkg/check/smoke/metrics.go @@ -15,6 +15,7 @@ type metrics struct { DownloadAttempts prometheus.Counter UploadDuration prometheus.Histogram DownloadDuration prometheus.Histogram + UploadSize prometheus.Gauge } func newMetrics(subsystem string) metrics { @@ -91,6 +92,14 @@ func newMetrics(subsystem string) metrics { Help: "Data download duration through the /bytes endpoint.", }, ), + UploadSize: prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: m.Namespace, + Subsystem: subsystem, + Name: "upload_size", + Help: "Amount of data uploaded per upload.", + }, + ), } } diff --git a/pkg/check/smoke/smoke.go b/pkg/check/smoke/smoke.go index c92d00aa0..ca6276300 100644 --- a/pkg/check/smoke/smoke.go +++ b/pkg/check/smoke/smoke.go @@ -91,6 +91,8 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int test := &test{clients: clients, logger: c.logger} + c.metrics.UploadSize.Set(float64(o.ContentSize)) + for i := 0; true; i++ { select { case <-ctx.Done():