diff --git a/config/config.yaml b/config/config.yaml index be2fe26ea..afdc6f0fd 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -297,6 +297,8 @@ checks: postage-depth: 20 nodes-sync-wait: 1m duration: 12h + upload-timeout: 5m + download-timeout: 5m timeout: 5m type: smoke load: diff --git a/config/local.yaml b/config/local.yaml index 4a92637a3..b23699279 100644 --- a/config/local.yaml +++ b/config/local.yaml @@ -340,6 +340,8 @@ checks: postage-amount: 1000000 postage-depth: 20 nodes-sync-wait: 1m + upload-timeout: 1m + download-timeout: 1m timeout: 5m type: smoke ci-soc: diff --git a/go.mod b/go.mod index 215b5a7e7..be34dbd67 100644 --- a/go.mod +++ b/go.mod @@ -53,12 +53,9 @@ require ( github.com/go-stack/stack v1.8.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/golang/snappy v0.0.4 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa // indirect github.com/googleapis/gnostic v0.5.5 // indirect - github.com/hashicorp/errwrap v1.0.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect @@ -102,7 +99,6 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.4.2 // indirect - github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect github.com/tklauser/go-sysconf v0.3.6 // indirect github.com/tklauser/numcpus v0.2.2 // indirect github.com/uber/jaeger-lib v2.2.0+incompatible // indirect diff --git a/go.sum b/go.sum index ee23ef89a..fd998f689 100644 --- a/go.sum +++ b/go.sum @@ -208,7 +208,6 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= @@ -257,11 +256,7 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= @@ -472,7 +467,6 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= -github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tklauser/go-sysconf v0.3.6 h1:oc1sJWvKkmvIxhDHeKWvZS4f6AW+YcoguSfRF2/Hmo4= github.com/tklauser/go-sysconf v0.3.6/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA= @@ -589,7 +583,6 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -666,7 +659,6 @@ golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/pkg/check/smoke/load.go b/pkg/check/smoke/load.go index 7a68bff81..15d0d20fc 100644 --- a/pkg/check/smoke/load.go +++ b/pkg/check/smoke/load.go @@ -59,7 +59,7 @@ func (c *LoadCheck) Run(ctx context.Context, cluster orchestration.Cluster, opts ctx, cancel := context.WithTimeout(ctx, o.Duration) defer cancel() - test := &test{opt: o, ctx: ctx, clients: clients, logger: c.logger} + test := &test{clients: clients, logger: c.logger} uploaders := selectNames(cluster, o.UploadGroups...) downloaders := selectNames(cluster, o.DownloadGroups...) @@ -125,7 +125,7 @@ func (c *LoadCheck) Run(ctx context.Context, cluster orchestration.Cluster, opts batches.Store(txName, batchID) } - address, duration, err = test.uploadWithBatch(txName, txData, batchID) + address, duration, err = test.upload(ctx, txName, txData, batchID) if err != nil { c.metrics.UploadErrors.Inc() c.logger.Infof("upload failed: %v", err) @@ -174,7 +174,7 @@ func (c *LoadCheck) Run(ctx context.Context, cluster orchestration.Cluster, opts c.metrics.DownloadAttempts.Inc() - rxData, rxDuration, err = test.download(rxName, address) + rxData, rxDuration, err = test.download(ctx, rxName, address) if err != nil { c.metrics.DownloadErrors.Inc() c.logger.Infof("download failed: %v", err) diff --git a/pkg/check/smoke/smoke.go b/pkg/check/smoke/smoke.go index 56c644118..925ac1fe1 100644 --- a/pkg/check/smoke/smoke.go +++ b/pkg/check/smoke/smoke.go @@ -18,14 +18,16 @@ import ( // Options represents smoke test options type Options struct { - ContentSize int64 - RndSeed int64 - PostageAmount int64 - PostageDepth uint64 - TxOnErrWait time.Duration - RxOnErrWait time.Duration - NodesSyncWait time.Duration - Duration time.Duration + ContentSize int64 + RndSeed int64 + PostageAmount int64 + PostageDepth uint64 + TxOnErrWait time.Duration + RxOnErrWait time.Duration + NodesSyncWait time.Duration + Duration time.Duration + UploadTimeout time.Duration + DownloadTimeout time.Duration // load test params UploaderCount int UploadGroups []string @@ -38,16 +40,18 @@ type Options struct { // NewDefaultOptions returns new default options func NewDefaultOptions() Options { return Options{ - ContentSize: 5000000, - RndSeed: time.Now().UnixNano(), - PostageAmount: 1000000, - PostageDepth: 20, - TxOnErrWait: 10 * time.Second, - RxOnErrWait: 10 * time.Second, - NodesSyncWait: time.Second * 30, - Duration: 12 * time.Hour, - GasPrice: "100000000000", - MaxUseBatch: time.Hour * 12, + ContentSize: 5000000, + RndSeed: time.Now().UnixNano(), + PostageAmount: 1000000, + PostageDepth: 20, + TxOnErrWait: 10 * time.Second, + RxOnErrWait: 10 * time.Second, + NodesSyncWait: time.Second * 30, + Duration: 12 * time.Hour, + UploadTimeout: 5 * time.Minute, + DownloadTimeout: 5 * time.Minute, + GasPrice: "100000000000", + MaxUseBatch: 12 * time.Hour, } } @@ -89,7 +93,7 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int batches := NewStore(o.MaxUseBatch) - test := &test{opt: o, ctx: ctx, clients: clients, logger: c.logger} + test := &test{clients: clients, logger: c.logger} for i := 0; true; i++ { select { @@ -129,7 +133,10 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int continue } + var txCancel context.CancelFunc = func() {} for retries := 0; retries < 3; retries++ { + txCancel() + select { case <-ctx.Done(): return nil @@ -138,6 +145,8 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int c.metrics.UploadAttempts.Inc() + ctx, txCancel = context.WithTimeout(ctx, o.UploadTimeout) + batchID := batches.Get(txName) if batchID == "" { batchID, err = clients[txName].CreatePostageBatch(ctx, o.PostageAmount, o.PostageDepth, o.GasPrice, "load-test", true) @@ -148,7 +157,7 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int batches.Store(txName, batchID) } - address, txDuration, err = test.uploadWithBatch(txName, txData, batchID) + address, txDuration, err = test.upload(ctx, txName, txData, batchID) if err != nil { c.metrics.UploadErrors.Inc() c.logger.Infof("upload failed: %v", err) @@ -157,6 +166,7 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int break } } + txCancel() if err != nil { continue // skip @@ -166,7 +176,10 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int time.Sleep(o.NodesSyncWait) + var rxCancel context.CancelFunc = func() {} for retries := 0; retries < 3; retries++ { + rxCancel() + select { case <-ctx.Done(): return nil @@ -175,7 +188,8 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int c.metrics.DownloadAttempts.Inc() - rxData, rxDuration, err = test.download(rxName, address) + ctx, rxCancel = context.WithTimeout(ctx, o.DownloadTimeout) + rxData, rxDuration, err = test.download(ctx, rxName, address) if err != nil { c.metrics.DownloadErrors.Inc() c.logger.Infof("download failed: %v", err) @@ -210,23 +224,22 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int } c.logger.Infof("data mismatch: found %d different bytes, ~%.2f%%", diff, float64(diff)/float64(txLen)*100) } + rxCancel() } return nil } type test struct { - opt Options - ctx context.Context clients map[string]*bee.Client logger logging.Logger } -func (t *test) uploadWithBatch(cName string, data []byte, batchID string) (swarm.Address, time.Duration, error) { +func (t *test) upload(ctx context.Context, cName string, data []byte, batchID string) (swarm.Address, time.Duration, error) { client := t.clients[cName] t.logger.Infof("node %s: uploading data, batch id %s", cName, batchID) start := time.Now() - addr, err := client.UploadBytes(t.ctx, data, api.UploadOptions{Pin: false, BatchID: batchID, Direct: true}) + addr, err := client.UploadBytes(ctx, data, api.UploadOptions{Pin: false, BatchID: batchID, Direct: true}) if err != nil { return swarm.ZeroAddress, 0, fmt.Errorf("upload to the node %s: %w", cName, err) } @@ -236,11 +249,11 @@ func (t *test) uploadWithBatch(cName string, data []byte, batchID string) (swarm return addr, txDuration, nil } -func (t *test) download(cName string, addr swarm.Address) ([]byte, time.Duration, error) { +func (t *test) download(ctx context.Context, cName string, addr swarm.Address) ([]byte, time.Duration, error) { client := t.clients[cName] t.logger.Infof("node %s: downloading address %s", cName, addr) start := time.Now() - data, err := client.DownloadBytes(t.ctx, addr) + data, err := client.DownloadBytes(ctx, addr) if err != nil { return nil, 0, fmt.Errorf("download from node %s: %w", cName, err) } @@ -249,21 +262,3 @@ func (t *test) download(cName string, addr swarm.Address) ([]byte, time.Duration return data, rxDuration, nil } - -// func (t *test) upload(cName string, data []byte) (swarm.Address, time.Duration, error) { -// client := t.clients[cName] -// batchID, err := client.GetOrCreateBatch(t.ctx, t.opt.PostageAmount, t.opt.PostageDepth, t.opt.GasPrice, "smoke-test") -// if err != nil { -// return swarm.ZeroAddress, 0, fmt.Errorf("node %s: unable to create batch id: %w", cName, err) -// } -// t.logger.Infof("node %s: uploading data, batch id %s", cName, batchID) -// start := time.Now() -// addr, err := client.UploadBytes(t.ctx, data, api.UploadOptions{Pin: false, BatchID: batchID, Direct: true}) -// if err != nil { -// return swarm.ZeroAddress, 0, fmt.Errorf("upload to the node %s: %w", cName, err) -// } -// txDuration := time.Since(start) -// t.logger.Infof("node %s: upload done in %s", cName, txDuration) - -// return addr, txDuration, nil -// }