diff --git a/.github/workflows/release-go-module.yml b/.github/workflows/release-go-module.yml index fc89a1620..c073dea05 100644 --- a/.github/workflows/release-go-module.yml +++ b/.github/workflows/release-go-module.yml @@ -47,7 +47,7 @@ jobs: VERSION=$(echo "$TAG_REF" | cut -d'/' -f2) # Find the latest tag for the same package that is not the current tag - LAST_TAG=$(git describe --abbrev=0 --match "$PACKAGE_NAME/v*" --tags $(git rev-list --tags --skip=1 --max-count=1)) + LAST_TAG=$(git describe --abbrev=0 --always --match "$PACKAGE_NAME/v*" --tags $(git rev-list --tags --skip=1 --max-count=1)) echo "Last tag: ${LAST_TAG}" # If no previous tag is found, use the initial commit as the reference diff --git a/havoc/go.mod b/havoc/go.mod index 600625e84..20bd593d2 100644 --- a/havoc/go.mod +++ b/havoc/go.mod @@ -2,8 +2,6 @@ module github.com/smartcontractkit/chainlink-testing-framework/havoc go 1.22.5 -exclude github.com/chaos-mesh/chaos-mesh/api/v1alpha1 v0.0.0-20220226050744-799408773657 - require ( github.com/chaos-mesh/chaos-mesh/api v0.0.0-20240821051457-da69c6d9617a github.com/pkg/errors v0.9.1 @@ -11,7 +9,7 @@ require ( github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0 k8s.io/api v0.31.0 k8s.io/client-go v0.31.0 - sigs.k8s.io/controller-runtime v0.16.2 + sigs.k8s.io/controller-runtime v0.19.0 ) require ( @@ -76,6 +74,4 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -replace sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.19.0 - retract [v1.999.0-test-release, v1.999.999-test-release] diff --git a/lib/go.mod b/lib/go.mod index 78513c12a..484178d9d 100644 --- a/lib/go.mod +++ b/lib/go.mod @@ -31,7 +31,7 @@ require ( github.com/prometheus/client_golang v1.19.1 github.com/prometheus/common v0.55.0 github.com/rs/zerolog v1.30.0 - github.com/slack-go/slack v0.12.2 + github.com/slack-go/slack v0.15.0 github.com/smartcontractkit/chainlink-testing-framework/seth v1.2.1-0.20240827112945-bd8c580392d6 github.com/smartcontractkit/chainlink-testing-framework/wasp v0.4.9 github.com/spf13/cobra v1.8.1 diff --git a/lib/go.sum b/lib/go.sum index b4b5fb9c6..0d4216ba1 100644 --- a/lib/go.sum +++ b/lib/go.sum @@ -1045,8 +1045,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/slack-go/slack v0.12.2 h1:x3OppyMyGIbbiyFhsBmpf9pwkUzMhthJMRNmNlA4LaQ= -github.com/slack-go/slack v0.12.2/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw= +github.com/slack-go/slack v0.15.0 h1:LE2lj2y9vqqiOf+qIIy0GvEoxgF1N5yLGZffmEZykt0= +github.com/slack-go/slack v0.15.0/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449 h1:fX/xmGm1GBsD1ZZnooNT+eWA0hiTAqFlHzOC5CY4dy8= github.com/smartcontractkit/chainlink-testing-framework/grafana v0.0.0-20240328204215-ac91f55f1449/go.mod h1:DC8sQMyTlI/44UCTL8QWFwb0bYNoXCfjwCv2hMivYZU= github.com/smartcontractkit/chainlink-testing-framework/seth v1.2.1-0.20240827112945-bd8c580392d6 h1:ItZ75xmt+VHR/lw+GJwSWj9XICpgZ94dJ+I/5jdet7c= diff --git a/lib/testreporters/slack_notification.go b/lib/testreporters/slack_notification.go index 1bdb92fc4..e020eefe6 100644 --- a/lib/testreporters/slack_notification.go +++ b/lib/testreporters/slack_notification.go @@ -22,7 +22,7 @@ var ( ) // Uploads a slack file to the designated channel using the API key -func UploadSlackFile(slackClient *slack.Client, uploadParams slack.FileUploadParameters) error { +func UploadSlackFile(slackClient *slack.Client, uploadParams slack.UploadFileV2Parameters) error { log.Info(). Str("Slack API Key", SlackAPIKey). Str("Slack Channel", SlackChannel). @@ -35,17 +35,22 @@ func UploadSlackFile(slackClient *slack.Client, uploadParams slack.FileUploadPar if SlackChannel == "" { return fmt.Errorf("unable to upload file without a Slack Channel") } - if uploadParams.Channels == nil || uploadParams.Channels[0] == "" { - uploadParams.Channels = []string{SlackChannel} + if uploadParams.Channel == "" { + uploadParams.Channel = SlackChannel } if uploadParams.File != "" { - if _, err := os.Stat(uploadParams.File); errors.Is(err, os.ErrNotExist) { + file, err := os.Stat(uploadParams.File) + if errors.Is(err, os.ErrNotExist) { return fmt.Errorf("unable to upload file as it does not exist: %w", err) } else if err != nil { return err } + // file size is now mandatory, so we need to set if it's empty + if uploadParams.FileSize == 0 { + uploadParams.FileSize = int(file.Size()) + } } - _, err := slackClient.UploadFile(uploadParams) + _, err := slackClient.UploadFileV2(uploadParams) return err } diff --git a/seth/README.md b/seth/README.md index 6b4a5ae87..56124691b 100644 --- a/seth/README.md +++ b/seth/README.md @@ -270,6 +270,8 @@ export SETH_ROOT_PRIVATE_KEY=ac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae7 alias seth="SETH_CONFIG_PATH=seth.toml go run cmd/seth/seth.go" # useful alias for CLI ``` +> Find the log level options [here](https://github.com/rs/zerolog?tab=readme-ov-file#leveled-logging) + Alternatively if you don't have a network defined in the TOML you can still use the CLI by providing these 2 key env vars: ```sh diff --git a/wasp/examples/scenario/vu.go b/wasp/examples/scenario/vu.go index 410efcde5..5bfd79fdf 100644 --- a/wasp/examples/scenario/vu.go +++ b/wasp/examples/scenario/vu.go @@ -24,7 +24,7 @@ func NewExampleScenario(target string) *VirtualUser { return &VirtualUser{ VUControl: wasp.NewVUControl(), target: target, - rl: ratelimit.New(10), + rl: ratelimit.New(10, ratelimit.WithoutSlack), client: resty.New().SetBaseURL(target), Data: make([]string, 0), } @@ -34,7 +34,7 @@ func (m *VirtualUser) Clone(_ *wasp.Generator) wasp.VirtualUser { return &VirtualUser{ VUControl: wasp.NewVUControl(), target: m.target, - rl: ratelimit.New(10), + rl: ratelimit.New(10, ratelimit.WithoutSlack), client: resty.New().SetBaseURL(m.target), Data: make([]string, 0), } diff --git a/wasp/wasp.go b/wasp/wasp.go index 0e9bef2c8..88cd6c6b3 100644 --- a/wasp/wasp.go +++ b/wasp/wasp.go @@ -195,6 +195,7 @@ type Stats struct { CurrentSegment atomic.Int64 `json:"current_schedule_segment"` SamplesRecorded atomic.Int64 `json:"samples_recorded"` SamplesSkipped atomic.Int64 `json:"samples_skipped"` + RunStarted atomic.Bool `json:"runStarted"` RunPaused atomic.Bool `json:"runPaused"` RunStopped atomic.Bool `json:"runStopped"` RunFailed atomic.Bool `json:"runFailed"` @@ -224,6 +225,7 @@ type Generator struct { labels model.LabelSet rl atomic.Pointer[ratelimit.Limiter] scheduleSegments []*Segment + currentSegmentMu *sync.Mutex currentSegment *Segment ResponsesWaitGroup *sync.WaitGroup dataWaitGroup *sync.WaitGroup @@ -298,6 +300,7 @@ func NewGenerator(cfg *Config) (*Generator, error) { Responses: NewResponses(rch), ResponsesChan: rch, labels: ls, + currentSegmentMu: &sync.Mutex{}, responsesData: &ResponseData{ okDataMu: &sync.Mutex{}, OKData: NewSliceBuffer[any](cfg.CallResultBufLen), @@ -323,16 +326,13 @@ func NewGenerator(cfg *Config) (*Generator, error) { return g, nil } -// setupSchedule set up initial data for both RPS and VirtualUser load types -func (g *Generator) setupSchedule() { +// runExecuteLoop set up initial data for both RPS and VirtualUser load types +func (g *Generator) runExecuteLoop() { g.currentSegment = g.scheduleSegments[0] g.stats.LastSegment.Store(int64(len(g.scheduleSegments))) switch g.Cfg.LoadType { case RPS: g.ResponsesWaitGroup.Add(1) - g.stats.CurrentRPS.Store(g.currentSegment.From) - newRateLimit := ratelimit.New(int(g.currentSegment.From), ratelimit.Per(g.Cfg.RateLimitUnitDuration)) - g.rl.Store(&newRateLimit) // we run pacedCall controlled by stats.CurrentRPS go func() { for { @@ -347,7 +347,9 @@ func (g *Generator) setupSchedule() { } }() case VU: + g.currentSegmentMu.Lock() g.stats.CurrentVUs.Store(g.currentSegment.From) + g.currentSegmentMu.Unlock() // we start all vus once vus := g.stats.CurrentVUs.Load() for i := 0; i < int(vus); i++ { @@ -449,6 +451,7 @@ func (g *Generator) runVU(vu VirtualUser) { // changing both internal and Stats values to report func (g *Generator) processSegment() bool { defer func() { + g.stats.RunStarted.Store(true) g.Log.Info(). Int64("Segment", g.stats.CurrentSegment.Load()). Int64("VUs", g.stats.CurrentVUs.Load()). @@ -458,11 +461,13 @@ func (g *Generator) processSegment() bool { if g.stats.CurrentSegment.Load() == g.stats.LastSegment.Load() { return true } + g.currentSegmentMu.Lock() g.currentSegment = g.scheduleSegments[g.stats.CurrentSegment.Load()] + g.currentSegmentMu.Unlock() g.stats.CurrentSegment.Add(1) switch g.Cfg.LoadType { case RPS: - newRateLimit := ratelimit.New(int(g.currentSegment.From), ratelimit.Per(g.Cfg.RateLimitUnitDuration)) + newRateLimit := ratelimit.New(int(g.currentSegment.From), ratelimit.Per(g.Cfg.RateLimitUnitDuration), ratelimit.WithoutSlack) g.rl.Store(&newRateLimit) g.stats.CurrentRPS.Store(g.currentSegment.From) case VU: @@ -491,9 +496,9 @@ func (g *Generator) processSegment() bool { return false } -// runSchedule runs scheduling loop +// runScheduleLoop runs scheduling loop // processing segments inside the whole schedule -func (g *Generator) runSchedule() { +func (g *Generator) runScheduleLoop() { go func() { for { select { @@ -583,11 +588,17 @@ func (g *Generator) collectVUResults() { // pacedCall calls a gun according to a scheduleSegments or plain RPS func (g *Generator) pacedCall() { - if g.stats.RunPaused.Load() || g.stats.RunStopped.Load() { + if !g.Stats().RunStarted.Load() { return } l := *g.rl.Load() l.Take() + if g.stats.RunPaused.Load() { + return + } + if g.stats.RunStopped.Load() { + return + } result := make(chan *Response) requestCtx, cancel := context.WithTimeout(context.Background(), g.Cfg.CallTimeout) callStartTS := time.Now() @@ -621,9 +632,9 @@ func (g *Generator) Run(wait bool) (interface{}, bool) { g.sendResponsesToLoki() g.sendStatsToLoki() } - g.setupSchedule() + g.runScheduleLoop() + g.runExecuteLoop() g.collectVUResults() - g.runSchedule() if wait { return g.Wait() } @@ -648,6 +659,7 @@ func (g *Generator) Stop() (interface{}, bool) { if g.stats.RunStopped.Load() { return nil, true } + g.stats.RunStarted.Store(false) g.stats.RunStopped.Store(true) g.stats.RunFailed.Store(true) g.Log.Warn().Msg("Graceful stop") diff --git a/wasp/wasp_bench_test.go b/wasp/wasp_bench_test.go index d5a7a6f73..bdbbfc55c 100644 --- a/wasp/wasp_bench_test.go +++ b/wasp/wasp_bench_test.go @@ -21,7 +21,7 @@ func BenchmarkPacedCall(b *testing.B) { Gun: NewMockGun(&MockGunConfig{}), }) require.NoError(b, err) - gen.setupSchedule() + gen.runExecuteLoop() b.ResetTimer() for i := 0; i < b.N; i++ { gen.pacedCall() diff --git a/wasp/wasp_test.go b/wasp/wasp_test.go index 202cf2735..f396455fe 100644 --- a/wasp/wasp_test.go +++ b/wasp/wasp_test.go @@ -164,7 +164,7 @@ func TestSmokeFailedOneRequest(t *testing.T) { _, failed := gen.Stop() require.Equal(t, true, failed) stats := gen.Stats() - require.GreaterOrEqual(t, stats.Failed.Load(), int64(2)) + require.GreaterOrEqual(t, stats.Failed.Load(), int64(1)) require.Equal(t, stats.RunFailed.Load(), true) require.Equal(t, stats.CurrentRPS.Load(), int64(1)) require.Equal(t, stats.Duration, gen.Cfg.duration.Nanoseconds()) @@ -172,15 +172,11 @@ func TestSmokeFailedOneRequest(t *testing.T) { okData, _, failResponses := convertResponsesData(gen) require.Empty(t, okData) require.GreaterOrEqual(t, failResponses[0].Duration, 50*time.Millisecond) - require.GreaterOrEqual(t, failResponses[1].Duration, 50*time.Millisecond) require.Equal(t, failResponses[0].Data.(string), "failedCallData") require.Equal(t, failResponses[0].Error, "error") - require.Equal(t, failResponses[1].Data.(string), "failedCallData") - require.Equal(t, failResponses[1].Error, "error") errs := gen.Errors() require.Equal(t, errs[0], "error") - require.Equal(t, errs[1], "error") - require.GreaterOrEqual(t, len(errs), 2) + require.GreaterOrEqual(t, len(errs), 1) } func TestSmokeGenCallTimeout(t *testing.T) { @@ -201,7 +197,7 @@ func TestSmokeGenCallTimeout(t *testing.T) { require.Equal(t, true, failed) stats := gen.Stats() require.GreaterOrEqual(t, stats.Success.Load(), int64(0)) - require.GreaterOrEqual(t, stats.CallTimeout.Load(), int64(2)) + require.GreaterOrEqual(t, stats.CallTimeout.Load(), int64(1)) require.Equal(t, stats.CurrentRPS.Load(), int64(1)) okData, _, failResponses := convertResponsesData(gen) @@ -209,7 +205,6 @@ func TestSmokeGenCallTimeout(t *testing.T) { require.Equal(t, failResponses[0].Data, nil) require.Equal(t, failResponses[0].Error, ErrCallTimeout.Error()) require.Equal(t, gen.Errors()[0], ErrCallTimeout.Error()) - require.Equal(t, gen.Errors()[1], ErrCallTimeout.Error()) } func TestSmokeVUCallTimeout(t *testing.T) { @@ -414,12 +409,11 @@ func TestSmokeCancelledBeforeDeadline(t *testing.T) { require.Greater(t, elapsed, 1050*time.Millisecond) require.Equal(t, true, failed) stats := gen.Stats() - require.GreaterOrEqual(t, stats.Success.Load(), int64(2)) + require.GreaterOrEqual(t, stats.Success.Load(), int64(1)) require.Equal(t, stats.CurrentRPS.Load(), int64(1)) okData, _, failResponses := convertResponsesData(gen) require.Equal(t, okData[0], "successCallData") - require.Equal(t, okData[1], "successCallData") require.Empty(t, failResponses) require.Empty(t, gen.Errors()) } @@ -455,13 +449,13 @@ func TestSmokeStaticRPSSchedulePrecision(t *testing.T) { require.NoError(t, err) _, failed := gen.Run(true) require.Equal(t, false, failed) - require.GreaterOrEqual(t, gen.Stats().Success.Load(), int64(990)) + require.GreaterOrEqual(t, gen.Stats().Success.Load(), int64(980)) require.LessOrEqual(t, gen.Stats().Success.Load(), int64(1010)) require.Equal(t, gen.Stats().Failed.Load(), int64(0)) require.Equal(t, gen.Stats().CallTimeout.Load(), int64(0)) okData, _, failResponses := convertResponsesData(gen) - require.GreaterOrEqual(t, len(okData), 990) + require.GreaterOrEqual(t, len(okData), 980) require.LessOrEqual(t, len(okData), 1010) require.Empty(t, failResponses) require.Empty(t, gen.Errors()) @@ -481,14 +475,14 @@ func TestSmokeCustomUnitPrecision(t *testing.T) { _, failed := gen.Run(true) require.Equal(t, false, failed) stats := gen.Stats() - require.GreaterOrEqual(t, stats.Success.Load(), int64(4990)) + require.GreaterOrEqual(t, stats.Success.Load(), int64(4970)) require.LessOrEqual(t, stats.Success.Load(), int64(5010)) require.Equal(t, stats.Failed.Load(), int64(0)) require.Equal(t, stats.CallTimeout.Load(), int64(0)) require.Equal(t, stats.CurrentTimeUnit, gen.Cfg.RateLimitUnitDuration.Nanoseconds()) okData, _, failResponses := convertResponsesData(gen) - require.GreaterOrEqual(t, len(okData), 4990) + require.GreaterOrEqual(t, len(okData), 4970) require.LessOrEqual(t, len(okData), 5010) require.Empty(t, failResponses) require.Empty(t, gen.Errors()) @@ -507,13 +501,13 @@ func TestSmokeStaticRPSScheduleIsNotBlocking(t *testing.T) { require.NoError(t, err) _, failed := gen.Run(true) require.Equal(t, false, failed) - require.GreaterOrEqual(t, gen.Stats().Success.Load(), int64(990)) + require.GreaterOrEqual(t, gen.Stats().Success.Load(), int64(980)) require.LessOrEqual(t, gen.Stats().Success.Load(), int64(1010)) require.Equal(t, gen.Stats().Failed.Load(), int64(0)) require.Equal(t, gen.Stats().CallTimeout.Load(), int64(0)) okData, _, failResponses := convertResponsesData(gen) - require.GreaterOrEqual(t, len(okData), 990) + require.GreaterOrEqual(t, len(okData), 980) require.LessOrEqual(t, len(okData), 1010) require.Empty(t, failResponses) require.Empty(t, gen.Errors()) @@ -1006,7 +1000,7 @@ func TestSmokePauseResumeGenerator(t *testing.T) { stats := gen.Stats() _, okResponses, failResponses := convertResponsesData(gen) require.Equal(t, int64(10), stats.CurrentRPS.Load()) - require.GreaterOrEqual(t, len(okResponses), 70) + require.GreaterOrEqual(t, len(okResponses), 60) require.Empty(t, failResponses) require.Empty(t, gen.Errors()) }) @@ -1039,3 +1033,39 @@ func TestSmokePauseResumeGenerator(t *testing.T) { require.Empty(t, gen.Errors()) }) } + +// regression + +func TestSmokeNoDuplicateRequestsOnceOnStart(t *testing.T) { + t.Parallel() + gen, err := NewGenerator(&Config{ + T: t, + LoadType: RPS, + StatsPollInterval: 1 * time.Second, + Schedule: Plain(1, 1*time.Second), + Gun: NewMockGun(&MockGunConfig{ + CallSleep: 50 * time.Millisecond, + }), + }) + require.NoError(t, err) + _, failed := gen.Run(false) + require.Equal(t, false, failed) + time.Sleep(950 * time.Millisecond) + _, _ = gen.Stop() + stats := gen.Stats() + require.Equal(t, stats.CurrentRPS.Load(), int64(1)) + require.Equal(t, stats.CurrentVUs.Load(), int64(0)) + require.GreaterOrEqual(t, stats.Success.Load(), int64(1)) + require.Equal(t, stats.CallTimeout.Load(), int64(0)) + require.Equal(t, stats.Failed.Load(), int64(0)) + require.Equal(t, stats.Duration, gen.Cfg.duration.Nanoseconds()) + + okData, okResponses, failResponses := convertResponsesData(gen) + require.GreaterOrEqual(t, len(okResponses), 1) + require.GreaterOrEqual(t, len(okData), 1) + require.Equal(t, okData[0], "successCallData") + require.GreaterOrEqual(t, okResponses[0].Duration, 50*time.Millisecond) + require.Equal(t, okResponses[0].Data.(string), "successCallData") + require.Empty(t, failResponses) + require.Empty(t, gen.Errors()) +}