From 1d5ca0c2d5eb78ffac5dc0dd6230e5566325cdca Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Tue, 3 Dec 2024 13:26:43 +0100 Subject: [PATCH] make the tool more abstract --- wasp/benchspy/basic.go | 107 +++++++ wasp/benchspy/loki.go | 147 +++++++++ wasp/benchspy/report.go | 114 +++++++ wasp/{comparator => benchspy}/resources.go | 6 +- wasp/{comparator => benchspy}/storage.go | 28 +- wasp/benchspy/types.go | 27 ++ wasp/benchspy_test.go | 111 +++++++ wasp/comparator/report.go | 246 ---------------- wasp/comparator_test.go | 82 ------ ...c5826a572c09f8b93df3b9f674113372ce924.json | 278 ++++++++++++++++++ 10 files changed, 809 insertions(+), 337 deletions(-) create mode 100644 wasp/benchspy/basic.go create mode 100644 wasp/benchspy/loki.go create mode 100644 wasp/benchspy/report.go rename wasp/{comparator => benchspy}/resources.go (98%) rename wasp/{comparator => benchspy}/storage.go (79%) create mode 100644 wasp/benchspy/types.go create mode 100644 wasp/benchspy_test.go delete mode 100644 wasp/comparator/report.go delete mode 100644 wasp/comparator_test.go create mode 100644 wasp/test_performance_reports/TestBenchSpyWithLokiQuery-e7fc5826a572c09f8b93df3b9f674113372ce924.json diff --git a/wasp/benchspy/basic.go b/wasp/benchspy/basic.go new file mode 100644 index 000000000..c3b414aa1 --- /dev/null +++ b/wasp/benchspy/basic.go @@ -0,0 +1,107 @@ +package benchspy + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/pkg/errors" + "github.com/smartcontractkit/chainlink-testing-framework/wasp" +) + +// BasicData is the basic data that is required for a report, common to all reports +type BasicData struct { + TestName string `json:"test_name"` + CommitOrTag string `json:"commit_or_tag"` + + // Test metrics + TestStart time.Time `json:"test_start_timestamp"` + TestEnd time.Time `json:"test_end_timestamp"` + + // all, generator settings, including segments + GeneratorConfigs map[string]*wasp.Config `json:"generator_configs"` +} + +func (b *BasicData) Validate() error { + if b.TestStart.IsZero() { + return errors.New("test start time is missing. We cannot query Loki without a time range. Please set it and try again") + } + if b.TestEnd.IsZero() { + return errors.New("test end time is missing. We cannot query Loki without a time range. Please set it and try again") + } + + if len(b.GeneratorConfigs) == 0 { + return errors.New("generator configs are missing. At least one is required. Please set them and try again") + } + + return nil +} + +func (b *BasicData) IsComparable(otherData BasicData) error { + // are all configs present? do they have the same schedule type? do they have the same segments? is call timeout the same? is rate limit timeout the same? + if len(b.GeneratorConfigs) != len(otherData.GeneratorConfigs) { + return fmt.Errorf("generator configs count is different. Expected %d, got %d", len(b.GeneratorConfigs), len(otherData.GeneratorConfigs)) + } + + for name1, cfg1 := range b.GeneratorConfigs { + if cfg2, ok := otherData.GeneratorConfigs[name1]; !ok { + return fmt.Errorf("generator config %s is missing from the other report", name1) + } else { + if err := compareGeneratorConfigs(cfg1, cfg2); err != nil { + return err + } + } + } + + for name2 := range otherData.GeneratorConfigs { + if _, ok := b.GeneratorConfigs[name2]; !ok { + return fmt.Errorf("generator config %s is missing from the current report", name2) + } + } + + // TODO: would be good to be able to check if Gun and VU are the same, but idk yet how we could do that easily [hash the code?] + + return nil +} + +func compareGeneratorConfigs(cfg1, cfg2 *wasp.Config) error { + if cfg1.LoadType != cfg2.LoadType { + return fmt.Errorf("load types are different. Expected %s, got %s", cfg1.LoadType, cfg2.LoadType) + } + + if len(cfg1.Schedule) != len(cfg2.Schedule) { + return fmt.Errorf("schedules are different. Expected %d, got %d", len(cfg1.Schedule), len(cfg2.Schedule)) + } + + for i, segment1 := range cfg1.Schedule { + segment2 := cfg2.Schedule[i] + if segment1 == nil { + return fmt.Errorf("schedule at index %d is nil in the current report", i) + } + if segment2 == nil { + return fmt.Errorf("schedule at index %d is nil in the other report", i) + } + if *segment1 != *segment2 { + return fmt.Errorf("schedules at index %d are different. Expected %s, got %s", i, mustMarshallSegment(segment1), mustMarshallSegment(segment2)) + } + } + + if cfg1.CallTimeout != cfg2.CallTimeout { + return fmt.Errorf("call timeouts are different. Expected %s, got %s", cfg1.CallTimeout, cfg2.CallTimeout) + } + + if cfg1.RateLimitUnitDuration != cfg2.RateLimitUnitDuration { + return fmt.Errorf("rate limit unit durations are different. Expected %s, got %s", cfg1.RateLimitUnitDuration, cfg2.RateLimitUnitDuration) + } + + return nil +} + +func mustMarshallSegment(segment *wasp.Segment) string { + segmentBytes, err := json.MarshalIndent(segment, "", " ") + if err != nil { + panic(err) + } + + return string(segmentBytes) +} diff --git a/wasp/benchspy/loki.go b/wasp/benchspy/loki.go new file mode 100644 index 000000000..3cad6cc8c --- /dev/null +++ b/wasp/benchspy/loki.go @@ -0,0 +1,147 @@ +package benchspy + +import ( + "context" + "fmt" + "net/url" + "reflect" + "strings" + "time" + + "github.com/pkg/errors" + "github.com/smartcontractkit/chainlink-testing-framework/lib/client" + "github.com/smartcontractkit/chainlink-testing-framework/wasp" +) + +func NewLokiQuery(queries map[string]string, lokiConfig *wasp.LokiConfig) *LokiQuery { + return &LokiQuery{ + Kind: "loki", + Queries: queries, + LokiConfig: lokiConfig, + QueryResults: make(map[string][]string), + } +} + +type LokiQuery struct { + Kind string `json:"kind"` + // Test metrics + StartTime time.Time `json:"start_time"` + EndTime time.Time `json:"end_time"` + + // Performance queries + // a map of name to query template, ex: "average cpu usage": "avg(rate(cpu_usage_seconds_total[5m]))" + Queries map[string]string `json:"queries"` + // Performance queries results + // can be anything, avg RPS, amount of errors, 95th percentile of CPU utilization, etc + QueryResults map[string][]string `json:"query_results"` + // In case something went wrong + Errors []error `json:"errors"` + + LokiConfig *wasp.LokiConfig `json:"-"` +} + +func (l *LokiQuery) Results() map[string][]string { + return l.QueryResults +} + +func (l *LokiQuery) IsComparable(otherQueryExecutor QueryExecutor) error { + otherType := reflect.TypeOf(otherQueryExecutor) + + if otherType != reflect.TypeOf(l) { + return fmt.Errorf("expected type %s, got %s", reflect.TypeOf(l), otherType) + } + + return l.compareLokiQueries(otherQueryExecutor.(*LokiQuery).Queries) +} + +func (l *LokiQuery) Validate() error { + if len(l.Queries) == 0 { + return errors.New("there are no Loki queries, there's nothing to fetch. Please set them and try again") + } + if l.LokiConfig == nil { + return errors.New("loki config is missing. Please set it and try again") + } + + return nil +} + +func (l *LokiQuery) Execute() error { + splitAuth := strings.Split(l.LokiConfig.BasicAuth, ":") + var basicAuth client.LokiBasicAuth + if len(splitAuth) == 2 { + basicAuth = client.LokiBasicAuth{ + Login: splitAuth[0], + Password: splitAuth[1], + } + } + + l.QueryResults = make(map[string][]string) + + // TODO this can also be parallelized, just remember to add a mutex for writing to results map + for name, query := range l.Queries { + queryParams := client.LokiQueryParams{ + Query: query, + StartTime: l.StartTime, + EndTime: l.EndTime, + Limit: 1000, //TODO make this configurable + } + + parsedLokiUrl, err := url.Parse(l.LokiConfig.URL) + if err != nil { + return errors.Wrapf(err, "failed to parse Loki URL %s", l.LokiConfig.URL) + } + + lokiUrl := parsedLokiUrl.Scheme + "://" + parsedLokiUrl.Host + lokiClient := client.NewLokiClient(lokiUrl, l.LokiConfig.TenantID, basicAuth, queryParams) + + ctx, cancelFn := context.WithTimeout(context.Background(), l.LokiConfig.Timeout) + rawLogs, err := lokiClient.QueryLogs(ctx) + if err != nil { + l.Errors = append(l.Errors, err) + cancelFn() + continue + } + + cancelFn() + l.QueryResults[name] = []string{} + for _, log := range rawLogs { + l.QueryResults[name] = append(l.QueryResults[name], log.Log) + } + } + + if len(l.Errors) > 0 { + return errors.New("there were errors while fetching the results. Please check the errors and try again") + } + + return nil +} + +func (l *LokiQuery) compareLokiQueries(other map[string]string) error { + this := l.Queries + if len(this) != len(other) { + return fmt.Errorf("queries count is different. Expected %d, got %d", len(this), len(other)) + } + + for name1, query1 := range this { + if query2, ok := other[name1]; !ok { + return fmt.Errorf("query %s is missing from the other report", name1) + } else { + if query1 != query2 { + return fmt.Errorf("query %s is different. Expected %s, got %s", name1, query1, query2) + } + } + } + + for name2 := range other { + if _, ok := this[name2]; !ok { + return fmt.Errorf("query %s is missing from the current report", name2) + } + } + + return nil +} + +func (l *LokiQuery) TimeRange(start, end time.Time) { + l.StartTime = start + l.EndTime = end +} diff --git a/wasp/benchspy/report.go b/wasp/benchspy/report.go new file mode 100644 index 000000000..5f6a0b658 --- /dev/null +++ b/wasp/benchspy/report.go @@ -0,0 +1,114 @@ +package benchspy + +import ( + "encoding/json" + "fmt" +) + +// StandardReport is a report that contains all the necessary data for a performance test +type StandardReport struct { + BasicData + LocalReportStorage + ResourceReporter + QueryExecutors []QueryExecutor `json:"query_executors"` +} + +func (b *StandardReport) Store() (string, error) { + return b.LocalReportStorage.Store(b.TestName, b.CommitOrTag, b) +} + +func (b *StandardReport) Load() error { + return b.LocalReportStorage.Load(b.TestName, b.CommitOrTag, b) +} + +func (b *StandardReport) Fetch() error { + basicErr := b.BasicData.Validate() + if basicErr != nil { + return basicErr + } + + // TODO parallelize it + for _, queryExecutor := range b.QueryExecutors { + queryExecutor.TimeRange(b.TestStart, b.TestEnd) + + if validateErr := queryExecutor.Validate(); validateErr != nil { + return validateErr + } + + if execErr := queryExecutor.Execute(); execErr != nil { + return execErr + } + } + + resourceErr := b.FetchResources() + if resourceErr != nil { + return resourceErr + } + + return nil +} + +func (b *StandardReport) IsComparable(otherReport StandardReport) error { + basicErr := b.BasicData.IsComparable(otherReport.BasicData) + if basicErr != nil { + return basicErr + } + + if resourceErr := b.CompareResources(&otherReport.ResourceReporter); resourceErr != nil { + return resourceErr + } + + for i, queryExecutor := range b.QueryExecutors { + queryErr := queryExecutor.IsComparable(otherReport.QueryExecutors[i]) + if queryErr != nil { + return queryErr + } + } + + return nil +} + +func (s *StandardReport) UnmarshalJSON(data []byte) error { + // Define a helper struct with QueryExecutors as json.RawMessage + type Alias StandardReport + var raw struct { + Alias + QueryExecutors []json.RawMessage `json:"query_executors"` + } + + // Unmarshal into the helper struct to populate other fields automatically + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + + var queryExecutors []QueryExecutor + + // Manually handle QueryExecutors + for _, rawExecutor := range raw.QueryExecutors { + var typeIndicator struct { + Kind string `json:"kind"` + } + if err := json.Unmarshal(rawExecutor, &typeIndicator); err != nil { + return err + } + + var executor QueryExecutor + switch typeIndicator.Kind { + case "loki": + executor = &LokiQuery{} + default: + return fmt.Errorf("unknown query executor type: %s", typeIndicator.Kind) + } + + if err := json.Unmarshal(rawExecutor, executor); err != nil { + return err + } + + queryExecutors = append(s.QueryExecutors, executor) + } + + // Copy the automatically unmarshalled fields back to the main struct + *s = StandardReport(raw.Alias) + s.QueryExecutors = queryExecutors + return nil +} diff --git a/wasp/comparator/resources.go b/wasp/benchspy/resources.go similarity index 98% rename from wasp/comparator/resources.go rename to wasp/benchspy/resources.go index 72abc86f0..24e016cef 100644 --- a/wasp/comparator/resources.go +++ b/wasp/benchspy/resources.go @@ -1,10 +1,11 @@ -package comparator +package benchspy import ( "context" "fmt" "os" "regexp" + "sync" "time" "github.com/docker/docker/api/types/container" @@ -115,6 +116,7 @@ func (r *ResourceReporter) fetchDockerResources() error { pattern := regexp.MustCompile(r.ResourceSelectionPattern) var dockerResources = make(map[string]*DockerResources) + resourceMutex := sync.Mutex{} for _, containerInfo := range containers { eg.Go(func() error { @@ -131,12 +133,14 @@ func (r *ResourceReporter) fetchDockerResources() error { } cancelFn() + resourceMutex.Lock() dockerResources[containerName] = &DockerResources{ NanoCPUs: info.HostConfig.NanoCPUs, CpuShares: info.HostConfig.CPUShares, Memory: info.HostConfig.Memory, MemorySwap: info.HostConfig.MemorySwap, } + resourceMutex.Unlock() return nil }) diff --git a/wasp/comparator/storage.go b/wasp/benchspy/storage.go similarity index 79% rename from wasp/comparator/storage.go rename to wasp/benchspy/storage.go index db0795bec..581250353 100644 --- a/wasp/comparator/storage.go +++ b/wasp/benchspy/storage.go @@ -1,4 +1,4 @@ -package comparator +package benchspy import ( "bytes" @@ -14,21 +14,32 @@ import ( "github.com/pkg/errors" ) -type LocalReportStorage struct{} +const DEFAULT_DIRECTORY = "performance_reports" + +type LocalReportStorage struct { + Directory string `json:"directory"` +} + +func (l *LocalReportStorage) defaultDirectoryIfEmpty() { + if l.Directory == "" { + l.Directory = DEFAULT_DIRECTORY + } +} func (l *LocalReportStorage) Store(testName, commitOrTag string, report interface{}) (string, error) { + l.defaultDirectoryIfEmpty() asJson, err := json.MarshalIndent(report, "", " ") if err != nil { return "", err } - if _, err := os.Stat(directory); os.IsNotExist(err) { - if err := os.MkdirAll(directory, 0755); err != nil { - return "", errors.Wrapf(err, "failed to create directory %s", directory) + if _, err := os.Stat(l.Directory); os.IsNotExist(err) { + if err := os.MkdirAll(l.Directory, 0755); err != nil { + return "", errors.Wrapf(err, "failed to create directory %s", l.Directory) } } - reportFilePath := filepath.Join(directory, fmt.Sprintf("%s-%s.json", testName, commitOrTag)) + reportFilePath := filepath.Join(l.Directory, fmt.Sprintf("%s-%s.json", testName, commitOrTag)) reportFile, err := os.Create(reportFilePath) if err != nil { return "", errors.Wrapf(err, "failed to create file %s", reportFilePath) @@ -50,12 +61,13 @@ func (l *LocalReportStorage) Store(testName, commitOrTag string, report interfac } func (l *LocalReportStorage) Load(testName, commitOrTag string, report interface{}) error { + l.defaultDirectoryIfEmpty() if testName == "" { return errors.New("test name is empty. Please set it and try again") } if commitOrTag == "" { - tagsOrCommits, tagErr := extractTagsOrCommits(directory) + tagsOrCommits, tagErr := extractTagsOrCommits(l.Directory) if tagErr != nil { return tagErr } @@ -66,7 +78,7 @@ func (l *LocalReportStorage) Load(testName, commitOrTag string, report interface } commitOrTag = latestCommit } - reportFilePath := filepath.Join(directory, fmt.Sprintf("%s-%s.json", testName, commitOrTag)) + reportFilePath := filepath.Join(l.Directory, fmt.Sprintf("%s-%s.json", testName, commitOrTag)) reportFile, err := os.Open(reportFilePath) if err != nil { diff --git a/wasp/benchspy/types.go b/wasp/benchspy/types.go new file mode 100644 index 000000000..6b38e08a1 --- /dev/null +++ b/wasp/benchspy/types.go @@ -0,0 +1,27 @@ +package benchspy + +import "time" + +type Report interface { + // Store stores the report in a persistent storage and returns the path to it, or an error + Store() (string, error) + // Load loads the report from a persistent storage and returns it, or an error + Load() error + // Fetch populates the report with the data from the test + Fetch() error + // IsComparable checks whether both reports can be compared (e.g. test config is the same, app's resources are the same, queries or metrics used are the same, etc.), and an error if any difference is found + IsComparable(otherReport Report) error +} + +type QueryExecutor interface { + // Validate checks if the QueryExecutor has all the necessary data and configuration to execute the queries + Validate() error + // Execute executes the queries and populates the QueryExecutor with the results + Execute() error + // Results returns the results of the queries, where key is the name of the query and value is the result + Results() map[string][]string + // IsComparable checks whether both QueryExecutors can be compared (e.g. they have the same type, queries are the same, etc.), and returns an error (if any difference is found) + IsComparable(other QueryExecutor) error + // TimeRange sets the time range for the queries + TimeRange(time.Time, time.Time) +} diff --git a/wasp/benchspy_test.go b/wasp/benchspy_test.go new file mode 100644 index 000000000..1bc955c88 --- /dev/null +++ b/wasp/benchspy_test.go @@ -0,0 +1,111 @@ +package wasp_test + +import ( + "fmt" + "strconv" + "testing" + "time" + + "github.com/smartcontractkit/chainlink-testing-framework/wasp" + "github.com/smartcontractkit/chainlink-testing-framework/wasp/benchspy" + "github.com/stretchr/testify/require" +) + +func TestBenchSpyWithLokiQuery(t *testing.T) { + label := "benchspy" + + gen, err := wasp.NewGenerator(&wasp.Config{ + T: t, + LokiConfig: wasp.NewEnvLokiConfig(), + GenName: "vu", + Labels: map[string]string{ + "branch": label, + "commit": label, + }, + CallTimeout: 100 * time.Millisecond, + LoadType: wasp.VU, + Schedule: wasp.CombineAndRepeat( + 2, + wasp.Steps(10, 1, 10, 10*time.Second), + wasp.Plain(30, 15*time.Second), + wasp.Steps(20, -1, 10, 5*time.Second), + ), + VU: wasp.NewMockVU(&wasp.MockVirtualUserConfig{ + CallSleep: 50 * time.Millisecond, + }), + }) + require.NoError(t, err) + + currentReport := benchspy.StandardReport{ + BasicData: benchspy.BasicData{ + GeneratorConfigs: map[string]*wasp.Config{ + gen.Cfg.GenName: gen.Cfg, + }, + TestName: t.Name(), + TestStart: time.Now(), + CommitOrTag: "e7fc5826a572c09f8b93df3b9f674113372ce925", + }, + ResourceReporter: benchspy.ResourceReporter{ + ExecutionEnvironment: benchspy.ExecutionEnvironment_Docker, + }, + } + + lokiQueryExecutor := benchspy.NewLokiQuery( + map[string]string{ + "vu_over_time": fmt.Sprintf("max_over_time({branch=~\"%s\", commit=~\"%s\", go_test_name=~\"%s\", test_data_type=~\"stats\", gen_name=~\"%s\"} | json | unwrap current_instances [10s]) by (node_id, go_test_name, gen_name)", label, label, t.Name(), gen.Cfg.GenName), + }, + gen.Cfg.LokiConfig) + + currentReport.QueryExecutors = append(currentReport.QueryExecutors, lokiQueryExecutor) + + gen.Run(true) + currentReport.TestEnd = time.Now() + + fetchErr := currentReport.Fetch() + require.NoError(t, fetchErr, "failed to fetch current report") + + // path, storeErr := currentReport.Store() + // require.NoError(t, storeErr, "failed to store current report", path) + + previousReport := benchspy.StandardReport{ + BasicData: benchspy.BasicData{ + TestName: t.Name(), + CommitOrTag: "e7fc5826a572c09f8b93df3b9f674113372ce924", + }, + LocalReportStorage: benchspy.LocalReportStorage{ + Directory: "test_performance_reports", + }, + } + loadErr := previousReport.Load() + require.NoError(t, loadErr, "failed to load previous report") + + isComparableErrs := previousReport.IsComparable(currentReport) + require.Empty(t, isComparableErrs, "reports were not comparable", isComparableErrs) + require.NotEmpty(t, currentReport.QueryExecutors[0].Results()["vu_over_time"], "vu_over_time results were missing from current report") + require.NotEmpty(t, previousReport.QueryExecutors[0].Results()["vu_over_time"], "vu_over_time results were missing from current report") + require.Equal(t, len(currentReport.QueryExecutors[0].Results()["vu_over_time"]), len(previousReport.QueryExecutors[0].Results()["vu_over_time"]), "vu_over_time results are not the same length") + + // compare each result entry individually + for i := range currentReport.QueryExecutors[0].Results()["vu_over_time"] { + require.Equal(t, currentReport.QueryExecutors[0].Results()["vu_over_time"][i], previousReport.QueryExecutors[0].Results()["vu_over_time"][i], "vu_over_time results are not the same for given index") + } + + //compare averages + var currentSum float64 + for _, value := range currentReport.QueryExecutors[0].Results()["vu_over_time"] { + asFloat, err := strconv.ParseFloat(value, 64) + require.NoError(t, err, "failed to parse float") + currentSum += asFloat + } + currentAverage := currentSum / float64(len(currentReport.QueryExecutors[0].Results()["vu_over_time"])) + + var previousSum float64 + for _, value := range previousReport.QueryExecutors[0].Results()["vu_over_time"] { + asFloat, err := strconv.ParseFloat(value, 64) + require.NoError(t, err, "failed to parse float") + previousSum += asFloat + } + previousAverage := currentSum / float64(len(previousReport.QueryExecutors[0].Results()["vu_over_time"])) + + require.Equal(t, currentAverage, previousAverage, "vu_over_time averages are not the same") +} diff --git a/wasp/comparator/report.go b/wasp/comparator/report.go deleted file mode 100644 index 8174597a2..000000000 --- a/wasp/comparator/report.go +++ /dev/null @@ -1,246 +0,0 @@ -package comparator - -import ( - "context" - "encoding/json" - "fmt" - "net/url" - "strings" - "time" - - "github.com/pkg/errors" - - "github.com/smartcontractkit/chainlink-testing-framework/lib/client" - "github.com/smartcontractkit/chainlink-testing-framework/wasp" -) - -type Report interface { - // Store stores the report in a persistent storage and returns the path to it, or an error - Store() (string, error) - // Load loads the report from a persistent storage and returns it, or an error - Load() error - // Fetch populates the report with the data from the test - Fetch() error - // IsComparable checks whether both reports can be compared (e.g. test config is the same, app's resources are the same, queries or metrics used are the same, etc.), and returns a map of the differences and an error (if any difference is found) - IsComparable(otherReport Report) (bool, map[string]string, error) -} - -var directory = "performance_reports" - -type BasicData struct { - TestName string `json:"test_name"` - CommitOrTag string `json:"commit_or_tag"` - - // Test metrics - TestStart time.Time `json:"test_start_timestamp"` - TestEnd time.Time `json:"test_end_timestamp"` - - // all, generator settings, including segments - GeneratorConfigs map[string]*wasp.Config `json:"generator_configs"` -} - -type BasicReport struct { - BasicData - LocalReportStorage - ResourceReporter - - // Performance queries - // a map of name to query template, ex: "average cpu usage": "avg(rate(cpu_usage_seconds_total[5m]))" - LokiQueries map[string]string `json:"loki_queries"` - // Performance queries results - // can be anything, avg RPS, amount of errors, 95th percentile of CPU utilization, etc - Results map[string][]string `json:"results"` - // In case something went wrong - Errors []error `json:"errors"` - - LokiConfig *wasp.LokiConfig `json:"-"` -} - -func (b *BasicReport) Store() (string, error) { - return b.LocalReportStorage.Store(b.TestName, b.CommitOrTag, b) -} - -func (b *BasicReport) Load() error { - return b.LocalReportStorage.Load(b.TestName, b.CommitOrTag, b) -} - -func (b *BasicReport) Fetch() error { - if len(b.LokiQueries) == 0 { - return errors.New("there are no Loki queries, there's nothing to fetch. Please set them and try again") - } - if b.LokiConfig == nil { - return errors.New("loki config is missing. Please set it and try again") - } - if b.TestStart.IsZero() { - return errors.New("test start time is missing. We cannot query Loki without a time range. Please set it and try again") - } - if b.TestEnd.IsZero() { - return errors.New("test end time is missing. We cannot query Loki without a time range. Please set it and try again") - } - - splitAuth := strings.Split(b.LokiConfig.BasicAuth, ":") - var basicAuth client.LokiBasicAuth - if len(splitAuth) == 2 { - basicAuth = client.LokiBasicAuth{ - Login: splitAuth[0], - Password: splitAuth[1], - } - } - - b.Results = make(map[string][]string) - - for name, query := range b.LokiQueries { - queryParams := client.LokiQueryParams{ - Query: query, - StartTime: b.TestStart, - EndTime: b.TestEnd, - Limit: 1000, //TODO make this configurable - } - - parsedLokiUrl, err := url.Parse(b.LokiConfig.URL) - if err != nil { - return errors.Wrapf(err, "failed to parse Loki URL %s", b.LokiConfig.URL) - } - - lokiUrl := parsedLokiUrl.Scheme + "://" + parsedLokiUrl.Host - lokiClient := client.NewLokiClient(lokiUrl, b.LokiConfig.TenantID, basicAuth, queryParams) - - ctx, cancelFn := context.WithTimeout(context.Background(), b.LokiConfig.Timeout) - rawLogs, err := lokiClient.QueryLogs(ctx) - if err != nil { - b.Errors = append(b.Errors, err) - cancelFn() - continue - } - - cancelFn() - b.Results[name] = []string{} - for _, log := range rawLogs { - b.Results[name] = append(b.Results[name], log.Log) - } - } - - if len(b.Errors) > 0 { - return errors.New("there were errors while fetching the results. Please check the errors and try again") - } - - resourceErr := b.FetchResources() - if resourceErr != nil { - return resourceErr - } - - return nil -} - -func (b *BasicReport) IsComparable(otherReport BasicReport) (bool, []error) { - // check if generator configs are the same - // are all configs present? do they have the same schedule type? do they have the same segments? - // is call timeout the same? - // is rate limit timeout the same? - // would be good to be able to check if Gun and VU are the same, but idk yet how we could do that easily [hash the code?] - - if len(b.GeneratorConfigs) != len(otherReport.GeneratorConfigs) { - return false, []error{fmt.Errorf("generator configs count is different. Expected %d, got %d", len(b.GeneratorConfigs), len(otherReport.GeneratorConfigs))} - } - - for name1, cfg1 := range b.GeneratorConfigs { - if cfg2, ok := otherReport.GeneratorConfigs[name1]; !ok { - return false, []error{fmt.Errorf("generator config %s is missing from the other report", name1)} - } else { - if err := compareGeneratorConfigs(cfg1, cfg2); err != nil { - return false, []error{err} - } - } - } - - for name2 := range otherReport.GeneratorConfigs { - if _, ok := b.GeneratorConfigs[name2]; !ok { - return false, []error{fmt.Errorf("generator config %s is missing from the current report", name2)} - } - } - - if b.ExecutionEnvironment != otherReport.ExecutionEnvironment { - return false, []error{fmt.Errorf("execution environments are different. Expected %s, got %s", b.ExecutionEnvironment, otherReport.ExecutionEnvironment)} - } - - // check if pods resources are the same - // are all pods present? do they have the same resources? - if resourceErr := b.CompareResources(&otherReport.ResourceReporter); resourceErr != nil { - return false, []error{resourceErr} - } - - // check if queries are the same - // are all queries present? do they have the same template? - lokiQueriesErr := compareLokiQueries(b.LokiQueries, otherReport.LokiQueries) - if lokiQueriesErr != nil { - return false, []error{lokiQueriesErr} - } - - return true, nil -} - -func compareLokiQueries(this, other map[string]string) error { - if len(this) != len(other) { - return fmt.Errorf("queries count is different. Expected %d, got %d", len(this), len(other)) - } - - for name1, query1 := range this { - if query2, ok := other[name1]; !ok { - return fmt.Errorf("query %s is missing from the other report", name1) - } else { - if query1 != query2 { - return fmt.Errorf("query %s is different. Expected %s, got %s", name1, query1, query2) - } - } - } - - for name2 := range other { - if _, ok := this[name2]; !ok { - return fmt.Errorf("query %s is missing from the current report", name2) - } - } - - return nil -} - -func compareGeneratorConfigs(cfg1, cfg2 *wasp.Config) error { - if cfg1.LoadType != cfg2.LoadType { - return fmt.Errorf("load types are different. Expected %s, got %s", cfg1.LoadType, cfg2.LoadType) - } - - if len(cfg1.Schedule) != len(cfg2.Schedule) { - return fmt.Errorf("schedules are different. Expected %d, got %d", len(cfg1.Schedule), len(cfg2.Schedule)) - } - - for i, segment1 := range cfg1.Schedule { - segment2 := cfg2.Schedule[i] - if segment1 == nil { - return fmt.Errorf("schedule at index %d is nil in the current report", i) - } - if segment2 == nil { - return fmt.Errorf("schedule at index %d is nil in the other report", i) - } - if *segment1 != *segment2 { - return fmt.Errorf("schedules at index %d are different. Expected %s, got %s", i, mustMarshallSegment(segment1), mustMarshallSegment(segment2)) - } - } - - if cfg1.CallTimeout != cfg2.CallTimeout { - return fmt.Errorf("call timeouts are different. Expected %s, got %s", cfg1.CallTimeout, cfg2.CallTimeout) - } - - if cfg1.RateLimitUnitDuration != cfg2.RateLimitUnitDuration { - return fmt.Errorf("rate limit unit durations are different. Expected %s, got %s", cfg1.RateLimitUnitDuration, cfg2.RateLimitUnitDuration) - } - - return nil -} - -func mustMarshallSegment(segment *wasp.Segment) string { - segmentBytes, err := json.MarshalIndent(segment, "", " ") - if err != nil { - panic(err) - } - - return string(segmentBytes) -} diff --git a/wasp/comparator_test.go b/wasp/comparator_test.go deleted file mode 100644 index b2cbff4cd..000000000 --- a/wasp/comparator_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package wasp_test - -import ( - "fmt" - "testing" - "time" - - "github.com/smartcontractkit/chainlink-testing-framework/wasp" - "github.com/smartcontractkit/chainlink-testing-framework/wasp/comparator" - "github.com/stretchr/testify/require" -) - -func TestLokiComparator(t *testing.T) { - label := "performance_comparator_tool" - - gen, err := wasp.NewGenerator(&wasp.Config{ - T: t, - LokiConfig: wasp.NewEnvLokiConfig(), - GenName: "vu", - Labels: map[string]string{ - "branch": label, - "commit": label, - }, - CallTimeout: 100 * time.Millisecond, - LoadType: wasp.VU, - Schedule: wasp.CombineAndRepeat( - 2, - wasp.Steps(10, 1, 10, 10*time.Second), - wasp.Plain(30, 15*time.Second), - wasp.Steps(20, -1, 10, 5*time.Second), - ), - VU: wasp.NewMockVU(&wasp.MockVirtualUserConfig{ - CallSleep: 50 * time.Millisecond, - }), - }) - require.NoError(t, err) - - currentReport := comparator.BasicReport{ - BasicData: comparator.BasicData{ - GeneratorConfigs: map[string]*wasp.Config{ - gen.Cfg.GenName: gen.Cfg, - }, - TestName: "TestLokiComparator", - TestStart: time.Now(), - CommitOrTag: "e7fc5826a572c09f8b93df3b9f674113372ce923", - }, - ResourceReporter: comparator.ResourceReporter{ - ExecutionEnvironment: comparator.ExecutionEnvironment_Docker, - }, - LokiConfig: gen.Cfg.LokiConfig, - LokiQueries: map[string]string{ - "vu_over_time": fmt.Sprintf("max_over_time({branch=~\"%s\", commit=~\"%s\", go_test_name=~\"%s\", test_data_type=~\"stats\", gen_name=~\"%s\"} | json | unwrap current_instances [10s]) by (node_id, go_test_name, gen_name)", label, label, t.Name(), gen.Cfg.GenName), - }, - } - - gen.Run(true) - currentReport.TestEnd = time.Now() - - fetchErr := currentReport.Fetch() - require.NoError(t, fetchErr, "failed to fetch current report") - - path, storeErr := currentReport.Store() - require.NoError(t, storeErr, "failed to store current report", path) - - previousReport := comparator.BasicReport{ - BasicData: comparator.BasicData{ - TestName: "TestLokiComparator", - }, - } - loadErr := previousReport.Load() - require.NoError(t, loadErr, "failed to load previous report") - - isComparable, isComparableErrs := previousReport.IsComparable(currentReport) - require.True(t, isComparable, "reports are not comparable", isComparableErrs) - require.Empty(t, isComparableErrs, "reports were not comparable", isComparableErrs) - require.Equal(t, len(currentReport.Results["vu_over_time"]), len(previousReport.Results["vu_over_time"]), "vu_over_time results are not the same length") - - // compare each result individually - for i := range currentReport.Results["vu_over_time"] { - require.Equal(t, currentReport.Results["vu_over_time"][i], previousReport.Results["vu_over_time"][i], "vu_over_time results are not the same for given index") - } -} diff --git a/wasp/test_performance_reports/TestBenchSpyWithLokiQuery-e7fc5826a572c09f8b93df3b9f674113372ce924.json b/wasp/test_performance_reports/TestBenchSpyWithLokiQuery-e7fc5826a572c09f8b93df3b9f674113372ce924.json new file mode 100644 index 000000000..c44a0d86d --- /dev/null +++ b/wasp/test_performance_reports/TestBenchSpyWithLokiQuery-e7fc5826a572c09f8b93df3b9f674113372ce924.json @@ -0,0 +1,278 @@ +{ + "test_name": "TestBenchSpyWithLokiQuery", + "commit_or_tag": "e7fc5826a572c09f8b93df3b9f674113372ce924", + "test_start_timestamp": "2024-12-03T12:49:39.719104+01:00", + "test_end_timestamp": "2024-12-03T12:50:44.736575+01:00", + "generator_configs": { + "vu": { + "generator_name": "vu", + "load_type": "vu_schedule", + "schedule": [ + { + "from": 10, + "duration": 1000000000 + }, + { + "from": 11, + "duration": 1000000000 + }, + { + "from": 12, + "duration": 1000000000 + }, + { + "from": 13, + "duration": 1000000000 + }, + { + "from": 14, + "duration": 1000000000 + }, + { + "from": 15, + "duration": 1000000000 + }, + { + "from": 16, + "duration": 1000000000 + }, + { + "from": 17, + "duration": 1000000000 + }, + { + "from": 18, + "duration": 1000000000 + }, + { + "from": 19, + "duration": 1000000000 + }, + { + "from": 30, + "duration": 15000000000 + }, + { + "from": 20, + "duration": 500000000 + }, + { + "from": 19, + "duration": 500000000 + }, + { + "from": 18, + "duration": 500000000 + }, + { + "from": 17, + "duration": 500000000 + }, + { + "from": 16, + "duration": 500000000 + }, + { + "from": 15, + "duration": 500000000 + }, + { + "from": 14, + "duration": 500000000 + }, + { + "from": 13, + "duration": 500000000 + }, + { + "from": 12, + "duration": 500000000 + }, + { + "from": 11, + "duration": 500000000 + }, + { + "from": 10, + "duration": 1000000000 + }, + { + "from": 11, + "duration": 1000000000 + }, + { + "from": 12, + "duration": 1000000000 + }, + { + "from": 13, + "duration": 1000000000 + }, + { + "from": 14, + "duration": 1000000000 + }, + { + "from": 15, + "duration": 1000000000 + }, + { + "from": 16, + "duration": 1000000000 + }, + { + "from": 17, + "duration": 1000000000 + }, + { + "from": 18, + "duration": 1000000000 + }, + { + "from": 19, + "duration": 1000000000 + }, + { + "from": 30, + "duration": 15000000000 + }, + { + "from": 20, + "duration": 500000000 + }, + { + "from": 19, + "duration": 500000000 + }, + { + "from": 18, + "duration": 500000000 + }, + { + "from": 17, + "duration": 500000000 + }, + { + "from": 16, + "duration": 500000000 + }, + { + "from": 15, + "duration": 500000000 + }, + { + "from": 14, + "duration": 500000000 + }, + { + "from": 13, + "duration": 500000000 + }, + { + "from": 12, + "duration": 500000000 + }, + { + "from": 11, + "duration": 500000000 + } + ], + "rate_limit_unit_duration": 1000000000, + "call_timeout": 100000000 + } + }, + "directory": "performance_reports", + "execution_environment": "docker", + "pods_resources": null, + "container_resources": { + "/compose-grafana-1": { + "NanoCPUs": 0, + "CpuShares": 0, + "Memory": 0, + "MemorySwap": 0 + }, + "/compose-loki-1": { + "NanoCPUs": 0, + "CpuShares": 0, + "Memory": 0, + "MemorySwap": 0 + } + }, + "resource_selection_pattern": "", + "query_executors": [ + { + "kind": "loki", + "start_time": "2024-12-03T12:49:39.719104+01:00", + "end_time": "2024-12-03T12:50:44.736575+01:00", + "queries": { + "vu_over_time": "max_over_time({branch=~\"benchspy\", commit=~\"benchspy\", go_test_name=~\"TestBenchSpyWithLokiQuery\", test_data_type=~\"stats\", gen_name=~\"vu\"} | json | unwrap current_instances [10s]) by (node_id, go_test_name, gen_name)" + }, + "query_results": { + "vu_over_time": [ + "14", + "14", + "14", + "14", + "14", + "19", + "19", + "19", + "19", + "19", + "30", + "30", + "30", + "30", + "30", + "30", + "30", + "30", + "30", + "30", + "30", + "30", + "30", + "30", + "30", + "30", + "30", + "30", + "30", + "30", + "14", + "14", + "14", + "14", + "14", + "19", + "19", + "19", + "19", + "19", + "30", + "30", + "30", + "30", + "30", + "30", + "30", + "30", + "30", + "30", + "30", + "30", + "30", + "30", + "30", + "30", + "30", + "30", + "30", + "30", + "11" + ] + }, + "errors": null + } + ] +} \ No newline at end of file