[TT-1741] performance comparison tool

lib/client/loki.go

@@ -171,7 +171,17 @@ func (lc *LokiClient) extractRawLogEntries(lokiResp LokiResponse) []LokiLogEntry
 
 	for _, result := range lokiResp.Data.Result {
 		for _, entry := range result.Values {
-			timestamp := entry[0].(string)
+			var timestamp string
+			if timestampString, ok := entry[0].(string); ok {
+				timestamp = timestampString
+			} else if timestampInt, ok := entry[0].(int); ok {
+				timestamp = fmt.Sprintf("%d", timestampInt)
+			} else if timestampFloat, ok := entry[0].(float64); ok {
+				timestamp = fmt.Sprintf("%f", timestampFloat)
+			} else {
+				lc.Logger.Error().Msg("Error parsing timestamp")
+				continue
+			}
 			logLine := entry[1].(string)
 			logEntries = append(logEntries, LokiLogEntry{
 				Timestamp: timestamp,

wasp/.gitignore

@@ -2,7 +2,7 @@ bin/
 .vscode/
 .idea/
 .direnv/
-
+performance_reports/
 k3dvolume/
 .private.env
 .envrc.ci

wasp/benchspy/TO_DO.md

@@ -0,0 +1,11 @@
+Known things to do:
+- [ ] add logger
+- [ ] add tests
+- [ ] write documentation
+- [ ] test with Docker app (focus: resources)
+- [ ] test with k8s app (focus: resources)
+- [ ] add report builder (?)
+- [ ] add wrapper function for executing some code and then creating a report
+- [ ] think what to do with errors... do we really need a slice?
+- [ ] if TestEnd is zero, default to `time.Now()`?
+- [ ] add helper method for a profile what would create a report based on all generators?

wasp/benchspy/basic.go

@@ -0,0 +1,107 @@
+package benchspy
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/smartcontractkit/chainlink-testing-framework/wasp"
+)
+
+// BasicData is the basic data that is required for a report, common to all reports
+type BasicData struct {
+	TestName    string `json:"test_name"`
+	CommitOrTag string `json:"commit_or_tag"`
+
+	// Test metrics
+	TestStart time.Time `json:"test_start_timestamp"`
+	TestEnd   time.Time `json:"test_end_timestamp"`
+
+	// all, generator settings, including segments
+	GeneratorConfigs map[string]*wasp.Config `json:"generator_configs"`
+}
+
+func (b *BasicData) Validate() error {
+	if b.TestStart.IsZero() {
+		return errors.New("test start time is missing. We cannot query Loki without a time range. Please set it and try again")
+	}
+	if b.TestEnd.IsZero() {
+		return errors.New("test end time is missing. We cannot query Loki without a time range. Please set it and try again")
+	}
+
+	if len(b.GeneratorConfigs) == 0 {
+		return errors.New("generator configs are missing. At least one is required. Please set them and try again")
+	}
+
+	return nil
+}
+
+func (b *BasicData) IsComparable(otherData BasicData) error {
+	// are all configs present? do they have the same schedule type? do they have the same segments? is call timeout the same? is rate limit timeout the same?
+	if len(b.GeneratorConfigs) != len(otherData.GeneratorConfigs) {
+		return fmt.Errorf("generator configs count is different. Expected %d, got %d", len(b.GeneratorConfigs), len(otherData.GeneratorConfigs))
+	}
+
+	for name1, cfg1 := range b.GeneratorConfigs {
+		if cfg2, ok := otherData.GeneratorConfigs[name1]; !ok {
+			return fmt.Errorf("generator config %s is missing from the other report", name1)
+		} else {
+			if err := compareGeneratorConfigs(cfg1, cfg2); err != nil {
+				return err
+			}
+		}
+	}
+
+	for name2 := range otherData.GeneratorConfigs {
+		if _, ok := b.GeneratorConfigs[name2]; !ok {
+			return fmt.Errorf("generator config %s is missing from the current report", name2)
+		}
+	}
+
+	// TODO: would be good to be able to check if Gun and VU are the same, but idk yet how we could do that easily [hash the code?]
+
+	return nil
+}
+
+func compareGeneratorConfigs(cfg1, cfg2 *wasp.Config) error {
+	if cfg1.LoadType != cfg2.LoadType {
+		return fmt.Errorf("load types are different. Expected %s, got %s", cfg1.LoadType, cfg2.LoadType)
+	}
+
+	if len(cfg1.Schedule) != len(cfg2.Schedule) {
+		return fmt.Errorf("schedules are different. Expected %d, got %d", len(cfg1.Schedule), len(cfg2.Schedule))
+	}
+
+	for i, segment1 := range cfg1.Schedule {
+		segment2 := cfg2.Schedule[i]
+		if segment1 == nil {
+			return fmt.Errorf("schedule at index %d is nil in the current report", i)
+		}
+		if segment2 == nil {
+			return fmt.Errorf("schedule at index %d is nil in the other report", i)
+		}
+		if *segment1 != *segment2 {
+			return fmt.Errorf("schedules at index %d are different. Expected %s, got %s", i, mustMarshallSegment(segment1), mustMarshallSegment(segment2))
+		}
+	}
+
+	if cfg1.CallTimeout != cfg2.CallTimeout {
+		return fmt.Errorf("call timeouts are different. Expected %s, got %s", cfg1.CallTimeout, cfg2.CallTimeout)
+	}
+
+	if cfg1.RateLimitUnitDuration != cfg2.RateLimitUnitDuration {
+		return fmt.Errorf("rate limit unit durations are different. Expected %s, got %s", cfg1.RateLimitUnitDuration, cfg2.RateLimitUnitDuration)
+	}
+
+	return nil
+}
+
+func mustMarshallSegment(segment *wasp.Segment) string {
+	segmentBytes, err := json.MarshalIndent(segment, "", " ")
+	if err != nil {
+		panic(err)
+	}
+
+	return string(segmentBytes)
+}

wasp/benchspy/log.go

@@ -0,0 +1,32 @@
+package benchspy
+
+import (
+	"os"
+
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+)
+
+const (
+	LogLevelEnvVar = "BENCHSPY_LOG_LEVEL"
+)
+
+var (
+	L zerolog.Logger
+)
+
+func init() {
+	initDefaultLogging()
+}
+
+func initDefaultLogging() {
+	lvlStr := os.Getenv(LogLevelEnvVar)
+	if lvlStr == "" {
+		lvlStr = "info"
+	}
+	lvl, err := zerolog.ParseLevel(lvlStr)
+	if err != nil {
+		panic(err)
+	}
+	L = log.Output(zerolog.ConsoleWriter{Out: os.Stderr}).Level(lvl)
+}

wasp/benchspy/loki.go

@@ -0,0 +1,161 @@
+package benchspy
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"reflect"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/smartcontractkit/chainlink-testing-framework/lib/client"
+	"github.com/smartcontractkit/chainlink-testing-framework/wasp"
+	"golang.org/x/sync/errgroup"
+)
+
+func NewLokiQueryExecutor(queries map[string]string, lokiConfig *wasp.LokiConfig) *LokiQueryExecutor {
+	return &LokiQueryExecutor{
+		Kind:         "loki",
+		Queries:      queries,
+		LokiConfig:   lokiConfig,
+		QueryResults: make(map[string][]string),
+	}
+}
+
+type LokiQueryExecutor struct {
+	Kind string `json:"kind"`
+	// Test metrics
+	StartTime time.Time `json:"start_time"`
+	EndTime   time.Time `json:"end_time"`
+
+	// Performance queries
+	// a map of name to query template, ex: "average cpu usage": "avg(rate(cpu_usage_seconds_total[5m]))"
+	Queries map[string]string `json:"queries"`
+	// Performance queries results
+	// can be anything, avg RPS, amount of errors, 95th percentile of CPU utilization, etc
+	QueryResults map[string][]string `json:"query_results"`
+	// In case something went wrong, but sure we need it
+	Errors []error `json:"errors"`
+
+	LokiConfig *wasp.LokiConfig `json:"-"`
+}
+
+func (l *LokiQueryExecutor) Results() map[string][]string {
+	return l.QueryResults
+}
+
+func (l *LokiQueryExecutor) IsComparable(otherQueryExecutor QueryExecutor) error {
+	otherType := reflect.TypeOf(otherQueryExecutor)
+
+	if otherType != reflect.TypeOf(l) {
+		return fmt.Errorf("expected type %s, got %s", reflect.TypeOf(l), otherType)
+	}
+
+	return l.compareLokiQueries(otherQueryExecutor.(*LokiQueryExecutor).Queries)
+}
+
+func (l *LokiQueryExecutor) Validate() error {
+	if len(l.Queries) == 0 {
+		return errors.New("there are no Loki queries, there's nothing to fetch. Please set them and try again")
+	}
+	if l.LokiConfig == nil {
+		return errors.New("loki config is missing. Please set it and try again")
+	}
+
+	return nil
+}
+
+func (l *LokiQueryExecutor) Execute(ctx context.Context) error {
+	splitAuth := strings.Split(l.LokiConfig.BasicAuth, ":")
+	var basicAuth client.LokiBasicAuth
+	if len(splitAuth) == 2 {
+		basicAuth = client.LokiBasicAuth{
+			Login:    splitAuth[0],
+			Password: splitAuth[1],
+		}
+	}
+
+	l.QueryResults = make(map[string][]string)
+	resultCh := make(chan map[string][]string, len(l.Queries))
+	errGroup, errCtx := errgroup.WithContext(ctx)
+
+	for name, query := range l.Queries {
+		errGroup.Go(func() error {
+			queryParams := client.LokiQueryParams{
+				Query:     query,
+				StartTime: l.StartTime,
+				EndTime:   l.EndTime,
+				Limit:     1000, //TODO make this configurable
+			}
+
+			parsedLokiUrl, err := url.Parse(l.LokiConfig.URL)
+			if err != nil {
+				return errors.Wrapf(err, "failed to parse Loki URL %s", l.LokiConfig.URL)
+			}
+
+			lokiUrl := parsedLokiUrl.Scheme + "://" + parsedLokiUrl.Host
+			lokiClient := client.NewLokiClient(lokiUrl, l.LokiConfig.TenantID, basicAuth, queryParams)
+
+			rawLogs, err := lokiClient.QueryLogs(errCtx)
+			if err != nil {
+				return errors.Wrapf(err, "failed to query logs for %s", name)
+			}
+
+			resultMap := make(map[string][]string)
+			for _, log := range rawLogs {
+				resultMap[name] = append(resultMap[name], log.Log)
+			}
+
+			select {
+			case resultCh <- resultMap:
+				return nil
+			case <-errCtx.Done():
+				return errCtx.Err() // Allows goroutine to exit if timeout occurs
+			}
+		})
+	}
+
+	if err := errGroup.Wait(); err != nil {
+		return errors.Wrap(err, "failed to execute Loki queries")
+	}
+
+	for i := 0; i < len(l.Queries); i++ {
+		result := <-resultCh
+		for name, logs := range result {
+			l.QueryResults[name] = logs
+		}
+	}
+
+	return nil
+}
+
+func (l *LokiQueryExecutor) compareLokiQueries(other map[string]string) error {
+	this := l.Queries
+	if len(this) != len(other) {
+		return fmt.Errorf("queries count is different. Expected %d, got %d", len(this), len(other))
+	}
+
+	for name1, query1 := range this {
+		if query2, ok := other[name1]; !ok {
+			return fmt.Errorf("query %s is missing from the other report", name1)
+		} else {
+			if query1 != query2 {
+				return fmt.Errorf("query %s is different. Expected %s, got %s", name1, query1, query2)
+			}
+		}
+	}
+
+	for name2 := range other {
+		if _, ok := this[name2]; !ok {
+			return fmt.Errorf("query %s is missing from the current report", name2)
+		}
+	}
+
+	return nil
+}
+
+func (l *LokiQueryExecutor) TimeRange(start, end time.Time) {
+	l.StartTime = start
+	l.EndTime = end
+}