Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

integration: properly parse output metrics and assert more things about them #44

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ go 1.21
toolchain go1.23.4

require (
github.com/prometheus/client_model v0.4.0
github.com/prometheus/common v0.42.0
github.com/sirupsen/logrus v1.9.3
github.com/spf13/afero v1.12.0
github.com/stretchr/testify v1.10.0
Expand All @@ -17,12 +19,14 @@ require (
github.com/fatih/color v1.18.0 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mstoykov/atlas v0.0.0-20220811071828-388f114305dd // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
go.opentelemetry.io/otel v1.29.0 // indirect
Expand Down
10 changes: 10 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
Expand All @@ -35,6 +38,8 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/mccutchen/go-httpbin v1.1.2-0.20190116014521-c5cb2f4802fa h1:lx8ZnNPwjkXSzOROz0cg69RlErRXs+L3eDkggASWKLo=
github.com/mccutchen/go-httpbin v1.1.2-0.20190116014521-c5cb2f4802fa/go.mod h1:fhpOYavp5g2K74XDl/ao2y4KvhqVtKlkg1e+0UaQv7I=
github.com/mstoykov/atlas v0.0.0-20220811071828-388f114305dd h1:AC3N94irbx2kWGA8f/2Ks7EQl2LxKIRQYuT9IJDwgiI=
Expand All @@ -43,6 +48,10 @@ github.com/mstoykov/envconfig v1.5.0 h1:E2FgWf73BQt0ddgn7aoITkQHmgwAcHup1s//MsS5
github.com/mstoykov/envconfig v1.5.0/go.mod h1:vk/d9jpexY2Z9Bb0uB4Ndesss1Sr0Z9ZiGUrg5o9VGk=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY=
github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM=
github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
Expand Down Expand Up @@ -77,6 +86,7 @@ golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc=
golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
Expand Down
206 changes: 147 additions & 59 deletions integration/integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,24 @@ package integration_test
import (
"bytes"
"context"
_ "embed"
"errors"
"io"
"os"
"os/exec"
"path/filepath"
"runtime"
"slices"
"testing"
"time"

prometheus "github.com/prometheus/client_model/go"
"github.com/prometheus/common/expfmt"
)

//go:embed test-script.js
var testScript []byte

func TestSMK6(t *testing.T) {
t.Parallel()

Expand All @@ -19,72 +29,150 @@ func TestSMK6(t *testing.T) {
smk6 = filepath.Join("..", "dist", "sm-k6-"+runtime.GOOS+"-"+runtime.GOARCH)
}

t.Run("metrics", func(t *testing.T) {
for _, tc := range []struct {
name string
script []byte
}{
{
name: "singleRequest",
script: scriptSingleRequest,
},
} {
tc := tc

t.Run(tc.name, func(t *testing.T) {
Comment on lines -22 to -34
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This used to create two subtests, one for metrics and then one for each script we run. As it turns out, we only test for metrics and we only run that script, so I removed these two subtests entirely and decreased indentation a bunch. Git does not do a good job displaying this indentation change.

Feel free to switch to the single commit view for a less noisy diff.

t.Parallel()

ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
t.Cleanup(cancel)

outFile := filepath.Join(t.TempDir(), "metrics.txt")

cmd := exec.CommandContext(ctx, smk6, "run", "-", "-o=sm="+outFile)
cmd.Stdin = bytes.NewReader(tc.script)
err := cmd.Run()
if err != nil {
t.Fatalf("running sm-k6: %v", err)
}
_, err := os.Stat(smk6)
if err != nil {
t.Fatalf("sm-k6 binary does not seem to exist, must be compiled before running this test: %v", err)
}

out, err := os.ReadFile(outFile)
if err != nil {
t.Fatalf("reading output metrics: %v", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
t.Cleanup(cancel)

outFile := filepath.Join(t.TempDir(), "metrics.txt")

cmd := exec.CommandContext(ctx, smk6, "run", "-", "-o=sm="+outFile)
cmd.Stdin = bytes.NewReader(testScript)
err = cmd.Run()
if err != nil {
t.Fatalf("running sm-k6: %v", err)
}

out, err := os.Open(outFile)
if err != nil {
t.Fatalf("reading output metrics: %v", err)
}

mfs := []*prometheus.MetricFamily{}
decoder := expfmt.NewDecoder(out, expfmt.FmtText)
for {
mf := &prometheus.MetricFamily{}
err := decoder.Decode(mf)
if errors.Is(err, io.EOF) {
break
}

if err != nil {
t.Fatalf("decoding metric: %v", err)
}

mfs = append(mfs, mf)

}
if len(mfs) == 0 {
t.Fatalf("no metrics decoded")
}

t.Run("wanted metrics are present", func(t *testing.T) {
t.Parallel()

wantedMetrics := []string{
"probe_checks_total",
"probe_data_received_bytes",
"probe_data_sent_bytes",
"probe_http_duration_seconds",
"probe_http_error_code",
"probe_http_got_expected_response",
"probe_http_info",
"probe_http_requests_failed_total",
"probe_http_requests_total",
"probe_http_ssl",
"probe_http_status_code",
"probe_http_total_duration_seconds",
"probe_http_version",
"probe_iteration_duration_seconds",
"probe_script_duration_seconds",
// Custom metrics:
"probe_waiting_time", "probe_my_counter", "probe_my_gauge",
}

for _, wanted := range wantedMetrics {
if !bytes.Contains(out, []byte(wanted+"{")) { // Add { to force an exact match.
t.Fatalf("Metric %q not found in output", wanted)
for _, wanted := range wantedMetrics {
if !slices.ContainsFunc(mfs, func(m *prometheus.MetricFamily) bool { return *m.Name == wanted }) {
t.Fatalf("Metric %q not found in output", wanted)
}
}
})

t.Run("unwanted metrics are not present", func(t *testing.T) {
t.Parallel()

// probe_ prefix is automatically checked.
unwantedMetrics := []string{
"probe_checks",
"probe_http_reqs", "probe_http_req_failed",
"probe_data_sent", "probe_data_received",
"http_req_duration", "iteration_duration",
"http_req_blocked", "http_req_connecting", "http_req_receiving", "http_req_sending", "http_req_tls_handshaking", "http_req_waiting",
"vus", "vus_max", "iterations",
}

for _, wanted := range unwantedMetrics {
if slices.ContainsFunc(mfs, func(m *prometheus.MetricFamily) bool { return *m.Name == wanted }) {
t.Fatalf("Metric %q not found in output", wanted)
}
}
})

t.Run("labels are present", func(t *testing.T) {
t.Parallel()

// requiredLabels maps metric names to label names that are required to be present on that metric.
requiredLabels := map[string][]string{
// FIXME: probe_http_info will not contain these labels if the request failed, so an instance of this metric
// fails this test.
//"probe_http_info": {"tls_version", "proto"},
"probe_http_duration_seconds": {"phase"},
"checks_total": {"result"},
}

for _, mf := range mfs {
for _, m := range mf.Metric {
required := requiredLabels[*mf.Name]
for _, req := range required {
if !slices.ContainsFunc(m.Label, func(lp *prometheus.LabelPair) bool { return *lp.Name == req }) {
t.Fatalf("metric %q does not contain label %q", *mf.Name, req)
}
}
})
}
}
})
}

var wantedMetrics = []string{
"probe_data_received_bytes",
"probe_data_sent_bytes",
"probe_http_duration_seconds",
"probe_http_error_code",
"probe_http_got_expected_response",
"probe_http_info",
"probe_http_requests_failed_total",
"probe_http_requests_total",
"probe_http_ssl",
"probe_http_status_code",
"probe_http_total_duration_seconds",
"probe_http_version",
"probe_iteration_duration_seconds",
"probe_script_duration_seconds",
}
t.Run("labels are not present", func(t *testing.T) {
t.Parallel()

var scriptSingleRequest = []byte(`
import http from 'k6/http';
// The keys of this map are the set of labels that are not allowed to be present in most metrics.
// The values represent the list of metrics that are _allowed_ to have this label, as an exception.
// If a metric contains a label (key), and that metric is not in the list (value), the test fails.
type exceptForMetrics []string
forbiddenLabels := map[string]exceptForMetrics{
"tls_version": {"probe_http_info"},
"proto": {"probe_http_info"},
"error": {"probe_http_info"},
"expected_response": {"probe_http_got_expected_response"},
"group": {},
}

export const options = {
iterations: 1,
};
for _, mf := range mfs {
for _, m := range mf.Metric {
for _, labelPair := range m.Label {
allowedMetrics, isForbidden := forbiddenLabels[*labelPair.Name]
if !isForbidden {
continue
}

export default function () {
const response = http.get('https://test-api.k6.io/public/crocodiles/');
}`)
if !slices.Contains(allowedMetrics, *mf.Name) {
t.Fatalf("%q should not contain label %q", *mf.Name, *labelPair.Name)
}
}
}
}
})
}
41 changes: 41 additions & 0 deletions integration/test-script.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
import { check } from 'k6';
import http from 'k6/http';
import { Trend, Counter, Gauge } from 'k6/metrics';

const testHost = __ENV.TEST_HOST ? __ENV.TEST_HOST : "test-api.k6.io";

const myTrend = new Trend('waiting_time');
const myCounter = new Counter('my_counter');
const myGauge = new Gauge('my_gauge');

export const options = {
iterations: 1,
};

export default function () {
myTrend.add(0.5);
myTrend.add(0.6);
myTrend.add(0.7);

myGauge.add(5);
myGauge.add(6); // Discards previous value.

myCounter.add(1);
myCounter.add(2);

check({}, {
'something': () => false,
}
);
check({}, {
'something': () => false,
}
);

http.get(`http://${testHost}`); // non-https.
http.get(`https://${testHost}/public/crocodiles/`);
http.get(`https://${testHost}/public/crocodiles2/`); // 404
http.get(`https://${testHost}/public/crocodiles3/`); // 404
http.get(`https://${testHost}/public/crocodiles4/`); // 404
http.get(`http://fail.internal/public/crocodiles4/`); // failed
}
Loading