metric: front-port a regression test

Front-ports some testing improvements from cockroachdb#88331.

Note that 22.2 and master don't exhibit the bug fixed in that
PR since we switched to using prometheus histograms in cockroachdb#86671.

Release note: None

pkg/util/metric/BUILD.bazel

@@ -42,15 +42,19 @@ go_test(
     size = "small",
     srcs = [
         "histogram_buckets_test.go",
+        "metric_ext_test.go",
         "metric_test.go",
         "prometheus_exporter_test.go",
         "prometheus_rule_exporter_test.go",
         "registry_test.go",
         "rule_test.go",
     ],
     args = ["-test.timeout=55s"],
+    data = glob(["testdata/**"]),
     embed = [":metric"],
     deps = [
+        "//pkg/testutils",
+        "//pkg/testutils/echotest",
         "//pkg/util/log",
         "@com_github_dustin_go_humanize//:go-humanize",
         "@com_github_kr_pretty//:pretty",

pkg/util/metric/aggmetric/BUILD.bazel

@@ -27,10 +27,13 @@ go_test(
     size = "small",
     srcs = ["agg_metric_test.go"],
     args = ["-test.timeout=55s"],
+    data = glob(["testdata/**"]),
     deps = [
         ":aggmetric",
         "//pkg/base",
         "//pkg/roachpb",
+        "//pkg/testutils",
+        "//pkg/testutils/echotest",
         "//pkg/util/leaktest",
         "//pkg/util/metric",
         "@com_github_prometheus_client_model//go",

pkg/util/metric/aggmetric/agg_metric_test.go

@@ -19,6 +19,8 @@ import (
 
 	"github.com/cockroachdb/cockroach/pkg/base"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
+	"github.com/cockroachdb/cockroach/pkg/testutils"
+	"github.com/cockroachdb/cockroach/pkg/testutils/echotest"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
 	"github.com/cockroachdb/cockroach/pkg/util/metric"
 	"github.com/cockroachdb/cockroach/pkg/util/metric/aggmetric"
@@ -88,102 +90,15 @@ func TestAggMetric(t *testing.T) {
 		f3.Update(2.5)
 		h2.RecordValue(10)
 		h3.RecordValue(90)
-		require.Equal(t,
-			`bar_gauge 4
-bar_gauge{tenant_id="2"} 2
-bar_gauge{tenant_id="3"} 2
-baz_gauge 4
-baz_gauge{tenant_id="2"} 1.5
-baz_gauge{tenant_id="3"} 2.5
-foo_counter 6
-foo_counter{tenant_id="2"} 2
-foo_counter{tenant_id="3"} 4
-histo_gram_bucket{le="+Inf"} 2
-histo_gram_bucket{le="1"} 0
-histo_gram_bucket{le="1024"} 2
-histo_gram_bucket{le="128"} 2
-histo_gram_bucket{le="16"} 1
-histo_gram_bucket{le="2"} 0
-histo_gram_bucket{le="256"} 2
-histo_gram_bucket{le="32"} 1
-histo_gram_bucket{le="4"} 0
-histo_gram_bucket{le="512"} 2
-histo_gram_bucket{le="64"} 1
-histo_gram_bucket{le="8"} 0
-histo_gram_bucket{tenant_id="2",le="+Inf"} 1
-histo_gram_bucket{tenant_id="2",le="1"} 0
-histo_gram_bucket{tenant_id="2",le="1024"} 1
-histo_gram_bucket{tenant_id="2",le="128"} 1
-histo_gram_bucket{tenant_id="2",le="16"} 1
-histo_gram_bucket{tenant_id="2",le="2"} 0
-histo_gram_bucket{tenant_id="2",le="256"} 1
-histo_gram_bucket{tenant_id="2",le="32"} 1
-histo_gram_bucket{tenant_id="2",le="4"} 0
-histo_gram_bucket{tenant_id="2",le="512"} 1
-histo_gram_bucket{tenant_id="2",le="64"} 1
-histo_gram_bucket{tenant_id="2",le="8"} 0
-histo_gram_bucket{tenant_id="3",le="+Inf"} 1
-histo_gram_bucket{tenant_id="3",le="1"} 0
-histo_gram_bucket{tenant_id="3",le="1024"} 1
-histo_gram_bucket{tenant_id="3",le="128"} 1
-histo_gram_bucket{tenant_id="3",le="16"} 0
-histo_gram_bucket{tenant_id="3",le="2"} 0
-histo_gram_bucket{tenant_id="3",le="256"} 1
-histo_gram_bucket{tenant_id="3",le="32"} 0
-histo_gram_bucket{tenant_id="3",le="4"} 0
-histo_gram_bucket{tenant_id="3",le="512"} 1
-histo_gram_bucket{tenant_id="3",le="64"} 0
-histo_gram_bucket{tenant_id="3",le="8"} 0
-histo_gram_count 2
-histo_gram_count{tenant_id="2"} 1
-histo_gram_count{tenant_id="3"} 1
-histo_gram_sum 100
-histo_gram_sum{tenant_id="2"} 10
-histo_gram_sum{tenant_id="3"} 90`,
-			writePrometheusMetrics(t))
+		echotest.Require(t, writePrometheusMetrics(t), testutils.TestDataPath(t, "basic.txt"))
 	})
 
 	t.Run("destroy", func(t *testing.T) {
 		g3.Destroy()
 		c2.Destroy()
 		f3.Destroy()
 		h3.Destroy()
-		require.Equal(t,
-			`bar_gauge 2
-bar_gauge{tenant_id="2"} 2
-baz_gauge 1.5
-baz_gauge{tenant_id="2"} 1.5
-foo_counter 6
-foo_counter{tenant_id="3"} 4
-histo_gram_bucket{le="+Inf"} 2
-histo_gram_bucket{le="1"} 0
-histo_gram_bucket{le="1024"} 2
-histo_gram_bucket{le="128"} 2
-histo_gram_bucket{le="16"} 1
-histo_gram_bucket{le="2"} 0
-histo_gram_bucket{le="256"} 2
-histo_gram_bucket{le="32"} 1
-histo_gram_bucket{le="4"} 0
-histo_gram_bucket{le="512"} 2
-histo_gram_bucket{le="64"} 1
-histo_gram_bucket{le="8"} 0
-histo_gram_bucket{tenant_id="2",le="+Inf"} 1
-histo_gram_bucket{tenant_id="2",le="1"} 0
-histo_gram_bucket{tenant_id="2",le="1024"} 1
-histo_gram_bucket{tenant_id="2",le="128"} 1
-histo_gram_bucket{tenant_id="2",le="16"} 1
-histo_gram_bucket{tenant_id="2",le="2"} 0
-histo_gram_bucket{tenant_id="2",le="256"} 1
-histo_gram_bucket{tenant_id="2",le="32"} 1
-histo_gram_bucket{tenant_id="2",le="4"} 0
-histo_gram_bucket{tenant_id="2",le="512"} 1
-histo_gram_bucket{tenant_id="2",le="64"} 1
-histo_gram_bucket{tenant_id="2",le="8"} 0
-histo_gram_count 2
-histo_gram_count{tenant_id="2"} 1
-histo_gram_sum 100
-histo_gram_sum{tenant_id="2"} 10`,
-			writePrometheusMetrics(t))
+		echotest.Require(t, writePrometheusMetrics(t), testutils.TestDataPath(t, "destroy.txt"))
 	})
 
 	t.Run("panic on already exists", func(t *testing.T) {
@@ -204,59 +119,7 @@ histo_gram_sum{tenant_id="2"} 10`,
 		c2 = c.AddChild(tenant2.String())
 		f3 = f.AddChild(tenant3.String())
 		h3 = h.AddChild(tenant3.String())
-		require.Equal(t,
-			`bar_gauge 2
-bar_gauge{tenant_id="2"} 2
-bar_gauge{tenant_id="3"} 0
-baz_gauge 1.5
-baz_gauge{tenant_id="2"} 1.5
-baz_gauge{tenant_id="3"} 0
-foo_counter 6
-foo_counter{tenant_id="2"} 0
-foo_counter{tenant_id="3"} 4
-histo_gram_bucket{le="+Inf"} 2
-histo_gram_bucket{le="1"} 0
-histo_gram_bucket{le="1024"} 2
-histo_gram_bucket{le="128"} 2
-histo_gram_bucket{le="16"} 1
-histo_gram_bucket{le="2"} 0
-histo_gram_bucket{le="256"} 2
-histo_gram_bucket{le="32"} 1
-histo_gram_bucket{le="4"} 0
-histo_gram_bucket{le="512"} 2
-histo_gram_bucket{le="64"} 1
-histo_gram_bucket{le="8"} 0
-histo_gram_bucket{tenant_id="2",le="+Inf"} 1
-histo_gram_bucket{tenant_id="2",le="1"} 0
-histo_gram_bucket{tenant_id="2",le="1024"} 1
-histo_gram_bucket{tenant_id="2",le="128"} 1
-histo_gram_bucket{tenant_id="2",le="16"} 1
-histo_gram_bucket{tenant_id="2",le="2"} 0
-histo_gram_bucket{tenant_id="2",le="256"} 1
-histo_gram_bucket{tenant_id="2",le="32"} 1
-histo_gram_bucket{tenant_id="2",le="4"} 0
-histo_gram_bucket{tenant_id="2",le="512"} 1
-histo_gram_bucket{tenant_id="2",le="64"} 1
-histo_gram_bucket{tenant_id="2",le="8"} 0
-histo_gram_bucket{tenant_id="3",le="+Inf"} 0
-histo_gram_bucket{tenant_id="3",le="1"} 0
-histo_gram_bucket{tenant_id="3",le="1024"} 0
-histo_gram_bucket{tenant_id="3",le="128"} 0
-histo_gram_bucket{tenant_id="3",le="16"} 0
-histo_gram_bucket{tenant_id="3",le="2"} 0
-histo_gram_bucket{tenant_id="3",le="256"} 0
-histo_gram_bucket{tenant_id="3",le="32"} 0
-histo_gram_bucket{tenant_id="3",le="4"} 0
-histo_gram_bucket{tenant_id="3",le="512"} 0
-histo_gram_bucket{tenant_id="3",le="64"} 0
-histo_gram_bucket{tenant_id="3",le="8"} 0
-histo_gram_count 2
-histo_gram_count{tenant_id="2"} 1
-histo_gram_count{tenant_id="3"} 0
-histo_gram_sum 100
-histo_gram_sum{tenant_id="2"} 10
-histo_gram_sum{tenant_id="3"} 0`,
-			writePrometheusMetrics(t))
+		echotest.Require(t, writePrometheusMetrics(t), testutils.TestDataPath(t, "add_after_destroy.txt"))
 	})
 
 	t.Run("panic on label length mismatch", func(t *testing.T) {

pkg/util/metric/aggmetric/testdata/add_after_destroy.txt

@@ -0,0 +1,53 @@
+echo
+----
+bar_gauge 2
+bar_gauge{tenant_id="2"} 2
+bar_gauge{tenant_id="3"} 0
+baz_gauge 1.5
+baz_gauge{tenant_id="2"} 1.5
+baz_gauge{tenant_id="3"} 0
+foo_counter 6
+foo_counter{tenant_id="2"} 0
+foo_counter{tenant_id="3"} 4
+histo_gram_bucket{le="+Inf"} 2
+histo_gram_bucket{le="1"} 0
+histo_gram_bucket{le="1024"} 2
+histo_gram_bucket{le="128"} 2
+histo_gram_bucket{le="16"} 1
+histo_gram_bucket{le="2"} 0
+histo_gram_bucket{le="256"} 2
+histo_gram_bucket{le="32"} 1
+histo_gram_bucket{le="4"} 0
+histo_gram_bucket{le="512"} 2
+histo_gram_bucket{le="64"} 1
+histo_gram_bucket{le="8"} 0
+histo_gram_bucket{tenant_id="2",le="+Inf"} 1
+histo_gram_bucket{tenant_id="2",le="1"} 0
+histo_gram_bucket{tenant_id="2",le="1024"} 1
+histo_gram_bucket{tenant_id="2",le="128"} 1
+histo_gram_bucket{tenant_id="2",le="16"} 1
+histo_gram_bucket{tenant_id="2",le="2"} 0
+histo_gram_bucket{tenant_id="2",le="256"} 1
+histo_gram_bucket{tenant_id="2",le="32"} 1
+histo_gram_bucket{tenant_id="2",le="4"} 0
+histo_gram_bucket{tenant_id="2",le="512"} 1
+histo_gram_bucket{tenant_id="2",le="64"} 1
+histo_gram_bucket{tenant_id="2",le="8"} 0
+histo_gram_bucket{tenant_id="3",le="+Inf"} 0
+histo_gram_bucket{tenant_id="3",le="1"} 0
+histo_gram_bucket{tenant_id="3",le="1024"} 0
+histo_gram_bucket{tenant_id="3",le="128"} 0
+histo_gram_bucket{tenant_id="3",le="16"} 0
+histo_gram_bucket{tenant_id="3",le="2"} 0
+histo_gram_bucket{tenant_id="3",le="256"} 0
+histo_gram_bucket{tenant_id="3",le="32"} 0
+histo_gram_bucket{tenant_id="3",le="4"} 0
+histo_gram_bucket{tenant_id="3",le="512"} 0
+histo_gram_bucket{tenant_id="3",le="64"} 0
+histo_gram_bucket{tenant_id="3",le="8"} 0
+histo_gram_count 2
+histo_gram_count{tenant_id="2"} 1
+histo_gram_count{tenant_id="3"} 0
+histo_gram_sum 100
+histo_gram_sum{tenant_id="2"} 10
+histo_gram_sum{tenant_id="3"} 0

pkg/util/metric/aggmetric/testdata/basic.txt

@@ -0,0 +1,53 @@
+echo
+----
+bar_gauge 4
+bar_gauge{tenant_id="2"} 2
+bar_gauge{tenant_id="3"} 2
+baz_gauge 4
+baz_gauge{tenant_id="2"} 1.5
+baz_gauge{tenant_id="3"} 2.5
+foo_counter 6
+foo_counter{tenant_id="2"} 2
+foo_counter{tenant_id="3"} 4
+histo_gram_bucket{le="+Inf"} 2
+histo_gram_bucket{le="1"} 0
+histo_gram_bucket{le="1024"} 2
+histo_gram_bucket{le="128"} 2
+histo_gram_bucket{le="16"} 1
+histo_gram_bucket{le="2"} 0
+histo_gram_bucket{le="256"} 2
+histo_gram_bucket{le="32"} 1
+histo_gram_bucket{le="4"} 0
+histo_gram_bucket{le="512"} 2
+histo_gram_bucket{le="64"} 1
+histo_gram_bucket{le="8"} 0
+histo_gram_bucket{tenant_id="2",le="+Inf"} 1
+histo_gram_bucket{tenant_id="2",le="1"} 0
+histo_gram_bucket{tenant_id="2",le="1024"} 1
+histo_gram_bucket{tenant_id="2",le="128"} 1
+histo_gram_bucket{tenant_id="2",le="16"} 1
+histo_gram_bucket{tenant_id="2",le="2"} 0
+histo_gram_bucket{tenant_id="2",le="256"} 1
+histo_gram_bucket{tenant_id="2",le="32"} 1
+histo_gram_bucket{tenant_id="2",le="4"} 0
+histo_gram_bucket{tenant_id="2",le="512"} 1
+histo_gram_bucket{tenant_id="2",le="64"} 1
+histo_gram_bucket{tenant_id="2",le="8"} 0
+histo_gram_bucket{tenant_id="3",le="+Inf"} 1
+histo_gram_bucket{tenant_id="3",le="1"} 0
+histo_gram_bucket{tenant_id="3",le="1024"} 1
+histo_gram_bucket{tenant_id="3",le="128"} 1
+histo_gram_bucket{tenant_id="3",le="16"} 0
+histo_gram_bucket{tenant_id="3",le="2"} 0
+histo_gram_bucket{tenant_id="3",le="256"} 1
+histo_gram_bucket{tenant_id="3",le="32"} 0
+histo_gram_bucket{tenant_id="3",le="4"} 0
+histo_gram_bucket{tenant_id="3",le="512"} 1
+histo_gram_bucket{tenant_id="3",le="64"} 0
+histo_gram_bucket{tenant_id="3",le="8"} 0
+histo_gram_count 2
+histo_gram_count{tenant_id="2"} 1
+histo_gram_count{tenant_id="3"} 1
+histo_gram_sum 100
+histo_gram_sum{tenant_id="2"} 10
+histo_gram_sum{tenant_id="3"} 90

pkg/util/metric/aggmetric/testdata/destroy.txt

@@ -0,0 +1,36 @@
+echo
+----
+bar_gauge 2
+bar_gauge{tenant_id="2"} 2
+baz_gauge 1.5
+baz_gauge{tenant_id="2"} 1.5
+foo_counter 6
+foo_counter{tenant_id="3"} 4
+histo_gram_bucket{le="+Inf"} 2
+histo_gram_bucket{le="1"} 0
+histo_gram_bucket{le="1024"} 2
+histo_gram_bucket{le="128"} 2
+histo_gram_bucket{le="16"} 1
+histo_gram_bucket{le="2"} 0
+histo_gram_bucket{le="256"} 2
+histo_gram_bucket{le="32"} 1
+histo_gram_bucket{le="4"} 0
+histo_gram_bucket{le="512"} 2
+histo_gram_bucket{le="64"} 1
+histo_gram_bucket{le="8"} 0
+histo_gram_bucket{tenant_id="2",le="+Inf"} 1
+histo_gram_bucket{tenant_id="2",le="1"} 0
+histo_gram_bucket{tenant_id="2",le="1024"} 1
+histo_gram_bucket{tenant_id="2",le="128"} 1
+histo_gram_bucket{tenant_id="2",le="16"} 1
+histo_gram_bucket{tenant_id="2",le="2"} 0
+histo_gram_bucket{tenant_id="2",le="256"} 1
+histo_gram_bucket{tenant_id="2",le="32"} 1
+histo_gram_bucket{tenant_id="2",le="4"} 0
+histo_gram_bucket{tenant_id="2",le="512"} 1
+histo_gram_bucket{tenant_id="2",le="64"} 1
+histo_gram_bucket{tenant_id="2",le="8"} 0
+histo_gram_count 2
+histo_gram_count{tenant_id="2"} 1
+histo_gram_sum 100
+histo_gram_sum{tenant_id="2"} 10

pkg/util/metric/metric_ext_test.go

@@ -0,0 +1,36 @@
+// Copyright 2022 The Cockroach Authors.
+//
+// Use of this software is governed by the Business Source License
+// included in the file licenses/BSL.txt.
+//
+// As of the Change Date specified in that file, in accordance with
+// the Business Source License, use of this software will be governed
+// by the Apache License, Version 2.0, included in the file
+// licenses/APL.txt.
+
+package metric_test
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/cockroachdb/cockroach/pkg/testutils"
+	"github.com/cockroachdb/cockroach/pkg/testutils/echotest"
+	"github.com/cockroachdb/cockroach/pkg/util/metric"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHistogramPrometheus(t *testing.T) {
+	// Regression test against https://github.com/cockroachdb/cockroach/pull/88331.
+	// The output includes buckets for which the upper bound equals the previous
+	// bucket's upper bound.
+	h := metric.NewHistogram(metric.Metadata{}, time.Second, []float64{1, 2, 3, 4, 5, 6, 10, 20, 30})
+	h.RecordValue(1)
+	h.RecordValue(5)
+	h.RecordValue(5)
+	h.RecordValue(10)
+	act, err := json.MarshalIndent(*h.ToPrometheusMetric().Histogram, "", "  ")
+	require.NoError(t, err)
+	echotest.Require(t, string(act), testutils.TestDataPath(t, "histogram.txt"))
+}