Add write ahead log to collector metrics exporter

exporter/collector/README.md

@@ -178,7 +178,11 @@ Note: These `retry_on_failure` and `sending_queue` are provided (and documented)
 Additional configuration for the metric exporter:
 
 - `metric.prefix` (optional): MetricPrefix overrides the prefix / namespace of the Google Cloud metric type identifier. If not set, defaults to "custom.googleapis.com/opencensus/"
-- `metric.skip_create_descriptor` (optional): Whether to skip creating the metric descriptor.
+- `metric.skip_create_descriptor` (optional): Whether to skip creating the
+  metric descriptor.
+- `metric.experimental_wal.experimental_directory` (optional): Path to local write-ahead-log file.
+- `metric.experimental_wal.experimental_max_backoff` (optional): Maximum duration to retry entries from
+  the WAL on network errors.
 
 Addition configuration for the logging exporter:
 

exporter/collector/config.go

@@ -46,8 +46,8 @@ type Config struct {
 	ProjectID               string            `mapstructure:"project"`
 	UserAgent               string            `mapstructure:"user_agent"`
 	ImpersonateConfig       ImpersonateConfig `mapstructure:"impersonate"`
-	LogConfig               LogConfig         `mapstructure:"log"`
 	TraceConfig             TraceConfig       `mapstructure:"trace"`
+	LogConfig               LogConfig         `mapstructure:"log"`
 	MetricConfig            MetricConfig      `mapstructure:"metric"`
 	DestinationProjectQuota bool              `mapstructure:"destination_project_quota"`
 }
@@ -90,11 +90,6 @@ type AttributeMapping struct {
 }
 
 type MetricConfig struct {
-	// GetMetricName is not settable in config files, but can be used by other
-	// exporters which extend the functionality of this exporter. It allows
-	// customizing the naming of metrics. baseName already includes type
-	// suffixes for summary metrics, but does not (yet) include the domain prefix
-	GetMetricName func(baseName string, metric pmetric.Metric) (string, error)
 	// MapMonitoredResource is not exposed as an option in the configuration, but
 	// can be used by other exporters to extend the functionality of this
 	// exporter. It allows overriding the function used to map otel resource to
@@ -103,7 +98,14 @@ type MetricConfig struct {
 	// ExtraMetrics is an extension point for exporters to add to the set
 	// of ResourceMetrics during a call to PushMetrics.
 	ExtraMetrics func(pmetric.Metrics) pmetric.ResourceMetricsSlice
-	Prefix       string `mapstructure:"prefix"`
+	// GetMetricName is not settable in config files, but can be used by other
+	// exporters which extend the functionality of this exporter. It allows
+	// customizing the naming of metrics. baseName already includes type
+	// suffixes for summary metrics, but does not (yet) include the domain prefix
+	GetMetricName func(baseName string, metric pmetric.Metric) (string, error)
+	// WALConfig holds configuration settings for the write ahead log.
+	WALConfig *WALConfig `mapstructure:"experimental_wal_config"`
+	Prefix    string     `mapstructure:"prefix"`
 	// KnownDomains contains a list of prefixes. If a metric already has one
 	// of these prefixes, the prefix is not added.
 	KnownDomains []string `mapstructure:"known_domains"`
@@ -140,6 +142,17 @@ type MetricConfig struct {
 	EnableSumOfSquaredDeviation bool `mapstructure:"sum_of_squared_deviation"`
 }
 
+// WALConfig defines settings for the write ahead log. WAL buffering writes data
+// points in-order to disk before reading and exporting them. This allows for
+// better retry logic when exporting fails (such as a network outage), because
+// it preserves both the data on disk and the order of the data points.
+type WALConfig struct {
+	// Directory is the location to store WAL files.
+	Directory string `mapstructure:"directory"`
+	// MaxBackoff sets the length of time to exponentially re-try failed exports.
+	MaxBackoff time.Duration `mapstructure:"emax_backoff"`
+}
+
 // ImpersonateConfig defines configuration for service account impersonation.
 type ImpersonateConfig struct {
 	TargetPrincipal string   `mapstructure:"target_principal"`

exporter/collector/go.mod

@@ -9,9 +9,11 @@ require (
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.15.0
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.39.0
 	github.com/census-instrumentation/opencensus-proto v0.4.1
+	github.com/fsnotify/fsnotify v1.6.0
 	github.com/google/go-cmp v0.5.9
 	github.com/googleapis/gax-go/v2 v2.7.0
 	github.com/stretchr/testify v1.8.3
+	github.com/tidwall/wal v1.1.7
 	go.opencensus.io v0.24.0
 	go.opentelemetry.io/collector/pdata v1.0.0-rcv0012
 	go.opentelemetry.io/collector/semconv v0.78.0
@@ -47,6 +49,10 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/tidwall/gjson v1.10.2 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
+	github.com/tidwall/tinylru v1.1.0 // indirect
 	go.opentelemetry.io/otel/metric v1.16.0 // indirect
 	golang.org/x/net v0.10.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect

exporter/collector/go.sum

@@ -30,6 +30,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
+github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=
 github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@@ -98,6 +100,16 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
 github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/tidwall/gjson v1.10.2 h1:APbLGOM0rrEkd8WBw9C24nllro4ajFuJu0Sc9hRz8Bo=
+github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/tinylru v1.1.0 h1:XY6IUfzVTU9rpwdhKUF6nQdChgCdGjkMfLzbWyiau6I=
+github.com/tidwall/tinylru v1.1.0/go.mod h1:3+bX+TJ2baOLMWTnlyNWHh4QMnFyARg2TLTQ6OFbzw8=
+github.com/tidwall/wal v1.1.7 h1:emc1TRjIVsdKKSnpwGBAcsAGg0767SvUk8+ygx7Bb+4=
+github.com/tidwall/wal v1.1.7/go.mod h1:r6lR1j27W9EPalgHiB7zLJDYu3mzW5BQP5KrzBpYY/E=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
@@ -156,6 +168,7 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

exporter/collector/integrationtest/cmd/recordfixtures/main.go

@@ -136,19 +136,18 @@ func (fr fixtureRecorder) recordLogs(ctx context.Context, t *FakeTesting, timest
 }
 
 func (fr fixtureRecorder) recordMetrics(ctx context.Context, t *FakeTesting, startTime, endTime time.Time) {
-	testServer, err := cloudmock.NewMetricTestServer()
-	if err != nil {
-		panic(err)
-	}
-	//nolint:errcheck
-	go testServer.Serve()
-	defer testServer.Shutdown()
-
 	for _, test := range testcases.MetricsTestCases {
 		if test.Skip {
 			continue
 		}
 
+		testServer, err := cloudmock.NewMetricTestServer()
+		if err != nil {
+			panic(err)
+		}
+		//nolint:errcheck
+		go testServer.Serve()
+
 		require.NoError(t, fr.checkDuplicate(test))
 
 		func() {
@@ -177,5 +176,7 @@ func (fr fixtureRecorder) recordMetrics(ctx context.Context, t *FakeTesting, sta
 			}
 			test.SaveRecordedMetricFixtures(t, fixture)
 		}()
+
+		testServer.Shutdown()
 	}
 }

exporter/collector/integrationtest/go.mod

@@ -41,6 +41,7 @@ require (
 	github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/go-kit/log v0.2.1 // indirect
 	github.com/go-logfmt/logfmt v0.5.1 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
@@ -76,6 +77,11 @@ require (
 	github.com/shoenig/go-m1cpu v0.1.5 // indirect
 	github.com/spf13/cobra v1.7.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/tidwall/gjson v1.10.2 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
+	github.com/tidwall/tinylru v1.1.0 // indirect
+	github.com/tidwall/wal v1.1.7 // indirect
 	github.com/tklauser/go-sysconf v0.3.11 // indirect
 	github.com/tklauser/numcpus v0.6.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect

exporter/collector/integrationtest/go.sum

@@ -111,6 +111,7 @@ github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
+github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -414,6 +415,16 @@ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o
 github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
 github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stvp/go-udp-testing v0.0.0-20201019212854-469649b16807/go.mod h1:7jxmlfBCDBXRzr0eAQJ48XC1hBu1np4CS5+cHEYfwpc=
+github.com/tidwall/gjson v1.10.2 h1:APbLGOM0rrEkd8WBw9C24nllro4ajFuJu0Sc9hRz8Bo=
+github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/tinylru v1.1.0 h1:XY6IUfzVTU9rpwdhKUF6nQdChgCdGjkMfLzbWyiau6I=
+github.com/tidwall/tinylru v1.1.0/go.mod h1:3+bX+TJ2baOLMWTnlyNWHh4QMnFyARg2TLTQ6OFbzw8=
+github.com/tidwall/wal v1.1.7 h1:emc1TRjIVsdKKSnpwGBAcsAGg0767SvUk8+ygx7Bb+4=
+github.com/tidwall/wal v1.1.7/go.mod h1:r6lR1j27W9EPalgHiB7zLJDYu3mzW5BQP5KrzBpYY/E=
 github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM=
 github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI=
 github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms=
@@ -642,6 +653,7 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=

exporter/collector/integrationtest/metrics_test.go

@@ -64,8 +64,15 @@ func TestCollectorMetrics(t *testing.T) {
 			}
 			require.NoError(t, testServerExporter.Shutdown(ctx))
 
-			expectFixture := test.LoadMetricExpectFixture(
+			if !test.ExpectRetries {
+				require.Zero(t, testServer.RetryCount, "Server returned >0 retries when not expected")
+			} else {
+				require.NotZero(t, testServer.RetryCount, "Server returned 0 retries when expected >0")
+			}
+
+			expectFixture := test.LoadMetricFixture(
 				t,
+				test.ExpectFixturePath,
 				startTime,
 				endTime,
 			)
@@ -114,6 +121,40 @@ func TestCollectorMetrics(t *testing.T) {
 					diff,
 				)
 			}
+
+			if len(test.CompareFixturePath) > 0 {
+				compareFixture := test.LoadMetricFixture(
+					t,
+					test.CompareFixturePath,
+					startTime,
+					endTime,
+				)
+				sort.Slice(compareFixture.CreateTimeSeriesRequests, func(i, j int) bool {
+					return compareFixture.CreateTimeSeriesRequests[i].Name < compareFixture.CreateTimeSeriesRequests[j].Name
+				})
+				sort.Slice(compareFixture.CreateMetricDescriptorRequests, func(i, j int) bool {
+					if compareFixture.CreateMetricDescriptorRequests[i].Name != compareFixture.CreateMetricDescriptorRequests[j].Name {
+						return compareFixture.CreateMetricDescriptorRequests[i].Name < compareFixture.CreateMetricDescriptorRequests[j].Name
+					}
+					return compareFixture.CreateMetricDescriptorRequests[i].MetricDescriptor.Name < compareFixture.CreateMetricDescriptorRequests[j].MetricDescriptor.Name
+				})
+				sort.Slice(compareFixture.CreateServiceTimeSeriesRequests, func(i, j int) bool {
+					return compareFixture.CreateServiceTimeSeriesRequests[i].Name < compareFixture.CreateServiceTimeSeriesRequests[j].Name
+				})
+
+				diff := DiffMetricProtos(
+					t,
+					fixture,
+					compareFixture,
+				)
+				if diff != "" {
+					require.Fail(
+						t,
+						"Expected requests fixture and actual GCM requests differ",
+						diff,
+					)
+				}
+			}
 		})
 	}
 }
@@ -158,8 +199,9 @@ func TestSDKMetrics(t *testing.T) {
 			}
 			require.NoError(t, testServerExporter.Shutdown(ctx))
 
-			expectFixture := test.LoadMetricExpectFixture(
+			expectFixture := test.LoadMetricFixture(
 				t,
+				test.ExpectFixturePath,
 				startTime,
 				endTime,
 			)
@@ -209,6 +251,40 @@ func TestSDKMetrics(t *testing.T) {
 					diff,
 				)
 			}
+
+			if len(test.CompareFixturePath) > 0 {
+				compareFixture := test.LoadMetricFixture(
+					t,
+					test.CompareFixturePath,
+					startTime,
+					endTime,
+				)
+				sort.Slice(compareFixture.CreateTimeSeriesRequests, func(i, j int) bool {
+					return compareFixture.CreateTimeSeriesRequests[i].Name < compareFixture.CreateTimeSeriesRequests[j].Name
+				})
+				sort.Slice(compareFixture.CreateMetricDescriptorRequests, func(i, j int) bool {
+					if compareFixture.CreateMetricDescriptorRequests[i].Name != compareFixture.CreateMetricDescriptorRequests[j].Name {
+						return compareFixture.CreateMetricDescriptorRequests[i].Name < compareFixture.CreateMetricDescriptorRequests[j].Name
+					}
+					return compareFixture.CreateMetricDescriptorRequests[i].MetricDescriptor.Name < compareFixture.CreateMetricDescriptorRequests[j].MetricDescriptor.Name
+				})
+				sort.Slice(compareFixture.CreateServiceTimeSeriesRequests, func(i, j int) bool {
+					return compareFixture.CreateServiceTimeSeriesRequests[i].Name < compareFixture.CreateServiceTimeSeriesRequests[j].Name
+				})
+
+				diff := DiffMetricProtos(
+					t,
+					fixture,
+					compareFixture,
+				)
+				if diff != "" {
+					require.Fail(
+						t,
+						"Expected requests fixture and actual GCM requests differ",
+						diff,
+					)
+				}
+			}
 		})
 	}
 }

exporter/collector/integrationtest/testcases/testcase.go

@@ -67,6 +67,9 @@ type TestCase struct {
 	// ExpectFixturePath is the path to the JSON encoded MetricExpectFixture
 	// (see fixtures.proto) that contains request messages the exporter is expected to send.
 	ExpectFixturePath string
+	// CompareFixturePath is a second output fixture that should be equal to this test's output fixture.
+	// Used for cross-referencing multiple tests that should have the same output, without overwriting the same fixtures.
+	CompareFixturePath string
 	// When testing the SDK metrics exporter (not collector), this is the options to use. Optional.
 	MetricSDKExporterOptions []metric.Option
 	// Skip, if true, skips this test case
@@ -75,6 +78,8 @@ type TestCase struct {
 	SkipForSDK bool
 	// ExpectErr sets whether the test is expected to fail
 	ExpectErr bool
+	// ExpectRetries sets whether the test expects the server to report multiple attempts
+	ExpectRetries bool
 }
 
 func (tc *TestCase) LoadOTLPTracesInput(
@@ -329,12 +334,13 @@ func (tc *TestCase) LoadOTLPMetricsInput(
 	return metrics
 }
 
-func (tc *TestCase) LoadMetricExpectFixture(
+func (tc *TestCase) LoadMetricFixture(
 	t testing.TB,
+	path string,
 	startTime time.Time,
 	endTime time.Time,
 ) *protos.MetricExpectFixture {
-	fixtureBytes, err := os.ReadFile(tc.ExpectFixturePath)
+	fixtureBytes, err := os.ReadFile(path)
 	require.NoError(t, err)
 	fixture := &protos.MetricExpectFixture{}
 	require.NoError(t, protojson.Unmarshal(fixtureBytes, fixture))