From 35413ab67b1e94d04049aec68fddb7e77ceabfa6 Mon Sep 17 00:00:00 2001 From: Juraj Michalek Date: Wed, 16 Oct 2024 18:25:45 +0200 Subject: [PATCH 1/9] feat: added translation rw2 translation for gauges Signed-off-by: Juraj Michalek --- .../prometheusremotewrite/helper_v2.go | 108 ++++++++++++++ .../prometheusremotewrite/helper_v2_test.go | 139 ++++++++++++++++++ .../metrics_to_prw_v2.go | 3 +- .../metrics_to_prw_v2_test.go | 2 +- .../number_data_points_v2.go | 21 +-- .../number_data_points_v2_test.go | 23 ++- .../prometheusremotewrite/testutils_test.go | 13 ++ 7 files changed, 295 insertions(+), 14 deletions(-) create mode 100644 pkg/translator/prometheusremotewrite/helper_v2.go create mode 100644 pkg/translator/prometheusremotewrite/helper_v2_test.go diff --git a/pkg/translator/prometheusremotewrite/helper_v2.go b/pkg/translator/prometheusremotewrite/helper_v2.go new file mode 100644 index 000000000000..ed36f0b9724c --- /dev/null +++ b/pkg/translator/prometheusremotewrite/helper_v2.go @@ -0,0 +1,108 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package prometheusremotewrite // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite" + +import ( + "fmt" + "log" + "slices" + + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/labels" + "go.opentelemetry.io/collector/pdata/pcommon" + conventions "go.opentelemetry.io/collector/semconv/v1.25.0" + + prometheustranslator "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus" +) + +// createAttributes creates a slice of Prometheus Labels with OTLP attributes and pairs of string values. +// Unpaired string values are ignored. String pairs overwrite OTLP labels if collisions happen and +// if logOnOverwrite is true, the overwrite is logged. Resulting label names are sanitized. +func createAttributesV2(resource pcommon.Resource, attributes pcommon.Map, externalLabels map[string]string, + ignoreAttrs []string, logOnOverwrite bool, extras ...string) labels.Labels { + resourceAttrs := resource.Attributes() + serviceName, haveServiceName := resourceAttrs.Get(conventions.AttributeServiceName) + instance, haveInstanceID := resourceAttrs.Get(conventions.AttributeServiceInstanceID) + + // Calculate the maximum possible number of labels we could return so we can preallocate l + maxLabelCount := attributes.Len() + len(externalLabels) + len(extras)/2 + + if haveServiceName { + maxLabelCount++ + } + + if haveInstanceID { + maxLabelCount++ + } + + // map ensures no duplicate label name + l := make(map[string]string, maxLabelCount) + + // Ensure attributes are sorted by key for consistent merging of keys which + // collide when sanitized. + tempSeriesLabels := labels.Labels{} + // XXX: Should we always drop service namespace/service name/service instance ID from the labels + // (as they get mapped to other Prometheus labels)? + attributes.Range(func(key string, value pcommon.Value) bool { + if !slices.Contains(ignoreAttrs, key) { + tempSeriesLabels = append(tempSeriesLabels, labels.Label{Name: key, Value: value.AsString()}) + } + return true + }) + // TODO New returns a sorted Labels from the given labels. The caller has to guarantee that all label names are unique. + seriesLabels := labels.New(tempSeriesLabels...) // This sorts by name + + for _, label := range seriesLabels { + var finalKey = prometheustranslator.NormalizeLabel(label.Name) + if existingValue, alreadyExists := l[finalKey]; alreadyExists { + l[finalKey] = existingValue + ";" + label.Value + } else { + l[finalKey] = label.Value + } + } + + // Map service.name + service.namespace to job + if haveServiceName { + val := serviceName.AsString() + if serviceNamespace, ok := resourceAttrs.Get(conventions.AttributeServiceNamespace); ok { + val = fmt.Sprintf("%s/%s", serviceNamespace.AsString(), val) + } + l[model.JobLabel] = val + } + // Map service.instance.id to instance + if haveInstanceID { + l[model.InstanceLabel] = instance.AsString() + } + for key, value := range externalLabels { + // External labels have already been sanitized + if _, alreadyExists := l[key]; alreadyExists { + // Skip external labels if they are overridden by metric attributes + continue + } + l[key] = value + } + + for i := 0; i < len(extras); i += 2 { + if i+1 >= len(extras) { + break + } + _, found := l[extras[i]] + if found && logOnOverwrite { + log.Println("label " + extras[i] + " is overwritten. Check if Prometheus reserved labels are used.") + } + // internal labels should be maintained + name := extras[i] + if !(len(name) > 4 && name[:2] == "__" && name[len(name)-2:] == "__") { + name = prometheustranslator.NormalizeLabel(name) + } + l[name] = extras[i+1] + } + + seriesLabels = seriesLabels[:0] + for k, v := range l { + seriesLabels = append(seriesLabels, labels.Label{Name: k, Value: v}) + } + + return seriesLabels +} diff --git a/pkg/translator/prometheusremotewrite/helper_v2_test.go b/pkg/translator/prometheusremotewrite/helper_v2_test.go new file mode 100644 index 000000000000..8f674eca5b6d --- /dev/null +++ b/pkg/translator/prometheusremotewrite/helper_v2_test.go @@ -0,0 +1,139 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package prometheusremotewrite + +import ( + "testing" + + "github.com/prometheus/prometheus/model/labels" + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/pdata/pcommon" +) + +// Test_createLabelSet checks resultant label names are sanitized and label in extra overrides label in labels if +// collision happens. It does not check whether labels are not sorted +func Test_createLabelSetV2(t *testing.T) { + tests := []struct { + name string + resource pcommon.Resource + orig pcommon.Map + externalLabels map[string]string + extras []string + want labels.Labels + }{ + { + "labels_clean", + pcommon.NewResource(), + lbs1, + map[string]string{}, + []string{label31, value31, label32, value32}, + getPromLabelsV2(label11, value11, label12, value12, label31, value31, label32, value32), + }, + { + "labels_with_resource", + func() pcommon.Resource { + res := pcommon.NewResource() + res.Attributes().PutStr("service.name", "prometheus") + res.Attributes().PutStr("service.instance.id", "127.0.0.1:8080") + return res + }(), + lbs1, + map[string]string{}, + []string{label31, value31, label32, value32}, + getPromLabelsV2(label11, value11, label12, value12, label31, value31, label32, value32, "job", "prometheus", "instance", "127.0.0.1:8080"), + }, + { + "labels_with_nonstring_resource", + func() pcommon.Resource { + res := pcommon.NewResource() + res.Attributes().PutInt("service.name", 12345) + res.Attributes().PutBool("service.instance.id", true) + return res + }(), + lbs1, + map[string]string{}, + []string{label31, value31, label32, value32}, + getPromLabelsV2(label11, value11, label12, value12, label31, value31, label32, value32, "job", "12345", "instance", "true"), + }, + { + "labels_duplicate_in_extras", + pcommon.NewResource(), + lbs1, + map[string]string{}, + []string{label11, value31}, + getPromLabelsV2(label11, value31, label12, value12), + }, + { + "labels_dirty", + pcommon.NewResource(), + lbs1Dirty, + map[string]string{}, + []string{label31 + dirty1, value31, label32, value32}, + getPromLabelsV2(label11+"_", value11, "key_"+label12, value12, label31+"_", value31, label32, value32), + }, + { + "no_original_case", + pcommon.NewResource(), + pcommon.NewMap(), + nil, + []string{label31, value31, label32, value32}, + getPromLabelsV2(label31, value31, label32, value32), + }, + { + "empty_extra_case", + pcommon.NewResource(), + lbs1, + map[string]string{}, + []string{"", ""}, + getPromLabelsV2(label11, value11, label12, value12, "", ""), + }, + { + "single_left_over_case", + pcommon.NewResource(), + lbs1, + map[string]string{}, + []string{label31, value31, label32}, + getPromLabelsV2(label11, value11, label12, value12, label31, value31), + }, + { + "valid_external_labels", + pcommon.NewResource(), + lbs1, + exlbs1, + []string{label31, value31, label32, value32}, + getPromLabelsV2(label11, value11, label12, value12, label41, value41, label31, value31, label32, value32), + }, + { + "overwritten_external_labels", + pcommon.NewResource(), + lbs1, + exlbs2, + []string{label31, value31, label32, value32}, + getPromLabelsV2(label11, value11, label12, value12, label31, value31, label32, value32), + }, + { + "colliding attributes", + pcommon.NewResource(), + lbsColliding, + nil, + []string{label31, value31, label32, value32}, + getPromLabelsV2(collidingSanitized, value11+";"+value12, label31, value31, label32, value32), + }, + { + "sanitize_labels_starts_with_underscore", + pcommon.NewResource(), + lbs3, + exlbs1, + []string{label31, value31, label32, value32}, + getPromLabelsV2(label11, value11, label12, value12, "key"+label51, value51, label41, value41, label31, value31, label32, value32), + }, + } + // run tests + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + res := createAttributesV2(tt.resource, tt.orig, tt.externalLabels, nil, true, tt.extras...) + assert.ElementsMatch(t, tt.want, res) + }) + } +} diff --git a/pkg/translator/prometheusremotewrite/metrics_to_prw_v2.go b/pkg/translator/prometheusremotewrite/metrics_to_prw_v2.go index 2beb7385fbe1..ccbc218effcd 100644 --- a/pkg/translator/prometheusremotewrite/metrics_to_prw_v2.go +++ b/pkg/translator/prometheusremotewrite/metrics_to_prw_v2.go @@ -49,6 +49,7 @@ func (c *prometheusConverterV2) fromMetrics(md pmetric.Metrics, settings Setting resourceMetricsSlice := md.ResourceMetrics() for i := 0; i < resourceMetricsSlice.Len(); i++ { resourceMetrics := resourceMetricsSlice.At(i) + resource := resourceMetrics.Resource() scopeMetricsSlice := resourceMetrics.ScopeMetrics() // keep track of the most recent timestamp in the ResourceMetrics for // use with the "target" info metric @@ -77,7 +78,7 @@ func (c *prometheusConverterV2) fromMetrics(md pmetric.Metrics, settings Setting errs = multierr.Append(errs, fmt.Errorf("empty data points. %s is dropped", metric.Name())) break } - c.addGaugeNumberDataPoints(dataPoints, promName) + c.addGaugeNumberDataPoints(dataPoints, resource, settings, promName) case pmetric.MetricTypeSum: // TODO implement case pmetric.MetricTypeHistogram: diff --git a/pkg/translator/prometheusremotewrite/metrics_to_prw_v2_test.go b/pkg/translator/prometheusremotewrite/metrics_to_prw_v2_test.go index 375738b8bbee..30cb6b826b2b 100644 --- a/pkg/translator/prometheusremotewrite/metrics_to_prw_v2_test.go +++ b/pkg/translator/prometheusremotewrite/metrics_to_prw_v2_test.go @@ -27,7 +27,7 @@ func TestFromMetricsV2(t *testing.T) { want := func() map[string]*writev2.TimeSeries { return map[string]*writev2.TimeSeries{ "0": { - LabelsRefs: []uint32{1, 2}, + LabelsRefs: []uint32{1, 2, 3, 4, 5, 6, 7, 8}, Samples: []writev2.Sample{ {Timestamp: convertTimeStamp(pcommon.Timestamp(ts)), Value: 1.23}, }, diff --git a/pkg/translator/prometheusremotewrite/number_data_points_v2.go b/pkg/translator/prometheusremotewrite/number_data_points_v2.go index 3a192efb9589..bb35df438f96 100644 --- a/pkg/translator/prometheusremotewrite/number_data_points_v2.go +++ b/pkg/translator/prometheusremotewrite/number_data_points_v2.go @@ -7,23 +7,26 @@ import ( "math" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/value" writev2 "github.com/prometheus/prometheus/prompb/io/prometheus/write/v2" + "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" ) -func (c *prometheusConverterV2) addGaugeNumberDataPoints(dataPoints pmetric.NumberDataPointSlice, name string) { +func (c *prometheusConverterV2) addGaugeNumberDataPoints(dataPoints pmetric.NumberDataPointSlice, + resource pcommon.Resource, settings Settings, name string) { for x := 0; x < dataPoints.Len(); x++ { pt := dataPoints.At(x) - // TODO implement support for labels - labels := labels.Labels{ - labels.Label{ - Name: model.MetricNameLabel, - Value: name, - }, - } + labels := createAttributesV2( + resource, + pt.Attributes(), + settings.ExternalLabels, + nil, + true, + model.MetricNameLabel, + name, + ) sample := &writev2.Sample{ // convert ns to ms diff --git a/pkg/translator/prometheusremotewrite/number_data_points_v2_test.go b/pkg/translator/prometheusremotewrite/number_data_points_v2_test.go index ce78b9c1f391..5b7c2cf377d9 100644 --- a/pkg/translator/prometheusremotewrite/number_data_points_v2_test.go +++ b/pkg/translator/prometheusremotewrite/number_data_points_v2_test.go @@ -108,8 +108,16 @@ func TestPrometheusConverterV2_addGaugeNumberDataPoints(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { metric := tt.metric() + settings := Settings{ + Namespace: "", + ExternalLabels: nil, + DisableTargetInfo: false, + ExportCreatedMetric: false, + AddMetricSuffixes: false, + SendMetadata: false, + } converter := newPrometheusConverterV2() - converter.addGaugeNumberDataPoints(metric.Gauge().DataPoints(), metric.Name()) + converter.addGaugeNumberDataPoints(metric.Gauge().DataPoints(), pcommon.NewResource(), settings, metric.Name()) w := tt.want() diff := cmp.Diff(w, converter.unique, cmpopts.EquateNaNs()) @@ -150,9 +158,18 @@ func TestPrometheusConverterV2_addGaugeNumberDataPointsDuplicate(t *testing.T) { } } + settings := Settings{ + Namespace: "", + ExternalLabels: nil, + DisableTargetInfo: false, + ExportCreatedMetric: false, + AddMetricSuffixes: false, + SendMetadata: false, + } + converter := newPrometheusConverterV2() - converter.addGaugeNumberDataPoints(metric1.Gauge().DataPoints(), metric1.Name()) - converter.addGaugeNumberDataPoints(metric2.Gauge().DataPoints(), metric2.Name()) + converter.addGaugeNumberDataPoints(metric1.Gauge().DataPoints(), pcommon.NewResource(), settings, metric1.Name()) + converter.addGaugeNumberDataPoints(metric2.Gauge().DataPoints(), pcommon.NewResource(), settings, metric2.Name()) assert.Equal(t, want(), converter.unique) diff --git a/pkg/translator/prometheusremotewrite/testutils_test.go b/pkg/translator/prometheusremotewrite/testutils_test.go index 49ef7a735081..d7c92d111829 100644 --- a/pkg/translator/prometheusremotewrite/testutils_test.go +++ b/pkg/translator/prometheusremotewrite/testutils_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/prompb" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/pdata/pcommon" @@ -136,6 +137,18 @@ func getAttributes(labels ...string) pcommon.Map { return attributeMap } +// Prometheus TimeSeries +func getPromLabelsV2(lbs ...string) labels.Labels { + pbLbs := labels.Labels{} + for i := 0; i < len(lbs); i += 2 { + pbLbs = append(pbLbs, labels.Label{ + Name: lbs[i], + Value: lbs[i+1], + }) + } + return pbLbs +} + // Prometheus TimeSeries func getPromLabels(lbs ...string) []prompb.Label { pbLbs := prompb.Labels{ From 250893cbe1ff36ee6a774c3c94dc80605a601ed3 Mon Sep 17 00:00:00 2001 From: Juraj Michalek Date: Sun, 20 Oct 2024 20:12:49 +0200 Subject: [PATCH 2/9] chore: implemented changes suggested in rw, reducing code duplication Signed-off-by: Juraj Michalek --- .../prometheusremotewrite/helper_v2.go | 108 -------------- .../prometheusremotewrite/helper_v2_test.go | 139 ------------------ .../metrics_to_prw_v2.go | 22 ++- .../number_data_points_v2.go | 2 +- .../prometheusremotewrite/testutils_test.go | 13 -- 5 files changed, 16 insertions(+), 268 deletions(-) delete mode 100644 pkg/translator/prometheusremotewrite/helper_v2.go delete mode 100644 pkg/translator/prometheusremotewrite/helper_v2_test.go diff --git a/pkg/translator/prometheusremotewrite/helper_v2.go b/pkg/translator/prometheusremotewrite/helper_v2.go deleted file mode 100644 index ed36f0b9724c..000000000000 --- a/pkg/translator/prometheusremotewrite/helper_v2.go +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 - -package prometheusremotewrite // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite" - -import ( - "fmt" - "log" - "slices" - - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/model/labels" - "go.opentelemetry.io/collector/pdata/pcommon" - conventions "go.opentelemetry.io/collector/semconv/v1.25.0" - - prometheustranslator "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus" -) - -// createAttributes creates a slice of Prometheus Labels with OTLP attributes and pairs of string values. -// Unpaired string values are ignored. String pairs overwrite OTLP labels if collisions happen and -// if logOnOverwrite is true, the overwrite is logged. Resulting label names are sanitized. -func createAttributesV2(resource pcommon.Resource, attributes pcommon.Map, externalLabels map[string]string, - ignoreAttrs []string, logOnOverwrite bool, extras ...string) labels.Labels { - resourceAttrs := resource.Attributes() - serviceName, haveServiceName := resourceAttrs.Get(conventions.AttributeServiceName) - instance, haveInstanceID := resourceAttrs.Get(conventions.AttributeServiceInstanceID) - - // Calculate the maximum possible number of labels we could return so we can preallocate l - maxLabelCount := attributes.Len() + len(externalLabels) + len(extras)/2 - - if haveServiceName { - maxLabelCount++ - } - - if haveInstanceID { - maxLabelCount++ - } - - // map ensures no duplicate label name - l := make(map[string]string, maxLabelCount) - - // Ensure attributes are sorted by key for consistent merging of keys which - // collide when sanitized. - tempSeriesLabels := labels.Labels{} - // XXX: Should we always drop service namespace/service name/service instance ID from the labels - // (as they get mapped to other Prometheus labels)? - attributes.Range(func(key string, value pcommon.Value) bool { - if !slices.Contains(ignoreAttrs, key) { - tempSeriesLabels = append(tempSeriesLabels, labels.Label{Name: key, Value: value.AsString()}) - } - return true - }) - // TODO New returns a sorted Labels from the given labels. The caller has to guarantee that all label names are unique. - seriesLabels := labels.New(tempSeriesLabels...) // This sorts by name - - for _, label := range seriesLabels { - var finalKey = prometheustranslator.NormalizeLabel(label.Name) - if existingValue, alreadyExists := l[finalKey]; alreadyExists { - l[finalKey] = existingValue + ";" + label.Value - } else { - l[finalKey] = label.Value - } - } - - // Map service.name + service.namespace to job - if haveServiceName { - val := serviceName.AsString() - if serviceNamespace, ok := resourceAttrs.Get(conventions.AttributeServiceNamespace); ok { - val = fmt.Sprintf("%s/%s", serviceNamespace.AsString(), val) - } - l[model.JobLabel] = val - } - // Map service.instance.id to instance - if haveInstanceID { - l[model.InstanceLabel] = instance.AsString() - } - for key, value := range externalLabels { - // External labels have already been sanitized - if _, alreadyExists := l[key]; alreadyExists { - // Skip external labels if they are overridden by metric attributes - continue - } - l[key] = value - } - - for i := 0; i < len(extras); i += 2 { - if i+1 >= len(extras) { - break - } - _, found := l[extras[i]] - if found && logOnOverwrite { - log.Println("label " + extras[i] + " is overwritten. Check if Prometheus reserved labels are used.") - } - // internal labels should be maintained - name := extras[i] - if !(len(name) > 4 && name[:2] == "__" && name[len(name)-2:] == "__") { - name = prometheustranslator.NormalizeLabel(name) - } - l[name] = extras[i+1] - } - - seriesLabels = seriesLabels[:0] - for k, v := range l { - seriesLabels = append(seriesLabels, labels.Label{Name: k, Value: v}) - } - - return seriesLabels -} diff --git a/pkg/translator/prometheusremotewrite/helper_v2_test.go b/pkg/translator/prometheusremotewrite/helper_v2_test.go deleted file mode 100644 index 8f674eca5b6d..000000000000 --- a/pkg/translator/prometheusremotewrite/helper_v2_test.go +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 - -package prometheusremotewrite - -import ( - "testing" - - "github.com/prometheus/prometheus/model/labels" - "github.com/stretchr/testify/assert" - "go.opentelemetry.io/collector/pdata/pcommon" -) - -// Test_createLabelSet checks resultant label names are sanitized and label in extra overrides label in labels if -// collision happens. It does not check whether labels are not sorted -func Test_createLabelSetV2(t *testing.T) { - tests := []struct { - name string - resource pcommon.Resource - orig pcommon.Map - externalLabels map[string]string - extras []string - want labels.Labels - }{ - { - "labels_clean", - pcommon.NewResource(), - lbs1, - map[string]string{}, - []string{label31, value31, label32, value32}, - getPromLabelsV2(label11, value11, label12, value12, label31, value31, label32, value32), - }, - { - "labels_with_resource", - func() pcommon.Resource { - res := pcommon.NewResource() - res.Attributes().PutStr("service.name", "prometheus") - res.Attributes().PutStr("service.instance.id", "127.0.0.1:8080") - return res - }(), - lbs1, - map[string]string{}, - []string{label31, value31, label32, value32}, - getPromLabelsV2(label11, value11, label12, value12, label31, value31, label32, value32, "job", "prometheus", "instance", "127.0.0.1:8080"), - }, - { - "labels_with_nonstring_resource", - func() pcommon.Resource { - res := pcommon.NewResource() - res.Attributes().PutInt("service.name", 12345) - res.Attributes().PutBool("service.instance.id", true) - return res - }(), - lbs1, - map[string]string{}, - []string{label31, value31, label32, value32}, - getPromLabelsV2(label11, value11, label12, value12, label31, value31, label32, value32, "job", "12345", "instance", "true"), - }, - { - "labels_duplicate_in_extras", - pcommon.NewResource(), - lbs1, - map[string]string{}, - []string{label11, value31}, - getPromLabelsV2(label11, value31, label12, value12), - }, - { - "labels_dirty", - pcommon.NewResource(), - lbs1Dirty, - map[string]string{}, - []string{label31 + dirty1, value31, label32, value32}, - getPromLabelsV2(label11+"_", value11, "key_"+label12, value12, label31+"_", value31, label32, value32), - }, - { - "no_original_case", - pcommon.NewResource(), - pcommon.NewMap(), - nil, - []string{label31, value31, label32, value32}, - getPromLabelsV2(label31, value31, label32, value32), - }, - { - "empty_extra_case", - pcommon.NewResource(), - lbs1, - map[string]string{}, - []string{"", ""}, - getPromLabelsV2(label11, value11, label12, value12, "", ""), - }, - { - "single_left_over_case", - pcommon.NewResource(), - lbs1, - map[string]string{}, - []string{label31, value31, label32}, - getPromLabelsV2(label11, value11, label12, value12, label31, value31), - }, - { - "valid_external_labels", - pcommon.NewResource(), - lbs1, - exlbs1, - []string{label31, value31, label32, value32}, - getPromLabelsV2(label11, value11, label12, value12, label41, value41, label31, value31, label32, value32), - }, - { - "overwritten_external_labels", - pcommon.NewResource(), - lbs1, - exlbs2, - []string{label31, value31, label32, value32}, - getPromLabelsV2(label11, value11, label12, value12, label31, value31, label32, value32), - }, - { - "colliding attributes", - pcommon.NewResource(), - lbsColliding, - nil, - []string{label31, value31, label32, value32}, - getPromLabelsV2(collidingSanitized, value11+";"+value12, label31, value31, label32, value32), - }, - { - "sanitize_labels_starts_with_underscore", - pcommon.NewResource(), - lbs3, - exlbs1, - []string{label31, value31, label32, value32}, - getPromLabelsV2(label11, value11, label12, value12, "key"+label51, value51, label41, value41, label31, value31, label32, value32), - }, - } - // run tests - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - res := createAttributesV2(tt.resource, tt.orig, tt.externalLabels, nil, true, tt.extras...) - assert.ElementsMatch(t, tt.want, res) - }) - } -} diff --git a/pkg/translator/prometheusremotewrite/metrics_to_prw_v2.go b/pkg/translator/prometheusremotewrite/metrics_to_prw_v2.go index ccbc218effcd..4af418374a29 100644 --- a/pkg/translator/prometheusremotewrite/metrics_to_prw_v2.go +++ b/pkg/translator/prometheusremotewrite/metrics_to_prw_v2.go @@ -8,7 +8,7 @@ import ( "fmt" "strconv" - "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/prompb" writev2 "github.com/prometheus/prometheus/prompb/io/prometheus/write/v2" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" @@ -108,16 +108,24 @@ func (c *prometheusConverterV2) timeSeries() []writev2.TimeSeries { return allTS } -func (c *prometheusConverterV2) addSample(sample *writev2.Sample, lbls labels.Labels) *writev2.TimeSeries { +func (c *prometheusConverterV2) addSample(sample *writev2.Sample, lbls []prompb.Label) *writev2.TimeSeries { if sample == nil || len(lbls) == 0 { // This shouldn't happen return nil } - ts := &writev2.TimeSeries{} - ts.LabelsRefs = c.symbolTable.SymbolizeLabels(lbls, ts.LabelsRefs) - ts.Samples = append(ts.Samples, *sample) - c.unique[lbls.Hash()] = ts + buf := make([]uint32, 0, len(lbls)*2) + for _, l := range lbls { + off := c.symbolTable.Symbolize(l.Name) + buf = append(buf, off) + off = c.symbolTable.Symbolize(l.Value) + buf = append(buf, off) + } + ts := writev2.TimeSeries{ + LabelsRefs: buf, + Samples: []writev2.Sample{*sample}, + } + c.unique[timeSeriesSignature(lbls)] = &ts - return ts + return &ts } diff --git a/pkg/translator/prometheusremotewrite/number_data_points_v2.go b/pkg/translator/prometheusremotewrite/number_data_points_v2.go index bb35df438f96..d2c11447a841 100644 --- a/pkg/translator/prometheusremotewrite/number_data_points_v2.go +++ b/pkg/translator/prometheusremotewrite/number_data_points_v2.go @@ -18,7 +18,7 @@ func (c *prometheusConverterV2) addGaugeNumberDataPoints(dataPoints pmetric.Numb for x := 0; x < dataPoints.Len(); x++ { pt := dataPoints.At(x) - labels := createAttributesV2( + labels := createAttributes( resource, pt.Attributes(), settings.ExternalLabels, diff --git a/pkg/translator/prometheusremotewrite/testutils_test.go b/pkg/translator/prometheusremotewrite/testutils_test.go index d7c92d111829..49ef7a735081 100644 --- a/pkg/translator/prometheusremotewrite/testutils_test.go +++ b/pkg/translator/prometheusremotewrite/testutils_test.go @@ -10,7 +10,6 @@ import ( "testing" "time" - "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/prompb" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/pdata/pcommon" @@ -137,18 +136,6 @@ func getAttributes(labels ...string) pcommon.Map { return attributeMap } -// Prometheus TimeSeries -func getPromLabelsV2(lbs ...string) labels.Labels { - pbLbs := labels.Labels{} - for i := 0; i < len(lbs); i += 2 { - pbLbs = append(pbLbs, labels.Label{ - Name: lbs[i], - Value: lbs[i+1], - }) - } - return pbLbs -} - // Prometheus TimeSeries func getPromLabels(lbs ...string) []prompb.Label { pbLbs := prompb.Labels{ From 5fb8e273ad813d0a95d1b7ad33e7fca91ed9fa82 Mon Sep 17 00:00:00 2001 From: Juraj Michalek Date: Sun, 20 Oct 2024 20:35:07 +0200 Subject: [PATCH 3/9] chore: added changelog entry Signed-off-by: Juraj Michalek --- ...ranslation-rw2-add-support-for-labels.yaml | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .chloggen/jm-prom-translation-rw2-add-support-for-labels.yaml diff --git a/.chloggen/jm-prom-translation-rw2-add-support-for-labels.yaml b/.chloggen/jm-prom-translation-rw2-add-support-for-labels.yaml new file mode 100644 index 000000000000..8c2a10f5d7b5 --- /dev/null +++ b/.chloggen/jm-prom-translation-rw2-add-support-for-labels.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: pkg/translator/prometheusremotewrite + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: RW2 translation added support for labels + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [33661] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] From ba45c6d417c531c4dd297584f44a10af76410f21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juraj=20Mich=C3=A1lek?= Date: Sun, 20 Oct 2024 23:21:13 +0200 Subject: [PATCH 4/9] Update .chloggen/jm-prom-translation-rw2-add-support-for-labels.yaml Co-authored-by: Arthur Silva Sens --- .chloggen/jm-prom-translation-rw2-add-support-for-labels.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.chloggen/jm-prom-translation-rw2-add-support-for-labels.yaml b/.chloggen/jm-prom-translation-rw2-add-support-for-labels.yaml index 8c2a10f5d7b5..8950159f043e 100644 --- a/.chloggen/jm-prom-translation-rw2-add-support-for-labels.yaml +++ b/.chloggen/jm-prom-translation-rw2-add-support-for-labels.yaml @@ -24,4 +24,4 @@ subtext: # Include 'user' if the change is relevant to end users. # Include 'api' if there is a change to a library API. # Default: '[user]' -change_logs: [] +change_logs: [api] From bc096d3fd279f8bacfaf3d5a7e012abd8327f854 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juraj=20Mich=C3=A1lek?= Date: Sun, 20 Oct 2024 23:21:32 +0200 Subject: [PATCH 5/9] Update .chloggen/jm-prom-translation-rw2-add-support-for-labels.yaml Co-authored-by: Arthur Silva Sens --- .chloggen/jm-prom-translation-rw2-add-support-for-labels.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.chloggen/jm-prom-translation-rw2-add-support-for-labels.yaml b/.chloggen/jm-prom-translation-rw2-add-support-for-labels.yaml index 8950159f043e..57473580a18a 100644 --- a/.chloggen/jm-prom-translation-rw2-add-support-for-labels.yaml +++ b/.chloggen/jm-prom-translation-rw2-add-support-for-labels.yaml @@ -7,7 +7,7 @@ change_type: enhancement component: pkg/translator/prometheusremotewrite # A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). -note: RW2 translation added support for labels +note: `FromMetricsV2` now transforms attributes into labels. # Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. issues: [33661] From 7446458ac987cf542747424f3b4e18f0f16af9dd Mon Sep 17 00:00:00 2001 From: Juraj Michalek Date: Sun, 20 Oct 2024 23:36:17 +0200 Subject: [PATCH 6/9] chore: address comment from PR Signed-off-by: Juraj Michalek --- pkg/translator/prometheusremotewrite/metrics_to_prw_v2.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/translator/prometheusremotewrite/metrics_to_prw_v2.go b/pkg/translator/prometheusremotewrite/metrics_to_prw_v2.go index 4af418374a29..5ccd4e166e8a 100644 --- a/pkg/translator/prometheusremotewrite/metrics_to_prw_v2.go +++ b/pkg/translator/prometheusremotewrite/metrics_to_prw_v2.go @@ -115,8 +115,9 @@ func (c *prometheusConverterV2) addSample(sample *writev2.Sample, lbls []prompb. } buf := make([]uint32, 0, len(lbls)*2) + var off uint32 for _, l := range lbls { - off := c.symbolTable.Symbolize(l.Name) + off = c.symbolTable.Symbolize(l.Name) buf = append(buf, off) off = c.symbolTable.Symbolize(l.Value) buf = append(buf, off) From e758fbbe13457424877017687e59004e418aed21 Mon Sep 17 00:00:00 2001 From: Juraj Michalek Date: Sun, 20 Oct 2024 23:46:55 +0200 Subject: [PATCH 7/9] chore: tests check contect of symbolTable Signed-off-by: Juraj Michalek --- .../prometheusremotewrite/metrics_to_prw_v2_test.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pkg/translator/prometheusremotewrite/metrics_to_prw_v2_test.go b/pkg/translator/prometheusremotewrite/metrics_to_prw_v2_test.go index 30cb6b826b2b..ee56b78428e4 100644 --- a/pkg/translator/prometheusremotewrite/metrics_to_prw_v2_test.go +++ b/pkg/translator/prometheusremotewrite/metrics_to_prw_v2_test.go @@ -34,11 +34,9 @@ func TestFromMetricsV2(t *testing.T) { }, } } + wantedSymbols := []string{"", "series_name_2", "value-2", "series_name_3", "value-3", "__name__", "gauge_1", "series_name_1", "value-1"} tsMap, symbolsTable, err := FromMetricsV2(payload.Metrics(), settings) - wanted := want() require.NoError(t, err) - require.NotNil(t, tsMap) - require.Equal(t, wanted, tsMap) - require.NotNil(t, symbolsTable) - + require.Equal(t, want(), tsMap) + require.ElementsMatch(t, wantedSymbols, symbolsTable.Symbols()) } From 275215166cec79f2927af257ec6a11a0121b8eb4 Mon Sep 17 00:00:00 2001 From: Juraj Michalek Date: Sun, 20 Oct 2024 23:52:01 +0200 Subject: [PATCH 8/9] chore: fix changelog entry Signed-off-by: Juraj Michalek --- .chloggen/jm-prom-translation-rw2-add-support-for-labels.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.chloggen/jm-prom-translation-rw2-add-support-for-labels.yaml b/.chloggen/jm-prom-translation-rw2-add-support-for-labels.yaml index 57473580a18a..8efbf25ef45e 100644 --- a/.chloggen/jm-prom-translation-rw2-add-support-for-labels.yaml +++ b/.chloggen/jm-prom-translation-rw2-add-support-for-labels.yaml @@ -7,7 +7,7 @@ change_type: enhancement component: pkg/translator/prometheusremotewrite # A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). -note: `FromMetricsV2` now transforms attributes into labels. +note: "`FromMetricsV2` now transforms attributes into labels." # Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. issues: [33661] From 83aa81ebb5102d39a09df9ce52e05cda2860eaa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juraj=20Mich=C3=A1lek?= Date: Mon, 21 Oct 2024 16:24:50 +0200 Subject: [PATCH 9/9] Update .chloggen/jm-prom-translation-rw2-add-support-for-labels.yaml Co-authored-by: David Ashpole --- .chloggen/jm-prom-translation-rw2-add-support-for-labels.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.chloggen/jm-prom-translation-rw2-add-support-for-labels.yaml b/.chloggen/jm-prom-translation-rw2-add-support-for-labels.yaml index 8efbf25ef45e..24e0251534c5 100644 --- a/.chloggen/jm-prom-translation-rw2-add-support-for-labels.yaml +++ b/.chloggen/jm-prom-translation-rw2-add-support-for-labels.yaml @@ -24,4 +24,4 @@ subtext: # Include 'user' if the change is relevant to end users. # Include 'api' if there is a change to a library API. # Default: '[user]' -change_logs: [api] +change_logs: []