From 642e7f7f4532338cd1981d1128c4bf089806c5f4 Mon Sep 17 00:00:00 2001 From: David Ashpole Date: Fri, 25 Aug 2023 17:45:51 -0400 Subject: [PATCH] Parse scope information from otel_scope_name, otel_scope_version, and otel_scope_info (#25898) **Description:** Implements this specification: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/compatibility/prometheus_and_openmetrics.md#instrumentation-scope, with https://github.com/open-telemetry/opentelemetry-specification/pull/3660 Instrumentation Scope > Each otel_scope_info metric point present in a batch of metrics SHOULD be dropped from the incoming scrape, and converted to an instrumentation scope. The otel_scope_name and otel_scope_version labels, if present, MUST be converted to the Name and Version of the Instrumentation Scope. Additional labels MUST be added as scope attributes, with keys and values unaltered. Other metrics in the batch which have otel_scope_name and otel_scope_version labels that match an instrumentation scope MUST be placed within the matching instrumentation scope, and MUST remove those labels. > Metrics which are not found to be associated with an instrumentation scope MUST all be placed within an empty instrumentation scope, and MUST not have any labels removed. It does this by: * For all metrics, use `otel_scope_name` and `otel_scope_version` to set the scope name and version. * For `otel_scope_info` metrics, use `otel_scope_name` and `otel_scope_version` as the scope name and version, and all other labels (other than `job`/`instance`/`__name__`) as scope attributes. * Change `map[metricName]metricFamily` to `map[scope][metricName]metricFamily` to sort metrics by instrumentation scope as they come in, and to make writing the scope easier on commit. **Link to tracking Issue:** Fixes https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/25870 --- .chloggen/prometheus-receiver-scope.yaml | 27 ++++ receiver/prometheusreceiver/README.md | 10 ++ .../internal/transaction.go | 152 +++++++++++++----- .../metrics_receiver_labels_test.go | 43 +++++ 4 files changed, 190 insertions(+), 42 deletions(-) create mode 100644 .chloggen/prometheus-receiver-scope.yaml diff --git a/.chloggen/prometheus-receiver-scope.yaml b/.chloggen/prometheus-receiver-scope.yaml new file mode 100644 index 000000000000..3bdb60395d7f --- /dev/null +++ b/.chloggen/prometheus-receiver-scope.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: prometheusreceiver + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: The otel_scope_name and otel_scope_version labels are used to populate scope name and version. otel_scope_info is used to populate scope attributes. + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [25870] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/receiver/prometheusreceiver/README.md b/receiver/prometheusreceiver/README.md index bc183ef996cc..1634d48f4981 100644 --- a/receiver/prometheusreceiver/README.md +++ b/receiver/prometheusreceiver/README.md @@ -149,3 +149,13 @@ This receiver accepts exemplars coming in Prometheus format and converts it to O [sc]: https://github.com/prometheus/prometheus/blob/v2.28.1/docs/configuration/configuration.md#scrape_config +## Resource and Scope + +This receiver drops the `target_info` prometheus metric, if present, and uses attributes on +that metric to populate the OpenTelemetry Resource. + +It drops `otel_scope_name` and `otel_scope_version` labels, if present, from metrics, and uses them to populate +the OpenTelemetry Instrumentation Scope name and version. It drops the `otel_scope_info` metric, +and uses attributes (other than `otel_scope_name` and `otel_scope_version`) to populate Scope +Attributes. + diff --git a/receiver/prometheusreceiver/internal/transaction.go b/receiver/prometheusreceiver/internal/transaction.go index 77fb8f5ffdfc..e1ac2012f213 100644 --- a/receiver/prometheusreceiver/internal/transaction.go +++ b/receiver/prometheusreceiver/internal/transaction.go @@ -27,27 +27,38 @@ import ( ) const ( - targetMetricName = "target_info" - receiverName = "otelcol/prometheusreceiver" + targetMetricName = "target_info" + scopeMetricName = "otel_scope_info" + scopeNameLabel = "otel_scope_name" + scopeVersionLabel = "otel_scope_version" + receiverName = "otelcol/prometheusreceiver" ) type transaction struct { - isNew bool - trimSuffixes bool - ctx context.Context - families map[string]*metricFamily - mc scrape.MetricMetadataStore - sink consumer.Metrics - externalLabels labels.Labels - nodeResource pcommon.Resource - logger *zap.Logger - buildInfo component.BuildInfo - metricAdjuster MetricsAdjuster - obsrecv *obsreport.Receiver + isNew bool + trimSuffixes bool + ctx context.Context + families map[scopeID]map[string]*metricFamily + mc scrape.MetricMetadataStore + sink consumer.Metrics + externalLabels labels.Labels + nodeResource pcommon.Resource + scopeAttributes map[scopeID]pcommon.Map + logger *zap.Logger + buildInfo component.BuildInfo + metricAdjuster MetricsAdjuster + obsrecv *obsreport.Receiver // Used as buffer to calculate series ref hash. bufBytes []byte } +var emptyScopeID scopeID + +type scopeID struct { + name string + version string +} + func newTransaction( ctx context.Context, metricAdjuster MetricsAdjuster, @@ -57,17 +68,18 @@ func newTransaction( obsrecv *obsreport.Receiver, trimSuffixes bool) *transaction { return &transaction{ - ctx: ctx, - families: make(map[string]*metricFamily), - isNew: true, - trimSuffixes: trimSuffixes, - sink: sink, - metricAdjuster: metricAdjuster, - externalLabels: externalLabels, - logger: settings.Logger, - buildInfo: settings.BuildInfo, - obsrecv: obsrecv, - bufBytes: make([]byte, 0, 1024), + ctx: ctx, + families: make(map[scopeID]map[string]*metricFamily), + isNew: true, + trimSuffixes: trimSuffixes, + sink: sink, + metricAdjuster: metricAdjuster, + externalLabels: externalLabels, + logger: settings.Logger, + buildInfo: settings.BuildInfo, + obsrecv: obsrecv, + bufBytes: make([]byte, 0, 1024), + scopeAttributes: make(map[scopeID]pcommon.Map), } } @@ -121,10 +133,17 @@ func (t *transaction) Append(_ storage.SeriesRef, ls labels.Labels, atMs int64, // For the `target_info` metric we need to convert it to resource attributes. if metricName == targetMetricName { - return 0, t.AddTargetInfo(ls) + t.AddTargetInfo(ls) + return 0, nil + } + + // For the `otel_scope_info` metric we need to convert it to scope attributes. + if metricName == scopeMetricName { + t.addScopeInfo(ls) + return 0, nil } - curMF := t.getOrCreateMetricFamily(metricName) + curMF := t.getOrCreateMetricFamily(getScopeID(ls), metricName) err := curMF.addSeries(t.getSeriesRef(ls, curMF.mtype), metricName, ls, atMs, val) if err != nil { t.logger.Warn("failed to add datapoint", zap.Error(err), zap.String("metric_name", metricName), zap.Any("labels", ls)) @@ -133,18 +152,22 @@ func (t *transaction) Append(_ storage.SeriesRef, ls labels.Labels, atMs int64, return 0, nil // never return errors, as that fails the whole scrape } -func (t *transaction) getOrCreateMetricFamily(mn string) *metricFamily { - curMf, ok := t.families[mn] +func (t *transaction) getOrCreateMetricFamily(scope scopeID, mn string) *metricFamily { + _, ok := t.families[scope] + if !ok { + t.families[scope] = make(map[string]*metricFamily) + } + curMf, ok := t.families[scope][mn] if !ok { fn := mn if _, ok := t.mc.GetMetadata(mn); !ok { fn = normalizeMetricName(mn) } - if mf, ok := t.families[fn]; ok && mf.includesMetric(mn) { + if mf, ok := t.families[scope][fn]; ok && mf.includesMetric(mn) { curMf = mf } else { curMf = newMetricFamily(mn, t.mc, t.logger) - t.families[curMf.name] = curMf + t.families[scope][curMf.name] = curMf } } return curMf @@ -174,7 +197,7 @@ func (t *transaction) AppendExemplar(_ storage.SeriesRef, l labels.Labels, e exe return 0, errMetricNameNotFound } - mf := t.getOrCreateMetricFamily(mn) + mf := t.getOrCreateMetricFamily(getScopeID(l), mn) mf.addExemplar(t.getSeriesRef(l, mf.mtype), e) return 0, nil @@ -201,18 +224,47 @@ func (t *transaction) getMetrics(resource pcommon.Resource) (pmetric.Metrics, er md := pmetric.NewMetrics() rms := md.ResourceMetrics().AppendEmpty() resource.CopyTo(rms.Resource()) - ils := rms.ScopeMetrics().AppendEmpty() - ils.Scope().SetName(receiverName) - ils.Scope().SetVersion(t.buildInfo.Version) - metrics := ils.Metrics() - for _, mf := range t.families { - mf.appendMetric(metrics, t.trimSuffixes) + for scope, mfs := range t.families { + ils := rms.ScopeMetrics().AppendEmpty() + // If metrics don't include otel_scope_name or otel_scope_version + // labels, use the receiver name and version. + if scope == emptyScopeID { + ils.Scope().SetName(receiverName) + ils.Scope().SetVersion(t.buildInfo.Version) + } else { + // Otherwise, use the scope that was provided with the metrics. + ils.Scope().SetName(scope.name) + ils.Scope().SetVersion(scope.version) + // If we got an otel_scope_info metric for that scope, get scope + // attributes from it. + attributes, ok := t.scopeAttributes[scope] + if ok { + attributes.CopyTo(ils.Scope().Attributes()) + } + } + metrics := ils.Metrics() + for _, mf := range mfs { + mf.appendMetric(metrics, t.trimSuffixes) + } } return md, nil } +func getScopeID(ls labels.Labels) scopeID { + var scope scopeID + for _, lbl := range ls { + if lbl.Name == scopeNameLabel { + scope.name = lbl.Value + } + if lbl.Name == scopeVersionLabel { + scope.version = lbl.Value + } + } + return scope +} + func (t *transaction) initTransaction(labels labels.Labels) error { target, ok := scrape.TargetFromContext(t.ctx) if !ok { @@ -268,18 +320,34 @@ func (t *transaction) UpdateMetadata(_ storage.SeriesRef, _ labels.Labels, _ met return 0, nil } -func (t *transaction) AddTargetInfo(labels labels.Labels) error { +func (t *transaction) AddTargetInfo(labels labels.Labels) { attrs := t.nodeResource.Attributes() - for _, lbl := range labels { if lbl.Name == model.JobLabel || lbl.Name == model.InstanceLabel || lbl.Name == model.MetricNameLabel { continue } - attrs.PutStr(lbl.Name, lbl.Value) } +} - return nil +func (t *transaction) addScopeInfo(labels labels.Labels) { + attrs := pcommon.NewMap() + scope := scopeID{} + for _, lbl := range labels { + if lbl.Name == model.JobLabel || lbl.Name == model.InstanceLabel || lbl.Name == model.MetricNameLabel { + continue + } + if lbl.Name == scopeNameLabel { + scope.name = lbl.Value + continue + } + if lbl.Name == scopeVersionLabel { + scope.version = lbl.Value + continue + } + attrs.PutStr(lbl.Name, lbl.Value) + } + t.scopeAttributes[scope] = attrs } func getSeriesRef(bytes []byte, ls labels.Labels, mtype pmetric.MetricType) (uint64, []byte) { diff --git a/receiver/prometheusreceiver/metrics_receiver_labels_test.go b/receiver/prometheusreceiver/metrics_receiver_labels_test.go index 6adfef5c1a4a..f4251962c0e9 100644 --- a/receiver/prometheusreceiver/metrics_receiver_labels_test.go +++ b/receiver/prometheusreceiver/metrics_receiver_labels_test.go @@ -737,3 +737,46 @@ func verifyTargetInfoResourceAttributes(t *testing.T, td *testData, rms []pmetri }), }) } + +const targetInstrumentationScopes = ` +# HELP jvm_memory_bytes_used Used bytes of a given JVM memory area. +# TYPE jvm_memory_bytes_used gauge +jvm_memory_bytes_used{area="heap", otel_scope_name="fake.scope.name", otel_scope_version="v0.1.0"} 100 +jvm_memory_bytes_used{area="heap", otel_scope_name="scope.with.attributes", otel_scope_version="v1.5.0"} 100 +jvm_memory_bytes_used{area="heap"} 100 +# TYPE otel_scope_info gauge +otel_scope_info{animal="bear", otel_scope_name="scope.with.attributes", otel_scope_version="v1.5.0"} 1 +` + +func TestScopeInfoScopeAttributes(t *testing.T) { + targets := []*testData{ + { + name: "target1", + pages: []mockPrometheusResponse{ + {code: 200, data: targetInstrumentationScopes}, + }, + validateFunc: verifyMultipleScopes, + }, + } + + testComponent(t, targets, false, false, "") +} + +func verifyMultipleScopes(t *testing.T, td *testData, rms []pmetric.ResourceMetrics) { + verifyNumValidScrapeResults(t, td, rms) + require.Greater(t, len(rms), 0, "At least one resource metric should be present") + + sms := rms[0].ScopeMetrics() + require.Equal(t, sms.Len(), 3, "At two scope metrics should be present") + require.Equal(t, sms.At(0).Scope().Name(), "fake.scope.name") + require.Equal(t, sms.At(0).Scope().Version(), "v0.1.0") + require.Equal(t, sms.At(0).Scope().Attributes().Len(), 0) + require.Equal(t, sms.At(1).Scope().Name(), "scope.with.attributes") + require.Equal(t, sms.At(1).Scope().Version(), "v1.5.0") + require.Equal(t, sms.At(1).Scope().Attributes().Len(), 1) + scopeAttrVal, found := sms.At(1).Scope().Attributes().Get("animal") + require.True(t, found) + require.Equal(t, scopeAttrVal.Str(), "bear") + require.Equal(t, sms.At(2).Scope().Name(), "otelcol/prometheusreceiver") + require.Equal(t, sms.At(2).Scope().Attributes().Len(), 0) +}