From 29786f52b02179a19125782bc1479b1aecc6eba1 Mon Sep 17 00:00:00 2001 From: Wassim DHIF Date: Mon, 13 Jan 2025 09:56:29 +0100 Subject: [PATCH] feat(dogstatsd): add cardinality common field Signed-off-by: Wassim DHIF --- comp/dogstatsd/server/enrich.go | 11 +++--- comp/dogstatsd/server/enrich_bench_test.go | 2 +- comp/dogstatsd/server/enrich_test.go | 39 ++++++++++++++++++- comp/dogstatsd/server/parse.go | 8 ++++ comp/dogstatsd/server/parse_events.go | 4 ++ comp/dogstatsd/server/parse_metrics.go | 2 + comp/dogstatsd/server/parse_service_checks.go | 4 ++ ...sd_cardinality_field-564b3d846700335c.yaml | 12 ++++++ 8 files changed, 74 insertions(+), 8 deletions(-) create mode 100644 releasenotes/notes/dogstatsd_cardinality_field-564b3d846700335c.yaml diff --git a/comp/dogstatsd/server/enrich.go b/comp/dogstatsd/server/enrich.go index df4d16fef553c6..8018ea50359801 100644 --- a/comp/dogstatsd/server/enrich.go +++ b/comp/dogstatsd/server/enrich.go @@ -37,7 +37,7 @@ type enrichConfig struct { } // extractTagsMetadata returns tags (client tags + host tag) and information needed to query tagger (origins, cardinality). -func extractTagsMetadata(tags []string, originFromUDS string, localData origindetection.LocalData, externalData origindetection.ExternalData, conf enrichConfig) ([]string, string, taggertypes.OriginInfo, metrics.MetricSource) { +func extractTagsMetadata(tags []string, originFromUDS string, localData origindetection.LocalData, externalData origindetection.ExternalData, cardinality string, conf enrichConfig) ([]string, string, taggertypes.OriginInfo, metrics.MetricSource) { host := conf.defaultHostname metricSource := metrics.MetricSourceDogstatsd origin := taggertypes.OriginInfo{ @@ -45,6 +45,7 @@ func extractTagsMetadata(tags []string, originFromUDS string, localData originde LocalData: localData, ExternalData: externalData, ProductOrigin: origindetection.ProductOriginDogStatsD, + Cardinality: cardinality, } n := 0 @@ -55,7 +56,7 @@ func extractTagsMetadata(tags []string, originFromUDS string, localData originde } else if strings.HasPrefix(tag, entityIDTagPrefix) { origin.LocalData.PodUID = tag[len(entityIDTagPrefix):] continue - } else if strings.HasPrefix(tag, CardinalityTagPrefix) { + } else if strings.HasPrefix(tag, CardinalityTagPrefix) && origin.Cardinality == "" { origin.Cardinality = tag[len(CardinalityTagPrefix):] continue } else if strings.HasPrefix(tag, jmxCheckNamePrefix) { @@ -112,7 +113,7 @@ func tsToFloatForSamples(ts time.Time) float64 { func enrichMetricSample(dest []metrics.MetricSample, ddSample dogstatsdMetricSample, origin string, listenerID string, conf enrichConfig) []metrics.MetricSample { metricName := ddSample.name - tags, hostnameFromTags, extractedOrigin, metricSource := extractTagsMetadata(ddSample.tags, origin, ddSample.localData, ddSample.externalData, conf) + tags, hostnameFromTags, extractedOrigin, metricSource := extractTagsMetadata(ddSample.tags, origin, ddSample.localData, ddSample.externalData, ddSample.cardinality, conf) if !isExcluded(metricName, conf.metricPrefix, conf.metricPrefixBlacklist) { metricName = conf.metricPrefix + metricName @@ -192,7 +193,7 @@ func enrichEventAlertType(dogstatsdAlertType alertType) metricsevent.AlertType { } func enrichEvent(event dogstatsdEvent, origin string, conf enrichConfig) *metricsevent.Event { - tags, hostnameFromTags, extractedOrigin, _ := extractTagsMetadata(event.tags, origin, event.localData, event.externalData, conf) + tags, hostnameFromTags, extractedOrigin, _ := extractTagsMetadata(event.tags, origin, event.localData, event.externalData, event.cardinality, conf) enrichedEvent := &metricsevent.Event{ Title: event.title, @@ -229,7 +230,7 @@ func enrichServiceCheckStatus(status serviceCheckStatus) servicecheck.ServiceChe } func enrichServiceCheck(serviceCheck dogstatsdServiceCheck, origin string, conf enrichConfig) *servicecheck.ServiceCheck { - tags, hostnameFromTags, extractedOrigin, _ := extractTagsMetadata(serviceCheck.tags, origin, serviceCheck.localData, serviceCheck.externalData, conf) + tags, hostnameFromTags, extractedOrigin, _ := extractTagsMetadata(serviceCheck.tags, origin, serviceCheck.localData, serviceCheck.externalData, serviceCheck.cardinality, conf) enrichedServiceCheck := &servicecheck.ServiceCheck{ CheckName: serviceCheck.name, diff --git a/comp/dogstatsd/server/enrich_bench_test.go b/comp/dogstatsd/server/enrich_bench_test.go index 2d4626fd5b7432..ef1517d7868da9 100644 --- a/comp/dogstatsd/server/enrich_bench_test.go +++ b/comp/dogstatsd/server/enrich_bench_test.go @@ -35,7 +35,7 @@ func BenchmarkExtractTagsMetadata(b *testing.B) { sb.ResetTimer() for n := 0; n < sb.N; n++ { - tags, _, _, _ = extractTagsMetadata(baseTags, "", origindetection.LocalData{}, origindetection.ExternalData{}, conf) + tags, _, _, _ = extractTagsMetadata(baseTags, "", origindetection.LocalData{}, origindetection.ExternalData{}, "", conf) } }) } diff --git a/comp/dogstatsd/server/enrich_test.go b/comp/dogstatsd/server/enrich_test.go index 13952d769428a9..a92ec54034cf6a 100644 --- a/comp/dogstatsd/server/enrich_test.go +++ b/comp/dogstatsd/server/enrich_test.go @@ -1115,6 +1115,7 @@ func TestEnrichTags(t *testing.T) { originFromMsg []byte localData origindetection.LocalData externalData origindetection.ExternalData + cardinality string conf enrichConfig } tests := []struct { @@ -1463,12 +1464,46 @@ func TestEnrichTags(t *testing.T) { }, wantedMetricSource: metrics.MetricSourceDogstatsd, }, + { + name: "cardinality field as none", + args: args{ + cardinality: types.NoneCardinalityString, + }, + wantedTags: nil, + wantedOrigin: taggertypes.OriginInfo{ + Cardinality: "none", + }, + wantedMetricSource: metrics.MetricSourceDogstatsd, + }, + { + name: "cardinality field as high", + args: args{ + cardinality: types.HighCardinalityString, + }, + wantedTags: nil, + wantedOrigin: taggertypes.OriginInfo{ + Cardinality: "high", + }, + wantedMetricSource: metrics.MetricSourceDogstatsd, + }, + { + name: "cardinality field with dd.internal.card", + args: args{ + tags: []string{"env:prod", "dd.internal.card:high"}, + cardinality: types.NoneCardinalityString, + }, + wantedTags: []string{"env:prod", "dd.internal.card:high"}, + wantedOrigin: taggertypes.OriginInfo{ + Cardinality: "none", + }, + wantedMetricSource: metrics.MetricSourceDogstatsd, + }, } for _, tt := range tests { tt.wantedOrigin.ProductOrigin = origindetection.ProductOriginDogStatsD t.Run(tt.name, func(t *testing.T) { - tags, host, origin, metricSource := extractTagsMetadata(tt.args.tags, tt.args.originFromUDS, tt.args.localData, tt.args.externalData, tt.args.conf) + tags, host, origin, metricSource := extractTagsMetadata(tt.args.tags, tt.args.originFromUDS, tt.args.localData, tt.args.externalData, tt.args.cardinality, tt.args.conf) assert.Equal(t, tt.wantedTags, tags) assert.Equal(t, tt.wantedHost, host) assert.Equal(t, tt.wantedOrigin, origin) @@ -1516,7 +1551,7 @@ func TestEnrichTagsWithJMXCheckName(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - tags, _, _, metricSource := extractTagsMetadata(tt.tags, "", origindetection.LocalData{}, origindetection.ExternalData{}, enrichConfig{}) + tags, _, _, metricSource := extractTagsMetadata(tt.tags, "", origindetection.LocalData{}, origindetection.ExternalData{}, "", enrichConfig{}) assert.Equal(t, tt.wantedTags, tags) assert.Equal(t, tt.wantedMetricSource, metricSource) assert.NotContains(t, tags, tt.jmxCheckName) diff --git a/comp/dogstatsd/server/parse.go b/comp/dogstatsd/server/parse.go index 8b7ae778af0503..e7b6b3b5123cf0 100644 --- a/comp/dogstatsd/server/parse.go +++ b/comp/dogstatsd/server/parse.go @@ -48,6 +48,9 @@ var ( // externalDataPrefix is the prefix for a common field which contains the external data for Origin Detection. externalDataPrefix = []byte("e:") + + // cardinalityPrefix is the prefix for a common field which contains the cardinality for Origin Detection. + cardinalityPrefix = []byte("card:") ) // parser parses dogstatsd messages @@ -174,6 +177,7 @@ func (p *parser) parseMetricSample(message []byte) (dogstatsdMetricSample, error var tags []string var localData origindetection.LocalData var externalData origindetection.ExternalData + var cardinality string var optionalField []byte var timestamp time.Time for message != nil { @@ -207,6 +211,9 @@ func (p *parser) parseMetricSample(message []byte) (dogstatsdMetricSample, error // external data case p.dsdOriginEnabled && bytes.HasPrefix(optionalField, externalDataPrefix): externalData = p.parseExternalData(optionalField[len(externalDataPrefix):]) + // cardinality + case p.dsdOriginEnabled && bytes.HasPrefix(optionalField, cardinalityPrefix): + cardinality = string(optionalField[len(cardinalityPrefix):]) } } @@ -220,6 +227,7 @@ func (p *parser) parseMetricSample(message []byte) (dogstatsdMetricSample, error tags: tags, localData: localData, externalData: externalData, + cardinality: cardinality, ts: timestamp, }, nil } diff --git a/comp/dogstatsd/server/parse_events.go b/comp/dogstatsd/server/parse_events.go index b84a530d180990..65e5393e2dad73 100644 --- a/comp/dogstatsd/server/parse_events.go +++ b/comp/dogstatsd/server/parse_events.go @@ -43,6 +43,8 @@ type dogstatsdEvent struct { localData origindetection.LocalData // externalData is used for Origin Detection externalData origindetection.ExternalData + // cardinality is used for Origin Detection + cardinality string } type eventHeader struct { @@ -170,6 +172,8 @@ func (p *parser) applyEventOptionalField(event dogstatsdEvent, optionalField []b newEvent.localData = p.parseLocalData(optionalField[len(localDataPrefix):]) case p.dsdOriginEnabled && bytes.HasPrefix(optionalField, externalDataPrefix): newEvent.externalData = p.parseExternalData(optionalField[len(externalDataPrefix):]) + case p.dsdOriginEnabled && bytes.HasPrefix(optionalField, cardinalityPrefix): + newEvent.cardinality = string(optionalField[len(cardinalityPrefix):]) } if err != nil { return event, err diff --git a/comp/dogstatsd/server/parse_metrics.go b/comp/dogstatsd/server/parse_metrics.go index 0c6da45f9d1205..8edba062bf863b 100644 --- a/comp/dogstatsd/server/parse_metrics.go +++ b/comp/dogstatsd/server/parse_metrics.go @@ -51,6 +51,8 @@ type dogstatsdMetricSample struct { localData origindetection.LocalData // externalData is used for Origin Detection externalData origindetection.ExternalData + // cardinality is used for Origin Detection + cardinality string // timestamp read in the message if any ts time.Time } diff --git a/comp/dogstatsd/server/parse_service_checks.go b/comp/dogstatsd/server/parse_service_checks.go index 6f4cc2239f2fa6..1c230d088e3739 100644 --- a/comp/dogstatsd/server/parse_service_checks.go +++ b/comp/dogstatsd/server/parse_service_checks.go @@ -34,6 +34,8 @@ type dogstatsdServiceCheck struct { localData origindetection.LocalData // externalData is used for Origin Detection externalData origindetection.ExternalData + // cardinality is used for Origin Detection + cardinality string } var ( @@ -104,6 +106,8 @@ func (p *parser) applyServiceCheckOptionalField(serviceCheck dogstatsdServiceChe newServiceCheck.localData = p.parseLocalData(optionalField[len(localDataPrefix):]) case p.dsdOriginEnabled && bytes.HasPrefix(optionalField, externalDataPrefix): newServiceCheck.externalData = p.parseExternalData(optionalField[len(externalDataPrefix):]) + case p.dsdOriginEnabled && bytes.HasPrefix(optionalField, cardinalityPrefix): + newServiceCheck.cardinality = string(optionalField[len(cardinalityPrefix):]) } if err != nil { return serviceCheck, err diff --git a/releasenotes/notes/dogstatsd_cardinality_field-564b3d846700335c.yaml b/releasenotes/notes/dogstatsd_cardinality_field-564b3d846700335c.yaml new file mode 100644 index 00000000000000..a5979aba81a64c --- /dev/null +++ b/releasenotes/notes/dogstatsd_cardinality_field-564b3d846700335c.yaml @@ -0,0 +1,12 @@ +# Each section from every release note are combined when the +# CHANGELOG.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +features: + - | + Add new `card:` common field to DogStatsD Datagram specification to allow + customer to specify the cardinality of the metric. This field is optional.