From 49b729d864db2a21edb97f144b4758f599a01215 Mon Sep 17 00:00:00 2001 From: Sean Porter Date: Wed, 8 Jan 2025 03:09:52 -0800 Subject: [PATCH] [processor/tailsampling] Added debug logging to sampling decision caches (#37038) #### Description This pull-request adds debug logging to the sampling decision caches, making it a little easier to debug decisions. "Trace ID is in the sampled cache" and "Trace ID is in the non-sampled cache" with the trace ID as a zap field. --------- Signed-off-by: Sean Porter --- ...plingprocessor-decision-cache-logging.yaml | 27 +++++++++++++++++++ processor/tailsamplingprocessor/processor.go | 4 ++- 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 .chloggen/tailsamplingprocessor-decision-cache-logging.yaml diff --git a/.chloggen/tailsamplingprocessor-decision-cache-logging.yaml b/.chloggen/tailsamplingprocessor-decision-cache-logging.yaml new file mode 100644 index 000000000000..4551603b6364 --- /dev/null +++ b/.chloggen/tailsamplingprocessor-decision-cache-logging.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: tailsamplingprocessor + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Added debug logging to the sampling decision caches. + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [37038] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/processor/tailsamplingprocessor/processor.go b/processor/tailsamplingprocessor/processor.go index 99b8a50152d7..572aa3e60160 100644 --- a/processor/tailsamplingprocessor/processor.go +++ b/processor/tailsamplingprocessor/processor.go @@ -450,6 +450,7 @@ func (tsp *tailSamplingSpanProcessor) processTraces(resourceSpans ptrace.Resourc for id, spans := range idToSpansAndScope { // If the trace ID is in the sampled cache, short circuit the decision if _, ok := tsp.sampledIDCache.Get(id); ok { + tsp.logger.Debug("Trace ID is in the sampled cache", zap.Stringer("id", id)) traceTd := ptrace.NewTraces() appendToTraces(traceTd, resourceSpans, spans) tsp.releaseSampledTrace(tsp.ctx, id, traceTd) @@ -460,6 +461,7 @@ func (tsp *tailSamplingSpanProcessor) processTraces(resourceSpans ptrace.Resourc } // If the trace ID is in the non-sampled cache, short circuit the decision if _, ok := tsp.nonSampledIDCache.Get(id); ok { + tsp.logger.Debug("Trace ID is in the non-sampled cache", zap.Stringer("id", id)) tsp.telemetry.ProcessorTailSamplingEarlyReleasesFromCacheDecision. Add(tsp.ctx, int64(len(spans)), attrSampledFalse) continue @@ -553,7 +555,7 @@ func (tsp *tailSamplingSpanProcessor) dropTrace(traceID pcommon.TraceID, deletio tsp.numTracesOnMap.Add(^uint64(0)) } if trace == nil { - tsp.logger.Debug("Attempt to delete traceID not on table") + tsp.logger.Debug("Attempt to delete trace ID not on table", zap.Stringer("id", traceID)) return }