From 361f6b5b56bb3017bd3dfee890c011a199deff13 Mon Sep 17 00:00:00 2001 From: Kirill 'Earwin' Zakharenko Date: Wed, 9 Oct 2024 06:34:41 +0100 Subject: [PATCH] [exporter/clickhouseexporter] Sort attribute maps before insertion (#33634) --- ...clickhouseexporter-ordered-attributes.yaml | 27 ++++++++++++++ exporter/clickhouseexporter/exporter_logs.go | 17 +++------ .../clickhouseexporter/exporter_logs_test.go | 9 ++--- .../clickhouseexporter/exporter_metrics.go | 2 +- .../clickhouseexporter/exporter_traces.go | 36 +++++++------------ exporter/clickhouseexporter/go.mod | 8 ++--- exporter/clickhouseexporter/go.sum | 22 ++++++------ .../internal/exponential_histogram_metrics.go | 15 ++++---- .../internal/gauge_metrics.go | 15 ++++---- .../internal/histogram_metrics.go | 15 ++++---- .../internal/metrics_model.go | 21 +++++------ .../internal/metrics_model_test.go | 21 +++++------ .../internal/sum_metrics.go | 15 ++++---- .../internal/summary_metrics.go | 15 ++++---- 14 files changed, 117 insertions(+), 121 deletions(-) create mode 100644 .chloggen/clickhouseexporter-ordered-attributes.yaml diff --git a/.chloggen/clickhouseexporter-ordered-attributes.yaml b/.chloggen/clickhouseexporter-ordered-attributes.yaml new file mode 100644 index 000000000000..2275aef4e0c7 --- /dev/null +++ b/.chloggen/clickhouseexporter-ordered-attributes.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: clickhouseexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: "Exporter now sorts attribute maps' keys during INSERT, yielding better compression and predictable aggregates" + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [33634] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/exporter/clickhouseexporter/exporter_logs.go b/exporter/clickhouseexporter/exporter_logs.go index a4987457420a..30894377d2e0 100644 --- a/exporter/clickhouseexporter/exporter_logs.go +++ b/exporter/clickhouseexporter/exporter_logs.go @@ -11,11 +11,11 @@ import ( _ "github.com/ClickHouse/clickhouse-go/v2" // For register database driver. "go.opentelemetry.io/collector/component" - "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/plog" conventions "go.opentelemetry.io/collector/semconv/v1.27.0" "go.uber.org/zap" + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter/internal" "github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/traceutil" ) @@ -77,7 +77,7 @@ func (e *logsExporter) pushLogsData(ctx context.Context, ld plog.Logs) error { logs := ld.ResourceLogs().At(i) res := logs.Resource() resURL := logs.SchemaUrl() - resAttr := attributesToMap(res.Attributes()) + resAttr := internal.AttributesToMap(res.Attributes()) if v, ok := res.Attributes().Get(conventions.AttributeServiceName); ok { serviceName = v.Str() } @@ -87,7 +87,7 @@ func (e *logsExporter) pushLogsData(ctx context.Context, ld plog.Logs) error { scopeURL := logs.ScopeLogs().At(j).SchemaUrl() scopeName := logs.ScopeLogs().At(j).Scope().Name() scopeVersion := logs.ScopeLogs().At(j).Scope().Version() - scopeAttr := attributesToMap(logs.ScopeLogs().At(j).Scope().Attributes()) + scopeAttr := internal.AttributesToMap(logs.ScopeLogs().At(j).Scope().Attributes()) for k := 0; k < rs.Len(); k++ { r := rs.At(k) @@ -97,7 +97,7 @@ func (e *logsExporter) pushLogsData(ctx context.Context, ld plog.Logs) error { timestamp = r.ObservedTimestamp() } - logAttr := attributesToMap(r.Attributes()) + logAttr := internal.AttributesToMap(r.Attributes()) _, err = statement.ExecContext(ctx, timestamp.AsTime(), traceutil.TraceIDToHexOrEmptyString(r.TraceID()), @@ -129,15 +129,6 @@ func (e *logsExporter) pushLogsData(ctx context.Context, ld plog.Logs) error { return err } -func attributesToMap(attributes pcommon.Map) map[string]string { - m := make(map[string]string, attributes.Len()) - attributes.Range(func(k string, v pcommon.Value) bool { - m[k] = v.AsString() - return true - }) - return m -} - const ( // language=ClickHouse SQL createLogsTableSQL = ` diff --git a/exporter/clickhouseexporter/exporter_logs_test.go b/exporter/clickhouseexporter/exporter_logs_test.go index 7388e68f243c..8ad4cc564fd1 100644 --- a/exporter/clickhouseexporter/exporter_logs_test.go +++ b/exporter/clickhouseexporter/exporter_logs_test.go @@ -12,6 +12,7 @@ import ( "testing" "time" + "github.com/ClickHouse/clickhouse-go/v2/lib/column/orderedmap" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/plog" @@ -94,9 +95,9 @@ func TestExporter_pushLogsData(t *testing.T) { initClickhouseTestServer(t, func(query string, values []driver.Value) error { if strings.HasPrefix(query, "INSERT") { require.Equal(t, "https://opentelemetry.io/schemas/1.4.0", values[8]) - require.Equal(t, map[string]string{ + require.Equal(t, orderedmap.FromMap(map[string]string{ "service.name": "test-service", - }, values[9]) + }), values[9]) } return nil }) @@ -109,9 +110,9 @@ func TestExporter_pushLogsData(t *testing.T) { require.Equal(t, "https://opentelemetry.io/schemas/1.7.0", values[10]) require.Equal(t, "io.opentelemetry.contrib.clickhouse", values[11]) require.Equal(t, "1.0.0", values[12]) - require.Equal(t, map[string]string{ + require.Equal(t, orderedmap.FromMap(map[string]string{ "lib": "clickhouse", - }, values[13]) + }), values[13]) } return nil }) diff --git a/exporter/clickhouseexporter/exporter_metrics.go b/exporter/clickhouseexporter/exporter_metrics.go index 66f67b5cc065..be5696a01855 100644 --- a/exporter/clickhouseexporter/exporter_metrics.go +++ b/exporter/clickhouseexporter/exporter_metrics.go @@ -77,7 +77,7 @@ func (e *metricsExporter) pushMetricsData(ctx context.Context, md pmetric.Metric metricsMap := internal.NewMetricsModel(e.tablesConfig) for i := 0; i < md.ResourceMetrics().Len(); i++ { metrics := md.ResourceMetrics().At(i) - resAttr := attributesToMap(metrics.Resource().Attributes()) + resAttr := metrics.Resource().Attributes() for j := 0; j < metrics.ScopeMetrics().Len(); j++ { rs := metrics.ScopeMetrics().At(j).Metrics() scopeInstr := metrics.ScopeMetrics().At(j).Scope() diff --git a/exporter/clickhouseexporter/exporter_traces.go b/exporter/clickhouseexporter/exporter_traces.go index 193a1ad0fd8e..39a706c60afd 100644 --- a/exporter/clickhouseexporter/exporter_traces.go +++ b/exporter/clickhouseexporter/exporter_traces.go @@ -11,11 +11,13 @@ import ( "time" _ "github.com/ClickHouse/clickhouse-go/v2" // For register database driver. + "github.com/ClickHouse/clickhouse-go/v2/lib/column" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/pdata/ptrace" conventions "go.opentelemetry.io/collector/semconv/v1.27.0" "go.uber.org/zap" + "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter/internal" "github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/traceutil" ) @@ -74,18 +76,15 @@ func (e *tracesExporter) pushTraceData(ctx context.Context, td ptrace.Traces) er for i := 0; i < td.ResourceSpans().Len(); i++ { spans := td.ResourceSpans().At(i) res := spans.Resource() - resAttr := attributesToMap(res.Attributes()) - var serviceName string - if v, ok := res.Attributes().Get(conventions.AttributeServiceName); ok { - serviceName = v.Str() - } + resAttr := internal.AttributesToMap(res.Attributes()) + serviceName, _ := res.Attributes().Get(conventions.AttributeServiceName) for j := 0; j < spans.ScopeSpans().Len(); j++ { rs := spans.ScopeSpans().At(j).Spans() scopeName := spans.ScopeSpans().At(j).Scope().Name() scopeVersion := spans.ScopeSpans().At(j).Scope().Version() for k := 0; k < rs.Len(); k++ { r := rs.At(k) - spanAttr := attributesToMap(r.Attributes()) + spanAttr := internal.AttributesToMap(r.Attributes()) status := r.Status() eventTimes, eventNames, eventAttrs := convertEvents(r.Events()) linksTraceIDs, linksSpanIDs, linksTraceStates, linksAttrs := convertLinks(r.Links()) @@ -97,7 +96,7 @@ func (e *tracesExporter) pushTraceData(ctx context.Context, td ptrace.Traces) er r.TraceState().AsRaw(), r.Name(), r.Kind().String(), - serviceName, + serviceName.AsString(), resAttr, scopeName, scopeVersion, @@ -127,36 +126,25 @@ func (e *tracesExporter) pushTraceData(ctx context.Context, td ptrace.Traces) er return err } -func convertEvents(events ptrace.SpanEventSlice) ([]time.Time, []string, []map[string]string) { - var ( - times []time.Time - names []string - attrs []map[string]string - ) +func convertEvents(events ptrace.SpanEventSlice) (times []time.Time, names []string, attrs []column.IterableOrderedMap) { for i := 0; i < events.Len(); i++ { event := events.At(i) times = append(times, event.Timestamp().AsTime()) names = append(names, event.Name()) - attrs = append(attrs, attributesToMap(event.Attributes())) + attrs = append(attrs, internal.AttributesToMap(event.Attributes())) } - return times, names, attrs + return } -func convertLinks(links ptrace.SpanLinkSlice) ([]string, []string, []string, []map[string]string) { - var ( - traceIDs []string - spanIDs []string - states []string - attrs []map[string]string - ) +func convertLinks(links ptrace.SpanLinkSlice) (traceIDs []string, spanIDs []string, states []string, attrs []column.IterableOrderedMap) { for i := 0; i < links.Len(); i++ { link := links.At(i) traceIDs = append(traceIDs, traceutil.TraceIDToHexOrEmptyString(link.TraceID())) spanIDs = append(spanIDs, traceutil.SpanIDToHexOrEmptyString(link.SpanID())) states = append(states, link.TraceState().AsRaw()) - attrs = append(attrs, attributesToMap(link.Attributes())) + attrs = append(attrs, internal.AttributesToMap(link.Attributes())) } - return traceIDs, spanIDs, states, attrs + return } const ( diff --git a/exporter/clickhouseexporter/go.mod b/exporter/clickhouseexporter/go.mod index 841fa738688d..5f7f1eb102f2 100644 --- a/exporter/clickhouseexporter/go.mod +++ b/exporter/clickhouseexporter/go.mod @@ -3,7 +3,7 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickh go 1.22.0 require ( - github.com/ClickHouse/clickhouse-go/v2 v2.29.0 + github.com/ClickHouse/clickhouse-go/v2 v2.30.0 github.com/cenkalti/backoff/v4 v4.3.0 github.com/jmoiron/sqlx v1.4.0 github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.111.0 @@ -25,7 +25,7 @@ require ( github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/ClickHouse/ch-go v0.61.5 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect - github.com/andybalholm/brotli v1.1.0 // indirect + github.com/andybalholm/brotli v1.1.1 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/platforms v0.2.1 // indirect github.com/cpuguy83/dockercfg v0.3.1 // indirect @@ -98,8 +98,8 @@ require ( go.opentelemetry.io/otel/trace v1.31.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.26.0 // indirect - golang.org/x/net v0.28.0 // indirect + golang.org/x/crypto v0.28.0 // indirect + golang.org/x/net v0.30.0 // indirect golang.org/x/sys v0.26.0 // indirect golang.org/x/text v0.19.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd // indirect diff --git a/exporter/clickhouseexporter/go.sum b/exporter/clickhouseexporter/go.sum index d1beff56654f..91068ae3d101 100644 --- a/exporter/clickhouseexporter/go.sum +++ b/exporter/clickhouseexporter/go.sum @@ -8,12 +8,12 @@ github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOEl github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4= github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg= -github.com/ClickHouse/clickhouse-go/v2 v2.29.0 h1:Dj1w59RssRyLgGHXtYaWU0eIM1pJsu9nGPi/btmvAqw= -github.com/ClickHouse/clickhouse-go/v2 v2.29.0/go.mod h1:bLookq6qZJ4Ush/6tOAnJGh1Sf3Sa/nQoMn71p7ZCUE= +github.com/ClickHouse/clickhouse-go/v2 v2.30.0 h1:AG4D/hW39qa58+JHQIFOSnxyL46H6h2lrmGGk17dhFo= +github.com/ClickHouse/clickhouse-go/v2 v2.30.0/go.mod h1:i9ZQAojcayW3RsdCb3YR+n+wC2h65eJsZCscZ1Z1wyo= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= -github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= @@ -165,6 +165,8 @@ github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9f github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -239,8 +241,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= -golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -248,8 +250,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -269,8 +271,8 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= -golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/exporter/clickhouseexporter/internal/exponential_histogram_metrics.go b/exporter/clickhouseexporter/internal/exponential_histogram_metrics.go index 82d93dcf366f..064e12a2b234 100644 --- a/exporter/clickhouseexporter/internal/exponential_histogram_metrics.go +++ b/exporter/clickhouseexporter/internal/exponential_histogram_metrics.go @@ -130,27 +130,24 @@ func (e *expHistogramMetrics) insert(ctx context.Context, db *sql.DB) error { }() for _, model := range e.expHistogramModels { - var serviceName string - if v, ok := model.metadata.ResAttr[conventions.AttributeServiceName]; ok { - serviceName = v - } + serviceName, _ := model.metadata.ResAttr.Get(conventions.AttributeServiceName) for i := 0; i < model.expHistogram.DataPoints().Len(); i++ { dp := model.expHistogram.DataPoints().At(i) attrs, times, values, traceIDs, spanIDs := convertExemplars(dp.Exemplars()) _, err = statement.ExecContext(ctx, - model.metadata.ResAttr, + AttributesToMap(model.metadata.ResAttr), model.metadata.ResURL, model.metadata.ScopeInstr.Name(), model.metadata.ScopeInstr.Version(), - attributesToMap(model.metadata.ScopeInstr.Attributes()), + AttributesToMap(model.metadata.ScopeInstr.Attributes()), model.metadata.ScopeInstr.DroppedAttributesCount(), model.metadata.ScopeURL, - serviceName, + serviceName.AsString(), model.metricName, model.metricDescription, model.metricUnit, - attributesToMap(dp.Attributes()), + AttributesToMap(dp.Attributes()), dp.StartTimestamp().AsTime(), dp.Timestamp().AsTime(), dp.Count(), @@ -190,7 +187,7 @@ func (e *expHistogramMetrics) insert(ctx context.Context, db *sql.DB) error { return nil } -func (e *expHistogramMetrics) Add(resAttr map[string]string, resURL string, scopeInstr pcommon.InstrumentationScope, scopeURL string, metrics any, name string, description string, unit string) error { +func (e *expHistogramMetrics) Add(resAttr pcommon.Map, resURL string, scopeInstr pcommon.InstrumentationScope, scopeURL string, metrics any, name string, description string, unit string) error { expHistogram, ok := metrics.(pmetric.ExponentialHistogram) if !ok { return fmt.Errorf("metrics param is not type of ExponentialHistogram") diff --git a/exporter/clickhouseexporter/internal/gauge_metrics.go b/exporter/clickhouseexporter/internal/gauge_metrics.go index 596f3fa654ed..e2fbfe2dc365 100644 --- a/exporter/clickhouseexporter/internal/gauge_metrics.go +++ b/exporter/clickhouseexporter/internal/gauge_metrics.go @@ -109,27 +109,24 @@ func (g *gaugeMetrics) insert(ctx context.Context, db *sql.DB) error { }() for _, model := range g.gaugeModels { - var serviceName string - if v, ok := model.metadata.ResAttr[conventions.AttributeServiceName]; ok { - serviceName = v - } + serviceName, _ := model.metadata.ResAttr.Get(conventions.AttributeServiceName) for i := 0; i < model.gauge.DataPoints().Len(); i++ { dp := model.gauge.DataPoints().At(i) attrs, times, values, traceIDs, spanIDs := convertExemplars(dp.Exemplars()) _, err = statement.ExecContext(ctx, - model.metadata.ResAttr, + AttributesToMap(model.metadata.ResAttr), model.metadata.ResURL, model.metadata.ScopeInstr.Name(), model.metadata.ScopeInstr.Version(), - attributesToMap(model.metadata.ScopeInstr.Attributes()), + AttributesToMap(model.metadata.ScopeInstr.Attributes()), model.metadata.ScopeInstr.DroppedAttributesCount(), model.metadata.ScopeURL, - serviceName, + serviceName.AsString(), model.metricName, model.metricDescription, model.metricUnit, - attributesToMap(dp.Attributes()), + AttributesToMap(dp.Attributes()), dp.StartTimestamp().AsTime(), dp.Timestamp().AsTime(), getValue(dp.IntValue(), dp.DoubleValue(), dp.ValueType()), @@ -155,7 +152,7 @@ func (g *gaugeMetrics) insert(ctx context.Context, db *sql.DB) error { return nil } -func (g *gaugeMetrics) Add(resAttr map[string]string, resURL string, scopeInstr pcommon.InstrumentationScope, scopeURL string, metrics any, name string, description string, unit string) error { +func (g *gaugeMetrics) Add(resAttr pcommon.Map, resURL string, scopeInstr pcommon.InstrumentationScope, scopeURL string, metrics any, name string, description string, unit string) error { gauge, ok := metrics.(pmetric.Gauge) if !ok { return fmt.Errorf("metrics param is not type of Gauge") diff --git a/exporter/clickhouseexporter/internal/histogram_metrics.go b/exporter/clickhouseexporter/internal/histogram_metrics.go index 544e019cf8a3..f3374b655ba2 100644 --- a/exporter/clickhouseexporter/internal/histogram_metrics.go +++ b/exporter/clickhouseexporter/internal/histogram_metrics.go @@ -121,27 +121,24 @@ func (h *histogramMetrics) insert(ctx context.Context, db *sql.DB) error { }() for _, model := range h.histogramModel { - var serviceName string - if v, ok := model.metadata.ResAttr[conventions.AttributeServiceName]; ok { - serviceName = v - } + serviceName, _ := model.metadata.ResAttr.Get(conventions.AttributeServiceName) for i := 0; i < model.histogram.DataPoints().Len(); i++ { dp := model.histogram.DataPoints().At(i) attrs, times, values, traceIDs, spanIDs := convertExemplars(dp.Exemplars()) _, err = statement.ExecContext(ctx, - model.metadata.ResAttr, + AttributesToMap(model.metadata.ResAttr), model.metadata.ResURL, model.metadata.ScopeInstr.Name(), model.metadata.ScopeInstr.Version(), - attributesToMap(model.metadata.ScopeInstr.Attributes()), + AttributesToMap(model.metadata.ScopeInstr.Attributes()), model.metadata.ScopeInstr.DroppedAttributesCount(), model.metadata.ScopeURL, - serviceName, + serviceName.AsString(), model.metricName, model.metricDescription, model.metricUnit, - attributesToMap(dp.Attributes()), + AttributesToMap(dp.Attributes()), dp.StartTimestamp().AsTime(), dp.Timestamp().AsTime(), dp.Count(), @@ -177,7 +174,7 @@ func (h *histogramMetrics) insert(ctx context.Context, db *sql.DB) error { return nil } -func (h *histogramMetrics) Add(resAttr map[string]string, resURL string, scopeInstr pcommon.InstrumentationScope, scopeURL string, metrics any, name string, description string, unit string) error { +func (h *histogramMetrics) Add(resAttr pcommon.Map, resURL string, scopeInstr pcommon.InstrumentationScope, scopeURL string, metrics any, name string, description string, unit string) error { histogram, ok := metrics.(pmetric.Histogram) if !ok { return fmt.Errorf("metrics param is not type of Histogram") diff --git a/exporter/clickhouseexporter/internal/metrics_model.go b/exporter/clickhouseexporter/internal/metrics_model.go index 978a36d75be8..a412051800c0 100644 --- a/exporter/clickhouseexporter/internal/metrics_model.go +++ b/exporter/clickhouseexporter/internal/metrics_model.go @@ -13,6 +13,8 @@ import ( "sync" "github.com/ClickHouse/clickhouse-go/v2" + "github.com/ClickHouse/clickhouse-go/v2/lib/column" + "github.com/ClickHouse/clickhouse-go/v2/lib/column/orderedmap" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" "go.uber.org/zap" @@ -38,14 +40,14 @@ type MetricTypeConfig struct { // any type of metrics need implement it. type MetricsModel interface { // Add used to bind MetricsMetaData to a specific metric then put them into a slice - Add(resAttr map[string]string, resURL string, scopeInstr pcommon.InstrumentationScope, scopeURL string, metrics any, name string, description string, unit string) error + Add(resAttr pcommon.Map, resURL string, scopeInstr pcommon.InstrumentationScope, scopeURL string, metrics any, name string, description string, unit string) error // insert is used to insert metric data to clickhouse insert(ctx context.Context, db *sql.DB) error } // MetricsMetaData contain specific metric data type MetricsMetaData struct { - ResAttr map[string]string + ResAttr pcommon.Map ResURL string ScopeURL string ScopeInstr pcommon.InstrumentationScope @@ -118,7 +120,7 @@ func convertExemplars(exemplars pmetric.ExemplarSlice) (clickhouse.ArraySet, cli ) for i := 0; i < exemplars.Len(); i++ { exemplar := exemplars.At(i) - attrs = append(attrs, attributesToMap(exemplar.FilteredAttributes())) + attrs = append(attrs, AttributesToMap(exemplar.FilteredAttributes())) times = append(times, exemplar.Timestamp().AsTime()) values = append(values, getValue(exemplar.IntValue(), exemplar.DoubleValue(), exemplar.ValueType())) @@ -165,13 +167,12 @@ func getValue(intValue int64, floatValue float64, dataType any) float64 { } } -func attributesToMap(attributes pcommon.Map) map[string]string { - m := make(map[string]string, attributes.Len()) - attributes.Range(func(k string, v pcommon.Value) bool { - m[k] = v.AsString() - return true - }) - return m +func AttributesToMap(attributes pcommon.Map) column.IterableOrderedMap { + return orderedmap.CollectN(func(yield func(string, string) bool) { + attributes.Range(func(k string, v pcommon.Value) bool { + return yield(k, v.AsString()) + }) + }, attributes.Len()) } func convertSliceToArraySet[T any](slice []T) clickhouse.ArraySet { diff --git a/exporter/clickhouseexporter/internal/metrics_model_test.go b/exporter/clickhouseexporter/internal/metrics_model_test.go index 83d983bb8987..49ac0cfe7d33 100644 --- a/exporter/clickhouseexporter/internal/metrics_model_test.go +++ b/exporter/clickhouseexporter/internal/metrics_model_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/ClickHouse/clickhouse-go/v2" + "github.com/ClickHouse/clickhouse-go/v2/lib/column/orderedmap" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" @@ -20,15 +21,15 @@ func Test_attributesToMap(t *testing.T) { attributes.PutBool("bool", true) attributes.PutInt("int", 0) attributes.PutDouble("double", 0.0) - result := attributesToMap(attributes) + result := AttributesToMap(attributes) require.Equal( t, - map[string]string{ + orderedmap.FromMap(map[string]string{ "key": "value", "bool": "true", "int": "0", "double": "0", - }, + }), result, ) } @@ -59,7 +60,7 @@ func Test_convertExemplars(t *testing.T) { exemplar.FilteredAttributes().PutStr("key2", "value2") attrs, times, values, traceIDs, spanIDs := convertExemplars(exemplars) - require.Equal(t, clickhouse.ArraySet{map[string]string{"key1": "value1", "key2": "value2"}}, attrs) + require.Equal(t, clickhouse.ArraySet{orderedmap.FromMap(map[string]string{"key1": "value1", "key2": "value2"})}, attrs) require.Equal(t, clickhouse.ArraySet{time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC)}, times) require.Equal(t, clickhouse.ArraySet{0.0}, values) require.Equal(t, clickhouse.ArraySet{"00000000000000000000000000000000"}, traceIDs) @@ -71,7 +72,7 @@ func Test_convertExemplars(t *testing.T) { exemplar.SetTimestamp(pcommon.NewTimestampFromTime(time.Unix(1672218930, 0))) attrs, times, values, traceIDs, spanIDs := convertExemplars(exemplars) - require.Equal(t, clickhouse.ArraySet{map[string]string{}}, attrs) + require.Equal(t, clickhouse.ArraySet{orderedmap.FromMap(map[string]string{})}, attrs) require.Equal(t, clickhouse.ArraySet{time.Unix(1672218930, 0).UTC()}, times) require.Equal(t, clickhouse.ArraySet{0.0}, values) require.Equal(t, clickhouse.ArraySet{"00000000000000000000000000000000"}, traceIDs) @@ -83,7 +84,7 @@ func Test_convertExemplars(t *testing.T) { exemplar.SetDoubleValue(15.0) attrs, times, values, traceIDs, spanIDs := convertExemplars(exemplars) - require.Equal(t, clickhouse.ArraySet{map[string]string{}}, attrs) + require.Equal(t, clickhouse.ArraySet{orderedmap.FromMap(map[string]string{})}, attrs) require.Equal(t, clickhouse.ArraySet{time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC)}, times) require.Equal(t, clickhouse.ArraySet{15.0}, values) require.Equal(t, clickhouse.ArraySet{"00000000000000000000000000000000"}, traceIDs) @@ -95,7 +96,7 @@ func Test_convertExemplars(t *testing.T) { exemplar.SetIntValue(20) attrs, times, values, traceIDs, spanIDs := convertExemplars(exemplars) - require.Equal(t, clickhouse.ArraySet{map[string]string{}}, attrs) + require.Equal(t, clickhouse.ArraySet{orderedmap.FromMap(map[string]string{})}, attrs) require.Equal(t, clickhouse.ArraySet{time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC)}, times) require.Equal(t, clickhouse.ArraySet{20.0}, values) require.Equal(t, clickhouse.ArraySet{"00000000000000000000000000000000"}, traceIDs) @@ -107,7 +108,7 @@ func Test_convertExemplars(t *testing.T) { exemplar.SetSpanID([8]byte{1, 2, 3, 4}) attrs, times, values, traceIDs, spanIDs := convertExemplars(exemplars) - require.Equal(t, clickhouse.ArraySet{map[string]string{}}, attrs) + require.Equal(t, clickhouse.ArraySet{orderedmap.FromMap(map[string]string{})}, attrs) require.Equal(t, clickhouse.ArraySet{time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC)}, times) require.Equal(t, clickhouse.ArraySet{0.0}, values) require.Equal(t, clickhouse.ArraySet{"00000000000000000000000000000000"}, traceIDs) @@ -119,7 +120,7 @@ func Test_convertExemplars(t *testing.T) { exemplar.SetTraceID([16]byte{1, 2, 3, 4}) attrs, times, values, traceIDs, spanIDs := convertExemplars(exemplars) - require.Equal(t, clickhouse.ArraySet{map[string]string{}}, attrs) + require.Equal(t, clickhouse.ArraySet{orderedmap.FromMap(map[string]string{})}, attrs) require.Equal(t, clickhouse.ArraySet{time.Date(1970, time.January, 1, 0, 0, 0, 0, time.UTC)}, times) require.Equal(t, clickhouse.ArraySet{0.0}, values) require.Equal(t, clickhouse.ArraySet{"01020304000000000000000000000000"}, traceIDs) @@ -146,7 +147,7 @@ func Test_convertExemplars(t *testing.T) { exemplar.SetTraceID([16]byte{1, 2, 3, 5}) attrs, times, values, traceIDs, spanIDs := convertExemplars(exemplars) - require.Equal(t, clickhouse.ArraySet{map[string]string{"key1": "value1", "key2": "value2"}, map[string]string{"key3": "value3", "key4": "value4"}}, attrs) + require.Equal(t, clickhouse.ArraySet{orderedmap.FromMap(map[string]string{"key1": "value1", "key2": "value2"}), orderedmap.FromMap(map[string]string{"key3": "value3", "key4": "value4"})}, attrs) require.Equal(t, clickhouse.ArraySet{time.Unix(1672218930, 0).UTC(), time.Unix(1672219930, 0).UTC()}, times) require.Equal(t, clickhouse.ArraySet{20.0, 16.0}, values) require.Equal(t, clickhouse.ArraySet{"01020304000000000000000000000000", "01020305000000000000000000000000"}, traceIDs) diff --git a/exporter/clickhouseexporter/internal/sum_metrics.go b/exporter/clickhouseexporter/internal/sum_metrics.go index 4da0faa5cb3a..89455f8e3048 100644 --- a/exporter/clickhouseexporter/internal/sum_metrics.go +++ b/exporter/clickhouseexporter/internal/sum_metrics.go @@ -113,27 +113,24 @@ func (s *sumMetrics) insert(ctx context.Context, db *sql.DB) error { }() for _, model := range s.sumModel { - var serviceName string - if v, ok := model.metadata.ResAttr[conventions.AttributeServiceName]; ok { - serviceName = v - } + serviceName, _ := model.metadata.ResAttr.Get(conventions.AttributeServiceName) for i := 0; i < model.sum.DataPoints().Len(); i++ { dp := model.sum.DataPoints().At(i) attrs, times, values, traceIDs, spanIDs := convertExemplars(dp.Exemplars()) _, err = statement.ExecContext(ctx, - model.metadata.ResAttr, + AttributesToMap(model.metadata.ResAttr), model.metadata.ResURL, model.metadata.ScopeInstr.Name(), model.metadata.ScopeInstr.Version(), - attributesToMap(model.metadata.ScopeInstr.Attributes()), + AttributesToMap(model.metadata.ScopeInstr.Attributes()), model.metadata.ScopeInstr.DroppedAttributesCount(), model.metadata.ScopeURL, - serviceName, + serviceName.AsString(), model.metricName, model.metricDescription, model.metricUnit, - attributesToMap(dp.Attributes()), + AttributesToMap(dp.Attributes()), dp.StartTimestamp().AsTime(), dp.Timestamp().AsTime(), getValue(dp.IntValue(), dp.DoubleValue(), dp.ValueType()), @@ -165,7 +162,7 @@ func (s *sumMetrics) insert(ctx context.Context, db *sql.DB) error { return nil } -func (s *sumMetrics) Add(resAttr map[string]string, resURL string, scopeInstr pcommon.InstrumentationScope, scopeURL string, metrics any, name string, description string, unit string) error { +func (s *sumMetrics) Add(resAttr pcommon.Map, resURL string, scopeInstr pcommon.InstrumentationScope, scopeURL string, metrics any, name string, description string, unit string) error { sum, ok := metrics.(pmetric.Sum) if !ok { return fmt.Errorf("metrics param is not type of Sum") diff --git a/exporter/clickhouseexporter/internal/summary_metrics.go b/exporter/clickhouseexporter/internal/summary_metrics.go index 3182ffee452c..d98197c12b2e 100644 --- a/exporter/clickhouseexporter/internal/summary_metrics.go +++ b/exporter/clickhouseexporter/internal/summary_metrics.go @@ -103,28 +103,25 @@ func (s *summaryMetrics) insert(ctx context.Context, db *sql.DB) error { _ = statement.Close() }() for _, model := range s.summaryModel { - var serviceName string - if v, ok := model.metadata.ResAttr[conventions.AttributeServiceName]; ok { - serviceName = v - } + serviceName, _ := model.metadata.ResAttr.Get(conventions.AttributeServiceName) for i := 0; i < model.summary.DataPoints().Len(); i++ { dp := model.summary.DataPoints().At(i) quantiles, values := convertValueAtQuantile(dp.QuantileValues()) _, err = statement.ExecContext(ctx, - model.metadata.ResAttr, + AttributesToMap(model.metadata.ResAttr), model.metadata.ResURL, model.metadata.ScopeInstr.Name(), model.metadata.ScopeInstr.Version(), - attributesToMap(model.metadata.ScopeInstr.Attributes()), + AttributesToMap(model.metadata.ScopeInstr.Attributes()), model.metadata.ScopeInstr.DroppedAttributesCount(), model.metadata.ScopeURL, - serviceName, + serviceName.AsString(), model.metricName, model.metricDescription, model.metricUnit, - attributesToMap(dp.Attributes()), + AttributesToMap(dp.Attributes()), dp.StartTimestamp().AsTime(), dp.Timestamp().AsTime(), dp.Count(), @@ -153,7 +150,7 @@ func (s *summaryMetrics) insert(ctx context.Context, db *sql.DB) error { return nil } -func (s *summaryMetrics) Add(resAttr map[string]string, resURL string, scopeInstr pcommon.InstrumentationScope, scopeURL string, metrics any, name string, description string, unit string) error { +func (s *summaryMetrics) Add(resAttr pcommon.Map, resURL string, scopeInstr pcommon.InstrumentationScope, scopeURL string, metrics any, name string, description string, unit string) error { summary, ok := metrics.(pmetric.Summary) if !ok { return fmt.Errorf("metrics param is not type of Summary")