From f81021f571d45909634e5943e4a94926ae59a3ff Mon Sep 17 00:00:00 2001 From: Spencer Torres Date: Wed, 24 Jul 2024 23:53:50 -0400 Subject: [PATCH 1/5] begin update trace table --- ...clickhouseexporter_update_trace_table.yaml | 27 ++++++ .../example/default_ddl/traces.sql | 94 ++++++++++++------- .../clickhouseexporter/exporter_traces.go | 6 +- 3 files changed, 90 insertions(+), 37 deletions(-) create mode 100644 .chloggen/clickhouseexporter_update_trace_table.yaml diff --git a/.chloggen/clickhouseexporter_update_trace_table.yaml b/.chloggen/clickhouseexporter_update_trace_table.yaml new file mode 100644 index 000000000000..540b7a35ce1d --- /dev/null +++ b/.chloggen/clickhouseexporter_update_trace_table.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: clickhouseexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Updated the default trace table + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/exporter/clickhouseexporter/example/default_ddl/traces.sql b/exporter/clickhouseexporter/example/default_ddl/traces.sql index a9ecb397bad0..9b05c3771712 100644 --- a/exporter/clickhouseexporter/example/default_ddl/traces.sql +++ b/exporter/clickhouseexporter/example/default_ddl/traces.sql @@ -1,40 +1,64 @@ -- Default Trace table DDL CREATE TABLE IF NOT EXISTS otel_traces ( - Timestamp DateTime64(9) CODEC(Delta, ZSTD(1)), - TraceId String CODEC(ZSTD(1)), - SpanId String CODEC(ZSTD(1)), - ParentSpanId String CODEC(ZSTD(1)), - TraceState String CODEC(ZSTD(1)), - SpanName LowCardinality(String) CODEC(ZSTD(1)), - SpanKind LowCardinality(String) CODEC(ZSTD(1)), - ServiceName LowCardinality(String) CODEC(ZSTD(1)), - ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)), - ScopeName String CODEC(ZSTD(1)), - ScopeVersion String CODEC(ZSTD(1)), - SpanAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)), - Duration Int64 CODEC(ZSTD(1)), - StatusCode LowCardinality(String) CODEC(ZSTD(1)), - StatusMessage String CODEC(ZSTD(1)), - Events Nested ( - Timestamp DateTime64(9), - Name LowCardinality(String), - Attributes Map(LowCardinality(String), String) - ) CODEC(ZSTD(1)), - Links Nested ( - TraceId String, - SpanId String, - TraceState String, - Attributes Map(LowCardinality(String), String) - ) CODEC(ZSTD(1)), - INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, - INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_duration Duration TYPE minmax GRANULARITY 1 + Timestamp DateTime64(9) CODEC(Delta, ZSTD(1)), + TimestampTime DateTime DEFAULT toDateTime(Timestamp), + TraceId String CODEC(ZSTD(1)), + SpanId String CODEC(ZSTD(1)), + ParentSpanId String CODEC(ZSTD(1)), + TraceState String CODEC(ZSTD(1)), + SpanName LowCardinality(String) CODEC(ZSTD(1)), + SpanKind LowCardinality(String) CODEC(ZSTD(1)), + ServiceName LowCardinality(String) CODEC(ZSTD(1)), + ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)), + ScopeName String CODEC(ZSTD(1)), + ScopeVersion String CODEC(ZSTD(1)), + SpanAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)), + Duration Int64 CODEC(ZSTD(1)), + StatusCode LowCardinality(String) CODEC(ZSTD(1)), + StatusMessage String CODEC(ZSTD(1)), + Events Nested ( + Timestamp DateTime64(9), + Name LowCardinality(String), + Attributes Map(LowCardinality(String), String) + ) CODEC(ZSTD(1)), + Links Nested ( + TraceId String, + SpanId String, + TraceState String, + Attributes Map(LowCardinality(String), String) + ) CODEC(ZSTD(1)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_duration Duration TYPE minmax GRANULARITY 1 ) ENGINE = MergeTree() -TTL toDateTime("Timestamp") + toIntervalDay(180) -PARTITION BY toDate(Timestamp) -ORDER BY (ServiceName, SpanName, toUnixTimestamp(Timestamp), TraceId) +PARTITION BY toDate(TimestampTime) +PRIMARY KEY (ServiceName, SpanName, TimestampTime) +ORDER BY (ServiceName, SpanName, TimestampTime, Timestamp, TraceId) +TTL TimestampTime + toIntervalDay(180) SETTINGS index_granularity=8192, ttl_only_drop_parts = 1; + + +CREATE TABLE IF NOT EXISTS otel_traces_trace_id_ts ( + TraceId String CODEC(ZSTD(1)), + Start DateTime64(9) CODEC(Delta, ZSTD(1)), + End DateTime64(9) CODEC(Delta, ZSTD(1)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1 +) ENGINE = MergeTree() +ORDER BY (TraceId, toUnixTimestamp(Start)) +TTL toDateTime(Start) + toIntervalDay(180) +SETTINGS index_granularity=8192; + + +CREATE MATERIALIZED VIEW IF NOT EXISTS otel_traces_trace_id_ts_mv +TO otel_traces_trace_id_ts +AS SELECT + TraceId, + min(Timestamp) as Start, + max(Timestamp) as End +FROM otel_traces +WHERE TraceId != '' +GROUP BY TraceId; diff --git a/exporter/clickhouseexporter/exporter_traces.go b/exporter/clickhouseexporter/exporter_traces.go index 6a04f4d157b0..cbec85c02840 100644 --- a/exporter/clickhouseexporter/exporter_traces.go +++ b/exporter/clickhouseexporter/exporter_traces.go @@ -164,6 +164,7 @@ const ( createTracesTableSQL = ` CREATE TABLE IF NOT EXISTS %s %s ( Timestamp DateTime64(9) CODEC(Delta, ZSTD(1)), + TimestampTime DateTime DEFAULT toDateTime(Timestamp), TraceId String CODEC(ZSTD(1)), SpanId String CODEC(ZSTD(1)), ParentSpanId String CODEC(ZSTD(1)), @@ -196,9 +197,10 @@ CREATE TABLE IF NOT EXISTS %s %s ( INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_duration Duration TYPE minmax GRANULARITY 1 ) ENGINE = %s +PARTITION BY toDate(TimestampTime) +PRIMARY KEY (ServiceName, SpanName, TimestampTime) +ORDER BY (ServiceName, SpanName, TimestampTime, Timestamp, TraceId) %s -PARTITION BY toDate(Timestamp) -ORDER BY (ServiceName, SpanName, toUnixTimestamp(Timestamp), TraceId) SETTINGS index_granularity=8192, ttl_only_drop_parts = 1; ` // language=ClickHouse SQL From fa92645c9bd7384d393bef6da383ac312f1fc1ea Mon Sep 17 00:00:00 2001 From: Spencer Torres Date: Wed, 24 Jul 2024 23:58:08 -0400 Subject: [PATCH 2/5] changelog issue ID --- .chloggen/clickhouseexporter_update_trace_table.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.chloggen/clickhouseexporter_update_trace_table.yaml b/.chloggen/clickhouseexporter_update_trace_table.yaml index 540b7a35ce1d..9cb329bfd913 100644 --- a/.chloggen/clickhouseexporter_update_trace_table.yaml +++ b/.chloggen/clickhouseexporter_update_trace_table.yaml @@ -10,7 +10,7 @@ component: clickhouseexporter note: Updated the default trace table # Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. -issues: [] +issues: [34245] # (Optional) One or more lines of additional information to render under the primary note. # These lines will be padded with 2 spaces and then inserted directly into the document. From ebceba50d26e442b740f4aed148576468655ca7d Mon Sep 17 00:00:00 2001 From: Spencer Torres Date: Fri, 30 Aug 2024 01:56:27 -0400 Subject: [PATCH 3/5] formatting, codec --- .../example/default_ddl/traces.sql | 8 +-- .../clickhouseexporter/exporter_traces.go | 66 +++++++++---------- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/exporter/clickhouseexporter/example/default_ddl/traces.sql b/exporter/clickhouseexporter/example/default_ddl/traces.sql index 9b05c3771712..a93b1837e618 100644 --- a/exporter/clickhouseexporter/example/default_ddl/traces.sql +++ b/exporter/clickhouseexporter/example/default_ddl/traces.sql @@ -14,7 +14,7 @@ CREATE TABLE IF NOT EXISTS otel_traces ( ScopeName String CODEC(ZSTD(1)), ScopeVersion String CODEC(ZSTD(1)), SpanAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)), - Duration Int64 CODEC(ZSTD(1)), + Duration UInt64 CODEC(ZSTD(1)), StatusCode LowCardinality(String) CODEC(ZSTD(1)), StatusMessage String CODEC(ZSTD(1)), Events Nested ( @@ -23,9 +23,9 @@ CREATE TABLE IF NOT EXISTS otel_traces ( Attributes Map(LowCardinality(String), String) ) CODEC(ZSTD(1)), Links Nested ( - TraceId String, - SpanId String, - TraceState String, + TraceId String CODEC(ZSTD(1)), + SpanId String CODEC(ZSTD(1)), + TraceState String CODEC(ZSTD(1)), Attributes Map(LowCardinality(String), String) ) CODEC(ZSTD(1)), INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, diff --git a/exporter/clickhouseexporter/exporter_traces.go b/exporter/clickhouseexporter/exporter_traces.go index cbec85c02840..6fd15e00a7a9 100644 --- a/exporter/clickhouseexporter/exporter_traces.go +++ b/exporter/clickhouseexporter/exporter_traces.go @@ -163,39 +163,39 @@ const ( // language=ClickHouse SQL createTracesTableSQL = ` CREATE TABLE IF NOT EXISTS %s %s ( - Timestamp DateTime64(9) CODEC(Delta, ZSTD(1)), - TimestampTime DateTime DEFAULT toDateTime(Timestamp), - TraceId String CODEC(ZSTD(1)), - SpanId String CODEC(ZSTD(1)), - ParentSpanId String CODEC(ZSTD(1)), - TraceState String CODEC(ZSTD(1)), - SpanName LowCardinality(String) CODEC(ZSTD(1)), - SpanKind LowCardinality(String) CODEC(ZSTD(1)), - ServiceName LowCardinality(String) CODEC(ZSTD(1)), - ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)), - ScopeName String CODEC(ZSTD(1)), - ScopeVersion String CODEC(ZSTD(1)), - SpanAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)), - Duration Int64 CODEC(ZSTD(1)), - StatusCode LowCardinality(String) CODEC(ZSTD(1)), - StatusMessage String CODEC(ZSTD(1)), - Events Nested ( - Timestamp DateTime64(9), - Name LowCardinality(String), - Attributes Map(LowCardinality(String), String) - ) CODEC(ZSTD(1)), - Links Nested ( - TraceId String, - SpanId String, - TraceState String, - Attributes Map(LowCardinality(String), String) - ) CODEC(ZSTD(1)), - INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, - INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_duration Duration TYPE minmax GRANULARITY 1 + Timestamp DateTime64(9) CODEC(Delta, ZSTD(1)), + TimestampTime DateTime DEFAULT toDateTime(Timestamp), + TraceId String CODEC(ZSTD(1)), + SpanId String CODEC(ZSTD(1)), + ParentSpanId String CODEC(ZSTD(1)), + TraceState String CODEC(ZSTD(1)), + SpanName LowCardinality(String) CODEC(ZSTD(1)), + SpanKind LowCardinality(String) CODEC(ZSTD(1)), + ServiceName LowCardinality(String) CODEC(ZSTD(1)), + ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)), + ScopeName String CODEC(ZSTD(1)), + ScopeVersion String CODEC(ZSTD(1)), + SpanAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)), + Duration UInt64 CODEC(ZSTD(1)), + StatusCode LowCardinality(String) CODEC(ZSTD(1)), + StatusMessage String CODEC(ZSTD(1)), + Events Nested ( + Timestamp DateTime64(9), + Name LowCardinality(String) CODEC(ZSTD(1)), + Attributes Map(LowCardinality(String), String) + ) CODEC(ZSTD(1)), + Links Nested ( + TraceId String CODEC(ZSTD(1)), + SpanId String CODEC(ZSTD(1)), + TraceState String CODEC(ZSTD(1)), + Attributes Map(LowCardinality(String), String) + ) CODEC(ZSTD(1)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_duration Duration TYPE minmax GRANULARITY 1 ) ENGINE = %s PARTITION BY toDate(TimestampTime) PRIMARY KEY (ServiceName, SpanName, TimestampTime) From 33a70e9e634535235491d2042069dab85bc22a0f Mon Sep 17 00:00:00 2001 From: Spencer Torres Date: Wed, 11 Sep 2024 23:48:34 -0400 Subject: [PATCH 4/5] remove TimestampTime column --- exporter/clickhouseexporter/README.md | 2 +- .../example/default_ddl/traces.sql | 13 +++++----- .../clickhouseexporter/exporter_traces.go | 26 +++++++++---------- 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/exporter/clickhouseexporter/README.md b/exporter/clickhouseexporter/README.md index 9f3a0e72ee40..5b5c165dace6 100644 --- a/exporter/clickhouseexporter/README.md +++ b/exporter/clickhouseexporter/README.md @@ -196,7 +196,7 @@ SELECT Timestamp as log_time, toString(Links.TraceId) FROM otel_traces WHERE ServiceName = 'clickhouse-exporter' - AND StatusCode = 'STATUS_CODE_ERROR' + AND StatusCode = 'Error' AND Timestamp >= NOW() - INTERVAL 1 HOUR Limit 100; ``` diff --git a/exporter/clickhouseexporter/example/default_ddl/traces.sql b/exporter/clickhouseexporter/example/default_ddl/traces.sql index a93b1837e618..f8fb00d03c33 100644 --- a/exporter/clickhouseexporter/example/default_ddl/traces.sql +++ b/exporter/clickhouseexporter/example/default_ddl/traces.sql @@ -2,7 +2,6 @@ CREATE TABLE IF NOT EXISTS otel_traces ( Timestamp DateTime64(9) CODEC(Delta, ZSTD(1)), - TimestampTime DateTime DEFAULT toDateTime(Timestamp), TraceId String CODEC(ZSTD(1)), SpanId String CODEC(ZSTD(1)), ParentSpanId String CODEC(ZSTD(1)), @@ -35,10 +34,9 @@ CREATE TABLE IF NOT EXISTS otel_traces ( INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_duration Duration TYPE minmax GRANULARITY 1 ) ENGINE = MergeTree() -PARTITION BY toDate(TimestampTime) -PRIMARY KEY (ServiceName, SpanName, TimestampTime) -ORDER BY (ServiceName, SpanName, TimestampTime, Timestamp, TraceId) -TTL TimestampTime + toIntervalDay(180) +PARTITION BY toDate(Timestamp) +ORDER BY (ServiceName, SpanName, Timestamp) +TTL toDateTime(Timestamp) + toIntervalDay(180) SETTINGS index_granularity=8192, ttl_only_drop_parts = 1; @@ -48,9 +46,10 @@ CREATE TABLE IF NOT EXISTS otel_traces_trace_id_ts ( End DateTime64(9) CODEC(Delta, ZSTD(1)), INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1 ) ENGINE = MergeTree() -ORDER BY (TraceId, toUnixTimestamp(Start)) +PARTITION BY toDate(Start) +ORDER BY (Start) TTL toDateTime(Start) + toIntervalDay(180) -SETTINGS index_granularity=8192; +SETTINGS index_granularity=8192, ttl_only_drop_parts = 1; CREATE MATERIALIZED VIEW IF NOT EXISTS otel_traces_trace_id_ts_mv diff --git a/exporter/clickhouseexporter/exporter_traces.go b/exporter/clickhouseexporter/exporter_traces.go index 6fd15e00a7a9..52be8496697e 100644 --- a/exporter/clickhouseexporter/exporter_traces.go +++ b/exporter/clickhouseexporter/exporter_traces.go @@ -164,7 +164,6 @@ const ( createTracesTableSQL = ` CREATE TABLE IF NOT EXISTS %s %s ( Timestamp DateTime64(9) CODEC(Delta, ZSTD(1)), - TimestampTime DateTime DEFAULT toDateTime(Timestamp), TraceId String CODEC(ZSTD(1)), SpanId String CODEC(ZSTD(1)), ParentSpanId String CODEC(ZSTD(1)), @@ -197,9 +196,9 @@ CREATE TABLE IF NOT EXISTS %s %s ( INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, INDEX idx_duration Duration TYPE minmax GRANULARITY 1 ) ENGINE = %s -PARTITION BY toDate(TimestampTime) -PRIMARY KEY (ServiceName, SpanName, TimestampTime) -ORDER BY (ServiceName, SpanName, TimestampTime, Timestamp, TraceId) +PARTITION BY toDate(Timestamp) +PRIMARY KEY (ServiceName, SpanName, Timestamp) +ORDER BY (ServiceName, SpanName, Timestamp) %s SETTINGS index_granularity=8192, ttl_only_drop_parts = 1; ` @@ -255,26 +254,27 @@ SETTINGS index_granularity=8192, ttl_only_drop_parts = 1; const ( createTraceIDTsTableSQL = ` -create table IF NOT EXISTS %s_trace_id_ts %s ( +CREATE TABLE IF NOT EXISTS %s_trace_id_ts %s ( TraceId String CODEC(ZSTD(1)), Start DateTime64(9) CODEC(Delta, ZSTD(1)), End DateTime64(9) CODEC(Delta, ZSTD(1)), INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1 ) ENGINE = %s +PARTITION BY toDate(Start) +ORDER BY (TraceId, Start) %s -ORDER BY (TraceId, toUnixTimestamp(Start)) -SETTINGS index_granularity=8192; +SETTINGS index_granularity=8192, ttl_only_drop_parts = 1; ` createTraceIDTsMaterializedViewSQL = ` CREATE MATERIALIZED VIEW IF NOT EXISTS %s_trace_id_ts_mv %s TO %s.%s_trace_id_ts AS SELECT -TraceId, -min(Timestamp) as Start, -max(Timestamp) as End + TraceId, + min(Timestamp) as Start, + max(Timestamp) as End FROM %s.%s -WHERE TraceId!='' +WHERE TraceId != '' GROUP BY TraceId; ` ) @@ -284,10 +284,10 @@ func createTracesTable(ctx context.Context, cfg *Config, db *sql.DB) error { return fmt.Errorf("exec create traces table sql: %w", err) } if _, err := db.ExecContext(ctx, renderCreateTraceIDTsTableSQL(cfg)); err != nil { - return fmt.Errorf("exec create traceIDTs table sql: %w", err) + return fmt.Errorf("exec create traceID timestamp table sql: %w", err) } if _, err := db.ExecContext(ctx, renderTraceIDTsMaterializedViewSQL(cfg)); err != nil { - return fmt.Errorf("exec create traceIDTs view sql: %w", err) + return fmt.Errorf("exec create traceID timestamp view sql: %w", err) } return nil } From 77496ad605799a6686709673cfd5df4eae6b0acd Mon Sep 17 00:00:00 2001 From: Spencer Torres Date: Sat, 14 Sep 2024 02:04:20 -0400 Subject: [PATCH 5/5] update table + readme --- ...clickhouseexporter_update_trace_table.yaml | 2 +- exporter/clickhouseexporter/README.md | 8 ++++---- .../example/default_ddl/traces.sql | 18 +++++++++--------- .../clickhouseexporter/exporter_traces.go | 19 +++++++++---------- .../clickhouseexporter/integration_test.go | 2 -- 5 files changed, 23 insertions(+), 26 deletions(-) diff --git a/.chloggen/clickhouseexporter_update_trace_table.yaml b/.chloggen/clickhouseexporter_update_trace_table.yaml index 9cb329bfd913..4bfe626d9968 100644 --- a/.chloggen/clickhouseexporter_update_trace_table.yaml +++ b/.chloggen/clickhouseexporter_update_trace_table.yaml @@ -15,7 +15,7 @@ issues: [34245] # (Optional) One or more lines of additional information to render under the primary note. # These lines will be padded with 2 spaces and then inserted directly into the document. # Use pipe (|) for multiline entries. -subtext: +subtext: Reduced data types, improved partitioning and time range queries # If your change doesn't affect end users or the exported elements of any package, # you should instead start your pull request title with [chore] or use the "Skip Changelog" label. diff --git a/exporter/clickhouseexporter/README.md b/exporter/clickhouseexporter/README.md index 5b5c165dace6..baffa15262a8 100644 --- a/exporter/clickhouseexporter/README.md +++ b/exporter/clickhouseexporter/README.md @@ -128,7 +128,7 @@ Limit 100; - Find spans with specific attribute. ```sql -SELECT Timestamp as log_time, +SELECT Timestamp, TraceId, SpanId, ParentSpanId, @@ -156,7 +156,7 @@ WITH '391dae938234560b16bb63f51501cb6f' as trace_id, (SELECT min(Start) FROM otel_traces_trace_id_ts WHERE TraceId = trace_id) as start, (SELECT max(End) + 1 FROM otel_traces_trace_id_ts WHERE TraceId = trace_id) as end -SELECT Timestamp as log_time, +SELECT Timestamp, TraceId, SpanId, ParentSpanId, @@ -180,7 +180,7 @@ Limit 100; - Find spans is error. ```sql -SELECT Timestamp as log_time, +SELECT Timestamp, TraceId, SpanId, ParentSpanId, @@ -204,7 +204,7 @@ Limit 100; - Find slow spans. ```sql -SELECT Timestamp as log_time, +SELECT Timestamp, TraceId, SpanId, ParentSpanId, diff --git a/exporter/clickhouseexporter/example/default_ddl/traces.sql b/exporter/clickhouseexporter/example/default_ddl/traces.sql index f8fb00d03c33..bc4896fcb8db 100644 --- a/exporter/clickhouseexporter/example/default_ddl/traces.sql +++ b/exporter/clickhouseexporter/example/default_ddl/traces.sql @@ -22,9 +22,9 @@ CREATE TABLE IF NOT EXISTS otel_traces ( Attributes Map(LowCardinality(String), String) ) CODEC(ZSTD(1)), Links Nested ( - TraceId String CODEC(ZSTD(1)), - SpanId String CODEC(ZSTD(1)), - TraceState String CODEC(ZSTD(1)), + TraceId String, + SpanId String, + TraceState String, Attributes Map(LowCardinality(String), String) ) CODEC(ZSTD(1)), INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, @@ -35,20 +35,20 @@ CREATE TABLE IF NOT EXISTS otel_traces ( INDEX idx_duration Duration TYPE minmax GRANULARITY 1 ) ENGINE = MergeTree() PARTITION BY toDate(Timestamp) -ORDER BY (ServiceName, SpanName, Timestamp) -TTL toDateTime(Timestamp) + toIntervalDay(180) +ORDER BY (ServiceName, SpanName, toDateTime(Timestamp)) +TTL toDate(Timestamp) + toIntervalDay(180) SETTINGS index_granularity=8192, ttl_only_drop_parts = 1; CREATE TABLE IF NOT EXISTS otel_traces_trace_id_ts ( TraceId String CODEC(ZSTD(1)), - Start DateTime64(9) CODEC(Delta, ZSTD(1)), - End DateTime64(9) CODEC(Delta, ZSTD(1)), + Start DateTime CODEC(Delta, ZSTD(1)), + End DateTime CODEC(Delta, ZSTD(1)), INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1 ) ENGINE = MergeTree() PARTITION BY toDate(Start) -ORDER BY (Start) -TTL toDateTime(Start) + toIntervalDay(180) +ORDER BY (TraceId, Start) +TTL toDate(Start) + toIntervalDay(180) SETTINGS index_granularity=8192, ttl_only_drop_parts = 1; diff --git a/exporter/clickhouseexporter/exporter_traces.go b/exporter/clickhouseexporter/exporter_traces.go index 52be8496697e..c6d2c1d71ea0 100644 --- a/exporter/clickhouseexporter/exporter_traces.go +++ b/exporter/clickhouseexporter/exporter_traces.go @@ -180,13 +180,13 @@ CREATE TABLE IF NOT EXISTS %s %s ( StatusMessage String CODEC(ZSTD(1)), Events Nested ( Timestamp DateTime64(9), - Name LowCardinality(String) CODEC(ZSTD(1)), + Name LowCardinality(String), Attributes Map(LowCardinality(String), String) ) CODEC(ZSTD(1)), Links Nested ( - TraceId String CODEC(ZSTD(1)), - SpanId String CODEC(ZSTD(1)), - TraceState String CODEC(ZSTD(1)), + TraceId String, + SpanId String, + TraceState String, Attributes Map(LowCardinality(String), String) ) CODEC(ZSTD(1)), INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, @@ -197,8 +197,7 @@ CREATE TABLE IF NOT EXISTS %s %s ( INDEX idx_duration Duration TYPE minmax GRANULARITY 1 ) ENGINE = %s PARTITION BY toDate(Timestamp) -PRIMARY KEY (ServiceName, SpanName, Timestamp) -ORDER BY (ServiceName, SpanName, Timestamp) +ORDER BY (ServiceName, SpanName, toDateTime(Timestamp)) %s SETTINGS index_granularity=8192, ttl_only_drop_parts = 1; ` @@ -256,8 +255,8 @@ const ( createTraceIDTsTableSQL = ` CREATE TABLE IF NOT EXISTS %s_trace_id_ts %s ( TraceId String CODEC(ZSTD(1)), - Start DateTime64(9) CODEC(Delta, ZSTD(1)), - End DateTime64(9) CODEC(Delta, ZSTD(1)), + Start DateTime CODEC(Delta, ZSTD(1)), + End DateTime CODEC(Delta, ZSTD(1)), INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1 ) ENGINE = %s PARTITION BY toDate(Start) @@ -297,12 +296,12 @@ func renderInsertTracesSQL(cfg *Config) string { } func renderCreateTracesTableSQL(cfg *Config) string { - ttlExpr := generateTTLExpr(cfg.TTL, "toDateTime(Timestamp)") + ttlExpr := generateTTLExpr(cfg.TTL, "toDate(Timestamp)") return fmt.Sprintf(createTracesTableSQL, cfg.TracesTableName, cfg.clusterString(), cfg.tableEngineString(), ttlExpr) } func renderCreateTraceIDTsTableSQL(cfg *Config) string { - ttlExpr := generateTTLExpr(cfg.TTL, "toDateTime(Start)") + ttlExpr := generateTTLExpr(cfg.TTL, "toDate(Start)") return fmt.Sprintf(createTraceIDTsTableSQL, cfg.TracesTableName, cfg.clusterString(), cfg.tableEngineString(), ttlExpr) } diff --git a/exporter/clickhouseexporter/integration_test.go b/exporter/clickhouseexporter/integration_test.go index cbcba99f732e..74ecd52dde6b 100644 --- a/exporter/clickhouseexporter/integration_test.go +++ b/exporter/clickhouseexporter/integration_test.go @@ -95,7 +95,6 @@ func verifyExportLog(t *testing.T, logExporter *logsExporter) { type log struct { Timestamp string `db:"Timestamp"` - TimestampDate string `db:"TimestampDate"` TimestampTime string `db:"TimestampTime"` TraceID string `db:"TraceId"` SpanID string `db:"SpanId"` @@ -117,7 +116,6 @@ func verifyExportLog(t *testing.T, logExporter *logsExporter) { expectLog := log{ Timestamp: "2023-12-25T09:53:49Z", - TimestampDate: "2023-12-25T00:00:00Z", TimestampTime: "2023-12-25T09:53:49Z", TraceID: "01020300000000000000000000000000", SpanID: "0102030000000000",