diff --git a/.chloggen/clickhouseexporter_update_trace_table.yaml b/.chloggen/clickhouseexporter_update_trace_table.yaml new file mode 100644 index 000000000000..4bfe626d9968 --- /dev/null +++ b/.chloggen/clickhouseexporter_update_trace_table.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: clickhouseexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Updated the default trace table + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [34245] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: Reduced data types, improved partitioning and time range queries + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/exporter/clickhouseexporter/README.md b/exporter/clickhouseexporter/README.md index 9f3a0e72ee40..baffa15262a8 100644 --- a/exporter/clickhouseexporter/README.md +++ b/exporter/clickhouseexporter/README.md @@ -128,7 +128,7 @@ Limit 100; - Find spans with specific attribute. ```sql -SELECT Timestamp as log_time, +SELECT Timestamp, TraceId, SpanId, ParentSpanId, @@ -156,7 +156,7 @@ WITH '391dae938234560b16bb63f51501cb6f' as trace_id, (SELECT min(Start) FROM otel_traces_trace_id_ts WHERE TraceId = trace_id) as start, (SELECT max(End) + 1 FROM otel_traces_trace_id_ts WHERE TraceId = trace_id) as end -SELECT Timestamp as log_time, +SELECT Timestamp, TraceId, SpanId, ParentSpanId, @@ -180,7 +180,7 @@ Limit 100; - Find spans is error. ```sql -SELECT Timestamp as log_time, +SELECT Timestamp, TraceId, SpanId, ParentSpanId, @@ -196,7 +196,7 @@ SELECT Timestamp as log_time, toString(Links.TraceId) FROM otel_traces WHERE ServiceName = 'clickhouse-exporter' - AND StatusCode = 'STATUS_CODE_ERROR' + AND StatusCode = 'Error' AND Timestamp >= NOW() - INTERVAL 1 HOUR Limit 100; ``` @@ -204,7 +204,7 @@ Limit 100; - Find slow spans. ```sql -SELECT Timestamp as log_time, +SELECT Timestamp, TraceId, SpanId, ParentSpanId, diff --git a/exporter/clickhouseexporter/example/default_ddl/traces.sql b/exporter/clickhouseexporter/example/default_ddl/traces.sql index a9ecb397bad0..bc4896fcb8db 100644 --- a/exporter/clickhouseexporter/example/default_ddl/traces.sql +++ b/exporter/clickhouseexporter/example/default_ddl/traces.sql @@ -1,40 +1,63 @@ -- Default Trace table DDL CREATE TABLE IF NOT EXISTS otel_traces ( - Timestamp DateTime64(9) CODEC(Delta, ZSTD(1)), - TraceId String CODEC(ZSTD(1)), - SpanId String CODEC(ZSTD(1)), - ParentSpanId String CODEC(ZSTD(1)), - TraceState String CODEC(ZSTD(1)), - SpanName LowCardinality(String) CODEC(ZSTD(1)), - SpanKind LowCardinality(String) CODEC(ZSTD(1)), - ServiceName LowCardinality(String) CODEC(ZSTD(1)), - ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)), - ScopeName String CODEC(ZSTD(1)), - ScopeVersion String CODEC(ZSTD(1)), - SpanAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)), - Duration Int64 CODEC(ZSTD(1)), - StatusCode LowCardinality(String) CODEC(ZSTD(1)), - StatusMessage String CODEC(ZSTD(1)), - Events Nested ( - Timestamp DateTime64(9), - Name LowCardinality(String), - Attributes Map(LowCardinality(String), String) - ) CODEC(ZSTD(1)), - Links Nested ( - TraceId String, - SpanId String, - TraceState String, - Attributes Map(LowCardinality(String), String) - ) CODEC(ZSTD(1)), - INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, - INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_duration Duration TYPE minmax GRANULARITY 1 + Timestamp DateTime64(9) CODEC(Delta, ZSTD(1)), + TraceId String CODEC(ZSTD(1)), + SpanId String CODEC(ZSTD(1)), + ParentSpanId String CODEC(ZSTD(1)), + TraceState String CODEC(ZSTD(1)), + SpanName LowCardinality(String) CODEC(ZSTD(1)), + SpanKind LowCardinality(String) CODEC(ZSTD(1)), + ServiceName LowCardinality(String) CODEC(ZSTD(1)), + ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)), + ScopeName String CODEC(ZSTD(1)), + ScopeVersion String CODEC(ZSTD(1)), + SpanAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)), + Duration UInt64 CODEC(ZSTD(1)), + StatusCode LowCardinality(String) CODEC(ZSTD(1)), + StatusMessage String CODEC(ZSTD(1)), + Events Nested ( + Timestamp DateTime64(9), + Name LowCardinality(String), + Attributes Map(LowCardinality(String), String) + ) CODEC(ZSTD(1)), + Links Nested ( + TraceId String, + SpanId String, + TraceState String, + Attributes Map(LowCardinality(String), String) + ) CODEC(ZSTD(1)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_duration Duration TYPE minmax GRANULARITY 1 ) ENGINE = MergeTree() -TTL toDateTime("Timestamp") + toIntervalDay(180) PARTITION BY toDate(Timestamp) -ORDER BY (ServiceName, SpanName, toUnixTimestamp(Timestamp), TraceId) +ORDER BY (ServiceName, SpanName, toDateTime(Timestamp)) +TTL toDate(Timestamp) + toIntervalDay(180) +SETTINGS index_granularity=8192, ttl_only_drop_parts = 1; + + +CREATE TABLE IF NOT EXISTS otel_traces_trace_id_ts ( + TraceId String CODEC(ZSTD(1)), + Start DateTime CODEC(Delta, ZSTD(1)), + End DateTime CODEC(Delta, ZSTD(1)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1 +) ENGINE = MergeTree() +PARTITION BY toDate(Start) +ORDER BY (TraceId, Start) +TTL toDate(Start) + toIntervalDay(180) SETTINGS index_granularity=8192, ttl_only_drop_parts = 1; + + +CREATE MATERIALIZED VIEW IF NOT EXISTS otel_traces_trace_id_ts_mv +TO otel_traces_trace_id_ts +AS SELECT + TraceId, + min(Timestamp) as Start, + max(Timestamp) as End +FROM otel_traces +WHERE TraceId != '' +GROUP BY TraceId; diff --git a/exporter/clickhouseexporter/exporter_traces.go b/exporter/clickhouseexporter/exporter_traces.go index 6a04f4d157b0..c6d2c1d71ea0 100644 --- a/exporter/clickhouseexporter/exporter_traces.go +++ b/exporter/clickhouseexporter/exporter_traces.go @@ -163,42 +163,42 @@ const ( // language=ClickHouse SQL createTracesTableSQL = ` CREATE TABLE IF NOT EXISTS %s %s ( - Timestamp DateTime64(9) CODEC(Delta, ZSTD(1)), - TraceId String CODEC(ZSTD(1)), - SpanId String CODEC(ZSTD(1)), - ParentSpanId String CODEC(ZSTD(1)), - TraceState String CODEC(ZSTD(1)), - SpanName LowCardinality(String) CODEC(ZSTD(1)), - SpanKind LowCardinality(String) CODEC(ZSTD(1)), - ServiceName LowCardinality(String) CODEC(ZSTD(1)), - ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)), - ScopeName String CODEC(ZSTD(1)), - ScopeVersion String CODEC(ZSTD(1)), - SpanAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)), - Duration Int64 CODEC(ZSTD(1)), - StatusCode LowCardinality(String) CODEC(ZSTD(1)), - StatusMessage String CODEC(ZSTD(1)), - Events Nested ( - Timestamp DateTime64(9), - Name LowCardinality(String), - Attributes Map(LowCardinality(String), String) - ) CODEC(ZSTD(1)), - Links Nested ( - TraceId String, - SpanId String, - TraceState String, - Attributes Map(LowCardinality(String), String) - ) CODEC(ZSTD(1)), - INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, - INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, - INDEX idx_duration Duration TYPE minmax GRANULARITY 1 + Timestamp DateTime64(9) CODEC(Delta, ZSTD(1)), + TraceId String CODEC(ZSTD(1)), + SpanId String CODEC(ZSTD(1)), + ParentSpanId String CODEC(ZSTD(1)), + TraceState String CODEC(ZSTD(1)), + SpanName LowCardinality(String) CODEC(ZSTD(1)), + SpanKind LowCardinality(String) CODEC(ZSTD(1)), + ServiceName LowCardinality(String) CODEC(ZSTD(1)), + ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)), + ScopeName String CODEC(ZSTD(1)), + ScopeVersion String CODEC(ZSTD(1)), + SpanAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)), + Duration UInt64 CODEC(ZSTD(1)), + StatusCode LowCardinality(String) CODEC(ZSTD(1)), + StatusMessage String CODEC(ZSTD(1)), + Events Nested ( + Timestamp DateTime64(9), + Name LowCardinality(String), + Attributes Map(LowCardinality(String), String) + ) CODEC(ZSTD(1)), + Links Nested ( + TraceId String, + SpanId String, + TraceState String, + Attributes Map(LowCardinality(String), String) + ) CODEC(ZSTD(1)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, + INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_duration Duration TYPE minmax GRANULARITY 1 ) ENGINE = %s -%s PARTITION BY toDate(Timestamp) -ORDER BY (ServiceName, SpanName, toUnixTimestamp(Timestamp), TraceId) +ORDER BY (ServiceName, SpanName, toDateTime(Timestamp)) +%s SETTINGS index_granularity=8192, ttl_only_drop_parts = 1; ` // language=ClickHouse SQL @@ -253,26 +253,27 @@ SETTINGS index_granularity=8192, ttl_only_drop_parts = 1; const ( createTraceIDTsTableSQL = ` -create table IF NOT EXISTS %s_trace_id_ts %s ( +CREATE TABLE IF NOT EXISTS %s_trace_id_ts %s ( TraceId String CODEC(ZSTD(1)), - Start DateTime64(9) CODEC(Delta, ZSTD(1)), - End DateTime64(9) CODEC(Delta, ZSTD(1)), + Start DateTime CODEC(Delta, ZSTD(1)), + End DateTime CODEC(Delta, ZSTD(1)), INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1 ) ENGINE = %s +PARTITION BY toDate(Start) +ORDER BY (TraceId, Start) %s -ORDER BY (TraceId, toUnixTimestamp(Start)) -SETTINGS index_granularity=8192; +SETTINGS index_granularity=8192, ttl_only_drop_parts = 1; ` createTraceIDTsMaterializedViewSQL = ` CREATE MATERIALIZED VIEW IF NOT EXISTS %s_trace_id_ts_mv %s TO %s.%s_trace_id_ts AS SELECT -TraceId, -min(Timestamp) as Start, -max(Timestamp) as End + TraceId, + min(Timestamp) as Start, + max(Timestamp) as End FROM %s.%s -WHERE TraceId!='' +WHERE TraceId != '' GROUP BY TraceId; ` ) @@ -282,10 +283,10 @@ func createTracesTable(ctx context.Context, cfg *Config, db *sql.DB) error { return fmt.Errorf("exec create traces table sql: %w", err) } if _, err := db.ExecContext(ctx, renderCreateTraceIDTsTableSQL(cfg)); err != nil { - return fmt.Errorf("exec create traceIDTs table sql: %w", err) + return fmt.Errorf("exec create traceID timestamp table sql: %w", err) } if _, err := db.ExecContext(ctx, renderTraceIDTsMaterializedViewSQL(cfg)); err != nil { - return fmt.Errorf("exec create traceIDTs view sql: %w", err) + return fmt.Errorf("exec create traceID timestamp view sql: %w", err) } return nil } @@ -295,12 +296,12 @@ func renderInsertTracesSQL(cfg *Config) string { } func renderCreateTracesTableSQL(cfg *Config) string { - ttlExpr := generateTTLExpr(cfg.TTL, "toDateTime(Timestamp)") + ttlExpr := generateTTLExpr(cfg.TTL, "toDate(Timestamp)") return fmt.Sprintf(createTracesTableSQL, cfg.TracesTableName, cfg.clusterString(), cfg.tableEngineString(), ttlExpr) } func renderCreateTraceIDTsTableSQL(cfg *Config) string { - ttlExpr := generateTTLExpr(cfg.TTL, "toDateTime(Start)") + ttlExpr := generateTTLExpr(cfg.TTL, "toDate(Start)") return fmt.Sprintf(createTraceIDTsTableSQL, cfg.TracesTableName, cfg.clusterString(), cfg.tableEngineString(), ttlExpr) } diff --git a/exporter/clickhouseexporter/integration_test.go b/exporter/clickhouseexporter/integration_test.go index cbcba99f732e..74ecd52dde6b 100644 --- a/exporter/clickhouseexporter/integration_test.go +++ b/exporter/clickhouseexporter/integration_test.go @@ -95,7 +95,6 @@ func verifyExportLog(t *testing.T, logExporter *logsExporter) { type log struct { Timestamp string `db:"Timestamp"` - TimestampDate string `db:"TimestampDate"` TimestampTime string `db:"TimestampTime"` TraceID string `db:"TraceId"` SpanID string `db:"SpanId"` @@ -117,7 +116,6 @@ func verifyExportLog(t *testing.T, logExporter *logsExporter) { expectLog := log{ Timestamp: "2023-12-25T09:53:49Z", - TimestampDate: "2023-12-25T00:00:00Z", TimestampTime: "2023-12-25T09:53:49Z", TraceID: "01020300000000000000000000000000", SpanID: "0102030000000000",