[exporter/clickhouse] Upgrade trace table

.chloggen/clickhouseexporter_update_trace_table.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: clickhouseexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Updated the default trace table
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [34245]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: Reduced data types, improved partitioning and time range queries
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

exporter/clickhouseexporter/README.md

@@ -128,7 +128,7 @@ Limit 100;
 - Find spans with specific attribute.
 
 ```sql
-SELECT Timestamp as log_time,
+SELECT Timestamp,
        TraceId,
        SpanId,
        ParentSpanId,
@@ -156,7 +156,7 @@ WITH
     '391dae938234560b16bb63f51501cb6f' as trace_id,
     (SELECT min(Start) FROM otel_traces_trace_id_ts WHERE TraceId = trace_id) as start,
     (SELECT max(End) + 1 FROM otel_traces_trace_id_ts WHERE TraceId = trace_id) as end
-SELECT Timestamp as log_time,
+SELECT Timestamp,
        TraceId,
        SpanId,
        ParentSpanId,
@@ -180,7 +180,7 @@ Limit 100;
 - Find spans is error.
 
 ```sql
-SELECT Timestamp as log_time,
+SELECT Timestamp,
        TraceId,
        SpanId,
        ParentSpanId,
@@ -196,15 +196,15 @@ SELECT Timestamp as log_time,
        toString(Links.TraceId)
 FROM otel_traces
 WHERE ServiceName = 'clickhouse-exporter'
-  AND StatusCode = 'STATUS_CODE_ERROR'
+  AND StatusCode = 'Error'
   AND Timestamp >= NOW() - INTERVAL 1 HOUR
 Limit 100;
 ```
 
 - Find slow spans.
 
 ```sql
-SELECT Timestamp as log_time,
+SELECT Timestamp,
        TraceId,
        SpanId,
        ParentSpanId,

exporter/clickhouseexporter/example/default_ddl/traces.sql

@@ -1,40 +1,63 @@
 -- Default Trace table DDL
 
 CREATE TABLE IF NOT EXISTS otel_traces (
-     Timestamp DateTime64(9) CODEC(Delta, ZSTD(1)),
-     TraceId String CODEC(ZSTD(1)),
-     SpanId String CODEC(ZSTD(1)),
-     ParentSpanId String CODEC(ZSTD(1)),
-     TraceState String CODEC(ZSTD(1)),
-     SpanName LowCardinality(String) CODEC(ZSTD(1)),
-     SpanKind LowCardinality(String) CODEC(ZSTD(1)),
-     ServiceName LowCardinality(String) CODEC(ZSTD(1)),
-     ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
-     ScopeName String CODEC(ZSTD(1)),
-     ScopeVersion String CODEC(ZSTD(1)),
-     SpanAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
-     Duration Int64 CODEC(ZSTD(1)),
-     StatusCode LowCardinality(String) CODEC(ZSTD(1)),
-     StatusMessage String CODEC(ZSTD(1)),
-     Events Nested (
-         Timestamp DateTime64(9),
-         Name LowCardinality(String),
-         Attributes Map(LowCardinality(String), String)
-     ) CODEC(ZSTD(1)),
-     Links Nested (
-         TraceId String,
-         SpanId String,
-         TraceState String,
-         Attributes Map(LowCardinality(String), String)
-     ) CODEC(ZSTD(1)),
-     INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
-     INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-     INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-     INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-     INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-     INDEX idx_duration Duration TYPE minmax GRANULARITY 1
+    Timestamp DateTime64(9) CODEC(Delta, ZSTD(1)),
+    TraceId String CODEC(ZSTD(1)),
+    SpanId String CODEC(ZSTD(1)),
+    ParentSpanId String CODEC(ZSTD(1)),
+    TraceState String CODEC(ZSTD(1)),
+    SpanName LowCardinality(String) CODEC(ZSTD(1)),
+    SpanKind LowCardinality(String) CODEC(ZSTD(1)),
+    ServiceName LowCardinality(String) CODEC(ZSTD(1)),
+    ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    ScopeName String CODEC(ZSTD(1)),
+    ScopeVersion String CODEC(ZSTD(1)),
+    SpanAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+    Duration UInt64 CODEC(ZSTD(1)),
+    StatusCode LowCardinality(String) CODEC(ZSTD(1)),
+    StatusMessage String CODEC(ZSTD(1)),
+    Events Nested (
+        Timestamp DateTime64(9),
+        Name LowCardinality(String),
+        Attributes Map(LowCardinality(String), String)
+    ) CODEC(ZSTD(1)),
+    Links Nested (
+        TraceId String,
+        SpanId String,
+        TraceState String,
+        Attributes Map(LowCardinality(String), String)
+    ) CODEC(ZSTD(1)),
+    INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
+    INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+    INDEX idx_duration Duration TYPE minmax GRANULARITY 1
 ) ENGINE = MergeTree()
-TTL toDateTime("Timestamp") + toIntervalDay(180)
 PARTITION BY toDate(Timestamp)
-ORDER BY (ServiceName, SpanName, toUnixTimestamp(Timestamp), TraceId)
+ORDER BY (ServiceName, SpanName, toDateTime(Timestamp))
+TTL toDate(Timestamp) + toIntervalDay(180)
+SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;
+
+
+CREATE TABLE IF NOT EXISTS otel_traces_trace_id_ts (
+     TraceId String CODEC(ZSTD(1)),
+     Start DateTime CODEC(Delta, ZSTD(1)),
+     End DateTime CODEC(Delta, ZSTD(1)),
+     INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1
+) ENGINE = MergeTree()
+PARTITION BY toDate(Start)
+ORDER BY (TraceId, Start)
+TTL toDate(Start) + toIntervalDay(180)
 SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;
+
+
+CREATE MATERIALIZED VIEW IF NOT EXISTS otel_traces_trace_id_ts_mv
+TO otel_traces_trace_id_ts
+AS SELECT
+    TraceId,
+    min(Timestamp) as Start,
+    max(Timestamp) as End
+FROM otel_traces
+WHERE TraceId != ''
+GROUP BY TraceId;

exporter/clickhouseexporter/exporter_traces.go

@@ -163,42 +163,42 @@ const (
 	// language=ClickHouse SQL
 	createTracesTableSQL = `
 CREATE TABLE IF NOT EXISTS %s %s (
-     Timestamp DateTime64(9) CODEC(Delta, ZSTD(1)),
-     TraceId String CODEC(ZSTD(1)),
-     SpanId String CODEC(ZSTD(1)),
-     ParentSpanId String CODEC(ZSTD(1)),
-     TraceState String CODEC(ZSTD(1)),
-     SpanName LowCardinality(String) CODEC(ZSTD(1)),
-     SpanKind LowCardinality(String) CODEC(ZSTD(1)),
-     ServiceName LowCardinality(String) CODEC(ZSTD(1)),
-     ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
-     ScopeName String CODEC(ZSTD(1)),
-     ScopeVersion String CODEC(ZSTD(1)),
-     SpanAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
-     Duration Int64 CODEC(ZSTD(1)),
-     StatusCode LowCardinality(String) CODEC(ZSTD(1)),
-     StatusMessage String CODEC(ZSTD(1)),
-     Events Nested (
-         Timestamp DateTime64(9),
-         Name LowCardinality(String),
-         Attributes Map(LowCardinality(String), String)
-     ) CODEC(ZSTD(1)),
-     Links Nested (
-         TraceId String,
-         SpanId String,
-         TraceState String,
-         Attributes Map(LowCardinality(String), String)
-     ) CODEC(ZSTD(1)),
-     INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
-     INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-     INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-     INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-     INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
-     INDEX idx_duration Duration TYPE minmax GRANULARITY 1
+	Timestamp DateTime64(9) CODEC(Delta, ZSTD(1)),
+	TraceId String CODEC(ZSTD(1)),
+	SpanId String CODEC(ZSTD(1)),
+	ParentSpanId String CODEC(ZSTD(1)),
+	TraceState String CODEC(ZSTD(1)),
+	SpanName LowCardinality(String) CODEC(ZSTD(1)),
+	SpanKind LowCardinality(String) CODEC(ZSTD(1)),
+	ServiceName LowCardinality(String) CODEC(ZSTD(1)),
+	ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+	ScopeName String CODEC(ZSTD(1)),
+	ScopeVersion String CODEC(ZSTD(1)),
+	SpanAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
+	Duration UInt64 CODEC(ZSTD(1)),
+	StatusCode LowCardinality(String) CODEC(ZSTD(1)),
+	StatusMessage String CODEC(ZSTD(1)),
+	Events Nested (
+		Timestamp DateTime64(9),
+		Name LowCardinality(String),
+		Attributes Map(LowCardinality(String), String)
+	) CODEC(ZSTD(1)),
+	Links Nested (
+		TraceId String,
+		SpanId String,
+		TraceState String,
+		Attributes Map(LowCardinality(String), String)
+	) CODEC(ZSTD(1)),
+	INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
+	INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+	INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+	INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+	INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
+	INDEX idx_duration Duration TYPE minmax GRANULARITY 1
 ) ENGINE = %s
-%s
 PARTITION BY toDate(Timestamp)
-ORDER BY (ServiceName, SpanName, toUnixTimestamp(Timestamp), TraceId)
+ORDER BY (ServiceName, SpanName, toDateTime(Timestamp))
+%s
 SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;
 `
 	// language=ClickHouse SQL
@@ -253,26 +253,27 @@ SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;
 
 const (
 	createTraceIDTsTableSQL = `
-create table IF NOT EXISTS %s_trace_id_ts %s (
+CREATE TABLE IF NOT EXISTS %s_trace_id_ts %s (
      TraceId String CODEC(ZSTD(1)),
-     Start DateTime64(9) CODEC(Delta, ZSTD(1)),
-     End DateTime64(9) CODEC(Delta, ZSTD(1)),
+     Start DateTime CODEC(Delta, ZSTD(1)),
+     End DateTime CODEC(Delta, ZSTD(1)),
      INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1
 ) ENGINE = %s
+PARTITION BY toDate(Start)
+ORDER BY (TraceId, Start)
 %s
-ORDER BY (TraceId, toUnixTimestamp(Start))
-SETTINGS index_granularity=8192;
+SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;
 `
 	createTraceIDTsMaterializedViewSQL = `
 CREATE MATERIALIZED VIEW IF NOT EXISTS %s_trace_id_ts_mv %s
 TO %s.%s_trace_id_ts
 AS SELECT
-TraceId,
-min(Timestamp) as Start,
-max(Timestamp) as End
+	TraceId,
+	min(Timestamp) as Start,
+	max(Timestamp) as End
 FROM
 %s.%s
-WHERE TraceId!=''
+WHERE TraceId != ''
 GROUP BY TraceId;
 `
 )
@@ -282,10 +283,10 @@ func createTracesTable(ctx context.Context, cfg *Config, db *sql.DB) error {
 		return fmt.Errorf("exec create traces table sql: %w", err)
 	}
 	if _, err := db.ExecContext(ctx, renderCreateTraceIDTsTableSQL(cfg)); err != nil {
-		return fmt.Errorf("exec create traceIDTs table sql: %w", err)
+		return fmt.Errorf("exec create traceID timestamp table sql: %w", err)
 	}
 	if _, err := db.ExecContext(ctx, renderTraceIDTsMaterializedViewSQL(cfg)); err != nil {
-		return fmt.Errorf("exec create traceIDTs view sql: %w", err)
+		return fmt.Errorf("exec create traceID timestamp view sql: %w", err)
 	}
 	return nil
 }
@@ -295,12 +296,12 @@ func renderInsertTracesSQL(cfg *Config) string {
 }
 
 func renderCreateTracesTableSQL(cfg *Config) string {
-	ttlExpr := generateTTLExpr(cfg.TTL, "toDateTime(Timestamp)")
+	ttlExpr := generateTTLExpr(cfg.TTL, "toDate(Timestamp)")
 	return fmt.Sprintf(createTracesTableSQL, cfg.TracesTableName, cfg.clusterString(), cfg.tableEngineString(), ttlExpr)
 }
 
 func renderCreateTraceIDTsTableSQL(cfg *Config) string {
-	ttlExpr := generateTTLExpr(cfg.TTL, "toDateTime(Start)")
+	ttlExpr := generateTTLExpr(cfg.TTL, "toDate(Start)")
 	return fmt.Sprintf(createTraceIDTsTableSQL, cfg.TracesTableName, cfg.clusterString(), cfg.tableEngineString(), ttlExpr)
 }
 

exporter/clickhouseexporter/integration_test.go

@@ -95,7 +95,6 @@ func verifyExportLog(t *testing.T, logExporter *logsExporter) {
 
 	type log struct {
 		Timestamp          string            `db:"Timestamp"`
-		TimestampDate      string            `db:"TimestampDate"`
 		TimestampTime      string            `db:"TimestampTime"`
 		TraceID            string            `db:"TraceId"`
 		SpanID             string            `db:"SpanId"`
@@ -117,7 +116,6 @@ func verifyExportLog(t *testing.T, logExporter *logsExporter) {
 
 	expectLog := log{
 		Timestamp:         "2023-12-25T09:53:49Z",
-		TimestampDate:     "2023-12-25T00:00:00Z",
 		TimestampTime:     "2023-12-25T09:53:49Z",
 		TraceID:           "01020300000000000000000000000000",
 		SpanID:            "0102030000000000",