update table + readme

open-telemetry · Sep 14, 2024 · f4bdbe4 · f4bdbe4
1 parent 6978697
commit f4bdbe4
Show file tree

Hide file tree

Showing 5 changed files with 23 additions and 26 deletions.
diff --git a/.chloggen/clickhouseexporter_update_trace_table.yaml b/.chloggen/clickhouseexporter_update_trace_table.yaml
@@ -15,7 +15,7 @@ issues: [34245]
 # (Optional) One or more lines of additional information to render under the primary note.
 # These lines will be padded with 2 spaces and then inserted directly into the document.
 # Use pipe (|) for multiline entries.
-subtext:
+subtext: Reduced data types, improved partitioning and time range queries
 
 # If your change doesn't affect end users or the exported elements of any package,
 # you should instead start your pull request title with [chore] or use the "Skip Changelog" label.

diff --git a/exporter/clickhouseexporter/README.md b/exporter/clickhouseexporter/README.md
@@ -128,7 +128,7 @@ Limit 100;
 - Find spans with specific attribute.
 
 ```sql
-SELECT Timestamp as log_time,
+SELECT Timestamp,
        TraceId,
        SpanId,
        ParentSpanId,
@@ -156,7 +156,7 @@ WITH
     '391dae938234560b16bb63f51501cb6f' as trace_id,
     (SELECT min(Start) FROM otel_traces_trace_id_ts WHERE TraceId = trace_id) as start,
     (SELECT max(End) + 1 FROM otel_traces_trace_id_ts WHERE TraceId = trace_id) as end
-SELECT Timestamp as log_time,
+SELECT Timestamp,
        TraceId,
        SpanId,
        ParentSpanId,
@@ -180,7 +180,7 @@ Limit 100;
 - Find spans is error.
 
 ```sql
-SELECT Timestamp as log_time,
+SELECT Timestamp,
        TraceId,
        SpanId,
        ParentSpanId,
@@ -204,7 +204,7 @@ Limit 100;
 - Find slow spans.
 
 ```sql
-SELECT Timestamp as log_time,
+SELECT Timestamp,
        TraceId,
        SpanId,
        ParentSpanId,

diff --git a/exporter/clickhouseexporter/example/default_ddl/traces.sql b/exporter/clickhouseexporter/example/default_ddl/traces.sql
@@ -22,9 +22,9 @@ CREATE TABLE IF NOT EXISTS otel_traces (
         Attributes Map(LowCardinality(String), String)
     ) CODEC(ZSTD(1)),
     Links Nested (
-        TraceId String CODEC(ZSTD(1)),
-        SpanId String CODEC(ZSTD(1)),
-        TraceState String CODEC(ZSTD(1)),
+        TraceId String,
+        SpanId String,
+        TraceState String,
         Attributes Map(LowCardinality(String), String)
     ) CODEC(ZSTD(1)),
     INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
@@ -35,20 +35,20 @@ CREATE TABLE IF NOT EXISTS otel_traces (
     INDEX idx_duration Duration TYPE minmax GRANULARITY 1
 ) ENGINE = MergeTree()
 PARTITION BY toDate(Timestamp)
-ORDER BY (ServiceName, SpanName, Timestamp)
-TTL toDateTime(Timestamp) + toIntervalDay(180)
+ORDER BY (ServiceName, SpanName, toDateTime(Timestamp))
+TTL toDate(Timestamp) + toIntervalDay(180)
 SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;
 
 
 CREATE TABLE IF NOT EXISTS otel_traces_trace_id_ts (
      TraceId String CODEC(ZSTD(1)),
-     Start DateTime64(9) CODEC(Delta, ZSTD(1)),
-     End DateTime64(9) CODEC(Delta, ZSTD(1)),
+     Start DateTime CODEC(Delta, ZSTD(1)),
+     End DateTime CODEC(Delta, ZSTD(1)),
      INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1
 ) ENGINE = MergeTree()
 PARTITION BY toDate(Start)
-ORDER BY (Start)
-TTL toDateTime(Start) + toIntervalDay(180)
+ORDER BY (TraceId, Start)
+TTL toDate(Start) + toIntervalDay(180)
 SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;
 
 

diff --git a/exporter/clickhouseexporter/exporter_traces.go b/exporter/clickhouseexporter/exporter_traces.go
@@ -180,13 +180,13 @@ CREATE TABLE IF NOT EXISTS %s %s (
 	StatusMessage String CODEC(ZSTD(1)),
 	Events Nested (
 		Timestamp DateTime64(9),
-		Name LowCardinality(String) CODEC(ZSTD(1)),
+		Name LowCardinality(String),
 		Attributes Map(LowCardinality(String), String)
 	) CODEC(ZSTD(1)),
 	Links Nested (
-		TraceId String CODEC(ZSTD(1)),
-		SpanId String CODEC(ZSTD(1)),
-		TraceState String CODEC(ZSTD(1)),
+		TraceId String,
+		SpanId String,
+		TraceState String,
 		Attributes Map(LowCardinality(String), String)
 	) CODEC(ZSTD(1)),
 	INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
@@ -197,8 +197,7 @@ CREATE TABLE IF NOT EXISTS %s %s (
 	INDEX idx_duration Duration TYPE minmax GRANULARITY 1
 ) ENGINE = %s
 PARTITION BY toDate(Timestamp)
-PRIMARY KEY (ServiceName, SpanName, Timestamp)
-ORDER BY (ServiceName, SpanName, Timestamp)
+ORDER BY (ServiceName, SpanName, toDateTime(Timestamp))
 %s
 SETTINGS index_granularity=8192, ttl_only_drop_parts = 1;
 `
@@ -256,8 +255,8 @@ const (
 	createTraceIDTsTableSQL = `
 CREATE TABLE IF NOT EXISTS %s_trace_id_ts %s (
      TraceId String CODEC(ZSTD(1)),
-     Start DateTime64(9) CODEC(Delta, ZSTD(1)),
-     End DateTime64(9) CODEC(Delta, ZSTD(1)),
+     Start DateTime CODEC(Delta, ZSTD(1)),
+     End DateTime CODEC(Delta, ZSTD(1)),
      INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1
 ) ENGINE = %s
 PARTITION BY toDate(Start)
@@ -297,12 +296,12 @@ func renderInsertTracesSQL(cfg *Config) string {
 }
 
 func renderCreateTracesTableSQL(cfg *Config) string {
-	ttlExpr := generateTTLExpr(cfg.TTL, "toDateTime(Timestamp)")
+	ttlExpr := generateTTLExpr(cfg.TTL, "toDate(Timestamp)")
 	return fmt.Sprintf(createTracesTableSQL, cfg.TracesTableName, cfg.clusterString(), cfg.tableEngineString(), ttlExpr)
 }
 
 func renderCreateTraceIDTsTableSQL(cfg *Config) string {
-	ttlExpr := generateTTLExpr(cfg.TTL, "toDateTime(Start)")
+	ttlExpr := generateTTLExpr(cfg.TTL, "toDate(Start)")
 	return fmt.Sprintf(createTraceIDTsTableSQL, cfg.TracesTableName, cfg.clusterString(), cfg.tableEngineString(), ttlExpr)
 }
 

diff --git a/exporter/clickhouseexporter/integration_test.go b/exporter/clickhouseexporter/integration_test.go
@@ -95,7 +95,6 @@ func verifyExportLog(t *testing.T, logExporter *logsExporter) {
 
 	type log struct {
 		Timestamp          string            `db:"Timestamp"`
-		TimestampDate      string            `db:"TimestampDate"`
 		TimestampTime      string            `db:"TimestampTime"`
 		TraceID            string            `db:"TraceId"`
 		SpanID             string            `db:"SpanId"`
@@ -117,7 +116,6 @@ func verifyExportLog(t *testing.T, logExporter *logsExporter) {
 
 	expectLog := log{
 		Timestamp:         "2023-12-25T09:53:49Z",
-		TimestampDate:     "2023-12-25T00:00:00Z",
 		TimestampTime:     "2023-12-25T09:53:49Z",
 		TraceID:           "01020300000000000000000000000000",
 		SpanID:            "0102030000000000",