diff --git a/docker/clickhouse/init_schema.sql b/docker/clickhouse/init_schema.sql index a73933c9f..803009665 100644 --- a/docker/clickhouse/init_schema.sql +++ b/docker/clickhouse/init_schema.sql @@ -33,6 +33,37 @@ INSERT INTO default.test_logs(event_time, content, level, id, label, detected_fi INSERT INTO default.test_logs(event_time, content, level, id, label, detected_field) SELECT toDateTime(now()+(number*10)) AS event_time, concat('Info Log line ', toString(number)) as content, 'Info' AS level, generateUUIDv4() as id, if(rand() % 2 = 1,'abc','cba') AS label, 1000000000.05 AS detected_field FROM numbers(1000); INSERT INTO default.test_logs(event_time, content, level, id, label, detected_field) SELECT toDateTime(now()+((500+number)*10)) AS event_time, concat('Unknown Log line ', toString(number)) as content, 'Unknown' AS level, generateUUIDv4() as id, if(rand() % 2 = 1,'abc','cba') AS label, 1000000000.05 AS detected_field FROM numbers(1000); +DROP TABLE IF EXISTS default.test_logs_with_complex_labels; +CREATE TABLE default.test_logs_with_complex_labels( + `_raw` String CODEC(ZSTD(1)), + `_time` DateTime64(3, 'Asia/Yekaterinburg') CODEC(ZSTD(1)), + `_map` Map(String, String), + `_db_time` DateTime DEFAULT now() CODEC(ZSTD(1)), + `_time_dec` Float64 DEFAULT toFloat64(_time) CODEC(DoubleDelta, Default), + `cluster_name` LowCardinality(String) DEFAULT JSONExtractString(_raw, 'cluster_name') CODEC(ZSTD(1)), + `host` LowCardinality(String) DEFAULT JSONExtractString(_raw, 'host') CODEC(ZSTD(1)), + `pod_namespace` LowCardinality(String) DEFAULT JSONExtractString(_raw, 'pod_namespace') CODEC(ZSTD(1)), + `pod_name` String DEFAULT JSONExtractString(_raw, 'pod_name') CODEC(ZSTD(1)), + `container_name` String DEFAULT JSONExtractString(_raw, 'container_name') CODEC(ZSTD(1)), + `container_image` String DEFAULT JSONExtractString(_raw, 'container_image') CODEC(ZSTD(1)), + `stream` LowCardinality(String) DEFAULT JSONExtractString(_raw, 'stream') CODEC(ZSTD(1)), + `source` LowCardinality(String) DEFAULT JSONExtractString(_raw, 'source') CODEC(ZSTD(1)), + `sourcetype` String DEFAULT JSONExtractString(_raw, 'source_type') CODEC(ZSTD(1)), + `message` String DEFAULT JSONExtractString(_raw, 'message') CODEC(ZSTD(1)), + `bu` LowCardinality(String) DEFAULT JSON_VALUE(_raw, '$.namespace_labels."business-unit-code"'), + INDEX message_ngram_bf message TYPE ngrambf_v1(4, 1024, 2, 0) GRANULARITY 1, + INDEX pod_name_token_bf pod_name TYPE tokenbf_v1(2048, 4, 0) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(_time) +ORDER BY (cluster_name, bu, pod_namespace, pod_name, container_name, _time); + +INSERT INTO default.test_logs_with_complex_labels(_raw, _time, _map) +SELECT '{"cluster_name":"test' || toString(number) || '","host":"test","pod_namespace":"test","pod_name":"test","container_name":"test"' || toString(number) || ',"container_image":"test","stream":"test","source":"test","source_type":"test","namespace_labels":{"business-unit-code":"test"}}' AS _raw, + now64() - INTERVAL number SECOND _time, + map('map_key' || toString(number),'map_value' ||toString(number)) AS _map +FROM numbers(100); + DROP TABLE IF EXISTS default.test_alerts; CREATE TABLE IF NOT EXISTS default.test_alerts ( diff --git a/docker/grafana/dashboards/test_logs_support.json b/docker/grafana/dashboards/test_logs_support.json index d2130e6aa..e9d60bd5f 100755 --- a/docker/grafana/dashboards/test_logs_support.json +++ b/docker/grafana/dashboards/test_logs_support.json @@ -21,21 +21,90 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 6, + "id": 39, "links": [], - "liveNow": false, "panels": [ + { + "datasource": { + "type": "vertamedia-clickhouse-datasource", + "uid": "P7E099F39B84EA795" + }, + "description": "reproduce https://github.com/Altinity/clickhouse-grafana/issues/672", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 22, + "x": 0, + "y": 0 + }, + "id": 4, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": true, + "showCommonLabels": true, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "adHocFilters": [ + { + "key": "default.test_logs.level", + "operator": "=", + "value": "Info" + } + ], + "adHocValuesQuery": "", + "add_metadata": true, + "contextWindowSize": "100", + "database": "default", + "datasource": { + "type": "vertamedia-clickhouse-datasource", + "uid": "P7E099F39B84EA795" + }, + "dateTimeColDataType": "_time", + "dateTimeType": "DATETIME64", + "editorMode": "sql", + "extrapolate": true, + "format": "logs", + "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", + "initialized": true, + "interval": "", + "intervalFactor": 1, + "query": "SELECT * FROM $table WHERE $timeFilter", + "rawQuery": " /* grafana dashboard=Test Logs support, user=0 */\n\nSELECT *\n\nFROM default.test_logs_with_complex_labels\n\nWHERE \"_time\" >= toDateTime64(1732352032, 3) AND \"_time\" <= toDateTime64(1732524832, 3)\n", + "refId": "A", + "round": "0s", + "showFormattedSQL": true, + "skip_comments": true, + "table": "test_logs_with_complex_labels" + } + ], + "title": "Logs WIth Map as Label", + "type": "logs" + }, { "datasource": { "type": "vertamedia-clickhouse-datasource", "uid": "P7E099F39B84EA795" }, "description": "reproduce https://github.com/Altinity/clickhouse-grafana/issues/125 and https://github.com/Altinity/clickhouse-grafana/issues/331", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "gridPos": { "h": 9, "w": 11, "x": 0, - "y": 0 + "y": 8 }, "id": 2, "options": { @@ -48,8 +117,19 @@ "sortOrder": "Descending", "wrapLogMessage": false }, + "pluginVersion": "11.3.1", "targets": [ { + "adHocFilters": [ + { + "key": "default.test_logs.level", + "operator": "=", + "value": "Info" + } + ], + "adHocValuesQuery": "", + "add_metadata": true, + "contextWindowSize": "20", "database": "default", "datasource": { "type": "vertamedia-clickhouse-datasource", @@ -60,15 +140,18 @@ "dateTimeColDataType": "event_time", "dateTimeType": "DATETIME", "datetimeLoading": false, + "editorMode": "sql", "extrapolate": true, "format": "logs", "formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t", + "initialized": true, "interval": "", "intervalFactor": 1, "query": "SELECT *\nFROM $table\n\nWHERE $timeFilter AND $adhoc\n$conditionalTest(AND content ILIKE ${filter:sqlstring},$filter)", - "rawQuery": "SELECT *\nFROM default.test_logs\n\nWHERE event_time >= toDateTime(1653997717) AND event_time <= toDateTime(1654019317) AND 1\n AND content ILIKE '%Info%line 11%' ", + "rawQuery": " /* grafana dashboard=Test Logs support, user=0 */\n\nSELECT *\n\nFROM default.test_logs\n\nWHERE\n event_time >= toDateTime(1732347051) AND event_time <= toDateTime(1732519851)\n AND (level = 'Info')", "refId": "A", "round": "0s", + "showFormattedSQL": true, "skip_comments": true, "table": "test_logs", "tableLoading": false @@ -83,11 +166,15 @@ "uid": "P4F4839B759FB0509" }, "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "gridPos": { "h": 9, "w": 11, "x": 11, - "y": 0 + "y": 8 }, "id": 3, "options": { @@ -100,6 +187,7 @@ "sortOrder": "Ascending", "wrapLogMessage": true }, + "pluginVersion": "11.3.1", "targets": [ { "builderOptions": { @@ -192,11 +280,13 @@ "type": "logs" } ], - "schemaVersion": 39, + "preload": false, + "schemaVersion": 40, "tags": [], "templating": { "list": [ { + "baseFilters": [], "datasource": { "type": "vertamedia-clickhouse-datasource", "uid": "P7E099F39B84EA795" @@ -208,18 +298,14 @@ "value": "Info" } ], - "hide": 0, "name": "adhoc_variable", - "skipUrlSync": false, "type": "adhoc" }, { "current": { - "selected": false, "text": "", "value": "" }, - "hide": 0, "name": "filter", "options": [ { @@ -229,7 +315,6 @@ } ], "query": "", - "skipUrlSync": false, "type": "textbox" } ] @@ -238,9 +323,8 @@ "from": "now-2d", "to": "now" }, - "timeRangeUpdatedDuringEditOrView": false, "timepicker": {}, - "timezone": "", + "timezone": "utc", "title": "Test Logs support", "uid": "VtsMXQl7z", "version": 1,