Skip to content

Commit

Permalink
add panel to reproduce #672
Browse files Browse the repository at this point in the history
  • Loading branch information
@Slach
Slach committed Nov 25, 2024
1 parent bf4072e commit 99d42aa
Show file tree
Hide file tree
Showing 2 changed files with 128 additions and 13 deletions.
31 changes: 31 additions & 0 deletions docker/clickhouse/init_schema.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,37 @@ INSERT INTO default.test_logs(event_time, content, level, id, label, detected_fi
INSERT INTO default.test_logs(event_time, content, level, id, label, detected_field) SELECT toDateTime(now()+(number*10)) AS event_time, concat('Info Log line ', toString(number)) as content, 'Info' AS level, generateUUIDv4() as id, if(rand() % 2 = 1,'abc','cba') AS label, 1000000000.05 AS detected_field FROM numbers(1000);
INSERT INTO default.test_logs(event_time, content, level, id, label, detected_field) SELECT toDateTime(now()+((500+number)*10)) AS event_time, concat('Unknown Log line ', toString(number)) as content, 'Unknown' AS level, generateUUIDv4() as id, if(rand() % 2 = 1,'abc','cba') AS label, 1000000000.05 AS detected_field FROM numbers(1000);

DROP TABLE IF EXISTS default.test_logs_with_complex_labels;
CREATE TABLE default.test_logs_with_complex_labels(
`_raw` String CODEC(ZSTD(1)),
`_time` DateTime64(3, 'Asia/Yekaterinburg') CODEC(ZSTD(1)),
`_map` Map(String, String),
`_db_time` DateTime DEFAULT now() CODEC(ZSTD(1)),
`_time_dec` Float64 DEFAULT toFloat64(_time) CODEC(DoubleDelta, Default),
`cluster_name` LowCardinality(String) DEFAULT JSONExtractString(_raw, 'cluster_name') CODEC(ZSTD(1)),
`host` LowCardinality(String) DEFAULT JSONExtractString(_raw, 'host') CODEC(ZSTD(1)),
`pod_namespace` LowCardinality(String) DEFAULT JSONExtractString(_raw, 'pod_namespace') CODEC(ZSTD(1)),
`pod_name` String DEFAULT JSONExtractString(_raw, 'pod_name') CODEC(ZSTD(1)),
`container_name` String DEFAULT JSONExtractString(_raw, 'container_name') CODEC(ZSTD(1)),
`container_image` String DEFAULT JSONExtractString(_raw, 'container_image') CODEC(ZSTD(1)),
`stream` LowCardinality(String) DEFAULT JSONExtractString(_raw, 'stream') CODEC(ZSTD(1)),
`source` LowCardinality(String) DEFAULT JSONExtractString(_raw, 'source') CODEC(ZSTD(1)),
`sourcetype` String DEFAULT JSONExtractString(_raw, 'source_type') CODEC(ZSTD(1)),
`message` String DEFAULT JSONExtractString(_raw, 'message') CODEC(ZSTD(1)),
`bu` LowCardinality(String) DEFAULT JSON_VALUE(_raw, '$.namespace_labels."business-unit-code"'),
INDEX message_ngram_bf message TYPE ngrambf_v1(4, 1024, 2, 0) GRANULARITY 1,
INDEX pod_name_token_bf pod_name TYPE tokenbf_v1(2048, 4, 0) GRANULARITY 4
)
ENGINE = MergeTree
PARTITION BY toDate(_time)
ORDER BY (cluster_name, bu, pod_namespace, pod_name, container_name, _time);

INSERT INTO default.test_logs_with_complex_labels(_raw, _time, _map)
SELECT '{"cluster_name":"test' || toString(number) || '","host":"test","pod_namespace":"test","pod_name":"test","container_name":"test"' || toString(number) || ',"container_image":"test","stream":"test","source":"test","source_type":"test","namespace_labels":{"business-unit-code":"test"}}' AS _raw,
now64() - INTERVAL number SECOND _time,
map('map_key' || toString(number),'map_value' ||toString(number)) AS _map
FROM numbers(100);

DROP TABLE IF EXISTS default.test_alerts;
CREATE TABLE IF NOT EXISTS default.test_alerts
(
Expand Down
110 changes: 97 additions & 13 deletions docker/grafana/dashboards/test_logs_support.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,21 +21,90 @@
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 6,
"id": 39,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "vertamedia-clickhouse-datasource",
"uid": "P7E099F39B84EA795"
},
"description": "reproduce https://github.com/Altinity/clickhouse-grafana/issues/672",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 22,
"x": 0,
"y": 0
},
"id": 4,
"options": {
"dedupStrategy": "none",
"enableLogDetails": true,
"prettifyLogMessage": true,
"showCommonLabels": true,
"showLabels": true,
"showTime": true,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"adHocFilters": [
{
"key": "default.test_logs.level",
"operator": "=",
"value": "Info"
}
],
"adHocValuesQuery": "",
"add_metadata": true,
"contextWindowSize": "100",
"database": "default",
"datasource": {
"type": "vertamedia-clickhouse-datasource",
"uid": "P7E099F39B84EA795"
},
"dateTimeColDataType": "_time",
"dateTimeType": "DATETIME64",
"editorMode": "sql",
"extrapolate": true,
"format": "logs",
"formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t",
"initialized": true,
"interval": "",
"intervalFactor": 1,
"query": "SELECT * FROM $table WHERE $timeFilter",
"rawQuery": " /* grafana dashboard=Test Logs support, user=0 */\n\nSELECT *\n\nFROM default.test_logs_with_complex_labels\n\nWHERE \"_time\" >= toDateTime64(1732352032, 3) AND \"_time\" <= toDateTime64(1732524832, 3)\n",
"refId": "A",
"round": "0s",
"showFormattedSQL": true,
"skip_comments": true,
"table": "test_logs_with_complex_labels"
}
],
"title": "Logs WIth Map as Label",
"type": "logs"
},
{
"datasource": {
"type": "vertamedia-clickhouse-datasource",
"uid": "P7E099F39B84EA795"
},
"description": "reproduce https://github.com/Altinity/clickhouse-grafana/issues/125 and https://github.com/Altinity/clickhouse-grafana/issues/331",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 11,
"x": 0,
"y": 0
"y": 8
},
"id": 2,
"options": {
Expand All @@ -48,8 +117,19 @@
"sortOrder": "Descending",
"wrapLogMessage": false
},
"pluginVersion": "11.3.1",
"targets": [
{
"adHocFilters": [
{
"key": "default.test_logs.level",
"operator": "=",
"value": "Info"
}
],
"adHocValuesQuery": "",
"add_metadata": true,
"contextWindowSize": "20",
"database": "default",
"datasource": {
"type": "vertamedia-clickhouse-datasource",
Expand All @@ -60,15 +140,18 @@
"dateTimeColDataType": "event_time",
"dateTimeType": "DATETIME",
"datetimeLoading": false,
"editorMode": "sql",
"extrapolate": true,
"format": "logs",
"formattedQuery": "SELECT $timeSeries as t, count() FROM $table WHERE $timeFilter GROUP BY t ORDER BY t",
"initialized": true,
"interval": "",
"intervalFactor": 1,
"query": "SELECT *\nFROM $table\n\nWHERE $timeFilter AND $adhoc\n$conditionalTest(AND content ILIKE ${filter:sqlstring},$filter)",
"rawQuery": "SELECT *\nFROM default.test_logs\n\nWHERE event_time >= toDateTime(1653997717) AND event_time <= toDateTime(1654019317) AND 1\n AND content ILIKE '%Info%line 11%' ",
"rawQuery": " /* grafana dashboard=Test Logs support, user=0 */\n\nSELECT *\n\nFROM default.test_logs\n\nWHERE\n event_time >= toDateTime(1732347051) AND event_time <= toDateTime(1732519851)\n AND (level = 'Info')",
"refId": "A",
"round": "0s",
"showFormattedSQL": true,
"skip_comments": true,
"table": "test_logs",
"tableLoading": false
Expand All @@ -83,11 +166,15 @@
"uid": "P4F4839B759FB0509"
},
"description": "",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 11,
"x": 11,
"y": 0
"y": 8
},
"id": 3,
"options": {
Expand All @@ -100,6 +187,7 @@
"sortOrder": "Ascending",
"wrapLogMessage": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"builderOptions": {
Expand Down Expand Up @@ -192,11 +280,13 @@
"type": "logs"
}
],
"schemaVersion": 39,
"preload": false,
"schemaVersion": 40,
"tags": [],
"templating": {
"list": [
{
"baseFilters": [],
"datasource": {
"type": "vertamedia-clickhouse-datasource",
"uid": "P7E099F39B84EA795"
Expand All @@ -208,18 +298,14 @@
"value": "Info"
}
],
"hide": 0,
"name": "adhoc_variable",
"skipUrlSync": false,
"type": "adhoc"
},
{
"current": {
"selected": false,
"text": "",
"value": ""
},
"hide": 0,
"name": "filter",
"options": [
{
Expand All @@ -229,7 +315,6 @@
}
],
"query": "",
"skipUrlSync": false,
"type": "textbox"
}
]
Expand All @@ -238,9 +323,8 @@
"from": "now-2d",
"to": "now"
},
"timeRangeUpdatedDuringEditOrView": false,
"timepicker": {},
"timezone": "",
"timezone": "utc",
"title": "Test Logs support",
"uid": "VtsMXQl7z",
"version": 1,
Expand Down

0 comments on commit 99d42aa

Please sign in to comment.