From 20c9606f6ea9c2bfe92dc05fcce2a91715168ea4 Mon Sep 17 00:00:00 2001 From: localhost Date: Wed, 27 Nov 2024 19:20:19 +0800 Subject: [PATCH] docs: add doc for loki ingester protocol (#1329) --- .typo.toml | 2 +- .../clients/otlp-logs-integration.md | 2 +- .../clients/otlp-metrics-integration.md | 2 +- .../ingest-data/for-observerbility/loki.md | 123 ++++++++++++++++++ .../for-observerbility/opentelemetry.md | 4 +- .../for-observerbility/overview.md | 1 + .../ingest-data/for-observerbility/loki.md | 122 +++++++++++++++++ .../for-observerbility/overview.md | 1 + .../ingest-data/for-observerbility/loki.md | 122 +++++++++++++++++ .../for-observerbility/overview.md | 1 + sidebars.ts | 1 + .../ingest-data/for-observerbility/loki.md | 123 ++++++++++++++++++ .../for-observerbility/overview.md | 1 + 13 files changed, 500 insertions(+), 5 deletions(-) create mode 100644 docs/user-guide/ingest-data/for-observerbility/loki.md create mode 100644 i18n/zh/docusaurus-plugin-content-docs/current/user-guide/ingest-data/for-observerbility/loki.md create mode 100644 i18n/zh/docusaurus-plugin-content-docs/version-0.10/user-guide/ingest-data/for-observerbility/loki.md create mode 100644 versioned_docs/version-0.10/user-guide/ingest-data/for-observerbility/loki.md diff --git a/.typo.toml b/.typo.toml index 05aafcc49..d1e18542e 100644 --- a/.typo.toml +++ b/.typo.toml @@ -1,2 +1,2 @@ [files] -extend-exclude = ["*/**/df-functions.md"] +extend-exclude = ["*/**/df-functions.md", "pnpm-lock.yaml"] diff --git a/docs/db-cloud-shared/clients/otlp-logs-integration.md b/docs/db-cloud-shared/clients/otlp-logs-integration.md index c3b315032..4a6191164 100644 --- a/docs/db-cloud-shared/clients/otlp-logs-integration.md +++ b/docs/db-cloud-shared/clients/otlp-logs-integration.md @@ -4,7 +4,7 @@ GreptimeDB is an observability backend to consume OpenTelemetry Logs natively vi To send OpenTelemetry Logs to GreptimeDB through OpenTelemetry SDK libraries, use the following information: -* URL: `https:///v1/otlp/v1/logs` +* URL: `http{s}:///v1/otlp/v1/logs` * Headers: * `X-Greptime-DB-Name`: `` * `Authorization`: `Basic` authentication, which is a Base64 encoded string of `:`. For more information, please refer to [Authentication](https://docs.greptime.com/user-guide/deployments/authentication/static/) and [HTTP API](https://docs.greptime.com/user-guide/protocols/http#authentication). diff --git a/docs/db-cloud-shared/clients/otlp-metrics-integration.md b/docs/db-cloud-shared/clients/otlp-metrics-integration.md index 033884021..7a8a31ecf 100644 --- a/docs/db-cloud-shared/clients/otlp-metrics-integration.md +++ b/docs/db-cloud-shared/clients/otlp-metrics-integration.md @@ -4,7 +4,7 @@ GreptimeDB is an observability backend to consume OpenTelemetry Metrics natively To send OpenTelemetry Metrics to GreptimeDB through OpenTelemetry SDK libraries, use the following information: -* URL: `https:///v1/otlp/v1/metrics` +* URL: `http{s}:///v1/otlp/v1/metrics` * Headers: * `X-Greptime-DB-Name`: `` * `Authorization`: `Basic` authentication, which is a Base64 encoded string of `:`. For more information, please refer to [Authentication](https://docs.greptime.com/user-guide/deployments/authentication/static/) and [HTTP API](https://docs.greptime.com/user-guide/protocols/http#authentication) diff --git a/docs/user-guide/ingest-data/for-observerbility/loki.md b/docs/user-guide/ingest-data/for-observerbility/loki.md new file mode 100644 index 000000000..f12299eb0 --- /dev/null +++ b/docs/user-guide/ingest-data/for-observerbility/loki.md @@ -0,0 +1,123 @@ +# Loki + +## Usage + +### API + +To send Logs to GreptimeDB through Raw HTTP API, use the following information: + +* URL: `http{s}:///v1/loki/api/v1/push` +* Headers: + * `X-Greptime-DB-Name`: `` + * `Authorization`: `Basic` authentication, which is a Base64 encoded string of `:`. For more information, please refer to [Authentication](https://docs.greptime.com/user-guide/deployments/authentication/static/) and [HTTP API](https://docs.greptime.com/user-guide/protocols/http#authentication). + * `X-Greptime-Log-Table-Name`: `` (optional) - The table name to store the logs. If not provided, the default table name is `loki_logs`. + + The request uses binary protobuf to encode the payload, The defined schema is the same as the [logproto.proto](https://github.com/grafana/loki/blob/main/pkg/logproto/logproto.proto). + +### Example Code + +[Grafana Alloy](https://grafana.com/docs/alloy/latest/) is a vendor-neutral distribution of the OpenTelemetry (OTel) Collector. Alloy uniquely combines the very best OSS observability signals in the community. + +It suplies a Loki exporter that can be used to send logs to GreptimeDB. Here is an example configuration: + +```hcl +loki.source.file "greptime" { + targets = [ + {__path__ = "/tmp/foo.txt"}, + ] + forward_to = [loki.write.greptime_loki.receiver] +} + +loki.write "greptime_loki" { + endpoint { + url = "${GREPTIME_SCHEME:=http}://${GREPTIME_HOST:=greptimedb}:${GREPTIME_PORT:=4000}/v1/loki/api/v1/push" + headers = { + "x-greptime-db-name" = "${GREPTIME_DB:=public}", + "x-greptime-log-table-name" = "${GREPTIME_LOG_TABLE_NAME:=loki_demo_logs}", + } + } + external_labels = { + "job" = "greptime" + "from" = "alloy" + } +} +``` + +We listen to the file `/tmp/foo.txt` and send the logs to GreptimeDB. The logs are stored in the table `loki_demo_logs` with the external labels `job` and `from`. + +For more information, please refer to the [Grafana Alloy loki.write documentation](https://grafana.com/docs/alloy/latest/reference/components/loki/loki.write/). + +You can run the following command to check the data in the table: + +```sql +SELECT * FROM loki_demo_logs; ++----------------------------+------------------------+--------------+-------+----------+ +| greptime_timestamp | line | filename | from | job | ++----------------------------+------------------------+--------------+-------+----------+ +| 2024-11-25 11:02:31.256251 | Greptime is very cool! | /tmp/foo.txt | alloy | greptime | ++----------------------------+------------------------+--------------+-------+----------+ +1 row in set (0.01 sec) +``` + +## Data Model + +The Loki logs data model is mapped to the GreptimeDB data model according to the following rules: + +Default table schema without external labels: + +```sql +DESC loki_demo_logs; ++--------------------+---------------------+------+------+---------+---------------+ +| Column | Type | Key | Null | Default | Semantic Type | ++--------------------+---------------------+------+------+---------+---------------+ +| greptime_timestamp | TimestampNanosecond | PRI | NO | | TIMESTAMP | +| line | String | | YES | | FIELD | ++--------------------+---------------------+------+------+---------+---------------+ +5 rows in set (0.00 sec) +``` + +- greptime_timestamp: The timestamp of the log. +- line: The log message. + +if you specify the external labels, we will add them as tags to the table schema. like `job` and `from` in the above example. +You can't specify tags manually, all labels are treated as tags and string type. +Please do not attempt to pre-create the table using SQL to specify tag columns, as this will cause a type mismatch and write failure. + +### Example + +The following is an example of the table schema: + +```sql +DESC loki_demo_logs; ++--------------------+---------------------+------+------+---------+---------------+ +| Column | Type | Key | Null | Default | Semantic Type | ++--------------------+---------------------+------+------+---------+---------------+ +| greptime_timestamp | TimestampNanosecond | PRI | NO | | TIMESTAMP | +| line | String | | YES | | FIELD | +| filename | String | PRI | YES | | TAG | +| from | String | PRI | YES | | TAG | +| job | String | PRI | YES | | TAG | ++--------------------+---------------------+------+------+---------+---------------+ +5 rows in set (0.00 sec) +``` + +```sql +SHOW CREATE TABLE loki_demo_logs\G +*************************** 1. row *************************** + Table: loki_demo_logs +Create Table: CREATE TABLE IF NOT EXISTS `loki_demo_logs` ( + `greptime_timestamp` TIMESTAMP(9) NOT NULL, + `line` STRING NULL, + `filename` STRING NULL, + `from` STRING NULL, + `job` STRING NULL, + TIME INDEX (`greptime_timestamp`), + PRIMARY KEY (`filename`, `from`, `job`) +) + +ENGINE=mito +WITH( + append_mode = 'true' +) +1 row in set (0.00 sec) +``` \ No newline at end of file diff --git a/docs/user-guide/ingest-data/for-observerbility/opentelemetry.md b/docs/user-guide/ingest-data/for-observerbility/opentelemetry.md index a2f1055b3..39143fed3 100644 --- a/docs/user-guide/ingest-data/for-observerbility/opentelemetry.md +++ b/docs/user-guide/ingest-data/for-observerbility/opentelemetry.md @@ -137,8 +137,8 @@ otelcol.exporter.otlphttp "greptimedb_logs" { endpoint = "${GREPTIME_SCHEME:=http}://${GREPTIME_HOST:=greptimedb}:${GREPTIME_PORT:=4000}/v1/otlp/" headers = { "X-Greptime-DB-Name" = "${GREPTIME_DB:=public}", - "x-greptime-log-table-name" = "demo_logs", - "x-greptime-log-extract-keys" = "filename,log.file.name,loki.attribute.labels", + "X-Greptime-Log-Table-Name" = "demo_logs", + "X-Greptime-Gog-Extract-Keys" = "filename,log.file.name,loki.attribute.labels", } auth = otelcol.auth.basic.credentials.handler } diff --git a/docs/user-guide/ingest-data/for-observerbility/overview.md b/docs/user-guide/ingest-data/for-observerbility/overview.md index f20030a56..39a9d5bd2 100644 --- a/docs/user-guide/ingest-data/for-observerbility/overview.md +++ b/docs/user-guide/ingest-data/for-observerbility/overview.md @@ -8,3 +8,4 @@ GreptimeDB integrates seamlessly with leading observability tools to provide a c - [Vector](vector.md): Use GreptimeDB as a sink for Vector, ideal for complex data pipelines and diverse data sources. - [OpenTelemetry](opentelemetry.md): Collect and export telemetry data to GreptimeDB for detailed observability insights. - [InfluxDB Line Protocol](influxdb-line-protocol.md): A widely-used protocol for time-series data, facilitating migration from InfluxDB to GreptimeDB. The Telegraf integration method is also introduced in this document. +- [Loki](loki.md): A widely-used log write protocol, facilitating migration from Loki to GreptimeDB. The Alloy integration method is also introduced in this document. diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/user-guide/ingest-data/for-observerbility/loki.md b/i18n/zh/docusaurus-plugin-content-docs/current/user-guide/ingest-data/for-observerbility/loki.md new file mode 100644 index 000000000..b0a7b5b93 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/current/user-guide/ingest-data/for-observerbility/loki.md @@ -0,0 +1,122 @@ +# Loki + +## 使用方法 + +### API + +要通过原始 HTTP API 将日志发送到 GreptimeDB，请使用以下信息： + +* URL: `http{s}:///v1/loki/api/v1/push` +* Headers: + * `X-Greptime-DB-Name`: `` + * `Authorization`: `Basic` 认证，这是一个 Base64 编码的 `:` 字符串。更多信息，请参考 [认证](https://docs.greptime.com/user-guide/deployments/authentication/static/) 和 [HTTP API](https://docs.greptime.com/user-guide/protocols/http#authentication)。 + * `X-Greptime-Log-Table-Name`: ``（可选）- 存储日志的表名。如果未提供，默认表名为 `loki_logs`。 + +请求使用二进制 protobuf 编码负载，定义的格式与 [logproto.proto](https://github.com/grafana/loki/blob/main/pkg/logproto/logproto.proto) 相同。 + +### 示例代码 + +[Grafana Alloy](https://grafana.com/docs/alloy/latest/) 是一个供应商中立的 OpenTelemetry (OTel) Collector 发行版。Alloy 独特地结合了社区中最好的开源可观测性信号。 + +它提供了一个 Loki 导出器，可以用来将日志发送到 GreptimeDB。以下是一个配置示例： + +```hcl +loki.source.file "greptime" { + targets = [ + {__path__ = "/tmp/foo.txt"}, + ] + forward_to = [loki.write.greptime_loki.receiver] +} + +loki.write "greptime_loki" { + endpoint { + url = "${GREPTIME_SCHEME:=http}://${GREPTIME_HOST:=greptimedb}:${GREPTIME_PORT:=4000}/v1/loki/api/v1/push" + headers = { + "x-greptime-db-name" = "${GREPTIME_DB:=public}", + "x-greptime-log-table-name" = "${GREPTIME_LOG_TABLE_NAME:=loki_demo_logs}", + } + } + external_labels = { + "job" = "greptime" + "from" = "alloy" + } +} +``` + +我们监听文件 `/tmp/foo.txt` 并将日志发送到 GreptimeDB。日志存储在表 `loki_demo_logs` 中，并带有 label `job` 和 `from`。 + +更多信息，请参考 [Grafana Alloy loki.write 文档](https://grafana.com/docs/alloy/latest/reference/components/loki/loki.write/)。 + +你可以运行以下命令来检查表中的数据： + +```sql +SELECT * FROM loki_demo_logs; ++----------------------------+------------------------+--------------+-------+----------+ +| greptime_timestamp | line | filename | from | job | ++----------------------------+------------------------+--------------+-------+----------+ +| 2024-11-25 11:02:31.256251 | Greptime is very cool! | /tmp/foo.txt | alloy | greptime | ++----------------------------+------------------------+--------------+-------+----------+ +1 row in set (0.01 sec) +``` + +## 数据模型 + +Loki 日志数据模型根据以下规则映射到 GreptimeDB 数据模型： + +没有 label 的默认表结构： + +```sql +DESC loki_demo_logs; ++--------------------+---------------------+------+------+---------+---------------+ +| Column | Type | Key | Null | Default | Semantic Type | ++--------------------+---------------------+------+------+---------+---------------+ +| greptime_timestamp | TimestampNanosecond | PRI | NO | | TIMESTAMP | +| line | String | | YES | | FIELD | ++--------------------+---------------------+------+------+---------+---------------+ +5 rows in set (0.00 sec) +``` + +- greptime_timestamp: 日志的时间戳。 +- line: 日志消息。 + +如果你指定了外部 label，我们会将它们添加为表结构中的 tag。例如上面的 `job` 和 `from`。 +在这种写入方式下不能手动指定，所有 label 都被视为 tag 并且类型为字符串。请不要尝试使用 SQL 提前创建表来指定 tag 列，这会导致类型不匹配而写入失败。 + +### 示例 + +以下是表结构的示例： + +```sql +DESC loki_demo_logs; ++--------------------+---------------------+------+------+---------+---------------+ +| Column | Type | Key | Null | Default | Semantic Type | ++--------------------+---------------------+------+------+---------+---------------+ +| greptime_timestamp | TimestampNanosecond | PRI | NO | | TIMESTAMP | +| line | String | | YES | | FIELD | +| filename | String | PRI | YES | | TAG | +| from | String | PRI | YES | | TAG | +| job | String | PRI | YES | | TAG | ++--------------------+---------------------+------+------+---------+---------------+ +5 rows in set (0.00 sec) +``` + +```sql +SHOW CREATE TABLE loki_demo_logs\G +*************************** 1. row *************************** + Table: loki_demo_logs +Create Table: CREATE TABLE IF NOT EXISTS `loki_demo_logs` ( + `greptime_timestamp` TIMESTAMP(9) NOT NULL, + `line` STRING NULL, + `filename` STRING NULL, + `from` STRING NULL, + `job` STRING NULL, + TIME INDEX (`greptime_timestamp`), + PRIMARY KEY (`filename`, `from`, `job`) +) + +ENGINE=mito +WITH( + append_mode = 'true' +) +1 row in set (0.00 sec) +``` \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/user-guide/ingest-data/for-observerbility/overview.md b/i18n/zh/docusaurus-plugin-content-docs/current/user-guide/ingest-data/for-observerbility/overview.md index 12bf6bbba..26a26fe0c 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/user-guide/ingest-data/for-observerbility/overview.md +++ b/i18n/zh/docusaurus-plugin-content-docs/current/user-guide/ingest-data/for-observerbility/overview.md @@ -7,4 +7,5 @@ GreptimeDB 与领先的可观测性工具无缝集成，为你提供系统健康 - [Vector](vector.md)：将 GreptimeDB 用作 Vector 的接收端，适用于复杂的数据流水线和多样化的数据源。 - [OpenTelemetry](opentelemetry.md)：将 telemetry 数据收集并导出到 GreptimeDB，以获取详细的可观测性洞察。 - [InfluxDB Line Protocol](influxdb-line-protocol.md)：一种广泛使用的时间序列数据协议，便于从 InfluxDB 迁移到 GreptimeDB。该文档同样介绍了 Telegraf 的集成方式。 +- [Loki](loki.md)：一种广泛使用的日志写入协议，便于从 Loki 迁移到 GreptimeDB。本文档还介绍了 Alloy 集成方法。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-0.10/user-guide/ingest-data/for-observerbility/loki.md b/i18n/zh/docusaurus-plugin-content-docs/version-0.10/user-guide/ingest-data/for-observerbility/loki.md new file mode 100644 index 000000000..b0a7b5b93 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/version-0.10/user-guide/ingest-data/for-observerbility/loki.md @@ -0,0 +1,122 @@ +# Loki + +## 使用方法 + +### API + +要通过原始 HTTP API 将日志发送到 GreptimeDB，请使用以下信息： + +* URL: `http{s}:///v1/loki/api/v1/push` +* Headers: + * `X-Greptime-DB-Name`: `` + * `Authorization`: `Basic` 认证，这是一个 Base64 编码的 `:` 字符串。更多信息，请参考 [认证](https://docs.greptime.com/user-guide/deployments/authentication/static/) 和 [HTTP API](https://docs.greptime.com/user-guide/protocols/http#authentication)。 + * `X-Greptime-Log-Table-Name`: ``（可选）- 存储日志的表名。如果未提供，默认表名为 `loki_logs`。 + +请求使用二进制 protobuf 编码负载，定义的格式与 [logproto.proto](https://github.com/grafana/loki/blob/main/pkg/logproto/logproto.proto) 相同。 + +### 示例代码 + +[Grafana Alloy](https://grafana.com/docs/alloy/latest/) 是一个供应商中立的 OpenTelemetry (OTel) Collector 发行版。Alloy 独特地结合了社区中最好的开源可观测性信号。 + +它提供了一个 Loki 导出器，可以用来将日志发送到 GreptimeDB。以下是一个配置示例： + +```hcl +loki.source.file "greptime" { + targets = [ + {__path__ = "/tmp/foo.txt"}, + ] + forward_to = [loki.write.greptime_loki.receiver] +} + +loki.write "greptime_loki" { + endpoint { + url = "${GREPTIME_SCHEME:=http}://${GREPTIME_HOST:=greptimedb}:${GREPTIME_PORT:=4000}/v1/loki/api/v1/push" + headers = { + "x-greptime-db-name" = "${GREPTIME_DB:=public}", + "x-greptime-log-table-name" = "${GREPTIME_LOG_TABLE_NAME:=loki_demo_logs}", + } + } + external_labels = { + "job" = "greptime" + "from" = "alloy" + } +} +``` + +我们监听文件 `/tmp/foo.txt` 并将日志发送到 GreptimeDB。日志存储在表 `loki_demo_logs` 中，并带有 label `job` 和 `from`。 + +更多信息，请参考 [Grafana Alloy loki.write 文档](https://grafana.com/docs/alloy/latest/reference/components/loki/loki.write/)。 + +你可以运行以下命令来检查表中的数据： + +```sql +SELECT * FROM loki_demo_logs; ++----------------------------+------------------------+--------------+-------+----------+ +| greptime_timestamp | line | filename | from | job | ++----------------------------+------------------------+--------------+-------+----------+ +| 2024-11-25 11:02:31.256251 | Greptime is very cool! | /tmp/foo.txt | alloy | greptime | ++----------------------------+------------------------+--------------+-------+----------+ +1 row in set (0.01 sec) +``` + +## 数据模型 + +Loki 日志数据模型根据以下规则映射到 GreptimeDB 数据模型： + +没有 label 的默认表结构： + +```sql +DESC loki_demo_logs; ++--------------------+---------------------+------+------+---------+---------------+ +| Column | Type | Key | Null | Default | Semantic Type | ++--------------------+---------------------+------+------+---------+---------------+ +| greptime_timestamp | TimestampNanosecond | PRI | NO | | TIMESTAMP | +| line | String | | YES | | FIELD | ++--------------------+---------------------+------+------+---------+---------------+ +5 rows in set (0.00 sec) +``` + +- greptime_timestamp: 日志的时间戳。 +- line: 日志消息。 + +如果你指定了外部 label，我们会将它们添加为表结构中的 tag。例如上面的 `job` 和 `from`。 +在这种写入方式下不能手动指定，所有 label 都被视为 tag 并且类型为字符串。请不要尝试使用 SQL 提前创建表来指定 tag 列，这会导致类型不匹配而写入失败。 + +### 示例 + +以下是表结构的示例： + +```sql +DESC loki_demo_logs; ++--------------------+---------------------+------+------+---------+---------------+ +| Column | Type | Key | Null | Default | Semantic Type | ++--------------------+---------------------+------+------+---------+---------------+ +| greptime_timestamp | TimestampNanosecond | PRI | NO | | TIMESTAMP | +| line | String | | YES | | FIELD | +| filename | String | PRI | YES | | TAG | +| from | String | PRI | YES | | TAG | +| job | String | PRI | YES | | TAG | ++--------------------+---------------------+------+------+---------+---------------+ +5 rows in set (0.00 sec) +``` + +```sql +SHOW CREATE TABLE loki_demo_logs\G +*************************** 1. row *************************** + Table: loki_demo_logs +Create Table: CREATE TABLE IF NOT EXISTS `loki_demo_logs` ( + `greptime_timestamp` TIMESTAMP(9) NOT NULL, + `line` STRING NULL, + `filename` STRING NULL, + `from` STRING NULL, + `job` STRING NULL, + TIME INDEX (`greptime_timestamp`), + PRIMARY KEY (`filename`, `from`, `job`) +) + +ENGINE=mito +WITH( + append_mode = 'true' +) +1 row in set (0.00 sec) +``` \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/version-0.10/user-guide/ingest-data/for-observerbility/overview.md b/i18n/zh/docusaurus-plugin-content-docs/version-0.10/user-guide/ingest-data/for-observerbility/overview.md index 12bf6bbba..26a26fe0c 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/version-0.10/user-guide/ingest-data/for-observerbility/overview.md +++ b/i18n/zh/docusaurus-plugin-content-docs/version-0.10/user-guide/ingest-data/for-observerbility/overview.md @@ -7,4 +7,5 @@ GreptimeDB 与领先的可观测性工具无缝集成，为你提供系统健康 - [Vector](vector.md)：将 GreptimeDB 用作 Vector 的接收端，适用于复杂的数据流水线和多样化的数据源。 - [OpenTelemetry](opentelemetry.md)：将 telemetry 数据收集并导出到 GreptimeDB，以获取详细的可观测性洞察。 - [InfluxDB Line Protocol](influxdb-line-protocol.md)：一种广泛使用的时间序列数据协议，便于从 InfluxDB 迁移到 GreptimeDB。该文档同样介绍了 Telegraf 的集成方式。 +- [Loki](loki.md)：一种广泛使用的日志写入协议，便于从 Loki 迁移到 GreptimeDB。本文档还介绍了 Alloy 集成方法。 diff --git a/sidebars.ts b/sidebars.ts index 9ee4ca35a..e75d1abfb 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -57,6 +57,7 @@ const sidebars: SidebarsConfig = { 'user-guide/ingest-data/for-observerbility/vector', 'user-guide/ingest-data/for-observerbility/opentelemetry', 'user-guide/ingest-data/for-observerbility/influxdb-line-protocol', + 'user-guide/ingest-data/for-observerbility/loki', ], }, { diff --git a/versioned_docs/version-0.10/user-guide/ingest-data/for-observerbility/loki.md b/versioned_docs/version-0.10/user-guide/ingest-data/for-observerbility/loki.md new file mode 100644 index 000000000..f12299eb0 --- /dev/null +++ b/versioned_docs/version-0.10/user-guide/ingest-data/for-observerbility/loki.md @@ -0,0 +1,123 @@ +# Loki + +## Usage + +### API + +To send Logs to GreptimeDB through Raw HTTP API, use the following information: + +* URL: `http{s}:///v1/loki/api/v1/push` +* Headers: + * `X-Greptime-DB-Name`: `` + * `Authorization`: `Basic` authentication, which is a Base64 encoded string of `:`. For more information, please refer to [Authentication](https://docs.greptime.com/user-guide/deployments/authentication/static/) and [HTTP API](https://docs.greptime.com/user-guide/protocols/http#authentication). + * `X-Greptime-Log-Table-Name`: `` (optional) - The table name to store the logs. If not provided, the default table name is `loki_logs`. + + The request uses binary protobuf to encode the payload, The defined schema is the same as the [logproto.proto](https://github.com/grafana/loki/blob/main/pkg/logproto/logproto.proto). + +### Example Code + +[Grafana Alloy](https://grafana.com/docs/alloy/latest/) is a vendor-neutral distribution of the OpenTelemetry (OTel) Collector. Alloy uniquely combines the very best OSS observability signals in the community. + +It suplies a Loki exporter that can be used to send logs to GreptimeDB. Here is an example configuration: + +```hcl +loki.source.file "greptime" { + targets = [ + {__path__ = "/tmp/foo.txt"}, + ] + forward_to = [loki.write.greptime_loki.receiver] +} + +loki.write "greptime_loki" { + endpoint { + url = "${GREPTIME_SCHEME:=http}://${GREPTIME_HOST:=greptimedb}:${GREPTIME_PORT:=4000}/v1/loki/api/v1/push" + headers = { + "x-greptime-db-name" = "${GREPTIME_DB:=public}", + "x-greptime-log-table-name" = "${GREPTIME_LOG_TABLE_NAME:=loki_demo_logs}", + } + } + external_labels = { + "job" = "greptime" + "from" = "alloy" + } +} +``` + +We listen to the file `/tmp/foo.txt` and send the logs to GreptimeDB. The logs are stored in the table `loki_demo_logs` with the external labels `job` and `from`. + +For more information, please refer to the [Grafana Alloy loki.write documentation](https://grafana.com/docs/alloy/latest/reference/components/loki/loki.write/). + +You can run the following command to check the data in the table: + +```sql +SELECT * FROM loki_demo_logs; ++----------------------------+------------------------+--------------+-------+----------+ +| greptime_timestamp | line | filename | from | job | ++----------------------------+------------------------+--------------+-------+----------+ +| 2024-11-25 11:02:31.256251 | Greptime is very cool! | /tmp/foo.txt | alloy | greptime | ++----------------------------+------------------------+--------------+-------+----------+ +1 row in set (0.01 sec) +``` + +## Data Model + +The Loki logs data model is mapped to the GreptimeDB data model according to the following rules: + +Default table schema without external labels: + +```sql +DESC loki_demo_logs; ++--------------------+---------------------+------+------+---------+---------------+ +| Column | Type | Key | Null | Default | Semantic Type | ++--------------------+---------------------+------+------+---------+---------------+ +| greptime_timestamp | TimestampNanosecond | PRI | NO | | TIMESTAMP | +| line | String | | YES | | FIELD | ++--------------------+---------------------+------+------+---------+---------------+ +5 rows in set (0.00 sec) +``` + +- greptime_timestamp: The timestamp of the log. +- line: The log message. + +if you specify the external labels, we will add them as tags to the table schema. like `job` and `from` in the above example. +You can't specify tags manually, all labels are treated as tags and string type. +Please do not attempt to pre-create the table using SQL to specify tag columns, as this will cause a type mismatch and write failure. + +### Example + +The following is an example of the table schema: + +```sql +DESC loki_demo_logs; ++--------------------+---------------------+------+------+---------+---------------+ +| Column | Type | Key | Null | Default | Semantic Type | ++--------------------+---------------------+------+------+---------+---------------+ +| greptime_timestamp | TimestampNanosecond | PRI | NO | | TIMESTAMP | +| line | String | | YES | | FIELD | +| filename | String | PRI | YES | | TAG | +| from | String | PRI | YES | | TAG | +| job | String | PRI | YES | | TAG | ++--------------------+---------------------+------+------+---------+---------------+ +5 rows in set (0.00 sec) +``` + +```sql +SHOW CREATE TABLE loki_demo_logs\G +*************************** 1. row *************************** + Table: loki_demo_logs +Create Table: CREATE TABLE IF NOT EXISTS `loki_demo_logs` ( + `greptime_timestamp` TIMESTAMP(9) NOT NULL, + `line` STRING NULL, + `filename` STRING NULL, + `from` STRING NULL, + `job` STRING NULL, + TIME INDEX (`greptime_timestamp`), + PRIMARY KEY (`filename`, `from`, `job`) +) + +ENGINE=mito +WITH( + append_mode = 'true' +) +1 row in set (0.00 sec) +``` \ No newline at end of file diff --git a/versioned_docs/version-0.10/user-guide/ingest-data/for-observerbility/overview.md b/versioned_docs/version-0.10/user-guide/ingest-data/for-observerbility/overview.md index f20030a56..39a9d5bd2 100644 --- a/versioned_docs/version-0.10/user-guide/ingest-data/for-observerbility/overview.md +++ b/versioned_docs/version-0.10/user-guide/ingest-data/for-observerbility/overview.md @@ -8,3 +8,4 @@ GreptimeDB integrates seamlessly with leading observability tools to provide a c - [Vector](vector.md): Use GreptimeDB as a sink for Vector, ideal for complex data pipelines and diverse data sources. - [OpenTelemetry](opentelemetry.md): Collect and export telemetry data to GreptimeDB for detailed observability insights. - [InfluxDB Line Protocol](influxdb-line-protocol.md): A widely-used protocol for time-series data, facilitating migration from InfluxDB to GreptimeDB. The Telegraf integration method is also introduced in this document. +- [Loki](loki.md): A widely-used log write protocol, facilitating migration from Loki to GreptimeDB. The Alloy integration method is also introduced in this document.