From 830efa2c07a99bc9afcf16ca59ba53fe7637effc Mon Sep 17 00:00:00 2001 From: Yiran Date: Thu, 2 Jan 2025 11:18:18 +0800 Subject: [PATCH] docs: add Alloy to open source GreptimeDB (#1413) --- .../clients/otlp-logs-integration.md | 22 ---- .../clients/otlp-metrics-integration.md | 18 --- .../ingest-data/for-observerbility/alloy.md | 120 ++++++++++++++++++ .../for-observerbility/opentelemetry.md | 98 +++++++------- .../for-observerbility/prometheus.md | 6 + docs/user-guide/integrations/alloy.md | 10 ++ .../clients/otlp-logs-integration.md | 22 ---- .../clients/otlp-metrics-integration.md | 18 --- .../ingest-data/for-observerbility/alloy.md | 120 ++++++++++++++++++ .../for-observerbility/opentelemetry.md | 99 +++++++-------- .../for-observerbility/prometheus.md | 6 + .../current/user-guide/integrations/alloy.md | 9 ++ sidebars.ts | 2 + 13 files changed, 368 insertions(+), 182 deletions(-) delete mode 100644 docs/db-cloud-shared/clients/otlp-logs-integration.md delete mode 100644 docs/db-cloud-shared/clients/otlp-metrics-integration.md create mode 100644 docs/user-guide/ingest-data/for-observerbility/alloy.md create mode 100644 docs/user-guide/integrations/alloy.md delete mode 100644 i18n/zh/docusaurus-plugin-content-docs/current/db-cloud-shared/clients/otlp-logs-integration.md delete mode 100644 i18n/zh/docusaurus-plugin-content-docs/current/db-cloud-shared/clients/otlp-metrics-integration.md create mode 100644 i18n/zh/docusaurus-plugin-content-docs/current/user-guide/ingest-data/for-observerbility/alloy.md create mode 100644 i18n/zh/docusaurus-plugin-content-docs/current/user-guide/integrations/alloy.md diff --git a/docs/db-cloud-shared/clients/otlp-logs-integration.md b/docs/db-cloud-shared/clients/otlp-logs-integration.md deleted file mode 100644 index 4a6191164..000000000 --- a/docs/db-cloud-shared/clients/otlp-logs-integration.md +++ /dev/null @@ -1,22 +0,0 @@ -GreptimeDB is an observability backend to consume OpenTelemetry Logs natively via [OTLP/HTTP](https://opentelemetry.io/docs/specs/otlp/#otlphttp) protocol. - -#### API - -To send OpenTelemetry Logs to GreptimeDB through OpenTelemetry SDK libraries, use the following information: - -* URL: `http{s}:///v1/otlp/v1/logs` -* Headers: - * `X-Greptime-DB-Name`: `` - * `Authorization`: `Basic` authentication, which is a Base64 encoded string of `:`. For more information, please refer to [Authentication](https://docs.greptime.com/user-guide/deployments/authentication/static/) and [HTTP API](https://docs.greptime.com/user-guide/protocols/http#authentication). - * `X-Greptime-Log-Table-Name`: `` (optional) - The table name to store the logs. If not provided, the default table name is `opentelemetry_logs`. - * `X-Greptime-Log-Extract-Keys`: `` (optional) - The keys to extract from the attributes. The keys should be separated by commas (`,`). For example, `key1,key2,key3` will extract the keys `key1`, `key2`, and `key3` from the attributes and promote them to the top level of the log, setting them as tags. If the field type is array, float, or object, an error will be returned. If a pipeline is provided, this setting will be ignored. - * `X-Greptime-Log-Pipeline-Name`: `` (optional) - The pipeline name to process the logs. If not provided, the extract keys will be used to process the logs. - * `X-Greptime-Log-Pipeline-Version`: `` (optional) - The pipeline version to process the logs. If not provided, the latest version of the pipeline will be used. - -The request uses binary protobuf to encode the payload, so you need to use packages that support `HTTP/protobuf`. - -:::tip NOTE -The package names may change according to OpenTelemetry, so we recommend that you refer to the official OpenTelemetry documentation for the most up-to-date information. -::: - -For more information about the OpenTelemetry SDK, please refer to the official documentation for your preferred programming language. diff --git a/docs/db-cloud-shared/clients/otlp-metrics-integration.md b/docs/db-cloud-shared/clients/otlp-metrics-integration.md deleted file mode 100644 index 7a8a31ecf..000000000 --- a/docs/db-cloud-shared/clients/otlp-metrics-integration.md +++ /dev/null @@ -1,18 +0,0 @@ -GreptimeDB is an observability backend to consume OpenTelemetry Metrics natively via [OTLP/HTTP](https://opentelemetry.io/docs/specs/otlp/#otlphttp) protocol. - -#### API - -To send OpenTelemetry Metrics to GreptimeDB through OpenTelemetry SDK libraries, use the following information: - -* URL: `http{s}:///v1/otlp/v1/metrics` -* Headers: - * `X-Greptime-DB-Name`: `` - * `Authorization`: `Basic` authentication, which is a Base64 encoded string of `:`. For more information, please refer to [Authentication](https://docs.greptime.com/user-guide/deployments/authentication/static/) and [HTTP API](https://docs.greptime.com/user-guide/protocols/http#authentication) - -The request uses binary protobuf to encode the payload, so you need to use packages that support `HTTP/protobuf`. For example, in Node.js, you can use [`exporter-trace-otlp-proto`](https://www.npmjs.com/package/@opentelemetry/exporter-trace-otlp-proto); in Go, you can use [`go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp`](https://pkg.go.dev/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp); in Java, you can use [`io.opentelemetry:opentelemetry-exporter-otlp`](https://mvnrepository.com/artifact/io.opentelemetry/opentelemetry-exporter-otlp); and in Python, you can use [`opentelemetry-exporter-otlp-proto-http`](https://pypi.org/project/opentelemetry-exporter-otlp-proto-http/). - -:::tip NOTE -The package names may change according to OpenTelemetry, so we recommend that you refer to the official OpenTelemetry documentation for the most up-to-date information. -::: - -For more information about the OpenTelemetry SDK, please refer to the official documentation for your preferred programming language. diff --git a/docs/user-guide/ingest-data/for-observerbility/alloy.md b/docs/user-guide/ingest-data/for-observerbility/alloy.md new file mode 100644 index 000000000..78ff91868 --- /dev/null +++ b/docs/user-guide/ingest-data/for-observerbility/alloy.md @@ -0,0 +1,120 @@ +--- +keywords: [Grafana Alloy, Prometheus Remote Write, OpenTelemetry, data pipeline] +description: Instructions on integrating GreptimeDB with Grafana Alloy for Prometheus Remote Write and OpenTelemetry. +--- + +# Grafana Alloy + +[Grafana Alloy](https://grafana.com/docs/alloy/latest/) is an observability data pipeline for OpenTelemetry (OTel), Prometheus, Pyroscope, Loki, and many other metrics, logs, traces, and profiling tools. +You can integrate GreptimeDB as a data sink for Alloy. + +## Prometheus Remote Write + +Configure GreptimeDB as remote write target: + +```hcl +prometheus.remote_write "greptimedb" { + endpoint { + url = "${GREPTIME_SCHEME:=http}://${GREPTIME_HOST:=greptimedb}:${GREPTIME_PORT:=4000}/v1/prometheus/write?db=${GREPTIME_DB:=public}" + + basic_auth { + username = "${GREPTIME_USERNAME}" + password = "${GREPTIME_PASSWORD}" + } + } +} +``` + +- `GREPTIME_HOST`: GreptimeDB host address, e.g., `localhost`. +- `GREPTIME_DB`: GreptimeDB database name, default is `public`. +- `GREPTIME_USERNAME` and `GREPTIME_PASSWORD`: The [authentication credentials](/user-guide/deployments/authentication/static.md) for GreptimeDB. + +For details on the data model transformation from Prometheus to GreptimeDB, refer to the [Data Model](/user-guide/ingest-data/for-observerbility/prometheus.md#data-model) section in the Prometheus Remote Write guide. + +## OpenTelemetry + +GreptimeDB can also be configured as OpenTelemetry collector. + +### Metrics + +```hcl +otelcol.exporter.otlphttp "greptimedb" { + client { + endpoint = "${GREPTIME_SCHEME:=http}://${GREPTIME_HOST:=greptimedb}:${GREPTIME_PORT:=4000}/v1/otlp/" + headers = { + "X-Greptime-DB-Name" = "${GREPTIME_DB:=public}", + } + auth = otelcol.auth.basic.credentials.handler + } +} + +otelcol.auth.basic "credentials" { + username = "${GREPTIME_USERNAME}" + password = "${GREPTIME_PASSWORD}" +} +``` + +- `GREPTIME_HOST`: GreptimeDB host address, e.g., `localhost`. +- `GREPTIME_DB`: GreptimeDB database name, default is `public`. +- `GREPTIME_USERNAME` and `GREPTIME_PASSWORD`: The [authentication credentials](/user-guide/deployments/authentication/static.md) for GreptimeDB. + +For details on the metrics data model transformation from OpenTelemetry to GreptimeDB, refer to the [Data Model](/user-guide/ingest-data/for-observerbility/opentelemetry.md#data-model) section in the OpenTelemetry guide. + +### Logs + +The following example setting up a logging pipeline using Loki and OpenTelemetry Collector (otelcol) to forward logs to a GreptimeDB: + +```hcl +loki.source.file "greptime" { + targets = [ + {__path__ = "/tmp/foo.txt"}, + ] + forward_to = [otelcol.receiver.loki.greptime.receiver] +} + +otelcol.receiver.loki "greptime" { + output { + logs = [otelcol.exporter.otlphttp.greptimedb_logs.input] + } +} + +otelcol.auth.basic "credentials" { + username = "${GREPTIME_USERNAME}" + password = "${GREPTIME_PASSWORD}" +} + +otelcol.exporter.otlphttp "greptimedb_logs" { + client { + endpoint = "${GREPTIME_SCHEME:=http}://${GREPTIME_HOST:=greptimedb}:${GREPTIME_PORT:=4000}/v1/otlp/" + headers = { + "X-Greptime-DB-Name" = "${GREPTIME_DB:=public}", + "X-Greptime-Log-Table-Name" = "demo_logs", + "X-Greptime-Gog-Extract-Keys" = "filename,log.file.name,loki.attribute.labels", + } + auth = otelcol.auth.basic.credentials.handler + } +} +``` + +- Loki Source Configuration + - The `loki.source.file "greptime"` block defines a source for Loki to read logs from a file located at `/tmp/foo.txt` + - The `forward_to` array indicates that the logs read from this file should be forwarded to the `otelcol.receiver.loki.greptime.receiver` +- OpenTelemetry Collector Receiver Configuration: + - The `otelcol.receiver.loki "greptime"` block sets up a receiver within the OpenTelemetry Collector to receive logs from Loki. + - The `output` section specifies that the received logs should be forwarded to the `otelcol.exporter.otlphttp.greptimedb_logs.input`. +- OpenTelemetry Collector Exporter Configuration: + - The `otelcol.exporter.otlphttp "greptimedb_logs"` block configures an HTTP exporter to send logs to GreptimeDB. + - `GREPTIME_HOST`: GreptimeDB host address, e.g., `localhost`. + - `GREPTIME_DB`: GreptimeDB database name, default is `public`. + - `GREPTIME_USERNAME` and `GREPTIME_PASSWORD`: The [authentication credentials](/user-guide/deployments/authentication/static.md) for GreptimeDB. + - `LOG_TABLE_NAME`: The name of the table to store logs, default table name is `opentelemetry_logs`. + - `EXTRACT_KEYS`: The keys to extract from the attributes, separated by commas (`,`), e.g., `filename,log.file.name,loki.attribute.labels`, see [HTTP API documentation](opentelemetry.md#otlphttp-api-1) for details. + +For details on the log data model transformation from OpenTelemetry to GreptimeDB, refer to the [Data Model](/user-guide/ingest-data/for-observerbility/opentelemetry.md#data-model-1) section in the OpenTelemetry guide. + +:::tip NOTE +The example codes above may be outdated according to OpenTelemetry. We recommend that you refer to the official OpenTelemetry documentation And Grafana Alloy for the most up-to-date information. +::: + +For more information on the example code, please refer to the official documentation for your preferred programming language. + diff --git a/docs/user-guide/ingest-data/for-observerbility/opentelemetry.md b/docs/user-guide/ingest-data/for-observerbility/opentelemetry.md index eeb48a18b..9ff771e85 100644 --- a/docs/user-guide/ingest-data/for-observerbility/opentelemetry.md +++ b/docs/user-guide/ingest-data/for-observerbility/opentelemetry.md @@ -7,15 +7,33 @@ description: Instructions for integrating OpenTelemetry with GreptimeDB, includi [OpenTelemetry](https://opentelemetry.io/) is a vendor-neutral open-source observability framework for instrumenting, generating, collecting, and exporting telemetry data such as traces, metrics, logs. The OpenTelemetry Protocol (OTLP) defines the encoding, transport, and delivery mechanism of telemetry data between telemetry sources, intermediate processes such as collectors and telemetry backends. +## OpenTelemetry Collectors + +You can easily configure GreptimeDB as the target for your OpenTelemetry collector. +For more information, please refer to the [Grafana Alloy](alloy.md) example. + ## Metrics -### OTLP/HTTP +GreptimeDB is an observability backend to consume OpenTelemetry Metrics natively via [OTLP/HTTP](https://opentelemetry.io/docs/specs/otlp/#otlphttp) protocol. + +### OTLP/HTTP API + +To send OpenTelemetry Metrics to GreptimeDB through OpenTelemetry SDK libraries, use the following information: -import Includeotlpmetrycsintegration from '../../../db-cloud-shared/clients/otlp-metrics-integration.md' +* URL: `http{s}:///v1/otlp/v1/metrics` +* Headers: + * `X-Greptime-DB-Name`: `` + * `Authorization`: `Basic` authentication, which is a Base64 encoded string of `:`. For more information, please refer to [Authentication](https://docs.greptime.com/user-guide/deployments/authentication/static/) and [HTTP API](https://docs.greptime.com/user-guide/protocols/http#authentication) + +The request uses binary protobuf to encode the payload, so you need to use packages that support `HTTP/protobuf`. For example, in Node.js, you can use [`exporter-trace-otlp-proto`](https://www.npmjs.com/package/@opentelemetry/exporter-trace-otlp-proto); in Go, you can use [`go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp`](https://pkg.go.dev/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp); in Java, you can use [`io.opentelemetry:opentelemetry-exporter-otlp`](https://mvnrepository.com/artifact/io.opentelemetry/opentelemetry-exporter-otlp); and in Python, you can use [`opentelemetry-exporter-otlp-proto-http`](https://pypi.org/project/opentelemetry-exporter-otlp-proto-http/). + +:::tip NOTE +The package names may change according to OpenTelemetry, so we recommend that you refer to the official OpenTelemetry documentation for the most up-to-date information. +::: - +For more information about the OpenTelemetry SDK, please refer to the official documentation for your preferred programming language. -#### Example Code +### Example Code Here are some example codes about how to setup the request in different languages: @@ -95,7 +113,7 @@ The example codes above may be outdated according to OpenTelemetry. We recommend For more information on the example code, please refer to the official documentation for your preferred programming language. -#### Data Model +### Data Model The OTLP metrics data model is mapped to the GreptimeDB data model according to the following rules: @@ -108,57 +126,34 @@ The OTLP metrics data model is mapped to the GreptimeDB data model according to ## Logs -### OTLP/HTTP - -import Includeotlplogintegration from '../../../db-cloud-shared/clients/otlp-logs-integration.md' - - - -#### Example Code - -Here are some example codes about how to use Grafana Alloy to send OpenTelemetry logs to GreptimeDB: - -```hcl -loki.source.file "greptime" { - targets = [ - {__path__ = "/tmp/foo.txt"}, - ] - forward_to = [otelcol.receiver.loki.greptime.receiver] -} - -otelcol.receiver.loki "greptime" { - output { - logs = [otelcol.exporter.otlphttp.greptimedb_logs.input] - } -} - -otelcol.auth.basic "credentials" { - username = "${GREPTIME_USERNAME}" - password = "${GREPTIME_PASSWORD}" -} - -otelcol.exporter.otlphttp "greptimedb_logs" { - client { - endpoint = "${GREPTIME_SCHEME:=http}://${GREPTIME_HOST:=greptimedb}:${GREPTIME_PORT:=4000}/v1/otlp/" - headers = { - "X-Greptime-DB-Name" = "${GREPTIME_DB:=public}", - "X-Greptime-Log-Table-Name" = "demo_logs", - "X-Greptime-Gog-Extract-Keys" = "filename,log.file.name,loki.attribute.labels", - } - auth = otelcol.auth.basic.credentials.handler - } -} -``` +GreptimeDB consumes OpenTelemetry Logs natively via [OTLP/HTTP](https://opentelemetry.io/docs/specs/otlp/#otlphttp) protocol. + +### OTLP/HTTP API API + +To send OpenTelemetry Logs to GreptimeDB through OpenTelemetry SDK libraries, use the following information: + +* URL: `http{s}:///v1/otlp/v1/logs` +* Headers: + * `X-Greptime-DB-Name`: `` + * `Authorization`: `Basic` authentication, which is a Base64 encoded string of `:`. For more information, please refer to [Authentication](/user-guide/deployments/authentication/static.md) and [HTTP API](/user-guide/protocols/http.md#authentication). + * `X-Greptime-Log-Table-Name`: `` (optional) - The table name to store the logs. If not provided, the default table name is `opentelemetry_logs`. + * `X-Greptime-Log-Extract-Keys`: `` (optional) - The keys to extract from the attributes. The keys should be separated by commas (`,`). For example, `key1,key2,key3` will extract the keys `key1`, `key2`, and `key3` from the attributes and promote them to the top level of the log, setting them as tags. If the field type is array, float, or object, an error will be returned. If a pipeline is provided, this setting will be ignored. + * `X-Greptime-Log-Pipeline-Name`: `` (optional) - The pipeline name to process the logs. If not provided, the extract keys will be used to process the logs. + * `X-Greptime-Log-Pipeline-Version`: `` (optional) - The pipeline version to process the logs. If not provided, the latest version of the pipeline will be used. -This example listens for changes to the file and sends the latest values to GreptimeDB via the otlp protocol. +The request uses binary protobuf to encode the payload, so you need to use packages that support `HTTP/protobuf`. :::tip NOTE -The example codes above may be outdated according to OpenTelemetry. We recommend that you refer to the official OpenTelemetry documentation And Grafana Alloy for the most up-to-date information. +The package names may change according to OpenTelemetry, so we recommend that you refer to the official OpenTelemetry documentation for the most up-to-date information. ::: -For more information on the example code, please refer to the official documentation for your preferred programming language. +For more information about the OpenTelemetry SDK, please refer to the official documentation for your preferred programming language. + +### Example Code -#### Data Model +Please refer to the [Alloy documentation](alloy.md#logs) for example code on how to send OpenTelemetry logs to GreptimeDB. + +### Data Model The OTLP logs data model is mapped to the GreptimeDB data model according to the following rules: @@ -188,4 +183,5 @@ Default table schema: - You can use `X-Greptime-Log-Table-Name` to specify the table name for storing the logs. If not provided, the default table name is `opentelemetry_logs`. - All attributes, including resource attributes, scope attributes, and log attributes, will be stored as a JSON column in the GreptimeDB table. -- The timestamp of the log will be used as the timestamp index in GreptimeDB, with the column name `timestamp`. It is preferred to use `time_unix_nano` as the timestamp column. If `time_unix_nano` is not provided, `observed_time_unix_nano` will be used instead. \ No newline at end of file +- The timestamp of the log will be used as the timestamp index in GreptimeDB, with the column name `timestamp`. It is preferred to use `time_unix_nano` as the timestamp column. If `time_unix_nano` is not provided, `observed_time_unix_nano` will be used instead. + diff --git a/docs/user-guide/ingest-data/for-observerbility/prometheus.md b/docs/user-guide/ingest-data/for-observerbility/prometheus.md index e8c780ef4..81b67d2c0 100644 --- a/docs/user-guide/ingest-data/for-observerbility/prometheus.md +++ b/docs/user-guide/ingest-data/for-observerbility/prometheus.md @@ -10,6 +10,8 @@ providing a seamless integration experience. ## Remote write configuration +### Prometheus configuration file + To configure Prometheus with GreptimeDB, update your [Prometheus configuration file](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#configuration-file) (`prometheus.yml`) as follows: @@ -33,6 +35,10 @@ remote_read: - The `db` parameter in the URL represents the database to which we want to write data. It is optional. By default, the database is set to `public`. - `basic_auth` is the authentication configuration. Fill in the username and password if GreptimeDB authentication is enabled. Please refer to the [authentication document](/user-guide/deployments/authentication/overview.md). +### Grafana Alloy configuration file + +If you are using Grafana Alloy, configure the remote write endpoint in the Alloy configuration file (`config.alloy`). For more information, refer to the [Alloy documentation](alloy.md#prometheus-remote-write). + ## Data Model In the [data model](/user-guide/concepts/data-model.md) of GreptimeDB, data is organized into tables with columns for tags, time index, and fields. diff --git a/docs/user-guide/integrations/alloy.md b/docs/user-guide/integrations/alloy.md new file mode 100644 index 000000000..52ea6dc75 --- /dev/null +++ b/docs/user-guide/integrations/alloy.md @@ -0,0 +1,10 @@ +--- +keywords: [Alloy, Grafana Alloy, GreptimeDB, Ingest Data] +description: Integrate GreptimeDB with Grafana Alloy. +--- + +# Grafana Alloy + +GreptimeDB can be set up as a data sink for Grafana Alloy. +For more information, please refer to the [Ingest Data through Grafana Alloy](/user-guide/ingest-data/for-observerbility/alloy.md) guide. + diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/db-cloud-shared/clients/otlp-logs-integration.md b/i18n/zh/docusaurus-plugin-content-docs/current/db-cloud-shared/clients/otlp-logs-integration.md deleted file mode 100644 index 58f9b233c..000000000 --- a/i18n/zh/docusaurus-plugin-content-docs/current/db-cloud-shared/clients/otlp-logs-integration.md +++ /dev/null @@ -1,22 +0,0 @@ -GreptimeDB 是一个可观测性后端，能够通过 [OTLP/HTTP](https://opentelemetry.io/docs/specs/otlp/#otlphttp) 协议原生地消费 OpenTelemetry 日志。 - -#### API - -要通过 OpenTelemetry SDK 库将 OpenTelemetry 日志发送到 GreptimeDB，请使用以下信息： - -* **URL:** `https:///v1/otlp/v1/logs` -* **Headers:** - * `X-Greptime-DB-Name`: `` - * `Authorization`: `Basic` 认证，这是一个 Base64 编码的 `:` 字符串。更多信息，请参考 [鉴权](https://docs.greptime.cn/user-guide/deployments/authentication/static/) 和 [HTTP API](https://docs.greptime.cn/user-guide/protocols/http#authentication)。 - * `X-Greptime-Log-Table-Name`: ``（可选）- 存储日志的表名。如果未提供，默认表名为 `opentelemetry_logs`。 - * `X-Greptime-Log-Extract-Keys`: ``（可选）- 从属性中提取对应 key 的值到表的顶级字段。key 应以逗号（`,`）分隔。例如，`key1,key2,key3` 将从属性中提取 `key1`、`key2` 和 `key3`，并将它们提升到日志的顶层，设置为标签。如果提取的字段类型是数组、浮点数或对象，将返回错误。如果提供了 pipeline name，此设置将被忽略。 - * `X-Greptime-Log-Pipeline-Name`: ``（可选）- 处理日志的 pipeline 名称。如果未提供，将使用 `X-Greptime-Log-Extract-Keys` 来处理日志。 - * `X-Greptime-Log-Pipeline-Version`: ``（可选）- 处理日志的 pipeline 的版本。如果未提供，将使用 pipeline 的最新版本。 - -请求使用二进制 protobuf 编码负载，因此您需要使用支持 `HTTP/protobuf` 的包。 - -:::tip 提示 -包名可能会根据 OpenTelemetry 的更新而变化，因此我们建议您参考官方 OpenTelemetry 文档以获取最新信息。 -::: - -有关 OpenTelemetry SDK 的更多信息，请参考您首选编程语言的官方文档。 \ No newline at end of file diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/db-cloud-shared/clients/otlp-metrics-integration.md b/i18n/zh/docusaurus-plugin-content-docs/current/db-cloud-shared/clients/otlp-metrics-integration.md deleted file mode 100644 index 6e3cd7b71..000000000 --- a/i18n/zh/docusaurus-plugin-content-docs/current/db-cloud-shared/clients/otlp-metrics-integration.md +++ /dev/null @@ -1,18 +0,0 @@ -GreptimeDB 通过原生支持 [OTLP/HTTP](https://opentelemetry.io/docs/specs/otlp/#otlphttp) 协议，可以作为后端存储服务来接收 OpenTelemetry 指标数据。 - -### API - -使用下面的信息通过 Opentelemetry SDK 库发送 Metrics 到 GreptimeDB： - -* URL: `https:///v1/otlp/v1/metrics` -* Headers: - * `X-Greptime-DB-Name`: `` -* `Authorization`: `Basic` 认证，是 `:` 的 Base64 编码字符串。更多信息请参考 [鉴权](https://docs.greptime.cn/user-guide/deployments/authentication/static/) 和 [HTTP API](https://docs.greptime.cn/user-guide/protocols/http#authentication)。 - -请求中使用 binary protobuf 编码 payload，因此你需要使用支持 `HTTP/protobuf` 的包。例如，在 Node.js 中，可以使用 [`exporter-trace-otlp-proto`](https://www.npmjs.com/package/@opentelemetry/exporter-trace-otlp-proto)；在 Go 中，可以使用 [`go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp`](https://pkg.go.dev/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp)；在 Java 中，可以使用 [`io.opentelemetry:opentelemetry-exporter-otlp`](https://mvnrepository.com/artifact/io.opentelemetry/opentelemetry-exporter-otlp)；在 Python 中，可以使用 [`opentelemetry-exporter-otlp-proto-http`](https://pypi.org/project/opentelemetry-exporter-otlp-proto-http/)。 - -:::tip 注意 -包名可能会根据 OpenTelemetry 的发展发生变化，因此建议你参考 OpenTelemetry 官方文档以获取最新信息。 -::: - -请参考 Opentelementry 的官方文档获取它所支持的编程语言的更多信息。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/user-guide/ingest-data/for-observerbility/alloy.md b/i18n/zh/docusaurus-plugin-content-docs/current/user-guide/ingest-data/for-observerbility/alloy.md new file mode 100644 index 000000000..7b33a5999 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/current/user-guide/ingest-data/for-observerbility/alloy.md @@ -0,0 +1,120 @@ +--- +keywords: [Grafana Alloy, Prometheus Remote Write, OpenTelemetry, 数据管道] +description: 绍了如何将 GreptimeDB 配置为 Grafana Alloy 的数据接收端，包括 Prometheus Remote Write 和 OpenTelemetry 的配置示例。通过这些配置，你可以将 GreptimeDB 集成到可观测性数据管道中，实现对指标和日志的高效管理和分析。 +--- + +# Grafana Alloy + +[Grafana Alloy](https://grafana.com/docs/alloy/latest/) 是一个用于 OpenTelemetry (OTel)、Prometheus、Pyroscope、Loki 等其他指标、日志、追踪和分析工具的可观测性数据管道。 +你可以将 GreptimeDB 集成为 Alloy 的数据接收端。 + +## Prometheus Remote Write + +将 GreptimeDB 配置为远程写入目标： + +```hcl +prometheus.remote_write "greptimedb" { + endpoint { + url = "${GREPTIME_SCHEME:=http}://${GREPTIME_HOST:=greptimedb}:${GREPTIME_PORT:=4000}/v1/prometheus/write?db=${GREPTIME_DB:=public}" + + basic_auth { + username = "${GREPTIME_USERNAME}" + password = "${GREPTIME_PASSWORD}" + } + } +} +``` + +- `GREPTIME_HOST`: GreptimeDB 主机地址，例如 `localhost`。 +- `GREPTIME_DB`: GreptimeDB 数据库名称，默认是 `public`。 +- `GREPTIME_USERNAME` 和 `GREPTIME_PASSWORD`: GreptimeDB的[鉴权认证信息](/user-guide/deployments/authentication/static.md)。 + +有关从 Prometheus 到 GreptimeDB 的数据模型转换的详细信息，请参阅 Prometheus Remote Write 指南中的[数据模型](/user-guide/ingest-data/for-observerbility/prometheus.md#数据模型)部分。 + +## OpenTelemetry + +GreptimeDB 也可以配置为 OpenTelemetry Collector 的目标。 + +### 指标 + +```hcl +otelcol.exporter.otlphttp "greptimedb" { + client { + endpoint = "${GREPTIME_SCHEME:=http}://${GREPTIME_HOST:=greptimedb}:${GREPTIME_PORT:=4000}/v1/otlp/" + headers = { + "X-Greptime-DB-Name" = "${GREPTIME_DB:=public}", + } + auth = otelcol.auth.basic.credentials.handler + } +} + +otelcol.auth.basic "credentials" { + username = "${GREPTIME_USERNAME}" + password = "${GREPTIME_PASSWORD}" +} +``` + +- `GREPTIME_HOST`: GreptimeDB 主机地址，例如 `localhost`。 +- `GREPTIME_DB`: GreptimeDB 数据库名称，默认是 `public`。 +- `GREPTIME_USERNAME` 和 `GREPTIME_PASSWORD`: GreptimeDB 的[鉴权认证信息](/user-guide/deployments/authentication/static.md)。 + +有关从 OpenTelemetry 到 GreptimeDB 的指标数据模型转换的详细信息，请参阅 OpenTelemetry 指南中的[数据模型](/user-guide/ingest-data/for-observerbility/opentelemetry.md#数据模型)部分。 + +### 日志 + +以下示例设置了一个使用 Loki 和 OpenTelemetry Collector (otelcol) 的日志管道，将日志转发到 GreptimeDB： + +```hcl +loki.source.file "greptime" { + targets = [ + {__path__ = "/tmp/foo.txt"}, + ] + forward_to = [otelcol.receiver.loki.greptime.receiver] +} + +otelcol.receiver.loki "greptime" { + output { + logs = [otelcol.exporter.otlphttp.greptimedb_logs.input] + } +} + +otelcol.auth.basic "credentials" { + username = "${GREPTIME_USERNAME}" + password = "${GREPTIME_PASSWORD}" +} + +otelcol.exporter.otlphttp "greptimedb_logs" { + client { + endpoint = "${GREPTIME_SCHEME:=http}://${GREPTIME_HOST:=greptimedb}:${GREPTIME_PORT:=4000}/v1/otlp/" + headers = { + "X-Greptime-DB-Name" = "${GREPTIME_DB:=public}", + "X-Greptime-Log-Table-Name" = "${LOG_TABLE_NAME}", + "X-Greptime-Gog-Extract-Keys" = "${EXTRACT_KEYS}", + } + auth = otelcol.auth.basic.credentials.handler + } +} +``` + +- Loki source 配置 + - `loki.source.file "greptime"` 块定义了 source，用于 Loki 从位于 `/tmp/foo.txt` 的文件中读取日志。 + - `forward_to` 数组指示从该文件读取的日志应转发到 `otelcol.receiver.loki.greptime.receiver`。 +- OpenTelemetry Collector Receiver 配置： + - `otelcol.receiver.loki "greptime"` 在 OpenTelemetry Collector 中设置了一个 receiver，以接收来自 Loki 的日志。 + - `output` 指定接收到的日志应转发到 `otelcol.exporter.otlphttp.greptimedb_logs.input`。 +- OpenTelemetry Collector Exporter 配置： + - `otelcol.exporter.otlphttp "greptimedb_logs"` 块配置了一个 HTTP Exporter，将日志发送到 GreptimeDB。 + - `GREPTIME_HOST`: GreptimeDB 主机地址，例如 `localhost`。 + - `GREPTIME_DB`: GreptimeDB 数据库名称，默认是 `public`。 + - `GREPTIME_USERNAME` 和 `GREPTIME_PASSWORD`: GreptimeDB 的[鉴权认证信息](/user-guide/deployments/authentication/static.md)。 + - `LOG_TABLE_NAME`: 存储日志的表名，默认表名为 `opentelemetry_logs`。 + - `EXTRACT_KEYS`: 从属性中提取对应 key 的值到表的顶级字段，用逗号分隔，例如 `filename,log.file.name,loki.attribute.labels`，详情请看 [HTTP API 文档](opentelemetry.md#otlphttp-api-1)。 + +有关从 OpenTelemetry 到 GreptimeDB 的日志数据模型转换的详细信息，请参阅 OpenTelemetry 指南中的[数据模型](/user-guide/ingest-data/for-observerbility/opentelemetry.md#数据模型-1)部分。 + +:::tip 提示 +上述示例代码可能会过时，请参考 OpenTelemetry 和 Grafana Alloy 的官方文档以获取最新信息。 +::: + +有关示例代码的更多信息，请参阅你首选编程语言的官方文档。 + diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/user-guide/ingest-data/for-observerbility/opentelemetry.md b/i18n/zh/docusaurus-plugin-content-docs/current/user-guide/ingest-data/for-observerbility/opentelemetry.md index 12a2d7ff4..f3b5e5b64 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/user-guide/ingest-data/for-observerbility/opentelemetry.md +++ b/i18n/zh/docusaurus-plugin-content-docs/current/user-guide/ingest-data/for-observerbility/opentelemetry.md @@ -7,15 +7,34 @@ description: 介绍如何使用 OpenTelemetry Protocol (OTLP) 将观测数据（ [OpenTelemetry](https://opentelemetry.io/) 是一个供应商中立的开源可观测性框架，用于检测、生成、收集和导出观测数据，例如 traces, metrics 和 logs。 OpenTelemetry Protocol (OTLP) 定义了观测数据在观测源和中间进程（例如收集器和观测后端）之间的编码、传输机制。 + +## OpenTelemetry Collectors + +你可以很简单地将 GreptimeDB 配置为 OpenTelemetry Collector 的目标。 +有关更多信息，请参阅 [Grafana Alloy](alloy.md) 示例。 + ## Metrics -### OTLP/HTTP +GreptimeDB 通过原生支持 [OTLP/HTTP](https://opentelemetry.io/docs/specs/otlp/#otlphttp) 协议，可以作为后端存储服务来接收 OpenTelemetry 指标数据。 + +### OTLP/HTTP API -import Includeotlpmetrycsintegration from '../../../db-cloud-shared/clients/otlp-metrics-integration.md' +使用下面的信息通过 Opentelemetry SDK 库发送 Metrics 到 GreptimeDB： + +* URL: `https:///v1/otlp/v1/metrics` +* Headers: + * `X-Greptime-DB-Name`: `` +* `Authorization`: `Basic` 认证，是 `:` 的 Base64 编码字符串。更多信息请参考 [鉴权](https://docs.greptime.cn/user-guide/deployments/authentication/static/) 和 [HTTP API](https://docs.greptime.cn/user-guide/protocols/http#authentication)。 + +请求中使用 binary protobuf 编码 payload，因此你需要使用支持 `HTTP/protobuf` 的包。例如，在 Node.js 中，可以使用 [`exporter-trace-otlp-proto`](https://www.npmjs.com/package/@opentelemetry/exporter-trace-otlp-proto)；在 Go 中，可以使用 [`go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp`](https://pkg.go.dev/go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp)；在 Java 中，可以使用 [`io.opentelemetry:opentelemetry-exporter-otlp`](https://mvnrepository.com/artifact/io.opentelemetry/opentelemetry-exporter-otlp)；在 Python 中，可以使用 [`opentelemetry-exporter-otlp-proto-http`](https://pypi.org/project/opentelemetry-exporter-otlp-proto-http/)。 + +:::tip 注意 +包名可能会根据 OpenTelemetry 的发展发生变化，因此建议你参考 OpenTelemetry 官方文档以获取最新信息。 +::: - +请参考 Opentelementry 的官方文档获取它所支持的编程语言的更多信息。 -#### 示例代码 +### 示例代码 下面是一些编程语言设置请求的示例代码： @@ -95,7 +114,7 @@ exporter = OTLPMetricExporter( 关于示例代码，请参考 Opentelementry 的官方文档获取它所支持的编程语言获取更多信息。 -#### 数据模型 +### 数据模型 OTLP 指标数据模型按照下方的规则被映射到 GreptimeDB 数据模型中： @@ -108,57 +127,34 @@ OTLP 指标数据模型按照下方的规则被映射到 GreptimeDB 数据模型 ## Logs -### OTLP/HTTP - -import Includeotlplogintegration from '../../../db-cloud-shared/clients/otlp-logs-integration.md' - - - -#### 示例代码 - -以下是一些关于如何使用 Grafana Alloy 将 OpenTelemetry 日志发送到 GreptimeDB 的示例代码： - -```hcl -loki.source.file "greptime" { - targets = [ - {__path__ = "/tmp/foo.txt"}, - ] - forward_to = [otelcol.receiver.loki.greptime.receiver] -} - -otelcol.receiver.loki "greptime" { - output { - logs = [otelcol.exporter.otlphttp.greptimedb_logs.input] - } -} - -otelcol.auth.basic "credentials" { - username = "${GREPTIME_USERNAME}" - password = "${GREPTIME_PASSWORD}" -} - -otelcol.exporter.otlphttp "greptimedb_logs" { - client { - endpoint = "${GREPTIME_SCHEME:=http}://${GREPTIME_HOST:=greptimedb}:${GREPTIME_PORT:=4000}/v1/otlp/" - headers = { - "X-Greptime-DB-Name" = "${GREPTIME_DB:=public}", - "x-greptime-log-table-name" = "demo_logs", - "x-greptime-log-extract-keys" = "filename,log.file.name,loki.attribute.labels", - } - auth = otelcol.auth.basic.credentials.handler - } -} -``` +GreptimeDB 是能够通过 [OTLP/HTTP](https://opentelemetry.io/docs/specs/otlp/#otlphttp) 协议原生地消费 OpenTelemetry 日志。 + +### OTLP/HTTP API + +要通过 OpenTelemetry SDK 库将 OpenTelemetry 日志发送到 GreptimeDB，请使用以下信息： -此示例监听文件的变化，并通过 OTLP 协议将最新的值发送到 GreptimeDB。 +* **URL:** `https:///v1/otlp/v1/logs` +* **Headers:** + * `X-Greptime-DB-Name`: `` + * `Authorization`: `Basic` 认证，这是一个 Base64 编码的 `:` 字符串。更多信息，请参考 [鉴权](/user-guide/deployments/authentication/static.md) 和 [HTTP API](/user-guide/protocols/http.md#鉴权)。 + * `X-Greptime-Log-Table-Name`: ``（可选）- 存储日志的表名。如果未提供，默认表名为 `opentelemetry_logs`。 + * `X-Greptime-Log-Extract-Keys`: ``（可选）- 从属性中提取对应 key 的值到表的顶级字段。key 应以逗号（`,`）分隔。例如，`key1,key2,key3` 将从属性中提取 `key1`、`key2` 和 `key3`，并将它们提升到日志的顶层，设置为标签。如果提取的字段类型是数组、浮点数或对象，将返回错误。如果提供了 pipeline name，此设置将被忽略。 + * `X-Greptime-Log-Pipeline-Name`: ``（可选）- 处理日志的 pipeline 名称。如果未提供，将使用 `X-Greptime-Log-Extract-Keys` 来处理日志。 + * `X-Greptime-Log-Pipeline-Version`: ``（可选）- 处理日志的 pipeline 的版本。如果未提供，将使用 pipeline 的最新版本。 + +请求使用二进制 protobuf 编码负载，因此您需要使用支持 `HTTP/protobuf` 的包。 :::tip 提示 -上述示例代码可能会因 OpenTelemetry 的更新而过时。我们建议您参考 OpenTelemetry 和 Grafana Alloy 的官方文档以获取最新信息。 +包名可能会根据 OpenTelemetry 的更新而变化，因此我们建议您参考官方 OpenTelemetry 文档以获取最新信息。 ::: -有关示例代码的更多信息，请参考您首选编程语言的官方文档。 +有关 OpenTelemetry SDK 的更多信息，请参考您首选编程语言的官方文档。 + +### 示例代码 -#### 数据模型 +请参考 [Alloy 文档](alloy.md#日志)中的示例代码，了解如何将 OpenTelemetry 日志发送到 GreptimeDB。 + +### 数据模型 OTLP 日志数据模型根据以下规则映射到 GreptimeDB 数据模型： @@ -188,4 +184,5 @@ OTLP 日志数据模型根据以下规则映射到 GreptimeDB 数据模型： - 您可以使用 `X-Greptime-Log-Table-Name` 指定存储日志的表名。如果未提供，默认表名为 `opentelemetry_logs`。 - 所有属性，包括资源属性、范围属性和日志属性，将作为 JSON 列存储在 GreptimeDB 表中。 -- 日志的时间戳将用作 GreptimeDB 中的时间戳索引，列名为 `timestamp`。建议使用 `time_unix_nano` 作为时间戳列。如果未提供 `time_unix_nano`，将使用 `observed_time_unix_nano`。 \ No newline at end of file +- 日志的时间戳将用作 GreptimeDB 中的时间戳索引，列名为 `timestamp`。建议使用 `time_unix_nano` 作为时间戳列。如果未提供 `time_unix_nano`，将使用 `observed_time_unix_nano`。 + diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/user-guide/ingest-data/for-observerbility/prometheus.md b/i18n/zh/docusaurus-plugin-content-docs/current/user-guide/ingest-data/for-observerbility/prometheus.md index 97bcfab11..184744eac 100644 --- a/i18n/zh/docusaurus-plugin-content-docs/current/user-guide/ingest-data/for-observerbility/prometheus.md +++ b/i18n/zh/docusaurus-plugin-content-docs/current/user-guide/ingest-data/for-observerbility/prometheus.md @@ -9,6 +9,8 @@ GreptimeDB 可以作为 Prometheus 的长期存储解决方案，提供无缝集 ## 配置 Remote Write +### Prometheus 配置文件 + 要将 GreptimeDB 集成到 Prometheus 中， 请按照以下步骤更新你的 [Prometheus 配置文件](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#configuration-file)（`prometheus.yml`）： @@ -32,6 +34,10 @@ remote_read: - URL 中的 `db` 参数表示要写入的数据库。它是可选的。默认情况下，数据库设置为 `public`。 - `basic_auth` 是身份鉴权配置。如果 GreptimeDB 启用了鉴权，请填写用户名和密码。请参阅 [鉴权认证文档](/user-guide/deployments/authentication/overview.md)。 +### Grafana Alloy 配置文件 + +如果你使用 Grafana Alloy，请在 Alloy 配置文件（`config.alloy`）中配置 Remote Write。有关更多信息，请参阅 [Alloy 文档](alloy.md#prometheus-remote-write)。 + ## 数据模型 在 GreptimeDB 的[数据模型](/user-guide/concepts/data-model.md)中，数据被组织成具有 tag、time index 和 field 的表。 diff --git a/i18n/zh/docusaurus-plugin-content-docs/current/user-guide/integrations/alloy.md b/i18n/zh/docusaurus-plugin-content-docs/current/user-guide/integrations/alloy.md new file mode 100644 index 000000000..40c3acc57 --- /dev/null +++ b/i18n/zh/docusaurus-plugin-content-docs/current/user-guide/integrations/alloy.md @@ -0,0 +1,9 @@ +--- +keywords: [Alloy, Grafana Alloy, GreptimeDB] +description: 将 GreptimeDB 与 Grafana Alloy 集成。 +--- + +# Grafana Alloy + +你可以将 GreptimeDB 设置为 Grafana Alloy 的数据接收端。 +更多信息，请参考[通过 Grafana Alloy 写入数据](/user-guide/ingest-data/for-observerbility/alloy.md)指南。 diff --git a/sidebars.ts b/sidebars.ts index d6f9b712f..6702c944f 100644 --- a/sidebars.ts +++ b/sidebars.ts @@ -58,6 +58,7 @@ const sidebars: SidebarsConfig = { 'user-guide/ingest-data/for-observerbility/opentelemetry', 'user-guide/ingest-data/for-observerbility/influxdb-line-protocol', 'user-guide/ingest-data/for-observerbility/loki', + 'user-guide/ingest-data/for-observerbility/alloy', ], }, { @@ -107,6 +108,7 @@ const sidebars: SidebarsConfig = { 'user-guide/integrations/metabase', 'user-guide/integrations/emqx', 'user-guide/integrations/dbeaver', + 'user-guide/integrations/alloy', ], }, {