diff --git a/README.md b/README.md index b157d05..3824697 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Scripts and samples to support Greptime Demos and Talks. Might be rough around t * [Kafka + Vector + GreptimeDB + Web UI](kafka-ingestion) * [Flight data ingestion and visualization](flight-data-ingester), including geospatial index with H3 -* [Grafana Alloy + GreptimeDB](grafana-alloy) +* [OpenTelemetry + Grafana Alloy + GreptimeDB](grafana-alloy) ### Data Migrations diff --git a/grafana-alloy/README.md b/grafana-alloy/README.md index de2452a..b738d7f 100644 --- a/grafana-alloy/README.md +++ b/grafana-alloy/README.md @@ -1,16 +1,23 @@ -# Grafana Alloy with GreptimeDB +# OpenTelemetry and Grafana Alloy with GreptimeDB This docker-compose file demos how to ingest data from Grafana Alloy to -GreptimeDB. +GreptimeDB, as well as using GreptimeDB as an OpenTelemetry data collector. It uses [Grafana Alloy](https://grafana.com/docs/alloy) as a Prometheus data -source (combination of Prometheus Node Exporter and Prometheus Agent), and use -remote write protocol to ingest data into GreptimeDB. +source (combination of Prometheus Node Exporter and Prometheus Agent), and +ingest data into GreptimeDB using following protocols: + +- Prometheus Remote Write +- OpenTelemetry OTLP + +In the real world, you will use Grafana Alloy as a Prometheus agent or an +OpenTelemetry data collector, with both sinks connected to GreptimeDB. ## How to run this demo Ensure you have `git`, `docker`, `docker-compose` and `mysql` client -installed. To run this demo: +installed. Docker Compose version 2.24 or higher is required. To run this +demo: ```shell git clone https://github.com/GreptimeTeam/demo-scene.git @@ -67,19 +74,39 @@ The topology is illustrated in this diagram. ```mermaid flowchart LR greptimedb[(GreptimeDB)] - alloy[Alloy] grafana[Grafana] - alloy --> greptimedb + subgraph alloy[Alloy] + prometheus_exporter_unix --> prometheus_relabel + prometheus_relabel --> prometheus_remote_write + + prometheus_relabel --> otelcol_receiver_prometheus + otelcol_receiver_prometheus --> otelcol_processor_transform + otelcol_processor_transform --> otelcol_exporter_otlphttp + end + + prometheus_remote_write --> |PRW| greptimedb + otelcol_exporter_otlphttp --> |OTLP HTTP/Protobuf| greptimedb + greptimedb --> grafana ``` +Grafana Alloy is a telemetry data pipeline that ingests data in Prometheus, +Loki, and OpenTelemetry formats. It also provides processing capabilities, such +as Prometheus relabeling and OpenTelemetry OTTL (OpenTelemetry Transformation +Language) functions. + +In this example, we generate metrics data from Alloy's built-in data source +called `prometheus.exporter.unix` and export the data to both a Prometheus +remote write sink and an OpenTelemetry-compatible collector, which is +GreptimeDB. + ## Run in GreptimeCloud By default, this example writes data into a GreptimeDB instance within the docker compose. It's also possible to write to your own GreptimeCloud instance by creating a `greptime.env` file from our sample `greptime.env.sample` and -providing your host, dbname and authentication information. Then use `docker +providing your host, dbname and authentication information.Then use `docker compose down` and `docker compose up` to recreate the compose cluster and apply new settings. diff --git a/grafana-alloy/config.alloy.tpl b/grafana-alloy/config.alloy.tpl index 0fe7677..3f41e04 100644 --- a/grafana-alloy/config.alloy.tpl +++ b/grafana-alloy/config.alloy.tpl @@ -13,16 +13,57 @@ prometheus.relabel "filter_metrics" { regex = "dev" } - forward_to = [prometheus.remote_write.metrics_service.receiver] + forward_to = [ + prometheus.remote_write.metrics_service.receiver, + otelcol.receiver.prometheus.metrics_prm_to_otel.receiver, + ] } prometheus.remote_write "metrics_service" { - endpoint { - url = "${GREPTIME_SCHEME:=http}://${GREPTIME_HOST:=greptimedb}:${GREPTIME_PORT:=4000}/v1/prometheus/write?db=${GREPTIME_DB:=public}" + endpoint { + url = "${GREPTIME_SCHEME:=http}://${GREPTIME_HOST:=greptimedb}:${GREPTIME_PORT:=4000}/v1/prometheus/write?db=${GREPTIME_DB:=public}" - basic_auth { - username = "${GREPTIME_USERNAME}" - password = "${GREPTIME_PASSWORD}" - } + basic_auth { + username = "${GREPTIME_USERNAME}" + password = "${GREPTIME_PASSWORD}" } + } +} + +// Additional example: +// Convert Prometheus metrics into OpenTelemetry format and ingest into GreptimeDB OTLP endpoint +// +// This is to demo the usage of using OpenTelemetry with GreptimeDB (we just use Prometheus as data source here) +otelcol.receiver.prometheus "metrics_prm_to_otel" { + output { + metrics = [otelcol.processor.transform.rename.input] + } +} + +otelcol.processor.transform "rename" { + metric_statements { + context = "metric" + statements = [ + "replace_pattern(name, \"(.*)\", \"otel_$$1\")", + ] + } + + output { + metrics = [otelcol.exporter.otlphttp.greptimedb.input] + } +} + +otelcol.exporter.otlphttp "greptimedb" { + client { + endpoint = "${GREPTIME_SCHEME:=http}://${GREPTIME_HOST:=greptimedb}:${GREPTIME_PORT:=4000}/v1/otlp/" + headers = { + "X-Greptime-DB-Name" = "${GREPTIME_DB:=public}", + } + auth = otelcol.auth.basic.credentials.handler + } +} + +otelcol.auth.basic "credentials" { + username = "${GREPTIME_USERNAME}" + password = "${GREPTIME_PASSWORD}" } diff --git a/grafana-alloy/docker-compose.yml b/grafana-alloy/docker-compose.yml index ed26a39..eedaa45 100644 --- a/grafana-alloy/docker-compose.yml +++ b/grafana-alloy/docker-compose.yml @@ -11,7 +11,7 @@ services: init: true greptimedb: - image: docker.io/greptime/greptimedb:v0.9.3 + image: docker.io/greptime/greptimedb:v0.10.0-nightly-20241007 command: standalone start --http-addr=0.0.0.0:4000 --rpc-addr=0.0.0.0:4001 --mysql-addr=0.0.0.0:4002 --postgres-addr 0.0.0.0:4003 ports: - 4000:4000 diff --git a/grafana-alloy/media/alloy-graph.png b/grafana-alloy/media/alloy-graph.png index 9e6be30..2680352 100644 Binary files a/grafana-alloy/media/alloy-graph.png and b/grafana-alloy/media/alloy-graph.png differ