diff --git a/docs/profiling/ingestion.md b/docs/profiling/ingestion.md index 9a4fa07..fb0f35a 100644 --- a/docs/profiling/ingestion.md +++ b/docs/profiling/ingestion.md @@ -13,7 +13,7 @@ The [OTEL Collector]([https://opentelemetry.io/docs/collector/](https://github.c This works with improved scalability and supports a large variety of open-source observability data formats _(e.g. pprof, etc.)_ and allows aggregating and sending profiling data to **qryn** using the _Pyroscope API_ -For large scale ingestion, we suggest using the dedicated [qryn opentelemetry distribution](https://github.com/metrico/otel-collector) for ClickHouse. +This guide is based on the [qryn opentelemetry distribution](https://github.com/metrico/otel-collector) for qryn + clickhouse. ### Examples @@ -25,126 +25,27 @@ otel-collector: volumes: - ./otel-collector-config.yaml:/etc/otel/config.yaml ports: - - "4317:4317" # OTLP gRPC receiver - - "4318:4318" # OTLP HTTP receiver - - "14250:14250" # Jaeger gRPC - - "14268:14268" # Jaeger thrift HTTP - - "9411:9411" # Zipkin port - - "24224:24224". # Fluent Forward + - "8062 :8062 " # Pyroscope gRPC receiver restart: on-failure ``` -###### qryn collector `config.taml` -The following example enables all the supported formats at once. Filter the required sections to build your configuration. +###### pyroscope `otel-collector-config.taml` +The following example enables Pyroscope ingestion using the qryn-collector. Integrate in your existing configuration. ```yml receivers: - otlp: - protocols: - grpc: - endpoint: 0.0.0.0:4317 - http: - endpoint: 0.0.0.0:4318 - jaeger: - protocols: - grpc: - endpoint: 0.0.0.0:14250 - thrift_http: - endpoint: 0.0.0.0:14268 - zipkin: - endpoint: 0.0.0.0:9411 - fluentforward: - endpoint: 0.0.0.0:24224 - prometheus: - config: - scrape_configs: - - job_name: 'otel-collector' - scrape_interval: 5s - static_configs: - - targets: ['exporter:8080'] -processors: - batch: - send_batch_size: 10000 - timeout: 5s - memory_limiter: - check_interval: 2s - limit_mib: 1800 - spike_limit_mib: 500 - resourcedetection/system: - detectors: ['system'] - system: - hostname_sources: ['os'] - resource: - attributes: - - key: service.name - value: "serviceName" - action: upsert - spanmetrics: - metrics_exporter: otlp/spanmetrics - latency_histogram_buckets: [100us, 1ms, 2ms, 6ms, 10ms, 100ms, 250ms] - dimensions_cache_size: 1500 - servicegraph: - metrics_exporter: otlp/spanmetrics - latency_histogram_buckets: [100us, 1ms, 2ms, 6ms, 10ms, 100ms, 250ms] - dimensions: [cluster, namespace] - store: - ttl: 2s - max_items: 200 - metricstransform: - transforms: - - include: calls_total - action: update - new_name: traces_spanmetrics_calls_total - - include: latency - action: update - new_name: traces_spanmetrics_latency + pyroscopereceiver: exporters: - qryn: - dsn: tcp://clickhouse-server:9000/cloki?username=default&password=************* - timeout: 10s - sending_queue: - queue_size: 100 - retry_on_failure: - enabled: true - initial_interval: 5s - max_interval: 30s - max_elapsed_time: 300s - logs: - format: json - otlp/spanmetrics: - endpoint: localhost:4317 - tls: - insecure: true -extensions: - health_check: - pprof: - zpages: - memory_ballast: - size_mib: 1000 - + clickhouseprofileexporter: + dsn: clickhouse://localhost:9000/qryn service: - extensions: [pprof, zpages, health_check] pipelines: logs: - receivers: [fluentforward, otlp] - processors: [memory_limiter, resourcedetection/system, resource, batch] - exporters: [qryn] - traces: - receivers: [otlp, jaeger, zipkin] - processors: [memory_limiter, resourcedetection/system, resource, spanmetrics, servicegraph, batch] - exporters: [qryn] - # for align with https://grafana.com/docs/tempo/latest/metrics-generator/span_metrics/#how-to-run - metrics/spanmetrics: - receivers: [otlp] - processors: [metricstransform] - exporters: [qryn] - metrics: - receivers: [prometheus] - processors: [memory_limiter, resourcedetection/system, resource, batch] - exporters: [qryn] + receivers: [pyroscopereceiver] + exporters: [clickhouseprofileexporter] ``` -?> _That's it!_ You're now _continuous profiling to **qryn** using OTLP Collector! +?> _That's it!_ You're now ready to ingest _continuous profiling into **qryn** using OTLP Collector!