From da581fa947a17fd6e1af8005a086ae476d94d5a4 Mon Sep 17 00:00:00 2001 From: Chris Elias Date: Wed, 1 May 2024 10:28:31 -0400 Subject: [PATCH 1/4] make tempo config tweaks for better performance --- monitoring/values-tempo.yaml | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/monitoring/values-tempo.yaml b/monitoring/values-tempo.yaml index 78fdb710..ac79c16f 100644 --- a/monitoring/values-tempo.yaml +++ b/monitoring/values-tempo.yaml @@ -22,10 +22,27 @@ tempo: enabled: true remoteWriteUrl: "http://v4m-prometheus.monitoring:9090/api/v1/write" - # global_overrides: - # metrics_generator_processors: - # - service-graphs + global_overrides: + # strategy: local + # limit_bytes: 15000000 + max_traces_per_user: 5000000 + ingestion_burst_size_bytes: 500000000 + ingestion_rate_limit_bytes: 500000000 + # metrics_generator_processors: + # - service-graphs + # - span-metrics + # overrides: + # defaults: + # ingestion: + # strategy: local + # limit_bytes: 15000000 + # max_traces_per_user: 10000 + # ingestion_burst_size_bytes: 100000000 + # ingestion_rate_limit_bytes: 50000000 + # metrics_generator_processors: + # - service-graphs + # - span-metrics retention: 24h # Tempo server configuration @@ -33,6 +50,19 @@ tempo: server: # -- HTTP server listen port http_listen_port: 3100 + grpc_server_max_recv_msg_size: 500000000 + grpc_server_max_send_msg_size: 500000000 + + querier: + frontend_worker: + grpc_client_config: + max_send_msg_size: 500000000 + trace_by_id: + query_timeout: "60s" + search: + query_timeout: "120s" + max_concurrent_queries: 50 + storage: trace: # tempo storage backend From 14fca25e2a124da433df6956fa5c48c3c07a5fd3 Mon Sep 17 00:00:00 2001 From: Chris Elias Date: Wed, 1 May 2024 10:29:31 -0400 Subject: [PATCH 2/4] cleanup --- monitoring/values-tempo.yaml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/monitoring/values-tempo.yaml b/monitoring/values-tempo.yaml index ac79c16f..0181da88 100644 --- a/monitoring/values-tempo.yaml +++ b/monitoring/values-tempo.yaml @@ -23,26 +23,10 @@ tempo: remoteWriteUrl: "http://v4m-prometheus.monitoring:9090/api/v1/write" global_overrides: - # strategy: local - # limit_bytes: 15000000 max_traces_per_user: 5000000 ingestion_burst_size_bytes: 500000000 ingestion_rate_limit_bytes: 500000000 - # metrics_generator_processors: - # - service-graphs - # - span-metrics - # overrides: - # defaults: - # ingestion: - # strategy: local - # limit_bytes: 15000000 - # max_traces_per_user: 10000 - # ingestion_burst_size_bytes: 100000000 - # ingestion_rate_limit_bytes: 50000000 - # metrics_generator_processors: - # - service-graphs - # - span-metrics retention: 24h # Tempo server configuration From a2c6aa39738e28e41b40df9a06a7e2334646367a Mon Sep 17 00:00:00 2001 From: Chris Elias Date: Wed, 1 May 2024 10:28:31 -0400 Subject: [PATCH 3/4] make tempo config tweaks for better performance --- monitoring/values-tempo.yaml | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/monitoring/values-tempo.yaml b/monitoring/values-tempo.yaml index 78fdb710..ac79c16f 100644 --- a/monitoring/values-tempo.yaml +++ b/monitoring/values-tempo.yaml @@ -22,10 +22,27 @@ tempo: enabled: true remoteWriteUrl: "http://v4m-prometheus.monitoring:9090/api/v1/write" - # global_overrides: - # metrics_generator_processors: - # - service-graphs + global_overrides: + # strategy: local + # limit_bytes: 15000000 + max_traces_per_user: 5000000 + ingestion_burst_size_bytes: 500000000 + ingestion_rate_limit_bytes: 500000000 + # metrics_generator_processors: + # - service-graphs + # - span-metrics + # overrides: + # defaults: + # ingestion: + # strategy: local + # limit_bytes: 15000000 + # max_traces_per_user: 10000 + # ingestion_burst_size_bytes: 100000000 + # ingestion_rate_limit_bytes: 50000000 + # metrics_generator_processors: + # - service-graphs + # - span-metrics retention: 24h # Tempo server configuration @@ -33,6 +50,19 @@ tempo: server: # -- HTTP server listen port http_listen_port: 3100 + grpc_server_max_recv_msg_size: 500000000 + grpc_server_max_send_msg_size: 500000000 + + querier: + frontend_worker: + grpc_client_config: + max_send_msg_size: 500000000 + trace_by_id: + query_timeout: "60s" + search: + query_timeout: "120s" + max_concurrent_queries: 50 + storage: trace: # tempo storage backend From ffbc269648ca3f758e332921611c9614b4da01a2 Mon Sep 17 00:00:00 2001 From: Chris Elias Date: Wed, 1 May 2024 10:29:31 -0400 Subject: [PATCH 4/4] cleanup --- monitoring/values-tempo.yaml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/monitoring/values-tempo.yaml b/monitoring/values-tempo.yaml index ac79c16f..0181da88 100644 --- a/monitoring/values-tempo.yaml +++ b/monitoring/values-tempo.yaml @@ -23,26 +23,10 @@ tempo: remoteWriteUrl: "http://v4m-prometheus.monitoring:9090/api/v1/write" global_overrides: - # strategy: local - # limit_bytes: 15000000 max_traces_per_user: 5000000 ingestion_burst_size_bytes: 500000000 ingestion_rate_limit_bytes: 500000000 - # metrics_generator_processors: - # - service-graphs - # - span-metrics - # overrides: - # defaults: - # ingestion: - # strategy: local - # limit_bytes: 15000000 - # max_traces_per_user: 10000 - # ingestion_burst_size_bytes: 100000000 - # ingestion_rate_limit_bytes: 50000000 - # metrics_generator_processors: - # - service-graphs - # - span-metrics retention: 24h # Tempo server configuration