From d676951a01a108293657dc09cc9bf3303d3713f9 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Fri, 17 May 2024 15:39:44 +0200 Subject: [PATCH 1/2] ref(rust): Backpressure metrics for threadpools It's hard to tell which strategy is the bottleneck, but we have a lot of different threadpools in the snuba rust consumers who all could be it. Emit backpressure metrics for both getting and receiving backpressure, it will help figuring out which strategy causes the most backpressure. --- rust-arroyo/src/processing/strategies/run_task_in_threads.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rust-arroyo/src/processing/strategies/run_task_in_threads.rs b/rust-arroyo/src/processing/strategies/run_task_in_threads.rs index d48a959b..97bf6770 100644 --- a/rust-arroyo/src/processing/strategies/run_task_in_threads.rs +++ b/rust-arroyo/src/processing/strategies/run_task_in_threads.rs @@ -12,7 +12,7 @@ use crate::processing::strategies::{ }; use crate::types::Message; use crate::utils::timing::Deadline; -use crate::{gauge, timer}; +use crate::{gauge, timer, counter}; use super::StrategyError; @@ -142,6 +142,7 @@ where Err(SubmitError::MessageRejected(MessageRejected { message: transformed_message, })) => { + counter!("arroyo.strategies.run_task_in_threads.got_backpressure", 1, "strategy_name" => self.metric_strategy_name); self.message_carried_over = Some(transformed_message); } Err(SubmitError::InvalidMessage(invalid_message)) => { @@ -190,6 +191,7 @@ where fn submit(&mut self, message: Message) -> Result<(), SubmitError> { if self.message_carried_over.is_some() { + counter!("arroyo.strategies.run_task_in_threads.giving_backpressure", 1, "strategy_name" => self.metric_strategy_name); return Err(SubmitError::MessageRejected(MessageRejected { message })); } From 513182aa9c3da3635266048baf507e3daeed3642 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Fri, 17 May 2024 15:43:12 +0200 Subject: [PATCH 2/2] fix formatting --- rust-arroyo/src/processing/strategies/run_task_in_threads.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-arroyo/src/processing/strategies/run_task_in_threads.rs b/rust-arroyo/src/processing/strategies/run_task_in_threads.rs index 97bf6770..b3130489 100644 --- a/rust-arroyo/src/processing/strategies/run_task_in_threads.rs +++ b/rust-arroyo/src/processing/strategies/run_task_in_threads.rs @@ -12,7 +12,7 @@ use crate::processing::strategies::{ }; use crate::types::Message; use crate::utils::timing::Deadline; -use crate::{gauge, timer, counter}; +use crate::{counter, gauge, timer}; use super::StrategyError;