From b9bf35f4b3b2a46c34da78d4059af1d7ad72bd36 Mon Sep 17 00:00:00 2001 From: Lawrence Jones Date: Fri, 6 Sep 2019 12:16:45 +0100 Subject: [PATCH] Timeout metric queries Ensure we never hurt the database with metric queries if they begin to take longer than 1s to execute. --- lib/que/middleware/queue_collector.rb | 34 ++++++++++++++++----------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/lib/que/middleware/queue_collector.rb b/lib/que/middleware/queue_collector.rb index 49ba52ac..cb8c7062 100644 --- a/lib/que/middleware/queue_collector.rb +++ b/lib/que/middleware/queue_collector.rb @@ -53,21 +53,27 @@ def call(env) # report metric values that are current in every scrape. Queued.values.each { |labels, _| Queued.set(0.0, labels: labels) } - # Now we can safely update our gauges, touching only those that exist in our queue - Que.execute(QUEUE_VIEW_SQL).each do |labels| - Queued.set( - labels["count"], - labels: { - queue: labels["queue"], - job_class: labels["job_class"], - priority: labels["priority"], - due: labels["due"], - }, - ) - end + Que.transaction do + # Ensure metric queries never take more than 500ms to execute, preventing our + # metric collector from hurting the database when it's already under pressure. + Que.execute("set local statement_timeout='500ms';") + + # Now we can safely update our gauges, touching only those that exist in our queue + Que.execute(QUEUE_VIEW_SQL).each do |labels| + Queued.set( + labels["count"], + labels: { + queue: labels["queue"], + job_class: labels["job_class"], + priority: labels["priority"], + due: labels["due"], + }, + ) + end - # DeadTuples has no labels, we can expect this to be a single numeric value - DeadTuples.set(Que.execute(DEAD_TUPLES_SQL).first&.fetch("n_dead_tup")) + # DeadTuples has no labels, we can expect this to be a single numeric value + DeadTuples.set(Que.execute(DEAD_TUPLES_SQL).first&.fetch("n_dead_tup")) + end @app.call(env) end