From 86a877050823ebeb5df0a6163780680e5fac8942 Mon Sep 17 00:00:00 2001 From: Remi Dettai Date: Wed, 31 Jul 2024 11:31:18 +0200 Subject: [PATCH] Add docs about situations where messages require a DLQ --- docs/configuration/source-config.md | 5 ++++- .../src/source/queue_sources/coordinator.rs | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/configuration/source-config.md b/docs/configuration/source-config.md index d9f0e374023..83bdace6f96 100644 --- a/docs/configuration/source-config.md +++ b/docs/configuration/source-config.md @@ -78,7 +78,10 @@ EOF - Quickwit does not automatically delete the source files after a successful ingestion. You can use [S3 object expiration](https://docs.aws.amazon.com/AmazonS3/latest/userguide/lifecycle-expire-general-considerations.html) to configure how long they should be retained in the bucket. - Configure the notification to only forward events of type `s3:ObjectCreated:*`. Other events are acknowledged by the source without further processing and an warning is logged. -- We strongly recommend using a [dead letter queue](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-dead-letter-queues.html) to receive all messages that couldn't be processed by the file source. A `maxReceiveCount` of 5 is a good default value. +- We strongly recommend using a [dead letter queue](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-dead-letter-queues.html) to receive all messages that couldn't be processed by the file source. A `maxReceiveCount` of 5 is a good default value. Here are some common situations where the notification message ends up in the dead letter queue: + - the notification message could not be parsed (e.g it is not a valid S3 notification) + - the file was not found + - the file is corrupted (e.g unexpected compression) ::: diff --git a/quickwit/quickwit-indexing/src/source/queue_sources/coordinator.rs b/quickwit/quickwit-indexing/src/source/queue_sources/coordinator.rs index 55212855928..bd00840f657 100644 --- a/quickwit/quickwit-indexing/src/source/queue_sources/coordinator.rs +++ b/quickwit/quickwit-indexing/src/source/queue_sources/coordinator.rs @@ -176,7 +176,7 @@ impl QueueCoordinator { limit_per_min = 10, count = format_errors.len(), last_err = ?format_errors.last().unwrap(), - "invalid message(s) not processed, use a dead letter queue to limit retries" + "invalid messages not processed, use a dead letter queue to limit retries" ); } if preprocessed_messages.is_empty() { @@ -268,7 +268,7 @@ impl QueueCoordinator { rate_limited_error!( limit_per_min = 5, err = ?err, - "failed to start processing message" + "failed to start message processing" ); } }