From 544512f3977e693fb05da4ac959d096e7b5abc31 Mon Sep 17 00:00:00 2001 From: Remi Dettai Date: Wed, 18 Sep 2024 10:22:39 +0200 Subject: [PATCH] Clarify what the shutdown needs to wait for --- distribution/ecs/quickwit/quickwit-indexer.tf | 5 +++-- docs/configuration/index-config.md | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/distribution/ecs/quickwit/quickwit-indexer.tf b/distribution/ecs/quickwit/quickwit-indexer.tf index ab3ca76a199..aea74b30aaa 100644 --- a/distribution/ecs/quickwit/quickwit-indexer.tf +++ b/distribution/ecs/quickwit/quickwit-indexer.tf @@ -20,8 +20,9 @@ module "quickwit_indexer" { enable_cloudwatch_logging = var.enable_cloudwatch_logging service_config = var.quickwit_indexer quickwit_index_s3_prefix = local.quickwit_index_s3_prefix - # Longer termination grace period for indexers because their ingest services - # need to commit their WALs. Should be larger than the largest commit timeout. + # Longer termination grace period for indexers because we are waiting for the + # data persisted in the ingesters to be indexed and committed. Should be + # larger than the largest commit timeout. stop_timeout = 120 } diff --git a/docs/configuration/index-config.md b/docs/configuration/index-config.md index 270bddcc482..e7376aaf4bc 100644 --- a/docs/configuration/index-config.md +++ b/docs/configuration/index-config.md @@ -586,9 +586,9 @@ This section describes indexing settings for a given index. :::note -Choosing an appropriate commit timeout is critical. With a shorter commit timeout, ingested data is more quickly queryable. But the published splits will be smaller, increasing the overhead associated with [merges](#merge-policies). +Choosing an appropriate commit timeout is critical. With a shorter commit timeout, ingested data is queryable faster. But the published splits will be smaller, increasing the overhead associated with [merges](#merge-policies). -When decommissioning definitively a indexer node that received data through the ingest API (including the [Elastic bulk API](/docs/reference/es_compatible_api) and the OTEL [log](/docs/log-management/otel-service.md) and [trace](/docs/distributed-tracing/otel-service.md) services), we need to make sure that all the data that was staged locally (Write Ahead Log) is indexed. After receiving the termination signal, the Quickwit process waits for the local indexing pipelines to complete. This can take as long as the longest commit timeout of all indexes. Make sure that the termination grace period of the infrastructure supporting the Quickwit indexer nodes is long enough (e.g [`terminationGracePeriodSeconds`](https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/) in Kubernetes or [`stopTimeout`](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definition_parameters.html) on AWS ECS). +When decommissioning definitively an indexer node that received data through the ingest API (including the [Elastic bulk API](/docs/reference/es_compatible_api) and the OTEL [log](/docs/log-management/otel-service.md) and [trace](/docs/distributed-tracing/otel-service.md) services), we need to make sure that all the data that was persisted locally (Write Ahead Log) is indexed and committed. After receiving the termination signal, the Quickwit process waits for the indexing pipelines to finish processing this local data. This can take as long as the longest commit timeout of all indexes. Make sure that the termination grace period of the infrastructure supporting the Quickwit indexer nodes is long enough (e.g [`terminationGracePeriodSeconds`](https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/) in Kubernetes or [`stopTimeout`](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definition_parameters.html) on AWS ECS). :::