From c5c2b701cdf1aee1ee41cbb7540069097690a872 Mon Sep 17 00:00:00 2001
From: Paul Masurel <paul@quickwit.io>
Date: Thu, 16 May 2024 10:53:56 +0900
Subject: [PATCH] The super large grace period of 1 day has proved to be
 harmful on (#4986)

Cicada.

This PR lowers it to 2h.
For reminder, starting the detection of the node as dead,
the node gets into a zombie state for 1h.

We do share its KVs.

From timeofdeath+1h to timeofdeath+2h, we won't share the node.

After 2h, we will delete the node from the state.
---
 quickwit/quickwit-cluster/src/lib.rs | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/quickwit/quickwit-cluster/src/lib.rs b/quickwit/quickwit-cluster/src/lib.rs
index 7174bde9abe..8077d0a229a 100644
--- a/quickwit/quickwit-cluster/src/lib.rs
+++ b/quickwit/quickwit-cluster/src/lib.rs
@@ -28,6 +28,7 @@ mod metrics;
 mod node;
 
 use std::net::SocketAddr;
+use std::time::Duration;
 
 use async_trait::async_trait;
 pub use chitchat::transport::ChannelTransport;
@@ -146,13 +147,17 @@ pub async fn start_cluster_service(node_config: &NodeConfig) -> anyhow::Result<C
         indexing_tasks,
         indexing_cpu_capacity,
     };
+    let failure_detector_config = FailureDetectorConfig {
+        dead_node_grace_period: Duration::from_secs(2 * 60 * 60), // 2 hours
+        ..Default::default()
+    };
     let cluster = Cluster::join(
         cluster_id,
         self_node,
         gossip_listen_addr,
         peer_seed_addrs,
         node_config.gossip_interval,
-        FailureDetectorConfig::default(),
+        failure_detector_config,
         &CountingUdpTransport,
     )
     .await?;