From b64e75d3d733e0e7573b31b13cd31a6d3f512512 Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Mon, 19 Feb 2024 11:18:32 +0900 Subject: [PATCH 1/4] Change in chitchat gossiping priority. Following the original paper, chitchat currently first shares nodes with the highest number of stale values. As a side effect, nodes that are not emitting many KVs are gossiped last. In quickwit, under a little bit of load (1000 indexes on 10 indexer), it has a very dire effect. Indexer that reconnect have to gossip the entire cluster state (~10MB) before being able to get any information about the metastore. Knowing at least one node with the metastore service is required for nodes to declare themselves as live. This PR changes the gossip order. It prioritizes nodes for which the node that originated the Syn message has not receives any KV yet. This is identified by the fact that either the node was not part of the digest at all, or the floor_version is equal to 0. The latter (floor_version == 0) should not happen today, but this case is done in preparation for another PR updating heartbeat on Syn. --- chitchat/src/state.rs | 284 ++++++++++++++++++++++++++++-------------- 1 file changed, 193 insertions(+), 91 deletions(-) diff --git a/chitchat/src/state.rs b/chitchat/src/state.rs index 7ceee74..0729991 100644 --- a/chitchat/src/state.rs +++ b/chitchat/src/state.rs @@ -1,3 +1,4 @@ +use std::cmp::Ordering; use std::collections::btree_map::Entry; use std::collections::{BTreeMap, HashSet}; use std::fmt::{Debug, Formatter}; @@ -463,20 +464,79 @@ impl ClusterState { } } +/// Score used to decide which member should be gossiped first. +/// /// Number of stale key-value pairs carried by the node. A key-value is considered stale if its /// local version is higher than the max version of the digest, also called "floor version". -type Staleness = usize; +#[derive(Clone, Copy, Eq, PartialEq)] +struct Staleness { + is_unknown: bool, + num_stale_records: usize, +} + +/// The ord should be considered a "priority". The higher the faster a node's +/// information is gossiped. +impl Ord for Staleness { + fn cmp(&self, other: &Self) -> Ordering { + // Nodes get gossiped in priority. + // Unknown nodes get gossiped first. + self.is_unknown.cmp(&other.is_unknown).then_with(|| { + // Then nodes with the highest number of stale records get higher priority. + self.num_stale_records.cmp(&other.num_stale_records) + }) + } +} + +impl PartialOrd for Staleness { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} /// Sorts the stale nodes in decreasing order of staleness. -#[derive(Debug, Default)] +#[derive(Default)] struct SortedStaleNodes<'a> { stale_nodes: BTreeMap>>, } +/// The `staleness_score` is used to decide which node should be gossiped first. +/// `floor_version` is the version (transmitted in the digest), below which +/// all of the records have already been received. +/// +/// There is no such thing as a KV for version 0. So if `floor_version == 0` +/// it means the node is entirely new. +/// We artificially prioritize those nodes to make sure their membership (in quickwit the service +/// key for instnace) and initial KVs spread rapidly. +/// +/// If no KV is stale, there is nothing to gossip, and we simply return `None`: +/// the node is not a candidate for gossip. +fn staleness_score(node_state: &NodeState, floor_version: u64) -> Option { + let is_unknown = floor_version == 0u64; + let num_stale_kv = if is_unknown { + node_state.num_key_values() + } else { + node_state.stale_key_values(floor_version).count() + }; + // We don't return None if the node is unknown yet because we want to make + // sure the node is added to the neighbors's list. + if !is_unknown && num_stale_kv == 0 { + return None; + } + Some(Staleness { + is_unknown, + num_stale_records: num_stale_kv, + }) +} + impl<'a> SortedStaleNodes<'a> { /// Adds a node to the list of stale nodes. fn insert(&mut self, chitchat_id: &'a ChitchatId, node_state: &'a NodeState) { - let staleness = node_state.num_key_values() + 1; // +1 for the heartbeat. + let Some(staleness) = staleness_score(node_state, 0u64) else { + // The node does not have any stale KV. + return; + }; + let heartbeat = node_state.heartbeat; + let floor_version = 0; let stale_node = StaleNode { chitchat_id, @@ -496,23 +556,20 @@ impl<'a> SortedStaleNodes<'a> { node_state: &'a NodeState, node_digest: &NodeDigest, ) { - let heartbeat: Heartbeat = node_digest.heartbeat.max(node_state.heartbeat); let floor_version = node_digest.max_version; - let mut staleness = node_state.stale_key_values(floor_version).count(); - if heartbeat > node_digest.heartbeat { - staleness += 1; - } - if staleness > 0 { - let stale_node = StaleNode { - chitchat_id, - node_state, - floor_version, - }; - self.stale_nodes - .entry(staleness) - .or_default() - .push(stale_node); - } + let Some(staleness) = staleness_score(node_state, floor_version) else { + // The node does not have any stale KV. + return; + }; + let stale_node = StaleNode { + chitchat_id, + node_state, + floor_version, + }; + self.stale_nodes + .entry(staleness) + .or_default() + .push(stale_node); } /// Returns an iterator over the stale nodes sorted in decreasing order of staleness. @@ -646,38 +703,40 @@ mod tests { let mut stale_nodes = SortedStaleNodes::default(); let node1 = ChitchatId::for_local_test(10_001); - let node1_state = NodeState::for_test(); - stale_nodes.insert(&node1, &node1_state); + let node2 = ChitchatId::for_local_test(10_002); + let node3 = ChitchatId::for_local_test(10_003); - let expected_staleness = 1; - assert_eq!(stale_nodes.stale_nodes[&expected_staleness].len(), 1); + // No stale KV. We still insert the node! + // That way it will get a node state, and be a candidate for gossip later. + let node_state1 = NodeState::for_test(); + stale_nodes.insert(&node1, &node_state1); + assert_eq!(stale_nodes.stale_nodes.len(), 1); - let node2 = ChitchatId::for_local_test(10_002); let mut node2_state = NodeState::for_test(); node2_state .key_values .insert("key_a".to_string(), VersionedValue::for_test("value_a", 1)); stale_nodes.insert(&node2, &node2_state); - - let expected_staleness = 2; + let expected_staleness = Staleness { + is_unknown: true, + num_stale_records: 1, + }; assert_eq!(stale_nodes.stale_nodes[&expected_staleness].len(), 1); - let node3 = ChitchatId::for_local_test(10_003); let mut node3_state = NodeState::for_test(); node3_state .key_values .insert("key_b".to_string(), VersionedValue::for_test("value_b", 2)); - stale_nodes.insert(&node3, &node3_state); - - let expected_staleness = 2; - assert_eq!(stale_nodes.stale_nodes[&expected_staleness].len(), 2); + node3_state + .key_values + .insert("key_c".to_string(), VersionedValue::for_test("value_c", 3)); - let num_nodes = stale_nodes - .stale_nodes - .values() - .map(|nodes| nodes.len()) - .sum::(); - assert_eq!(num_nodes, 3); + stale_nodes.insert(&node3, &node3_state); + let expected_staleness = Staleness { + is_unknown: true, + num_stale_records: 2, + }; + assert_eq!(stale_nodes.stale_nodes[&expected_staleness].len(), 1); } #[test] @@ -686,75 +745,120 @@ mod tests { let node1 = ChitchatId::for_local_test(10_001); let node1_state = NodeState::for_test(); - stale_nodes.offer(&node1, &node1_state, &NodeDigest::new(Heartbeat(0), 0)); - - assert_eq!(stale_nodes.stale_nodes.len(), 0); + stale_nodes.offer(&node1, &node1_state, &NodeDigest::new(Heartbeat(0), 1)); + // No stale records. This is not a candidate for gossip. + assert!(stale_nodes.stale_nodes.is_empty()); let node2 = ChitchatId::for_local_test(10_002); let mut node2_state = NodeState::for_test(); - node2_state.heartbeat = Heartbeat(1); - stale_nodes.offer(&node2, &node2_state, &NodeDigest::new(Heartbeat(0), 0)); - - let expected_staleness = 1; - assert_eq!(stale_nodes.stale_nodes[&expected_staleness].len(), 1); + node2_state + .key_values + .insert("key_a".to_string(), VersionedValue::for_test("value_a", 1)); + stale_nodes.offer(&node2, &node2_state, &NodeDigest::new(Heartbeat(0), 1)); + // No stale records (due to the floor version). This is not a candidate for gossip. + assert!(stale_nodes.stale_nodes.is_empty()); - let node3 = ChitchatId::for_local_test(10_003); + let node3 = ChitchatId::for_local_test(10_002); let mut node3_state = NodeState::for_test(); node3_state .key_values .insert("key_a".to_string(), VersionedValue::for_test("value_a", 1)); - stale_nodes.offer(&node3, &node3_state, &NodeDigest::new(Heartbeat(0), 0)); - - let expected_staleness = 1; - assert_eq!(stale_nodes.stale_nodes[&expected_staleness].len(), 2); - - let node4 = ChitchatId::for_local_test(10_004); - let mut node4_state = NodeState::for_test(); - node4_state.heartbeat = Heartbeat(1); - node4_state + node3_state .key_values - .insert("key_a".to_string(), VersionedValue::for_test("value_a", 1)); - stale_nodes.offer(&node4, &node4_state, &NodeDigest::new(Heartbeat(0), 0)); - - let expected_staleness = 2; + .insert("key_b".to_string(), VersionedValue::for_test("value_b", 2)); + node3_state + .key_values + .insert("key_c".to_string(), VersionedValue::for_test("value_c", 3)); + stale_nodes.offer(&node3, &node3_state, &NodeDigest::new(Heartbeat(0), 1)); + assert_eq!(stale_nodes.stale_nodes.len(), 1); + let expected_staleness = Staleness { + is_unknown: false, + num_stale_records: 2, + }; assert_eq!(stale_nodes.stale_nodes[&expected_staleness].len(), 1); } #[test] fn test_sorted_stale_nodes_into_iter() { let mut stale_nodes = SortedStaleNodes::default(); - let stale_node1 = StaleNode { - chitchat_id: &ChitchatId::for_local_test(10_001), - node_state: &NodeState::for_test(), - floor_version: 0, - }; - stale_nodes.stale_nodes.insert(1, vec![stale_node1]); + let node1 = ChitchatId::for_local_test(10_001); + let mut node_state1 = NodeState::for_test(); + node_state1 + .key_values + .insert("key_a".to_string(), VersionedValue::for_test("value_a", 1)); + node_state1 + .key_values + .insert("key_b".to_string(), VersionedValue::for_test("value_b", 2)); + node_state1 + .key_values + .insert("key_c".to_string(), VersionedValue::for_test("value_c", 3)); + stale_nodes.offer(&node1, &node_state1, &NodeDigest::new(Heartbeat(0), 1)); + // 2 stale values. - let stale_node2 = StaleNode { - chitchat_id: &ChitchatId::for_local_test(10_002), - node_state: &NodeState::for_test(), - floor_version: 0, - }; - let stale_node3 = StaleNode { - chitchat_id: &ChitchatId::for_local_test(10_003), - node_state: &NodeState::for_test(), - floor_version: 0, - }; - let stale_node4 = StaleNode { - chitchat_id: &ChitchatId::for_local_test(10_004), - node_state: &NodeState::for_test(), - floor_version: 0, - }; - stale_nodes - .stale_nodes - .insert(2, vec![stale_node2, stale_node3, stale_node4]); + let node2 = ChitchatId::for_local_test(10_002); + let mut node_state2 = NodeState::for_test(); + node_state2 + .key_values + .insert("key_a".to_string(), VersionedValue::for_test("value_a", 1)); + node_state2 + .key_values + .insert("key_b".to_string(), VersionedValue::for_test("value_b", 2)); + node_state2 + .key_values + .insert("key_c".to_string(), VersionedValue::for_test("value_c", 5)); + stale_nodes.offer(&node2, &node_state2, &NodeDigest::new(Heartbeat(0), 2)); + // 1 stale value. + + let node3 = ChitchatId::for_local_test(10_003); + let mut node_state3 = NodeState::for_test(); + node_state3 + .key_values + .insert("key_a".to_string(), VersionedValue::for_test("value_a", 1)); + node_state3 + .key_values + .insert("key_b".to_string(), VersionedValue::for_test("value_b", 2)); + node_state3 + .key_values + .insert("key_c".to_string(), VersionedValue::for_test("value_c", 3)); + stale_nodes.offer(&node3, &node_state3, &NodeDigest::new(Heartbeat(0), 7)); + // 0 stale values. + + let node4 = ChitchatId::for_local_test(10_004); + let mut node_state4 = NodeState::for_test(); + node_state4 + .key_values + .insert("key_a".to_string(), VersionedValue::for_test("value_a", 1)); + node_state4 + .key_values + .insert("key_b".to_string(), VersionedValue::for_test("value_b", 2)); + node_state4 + .key_values + .insert("key_c".to_string(), VersionedValue::for_test("value_c", 5)); + node_state4 + .key_values + .insert("key_d".to_string(), VersionedValue::for_test("value_d", 7)); + stale_nodes.offer(&node4, &node_state4, &NodeDigest::new(Heartbeat(0), 1)); + + // 3 stale values + let node5 = ChitchatId::for_local_test(10_005); + let node_state5 = NodeState::for_test(); + stale_nodes.insert(&node5, &node_state5); + + // 0 stale values + let node6 = ChitchatId::for_local_test(10_006); + let mut node_state6 = NodeState::for_test(); + node_state6 + .key_values + .insert("key_a".to_string(), VersionedValue::for_test("value_a", 1)); + stale_nodes.insert(&node6, &node_state6); + // 1 stale values assert_eq!( stale_nodes .into_iter() .map(|stale_node| stale_node.chitchat_id.gossip_advertise_addr.port()) .collect::>(), - vec![10_003, 10_002, 10_004, 10_001] + vec![10_006, 10_005, 10_004, 10_001, 10_002] ); } @@ -1155,7 +1259,6 @@ mod tests { { let mut digest = Digest::default(); - let node1 = ChitchatId::for_local_test(10_001); digest.add_node(node1.clone(), Heartbeat(0), 1); let delta = cluster_state.compute_partial_delta_respecting_mtu( &digest, @@ -1164,10 +1267,10 @@ mod tests { ); assert!(delta.nodes_to_reset.is_empty()); let mut expected_delta = Delta::default(); - expected_delta.add_node(node1.clone(), 0u64); - expected_delta.add_kv(&node1, "key_b", "2", 2, false); expected_delta.add_node(node2.clone(), 0u64); expected_delta.add_kv(&node2.clone(), "key_c", "3", 2, false); + expected_delta.add_node(node1.clone(), 0u64); + expected_delta.add_kv(&node1, "key_b", "2", 2, false); expected_delta.set_serialized_len(73); assert_eq!(delta, expected_delta); } @@ -1189,12 +1292,12 @@ mod tests { ); assert!(delta.nodes_to_reset.is_empty()); let mut expected_delta = Delta::default(); + expected_delta.add_node(node2.clone(), 0u64); + expected_delta.add_kv(&node2.clone(), "key_c", "3", 2, false); expected_delta.add_node(node1.clone(), 0u64); expected_delta.add_kv(&node1, "key_b", "2", 2, false); expected_delta.add_kv(&node1, "key_a", "", 3, true); - expected_delta.add_node(node2.clone(), 0u64); - expected_delta.add_kv(&node2.clone(), "key_c", "3", 2, false); - expected_delta.set_serialized_len(83); + expected_delta.set_serialized_len(87); assert_eq!(delta, expected_delta); } @@ -1220,7 +1323,6 @@ mod tests { expected_delta.add_node(node2.clone(), 0u64); expected_delta.add_kv(&node2.clone(), "key_c", "3", 2, false); expected_delta.set_serialized_len(80); - assert_eq!(delta, expected_delta); } } From 5c935e9399febaccc7be649b73cdf54519cddbf5 Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Fri, 23 Feb 2024 11:40:10 +0900 Subject: [PATCH 2/4] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrien "Code Monkey" Guillo Co-authored-by: François Massot --- chitchat/src/state.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/chitchat/src/state.rs b/chitchat/src/state.rs index 0729991..9d575d7 100644 --- a/chitchat/src/state.rs +++ b/chitchat/src/state.rs @@ -471,14 +471,14 @@ impl ClusterState { #[derive(Clone, Copy, Eq, PartialEq)] struct Staleness { is_unknown: bool, - num_stale_records: usize, + num_stale_key_values: usize, } -/// The ord should be considered a "priority". The higher the faster a node's +/// The ord should be considered a "priority". The higher, the faster a node's /// information is gossiped. impl Ord for Staleness { fn cmp(&self, other: &Self) -> Ordering { - // Nodes get gossiped in priority. + // Nodes are gossiped in order of priority: // Unknown nodes get gossiped first. self.is_unknown.cmp(&other.is_unknown).then_with(|| { // Then nodes with the highest number of stale records get higher priority. @@ -501,12 +501,12 @@ struct SortedStaleNodes<'a> { /// The `staleness_score` is used to decide which node should be gossiped first. /// `floor_version` is the version (transmitted in the digest), below which -/// all of the records have already been received. +/// all the records have already been received. /// -/// There is no such thing as a KV for version 0. So if `floor_version == 0` +/// There is no such thing as a KV for version 0. So if `floor_version == 0`, /// it means the node is entirely new. /// We artificially prioritize those nodes to make sure their membership (in quickwit the service -/// key for instnace) and initial KVs spread rapidly. +/// key for instance) and initial KVs spread rapidly. /// /// If no KV is stale, there is nothing to gossip, and we simply return `None`: /// the node is not a candidate for gossip. From 257f81e2df2712869eaa2876d473c00fa002c5e8 Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Fri, 23 Feb 2024 11:42:04 +0900 Subject: [PATCH 3/4] Removing unhelpful comment --- chitchat/src/state.rs | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/chitchat/src/state.rs b/chitchat/src/state.rs index 9d575d7..17fe6d0 100644 --- a/chitchat/src/state.rs +++ b/chitchat/src/state.rs @@ -482,7 +482,7 @@ impl Ord for Staleness { // Unknown nodes get gossiped first. self.is_unknown.cmp(&other.is_unknown).then_with(|| { // Then nodes with the highest number of stale records get higher priority. - self.num_stale_records.cmp(&other.num_stale_records) + self.num_stale_key_values.cmp(&other.num_stale_key_values) }) } } @@ -512,19 +512,17 @@ struct SortedStaleNodes<'a> { /// the node is not a candidate for gossip. fn staleness_score(node_state: &NodeState, floor_version: u64) -> Option { let is_unknown = floor_version == 0u64; - let num_stale_kv = if is_unknown { + let num_stale_key_values = if is_unknown { node_state.num_key_values() } else { node_state.stale_key_values(floor_version).count() }; - // We don't return None if the node is unknown yet because we want to make - // sure the node is added to the neighbors's list. - if !is_unknown && num_stale_kv == 0 { + if !is_unknown && num_stale_key_values == 0 { return None; } Some(Staleness { is_unknown, - num_stale_records: num_stale_kv, + num_stale_key_values, }) } @@ -719,7 +717,7 @@ mod tests { stale_nodes.insert(&node2, &node2_state); let expected_staleness = Staleness { is_unknown: true, - num_stale_records: 1, + num_stale_key_values: 1, }; assert_eq!(stale_nodes.stale_nodes[&expected_staleness].len(), 1); @@ -734,7 +732,7 @@ mod tests { stale_nodes.insert(&node3, &node3_state); let expected_staleness = Staleness { is_unknown: true, - num_stale_records: 2, + num_stale_key_values: 2, }; assert_eq!(stale_nodes.stale_nodes[&expected_staleness].len(), 1); } @@ -773,7 +771,7 @@ mod tests { assert_eq!(stale_nodes.stale_nodes.len(), 1); let expected_staleness = Staleness { is_unknown: false, - num_stale_records: 2, + num_stale_key_values: 2, }; assert_eq!(stale_nodes.stale_nodes[&expected_staleness].len(), 1); } From 728e715607304c2625e34919e83f92faccedd537 Mon Sep 17 00:00:00 2001 From: Paul Masurel Date: Fri, 23 Feb 2024 12:06:37 +0900 Subject: [PATCH 4/4] Updating policy --- chitchat/src/state.rs | 57 +++++++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/chitchat/src/state.rs b/chitchat/src/state.rs index 17fe6d0..032b5b4 100644 --- a/chitchat/src/state.rs +++ b/chitchat/src/state.rs @@ -251,12 +251,12 @@ impl NodeState { } } - fn set_with_version(&mut self, key: String, value: String, version: Version) { + fn set_with_version(&mut self, key: impl ToString, value: impl ToString, version: Version) { assert!(version > self.max_version); self.set_versioned_value( - key, + key.to_string(), VersionedValue { - value, + value: value.to_string(), version, tombstone: None, }, @@ -468,9 +468,10 @@ impl ClusterState { /// /// Number of stale key-value pairs carried by the node. A key-value is considered stale if its /// local version is higher than the max version of the digest, also called "floor version". -#[derive(Clone, Copy, Eq, PartialEq)] +#[derive(Clone, Copy, Eq, PartialEq, Debug)] struct Staleness { is_unknown: bool, + max_version: u64, num_stale_key_values: usize, } @@ -478,11 +479,21 @@ struct Staleness { /// information is gossiped. impl Ord for Staleness { fn cmp(&self, other: &Self) -> Ordering { - // Nodes are gossiped in order of priority: + // Nodes get gossiped in priority. // Unknown nodes get gossiped first. + // If several nodes are unknown, the one with the lowest max_version gets gossiped first. + // This is a bit of a hack to make sure we know about the metastore + // as soon as possible in quickwit, even when the indexer's chitchat state is bloated. + // + // Within known nodes, the one with the highest number of stale records gets gossiped first, + // as described in the scuttlebutt paper. self.is_unknown.cmp(&other.is_unknown).then_with(|| { - // Then nodes with the highest number of stale records get higher priority. - self.num_stale_key_values.cmp(&other.num_stale_key_values) + if self.is_unknown { + self.max_version.cmp(&other.max_version).reverse() + } else { + // Then nodes with the highest number of stale records get higher priority. + self.num_stale_key_values.cmp(&other.num_stale_key_values) + } }) } } @@ -522,6 +533,7 @@ fn staleness_score(node_state: &NodeState, floor_version: u64) -> Option SortedStaleNodes<'a> { /// Adds a node to the list of stale nodes. fn insert(&mut self, chitchat_id: &'a ChitchatId, node_state: &'a NodeState) { let Some(staleness) = staleness_score(node_state, 0u64) else { - // The node does not have any stale KV. + // The node does not have any stale key values. return; }; - let heartbeat = node_state.heartbeat; let floor_version = 0; let stale_node = StaleNode { @@ -711,28 +722,24 @@ mod tests { assert_eq!(stale_nodes.stale_nodes.len(), 1); let mut node2_state = NodeState::for_test(); - node2_state - .key_values - .insert("key_a".to_string(), VersionedValue::for_test("value_a", 1)); + node2_state.set_with_version("key_a", "value_a", 1); stale_nodes.insert(&node2, &node2_state); let expected_staleness = Staleness { is_unknown: true, - num_stale_key_values: 1, + max_version: 0, + num_stale_key_values: 0, }; assert_eq!(stale_nodes.stale_nodes[&expected_staleness].len(), 1); let mut node3_state = NodeState::for_test(); - node3_state - .key_values - .insert("key_b".to_string(), VersionedValue::for_test("value_b", 2)); - node3_state - .key_values - .insert("key_c".to_string(), VersionedValue::for_test("value_c", 3)); + node3_state.set_with_version("key_b", "value_b", 2); + node3_state.set_with_version("key_c", "value_c", 3); stale_nodes.insert(&node3, &node3_state); let expected_staleness = Staleness { is_unknown: true, - num_stale_key_values: 2, + max_version: 3, + num_stale_key_values: 3, }; assert_eq!(stale_nodes.stale_nodes[&expected_staleness].len(), 1); } @@ -771,6 +778,7 @@ mod tests { assert_eq!(stale_nodes.stale_nodes.len(), 1); let expected_staleness = Staleness { is_unknown: false, + max_version: 1, num_stale_key_values: 2, }; assert_eq!(stale_nodes.stale_nodes[&expected_staleness].len(), 1); @@ -856,7 +864,7 @@ mod tests { .into_iter() .map(|stale_node| stale_node.chitchat_id.gossip_advertise_addr.port()) .collect::>(), - vec![10_006, 10_005, 10_004, 10_001, 10_002] + vec![10_005, 10_006, 10_004, 10_001, 10_002] ); } @@ -1315,12 +1323,13 @@ mod tests { &HashSet::new(), ); let mut expected_delta = Delta::default(); + expected_delta.add_node(node2.clone(), 0u64); + expected_delta.add_kv(&node2.clone(), "key_c", "3", 2, false); expected_delta.add_node_to_reset(node1.clone()); expected_delta.add_node(node1.clone(), 3u64); expected_delta.add_kv(&node1, "key_b", "2", 2, false); - expected_delta.add_node(node2.clone(), 0u64); - expected_delta.add_kv(&node2.clone(), "key_c", "3", 2, false); - expected_delta.set_serialized_len(80); + expected_delta.set_serialized_len(83); + assert_eq!(&delta, &expected_delta); } }