diff --git a/add_new_dc_test.py b/add_new_dc_test.py index dbb2d1d510..b8a4a7bc8e 100644 --- a/add_new_dc_test.py +++ b/add_new_dc_test.py @@ -125,34 +125,65 @@ def test_add_new_dc_with_zero_nodes(self): # check that raft quorum is not lost new_node.raft.call_read_barrier() # restore dc1 - new_node.run_nodetool( - sub_cmd=f"removenode {node_host_ids[0]} --ignore-dead-nodes {','.join(node_host_ids[1:])}") - self.replace_cluster_node(new_node, - node_host_ids[1], - nodes_to_region[target_dc_name][-1].dc_idx, - dead_node_hostids=node_host_ids[2]) - - self.replace_cluster_node(new_node, - node_host_ids[2], - nodes_to_region[target_dc_name][-1].dc_idx) - - # bootstrap new node in 1st dc - new_data_node = self.add_node_in_new_dc(nodes_to_region[target_dc_name][-1].dc_idx, 3) - for node in node_for_termination: - self.db_cluster.terminate_node(node) - - self.db_cluster.wait_all_nodes_un() - status = self.db_cluster.get_nodetool_status() - self.log.info("Running rebuild in restored DC") - new_data_node.run_nodetool(sub_cmd=f"rebuild -- {list(status.keys())[-1]}", publish_event=True) - - self.log.info("Running repair on all nodes") - for node in self.db_cluster.nodes: - node.run_nodetool(sub_cmd="repair -pr", publish_event=True) - - self.verify_data_can_be_read_from_new_dc(new_data_node) - self.log.info("Test completed.") + def remove_one_replace_other_nodes_in_DC1(): + new_node.run_nodetool( + sub_cmd=f"removenode {node_host_ids[0]} --ignore-dead-nodes {','.join(node_host_ids[1:])}") + + self.replace_cluster_node(new_node, + node_host_ids[1], + nodes_to_region[target_dc_name][-1].dc_idx, + dead_node_hostids=node_host_ids[2]) + + self.replace_cluster_node(new_node, + node_host_ids[2], + nodes_to_region[target_dc_name][-1].dc_idx) + + # bootstrap new node in 1st dc + new_data_node = self.add_node_in_new_dc(nodes_to_region[target_dc_name][-1].dc_idx, 3) + for node in node_for_termination: + self.db_cluster.terminate_node(node) + + self.db_cluster.wait_all_nodes_un() + status = self.db_cluster.get_nodetool_status() + self.log.info("Running rebuild in restored DC") + new_data_node.run_nodetool(sub_cmd=f"rebuild -- {list(status.keys())[-1]}", publish_event=True) + + self.log.info("Running repair on all nodes") + for node in self.db_cluster.nodes: + node.run_nodetool(sub_cmd="repair -pr", publish_event=True) + + self.verify_data_can_be_read_from_new_dc(new_data_node) + self.log.info("Test completed.") + + def remove_all_add_new_in_DC1(): + # remove all nodes from DC1 + while node_host_ids: + remove_host_id = node_host_ids.pop(0) + if node_host_ids: + dead_nodes_param = f" --ignore-dead-nodes {','.join(node_host_ids)}" + else: + dead_nodes_param = "" + + new_node.run_nodetool( + sub_cmd=f"removenode {remove_host_id}{dead_nodes_param}") + + # bootstrap new node in 1st dc + new_data_node1 = self.add_node_in_new_dc(nodes_to_region[target_dc_name][-1].dc_idx, 1) + new_data_node2 = self.add_node_in_new_dc(nodes_to_region[target_dc_name][-1].dc_idx, 2) + new_data_node3 = self.add_node_in_new_dc(nodes_to_region[target_dc_name][-1].dc_idx, 3) + + for node in node_for_termination: + self.db_cluster.terminate_node(node) + + self.db_cluster.wait_all_nodes_un() + status = self.db_cluster.get_nodetool_status() + self.log.info("Running rebuild in restored DC") + new_data_node1.run_nodetool(sub_cmd=f"rebuild -- {list(status.keys())[-1]}", publish_event=True) + new_data_node2.run_nodetool(sub_cmd=f"rebuild -- {list(status.keys())[-1]}", publish_event=True) + new_data_node3.run_nodetool(sub_cmd=f"rebuild -- {list(status.keys())[-1]}", publish_event=True) + + remove_all_add_new_in_DC1() def reconfigure_keyspaces_to_use_network_topology_strategy(self, keyspaces: List[str], replication_factors: dict[str, int]) -> None: node = self.db_cluster.nodes[0]