From 6c2465ddf14d4e07084716d22041148b8f4ad64e Mon Sep 17 00:00:00 2001 From: Jesse S Date: Tue, 17 Oct 2023 12:57:34 -0700 Subject: [PATCH] feat: TOOLS-2700 handle server removal of 'bins' info cmd (#215) --- lib/health/query.py | 1 + lib/live_cluster/get_controller.py | 10 +++--- lib/live_cluster/show_controller.py | 32 ++++++++++++++----- lib/utils/constants.py | 3 +- .../unit/live_cluster/test_show_controller.py | 30 ++++++++++++++++- 5 files changed, 61 insertions(+), 15 deletions(-) diff --git a/lib/health/query.py b/lib/health/query.py index 1d16e6e4..43d32a2c 100644 --- a/lib/health/query.py +++ b/lib/health/query.py @@ -198,6 +198,7 @@ /* NB : ADD CHECKS IF NODES ARE NOT HOMOGENOUS MEM / NUM CPU etc */ +SET CONSTRAINT VERSION < 7.0 s = select "available_bin_names", "available-bin-names" from NAMESPACE save; r = group by NAMESPACE do s > 3200; diff --git a/lib/live_cluster/get_controller.py b/lib/live_cluster/get_controller.py index 504109ce..6f2d87ae 100644 --- a/lib/live_cluster/get_controller.py +++ b/lib/live_cluster/get_controller.py @@ -695,7 +695,7 @@ async def get_bins( bin_stats = await self.cluster.info_bin_statistics(nodes=nodes) new_bin_stats = {} - for node_id, bin_stat in bin_stats.items(): + for node, bin_stat in bin_stats.items(): if not bin_stat or isinstance(bin_stat, Exception): continue @@ -708,9 +708,9 @@ async def get_bins( new_bin_stats[namespace] = {} ns_stats = new_bin_stats[namespace] - if node_id not in ns_stats: - ns_stats[node_id] = {} - node_stats = ns_stats[node_id] + if node not in ns_stats: + ns_stats[node] = {} + node_stats = ns_stats[node] node_stats.update(stats) @@ -838,7 +838,7 @@ class GetClusterMetadataController: def __init__(self, cluster): self.cluster = cluster - async def get_builds(self, nodes="all"): + async def get_builds(self, nodes="all") -> NodeDict[str]: builds = await self.cluster.info_build(nodes=nodes) return util.filter_exceptions(builds) diff --git a/lib/live_cluster/show_controller.py b/lib/live_cluster/show_controller.py index 46dd0078..c696cd75 100644 --- a/lib/live_cluster/show_controller.py +++ b/lib/live_cluster/show_controller.py @@ -16,6 +16,7 @@ from lib.utils import common, util, version, constants from lib.utils.constants import ModifierUsage, Modifiers from lib.live_cluster.get_controller import ( + GetClusterMetadataController, GetConfigController, GetDistributionController, GetJobsController, @@ -892,7 +893,8 @@ async def do_node(self, line): class ShowStatisticsController(LiveClusterCommandController): def __init__(self): self.modifiers = set([Modifiers.WITH, Modifiers.LIKE, Modifiers.FOR]) - self.getter = GetStatisticsController(self.cluster) + self.stat_getter = GetStatisticsController(self.cluster) + self.meta_getter = GetClusterMetadataController(self.cluster) self.controller_map = {"xdr": ShowStatisticsXDRController} @CommandHelp( @@ -945,7 +947,7 @@ async def do_service(self, line): mods=self.mods, ) - service_stats = await self.getter.get_service(nodes=self.nodes) + service_stats = await self.stat_getter.get_service(nodes=self.nodes) return util.callable( self.view.show_stats, @@ -998,7 +1000,7 @@ async def do_namespace(self, line): mods=self.mods, ) - ns_stats = await self.getter.get_namespace( + ns_stats = await self.stat_getter.get_namespace( flip=True, nodes=self.nodes, for_mods=self.mods[Modifiers.FOR] ) @@ -1059,7 +1061,7 @@ async def do_sindex(self, line): mods=self.mods, ) - sindex_stats = await self.getter.get_sindex( + sindex_stats = await self.stat_getter.get_sindex( flip=True, nodes=self.nodes, for_mods=self.mods[Modifiers.FOR] ) @@ -1120,7 +1122,7 @@ async def do_sets(self, line): mods=self.mods, ) - set_stats = await self.getter.get_sets( + set_stats = await self.stat_getter.get_sets( flip=True, nodes=self.nodes, for_mods=self.mods[Modifiers.FOR] ) @@ -1178,10 +1180,24 @@ async def do_bins(self, line): mods=self.mods, ) - new_bin_stats = await self.getter.get_bins( - flip=True, nodes=self.nodes, for_mods=self.mods[Modifiers.FOR] + new_bin_stats, builds = await asyncio.gather( + self.stat_getter.get_bins( + flip=True, nodes=self.nodes, for_mods=self.mods[Modifiers.FOR] + ), + self.meta_getter.get_builds(nodes=self.nodes), ) + if any( + [ + version.LooseVersion(build) + >= version.LooseVersion(constants.SERVER_INFO_BINS_REMOVAL_VERSION) + for build in builds.values() + ] + ): + self.logger.error( + f"Server version {constants.SERVER_INFO_BINS_REMOVAL_VERSION} removed namespace bin-name limits and statistics." + ) + return [ util.callable( self.view.show_stats, @@ -1237,7 +1253,7 @@ async def do_dc(self, line): mods=self.mods, ) - dc_stats = await self.getter.get_xdr_dcs(nodes=self.nodes) + dc_stats = await self.stat_getter.get_xdr_dcs(nodes=self.nodes) futures = [ util.callable( diff --git a/lib/utils/constants.py b/lib/utils/constants.py index dd1c47cb..9e289aa4 100644 --- a/lib/utils/constants.py +++ b/lib/utils/constants.py @@ -153,6 +153,7 @@ class JobType: # server versions with critical changes # TODO: Change to functions on the node +SERVER_INFO_BINS_REMOVAL_VERSION = "7.0" SERVER_SINDEX_ON_CDT_FIRST_VERSION = "6.1" SERVER_QUERIES_ABORT_ALL_FIRST_VERSION = "6.0" SERVER_39_BYTE_OVERHEAD_FIRST_VERSION = "6.0" @@ -164,7 +165,7 @@ class JobType: SERVER_NEW_XDR5_VERSION = "5.0" SERVER_NEW_HISTOGRAM_FIRST_VERSION = "4.2" -# (inclusive, exclusive) +# [inclusive, exclusive) SERVER_TRUNCATE_NAMESPACE_CMD_FIRST_VERSIONS = [ ("4.3.1.11", "4.3.2.0"), ("4.4.0.11", "4.4.1.0"), diff --git a/test/unit/live_cluster/test_show_controller.py b/test/unit/live_cluster/test_show_controller.py index a580ef06..6b6818bb 100644 --- a/test/unit/live_cluster/test_show_controller.py +++ b/test/unit/live_cluster/test_show_controller.py @@ -18,6 +18,7 @@ from mock.mock import call from lib.live_cluster.client.cluster import Cluster from lib.live_cluster.get_controller import ( + GetClusterMetadataController, GetConfigController, GetJobsController, GetStatisticsController, @@ -288,9 +289,13 @@ def setUp(self) -> None: warnings.filterwarnings("error", category=PytestUnraisableExceptionWarning) self.controller = ShowStatisticsController() self.cluster_mock = self.controller.cluster = create_autospec(Cluster) - self.getter_mock = self.controller.getter = create_autospec( + self.getter_mock = self.controller.stat_getter = create_autospec( GetStatisticsController ) + self.meta_mock = self.controller.meta_getter = create_autospec( + GetClusterMetadataController + ) + self.logger_mock = patch("lib.base_controller.BaseController.logger").start() self.view_mock = self.controller.view = create_autospec(CliView) self.controller.mods = ( {} @@ -462,6 +467,7 @@ async def test_do_bins(self): "ns3": "stats3", } self.getter_mock.get_bins.return_value = stats + self.meta_mock.get_builds.return_value = {"1.2.3.4": "6.4.0.0"} await self.controller.execute(line) @@ -481,6 +487,28 @@ async def test_do_bins(self): **mods, ) + async def test_do_bins_logs_error(self): + line = [ + "bins", + ] + mods = {"like": [], "with": [], "for": [], "line": []} + stats = { + "ns1": "stats1", + "ns2": "stats2", + "ns3": "stats3", + } + self.getter_mock.get_bins.return_value = stats + self.meta_mock.get_builds.return_value = { + "1.1.1.1": "6.4.0.0", + "2.2.2.2": "7.0.0.0", + } + + await self.controller.execute(line) + + self.logger_mock.error.assert_called_once_with( + "Server version 7.0 removed namespace bin-name limits and statistics." + ) + class ShowStatisticsXDRControllerTest(asynctest.TestCase): def setUp(self) -> None: