From 2e087237ec0f8f5b8b17ca07faee7bdb1578fd1f Mon Sep 17 00:00:00 2001 From: vivekverma Date: Wed, 7 Feb 2024 03:15:14 -0800 Subject: [PATCH 1/3] Added CLI support for aggregate VOQ counters --- scripts/queuestat | 86 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 80 insertions(+), 6 deletions(-) diff --git a/scripts/queuestat b/scripts/queuestat index 8f95554481..1f4c909096 100755 --- a/scripts/queuestat +++ b/scripts/queuestat @@ -11,11 +11,14 @@ import argparse import datetime import os.path import sys +import operator as op +import re from collections import namedtuple, OrderedDict from natsort import natsorted from tabulate import tabulate -from sonic_py_common import multi_asic +from sonic_py_common import multi_asic, device_info +from swsscommon import swsscommon # mock the redis for unit test purposes # try: @@ -61,6 +64,7 @@ SAI_QUEUE_TYPE_UNICAST_VOQ = "SAI_QUEUE_TYPE_UNICAST_VOQ" SAI_QUEUE_TYPE_ALL = "SAI_QUEUE_TYPE_ALL" COUNTER_TABLE_PREFIX = "COUNTERS:" +VOQ_COUNTERS_PREFIX = "COUNTERS|" COUNTERS_PORT_NAME_MAP = "COUNTERS_PORT_NAME_MAP" COUNTERS_SYSTEM_PORT_NAME_MAP = "COUNTERS_SYSTEM_PORT_NAME_MAP" COUNTERS_QUEUE_NAME_MAP = "COUNTERS_QUEUE_NAME_MAP" @@ -72,6 +76,28 @@ COUNTERS_QUEUE_PORT_MAP = "COUNTERS_QUEUE_PORT_MAP" cnstat_dir = 'N/A' cnstat_fqn_file = 'N/A' +class voqCounterUtil(): + @staticmethod + def parse_key(key): + # COUNTERS|nfc406-7|Asic0|Ethernet164@nfc406-9|Asic0:3 + _, sysPort, fapId, voqIdx = re.search(r"(COUNTERS).([a-zA-Z0-9\-\|]+).([a-zA-Z0-9\-\|]+).(\d)",key).groups() + return sysPort, fapId, voqIdx + # sysPort = nfc406-7|Asic0|Ethernet164 + # fapId = nfc406-9:Asic0 + + @staticmethod + def get_key(sysPort, fapId, voqIdx): + return VOQ_COUNTERS_PREFIX + sysPort + "@" + fapId + ":" + voqIdx + + @staticmethod + def process_keys(keys): + metaData = {} + ports = set() + for key in keys: + sysPort, fapId, voqIdx = voqCounterUtil.parse_key(key) + ports.add(sysPort) + metaData.setdefault(sysPort,{}).setdefault(fapId,set()).add(voqIdx) + return metaData, ports def build_json(port, cnstat): def ports_stats(k): @@ -99,7 +125,10 @@ class Queuestat(object): self.db = multi_asic.connect_to_all_dbs_for_ns(ns) else: self.db = SonicV2Connector(use_unix_socket_path=False) - self.db.connect(self.db.COUNTERS_DB) + if voq and device_info.is_supervisor(): + self.db.connect(self.db.CHASSIS_APP_DB) + else: + self.db.connect(self.db.COUNTERS_DB) self.voq = voq def get_queue_port(table_id): @@ -111,7 +140,10 @@ class Queuestat(object): return port_table_id # Get all ports - if voq: + if voq and device_info.is_supervisor(): + # counter_port_name_map is assigned later for SSI as a list + self.counter_port_name_map = [] + elif voq: self.counter_port_name_map = self.db.get_all(self.db.COUNTERS_DB, COUNTERS_SYSTEM_PORT_NAME_MAP) else: self.counter_port_name_map = self.db.get_all(self.db.COUNTERS_DB, COUNTERS_PORT_NAME_MAP) @@ -122,12 +154,21 @@ class Queuestat(object): self.port_queues_map = {} self.port_name_map = {} + self.ports = {} + self.voqIdxs = {} + self.metaData = {} for port in self.counter_port_name_map: self.port_queues_map[port] = {} self.port_name_map[self.counter_port_name_map[port]] = port counter_queue_name_map = None + if voq and device_info.is_supervisor(): + keys = self.db.keys(self.db.CHASSIS_APP_DB, VOQ_COUNTERS_PREFIX + "*") + self.metaData, self.ports = voqCounterUtil.process_keys(keys) + self.counter_port_name_map = self.ports + return + # Get Queues for each port if voq: counter_queue_name_map = self.db.get_all(self.db.COUNTERS_DB, COUNTERS_VOQ_NAME_MAP) @@ -142,6 +183,31 @@ class Queuestat(object): port = self.port_name_map[get_queue_port(counter_queue_name_map[queue])] self.port_queues_map[port][queue] = counter_queue_name_map[queue] + def getAggregatePortStats( self, port ): + voqCnt = {} + + for fapId in self.metaData[port]: + for idx in self.metaData[port][fapId]: + key = voqCounterUtil.get_key(port,fapId,idx) + for counter_name in counter_bucket_dict: + voqCnt.setdefault(idx,{}).setdefault(counter_name,0) + counter_data = self.db.get(self.db.CHASSIS_APP_DB, key, counter_name) + if counter_data is not None: + voqCnt[idx][counter_name] += int(counter_data) + + # Build a dictionary of stats + cnstat_dict = OrderedDict() + cnstat_dict['time'] = datetime.datetime.now() + for idx in sorted(voqCnt.keys()): + fields = ["0"]*6 + fields[0] = idx + fields[1] = QUEUE_TYPE_VOQ + for counter_name, pos in counter_bucket_dict.items(): + fields[pos] = str(voqCnt[idx][counter_name]) + cntr = QueueStats._make(fields)._asdict() + cnstat_dict[port+":"+idx] = cntr + return cnstat_dict + def get_cnstat(self, queue_map): """ Get the counters info from database. @@ -278,7 +344,11 @@ class Queuestat(object): json_output = {} for port in natsorted(self.counter_port_name_map): json_output[port] = {} - cnstat_dict = self.get_cnstat(self.port_queues_map[port]) + if self.voq and device_info.is_supervisor(): + cnstat_dict = self.getAggregatePortStats(port) + + else: + cnstat_dict = self.get_cnstat(self.port_queues_map[port]) cnstat_fqn_file_name = cnstat_fqn_file + port if os.path.isfile(cnstat_fqn_file_name): @@ -305,12 +375,16 @@ class Queuestat(object): Get stat for the port If JSON option is True print data in JSON format """ - if not port in self.port_queues_map: + if not (port in self.port_queues_map or port in self.ports): print("Port doesn't exist!", port) sys.exit(1) # Get stat for the port queried - cnstat_dict = self.get_cnstat(self.port_queues_map[port]) + if self.voq and device_info.is_supervisor(): + cnstat_dict = self.getAggregatePortStats(port) + else: + cnstat_dict = self.get_cnstat(self.port_queues_map[port]) + cnstat_fqn_file_name = cnstat_fqn_file + port json_output = {} json_output[port] = {} From 3fbcb4c0f4aaa3b72d796c241b5af89fddb80c67 Mon Sep 17 00:00:00 2001 From: "vivekverma@arista.com" Date: Wed, 28 Aug 2024 09:37:56 -0700 Subject: [PATCH 2/3] Updated queustat in accordance to the new design --- scripts/queuestat | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/queuestat b/scripts/queuestat index 16112d7af3..6e8fe116f7 100755 --- a/scripts/queuestat +++ b/scripts/queuestat @@ -18,7 +18,6 @@ from collections import namedtuple, OrderedDict from natsort import natsorted from tabulate import tabulate from sonic_py_common import multi_asic, device_info -from swsscommon import swsscommon # mock the redis for unit test purposes # try: @@ -68,7 +67,7 @@ SAI_QUEUE_TYPE_UNICAST_VOQ = "SAI_QUEUE_TYPE_UNICAST_VOQ" SAI_QUEUE_TYPE_ALL = "SAI_QUEUE_TYPE_ALL" COUNTER_TABLE_PREFIX = "COUNTERS:" -VOQ_COUNTERS_PREFIX = "COUNTERS|" +VOQ_COUNTERS_PREFIX = "COUNTERS_VOQ:" COUNTERS_PORT_NAME_MAP = "COUNTERS_PORT_NAME_MAP" COUNTERS_SYSTEM_PORT_NAME_MAP = "COUNTERS_SYSTEM_PORT_NAME_MAP" COUNTERS_QUEUE_NAME_MAP = "COUNTERS_QUEUE_NAME_MAP" @@ -84,7 +83,7 @@ class voqCounterUtil(): @staticmethod def parse_key(key): # COUNTERS|nfc406-7|Asic0|Ethernet164@nfc406-9|Asic0:3 - _, sysPort, fapId, voqIdx = re.search(r"(COUNTERS).([a-zA-Z0-9\-\|]+).([a-zA-Z0-9\-\|]+).(\d)",key).groups() + _, sysPort, fapId, voqIdx = re.search(r"(COUNTERS_VOQ).([a-zA-Z0-9\-\|]+).([a-zA-Z0-9\-\|]+).(\d)",key).groups() return sysPort, fapId, voqIdx # sysPort = nfc406-7|Asic0|Ethernet164 # fapId = nfc406-9:Asic0 @@ -139,7 +138,7 @@ class Queuestat(object): else: self.db = SonicV2Connector(use_unix_socket_path=False) if voq and device_info.is_supervisor(): - self.db.connect(self.db.CHASSIS_APP_DB) + self.db.connect(self.db.CHASSIS_COUNTERS_DB) else: self.db.connect(self.db.COUNTERS_DB) self.voq = voq @@ -175,9 +174,12 @@ class Queuestat(object): self.port_queues_map[port] = {} self.port_name_map[self.counter_port_name_map[port]] = port + if self.voq: + counter_bucket_dict.update(voq_counter_bucket_dict) + counter_queue_name_map = None if voq and device_info.is_supervisor(): - keys = self.db.keys(self.db.CHASSIS_APP_DB, VOQ_COUNTERS_PREFIX + "*") + keys = self.db.keys(self.db.CHASSIS_COUNTERS_DB, VOQ_COUNTERS_PREFIX + "*") self.metaData, self.ports = voqCounterUtil.process_keys(keys) self.counter_port_name_map = self.ports return @@ -204,7 +206,7 @@ class Queuestat(object): key = voqCounterUtil.get_key(port,fapId,idx) for counter_name in counter_bucket_dict: voqCnt.setdefault(idx,{}).setdefault(counter_name,0) - counter_data = self.db.get(self.db.CHASSIS_APP_DB, key, counter_name) + counter_data = self.db.get(self.db.CHASSIS_COUNTERS_DB, key, counter_name) if counter_data is not None: voqCnt[idx][counter_name] += int(counter_data) @@ -212,12 +214,12 @@ class Queuestat(object): cnstat_dict = OrderedDict() cnstat_dict['time'] = datetime.datetime.now() for idx in sorted(voqCnt.keys()): - fields = ["0"]*6 + fields = ["0"]*len(voq_header) fields[0] = idx fields[1] = QUEUE_TYPE_VOQ for counter_name, pos in counter_bucket_dict.items(): fields[pos] = str(voqCnt[idx][counter_name]) - cntr = QueueStats._make(fields)._asdict() + cntr = VoqStats._make(fields)._asdict() cnstat_dict[port+":"+idx] = cntr return cnstat_dict @@ -255,16 +257,14 @@ class Queuestat(object): sys.exit(1) if self.voq: - fields = ["0","0","0","0","0","0","0"] + fields = ["0"]*len(voq_header) else: - fields = ["0","0","0","0","0","0"] + fields = ["0"]*len(header) fields[0] = get_queue_index(table_id) fields[1] = get_queue_type(table_id) counter_dict = {} counter_dict.update(counter_bucket_dict) - if self.voq: - counter_dict.update(voq_counter_bucket_dict) for counter_name, pos in counter_dict.items(): full_table_id = COUNTER_TABLE_PREFIX + table_id From fe878ab2aec30f7a2d067c5dad1eaff3774be0b0 Mon Sep 17 00:00:00 2001 From: Vivek Verma <137406113+vivekverma-arista@users.noreply.github.com> Date: Tue, 24 Sep 2024 19:08:54 +0530 Subject: [PATCH 3/3] Update queuestat --- scripts/queuestat | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/scripts/queuestat b/scripts/queuestat index 6e8fe116f7..c70bec9148 100755 --- a/scripts/queuestat +++ b/scripts/queuestat @@ -83,23 +83,23 @@ class voqCounterUtil(): @staticmethod def parse_key(key): # COUNTERS|nfc406-7|Asic0|Ethernet164@nfc406-9|Asic0:3 - _, sysPort, fapId, voqIdx = re.search(r"(COUNTERS_VOQ).([a-zA-Z0-9\-\|]+).([a-zA-Z0-9\-\|]+).(\d)",key).groups() - return sysPort, fapId, voqIdx + _, sysPort, asic, voqIdx = re.search(r"(COUNTERS_VOQ).([a-zA-Z0-9\-\|]+).([a-zA-Z0-9\-\|]+).(\d)",key).groups() + return sysPort, asic, voqIdx # sysPort = nfc406-7|Asic0|Ethernet164 - # fapId = nfc406-9:Asic0 + # asic = nfc406-9:Asic0 @staticmethod - def get_key(sysPort, fapId, voqIdx): - return VOQ_COUNTERS_PREFIX + sysPort + "@" + fapId + ":" + voqIdx + def get_key(sysPort, asic, voqIdx): + return VOQ_COUNTERS_PREFIX + sysPort + "@" + asic + ":" + voqIdx @staticmethod def process_keys(keys): metaData = {} ports = set() for key in keys: - sysPort, fapId, voqIdx = voqCounterUtil.parse_key(key) + sysPort, asic, voqIdx = voqCounterUtil.parse_key(key) ports.add(sysPort) - metaData.setdefault(sysPort,{}).setdefault(fapId,set()).add(voqIdx) + metaData.setdefault(sysPort,{}).setdefault(asic,set()).add(voqIdx) return metaData, ports def build_json(port, cnstat, voq=False): @@ -201,9 +201,9 @@ class Queuestat(object): def getAggregatePortStats( self, port ): voqCnt = {} - for fapId in self.metaData[port]: - for idx in self.metaData[port][fapId]: - key = voqCounterUtil.get_key(port,fapId,idx) + for asic in self.metaData[port]: + for idx in self.metaData[port][asic]: + key = voqCounterUtil.get_key(port,asic,idx) for counter_name in counter_bucket_dict: voqCnt.setdefault(idx,{}).setdefault(counter_name,0) counter_data = self.db.get(self.db.CHASSIS_COUNTERS_DB, key, counter_name)