Multi-asic Support for Telemetry Test (sonic-net#14982)

Description of PR
Add multi-asic support for telemetry test. This does not include multi-asic support for events (bgp, swss etc) as there are issues with multi-asic support on server side. A different PR sonic-net#13826 will handle telemetry event multi-asic support on mgmt.

Approach
What is the motivation for this PR?
Multi-asic support for telemetry test

How did you do it?
How did you verify/test it?
Whole telemetry OC testd pass with the changes (exclude events test).

co-authorized by: [email protected]

.azure-pipelines/pr_test_scripts.yaml

@@ -447,6 +447,8 @@ multi-asic-t1-lag:
   - process_monitoring/test_critical_process_monitoring.py
   - container_checker/test_container_checker.py
   - http/test_http_copy.py
+  - telemetry/test_telemetry_cert_rotation.py
+  - telemetry/test_telemetry.py
 
 dpu:
   - dash/test_dash_vnet.py

tests/common/devices/multi_asic.py

@@ -286,6 +286,12 @@ def get_linux_ip_cmd_for_namespace(self, cmd, namespace):
         ns_cmd = cmd.replace('ip', 'ip -n {}'.format(namespace))
         return ns_cmd
 
+    def get_cli_cmd_for_namespace(self, cmd, namespace):
+        if not namespace:
+            return cmd
+        ns_cmd = cmd.replace('sonic-db-cli', 'sonic-db-cli -n {}'.format(namespace))
+        return ns_cmd
+
     @property
     def ttl_decr_value(self):
         """
@@ -520,9 +526,10 @@ def modify_syslog_rate_limit(self, feature, rl_option='disable'):
             cmds.append(cmd_reload.format(docker))
             self.sonichost.shell_cmds(cmds=cmds)
 
-    def get_bgp_neighbors(self):
+    def get_bgp_neighbors(self, namespace=None):
         """
-        Get a diction of BGP neighbor states
+        Get a diction of BGP neighbor states. If namespace is not None
+        will get a dictionary of BGP neighbor states for that namespace
 
         Args: None
 
@@ -531,8 +538,9 @@ def get_bgp_neighbors(self):
         """
         bgp_neigh = {}
         for asic in self.asics:
-            bgp_info = asic.bgp_facts()
-            bgp_neigh.update(bgp_info["ansible_facts"]["bgp_neighbors"])
+            if namespace is None or asic.namespace == namespace:
+                bgp_info = asic.bgp_facts()
+                bgp_neigh.update(bgp_info["ansible_facts"]["bgp_neighbors"])
 
         return bgp_neigh
 

tests/common/devices/sonic.py

@@ -1378,17 +1378,22 @@ def get_intf_link_local_ipv6_addr(self, intf):
         addr = self.shell(cmd)["stdout"]
         return addr
 
-    def get_bgp_neighbor_info(self, neighbor_ip):
+    def get_bgp_neighbor_info(self, neighbor_ip, asic_id=None):
         """
         @summary: return bgp neighbor info
 
         @param neighbor_ip: bgp neighbor IP
         """
         nbip = ipaddress.ip_address(neighbor_ip)
+        vtysh = "vtysh"
+        if asic_id is not None:
+            vtysh = "vtysh -n {}".format(asic_id)
+
         if nbip.version == 4:
-            out = self.command("vtysh -c \"show ip bgp neighbor {} json\"".format(neighbor_ip))
+            out = self.command("{} -c \"show ip bgp neighbor {} json\"".format(vtysh, neighbor_ip))
         else:
-            out = self.command("vtysh -c \"show bgp ipv6 neighbor {} json\"".format(neighbor_ip))
+            out = self.command("{} -c \"show bgp ipv6 neighbor {} json\"".format(vtysh, neighbor_ip))
+
         nbinfo = json.loads(re.sub(r"\\\"", '"', re.sub(r"\\n", "", out['stdout'])))
         logging.info("bgp neighbor {} info {}".format(neighbor_ip, nbinfo))
 

tests/common/plugins/conditional_mark/tests_mark_conditions.yaml

@@ -1890,10 +1890,11 @@ telemetry/test_telemetry.py:
 telemetry/test_telemetry.py::test_telemetry_queue_buffer_cnt:
   skip:
     conditions_logical_operator: or
-    reason: "Testcase ignored due to switch type is voq / Unsupported in MGFX topos"
+    reason: "Testcase ignored due to switch type is voq / Unsupported in MGFX topos / multi-asic issue 15393"
     conditions:
       - "(switch_type=='voq')"
       - "topo_type in ['m0', 'mx']"
+      - "(is_multi_asic==True) and https://github.com/sonic-net/sonic-mgmt/issues/15393"
 
 #######################################
 #####         pktgen              #####

tests/telemetry/telemetry_utils.py

@@ -106,7 +106,7 @@ def trigger_logger(duthost, log, process, container="", priority="local0.notice"
 
 def generate_client_cli(duthost, gnxi_path, method=METHOD_GET, xpath="COUNTERS/Ethernet0", target="COUNTERS_DB",
                         subscribe_mode=SUBSCRIBE_MODE_STREAM, submode=SUBMODE_SAMPLE,
-                        intervalms=0, update_count=3, create_connections=1, filter_event_regex="",
+                        intervalms=0, update_count=3, create_connections=1, filter_event_regex="", namespace=None,
                         timeout=-1):
     """ Generate the py_gnmicli command line based on the given params.
     t                      --target: gNMI target; required
@@ -121,11 +121,15 @@ def generate_client_cli(duthost, gnxi_path, method=METHOD_GET, xpath="COUNTERS/E
     update_count:          Max number of streaming updates to receive. 0 means no limit. default 0
     create_connections:    Creates TCP connections with gNMI server; default 1; -1 for infinite connections
     filter_event_regex:    Regex to filter event when querying events path
+    namespace:             namespace for multi-asic
     timeout:               Subscription duration in seconds; After X seconds, request terminates; default none
     """
     env = GNMIEnvironment(duthost, GNMIEnvironment.TELEMETRY_MODE)
-    cmdFormat = 'python ' + gnxi_path + 'gnmi_cli_py/py_gnmicli.py -g -t {0} -p {1} -m {2} -x {3} -xt {4} -o {5}'
-    cmd = cmdFormat.format(duthost.mgmt_ip, env.gnmi_port, method, xpath, target, "ndastreamingservertest")
+    ns = ""
+    if namespace is not None:
+        ns = "/{}".format(namespace)
+    cmdFormat = 'python ' + gnxi_path + 'gnmi_cli_py/py_gnmicli.py -g -t {0} -p {1} -m {2} -x {3} -xt {4}{5} -o {6}'
+    cmd = cmdFormat.format(duthost.mgmt_ip, env.gnmi_port, method, xpath, target, ns, "ndastreamingservertest")
 
     if method == METHOD_SUBSCRIBE:
         cmd += " --subscribe_mode {0} --submode {1} --interval {2} --update_count {3} --create_connections {4}".format(

tests/telemetry/test_telemetry.py

@@ -281,33 +281,44 @@ def test_on_change_updates(duthosts, enum_rand_one_per_hwsku_hostname, ptfhost,
     logger.info("Testing on change update notifications")
 
     duthost = duthosts[enum_rand_one_per_hwsku_hostname]
+    if duthost.is_supervisor_node():
+        pytest.skip(
+            "Skipping test as no Ethernet0 frontpanel port on supervisor")
     skip_201911_and_older(duthost)
-    cmd = generate_client_cli(duthost=duthost, gnxi_path=gnxi_path, method=METHOD_SUBSCRIBE,
-                              submode=SUBMODE_ONCHANGE, update_count=2, xpath="NEIGH_STATE_TABLE",
-                              target="STATE_DB")
 
-    bgp_nbrs = list(duthost.get_bgp_neighbors().keys())
+    nslist = duthost.get_asic_namespace_list()
+    ns = random.choice(nslist)
+    bgp_nbrs = list(duthost.get_bgp_neighbors(ns).keys())
     bgp_neighbor = random.choice(bgp_nbrs)
-    bgp_info = duthost.get_bgp_neighbor_info(bgp_neighbor)
+    asic_id = duthost.get_asic_id_from_namespace(ns)
+    bgp_info = duthost.get_bgp_neighbor_info(bgp_neighbor, asic_id)
     original_state = bgp_info["bgpState"]
     new_state = "Established" if original_state.lower() == "active" else "Active"
 
+    cmd = generate_client_cli(duthost=duthost, gnxi_path=gnxi_path, method=METHOD_SUBSCRIBE,
+                              submode=SUBMODE_ONCHANGE, update_count=2, xpath="NEIGH_STATE_TABLE",
+                              target="STATE_DB", namespace=ns)
+
     def callback(result):
         logger.info("Assert that ptf client output is non empty and contains on change update")
         try:
             assert result != "", "Did not get output from PTF client"
         finally:
-            duthost.shell("sonic-db-cli STATE_DB HSET \"NEIGH_STATE_TABLE|{}\" \"state\" {}".format(bgp_neighbor,
-                                                                                                    original_state))
+            ccmd = "sonic-db-cli STATE_DB HSET \"NEIGH_STATE_TABLE|{}\" \"state\" {}".format(bgp_neighbor,
+                                                                                             original_state)
+            ccmd = duthost.get_cli_cmd_for_namespace(ccmd, ns)
+            duthost.shell(ccmd)
         ret = parse_gnmi_output(result, 1, bgp_neighbor)
         assert ret is True, "Did not find key in update"
 
     client_thread = threading.Thread(target=invoke_py_cli_from_ptf, args=(ptfhost, cmd, callback,))
     client_thread.start()
 
     wait_until(5, 1, 0, check_gnmi_cli_running, ptfhost)
-    duthost.shell("sonic-db-cli STATE_DB HSET \"NEIGH_STATE_TABLE|{}\" \"state\" {}".format(bgp_neighbor,
-                                                                                            new_state))
+    cmd = "sonic-db-cli STATE_DB HSET \"NEIGH_STATE_TABLE|{}\" \"state\" {}".format(bgp_neighbor,
+                                                                                    new_state)
+    cmd = duthost.get_cli_cmd_for_namespace(cmd, ns)
+    duthost.shell(cmd)
     client_thread.join(60)  # max timeout of 60s, expect update to come in <=30s