From c6151cf376061b183f5515683a22746b68ad7f68 Mon Sep 17 00:00:00 2001 From: Chenyang Wang <49756587+cyw233@users.noreply.github.com> Date: Fri, 29 Nov 2024 15:56:22 +1100 Subject: [PATCH] refactor: use separate LCs for BFD traffic test (#15787) Description of PR Use upstream and downstream LCs for BFD traffic test so it can cover the "port channel down but BFD not down" issue. Summary: Fixes # (issue) Microsoft ADO 30112186 Approach What is the motivation for this PR? In bfd/test_bfd_traffic.py, we want to pick 2 LCs, where one connected to T1 (downstream LC) and other one connected to T3 (upstream LC), because if we pick only 1 LC or pick 2 LCs but both are downstream LCs (or upstream LCs), we will not cover the issue of "port channel down but BFD not down". How did you do it? Randomly pick one upstream LC and one downstream LC. How did you verify/test it? I ran the updated code and can confirm that it works as expected. co-authorized by: jianquanye@microsoft.com --- tests/bfd/bfd_helpers.py | 92 +++++---- tests/bfd/conftest.py | 15 +- tests/bfd/test_bfd_traffic.py | 351 ++++++++++++++++------------------ 3 files changed, 235 insertions(+), 223 deletions(-) diff --git a/tests/bfd/bfd_helpers.py b/tests/bfd/bfd_helpers.py index 3f9d3b7a865..792b784b22e 100644 --- a/tests/bfd/bfd_helpers.py +++ b/tests/bfd/bfd_helpers.py @@ -574,7 +574,8 @@ def send_packets_batch_from_ptf( def get_backend_interface_in_use_by_counter( - dut, + src_dut, + dst_dut, packet_count, version, src_asic_router_mac, @@ -584,10 +585,13 @@ def get_backend_interface_in_use_by_counter( src_asic_index, dst_asic_index, ): - clear_interface_counters(dut) + with SafeThreadPoolExecutor(max_workers=8) as executor: + for dut in [src_dut, dst_dut]: + executor.submit(clear_interface_counters, dut) + send_packets_batch_from_ptf(packet_count, version, src_asic_router_mac, ptfadapter, ptf_src_port, dst_neighbor_ip) - src_output = dut.show_and_parse("show int counters -n asic{} -d all".format(src_asic_index)) - dst_output = dut.show_and_parse("show int counters -n asic{} -d all".format(dst_asic_index)) + src_output = src_dut.show_and_parse("show int counters -n asic{} -d all".format(src_asic_index)) + dst_output = dst_dut.show_and_parse("show int counters -n asic{} -d all".format(dst_asic_index)) src_bp_iface = None for item in src_output: if "BP" in item.get("iface", "") and int(item.get("tx_ok", "0").replace(',', '')) >= packet_count: @@ -604,24 +608,25 @@ def get_backend_interface_in_use_by_counter( return src_bp_iface, dst_bp_iface -def get_src_dst_asic_next_hops(version, dut, src_asic, dst_asic, request, backend_port_channels): - src_asic_next_hops = extract_ip_addresses_for_backend_portchannels(dut, dst_asic, version, backend_port_channels) - assert len(src_asic_next_hops) != 0, "Source next hops are empty" - dst_asic_next_hops = extract_ip_addresses_for_backend_portchannels(dut, src_asic, version, backend_port_channels) - assert len(dst_asic_next_hops) != 0, "Destination next hops are empty" - - dut_asic_static_routes = get_dut_asic_static_routes(version, dut) - - # Picking a static route to delete its BFD session - src_prefix = selecting_route_to_delete(dut_asic_static_routes, src_asic_next_hops.values()) - request.config.src_prefix = src_prefix - assert src_prefix is not None and src_prefix != "", "Source prefix not found" +def get_src_dst_asic_next_hops(version, src_dut, src_asic, src_backend_port_channels, dst_dut, dst_asic, + dst_backend_port_channels): + src_asic_next_hops = extract_ip_addresses_for_backend_portchannels( + dst_dut, + dst_asic, + version, + backend_port_channels=dst_backend_port_channels, + ) - dst_prefix = selecting_route_to_delete(dut_asic_static_routes, dst_asic_next_hops.values()) - request.config.dst_prefix = dst_prefix - assert dst_prefix is not None and dst_prefix != "", "Destination prefix not found" + assert len(src_asic_next_hops) != 0, "Source next hops are empty" + dst_asic_next_hops = extract_ip_addresses_for_backend_portchannels( + src_dut, + src_asic, + version, + backend_port_channels=src_backend_port_channels, + ) - return src_asic_next_hops, dst_asic_next_hops, src_prefix, dst_prefix + assert len(dst_asic_next_hops) != 0, "Destination next hops are empty" + return src_asic_next_hops, dst_asic_next_hops def get_port_channel_by_member(backend_port_channels, member): @@ -641,6 +646,7 @@ def toggle_port_channel_or_member( ): request.config.portchannels_on_dut = "dut" request.config.selected_portchannels = [target_to_toggle] + request.config.dut = dut request.config.asic = asic batch_control_interface_state(dut, asic, [target_to_toggle], action) @@ -655,13 +661,14 @@ def assert_bp_iface_after_shutdown( dst_bp_iface_after_shutdown, src_asic_index, dst_asic_index, - dut_hostname, + src_dut_hostname, + dst_dut_hostname, ): if src_bp_iface_before_shutdown == src_bp_iface_after_shutdown: pytest.fail( "Source backend interface in use on asic{} of dut {} does not change after shutdown".format( src_asic_index, - dut_hostname, + src_dut_hostname, ) ) @@ -669,7 +676,7 @@ def assert_bp_iface_after_shutdown( pytest.fail( "Destination backend interface in use on asic{} of dut {} does not change after shutdown".format( dst_asic_index, - dut_hostname, + dst_dut_hostname, ) ) @@ -681,13 +688,14 @@ def assert_port_channel_after_shutdown( dst_port_channel_after_shutdown, src_asic_index, dst_asic_index, - dut_hostname, + src_dut_hostname, + dst_dut_hostname, ): if src_port_channel_before_shutdown == src_port_channel_after_shutdown: pytest.fail( "Source port channel in use on asic{} of dut {} does not change after shutdown".format( src_asic_index, - dut_hostname, + src_dut_hostname, ) ) @@ -695,7 +703,7 @@ def assert_port_channel_after_shutdown( pytest.fail( "Destination port channel in use on asic{} of dut {} does not change after shutdown".format( dst_asic_index, - dut_hostname, + dst_dut_hostname, ) ) @@ -715,8 +723,10 @@ def wait_until_given_bfd_down(next_hops, port_channel, asic_index, dut): def assert_traffic_switching( - dut, - backend_port_channels, + src_dut, + dst_dut, + src_backend_port_channels, + dst_backend_port_channels, src_asic_index, src_bp_iface_before_shutdown, src_bp_iface_after_shutdown, @@ -733,16 +743,17 @@ def assert_traffic_switching( dst_bp_iface_after_shutdown, src_asic_index, dst_asic_index, - dut.hostname, + src_dut.hostname, + dst_dut.hostname, ) src_port_channel_after_shutdown = get_port_channel_by_member( - backend_port_channels, + src_backend_port_channels, src_bp_iface_after_shutdown, ) dst_port_channel_after_shutdown = get_port_channel_by_member( - backend_port_channels, + dst_backend_port_channels, dst_bp_iface_after_shutdown, ) @@ -753,5 +764,22 @@ def assert_traffic_switching( dst_port_channel_after_shutdown, src_asic_index, dst_asic_index, - dut.hostname, + src_dut.hostname, + dst_dut.hostname, ) + + +def get_upstream_and_downstream_dut_pool(frontend_nodes): + upstream_dut_pool = [] + downstream_dut_pool = [] + for node in frontend_nodes: + bgp_neighbors = node.get_bgp_neighbors() + for neighbor_info in bgp_neighbors.values(): + if "t3" in neighbor_info["description"].lower(): + upstream_dut_pool.append(node) + break + elif "t1" in neighbor_info["description"].lower(): + downstream_dut_pool.append(node) + break + + return upstream_dut_pool, downstream_dut_pool diff --git a/tests/bfd/conftest.py b/tests/bfd/conftest.py index f69f7170d31..fcabab697a8 100644 --- a/tests/bfd/conftest.py +++ b/tests/bfd/conftest.py @@ -48,24 +48,25 @@ def bfd_cleanup_db(request, duthosts, enum_supervisor_dut_hostname): # 120, 4, 0, check_orch_cpu_utilization, dut, orch_cpu_threshold # ), "Orch CPU utilization exceeds orch cpu threshold {} after finishing the test".format(orch_cpu_threshold) - logger.info("Verifying swss container status on RP") rp = duthosts[enum_supervisor_dut_hostname] container_status = True if hasattr(request.config, "rp_asic_ids"): + logger.info("Verifying swss container status on RP") for id in request.config.rp_asic_ids: docker_output = rp.shell( "docker ps | grep swss{} | awk '{{print $NF}}'".format(id) )["stdout"] if len(docker_output) == 0: container_status = False + if not container_status: - config_reload(rp) + logger.error("swss container is not running on RP, so running config reload") + config_reload(rp, safe_reload=True) if hasattr(request.config, "src_dut") and hasattr(request.config, "dst_dut"): clear_bfd_configs(request.config.src_dut, request.config.src_asic.asic_index, request.config.src_prefix) clear_bfd_configs(request.config.dst_dut, request.config.dst_asic.asic_index, request.config.dst_prefix) - logger.info("Bringing up portchannels or respective members") portchannels_on_dut = None if hasattr(request.config, "portchannels_on_dut"): portchannels_on_dut = request.config.portchannels_on_dut @@ -74,12 +75,10 @@ def bfd_cleanup_db(request, duthosts, enum_supervisor_dut_hostname): portchannels_on_dut = request.config.portchannels_on_dut selected_interfaces = request.config.selected_portchannel_members else: - logger.info( - "None of the portchannels are selected to flap. So skipping portchannel interface check" - ) selected_interfaces = [] if selected_interfaces: + logger.info("Bringing up portchannels or respective members") if portchannels_on_dut == "src": dut = request.config.src_dut elif portchannels_on_dut == "dst": @@ -95,3 +94,7 @@ def bfd_cleanup_db(request, duthosts, enum_supervisor_dut_hostname): asic = request.config.asic ensure_interfaces_are_up(dut, asic, selected_interfaces) + else: + logger.info( + "None of the portchannels are selected to flap. So skipping portchannel interface check" + ) diff --git a/tests/bfd/test_bfd_traffic.py b/tests/bfd/test_bfd_traffic.py index 341eca7eb11..59fb8a9c456 100644 --- a/tests/bfd/test_bfd_traffic.py +++ b/tests/bfd/test_bfd_traffic.py @@ -6,7 +6,7 @@ from tests.bfd.bfd_helpers import get_ptf_src_port, get_backend_interface_in_use_by_counter, \ get_random_bgp_neighbor_ip_of_asic, toggle_port_channel_or_member, get_port_channel_by_member, \ wait_until_given_bfd_down, assert_traffic_switching, verify_bfd_only, extract_backend_portchannels, \ - get_src_dst_asic_next_hops + get_src_dst_asic_next_hops, get_upstream_and_downstream_dut_pool from tests.common.helpers.multi_thread_utils import SafeThreadPoolExecutor pytestmark = [pytest.mark.topology("t2")] @@ -18,127 +18,111 @@ class TestBfdTraffic: PACKET_COUNT = 10000 @pytest.fixture(scope="class") - def select_dut_and_src_dst_asic_index(self, duthosts): + def get_src_dst_asic(self, request, duthosts): if not duthosts.frontend_nodes: pytest.skip("DUT does not have any frontend nodes") - dut_index = random.choice(list(range(len(duthosts.frontend_nodes)))) - asic_namespace_list = duthosts.frontend_nodes[dut_index].get_asic_namespace_list() - if len(asic_namespace_list) < 2: - pytest.skip("DUT does not have more than one ASICs") - - # Random selection of src asic & dst asic on DUT - src_asic_namespace, dst_asic_namespace = random.sample(asic_namespace_list, 2) - src_asic_index = src_asic_namespace.split("asic")[1] - dst_asic_index = dst_asic_namespace.split("asic")[1] + src_dut_pool, dst_dut_pool = get_upstream_and_downstream_dut_pool(duthosts.frontend_nodes) + if not src_dut_pool or not dst_dut_pool: + pytest.skip("No upstream or downstream DUTs found") + + src_dut_index = random.choice(list(range(len(src_dut_pool)))) + dst_dut_index = random.choice(list(range(len(dst_dut_pool)))) + src_dut = src_dut_pool[src_dut_index] + dst_dut = dst_dut_pool[dst_dut_index] + src_asic_namespace_list = src_dut.get_asic_namespace_list() + dst_asic_namespace_list = dst_dut.get_asic_namespace_list() + if not src_asic_namespace_list or not dst_asic_namespace_list: + pytest.skip("No asic namespaces found on source or destination DUT") + + src_asic_namespace = random.choice(src_asic_namespace_list) + dst_asic_namespace = random.choice(dst_asic_namespace_list) + src_asic_index = int(src_asic_namespace.split("asic")[1]) + dst_asic_index = int(dst_asic_namespace.split("asic")[1]) + src_asic = src_dut.asics[src_asic_index] + dst_asic = dst_dut.asics[dst_asic_index] yield { - "dut_index": dut_index, - "src_asic_index": int(src_asic_index), - "dst_asic_index": int(dst_asic_index), - } - - @pytest.fixture(scope="class") - def get_src_dst_asic(self, request, duthosts, select_dut_and_src_dst_asic_index): - logger.info("Printing select_dut_and_src_dst_asic_index") - logger.info(select_dut_and_src_dst_asic_index) - - logger.info("Printing duthosts.frontend_nodes") - logger.info(duthosts.frontend_nodes) - dut = duthosts.frontend_nodes[select_dut_and_src_dst_asic_index["dut_index"]] - - logger.info("Printing dut asics") - logger.info(dut.asics) - - src_asic = dut.asics[select_dut_and_src_dst_asic_index["src_asic_index"]] - dst_asic = dut.asics[select_dut_and_src_dst_asic_index["dst_asic_index"]] - - request.config.src_asic = src_asic - request.config.dst_asic = dst_asic - request.config.dut = dut - - rtn_dict = { + "src_dut": src_dut, "src_asic": src_asic, + "src_asic_index": src_asic_index, + "dst_dut": dst_dut, "dst_asic": dst_asic, - "dut": dut, + "dst_asic_index": dst_asic_index, } - rtn_dict.update(select_dut_and_src_dst_asic_index) - yield rtn_dict - @pytest.fixture(scope="class", params=["ipv4", "ipv6"]) def prepare_traffic_test_variables(self, get_src_dst_asic, request): version = request.param logger.info("Version: %s", version) - dut = get_src_dst_asic["dut"] + src_dut = get_src_dst_asic["src_dut"] src_asic = get_src_dst_asic["src_asic"] src_asic_index = get_src_dst_asic["src_asic_index"] + dst_dut = get_src_dst_asic["dst_dut"] dst_asic = get_src_dst_asic["dst_asic"] dst_asic_index = get_src_dst_asic["dst_asic_index"] logger.info( - "DUT: {}, src_asic_index: {}, dst_asic_index: {}".format(dut.hostname, src_asic_index, dst_asic_index) + "src_dut: {}, src_asic_index: {}, dst_dut: {}, dst_asic_index: {}".format( + src_dut.hostname, + src_asic_index, + dst_dut.hostname, + dst_asic_index, + ) ) - backend_port_channels = extract_backend_portchannels(dut) - src_asic_next_hops, dst_asic_next_hops, src_prefix, dst_prefix = get_src_dst_asic_next_hops( + src_backend_port_channels = extract_backend_portchannels(src_dut) + dst_backend_port_channels = extract_backend_portchannels(dst_dut) + src_asic_next_hops, dst_asic_next_hops = get_src_dst_asic_next_hops( version, - dut, + src_dut, src_asic, + src_backend_port_channels, + dst_dut, dst_asic, - request, - backend_port_channels, + dst_backend_port_channels, ) src_asic_router_mac = src_asic.get_router_mac() yield { - "dut": dut, + "src_dut": src_dut, "src_asic": src_asic, "src_asic_index": src_asic_index, + "dst_dut": dst_dut, "dst_asic": dst_asic, "dst_asic_index": dst_asic_index, "src_asic_next_hops": src_asic_next_hops, "dst_asic_next_hops": dst_asic_next_hops, - "src_prefix": src_prefix, - "dst_prefix": dst_prefix, "src_asic_router_mac": src_asic_router_mac, - "backend_port_channels": backend_port_channels, + "src_backend_port_channels": src_backend_port_channels, + "dst_backend_port_channels": dst_backend_port_channels, "version": version, } - def test_bfd_traffic_remote_port_channel_shutdown( - self, - request, - tbinfo, - ptfadapter, - prepare_traffic_test_variables, - bfd_cleanup_db, - ): - dut = prepare_traffic_test_variables["dut"] + def test_bfd_traffic_remote_port_channel_shutdown(self, request, tbinfo, ptfadapter, + prepare_traffic_test_variables, bfd_cleanup_db): + src_dut = prepare_traffic_test_variables["src_dut"] src_asic = prepare_traffic_test_variables["src_asic"] src_asic_index = prepare_traffic_test_variables["src_asic_index"] + dst_dut = prepare_traffic_test_variables["dst_dut"] dst_asic = prepare_traffic_test_variables["dst_asic"] dst_asic_index = prepare_traffic_test_variables["dst_asic_index"] src_asic_next_hops = prepare_traffic_test_variables["src_asic_next_hops"] dst_asic_next_hops = prepare_traffic_test_variables["dst_asic_next_hops"] - src_prefix = prepare_traffic_test_variables["src_prefix"] - dst_prefix = prepare_traffic_test_variables["dst_prefix"] src_asic_router_mac = prepare_traffic_test_variables["src_asic_router_mac"] - backend_port_channels = prepare_traffic_test_variables["backend_port_channels"] + src_backend_port_channels = prepare_traffic_test_variables["src_backend_port_channels"] + dst_backend_port_channels = prepare_traffic_test_variables["dst_backend_port_channels"] version = prepare_traffic_test_variables["version"] - src_dst_context = [ - ("src", src_asic, src_prefix, src_asic_next_hops), - ("dst", dst_asic, dst_prefix, dst_asic_next_hops), - ] - dst_neighbor_ip = get_random_bgp_neighbor_ip_of_asic(dut, dst_asic_index, version) + dst_neighbor_ip = get_random_bgp_neighbor_ip_of_asic(dst_dut, dst_asic_index, version) if not dst_neighbor_ip: - pytest.skip("No BGP neighbor found on asic{} of dut {}".format(dst_asic_index, dut.hostname)) + pytest.skip("No BGP neighbor found on asic{} of dut {}".format(dst_asic_index, dst_dut.hostname)) ptf_src_port = get_ptf_src_port(src_asic, tbinfo) src_bp_iface_before_shutdown, dst_bp_iface_before_shutdown = get_backend_interface_in_use_by_counter( - dut, + src_dut, + dst_dut, self.PACKET_COUNT, version, src_asic_router_mac, @@ -150,7 +134,7 @@ def test_bfd_traffic_remote_port_channel_shutdown( ) dst_port_channel_before_shutdown = get_port_channel_by_member( - backend_port_channels, + dst_backend_port_channels, dst_bp_iface_before_shutdown, ) @@ -159,26 +143,27 @@ def test_bfd_traffic_remote_port_channel_shutdown( toggle_port_channel_or_member( dst_port_channel_before_shutdown, - dut, + dst_dut, dst_asic, request, "shutdown", ) src_port_channel_before_shutdown = get_port_channel_by_member( - backend_port_channels, + src_backend_port_channels, src_bp_iface_before_shutdown, ) with SafeThreadPoolExecutor(max_workers=8) as executor: - for next_hops, port_channel, asic_index in [ - (src_asic_next_hops, dst_port_channel_before_shutdown, src_asic_index), - (dst_asic_next_hops, src_port_channel_before_shutdown, dst_asic_index), + for next_hops, port_channel, asic_index, dut in [ + (src_asic_next_hops, dst_port_channel_before_shutdown, src_asic_index, src_dut), + (dst_asic_next_hops, src_port_channel_before_shutdown, dst_asic_index, dst_dut), ]: executor.submit(wait_until_given_bfd_down, next_hops, port_channel, asic_index, dut) src_bp_iface_after_shutdown, dst_bp_iface_after_shutdown = get_backend_interface_in_use_by_counter( - dut, + src_dut, + dst_dut, self.PACKET_COUNT, version, src_asic_router_mac, @@ -190,8 +175,10 @@ def test_bfd_traffic_remote_port_channel_shutdown( ) assert_traffic_switching( - dut, - backend_port_channels, + src_dut, + dst_dut, + src_backend_port_channels, + dst_backend_port_channels, src_asic_index, src_bp_iface_before_shutdown, src_bp_iface_after_shutdown, @@ -204,48 +191,42 @@ def test_bfd_traffic_remote_port_channel_shutdown( toggle_port_channel_or_member( dst_port_channel_before_shutdown, - dut, + dst_dut, dst_asic, request, "startup", ) with SafeThreadPoolExecutor(max_workers=8) as executor: - for _, asic, _, next_hops in src_dst_context: + for dut, next_hops, asic in [ + (src_dut, src_asic_next_hops, src_asic), + (dst_dut, dst_asic_next_hops, dst_asic), + ]: executor.submit(verify_bfd_only, dut, next_hops, asic, "Up") - def test_bfd_traffic_local_port_channel_shutdown( - self, - request, - tbinfo, - ptfadapter, - prepare_traffic_test_variables, - bfd_cleanup_db, - ): - dut = prepare_traffic_test_variables["dut"] + def test_bfd_traffic_local_port_channel_shutdown(self, request, tbinfo, ptfadapter, + prepare_traffic_test_variables, bfd_cleanup_db): + src_dut = prepare_traffic_test_variables["src_dut"] src_asic = prepare_traffic_test_variables["src_asic"] src_asic_index = prepare_traffic_test_variables["src_asic_index"] + dst_dut = prepare_traffic_test_variables["dst_dut"] dst_asic = prepare_traffic_test_variables["dst_asic"] dst_asic_index = prepare_traffic_test_variables["dst_asic_index"] src_asic_next_hops = prepare_traffic_test_variables["src_asic_next_hops"] dst_asic_next_hops = prepare_traffic_test_variables["dst_asic_next_hops"] - src_prefix = prepare_traffic_test_variables["src_prefix"] - dst_prefix = prepare_traffic_test_variables["dst_prefix"] src_asic_router_mac = prepare_traffic_test_variables["src_asic_router_mac"] - backend_port_channels = prepare_traffic_test_variables["backend_port_channels"] + src_backend_port_channels = prepare_traffic_test_variables["src_backend_port_channels"] + dst_backend_port_channels = prepare_traffic_test_variables["dst_backend_port_channels"] version = prepare_traffic_test_variables["version"] - src_dst_context = [ - ("src", src_asic, src_prefix, src_asic_next_hops), - ("dst", dst_asic, dst_prefix, dst_asic_next_hops), - ] - dst_neighbor_ip = get_random_bgp_neighbor_ip_of_asic(dut, dst_asic_index, version) + dst_neighbor_ip = get_random_bgp_neighbor_ip_of_asic(dst_dut, dst_asic_index, version) if not dst_neighbor_ip: - pytest.skip("No BGP neighbor found on asic{} of dut {}".format(dst_asic_index, dut.hostname)) + pytest.skip("No BGP neighbor found on asic{} of dut {}".format(dst_asic_index, dst_dut.hostname)) ptf_src_port = get_ptf_src_port(src_asic, tbinfo) src_bp_iface_before_shutdown, dst_bp_iface_before_shutdown = get_backend_interface_in_use_by_counter( - dut, + src_dut, + dst_dut, self.PACKET_COUNT, version, src_asic_router_mac, @@ -257,7 +238,7 @@ def test_bfd_traffic_local_port_channel_shutdown( ) src_port_channel_before_shutdown = get_port_channel_by_member( - backend_port_channels, + src_backend_port_channels, src_bp_iface_before_shutdown, ) @@ -266,26 +247,27 @@ def test_bfd_traffic_local_port_channel_shutdown( toggle_port_channel_or_member( src_port_channel_before_shutdown, - dut, + src_dut, src_asic, request, "shutdown", ) dst_port_channel_before_shutdown = get_port_channel_by_member( - backend_port_channels, + dst_backend_port_channels, dst_bp_iface_before_shutdown, ) with SafeThreadPoolExecutor(max_workers=8) as executor: - for next_hops, port_channel, asic_index in [ - (src_asic_next_hops, dst_port_channel_before_shutdown, src_asic_index), - (dst_asic_next_hops, src_port_channel_before_shutdown, dst_asic_index), + for next_hops, port_channel, asic_index, dut in [ + (src_asic_next_hops, dst_port_channel_before_shutdown, src_asic_index, src_dut), + (dst_asic_next_hops, src_port_channel_before_shutdown, dst_asic_index, dst_dut), ]: executor.submit(wait_until_given_bfd_down, next_hops, port_channel, asic_index, dut) src_bp_iface_after_shutdown, dst_bp_iface_after_shutdown = get_backend_interface_in_use_by_counter( - dut, + src_dut, + dst_dut, self.PACKET_COUNT, version, src_asic_router_mac, @@ -297,8 +279,10 @@ def test_bfd_traffic_local_port_channel_shutdown( ) assert_traffic_switching( - dut, - backend_port_channels, + src_dut, + dst_dut, + src_backend_port_channels, + dst_backend_port_channels, src_asic_index, src_bp_iface_before_shutdown, src_bp_iface_after_shutdown, @@ -311,48 +295,42 @@ def test_bfd_traffic_local_port_channel_shutdown( toggle_port_channel_or_member( src_port_channel_before_shutdown, - dut, + src_dut, src_asic, request, "startup", ) with SafeThreadPoolExecutor(max_workers=8) as executor: - for _, asic, _, next_hops in src_dst_context: + for dut, next_hops, asic in [ + (src_dut, src_asic_next_hops, src_asic), + (dst_dut, dst_asic_next_hops, dst_asic), + ]: executor.submit(verify_bfd_only, dut, next_hops, asic, "Up") - def test_bfd_traffic_remote_port_channel_member_shutdown( - self, - request, - tbinfo, - ptfadapter, - prepare_traffic_test_variables, - bfd_cleanup_db, - ): - dut = prepare_traffic_test_variables["dut"] + def test_bfd_traffic_remote_port_channel_member_shutdown(self, request, tbinfo, ptfadapter, + prepare_traffic_test_variables, bfd_cleanup_db): + src_dut = prepare_traffic_test_variables["src_dut"] src_asic = prepare_traffic_test_variables["src_asic"] src_asic_index = prepare_traffic_test_variables["src_asic_index"] + dst_dut = prepare_traffic_test_variables["dst_dut"] dst_asic = prepare_traffic_test_variables["dst_asic"] dst_asic_index = prepare_traffic_test_variables["dst_asic_index"] src_asic_next_hops = prepare_traffic_test_variables["src_asic_next_hops"] dst_asic_next_hops = prepare_traffic_test_variables["dst_asic_next_hops"] - src_prefix = prepare_traffic_test_variables["src_prefix"] - dst_prefix = prepare_traffic_test_variables["dst_prefix"] src_asic_router_mac = prepare_traffic_test_variables["src_asic_router_mac"] - backend_port_channels = prepare_traffic_test_variables["backend_port_channels"] + src_backend_port_channels = prepare_traffic_test_variables["src_backend_port_channels"] + dst_backend_port_channels = prepare_traffic_test_variables["dst_backend_port_channels"] version = prepare_traffic_test_variables["version"] - src_dst_context = [ - ("src", src_asic, src_prefix, src_asic_next_hops), - ("dst", dst_asic, dst_prefix, dst_asic_next_hops), - ] - dst_neighbor_ip = get_random_bgp_neighbor_ip_of_asic(dut, dst_asic_index, version) + dst_neighbor_ip = get_random_bgp_neighbor_ip_of_asic(dst_dut, dst_asic_index, version) if not dst_neighbor_ip: - pytest.skip("No BGP neighbor found on asic{} of dut {}".format(dst_asic_index, dut.hostname)) + pytest.skip("No BGP neighbor found on asic{} of dut {}".format(dst_asic_index, dst_dut.hostname)) ptf_src_port = get_ptf_src_port(src_asic, tbinfo) src_bp_iface_before_shutdown, dst_bp_iface_before_shutdown = get_backend_interface_in_use_by_counter( - dut, + src_dut, + dst_dut, self.PACKET_COUNT, version, src_asic_router_mac, @@ -363,36 +341,37 @@ def test_bfd_traffic_remote_port_channel_member_shutdown( dst_asic_index, ) - toggle_port_channel_or_member( - dst_bp_iface_before_shutdown, - dut, - dst_asic, - request, - "shutdown", - ) - src_port_channel_before_shutdown = get_port_channel_by_member( - backend_port_channels, + src_backend_port_channels, src_bp_iface_before_shutdown, ) dst_port_channel_before_shutdown = get_port_channel_by_member( - backend_port_channels, + dst_backend_port_channels, dst_bp_iface_before_shutdown, ) if not src_port_channel_before_shutdown or not dst_port_channel_before_shutdown: pytest.fail("No port channel found with interface in use") + toggle_port_channel_or_member( + dst_bp_iface_before_shutdown, + dst_dut, + dst_asic, + request, + "shutdown", + ) + with SafeThreadPoolExecutor(max_workers=8) as executor: - for next_hops, port_channel, asic_index in [ - (src_asic_next_hops, dst_port_channel_before_shutdown, src_asic_index), - (dst_asic_next_hops, src_port_channel_before_shutdown, dst_asic_index), + for next_hops, port_channel, asic_index, dut in [ + (src_asic_next_hops, dst_port_channel_before_shutdown, src_asic_index, src_dut), + (dst_asic_next_hops, src_port_channel_before_shutdown, dst_asic_index, dst_dut), ]: executor.submit(wait_until_given_bfd_down, next_hops, port_channel, asic_index, dut) src_bp_iface_after_shutdown, dst_bp_iface_after_shutdown = get_backend_interface_in_use_by_counter( - dut, + src_dut, + dst_dut, self.PACKET_COUNT, version, src_asic_router_mac, @@ -404,8 +383,10 @@ def test_bfd_traffic_remote_port_channel_member_shutdown( ) assert_traffic_switching( - dut, - backend_port_channels, + src_dut, + dst_dut, + src_backend_port_channels, + dst_backend_port_channels, src_asic_index, src_bp_iface_before_shutdown, src_bp_iface_after_shutdown, @@ -418,48 +399,42 @@ def test_bfd_traffic_remote_port_channel_member_shutdown( toggle_port_channel_or_member( dst_bp_iface_before_shutdown, - dut, + dst_dut, dst_asic, request, "startup", ) with SafeThreadPoolExecutor(max_workers=8) as executor: - for _, asic, _, next_hops in src_dst_context: + for dut, next_hops, asic in [ + (src_dut, src_asic_next_hops, src_asic), + (dst_dut, dst_asic_next_hops, dst_asic), + ]: executor.submit(verify_bfd_only, dut, next_hops, asic, "Up") - def test_bfd_traffic_local_port_channel_member_shutdown( - self, - request, - tbinfo, - ptfadapter, - prepare_traffic_test_variables, - bfd_cleanup_db, - ): - dut = prepare_traffic_test_variables["dut"] + def test_bfd_traffic_local_port_channel_member_shutdown(self, request, tbinfo, ptfadapter, + prepare_traffic_test_variables, bfd_cleanup_db): + src_dut = prepare_traffic_test_variables["src_dut"] src_asic = prepare_traffic_test_variables["src_asic"] src_asic_index = prepare_traffic_test_variables["src_asic_index"] + dst_dut = prepare_traffic_test_variables["dst_dut"] dst_asic = prepare_traffic_test_variables["dst_asic"] dst_asic_index = prepare_traffic_test_variables["dst_asic_index"] src_asic_next_hops = prepare_traffic_test_variables["src_asic_next_hops"] dst_asic_next_hops = prepare_traffic_test_variables["dst_asic_next_hops"] - src_prefix = prepare_traffic_test_variables["src_prefix"] - dst_prefix = prepare_traffic_test_variables["dst_prefix"] src_asic_router_mac = prepare_traffic_test_variables["src_asic_router_mac"] - backend_port_channels = prepare_traffic_test_variables["backend_port_channels"] + src_backend_port_channels = prepare_traffic_test_variables["src_backend_port_channels"] + dst_backend_port_channels = prepare_traffic_test_variables["dst_backend_port_channels"] version = prepare_traffic_test_variables["version"] - src_dst_context = [ - ("src", src_asic, src_prefix, src_asic_next_hops), - ("dst", dst_asic, dst_prefix, dst_asic_next_hops), - ] - dst_neighbor_ip = get_random_bgp_neighbor_ip_of_asic(dut, dst_asic_index, version) + dst_neighbor_ip = get_random_bgp_neighbor_ip_of_asic(dst_dut, dst_asic_index, version) if not dst_neighbor_ip: - pytest.skip("No BGP neighbor found on asic{} of dut {}".format(dst_asic_index, dut.hostname)) + pytest.skip("No BGP neighbor found on asic{} of dut {}".format(dst_asic_index, dst_dut.hostname)) ptf_src_port = get_ptf_src_port(src_asic, tbinfo) src_bp_iface_before_shutdown, dst_bp_iface_before_shutdown = get_backend_interface_in_use_by_counter( - dut, + src_dut, + dst_dut, self.PACKET_COUNT, version, src_asic_router_mac, @@ -470,36 +445,37 @@ def test_bfd_traffic_local_port_channel_member_shutdown( dst_asic_index, ) - toggle_port_channel_or_member( - src_bp_iface_before_shutdown, - dut, - src_asic, - request, - "shutdown", - ) - src_port_channel_before_shutdown = get_port_channel_by_member( - backend_port_channels, + src_backend_port_channels, src_bp_iface_before_shutdown, ) dst_port_channel_before_shutdown = get_port_channel_by_member( - backend_port_channels, + dst_backend_port_channels, dst_bp_iface_before_shutdown, ) if not src_port_channel_before_shutdown or not dst_port_channel_before_shutdown: pytest.fail("No port channel found with interface in use") + toggle_port_channel_or_member( + src_bp_iface_before_shutdown, + src_dut, + src_asic, + request, + "shutdown", + ) + with SafeThreadPoolExecutor(max_workers=8) as executor: - for next_hops, port_channel, asic_index in [ - (src_asic_next_hops, dst_port_channel_before_shutdown, src_asic_index), - (dst_asic_next_hops, src_port_channel_before_shutdown, dst_asic_index), + for next_hops, port_channel, asic_index, dut in [ + (src_asic_next_hops, dst_port_channel_before_shutdown, src_asic_index, src_dut), + (dst_asic_next_hops, src_port_channel_before_shutdown, dst_asic_index, dst_dut), ]: executor.submit(wait_until_given_bfd_down, next_hops, port_channel, asic_index, dut) src_bp_iface_after_shutdown, dst_bp_iface_after_shutdown = get_backend_interface_in_use_by_counter( - dut, + src_dut, + dst_dut, self.PACKET_COUNT, version, src_asic_router_mac, @@ -511,8 +487,10 @@ def test_bfd_traffic_local_port_channel_member_shutdown( ) assert_traffic_switching( - dut, - backend_port_channels, + src_dut, + dst_dut, + src_backend_port_channels, + dst_backend_port_channels, src_asic_index, src_bp_iface_before_shutdown, src_bp_iface_after_shutdown, @@ -525,12 +503,15 @@ def test_bfd_traffic_local_port_channel_member_shutdown( toggle_port_channel_or_member( src_bp_iface_before_shutdown, - dut, + src_dut, src_asic, request, "startup", ) with SafeThreadPoolExecutor(max_workers=8) as executor: - for _, asic, _, next_hops in src_dst_context: + for dut, next_hops, asic in [ + (src_dut, src_asic_next_hops, src_asic), + (dst_dut, dst_asic_next_hops, dst_asic), + ]: executor.submit(verify_bfd_only, dut, next_hops, asic, "Up")