Merge branch 'sonic-net:master' into master

.azure-pipelines/pr_test_scripts.yaml

@@ -147,7 +147,7 @@ t0:
   - snmp/test_snmp_default_route.py
   - snmp/test_snmp_fdb.py
   - snmp/test_snmp_interfaces.py
-  - snmp/test_snmp_link_local_ip.py
+  - snmp/test_snmp_link_local.py
   - snmp/test_snmp_lldp.py
   - snmp/test_snmp_loopback.py
   - snmp/test_snmp_memory.py
@@ -429,7 +429,7 @@ multi-asic-t1-lag:
   - bgp/test_bgp_fact.py
   - bgp/test_bgp_update_timer.py
   - snmp/test_snmp_default_route.py
-  - snmp/test_snmp_link_local_ip.py
+  - snmp/test_snmp_link_local.py
   - snmp/test_snmp_loopback.py
   - snmp/test_snmp_pfc_counters.py
   - snmp/test_snmp_queue.py

.azure-pipelines/testscripts_analyse/analyse_testscripts.py

@@ -42,16 +42,19 @@
 
 
 def topo_name_to_type(topo_name):
-    pattern = re.compile(r'^(wan|t0|t1|ptf|fullmesh|dualtor|t2|tgen|multidut-tgen|mgmttor|m0|mc0|mx|dpu|any|snappi)')
+    pattern = re.compile(r'^(wan|t0|t1|ptf|fullmesh|dualtor|t2|tgen|multidut-tgen|mgmttor'
+                         r'|m0|mc0|mx|dpu|any|snappi|t0-sonic|t1-multi-asic|t0-2vlans)')
     match = pattern.match(topo_name)
     if match is None:
         logging.warning("Unsupported testbed type - {}".format(topo_name))
         return topo_name
 
     topo_type = match.group()
-    if topo_type in ['mgmttor', 'dualtor', 'm0', 'mc0', 'mx']:
+    if topo_type in ['mgmttor', 'dualtor', 'm0', 'mc0', 'mx', 't0-sonic', 't0-2vlans']:
         # certain testbed types are in 't0' category with different names.
         topo_type = 't0'
+    if topo_type in ['t1-multi-asic']:
+        topo_type = 't1'
     if topo_type in ['multidut-tgen']:
         topo_type = 'tgen'
     return topo_type

tests/bgp/test_bgp_bbr.py

@@ -25,7 +25,7 @@
 
 
 pytestmark = [
-    pytest.mark.topology('t1'),
+    pytest.mark.topology('t1', 't1-multi-asic'),
     pytest.mark.device_type('vs')
 ]
 

tests/bgp/test_bgp_fact.py

@@ -2,7 +2,7 @@
 from tests.common.helpers.bgp import run_bgp_facts
 
 pytestmark = [
-    pytest.mark.topology('any'),
+    pytest.mark.topology('any', 't0-sonic', 't1-multi-asic'),
     pytest.mark.device_type('vs')
 ]
 

tests/bgp/test_bgp_update_timer.py

@@ -25,7 +25,7 @@
 
 
 pytestmark = [
-    pytest.mark.topology("any"),
+    pytest.mark.topology("any", "t1-multi-asic"),
 ]
 
 PEER_COUNT = 2

tests/cacl/test_cacl_function.py

@@ -12,7 +12,7 @@
 
 pytestmark = [
     pytest.mark.disable_loganalyzer,  # disable automatic loganalyzer globally
-    pytest.mark.topology('any'),
+    pytest.mark.topology('any', 't1-multi-asic'),
     pytest.mark.device_type('vs')
 ]
 

tests/conftest.py

@@ -1492,7 +1492,7 @@ def generate_dut_backend_asics(request, duts_selected):
     return dut_asic_list
 
 
-def generate_priority_lists(request, prio_scope):
+def generate_priority_lists(request, prio_scope, with_completeness_level=False):
     empty = []
 
     tbname = request.config.getoption("--testbed")
@@ -1518,6 +1518,22 @@ def generate_priority_lists(request, prio_scope):
         for p in priorities:
             ret.append('{}|{}'.format(dut, p))
 
+    if with_completeness_level:
+        completeness_level = get_completeness_level_metadata(request)
+        # if completeness_level in ["debug", "basic", "confident"],
+        # select a small subnet to save test time
+        # if completeness_level in ["debug"], only select one item
+        # if completeness_level in ["basic", "confident"], select 1 priority per DUT
+
+        if completeness_level in ["debug"]:
+            ret = random.sample(ret, 1)
+        elif completeness_level in ["basic", "confident"]:
+            ret = []
+            for dut, priorities in list(dut_prio.items()):
+                if priorities:
+                    p = random.choice(priorities)
+                    ret.append('{}|{}'.format(dut, p))
+
     return ret if ret else empty
 
 
@@ -1717,8 +1733,14 @@ def format_portautoneg_test_id(param):
         metafunc.parametrize("enum_dut_all_prio", generate_priority_lists(metafunc, 'all'))
     if 'enum_dut_lossless_prio' in metafunc.fixturenames:
         metafunc.parametrize("enum_dut_lossless_prio", generate_priority_lists(metafunc, 'lossless'))
+    if 'enum_dut_lossless_prio_with_completeness_level' in metafunc.fixturenames:
+        metafunc.parametrize("enum_dut_lossless_prio_with_completeness_level",
+                             generate_priority_lists(metafunc, 'lossless', with_completeness_level=True))
     if 'enum_dut_lossy_prio' in metafunc.fixturenames:
         metafunc.parametrize("enum_dut_lossy_prio", generate_priority_lists(metafunc, 'lossy'))
+    if 'enum_dut_lossy_prio_with_completeness_level' in metafunc.fixturenames:
+        metafunc.parametrize("enum_dut_lossy_prio_with_completeness_level",
+                             generate_priority_lists(metafunc, 'lossy', with_completeness_level=True))
     if 'enum_pfc_pause_delay_test_params' in metafunc.fixturenames:
         metafunc.parametrize("enum_pfc_pause_delay_test_params", pfc_pause_delay_test_params(metafunc))
 

tests/container_checker/test_container_checker.py

@@ -19,7 +19,7 @@
 logger = logging.getLogger(__name__)
 
 pytestmark = [
-    pytest.mark.topology('any'),
+    pytest.mark.topology('any', 't1-multi-asic'),
     pytest.mark.disable_loganalyzer
 ]
 

tests/dhcp_relay/test_dhcp_relay.py

@@ -21,7 +21,7 @@
 from tests.dhcp_relay.dhcp_relay_utils import check_routes_to_dhcp_server, restart_dhcp_service
 
 pytestmark = [
-    pytest.mark.topology('t0', 'm0'),
+    pytest.mark.topology('t0', 'm0', 't0-2vlans'),
     pytest.mark.device_type('vs')
 ]
 

tests/dhcp_relay/test_dhcpv6_relay.py

@@ -21,7 +21,7 @@
 from tests.common.dualtor.dual_tor_common import active_active_ports                                        # noqa F401
 
 pytestmark = [
-    pytest.mark.topology('t0', 'm0', 'mx'),
+    pytest.mark.topology('t0', 'm0', 'mx', 't0-2vlans'),
     pytest.mark.device_type('vs')
 ]
 

tests/http/test_http_copy.py

@@ -5,7 +5,7 @@
 
 pytestmark = [
     pytest.mark.disable_loganalyzer,
-    pytest.mark.topology("any"),
+    pytest.mark.topology("any", "t1-multi-asic"),
     pytest.mark.device_type("vs"),
 ]
 

tests/iface_namingmode/test_iface_namingmode.py

@@ -10,7 +10,7 @@
 from tests.common.helpers.sonic_db import redis_get_keys
 
 pytestmark = [
-    pytest.mark.topology('any')
+    pytest.mark.topology('any', 't1-multi-asic')
 ]
 
 logger = logging.getLogger(__name__)
@@ -317,7 +317,7 @@ def test_show_interfaces_counter(self, setup, setup_config_mode):
             if regex_int.match(line):
                 interfaces.append(regex_int.match(line).group(0))
 
-        assert(len(interfaces) > 0)
+        assert (len(interfaces) > 0)
 
         for item in interfaces:
             if mode == 'alias':
@@ -551,7 +551,7 @@ def test_show_queue_counters(self, setup, setup_config_mode, duthosts, enum_rand
                 intfsChecked += 1
 
         # At least one interface should have been checked to have a valid result
-        assert(intfsChecked > 0)
+        assert (intfsChecked > 0)
 
     def test_show_queue_counters_interface(self, setup_config_mode, sample_intf):
         """

tests/macsec/test_controlplane.py

@@ -13,7 +13,7 @@
 
 pytestmark = [
     pytest.mark.macsec_required,
-    pytest.mark.topology("t0", "t2"),
+    pytest.mark.topology("t0", "t2", "t0-sonic"),
 ]
 
 

tests/macsec/test_dataplane.py

@@ -15,7 +15,7 @@
 
 pytestmark = [
     pytest.mark.macsec_required,
-    pytest.mark.topology("t0", "t2"),
+    pytest.mark.topology("t0", "t2", "t0-sonic"),
 ]
 
 

tests/macsec/test_deployment.py

@@ -8,7 +8,7 @@
 
 pytestmark = [
     pytest.mark.macsec_required,
-    pytest.mark.topology("t0", "t2"),
+    pytest.mark.topology("t0", "t2", "t0-sonic"),
 ]
 
 

tests/macsec/test_fault_handling.py

@@ -12,7 +12,7 @@
 
 pytestmark = [
     pytest.mark.macsec_required,
-    pytest.mark.topology("t0", "t2"),
+    pytest.mark.topology("t0", "t2", "t0-sonic"),
 ]
 
 

tests/macsec/test_interop_protocol.py

@@ -12,7 +12,7 @@
 
 pytestmark = [
     pytest.mark.macsec_required,
-    pytest.mark.topology("t0", "t2"),
+    pytest.mark.topology("t0", "t2", "t0-sonic"),
 ]
 
 

tests/monit/test_monit_status.py

@@ -13,7 +13,7 @@
 logger = logging.getLogger(__name__)
 
 pytestmark = [
-    pytest.mark.topology('any'),
+    pytest.mark.topology('any', 't1-multi-asic'),
     pytest.mark.disable_loganalyzer
 ]
 

tests/pc/test_retry_count.py

@@ -15,7 +15,7 @@
 logger = logging.getLogger(__name__)
 
 pytestmark = [
-    pytest.mark.topology("t0", "t1")
+    pytest.mark.topology("t0", "t1", "t0-sonic")
 ]
 
 

tests/platform_tests/test_advanced_reboot.py

@@ -18,7 +18,7 @@
 
 pytestmark = [
     pytest.mark.disable_loganalyzer,
-    pytest.mark.topology('t0'),
+    pytest.mark.topology('t0', "t0-sonic"),
     pytest.mark.skip_check_dut_health
 ]
 

tests/platform_tests/test_intf_fec.py

@@ -1,7 +1,7 @@
 import logging
 import pytest
 
-from tests.common.utilities import skip_release
+from tests.common.utilities import skip_release, wait_until
 
 pytestmark = [
     pytest.mark.disable_loganalyzer,  # disable automatic loganalyzer
@@ -80,7 +80,7 @@ def test_config_fec_oper_mode(duthosts, enum_rand_one_per_hwsku_frontend_hostnam
         config_status = duthost.command("sudo config interface fec {} rs"
                                         .format(intf['interface']))
         if config_status:
-            duthost.command("sleep 2")
+            wait_until(30, 2, 0, duthost.is_interface_status_up, intf["interface"])
             # Verify the FEC operational mode is restored
             logging.info("Get output of '{} {}'".format("show interfaces fec status", intf['interface']))
             fec_status = duthost.show_and_parse("show interfaces fec status {}".format(intf['interface']))

tests/process_monitoring/test_critical_process_monitoring.py

@@ -23,7 +23,7 @@
 logger = logging.getLogger(__name__)
 
 pytestmark = [
-    pytest.mark.topology('any'),
+    pytest.mark.topology('any', 't1-multi-asic'),
     pytest.mark.disable_loganalyzer
 ]
 

tests/pytest.ini

@@ -11,7 +11,7 @@ markers:
     pretest: tests are run before feature/regression test cases to prepare the DUT/environment.
     posttest: tests are run after feature/regression test cases to cleanup the DUT/environment and/or collect diagnostics.
     sanity_check: override the default sanity check settings
-    topology: specify which topology testcase can be executed on: (t0, t1, ptf, etc)
+    topology: specify which topology testcase can be executed on: (t0, t1, ptf, etc). For subtype topology marks, such as `t0-2vlans`, `t1-multi-asic`, plaese add them only when you confirm that the test script needs to run on this topology.
     platform: specify which platform testcase can be executed on: (physical, virtual, broadcom, mellanox, etc)
     supported_completeness_level: test supported levels of completeness (coverage) level (Debug, Basic, Confident, Thorough)
     skip_check_dut_health: skip default execution of check_dut_health_status fixture

tests/scp/test_scp_copy.py

@@ -3,7 +3,7 @@
 
 pytestmark = [
     pytest.mark.disable_loganalyzer,
-    pytest.mark.topology("any"),
+    pytest.mark.topology("any", "t1-multi-asic"),
     pytest.mark.device_type("vs"),
 ]
 

tests/snappi_tests/multidut/pfc/test_multidut_pfc_pause_lossless_with_snappi.py

@@ -191,7 +191,7 @@ def test_pfc_pause_single_lossless_prio_reboot(snappi_api,                  # no
                                                fanout_graph_facts_multidut,          # noqa: F811
                                                duthosts,
                                                localhost,
-                                               enum_dut_lossless_prio,    # noqa: F811
+                                               enum_dut_lossless_prio_with_completeness_level,    # noqa: F811
                                                prio_dscp_map,            # noqa: F811
                                                lossless_prio_list,         # noqa: F811
                                                all_prio_list,        # noqa: F811
@@ -210,7 +210,7 @@ def test_pfc_pause_single_lossless_prio_reboot(snappi_api,                  # no
         localhost (pytest fixture): localhost handle
         all_prio_list (pytest fixture): list of all the priorities
         prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority).
-        lossless_prio_list (pytest fixture): list of all the lossless priorities
+        enum_dut_lossless_prio_with_completeness_level (str): lossless priority to test, e.g., 's6100-1|3'
         reboot_type (str): reboot type to be issued on the DUT
         tbinfo (pytest fixture): fixture provides information about testbed
         get_snappi_ports (pytest fixture): gets snappi ports and connected DUT port info and returns as a list
@@ -246,7 +246,7 @@ def test_pfc_pause_single_lossless_prio_reboot(snappi_api,                  # no
     skip_warm_reboot(snappi_ports[0]['duthost'], reboot_type)
     skip_warm_reboot(snappi_ports[1]['duthost'], reboot_type)
 
-    _, lossless_prio = enum_dut_lossless_prio.split('|')
+    _, lossless_prio = enum_dut_lossless_prio_with_completeness_level.split('|')
     lossless_prio = int(lossless_prio)
     pause_prio_list = [lossless_prio]
     test_prio_list = [lossless_prio]
@@ -257,7 +257,7 @@ def test_pfc_pause_single_lossless_prio_reboot(snappi_api,                  # no
     snappi_extra_params = SnappiTestParams()
     snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports
 
-    for duthost in [snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]:
+    for duthost in set([snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]):
         logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost.hostname))
         reboot(duthost, localhost, reboot_type=reboot_type, safe_reboot=True)
         logger.info("Wait until the system is stable")
@@ -347,7 +347,7 @@ def test_pfc_pause_multi_lossless_prio_reboot(snappi_api,                  # noq
     snappi_extra_params = SnappiTestParams()
     snappi_extra_params.multi_dut_params.multi_dut_ports = snappi_ports
 
-    for duthost in [snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]:
+    for duthost in set([snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]):
         logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost.hostname))
         reboot(duthost, localhost, reboot_type=reboot_type, safe_reboot=True)
         logger.info("Wait until the system is stable")

tests/snappi_tests/multidut/pfc/test_multidut_pfc_pause_lossy_with_snappi.py

@@ -185,7 +185,7 @@ def test_pfc_pause_single_lossy_prio_reboot(snappi_api,             # noqa: F811
                                             fanout_graph_facts_multidut,     # noqa: F811
                                             duthosts,
                                             localhost,
-                                            enum_dut_lossy_prio,
+                                            enum_dut_lossy_prio_with_completeness_level,
                                             prio_dscp_map,                   # noqa: F811
                                             lossy_prio_list,              # noqa: F811
                                             all_prio_list,                   # noqa: F811
@@ -202,7 +202,7 @@ def test_pfc_pause_single_lossy_prio_reboot(snappi_api,             # noqa: F811
         fanout_graph_facts (pytest fixture): fanout graph
         duthosts (pytest fixture): list of DUTs
         localhost (pytest fixture): localhost handle
-        enum_dut_lossy_prio (str): name of lossy priority to test, e.g., 's6100-1|2'
+        enum_dut_lossy_prio_with_completeness_level (str): lossy priority to test, e.g., 's6100-1|2'
         prio_dscp_map (pytest fixture): priority vs. DSCP map (key = priority).
         lossy_prio_list (pytest fixture): list of all the lossy priorities
         all_prio_list (pytest fixture): list of all the priorities
@@ -241,14 +241,14 @@ def test_pfc_pause_single_lossy_prio_reboot(snappi_api,             # noqa: F811
     skip_warm_reboot(snappi_ports[0]['duthost'], reboot_type)
     skip_warm_reboot(snappi_ports[1]['duthost'], reboot_type)
 
-    _, lossy_prio = enum_dut_lossy_prio.split('|')
+    _, lossy_prio = enum_dut_lossy_prio_with_completeness_level.split('|')
     lossy_prio = int(lossy_prio)
     pause_prio_list = [lossy_prio]
     test_prio_list = [lossy_prio]
     bg_prio_list = [p for p in all_prio_list]
     bg_prio_list.remove(lossy_prio)
 
-    for duthost in [snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]:
+    for duthost in set([snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]):
         logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost.hostname))
         reboot(duthost, localhost, reboot_type=reboot_type, safe_reboot=True)
         logger.info("Wait until the system is stable")
@@ -338,7 +338,7 @@ def test_pfc_pause_multi_lossy_prio_reboot(snappi_api,          # noqa: F811
     test_prio_list = lossy_prio_list
     bg_prio_list = lossless_prio_list
 
-    for duthost in [snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]:
+    for duthost in set([snappi_ports[0]['duthost'], snappi_ports[1]['duthost']]):
         logger.info("Issuing a {} reboot on the dut {}".format(reboot_type, duthost.hostname))
         reboot(duthost, localhost, reboot_type=reboot_type, safe_reboot=True)
         logger.info("Wait until the system is stable")