tests: Modify tests to allow random order running

You can achieve this now by installing pytest-random-order:

pip3 install pytest-random-order

Then running pytest with this:

python3 -m pytest --random-order=package -nauto --dist=loadfile

Tests will be run in a random order.

Signed-off-by: Donald Sharp <[email protected]>

tests/topotests/all_protocol_startup/test_all_protocol_startup.py

@@ -21,7 +21,7 @@
 import glob
 from time import sleep
 
-pytestmark = [
+pytestmark = [pytest.mark.random_order(disabled=True),
     pytest.mark.babeld,
     pytest.mark.bgpd,
     pytest.mark.isisd,

tests/topotests/babel_topo1/test_babel_topo1.py

@@ -20,7 +20,7 @@
 import json
 from functools import partial
 
-pytestmark = [pytest.mark.babeld]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.babeld]
 
 # Save the Current Working Directory to find configuration files.
 CWD = os.path.dirname(os.path.realpath(__file__))

tests/topotests/bfd_bgp_cbit_topo3/test_bfd_bgp_cbit_topo3.py

@@ -28,7 +28,7 @@
 from lib.topogen import Topogen, TopoRouter, get_topogen
 from lib.topolog import logger
 
-pytestmark = [pytest.mark.bgpd, pytest.mark.bfdd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd, pytest.mark.bfdd]
 
 
 def setup_module(mod):

tests/topotests/bfd_isis_topo1/test_bfd_isis_topo1.py

@@ -71,7 +71,7 @@
 from lib.topogen import Topogen, TopoRouter, get_topogen
 from lib.topolog import logger
 
-pytestmark = [pytest.mark.bfdd, pytest.mark.isisd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bfdd, pytest.mark.isisd]
 
 
 def setup_module(mod):

tests/topotests/bfd_ospf_topo1/test_bfd_ospf_topo1.py

@@ -71,7 +71,7 @@
 from lib.topogen import Topogen, TopoRouter, get_topogen
 from lib.topolog import logger
 
-pytestmark = [pytest.mark.bfdd, pytest.mark.ospfd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bfdd, pytest.mark.ospfd]
 
 
 def setup_module(mod):

tests/topotests/bfd_profiles_topo1/test_bfd_profiles_topo1.py

@@ -29,7 +29,7 @@
 from lib.topogen import Topogen, TopoRouter, get_topogen
 from lib.topolog import logger
 
-pytestmark = [pytest.mark.bfdd, pytest.mark.bgpd, pytest.mark.isisd, pytest.mark.ospfd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bfdd, pytest.mark.bgpd, pytest.mark.isisd, pytest.mark.ospfd]
 
 
 def setup_module(mod):

tests/topotests/bfd_topo1/test_bfd_topo1.py

@@ -29,7 +29,7 @@
 from lib.topogen import Topogen, TopoRouter, get_topogen
 from lib.topolog import logger
 
-pytestmark = [pytest.mark.bfdd, pytest.mark.bgpd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bfdd, pytest.mark.bgpd]
 
 
 def setup_module(mod):

tests/topotests/bfd_topo2/test_bfd_topo2.py

@@ -30,7 +30,7 @@
 from lib.topogen import Topogen, TopoRouter, get_topogen
 from lib.topolog import logger
 
-pytestmark = [pytest.mark.bfdd, pytest.mark.bgpd, pytest.mark.ospfd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bfdd, pytest.mark.bgpd, pytest.mark.ospfd]
 
 
 def setup_module(mod):

tests/topotests/bfd_topo3/test_bfd_topo3.py

@@ -29,7 +29,7 @@
 from lib.topogen import Topogen, TopoRouter, get_topogen
 from lib.topolog import logger
 
-pytestmark = [pytest.mark.bfdd, pytest.mark.bgpd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bfdd, pytest.mark.bgpd]
 
 
 def setup_module(mod):

tests/topotests/bfd_vrf_topo1/test_bfd_vrf_topo1.py

@@ -32,7 +32,7 @@
 
 # Required to instantiate the topology builder class.
 
-pytestmark = [pytest.mark.bfdd, pytest.mark.bgpd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bfdd, pytest.mark.bgpd]
 
 
 def build_topo(tgen):

tests/topotests/bfd_vrflite_topo1/test_bfd_vrflite_topo1.py

@@ -32,7 +32,7 @@
 
 # Required to instantiate the topology builder class.
 
-pytestmark = [pytest.mark.bfdd, pytest.mark.bgpd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bfdd, pytest.mark.bgpd]
 
 
 def build_topo(tgen):

tests/topotests/bgp_6vpe_ebgp_topo1/test_bgp_6vpe_ebgp_topo1.py

@@ -26,7 +26,7 @@
 from lib.topolog import logger
 from lib.checkping import check_ping
 
-pytestmark = [pytest.mark.bgpd, pytest.mark.isisd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd, pytest.mark.isisd]
 
 
 def build_topo(tgen):

tests/topotests/bgp_accept_own/test_bgp_accept_own.py

@@ -24,7 +24,7 @@
 from lib.topogen import Topogen, TopoRouter, get_topogen
 from lib.common_config import step
 
-pytestmark = [pytest.mark.bgpd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd]
 
 
 def build_topo(tgen):

tests/topotests/bgp_addpath_best_selected/test_bgp_addpath_best_selected.py

@@ -22,7 +22,7 @@
 from lib import topotest
 from lib.topogen import Topogen, TopoRouter, get_topogen
 
-pytestmark = [pytest.mark.bgpd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd]
 
 
 def build_topo(tgen):

tests/topotests/bgp_addpath_paths_limit/test_bgp_addpath_paths_limit.py

@@ -22,7 +22,7 @@
 from lib import topotest
 from lib.topogen import Topogen, get_topogen
 
-pytestmark = [pytest.mark.bgpd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd]
 
 
 def build_topo(tgen):

tests/topotests/bgp_aggregate_address_matching_med/test_bgp_aggregate_address_matching_med.py

@@ -24,7 +24,7 @@
 from lib import topotest
 from lib.topogen import Topogen, TopoRouter, get_topogen
 
-pytestmark = [pytest.mark.bgpd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd]
 
 CWD = os.path.dirname(os.path.realpath(__file__))
 sys.path.append(os.path.join(CWD, "../"))

tests/topotests/bgp_aggregate_address_origin/test_bgp_aggregate-address_origin.py

@@ -31,7 +31,7 @@
 from lib import topotest
 from lib.topogen import Topogen, TopoRouter, get_topogen
 
-pytestmark = [pytest.mark.bgpd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd]
 
 
 def build_topo(tgen):

tests/topotests/bgp_aggregate_address_route_map/test_bgp_aggregate-address_route-map.py

@@ -34,7 +34,7 @@
 from lib import topotest
 from lib.topogen import Topogen, TopoRouter, get_topogen
 
-pytestmark = [pytest.mark.bgpd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd]
 
 
 def build_topo(tgen):

tests/topotests/bgp_aggregate_address_topo1/test_bgp_aggregate_address_topo1.py

@@ -26,7 +26,7 @@
 from lib.topogen import Topogen, TopoRouter, get_topogen
 from lib.topolog import logger
 
-pytestmark = [pytest.mark.bgpd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd]
 
 
 def build_topo(tgen):

tests/topotests/bgp_aggregator_zero/test_bgp_aggregator_zero.py

@@ -24,7 +24,7 @@
 from lib import topotest
 from lib.topogen import Topogen, TopoRouter, get_topogen
 
-pytestmark = [pytest.mark.bgpd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd]
 
 
 def build_topo(tgen):

tests/topotests/bgp_aigp/test_bgp_aigp.py

@@ -30,7 +30,7 @@
 from lib import topotest
 from lib.topogen import Topogen, TopoRouter, get_topogen
 
-pytestmark = [pytest.mark.bgpd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd]
 
 
 def build_topo(tgen):

tests/topotests/bgp_always_compare_med/test_bgp_always_compare_med_topo1.py

@@ -55,7 +55,7 @@
 from lib.bgp import verify_bgp_convergence, verify_bgp_rib, create_router_bgp, clear_bgp
 from lib.topojson import build_config_from_json
 
-pytestmark = [pytest.mark.bgpd, pytest.mark.staticd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd, pytest.mark.staticd]
 
 # Reading the data from JSON File for topology creation
 topo = None

tests/topotests/bgp_as_allow_in/test_bgp_as_allow_in.py

@@ -60,7 +60,7 @@
 )
 from lib.topojson import build_config_from_json
 
-pytestmark = [pytest.mark.bgpd, pytest.mark.staticd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd, pytest.mark.staticd]
 
 # Global variables
 BGP_CONVERGENCE = False

tests/topotests/bgp_as_override/test_bgp_as_override.py

@@ -23,7 +23,7 @@
 from lib.topogen import Topogen, TopoRouter, get_topogen
 from lib.common_config import step
 
-pytestmark = [pytest.mark.bgpd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd]
 
 
 def build_topo(tgen):

tests/topotests/bgp_as_wide_bgp_identifier/test_bgp_as_wide_bgp_identifier.py

@@ -29,7 +29,7 @@
 from lib import topotest
 from lib.topogen import Topogen, TopoRouter, get_topogen
 
-pytestmark = [pytest.mark.bgpd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd]
 
 
 def build_topo(tgen):

tests/topotests/bgp_asdot_regex/test_bgp_asdot_regex.py

@@ -34,7 +34,7 @@
 from functools import partial
 
 # add after imports, before defining classes or functions:
-pytestmark = [pytest.mark.bgpd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd]
 
 CWD = os.path.dirname(os.path.realpath(__file__))
 sys.path.append(os.path.join(CWD, "../"))

tests/topotests/bgp_aspath_zero/test_bgp_aspath_zero.py

@@ -24,7 +24,7 @@
 from lib import topotest
 from lib.topogen import Topogen, TopoRouter, get_topogen
 
-pytestmark = [pytest.mark.bgpd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd]
 
 
 def build_topo(tgen):

tests/topotests/bgp_auth/test_bgp_auth1.py

@@ -50,7 +50,7 @@
 )
 from lib.topogen import Topogen, TopoRouter, get_topogen
 
-pytestmark = [pytest.mark.bgpd, pytest.mark.ospfd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd, pytest.mark.ospfd]
 
 CWD = os.path.dirname(os.path.realpath(__file__))
 

tests/topotests/bgp_auth/test_bgp_auth2.py

@@ -50,7 +50,7 @@
 )
 from lib.topogen import Topogen, TopoRouter, get_topogen
 
-pytestmark = [pytest.mark.bgpd, pytest.mark.ospfd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd, pytest.mark.ospfd]
 
 CWD = os.path.dirname(os.path.realpath(__file__))
 

tests/topotests/bgp_auth/test_bgp_auth3.py

@@ -49,7 +49,7 @@
 )
 from lib.topogen import Topogen, TopoRouter, get_topogen
 
-pytestmark = [pytest.mark.bgpd, pytest.mark.ospfd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd, pytest.mark.ospfd]
 
 CWD = os.path.dirname(os.path.realpath(__file__))
 

tests/topotests/bgp_auth/test_bgp_auth4.py

@@ -49,7 +49,7 @@
 )
 from lib.topogen import Topogen, TopoRouter, get_topogen
 
-pytestmark = [pytest.mark.bgpd, pytest.mark.ospfd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd, pytest.mark.ospfd]
 
 CWD = os.path.dirname(os.path.realpath(__file__))
 

tests/topotests/bgp_basic_functionality_topo1/test_bgp_basic_functionality.py

@@ -78,7 +78,7 @@
 from lib.topojson import build_config_from_json
 from lib.topolog import logger
 
-pytestmark = [pytest.mark.bgpd, pytest.mark.staticd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd, pytest.mark.staticd]
 
 
 # Global Variable

tests/topotests/bgp_bfd_down_cease_notification/test_bgp_bfd_down_cease_notification.py

@@ -27,7 +27,7 @@
 from lib.topogen import Topogen, TopoRouter, get_topogen
 from lib.common_config import kill_router_daemons, step
 
-pytestmark = [pytest.mark.bfdd, pytest.mark.bgpd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bfdd, pytest.mark.bgpd]
 
 
 def build_topo(tgen):

tests/topotests/bgp_bfd_down_cease_notification/test_bgp_bfd_down_cease_notification_shutdown.py

@@ -27,7 +27,7 @@
 from lib.topogen import Topogen, TopoRouter, get_topogen
 from lib.common_config import kill_router_daemons, step
 
-pytestmark = [pytest.mark.bfdd, pytest.mark.bgpd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bfdd, pytest.mark.bgpd]
 
 
 def build_topo(tgen):

tests/topotests/bgp_blackhole_community/test_bgp_blackhole_community.py

@@ -25,7 +25,7 @@
 from lib.topogen import Topogen, TopoRouter, get_topogen
 from lib.common_config import step
 
-pytestmark = [pytest.mark.bgpd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd]
 
 
 def build_topo(tgen):

tests/topotests/bgp_bmp/test_bgp_bmp.py

@@ -41,7 +41,7 @@
 from lib.topogen import Topogen, TopoRouter, get_topogen
 from lib.topolog import logger
 
-pytestmark = [pytest.mark.bgpd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd]
 
 # remember the last sequence number of the logging messages
 SEQ = 0

tests/topotests/bgp_color_extcommunities/test_bgp_color_extcommunities.py

@@ -28,7 +28,7 @@
 from lib.topogen import Topogen, TopoRouter, get_topogen
 from lib.topolog import logger
 
-pytestmark = [pytest.mark.bgpd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd]
 
 
 def build_topo(tgen):

tests/topotests/bgp_comm_list_delete/test_bgp_comm-list_delete.py

@@ -30,7 +30,7 @@
 from lib import topotest
 from lib.topogen import Topogen, TopoRouter, get_topogen
 
-pytestmark = [pytest.mark.bgpd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd]
 
 
 def build_topo(tgen):

tests/topotests/bgp_comm_list_match/test_bgp_comm_list_match.py

@@ -35,7 +35,7 @@
 from lib.topogen import Topogen, TopoRouter, get_topogen
 from lib.common_config import step
 
-pytestmark = [pytest.mark.bgpd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd]
 
 
 def build_topo(tgen):

tests/topotests/bgp_communities_topo1/test_bgp_communities.py

@@ -50,7 +50,7 @@
 from lib.topojson import build_config_from_json
 from copy import deepcopy
 
-pytestmark = [pytest.mark.bgpd, pytest.mark.staticd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd, pytest.mark.staticd]
 
 
 # Global variables

tests/topotests/bgp_communities_topo1/test_bgp_communities_topo2.py

@@ -52,7 +52,7 @@
 )
 from lib.topojson import build_config_from_json
 
-pytestmark = [pytest.mark.bgpd, pytest.mark.staticd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd, pytest.mark.staticd]
 
 
 # Global variables

tests/topotests/bgp_community_change_update/test_bgp_community_change_update.py

@@ -44,7 +44,7 @@
 from lib.common_config import step
 from time import sleep
 
-pytestmark = [pytest.mark.bgpd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd]
 
 
 def build_topo(tgen):

tests/topotests/bgp_conditional_advertisement/test_bgp_conditional_advertisement.py

@@ -130,7 +130,7 @@
 from lib.topogen import Topogen, TopoRouter, get_topogen
 from lib.topolog import logger
 
-pytestmark = [pytest.mark.bgpd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd]
 
 
 def build_topo(tgen):

tests/topotests/bgp_conditional_advertisement_static_route/test_bgp_conditional_advertisement_static_route.py

@@ -16,7 +16,7 @@
 import pytest
 import functools
 
-pytestmark = [pytest.mark.bgpd]
+pytestmark = [pytest.mark.random_order(disabled=True), pytest.mark.bgpd]
 
 CWD = os.path.dirname(os.path.realpath(__file__))
 sys.path.append(os.path.join(CWD, "../"))