Merge pull request #15565 from LabNConsulting/chopps/code-cover

tests: enable code coverage reporting with topotests

doc/developer/topotests.rst

@@ -704,6 +704,44 @@ Here's an example of collecting ``rr`` execution state from ``mgmtd`` on router
 To specify additional arguments for ``rr record``, one can use the
 ``--rr-options``.
 
+.. _code_coverage:
+
+Code coverage
+"""""""""""""
+Code coverage reporting requires installation of the ``gcov`` and ``lcov``
+packages.
+
+Code coverage can automatically be gathered for any topotest run. To support
+this FRR must first be compiled with the ``--enable-gcov`` configure option.
+This will cause *.gnco files to be created during the build. When topotests are
+run the statistics are generated and stored in *.gcda files. Topotest
+infrastructure will gather these files, capture the information into a
+``coverage.info`` ``lcov`` file and also report the coverage summary.
+
+To enable code coverage support pass the ``--cov-topotest`` argument to pytest.
+If you build your FRR in a directory outside of the FRR source directory you
+will also need to pass the ``--cov-frr-build-dir`` argument specifying the build
+directory location.
+
+During the topotest run the *.gcda files are generated into a ``gcda``
+sub-directory of the top-level run directory (i.e., normally
+``/tmp/topotests/gcda``). These files will then be copied at the end of the
+topotest run into the FRR build directory where the ``gcov`` and ``lcov``
+utilities expect to find them. This is done to deal with the various different
+file ownership and permissions.
+
+At the end of the run ``lcov`` will be run to capture all of the coverage data
+into a ``coverage.info`` file. This file will be located in the top-level run
+directory (i.e., normally ``/tmp/topotests/coverage.info``).
+
+The ``coverage.info`` file can then be used to generate coverage reports or file
+markup (e.g., using the ``genhtml`` utility) or enable markup within your
+IDE/editor if supported (e.g., the emacs ``cov-mode`` package)
+
+NOTE: the *.gcda files in ``/tmp/topotests/gcda`` are cumulative so if you do
+not remove them they will aggregate data across multiple topotest runs.
+
+
 .. _topotests_docker:
 
 Running Tests with Docker

lib/libfrr.c

@@ -319,7 +319,12 @@ void frr_preinit(struct frr_daemon_info *daemon, int argc, char **argv)
 	char *p = strrchr(argv[0], '/');
 	di->progname = p ? p + 1 : argv[0];
 
-	umask(0027);
+	if (!getenv("GCOV_PREFIX"))
+		umask(0027);
+	else {
+		/* If we are profiling use a more generous umask */
+		umask(0002);
+	}
 
 	log_args_init(daemon->early_logging);
 

tests/topotests/conftest.py

@@ -68,6 +68,10 @@ def log_handler(basename, logpath):
         topolog.logfinish(basename, logpath)
 
 
+def is_main_runner():
+    return "PYTEST_XDIST_WORKER" not in os.environ
+
+
 def pytest_addoption(parser):
     """
     Add topology-only option to the topology tester. This option makes pytest
@@ -85,6 +89,17 @@ def pytest_addoption(parser):
         help="Mininet cli on test failure",
     )
 
+    parser.addoption(
+        "--cov-topotest",
+        action="store_true",
+        help="Enable reporting of coverage",
+    )
+
+    parser.addoption(
+        "--cov-frr-build-dir",
+        help="Dir of coverage-enable build being run, default is the source dir",
+    )
+
     parser.addoption(
         "--gdb-breakpoints",
         metavar="SYMBOL[,SYMBOL...]",
@@ -456,6 +471,37 @@ def pytest_assertrepr_compare(op, left, right):
     return json_result.gen_report()
 
 
+def setup_coverage(config):
+    commander = Commander("pytest")
+    if config.option.cov_frr_build_dir:
+        bdir = Path(config.option.cov_frr_build_dir).resolve()
+        output = commander.cmd_raises(f"find {bdir} -name zebra_nb.gcno").strip()
+    else:
+        # Support build sub-directory of main source dir
+        bdir = Path(__file__).resolve().parent.parent.parent
+        output = commander.cmd_raises(f"find {bdir} -name zebra_nb.gcno").strip()
+    m = re.match(f"({bdir}.*)/zebra/zebra_nb.gcno", output)
+    if not m:
+        logger.warning(
+            "No coverage data files (*.gcno) found, try specifying --cov-frr-build-dir"
+        )
+        return
+
+    bdir = Path(m.group(1))
+    # Save so we can get later from g_pytest_config
+    rundir = Path(config.option.rundir).resolve()
+    gcdadir = rundir / "gcda"
+    os.environ["FRR_BUILD_DIR"] = str(bdir)
+    os.environ["GCOV_PREFIX_STRIP"] = str(len(bdir.parts) - 1)
+    os.environ["GCOV_PREFIX"] = str(gcdadir)
+
+    if is_main_runner():
+        commander.cmd_raises(f"find {bdir} -name '*.gc??' -exec chmod o+rw {{}} +")
+        commander.cmd_raises(f"mkdir -p {gcdadir}")
+        commander.cmd_raises(f"chown -R root:frr {gcdadir}")
+        commander.cmd_raises(f"chmod 2775 {gcdadir}")
+
+
 def pytest_configure(config):
     """
     Assert that the environment is correctly configured, and get extra config.
@@ -556,8 +602,6 @@ def assert_feature_windows(b, feature):
     if config.option.topology_only and is_xdist:
         pytest.exit("Cannot use --topology-only with distributed test mode")
 
-        pytest.exit("Cannot use --topology-only with distributed test mode")
-
     # Check environment now that we have config
     if not diagnose_env(rundir):
         pytest.exit("environment has errors, please read the logs in %s" % rundir)
@@ -572,27 +616,25 @@ def assert_feature_windows(b, feature):
         if "TOPOTESTS_CHECK_STDERR" in os.environ:
             del os.environ["TOPOTESTS_CHECK_STDERR"]
 
+    if config.option.cov_topotest:
+        setup_coverage(config)
+
 
 @pytest.fixture(autouse=True, scope="session")
-def setup_session_auto():
+def session_autouse():
     # Aligns logs nicely
     logging.addLevelName(logging.WARNING, " WARN")
     logging.addLevelName(logging.INFO, " INFO")
 
-    if "PYTEST_TOPOTEST_WORKER" not in os.environ:
-        is_worker = False
-    elif not os.environ["PYTEST_TOPOTEST_WORKER"]:
-        is_worker = False
-    else:
-        is_worker = True
+    is_main = is_main_runner()
 
-    logger.debug("Before the run (is_worker: %s)", is_worker)
-    if not is_worker:
+    logger.debug("Before the run (is_main: %s)", is_main)
+    if is_main:
         cleanup_previous()
     yield
-    if not is_worker:
+    if is_main:
         cleanup_current()
-    logger.debug("After the run (is_worker: %s)", is_worker)
+    logger.debug("After the run (is_main: %s)", is_main)
 
 
 def pytest_runtest_setup(item):
@@ -719,6 +761,42 @@ def pytest_runtest_makereport(item, call):
         pause_test()
 
 
+def coverage_finish(terminalreporter, config):
+    commander = Commander("pytest")
+    rundir = Path(config.option.rundir).resolve()
+    bdir = Path(os.environ["FRR_BUILD_DIR"])
+    gcdadir = Path(os.environ["GCOV_PREFIX"])
+
+    # Get the data files into the build directory
+    logger.info("Copying gcda files from '%s' to '%s'", gcdadir, bdir)
+    user = os.environ.get("SUDO_USER", os.environ["USER"])
+    commander.cmd_raises(f"chmod -R ugo+r {gcdadir}")
+    commander.cmd_raises(
+        f"tar -C {gcdadir} -cf - . | su {user} -c 'tar -C {bdir} -xf -'"
+    )
+
+    # Get the results into a summary file
+    data_file = rundir / "coverage.info"
+    logger.info("Gathering coverage data into: %s", data_file)
+    commander.cmd_raises(f"lcov --directory {bdir} --capture --output-file {data_file}")
+
+    # Get coverage info filtered to a specific set of files
+    report_file = rundir / "coverage.info"
+    logger.debug("Generating coverage summary from: %s\n%s", report_file)
+    output = commander.cmd_raises(f"lcov --summary {data_file}")
+    logger.info("\nCOVERAGE-SUMMARY-START\n%s\nCOVERAGE-SUMMARY-END", output)
+    terminalreporter.write(
+        f"\nCOVERAGE-SUMMARY-START\n{output}\nCOVERAGE-SUMMARY-END\n"
+    )
+
+
+def pytest_terminal_summary(terminalreporter, exitstatus, config):
+    # Only run if we are the top level test runner
+    is_xdist_worker = "PYTEST_XDIST_WORKER" in os.environ
+    if config.option.cov_topotest and not is_xdist_worker:
+        coverage_finish(terminalreporter, config)
+
+
 #
 # Add common fixtures available to all tests as parameters
 #

tests/topotests/lib/topotest.py

@@ -27,6 +27,7 @@
 import logging
 from collections.abc import Mapping
 from copy import deepcopy
+from pathlib import Path
 
 import lib.topolog as topolog
 from lib.micronet_compat import Node
@@ -1262,8 +1263,8 @@ def rlimit_atleast(rname, min_value, raises=False):
 
 def fix_netns_limits(ns):
     # Maximum read and write socket buffer sizes
-    sysctl_atleast(ns, "net.ipv4.tcp_rmem", [10 * 1024, 87380, 16 * 2 ** 20])
-    sysctl_atleast(ns, "net.ipv4.tcp_wmem", [10 * 1024, 87380, 16 * 2 ** 20])
+    sysctl_atleast(ns, "net.ipv4.tcp_rmem", [10 * 1024, 87380, 16 * 2**20])
+    sysctl_atleast(ns, "net.ipv4.tcp_wmem", [10 * 1024, 87380, 16 * 2**20])
 
     sysctl_assure(ns, "net.ipv4.conf.all.rp_filter", 0)
     sysctl_assure(ns, "net.ipv4.conf.default.rp_filter", 0)
@@ -1322,8 +1323,8 @@ def fix_host_limits():
     sysctl_atleast(None, "net.core.netdev_max_backlog", 4 * 1024)
 
     # Maximum read and write socket buffer sizes
-    sysctl_atleast(None, "net.core.rmem_max", 16 * 2 ** 20)
-    sysctl_atleast(None, "net.core.wmem_max", 16 * 2 ** 20)
+    sysctl_atleast(None, "net.core.rmem_max", 16 * 2**20)
+    sysctl_atleast(None, "net.core.wmem_max", 16 * 2**20)
 
     # Garbage Collection Settings for ARP and Neighbors
     sysctl_atleast(None, "net.ipv4.neigh.default.gc_thresh2", 4 * 1024)
@@ -1523,7 +1524,7 @@ def listDaemons(self):
                 pass
         return ret
 
-    def stopRouter(self, assertOnError=True, minErrorVersion="5.1"):
+    def stopRouter(self, assertOnError=True):
         # Stop Running FRR Daemons
         running = self.listDaemons()
         if not running:
@@ -1570,9 +1571,6 @@ def stopRouter(self, assertOnError=True, minErrorVersion="5.1"):
             )
 
         errors = self.checkRouterCores(reportOnce=True)
-        if self.checkRouterVersion("<", minErrorVersion):
-            # ignore errors in old versions
-            errors = ""
         if assertOnError and (errors is not None) and len(errors) > 0:
             assert "Errors found - details follow:" == 0, errors
         return errors
@@ -1803,6 +1801,8 @@ def startRouterDaemons(self, daemons=None, tgen=None):
         "Starts FRR daemons for this router."
 
         asan_abort = bool(g_pytest_config.option.asan_abort)
+        cov_option = bool(g_pytest_config.option.cov_topotest)
+        cov_dir = Path(g_pytest_config.option.rundir) / "gcda"
         gdb_breakpoints = g_pytest_config.get_option_list("--gdb-breakpoints")
         gdb_daemons = g_pytest_config.get_option_list("--gdb-daemons")
         gdb_routers = g_pytest_config.get_option_list("--gdb-routers")
@@ -1836,13 +1836,6 @@ def startRouterDaemons(self, daemons=None, tgen=None):
         # Re-enable to allow for report per run
         self.reportCores = True
 
-        # XXX: glue code forward ported from removed function.
-        if self.version is None:
-            self.version = self.cmd(
-                os.path.join(self.daemondir, "bgpd") + " -v"
-            ).split()[2]
-            logger.info("{}: running version: {}".format(self.name, self.version))
-
         perfds = {}
         perf_options = g_pytest_config.get_option("--perf-options", "-g")
         for perf in g_pytest_config.get_option("--perf", []):
@@ -1928,6 +1921,10 @@ def do_gdb_or_rr(gdb):
                     self.logdir, self.name, daemon
                 )
 
+                if cov_option:
+                    scount = os.environ["GCOV_PREFIX_STRIP"]
+                    cmdenv += f"GCOV_PREFIX_STRIP={scount} GCOV_PREFIX={cov_dir}"
+
                 if valgrind_memleaks:
                     this_dir = os.path.dirname(
                         os.path.abspath(os.path.realpath(__file__))
@@ -2277,9 +2274,7 @@ def pid_exists(self, pid):
         rc, o, e = self.cmd_status("kill -0 " + str(pid), warn=False)
         return rc == 0 or "No such process" not in e
 
-    def killRouterDaemons(
-        self, daemons, wait=True, assertOnError=True, minErrorVersion="5.1"
-    ):
+    def killRouterDaemons(self, daemons, wait=True, assertOnError=True):
         # Kill Running FRR
         # Daemons(user specified daemon only) using SIGKILL
         rundaemons = self.cmd("ls -1 /var/run/%s/*.pid" % self.routertype)
@@ -2339,9 +2334,6 @@ def killRouterDaemons(
                         self.cmd("rm -- {}".format(daemonpidfile))
                     if wait:
                         errors = self.checkRouterCores(reportOnce=True)
-                        if self.checkRouterVersion("<", minErrorVersion):
-                            # ignore errors in old versions
-                            errors = ""
                         if assertOnError and len(errors) > 0:
                             assert "Errors found - details follow:" == 0, errors
             else: