diff --git a/doc/developer/topotests.rst b/doc/developer/topotests.rst index 3af4048ed4cb..0511353f3f39 100644 --- a/doc/developer/topotests.rst +++ b/doc/developer/topotests.rst @@ -704,6 +704,44 @@ Here's an example of collecting ``rr`` execution state from ``mgmtd`` on router To specify additional arguments for ``rr record``, one can use the ``--rr-options``. +.. _code_coverage: + +Code coverage +""""""""""""" +Code coverage reporting requires installation of the ``gcov`` and ``lcov`` +packages. + +Code coverage can automatically be gathered for any topotest run. To support +this FRR must first be compiled with the ``--enable-gcov`` configure option. +This will cause *.gnco files to be created during the build. When topotests are +run the statistics are generated and stored in *.gcda files. Topotest +infrastructure will gather these files, capture the information into a +``coverage.info`` ``lcov`` file and also report the coverage summary. + +To enable code coverage support pass the ``--cov-topotest`` argument to pytest. +If you build your FRR in a directory outside of the FRR source directory you +will also need to pass the ``--cov-frr-build-dir`` argument specifying the build +directory location. + +During the topotest run the *.gcda files are generated into a ``gcda`` +sub-directory of the top-level run directory (i.e., normally +``/tmp/topotests/gcda``). These files will then be copied at the end of the +topotest run into the FRR build directory where the ``gcov`` and ``lcov`` +utilities expect to find them. This is done to deal with the various different +file ownership and permissions. + +At the end of the run ``lcov`` will be run to capture all of the coverage data +into a ``coverage.info`` file. This file will be located in the top-level run +directory (i.e., normally ``/tmp/topotests/coverage.info``). + +The ``coverage.info`` file can then be used to generate coverage reports or file +markup (e.g., using the ``genhtml`` utility) or enable markup within your +IDE/editor if supported (e.g., the emacs ``cov-mode`` package) + +NOTE: the *.gcda files in ``/tmp/topotests/gcda`` are cumulative so if you do +not remove them they will aggregate data across multiple topotest runs. + + .. _topotests_docker: Running Tests with Docker diff --git a/lib/libfrr.c b/lib/libfrr.c index 2861ebe0b933..c5c7e7837a79 100644 --- a/lib/libfrr.c +++ b/lib/libfrr.c @@ -319,7 +319,12 @@ void frr_preinit(struct frr_daemon_info *daemon, int argc, char **argv) char *p = strrchr(argv[0], '/'); di->progname = p ? p + 1 : argv[0]; - umask(0027); + if (!getenv("GCOV_PREFIX")) + umask(0027); + else { + /* If we are profiling use a more generous umask */ + umask(0002); + } log_args_init(daemon->early_logging); diff --git a/tests/topotests/conftest.py b/tests/topotests/conftest.py index 23eab68db426..c6f038b7f670 100755 --- a/tests/topotests/conftest.py +++ b/tests/topotests/conftest.py @@ -68,6 +68,10 @@ def log_handler(basename, logpath): topolog.logfinish(basename, logpath) +def is_main_runner(): + return "PYTEST_XDIST_WORKER" not in os.environ + + def pytest_addoption(parser): """ Add topology-only option to the topology tester. This option makes pytest @@ -85,6 +89,17 @@ def pytest_addoption(parser): help="Mininet cli on test failure", ) + parser.addoption( + "--cov-topotest", + action="store_true", + help="Enable reporting of coverage", + ) + + parser.addoption( + "--cov-frr-build-dir", + help="Dir of coverage-enable build being run, default is the source dir", + ) + parser.addoption( "--gdb-breakpoints", metavar="SYMBOL[,SYMBOL...]", @@ -456,6 +471,37 @@ def pytest_assertrepr_compare(op, left, right): return json_result.gen_report() +def setup_coverage(config): + commander = Commander("pytest") + if config.option.cov_frr_build_dir: + bdir = Path(config.option.cov_frr_build_dir).resolve() + output = commander.cmd_raises(f"find {bdir} -name zebra_nb.gcno").strip() + else: + # Support build sub-directory of main source dir + bdir = Path(__file__).resolve().parent.parent.parent + output = commander.cmd_raises(f"find {bdir} -name zebra_nb.gcno").strip() + m = re.match(f"({bdir}.*)/zebra/zebra_nb.gcno", output) + if not m: + logger.warning( + "No coverage data files (*.gcno) found, try specifying --cov-frr-build-dir" + ) + return + + bdir = Path(m.group(1)) + # Save so we can get later from g_pytest_config + rundir = Path(config.option.rundir).resolve() + gcdadir = rundir / "gcda" + os.environ["FRR_BUILD_DIR"] = str(bdir) + os.environ["GCOV_PREFIX_STRIP"] = str(len(bdir.parts) - 1) + os.environ["GCOV_PREFIX"] = str(gcdadir) + + if is_main_runner(): + commander.cmd_raises(f"find {bdir} -name '*.gc??' -exec chmod o+rw {{}} +") + commander.cmd_raises(f"mkdir -p {gcdadir}") + commander.cmd_raises(f"chown -R root:frr {gcdadir}") + commander.cmd_raises(f"chmod 2775 {gcdadir}") + + def pytest_configure(config): """ Assert that the environment is correctly configured, and get extra config. @@ -556,8 +602,6 @@ def assert_feature_windows(b, feature): if config.option.topology_only and is_xdist: pytest.exit("Cannot use --topology-only with distributed test mode") - pytest.exit("Cannot use --topology-only with distributed test mode") - # Check environment now that we have config if not diagnose_env(rundir): pytest.exit("environment has errors, please read the logs in %s" % rundir) @@ -572,27 +616,25 @@ def assert_feature_windows(b, feature): if "TOPOTESTS_CHECK_STDERR" in os.environ: del os.environ["TOPOTESTS_CHECK_STDERR"] + if config.option.cov_topotest: + setup_coverage(config) + @pytest.fixture(autouse=True, scope="session") -def setup_session_auto(): +def session_autouse(): # Aligns logs nicely logging.addLevelName(logging.WARNING, " WARN") logging.addLevelName(logging.INFO, " INFO") - if "PYTEST_TOPOTEST_WORKER" not in os.environ: - is_worker = False - elif not os.environ["PYTEST_TOPOTEST_WORKER"]: - is_worker = False - else: - is_worker = True + is_main = is_main_runner() - logger.debug("Before the run (is_worker: %s)", is_worker) - if not is_worker: + logger.debug("Before the run (is_main: %s)", is_main) + if is_main: cleanup_previous() yield - if not is_worker: + if is_main: cleanup_current() - logger.debug("After the run (is_worker: %s)", is_worker) + logger.debug("After the run (is_main: %s)", is_main) def pytest_runtest_setup(item): @@ -719,6 +761,42 @@ def pytest_runtest_makereport(item, call): pause_test() +def coverage_finish(terminalreporter, config): + commander = Commander("pytest") + rundir = Path(config.option.rundir).resolve() + bdir = Path(os.environ["FRR_BUILD_DIR"]) + gcdadir = Path(os.environ["GCOV_PREFIX"]) + + # Get the data files into the build directory + logger.info("Copying gcda files from '%s' to '%s'", gcdadir, bdir) + user = os.environ.get("SUDO_USER", os.environ["USER"]) + commander.cmd_raises(f"chmod -R ugo+r {gcdadir}") + commander.cmd_raises( + f"tar -C {gcdadir} -cf - . | su {user} -c 'tar -C {bdir} -xf -'" + ) + + # Get the results into a summary file + data_file = rundir / "coverage.info" + logger.info("Gathering coverage data into: %s", data_file) + commander.cmd_raises(f"lcov --directory {bdir} --capture --output-file {data_file}") + + # Get coverage info filtered to a specific set of files + report_file = rundir / "coverage.info" + logger.debug("Generating coverage summary from: %s\n%s", report_file) + output = commander.cmd_raises(f"lcov --summary {data_file}") + logger.info("\nCOVERAGE-SUMMARY-START\n%s\nCOVERAGE-SUMMARY-END", output) + terminalreporter.write( + f"\nCOVERAGE-SUMMARY-START\n{output}\nCOVERAGE-SUMMARY-END\n" + ) + + +def pytest_terminal_summary(terminalreporter, exitstatus, config): + # Only run if we are the top level test runner + is_xdist_worker = "PYTEST_XDIST_WORKER" in os.environ + if config.option.cov_topotest and not is_xdist_worker: + coverage_finish(terminalreporter, config) + + # # Add common fixtures available to all tests as parameters # diff --git a/tests/topotests/lib/topotest.py b/tests/topotests/lib/topotest.py index 0a5be970b8b5..2f69f7364b21 100644 --- a/tests/topotests/lib/topotest.py +++ b/tests/topotests/lib/topotest.py @@ -27,6 +27,7 @@ import logging from collections.abc import Mapping from copy import deepcopy +from pathlib import Path import lib.topolog as topolog from lib.micronet_compat import Node @@ -1262,8 +1263,8 @@ def rlimit_atleast(rname, min_value, raises=False): def fix_netns_limits(ns): # Maximum read and write socket buffer sizes - sysctl_atleast(ns, "net.ipv4.tcp_rmem", [10 * 1024, 87380, 16 * 2 ** 20]) - sysctl_atleast(ns, "net.ipv4.tcp_wmem", [10 * 1024, 87380, 16 * 2 ** 20]) + sysctl_atleast(ns, "net.ipv4.tcp_rmem", [10 * 1024, 87380, 16 * 2**20]) + sysctl_atleast(ns, "net.ipv4.tcp_wmem", [10 * 1024, 87380, 16 * 2**20]) sysctl_assure(ns, "net.ipv4.conf.all.rp_filter", 0) sysctl_assure(ns, "net.ipv4.conf.default.rp_filter", 0) @@ -1322,8 +1323,8 @@ def fix_host_limits(): sysctl_atleast(None, "net.core.netdev_max_backlog", 4 * 1024) # Maximum read and write socket buffer sizes - sysctl_atleast(None, "net.core.rmem_max", 16 * 2 ** 20) - sysctl_atleast(None, "net.core.wmem_max", 16 * 2 ** 20) + sysctl_atleast(None, "net.core.rmem_max", 16 * 2**20) + sysctl_atleast(None, "net.core.wmem_max", 16 * 2**20) # Garbage Collection Settings for ARP and Neighbors sysctl_atleast(None, "net.ipv4.neigh.default.gc_thresh2", 4 * 1024) @@ -1523,7 +1524,7 @@ def listDaemons(self): pass return ret - def stopRouter(self, assertOnError=True, minErrorVersion="5.1"): + def stopRouter(self, assertOnError=True): # Stop Running FRR Daemons running = self.listDaemons() if not running: @@ -1570,9 +1571,6 @@ def stopRouter(self, assertOnError=True, minErrorVersion="5.1"): ) errors = self.checkRouterCores(reportOnce=True) - if self.checkRouterVersion("<", minErrorVersion): - # ignore errors in old versions - errors = "" if assertOnError and (errors is not None) and len(errors) > 0: assert "Errors found - details follow:" == 0, errors return errors @@ -1803,6 +1801,8 @@ def startRouterDaemons(self, daemons=None, tgen=None): "Starts FRR daemons for this router." asan_abort = bool(g_pytest_config.option.asan_abort) + cov_option = bool(g_pytest_config.option.cov_topotest) + cov_dir = Path(g_pytest_config.option.rundir) / "gcda" gdb_breakpoints = g_pytest_config.get_option_list("--gdb-breakpoints") gdb_daemons = g_pytest_config.get_option_list("--gdb-daemons") gdb_routers = g_pytest_config.get_option_list("--gdb-routers") @@ -1836,13 +1836,6 @@ def startRouterDaemons(self, daemons=None, tgen=None): # Re-enable to allow for report per run self.reportCores = True - # XXX: glue code forward ported from removed function. - if self.version is None: - self.version = self.cmd( - os.path.join(self.daemondir, "bgpd") + " -v" - ).split()[2] - logger.info("{}: running version: {}".format(self.name, self.version)) - perfds = {} perf_options = g_pytest_config.get_option("--perf-options", "-g") for perf in g_pytest_config.get_option("--perf", []): @@ -1928,6 +1921,10 @@ def do_gdb_or_rr(gdb): self.logdir, self.name, daemon ) + if cov_option: + scount = os.environ["GCOV_PREFIX_STRIP"] + cmdenv += f"GCOV_PREFIX_STRIP={scount} GCOV_PREFIX={cov_dir}" + if valgrind_memleaks: this_dir = os.path.dirname( os.path.abspath(os.path.realpath(__file__)) @@ -2277,9 +2274,7 @@ def pid_exists(self, pid): rc, o, e = self.cmd_status("kill -0 " + str(pid), warn=False) return rc == 0 or "No such process" not in e - def killRouterDaemons( - self, daemons, wait=True, assertOnError=True, minErrorVersion="5.1" - ): + def killRouterDaemons(self, daemons, wait=True, assertOnError=True): # Kill Running FRR # Daemons(user specified daemon only) using SIGKILL rundaemons = self.cmd("ls -1 /var/run/%s/*.pid" % self.routertype) @@ -2339,9 +2334,6 @@ def killRouterDaemons( self.cmd("rm -- {}".format(daemonpidfile)) if wait: errors = self.checkRouterCores(reportOnce=True) - if self.checkRouterVersion("<", minErrorVersion): - # ignore errors in old versions - errors = "" if assertOnError and len(errors) > 0: assert "Errors found - details follow:" == 0, errors else: