From 135da45180f2880a3c9b8af2538cb11b02ebd555 Mon Sep 17 00:00:00 2001 From: Volodymyr Huti Date: Fri, 20 Jan 2023 02:03:43 +0200 Subject: [PATCH] [QPPB] Implement test suit with common setups, based on BCC --- .../topotests/bgp_qppb_vyos_flow/__init__.py | 132 ++++ .../bgp_qppb_vyos_flow/bgp_ipv4_nh.ref | 10 + tests/topotests/bgp_qppb_vyos_flow/ns.py | 142 ++++ .../bgp_qppb_vyos_flow/test_bgp_qppb.py | 641 ++++++++++++++++++ .../bgp_qppb_vyos_flow/topojson.json | 383 +++++++++++ tests/topotests/bgp_qppb_vyos_flow/xdp_qppb.c | 239 +++++++ tests/topotests/lib/common_config.py | 17 +- tests/topotests/lib/micronet.py | 12 +- tests/topotests/lib/topogen.py | 14 +- tests/topotests/lib/topotest.py | 32 +- 10 files changed, 1607 insertions(+), 15 deletions(-) create mode 100755 tests/topotests/bgp_qppb_vyos_flow/__init__.py create mode 100644 tests/topotests/bgp_qppb_vyos_flow/bgp_ipv4_nh.ref create mode 100644 tests/topotests/bgp_qppb_vyos_flow/ns.py create mode 100644 tests/topotests/bgp_qppb_vyos_flow/test_bgp_qppb.py create mode 100644 tests/topotests/bgp_qppb_vyos_flow/topojson.json create mode 100644 tests/topotests/bgp_qppb_vyos_flow/xdp_qppb.c diff --git a/tests/topotests/bgp_qppb_vyos_flow/__init__.py b/tests/topotests/bgp_qppb_vyos_flow/__init__.py new file mode 100755 index 000000000000..457b60848326 --- /dev/null +++ b/tests/topotests/bgp_qppb_vyos_flow/__init__.py @@ -0,0 +1,132 @@ +import os +import sys +import json + +from bcc import BPF, DEBUG_PREPROCESSOR, DEBUG_SOURCE, DEBUG_BPF, DEBUG_BTF +from .ns import pushns, popns +from lib.topolog import logger +from lib.common_config import ( + start_router_daemons, + kill_router_daemons, +) + +CWD = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(CWD, "../")) +sys.path.append(os.path.join(CWD, "../lib/")) + +# from pyroute2.netns import pushns, popns +def router_attach_xdp(rnode, iface): + """ + - swap netns to rnode, + - attach `xdp_qppb` to `iface` + - switch back to root ns + """ + ns = "/proc/%d/ns/net" % rnode.net.pid + qppb_fn = rnode.bpf.funcs[b"xdp_qppb"] + + pushns(ns) + logger.debug("Attach XDP handler '{}'\nNetNS --> {})".format(iface, ns)) + rnode.bpf.attach_xdp(iface, qppb_fn, BPF.XDP_FLAGS_DRV_MODE) + popns() + + +def router_remove_xdp(rnode, iface): + " XXX: remove xdp hook ... " + pushns("/proc/%d/ns/net" % rnode.net.pid) + logger.debug("Removing XDP handler for {}:{}".format(rnode.name, iface)) + rnode.bpf.remove_xdp(iface) + popns() + + +def load_qppb_plugin(tgen, rnode, vtysh_cmd=None, cflags=None, debug_on=True): + """ + Initialize rnode XDP hooks and BPF mapping handlers + - compile xdp handlers from `xdp_qppb.c` + - load `xdp_qppb` and `xdp_tc_mark` hooks + - restart router with QPPB plugin + - execute optional vtysh configuration + + Parameters + ---------- + * `tgen`: topogen object + * `vtysh_cmd`: any extra configuration, typically not present in json + * `cflags`: processing mode required, any other optional + * `debug_on`: enable all BPF debug logs + + Usage + --------- + load_qppb_plugin(tgen, r1, SET_QPPB_TABLE_MAP, cflags=["-DMARK_SKB"]) + + Returns -> None (XXX) + """ + debug_flags = DEBUG_BPF | DEBUG_PREPROCESSOR | DEBUG_SOURCE | DEBUG_BTF + debug = debug_flags if debug_on else 0 + bpf_flags = [ + '-DBPF_PIN_DIR="{}"'.format(rnode.bpfdir), + "-DRESPECT_TOS", + "-DLOG_QPPB", + "-DLOG_TC", + "-w", + ] + + bpf_flags.extend(cflags) + for mode in ["MARK_SKB", "MARK_META"]: + if ("-D%s" % mode) in bpf_flags: + bpf_flags.append('-DMODE_STR="%s"' % mode) + + try: + src_file = "%s/xdp_qppb.c" % CWD + logger.info("Preparing the XDP src: " + src_file) + b = BPF(src_file=src_file, cflags=bpf_flags, debug=debug) + + logger.info("Loading XDP hooks -- xdp_qppb, xdp_tc_mark") + b.load_func(b"xdp_qppb", BPF.XDP) + b.load_func(b"xdp_tc_mark", BPF.SCHED_CLS) + rnode.bpf = b + except Exception as e: + import pytest # XXX: proper error handling + pytest.skip("Failed to configure XDP environment -- \n%s", str(e)) + + qppb_module = "-M vyos_qppb:" + rnode.bpfdir + logger.info( + "Restart {}, XDP hooks loading...\nPlugin args:: {}".format( + rnode.name, qppb_module + ) + ) + kill_router_daemons(tgen, rnode.name, ["bgpd"]) + start_router_daemons(tgen, rnode.name, ["bgpd"], {"bgpd": qppb_module}) + if vtysh_cmd: + rnode.vtysh_cmd(vtysh_cmd) + + +def assert_bw(out, bw_target, tolerance, time=10): + " Assert that connection matches BW in Mbits +- %tolerance" + _min = bw_target * (1 - tolerance) + _max = bw_target * (1 + tolerance) + half_samples = time / 2 + data = json.loads(out) + bws = [] + + for sample in data["intervals"]: + bits = int(sample["sum"]["bits_per_second"]) + mbits = bits / 1024 / 1024 + bw = mbits / 8 + logger.debug("BW sample [{} <= {} <= {}]".format(_min, bw, _max)) + if _min <= bw <= _max: + bws.append(bw) + + _len = len(bws) + assert ( + _len >= half_samples + ), "Only {} samples are within targeted BW [{}:{}%]".format( + _len, bw_target, tolerance * 100 + ) + + +def bpf_print_trace(b): + " XXX: Call this from debugger, to avoid blocking / IO collissions " + logger.debug("===================\nDump bpf log buffer:\n") + line = b.trace_readline(nonblocking=True) + while line: + logger.debug(line) + line = b.trace_readline(nonblocking=True) diff --git a/tests/topotests/bgp_qppb_vyos_flow/bgp_ipv4_nh.ref b/tests/topotests/bgp_qppb_vyos_flow/bgp_ipv4_nh.ref new file mode 100644 index 000000000000..4db9698d2168 --- /dev/null +++ b/tests/topotests/bgp_qppb_vyos_flow/bgp_ipv4_nh.ref @@ -0,0 +1,10 @@ +BGP routing table entry for 10.61.0.1/32, version XX +Paths: (1 available, best #1, table default) + Advertised to non peer-group peers: + 10.0.0.2 + 10 60 + 10.0.0.2 from 10.0.0.2 (1.0.2.17) + Origin incomplete, valid, external, best (First path received) + Community: 60:1 + QOS: Precedence af11 (10) + Last update: XXXX diff --git a/tests/topotests/bgp_qppb_vyos_flow/ns.py b/tests/topotests/bgp_qppb_vyos_flow/ns.py new file mode 100644 index 000000000000..48cc529af50c --- /dev/null +++ b/tests/topotests/bgp_qppb_vyos_flow/ns.py @@ -0,0 +1,142 @@ +import platform +import ctypes +import os +import io + +""" +XXX: + +The `xdp_attach` method expects target dev to be visible (to be in local netns). +It is possible to run some helper via `rnode.popen("bcc_script.py", ...)`, +but we would still need to initialized a separate BPF handler in the root ns. + + +The functionality is provided by `pyroute2` package - `setns/pushns/popns` +Either we can specify pyroute2 as an required dependencies, or just +adopt the relevant methods, avoiding any extra packages. + +/usr/lib/python3/dist-packages/pyroute2/netns/__init__.py +/usr/lib/python3/dist-packages/pyroute2/netns/nslink.py + +from pyroute2 import IPRoute, NetNS, IPDB, NSPopen +""" + +file = io.IOBase +CLONE_NEWNET = 0x40000000 +NETNS_RUN_DIR = '/var/run/netns' +__saved_ns = [] + +machine = platform.machine() +arch = platform.architecture()[0] +__NR = {'x86_': {'64bit': 308}, + 'i386': {'32bit': 346}, + 'i686': {'32bit': 346}, + 'mips': {'32bit': 4344, + '64bit': 5303}, # FIXME: NABI32? + 'armv': {'32bit': 375}, + 'aarc': {'32bit': 375, + '64bit': 268}, # FIXME: EABI vs. OABI? + 'ppc6': {'64bit': 350}, + 's390': {'64bit': 339}} +__NR_setns = __NR.get(machine[:4], {}).get(arch, 308) + + +def setns(netns, flags=os.O_CREAT, libc=None): + ''' + Set netns for the current process. + + The flags semantics is the same as for the `open(2)` + call: + + - O_CREAT -- create netns, if doesn't exist + - O_CREAT | O_EXCL -- create only if doesn't exist + + Note that "main" netns has no name. But you can access it with:: + + setns('foo') # move to netns foo + setns('/proc/1/ns/net') # go back to default netns + + See also `pushns()`/`popns()`/`dropns()` + + Changed in 0.5.1: the routine closes the ns fd if it's + not provided via arguments. + ''' + newfd = False + basestring = (str, bytes) + libc = libc or ctypes.CDLL(ctypes.util.find_library('c'), use_errno=True) + if isinstance(netns, basestring): + netnspath = _get_netnspath(netns) + if os.path.basename(netns) in listnetns(os.path.dirname(netns)): + if flags & (os.O_CREAT | os.O_EXCL) == (os.O_CREAT | os.O_EXCL): + raise OSError(errno.EEXIST, 'netns exists', netns) + else: + if flags & os.O_CREAT: + create(netns, libc=libc) + nsfd = os.open(netnspath, os.O_RDONLY) + newfd = True + elif isinstance(netns, file): + nsfd = netns.fileno() + elif isinstance(netns, int): + nsfd = netns + else: + raise RuntimeError('netns should be a string or an open fd') + error = libc.syscall(__NR_setns, nsfd, CLONE_NEWNET) + if newfd: + os.close(nsfd) + if error != 0: + raise OSError(ctypes.get_errno(), 'failed to open netns', netns) + +def _get_netnspath(name): + netnspath = name + dirname = os.path.dirname(name) + if not dirname: + netnspath = '%s/%s' % (NETNS_RUN_DIR, name) + if hasattr(netnspath, 'encode'): + netnspath = netnspath.encode('ascii') + return netnspath + +def listnetns(nspath=None): + ''' + List available network namespaces. + ''' + if nspath: + nsdir = nspath + else: + nsdir = NETNS_RUN_DIR + try: + return os.listdir(nsdir) + except OSError as e: + if e.errno == errno.ENOENT: + return [] + else: + raise + +def pushns(newns=None, libc=None): + ''' + Save the current netns in order to return to it later. If newns is + specified, change to it:: + + # --> the script in the "main" netns + netns.pushns("test") + # --> changed to "test", the "main" is saved + netns.popns() + # --> "test" is dropped, back to the "main" + ''' + global __saved_ns + __saved_ns.append(os.open('/proc/self/ns/net', os.O_RDONLY)) + if newns is not None: + setns(newns, libc=libc) + +def popns(libc=None): + ''' + Restore the previously saved netns. + ''' + global __saved_ns + fd = __saved_ns.pop() + try: + setns(fd, libc=libc) + except Exception: + __saved_ns.append(fd) + raise + os.close(fd) + diff --git a/tests/topotests/bgp_qppb_vyos_flow/test_bgp_qppb.py b/tests/topotests/bgp_qppb_vyos_flow/test_bgp_qppb.py new file mode 100644 index 000000000000..1cbaf228f752 --- /dev/null +++ b/tests/topotests/bgp_qppb_vyos_flow/test_bgp_qppb.py @@ -0,0 +1,641 @@ +#!/usr/bin/env python +# -*- coding: utf-8 eval: (blacken-mode 1) -*- +# +# XXX: fix copyright +# Part of QPPB Topology Tests +# +# Copyright (c) 2023 by Volodymyr Huti (@VyOS Inc.) +# XXX: Network Device Education Foundation, Inc. ("NetDEF") +# +# Permission to use, copy, modify, and/or distribute this software +# for any purpose with or without fee is hereby granted, provided +# that the above copyright notice and this permission notice appear +# in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND NETDEF DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NETDEF BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY +# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS +# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE +# OF THIS SOFTWARE. +# + +""" +test_bgp_qppb.py: + + 20...1 + +------+ 20...2 + | h1 |----------+ AS30 AS10 AS10 + +------+ ++------+ +------+ +------+ + | | | | | | + +------+ 20..1.2 | R1 | | R2 | | R3 | + | h2 |-----------| |------| |------| | + +------+ +------+ +------+ +------+ + 20..1.1 lo:1.0.1.17 lo:1.0.2.17 | lo:1.0.3.17 + QPPB Router | + | + lo: +------+ + | 1.0.4.17 | | | + | 10.61.0.1 | | R4 | + | ***** | | | + | 10.66.0.1 | +------+ + AS60 + +""" +import os +import re +import sys +import time +import pytest +import functools +import subprocess +import pysnooper + +from lib import topotest +from lib.topolog import logger +from lib.topojson import build_config_from_json +from lib.topogen import Topogen, TopoRouter, get_topogen +from lib.bgp import verify_bgp_convergence +from lib.common_config import ( + create_debug_log_config, + apply_raw_config, + start_topology, + TcpDumpHelper, + IPerfHelper, + step, +) + +from ctypes import Structure, c_int, c_uint, c_ubyte +from bgp_qppb_vyos_flow import * + +pytestmark = [pytest.mark.bgpd] + +# Globals +# ------------------------------------------------------- +EXTRA_DEBUG = True # XXX +os.environ["PYTHONBREAKPOINT"] = "pudb.set_trace" +SET_QPPB_TABLE_MAP = f""" + configure + router bgp 30 + table-map QPPB +""" + +BGP_POLICY_NONE = c_int(0) +BGP_POLICY_DST = c_int(1) +BGP_POLICY_SRC = c_int(2) + +MARK_META_MODE = 1 +MARK_SKB_MODE = 2 +mode = MARK_META_MODE + +af21_tag = c_ubyte(0x12) +af12_tag = c_ubyte(0x0C) +zero_tag = c_ubyte(0) + +from lib.topotest import version_cmp, interface_to_ifindex +xdp_ifindex = lambda host, iface: c_uint(interface_to_ifindex(host, iface)) + +class KeyV4(Structure): + _fields_ = [("prefixlen", c_uint), ("data", c_ubyte * 4)] + +# Helpers +# ------------------------------------------------------- +def setup_test_hosts(tgen, router): + """ + Setup client hosts to test traffic forwarding + NOTE, networks are overlaping for the purpose of lpm_overlap TC + privateDirs empty, so you can str(host) + """ + h1 = tgen.add_host("h1", "20.0.0.1", "dev h1-eth0", privateDirs="") + h2 = tgen.add_host("h2", "20.0.1.1", "dev h2-eth0", privateDirs="") + router.add_link(h1) + router.add_link(h2) + tgen.net.configure_hosts() + + ip_cmd = "ip addr add {} {}" + # XXX: will be used for overlap testing + router.cmd_raises(ip_cmd.format("20.0.0.2/16", "dev " + router.name + "-eth0")) + router.cmd_raises(ip_cmd.format("20.0.1.2/24", "dev " + router.name + "-eth1")) + # XXX: do we really need this? + router.cmd_raises("sysctl -w net.ipv4.conf.all.proxy_arp=1") + + +def check_ping4(rnode, dst, connected=True, src=None, tos=None, count=10, timeout=0): + ping = "" + if timeout: + ping = "timeout {} ".format(timeout) + ping += "ping {} -c{}".format(dst, count) + if src: + ping = "{} -I{}".format(ping, src) + if tos: + ping = "{} -Q{}".format(ping, src) + + match = ", {} packet loss".format("100%" if connected else "0%") + logger.info( + "[+] {} ping -> {}, connection expected -> {}".format( + rnode, dst, "up" if connected else "down" + ) + ) + logger.debug("Executing the ping -> {}".format(ping)) + + def _match_missing(rnode, dst, match): + output = rnode.run(ping) + logger.info(output) + return match not in output + + func = functools.partial(_match_missing, rnode, dst, match) + success, result = topotest.run_and_expect(func, True, count, wait=1) + assert result is True + + +# Module +# ------------------------------------------------------- +def teardown_module(_mod): + "Teardown the pytest environment" + tgen = get_topogen() + tgen.stop_topology() + # iperf_helper.cleanup() + # tcpdumpf_helper.cleanup() + + +def setup_module(mod): + # XXX: write down [ requirement:verion, ... ] + # result |= required_linux_kernel_version("5+") + # result |= required_linux_kernel_features("BPF") + # result |= required_package_version(bcc, dev) + # ... + # if result is not True: + # pytest.skip("Kernel requirements are not met") + # XXX(?): verify that user XPD env doesn't overlap with test + + testsuite_run_time = time.asctime(time.localtime(time.time())) + logger.info("Testsuite start time: {}".format(testsuite_run_time)) + logger.info("=" * 40) + + logger.info("Running setup_module to create topology") + + json_file = "{}/topojson.json".format(CWD) + tgen = Topogen(json_file, mod.__name__) + global topo + topo = tgen.json_topo + + start_topology(tgen) + build_config_from_json(tgen, topo) + if tgen.routers_have_failure(): + pytest.skip(tgen.errors) + + global BGP_CONVERGENCE + BGP_CONVERGENCE = verify_bgp_convergence(tgen, topo) + assert BGP_CONVERGENCE is True, "setup_module :Failed \n Error: {}".format( + BGP_CONVERGENCE + ) + + # Extra test configuration + # ----------------------------------------------------------------------- + debug_rmap_dict = {"r1": {"raw_config": ["end", "debug route-map"]}} + debug_config_dict = { + "r1": {"debug": {"log_file": "debug.log", "enable": ["bgpd", "zebra"]}} + } + if EXTRA_DEBUG: + create_debug_log_config(tgen, debug_config_dict) + apply_raw_config(tgen, debug_rmap_dict) + + + r4 = tgen.gears["r4"] + # each address will receive different marking (tier of preference) + lo_ip_add = "ip address add dev lo 10.6{0}.0.1/32" + [r4.cmd_raises(lo_ip_add.format(n)) for n in range(1, 7)] + + r1 = tgen.gears["r1"] + setup_test_hosts(tgen, r1) + load_qppb_plugin(tgen, r1, SET_QPPB_TABLE_MAP, cflags=["-DMARK_SKB"]) + + +@pytest.mark.skip +# @pysnooper.snoop(depth=3) +def test_xdp_lpm(tgen): + """ + Manually setup the XDP mappings, no route destribution. + Assume that R1 is pinging managment interface on R3 [lo(1.0.3.17)] + The R2 is marking/forwarding based on QPPB mappings: + R1 [sender,eth0] <-> R2 [eth0,forward,eth1] <-> R3 [eth0,receiver] + + The packet marking happens on R2 ingress XDP hook, as follows: + ----------------------------------------- + xdp_qppb(xdp_md *skb): + switch qppb_map[in_ifindex]: + BGP_POLICY_SRC: mark = dscp_map[(skb.src, 32)] + BGP_POLICY_DST: mark = dscp_map[(skb.dst, 32)] + NONE: return pass + + if MARK_SKB: skb->tos = mark + if MARK_META: skb->classid = mark + return pass + ----------------------------------------- + """ + h1 = tgen.gears["h1"] + r1 = tgen.gears["r1"] + r4 = tgen.gears["r4"] + + qppb_map = r1.bpf[b"qppb_mode_map"] + dscp_map = r1.bpf[b"dscp_map"] + + STEP = """ + (On R2) Verify that XDP was loaded and markings work properly + 1. Attach XDP hook to ingress iface in MARK_SKB mode + 2. Set BGP policy in DST mode + 3. Initialize LPM with prefix of targeted iface + 4. Send ping, verify that traffic was marked + """ + r1_eth0_idx = xdp_ifindex(r1, "r1-eth0") + qppb_map[r1_eth0_idx] = BGP_POLICY_DST + router_attach_xdp(r1, b"r1-eth0") + + step(STEP) + # -------------------------------------------------------------------------------- + tcpdump = TcpDumpHelper(tgen, "icmp[0] == 8") # ICMP Echo requst + r4_lo_key = KeyV4(32, (1, 0, 4, 17)) + h1_key = KeyV4(32, (20, 0, 0, 1)) + r4_lo_ip = "1.0.4.17" + PINGS = 10 + + def _check(sender, receiver, dst, cap_iface, tos, src=None, ping_tos=None): + p1 = tcpdump.capture_start(receiver, cap_iface, background=True, timeout=PINGS) + assert p1, "Failed to run tcpdump on {}:\n{}".format(sender.name, p1) + + p2 = check_ping4(sender, dst, src=src, count=PINGS, timeout=PINGS, tos=ping_tos) + time.sleep(1.5) + return tcpdump.find_msg(receiver, "tos 0x%x" % tos.value) + + check_connection = functools.partial(_check, h1, r4, r4_lo_ip, "r4-r3-eth0") + h1.run("ping -c 3 -w 3 " + r4_lo_ip) # refresh arp cache, etc ... + time.sleep(2) + # -------------------------------------------------------------------------------- + dscp_map[r4_lo_key] = af21_tag + found, matches = check_connection(af21_tag) + assert found and matches >= ( + PINGS - 1 # XXX: first packet is not tagged - caching issues? + ), "LPM doesn't work as expected, mark detected only {} times ".format(matches) + + # -------------------------------------------------------------------------------- + dscp_map[r4_lo_key] = af12_tag + found, matches = check_connection(af12_tag) + assert found and matches >= ( + PINGS - 1 + ), "LPM doesn't work as expected, mark detected only {} times ".format(matches) + + # --------------------------------------------------------------------------------- + dscp_map[h1_key] = af12_tag + dscp_map[r4_lo_key] = zero_tag + qppb_map[r1_eth0_idx] = BGP_POLICY_SRC + found, matches = check_connection(af12_tag) + assert found and matches >= ( + PINGS - 1 + ), "LPM doesn't work as expected, mark detected only {} times ".format(matches) + + # -------------------------------------------------------------------------------- + # XXX: Run some flows into opposite directions + # XXX: Use ping with custom tos ... + # XXX: Try using invalid values, i.e. tos > 64 + # ... + # -------------------------------------------------------------------------------- + qppb_map.clear() + dscp_map[h1_key] = af21_tag + dscp_map[r4_lo_key] = af12_tag + found, _ = check_connection(af12_tag) + assert not found, "LPM misbehaviour, markings not expected after clearing dscp map" + + # cleanup used resources + router_remove_xdp(r1, b"r1-eth0") + dscp_map.pop(h1_key) + dscp_map.pop(r4_lo_key) + # XXX dscp_map.clear() - clears the initial config, used by the following test + # bpf_print_trace(bpf) + # breakpoint() + # -------------------------------------------------------------------------------- + """ + TBD/XXX: Implement test logic to verify stats tracking using one of these + + There is a range of utilities that can be used to interact with XDP/BPF + Implement an example of using some of these, for the illustration purposes + bpftool | tool for interacting with BPF mappings + xdp-dump | tool for xdp troubleshooting, i.e. collecting stats for packets + xdp_stats | a minimal packet tracking tool from the tutorial + ip-route | attaching the XDP hook + ip-tc | attaching the SCH hook + """ + + +# @pytest.mark.skip +def test_nh_dscp_displayed(tgen): + """ + Verify that QoS group is displayed for a marked prefix + """ + nhFile = "{}/bgp_ipv4_nh.ref".format(CWD) + expected = open(nhFile).read().rstrip() + expected = ("\n".join(expected.splitlines()) + "\n").rstrip() + + def check_dscp_displayed(): + r1 = tgen.gears["r1"] + actual = r1.vtysh_cmd("show bgp ipv4 10.61.0.1") + actual = ("\n".join(actual.splitlines()) + "\n").rstrip() + actual = re.sub(r" version [0-9]+", " version XX", actual) + actual = re.sub(r"Last update: .*", "Last update: XXXX", actual) + return topotest.get_textdiff( + actual, expected, title1="Actual bgp nh show", title2="Expected bgp nh show" + ) + + ok, result = topotest.run_and_expect(check_dscp_displayed, "", count=5, wait=1) + assert ok, result + + +# @pytest.mark.skip +# @pysnooper.snoop(depth=2) +# @pytest.parametrize(markers=[TOS,QOS_GROUP]) +def test_qos_topo(tgen): + """ + - setup iproute tc tree + - limit bandwidth per interface + - run list of iperf helpers + - verify traffic bandwidth within specified limits + + Allocate 10Mbit queue and flows with different preference + dscp | bw Mbytes + 10 | 7.5 + 20 | 5.0 + 30 | 2.5 + 40 | 2.5 + * | 1 + + Mostly addopted from: + ipmininet/tests/test_tc.py + mininet/examples/test/test_simpleperf.py + mininet/examples/test/test_intfoptions.py + mininet/examples/test/test_walkthrough.py + mininet/mininet/link.py -> class TCIntf + http://luxik.cdi.cz/~devik/qos/htb/manual/userg.htm + """ + r1 = tgen.gears["r1"] + r4 = tgen.gears["r4"] + h1 = tgen.gears["h1"] + xdp_dscp = lambda x: c_ubyte(dscp_tos(x)) + dscp_tos = lambda x: x << 2 + + def tc(rnode, cmd): + logger.debug("TC cmd: " + cmd) + return rnode.cmd_raises("tc " + cmd) + + def tc_check(host, cmds): + tcoutputs = [tc(host, cmd) for cmd in cmds] + for output in tcoutputs: + if output != "": + logger.debug("TC: " + output) + + def tc_log_stats(host, iface): + if not EXTRA_DEBUG: + return + tc_flags = "-g -s -d -p -col" + tc_check( + host, + [ + tc_flags + " filter ls dev " + iface, + tc_flags + " class ls dev " + iface, + tc_flags + " qdisc ls ", + ], + ) + + def tc_bpf_filter(rnode, ifid): + """ Attach tc bpf filter, depends on pyroute2 package """ + import pyroute2 + from pyroute2.netns import pushns, popns + + tc_fn = rnode.bpf.funcs[b"xdp_tc_mark"] + rnode_ns = "/proc/{}/ns/net".format(rnode.net.pid) + + logger.debug("Attach TC-BPF handler '{}'\nNetNS --> {})".format(ifid, rnode_ns)) + # ip.tc("add", "clsact", ifid, "1:") + pushns(rnode_ns) + ip = pyroute2.IPRoute() + ip.tc( + "add-filter", + "bpf", + ifid, + 20, # XXX: + fd=tc_fn.fd, + name=tc_fn.name, + parent=0x10000, + classid=0x10030, # XXX: should be default? default is taken from htb + direct_action=True, + ) + popns() + + _class = "class add dev r1-r2-eth0 parent 1:1 " + _filter = "filter add dev r1-r2-eth0 parent 1:0 " + u32_fmt = "prio %d protocol ip u32 match ip tos %d 0xff classid %s" + tc_setup = [ + "qdisc replace dev r1-r2-eth0 root handle 1:0 htb default 50", + "class add dev r1-r2-eth0 parent 1:0 classid 1:1 htb rate 10Mbps", + _class + "classid 1:10 htb rate 7.5Mbps", + _class + "classid 1:20 htb rate 5.0Mbps", + _class + "classid 1:30 htb rate 2.5Mbps", + _class + "classid 1:40 htb rate 0.5Mbps", + _class + "classid 1:50 htb rate 100kbps", + ] + tc_filters = [ + _filter + u32_fmt % (1, dscp_tos(10), "1:10"), + _filter + u32_fmt % (2, dscp_tos(20), "1:20"), + _filter + u32_fmt % (3, dscp_tos(30), "1:30"), + _filter + u32_fmt % (4, dscp_tos(40), "1:40"), + ] + + tc_check(r1, tc_setup) + if mode == MARK_META_MODE: + tc_egress_idx = interface_to_ifindex(r1, "r1-r2-eth0") + load_qppb_plugin(tgen, r1, cflags=["-DMARK_META"]) + tc_bpf_filter(r1, tc_egress_idx) + elif mode == MARK_SKB_MODE: + tc_check(r1, tc_filters) + router_attach_xdp(r1, b"r1-eth0") + + bw = 7.5 + TIME_OUT = 8 + tolerance = 0.20 # 20% slippage, for short lived connection + servers = clients = [] + iph = IPerfHelper(tgen) + start_client = functools.partial(iph.iperf, json=True, length=TIME_OUT) + start_server = functools.partial(iph.iperf, server=True, background=True) + for i in range(1, 7): + server = start_server(r4, bind_addr="10.6%d.0.1" % i, port=5200 + i) + servers.append(server) + + r1_eth0_idx = xdp_ifindex(r1, "r1-eth0") + qppb_map = r1.bpf[b"qppb_mode_map"] + dscp_map = r1.bpf[b"dscp_map"] + R4_L0_61 = "10.61.0.1" + + # TC1: BGP_POLICT_DST + # ----------------------------------------------------------------- + tc_log_stats(r1, "r1-r2-eth0") + qppb_map[r1_eth0_idx] = BGP_POLICY_DST + client = start_client(h1, dst=R4_L0_61) + tc_log_stats(r1, "r1-r2-eth0") + + out, err = client.communicate() + assert_bw(out, bw, tolerance, time=TIME_OUT) + + # TC2: Respect DSCP + # ----------------------------------------------------------------- + tc_log_stats(r1, "r1-r2-eth0") + client = start_client(h1, dst=R4_L0_61, dscp=20) + tc_log_stats(r1, "r1-r2-eth0") + # bpf_print_trace(r1.bpf) + + bw = 5 + out, err = client.communicate() + assert_bw(out, bw, tolerance, time=TIME_OUT) + + # TC3: BGP_POLICT_SRC, swap host roles + # ----------------------------------------------------------------- + h1_key = KeyV4(24, (20, 0, 0, 1)) + dscp_map[h1_key] = xdp_dscp(10) + qppb_map[r1_eth0_idx] = BGP_POLICY_SRC + tc_log_stats(r1, "r1-r2-eth0") + client = start_client(h1, dst=R4_L0_61) + tc_log_stats(r1, "r1-r2-eth0") + + bw = 7.5 + out, err = client.communicate() + assert_bw(out, bw, tolerance, time=TIME_OUT) + + # TC4: verify bw rebalancing + # - setup all without max prio (10) + # - start max prio + # - verify BW is realocated -> max prio takes full link + # - verify lowest prio gets no BW + # ----------------------------------------------------------------- + qppb_map[r1_eth0_idx] = BGP_POLICY_DST + for i in range(2, 7): + client = start_client( + h1, dst="10.6%d.0.1" % i, port=5200 + i, timeout=10, background=True + ) + clients.append(client) + + # kill the second highest prio + # XXX: probably doesn't work on nohup, actual process is detached from the python object + # - need to setup a separate sighandler (?) + high_prio = clients[0] + high_prio.kill() + + # max prio flow, should receive the best treatment + client = start_client(h1, dst=R4_L0_61) + out, err = client.communicate() + assert_bw(out, bw, tolerance, time=TIME_OUT) + + time.sleep(5) # XXX: + # iph.stop_host("h1") # does not work with background processes + iperf_json = "%s/iperf_h1_10.66.0.1_client.json" % tgen.logdir + with open(iperf_json) as file: + out = file.read() + # the lowest prio should be ~0.5mbps + assert_bw(out, 0.25, 1, time=TIME_OUT) + + # XXX: + # - swap priority on the fly + # + swap filter priorty + # + swap lpm entries + # + swap direction + # .... + # success .................... + + +@pytest.mark.skip +def test_xdp_network_overlap(tgen): + """ + The feature configuration involves quite a bit of steps making it quite + easy to messup and accidentally leak traffic, apply wrong preference, etc .. + I`m assuming the following scenarios that may require special handling on xdp side + * configuration mistakes, network loops + * delays during (re)convergance (I guess?) + * using overlaping network ranges (router cascading ?) + * external events (malformed update packets / fail in processing ...?) + * sidefects of admin involvment ... + # XXX(?): how this would work with different kinds of NAT + # XXX: overlaping vs router cascading scenario + Topo: + 20.0.0.0/16 h1-eth1, learned (dscp af22) + h1 <-----++ + r2 <-- r3 <-- r4 + h2 <-----++ + 20.0.1.0/24 h2-eth2, installed by admin + + Admin should be aware that subnetwork 20..1. will receive the QOS treatment from 20.../16, + even though it was not explicitly configured for the new network segment + """ + # XXX,TBD: not sure how common/critical such issues would be + # XXX: likely, there will be many ways to acidentally leak traffic (; + # any tools to detect leaks? i.e. some packet processing stats to look at + r1 = tgen.gears["r1"] + qppb_map = r1.bpf[b"qppb_mode_map"] + dscp_map = r1.bpf[b"dscp_map"] + + r1_eth0_idx = xdp_ifindex(r1, "r1-r2-eth0") + qppb_map[r1_eth0_idx] = BGP_POLICY_DST + h1_key = KeyV4(16, (20, 0, 0, 0)) + dscp_map[h1_key] = af21_tag + router_attach_xdp(r1, b"r1-r2-eth0") + + r4 = tgen.gears["r4"] + r4.run("ping -c 3 20.0.0.1") # is tagged with 0x12 + r4.run("ping -c 3 20.0.1.1") # is tagged as well + + # Admin can dissable marking/processing by manually inserting prefix without policy + # The LPM will lookup the more specific prefix and ignore it + h2_key = KeyV4(24, (20, 0, 1, 0)) + dscp_map[h2_key] = zero_tag + r4.run("ping -c 3 20.0.1.1") # should not be tagged any more + + # ::TBD:: + breakpoint() + + +if __name__ == "__main__": + args = ["-s"] + sys.argv[1:] + sys.exit(pytest.main(args)) + + +@pytest.mark.skip +def test_get_version(tgen): + "Sanity testing, triggers breapoint " + r1 = tgen.gears["r1"] + r2 = tgen.gears["r2"] + r3 = tgen.gears["r3"] + r4 = tgen.gears["r4"] + version = r1.vtysh_cmd("show version") + logger.info("FRR version is: " + version) + + for host in ["h1", "r1", "r4"]: + tgen.net.hosts[host].run_in_window("bash") + + bpf_print_trace(r1.bpf) + breakpoint() + + +# def test_dscp_to_vrf (TBD: next in the priority) +# def test_qos_over_l3_dev (vrf/bridge/lag/..., check what cumulus/melanox for examples) +# def test_with_fragmentation +# def test_with_tunnels +# def test_defaulte_route_mark (does this make sense?) +# def test_loader (assuming there will some kind of loader choosen) +# +# def test_scalability (TBD) +# def test_limits | what table sizes should be? lpm/qppb_mode +# | what if we run out of prefixes in lpm (abort mechanism?) +# +# +# @pytest.skipIf( '-quick' in sys.argv, 'long test' ) +# def test_stability() +# r1[iperf] (5 min) -> r3 +# assert all_packets_marked +# ...... diff --git a/tests/topotests/bgp_qppb_vyos_flow/topojson.json b/tests/topotests/bgp_qppb_vyos_flow/topojson.json new file mode 100644 index 000000000000..c9bbd506441c --- /dev/null +++ b/tests/topotests/bgp_qppb_vyos_flow/topojson.json @@ -0,0 +1,383 @@ +{ + "address_types": ["ipv4"], + "ipv4base": "10.0.0.0", + "ipv4mask": 30, + "ipv6base": "fd00::", + "ipv6mask": 64, + "link_ip_start": { + "ipv4": "10.0.0.0", + "v4mask": 30, + "ipv6": "fd00::", + "v6mask": 64 + }, + "lo_prefix": { + "ipv4": "1.0.", + "v4mask": 32, + "ipv6": "2001:DB8:F::", + "v6mask": 128 + }, + "routers": { + "r1": { + "links": { + "lo": { + "ipv4": "auto", + "ipv6": "auto", + "type": "loopback" + }, + "r2": { + "ipv4": "auto", + "ipv6": "auto" + } + }, + "bgp": { + "local_as": "30", + "address_family": { + "ipv4": { + "unicast": { + "redistribute": [ + { "redist_type": "static" }, + { "redist_type": "connected" } + ], + "neighbor": { + "r2": { + "dest_link": { + "r1": {} + } + } + } + } + } + } + }, + "bgp_community_lists": [ + { + "community_type": "standard", + "action": "permit", + "name": "comm_list_1", + "value": "60:1" + }, + { + "community_type": "standard", + "action": "permit", + "name": "comm_list_2", + "value": "60:2" + }, + { + "community_type": "standard", + "action": "permit", + "name": "comm_list_3", + "value": "60:3" + }, + { + "community_type": "standard", + "action": "permit", + "name": "comm_list_4", + "value": "60:4" + }, + { + "community_type": "standard", + "action": "permit", + "name": "comm_list_5", + "value": "60:5" + }, + { + "community_type": "standard", + "action": "permit", + "name": "comm_list_6", + "value": "60:6" + }, + { + "community_type": "standard", + "action": "permit", + "name": "comm_list_7", + "value": "60:7" + } + ], + "route_maps": { + "QPPB": [ + { + "action": "permit", + "match": { "community_list": { "id": "comm_list_1" } }, + "set": { "dscp": "10" } + }, + { + "action": "permit", + "match": { "community_list": { "id": "comm_list_2" } }, + "set": { "dscp": "20" } + }, + { + "action": "permit", + "match": { "community_list": { "id": "comm_list_3" } }, + "set": { "dscp": "30" } + }, + { + "action": "permit", + "match": { "community_list": { "id": "comm_list_4" } }, + "set": { "dscp": "40" } + }, + { + "action": "permit", + "match": { "community_list": { "id": "comm_list_5" } }, + "set": { "dscp": "50" } + }, + { + "action": "permit", + "match": { "community_list": { "id": "comm_list_6" } }, + "set": { "dscp": "15" } + }, + { + "action": "permit", + "match": { "community_list": { "id": "comm_list_7" } }, + "set": { "dscp": "25" } + }, + { + "action": "permit", + "match": { "ipv4": {"prefix_lists": "pf_list_1"} }, + "set": { "dscp": "63" } + }, + { "action": "permit" } + ] + }, + "prefix_lists": { + "ipv4": { + "pf_list_1": [ + { + "seqid": "69", + "network": "10.69.0.0/30", + "action": "permit" + } + ] + } + } + }, + "r2": { + "links": { + "lo": { + "ipv4": "auto", + "ipv6": "auto", + "type": "loopback" + }, + "r1": { + "ipv4": "auto", + "ipv6": "auto" + }, + "r3": { + "ipv4": "auto", + "ipv6": "auto" + } + }, + "bgp": { + "local_as": "10", + "address_family": { + "ipv4": { + "unicast": { + "redistribute": [ + { "redist_type": "static" }, + { "redist_type": "connected" } + ], + "neighbor": { + "r1": { + "dest_link": { + "r2": { + "route_maps": [{ + "name": "send_community", + "direction": "out" + }] + } + } + }, + "r3": { + "dest_link": { + "r2": {} + } + } + } + } + } + } + }, + "route_maps": { + "send_community": [ + { + "action": "permit", + "match": { "ipv4": {"prefix_lists": "pf_list_1"} }, + "set": { + "community": { + "num": "60:1", + "action": "additive" + } + } + }, + { + "action": "permit", + "match": { "ipv4": {"prefix_lists": "pf_list_2"} }, + "set": { + "community": { + "num": "60:2", + "action": "additive" + } + } + }, + { + "action": "permit", + "match": { "ipv4": {"prefix_lists": "pf_list_3"} }, + "set": { + "community": { + "num": "60:3", + "action": "additive" + } + } + }, + { + "action": "permit", + "match": { "ipv4": {"prefix_lists": "pf_list_4"} }, + "set": { + "community": { + "num": "60:4", + "action": "additive" + } + } + }, + { + "action": "permit", + "match": { "ipv4": {"prefix_lists": "pf_list_5"} }, + "set": { + "community": { + "num": "60:5", + "action": "additive" + } + } + }, + { + "action": "permit", + "match": { "ipv4": {"prefix_lists": "pf_list_6"} }, + "set": { + "community": { + "num": "60:6", + "action": "additive" + } + } + }, + { + "action": "permit", + "match": { "ipv4": {"prefix_lists": "pf_list_7"} }, + "set": { + "community": { + "num": "60:7", + "action": "additive" + } + } + }, + { "action": "permit" } + ] + }, + "prefix_lists": { + "ipv4": { + "pf_list_1": [{ + "network": "10.61.0.1/32", + "action": "permit" + }], + "pf_list_2": [{ + "network": "10.62.0.1/32", + "action": "permit" + }], + "pf_list_3": [{ + "network": "10.63.0.1/32", + "action": "permit" + }], + "pf_list_4": [{ + "network": "10.64.0.1/32", + "action": "permit" + }], + "pf_list_5": [{ + "network": "10.65.0.1/32", + "action": "permit" + }], + "pf_list_6": [{ + "network": "10.66.0.1/32", + "action": "permit" + }], + "pf_list_7": [{ + "network": "10.67.0.1/32", + "action": "permit" + }] + } + } + }, + "r3": { + "links": { + "lo": { + "ipv4": "auto", + "ipv6": "auto", + "type": "loopback" + }, + "r2": { + "ipv4": "auto", + "ipv6": "auto" + }, + "r4": { + "ipv4": "auto", + "ipv6": "auto" + } + }, + "bgp": { + "local_as": "10", + "address_family": { + "ipv4": { + "unicast": { + "redistribute": [ + { "redist_type": "static" }, + { "redist_type": "connected" } + ], + "neighbor": { + "r2": { + "dest_link": { + "r3": {} + } + }, + "r4": { + "dest_link": { + "r3": {} + } + } + } + } + } + } + } + }, + "r4": { + "links": { + "lo": { + "ipv4": "auto", + "ipv6": "auto", + "type": "loopback" + }, + "r3": { + "ipv4": "auto", + "ipv6": "auto" + } + }, + "bgp": { + "local_as": "60", + "address_family": { + "ipv4": { + "unicast": { + "redistribute": [ + { "redist_type": "static" }, + { "redist_type": "connected" } + ], + "neighbor": { + "r3": { + "dest_link": { + "r4": {} + } + } + } + } + } + } + } + } + } +} diff --git a/tests/topotests/bgp_qppb_vyos_flow/xdp_qppb.c b/tests/topotests/bgp_qppb_vyos_flow/xdp_qppb.c new file mode 100644 index 000000000000..b2bc88b95d7a --- /dev/null +++ b/tests/topotests/bgp_qppb_vyos_flow/xdp_qppb.c @@ -0,0 +1,239 @@ +#include +#include +#include +#include +#include + +/* REFERENCES: + * linux/samples/bpf/xdp_fwd_kernel.c + * linux/samples/bpf/xdp_router_ipv4.bpf.c + * linux/samples/bpf/xdp2skb_meta_kern.c + * xdp-tutorial/packet-solutions/xdp_prog_kern_03.c + * bcc/examples/networking/xdp_drop_count.py + * bcc/examples/networking/tc_perf_event.py + * xdp-cpumap-tc/src/tc_classify_kern.c + */ + +/* #if !defined(IFACE) */ +/* #error The iface/mode should be specified */ +#if (!defined(MARK_SKB) && !defined(MARK_META)) +#error Specify marking mode to be used +#elif (defined(MARK_SKB) && defined(MARK_META)) +#error Specify single mode only +#elif (!defined(MODE_STR)) +#warn XXX: Poor config +#endif + +struct datarec { + __u64 rx_packets; + __u64 rx_bytes; +}; + +struct lpm_key4 { + __u32 prefixlen; + __u32 src; +}; + +union lpm_key4_u { + __u32 b32[2]; + __u8 b8[8]; +}; + +#if !defined(XDP_ACTION_MAX) +#define XDP_ACTION_MAX (XDP_REDIRECT + 1) +#endif +#if !defined(BPF_PIN_DIR) +#define BPF_PIN_DIR "/sys/fs/bpf" +#endif + +#define DSCP_PIN BPF_PIN_DIR "/dscp_map" +#define QPPB_PIN BPF_PIN_DIR "/qppb_mode_map" +#if !defined(IFACE) +#define STAT_PIN BPF_PIN_DIR "/xdp_stats_map" +#else +#define STAT_PIN BPF_PIN_DIR "/" IFACE "/xdp_stats_map" +#endif +// type : key : leaf : name : size : pin_dir : flags +BPF_TABLE_PINNED("percpu_array", u32, struct datarec, xdp_stats_map, XDP_ACTION_MAX, STAT_PIN); +BPF_TABLE_PINNED("lpm_trie", struct lpm_key4, u8, dscp_map, 10240, DSCP_PIN, BPF_F_NO_PREALLOC); +BPF_TABLE_PINNED("array", u32 /*iface_id*/, u32 /*qppb_bgp_policy*/, qppb_mode_map, 64, QPPB_PIN); +// XXX: choose table size limits (read them from sysctl?) + +enum qppb_bgp_policy { + BGP_POLICY_NONE = 0, + BGP_POLICY_DST = 1, + BGP_POLICY_SRC = 2, + BGP_POLICY_MAX +}; + +static __always_inline +__u32 xdp_stats_record_action(struct xdp_md *ctx, u32 action) +{ + if (action >= XDP_ACTION_MAX) + return XDP_ABORTED; + + struct datarec *rec = xdp_stats_map.lookup(&action); + if (!rec) + return XDP_ABORTED; + rec->rx_packets++; + rec->rx_bytes += (ctx->data_end - ctx->data); + return action; +} + +/* Taken from include/net/dsfield.h */ +static __always_inline +void ipv4_change_dsfield(struct iphdr *iph, __u8 mask, __u8 value) +{ + __u32 check = bpf_ntohs((__be16)iph->check); + __u8 dsfield; + + dsfield = (iph->tos & mask) | value; + check += iph->tos; + if ((check+1) >> 16) check = (check+1) & 0xffff; + check -= dsfield; + check += check >> 16; /* adjust carry */ + iph->check = (__sum16)bpf_htons(check); + iph->tos = dsfield; +} + +struct meta_info { + __u8 mark; +} __attribute__((aligned(4))); + +int xdp_qppb(struct xdp_md *ctx) +{ + int rc, action = XDP_PASS; +#if defined(MARK_META) + struct meta_info *meta; + rc = bpf_xdp_adjust_meta(ctx, -(int)sizeof(*meta)); + if (rc < 0) + goto aborted; +#endif + + void *data = (void *)(long)ctx->data; + void *data_end = (void *)(long)ctx->data_end; + struct iphdr *iph = data + sizeof(struct ethhdr); + __be16 h_proto = ((struct ethhdr *)data)->h_proto; + __u64 nh_off = sizeof(struct ethhdr); + struct bpf_fib_lookup fib_params; + union lpm_key4_u key4; + __u8 *mark, qppb_mode; + __u32 *qppb_mkey; + + if (data + nh_off > data_end) + goto drop; + if ((void *)(iph + 1) > data_end) + goto drop; +#if defined(MARK_META) + meta = (void *)(long)ctx->data_meta; + if ((void *)(meta + 1) > data) + goto aborted; +#endif +#if defined(RESPECT_TOS) + if (iph->tos) { + #if defined(MARK_META) + meta->mark = iph->tos; + #if defined(LOG_QPPB) + bpf_trace_printk("XDP ignore marked packet [%d|%d]", iph->tos, meta->mark); + #endif + #endif + goto skip; + } +#endif + if (iph->ttl <= 1) + goto skip; + if (h_proto != bpf_htons(ETH_P_IP)) + goto skip; + + __builtin_memset(&fib_params, 0, sizeof(fib_params)); + fib_params.tot_len = bpf_ntohs(iph->tot_len); + fib_params.ifindex = ctx->ingress_ifindex; + fib_params.l4_protocol = iph->protocol; + fib_params.ipv4_src = iph->saddr; + fib_params.ipv4_dst = iph->daddr; + fib_params.tos = iph->tos; + fib_params.family = AF_INET; + + qppb_mkey = qppb_mode_map.lookup(&fib_params.ifindex); + qppb_mode = qppb_mkey ? *qppb_mkey : BGP_POLICY_NONE; + if (qppb_mode == BGP_POLICY_NONE) + goto skip; + + rc = bpf_fib_lookup(ctx, &fib_params, sizeof(fib_params), 0); + if (rc != BPF_FIB_LKUP_RET_SUCCESS) + goto out; + + key4.b32[0] = 32; + switch (qppb_mode) { + case BGP_POLICY_DST: + key4.b8[4] = iph->daddr & 0xff; + key4.b8[5] = (iph->daddr >> 8) & 0xff; + key4.b8[6] = (iph->daddr >> 16) & 0xff; + key4.b8[7] = (iph->daddr >> 24) & 0xff; + break; + case BGP_POLICY_SRC: + key4.b8[4] = iph->saddr & 0xff; + key4.b8[5] = (iph->saddr >> 8) & 0xff; + key4.b8[6] = (iph->saddr >> 16) & 0xff; + key4.b8[7] = (iph->saddr >> 24) & 0xff; + break; + default: + goto out; + } + + mark = dscp_map.lookup((struct lpm_key4 *)&key4); + if (!mark) + goto out; +#if defined(MARK_SKB) + ipv4_change_dsfield(iph, 0, *mark); +#elif defined(MARK_META) + meta->mark = *mark; +#endif +#if defined(LOG_QPPB) + bpf_trace_printk("XDP Mark detected [%d]\n", *mark); + #if 0 + const char *MODES[] = { "MARK_SKB\0", "MARK_META\0", NULL }; + bpf_trace_printk("Mode [%s]\n", MODES[0] + ->"Mode " ... + + Lookslike bpf version of printk handles only first string arguments (?) + Need to retest with updated kernel + https://nakryiko.com/posts/bpf-tips-printk/ + #endif +#endif +out: return xdp_stats_record_action(ctx, action); +drop: return xdp_stats_record_action(ctx, XDP_DROP); +aborted: return xdp_stats_record_action(ctx, XDP_ABORTED); // packet is dropped +skip: return action; +} + +int xdp_tc_mark(struct __sk_buff *skb) +{ + void *data = (void *)(long)skb->data; + void *data_meta = (void *)(long)skb->data_meta; + struct meta_info *meta = data_meta; + + // Default priority + skb->tc_classid = 0x50; + // Check XDP gave us some data_meta + if ((void*)(meta + 1) > data) + return TC_ACT_OK; + if (!meta->mark) + return TC_ACT_OK; + + /* skb->mark = meta->mark; // Firewall fw mark */ + /* skb->priority = meta->mark; */ + switch(meta->mark >> 2) { + case 10: skb->tc_classid = 0x10; break; + case 20: skb->tc_classid = 0x20; break; + case 30: skb->tc_classid = 0x30; break; + case 40: skb->tc_classid = 0x40; break; + defaut: break; + } + +#if defined(LOG_TC) + bpf_trace_printk("TC Mark detected [%d|%d|%d]", + meta->mark, meta->mark >> 2, skb->tc_classid); +#endif + return TC_ACT_OK; +} diff --git a/tests/topotests/lib/common_config.py b/tests/topotests/lib/common_config.py index d58b553e7508..a5133530d7b0 100644 --- a/tests/topotests/lib/common_config.py +++ b/tests/topotests/lib/common_config.py @@ -21,6 +21,7 @@ import ipaddress import json import os +import re import platform import socket import subprocess @@ -399,7 +400,7 @@ def kill_router_daemons(tgen, router, daemons, save_config=True): return errormsg -def start_router_daemons(tgen, router, daemons): +def start_router_daemons(tgen, router, daemons, plugins=None): """ Daemons defined by user would be started * `tgen` : topogen object @@ -413,7 +414,7 @@ def start_router_daemons(tgen, router, daemons): router_list = tgen.routers() # Start daemons - res = router_list[router].startDaemons(daemons) + res = router_list[router].startDaemons(daemons, plugins) except Exception as e: errormsg = traceback.format_exc() @@ -2301,6 +2302,8 @@ def create_route_maps(tgen, input_dict, build=False): # med: metric value advertised for AS # aspath: set AS path value # weight: weight for the route + # dscp: dscp for the routed traffic, int or code point name + # XXX: check how error get`s handled on EINVALID/EOUTRANGE cases(?) # community: standard community value to be attached # large_community: large community value to be attached # community_additive: if set to "additive", adds community/large-community @@ -2333,6 +2336,7 @@ def create_route_maps(tgen, input_dict, build=False): "set": { "locPrf": 150, "metric": 30, + "dscp": "af21", "path": { "num": 20000, "action": "prepend", @@ -2432,6 +2436,7 @@ def create_route_maps(tgen, input_dict, build=False): ipv6_data = set_data.setdefault("ipv6", {}) local_preference = set_data.setdefault("locPrf", None) metric = set_data.setdefault("metric", None) + dscp = set_data.setdefault("dscp", None) metric_type = set_data.setdefault("metric-type", None) as_path = set_data.setdefault("path", {}) weight = set_data.setdefault("weight", None) @@ -2462,6 +2467,14 @@ def create_route_maps(tgen, input_dict, build=False): else: rmap_data.append("set metric {}".format(metric)) + # Dscp + if dscp: + del_comm = set_data.setdefault("delete", None) + if del_comm: + rmap_data.append("no set dscp {}".format(dscp)) + else: + rmap_data.append("set dscp {}".format(dscp)) + # Origin if origin: rmap_data.append("set origin {} \n".format(origin)) diff --git a/tests/topotests/lib/micronet.py b/tests/topotests/lib/micronet.py index dfa10ccb2fb1..33bf30e6d7ac 100644 --- a/tests/topotests/lib/micronet.py +++ b/tests/topotests/lib/micronet.py @@ -448,18 +448,18 @@ def __init__( Args: name: Internal name for the namespace. net: Create network namespace. - mount: Create network namespace. + mount: Create mount namespace. uts: Create UTS (hostname) namespace. cgroup: Create cgroup namespace. ipc: Create IPC namespace. pid: Create PID namespace, also mounts new /proc. time: Create time namespace. user: Create user namespace, also keeps capabilities. - set_hostname: Set the hostname to `name`, uts must also be True. - private_mounts: List of strings of the form - "[/external/path:]/internal/path. If no external path is specified a - tmpfs is mounted on the internal path. Any paths specified are first - passed to `mkdir -p`. + set_hostname: Set the hostname to `name`, uts must also be True. + private_mounts: List of strings of the form + "[/external/path:]/internal/path. If no external path is specified a + tmpfs is mounted on the internal path. Any paths specified are first + passed to `mkdir -p`. logger: Passed to superclass. """ super(LinuxNamespace, self).__init__(name, logger) diff --git a/tests/topotests/lib/topogen.py b/tests/topotests/lib/topogen.py index 04712eda8740..52cb7116e329 100644 --- a/tests/topotests/lib/topogen.py +++ b/tests/topotests/lib/topogen.py @@ -357,7 +357,7 @@ def add_exabgp_peer(self, name, ip, defaultRoute): self.peern += 1 return self.gears[name] - def add_host(self, name, ip, defaultRoute): + def add_host(self, name, ip, defaultRoute, **kwargs): """ Adds a new host to the topology. This function has the following parameters: @@ -369,7 +369,7 @@ def add_host(self, name, ip, defaultRoute): if name in self.gears: raise KeyError("host already exists") - self.gears[name] = TopoHost(self, name, ip=ip, defaultRoute=defaultRoute) + self.gears[name] = TopoHost(self, name, ip=ip, defaultRoute=defaultRoute, **kwargs) self.peern += 1 return self.gears[name] @@ -703,6 +703,7 @@ class TopoRouter(TopoGear): "/etc/snmp", "/var/run/frr", "/var/log", + # XXX: need to rbind(gearlog, /sys/fs/bpf) ] # Router Daemon enumeration definition. @@ -772,6 +773,8 @@ def __init__(self, tgen, cls, name, **params): tgen.net.add_host(self.name, cls=cls, **params) topotest.fix_netns_limits(tgen.net[name]) + # XXX: for now keep for debugging? + self.bpfdir = "{}/bpf".format(self.gearlogdir, name) # Mount gear log directory on a common path self.net.bind_mount(self.gearlogdir, "/tmp/gearlogdir") @@ -894,7 +897,7 @@ def stop(self): self.logger.debug("stopping (no assert)") return self.net.stopRouter(False) - def startDaemons(self, daemons): + def startDaemons(self, daemons, plugins=None): """ Start Daemons: to start specific daemon(user defined daemon only) * Start daemons (e.g. FRR) @@ -902,7 +905,7 @@ def startDaemons(self, daemons): """ self.logger.debug("starting") nrouter = self.net - result = nrouter.startRouterDaemons(daemons) + result = nrouter.startRouterDaemons(daemons, plugins=plugins) if daemons is None: daemons = nrouter.daemons.keys() @@ -1125,6 +1128,9 @@ def __init__(self, tgen, name, **params): # Mount gear log directory on a common path self.net.bind_mount(self.gearlogdir, "/tmp/gearlogdir") + # Ensure pid file + with open(os.path.join(self.logdir, self.name + ".pid"), "w") as f: + f.write(str(self.net.pid) + "\n") def __str__(self): gear = super(TopoHost, self).__str__() diff --git a/tests/topotests/lib/topotest.py b/tests/topotests/lib/topotest.py index 61cf16944ff5..9f72a2c34201 100644 --- a/tests/topotests/lib/topotest.py +++ b/tests/topotests/lib/topotest.py @@ -679,7 +679,18 @@ def version_cmp(v1, v2): return 0 -def interface_set_status(node, ifacename, ifaceaction=False, vrf_name=None): +def interface_to_ifindex(node, iface): + """ + Gets the interface index using its name. Returns None on failure. + """ + interfaces = json.loads(node.cmd_raises("ip -j link show")) + for interface in interfaces: + if interface["ifname"] == iface: + return int(interface["ifindex"]) + + return None + +def interface_set_status(node, ifacename, ifaceaction=False, vrf_name=None): #qqq if ifaceaction: str_ifaceaction = "no shutdown" else: @@ -1335,6 +1346,14 @@ def __init__(self, name, **params): l = topolog.get_logger(name, log_level="debug", target=logfile) params["logger"] = l + self.bpfdir = "{}/{}/bpf".format(self.logdir, name) + # XXX: unmount on router cleanup + # XXX: mouning before `unshare`, this way - + # - pytest keeps BPF object handle + # - namespace inherits the fs mappings + subprocess.check_call( + "mkdir -p {0} && mount -t bpf bpf {0} && mount --make-shared {0}".format(self.bpfdir), shell=True + ) super(Router, self).__init__(name, **params) self.daemondir = None @@ -1556,7 +1575,7 @@ def loadConf(self, daemon, source=None, param=None): if daemon == "frr" or not self.unified_config: self.cmd_raises("rm -f " + conf_file) self.cmd_raises("touch " + conf_file) - else: + elif not source == conf_file: self.cmd_raises("cp {} {}".format(source, conf_file)) if not self.unified_config or daemon == "frr": @@ -1684,7 +1703,7 @@ def getStdOut(self, daemon): def getLog(self, log, daemon): return self.cmd("cat {}/{}/{}.{}".format(self.logdir, self.name, daemon, log)) - def startRouterDaemons(self, daemons=None, tgen=None): + def startRouterDaemons(self, daemons=None, tgen=None, plugins=None): "Starts FRR daemons for this router." asan_abort = g_extra_config["asan_abort"] @@ -1824,6 +1843,13 @@ def start_daemon(daemon, extra_opts=None): else: logger.info("%s: %s %s started", self, self.routertype, daemon) + # XXX: handle plugins properly - per daemon + bgpd_plugins = (plugins.get("bgpd") if plugins else None) + if "bgpd" in daemons_list and bgpd_plugins: + start_daemon("bgpd", bgpd_plugins) + while "bgpd" in daemons_list: + daemons_list.remove("bgpd") + # Start Zebra first if "zebra" in daemons_list: start_daemon("zebra", "-s 90000000")