From 909353effb33c5fb3cfda2addd89bc63170055d1 Mon Sep 17 00:00:00 2001 From: Volodymyr Huti Date: Fri, 20 Jan 2023 02:03:43 +0200 Subject: [PATCH] tests: Implement test suit with common setups, based on BCC Check the documentation for more details: https://phabricator.vyos.net/T4180 Signed-off-by: Volodymyr Huti --- tests/topotests/bgp_qppb_flow/__init__.py | 201 ++++++ tests/topotests/bgp_qppb_flow/bgp_ipv4_nh.ref | 10 + tests/topotests/bgp_qppb_flow/bgp_xdp_qppb.c | 1 + .../topotests/bgp_qppb_flow/test_bgp_qppb.py | 580 ++++++++++++++++++ tests/topotests/bgp_qppb_flow/topo_cisco.json | 383 ++++++++++++ tests/topotests/lib/common_config.py | 18 +- tests/topotests/lib/topogen.py | 10 +- tests/topotests/lib/topotest.py | 21 +- 8 files changed, 1217 insertions(+), 7 deletions(-) create mode 100755 tests/topotests/bgp_qppb_flow/__init__.py create mode 100644 tests/topotests/bgp_qppb_flow/bgp_ipv4_nh.ref create mode 120000 tests/topotests/bgp_qppb_flow/bgp_xdp_qppb.c create mode 100644 tests/topotests/bgp_qppb_flow/test_bgp_qppb.py create mode 100644 tests/topotests/bgp_qppb_flow/topo_cisco.json diff --git a/tests/topotests/bgp_qppb_flow/__init__.py b/tests/topotests/bgp_qppb_flow/__init__.py new file mode 100755 index 000000000000..3b9ac16575ca --- /dev/null +++ b/tests/topotests/bgp_qppb_flow/__init__.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python +# +# SPDX-License-Identifier: ISC +# Copyright (c) 2023 VyOS Inc. +# Volodymyr Huti +# + +import os +import sys +import json +import pytest + +from lib.topolog import logger +from lib.common_config import ( + start_router_daemons, + kill_router_daemons, +) + +from bcc import BPF, DEBUG_PREPROCESSOR, DEBUG_SOURCE, DEBUG_BPF, DEBUG_BTF +from pyroute2.netns import pushns, popns +from pyroute2 import IPRoute +from ctypes import Structure, c_int, c_uint, c_ubyte +from enum import Enum + +CWD = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(CWD, "../")) +sys.path.append(os.path.join(CWD, "../lib/")) + +# os.environ["PYTHONBREAKPOINT"] = "pudb.set_trace" +DEV_DEBUG = False + + +class BgpPolicy(Enum): + NONE = c_int(0) + Dst = c_int(1) + Src = c_int(2) + + +class XdpMode(str, Enum): + META = "MARK_META" + SKB = "MARK_SKB" + + +class KeyV4(Structure): + _fields_ = [("prefixlen", c_uint), ("data", c_ubyte * 4)] + + +def router_attach_xdp(rnode, iface): + """ + - swap netns to rnode, + - attach `xdp_qppb` to `iface` + - switch back to root ns + """ + ns = "/proc/%d/ns/net" % rnode.net.pid + qppb_fn = rnode.bpf.funcs[b"xdp_qppb"] + + pushns(ns) + logger.debug("Attach XDP handler '{}'\nNetNS --> {})".format(iface, ns)) + rnode.bpf.attach_xdp(iface, qppb_fn, BPF.XDP_FLAGS_DRV_MODE) + popns() + + +def router_remove_xdp(rnode, iface): + pushns("/proc/%d/ns/net" % rnode.net.pid) + logger.debug("Removing XDP handler for {}:{}".format(rnode.name, iface)) + rnode.bpf.remove_xdp(iface) + popns() + + +def load_qppb_plugin(tgen, rnode, mode=XdpMode.SKB, debug_on=DEV_DEBUG): + """ + Initialize rnode XDP hooks and BPF mapping handlers + - compile xdp handlers from `xdp_qppb.c` in specified `mode` + - load `xdp_qppb` and `xdp_tc_mark` hooks + - restart router with QPPB plugin + + Parameters + ---------- + * `tgen`: topogen object + * `rnode`: router object + * `mode`: xdp processing mode required + * `debug_on`: enable debug logs for bpf compilation / xdp handlers + + Usage + --------- + load_qppb_plugin(tgen, r1, mode=XdpMode.META) + Returns -> None (XXX) + """ + debug_flags = DEBUG_BPF | DEBUG_PREPROCESSOR | DEBUG_SOURCE | DEBUG_BTF + debug = debug_flags if debug_on else 0 + src_file = CWD + "/bgp_xdp_qppb.c" + bpf_flags = [ + '-DMODE_STR="{}"'.format(mode), + "-D{}".format(mode.value), + "-DRESPECT_TOS", + "-w", + ] + if debug_on: + bpf_flags.append("-DLOG_QPPB") + bpf_flags.append("-DLOG_TC") + + try: + logger.info("Preparing the XDP src: " + src_file) + b = BPF(src_file=src_file.encode(), cflags=bpf_flags, debug=debug) + + logger.info("Loading XDP hooks -- xdp_qppb, xdp_tc_mark") + b.load_func(b"xdp_qppb", BPF.XDP) + b.load_func(b"xdp_tc_mark", BPF.SCHED_CLS) + rnode.bpf = b + except Exception as e: + pytest.skip("Failed to configure XDP environment -- \n" + str(e)) + + qppb_module = "-M vyos_qppb" + logger.info( + "Restart {}, XDP hooks loading...\nPlugin :: {}".format(rnode.name, qppb_module) + ) + kill_router_daemons(tgen, rnode.name, ["bgpd"]) + start_router_daemons(tgen, rnode.name, ["bgpd"], {"bgpd": qppb_module}) + + +def tc_bpf_filter(rnode, ifid): + "Attach tc bpf filter, depends on pyroute2 package" + tc_fn = rnode.bpf.funcs[b"xdp_tc_mark"] + rnode_ns = "/proc/{}/ns/net".format(rnode.net.pid) + + logger.debug("Attach TC-BPF handler '{}'\nNetNS --> {})".format(ifid, rnode_ns)) + # ip.tc("add", "clsact", ifid, "1:") + pushns(rnode_ns) + ip = IPRoute() + ip.tc( + "add-filter", + "bpf", + ifid, + 20, # XXX: + fd=tc_fn.fd, + name=tc_fn.name, + parent=0x10000, + classid=0x10030, # XXX: should be default? default is taken from htb + direct_action=True, + ) + popns() + + +def assert_bw(out, bw_target, tolerance, time=10): + "Assert that connection matches BW in Mbits +- %tolerance" + _min = bw_target * (1 - tolerance) + _max = bw_target * (1 + tolerance) + half_samples = time / 2 + data = json.loads(out) + bws = [] + + for sample in data["intervals"]: + bits = int(sample["sum"]["bits_per_second"]) + mbits = bits / 1024 / 1024 + bw = mbits / 8 + logger.debug("BW sample [{} <= {} <= {}]".format(_min, bw, _max)) + if _min <= bw <= _max: + bws.append(bw) + + _len = len(bws) + assert ( + _len >= half_samples + ), "Only {} samples are within targeted BW [{}:{}%]".format( + _len, bw_target, tolerance * 100 + ) + + +def bpf_print_trace(b): + "XXX: Call this from debugger, to avoid blocking / IO collissions" + logger.info("=" * 40) + logger.debug("Dump bpf log buffer:\n") + line = b.trace_readline(nonblocking=True) + while line: + logger.debug(line) + line = b.trace_readline(nonblocking=True) + + +def tc(rnode, cmd): + logger.debug("TC cmd: " + cmd) + return rnode.cmd_raises("tc " + cmd) + + +def tc_check(host, cmds): + tcoutputs = [tc(host, cmd) for cmd in cmds] + for output in tcoutputs: + if output != "": + logger.debug("TC: " + output) + + +def tc_log_stats(host, iface): + if not DEV_DEBUG: + return + tc_flags = "-g -s -d -p -col" + tc_check( + host, + [ + tc_flags + " filter ls dev " + iface, + tc_flags + " class ls dev " + iface, + tc_flags + " qdisc ls ", + ], + ) diff --git a/tests/topotests/bgp_qppb_flow/bgp_ipv4_nh.ref b/tests/topotests/bgp_qppb_flow/bgp_ipv4_nh.ref new file mode 100644 index 000000000000..4db9698d2168 --- /dev/null +++ b/tests/topotests/bgp_qppb_flow/bgp_ipv4_nh.ref @@ -0,0 +1,10 @@ +BGP routing table entry for 10.61.0.1/32, version XX +Paths: (1 available, best #1, table default) + Advertised to non peer-group peers: + 10.0.0.2 + 10 60 + 10.0.0.2 from 10.0.0.2 (1.0.2.17) + Origin incomplete, valid, external, best (First path received) + Community: 60:1 + QOS: Precedence af11 (10) + Last update: XXXX diff --git a/tests/topotests/bgp_qppb_flow/bgp_xdp_qppb.c b/tests/topotests/bgp_qppb_flow/bgp_xdp_qppb.c new file mode 120000 index 000000000000..b12e30f0f728 --- /dev/null +++ b/tests/topotests/bgp_qppb_flow/bgp_xdp_qppb.c @@ -0,0 +1 @@ +../../../bgpd/bgp_xdp_qppb.c \ No newline at end of file diff --git a/tests/topotests/bgp_qppb_flow/test_bgp_qppb.py b/tests/topotests/bgp_qppb_flow/test_bgp_qppb.py new file mode 100644 index 000000000000..ff55a18cb591 --- /dev/null +++ b/tests/topotests/bgp_qppb_flow/test_bgp_qppb.py @@ -0,0 +1,580 @@ +#!/usr/bin/env python +# +# SPDX-License-Identifier: ISC +# +# test_bgp_qppb.py +# +# Copyright (c) 2023 VyOS Inc. +# Volodymyr Huti +# + +""" +Test QPPB plugin functionality: +- verify bpf map manipulations affect xdp processing properly +- dscp tag is displayed for nexthop entry +- QOS setup balances the traffic throughput via plugin +- LPM overlapping setup + +TODO: +- redirection to different l3 iface based on configured marking +- layer 3 devices are functional +- fragmentation/scalability +""" + +import pytest + +pytestmark = [pytest.mark.bgpd] + +""" +test_bgp_qppb.py: + + 20...1 + +------+ 20...2 + | h1 |----------+ AS30 AS10 AS10 + +------+ ++------+ +------+ +------+ + | | | | | | + +------+ 20..1.2 | R1 | | R2 | | R3 | + | h2 |-----------| |------| |------| | + +------+ +------+ +------+ +------+ + 20..1.1 lo:1.0.1.17 lo:1.0.2.17 | lo:1.0.3.17 + QPPB Router | + | + lo: +------+ + | 1.0.4.17 | | + | 10.61.0.1 | R4 | + | ***** | | + | 10.66.0.1 +------+ + AS60 +""" + +import os +import re +import sys +import time +import functools +import subprocess + +from lib import topotest +from lib.topolog import logger +from lib.topojson import build_config_from_json +from lib.topogen import Topogen, TopoRouter, get_topogen +from lib.bgp import verify_bgp_convergence +from lib.common_config import ( + create_debug_log_config, + apply_raw_config, + start_topology, + TcpDumpHelper, + IPerfHelper, + step, +) + +from bgp_qppb_flow import * +from lib.topotest import version_cmp, interface_to_ifindex +import ctypes + + +xdp_ifindex = lambda host, iface: c_uint(interface_to_ifindex(host, iface)) +af21_tag = c_ubyte(0x12) +af12_tag = c_ubyte(0x0C) +zero_tag = c_ubyte(0) + + +# Helpers +# ------------------------------------------------------- +def setup_test_hosts(tgen, router): + """ + Setup client hosts to test traffic forwarding + NOTE, networks are overlaping for the purpose of lpm_overlap TC + privateDirs empty, so you can str(host) + """ + h1 = tgen.add_host("h1", "20.0.0.1", "dev h1-eth0", private_mounts="") + h2 = tgen.add_host("h2", "20.0.1.1", "dev h2-eth0", private_mounts="") + router.add_link(h1) + router.add_link(h2) + tgen.net.configure_hosts() + + ip_cmd = "ip addr add {} {}" + # XXX: will be used for overlap testing + router.cmd_raises(ip_cmd.format("20.0.0.2/16", "dev " + router.name + "-eth0")) + router.cmd_raises(ip_cmd.format("20.0.1.2/24", "dev " + router.name + "-eth1")) + # XXX: do we really need this? + router.cmd_raises("sysctl -w net.ipv4.conf.all.proxy_arp=1") + + +def check_ping4(rnode, dst, connected=True, src=None, tos=None, count=10, timeout=0): + ping = "" + if timeout: + ping = "timeout {} ".format(timeout) + ping += "ping {} -c{}".format(dst, count) + if src: + ping = "{} -I{}".format(ping, src) + if tos: + ping = "{} -Q{}".format(ping, src) + + match = ", {} packet loss".format("100%" if connected else "0%") + logger.info( + "[+] {} ping -> {}, connection expected -> {}".format( + rnode, dst, "up" if connected else "down" + ) + ) + logger.debug("Executing the ping -> {}".format(ping)) + + def _match_missing(rnode, dst, match): + output = rnode.run(ping) + logger.info(output) + return match not in output + + func = functools.partial(_match_missing, rnode, dst, match) + success, result = topotest.run_and_expect(func, True, count, wait=1) + assert result is True + + +# Module +# ------------------------------------------------------- +def teardown_module(_mod): + "Teardown the pytest environment" + tgen = get_topogen() + tgen.stop_topology() + # iperf_helper.cleanup() + # tcpdumpf_helper.cleanup() + + +def setup_module(mod): + # XXX: write down [ requirement:verion, ... ] + # result |= required_linux_kernel_version("5+") + # result |= required_linux_kernel_features("BPF") + # result |= required_package_version(bcc, dev) + # ... + # if result is not True: + # pytest.skip("Kernel requirements are not met") + # XXX(?): verify that user XPD env doesn't overlap with test + + testsuite_run_time = time.asctime(time.localtime(time.time())) + logger.info("Testsuite start time: {}".format(testsuite_run_time)) + logger.info("=" * 40) + + logger.info("Running setup_module to create topology") + + json_file = f"{CWD}/topo_cisco.json" + tgen = Topogen(json_file, mod.__name__) + global topo + topo = tgen.json_topo + + start_topology(tgen) + build_config_from_json(tgen, topo) + if tgen.routers_have_failure(): + pytest.skip(tgen.errors) + + global BGP_CONVERGENCE + BGP_CONVERGENCE = verify_bgp_convergence(tgen, topo) + assert BGP_CONVERGENCE is True, "setup_module :Failed \n Error: {}".format( + BGP_CONVERGENCE + ) + + # Extra setup steps + # ----------------------------------------------------------------------- + r4 = tgen.gears["r4"] + r1 = tgen.gears["r1"] + + debug_rmap_dict = {"r1": {"raw_config": ["end", "debug route-map"]}} + debug_config_dict = { + "r1": {"debug": {"log_file": "debug.log", "enable": ["bgpd", "zebra"]}} + } + if DEV_DEBUG: + create_debug_log_config(tgen, debug_config_dict) + apply_raw_config(tgen, debug_rmap_dict) + + setup_test_hosts(tgen, r1) + r1.vtysh_cmd( + """ + configure + router bgp 30 + table-map QPPB + """ + ) + # each address will receive different marking (tier of preference) + lo_ip_add = "ip address add dev lo 10.6{0}.0.1/32" + [r4.cmd_raises(lo_ip_add.format(n)) for n in range(1, 7)] + + # Initializing BPF objects + # ----------------------------------------------------------------------- + # NOTE: we need to switch mnt namespace to instantiate BPF mappings + # XXX: python3.12 introduces os.setns, for now use libc directly + ns = "/proc/%d/ns/mnt" % r1.net.pid + nsfd = os.open(ns, os.O_RDONLY) + + libc = ctypes.CDLL("libc.so.6", use_errno=True) + libc.setns(nsfd, 0) + + r1.cmd_raises( + """ + mkdir -p /sys/fs/bpf + mount -t bpf bpf /sys/fs/bpf + """ + ) + load_qppb_plugin(tgen, r1) + + +# Test Cases +# ------------------------------------------------------- +# @pytest.mark.skip +def test_xdp_lpm(tgen): + """ + Manually setup the XDP mappings, without route destribution + Assume that H1 is pinging the managment interface on R4 [lo(1.0.4.17)] + The R1 is marking/forwarding based on QPPB mappings: + qppb_router + h1 -> [ R1 ] -> .... -> R4 + eth0 r1-r2-eth0 + The packet marking happens as follows: + ----------------------------------------- + xdp_qppb(xdp_md *skb): + switch qppb_map[iif]: // idx for eth0 + BgpPolicy.Src: mark = dscp_map[(skb.src, 32)] + BgpPolicy.Dst: mark = dscp_map[(skb.dst, 32)] + NONE: return pass + + if MARK_SKB: skb->tos = mark + if MARK_META: skb->classid = mark + return pass + ----------------------------------------- + """ + h1 = tgen.gears["h1"] + r1 = tgen.gears["r1"] + r4 = tgen.gears["r4"] + + qppb_map = r1.bpf[b"qppb_mode_map"] + dscp_map = r1.bpf[b"dscp_map"] + + r1_eth0_idx = xdp_ifindex(r1, "r1-eth0") + qppb_map[r1_eth0_idx] = BgpPolicy.Dst.value + router_attach_xdp(r1, b"r1-eth0") + + # -------------------------------------------------------------------------------- + tcpdump = TcpDumpHelper(tgen, "icmp[0] == 8") # ICMP Echo requst + r4_lo_key = KeyV4(32, (1, 0, 4, 17)) + h1_key = KeyV4(32, (20, 0, 0, 1)) + r4_lo_ip = "1.0.4.17" + PINGS = 10 + + def _check(sender, receiver, dst, cap_iface, tos, src=None, ping_tos=None): + p1 = tcpdump.capture_start(receiver, cap_iface, background=True, timeout=PINGS) + assert p1, "Failed to run tcpdump on {}:\n{}".format(sender.name, p1) + + check_ping4(sender, dst, src=src, count=PINGS, timeout=PINGS, tos=ping_tos) + time.sleep(1.5) + return tcpdump.find_msg(receiver, "tos 0x%x" % tos.value) + + check_connection = functools.partial(_check, h1, r4, r4_lo_ip, "r4-r3-eth0") + h1.run("ping -c 3 -w 3 " + r4_lo_ip) # refresh arp cache, etc ... + time.sleep(2) + # -------------------------------------------------------------------------------- + dscp_map[r4_lo_key] = af21_tag + found, matches = check_connection(af21_tag) + assert found and matches >= ( + PINGS - 1 # XXX: first packet is not tagged - caching issues? + ), "LPM doesn't work as expected, mark detected only {} times ".format(matches) + + # -------------------------------------------------------------------------------- + dscp_map[r4_lo_key] = af12_tag + found, matches = check_connection(af12_tag) + assert found and matches >= ( + PINGS - 1 + ), "LPM doesn't work as expected, mark detected only {} times ".format(matches) + + # --------------------------------------------------------------------------------- + dscp_map[h1_key] = af12_tag + dscp_map[r4_lo_key] = zero_tag + qppb_map[r1_eth0_idx] = BgpPolicy.Src.value + found, matches = check_connection(af12_tag) + assert found and matches >= ( + PINGS - 1 + ), "LPM doesn't work as expected, mark detected only {} times ".format(matches) + + # -------------------------------------------------------------------------------- + # XXX: Run some flows into opposite directions + # XXX: Use ping with custom tos ... + # XXX: Try using invalid values, i.e. tos > 64 + # ... + # -------------------------------------------------------------------------------- + qppb_map.clear() + dscp_map[h1_key] = af21_tag + dscp_map[r4_lo_key] = af12_tag + found, _ = check_connection(af12_tag) + assert not found, "LPM misbehaviour, markings not expected after clearing dscp map" + + # cleanup used resources + router_remove_xdp(r1, b"r1-eth0") + dscp_map.pop(h1_key) + dscp_map.pop(r4_lo_key) + # XXX dscp_map.clear() - clears the initial config, used by the following test + # bpf_print_trace(bpf) + # breakpoint() + # -------------------------------------------------------------------------------- + + +def test_nh_dscp_displayed(tgen): + """ + Verify that QoS group is displayed for the marked prefix + """ + nhFile = "{}/bgp_ipv4_nh.ref".format(CWD) + expected = open(nhFile).read().rstrip() + expected = ("\n".join(expected.splitlines()) + "\n").rstrip() + + def check_dscp_displayed(): + r1 = tgen.gears["r1"] + actual = r1.vtysh_cmd("show bgp ipv4 10.61.0.1") + actual = ("\n".join(actual.splitlines()) + "\n").rstrip() + actual = re.sub(r" version [0-9]+", " version XX", actual) + actual = re.sub(r"Last update: .*", "Last update: XXXX", actual) + return topotest.get_textdiff( + actual, expected, title1="Actual bgp nh show", title2="Expected bgp nh show" + ) + + ok, result = topotest.run_and_expect(check_dscp_displayed, "", count=5, wait=1) + assert ok, result + + +def test_qos_topo(tgen): + """ + Setup QOS topology and verify traffic prioritization works as expected + + Steps: + --------------------------------------------- + - setup tc on qppb router (r1) + * 10Mbit htb queue + * bandwidth classes + - setup iperf servers + - choose processing mode SKB / META + * attach tc filters + - for SKB, use tc binary + - for META, use pyroute tc func + * run traffic in Dst mode + * run traffic with custom tos + verify, it is respected + * run traffic in Src mode + * flood link with different prio traffic + verify rebalancing works + + dscp | bw Mbytes + ------+------------ + 10 | 7.5 + 20 | 5.0 + 30 | 2.5 + 40 | 2.5 + * | 1 + --------------------------------------------- + Refences: + - ipmininet/tests/test_tc.py + - mininet/examples/test/test_simpleperf.py + - mininet/examples/test/test_intfoptions.py + - mininet/examples/test/test_walkthrough.py + - mininet/mininet/link.py -> class TCIntf + - http://luxik.cdi.cz/~devik/qos/htb/manual/userg.htm + --------------------------------------------- + """ + xdp_dscp = lambda x: c_ubyte(dscp_tos(x)) + dscp_tos = lambda x: x << 2 + h1 = tgen.gears["h1"] + r1 = tgen.gears["r1"] + r4 = tgen.gears["r4"] + + tc_egress_idx = interface_to_ifindex(r1, "r1-r2-eth0") + r1_eth0_idx = xdp_ifindex(r1, "r1-eth0") + qppb_map = r1.bpf[b"qppb_mode_map"] + dscp_map = r1.bpf[b"dscp_map"] + h1_key = KeyV4(24, (20, 0, 0, 1)) + R4_L0_61 = "10.61.0.1" + tolerance = 0.20 # 20% slippage, for short lived connection + TIME_OUT = 8 + bw = 7.5 + + # TC setup + # --------------------------------------------------------------- + _class = "class add dev r1-r2-eth0 parent 1:1 " + _filter = "filter add dev r1-r2-eth0 parent 1:0 " + u32_fmt = "prio %d protocol ip u32 match ip tos %d 0xff classid %s" + tc_setup = [ + "qdisc replace dev r1-r2-eth0 root handle 1:0 htb default 50", + "class add dev r1-r2-eth0 parent 1:0 classid 1:1 htb rate 10Mbps", + _class + "classid 1:10 htb rate 7.5Mbps", + _class + "classid 1:20 htb rate 5.0Mbps", + _class + "classid 1:30 htb rate 2.5Mbps", + _class + "classid 1:40 htb rate 0.5Mbps", + _class + "classid 1:50 htb rate 100kbps", + ] + tc_filters = [ + _filter + u32_fmt % (1, dscp_tos(10), "1:10"), + _filter + u32_fmt % (2, dscp_tos(20), "1:20"), + _filter + u32_fmt % (3, dscp_tos(30), "1:30"), + _filter + u32_fmt % (4, dscp_tos(40), "1:40"), + ] + + # Setup iperf server/client helpers + # --------------------------------------------------------------- + servers = clients = [] + iph = IPerfHelper(tgen) + start_client = functools.partial(iph.iperf, json=True, length=TIME_OUT) + start_server = functools.partial(iph.iperf, server=True, background=True) + for i in range(1, 7): + server = start_server(r4, bind_addr="10.6%d.0.1" % i, port=5200 + i) + servers.append(server) + + tc_check(r1, tc_setup) + for mode in [XdpMode.META, XdpMode.SKB]: + # reset tc filters/ xdp handlers + r1.run("tc filter del dev r1-r2-eth0") + router_remove_xdp(r1, b"r1-eth0") + load_qppb_plugin(tgen, r1, mode=mode) + router_attach_xdp(r1, b"r1-eth0") + + if mode == XdpMode.SKB: + tc_check(r1, tc_filters) + elif mode == XdpMode.META: + tc_bpf_filter(r1, tc_egress_idx) + # refresh arp cache, etc ... + h1.run("ping -c 3 " + R4_L0_61) + time.sleep(1) + # breakpoint() + # TC1: BGP_POLICT_DST + # ----------------------------------------------------------------- + tc_log_stats(r1, "r1-r2-eth0") + qppb_map[r1_eth0_idx] = BgpPolicy.Dst.value + client = start_client(h1, dst=R4_L0_61) + tc_log_stats(r1, "r1-r2-eth0") + + out, err = client.communicate() + assert_bw(out, bw, tolerance, time=TIME_OUT) + + # TC2: Respect TOS + # ----------------------------------------------------------------- + tc_log_stats(r1, "r1-r2-eth0") + client = start_client(h1, dst=R4_L0_61, dscp=20) + tc_log_stats(r1, "r1-r2-eth0") + # bpf_print_trace(r1.bpf) + + bw = 5 + out, err = client.communicate() + assert_bw(out, bw, tolerance, time=TIME_OUT) + + # TC3: BGP_POLICT_SRC, swap host roles + # ----------------------------------------------------------------- + qppb_map[r1_eth0_idx] = BgpPolicy.Src.value + dscp_map[h1_key] = xdp_dscp(10) + tc_log_stats(r1, "r1-r2-eth0") + client = start_client(h1, dst=R4_L0_61) + tc_log_stats(r1, "r1-r2-eth0") + + bw = 7.5 + out, err = client.communicate() + assert_bw(out, bw, tolerance, time=TIME_OUT) + dscp_map[h1_key] = zero_tag + + # TC4: verify bw rebalancing + # - setup all without max prio (10) + # - start max prio + # - verify BW is realocated -> max prio takes full link + # - verify lowest prio gets no BW + # ----------------------------------------------------------------- + # breakpoint() + qppb_map[r1_eth0_idx] = BgpPolicy.Dst.value + for i in range(2, 7): + client = start_client( + h1, dst="10.6%d.0.1" % i, port=5200 + i, timeout=10, background=True + ) + clients.append(client) + + # kill the second highest prio + high_prio = clients[0] + high_prio.kill() + + # max prio flow, should receive the best treatment + client = start_client(h1, dst=R4_L0_61) + out, err = client.communicate() + assert_bw(out, bw, tolerance, time=TIME_OUT) + + time.sleep(5) # XXX: + # iph.stop_host("h1") # does not work with background processes + iperf_json = "%s/iperf_h1_10.66.0.1_client.json" % tgen.logdir + with open(iperf_json) as file: + out = file.read() + # the lowest prio should be ~0.5mbps + assert_bw(out, 0.25, 1, time=TIME_OUT) + + # XXX: swap priority on the fly + # + swap filter priorty + # + swap lpm entries + # + swap direction + # .... + + +@pytest.mark.skip +def test_xdp_network_overlap(tgen): + """ + The feature configuration involves many steps making it quite easy to messup + and accidentally leak traffic, apply wrong preference, etc .. + I`m assuming the following scenarios that may require special handling on xdp side + * configuration mistakes, network loops + * delays during (re)convergance (I guess?) + * using overlaping network ranges (router cascading ?) + * external events (malformed update packets / fail in processing ...?) + * sidefects of admin involvment ... + # XXX(?): how this would work with different kinds of NAT + # XXX: overlaping vs router cascading scenario + Topo: + 20.0.0.0/16 h1-eth1, learned (dscp af22) + h1 <-----++ + r2 <-- r3 <-- r4 + h2 <-----++ + 20.0.1.0/24 h2-eth2, installed by admin + + Admin should be aware that subnetwork 20..1. will receive the QOS treatment from 20.../16, + even though it was not explicitly configured for the new network segment + """ + # XXX,TBD: not sure how common/critical such issues would be + # XXX: likely, there will be many ways to acidentally leak traffic (; + # any tools to detect leaks? i.e. some packet processing stats to look at + r1 = tgen.gears["r1"] + qppb_map = r1.bpf[b"qppb_mode_map"] + dscp_map = r1.bpf[b"dscp_map"] + + r1_eth0_idx = xdp_ifindex(r1, "r1-r2-eth0") + qppb_map[r1_eth0_idx] = BgpPolicy.Dst.value + h1_key = KeyV4(16, (20, 0, 0, 0)) + dscp_map[h1_key] = af21_tag + router_attach_xdp(r1, b"r1-r2-eth0") + + r4 = tgen.gears["r4"] + r4.run("ping -c 3 20.0.0.1") # is tagged with 0x12 + r4.run("ping -c 3 20.0.1.1") # is tagged as well + + # Admin can dissable marking/processing by manually inserting prefix without policy + # The LPM will lookup the more specific prefix and ignore it + h2_key = KeyV4(24, (20, 0, 1, 0)) + dscp_map[h2_key] = zero_tag + r4.run("ping -c 3 20.0.1.1") # should not be tagged any more + + # ::TBD:: + breakpoint() + + +@pytest.mark.skip +def test_get_version(tgen): + "Sanity testing, triggers breapoint" + r1 = tgen.gears["r1"] + r2 = tgen.gears["r2"] + r3 = tgen.gears["r3"] + r4 = tgen.gears["r4"] + version = r1.vtysh_cmd("show version") + logger.info("FRR version is: " + version) + + for host in ["h1", "r1", "r4"]: + tgen.net.hosts[host].run_in_window("bash") + + bpf_print_trace(r1.bpf) + breakpoint() + + +if __name__ == "__main__": + args = ["-s"] + sys.argv[1:] + sys.exit(pytest.main(args)) diff --git a/tests/topotests/bgp_qppb_flow/topo_cisco.json b/tests/topotests/bgp_qppb_flow/topo_cisco.json new file mode 100644 index 000000000000..c9bbd506441c --- /dev/null +++ b/tests/topotests/bgp_qppb_flow/topo_cisco.json @@ -0,0 +1,383 @@ +{ + "address_types": ["ipv4"], + "ipv4base": "10.0.0.0", + "ipv4mask": 30, + "ipv6base": "fd00::", + "ipv6mask": 64, + "link_ip_start": { + "ipv4": "10.0.0.0", + "v4mask": 30, + "ipv6": "fd00::", + "v6mask": 64 + }, + "lo_prefix": { + "ipv4": "1.0.", + "v4mask": 32, + "ipv6": "2001:DB8:F::", + "v6mask": 128 + }, + "routers": { + "r1": { + "links": { + "lo": { + "ipv4": "auto", + "ipv6": "auto", + "type": "loopback" + }, + "r2": { + "ipv4": "auto", + "ipv6": "auto" + } + }, + "bgp": { + "local_as": "30", + "address_family": { + "ipv4": { + "unicast": { + "redistribute": [ + { "redist_type": "static" }, + { "redist_type": "connected" } + ], + "neighbor": { + "r2": { + "dest_link": { + "r1": {} + } + } + } + } + } + } + }, + "bgp_community_lists": [ + { + "community_type": "standard", + "action": "permit", + "name": "comm_list_1", + "value": "60:1" + }, + { + "community_type": "standard", + "action": "permit", + "name": "comm_list_2", + "value": "60:2" + }, + { + "community_type": "standard", + "action": "permit", + "name": "comm_list_3", + "value": "60:3" + }, + { + "community_type": "standard", + "action": "permit", + "name": "comm_list_4", + "value": "60:4" + }, + { + "community_type": "standard", + "action": "permit", + "name": "comm_list_5", + "value": "60:5" + }, + { + "community_type": "standard", + "action": "permit", + "name": "comm_list_6", + "value": "60:6" + }, + { + "community_type": "standard", + "action": "permit", + "name": "comm_list_7", + "value": "60:7" + } + ], + "route_maps": { + "QPPB": [ + { + "action": "permit", + "match": { "community_list": { "id": "comm_list_1" } }, + "set": { "dscp": "10" } + }, + { + "action": "permit", + "match": { "community_list": { "id": "comm_list_2" } }, + "set": { "dscp": "20" } + }, + { + "action": "permit", + "match": { "community_list": { "id": "comm_list_3" } }, + "set": { "dscp": "30" } + }, + { + "action": "permit", + "match": { "community_list": { "id": "comm_list_4" } }, + "set": { "dscp": "40" } + }, + { + "action": "permit", + "match": { "community_list": { "id": "comm_list_5" } }, + "set": { "dscp": "50" } + }, + { + "action": "permit", + "match": { "community_list": { "id": "comm_list_6" } }, + "set": { "dscp": "15" } + }, + { + "action": "permit", + "match": { "community_list": { "id": "comm_list_7" } }, + "set": { "dscp": "25" } + }, + { + "action": "permit", + "match": { "ipv4": {"prefix_lists": "pf_list_1"} }, + "set": { "dscp": "63" } + }, + { "action": "permit" } + ] + }, + "prefix_lists": { + "ipv4": { + "pf_list_1": [ + { + "seqid": "69", + "network": "10.69.0.0/30", + "action": "permit" + } + ] + } + } + }, + "r2": { + "links": { + "lo": { + "ipv4": "auto", + "ipv6": "auto", + "type": "loopback" + }, + "r1": { + "ipv4": "auto", + "ipv6": "auto" + }, + "r3": { + "ipv4": "auto", + "ipv6": "auto" + } + }, + "bgp": { + "local_as": "10", + "address_family": { + "ipv4": { + "unicast": { + "redistribute": [ + { "redist_type": "static" }, + { "redist_type": "connected" } + ], + "neighbor": { + "r1": { + "dest_link": { + "r2": { + "route_maps": [{ + "name": "send_community", + "direction": "out" + }] + } + } + }, + "r3": { + "dest_link": { + "r2": {} + } + } + } + } + } + } + }, + "route_maps": { + "send_community": [ + { + "action": "permit", + "match": { "ipv4": {"prefix_lists": "pf_list_1"} }, + "set": { + "community": { + "num": "60:1", + "action": "additive" + } + } + }, + { + "action": "permit", + "match": { "ipv4": {"prefix_lists": "pf_list_2"} }, + "set": { + "community": { + "num": "60:2", + "action": "additive" + } + } + }, + { + "action": "permit", + "match": { "ipv4": {"prefix_lists": "pf_list_3"} }, + "set": { + "community": { + "num": "60:3", + "action": "additive" + } + } + }, + { + "action": "permit", + "match": { "ipv4": {"prefix_lists": "pf_list_4"} }, + "set": { + "community": { + "num": "60:4", + "action": "additive" + } + } + }, + { + "action": "permit", + "match": { "ipv4": {"prefix_lists": "pf_list_5"} }, + "set": { + "community": { + "num": "60:5", + "action": "additive" + } + } + }, + { + "action": "permit", + "match": { "ipv4": {"prefix_lists": "pf_list_6"} }, + "set": { + "community": { + "num": "60:6", + "action": "additive" + } + } + }, + { + "action": "permit", + "match": { "ipv4": {"prefix_lists": "pf_list_7"} }, + "set": { + "community": { + "num": "60:7", + "action": "additive" + } + } + }, + { "action": "permit" } + ] + }, + "prefix_lists": { + "ipv4": { + "pf_list_1": [{ + "network": "10.61.0.1/32", + "action": "permit" + }], + "pf_list_2": [{ + "network": "10.62.0.1/32", + "action": "permit" + }], + "pf_list_3": [{ + "network": "10.63.0.1/32", + "action": "permit" + }], + "pf_list_4": [{ + "network": "10.64.0.1/32", + "action": "permit" + }], + "pf_list_5": [{ + "network": "10.65.0.1/32", + "action": "permit" + }], + "pf_list_6": [{ + "network": "10.66.0.1/32", + "action": "permit" + }], + "pf_list_7": [{ + "network": "10.67.0.1/32", + "action": "permit" + }] + } + } + }, + "r3": { + "links": { + "lo": { + "ipv4": "auto", + "ipv6": "auto", + "type": "loopback" + }, + "r2": { + "ipv4": "auto", + "ipv6": "auto" + }, + "r4": { + "ipv4": "auto", + "ipv6": "auto" + } + }, + "bgp": { + "local_as": "10", + "address_family": { + "ipv4": { + "unicast": { + "redistribute": [ + { "redist_type": "static" }, + { "redist_type": "connected" } + ], + "neighbor": { + "r2": { + "dest_link": { + "r3": {} + } + }, + "r4": { + "dest_link": { + "r3": {} + } + } + } + } + } + } + } + }, + "r4": { + "links": { + "lo": { + "ipv4": "auto", + "ipv6": "auto", + "type": "loopback" + }, + "r3": { + "ipv4": "auto", + "ipv6": "auto" + } + }, + "bgp": { + "local_as": "60", + "address_family": { + "ipv4": { + "unicast": { + "redistribute": [ + { "redist_type": "static" }, + { "redist_type": "connected" } + ], + "neighbor": { + "r3": { + "dest_link": { + "r4": {} + } + } + } + } + } + } + } + } + } +} diff --git a/tests/topotests/lib/common_config.py b/tests/topotests/lib/common_config.py index 55efb1b09502..12c719f6273b 100644 --- a/tests/topotests/lib/common_config.py +++ b/tests/topotests/lib/common_config.py @@ -9,6 +9,7 @@ import ipaddress import json import os +import re import platform import socket import subprocess @@ -137,6 +138,7 @@ ], } + def is_string(value): try: return isinstance(value, basestring) @@ -408,7 +410,7 @@ def kill_router_daemons(tgen, router, daemons, save_config=True): return errormsg -def start_router_daemons(tgen, router, daemons): +def start_router_daemons(tgen, router, daemons, plugins=None): """ Daemons defined by user would be started * `tgen` : topogen object @@ -422,7 +424,7 @@ def start_router_daemons(tgen, router, daemons): router_list = tgen.routers() # Start daemons - res = router_list[router].startDaemons(daemons) + res = router_list[router].startDaemons(daemons, plugins) except Exception as e: errormsg = traceback.format_exc() @@ -2322,6 +2324,7 @@ def create_route_maps(tgen, input_dict, build=False): # med: metric value advertised for AS # aspath: set AS path value # weight: weight for the route + # dscp: dscp for the routed traffic, int or code point name # community: standard community value to be attached # large_community: large community value to be attached # community_additive: if set to "additive", adds community/large-community @@ -2354,6 +2357,7 @@ def create_route_maps(tgen, input_dict, build=False): "set": { "locPrf": 150, "metric": 30, + "dscp": "af21", "path": { "num": 20000, "action": "prepend", @@ -2452,6 +2456,7 @@ def create_route_maps(tgen, input_dict, build=False): ipv6_data = set_data.setdefault("ipv6", {}) local_preference = set_data.setdefault("locPrf", None) metric = set_data.setdefault("metric", None) + dscp = set_data.setdefault("dscp", None) metric_type = set_data.setdefault("metric-type", None) as_path = set_data.setdefault("path", {}) weight = set_data.setdefault("weight", None) @@ -2482,6 +2487,14 @@ def create_route_maps(tgen, input_dict, build=False): else: rmap_data.append("set metric {}".format(metric)) + # Dscp + if dscp: + del_comm = set_data.setdefault("delete", None) + if del_comm: + rmap_data.append("no set dscp {}".format(dscp)) + else: + rmap_data.append("set dscp {}".format(dscp)) + # Origin if origin: rmap_data.append("set origin {} \n".format(origin)) @@ -4879,6 +4892,7 @@ def find_msg(self, host, message, count=0): ) return matches != 0, matches + def verify_ip_nht(tgen, input_dict): """ Running "show ip nht" command and verifying given nexthop resolution diff --git a/tests/topotests/lib/topogen.py b/tests/topotests/lib/topogen.py index 48caf6f03a54..d0e486397c8f 100644 --- a/tests/topotests/lib/topogen.py +++ b/tests/topotests/lib/topogen.py @@ -347,7 +347,7 @@ def add_exabgp_peer(self, name, ip, defaultRoute): self.peern += 1 return self.gears[name] - def add_host(self, name, ip, defaultRoute): + def add_host(self, name, ip, defaultRoute, **kwargs): """ Adds a new host to the topology. This function has the following parameters: @@ -359,7 +359,9 @@ def add_host(self, name, ip, defaultRoute): if name in self.gears: raise KeyError("host already exists") - self.gears[name] = TopoHost(self, name, ip=ip, defaultRoute=defaultRoute) + self.gears[name] = TopoHost( + self, name, ip=ip, defaultRoute=defaultRoute, **kwargs + ) self.peern += 1 return self.gears[name] @@ -915,7 +917,7 @@ def stop(self): self.logger.debug("stopping (no assert)") return self.net.stopRouter(False) - def startDaemons(self, daemons): + def startDaemons(self, daemons, plugins=None): """ Start Daemons: to start specific daemon(user defined daemon only) * Start daemons (e.g. FRR) @@ -923,7 +925,7 @@ def startDaemons(self, daemons): """ self.logger.debug("starting") nrouter = self.net - result = nrouter.startRouterDaemons(daemons) + result = nrouter.startRouterDaemons(daemons, plugins=plugins) if daemons is None: daemons = nrouter.daemons.keys() diff --git a/tests/topotests/lib/topotest.py b/tests/topotests/lib/topotest.py index 3eb808ac4f45..0b96a8f57dab 100644 --- a/tests/topotests/lib/topotest.py +++ b/tests/topotests/lib/topotest.py @@ -734,6 +734,18 @@ def version_cmp(v1, v2): return 0 +def interface_to_ifindex(node, iface): + """ + Gets the interface index using its name. Returns None on failure. + """ + interfaces = json.loads(node.cmd_raises("ip -j link show")) + for interface in interfaces: + if interface["ifname"] == iface: + return int(interface["ifindex"]) + + return None + + def interface_set_status(node, ifacename, ifaceaction=False, vrf_name=None): if ifaceaction: str_ifaceaction = "no shutdown" @@ -1797,7 +1809,7 @@ def getLog(self, log, daemon): log = file.read() return log - def startRouterDaemons(self, daemons=None, tgen=None): + def startRouterDaemons(self, daemons=None, tgen=None, plugins=None): "Starts FRR daemons for this router." asan_abort = bool(g_pytest_config.option.asan_abort) @@ -2162,6 +2174,13 @@ def emacs_gdb_ready(): while "mgmtd" in daemons_list: daemons_list.remove("mgmtd") + # XXX: handle plugins properly - per daemon + bgpd_plugins = plugins.get("bgpd") if plugins else None + if "bgpd" in daemons_list and bgpd_plugins: + start_daemon("bgpd", bgpd_plugins) + while "bgpd" in daemons_list: + daemons_list.remove("bgpd") + # Start Zebra after mgmtd if "zebra" in daemons_list: start_daemon("zebra", "-s 90000000")