bgpd, zebra, tests: move plugin down to zebra

Move plugin invocation lower in the demon stack.
This way, dscp tag is exposed for usage to the rest of demons.

Signed-off-by: Volodymyr Huti <[email protected]>

bgpd/bgp_zebra.c

@@ -63,8 +63,6 @@ static bool bgp_zebra_label_manager_connect(void);
 /* hook to indicate vrf status change for SNMP */
 DEFINE_HOOK(bgp_vrf_status_changed, (struct bgp *bgp, struct interface *ifp),
 	    (bgp, ifp));
-DEFINE_HOOK(bgp_qppb_mark_prefix,
-	    (const struct prefix *p, uint8_t dscp, bool add), (p, dscp, add));
 
 DEFINE_MTYPE_STATIC(BGPD, BGP_IF_INFO, "BGP interface context");
 
@@ -1647,6 +1645,11 @@ bgp_zebra_announce_actual(struct bgp_dest *dest, struct bgp_path_info *info,
 	SET_FLAG(api.message, ZAPI_MESSAGE_METRIC);
 	api.metric = metric;
 
+	if (dscp) {
+		SET_FLAG(api.message, ZAPI_MESSAGE_DSCP);
+		api.dscp = dscp;
+	}
+
 	if (tag) {
 		SET_FLAG(api.message, ZAPI_MESSAGE_TAG);
 		api.tag = tag;
@@ -1672,7 +1675,6 @@ bgp_zebra_announce_actual(struct bgp_dest *dest, struct bgp_path_info *info,
 		zlog_debug("%s: %pFX: announcing to zebra (recursion %sset)",
 			   __func__, p, (recursion_flag ? "" : "NOT "));
 	}
-	hook_call(bgp_qppb_mark_prefix, p, dscp, is_add);
 	return zclient_route_send(is_add ? ZEBRA_ROUTE_ADD : ZEBRA_ROUTE_DELETE,
 				  zclient, &api);
 }

bgpd/bgpd.h

@@ -2753,8 +2753,6 @@ DECLARE_HOOK(bgp_rpki_prefix_status,
 	     (struct peer * peer, struct attr *attr,
 	      const struct prefix *prefix),
 	     (peer, attr, prefix));
-DECLARE_HOOK(bgp_qppb_mark_prefix,
-	     (const struct prefix *p, uint8_t dscp, bool add), (p, dscp, add));
 
 void peer_nsf_stop(struct peer *peer);
 

bgpd/subdir.am

@@ -9,9 +9,6 @@ noinst_PROGRAMS += bgpd/bgp_btoa
 
 vtysh_daemons += bgpd
 
-if QPPB
-module_LTLIBRARIES += bgpd/bgpd_qppb.la
-endif
 if SNMP
 module_LTLIBRARIES += bgpd/bgpd_snmp.la
 endif
@@ -209,10 +206,6 @@ bgpd_bgpd_bmp_la_SOURCES = bgpd/bgp_bmp.c
 bgpd_bgpd_bmp_la_LIBADD = lib/libfrrcares.la
 bgpd_bgpd_bmp_la_LDFLAGS = $(MODULE_LDFLAGS)
 
-bgpd_bgpd_qppb_la_SOURCES = bgpd/bgp_qppb_private.c
-bgpd_bgpd_qppb_la_LIBADD = lib/libfrrcares.la
-bgpd_bgpd_qppb_la_LDFLAGS = $(MODULE_LDFLAGS) -lbpf
-
 clippy_scan += \
 	bgpd/bgp_bmp.c \
 	bgpd/bgp_debug.c \

lib/zclient.c

@@ -1337,6 +1337,8 @@ int zapi_route_encode(uint8_t cmd, struct stream *s, struct zapi_route *api)
 		stream_putc(s, api->distance);
 	if (CHECK_FLAG(api->message, ZAPI_MESSAGE_METRIC))
 		stream_putl(s, api->metric);
+	if (CHECK_FLAG(api->message, ZAPI_MESSAGE_DSCP))
+		stream_putl(s, api->dscp);
 	if (CHECK_FLAG(api->message, ZAPI_MESSAGE_TAG))
 		stream_putl(s, api->tag);
 	if (CHECK_FLAG(api->message, ZAPI_MESSAGE_MTU))
@@ -1590,6 +1592,8 @@ int zapi_route_decode(struct stream *s, struct zapi_route *api)
 		STREAM_GETC(s, api->distance);
 	if (CHECK_FLAG(api->message, ZAPI_MESSAGE_METRIC))
 		STREAM_GETL(s, api->metric);
+	if (CHECK_FLAG(api->message, ZAPI_MESSAGE_DSCP))
+		STREAM_GETL(s, api->dscp);
 	if (CHECK_FLAG(api->message, ZAPI_MESSAGE_TAG))
 		STREAM_GETL(s, api->tag);
 	if (CHECK_FLAG(api->message, ZAPI_MESSAGE_MTU))

lib/zclient.h

@@ -406,6 +406,7 @@ extern int zclient_bfd_session_update(ZAPI_CALLBACK_ARGS);
 #define ZAPI_MESSAGE_TABLEID 0x0100
 #define ZAPI_MESSAGE_SRTE 0x0200
 #define ZAPI_MESSAGE_OPAQUE 0x0400
+#define ZAPI_MESSAGE_DSCP 0x0800
 
 #define ZSERV_VERSION 6
 /* Zserv protocol message header */
@@ -591,6 +592,8 @@ struct zapi_route {
 
 	uint32_t metric;
 
+	uint32_t dscp;
+
 	route_tag_t tag;
 
 	uint32_t mtu;

tests/topotests/bgp_qppb_flow/__init__.py

@@ -88,7 +88,7 @@ def load_qppb_plugin(tgen, rnode, mode=XdpMode.SKB, debug_on=DEV_DEBUG):
     """
     debug_flags = DEBUG_BPF | DEBUG_PREPROCESSOR | DEBUG_SOURCE | DEBUG_BTF
     debug = debug_flags if debug_on else 0
-    src_file = CWD + "/bgp_xdp_qppb.c"
+    src_file = CWD + "/zebra_xdp_qppb.c"
     bpf_flags = [
         '-DMODE_STR="{}"'.format(mode),
         "-D{}".format(mode.value),
@@ -110,13 +110,6 @@ def load_qppb_plugin(tgen, rnode, mode=XdpMode.SKB, debug_on=DEV_DEBUG):
     except Exception as e:
         pytest.skip("Failed to configure XDP environment -- \n" + str(e))
 
-    qppb_module = "-M vyos_qppb"
-    logger.info(
-        "Restart {}, XDP hooks loading...\nPlugin :: {}".format(rnode.name, qppb_module)
-    )
-    kill_router_daemons(tgen, rnode.name, ["bgpd"])
-    start_router_daemons(tgen, rnode.name, ["bgpd"], {"bgpd": qppb_module})
-
 
 def tc_bpf_filter(rnode, ifid):
     "Attach tc bpf filter, depends on pyroute2 package"

tests/topotests/bgp_qppb_flow/test_bgp_qppb.py

@@ -160,6 +160,27 @@ def setup_module(mod):
     global topo
     topo = tgen.json_topo
 
+    r1 = tgen.gears["r1"]
+    r4 = tgen.gears["r4"]
+    # Initializing BPF objects
+    # -----------------------------------------------------------------------
+    # NOTE: we need to switch mnt namespace to instantiate BPF mappings
+    # XXX: python3.12 introduces os.setns, for now use libc directly
+    ns = "/proc/%d/ns/mnt" % r1.net.pid
+    nsfd = os.open(ns, os.O_RDONLY)
+
+    libc = ctypes.CDLL("libc.so.6", use_errno=True)
+    libc.setns(nsfd, 0)
+
+    tgen.qppb_nodes.append("r1")
+    r1.cmd_raises(
+        """
+        mkdir -p /sys/fs/bpf
+        mount -t bpf bpf /sys/fs/bpf
+    """
+    )
+    load_qppb_plugin(tgen, r1)
+
     start_topology(tgen)
     build_config_from_json(tgen, topo)
     if tgen.routers_have_failure():
@@ -173,9 +194,6 @@ def setup_module(mod):
 
     # Extra setup steps
     # -----------------------------------------------------------------------
-    r4 = tgen.gears["r4"]
-    r1 = tgen.gears["r1"]
-
     debug_rmap_dict = {"r1": {"raw_config": ["end", "debug route-map"]}}
     debug_config_dict = {
         "r1": {"debug": {"log_file": "debug.log", "enable": ["bgpd", "zebra"]}}
@@ -196,23 +214,6 @@ def setup_module(mod):
     lo_ip_add = "ip address add dev lo 10.6{0}.0.1/32"
     [r4.cmd_raises(lo_ip_add.format(n)) for n in range(1, 7)]
 
-    # Initializing BPF objects
-    # -----------------------------------------------------------------------
-    # NOTE: we need to switch mnt namespace to instantiate BPF mappings
-    # XXX: python3.12 introduces os.setns, for now use libc directly
-    ns = "/proc/%d/ns/mnt" % r1.net.pid
-    nsfd = os.open(ns, os.O_RDONLY)
-
-    libc = ctypes.CDLL("libc.so.6", use_errno=True)
-    libc.setns(nsfd, 0)
-
-    r1.cmd_raises(
-        """
-        mkdir -p /sys/fs/bpf
-        mount -t bpf bpf /sys/fs/bpf
-    """
-    )
-    load_qppb_plugin(tgen, r1)
 
 
 # Test Cases

tests/topotests/bgp_qppb_flow/zebra_xdp_qppb.c

@@ -0,0 +1 @@
+../../../zebra/zebra_xdp_qppb.c

tests/topotests/lib/topogen.py

@@ -154,6 +154,7 @@ def __init__(self, topodef, modname="unnamed"):
         self.peern = 1
         self.cfg_gen = 0
         self.exabgp_cmd = None
+        self.qppb_nodes = []
         self._init_topo(topodef)
 
         logger.info("loading topology: {}".format(self.modname))

tests/topotests/lib/topotest.py

@@ -2209,19 +2209,16 @@ def emacs_gdb_ready():
             while "mgmtd" in daemons_list:
                 daemons_list.remove("mgmtd")
 
-        # XXX: handle plugins properly - per daemon
-        bgpd_plugins = plugins.get("bgpd") if plugins else None
-        if "bgpd" in daemons_list and bgpd_plugins:
-            start_daemon("bgpd", bgpd_plugins)
-            while "bgpd" in daemons_list:
-                daemons_list.remove("bgpd")
-
         # Start Zebra after mgmtd
+        zebra_plugins = ""
+        if tgen and self.name in tgen.qppb_nodes:
+            zebra_plugins = "-M zebra_qppb"
         if "zebra" in daemons_list:
-            start_daemon("zebra", "-s 90000000")
+            start_daemon("zebra", "-s 90000000 " + zebra_plugins)
             while "zebra" in daemons_list:
                 daemons_list.remove("zebra")
 
+
         # Start staticd next if required
         if "staticd" in daemons_list:
             start_daemon("staticd")

zebra/rib.h

@@ -103,6 +103,9 @@ struct route_entry {
 	/* Metric */
 	uint32_t metric;
 
+	/* DSCP */
+	uint32_t dscp;
+
 	/* MTU */
 	uint32_t mtu;
 	uint32_t nexthop_mtu;

zebra/subdir.am

@@ -18,6 +18,9 @@ endif
 if LINUX
 module_LTLIBRARIES += zebra/zebra_cumulus_mlag.la
 endif
+if QPPB
+module_LTLIBRARIES += zebra/zebra_qppb.la
+endif
 
 #if FPM_LISTENER
 sbin_PROGRAMS += zebra/fpm_listener
@@ -204,6 +207,10 @@ noinst_HEADERS += \
 	zebra/dpdk/zebra_dplane_dpdk_private.h \
 	# end
 
+zebra_zebra_qppb_la_SOURCES = zebra/zebra_qppb_private.c
+zebra_zebra_qppb_la_LIBADD = lib/libfrrcares.la
+zebra_zebra_qppb_la_LDFLAGS = $(MODULE_LDFLAGS) -lbpf
+
 zebra_zebra_irdp_la_SOURCES = \
 	zebra/irdp_interface.c \
 	zebra/irdp_main.c \

zebra/zapi_msg.c

@@ -55,6 +55,10 @@
 
 DEFINE_MTYPE_STATIC(ZEBRA, RE_OPAQUE, "Route Opaque Data");
 
+DEFINE_HOOK(zebra_qppb_mark_prefix,
+	    (const struct prefix *p, uint8_t dscp, bool add), (p, dscp, add));
+
+
 static int zapi_nhg_decode(struct stream *s, int cmd, struct zapi_nhg *api_nhg);
 
 /* Encoding helpers -------------------------------------------------------- */
@@ -604,6 +608,10 @@ int zsend_redistribute_route(int cmd, struct zserv *client,
 		api.distance = re->distance;
 	SET_FLAG(api.message, ZAPI_MESSAGE_METRIC);
 	api.metric = re->metric;
+	if (re->dscp) {
+		SET_FLAG(api.message, ZAPI_MESSAGE_DSCP);
+		api.dscp = re->dscp;
+	}
 	if (re->tag) {
 		SET_FLAG(api.message, ZAPI_MESSAGE_TAG);
 		api.tag = re->tag;
@@ -2182,6 +2190,8 @@ static void zread_route_add(ZAPI_HANDLER_ARGS)
 		XFREE(MTYPE_RE, re);
 	}
 
+	hook_call(zebra_qppb_mark_prefix, &api.prefix, api.dscp, true);
+
 	/* At this point, these allocations are not needed: 're' has been
 	 * retained or freed, and if 're' still exists, it is using
 	 * a reference to a shared group object.
@@ -2248,6 +2258,8 @@ static void zread_route_del(ZAPI_HANDLER_ARGS)
 		   api.flags, &api.prefix, src_p, NULL, 0, table_id, api.metric,
 		   api.distance, false);
 
+	hook_call(zebra_qppb_mark_prefix, &api.prefix, api.dscp, false);
+
 	/* Stats */
 	switch (api.prefix.family) {
 	case AF_INET:

bgpd/bgp_qppb_private.c → zebra/zebra_qppb_private.c

@@ -1,18 +1,19 @@
 // SPDX-License-Identifier: GPL-2.0-or-later
 /*
- * BGP QPPB support
+ * zebra QPPB support
  * Copyright (C) 2023 VyOS Inc.
  * Volodymyr Huti
  */
 
 #include <zebra.h>
 #include <bpf/bpf.h>
 #include <bpf/libbpf.h>
-#include "bgpd/bgpd.h"
 #include "log.h"
 #include "prefix.h"
 #include "privs.h"
 
+extern struct zebra_privs_t zserv_privs;
+
 #define BPF_DSCP_MAP "dscp_map"
 #define BPF_PIN_DIR  "/sys/fs/bpf/"
 /*
@@ -24,7 +25,8 @@
  */
 #define BPF_LPM_KEY_SIZE (sizeof(struct bpf_lpm_trie_key) + sizeof(__u32))
 
-extern struct zebra_privs_t bgpd_privs;
+DECLARE_HOOK(zebra_qppb_mark_prefix,
+	     (const struct prefix *p, uint8_t dscp, bool add), (p, dscp, add));
 static int dscp_map_fd;
 
 static int open_bpf_map_file(const char *pin_dir, const char *mapname)
@@ -45,26 +47,26 @@ static int open_bpf_map_file(const char *pin_dir, const char *mapname)
 	return fd;
 }
 
-static void bgp_qppb_map_init(void)
+static void zebra_qppb_map_init(void)
 {
 	const char *pin_dir = THIS_MODULE->load_args ?: BPF_PIN_DIR;
 
 	dscp_map_fd = open_bpf_map_file(pin_dir, BPF_DSCP_MAP);
 }
 
-static int bgp_qppb_mark_prefix(const struct prefix *p, uint8_t dscp, bool add)
+static int zebra_qppb_mark_prefix(const struct prefix *p, uint8_t dscp, bool add)
 {
 	struct bpf_lpm_trie_key *key_ipv4;
 	int err = 0;
 
-	if (dscp_map_fd < 0)
+	if (dscp_map_fd < 0 || !dscp)
 		return err;
 
 	key_ipv4 = alloca(BPF_LPM_KEY_SIZE);
 	key_ipv4->prefixlen = p->prefixlen;
 	memcpy(key_ipv4->data, &p->u.prefix4, sizeof(struct in_addr));
 
-	frr_with_privs (&bgpd_privs) {
+	frr_with_privs (&zserv_privs) {
 		err = add ? bpf_map_update_elem(dscp_map_fd, key_ipv4, &dscp, 0)
 			  : bpf_map_delete_elem(dscp_map_fd, key_ipv4);
 	}
@@ -73,13 +75,13 @@ static int bgp_qppb_mark_prefix(const struct prefix *p, uint8_t dscp, bool add)
 	return err;
 }
 
-static int bgp_qppb_module_init(void)
+static int zebra_qppb_module_init(void)
 {
-	bgp_qppb_map_init();
-	hook_register(bgp_qppb_mark_prefix, bgp_qppb_mark_prefix);
+	zebra_qppb_map_init();
+	hook_register(zebra_qppb_mark_prefix, zebra_qppb_mark_prefix);
 	return 0;
 }
 
-FRR_MODULE_SETUP(.name = "bgp_vyos_qppb", .version = "0.0.1",
-		 .description = "bgp QPPB implementation for VyOS",
-		 .init = bgp_qppb_module_init);
+FRR_MODULE_SETUP(.name = "zebra_vyos_qppb", .version = "0.0.1",
+		 .description = "zebra QPPB plugin for VyOS",
+		 .init = zebra_qppb_module_init);