EVPN L2VNI/L3VNI Optimize inline Global walk for remote route installations

bgpd/bgp_evpn.h

@@ -200,4 +200,6 @@ bool bgp_evpn_skip_vrf_import_of_local_es(struct bgp *bgp_vrf, const struct pref
 int uninstall_evpn_route_entry_in_vrf(struct bgp *bgp_vrf, const struct prefix_evpn *evp,
 				      struct bgp_path_info *parent_pi);
 extern void bgp_zebra_evpn_pop_items_from_announce_fifo(struct bgpevpn *vpn);
+extern int install_uninstall_routes_for_vni(struct bgp *bgp, struct bgpevpn *vpn, bool install);
+extern int install_uninstall_routes_for_vrf(struct bgp *bgp_vrf, bool install);
 #endif /* _QUAGGA_BGP_EVPN_H */

bgpd/bgp_evpn_private.h

@@ -32,6 +32,13 @@
 #define BGP_EVPN_TYPE4_V4_PSIZE 23
 #define BGP_EVPN_TYPE4_V6_PSIZE 34
 
+static const struct message bgp_evpn_route_type_str[] = { { BGP_EVPN_AD_ROUTE, "AD" },
+							  { BGP_EVPN_MAC_IP_ROUTE, "MACIP" },
+							  { BGP_EVPN_IMET_ROUTE, "IMET" },
+							  { BGP_EVPN_ES_ROUTE, "ES" },
+							  { BGP_EVPN_IP_PREFIX_ROUTE, "IP-PREFIX" },
+							  { 0 } };
+
 RB_HEAD(bgp_es_evi_rb_head, bgp_evpn_es_evi);
 RB_PROTOTYPE(bgp_es_evi_rb_head, bgp_evpn_es_evi, rb_node,
 		bgp_es_evi_rb_cmp);
@@ -53,8 +60,9 @@ struct bgpevpn {
 #define VNI_FLAG_RD_CFGD           0x4  /* RD is user configured. */
 #define VNI_FLAG_IMPRT_CFGD        0x8  /* Import RT is user configured */
 #define VNI_FLAG_EXPRT_CFGD        0x10 /* Export RT is user configured */
-#define VNI_FLAG_USE_TWO_LABELS    0x20 /* Attach both L2-VNI and L3-VNI if
-					   needed for this VPN */
+/* Attach both L2-VNI and L3-VNI if needed for this VPN */
+#define VNI_FLAG_USE_TWO_LABELS 0x20
+#define VNI_FLAG_ADD		0x40 /* L2VNI Add */
 
 	struct bgp *bgp_vrf; /* back pointer to the vrf instance */
 
@@ -108,11 +116,15 @@ struct bgpevpn {
 	/* List of local ESs */
 	struct list *local_es_evi_list;
 
+	struct zebra_l2_vni_item zl2vni;
+
 	QOBJ_FIELDS;
 };
 
 DECLARE_QOBJ_TYPE(bgpevpn);
 
+DECLARE_LIST(zebra_l2_vni, struct bgpevpn, zl2vni);
+
 /* Mapping of Import RT to VNIs.
  * The Import RTs of all VNIs are maintained in a hash table with each
  * RT linking to all VNIs that will import routes matching this RT.

bgpd/bgp_main.c

@@ -207,6 +207,8 @@ static __attribute__((__noreturn__)) void bgp_exit(int status)
 	bgp_nhg_finish();
 
 	zebra_announce_fini(&bm->zebra_announce_head);
+	zebra_l2_vni_fini(&bm->zebra_l2_vni_head);
+	zebra_l3_vni_fini(&bm->zebra_l3_vni_head);
 
 	/* reverse bgp_dump_init */
 	bgp_dump_finish();

bgpd/bgp_vty.c

@@ -1696,8 +1696,13 @@ DEFUN (no_router_bgp,
 		}
 
 		if (bgp->l3vni) {
-			vty_out(vty, "%% Please unconfigure l3vni %u\n",
-				bgp->l3vni);
+			if (CHECK_FLAG(bgp->flags, BGP_FLAG_L3VNI_SCHEDULE_FOR_DELETE))
+				vty_out(vty,
+					"%% L3VNI %u is scheduled to be deleted. Please give it few secs and retry the command\n",
+					bgp->l3vni);
+			else
+				vty_out(vty, "%% Please unconfigure l3vni %u\n", bgp->l3vni);
+
 			return CMD_WARNING_CONFIG_FAILED;
 		}
 

bgpd/bgp_zebra.c

@@ -3029,6 +3029,48 @@ static void bgp_zebra_connected(struct zclient *zclient)
 	BGP_GR_ROUTER_DETECT_AND_SEND_CAPABILITY_TO_ZEBRA(bgp, bgp->peer);
 }
 
+void bgp_zebra_process_remote_routes_for_l2vni(struct event *e)
+{
+	/*
+	 * If we have learnt and retained remote routes (VTEPs, MACs)
+	 * for this VNI, install them.
+	 */
+	install_uninstall_routes_for_vni(NULL, NULL, true);
+
+	/*
+	 * If there are VNIs still pending to be processed, schedule them
+	 * after a small sleep so that CPU can be used for other purposes.
+	 */
+	if (zebra_l2_vni_count(&bm->zebra_l2_vni_head))
+		event_add_timer_msec(bm->master, bgp_zebra_process_remote_routes_for_l2vni, NULL,
+				     20, &bm->t_bgp_zebra_l2_vni);
+}
+
+void bgp_zebra_process_remote_routes_for_l3vrf(struct event *e)
+{
+	/*
+	 * Install/Uninstall all remote routes belonging to l3vni
+	 *
+	 * NOTE:
+	 *  - At this point it does not matter whether we call
+	 *    install_routes_for_vrf/uninstall_routes_for_vrf.
+	 *  - Since we pass struct bgp as NULL,
+	 *      * we iterate the bm FIFO list
+	 *      * the second variable (true) is ignored as well and
+	 *        calculated based on the BGP-VRFs flags for ADD/DELETE.
+	 */
+	install_uninstall_routes_for_vrf(NULL, true);
+
+	/*
+	 * If there are L3VNIs still pending to be processed, schedule them
+	 * after a small sleep so that CPU can be used for other purposes.
+	 */
+	if (zebra_l3_vni_count(&bm->zebra_l3_vni_head)) {
+		event_add_timer_msec(bm->master, bgp_zebra_process_remote_routes_for_l3vrf, NULL,
+				     20, &bm->t_bgp_zebra_l3_vni);
+	}
+}
+
 static int bgp_zebra_process_local_es_add(ZAPI_CALLBACK_ARGS)
 {
 	esi_t esi;

bgpd/bgp_zebra.h

@@ -135,4 +135,6 @@ extern void bgp_zebra_release_label_range(uint32_t start, uint32_t end);
 extern enum zclient_send_status
 bgp_zebra_withdraw_actual(struct bgp_dest *dest, struct bgp_path_info *info,
 			  struct bgp *bgp);
+extern void bgp_zebra_process_remote_routes_for_l2vni(struct event *e);
+extern void bgp_zebra_process_remote_routes_for_l3vrf(struct event *e);
 #endif /* _QUAGGA_BGP_ZEBRA_H */

bgpd/bgpd.c

@@ -3966,19 +3966,24 @@ int bgp_delete(struct bgp *bgp)
 	afi_t afi;
 	safi_t safi;
 	int i;
+	uint32_t vni_count;
+	struct bgpevpn *vpn = NULL;
 	struct bgp_dest *dest = NULL;
 	struct bgp_dest *dest_next = NULL;
 	struct bgp_table *dest_table = NULL;
 	struct graceful_restart_info *gr_info;
-	uint32_t cnt_before, cnt_after;
+	uint32_t b_ann_cnt = 0, b_l2_cnt = 0, b_l3_cnt = 0;
+	uint32_t a_ann_cnt = 0, a_l2_cnt = 0, a_l3_cnt = 0;
+	struct bgp *bgp_to_proc = NULL;
+	struct bgp *bgp_to_proc_next = NULL;
 
 	assert(bgp);
 
 	/*
 	 * Iterate the pending dest list and remove all the dest pertaininig to
 	 * the bgp under delete.
 	 */
-	cnt_before = zebra_announce_count(&bm->zebra_announce_head);
+	b_ann_cnt = zebra_announce_count(&bm->zebra_announce_head);
 	for (dest = zebra_announce_first(&bm->zebra_announce_head); dest;
 	     dest = dest_next) {
 		dest_next = zebra_announce_next(&bm->zebra_announce_head, dest);
@@ -3990,10 +3995,36 @@ int bgp_delete(struct bgp *bgp)
 		}
 	}
 
-	cnt_after = zebra_announce_count(&bm->zebra_announce_head);
-	if (BGP_DEBUG(zebra, ZEBRA))
-		zlog_debug("Zebra Announce Fifo cleanup count before %u and after %u during BGP %s deletion",
-			   cnt_before, cnt_after, bgp->name_pretty);
+	/*
+	 * Pop all VPNs yet to be processed for remote routes install if the
+	 * bgp-evpn instance is getting deleted
+	 */
+	if (bgp == bgp_get_evpn()) {
+		b_l2_cnt = zebra_l2_vni_count(&bm->zebra_l2_vni_head);
+		vni_count = b_l2_cnt;
+		while (vni_count) {
+			vpn = zebra_l2_vni_pop(&bm->zebra_l2_vni_head);
+			UNSET_FLAG(vpn->flags, VNI_FLAG_ADD);
+			vni_count--;
+		}
+	}
+
+	b_l3_cnt = zebra_l3_vni_count(&bm->zebra_l3_vni_head);
+	for (bgp_to_proc = zebra_l3_vni_first(&bm->zebra_l3_vni_head); bgp_to_proc;
+	     bgp_to_proc = bgp_to_proc_next) {
+		bgp_to_proc_next = zebra_l3_vni_next(&bm->zebra_l3_vni_head, bgp_to_proc);
+		if (bgp_to_proc == bgp)
+			zebra_l3_vni_del(&bm->zebra_l3_vni_head, bgp_to_proc);
+	}
+
+	if (BGP_DEBUG(zebra, ZEBRA)) {
+		a_ann_cnt = zebra_announce_count(&bm->zebra_announce_head);
+		a_l2_cnt = zebra_l2_vni_count(&bm->zebra_l2_vni_head);
+		a_l3_cnt = zebra_l3_vni_count(&bm->zebra_l3_vni_head);
+		zlog_debug("BGP %s deletion FIFO cnt Zebra_Ann before %u after %u, L2_VNI before %u after, %u L3_VNI before %u after %u",
+			   bgp->name_pretty, b_ann_cnt, a_ann_cnt, b_l2_cnt, a_l2_cnt, b_l3_cnt,
+			   a_l3_cnt);
+	}
 
 	bgp_soft_reconfig_table_task_cancel(bgp, NULL, NULL);
 
@@ -8492,6 +8523,8 @@ void bgp_master_init(struct event_loop *master, const int buffer_size,
 	bm = &bgp_master;
 
 	zebra_announce_init(&bm->zebra_announce_head);
+	zebra_l2_vni_init(&bm->zebra_l2_vni_head);
+	zebra_l3_vni_init(&bm->zebra_l3_vni_head);
 	bm->bgp = list_new();
 	bm->listen_sockets = list_new();
 	bm->port = BGP_PORT_DEFAULT;
@@ -8515,6 +8548,8 @@ void bgp_master_init(struct event_loop *master, const int buffer_size,
 	bm->stalepath_time = BGP_DEFAULT_STALEPATH_TIME;
 	bm->select_defer_time = BGP_DEFAULT_SELECT_DEFERRAL_TIME;
 	bm->rib_stale_time = BGP_DEFAULT_RIB_STALE_TIME;
+	bm->t_bgp_zebra_l2_vni = NULL;
+	bm->t_bgp_zebra_l3_vni = NULL;
 
 	bgp_mac_init();
 	/* init the rd id space.
@@ -8762,6 +8797,8 @@ void bgp_terminate(void)
 	EVENT_OFF(bm->t_bgp_sync_label_manager);
 	EVENT_OFF(bm->t_bgp_start_label_manager);
 	EVENT_OFF(bm->t_bgp_zebra_route);
+	EVENT_OFF(bm->t_bgp_zebra_l2_vni);
+	EVENT_OFF(bm->t_bgp_zebra_l3_vni);
 
 	bgp_mac_finish();
 }

bgpd/bgpd.h

@@ -19,6 +19,8 @@
 #include "asn.h"
 
 PREDECL_LIST(zebra_announce);
+PREDECL_LIST(zebra_l2_vni);
+PREDECL_LIST(zebra_l3_vni);
 
 /* For union sockunion.  */
 #include "queue.h"
@@ -204,6 +206,14 @@ struct bgp_master {
 	/* To preserve ordering of installations into zebra across all Vrfs */
 	struct zebra_announce_head zebra_announce_head;
 
+	struct event *t_bgp_zebra_l2_vni;
+	/* To preserve ordering of processing of L2 VNIs in BGP */
+	struct zebra_l2_vni_head zebra_l2_vni_head;
+
+	struct event *t_bgp_zebra_l3_vni;
+	/* To preserve ordering of processing of BGP-VRFs for L3 VNIs */
+	struct zebra_l3_vni_head zebra_l3_vni_head;
+
 	QOBJ_FIELDS;
 };
 DECLARE_QOBJ_TYPE(bgp_master);
@@ -554,6 +564,8 @@ struct bgp {
 #define BGP_FLAG_INSTANCE_HIDDEN	 (1ULL << 39)
 /* Prohibit BGP from enabling IPv6 RA on interfaces */
 #define BGP_FLAG_IPV6_NO_AUTO_RA (1ULL << 40)
+#define BGP_FLAG_L3VNI_SCHEDULE_FOR_INSTALL (1ULL << 41)
+#define BGP_FLAG_L3VNI_SCHEDULE_FOR_DELETE  (1ULL << 42)
 
 	/* BGP default address-families.
 	 * New peers inherit enabled afi/safis from bgp instance.
@@ -868,10 +880,14 @@ struct bgp {
 	uint64_t node_already_on_queue;
 	uint64_t node_deferred_on_queue;
 
+	struct zebra_l3_vni_item zl3vni;
+
 	QOBJ_FIELDS;
 };
 DECLARE_QOBJ_TYPE(bgp);
 
+DECLARE_LIST(zebra_l3_vni, struct bgp, zl3vni);
+
 struct bgp_interface {
 #define BGP_INTERFACE_MPLS_BGP_FORWARDING (1 << 0)
 /* L3VPN multi domain switching */

tests/topotests/evpn_type5_test_topo1/test_evpn_type5_chaos_topo1.py

@@ -21,6 +21,8 @@
 import time
 import pytest
 import platform
+import functools
+from lib import topotest
 from copy import deepcopy
 
 
@@ -539,6 +541,16 @@ def test_RT_verification_auto_p0(request):
     result = create_vrf_cfg(tgen, topo, input_dict=input_dict_vni)
     assert result is True, "Testcase {} :Failed \n Error: {}".format(tc_name, result)
 
+    expected = {"numL3Vnis": 0}
+    test_func = functools.partial(
+        topotest.router_json_cmp,
+        tgen.gears["e1"],
+        "show bgp l2vpn evpn vni json",
+        expected,
+    )
+    _, result = topotest.run_and_expect(test_func, None, count=5, wait=3)
+    assert result is None, "Testcase {} :Failed \n Error: {}".format(tc_name, result)
+
     input_dict_2 = {}
     for dut in ["e1"]:
         temp = {dut: {"bgp": []}}

tests/topotests/evpn_type5_test_topo1/test_evpn_type5_topo1.py

@@ -25,6 +25,8 @@
 import time
 import pytest
 import platform
+import functools
+from lib import topotest
 from copy import deepcopy
 
 
@@ -1124,7 +1126,6 @@ def test_active_standby_evpn_implementation_p1(request):
     )
 
     for addr_type in ADDR_TYPES:
-
         logger.info("Verifying only ipv4 routes")
         if addr_type != "ipv4":
             continue
@@ -2050,6 +2051,18 @@ def test_bgp_attributes_for_evpn_address_family_p1(request, attribute):
             tc_name, result
         )
 
+        expected = {"numL3Vnis": 0}
+        test_func = functools.partial(
+            topotest.router_json_cmp,
+            tgen.gears["d1"],
+            "show bgp l2vpn evpn vni json",
+            expected,
+        )
+        _, result = topotest.run_and_expect(test_func, None, count=5, wait=3)
+        assert result is None, "Testcase {} :Failed \n Error: {}".format(
+            tc_name, result
+        )
+
         input_dict_2 = {}
         for dut in ["d1"]:
             temp = {dut: {"bgp": []}}