From 69f07fab28b32846a95571eb7404ef870cc3784c Mon Sep 17 00:00:00 2001
From: Alexander Skorichenko <askorichenko@netgate.com>
Date: Wed, 28 Feb 2024 20:34:06 +0100
Subject: [PATCH] zebra: fix route deletion during zebra shutdown

Split zebra's vrf_terminate() into disable() and delete() stages.
The former enqueues all events for the dplane thread.
Memory freeing is performed in the second stage.

Signed-off-by: Alexander Skorichenko <askorichenko@netgate.com>
(cherry picked from commit 444ce317b2af491b5cdc321286772627a5d4c8ea)
---
 lib/vrf.c    | 49 ++++++++++++++++++++++++++++---------------------
 lib/vrf.h    |  6 ++++++
 zebra/main.c |  6 ++++++
 3 files changed, 40 insertions(+), 21 deletions(-)

diff --git a/lib/vrf.c b/lib/vrf.c
index b279e8b7bf48..01bcd85d4661 100644
--- a/lib/vrf.c
+++ b/lib/vrf.c
@@ -325,6 +325,33 @@ void vrf_disable(struct vrf *vrf)
 		(*vrf_master.vrf_disable_hook)(vrf);
 }
 
+void vrf_iterate(vrf_iter_func fnc)
+{
+	struct vrf *vrf, *tmp;
+
+	if (debug_vrf)
+		zlog_debug("%s:  vrf subsystem iteration", __func__);
+
+	RB_FOREACH_SAFE (vrf, vrf_id_head, &vrfs_by_id, tmp) {
+		if (vrf->vrf_id == VRF_DEFAULT)
+			continue;
+
+		fnc(vrf);
+	}
+
+	RB_FOREACH_SAFE (vrf, vrf_name_head, &vrfs_by_name, tmp) {
+		if (vrf->vrf_id == VRF_DEFAULT)
+			continue;
+
+		fnc(vrf);
+	}
+
+	/* Finally process default VRF */
+	vrf = vrf_lookup_by_id(VRF_DEFAULT);
+	if (vrf)
+		fnc(vrf);
+}
+
 const char *vrf_id_to_name(vrf_id_t vrf_id)
 {
 	struct vrf *vrf;
@@ -541,32 +568,12 @@ static void vrf_terminate_single(struct vrf *vrf)
 	vrf_delete(vrf);
 }
 
-/* Terminate VRF module. */
 void vrf_terminate(void)
 {
-	struct vrf *vrf, *tmp;
-
 	if (debug_vrf)
 		zlog_debug("%s: Shutting down vrf subsystem", __func__);
 
-	RB_FOREACH_SAFE (vrf, vrf_id_head, &vrfs_by_id, tmp) {
-		if (vrf->vrf_id == VRF_DEFAULT)
-			continue;
-
-		vrf_terminate_single(vrf);
-	}
-
-	RB_FOREACH_SAFE (vrf, vrf_name_head, &vrfs_by_name, tmp) {
-		if (vrf->vrf_id == VRF_DEFAULT)
-			continue;
-
-		vrf_terminate_single(vrf);
-	}
-
-	/* Finally terminate default VRF */
-	vrf = vrf_lookup_by_id(VRF_DEFAULT);
-	if (vrf)
-		vrf_terminate_single(vrf);
+	vrf_iterate(vrf_terminate_single);
 }
 
 int vrf_socket(int domain, int type, int protocol, vrf_id_t vrf_id,
diff --git a/lib/vrf.h b/lib/vrf.h
index f66a9e6b325e..4be57d58114e 100644
--- a/lib/vrf.h
+++ b/lib/vrf.h
@@ -201,6 +201,12 @@ extern void vrf_init(int (*create)(struct vrf *vrf),
 		     int (*disable)(struct vrf *vrf),
 		     int (*destroy)(struct vrf *vrf));
 
+/*
+ * Iterate over custom VRFs and round up by processing the default VRF.
+ */
+typedef void (*vrf_iter_func)(struct vrf *vrf);
+extern void vrf_iterate(vrf_iter_func fnc);
+
 /*
  * Call vrf_terminate when the protocol is being shutdown
  */
diff --git a/zebra/main.c b/zebra/main.c
index 1e833ce7f182..b66011882497 100644
--- a/zebra/main.c
+++ b/zebra/main.c
@@ -195,6 +195,12 @@ static void sigint(void)
 
 	list_delete(&zrouter.client_list);
 
+	/*
+	 * Besides other clean-ups zebra's vrf_disable() also enqueues installed
+	 * routes for removal from the kernel, unless ZEBRA_VRF_RETAIN is set.
+	 */
+	vrf_iterate(vrf_disable);
+
 	/* Indicate that all new dplane work has been enqueued. When that
 	 * work is complete, the dataplane will enqueue an event
 	 * with the 'finalize' function.