WIP

P# Please enter the commit message for your changes. Lines starting

bgpd/bgp_main.c

@@ -182,6 +182,7 @@ static __attribute__((__noreturn__)) void bgp_exit(int status)
 {
 	struct bgp *bgp, *bgp_default, *bgp_evpn;
 	struct listnode *node, *nnode;
+	struct bgp_dest *dest;
 
 	/* it only makes sense for this to be called on a clean exit */
 	assert(status == 0);
@@ -204,6 +205,13 @@ static __attribute__((__noreturn__)) void bgp_exit(int status)
 	if (bgp_default)
 		bgp_delete(bgp_default);
 
+	while (zebra_announce_count(&bm->zebra_announce_head)) {
+		dest = zebra_announce_pop(&bm->zebra_announce_head);
+		bgp_path_info_unlock(dest->za_pi);
+		bgp_dest_unlock_node(dest);
+	}
+	zebra_announce_fini(&bm->zebra_announce_head);
+
 	bgp_evpn_mh_finish();
 	bgp_nhg_finish();
 

bgpd/bgp_route.c

@@ -3398,8 +3398,8 @@ static void bgp_process_main_one(struct bgp *bgp, struct bgp_dest *dest,
 					|| new_select->sub_type
 						   == BGP_ROUTE_IMPORTED))
 
-					bgp_zebra_announce(dest, old_select,
-							   bgp);
+					bgp_zebra_route_install(dest, old_select,
+								bgp, true);
 			}
 		}
 
@@ -3516,17 +3516,19 @@ static void bgp_process_main_one(struct bgp *bgp, struct bgp_dest *dest,
 			 */
 			if (old_select &&
 			    is_route_parent_evpn(old_select))
-				bgp_zebra_withdraw(dest, old_select, bgp);
+				bgp_zebra_route_install(dest, old_select, bgp,
+							false);
 
-			bgp_zebra_announce(dest, new_select, bgp);
+			bgp_zebra_route_install(dest, new_select, bgp, true);
 		} else {
 			/* Withdraw the route from the kernel. */
 			if (old_select && old_select->type == ZEBRA_ROUTE_BGP
 			    && (old_select->sub_type == BGP_ROUTE_NORMAL
 				|| old_select->sub_type == BGP_ROUTE_AGGREGATE
 				|| old_select->sub_type == BGP_ROUTE_IMPORTED))
 
-				bgp_zebra_withdraw(dest, old_select, bgp);
+				bgp_zebra_route_install(dest, old_select, bgp,
+							false);
 		}
 	}
 
@@ -4427,7 +4429,7 @@ void bgp_update(struct peer *peer, const struct prefix *p, uint32_t addpath_id,
 	if (pi && pi->attr->rmap_table_id != new_attr.rmap_table_id) {
 		if (CHECK_FLAG(pi->flags, BGP_PATH_SELECTED))
 			/* remove from RIB previous entry */
-			bgp_zebra_withdraw(dest, pi, bgp);
+			bgp_zebra_route_install(dest, pi, bgp, false);
 	}
 
 	if (peer->sort == BGP_PEER_EBGP) {
@@ -6053,7 +6055,8 @@ static void bgp_cleanup_table(struct bgp *bgp, struct bgp_table *table,
 				|| pi->sub_type == BGP_ROUTE_IMPORTED)) {
 
 				if (bgp_fibupd_safi(safi))
-					bgp_zebra_withdraw(dest, pi, bgp);
+					bgp_zebra_route_install(dest, pi, bgp,
+								false);
 			}
 
 			dest = bgp_path_info_reap(dest, pi);

bgpd/bgp_table.h

@@ -77,6 +77,7 @@ struct bgp_dest {
 	STAILQ_ENTRY(bgp_dest) pq;
 
 	struct zebra_announce_item zat;
+	struct bgp_path_info *za_pi; // HAHAHAHAA
 
 	uint64_t version;
 
@@ -93,6 +94,8 @@ struct bgp_dest {
 #define BGP_NODE_LABEL_REQUESTED        (1 << 7)
 #define BGP_NODE_SOFT_RECONFIG (1 << 8)
 #define BGP_NODE_PROCESS_CLEAR (1 << 9)
+#define BGP_NODE_SCHEDULE_FOR_INSTALL	(1 << 10)
+#define BGP_NODE_SCHEDULE_FOR_DELETE	(1 << 11)
 
 	struct bgp_addpath_node_data tx_addpath;
 

bgpd/bgp_zebra.c

@@ -1507,8 +1507,9 @@ static void bgp_debug_zebra_nh(struct zapi_route *api)
 	}
 }
 
-void bgp_zebra_announce(struct bgp_dest *dest, struct bgp_path_info *info,
-			struct bgp *bgp)
+static enum zclient_send_status
+bgp_zebra_announce_actual(struct bgp_dest *dest, struct bgp_path_info *info,
+			  struct bgp *bgp)
 {
 	struct zapi_route api = { 0 };
 	unsigned int valid_nh_count = 0;
@@ -1525,27 +1526,11 @@ void bgp_zebra_announce(struct bgp_dest *dest, struct bgp_path_info *info,
 
 	bti = bgp_dest_table(dest);
 
-	/*
-	 * BGP is installing this route and bgp has been configured
-	 * to suppress announcements until the route has been installed
-	 * let's set the fact that we expect this route to be installed
-	 */
-	if (BGP_SUPPRESS_FIB_ENABLED(bgp))
-		SET_FLAG(dest->flags, BGP_NODE_FIB_INSTALL_PENDING);
-
-	/* Don't try to install if we're not connected to Zebra or Zebra doesn't
-	 * know of this instance.
-	 */
-	if (!bgp_install_info_to_zebra(bgp))
-		return;
-
-	if (bgp->main_zebra_update_hold)
-		return;
-
 	if (bti->safi == SAFI_FLOWSPEC) {
 		bgp_pbr_update_entry(bgp, bgp_dest_get_prefix(dest), info,
 				     bti->afi, bti->safi, true);
-		return;
+
+		return ZCLIENT_SEND_SUCCESS;
 	}
 
 	/* Make Zebra API structure. */
@@ -1668,10 +1653,11 @@ void bgp_zebra_announce(struct bgp_dest *dest, struct bgp_path_info *info,
 		zlog_debug("%s: %pFX: announcing to zebra (recursion %sset)",
 			   __func__, p, (recursion_flag ? "" : "NOT "));
 	}
-	zclient_route_send(is_add ? ZEBRA_ROUTE_ADD : ZEBRA_ROUTE_DELETE,
-			   zclient, &api);
+	return zclient_route_send(is_add ? ZEBRA_ROUTE_ADD : ZEBRA_ROUTE_DELETE,
+				  zclient, &api);
 }
 
+
 /* Announce all routes of a table to zebra */
 void bgp_zebra_announce_table(struct bgp *bgp, afi_t afi, safi_t safi)
 {
@@ -1697,7 +1683,7 @@ void bgp_zebra_announce_table(struct bgp *bgp, afi_t afi, safi_t safi)
 			     && (pi->sub_type == BGP_ROUTE_NORMAL
 				 || pi->sub_type == BGP_ROUTE_IMPORTED)))
 
-				bgp_zebra_announce(dest, pi, bgp);
+				bgp_zebra_route_install(dest, pi, bgp, true);
 }
 
 /* Announce routes of any bgp subtype of a table to zebra */
@@ -1719,34 +1705,24 @@ void bgp_zebra_announce_table_all_subtypes(struct bgp *bgp, afi_t afi,
 		for (pi = bgp_dest_get_bgp_path_info(dest); pi; pi = pi->next)
 			if (CHECK_FLAG(pi->flags, BGP_PATH_SELECTED) &&
 			    pi->type == ZEBRA_ROUTE_BGP)
-				bgp_zebra_announce(dest, pi, bgp);
+				bgp_zebra_route_install(dest, pi, bgp, true);
 }
 
-void bgp_zebra_withdraw(struct bgp_dest *dest, struct bgp_path_info *info,
-			struct bgp *bgp)
+static enum zclient_send_status
+bgp_zebra_withdraw_actual(struct bgp_dest *dest, struct bgp_path_info *info,
+			  struct bgp *bgp)
 {
 	struct zapi_route api;
 	struct peer *peer;
 	struct bgp_table *bti = bgp_dest_table(dest);
 	const struct prefix *p = bgp_dest_get_prefix(dest);
 
-	/*
-	 * If we are withdrawing the route, we don't need to have this
-	 * flag set.  So unset it.
-	 */
-	UNSET_FLAG(dest->flags, BGP_NODE_FIB_INSTALL_PENDING);
-
-	/* Don't try to install if we're not connected to Zebra or Zebra doesn't
-	 * know of this instance.
-	 */
-	if (!bgp_install_info_to_zebra(bgp))
-		return;
-
 	if (bti->safi == SAFI_FLOWSPEC) {
 		peer = info->peer;
 		bgp_pbr_update_entry(peer->bgp, p, info, bti->afi, bti->safi,
 				     false);
-		return;
+
+		return ZCLIENT_SEND_SUCCESS;
 	}
 
 	memset(&api, 0, sizeof(api));
@@ -1764,7 +1740,152 @@ void bgp_zebra_withdraw(struct bgp_dest *dest, struct bgp_path_info *info,
 		zlog_debug("Tx route delete VRF %u %pFX", bgp->vrf_id,
 			   &api.prefix);
 
-	zclient_route_send(ZEBRA_ROUTE_DELETE, zclient, &api);
+	return zclient_route_send(ZEBRA_ROUTE_DELETE, zclient, &api);
+}
+
+#define ZEBRA_LOOP 1000
+static void bgp_zebra_route_senddown(struct event *e)
+{
+	struct bgp_dest *dest;
+	uint32_t count = 0;
+	enum zclient_send_status status = ZCLIENT_SEND_SUCCESS;
+	struct bgp_table *bti;
+
+	while (count < ZEBRA_LOOP) {
+		dest = zebra_announce_pop(&bm->zebra_announce_head);
+
+		if (!dest)
+			break;
+
+		bti = bgp_dest_table(dest);
+
+		if (CHECK_FLAG(dest->flags, BGP_NODE_SCHEDULE_FOR_INSTALL)) {
+			status = bgp_zebra_announce_actual(dest, dest->za_pi,
+							   bti->bgp);
+			UNSET_FLAG(dest->flags, BGP_NODE_SCHEDULE_FOR_INSTALL);
+		} else if (CHECK_FLAG(dest->flags,
+				      BGP_NODE_SCHEDULE_FOR_DELETE)) {
+			status = bgp_zebra_withdraw_actual(dest, dest->za_pi,
+							   bti->bgp);
+			UNSET_FLAG(dest->flags, BGP_NODE_SCHEDULE_FOR_DELETE);
+		}
+
+		bgp_path_info_unlock(dest->za_pi);
+		dest->za_pi = NULL;
+		bgp_dest_unlock_node(dest);
+
+		if (status == ZCLIENT_SEND_BUFFERED)
+			break;
+
+		count++;
+	}
+
+	if (status != ZCLIENT_SEND_BUFFERED &&
+	    zebra_announce_count(&bm->zebra_announce_head))
+		event_add_event(bm->master, bgp_zebra_route_senddown, NULL, 0,
+				&bm->e_bgp_zebra_route);
+}
+
+static void bgp_zebra_buffer_write_ready(void)
+{
+	bgp_zebra_route_senddown(NULL);
+}
+
+/*
+ * BGP is now keeping a list of dests with the dest having a pointer
+ * to the bgp_path_info that it will be working on.
+ * Here is the sequence of events that should happen:
+ *
+ *  Current State      New State       Action
+ *  -------------      ---------       ------
+ *      ----           Install         Place dest on list, save pi, mark
+ *                                     as going to be installed
+ *      ----           Withdrawal      Place dest on list, save pi, mark
+ *                                     as going to be deleted
+ *
+ *    Install          Install         Leave dest on list, release old pi,
+ *                                     save new pi, mark as going to be
+ *                                     Installed
+ *    Install          Withdrawal      Leave dest on list, release old pi,
+ *                                     save new pi, mark as going to be
+ *                                     withdrawan, remove install flag
+ *
+ *    Withdrawal       Install         Special case, send withdrawal immediately
+ *                                     Leave dest on list, release old pi,
+ *                                     save new pi, mark as going to be
+ *                                     installed.  <see note about evpn
+ *                                     in bgp_route.c in bgp_process_main_one>
+ *    Withdrawal       Withdrawal      Leave dest on list, release old pi,
+ *                                     save new pi, mark as going to be
+ *                                     withdrawn.
+ */
+void bgp_zebra_route_install(struct bgp_dest *dest, struct bgp_path_info *info,
+			     struct bgp *bgp, bool install)
+{
+	/*
+	 * BGP is installing this route and bgp has been configured
+	 * to suppress announcements until the route has been installed
+	 * let's set the fact that we expect this route to be installed
+	 */
+	if (install) {
+		if (BGP_SUPPRESS_FIB_ENABLED(bgp))
+			SET_FLAG(dest->flags, BGP_NODE_FIB_INSTALL_PENDING);
+
+		/* Don't try to install if we're not connected to Zebra or Zebra doesn't
+		 * know of this instance.
+		 */
+		if (!bgp_install_info_to_zebra(bgp))
+			return;
+
+		if (bgp->main_zebra_update_hold)
+			return;
+	} else {
+		UNSET_FLAG(dest->flags, BGP_NODE_FIB_INSTALL_PENDING);
+
+		/* Don't try to install if we're not connected to Zebra or Zebra doesn't
+		 * know of this instance.
+		 */
+		if (!bgp_install_info_to_zebra(bgp))
+			return;
+	}
+
+	if (!CHECK_FLAG(dest->flags, BGP_NODE_SCHEDULE_FOR_INSTALL) &&
+	    !CHECK_FLAG(dest->flags, BGP_NODE_SCHEDULE_FOR_DELETE)) {
+		zebra_announce_add_tail(&bm->zebra_announce_head, dest);
+
+		/*
+		 * This is a bug.  If neither flag is set and
+		 * za_pi is not set then there is a problem
+		 */
+		assert(!dest->za_pi);
+		dest->za_pi = info;
+		bgp_path_info_lock(info);
+		bgp_dest_lock_node(dest);
+	} else if (CHECK_FLAG(dest->flags, BGP_NODE_SCHEDULE_FOR_INSTALL)) {
+		assert(dest->za_pi);
+		bgp_path_info_lock(info);
+		bgp_path_info_unlock(dest->za_pi);
+		dest->za_pi = info;
+	} else if (CHECK_FLAG(dest->flags, BGP_NODE_SCHEDULE_FOR_DELETE)) {
+		assert(dest->za_pi);
+		if (install)
+			bgp_zebra_withdraw_actual(dest, dest->za_pi, bgp);
+
+		bgp_path_info_lock(info);
+		bgp_path_info_unlock(dest->za_pi);
+		dest->za_pi = info;
+	}
+
+	if (install) {
+		UNSET_FLAG(dest->flags, BGP_NODE_SCHEDULE_FOR_DELETE);
+		SET_FLAG(dest->flags, BGP_NODE_SCHEDULE_FOR_INSTALL);
+	} else {
+		UNSET_FLAG(dest->flags, BGP_NODE_SCHEDULE_FOR_INSTALL);
+		SET_FLAG(dest->flags, BGP_NODE_SCHEDULE_FOR_DELETE);
+	}
+
+	event_add_event(bm->master, bgp_zebra_route_senddown, NULL, 0,
+			&bm->e_bgp_zebra_route);
 }
 
 /* Withdraw all entries in a BGP instances RIB table from Zebra */
@@ -1785,7 +1906,7 @@ void bgp_zebra_withdraw_table_all_subtypes(struct bgp *bgp, afi_t afi, safi_t sa
 		for (pi = bgp_dest_get_bgp_path_info(dest); pi; pi = pi->next) {
 			if (CHECK_FLAG(pi->flags, BGP_PATH_SELECTED)
 			    && (pi->type == ZEBRA_ROUTE_BGP))
-				bgp_zebra_withdraw(dest, pi, bgp);
+				bgp_zebra_route_install(dest, pi, bgp, false);
 		}
 	}
 }
@@ -3451,6 +3572,7 @@ void bgp_zebra_init(struct event_loop *master, unsigned short instance)
 	zclient = zclient_new(master, &zclient_options_default, bgp_handlers,
 			      array_size(bgp_handlers));
 	zclient_init(zclient, ZEBRA_ROUTE_BGP, 0, &bgpd_privs);
+	zclient->zebra_buffer_write_ready = bgp_zebra_buffer_write_ready;
 	zclient->zebra_connected = bgp_zebra_connected;
 	zclient->zebra_capabilities = bgp_zebra_capabilities;
 	zclient->nexthop_update = bgp_nexthop_update;

bgpd/bgp_zebra.h

@@ -28,11 +28,10 @@ extern void bgp_zebra_destroy(void);
 extern int bgp_zebra_get_table_range(struct zclient *zc, uint32_t chunk_size,
 				     uint32_t *start, uint32_t *end);
 extern int bgp_if_update_all(void);
-extern void bgp_zebra_announce(struct bgp_dest *dest,
-			       struct bgp_path_info *path, struct bgp *bgp);
+extern void bgp_zebra_route_install(struct bgp_dest *dest,
+				    struct bgp_path_info *path, struct bgp *bgp,
+				    bool install);
 extern void bgp_zebra_announce_table(struct bgp *bgp, afi_t afi, safi_t safi);
-extern void bgp_zebra_withdraw(struct bgp_dest *dest,
-			       struct bgp_path_info *path, struct bgp *bgp);
 
 /* Announce routes of any bgp subtype of a table to zebra */
 extern void bgp_zebra_announce_table_all_subtypes(struct bgp *bgp, afi_t afi,