From ab2eed41cb6d9480f6d6127e5093e4ba53fe50bd Mon Sep 17 00:00:00 2001 From: Shaun Crampton Date: Mon, 25 Nov 2024 16:02:15 +0000 Subject: [PATCH] Free dedupe buffer's tracking map after shrink Go doesn't free map blocks, even after a map shrinks considerably. The dedupe buffer tends to store a lot of keys for a start-of-day snapshot, make sure we clean up the leaked map capacity once we're back down to zero. Upstream issue: https://github.com/golang/go/issues/20135 --- .../backend/syncersv1/dedupebuffer/dedupe_buffer.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/libcalico-go/lib/backend/syncersv1/dedupebuffer/dedupe_buffer.go b/libcalico-go/lib/backend/syncersv1/dedupebuffer/dedupe_buffer.go index 2da6c02a621..38b6a9cd408 100644 --- a/libcalico-go/lib/backend/syncersv1/dedupebuffer/dedupe_buffer.go +++ b/libcalico-go/lib/backend/syncersv1/dedupebuffer/dedupe_buffer.go @@ -45,7 +45,9 @@ type DedupeBuffer struct { // keyToPendingUpdate holds an entry for each updateWithStringKey in the // pendingUpdates queue - keyToPendingUpdate map[string]*list.Element + keyToPendingUpdate map[string]*list.Element + peakPendingUpdatesLen int + // liveResourceKeys Contains an entry for every key that we have sent to // the consumer and that we have not subsequently sent a deletion for. liveResourceKeys set.Set[string] @@ -176,6 +178,7 @@ func (d *DedupeBuffer) OnUpdatesKeysKnown(updates []api.Update, keys []string) { update: u, }) d.keyToPendingUpdate[key] = element + d.peakPendingUpdatesLen = max(len(d.keyToPendingUpdate), d.peakPendingUpdatesLen) } } queueNowEmpty := d.pendingUpdates.Len() == 0 @@ -252,6 +255,14 @@ func (d *DedupeBuffer) pullNextBatch(buf []any, batchSize int) []any { if u, ok := first.Value.(updateWithStringKey); ok { key := u.key delete(d.keyToPendingUpdate, key) + if len(d.keyToPendingUpdate) == 0 && d.peakPendingUpdatesLen > 100 { + // Map blocks never get freed when a map is scaled down. + // https://github.com/golang/go/issues/20135 + // Opportunistically free the map when it's empty. This can + // free a good amount of RAM after loading a large snapshot. + d.keyToPendingUpdate = map[string]*list.Element{} + d.peakPendingUpdatesLen = 0 + } // Update liveResourceKeys now, before we drop the lock. Once we drop // the lock we're committed to sending these keys. if u.update.Value == nil {