diff --git a/controller/controller.go b/controller/controller.go
index bf966e0..13f112b 100644
--- a/controller/controller.go
+++ b/controller/controller.go
@@ -301,6 +301,9 @@ func (c *Controller) subscribe(ctx context.Context) {
 					OldObject: oldObj,
 				}
 				_, exists := newObjs[uid]
+				if !exists {
+					delete(oldObjs, uid)
+				}
 				return event, !exists
 			})