diff --git a/shared/src/main/java/me/xginko/aef/utils/CachingPermTool.java b/shared/src/main/java/me/xginko/aef/utils/CachingPermTool.java
index bbd63bdf..75479cb5 100644
--- a/shared/src/main/java/me/xginko/aef/utils/CachingPermTool.java
+++ b/shared/src/main/java/me/xginko/aef/utils/CachingPermTool.java
@@ -54,15 +54,21 @@ public static boolean hasPermission(AEFPermission permission, HumanEntity human)
 
     @EventHandler(priority = EventPriority.MONITOR, ignoreCancelled = true)
     private void onLeave(PlayerQuitEvent event) {
-        Cache<AEFPermission, Boolean> cache = permissionCacheMap.remove(event.getPlayer().getUniqueId());
-        cache.invalidateAll();
-        cache.cleanUp();
+        if (permissionCacheMap.containsKey(event.getPlayer().getUniqueId())) {
+            Cache<AEFPermission, Boolean> cache = permissionCacheMap.get(event.getPlayer().getUniqueId());
+            cache.invalidateAll();
+            cache.cleanUp();
+            permissionCacheMap.remove(event.getPlayer().getUniqueId());
+        }
     }
 
     @EventHandler(priority = EventPriority.MONITOR, ignoreCancelled = true)
     private void onKick(PlayerKickEvent event) {
-        Cache<AEFPermission, Boolean> cache = permissionCacheMap.remove(event.getPlayer().getUniqueId());
-        cache.invalidateAll();
-        cache.cleanUp();
+        if (permissionCacheMap.containsKey(event.getPlayer().getUniqueId())) {
+            Cache<AEFPermission, Boolean> cache = permissionCacheMap.get(event.getPlayer().getUniqueId());
+            cache.invalidateAll();
+            cache.cleanUp();
+            permissionCacheMap.remove(event.getPlayer().getUniqueId());
+        }
     }
 }