diff --git a/tosfs/core.py b/tosfs/core.py
index 9bf5eb2..1f88370 100644
--- a/tosfs/core.py
+++ b/tosfs/core.py
@@ -239,7 +239,7 @@ def rmdir(self, path: str) -> None:
         if not self.isdir(path):
             raise NotADirectoryError(f"{path} is not a directory.")
 
-        if len(self.ls(path, refresh=True, detail=False)) > 0:
+        if len(self._listdir(bucket, max_items=1, prefix=key.rstrip("/") + "/")) > 0:
             raise TosfsError(f"Directory {path} is not empty.")
 
         try: