diff --git a/a_sync/executor.py b/a_sync/executor.py
index cb731961..62f2037d 100644
--- a/a_sync/executor.py
+++ b/a_sync/executor.py
@@ -324,7 +324,9 @@ def _worker(
 
                 with executor._adjusting_lock:  # NOTE: NEW
                     # NOTE: We keep a minimum of one thread active to prevent locks
-                    if len(executor) > 1:  # NOTE: NEW
+                    if len(executor) > 1 or executor._idle_semaphore.acquire(
+                        blocking=False
+                    ):  # NOTE: NEW
                         t = threading.current_thread()  # NOTE: NEW
                         executor._threads.remove(t)  # NOTE: NEW
                         thread._threads_queues.pop(t)  # NOTE: NEW