Skip nested directories when using S3DirectoryLoader

langchain-ai · Feb 20, 2024 · 342e3a4 · 342e3a4
1 parent 223e5ef
commit 342e3a4
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/libs/community/langchain_community/document_loaders/s3_directory.py b/libs/community/langchain_community/document_loaders/s3_directory.py
@@ -120,6 +120,9 @@ def load(self) -> List[Document]:
         bucket = s3.Bucket(self.bucket)
         docs = []
         for obj in bucket.objects.filter(Prefix=self.prefix):
+            # Skip directories
+            if obj.size == 0 and obj.key.endswith('/'):
+                continue
             loader = S3FileLoader(
                 self.bucket,
                 obj.key,