skip snapshots for ublad corpus

CentreForDigitalHumanities · May 24, 2024 · e5d8971 · e5d8971
1 parent fb04081
commit e5d8971
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/backend/corpora/ublad/ublad.py b/backend/corpora/ublad/ublad.py
@@ -72,6 +72,10 @@ def es_settings(self):
 
     def sources(self, start=min_date, end=max_date):
         for directory, _, filenames in os.walk(self.data_directory):
+            _body, tail = os.path.split(directory)
+            if tail.startswith("."):
+                # don't go through directories from snapshots
+                continue
             for filename in filenames:
                 if filename != '.DS_Store':
                     full_path = join(directory, filename)