Use nltk.download('punkt_tab'), pin nltk>=3.9

deepset-ai · Aug 20, 2024 · 8ba72dc · 8ba72dc
1 parent 9b7f9fb
commit 8ba72dc
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/haystack/nodes/preprocessor/preprocessor.py b/haystack/nodes/preprocessor/preprocessor.py
@@ -120,7 +120,7 @@ def __init__(
             nltk.data.find("tokenizers/punkt")
         except LookupError:
             try:
-                nltk.download("punkt")
+                nltk.download("punkt_tab")
             except FileExistsError as error:
                 logger.debug("NLTK punkt tokenizer seems to be already downloaded. Error message: %s", error)
                 pass

diff --git a/pyproject.toml b/pyproject.toml
@@ -147,7 +147,7 @@ crawler = [
   "selenium>=4.11.0"
 ]
 preprocessing = [
-  "nltk",
+  "nltk>=3.9",
   "langdetect",  # for language classification
 ]
 file-conversion = [