fix caching

langchain-ai · Oct 23, 2024 · 2ab40c5 · 2ab40c5
1 parent 10a613b
commit 2ab40c5
Show file tree

Hide file tree

Showing 6 changed files with 7 additions and 11 deletions.
diff --git a/docs/cassettes/document_loader_web_4211b1a6-e636-415b-a556-ae01969399a7.msgpack.zlib b/docs/cassettes/document_loader_web_4211b1a6-e636-415b-a556-ae01969399a7.msgpack.zlib
diff --git a/docs/cassettes/document_loader_web_5eeaeb54-ea03-4634-8a79-b60c22ab2b66.msgpack.zlib b/docs/cassettes/document_loader_web_5eeaeb54-ea03-4634-8a79-b60c22ab2b66.msgpack.zlib
diff --git a/docs/cassettes/document_loader_web_793018fd-1365-4a8a-8690-6d51dad2e1cf.msgpack.zlib b/docs/cassettes/document_loader_web_793018fd-1365-4a8a-8690-6d51dad2e1cf.msgpack.zlib
diff --git a/docs/cassettes/document_loader_web_7a6bbfef-ebd5-4357-a7f5-9c989dda092d.msgpack.zlib b/docs/cassettes/document_loader_web_7a6bbfef-ebd5-4357-a7f5-9c989dda092d.msgpack.zlib
diff --git a/docs/cassettes/document_loader_web_7faeccbc-4e56-4b88-99db-2274ed0680c1.msgpack.zlib b/docs/cassettes/document_loader_web_7faeccbc-4e56-4b88-99db-2274ed0680c1.msgpack.zlib
diff --git a/docs/scripts/cache_data.py b/docs/scripts/cache_data.py
@@ -1,16 +1,12 @@
-import nltk
 import tiktoken
+from unstructured.nlp.tokenize import download_nltk_packages
 
 
 def download_tiktoken_data():
     # This will trigger the download and caching of the necessary files
     _ = tiktoken.encoding_for_model("gpt-3.5-turbo")
 
 
-def download_nltk_data():
-    nltk.download("punkt")
-
-
 if __name__ == "__main__":
     download_tiktoken_data()
-    download_nltk_data()
+    download_nltk_packages()