Merge branch 'master' into erick/google-vertexai-patch-serializable-c…

…itation-metadata
langchain-ai · Feb 7, 2024 · 3b1e3b8 · 3b1e3b8
2 parents 96aeb2b + 6f1403b
commit 3b1e3b8
Show file tree

Hide file tree

Showing 41 changed files with 3,273 additions and 897 deletions.
diff --git a/.github/workflows/_release.yml b/.github/workflows/_release.yml
@@ -182,12 +182,18 @@ jobs:
         working-directory: ${{ inputs.working-directory }}
 
       - name: Get minimum versions
-        id: check-version
-        run: echo "min-versions=$(python $GITHUB_WORKSPACE/.github/scripts/get_min_versions.py pyproject.toml)" >> $GITHUB_OUTPUT
+        working-directory: ${{ inputs.working-directory }}
+        id: min-version
+        run: |
+          poetry run pip install packaging
+          min_versions="$(poetry run python $GITHUB_WORKSPACE/.github/scripts/get_min_versions.py pyproject.toml)"
+          echo "min-versions=$min_versions" >> "$GITHUB_OUTPUT"
+          echo "min-versions=$min_versions"
 
       - name: Run unit tests with minimum dependency versions
+        if: ${{ steps.min-version.outputs.min-versions != '' }}
         env:
-          MIN_VERSIONS: ${{ steps.check-version.outputs.min-versions }}
+          MIN_VERSIONS: ${{ steps.min-version.outputs.min-versions }}
         run: |
           poetry run pip install $MIN_VERSIONS
           make tests

diff --git a/docs/docs/additional_resources/tutorials.mdx b/docs/docs/additional_resources/tutorials.mdx
@@ -2,33 +2,37 @@
 
 Below are links to tutorials and courses on LangChain. For written guides on common use cases for LangChain, check out the [use cases guides](/docs/use_cases).
 
-⛓ icon marks a new addition [last update 2023-09-21]
+⛓ icon marks a new addition [last update 2024-02-06]
 
 ---------------------
 
 ### [LangChain](https://en.wikipedia.org/wiki/LangChain) on Wikipedia
 
 ### Books
 
-#### ⛓[Generative AI with LangChain](https://www.amazon.com/Generative-AI-LangChain-language-ChatGPT/dp/1835083463/ref=sr_1_1?crid=1GMOMH0G7GLR&keywords=generative+ai+with+langchain&qid=1703247181&sprefix=%2Caps%2C298&sr=8-1) by [Ben Auffrath](https://www.amazon.com/stores/Ben-Auffarth/author/B08JQKSZ7D?ref=ap_rdr&store_ref=ap_rdr&isDramIntegrated=true&shoppingPortalEnabled=true), ©️ 2023 Packt Publishing
+#### [Generative AI with LangChain](https://www.amazon.com/Generative-AI-LangChain-language-ChatGPT/dp/1835083463/ref=sr_1_1?crid=1GMOMH0G7GLR&keywords=generative+ai+with+langchain&qid=1703247181&sprefix=%2Caps%2C298&sr=8-1) by [Ben Auffrath](https://www.amazon.com/stores/Ben-Auffarth/author/B08JQKSZ7D?ref=ap_rdr&store_ref=ap_rdr&isDramIntegrated=true&shoppingPortalEnabled=true), ©️ 2023 Packt Publishing
 
 
 ### DeepLearning.AI courses
  by [Harrison Chase](https://en.wikipedia.org/wiki/LangChain) and [Andrew Ng](https://en.wikipedia.org/wiki/Andrew_Ng)
 - [LangChain for LLM Application Development](https://learn.deeplearning.ai/langchain)
 - [LangChain Chat with Your Data](https://learn.deeplearning.ai/langchain-chat-with-your-data)
-- ⛓ [Functions, Tools and Agents with LangChain](https://learn.deeplearning.ai/functions-tools-agents-langchain)
+- [Functions, Tools and Agents with LangChain](https://learn.deeplearning.ai/functions-tools-agents-langchain)
 
 ### Handbook
 [LangChain AI Handbook](https://www.pinecone.io/learn/langchain/) By **James Briggs** and **Francisco Ingham**
 
+⛓ [LangChain Cheatsheet](https://pub.towardsai.net/langchain-cheatsheet-all-secrets-on-a-single-page-8be26b721cde) by **Ivan Reznikov**
+
 ### Short Tutorials
 [LangChain Explained in 13 Minutes | QuickStart Tutorial for Beginners](https://youtu.be/aywZrzNaKjs) by [Rabbitmetrics](https://www.youtube.com/@rabbitmetrics)
 
 [LangChain Crash Course: Build an AutoGPT app in 25 minutes](https://youtu.be/MlK6SIjcjE8) by [Nicholas Renotte](https://www.youtube.com/@NicholasRenotte)
 
 [LangChain Crash Course - Build apps with language models](https://youtu.be/LbT1yp6quS8) by [Patrick Loeber](https://www.youtube.com/@patloeber)
 
+⛓ [LangChain 101 Course](https://medium.com/@ivanreznikov/langchain-101-course-updated-668f7b41d6cb) by **Ivan Reznikov**
+
 ##  Tutorials
 
 ### [LangChain for Gen AI and LLMs](https://www.youtube.com/playlist?list=PLIUOU7oqGTLieV9uTIFMm6_4PXg-hlN6F) by [James Briggs](https://www.youtube.com/@jamesbriggs)
@@ -44,8 +48,8 @@ Below are links to tutorials and courses on LangChain. For written guides on com
 - #9 [Build Conversational Agents with Vector DBs](https://youtu.be/H6bCqqw9xyI)
 - [Using NEW `MPT-7B` in Hugging Face and LangChain](https://youtu.be/DXpk9K7DgMo)
 - [`MPT-30B` Chatbot with LangChain](https://youtu.be/pnem-EhT6VI)
-- ⛓ [Fine-tuning OpenAI's `GPT 3.5` for LangChain Agents](https://youtu.be/boHXgQ5eQic?si=OOOfK-GhsgZGBqSr)
-- ⛓ [Chatbots with `RAG`: LangChain Full Walkthrough](https://youtu.be/LhnCsygAvzY?si=N7k6xy4RQksbWwsQ)
+- [Fine-tuning OpenAI's `GPT 3.5` for LangChain Agents](https://youtu.be/boHXgQ5eQic?si=OOOfK-GhsgZGBqSr)
+- [Chatbots with `RAG`: LangChain Full Walkthrough](https://youtu.be/LhnCsygAvzY?si=N7k6xy4RQksbWwsQ)
 
 
 ### [LangChain 101](https://www.youtube.com/playlist?list=PLqZXAkvF1bPNQER9mLmDbntNfSpzdDIU5) by [Greg Kamradt (Data Indy)](https://www.youtube.com/@DataIndependent)
@@ -109,16 +113,16 @@ Below are links to tutorials and courses on LangChain. For written guides on com
 - [What can you do with 16K tokens in LangChain?](https://youtu.be/z2aCZBAtWXs)
 - [Tagging and Extraction - Classification using `OpenAI Functions`](https://youtu.be/a8hMgIcUEnE)
 - [HOW to Make Conversational Form with LangChain](https://youtu.be/IT93On2LB5k)
-- ⛓ [`Claude-2` meets LangChain!](https://youtu.be/Hb_D3p0bK2U?si=j96Kc7oJoeRI5-iC)
-- ⛓ [`PaLM 2` Meets LangChain](https://youtu.be/orPwLibLqm4?si=KgJjpEbAD9YBPqT4)
-- ⛓ [`LLaMA2` with LangChain - Basics | LangChain TUTORIAL](https://youtu.be/cIRzwSXB4Rc?si=v3Hwxk1m3fksBIHN)
-- ⛓ [Serving `LLaMA2` with `Replicate`](https://youtu.be/JIF4nNi26DE?si=dSazFyC4UQmaR-rJ)
-- ⛓ [NEW LangChain Expression Language](https://youtu.be/ud7HJ2p3gp0?si=8pJ9O6hGbXrCX5G9)
-- ⛓ [Building a RCI Chain for Agents with LangChain Expression Language](https://youtu.be/QaKM5s0TnsY?si=0miEj-o17AHcGfLG)
-- ⛓ [How to Run `LLaMA-2-70B` on the `Together AI`](https://youtu.be/Tc2DHfzHeYE?si=Xku3S9dlBxWQukpe)
-- ⛓ [`RetrievalQA` with `LLaMA 2 70b` & `Chroma` DB](https://youtu.be/93yueQQnqpM?si=ZMwj-eS_CGLnNMXZ)
-- ⛓ [How to use `BGE Embeddings` for LangChain](https://youtu.be/sWRvSG7vL4g?si=85jnvnmTCF9YIWXI)
-- ⛓ [How to use Custom Prompts for `RetrievalQA` on `LLaMA-2 7B`](https://youtu.be/PDwUKves9GY?si=sMF99TWU0p4eiK80)
+- [`Claude-2` meets LangChain!](https://youtu.be/Hb_D3p0bK2U?si=j96Kc7oJoeRI5-iC)
+- [`PaLM 2` Meets LangChain](https://youtu.be/orPwLibLqm4?si=KgJjpEbAD9YBPqT4)
+- [`LLaMA2` with LangChain - Basics | LangChain TUTORIAL](https://youtu.be/cIRzwSXB4Rc?si=v3Hwxk1m3fksBIHN)
+- [Serving `LLaMA2` with `Replicate`](https://youtu.be/JIF4nNi26DE?si=dSazFyC4UQmaR-rJ)
+- [NEW LangChain Expression Language](https://youtu.be/ud7HJ2p3gp0?si=8pJ9O6hGbXrCX5G9)
+- [Building a RCI Chain for Agents with LangChain Expression Language](https://youtu.be/QaKM5s0TnsY?si=0miEj-o17AHcGfLG)
+- [How to Run `LLaMA-2-70B` on the `Together AI`](https://youtu.be/Tc2DHfzHeYE?si=Xku3S9dlBxWQukpe)
+- [`RetrievalQA` with `LLaMA 2 70b` & `Chroma` DB](https://youtu.be/93yueQQnqpM?si=ZMwj-eS_CGLnNMXZ)
+- [How to use `BGE Embeddings` for LangChain](https://youtu.be/sWRvSG7vL4g?si=85jnvnmTCF9YIWXI)
+- [How to use Custom Prompts for `RetrievalQA` on `LLaMA-2 7B`](https://youtu.be/PDwUKves9GY?si=sMF99TWU0p4eiK80)
 
 
 ### [LangChain](https://www.youtube.com/playlist?list=PLVEEucA9MYhOu89CX8H3MBZqayTbcCTMr) by [Prompt Engineering](https://www.youtube.com/@engineerprompt)
@@ -131,8 +135,8 @@ Below are links to tutorials and courses on LangChain. For written guides on com
 - [LangChain: Giving Memory to LLMs](https://youtu.be/dxO6pzlgJiY)
 - [BEST OPEN Alternative to `OPENAI's EMBEDDINGs` for Retrieval QA: LangChain](https://youtu.be/ogEalPMUCSY)
 - [LangChain: Run Language Models Locally - `Hugging Face Models`](https://youtu.be/Xxxuw4_iCzw) 
-- ⛓ [Slash API Costs: Mastering Caching for LLM Applications](https://youtu.be/EQOznhaJWR0?si=AXoI7f3-SVFRvQUl)
-- ⛓ [Avoid PROMPT INJECTION with `Constitutional AI` - LangChain](https://youtu.be/tyKSkPFHVX8?si=9mgcB5Y1kkotkBGB)
+- [Slash API Costs: Mastering Caching for LLM Applications](https://youtu.be/EQOznhaJWR0?si=AXoI7f3-SVFRvQUl)
+- [Avoid PROMPT INJECTION with `Constitutional AI` - LangChain](https://youtu.be/tyKSkPFHVX8?si=9mgcB5Y1kkotkBGB)
 
 
 ### LangChain by [Chat with data](https://www.youtube.com/@chatwithdata)
@@ -148,4 +152,4 @@ Below are links to tutorials and courses on LangChain. For written guides on com
 
 
 ---------------------
-⛓ icon marks a new addition [last update 2023-09-21]
+⛓ icon marks a new addition [last update 2024-02-061]
diff --git a/docs/docs/integrations/platforms/index.mdx b/docs/docs/integrations/platforms/index.mdx
@@ -1,3 +1,8 @@
+---
+sidebar_position: 0
+sidebar_class_name: hidden
+---
+
 # Providers
 
 LangChain integrates with many providers

diff --git a/docs/sidebars.js b/docs/sidebars.js
@@ -101,9 +101,9 @@ module.exports = {
       items: [
         { type: "category", label: "LLMs", collapsed: true, items: [{type:"autogenerated", dirName: "integrations/llms" }], link: { type: 'doc', id: "integrations/llms/index"}},
         { type: "category", label: "Chat models", collapsed: true, items: [{type:"autogenerated", dirName: "integrations/chat" }], link: { type: 'doc', id: "integrations/chat/index"}},
+        { type: "category", label: "Embedding models", collapsed: true, items: [{type: "autogenerated", dirName: "integrations/text_embedding" }], link: {type: "generated-index", slug: "integrations/text_embedding" }},
         { type: "category", label: "Document loaders", collapsed: true, items: [{type:"autogenerated", dirName: "integrations/document_loaders" }], link: {type: "generated-index", slug: "integrations/document_loaders" }},
         { type: "category", label: "Document transformers", collapsed: true, items: [{type: "autogenerated", dirName: "integrations/document_transformers" }], link: {type: "generated-index", slug: "integrations/document_transformers" }},
-        { type: "category", label: "Text embedding models", collapsed: true, items: [{type: "autogenerated", dirName: "integrations/text_embedding" }], link: {type: "generated-index", slug: "integrations/text_embedding" }},
         { type: "category", label: "Vector stores", collapsed: true, items: [{type: "autogenerated", dirName: "integrations/vectorstores" }], link: {type: "generated-index", slug: "integrations/vectorstores" }},
         { type: "category", label: "Retrievers", collapsed: true, items: [{type: "autogenerated", dirName: "integrations/retrievers" }], link: {type: "generated-index", slug: "integrations/retrievers" }},
         { type: "category", label: "Tools", collapsed: true, items: [{type: "autogenerated", dirName: "integrations/tools" }], link: {type: "generated-index", slug: "integrations/tools" }},

diff --git a/libs/cli/langchain_cli/integration_template/LICENSE b/libs/cli/langchain_cli/integration_template/LICENSE
@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2023 LangChain, Inc.
+Copyright (c) 2024 LangChain, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

diff --git a/libs/cli/langchain_cli/package_template/LICENSE b/libs/cli/langchain_cli/package_template/LICENSE
@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2023 LangChain, Inc.
+Copyright (c) 2024 LangChain, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

diff --git a/libs/community/langchain_community/document_loaders/recursive_url_loader.py b/libs/community/langchain_community/document_loaders/recursive_url_loader.py
@@ -93,6 +93,7 @@ def __init__(
         link_regex: Union[str, re.Pattern, None] = None,
         headers: Optional[dict] = None,
         check_response_status: bool = False,
+        continue_on_failure: bool = True,
     ) -> None:
         """Initialize with URL to crawl and any subdirectories to exclude.
 
@@ -117,6 +118,8 @@ def __init__(
             link_regex: Regex for extracting sub-links from the raw html of a web page.
             check_response_status: If True, check HTTP response status and skip
                 URLs with error responses (400-599).
+            continue_on_failure: If True, continue if getting or parsing a link raises
+                an exception. Otherwise, raise the exception.
         """
 
         self.url = url
@@ -142,6 +145,7 @@ def __init__(
         self._lock = asyncio.Lock() if self.use_async else None
         self.headers = headers
         self.check_response_status = check_response_status
+        self.continue_on_failure = continue_on_failure
 
     def _get_child_links_recursive(
         self, url: str, visited: Set[str], *, depth: int = 0
@@ -164,11 +168,14 @@ def _get_child_links_recursive(
             if self.check_response_status and 400 <= response.status_code <= 599:
                 raise ValueError(f"Received HTTP status {response.status_code}")
         except Exception as e:
-            logger.warning(
-                f"Unable to load from {url}. Received error {e} of type "
-                f"{e.__class__.__name__}"
-            )
-            return
+            if self.continue_on_failure:
+                logger.warning(
+                    f"Unable to load from {url}. Received error {e} of type "
+                    f"{e.__class__.__name__}"
+                )
+                return
+            else:
+                raise e
         content = self.extractor(response.text)
         if content:
             yield Document(
@@ -184,6 +191,7 @@ def _get_child_links_recursive(
             pattern=self.link_regex,
             prevent_outside=self.prevent_outside,
             exclude_prefixes=self.exclude_dirs,
+            continue_on_failure=self.continue_on_failure,
         )
         for link in sub_links:
             # Check all unvisited links
@@ -237,13 +245,16 @@ async def _async_get_child_links_recursive(
                 if self.check_response_status and 400 <= response.status <= 599:
                     raise ValueError(f"Received HTTP status {response.status}")
         except (aiohttp.client_exceptions.InvalidURL, Exception) as e:
-            logger.warning(
-                f"Unable to load {url}. Received error {e} of type "
-                f"{e.__class__.__name__}"
-            )
             if close_session:
                 await session.close()
-            return []
+            if self.continue_on_failure:
+                logger.warning(
+                    f"Unable to load {url}. Received error {e} of type "
+                    f"{e.__class__.__name__}"
+                )
+                return []
+            else:
+                raise e
         results = []
         content = self.extractor(text)
         if content:
@@ -261,6 +272,7 @@ async def _async_get_child_links_recursive(
                 pattern=self.link_regex,
                 prevent_outside=self.prevent_outside,
                 exclude_prefixes=self.exclude_dirs,
+                continue_on_failure=self.continue_on_failure,
             )
 
             # Recursively call the function to get the children of the children

diff --git a/libs/community/langchain_community/vectorstores/kdbai.py b/libs/community/langchain_community/vectorstores/kdbai.py
@@ -14,7 +14,7 @@
 
 
 class KDBAI(VectorStore):
-    """`KDB.AI` vector store [https://kdb.ai](https://kdb.ai)
+    """`KDB.AI` vector store.
 
     To use, you should have the `kdbai_client` python package installed.
 
@@ -25,7 +25,7 @@ class KDBAI(VectorStore):
         distance_strategy: One option from DistanceStrategy.EUCLIDEAN_DISTANCE,
             DistanceStrategy.DOT_PRODUCT or DistanceStrategy.COSINE.
 
-    See the example [notebook](https://github.com/KxSystems/langchain/blob/KDB.AI/docs/docs/integrations/vectorstores/kdbai.ipynb).
+    See the example https://github.com/KxSystems/langchain/blob/KDB.AI/docs/docs/integrations/vectorstores/kdbai.ipynb.
     """
 
     def __init__(