Merge branch 'langchain-ai:master' into master

gitchrisqueen · Jan 10, 2024 · 28a8ece · 28a8ece
2 parents 5b26987 + 85a4594
commit 28a8ece
Show file tree

Hide file tree

Showing 1,846 changed files with 93,100 additions and 59,816 deletions.
diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml
@@ -1,6 +1,9 @@
 blank_issues_enabled: true
 version: 2.1
 contact_links:
+  - name: 🤔 Question or Problem
+    about: Ask a question or ask about a problem in GitHub Discussions.
+    url: https://github.com/langchain-ai/langchain/discussions
   - name: Discord
     url: https://discord.gg/6adMQxSpJS
     about: General community discussions
diff --git a/.github/scripts/check_diff.py b/.github/scripts/check_diff.py
@@ -13,6 +13,10 @@
     files = sys.argv[1:]
     dirs_to_run = set()
 
+    if len(files) == 300:
+        # max diff length is 300 files - there are likely files missing
+        raise ValueError("Max diff reached. Please manually run CI on changed libs.")
+
     for file in files:
         if any(
             file.startswith(dir_)
@@ -48,4 +52,5 @@
             dirs_to_run.update(LANGCHAIN_DIRS)
         else:
             pass
-    print(json.dumps(list(dirs_to_run)))
+    json_output = json.dumps(list(dirs_to_run))
+    print(f"dirs-to-run={json_output}")
diff --git a/.github/workflows/_integration_test.yml b/.github/workflows/_integration_test.yml
@@ -37,13 +37,20 @@ jobs:
         shell: bash
         run: poetry install --with test,test_integration
 
+      - name: 'Authenticate to Google Cloud'
+        id: 'auth'
+        uses: google-github-actions/auth@v2
+        with:
+          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
+
       - name: Run integration tests
         shell: bash
         env:
           GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
           MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
           TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         run: |
           make integration_tests
 

diff --git a/.github/workflows/_release.yml b/.github/workflows/_release.yml
@@ -1,5 +1,5 @@
 name: release
-
+run-name: Release ${{ inputs.working-directory }} by @${{ github.actor }}
 on:
   workflow_call:
     inputs:
@@ -149,16 +149,29 @@ jobs:
         run: make tests
         working-directory: ${{ inputs.working-directory }}
 
+      - name: 'Authenticate to Google Cloud'
+        id: 'auth'
+        uses: google-github-actions/auth@v2
+        with:
+          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
+
       - name: Run integration tests
         if: ${{ startsWith(inputs.working-directory, 'libs/partners/') }}
         env:
           GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
           MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
           TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         run: make integration_tests
         working-directory: ${{ inputs.working-directory }}
 
+      - name: Run unit tests with minimum dependency versions
+        if: ${{ (inputs.working-directory == 'libs/langchain') || (inputs.working-directory == 'libs/community') || (inputs.working-directory == 'libs/experimental') }}
+        run: |
+          poetry run pip install -r _test_minimum_requirements.txt
+          make tests
+        working-directory: ${{ inputs.working-directory }}
 
   publish:
     needs:

diff --git a/.github/workflows/check_diffs.yml b/.github/workflows/check_diffs.yml
@@ -27,7 +27,8 @@ jobs:
       - id: files
         uses: Ana06/[email protected]
       - id: set-matrix
-        run: echo "dirs-to-run=$(python .github/scripts/check_diff.py ${{ steps.files.outputs.all }})" >> $GITHUB_OUTPUT
+        run: |
+          python .github/scripts/check_diff.py ${{ steps.files.outputs.all }} >> $GITHUB_OUTPUT
     outputs:
       dirs-to-run: ${{ steps.set-matrix.outputs.dirs-to-run }}
   ci:

diff --git a/cookbook/LLaMA2_sql_chat.ipynb b/cookbook/LLaMA2_sql_chat.ipynb
@@ -61,13 +61,13 @@
    ],
    "source": [
     "# Local\n",
-    "from langchain.chat_models import ChatOllama\n",
+    "from langchain_community.chat_models import ChatOllama\n",
     "\n",
     "llama2_chat = ChatOllama(model=\"llama2:13b-chat\")\n",
     "llama2_code = ChatOllama(model=\"codellama:7b-instruct\")\n",
     "\n",
     "# API\n",
-    "from langchain.llms import Replicate\n",
+    "from langchain_community.llms import Replicate\n",
     "\n",
     "# REPLICATE_API_TOKEN = getpass()\n",
     "# os.environ[\"REPLICATE_API_TOKEN\"] = REPLICATE_API_TOKEN\n",
@@ -107,7 +107,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from langchain.utilities import SQLDatabase\n",
+    "from langchain_community.utilities import SQLDatabase\n",
     "\n",
     "db = SQLDatabase.from_uri(\"sqlite:///nba_roster.db\", sample_rows_in_table_info=0)\n",
     "\n",
@@ -149,7 +149,7 @@
    ],
    "source": [
     "# Prompt\n",
-    "from langchain.prompts import ChatPromptTemplate\n",
+    "from langchain_core.prompts import ChatPromptTemplate\n",
     "\n",
     "# Update the template based on the type of SQL Database like MySQL, Microsoft SQL Server and so on\n",
     "template = \"\"\"Based on the table schema below, write a SQL query that would answer the user's question:\n",
@@ -278,7 +278,7 @@
    "source": [
     "# Prompt\n",
     "from langchain.memory import ConversationBufferMemory\n",
-    "from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder\n",
+    "from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder\n",
     "\n",
     "template = \"\"\"Given an input question, convert it to a SQL query. No pre-amble. Based on the table schema below, write a SQL query that would answer the user's question:\n",
     "{schema}\n",

diff --git a/cookbook/Multi_modal_RAG.ipynb b/cookbook/Multi_modal_RAG.ipynb
@@ -101,7 +101,7 @@
     "If you want to use the provided folder, then simply opt for a [pdf loader](https://python.langchain.com/docs/modules/data_connection/document_loaders/pdf) for the document:\n",
     "\n",
     "```\n",
-    "from langchain.document_loaders import PyPDFLoader\n",
+    "from langchain_community.document_loaders import PyPDFLoader\n",
     "loader = PyPDFLoader(path + fname)\n",
     "docs = loader.load()\n",
     "tables = [] # Ignore w/ basic pdf loader\n",
@@ -198,9 +198,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from langchain.chat_models import ChatOpenAI\n",
-    "from langchain.prompts import ChatPromptTemplate\n",
     "from langchain_core.output_parsers import StrOutputParser\n",
+    "from langchain_core.prompts import ChatPromptTemplate\n",
+    "from langchain_openai import ChatOpenAI\n",
     "\n",
     "\n",
     "# Generate summaries of text elements\n",
@@ -353,11 +353,11 @@
    "source": [
     "import uuid\n",
     "\n",
-    "from langchain.embeddings import OpenAIEmbeddings\n",
     "from langchain.retrievers.multi_vector import MultiVectorRetriever\n",
     "from langchain.storage import InMemoryStore\n",
-    "from langchain.vectorstores import Chroma\n",
+    "from langchain_community.vectorstores import Chroma\n",
     "from langchain_core.documents import Document\n",
+    "from langchain_openai import OpenAIEmbeddings\n",
     "\n",
     "\n",
     "def create_multi_vector_retriever(\n",

diff --git a/cookbook/Multi_modal_RAG_google.ipynb b/cookbook/Multi_modal_RAG_google.ipynb
@@ -93,7 +93,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from langchain.document_loaders import PyPDFLoader\n",
+    "from langchain_community.document_loaders import PyPDFLoader\n",
     "\n",
     "loader = PyPDFLoader(\"./cj/cj.pdf\")\n",
     "docs = loader.load()\n",
@@ -158,11 +158,11 @@
     }
    ],
    "source": [
-    "from langchain.chat_models import ChatVertexAI\n",
-    "from langchain.llms import VertexAI\n",
     "from langchain.prompts import PromptTemplate\n",
-    "from langchain.schema.output_parser import StrOutputParser\n",
+    "from langchain_community.chat_models import ChatVertexAI\n",
+    "from langchain_community.llms import VertexAI\n",
     "from langchain_core.messages import AIMessage\n",
+    "from langchain_core.output_parsers import StrOutputParser\n",
     "from langchain_core.runnables import RunnableLambda\n",
     "\n",
     "\n",
@@ -243,7 +243,7 @@
     "import base64\n",
     "import os\n",
     "\n",
-    "from langchain.schema.messages import HumanMessage\n",
+    "from langchain_core.messages import HumanMessage\n",
     "\n",
     "\n",
     "def encode_image(image_path):\n",
@@ -342,11 +342,11 @@
    "source": [
     "import uuid\n",
     "\n",
-    "from langchain.embeddings import VertexAIEmbeddings\n",
     "from langchain.retrievers.multi_vector import MultiVectorRetriever\n",
-    "from langchain.schema.document import Document\n",
     "from langchain.storage import InMemoryStore\n",
-    "from langchain.vectorstores import Chroma\n",
+    "from langchain_community.embeddings import VertexAIEmbeddings\n",
+    "from langchain_community.vectorstores import Chroma\n",
+    "from langchain_core.documents import Document\n",
     "\n",
     "\n",
     "def create_multi_vector_retriever(\n",
@@ -440,7 +440,7 @@
     "import re\n",
     "\n",
     "from IPython.display import HTML, display\n",
-    "from langchain.schema.runnable import RunnableLambda, RunnablePassthrough\n",
+    "from langchain_core.runnables import RunnableLambda, RunnablePassthrough\n",
     "from PIL import Image\n",
     "\n",
     "\n",

diff --git a/cookbook/Semi_Structured_RAG.ipynb b/cookbook/Semi_Structured_RAG.ipynb
@@ -235,9 +235,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from langchain.chat_models import ChatOpenAI\n",
-    "from langchain.prompts import ChatPromptTemplate\n",
-    "from langchain_core.output_parsers import StrOutputParser"
+    "from langchain_core.output_parsers import StrOutputParser\n",
+    "from langchain_core.prompts import ChatPromptTemplate\n",
+    "from langchain_openai import ChatOpenAI"
    ]
   },
   {
@@ -318,11 +318,11 @@
    "source": [
     "import uuid\n",
     "\n",
-    "from langchain.embeddings import OpenAIEmbeddings\n",
     "from langchain.retrievers.multi_vector import MultiVectorRetriever\n",
     "from langchain.storage import InMemoryStore\n",
-    "from langchain.vectorstores import Chroma\n",
+    "from langchain_community.vectorstores import Chroma\n",
     "from langchain_core.documents import Document\n",
+    "from langchain_openai import OpenAIEmbeddings\n",
     "\n",
     "# The vectorstore to use to index the child chunks\n",
     "vectorstore = Chroma(collection_name=\"summaries\", embedding_function=OpenAIEmbeddings())\n",

diff --git a/cookbook/Semi_structured_and_multi_modal_RAG.ipynb b/cookbook/Semi_structured_and_multi_modal_RAG.ipynb
@@ -211,9 +211,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from langchain.chat_models import ChatOpenAI\n",
-    "from langchain.prompts import ChatPromptTemplate\n",
-    "from langchain_core.output_parsers import StrOutputParser"
+    "from langchain_core.output_parsers import StrOutputParser\n",
+    "from langchain_core.prompts import ChatPromptTemplate\n",
+    "from langchain_openai import ChatOpenAI"
    ]
   },
   {
@@ -373,11 +373,11 @@
    "source": [
     "import uuid\n",
     "\n",
-    "from langchain.embeddings import OpenAIEmbeddings\n",
     "from langchain.retrievers.multi_vector import MultiVectorRetriever\n",
     "from langchain.storage import InMemoryStore\n",
-    "from langchain.vectorstores import Chroma\n",
+    "from langchain_community.vectorstores import Chroma\n",
     "from langchain_core.documents import Document\n",
+    "from langchain_openai import OpenAIEmbeddings\n",
     "\n",
     "# The vectorstore to use to index the child chunks\n",
     "vectorstore = Chroma(collection_name=\"summaries\", embedding_function=OpenAIEmbeddings())\n",

diff --git a/cookbook/Semi_structured_multi_modal_RAG_LLaMA2.ipynb b/cookbook/Semi_structured_multi_modal_RAG_LLaMA2.ipynb
@@ -209,9 +209,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from langchain.chat_models import ChatOllama\n",
-    "from langchain.prompts import ChatPromptTemplate\n",
-    "from langchain_core.output_parsers import StrOutputParser"
+    "from langchain_community.chat_models import ChatOllama\n",
+    "from langchain_core.output_parsers import StrOutputParser\n",
+    "from langchain_core.prompts import ChatPromptTemplate"
    ]
   },
   {
@@ -376,10 +376,10 @@
    "source": [
     "import uuid\n",
     "\n",
-    "from langchain.embeddings import GPT4AllEmbeddings\n",
     "from langchain.retrievers.multi_vector import MultiVectorRetriever\n",
     "from langchain.storage import InMemoryStore\n",
-    "from langchain.vectorstores import Chroma\n",
+    "from langchain_community.embeddings import GPT4AllEmbeddings\n",
+    "from langchain_community.vectorstores import Chroma\n",
     "from langchain_core.documents import Document\n",
     "\n",
     "# The vectorstore to use to index the child chunks\n",

diff --git a/cookbook/advanced_rag_eval.ipynb b/cookbook/advanced_rag_eval.ipynb
@@ -62,7 +62,7 @@
     "path = \"/Users/rlm/Desktop/cpi/\"\n",
     "\n",
     "# Load\n",
-    "from langchain.document_loaders import PyPDFLoader\n",
+    "from langchain_community.document_loaders import PyPDFLoader\n",
     "\n",
     "loader = PyPDFLoader(path + \"cpi.pdf\")\n",
     "pdf_pages = loader.load()\n",
@@ -132,8 +132,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from langchain.embeddings import OpenAIEmbeddings\n",
-    "from langchain.vectorstores import Chroma\n",
+    "from langchain_community.vectorstores import Chroma\n",
+    "from langchain_openai import OpenAIEmbeddings\n",
     "\n",
     "baseline = Chroma.from_texts(\n",
     "    texts=all_splits_pypdf_texts,\n",
@@ -160,9 +160,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from langchain.chat_models import ChatOpenAI\n",
-    "from langchain.prompts import ChatPromptTemplate\n",
     "from langchain_core.output_parsers import StrOutputParser\n",
+    "from langchain_core.prompts import ChatPromptTemplate\n",
+    "from langchain_openai import ChatOpenAI\n",
     "\n",
     "# Prompt\n",
     "prompt_text = \"\"\"You are an assistant tasked with summarizing tables and text for retrieval. \\\n",

diff --git a/cookbook/agent_vectorstore.ipynb b/cookbook/agent_vectorstore.ipynb
@@ -28,10 +28,9 @@
    "outputs": [],
    "source": [
     "from langchain.chains import RetrievalQA\n",
-    "from langchain.embeddings.openai import OpenAIEmbeddings\n",
-    "from langchain.llms import OpenAI\n",
     "from langchain.text_splitter import CharacterTextSplitter\n",
-    "from langchain.vectorstores import Chroma\n",
+    "from langchain_community.vectorstores import Chroma\n",
+    "from langchain_openai import OpenAI, OpenAIEmbeddings\n",
     "\n",
     "llm = OpenAI(temperature=0)"
    ]
@@ -69,7 +68,7 @@
     }
    ],
    "source": [
-    "from langchain.document_loaders import TextLoader\n",
+    "from langchain_community.document_loaders import TextLoader\n",
     "\n",
     "loader = TextLoader(doc_path)\n",
     "documents = loader.load()\n",
@@ -99,7 +98,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from langchain.document_loaders import WebBaseLoader"
+    "from langchain_community.document_loaders import WebBaseLoader"
    ]
   },
   {
@@ -161,7 +160,7 @@
    "source": [
     "# Import things that are needed generically\n",
     "from langchain.agents import AgentType, Tool, initialize_agent\n",
-    "from langchain.llms import OpenAI"
+    "from langchain_openai import OpenAI"
    ]
   },
   {

diff --git a/cookbook/analyze_document.ipynb b/cookbook/analyze_document.ipynb
@@ -29,7 +29,7 @@
    "outputs": [],
    "source": [
     "from langchain.chains import AnalyzeDocumentChain\n",
-    "from langchain.chat_models import ChatOpenAI\n",
+    "from langchain_openai import ChatOpenAI\n",
     "\n",
     "llm = ChatOpenAI(model=\"gpt-3.5-turbo\", temperature=0)"
    ]