Merge branch 'master' into erick/docs-throw-on-broken-anchors

.github/scripts/check_diff.py

@@ -300,6 +300,8 @@ def _get_configs_for_multi_dirs(
             ] != ["README.md"]:
                 dirs_to_run["test"].add(f"libs/partners/{partner_dir}")
             # Skip if the directory was deleted or is just a tombstone readme
+        elif file == "libs/packages.yml":
+            continue
         elif file.startswith("libs/"):
             raise ValueError(
                 f"Unknown lib: {file}. check_diff.py likely needs "

.github/scripts/prep_api_docs_build.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+"""Script to sync libraries from various repositories into the main langchain repository."""
+
+import os
+import shutil
+import yaml
+from pathlib import Path
+from typing import Dict, Any
+
+
+def load_packages_yaml() -> Dict[str, Any]:
+    """Load and parse the packages.yml file."""
+    with open("langchain/libs/packages.yml", "r") as f:
+        return yaml.safe_load(f)
+
+
+def get_target_dir(package_name: str) -> Path:
+    """Get the target directory for a given package."""
+    package_name_short = package_name.replace("langchain-", "")
+    base_path = Path("langchain/libs")
+    if package_name_short == "experimental":
+        return base_path / "experimental"
+    return base_path / "partners" / package_name_short
+
+
+def clean_target_directories(packages: Dict[str, Any]) -> None:
+    """Remove old directories that will be replaced."""
+    for package in packages["packages"]:
+        if package["repo"] != "langchain-ai/langchain":
+            target_dir = get_target_dir(package["name"])
+            if target_dir.exists():
+                print(f"Removing {target_dir}")
+                shutil.rmtree(target_dir)
+
+
+def move_libraries(packages: Dict[str, Any]) -> None:
+    """Move libraries from their source locations to the target directories."""
+    for package in packages["packages"]:
+        # Skip if it's the main langchain repo or disabled
+        if package["repo"] == "langchain-ai/langchain" or package.get(
+            "disabled", False
+        ):
+            continue
+
+        repo_name = package["repo"].split("/")[1]
+        source_path = package["path"]
+        target_dir = get_target_dir(package["name"])
+
+        # Handle root path case
+        if source_path == ".":
+            source_dir = repo_name
+        else:
+            source_dir = f"{repo_name}/{source_path}"
+
+        print(f"Moving {source_dir} to {target_dir}")
+
+        # Ensure target directory exists
+        os.makedirs(os.path.dirname(target_dir), exist_ok=True)
+
+        try:
+            # Move the directory
+            shutil.move(source_dir, target_dir)
+        except Exception as e:
+            print(f"Error moving {source_dir} to {target_dir}: {e}")
+
+
+def main():
+    """Main function to orchestrate the library sync process."""
+    try:
+        # Load packages configuration
+        packages = load_packages_yaml()
+
+        # Clean target directories
+        clean_target_directories(packages)
+
+        # Move libraries to their new locations
+        move_libraries(packages)
+
+        print("Library sync completed successfully!")
+
+    except Exception as e:
+        print(f"Error during library sync: {e}")
+        raise
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/_integration_test.yml

@@ -81,7 +81,6 @@ jobs:
           ES_URL: ${{ secrets.ES_URL }}
           ES_CLOUD_ID: ${{ secrets.ES_CLOUD_ID }}
           ES_API_KEY: ${{ secrets.ES_API_KEY }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # for airbyte
           MONGODB_ATLAS_URI: ${{ secrets.MONGODB_ATLAS_URI }}
           VOYAGE_API_KEY: ${{ secrets.VOYAGE_API_KEY }}
           COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}

.github/workflows/_release.yml

@@ -289,7 +289,6 @@ jobs:
           ES_URL: ${{ secrets.ES_URL }}
           ES_CLOUD_ID: ${{ secrets.ES_CLOUD_ID }}
           ES_API_KEY: ${{ secrets.ES_API_KEY }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # for airbyte
           MONGODB_ATLAS_URI: ${{ secrets.MONGODB_ATLAS_URI }}
           VOYAGE_API_KEY: ${{ secrets.VOYAGE_API_KEY }}
           UPSTAGE_API_KEY: ${{ secrets.UPSTAGE_API_KEY }}

.github/workflows/api_doc_build.yml

@@ -22,133 +22,29 @@ jobs:
           repository: langchain-ai/langchain-api-docs-html
           path: langchain-api-docs-html
           token: ${{ secrets.TOKEN_GITHUB_API_DOCS_HTML }}
-      - uses: actions/checkout@v4
-        with:
-          repository: langchain-ai/langchain-google
-          path: langchain-google
-      - uses: actions/checkout@v4
-        with:
-          repository: langchain-ai/langchain-datastax
-          path: langchain-datastax
-      - uses: actions/checkout@v4
-        with:
-          repository: langchain-ai/langchain-nvidia
-          path: langchain-nvidia
-      - uses: actions/checkout@v4
-        with:
-          repository: langchain-ai/langchain-cohere
-          path: langchain-cohere
-      - uses: actions/checkout@v4
-        with:
-          repository: langchain-ai/langchain-elastic
-          path: langchain-elastic
-      - uses: actions/checkout@v4
-        with:
-          repository: langchain-ai/langchain-postgres
-          path: langchain-postgres
-      - uses: actions/checkout@v4
-        with:
-          repository: langchain-ai/langchain-aws
-          path: langchain-aws
-      - uses: actions/checkout@v4
-        with:
-          repository: langchain-ai/langchain-weaviate
-          path: langchain-weaviate
-      - uses: actions/checkout@v4
-        with:
-          repository: langchain-ai/langchain-ai21
-          path: langchain-ai21
-      - uses: actions/checkout@v4
-        with:
-          repository: langchain-ai/langchain-together
-          path: langchain-together
-      - uses: actions/checkout@v4
-        with:
-          repository: langchain-ai/langchain-experimental
-          path: langchain-experimental
-      - uses: actions/checkout@v4
-        with:
-          repository: langchain-ai/langchain-milvus
-          path: langchain-milvus
-      - uses: actions/checkout@v4
-        with:
-          repository: langchain-ai/langchain-unstructured
-          path: langchain-unstructured
-      - uses: actions/checkout@v4
-        with:
-          repository: langchain-ai/langchain-databricks
-          path: langchain-databricks
-      - uses: actions/checkout@v4
-        with:
-          repository: langchain-ai/langchain-ibm
-          path: langchain-ibm
-      - uses: actions/checkout@v4
-        with:
-          repository: langchain-ai/langchain-azure
-          path: langchain-azure
-      - uses: actions/checkout@v4
-        with:
-          repository: langchain-ai/langchain-mongodb
-          path: langchain-mongodb
-      - uses: actions/checkout@v4
-        with:
-          repository: langchain-ai/langchain-redis
-          path: langchain-redis
-
 
+      - name: Get repos with yq
+        id: get-unsorted-repos
+        uses: mikefarah/yq@master
+        with:
+          cmd: yq '.packages[].repo' langchain/libs/packages.yml
 
-      - name: Set Git config
-        working-directory: langchain
+      - name: Parse YAML and checkout repos
+        env:
+          REPOS_UNSORTED: ${{ steps.get-unsorted-repos.outputs.result }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          git config --local user.email "[email protected]"
-          git config --local user.name "Github Actions"
+          # Get unique repositories
+          REPOS=$(echo "$REPOS_UNSORTED" | sort -u)
           
-      - name: Move libs
-        run: |
-          rm -rf \
-            langchain/libs/partners/google-genai \
-            langchain/libs/partners/google-vertexai \
-            langchain/libs/partners/astradb \
-            langchain/libs/partners/nvidia-trt \
-            langchain/libs/partners/nvidia-ai-endpoints \
-            langchain/libs/partners/cohere \
-            langchain/libs/partners/elasticsearch \
-            langchain/libs/partners/upstage \
-            langchain/libs/partners/ai21 \
-            langchain/libs/partners/together \
-            langchain/libs/standard-tests \
-            langchain/libs/experimental \
-            langchain/libs/partners/milvus \
-            langchain/libs/partners/unstructured \
-            langchain/libs/partners/databricks \
-            langchain/libs/partners/ibm \
-            langchain/libs/partners/azure-dynamic-sessions \
-            langchain/libs/partners/mongodb \
-            langchain/libs/partners/redis
-          mv langchain-google/libs/genai langchain/libs/partners/google-genai
-          mv langchain-google/libs/vertexai langchain/libs/partners/google-vertexai
-          mv langchain-google/libs/community langchain/libs/partners/google-community
-          mv langchain-datastax/libs/astradb langchain/libs/partners/astradb
-          mv langchain-nvidia/libs/ai-endpoints langchain/libs/partners/nvidia-ai-endpoints
-          mv langchain-cohere/libs/cohere langchain/libs/partners/cohere
-          mv langchain-elastic/libs/elasticsearch langchain/libs/partners/elasticsearch
-          mv langchain-postgres langchain/libs/partners/postgres
-          mv langchain-aws/libs/aws langchain/libs/partners/aws
-          mv langchain-weaviate/libs/weaviate langchain/libs/partners/weaviate
-          mv langchain-ai21/libs/ai21 langchain/libs/partners/ai21
-          mv langchain-together/libs/together langchain/libs/partners/together
-          mv langchain-experimental/libs/experimental langchain/libs/experimental
-          mv langchain-milvus/libs/milvus langchain/libs/partners/milvus
-          mv langchain-unstructured/libs/unstructured langchain/libs/partners/unstructured
-          mv langchain-databricks/libs/databricks langchain/libs/partners/databricks
-          mv langchain-ibm/libs/ibm langchain/libs/partners/ibm
-          mv langchain-azure/libs/azure-dynamic-sessions langchain/libs/partners/azure-dynamic-sessions
-          mv langchain-mongodb/libs/mongodb langchain/libs/partners/mongodb
-          mv langchain-redis/libs/redis langchain/libs/partners/redis
-
-      - name: Rm old html
-        run:
-          rm -rf langchain-api-docs-html/api_reference_build/html
+          # Checkout each unique repository
+          for repo in $REPOS; do
+            if [ "$repo" != "langchain-ai/langchain" ]; then
+              REPO_NAME=$(echo $repo | cut -d'/' -f2)
+              echo "Checking out $repo to $REPO_NAME"
+              git clone --depth 1 https://github.com/$repo.git $REPO_NAME
+            fi
+          done
 
       - name: Set up Python ${{ env.PYTHON_VERSION }} + Poetry ${{ env.POETRY_VERSION }}
         uses: "./langchain/.github/actions/poetry_setup"
@@ -158,16 +54,34 @@ jobs:
           cache-key: api-docs
           working-directory: langchain
 
-      - name: Install dependencies
+      - name: Install initial py deps
         working-directory: langchain
         run: |
           python -m pip install -U uv
-          python -m uv pip install --upgrade --no-cache-dir pip setuptools
-          # skip airbyte due to pandas dependency issue
-          python -m uv pip install $(ls ./libs/partners | grep -vE "airbyte" | xargs -I {} echo "./libs/partners/{}")
+          python -m uv pip install --upgrade --no-cache-dir pip setuptools pyyaml
+          
+      - name: Move libs with script
+        run: python langchain/.github/scripts/prep_api_docs_build.py
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Rm old html
+        run:
+          rm -rf langchain-api-docs-html/api_reference_build/html
+
+      - name: Install dependencies
+        working-directory: langchain
+        run: |
+          python -m uv pip install $(ls ./libs/partners | xargs -I {} echo "./libs/partners/{}")
           python -m uv pip install libs/core libs/langchain libs/text-splitters libs/community libs/experimental
           python -m uv pip install -r docs/api_reference/requirements.txt
 
+      - name: Set Git config
+        working-directory: langchain
+        run: |
+          git config --local user.email "[email protected]"
+          git config --local user.name "Github Actions"
+
       - name: Build docs
         working-directory: langchain
         run: |
@@ -182,4 +96,4 @@ jobs:
       - uses: EndBug/add-and-commit@v9
         with:
           cwd: langchain-api-docs-html
-          message: 'Update API docs build'
+          message: 'Update API docs build'

docs/api_reference/create_api_rst.py

@@ -530,7 +530,6 @@ def _out_file_path(package_name: str) -> Path:
 
 def _build_index(dirs: List[str]) -> None:
     custom_names = {
-        "airbyte": "Airbyte",
         "aws": "AWS",
         "ai21": "AI21",
         "ibm": "IBM",
@@ -601,9 +600,11 @@ def _build_index(dirs: List[str]) -> None:
         ]
         for header_name, dir_ in sorted(
             zip(integration_headers, integrations),
-            key=lambda h_d: integrations_to_show.index(h_d[1])
-            if h_d[1] in integrations_to_show
-            else len(integrations_to_show),
+            key=lambda h_d: (
+                integrations_to_show.index(h_d[1])
+                if h_d[1] in integrations_to_show
+                else len(integrations_to_show)
+            ),
         )[: len(integrations_to_show)]:
             integration_grid += f'\n- header: "**{header_name}**"\n  content: {_package_namespace(dir_).replace("_", "-")} {_get_package_version(_package_dir(dir_))}\n  link: {dir_.replace("-", "_")}/index.html'
         doc += f"""## Integrations
@@ -648,7 +649,7 @@ def main(dirs: Optional[list] = None) -> None:
         dirs = [
             dir_
             for dir_ in os.listdir(ROOT_DIR / "libs")
-            if dir_ not in ("cli", "partners", "standard-tests")
+            if dir_ not in ("cli", "partners", "standard-tests", "packages.yml")
         ]
         dirs += [
             dir_

docs/docs/concepts/async.mdx

@@ -73,7 +73,7 @@ in certain scenarios.
 
 If you are experiencing issues with streaming, callbacks or tracing in async code and are using Python 3.9 or 3.10, this is a likely cause.
 
-Please read [Propagation RunnableConfig](/docs/concepts/runnables#propagation-RunnableConfig) for more details to learn how to propagate the `RunnableConfig` down the call chain manually (or upgrade to Python 3.11 where this is no longer an issue).
+Please read [Propagation RunnableConfig](/docs/concepts/runnables/#propagation-of-runnableconfig) for more details to learn how to propagate the `RunnableConfig` down the call chain manually (or upgrade to Python 3.11 where this is no longer an issue).
 
 ## How to use in ipython and jupyter notebooks
 

docs/docs/concepts/chat_history.mdx

@@ -24,7 +24,7 @@ So a full conversation often involves a combination of two patterns of alternati
 
 ## Managing chat history
 
-Since chat models have a maximum limit on input size, it's important to manage chat history and trim it as needed to avoid exceeding the [context window](/docs/concepts/chat_models#context_window).
+Since chat models have a maximum limit on input size, it's important to manage chat history and trim it as needed to avoid exceeding the [context window](/docs/concepts/chat_models/#context-window).
 
 While processing chat history, it's essential to preserve a correct conversation structure.