Merge branch 'master' into falkordbvectorcreation

.github/CODEOWNERS

@@ -0,0 +1,2 @@
+/.github/   @efriis @baskaryan @ccurme
+/libs/packages.yml   @efriis

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,7 +1,7 @@
 Thank you for contributing to LangChain!
 
 - [ ] **PR title**: "package: description"
-  - Where "package" is whichever of langchain, community, core, etc. is being modified. Use "docs: ..." for purely docs changes, "templates: ..." for template changes, "infra: ..." for CI changes.
+  - Where "package" is whichever of langchain, community, core, etc. is being modified. Use "docs: ..." for purely docs changes, "infra: ..." for CI changes.
   - Example: "community: add foobar LLM"
 
 

.github/scripts/check_diff.py

@@ -32,6 +32,21 @@
     "huggingface",
 ]
 
+# Cap python version at 3.12 for some packages with dependencies that are not yet
+# compatible with python 3.13 (mostly hf tokenizers).
+PY_312_MAX_PACKAGES = [
+    f"libs/partners/{integration}"
+    for integration in [
+        "anthropic",
+        "chroma",
+        "couchbase",
+        "huggingface",
+        "mistralai",
+        "nomic",
+        "qdrant",
+    ]
+]
+
 
 def all_package_dirs() -> Set[str]:
     return {
@@ -117,9 +132,7 @@ def _get_configs_for_single_dir(job: str, dir_: str) -> List[Dict[str, str]]:
         # declare deps in funny way
         py_versions = ["3.9", "3.11"]
 
-    elif dir_ == "libs/partners/huggingface":
-        # huggingface tokenizers doesn't support 3.13 yet
-        # because of pep517 builds
+    elif dir_ in PY_312_MAX_PACKAGES:
         py_versions = ["3.9", "3.12"]
 
     elif dir_ in ["libs/community", "libs/langchain"] and job == "extended-tests":
@@ -287,12 +300,14 @@ def _get_configs_for_multi_dirs(
             ] != ["README.md"]:
                 dirs_to_run["test"].add(f"libs/partners/{partner_dir}")
             # Skip if the directory was deleted or is just a tombstone readme
+        elif file == "libs/packages.yml":
+            continue
         elif file.startswith("libs/"):
             raise ValueError(
                 f"Unknown lib: {file}. check_diff.py likely needs "
                 "an update for this new library!"
             )
-        elif any(file.startswith(p) for p in ["docs/", "templates/", "cookbook/"]):
+        elif any(file.startswith(p) for p in ["docs/", "cookbook/"]):
             if file.startswith("docs/"):
                 docs_edited = True
             dirs_to_run["lint"].add(".")

.github/scripts/get_min_versions.py

@@ -7,12 +7,17 @@
     # for python 3.10 and below, which doesnt have stdlib tomllib
     import tomli as tomllib
 
-from packaging.version import parse as parse_version
 from packaging.specifiers import SpecifierSet
 from packaging.version import Version
 
+
+import requests
+from packaging.version import parse
+from typing import List
+
 import re
 
+
 MIN_VERSION_LIBS = [
     "langchain-core",
     "langchain-community",
@@ -31,29 +36,61 @@
 ]
 
 
-def get_min_version(version: str) -> str:
-    # base regex for x.x.x with cases for rc/post/etc
-    # valid strings: https://peps.python.org/pep-0440/#public-version-identifiers
-    vstring = r"\d+(?:\.\d+){0,2}(?:(?:a|b|rc|\.post|\.dev)\d+)?"
-    # case ^x.x.x
-    _match = re.match(f"^\\^({vstring})$", version)
-    if _match:
-        return _match.group(1)
+def get_pypi_versions(package_name: str) -> List[str]:
+    """
+    Fetch all available versions for a package from PyPI.
+
+    Args:
+        package_name (str): Name of the package
+
+    Returns:
+        List[str]: List of all available versions
 
-    # case >=x.x.x,<y.y.y
-    _match = re.match(f"^>=({vstring}),<({vstring})$", version)
-    if _match:
-        _min = _match.group(1)
-        _max = _match.group(2)
-        assert parse_version(_min) < parse_version(_max)
-        return _min
+    Raises:
+        requests.exceptions.RequestException: If PyPI API request fails
+        KeyError: If package not found or response format unexpected
+    """
+    pypi_url = f"https://pypi.org/pypi/{package_name}/json"
+    response = requests.get(pypi_url)
+    response.raise_for_status()
+    return list(response.json()["releases"].keys())
 
-    # case x.x.x
-    _match = re.match(f"^({vstring})$", version)
-    if _match:
-        return _match.group(1)
 
-    raise ValueError(f"Unrecognized version format: {version}")
+def get_minimum_version(package_name: str, spec_string: str) -> Optional[str]:
+    """
+    Find the minimum published version that satisfies the given constraints.
+
+    Args:
+        package_name (str): Name of the package
+        spec_string (str): Version specification string (e.g., ">=0.2.43,<0.4.0,!=0.3.0")
+
+    Returns:
+        Optional[str]: Minimum compatible version or None if no compatible version found
+    """
+    # rewrite occurrences of ^0.0.z to 0.0.z (can be anywhere in constraint string)
+    spec_string = re.sub(r"\^0\.0\.(\d+)", r"0.0.\1", spec_string)
+    # rewrite occurrences of ^0.y.z to >=0.y.z,<0.y+1 (can be anywhere in constraint string)
+    for y in range(1, 10):
+        spec_string = re.sub(rf"\^0\.{y}\.(\d+)", rf">=0.{y}.\1,<0.{y+1}", spec_string)
+    # rewrite occurrences of ^x.y.z to >=x.y.z,<x+1.0.0 (can be anywhere in constraint string)
+    for x in range(1, 10):
+        spec_string = re.sub(
+            rf"\^{x}\.(\d+)\.(\d+)", rf">={x}.\1.\2,<{x+1}", spec_string
+        )
+
+    spec_set = SpecifierSet(spec_string)
+    all_versions = get_pypi_versions(package_name)
+
+    valid_versions = []
+    for version_str in all_versions:
+        try:
+            version = parse(version_str)
+            if spec_set.contains(version):
+                valid_versions.append(version)
+        except ValueError:
+            continue
+
+    return str(min(valid_versions)) if valid_versions else None
 
 
 def get_min_version_from_toml(
@@ -96,7 +133,7 @@ def get_min_version_from_toml(
                 ][0]["version"]
 
             # Use parse_version to get the minimum supported version from version_string
-            min_version = get_min_version(version_string)
+            min_version = get_minimum_version(lib, version_string)
 
             # Store the minimum version in the min_versions dictionary
             min_versions[lib] = min_version
@@ -112,6 +149,20 @@ def check_python_version(version_string, constraint_string):
     :param constraint_string: A string representing the package's Python version constraints (e.g. ">=3.6, <4.0").
     :return: True if the version matches the constraints, False otherwise.
     """
+
+    # rewrite occurrences of ^0.0.z to 0.0.z (can be anywhere in constraint string)
+    constraint_string = re.sub(r"\^0\.0\.(\d+)", r"0.0.\1", constraint_string)
+    # rewrite occurrences of ^0.y.z to >=0.y.z,<0.y+1.0 (can be anywhere in constraint string)
+    for y in range(1, 10):
+        constraint_string = re.sub(
+            rf"\^0\.{y}\.(\d+)", rf">=0.{y}.\1,<0.{y+1}.0", constraint_string
+        )
+    # rewrite occurrences of ^x.y.z to >=x.y.z,<x+1.0.0 (can be anywhere in constraint string)
+    for x in range(1, 10):
+        constraint_string = re.sub(
+            rf"\^{x}\.0\.(\d+)", rf">={x}.0.\1,<{x+1}.0.0", constraint_string
+        )
+
     try:
         version = Version(version_string)
         constraints = SpecifierSet(constraint_string)

.github/scripts/prep_api_docs_build.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+"""Script to sync libraries from various repositories into the main langchain repository."""
+
+import os
+import shutil
+import yaml
+from pathlib import Path
+from typing import Dict, Any
+
+
+def load_packages_yaml() -> Dict[str, Any]:
+    """Load and parse the packages.yml file."""
+    with open("langchain/libs/packages.yml", "r") as f:
+        return yaml.safe_load(f)
+
+
+def get_target_dir(package_name: str) -> Path:
+    """Get the target directory for a given package."""
+    package_name_short = package_name.replace("langchain-", "")
+    base_path = Path("langchain/libs")
+    if package_name_short == "experimental":
+        return base_path / "experimental"
+    return base_path / "partners" / package_name_short
+
+
+def clean_target_directories(packages: Dict[str, Any]) -> None:
+    """Remove old directories that will be replaced."""
+    for package in packages["packages"]:
+        if package["repo"] != "langchain-ai/langchain":
+            target_dir = get_target_dir(package["name"])
+            if target_dir.exists():
+                print(f"Removing {target_dir}")
+                shutil.rmtree(target_dir)
+
+
+def move_libraries(packages: Dict[str, Any]) -> None:
+    """Move libraries from their source locations to the target directories."""
+    for package in packages["packages"]:
+        # Skip if it's the main langchain repo or disabled
+        if package["repo"] == "langchain-ai/langchain" or package.get(
+            "disabled", False
+        ):
+            continue
+
+        repo_name = package["repo"].split("/")[1]
+        source_path = package["path"]
+        target_dir = get_target_dir(package["name"])
+
+        # Handle root path case
+        if source_path == ".":
+            source_dir = repo_name
+        else:
+            source_dir = f"{repo_name}/{source_path}"
+
+        print(f"Moving {source_dir} to {target_dir}")
+
+        # Ensure target directory exists
+        os.makedirs(os.path.dirname(target_dir), exist_ok=True)
+
+        try:
+            # Move the directory
+            shutil.move(source_dir, target_dir)
+        except Exception as e:
+            print(f"Error moving {source_dir} to {target_dir}: {e}")
+
+
+def main():
+    """Main function to orchestrate the library sync process."""
+    try:
+        # Load packages configuration
+        packages = load_packages_yaml()
+
+        # Clean target directories
+        clean_target_directories(packages)
+
+        # Move libraries to their new locations
+        move_libraries(packages)
+
+        print("Library sync completed successfully!")
+
+    except Exception as e:
+        print(f"Error during library sync: {e}")
+        raise
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/_integration_test.yml

@@ -41,12 +41,6 @@ jobs:
         shell: bash
         run: poetry run pip install "boto3<2" "google-cloud-aiplatform<2"
 
-      - name: 'Authenticate to Google Cloud'
-        id: 'auth'
-        uses: google-github-actions/auth@v2
-        with:
-          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
-
       - name: Run integration tests
         shell: bash
         env:
@@ -81,7 +75,6 @@ jobs:
           ES_URL: ${{ secrets.ES_URL }}
           ES_CLOUD_ID: ${{ secrets.ES_CLOUD_ID }}
           ES_API_KEY: ${{ secrets.ES_API_KEY }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # for airbyte
           MONGODB_ATLAS_URI: ${{ secrets.MONGODB_ATLAS_URI }}
           VOYAGE_API_KEY: ${{ secrets.VOYAGE_API_KEY }}
           COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}

.github/workflows/_release.yml

@@ -95,9 +95,30 @@ jobs:
           PKG_NAME: ${{ needs.build.outputs.pkg-name }}
           VERSION: ${{ needs.build.outputs.version }}
         run: |
-          REGEX="^$PKG_NAME==\\d+\\.\\d+\\.\\d+\$"
-          echo $REGEX
-          PREV_TAG=$(git tag --sort=-creatordate | grep -P $REGEX || true | head -1)
+          PREV_TAG="$PKG_NAME==${VERSION%.*}.$(( ${VERSION##*.} - 1 ))"; [[ "${VERSION##*.}" -eq 0 ]] && PREV_TAG=""
+
+          # backup case if releasing e.g. 0.3.0, looks up last release
+          # note if last release (chronologically) was e.g. 0.1.47 it will get 
+          # that instead of the last 0.2 release
+          if [ -z "$PREV_TAG" ]; then
+            REGEX="^$PKG_NAME==\\d+\\.\\d+\\.\\d+\$"
+            echo $REGEX
+            PREV_TAG=$(git tag --sort=-creatordate | (grep -P $REGEX || true) | head -1)
+          fi
+
+          # if PREV_TAG is empty, let it be empty
+          if [ -z "$PREV_TAG" ]; then
+            echo "No previous tag found - first release"
+          else
+            # confirm prev-tag actually exists in git repo with git tag
+            GIT_TAG_RESULT=$(git tag -l "$PREV_TAG")
+            if [ -z "$GIT_TAG_RESULT" ]; then
+              echo "Previous tag $PREV_TAG not found in git repo"
+              exit 1
+            fi
+          fi
+
+
           TAG="${PKG_NAME}==${VERSION}"
           if [ "$TAG" == "$PREV_TAG" ]; then
             echo "No new version to release"
@@ -231,7 +252,7 @@ jobs:
         working-directory: ${{ inputs.working-directory }}
         id: min-version
         run: |
-          poetry run pip install packaging
+          poetry run pip install packaging requests
           python_version="$(poetry run python --version | awk '{print $2}')"
           min_versions="$(poetry run python $GITHUB_WORKSPACE/.github/scripts/get_min_versions.py pyproject.toml release $python_version)"
           echo "min-versions=$min_versions" >> "$GITHUB_OUTPUT"
@@ -246,12 +267,6 @@ jobs:
           make tests
         working-directory: ${{ inputs.working-directory }}
 
-      - name: 'Authenticate to Google Cloud'
-        id: 'auth'
-        uses: google-github-actions/auth@v2
-        with:
-          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
-
       - name: Import integration test dependencies
         run: poetry install --with test,test_integration
         working-directory: ${{ inputs.working-directory }}
@@ -289,7 +304,6 @@ jobs:
           ES_URL: ${{ secrets.ES_URL }}
           ES_CLOUD_ID: ${{ secrets.ES_CLOUD_ID }}
           ES_API_KEY: ${{ secrets.ES_API_KEY }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # for airbyte
           MONGODB_ATLAS_URI: ${{ secrets.MONGODB_ATLAS_URI }}
           VOYAGE_API_KEY: ${{ secrets.VOYAGE_API_KEY }}
           UPSTAGE_API_KEY: ${{ secrets.UPSTAGE_API_KEY }}
@@ -338,6 +352,8 @@ jobs:
           packages-dir: ${{ inputs.working-directory }}/dist/
           verbose: true
           print-hash: true
+          # Temp workaround since attestations are on by default as of gh-action-pypi-publish v1.11.0
+          attestations: false
 
   mark-release:
     needs:

.github/workflows/_test.yml

@@ -47,7 +47,7 @@ jobs:
         id: min-version
         shell: bash
         run: |
-          poetry run pip install packaging tomli
+          poetry run pip install packaging tomli requests
           python_version="$(poetry run python --version | awk '{print $2}')"
           min_versions="$(poetry run python $GITHUB_WORKSPACE/.github/scripts/get_min_versions.py pyproject.toml pull_request $python_version)"
           echo "min-versions=$min_versions" >> "$GITHUB_OUTPUT"

.github/workflows/_test_release.yml

@@ -98,3 +98,5 @@ jobs:
           # This is *only for CI use* and is *extremely dangerous* otherwise!
           # https://github.com/pypa/gh-action-pypi-publish#tolerating-release-package-file-duplicates
           skip-existing: true
+          # Temp workaround since attestations are on by default as of gh-action-pypi-publish v1.11.0
+          attestations: false