voyageai[patch]: init package (#19098)

Co-authored-by: fodizoltan <[email protected]>
Co-authored-by: Yujie Qian <[email protected]>
Co-authored-by: fzowl <[email protected]>

.github/workflows/_integration_test.yml

@@ -75,6 +75,7 @@ jobs:
           ES_API_KEY: ${{ secrets.ES_API_KEY }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # for airbyte
           MONGODB_ATLAS_URI: ${{ secrets.MONGODB_ATLAS_URI }}
+          VOYAGE_API_KEY: ${{ secrets.VOYAGE_API_KEY }}
         run: |
           make integration_tests
 

.github/workflows/_release.yml

@@ -196,6 +196,7 @@ jobs:
           ES_API_KEY: ${{ secrets.ES_API_KEY }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # for airbyte
           MONGODB_ATLAS_URI: ${{ secrets.MONGODB_ATLAS_URI }}
+          VOYAGE_API_KEY: ${{ secrets.VOYAGE_API_KEY }}
         run: make integration_tests
         working-directory: ${{ inputs.working-directory }}
 

docs/docs/integrations/platforms/voyageai.mdx

@@ -0,0 +1,24 @@
+# VoyageAI
+
+All functionality related to VoyageAI
+
+>[VoyageAI](https://www.voyageai.com/) Voyage AI builds embedding models, customized for your domain and company, for better retrieval quality.
+> customized for your domain and company, for better retrieval quality.
+
+## Installation and Setup
+
+Install the integration package with
+```bash
+pip install langchain-voyageai
+```
+
+Get an VoyageAI api key and set it as an environment variable (`VOYAGE_API_KEY`)
+
+
+## Text Embedding Model
+
+See a [usage example](/docs/integrations/text_embedding/voyageai)
+
+```python
+from langchain_voyageai import VoyageAIEmbeddings
+```

docs/docs/integrations/text_embedding/voyageai.ipynb

@@ -9,7 +9,7 @@
     "\n",
     ">[Voyage AI](https://www.voyageai.com/) provides cutting-edge embedding/vectorizations models.\n",
     "\n",
-    "Let's load the Voyage Embedding class."
+    "Let's load the Voyage Embedding class. (Install the LangChain partner package with `pip install langchain-voyageai`)"
    ]
   },
   {
@@ -19,7 +19,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from langchain_community.embeddings import VoyageEmbeddings"
+    "from langchain_voyageai import VoyageAIEmbeddings"
    ]
   },
   {
@@ -37,7 +37,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "embeddings = VoyageEmbeddings(\n",
+    "embeddings = VoyageAIEmbeddings(\n",
     "    voyage_api_key=\"[ Your Voyage API key ]\", model=\"voyage-2\"\n",
     ")"
    ]

libs/community/langchain_community/embeddings/voyageai.py

@@ -14,6 +14,7 @@
 )
 
 import requests
+from langchain_core._api.deprecation import deprecated
 from langchain_core.embeddings import Embeddings
 from langchain_core.pydantic_v1 import BaseModel, Extra, SecretStr, root_validator
 from langchain_core.utils import convert_to_secret_str, get_from_dict_or_env
@@ -58,6 +59,11 @@ def _embed_with_retry(**kwargs: Any) -> Any:
     return _embed_with_retry(**kwargs)
 
 
+@deprecated(
+    since="0.0.29",
+    removal="0.2",
+    alternative_import="langchain_voyageai.VoyageAIEmbeddings",
+)
 class VoyageEmbeddings(BaseModel, Embeddings):
     """Voyage embedding models.
 

libs/partners/voyageai/.gitignore

@@ -0,0 +1 @@
+__pycache__

libs/partners/voyageai/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 LangChain, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

libs/partners/voyageai/Makefile

@@ -0,0 +1,57 @@
+.PHONY: all format lint test tests integration_tests docker_tests help extended_tests
+
+# Default target executed when no arguments are given to make.
+all: help
+
+# Define a variable for the test file path.
+TEST_FILE ?= tests/unit_tests/
+integration_test integration_tests: TEST_FILE=tests/integration_tests/
+
+test tests integration_test integration_tests:
+	poetry run pytest $(TEST_FILE)
+
+
+######################
+# LINTING AND FORMATTING
+######################
+
+# Define a variable for Python and notebook files.
+PYTHON_FILES=.
+MYPY_CACHE=.mypy_cache
+lint format: PYTHON_FILES=.
+lint_diff format_diff: PYTHON_FILES=$(shell git diff --relative=libs/partners/voyageai --name-only --diff-filter=d master | grep -E '\.py$$|\.ipynb$$')
+lint_package: PYTHON_FILES=langchain_voyageai
+lint_tests: PYTHON_FILES=tests
+lint_tests: MYPY_CACHE=.mypy_cache_test
+
+lint lint_diff lint_package lint_tests:
+	poetry run ruff .
+	poetry run ruff format $(PYTHON_FILES) --diff
+	poetry run ruff --select I $(PYTHON_FILES)
+	mkdir $(MYPY_CACHE); poetry run mypy $(PYTHON_FILES) --cache-dir $(MYPY_CACHE)
+
+format format_diff:
+	poetry run ruff format $(PYTHON_FILES)
+	poetry run ruff --select I --fix $(PYTHON_FILES)
+
+spell_check:
+	poetry run codespell --toml pyproject.toml
+
+spell_fix:
+	poetry run codespell --toml pyproject.toml -w
+
+check_imports: $(shell find langchain_voyageai -name '*.py')
+	poetry run python ./scripts/check_imports.py $^
+
+######################
+# HELP
+######################
+
+help:
+	@echo '----'
+	@echo 'check_imports				- check imports'
+	@echo 'format                       - run code formatters'
+	@echo 'lint                         - run linters'
+	@echo 'test                         - run unit tests'
+	@echo 'tests                        - run unit tests'
+	@echo 'test TEST_FILE=<test_file>   - run all tests in file'

libs/partners/voyageai/README.md

@@ -0,0 +1,21 @@
+# langchain-voyageai
+
+This package contains the LangChain integrations for VoyageAI through their `voyageai` client package.
+
+## Installation and Setup
+
+- Install the LangChain partner package
+```bash
+pip install langchain-voyageai
+```
+- Get an VoyageAI api key and set it as an environment variable (`VOYAGE_API_KEY`) or use the API key as a parameter in the Client.
+
+
+
+## Text Embedding Model
+
+See a [usage example](https://python.langchain.com/docs/integrations/text_embedding/voyageai)
+
+```python
+from langchain_voyageai import VoyageAIEmbeddings
+```

libs/partners/voyageai/langchain_voyageai/__init__.py

@@ -0,0 +1,5 @@
+from langchain_voyageai.embeddings import VoyageAIEmbeddings
+
+__all__ = [
+    "VoyageAIEmbeddings",
+]

libs/partners/voyageai/langchain_voyageai/embeddings.py

@@ -0,0 +1,130 @@
+import logging
+import os
+from typing import Iterable, List, Optional
+
+import voyageai  # type: ignore
+from langchain_core.embeddings import Embeddings
+from langchain_core.pydantic_v1 import (
+    BaseModel,
+    Extra,
+    Field,
+    SecretStr,
+    root_validator,
+)
+from langchain_core.utils import convert_to_secret_str
+
+logger = logging.getLogger(__name__)
+
+
+class VoyageAIEmbeddings(BaseModel, Embeddings):
+    """VoyageAIEmbeddings embedding model.
+
+    Example:
+        .. code-block:: python
+
+            from langchain_voyageai import VoyageAIEmbeddings
+
+            model = VoyageAIEmbeddings()
+    """
+
+    _client: voyageai.Client = Field(exclude=True)
+    _aclient: voyageai.client_async.AsyncClient = Field(exclude=True)
+    model: str
+    batch_size: int
+    show_progress_bar: bool = False
+    truncation: Optional[bool] = None
+    voyage_api_key: Optional[SecretStr] = None
+
+    class Config:
+        extra = Extra.forbid
+
+    @root_validator(pre=True)
+    def default_values(cls, values: dict) -> dict:
+        """Set default batch size based on model"""
+
+        model = values.get("model")
+        batch_size = values.get("batch_size")
+        if batch_size is None:
+            print("batch size", batch_size)
+            values["batch_size"] = 72 if model in ["voyage-2", "voyage-02"] else 7
+        return values
+
+    @root_validator()
+    def validate_environment(cls, values: dict) -> dict:
+        """Validate that VoyageAI credentials exist in environment."""
+        voyage_api_key = values.get("voyage_api_key") or os.getenv(
+            "VOYAGE_API_KEY", None
+        )
+        if voyage_api_key:
+            api_key_secretstr = convert_to_secret_str(voyage_api_key)
+            values["voyage_api_key"] = api_key_secretstr
+
+            api_key_str = api_key_secretstr.get_secret_value()
+        else:
+            api_key_str = None
+        values["_client"] = voyageai.Client(api_key=api_key_str)
+        values["_aclient"] = voyageai.client_async.AsyncClient(api_key=api_key_str)
+        return values
+
+    def _get_batch_iterator(self, texts: List[str]) -> Iterable:
+        if self.show_progress_bar:
+            try:
+                from tqdm.auto import tqdm  # type: ignore
+            except ImportError as e:
+                raise ImportError(
+                    "Must have tqdm installed if `show_progress_bar` is set to True. "
+                    "Please install with `pip install tqdm`."
+                ) from e
+
+            _iter = tqdm(range(0, len(texts), self.batch_size))
+        else:
+            _iter = range(0, len(texts), self.batch_size)  # type: ignore
+
+        return _iter
+
+    def embed_documents(self, texts: List[str]) -> List[List[float]]:
+        """Embed search docs."""
+        embeddings: List[List[float]] = []
+
+        _iter = self._get_batch_iterator(texts)
+        for i in _iter:
+            embeddings.extend(
+                self._client.embed(
+                    texts[i : i + self.batch_size],
+                    model=self.model,
+                    input_type="document",
+                    truncation=self.truncation,
+                ).embeddings
+            )
+
+        return embeddings
+
+    def embed_query(self, text: str) -> List[float]:
+        """Embed query text."""
+        return self._client.embed(
+            [text], model=self.model, input_type="query", truncation=self.truncation
+        ).embeddings[0]
+
+    async def aembed_documents(self, texts: List[str]) -> List[List[float]]:
+        embeddings: List[List[float]] = []
+
+        _iter = self._get_batch_iterator(texts)
+        for i in _iter:
+            r = await self._aclient.embed(
+                texts[i : i + self.batch_size],
+                model=self.model,
+                input_type="document",
+                truncation=self.truncation,
+            )
+            embeddings.extend(r.embeddings)
+
+        return embeddings
+
+    async def aembed_query(self, text: str) -> List[float]:
+        r = await self._aclient.embed(
+            [text],
+            model=self.model,
+            input_type="query",
+            truncation=self.truncation,
+        )
+        return r.embeddings[0]