Multi-modal RAG template (#14186)

* OpenCLIP embeddings
* GPT-4V

---------

Co-authored-by: Erick Friis <[email protected]>

libs/experimental/langchain_experimental/open_clip/open_clip.py

@@ -8,19 +8,21 @@ class OpenCLIPEmbeddings(BaseModel, Embeddings):
     model: Any
     preprocess: Any
     tokenizer: Any
+    # Select model: https://github.com/mlfoundations/open_clip
+    model_name: str = "ViT-H-14"
+    checkpoint: str = "laion2b_s32b_b79k"
 
     @root_validator()
     def validate_environment(cls, values: Dict) -> Dict:
         """Validate that open_clip and torch libraries are installed."""
         try:
             import open_clip
 
-            ### Smaller, less performant
-            # model_name = "ViT-B-32"
-            # checkpoint = "laion2b_s34b_b79k"
-            ### Larger, more performant
-            model_name = "ViT-H-14"
-            checkpoint = "laion2b_s32b_b79k"
+            # Fall back to class defaults if not provided
+            model_name = values.get("model_name", cls.__fields__["model_name"].default)
+            checkpoint = values.get("checkpoint", cls.__fields__["checkpoint"].default)
+
+            # Load model
             model, _, preprocess = open_clip.create_model_and_transforms(
                 model_name=model_name, pretrained=checkpoint
             )

libs/langchain/langchain/vectorstores/chroma.py

@@ -176,7 +176,7 @@ def add_images(
         """Run more images through the embeddings and add to the vectorstore.
 
         Args:
-            images (List[List[float]]): Images to add to the vectorstore.
+            uris List[str]: File path to the image.
             metadatas (Optional[List[dict]], optional): Optional list of metadatas.
             ids (Optional[List[str]], optional): Optional list of IDs.
 

templates/rag-chroma-multi-modal/.gitignore

@@ -0,0 +1 @@
+docs/img_*.jpg

templates/rag-chroma-multi-modal/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 LangChain, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

templates/rag-chroma-multi-modal/README.md

@@ -0,0 +1,106 @@
+
+# rag-chroma-multi-modal
+
+Presentations (slide decks, etc) contain visual content that challenges conventional RAG.
+
+Multi-modal LLMs unlock new ways to build apps over visual content like presentations.
+
+This template performs multi-modal RAG using Chroma with multi-modal OpenCLIP embeddings and OpenAI GPT-4V.
+
+## Input
+
+Supply a slide deck as pdf in the `/docs` directory. 
+
+Create your vectorstore with: 
+
+```
+poetry install
+python ingest.py
+```
+
+## Embeddings
+
+This template will use [OpenCLIP](https://github.com/mlfoundations/open_clip) multi-modal embeddings.
+
+You can select different options (see results [here](https://github.com/mlfoundations/open_clip/blob/main/docs/openclip_results.csv)).
+
+The first time you run the app, it will automatically download the multimodal embedding model.
+
+By default, LangChain will use an embedding model with reasonably strong performance, `ViT-H-14`.
+
+You can choose alternative `OpenCLIPEmbeddings` models in `rag_chroma_multi_modal/ingest.py`:
+```
+vectorstore_mmembd = Chroma(
+    collection_name="multi-modal-rag",
+    persist_directory=str(re_vectorstore_path),
+    embedding_function=OpenCLIPEmbeddings(
+        model_name="ViT-H-14", checkpoint="laion2b_s32b_b79k"
+    ),
+)
+```
+
+## LLM
+
+The app will retrieve images using multi-modal embeddings, and pass them to GPT-4V.
+
+## Environment Setup
+
+Set the `OPENAI_API_KEY` environment variable to access the OpenAI GPT-4V.
+
+## Usage
+
+To use this package, you should first have the LangChain CLI installed:
+
+```shell
+pip install -U langchain-cli
+```
+
+To create a new LangChain project and install this as the only package, you can do:
+
+```shell
+langchain app new my-app --package rag-chroma-multi-modal
+```
+
+If you want to add this to an existing project, you can just run:
+
+```shell
+langchain app add rag-chroma-multi-modal
+```
+
+And add the following code to your `server.py` file:
+```python
+from rag_chroma import chain as rag_chroma_chain
+
+add_routes(app, rag_chroma_chain, path="/rag-chroma-multi-modal")
+```
+
+(Optional) Let's now configure LangSmith. 
+LangSmith will help us trace, monitor and debug LangChain applications. 
+LangSmith is currently in private beta, you can sign up [here](https://smith.langchain.com/). 
+If you don't have access, you can skip this section
+
+```shell
+export LANGCHAIN_TRACING_V2=true
+export LANGCHAIN_API_KEY=<your-api-key>
+export LANGCHAIN_PROJECT=<your-project>  # if not specified, defaults to "default"
+```
+
+If you are inside this directory, then you can spin up a LangServe instance directly by:
+
+```shell
+langchain serve
+```
+
+This will start the FastAPI app with a server is running locally at 
+[http://localhost:8000](http://localhost:8000)
+
+We can see all templates at [http://127.0.0.1:8000/docs](http://127.0.0.1:8000/docs)
+We can access the playground at [http://127.0.0.1:8000/rag-chroma-multi-modal/playground](http://127.0.0.1:8000/rag-chroma-multi-modal/playground)  
+
+We can access the template from code with:
+
+```python
+from langserve.client import RemoteRunnable
+
+runnable = RemoteRunnable("http://localhost:8000/rag-chroma-multi-modal")
+```

templates/rag-chroma-multi-modal/ingest.py

@@ -0,0 +1,58 @@
+import os
+from pathlib import Path
+
+import pypdfium2 as pdfium
+from langchain.vectorstores import Chroma
+from langchain_experimental.open_clip import OpenCLIPEmbeddings
+
+
+def get_images_from_pdf(pdf_path, img_dump_path):
+    """
+    Extract images from each page of a PDF document and save as JPEG files.
+
+    :param pdf_path: A string representing the path to the PDF file.
+    :param img_dump_path: A string representing the path to dummp images.
+    """
+    pdf = pdfium.PdfDocument(pdf_path)
+    n_pages = len(pdf)
+    for page_number in range(n_pages):
+        page = pdf.get_page(page_number)
+        bitmap = page.render(scale=1, rotation=0, crop=(0, 0, 0, 0))
+        pil_image = bitmap.to_pil()
+        pil_image.save(f"{img_dump_path}/img_{page_number + 1}.jpg", format="JPEG")
+
+
+# Load PDF
+doc_path = Path(__file__).parent / "docs/DDOG_Q3_earnings_deck.pdf"
+img_dump_path = Path(__file__).parent / "docs/"
+rel_doc_path = doc_path.relative_to(Path.cwd())
+rel_img_dump_path = img_dump_path.relative_to(Path.cwd())
+print("pdf index")
+pil_images = get_images_from_pdf(rel_doc_path, rel_img_dump_path)
+print("done")
+vectorstore = Path(__file__).parent / "chroma_db_multi_modal"
+re_vectorstore_path = vectorstore.relative_to(Path.cwd())
+
+# Load embedding function
+print("Loading embedding function")
+embedding = OpenCLIPEmbeddings(model_name="ViT-H-14", checkpoint="laion2b_s32b_b79k")
+
+# Create chroma
+vectorstore_mmembd = Chroma(
+    collection_name="multi-modal-rag",
+    persist_directory=str(Path(__file__).parent / "chroma_db_multi_modal"),
+    embedding_function=embedding,
+)
+
+# Get image URIs
+image_uris = sorted(
+    [
+        os.path.join(rel_img_dump_path, image_name)
+        for image_name in os.listdir(rel_img_dump_path)
+        if image_name.endswith(".jpg")
+    ]
+)
+
+# Add images
+print("Embedding images")
+vectorstore_mmembd.add_images(uris=image_uris)