diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000000..0430c11987032 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,129 @@ +repos: +- repo: local + hooks: + - id: core + name: format core + language: system + entry: make -C libs/core format + files: ^libs/core/ + pass_filenames: false + - id: community + name: format community + language: system + entry: make -C libs/community format + files: ^libs/community/ + pass_filenames: false + - id: langchain + name: format langchain + language: system + entry: make -C libs/langchain format + files: ^libs/langchain/ + pass_filenames: false + - id: standard-tests + name: format standard-tests + language: system + entry: make -C libs/standard-tests format + files: ^libs/standard-tests/ + pass_filenames: false + - id: text-splitters + name: format text-splitters + language: system + entry: make -C libs/text-splitters format + files: ^libs/text-splitters/ + pass_filenames: false + - id: anthropic + name: format partners/anthropic + language: system + entry: make -C libs/partners/anthropic format + files: ^libs/partners/anthropic/ + pass_filenames: false + - id: chroma + name: format partners/chroma + language: system + entry: make -C libs/partners/chroma format + files: ^libs/partners/chroma/ + pass_filenames: false + - id: couchbase + name: format partners/couchbase + language: system + entry: make -C libs/partners/couchbase format + files: ^libs/partners/couchbase/ + pass_filenames: false + - id: exa + name: format partners/exa + language: system + entry: make -C libs/partners/exa format + files: ^libs/partners/exa/ + pass_filenames: false + - id: fireworks + name: format partners/fireworks + language: system + entry: make -C libs/partners/fireworks format + files: ^libs/partners/fireworks/ + pass_filenames: false + - id: groq + name: format partners/groq + language: system + entry: make -C libs/partners/groq format + files: ^libs/partners/groq/ + pass_filenames: false + - id: huggingface + name: format partners/huggingface + language: system + entry: make -C libs/partners/huggingface format + files: ^libs/partners/huggingface/ + pass_filenames: false + - id: mistralai + name: format partners/mistralai + language: system + entry: make -C libs/partners/mistralai format + files: ^libs/partners/mistralai/ + pass_filenames: false + - id: nomic + name: format partners/nomic + language: system + entry: make -C libs/partners/nomic format + files: ^libs/partners/nomic/ + pass_filenames: false + - id: ollama + name: format partners/ollama + language: system + entry: make -C libs/partners/ollama format + files: ^libs/partners/ollama/ + pass_filenames: false + - id: openai + name: format partners/openai + language: system + entry: make -C libs/partners/openai format + files: ^libs/partners/openai/ + pass_filenames: false + - id: pinecone + name: format partners/pinecone + language: system + entry: make -C libs/partners/pinecone format + files: ^libs/partners/pinecone/ + pass_filenames: false + - id: prompty + name: format partners/prompty + language: system + entry: make -C libs/partners/prompty format + files: ^libs/partners/prompty/ + pass_filenames: false + - id: qdrant + name: format partners/qdrant + language: system + entry: make -C libs/partners/qdrant format + files: ^libs/partners/qdrant/ + pass_filenames: false + - id: voyageai + name: format partners/voyageai + language: system + entry: make -C libs/partners/voyageai format + files: ^libs/partners/voyageai/ + pass_filenames: false + - id: root + name: format docs, cookbook + language: system + entry: make format + files: ^(docs|cookbook)/ + pass_filenames: false diff --git a/README.md b/README.md index dd8643d4b0a40..efaba73deaf37 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ For these applications, LangChain simplifies the entire application lifecycle: **🧱 Extracting structured output** - [Documentation](https://python.langchain.com/docs/tutorials/extraction/) -- End-to-end Example: [SQL Llama2 Template](https://github.com/langchain-ai/langchain-extract/) +- End-to-end Example: [LangChain Extract](https://github.com/langchain-ai/langchain-extract/) **🤖 Chatbots** diff --git a/docs/docs/additional_resources/tutorials.mdx b/docs/docs/additional_resources/tutorials.mdx index 1b98d8c31af7a..871b12e40812f 100644 --- a/docs/docs/additional_resources/tutorials.mdx +++ b/docs/docs/additional_resources/tutorials.mdx @@ -12,6 +12,7 @@ ### [by Mayo Oshin](https://www.youtube.com/@chatwithdata/search?query=langchain) ### [by 1 little Coder](https://www.youtube.com/playlist?list=PLpdmBGJ6ELUK-v0MK-t4wZmVEbxM5xk6L) ### [by BobLin (Chinese language)](https://www.youtube.com/playlist?list=PLbd7ntv6PxC3QMFQvtWfk55p-Op_syO1C) +### [by Total Technology Zonne](https://youtube.com/playlist?list=PLI8raxzYtfGyE02fAxiM1CPhLUuqcTLWg&si=fkAye16rQKBJVHc9) ## Courses diff --git a/docs/docs/concepts/index.mdx b/docs/docs/concepts/index.mdx index 6a9d8d0db8c69..e7b86f41e4e0f 100644 --- a/docs/docs/concepts/index.mdx +++ b/docs/docs/concepts/index.mdx @@ -48,7 +48,7 @@ The conceptual guide does not cover step-by-step instructions or specific implem - **[AIMessage](/docs/concepts/messages#aimessage)**: Represents a complete response from an AI model. - **[astream_events](/docs/concepts/chat_models#key-methods)**: Stream granular information from [LCEL](/docs/concepts/lcel) chains. - **[BaseTool](/docs/concepts/tools/#tool-interface)**: The base class for all tools in LangChain. -- **[batch](/docs/concepts/runnables)**: Use to execute a runnable with batch inputs a Runnable. +- **[batch](/docs/concepts/runnables)**: Use to execute a runnable with batch inputs. - **[bind_tools](/docs/concepts/tool_calling/#tool-binding)**: Allows models to interact with tools. - **[Caching](/docs/concepts/chat_models#caching)**: Storing results to avoid redundant calls to a chat model. - **[Chat models](/docs/concepts/multimodality/#multimodality-in-chat-models)**: Chat models that handle multiple data modalities. @@ -70,7 +70,7 @@ The conceptual guide does not cover step-by-step instructions or specific implem - **[langchain-core](/docs/concepts/architecture#langchain-core)**: Core langchain package. Includes base interfaces and in-memory implementations. - **[langchain](/docs/concepts/architecture#langchain)**: A package for higher level components (e.g., some pre-built chains). - **[langgraph](/docs/concepts/architecture#langgraph)**: Powerful orchestration layer for LangChain. Use to build complex pipelines and workflows. -- **[langserve](/docs/concepts/architecture#langserve)**: Use to deploy LangChain Runnables as REST endpoints. Uses FastAPI. Works primarily for LangChain Runnables, does not currently integrate with LangGraph. +- **[langserve](/docs/concepts/architecture#langserve)**: Used to deploy LangChain Runnables as REST endpoints. Uses FastAPI. Works primarily for LangChain Runnables, does not currently integrate with LangGraph. - **[LLMs (legacy)](/docs/concepts/text_llms)**: Older language models that take a string as input and return a string as output. - **[Managing chat history](/docs/concepts/chat_history#managing-chat-history)**: Techniques to maintain and manage the chat history. - **[OpenAI format](/docs/concepts/messages#openai-format)**: OpenAI's message format for chat models. @@ -79,7 +79,7 @@ The conceptual guide does not cover step-by-step instructions or specific implem - **[RemoveMessage](/docs/concepts/messages/#removemessage)**: An abstraction used to remove a message from chat history, used primarily in LangGraph. - **[role](/docs/concepts/messages#role)**: Represents the role (e.g., user, assistant) of a chat message. - **[RunnableConfig](/docs/concepts/runnables/#runnableconfig)**: Use to pass run time information to Runnables (e.g., `run_name`, `run_id`, `tags`, `metadata`, `max_concurrency`, `recursion_limit`, `configurable`). -- **[Standard parameters for chat models](/docs/concepts/chat_models#standard-parameters)**: Parameters such as API key, `temperature`, and `max_tokens`, +- **[Standard parameters for chat models](/docs/concepts/chat_models#standard-parameters)**: Parameters such as API key, `temperature`, and `max_tokens`. - **[Standard tests](/docs/concepts/testing#standard-tests)**: A defined set of unit and integration tests that all integrations must pass. - **[stream](/docs/concepts/streaming)**: Use to stream output from a Runnable or a graph. - **[Tokenization](/docs/concepts/tokens)**: The process of converting data into tokens and vice versa. diff --git a/docs/docs/concepts/runnables.mdx b/docs/docs/concepts/runnables.mdx index 93af2cecadc32..0ba5e0027f3f0 100644 --- a/docs/docs/concepts/runnables.mdx +++ b/docs/docs/concepts/runnables.mdx @@ -114,7 +114,7 @@ Please see the [Configurable Runnables](#configurable-runnables) section for mor | Method | Description | |-------------------------|------------------------------------------------------------------| | `get_input_schema` | Gives the Pydantic Schema of the input schema for the Runnable. | -| `get_output_chema` | Gives the Pydantic Schema of the output schema for the Runnable. | +| `get_output_schema` | Gives the Pydantic Schema of the output schema for the Runnable. | | `config_schema` | Gives the Pydantic Schema of the config schema for the Runnable. | | `get_input_jsonschema` | Gives the JSONSchema of the input schema for the Runnable. | | `get_output_jsonschema` | Gives the JSONSchema of the output schema for the Runnable. | @@ -323,7 +323,7 @@ multiple Runnables and you need to add custom processing logic in one of the ste There are two ways to create a custom Runnable from a function: -* `RunnableLambda`: Use this simple transformations where streaming is not required. +* `RunnableLambda`: Use this for simple transformations where streaming is not required. * `RunnableGenerator`: use this for more complex transformations when streaming is needed. See the [How to run custom functions](/docs/how_to/functions) guide for more information on how to use `RunnableLambda` and `RunnableGenerator`. @@ -347,6 +347,6 @@ Sometimes you may want to experiment with, or even expose to the end user, multi To simplify this process, the Runnable interface provides two methods for creating configurable Runnables at runtime: * `configurable_fields`: This method allows you to configure specific **attributes** in a Runnable. For example, the `temperature` attribute of a chat model. -* `configurable_alternatives`: This method enables you to specify **alternative** Runnables that can be run during run time. For example, you could specify a list of different chat models that can be used. +* `configurable_alternatives`: This method enables you to specify **alternative** Runnables that can be run during runtime. For example, you could specify a list of different chat models that can be used. See the [How to configure runtime chain internals](/docs/how_to/configure) guide for more information on how to configure runtime chain internals. diff --git a/docs/docs/contributing/how_to/integrations/package.mdx b/docs/docs/contributing/how_to/integrations/package.mdx index 990f097fd2a19..e824a0ffa266f 100644 --- a/docs/docs/contributing/how_to/integrations/package.mdx +++ b/docs/docs/contributing/how_to/integrations/package.mdx @@ -291,37 +291,8 @@ import VectorstoreSource from '../../../../src/theme/integration_template/integr Embeddings are used to convert `str` objects from `Document.page_content` fields into a vector representation (represented as a list of floats). -The `Embeddings` class must inherit from the [Embeddings](https://python.langchain.com/api_reference/core/embeddings/langchain_core.embeddings.embeddings.Embeddings.html#langchain_core.embeddings.embeddings.Embeddings) -base class. This interface has 5 methods that can be implemented. - -| Method/Property | Description | -|------------------------ |------------------------------------------------------| -| `__init__` | Initialize the embeddings object. (optional) | -| `embed_query` | Embed a list of texts. (required) | -| `embed_documents` | Embed a list of documents. (required) | -| `aembed_query` | Asynchronously embed a list of texts. (optional) | -| `aembed_documents` | Asynchronously embed a list of documents. (optional) | - -### Constructor - -The `__init__` constructor is optional but common, but can be used to set up any necessary attributes -that a user can pass in when initializing the embeddings object. Common attributes include - -- `model` - the id of the model to use for embeddings - -### Embedding queries vs documents - -The `embed_query` and `embed_documents` methods are required. These methods both operate -on string inputs (the accessing of `Document.page_content` attributes) is handled -by the VectorStore using the embedding model for legacy reasons. - -`embed_query` takes in a single string and returns a single embedding as a list of floats. -If your model has different modes for embedding queries vs the underlying documents, you can -implement this method to handle that. - -`embed_documents` takes in a list of strings and returns a list of embeddings as a list of lists of floats. - -### Implementation +Refer to the [Custom Embeddings Guide](/docs/how_to/custom_embeddings) guide for +detail on a starter embeddings [implementation](/docs/how_to/custom_embeddings/#implementation). You can start from the following template or langchain-cli command: diff --git a/docs/docs/contributing/index.mdx b/docs/docs/contributing/index.mdx index d2c1231789d8d..90e677bcf5f77 100644 --- a/docs/docs/contributing/index.mdx +++ b/docs/docs/contributing/index.mdx @@ -37,7 +37,11 @@ If you are able to help answer questions, please do so! This will allow the main Our [issues](https://github.com/langchain-ai/langchain/issues) page is kept up to date with bugs, improvements, and feature requests. -There is a taxonomy of labels to help with sorting and discovery of issues of interest. Please use these to help organize issues. +There is a [taxonomy of labels](https://github.com/langchain-ai/langchain/labels?sort=count-desc) +to help with sorting and discovery of issues of interest. Please use these to help +organize issues. Check out the [Help Wanted](https://github.com/langchain-ai/langchain/labels/help%20wanted) +and [Good First Issue](https://github.com/langchain-ai/langchain/labels/good%20first%20issue) +tags for recommendations. If you start working on an issue, please assign it to yourself. diff --git a/docs/docs/how_to/custom_embeddings.ipynb b/docs/docs/how_to/custom_embeddings.ipynb new file mode 100644 index 0000000000000..007539adcfac1 --- /dev/null +++ b/docs/docs/how_to/custom_embeddings.ipynb @@ -0,0 +1,222 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "id": "c160026f-aadb-4e9f-8642-b4a9e8479d77", + "metadata": {}, + "source": [ + "# Custom Embeddings\n", + "\n", + "LangChain is integrated with many [3rd party embedding models](/docs/integrations/text_embedding/). In this guide we'll show you how to create a custom Embedding class, in case a built-in one does not already exist. Embeddings are critical in natural language processing applications as they convert text into a numerical form that algorithms can understand, thereby enabling a wide range of applications such as similarity search, text classification, and clustering.\n", + "\n", + "Implementing embeddings using the standard [Embeddings](https://python.langchain.com/api_reference/core/embeddings/langchain_core.embeddings.embeddings.Embeddings.html) interface will allow your embeddings to be utilized in existing `LangChain` abstractions (e.g., as the embeddings powering a [VectorStore](https://python.langchain.com/api_reference/core/vectorstores/langchain_core.vectorstores.base.VectorStore.html) or cached using [CacheBackedEmbeddings](/docs/how_to/caching_embeddings/)).\n", + "\n", + "## Interface\n", + "\n", + "The current `Embeddings` abstraction in LangChain is designed to operate on text data. In this implementation, the inputs are either single strings or lists of strings, and the outputs are lists of numerical arrays (vectors), where each vector represents\n", + "an embedding of the input text into some n-dimensional space.\n", + "\n", + "Your custom embedding must implement the following methods:\n", + "\n", + "| Method/Property | Description | Required/Optional |\n", + "|---------------------------------|----------------------------------------------------------------------------|-------------------|\n", + "| `embed_documents(texts)` | Generates embeddings for a list of strings. | Required |\n", + "| `embed_query(text)` | Generates an embedding for a single text query. | Required |\n", + "| `aembed_documents(texts)` | Asynchronously generates embeddings for a list of strings. | Optional |\n", + "| `aembed_query(text)` | Asynchronously generates an embedding for a single text query. | Optional |\n", + "\n", + "These methods ensure that your embedding model can be integrated seamlessly into the LangChain framework, providing both synchronous and asynchronous capabilities for scalability and performance optimization.\n", + "\n", + "\n", + ":::note\n", + "`Embeddings` do not currently implement the [Runnable](/docs/concepts/runnables/) interface and are also **not** instances of pydantic `BaseModel`.\n", + ":::\n", + "\n", + "### Embedding queries vs documents\n", + "\n", + "The `embed_query` and `embed_documents` methods are required. These methods both operate\n", + "on string inputs. The accessing of `Document.page_content` attributes is handled\n", + "by the vector store using the embedding model for legacy reasons.\n", + "\n", + "`embed_query` takes in a single string and returns a single embedding as a list of floats.\n", + "If your model has different modes for embedding queries vs the underlying documents, you can\n", + "implement this method to handle that. \n", + "\n", + "`embed_documents` takes in a list of strings and returns a list of embeddings as a list of lists of floats.\n", + "\n", + ":::note\n", + "`embed_documents` takes in a list of plain text, not a list of LangChain `Document` objects. The name of this method\n", + "may change in future versions of LangChain.\n", + ":::" + ] + }, + { + "cell_type": "markdown", + "id": "2162547f-4577-47e8-b12f-e9aa3c243797", + "metadata": {}, + "source": [ + "## Implementation\n", + "\n", + "As an example, we'll implement a simple embeddings model that returns a constant vector. This model is for illustrative purposes only." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "6b838062-552c-43f8-94f8-d17e4ae4c221", + "metadata": {}, + "outputs": [], + "source": [ + "from typing import List\n", + "\n", + "from langchain_core.embeddings import Embeddings\n", + "\n", + "\n", + "class ParrotLinkEmbeddings(Embeddings):\n", + " \"\"\"ParrotLink embedding model integration.\n", + "\n", + " # TODO: Populate with relevant params.\n", + " Key init args — completion params:\n", + " model: str\n", + " Name of ParrotLink model to use.\n", + "\n", + " See full list of supported init args and their descriptions in the params section.\n", + "\n", + " # TODO: Replace with relevant init params.\n", + " Instantiate:\n", + " .. code-block:: python\n", + "\n", + " from langchain_parrot_link import ParrotLinkEmbeddings\n", + "\n", + " embed = ParrotLinkEmbeddings(\n", + " model=\"...\",\n", + " # api_key=\"...\",\n", + " # other params...\n", + " )\n", + "\n", + " Embed single text:\n", + " .. code-block:: python\n", + "\n", + " input_text = \"The meaning of life is 42\"\n", + " embed.embed_query(input_text)\n", + "\n", + " .. code-block:: python\n", + "\n", + " # TODO: Example output.\n", + "\n", + " # TODO: Delete if token-level streaming isn't supported.\n", + " Embed multiple text:\n", + " .. code-block:: python\n", + "\n", + " input_texts = [\"Document 1...\", \"Document 2...\"]\n", + " embed.embed_documents(input_texts)\n", + "\n", + " .. code-block:: python\n", + "\n", + " # TODO: Example output.\n", + "\n", + " # TODO: Delete if native async isn't supported.\n", + " Async:\n", + " .. code-block:: python\n", + "\n", + " await embed.aembed_query(input_text)\n", + "\n", + " # multiple:\n", + " # await embed.aembed_documents(input_texts)\n", + "\n", + " .. code-block:: python\n", + "\n", + " # TODO: Example output.\n", + "\n", + " \"\"\"\n", + "\n", + " def __init__(self, model: str):\n", + " self.model = model\n", + "\n", + " def embed_documents(self, texts: List[str]) -> List[List[float]]:\n", + " \"\"\"Embed search docs.\"\"\"\n", + " return [[0.5, 0.6, 0.7] for _ in texts]\n", + "\n", + " def embed_query(self, text: str) -> List[float]:\n", + " \"\"\"Embed query text.\"\"\"\n", + " return self.embed_documents([text])[0]\n", + "\n", + " # optional: add custom async implementations here\n", + " # you can also delete these, and the base class will\n", + " # use the default implementation, which calls the sync\n", + " # version in an async executor:\n", + "\n", + " # async def aembed_documents(self, texts: List[str]) -> List[List[float]]:\n", + " # \"\"\"Asynchronous Embed search docs.\"\"\"\n", + " # ...\n", + "\n", + " # async def aembed_query(self, text: str) -> List[float]:\n", + " # \"\"\"Asynchronous Embed query text.\"\"\"\n", + " # ..." + ] + }, + { + "cell_type": "markdown", + "id": "47a19044-5c3f-40da-889a-1a1cfffc137c", + "metadata": {}, + "source": [ + "### Let's test it 🧪" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "21c218fe-8f91-437f-b523-c2b6e5cf749e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[[0.5, 0.6, 0.7], [0.5, 0.6, 0.7]]\n", + "[0.5, 0.6, 0.7]\n" + ] + } + ], + "source": [ + "embeddings = ParrotLinkEmbeddings(\"test-model\")\n", + "print(embeddings.embed_documents([\"Hello\", \"world\"]))\n", + "print(embeddings.embed_query(\"Hello\"))" + ] + }, + { + "cell_type": "markdown", + "id": "de50f690-178e-4561-af98-14967b3c8501", + "metadata": {}, + "source": [ + "## Contributing\n", + "\n", + "We welcome contributions of Embedding models to the LangChain code base.\n", + "\n", + "If you aim to contribute an embedding model for a new provider (e.g., with a new set of dependencies or SDK), we encourage you to publish your implementation in a separate `langchain-*` integration package. This will enable you to appropriately manage dependencies and version your package. Please refer to our [contributing guide](/docs/contributing/how_to/integrations/) for a walkthrough of this process." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/docs/how_to/custom_llm.ipynb b/docs/docs/how_to/custom_llm.ipynb index d56380332f4b3..8accfd3040755 100644 --- a/docs/docs/how_to/custom_llm.ipynb +++ b/docs/docs/how_to/custom_llm.ipynb @@ -9,10 +9,16 @@ "\n", "This notebook goes over how to create a custom LLM wrapper, in case you want to use your own LLM or a different wrapper than one that is supported in LangChain.\n", "\n", - "Wrapping your LLM with the standard `LLM` interface allow you to use your LLM in existing LangChain programs with minimal code modifications!\n", + "Wrapping your LLM with the standard `LLM` interface allow you to use your LLM in existing LangChain programs with minimal code modifications.\n", "\n", "As an bonus, your LLM will automatically become a LangChain `Runnable` and will benefit from some optimizations out of the box, async support, the `astream_events` API, etc.\n", "\n", + ":::caution\n", + "You are currently on a page documenting the use of [text completion models](/docs/concepts/text_llms). Many of the latest and most popular models are [chat completion models](/docs/concepts/chat_models).\n", + "\n", + "Unless you are specifically using more advanced prompting techniques, you are probably looking for [this page instead](/docs/how_to/custom_chat_model/).\n", + ":::\n", + "\n", "## Implementation\n", "\n", "There are only two required things that a custom LLM needs to implement:\n", diff --git a/docs/docs/how_to/index.mdx b/docs/docs/how_to/index.mdx index 1ce6cc2737a57..51f906c434b43 100644 --- a/docs/docs/how_to/index.mdx +++ b/docs/docs/how_to/index.mdx @@ -159,6 +159,7 @@ See [supported integrations](/docs/integrations/text_embedding/) for details on - [How to: embed text data](/docs/how_to/embed_text) - [How to: cache embedding results](/docs/how_to/caching_embeddings) +- [How to: create a custom embeddings class](/docs/how_to/custom_embeddings) ### Vector stores @@ -244,6 +245,7 @@ All of LangChain components can easily be extended to support your own versions. - [How to: create a custom chat model class](/docs/how_to/custom_chat_model) - [How to: create a custom LLM class](/docs/how_to/custom_llm) +- [How to: create a custom embeddings class](/docs/how_to/custom_embeddings) - [How to: write a custom retriever class](/docs/how_to/custom_retriever) - [How to: write a custom document loader](/docs/how_to/document_loader_custom) - [How to: write a custom output parser class](/docs/how_to/output_parser_custom) diff --git a/docs/docs/how_to/indexing.ipynb b/docs/docs/how_to/indexing.ipynb index e3e6ec8aef6d7..4e6fed96abdcd 100644 --- a/docs/docs/how_to/indexing.ipynb +++ b/docs/docs/how_to/indexing.ipynb @@ -39,19 +39,20 @@ "| None | ✅ | ✅ | ❌ | ❌ | - |\n", "| Incremental | ✅ | ✅ | ❌ | ✅ | Continuously |\n", "| Full | ✅ | ❌ | ✅ | ✅ | At end of indexing |\n", + "| Scoped_Full | ✅ | ✅ | ❌ | ✅ | At end of indexing |\n", "\n", "\n", "`None` does not do any automatic clean up, allowing the user to manually do clean up of old content. \n", "\n", - "`incremental` and `full` offer the following automated clean up:\n", + "`incremental`, `full` and `scoped_full` offer the following automated clean up:\n", "\n", - "* If the content of the source document or derived documents has **changed**, both `incremental` or `full` modes will clean up (delete) previous versions of the content.\n", - "* If the source document has been **deleted** (meaning it is not included in the documents currently being indexed), the `full` cleanup mode will delete it from the vector store correctly, but the `incremental` mode will not.\n", + "* If the content of the source document or derived documents has **changed**, all 3 modes will clean up (delete) previous versions of the content.\n", + "* If the source document has been **deleted** (meaning it is not included in the documents currently being indexed), the `full` cleanup mode will delete it from the vector store correctly, but the `incremental` and `scoped_full` mode will not.\n", "\n", "When content is mutated (e.g., the source PDF file was revised) there will be a period of time during indexing when both the new and old versions may be returned to the user. This happens after the new content was written, but before the old version was deleted.\n", "\n", "* `incremental` indexing minimizes this period of time as it is able to do clean up continuously, as it writes.\n", - "* `full` mode does the clean up after all batches have been written.\n", + "* `full` and `scoped_full` mode does the clean up after all batches have been written.\n", "\n", "## Requirements\n", "\n", @@ -64,7 +65,7 @@ " \n", "## Caution\n", "\n", - "The record manager relies on a time-based mechanism to determine what content can be cleaned up (when using `full` or `incremental` cleanup modes).\n", + "The record manager relies on a time-based mechanism to determine what content can be cleaned up (when using `full` or `incremental` or `scoped_full` cleanup modes).\n", "\n", "If two tasks run back-to-back, and the first task finishes before the clock time changes, then the second task may not be able to clean up content.\n", "\n", diff --git a/docs/docs/how_to/markdown_header_metadata_splitter.ipynb b/docs/docs/how_to/markdown_header_metadata_splitter.ipynb index ad33be83991cf..a5257dc5e0e9e 100644 --- a/docs/docs/how_to/markdown_header_metadata_splitter.ipynb +++ b/docs/docs/how_to/markdown_header_metadata_splitter.ipynb @@ -162,6 +162,18 @@ "md_header_splits" ] }, + { + "cell_type": "markdown", + "id": "fb3f834a", + "metadata": {}, + "source": [ + ":::note\n", + "\n", + "The default `MarkdownHeaderTextSplitter` strips white spaces and new lines. To preserve the original formatting of your Markdown documents, check out [ExperimentalMarkdownSyntaxTextSplitter](https://python.langchain.com/api_reference/text_splitters/markdown/langchain_text_splitters.markdown.ExperimentalMarkdownSyntaxTextSplitter.html).\n", + "\n", + ":::" + ] + }, { "cell_type": "markdown", "id": "aa67e0cc-d721-4536-9c7a-9fa3a7a69cbe", diff --git a/docs/docs/how_to/semantic-chunker.ipynb b/docs/docs/how_to/semantic-chunker.ipynb index 35889ec53afd0..aa71aeafcc115 100644 --- a/docs/docs/how_to/semantic-chunker.ipynb +++ b/docs/docs/how_to/semantic-chunker.ipynb @@ -125,9 +125,11 @@ "\n", "There are a few ways to determine what that threshold is, which are controlled by the `breakpoint_threshold_type` kwarg.\n", "\n", + "Note: if the resulting chunk sizes are too small/big, the additional kwargs `breakpoint_threshold_amount` and `min_chunk_size` can be used for adjustments.\n", + "\n", "### Percentile\n", "\n", - "The default way to split is based on percentile. In this method, all differences between sentences are calculated, and then any difference greater than the X percentile is split." + "The default way to split is based on percentile. In this method, all differences between sentences are calculated, and then any difference greater than the X percentile is split. The default value for X is 95.0 and can be adjusted by the keyword argument `breakpoint_threshold_amount` which expects a number between 0.0 and 100.0." ] }, { @@ -186,7 +188,7 @@ "source": [ "### Standard Deviation\n", "\n", - "In this method, any difference greater than X standard deviations is split." + "In this method, any difference greater than X standard deviations is split. The default value for X is 3.0 and can be adjusted by the keyword argument `breakpoint_threshold_amount`." ] }, { @@ -245,7 +247,7 @@ "source": [ "### Interquartile\n", "\n", - "In this method, the interquartile distance is used to split chunks." + "In this method, the interquartile distance is used to split chunks. The interquartile range can be scaled by the keyword argument `breakpoint_threshold_amount`, the default value is 1.5." ] }, { @@ -306,8 +308,8 @@ "source": [ "### Gradient\n", "\n", - "In this method, the gradient of distance is used to split chunks along with the percentile method.\n", - "This method is useful when chunks are highly correlated with each other or specific to a domain e.g. legal or medical. The idea is to apply anomaly detection on gradient array so that the distribution become wider and easy to identify boundaries in highly semantic data." + "In this method, the gradient of distance is used to split chunks along with the percentile method. This method is useful when chunks are highly correlated with each other or specific to a domain e.g. legal or medical. The idea is to apply anomaly detection on gradient array so that the distribution become wider and easy to identify boundaries in highly semantic data.\n", + "Similar to the percentile method, the split can be adjusted by the keyword argument `breakpoint_threshold_amount` which expects a number between 0.0 and 100.0, the default value is 95.0." ] }, { diff --git a/docs/docs/integrations/chat/cerebras.ipynb b/docs/docs/integrations/chat/cerebras.ipynb index e6ad0cf8938e7..1713681f0100c 100644 --- a/docs/docs/integrations/chat/cerebras.ipynb +++ b/docs/docs/integrations/chat/cerebras.ipynb @@ -139,7 +139,7 @@ "from langchain_cerebras import ChatCerebras\n", "\n", "llm = ChatCerebras(\n", - " model=\"llama3.1-70b\",\n", + " model=\"llama-3.3-70b\",\n", " # other params...\n", ")" ] @@ -215,7 +215,7 @@ "from langchain_core.prompts import ChatPromptTemplate\n", "\n", "llm = ChatCerebras(\n", - " model=\"llama3.1-70b\",\n", + " model=\"llama-3.3-70b\",\n", " # other params...\n", ")\n", "\n", @@ -280,7 +280,7 @@ "from langchain_core.prompts import ChatPromptTemplate\n", "\n", "llm = ChatCerebras(\n", - " model=\"llama3.1-70b\",\n", + " model=\"llama-3.3-70b\",\n", " # other params...\n", ")\n", "\n", @@ -324,7 +324,7 @@ "from langchain_core.prompts import ChatPromptTemplate\n", "\n", "llm = ChatCerebras(\n", - " model=\"llama3.1-70b\",\n", + " model=\"llama-3.3-70b\",\n", " # other params...\n", ")\n", "\n", @@ -371,7 +371,7 @@ "from langchain_core.prompts import ChatPromptTemplate\n", "\n", "llm = ChatCerebras(\n", - " model=\"llama3.1-70b\",\n", + " model=\"llama-3.3-70b\",\n", " # other params...\n", ")\n", "\n", diff --git a/docs/docs/integrations/chat/litellm_router.ipynb b/docs/docs/integrations/chat/litellm_router.ipynb index 7a7d0fd1218c8..af56c657547a0 100644 --- a/docs/docs/integrations/chat/litellm_router.ipynb +++ b/docs/docs/integrations/chat/litellm_router.ipynb @@ -63,9 +63,9 @@ " },\n", " },\n", " {\n", - " \"model_name\": \"gpt-4\",\n", + " \"model_name\": \"gpt-35-turbo\",\n", " \"litellm_params\": {\n", - " \"model\": \"azure/gpt-4-1106-preview\",\n", + " \"model\": \"azure/gpt-35-turbo\",\n", " \"api_key\": \"\",\n", " \"api_version\": \"2023-05-15\",\n", " \"api_base\": \"https://.openai.azure.com/\",\n", @@ -73,7 +73,7 @@ " },\n", "]\n", "litellm_router = Router(model_list=model_list)\n", - "chat = ChatLiteLLMRouter(router=litellm_router)" + "chat = ChatLiteLLMRouter(router=litellm_router, model_name=\"gpt-35-turbo\")" ] }, { @@ -177,6 +177,7 @@ "source": [ "chat = ChatLiteLLMRouter(\n", " router=litellm_router,\n", + " model_name=\"gpt-35-turbo\",\n", " streaming=True,\n", " verbose=True,\n", " callback_manager=CallbackManager([StreamingStdOutCallbackHandler()]),\n", @@ -209,7 +210,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.13" + "version": "3.11.9" } }, "nbformat": 4, diff --git a/docs/docs/integrations/chat/oci_generative_ai.ipynb b/docs/docs/integrations/chat/oci_generative_ai.ipynb index d5dc26cf25eba..49efa3a57522d 100644 --- a/docs/docs/integrations/chat/oci_generative_ai.ipynb +++ b/docs/docs/integrations/chat/oci_generative_ai.ipynb @@ -26,14 +26,14 @@ "## Overview\n", "### Integration details\n", "\n", - "| Class | Package | Local | Serializable | [JS support](https://js.langchain.com/docs/integrations/chat/oci_generative_ai) | Package downloads | Package latest |\n", - "| :--- | :--- | :---: | :---: | :---: | :---: | :---: |\n", - "| [ChatOCIGenAI](https://python.langchain.com/api_reference/community/chat_models/langchain_community.chat_models.oci_generative_ai.ChatOCIGenAI.html) | [langchain-community](https://python.langchain.com/api_reference/community/index.html) | ❌ | ❌ | ❌ | ![PyPI - Downloads](https://img.shields.io/pypi/dm/langchain-oci-generative-ai?style=flat-square&label=%20) | ![PyPI - Version](https://img.shields.io/pypi/v/langchain-oci-generative-ai?style=flat-square&label=%20) |\n", + "| Class | Package | Local | Serializable | [JS support](https://js.langchain.com/docs/integrations/chat/oci_generative_ai) |\n", + "| :--- | :--- | :---: | :---: | :---: |\n", + "| [ChatOCIGenAI](https://python.langchain.com/api_reference/community/chat_models/langchain_community.chat_models.oci_generative_ai.ChatOCIGenAI.html) | [langchain-community](https://python.langchain.com/api_reference/community/index.html) | ❌ | ❌ | ❌ |\n", "\n", "### Model features\n", - "| [Tool calling](/docs/how_to/tool_calling/) | [Structured output](/docs/how_to/structured_output/) | JSON mode | [Image input](/docs/how_to/multimodal_inputs/) | Audio input | Video input | [Token-level streaming](/docs/how_to/chat_streaming/) | Native async | [Token usage](/docs/how_to/chat_token_usage_tracking/) | [Logprobs](/docs/how_to/logprobs/) |\n", + "| [Tool calling](/docs/how_to/tool_calling/) | [Structured output](/docs/how_to/structured_output/) | [JSON mode](/docs/how_to/structured_output/#advanced-specifying-the-method-for-structuring-outputs) | [Image input](/docs/how_to/multimodal_inputs/) | Audio input | Video input | [Token-level streaming](/docs/how_to/chat_streaming/) | Native async | [Token usage](/docs/how_to/chat_token_usage_tracking/) | [Logprobs](/docs/how_to/logprobs/) |\n", "| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |\n", - "| ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | \n", + "| ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | \n", "\n", "## Setup\n", "\n", diff --git a/docs/docs/integrations/chat/ollama.ipynb b/docs/docs/integrations/chat/ollama.ipynb index 1d9b5de040354..daf179fe7f09c 100644 --- a/docs/docs/integrations/chat/ollama.ipynb +++ b/docs/docs/integrations/chat/ollama.ipynb @@ -34,8 +34,8 @@ "\n", "### Model features\n", "| [Tool calling](/docs/how_to/tool_calling/) | [Structured output](/docs/how_to/structured_output/) | JSON mode | [Image input](/docs/how_to/multimodal_inputs/) | Audio input | Video input | [Token-level streaming](/docs/how_to/chat_streaming/) | Native async | [Token usage](/docs/how_to/chat_token_usage_tracking/) | [Logprobs](/docs/how_to/logprobs/) |\n", - "| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |\n", - "| ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | \n", + "| :---: |:----------------------------------------------------:| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |\n", + "| ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ |\n", "\n", "## Setup\n", "\n", @@ -96,6 +96,20 @@ "%pip install -qU langchain-ollama" ] }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "Make sure you're using the latest Ollama version for structured outputs. Update by running:", + "id": "b18bd692076f7cf7" + }, + { + "metadata": {}, + "cell_type": "code", + "outputs": [], + "execution_count": null, + "source": "%pip install -U ollama", + "id": "b7a05cba95644c2e" + }, { "cell_type": "markdown", "id": "a38cde65-254d-4219-a441-068766c0d4b5", diff --git a/docs/docs/integrations/chat/sambastudio.ipynb b/docs/docs/integrations/chat/sambastudio.ipynb index 1ad3ff2805049..5527baa6417d8 100644 --- a/docs/docs/integrations/chat/sambastudio.ipynb +++ b/docs/docs/integrations/chat/sambastudio.ipynb @@ -34,7 +34,7 @@ "\n", "| [Tool calling](/docs/how_to/tool_calling) | [Structured output](/docs/how_to/structured_output/) | JSON mode | [Image input](/docs/how_to/multimodal_inputs/) | Audio input | Video input | [Token-level streaming](/docs/how_to/chat_streaming/) | Native async | [Token usage](/docs/how_to/chat_token_usage_tracking/) | [Logprobs](/docs/how_to/logprobs/) |\n", "| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |\n", - "| ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | \n", + "| ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | \n", "\n", "## Setup\n", "\n", @@ -119,20 +119,20 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from langchain_community.chat_models.sambanova import ChatSambaStudio\n", "\n", "llm = ChatSambaStudio(\n", - " model=\"Meta-Llama-3-70B-Instruct-4096\", # set if using a CoE endpoint\n", + " model=\"Meta-Llama-3-70B-Instruct-4096\", # set if using a Bundle endpoint\n", " max_tokens=1024,\n", " temperature=0.7,\n", " top_k=1,\n", " top_p=0.01,\n", " do_sample=True,\n", - " process_prompt=\"True\", # set if using a CoE endpoint\n", + " process_prompt=\"True\", # set if using a Bundle endpoint\n", ")" ] }, @@ -349,6 +349,134 @@ " print(chunk.content, end=\"\", flush=True)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Tool calling" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import datetime\n", + "\n", + "from langchain_core.messages import HumanMessage, SystemMessage, ToolMessage\n", + "from langchain_core.tools import tool\n", + "\n", + "\n", + "@tool\n", + "def get_time(kind: str = \"both\") -> str:\n", + " \"\"\"Returns current date, current time or both.\n", + " Args:\n", + " kind: date, time or both\n", + " \"\"\"\n", + " if kind == \"date\":\n", + " date = datetime.now().strftime(\"%m/%d/%Y\")\n", + " return f\"Current date: {date}\"\n", + " elif kind == \"time\":\n", + " time = datetime.now().strftime(\"%H:%M:%S\")\n", + " return f\"Current time: {time}\"\n", + " else:\n", + " date = datetime.now().strftime(\"%m/%d/%Y\")\n", + " time = datetime.now().strftime(\"%H:%M:%S\")\n", + " return f\"Current date: {date}, Current time: {time}\"\n", + "\n", + "\n", + "tools = [get_time]\n", + "\n", + "\n", + "def invoke_tools(tool_calls, messages):\n", + " available_functions = {tool.name: tool for tool in tools}\n", + " for tool_call in tool_calls:\n", + " selected_tool = available_functions[tool_call[\"name\"]]\n", + " tool_output = selected_tool.invoke(tool_call[\"args\"])\n", + " print(f\"Tool output: {tool_output}\")\n", + " messages.append(ToolMessage(tool_output, tool_call_id=tool_call[\"id\"]))\n", + " return messages" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "llm_with_tools = llm.bind_tools(tools=tools)\n", + "messages = [\n", + " HumanMessage(\n", + " content=\"I need to schedule a meeting for two weeks from today. Can you tell me the exact date of the meeting?\"\n", + " )\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Intermediate model response: [{'name': 'get_time', 'args': {'kind': 'date'}, 'id': 'call_4092d5dd21cd4eb494', 'type': 'tool_call'}]\n", + "Tool output: Current date: 11/07/2024\n", + "final response: The meeting will be exactly two weeks from today, which would be 25/07/2024.\n" + ] + } + ], + "source": [ + "response = llm_with_tools.invoke(messages)\n", + "while len(response.tool_calls) > 0:\n", + " print(f\"Intermediate model response: {response.tool_calls}\")\n", + " messages.append(response)\n", + " messages = invoke_tools(response.tool_calls, messages)\n", + "response = llm_with_tools.invoke(messages)\n", + "\n", + "print(f\"final response: {response.content}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Structured Outputs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Joke(setup='Why did the cat join a band?', punchline='Because it wanted to be the purr-cussionist!')" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from pydantic import BaseModel, Field\n", + "\n", + "\n", + "class Joke(BaseModel):\n", + " \"\"\"Joke to tell user.\"\"\"\n", + "\n", + " setup: str = Field(description=\"The setup of the joke\")\n", + " punchline: str = Field(description=\"The punchline to the joke\")\n", + "\n", + "\n", + "structured_llm = llm.with_structured_output(Joke)\n", + "\n", + "structured_llm.invoke(\"Tell me a joke about cats\")" + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/docs/docs/integrations/document_loaders/confluence.ipynb b/docs/docs/integrations/document_loaders/confluence.ipynb index c7f04f0bae558..d325e129d08d7 100644 --- a/docs/docs/integrations/document_loaders/confluence.ipynb +++ b/docs/docs/integrations/document_loaders/confluence.ipynb @@ -11,7 +11,7 @@ "A loader for `Confluence` pages.\n", "\n", "\n", - "This currently supports `username/api_key`, `Oauth2 login`. Additionally, on-prem installations also support `token` authentication. \n", + "This currently supports `username/api_key`, `Oauth2 login`, `cookies`. Additionally, on-prem installations also support `token` authentication. \n", "\n", "\n", "Specify a list `page_id`-s and/or `space_key` to load in the corresponding pages into Document objects, if both are specified the union of both sets will be returned.\n", diff --git a/docs/docs/integrations/document_loaders/yt_dlp.ipynb b/docs/docs/integrations/document_loaders/yt_dlp.ipynb new file mode 100644 index 0000000000000..c27685d90a0c0 --- /dev/null +++ b/docs/docs/integrations/document_loaders/yt_dlp.ipynb @@ -0,0 +1,149 @@ +{ + "cells": [ + { + "cell_type": "raw", + "metadata": {}, + "source": [ + "---\n", + "sidebar_label: YoutubeLoaderDL\n", + "---" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# YoutubeLoaderDL\n", + "\n", + "Loader for Youtube leveraging the `yt-dlp` library.\n", + "\n", + "This package implements a [document loader](/docs/concepts/document_loaders/) for Youtube. In contrast to the [YoutubeLoader](https://python.langchain.com/api_reference/community/document_loaders/langchain_community.document_loaders.youtube.YoutubeLoader.html) of `langchain-community`, which relies on `pytube`, `YoutubeLoaderDL` is able to fetch YouTube metadata. `langchain-yt-dlp` leverages the robust `yt-dlp` library, providing a more reliable and feature-rich YouTube document loader.\n", + "\n", + "## Overview\n", + "### Integration details\n", + "\n", + "| Class | Package | Local | Serializable | JS Support |\n", + "| :--- | :--- | :---: | :---: | :---: |\n", + "| YoutubeLoader | langchain-yt-dlp | ✅ | ✅ | ❌ |\n", + "\n", + "## Setup\n", + "\n", + "### Installation\n", + "\n", + "```bash\n", + "pip install langchain-yt-dlp\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Initialization" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "from langchain_yt_dlp.youtube_loader import YoutubeLoaderDL\n", + "\n", + "# Basic transcript loading\n", + "loader = YoutubeLoaderDL.from_youtube_url(\n", + " \"https://www.youtube.com/watch?v=dQw4w9WgXcQ\", add_video_info=True\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Load" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "documents = loader.load()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'source': 'dQw4w9WgXcQ',\n", + " 'title': 'Rick Astley - Never Gonna Give You Up (Official Music Video)',\n", + " 'description': 'The official video for “Never Gonna Give You Up” by Rick Astley. \\n\\nNever: The Autobiography 📚 OUT NOW! \\nFollow this link to get your copy and listen to Rick’s ‘Never’ playlist ❤️ #RickAstleyNever\\nhttps://linktr.ee/rickastleynever\\n\\n“Never Gonna Give You Up” was a global smash on its release in July 1987, topping the charts in 25 countries including Rick’s native UK and the US Billboard Hot 100. It also won the Brit Award for Best single in 1988. Stock Aitken and Waterman wrote and produced the track which was the lead-off single and lead track from Rick’s debut LP “Whenever You Need Somebody”. The album was itself a UK number one and would go on to sell over 15 million copies worldwide.\\n\\nThe legendary video was directed by Simon West – who later went on to make Hollywood blockbusters such as Con Air, Lara Croft – Tomb Raider and The Expendables 2. The video passed the 1bn YouTube views milestone on 28 July 2021.\\n\\nSubscribe to the official Rick Astley YouTube channel: https://RickAstley.lnk.to/YTSubID\\n\\nFollow Rick Astley:\\nFacebook: https://RickAstley.lnk.to/FBFollowID \\nTwitter: https://RickAstley.lnk.to/TwitterID \\nInstagram: https://RickAstley.lnk.to/InstagramID \\nWebsite: https://RickAstley.lnk.to/storeID \\nTikTok: https://RickAstley.lnk.to/TikTokID\\n\\nListen to Rick Astley:\\nSpotify: https://RickAstley.lnk.to/SpotifyID \\nApple Music: https://RickAstley.lnk.to/AppleMusicID \\nAmazon Music: https://RickAstley.lnk.to/AmazonMusicID \\nDeezer: https://RickAstley.lnk.to/DeezerID \\n\\nLyrics:\\nWe’re no strangers to love\\nYou know the rules and so do I\\nA full commitment’s what I’m thinking of\\nYou wouldn’t get this from any other guy\\n\\nI just wanna tell you how I’m feeling\\nGotta make you understand\\n\\nNever gonna give you up\\nNever gonna let you down\\nNever gonna run around and desert you\\nNever gonna make you cry\\nNever gonna say goodbye\\nNever gonna tell a lie and hurt you\\n\\nWe’ve known each other for so long\\nYour heart’s been aching but you’re too shy to say it\\nInside we both know what’s been going on\\nWe know the game and we’re gonna play it\\n\\nAnd if you ask me how I’m feeling\\nDon’t tell me you’re too blind to see\\n\\nNever gonna give you up\\nNever gonna let you down\\nNever gonna run around and desert you\\nNever gonna make you cry\\nNever gonna say goodbye\\nNever gonna tell a lie and hurt you\\n\\n#RickAstley #NeverGonnaGiveYouUp #WheneverYouNeedSomebody #OfficialMusicVideo',\n", + " 'view_count': 1603360806,\n", + " 'publish_date': datetime.datetime(2009, 10, 25, 0, 0),\n", + " 'length': 212,\n", + " 'author': 'Rick Astley',\n", + " 'channel_id': 'UCuAXFkgsw1L7xaCfnd5JJOw',\n", + " 'webpage_url': 'https://www.youtube.com/watch?v=dQw4w9WgXcQ'}" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "documents[0].metadata" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Lazy Load" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- No lazy loading is implemented" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## API reference:\n", + "\n", + "- [Github](https://github.com/aqib0770/langchain-yt-dlp)\n", + "- [PyPi](https://pypi.org/project/langchain-yt-dlp/)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.4" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/docs/docs/integrations/llm_caching.ipynb b/docs/docs/integrations/llm_caching.ipynb index 4ba1901613ac2..3788ad5bfde39 100644 --- a/docs/docs/integrations/llm_caching.ipynb +++ b/docs/docs/integrations/llm_caching.ipynb @@ -12,6 +12,16 @@ "First, let's install some dependencies" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "bedbf252-4ea5-4eea-a3dc-d18ccc84aca3", + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -qU langchain-openai langchain-community" + ] + }, { "cell_type": "code", "execution_count": 2, @@ -19,8 +29,6 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install -qU langchain-openai langchain-community\n", - "\n", "import os\n", "from getpass import getpass\n", "\n", @@ -55,7 +63,7 @@ "tags": [] }, "source": [ - "## `In Memory` Cache" + "## `In Memory` cache" ] }, { @@ -139,7 +147,7 @@ "tags": [] }, "source": [ - "## `SQLite` Cache" + "## `SQLite` cache" ] }, { @@ -234,7 +242,7 @@ "id": "e71273ab", "metadata": {}, "source": [ - "## `Upstash Redis` Cache" + "## `Upstash Redis` caches" ] }, { @@ -242,7 +250,7 @@ "id": "f10dabef", "metadata": {}, "source": [ - "### Standard Cache\n", + "### Standard cache\n", "Use [Upstash Redis](https://upstash.com) to cache prompts and responses with a serverless HTTP API." ] }, @@ -340,7 +348,8 @@ "id": "b29dd776", "metadata": {}, "source": [ - "### Semantic Cache\n", + "### Semantic cache\n", + "\n", "Use [Upstash Vector](https://upstash.com/docs/vector/overall/whatisvector) to do a semantic similarity search and cache the most similar response in the database. The vectorization is automatically done by the selected embedding model while creating Upstash Vector database. " ] }, @@ -454,11 +463,10 @@ "cell_type": "markdown", "id": "278ad7ae", "metadata": { - "jp-MarkdownHeadingCollapsed": true, "tags": [] }, "source": [ - "## `Redis` Cache\n", + "## `Redis` caches\n", "\n", "See the main [Redis cache docs](/docs/integrations/caches/redis_llm_caching/) for detail." ] @@ -468,7 +476,7 @@ "id": "c5c9a4d5", "metadata": {}, "source": [ - "### Standard Cache\n", + "### Standard cache\n", "Use [Redis](/docs/integrations/providers/redis) to cache prompts and responses." ] }, @@ -564,7 +572,7 @@ "id": "82be23f6", "metadata": {}, "source": [ - "### Semantic Cache\n", + "### Semantic cache\n", "Use [Redis](/docs/integrations/providers/redis) to cache prompts and responses and evaluate hits based on semantic similarity." ] }, @@ -660,7 +668,6 @@ "cell_type": "markdown", "id": "684eab55", "metadata": { - "jp-MarkdownHeadingCollapsed": true, "tags": [] }, "source": [ @@ -905,7 +912,7 @@ "id": "9b2b2777", "metadata": {}, "source": [ - "## `MongoDB Atlas` Cache\n", + "## `MongoDB Atlas` caches\n", "\n", "[MongoDB Atlas](https://www.mongodb.com/docs/atlas/) is a fully-managed cloud database available in AWS, Azure, and GCP. It has native support for \n", "Vector Search on the MongoDB document data.\n", @@ -917,8 +924,9 @@ "id": "ecdc2a0a", "metadata": {}, "source": [ - "### `MongoDBCache`\n", - "An abstraction to store a simple cache in MongoDB. This does not use Semantic Caching, nor does it require an index to be made on the collection before generation.\n", + "### Standard cache\n", + "\n", + "Standard cache is a simple cache in MongoDB. It does not use Semantic Caching, nor does it require an index to be made on the collection before generation.\n", "\n", "To import this cache, first install the required dependency:\n", "\n", @@ -950,8 +958,9 @@ "```\n", "\n", "\n", - "### `MongoDBAtlasSemanticCache`\n", - "Semantic caching allows users to retrieve cached prompts based on semantic similarity between the user input and previously cached results. Under the hood it blends MongoDBAtlas as both a cache and a vectorstore.\n", + "### Semantic cache\n", + "\n", + "Semantic caching allows retrieval of cached prompts based on semantic similarity between the user input and previously cached results. Under the hood, it blends MongoDBAtlas as both a cache and a vectorstore.\n", "The MongoDBAtlasSemanticCache inherits from `MongoDBAtlasVectorSearch` and needs an Atlas Vector Search Index defined to work. Please look at the [usage example](/docs/integrations/vectorstores/mongodb_atlas) on how to set up the index.\n", "\n", "To import this cache:\n", @@ -985,14 +994,13 @@ "cell_type": "markdown", "id": "726fe754", "metadata": { - "jp-MarkdownHeadingCollapsed": true, "tags": [] }, "source": [ - "## `Momento` Cache\n", + "## `Momento` cache\n", "Use [Momento](/docs/integrations/providers/momento) to cache prompts and responses.\n", "\n", - "Requires momento to use, uncomment below to install:" + "Requires installing the `momento` package:" ] }, { @@ -1096,13 +1104,14 @@ "cell_type": "markdown", "id": "934943dc", "metadata": { - "jp-MarkdownHeadingCollapsed": true, "tags": [] }, "source": [ - "## `SQLAlchemy` Cache\n", + "## `SQLAlchemy` cache\n", + "\n", + "You can use `SQLAlchemyCache` to cache with any SQL database supported by `SQLAlchemy`.\n", "\n", - "You can use `SQLAlchemyCache` to cache with any SQL database supported by `SQLAlchemy`." + "### Standard cache" ] }, { @@ -1112,11 +1121,11 @@ "metadata": {}, "outputs": [], "source": [ - "# from langchain.cache import SQLAlchemyCache\n", - "# from sqlalchemy import create_engine\n", + "from langchain.cache import SQLAlchemyCache\n", + "from sqlalchemy import create_engine\n", "\n", - "# engine = create_engine(\"postgresql://postgres:postgres@localhost:5432/postgres\")\n", - "# set_llm_cache(SQLAlchemyCache(engine))" + "engine = create_engine(\"postgresql://postgres:postgres@localhost:5432/postgres\")\n", + "set_llm_cache(SQLAlchemyCache(engine))" ] }, { @@ -1124,7 +1133,9 @@ "id": "0959d640", "metadata": {}, "source": [ - "### Custom SQLAlchemy Schemas" + "### Custom SQLAlchemy schemas\n", + "\n", + "You can define your own declarative `SQLAlchemyCache` child class to customize the schema used for caching. For example, to support high-speed fulltext prompt indexing with `Postgres`, use:" ] }, { @@ -1134,8 +1145,6 @@ "metadata": {}, "outputs": [], "source": [ - "# You can define your own declarative SQLAlchemyCache child class to customize the schema used for caching. For example, to support high-speed fulltext prompt indexing with Postgres, use:\n", - "\n", "from langchain_community.cache import SQLAlchemyCache\n", "from sqlalchemy import Column, Computed, Index, Integer, Sequence, String, create_engine\n", "from sqlalchemy.ext.declarative import declarative_base\n", @@ -1185,7 +1194,7 @@ "id": "6cf6acb4-1bc4-4c4b-9325-2420c17e5e2b", "metadata": {}, "source": [ - "### Required dependency" + "Required dependency:" ] }, { @@ -1203,7 +1212,7 @@ "id": "a4a6725d", "metadata": {}, "source": [ - "### Connect to the DB\n", + "### Connecting to the DB\n", "\n", "The Cassandra caches shown in this page can be used with Cassandra as well as other derived databases, such as Astra DB, which use the CQL (Cassandra Query Language) protocol.\n", "\n", @@ -1217,7 +1226,7 @@ "id": "15735abe-2567-43ce-aa91-f253b33b5a88", "metadata": {}, "source": [ - "#### Connecting to a Cassandra cluster\n", + "#### to a Cassandra cluster\n", "\n", "You first need to create a `cassandra.cluster.Session` object, as described in the [Cassandra driver documentation](https://docs.datastax.com/en/developer/python-driver/latest/api/cassandra/cluster/#module-cassandra.cluster). The details vary (e.g. with network settings and authentication), but this might be something like:" ] @@ -1270,7 +1279,7 @@ "id": "2cc7ba29-8f84-4fbf-aaf7-3daa1be7e7b0", "metadata": {}, "source": [ - "#### Connecting to Astra DB through CQL\n", + "#### to Astra DB through CQL\n", "\n", "In this case you initialize CassIO with the following connection parameters:\n", "\n", @@ -1329,7 +1338,7 @@ "id": "8665664a", "metadata": {}, "source": [ - "### Cassandra: Exact cache\n", + "### Standard cache\n", "\n", "This will avoid invoking the LLM when the supplied prompt is _exactly_ the same as one encountered already:" ] @@ -1400,7 +1409,7 @@ "id": "8fc4d017", "metadata": {}, "source": [ - "### Cassandra: Semantic cache\n", + "### Semantic cache\n", "\n", "This cache will do a semantic similarity search and return a hit if it finds a cached entry that is similar enough, For this, you need to provide an `Embeddings` instance of your choice." ] @@ -1488,9 +1497,9 @@ "id": "55dc84b3-37cb-4f19-b175-40e18e06f83f", "metadata": {}, "source": [ - "#### Attribution statement\n", + "**Attribution statement:**\n", "\n", - ">Apache Cassandra, Cassandra and Apache are either registered trademarks or trademarks of the [Apache Software Foundation](http://www.apache.org/) in the United States and/or other countries." + ">`Apache Cassandra`, `Cassandra` and `Apache` are either registered trademarks or trademarks of the [Apache Software Foundation](http://www.apache.org/) in the United States and/or other countries." ] }, { @@ -1498,7 +1507,7 @@ "id": "8712f8fc-bb89-4164-beb9-c672778bbd91", "metadata": {}, "source": [ - "## `Astra DB` Caches" + "## `Astra DB` caches" ] }, { @@ -1543,7 +1552,7 @@ "id": "ee6d587f-4b7c-43f4-9e90-5129c842a143", "metadata": {}, "source": [ - "### Astra DB exact LLM cache\n", + "### Standard cache\n", "\n", "This will avoid invoking the LLM when the supplied prompt is _exactly_ the same as one encountered already:" ] @@ -1619,7 +1628,7 @@ "id": "524b94fa-6162-4880-884d-d008749d14e2", "metadata": {}, "source": [ - "### Astra DB Semantic cache\n", + "### Semantic cache\n", "\n", "This cache will do a semantic similarity search and return a hit if it finds a cached entry that is similar enough, For this, you need to provide an `Embeddings` instance of your choice." ] @@ -1713,7 +1722,7 @@ } }, "source": [ - "## Azure Cosmos DB Semantic Cache\n", + "## `Azure Cosmos DB` semantic cache\n", "\n", "You can use this integrated [vector database](https://learn.microsoft.com/en-us/azure/cosmos-db/vector-database) for caching." ] @@ -1848,18 +1857,162 @@ "output_type": "execute_result" } ], + "source": [ + "%%time\n", + "# The second time it is, so it goes faster\n", + "llm.invoke(\"Tell me a joke\")" + ] + }, + { + "cell_type": "markdown", + "id": "235ff73bf7143f13", + "metadata": {}, + "source": [ + "## `Azure Cosmos DB NoSql` semantic cache\n", + "\n", + "You can use this integrated [vector database](https://learn.microsoft.com/en-us/azure/cosmos-db/vector-database) for caching." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41fea5aa7b2153ca", + "metadata": { + "ExecuteTime": { + "end_time": "2024-12-06T00:55:38.648972Z", + "start_time": "2024-12-06T00:55:38.290541Z" + } + }, + "outputs": [], + "source": [ + "from typing import Any, Dict\n", + "\n", + "from azure.cosmos import CosmosClient, PartitionKey\n", + "from langchain_community.cache import AzureCosmosDBNoSqlSemanticCache\n", + "from langchain_openai import OpenAIEmbeddings\n", + "\n", + "HOST = \"COSMOS_DB_URI\"\n", + "KEY = \"COSMOS_DB_KEY\"\n", + "\n", + "cosmos_client = CosmosClient(HOST, KEY)\n", + "\n", + "\n", + "def get_vector_indexing_policy() -> dict:\n", + " return {\n", + " \"indexingMode\": \"consistent\",\n", + " \"includedPaths\": [{\"path\": \"/*\"}],\n", + " \"excludedPaths\": [{\"path\": '/\"_etag\"/?'}],\n", + " \"vectorIndexes\": [{\"path\": \"/embedding\", \"type\": \"diskANN\"}],\n", + " }\n", + "\n", + "\n", + "def get_vector_embedding_policy() -> dict:\n", + " return {\n", + " \"vectorEmbeddings\": [\n", + " {\n", + " \"path\": \"/embedding\",\n", + " \"dataType\": \"float32\",\n", + " \"dimensions\": 1536,\n", + " \"distanceFunction\": \"cosine\",\n", + " }\n", + " ]\n", + " }\n", + "\n", + "\n", + "cosmos_container_properties_test = {\"partition_key\": PartitionKey(path=\"/id\")}\n", + "cosmos_database_properties_test: Dict[str, Any] = {}\n", + "\n", + "set_llm_cache(\n", + " AzureCosmosDBNoSqlSemanticCache(\n", + " cosmos_client=cosmos_client,\n", + " embedding=OpenAIEmbeddings(),\n", + " vector_embedding_policy=get_vector_embedding_policy(),\n", + " indexing_policy=get_vector_indexing_policy(),\n", + " cosmos_container_properties=cosmos_container_properties_test,\n", + " cosmos_database_properties=cosmos_database_properties_test,\n", + " )\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "1e1cd93819921bf6", + "metadata": { + "ExecuteTime": { + "end_time": "2024-12-06T00:55:44.513080Z", + "start_time": "2024-12-06T00:55:41.353843Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 374 ms, sys: 34.2 ms, total: 408 ms\n", + "Wall time: 3.15 s\n" + ] + }, + { + "data": { + "text/plain": [ + "\"\\n\\nWhy couldn't the bicycle stand up by itself? Because it was two-tired!\"" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "%%time\n", "# The first time, it is not yet in cache, so it should take longer\n", "llm.invoke(\"Tell me a joke\")" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "576ce24c1244812a", + "metadata": { + "ExecuteTime": { + "end_time": "2024-12-06T00:55:50.925865Z", + "start_time": "2024-12-06T00:55:50.548520Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CPU times: user 17.7 ms, sys: 2.88 ms, total: 20.6 ms\n", + "Wall time: 373 ms\n" + ] + }, + { + "data": { + "text/plain": [ + "\"\\n\\nWhy couldn't the bicycle stand up by itself? Because it was two-tired!\"" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%%time\n", + "# The second time it is, so it goes faster\n", + "llm.invoke(\"Tell me a joke\")" + ] + }, { "cell_type": "markdown", "id": "306ff47b", "metadata": {}, "source": [ - "## `Elasticsearch` Cache\n", + "## `Elasticsearch` caches\n", + "\n", "A caching layer for LLMs that uses Elasticsearch.\n", "\n", "First install the LangChain integration with Elasticsearch." @@ -1880,6 +2033,8 @@ "id": "9e70b0a0", "metadata": {}, "source": [ + "### Standard cache\n", + "\n", "Use the class `ElasticsearchCache`.\n", "\n", "Simple example:" @@ -1987,6 +2142,26 @@ "please only make additive modifications, keeping the base mapping intact." ] }, + { + "cell_type": "markdown", + "id": "3dc15b3c-8793-432e-98c3-d2726d497a5a", + "metadata": {}, + "source": [ + "### Embedding cache\n", + "\n", + "An Elasticsearch store for caching embeddings." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b0ae5bd3-517d-470f-9b44-14d9359e6940", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain_elasticsearch import ElasticsearchEmbeddingsCache" + ] + }, { "cell_type": "markdown", "id": "0c69d84d", @@ -1994,8 +2169,9 @@ "tags": [] }, "source": [ - "## Optional Caching\n", - "You can also turn off caching for specific LLMs should you choose. In the example below, even though global caching is enabled, we turn it off for a specific LLM" + "## LLM-specific optional caching\n", + "\n", + "You can also turn off caching for specific LLMs. In the example below, even though global caching is enabled, we turn it off for a specific LLM." ] }, { @@ -2075,7 +2251,8 @@ "tags": [] }, "source": [ - "## Optional Caching in Chains\n", + "## Optional caching in Chains\n", + "\n", "You can also turn off caching for particular nodes in chains. Note that because of certain interfaces, its often easier to construct the chain first, and then edit the LLM afterwards.\n", "\n", "As an example, we will load a summarizer map-reduce chain. We will cache results for the map-step, but then not freeze it for the combine step." @@ -2242,7 +2419,7 @@ "id": "9ecfa565038eff71", "metadata": {}, "source": [ - "## OpenSearch Semantic Cache\n", + "## `OpenSearch` semantic cache\n", "Use [OpenSearch](https://python.langchain.com/docs/integrations/vectorstores/opensearch/) as a semantic cache to cache prompts and responses and evaluate hits based on semantic similarity." ] }, @@ -2346,7 +2523,8 @@ "id": "2ac1a8c7", "metadata": {}, "source": [ - "## SingleStoreDB Semantic Cache\n", + "## `SingleStoreDB` semantic cache\n", + "\n", "You can use [SingleStoreDB](https://python.langchain.com/docs/integrations/vectorstores/singlestoredb/) as a semantic cache to cache prompts and responses." ] }, @@ -2373,7 +2551,7 @@ "id": "7e6b9b1a", "metadata": {}, "source": [ - "## `Memcached` Cache\n", + "## `Memcached` cache\n", "You can use [Memcached](https://www.memcached.org/) as a cache to cache prompts and responses through [pymemcache](https://github.com/pinterest/pymemcache).\n", "\n", "This cache requires the pymemcache dependency to be installed:" @@ -2469,7 +2647,7 @@ "id": "7019c991-0101-4f9c-b212-5729a5471293", "metadata": {}, "source": [ - "## Couchbase Caches\n", + "## `Couchbase` caches\n", "\n", "Use [Couchbase](https://couchbase.com/) as a cache for prompts and responses." ] @@ -2479,7 +2657,7 @@ "id": "d6aac680-ba32-4c19-8864-6471cf0e7d5a", "metadata": {}, "source": [ - "### Couchbase Cache\n", + "### Standard cache\n", "\n", "The standard cache that looks for an exact match of the user prompt." ] @@ -2613,7 +2791,7 @@ "id": "1dca39d8-233a-45ba-ad7d-0920dfbc4a50", "metadata": {}, "source": [ - "### Specifying a Time to Live (TTL) for the Cached entries\n", + "#### Time to Live (TTL) for the cached entries\n", "The Cached documents can be deleted after a specified time automatically by specifying a `ttl` parameter along with the initialization of the Cache." ] }, @@ -2642,7 +2820,7 @@ "id": "43626f33-d184-4260-b641-c9341cef5842", "metadata": {}, "source": [ - "### Couchbase Semantic Cache\n", + "### Semantic cache\n", "Semantic caching allows users to retrieve cached prompts based on semantic similarity between the user input and previously cached inputs. Under the hood it uses Couchbase as both a cache and a vectorstore. This needs an appropriate Vector Search Index defined to work. Please look at the usage example on how to set up the index." ] }, @@ -2685,7 +2863,9 @@ "- The search index for the semantic cache needs to be defined before using the semantic cache. \n", "- The optional parameter, `score_threshold` in the Semantic Cache that you can use to tune the results of the semantic search.\n", "\n", - "### How to Import an Index to the Full Text Search service?\n", + "#### Index to the Full Text Search service\n", + "\n", + "How to Import an Index to the Full Text Search service?\n", " - [Couchbase Server](https://docs.couchbase.com/server/current/search/import-search-index.html)\n", " - Click on Search -> Add Index -> Import\n", " - Copy the following Index definition in the Import screen\n", @@ -2695,7 +2875,8 @@ " - Import the file in Capella using the instructions in the documentation.\n", " - Click on Create Index to create the index.\n", "\n", - "#### Example index for the vector search. \n", + "**Example index for the vector search:**\n", + "\n", " ```\n", " {\n", " \"type\": \"fulltext-index\",\n", @@ -2855,7 +3036,8 @@ "id": "f6f674fa-70b5-4cf9-a208-992aad2c3c89", "metadata": {}, "source": [ - "### Specifying a Time to Live (TTL) for the Cached entries\n", + "#### Time to Live (TTL) for the cached entries\n", + "\n", "The Cached documents can be deleted after a specified time automatically by specifying a `ttl` parameter along with the initialization of the Cache." ] }, @@ -2913,10 +3095,10 @@ "source": [ "**Cache** classes are implemented by inheriting the [BaseCache](https://python.langchain.com/api_reference/core/caches/langchain_core.caches.BaseCache.html) class.\n", "\n", - "This table lists all 21 derived classes with links to the API Reference.\n", + "This table lists all derived classes with links to the API Reference.\n", "\n", "\n", - "| Namespace 🔻 | Class |\n", + "| Namespace | Class 🔻 |\n", "|------------|---------|\n", "| langchain_astradb.cache | [AstraDBCache](https://python.langchain.com/api_reference/astradb/cache/langchain_astradb.cache.AstraDBCache.html) |\n", "| langchain_astradb.cache | [AstraDBSemanticCache](https://python.langchain.com/api_reference/astradb/cache/langchain_astradb.cache.AstraDBSemanticCache.html) |\n", @@ -2925,22 +3107,31 @@ "| langchain_community.cache | [AzureCosmosDBSemanticCache](https://python.langchain.com/api_reference/community/cache/langchain_community.cache.AzureCosmosDBSemanticCache.html) |\n", "| langchain_community.cache | [CassandraCache](https://python.langchain.com/api_reference/community/cache/langchain_community.cache.CassandraCache.html) |\n", "| langchain_community.cache | [CassandraSemanticCache](https://python.langchain.com/api_reference/community/cache/langchain_community.cache.CassandraSemanticCache.html) |\n", + "| langchain_couchbase.cache | [CouchbaseCache](https://python.langchain.com/api_reference/couchbase/cache/langchain_couchbase.cache.CouchbaseCache.html) |\n", + "| langchain_couchbase.cache | [CouchbaseSemanticCache](https://python.langchain.com/api_reference/couchbase/cache/langchain_couchbase.cache.CouchbaseSemanticCache.html) |\n", + "| langchain_elasticsearch.cache | [ElasticsearchCache](https://python.langchain.com/api_reference/elasticsearch/cache/langchain_elasticsearch.cache.AsyncElasticsearchCache.html) |\n", + "| langchain_elasticsearch.cache | [ElasticsearchEmbeddingsCache](https://python.langchain.com/api_reference/elasticsearch/cache/langchain_elasticsearch.cache.AsyncElasticsearchEmbeddingsCache.html) |\n", "| langchain_community.cache | [GPTCache](https://python.langchain.com/api_reference/community/cache/langchain_community.cache.GPTCache.html) |\n", + "| langchain_core.caches | [InMemoryCache](https://python.langchain.com/api_reference/core/caches/langchain_core.caches.InMemoryCache.html) |\n", "| langchain_community.cache | [InMemoryCache](https://python.langchain.com/api_reference/community/cache/langchain_community.cache.InMemoryCache.html) |\n", "| langchain_community.cache | [MomentoCache](https://python.langchain.com/api_reference/community/cache/langchain_community.cache.MomentoCache.html) |\n", + "| langchain_mongodb.cache | [MongoDBAtlasSemanticCache](https://python.langchain.com/api_reference/mongodb/cache/langchain_mongodb.cache.MongoDBAtlasSemanticCache.html) |\n", + "| langchain_mongodb.cache | [MongoDBCache](https://python.langchain.com/api_reference/mongodb/cache/langchain_mongodb.cache.MongoDBCache.html) |\n", "| langchain_community.cache | [OpenSearchSemanticCache](https://python.langchain.com/api_reference/community/cache/langchain_community.cache.OpenSearchSemanticCache.html) |\n", "| langchain_community.cache | [RedisSemanticCache](https://python.langchain.com/api_reference/community/cache/langchain_community.cache.RedisSemanticCache.html) |\n", "| langchain_community.cache | [SingleStoreDBSemanticCache](https://python.langchain.com/api_reference/community/cache/langchain_community.cache.SingleStoreDBSemanticCache.html) |\n", "| langchain_community.cache | [SQLAlchemyCache](https://python.langchain.com/api_reference/community/cache/langchain_community.cache.SQLAlchemyCache.html) |\n", "| langchain_community.cache | [SQLAlchemyMd5Cache](https://python.langchain.com/api_reference/community/cache/langchain_community.cache.SQLAlchemyMd5Cache.html) |\n", - "| langchain_community.cache | [UpstashRedisCache](https://python.langchain.com/api_reference/community/cache/langchain_community.cache.UpstashRedisCache.html) |\n", - "| langchain_core.caches | [InMemoryCache](https://python.langchain.com/api_reference/core/caches/langchain_core.caches.InMemoryCache.html) |\n", - "| langchain_elasticsearch.cache | [ElasticsearchCache](https://python.langchain.com/api_reference/elasticsearch/cache/langchain_elasticsearch.cache.ElasticsearchCache.html) |\n", - "| langchain_mongodb.cache | [MongoDBAtlasSemanticCache](https://python.langchain.com/api_reference/mongodb/cache/langchain_mongodb.cache.MongoDBAtlasSemanticCache.html) |\n", - "| langchain_mongodb.cache | [MongoDBCache](https://python.langchain.com/api_reference/mongodb/cache/langchain_mongodb.cache.MongoDBCache.html) |\n", - "| langchain_couchbase.cache | [CouchbaseCache](https://python.langchain.com/api_reference/couchbase/cache/langchain_couchbase.cache.CouchbaseCache.html) |\n", - "| langchain_couchbase.cache | [CouchbaseSemanticCache](https://python.langchain.com/api_reference/couchbase/cache/langchain_couchbase.cache.CouchbaseSemanticCache.html) |\n" + "| langchain_community.cache | [UpstashRedisCache](https://python.langchain.com/api_reference/community/cache/langchain_community.cache.UpstashRedisCache.html) |\n" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1ef1a2d4-da2e-4fb1-aae4-ffc4aef6c3ad", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { @@ -2959,7 +3150,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.13" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/docs/docs/integrations/llms/openllm.ipynb b/docs/docs/integrations/llms/openllm.ipynb index 279ba6435bce9..8a24ef8d0a25e 100644 --- a/docs/docs/integrations/llms/openllm.ipynb +++ b/docs/docs/integrations/llms/openllm.ipynb @@ -7,7 +7,14 @@ "source": [ "# OpenLLM\n", "\n", - "[🦾 OpenLLM](https://github.com/bentoml/OpenLLM) is an open platform for operating large language models (LLMs) in production. It enables developers to easily run inference with any open-source LLMs, deploy to the cloud or on-premises, and build powerful AI apps." + "[🦾 OpenLLM](https://github.com/bentoml/OpenLLM) lets developers run any **open-source LLMs** as **OpenAI-compatible API** endpoints with **a single command**.\n", + "\n", + "- 🔬 Build for fast and production usages\n", + "- 🚂 Support llama3, qwen2, gemma, etc, and many **quantized** versions [full list](https://github.com/bentoml/openllm-models)\n", + "- ⛓️ OpenAI-compatible API\n", + "- 💬 Built-in ChatGPT like UI\n", + "- 🔥 Accelerated LLM decoding with state-of-the-art inference backends\n", + "- 🌥️ Ready for enterprise-grade cloud deployment (Kubernetes, Docker and BentoCloud)" ] }, { @@ -37,10 +44,10 @@ "source": [ "## Launch OpenLLM server locally\n", "\n", - "To start an LLM server, use `openllm start` command. For example, to start a dolly-v2 server, run the following command from a terminal:\n", + "To start an LLM server, use `openllm hello` command:\n", "\n", "```bash\n", - "openllm start dolly-v2\n", + "openllm hello\n", "```\n", "\n", "\n", @@ -57,83 +64,18 @@ "from langchain_community.llms import OpenLLM\n", "\n", "server_url = \"http://localhost:3000\" # Replace with remote host if you are running on a remote server\n", - "llm = OpenLLM(server_url=server_url)" - ] - }, - { - "cell_type": "markdown", - "id": "4f830f9d", - "metadata": {}, - "source": [ - "### Optional: Local LLM Inference\n", - "\n", - "You may also choose to initialize an LLM managed by OpenLLM locally from current process. This is useful for development purpose and allows developers to quickly try out different types of LLMs.\n", - "\n", - "When moving LLM applications to production, we recommend deploying the OpenLLM server separately and access via the `server_url` option demonstrated above.\n", - "\n", - "To load an LLM locally via the LangChain wrapper:" + "llm = OpenLLM(base_url=server_url, api_key=\"na\")" ] }, { "cell_type": "code", "execution_count": null, - "id": "82c392b6", + "id": "56cb4bc0", "metadata": {}, "outputs": [], "source": [ - "from langchain_community.llms import OpenLLM\n", - "\n", - "llm = OpenLLM(\n", - " model_name=\"dolly-v2\",\n", - " model_id=\"databricks/dolly-v2-3b\",\n", - " temperature=0.94,\n", - " repetition_penalty=1.2,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "f15ebe0d", - "metadata": {}, - "source": [ - "### Integrate with a LLMChain" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "8b02a97a", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "iLkb\n" - ] - } - ], - "source": [ - "from langchain.chains import LLMChain\n", - "from langchain_core.prompts import PromptTemplate\n", - "\n", - "template = \"What is a good name for a company that makes {product}?\"\n", - "\n", - "prompt = PromptTemplate.from_template(template)\n", - "\n", - "llm_chain = LLMChain(prompt=prompt, llm=llm)\n", - "\n", - "generated = llm_chain.run(product=\"mechanical keyboard\")\n", - "print(generated)" + "llm(\"To build a LLM from scratch, the following are the steps:\")" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "56cb4bc0", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -152,7 +94,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.10" + "version": "3.11.9" } }, "nbformat": 4, diff --git a/docs/docs/integrations/providers/alibaba_cloud.mdx b/docs/docs/integrations/providers/alibaba_cloud.mdx index 74c3045a6424f..baf0fe9fe0768 100644 --- a/docs/docs/integrations/providers/alibaba_cloud.mdx +++ b/docs/docs/integrations/providers/alibaba_cloud.mdx @@ -89,3 +89,11 @@ See [installation instructions and a usage example](/docs/integrations/vectorsto ```python from langchain_community.vectorstores import Hologres ``` + +### Tablestore + +See [installation instructions and a usage example](/docs/integrations/vectorstores/tablestore). + +```python +from langchain_community.vectorstores import TablestoreVectorStore +``` \ No newline at end of file diff --git a/docs/docs/integrations/providers/couchbase.mdx b/docs/docs/integrations/providers/couchbase.mdx index 906fbda6b28b3..35ef674806967 100644 --- a/docs/docs/integrations/providers/couchbase.mdx +++ b/docs/docs/integrations/providers/couchbase.mdx @@ -33,7 +33,7 @@ from langchain_community.document_loaders.couchbase import CouchbaseLoader ### CouchbaseCache Use Couchbase as a cache for prompts and responses. -See a [usage example](/docs/integrations/llm_caching/#couchbase-cache). +See a [usage example](/docs/integrations/llm_caching/#couchbase-caches). To import this cache: ```python @@ -61,7 +61,7 @@ set_llm_cache( Semantic caching allows users to retrieve cached prompts based on the semantic similarity between the user input and previously cached inputs. Under the hood it uses Couchbase as both a cache and a vectorstore. The CouchbaseSemanticCache needs a Search Index defined to work. Please look at the [usage example](/docs/integrations/vectorstores/couchbase) on how to set up the index. -See a [usage example](/docs/integrations/llm_caching/#couchbase-semantic-cache). +See a [usage example](/docs/integrations/llm_caching/#couchbase-caches). To import this cache: ```python diff --git a/docs/docs/integrations/providers/dataherald.mdx b/docs/docs/integrations/providers/dataherald.mdx index 9b589f6ae6cd3..2b1c276df526b 100644 --- a/docs/docs/integrations/providers/dataherald.mdx +++ b/docs/docs/integrations/providers/dataherald.mdx @@ -41,7 +41,7 @@ tool = DataheraldTextToSQL(api_wrapper=api_wrapper) llm = ChatOpenAI(model="gpt-3.5-turbo", temperature=0) prompt = hub.pull("hwchase17/react") agent = create_react_agent(llm, tools, prompt) -agent_executor = AgentExecutor(agent=agent, tools=tools, verbose=True) +agent_executor = AgentExecutor(agent=agent, tools=tools, verbose=True, handle_parsing_errors=True) agent_executor.invoke({"input":"Return the sql for this question: How many employees are in the company?"}) ``` diff --git a/docs/docs/integrations/providers/elasticsearch.mdx b/docs/docs/integrations/providers/elasticsearch.mdx index c3b123d47b80f..734ef9d46ce8e 100644 --- a/docs/docs/integrations/providers/elasticsearch.mdx +++ b/docs/docs/integrations/providers/elasticsearch.mdx @@ -84,7 +84,7 @@ from langchain_elasticsearch import ElasticsearchChatMessageHistory ## LLM cache -See a [usage example](/docs/integrations/llm_caching/#elasticsearch-cache). +See a [usage example](/docs/integrations/llm_caching/#elasticsearch-caches). ```python from langchain_elasticsearch import ElasticsearchCache diff --git a/docs/docs/integrations/providers/oceanbase.mdx b/docs/docs/integrations/providers/oceanbase.mdx new file mode 100644 index 0000000000000..30537f1391004 --- /dev/null +++ b/docs/docs/integrations/providers/oceanbase.mdx @@ -0,0 +1,31 @@ +# OceanBase + +[OceanBase Database](https://github.com/oceanbase/oceanbase) is a distributed relational database. +It is developed entirely by Ant Group. The OceanBase Database is built on a common server cluster. +Based on the Paxos protocol and its distributed structure, the OceanBase Database provides high availability and linear scalability. + +OceanBase currently has the ability to store vectors. Users can easily perform the following operations with SQL: + +- Create a table containing vector type fields; +- Create a vector index table based on the HNSW algorithm; +- Perform vector approximate nearest neighbor queries; +- ... + +## Installation + +```bash +pip install -U langchain-oceanbase +``` + +We recommend using Docker to deploy OceanBase: + +```shell +docker run --name=ob433 -e MODE=slim -p 2881:2881 -d oceanbase/oceanbase-ce:4.3.3.0-100000132024100711 +``` + +[More methods to deploy OceanBase cluster](https://github.com/oceanbase/oceanbase-doc/blob/V4.3.1/en-US/400.deploy/500.deploy-oceanbase-database-community-edition/100.deployment-overview.md) + +### Usage + +For a more detailed walkthrough of the OceanBase Wrapper, see [this notebook](https://github.com/oceanbase/langchain-oceanbase/blob/main/docs/vectorstores.ipynb) + diff --git a/docs/docs/integrations/providers/openllm.mdx b/docs/docs/integrations/providers/openllm.mdx index 92bdca1242cf3..dd09257954774 100644 --- a/docs/docs/integrations/providers/openllm.mdx +++ b/docs/docs/integrations/providers/openllm.mdx @@ -1,11 +1,17 @@ +--- +keywords: [openllm] +--- + # OpenLLM -This page demonstrates how to use [OpenLLM](https://github.com/bentoml/OpenLLM) -with LangChain. +OpenLLM lets developers run any **open-source LLMs** as **OpenAI-compatible API** endpoints with **a single command**. -`OpenLLM` is an open platform for operating large language models (LLMs) in -production. It enables developers to easily run inference with any open-source -LLMs, deploy to the cloud or on-premises, and build powerful AI apps. +- 🔬 Build for fast and production usages +- 🚂 Support llama3, qwen2, gemma, etc, and many **quantized** versions [full list](https://github.com/bentoml/openllm-models) +- ⛓️ OpenAI-compatible API +- 💬 Built-in ChatGPT like UI +- 🔥 Accelerated LLM decoding with state-of-the-art inference backends +- 🌥️ Ready for enterprise-grade cloud deployment (Kubernetes, Docker and BentoCloud) ## Installation and Setup @@ -23,8 +29,7 @@ are pre-optimized for OpenLLM. ## Wrappers -There is a OpenLLM Wrapper which supports loading LLM in-process or accessing a -remote OpenLLM server: +There is a OpenLLM Wrapper which supports interacting with running server with OpenLLM: ```python from langchain_community.llms import OpenLLM @@ -32,13 +37,12 @@ from langchain_community.llms import OpenLLM ### Wrapper for OpenLLM server -This wrapper supports connecting to an OpenLLM server via HTTP or gRPC. The -OpenLLM server can run either locally or on the cloud. +This wrapper supports interacting with OpenLLM's OpenAI-compatible endpoint. -To try it out locally, start an OpenLLM server: +To run a model, do: ```bash -openllm start flan-t5 +openllm hello ``` Wrapper usage: @@ -46,20 +50,7 @@ Wrapper usage: ```python from langchain_community.llms import OpenLLM -llm = OpenLLM(server_url='http://localhost:3000') - -llm("What is the difference between a duck and a goose? And why there are so many Goose in Canada?") -``` - -### Wrapper for Local Inference - -You can also use the OpenLLM wrapper to load LLM in current Python process for -running inference. - -```python -from langchain_community.llms import OpenLLM - -llm = OpenLLM(model_name="dolly-v2", model_id='databricks/dolly-v2-7b') +llm = OpenLLM(base_url="http://localhost:3000/v1", api_key="na") llm("What is the difference between a duck and a goose? And why there are so many Goose in Canada?") ``` diff --git a/docs/docs/integrations/providers/wandb.mdx b/docs/docs/integrations/providers/wandb.mdx new file mode 100644 index 0000000000000..8eb85116ff02f --- /dev/null +++ b/docs/docs/integrations/providers/wandb.mdx @@ -0,0 +1,42 @@ +# Weights & Biases + +>[Weights & Biases](https://wandb.ai/) is provider of the AI developer platform to train and +> fine-tune AI models and develop AI applications. + +`Weights & Biase` products can be used to log metrics and artifacts during training, +and to trace the execution of your code. + +There are several main ways to use `Weights & Biases` products within LangChain: +- with `wandb_tracing_enabled` +- with `Weave` lightweight toolkit +- with `WandbCallbackHandler` (deprecated) + + +## wandb_tracing_enabled + +See a [usage example](/docs/integrations/providers/wandb_tracing). + +See in the [W&B documentation](https://docs.wandb.ai/guides/integrations/langchain). + +```python +from langchain_community.callbacks import wandb_tracing_enabled +``` + +## Weave + +See in the [W&B documentation](https://weave-docs.wandb.ai/guides/integrations/langchain). + + +## WandbCallbackHandler + +**Note:** the `WandbCallbackHandler` is being deprecated in favour of the `wandb_tracing_enabled`. + +See a [usage example](/docs/integrations/providers/wandb_tracking). + +See in the [W&B documentation](https://docs.wandb.ai/guides/integrations/langchain). + +```python +from langchain_community.callbacks import WandbCallbackHandler +``` + + diff --git a/docs/docs/integrations/providers/wandb_tracing.ipynb b/docs/docs/integrations/providers/wandb_tracing.ipynb index 89b0dd8d3b727..ce6480d1a0d06 100644 --- a/docs/docs/integrations/providers/wandb_tracing.ipynb +++ b/docs/docs/integrations/providers/wandb_tracing.ipynb @@ -5,7 +5,7 @@ "id": "5371a9bb", "metadata": {}, "source": [ - "# WandB Tracing\n", + "# Weights & Biases tracing\n", "\n", "There are two recommended ways to trace your LangChains:\n", "\n", @@ -165,7 +165,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.3" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/docs/docs/integrations/providers/wandb_tracking.ipynb b/docs/docs/integrations/providers/wandb_tracking.ipynb index 5f034619867a1..ea49435e24efd 100644 --- a/docs/docs/integrations/providers/wandb_tracking.ipynb +++ b/docs/docs/integrations/providers/wandb_tracking.ipynb @@ -6,19 +6,24 @@ "id": "e43f4ea0", "metadata": {}, "source": [ - "# Weights & Biases\n", + "# Weights & Biases tracking\n", "\n", - "This notebook goes over how to track your LangChain experiments into one centralized Weights and Biases dashboard. To learn more about prompt engineering and the callback please refer to this Report which explains both alongside the resultant dashboards you can expect to see.\n", + "This notebook goes over how to track your LangChain experiments into one centralized `Weights and Biases` dashboard. \n", "\n", + "To learn more about prompt engineering and the callback please refer to this notebook which explains both alongside the resultant dashboards you can expect to see:\n", "\n", "\"Open\n", "\n", "\n", - "[View Report](https://wandb.ai/a-sh0ts/langchain_callback_demo/reports/Prompt-Engineering-LLMs-with-LangChain-and-W-B--VmlldzozNjk1NTUw#👋-how-to-build-a-callback-in-langchain-for-better-prompt-engineering\n", - ") \n", + "View a detailed description and examples in the [W&B article](https://wandb.ai/a-sh0ts/langchain_callback_demo/reports/Prompt-Engineering-LLMs-with-LangChain-and-W-B--VmlldzozNjk1NTUw#👋-how-to-build-a-callback-in-langchain-for-better-prompt-engineering\n", + "). \n", "\n", "\n", - "**Note**: _the `WandbCallbackHandler` is being deprecated in favour of the `WandbTracer`_ . In future please use the `WandbTracer` as it is more flexible and allows for more granular logging. To know more about the `WandbTracer` refer to the [agent_with_wandb_tracing](/docs/integrations/providers/wandb_tracing) notebook or use the following [colab notebook](http://wandb.me/prompts-quickstart). To know more about Weights & Biases Prompts refer to the following [prompts documentation](https://docs.wandb.ai/guides/prompts)." + "**Note**: _the `WandbCallbackHandler` is being deprecated in favour of the `WandbTracer`_ . In future please use the `WandbTracer` as it is more flexible and allows for more granular logging. \n", + "\n", + "To know more about the `WandbTracer` refer to the [agent_with_wandb_tracing](/docs/integrations/providers/wandb_tracing) notebook or use the following [colab notebook](http://wandb.me/prompts-quickstart). \n", + "\n", + "To know more about Weights & Biases Prompts refer to the following [prompts documentation](https://docs.wandb.ai/guides/prompts)." ] }, { @@ -248,6 +253,38 @@ "The `flush_tracker` function is used to log LangChain sessions to Weights & Biases. It takes in the LangChain module or agent, and logs at minimum the prompts and generations alongside the serialized form of the LangChain module to the specified Weights & Biases project. By default we reset the session as opposed to concluding the session outright." ] }, + { + "cell_type": "markdown", + "id": "483eefd4-633e-4686-8730-944705fe8a80", + "metadata": { + "execution": { + "iopub.execute_input": "2024-11-12T18:20:31.003316Z", + "iopub.status.busy": "2024-11-12T18:20:31.003152Z", + "iopub.status.idle": "2024-11-12T18:20:31.006033Z", + "shell.execute_reply": "2024-11-12T18:20:31.005546Z", + "shell.execute_reply.started": "2024-11-12T18:20:31.003303Z" + } + }, + "source": [ + "## Usage Scenarios" + ] + }, + { + "cell_type": "markdown", + "id": "066adc04-8180-4936-8d75-5c3660b61de7", + "metadata": { + "execution": { + "iopub.execute_input": "2024-11-12T18:20:45.826326Z", + "iopub.status.busy": "2024-11-12T18:20:45.825714Z", + "iopub.status.idle": "2024-11-12T18:20:45.830483Z", + "shell.execute_reply": "2024-11-12T18:20:45.830037Z", + "shell.execute_reply.started": "2024-11-12T18:20:45.826279Z" + } + }, + "source": [ + "### With LLM" + ] + }, { "cell_type": "code", "execution_count": 4, @@ -373,6 +410,14 @@ "wandb_callback.flush_tracker(llm, name=\"simple_sequential\")" ] }, + { + "cell_type": "markdown", + "id": "7dcfc24b-f0b0-48ec-89ce-beeb2fb763d5", + "metadata": {}, + "source": [ + "### Within Chains" + ] + }, { "cell_type": "code", "execution_count": 5, @@ -524,6 +569,14 @@ "wandb_callback.flush_tracker(synopsis_chain, name=\"agent\")" ] }, + { + "cell_type": "markdown", + "id": "533cd4c9-56e2-4ad9-a83e-4653bfcf322f", + "metadata": {}, + "source": [ + "### With Agents" + ] + }, { "cell_type": "code", "execution_count": 7, @@ -646,7 +699,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.1" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/docs/docs/integrations/text_embedding/localai.ipynb b/docs/docs/integrations/text_embedding/localai.ipynb index 308d1ca67de91..263a3b346f56c 100644 --- a/docs/docs/integrations/text_embedding/localai.ipynb +++ b/docs/docs/integrations/text_embedding/localai.ipynb @@ -8,22 +8,40 @@ "source": [ "# LocalAI\n", "\n", + ":::info\n", + "\n", + "`langchain-localai` is a 3rd party integration package for LocalAI. It provides a simple way to use LocalAI services in Langchain.\n", + "\n", + "The source code is available on [Github](https://github.com/mkhludnev/langchain-localai)\n", + "\n", + ":::\n", + "\n", "Let's load the LocalAI Embedding class. In order to use the LocalAI Embedding class, you need to have the LocalAI service hosted somewhere and configure the embedding models. See the documentation at https://localai.io/basics/getting_started/index.html and https://localai.io/features/embeddings/index.html." ] }, { "cell_type": "code", - "execution_count": 1, - "id": "0be1af71", + "execution_count": null, + "id": "799d1f77", "metadata": {}, "outputs": [], "source": [ - "from langchain_community.embeddings import LocalAIEmbeddings" + "%pip install -U langchain-localai" ] }, { "cell_type": "code", "execution_count": 2, + "id": "0be1af71", + "metadata": {}, + "outputs": [], + "source": [ + "from langchain_localai import LocalAIEmbeddings" + ] + }, + { + "cell_type": "code", + "execution_count": 3, "id": "2c66e5da", "metadata": {}, "outputs": [], @@ -35,7 +53,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "id": "01370375", "metadata": {}, "outputs": [], @@ -45,7 +63,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "bfb6142c", "metadata": {}, "outputs": [], @@ -140,7 +158,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": ".venv", "language": "python", "name": "python3" }, @@ -154,12 +172,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" - }, - "vscode": { - "interpreter": { - "hash": "e971737741ff4ec9aff7dc6155a1060a59a8a6d52c757dbbe66bf8ee389494b1" - } + "version": "3.11.4" } }, "nbformat": 4, diff --git a/docs/docs/integrations/tools/wikidata.ipynb b/docs/docs/integrations/tools/wikidata.ipynb index f316108b80a13..8ec1d88e3acdf 100644 --- a/docs/docs/integrations/tools/wikidata.ipynb +++ b/docs/docs/integrations/tools/wikidata.ipynb @@ -15,7 +15,7 @@ { "cell_type": "code", "execution_count": null, - "id": "3d9195d4", + "id": "d622c581", "metadata": { "vscode": { "languageId": "shellscript" @@ -23,7 +23,7 @@ }, "outputs": [], "source": [ - "%pip install --upgrade --quiet \"wikibase-rest-api-client<0.2\" mediawikiapi" + "%pip install --upgrade --quiet wikibase-rest-api-client mediawikiapi" ] }, { @@ -110,7 +110,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.5" + "version": "3.12.5" } }, "nbformat": 4, diff --git a/docs/docs/integrations/vectorstores/aerospike.ipynb b/docs/docs/integrations/vectorstores/aerospike.ipynb index b2ad324b0554f..17f9fda39f930 100644 --- a/docs/docs/integrations/vectorstores/aerospike.ipynb +++ b/docs/docs/integrations/vectorstores/aerospike.ipynb @@ -29,8 +29,8 @@ "metadata": {}, "outputs": [], "source": [ - "PROXIMUS_HOST = \"\"\n", - "PROXIMUS_PORT = 5000" + "AVS_HOST = \"\"\n", + "AVS_PORT = 5000" ] }, { @@ -51,7 +51,7 @@ }, "outputs": [], "source": [ - "!pip install --upgrade --quiet aerospike-vector-search==0.6.1 langchain-community sentence-transformers langchain" + "!pip install --upgrade --quiet aerospike-vector-search==3.0.1 langchain-community sentence-transformers langchain" ] }, { @@ -369,7 +369,7 @@ "from langchain_community.vectorstores import Aerospike\n", "\n", "# Here we are using the AVS host and port you configured earlier\n", - "seed = HostPort(host=PROXIMUS_HOST, port=PROXIMUS_PORT)\n", + "seed = HostPort(host=AVS_HOST, port=AVS_PORT)\n", "\n", "# The namespace of where to place our vectors. This should match the vector configured in your docstore.conf file.\n", "NAMESPACE = \"test\"\n", @@ -401,7 +401,7 @@ " vector_field=VECTOR_KEY,\n", " vector_distance_metric=MODEL_DISTANCE_CALC,\n", " dimensions=MODEL_DIM,\n", - " index_meta_data={\n", + " index_labels={\n", " \"model\": \"miniLM-L6-v2\",\n", " \"date\": \"05/04/2024\",\n", " \"dim\": str(MODEL_DIM),\n", diff --git a/docs/docs/integrations/vectorstores/azure_cosmos_db_no_sql.ipynb b/docs/docs/integrations/vectorstores/azure_cosmos_db_no_sql.ipynb index 9e3109a01d89f..7a231f764e276 100644 --- a/docs/docs/integrations/vectorstores/azure_cosmos_db_no_sql.ipynb +++ b/docs/docs/integrations/vectorstores/azure_cosmos_db_no_sql.ipynb @@ -11,7 +11,12 @@ " \n", "Azure Cosmos DB is the database that powers OpenAI's ChatGPT service. It offers single-digit millisecond response times, automatic and instant scalability, along with guaranteed speed at any scale. \n", "\n", - "[Azure Cosmos DB for NoSQL](https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/vector-search) now offers vector indexing and search in preview. This feature is designed to handle high-dimensional vectors, enabling efficient and accurate vector search at any scale. You can now store vectors directly in the documents alongside your data. This means that each document in your database can contain not only traditional schema-free data, but also high-dimensional vectors as other properties of the documents. This colocation of data and vectors allows for efficient indexing and searching, as the vectors are stored in the same logical unit as the data they represent. This simplifies data management, AI application architectures, and the efficiency of vector-based operations.\n", + "Azure Cosmos DB for NoSQL now offers vector indexing and search in preview. This feature is designed to handle high-dimensional vectors, enabling efficient and accurate vector search at any scale. You can now store vectors directly in the documents alongside your data. This means that each document in your database can contain not only traditional schema-free data, but also high-dimensional vectors as other properties of the documents. This colocation of data and vectors allows for efficient indexing and searching, as the vectors are stored in the same logical unit as the data they represent. This simplifies data management, AI application architectures, and the efficiency of vector-based operations.\n", + "\n", + "Please refer here for more details:\n", + "- [Vector Search](https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/vector-search)\n", + "- [Full Text Search](https://learn.microsoft.com/en-us/azure/cosmos-db/gen-ai/full-text-search)\n", + "- [Hybrid Search](https://learn.microsoft.com/en-us/azure/cosmos-db/gen-ai/hybrid-search)\n", "\n", "[Sign Up](https://azure.microsoft.com/en-us/free/) for lifetime free access to get started today." ] @@ -36,12 +41,12 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 1, "id": "c507b0e8", "metadata": { "ExecuteTime": { - "end_time": "2024-05-25T01:36:53.595385Z", - "start_time": "2024-05-25T01:36:53.571737Z" + "end_time": "2024-12-13T21:14:23.285152Z", + "start_time": "2024-12-13T21:14:23.277816Z" } }, "outputs": [], @@ -64,12 +69,12 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 2, "id": "8205cd27", "metadata": { "ExecuteTime": { - "end_time": "2024-05-25T01:43:02.731634Z", - "start_time": "2024-05-25T01:43:00.383956Z" + "end_time": "2024-12-13T21:14:36.109885Z", + "start_time": "2024-12-13T21:14:30.764272Z" } }, "outputs": [], @@ -83,12 +88,12 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 3, "id": "8d33cceb", "metadata": { "ExecuteTime": { - "end_time": "2024-05-25T01:43:02.787966Z", - "start_time": "2024-05-25T01:43:02.763502Z" + "end_time": "2024-12-13T21:14:38.909571Z", + "start_time": "2024-12-13T21:14:38.813059Z" } }, "outputs": [], @@ -101,12 +106,12 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 4, "id": "6a80f1c2", "metadata": { "ExecuteTime": { - "end_time": "2024-05-25T01:43:04.582560Z", - "start_time": "2024-05-25T01:43:04.578948Z" + "end_time": "2024-12-13T21:14:41.079019Z", + "start_time": "2024-12-13T21:14:41.075776Z" } }, "outputs": [ @@ -114,7 +119,22 @@ "name": "stdout", "output_type": "stream", "text": [ - "page_content='GPT-4 Technical Report\\nOpenAI∗\\nAbstract\\nWe report the development of GPT-4, a large-scale, multimodal model which can\\naccept image and text inputs and produce text outputs. While less capable than\\nhumans in many real-world scenarios, GPT-4 exhibits human-level performance\\non various professional and academic benchmarks, including passing a simulated\\nbar exam with a score around the top 10% of test takers. GPT-4 is a Transformer-\\nbased model pre-trained to predict the next token in a document. The post-training\\nalignment process results in improved performance on measures of factuality and\\nadherence to desired behavior. A core component of this project was developing\\ninfrastructure and optimization methods that behave predictably across a wide\\nrange of scales. This allowed us to accurately predict some aspects of GPT-4’s\\nperformance based on models trained with no more than 1/1,000th the compute of\\nGPT-4.\\n1 Introduction' metadata={'source': 'https://arxiv.org/pdf/2303.08774.pdf', 'page': 0}\n" + "page_content='GPT-4 Technical Report\n", + "OpenAI∗\n", + "Abstract\n", + "We report the development of GPT-4, a large-scale, multimodal model which can\n", + "accept image and text inputs and produce text outputs. While less capable than\n", + "humans in many real-world scenarios, GPT-4 exhibits human-level performance\n", + "on various professional and academic benchmarks, including passing a simulated\n", + "bar exam with a score around the top 10% of test takers. GPT-4 is a Transformer-\n", + "based model pre-trained to predict the next token in a document. The post-training\n", + "alignment process results in improved performance on measures of factuality and\n", + "adherence to desired behavior. A core component of this project was developing\n", + "infrastructure and optimization methods that behave predictably across a wide\n", + "range of scales. This allowed us to accurately predict some aspects of GPT-4’s\n", + "performance based on models trained with no more than 1/1,000th the compute of\n", + "GPT-4.\n", + "1 Introduction' metadata={'source': 'https://arxiv.org/pdf/2303.08774.pdf', 'page': 0}\n" ] } ], @@ -132,12 +152,12 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 5, "id": "04c72ccc", "metadata": { "ExecuteTime": { - "end_time": "2024-05-25T01:43:13.279497Z", - "start_time": "2024-05-25T01:43:13.275379Z" + "end_time": "2024-12-13T21:15:39.050171Z", + "start_time": "2024-12-13T21:15:39.046271Z" } }, "outputs": [], @@ -146,7 +166,8 @@ " \"indexingMode\": \"consistent\",\n", " \"includedPaths\": [{\"path\": \"/*\"}],\n", " \"excludedPaths\": [{\"path\": '/\"_etag\"/?'}],\n", - " \"vectorIndexes\": [{\"path\": \"/embedding\", \"type\": \"quantizedFlat\"}],\n", + " \"vectorIndexes\": [{\"path\": \"/embedding\", \"type\": \"diskANN\"}],\n", + " \"fullTextIndexes\": [{\"path\": \"/text\"}],\n", "}\n", "\n", "vector_embedding_policy = {\n", @@ -158,17 +179,22 @@ " \"dimensions\": 1536,\n", " }\n", " ]\n", + "}\n", + "\n", + "full_text_policy = {\n", + " \"defaultLanguage\": \"en-US\",\n", + " \"fullTextPaths\": [{\"path\": \"/text\", \"language\": \"en-US\"}],\n", "}" ] }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 10, "id": "4ebad8ef01a6c04f", "metadata": { "ExecuteTime": { - "end_time": "2024-05-25T01:48:42.981276Z", - "start_time": "2024-05-25T01:44:55.468667Z" + "end_time": "2024-12-13T21:24:55.719198Z", + "start_time": "2024-12-13T21:22:47.334006Z" } }, "outputs": [], @@ -177,7 +203,7 @@ "from langchain_community.vectorstores.azure_cosmos_db_no_sql import (\n", " AzureCosmosDBNoSqlVectorSearch,\n", ")\n", - "from langchain_openai import AzureOpenAIEmbeddings\n", + "from langchain_openai import OpenAIEmbeddings\n", "\n", "HOST = \"AZURE_COSMOS_DB_ENDPOINT\"\n", "KEY = \"AZURE_COSMOS_DB_KEY\"\n", @@ -188,11 +214,11 @@ "partition_key = PartitionKey(path=\"/id\")\n", "cosmos_container_properties = {\"partition_key\": partition_key}\n", "\n", - "openai_embeddings = AzureOpenAIEmbeddings(\n", - " azure_deployment=OPENAI_EMBEDDINGS_MODEL_DEPLOYMENT,\n", - " api_version=OPENAI_API_VERSION,\n", - " azure_endpoint=OPENAI_API_BASE,\n", - " openai_api_key=OPENAI_API_KEY,\n", + "openai_embeddings = OpenAIEmbeddings(\n", + " deployment=\"smart-agent-embedding-ada\",\n", + " model=\"text-embedding-ada-002\",\n", + " chunk_size=1,\n", + " openai_api_key=\"OPENAI_API_KEY\",\n", ")\n", "\n", "# insert the documents in AzureCosmosDBNoSql with their embedding\n", @@ -203,8 +229,11 @@ " database_name=database_name,\n", " container_name=container_name,\n", " vector_embedding_policy=vector_embedding_policy,\n", + " full_text_policy=full_text_policy,\n", " indexing_policy=indexing_policy,\n", " cosmos_container_properties=cosmos_container_properties,\n", + " cosmos_database_properties={},\n", + " full_text_search_enabled=True,\n", ")" ] }, @@ -212,18 +241,16 @@ "cell_type": "markdown", "id": "3f5ff6adb7c8ad48", "metadata": {}, - "source": [ - "## Querying Data" - ] + "source": "## Vector Search" }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 22, "id": "7cf7a2cc", "metadata": { "ExecuteTime": { - "end_time": "2024-05-25T01:48:47.576539Z", - "start_time": "2024-05-25T01:48:46.693940Z" + "end_time": "2024-12-13T21:34:31.438548Z", + "start_time": "2024-12-13T21:34:30.456291Z" } }, "outputs": [ @@ -248,6 +275,8 @@ ], "source": [ "# Perform a similarity search between the embedding of the query and the embeddings of the documents\n", + "import json\n", + "\n", "query = \"What were the compute requirements for training GPT 4\"\n", "results = vector_search.similarity_search(query)\n", "\n", @@ -258,18 +287,75 @@ "cell_type": "markdown", "id": "30d77484c71fe192", "metadata": {}, + "source": "## Vector Search with Score" + }, + { + "cell_type": "code", + "execution_count": 44, + "id": "6371c107306cd76d", + "metadata": { + "ExecuteTime": { + "end_time": "2024-12-13T21:45:46.356588Z", + "start_time": "2024-12-13T21:45:45.770116Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Result 1: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":0,\"id\":\"9d59c3ed-deac-48cb-9382-a8ab079334e5\"},\"page_content\":\"performance based on models trained with no more than 1/1,000th the compute of\\nGPT-4.\\n1 Introduction\\nThis technical report presents GPT-4, a large multimodal model capable of processing image and\\ntext inputs and producing text outputs. Such models are an important area of study as they have the\\npotential to be used in a wide range of applications, such as dialogue systems, text summarization,\\nand machine translation. As such, they have been the subject of substantial interest and progress in\\nrecent years [1–34].\\nOne of the main goals of developing such models is to improve their ability to understand and generate\\nnatural language text, particularly in more complex and nuanced scenarios. To test its capabilities\\nin such scenarios, GPT-4 was evaluated on a variety of exams originally designed for humans. In\\nthese evaluations it performs quite well and often outscores the vast majority of human test takers.\",\"type\":\"Document\"}\n", + "Score 1: 0.8394796122122777\n", + "\n", + "\n", + "Result 2: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":43,\"id\":\"e5610de3-8af6-43b9-8266-51c26d76eaa3\"},\"page_content\":\"2 GPT-4 Observed Safety Challenges\\nGPT-4 demonstrates increased performance in areas such as reasoning, knowledge retention, and\\ncoding, compared to earlier models such as GPT-2[ 22] and GPT-3.[ 10] Many of these improvements\\nalso present new safety challenges, which we highlight in this section.\\nWe conducted a range of qualitative and quantitative evaluations of GPT-4. These evaluations\\nhelped us gain an understanding of GPT-4’s capabilities, limitations, and risks; prioritize our\\nmitigation efforts; and iteratively test and build safer versions of the model. Some of the specific\\nrisks we explored are:6\\n•Hallucinations\\n•Harmful content\\n•Harms of representation, allocation, and quality of service\\n•Disinformation and influence operations\\n•Proliferation of conventional and unconventional weapons\\n•Privacy\\n•Cybersecurity\\n•Potential for risky emergent behaviors\\n•Interactions with other systems\\n•Economic impacts\\n•Acceleration\\n•Overreliance\",\"type\":\"Document\"}\n", + "Score 2: 0.8299261339098007\n", + "\n", + "\n", + "Result 3: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":0,\"id\":\"cddfb7ac-d953-46f4-8a48-76655f116bcf\"},\"page_content\":\"GPT-4 Technical Report\\nOpenAI∗\\nAbstract\\nWe report the development of GPT-4, a large-scale, multimodal model which can\\naccept image and text inputs and produce text outputs. While less capable than\\nhumans in many real-world scenarios, GPT-4 exhibits human-level performance\\non various professional and academic benchmarks, including passing a simulated\\nbar exam with a score around the top 10% of test takers. GPT-4 is a Transformer-\\nbased model pre-trained to predict the next token in a document. The post-training\\nalignment process results in improved performance on measures of factuality and\\nadherence to desired behavior. A core component of this project was developing\\ninfrastructure and optimization methods that behave predictably across a wide\\nrange of scales. This allowed us to accurately predict some aspects of GPT-4’s\\nperformance based on models trained with no more than 1/1,000th the compute of\\nGPT-4.\\n1 Introduction\",\"type\":\"Document\"}\n", + "Score 3: 0.8286253517208215\n", + "\n", + "\n", + "Result 4: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":3,\"id\":\"4f3152cd-c543-4a4f-b94e-c52c4139c4a8\"},\"page_content\":\"plan to refine these methods and register performance predictions across various capabilities before\\nlarge model training begins, and we hope this becomes a common goal in the field.\\n4 Capabilities\\nWe tested GPT-4 on a diverse set of benchmarks, including simulating exams that were originally\\ndesigned for humans.4We did no specific training for these exams. A minority of the problems in the\\nexams were seen by the model during training; for each exam we run a variant with these questions\\nremoved and report the lower score of the two. We believe the results to be representative. For further\\ndetails on contamination (methodology and per-exam statistics), see Appendix C.\\nExams were sourced from publicly-available materials. Exam questions included both multiple-\\nchoice and free-response questions; we designed separate prompts for each format, and images were\\nincluded in the input for questions which required it. The evaluation setup was designed based\",\"type\":\"Document\"}\n", + "Score 4: 0.8278858118680015\n", + "\n", + "\n", + "Result 5: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":28,\"id\":\"18b4f43d-27d2-404e-9d66-f9328a3588c6\"},\"page_content\":\"overall GPT-4 training budget. When mixing in data from these math benchmarks, a portion of the\\ntraining data was held back, so each individual training example may or may not have been seen by\\nGPT-4 during training.\\nWe conducted contamination checking to verify the test set for GSM-8K is not included in the training\\nset (see Appendix D). We recommend interpreting the performance results reported for GPT-4\\nGSM-8K in Table 2 as something in-between true few-shot transfer and full benchmark-specific\\ntuning.\\nF Multilingual MMLU\\nWe translated all questions and answers from MMLU [ 49] using Azure Translate. We used an\\nexternal model to perform the translation, instead of relying on GPT-4 itself, in case the model had\\nunrepresentative performance for its own translations. We selected a range of languages that cover\\ndifferent geographic regions and scripts, we show an example question taken from the astronomy\",\"type\":\"Document\"}\n", + "Score 5: 0.8272138555588135\n", + "\n", + "\n" + ] + } + ], "source": [ - "## Similarity Search with Score" + "query = \"What were the compute requirements for training GPT 4\"\n", + "\n", + "results = vector_search.similarity_search_with_score(\n", + " query=query,\n", + " k=5,\n", + ")\n", + "\n", + "# Display results\n", + "for i in range(0, len(results)):\n", + " print(f\"Result {i+1}: \", results[i][0].json())\n", + " print(f\"Score {i+1}: \", results[i][1])\n", + " print(\"\\n\")" ] }, + { + "cell_type": "markdown", + "id": "9c4ffb492375192d", + "metadata": {}, + "source": "## Vector Search with filtering" + }, { "cell_type": "code", - "execution_count": 43, - "id": "6371c107306cd76d", + "execution_count": 50, + "id": "461c6ac2ba3cee2a", "metadata": { "ExecuteTime": { - "end_time": "2024-05-25T01:51:15.012519Z", - "start_time": "2024-05-25T01:51:13.597319Z" + "end_time": "2024-12-13T21:53:49.420189Z", + "start_time": "2024-12-13T21:53:47.779582Z" } }, "outputs": [ @@ -277,37 +363,303 @@ "name": "stdout", "output_type": "stream", "text": [ - "(Document(page_content='performance based on models trained with no more than 1/1,000th the compute of\\nGPT-4.\\n1 Introduction\\nThis technical report presents GPT-4, a large multimodal model capable of processing image and\\ntext inputs and producing text outputs. Such models are an important area of study as they have the\\npotential to be used in a wide range of applications, such as dialogue systems, text summarization,\\nand machine translation. As such, they have been the subject of substantial interest and progress in\\nrecent years [1–34].\\nOne of the main goals of developing such models is to improve their ability to understand and generate\\nnatural language text, particularly in more complex and nuanced scenarios. To test its capabilities\\nin such scenarios, GPT-4 was evaluated on a variety of exams originally designed for humans. In\\nthese evaluations it performs quite well and often outscores the vast majority of human test takers.', metadata={'id': '95cc87c2-27f5-40d2-afc2-a354e8f339e4', 'embedding': [-0.025839418172836304, -0.004486586898565292, -0.003380518639460206, -0.014719882979989052, 0.01754477061331272, 0.01754477061331272, -0.011216467246413231, 0.006923745386302471, -0.018334077671170235, -0.027348794043064117, 0.011860375292599201, 0.030990684404969215, -0.03522801771759987, -0.020854320377111435, -0.008370808325707912, 0.022931445389986038, 0.016727767884731293, -0.007706128526479006, 0.0006555921281687915, -0.017406295984983444, 0.014020584523677826, 0.00848851166665554, -0.018237145617604256, -0.018320230767130852, 0.010136363096535206, 0.011811909265816212, 0.031156854704022408, -0.037000495940446854, -0.014719882979989052, 0.0030897213146090508, 0.015052222646772861, -0.016893938183784485, -0.022045204415917397, -0.0068510458804667, 0.0028629687149077654, -0.007408407516777515, 0.009291666559875011, -0.010489474050700665, 0.004389654379338026, -0.004638909362256527, -0.0011874223127961159, 0.015647664666175842, 0.02370690368115902, -0.011583426035940647, 0.0017006449634209275, 0.0028266189619898796, 0.01800173707306385, -0.0284427460283041, 0.0016772772651165724, 0.004631985444575548, 0.018237145617604256, 0.03400943800806999, -0.003780364990234375, 0.0003414271923247725, -0.018791044130921364, 0.007733823731541634, -0.00014767050743103027, 0.00948553066700697, 0.017974043264985085, 0.0010601985268294811, -0.011922689154744148, -0.00823233276605606, -0.024759313091635704, 0.03373248875141144, -0.005840179044753313, -0.01509376522153616, -0.037055883556604385, 0.015924613922834396, -0.00361419515684247, -0.0017569003393873572, -0.0026569871697574854, 0.008675453253090382, 0.0020771236158907413, -0.025077804923057556, 0.037305139005184174, -0.010510245338082314, 0.0018901824951171875, -0.009921726770699024, -0.0230422243475914, 0.012552750296890736, 0.018749501556158066, -0.009520149789750576, 0.014103669673204422, 0.03403713181614876, 0.005663623567670584, -0.004663142375648022, 0.01595230959355831, -0.003087990451604128, -0.01701856590807438, -0.0017421874217689037, 0.017157040536403656, -0.0024908173363655806, 0.017710940912365913, 0.014096745289862156, -0.015329171903431416, 0.01995423436164856, 0.0031260710675269365, 0.02880278043448925, -0.014346000738441944, -0.019871149212121964, -0.011043373495340347, 0.0028923945501446724, -0.020341964438557625, -0.014415238052606583, -0.023208394646644592, 0.008363883942365646, 0.018167907372117043, 0.00011521545093273744, 0.0036834324710071087, 0.005642852280288935, -0.02021733671426773, 0.02186518721282482, -0.011486493051052094, -0.010780271142721176, -0.01595230959355831, 0.00623483257368207, 0.020868169143795967, -0.008017697371542454, 0.0008421005331911147, -0.0020546214655041695, 0.02577017992734909, -0.016783159226179123, 0.014359847642481327, 0.0021827106829732656, 0.026822589337825775, 0.006653719116002321, -0.018140213564038277, -0.031101463362574577, 0.003544957609847188, -0.0004048227274324745, 0.012711996212601662, 0.002544476417824626, 0.0274041835218668, -0.007678433787077665, -0.03085220977663994, 0.010191753506660461, -0.024496210739016533, 0.005992501508444548, -0.038440633565187454, -0.018278688192367554, 0.017461685463786125, 0.02776421792805195, -0.015924613922834396, -0.009000868536531925, -0.007879221811890602, 0.028415050357580185, 0.015661511570215225, 0.010960289277136326, 0.011770366691052914, 0.011708053760230541, 0.020134251564741135, -0.01779402606189251, -0.008973173797130585, 0.027736524119973183, 0.011631892062723637, -0.011396484449505806, 0.00421656109392643, 0.02277912199497223, -0.009499378502368927, -0.024662381038069725, -0.01769709214568138, 0.018860282376408577, -0.005151266697794199, 0.006986059248447418, 0.01851409487426281, 0.019871149212121964, 0.00910472497344017, 0.02027272619307041, -4.362500476418063e-05, -0.005341669544577599, -0.00907703023403883, 0.02448236383497715, -0.033870961517095566, 0.02351303957402706, -0.013265895657241344, 0.0006915090489201248, 0.013134344480931759, -0.007366864942014217, 0.0021740561351180077, -0.016340039670467377, 0.008142324164509773, 0.018707958981394768, -0.00318838469684124, -0.004337726626545191, -0.015619969926774502, -0.02711338736116886, 0.013979041948914528, -0.0014903361443430185, -0.01794634759426117, -0.01959419995546341, 0.01985730230808258, 0.0018174831056967378, 0.004143861588090658, -0.007408407516777515, -0.6279006004333496, -0.010441008023917675, -0.0062002139165997505, -0.015107612125575542, 0.025797875598073006, 0.013930575922131538, 0.014290610328316689, -0.0011727093951776624, -0.024191565811634064, 0.020646609365940094, -0.008758537471294403, -0.015287629328668118, -0.01267737802118063, -0.025216281414031982, -0.009201657958328724, -0.03353862091898918, -0.017420142889022827, -0.018237145617604256, 0.0038149836473166943, 0.02046659030020237, -0.02921820618212223, 0.014706035144627094, 0.0013007986126467586, 0.019774217158555984, -0.0291628148406744, 0.0003647948324214667, -0.00884854607284069, -0.013736710883677006, 0.005535534583032131, 0.01609078422188759, -0.03594808652997017, 0.023263784125447273, 0.0034341777209192514, -0.004334264434874058, 0.04134860634803772, -0.0021809798199683428, -0.029550544917583466, 0.014006736688315868, -0.008481588214635849, 0.026656419038772583, -0.029550544917583466, -0.012739690952003002, -0.0017214161343872547, 0.008675453253090382, -0.013868262059986591, 0.037166666239500046, 0.0016314075328409672, -0.014899900183081627, -0.006425235886126757, -0.01445678062736988, -0.012130402028560638, 0.010468702763319016, 0.024302346631884575, -0.0021619393955916166, -0.01093259360641241, 0.00438619265332818, 0.03539418429136276, 0.001936917775310576, -0.015412257052958012, -0.028553524985909462, -0.0023211855441331863, 0.005892107263207436, -0.003551881294697523, -0.0402408093214035, -0.01887412928044796, 0.019109537824988365, 0.008024620823562145, -0.0028196952771395445, 0.0032281961757689714, -0.041071657091379166, -0.010413313284516335, 0.019940387457609177, 0.0026310232933610678, -0.008890088647603989, 0.003956920467317104, 0.005310512613505125, 0.021879035979509354, -0.016963176429271698, 0.0009398984257131815, 0.026933370158076286, 0.011631892062723637, -0.014152135699987411, 0.005313974339514971, -0.00042927221511490643, 0.019303401932120323, 0.009263970889151096, -0.005275893956422806, 0.015066069550812244, 0.024468515068292618, 0.006937592756003141, 0.007858450524508953, 0.024565448984503746, -0.020480439066886902, -0.023803835734725, 0.012151173315942287, -0.0008226275094784796, -0.02891356125473976, -0.0052412752993404865, 0.005030101165175438, -0.010468702763319016, -0.009430141188204288, -0.024662381038069725, 0.03533879667520523, 0.028359660878777504, 0.03508954122662544, 0.017475532367825508, -0.014706035144627094, 0.0012436778051778674, -0.006013272795826197, -0.006767961196601391, -0.008107705973088741, 0.002279643202200532, -0.006574096158146858, -0.009610158391296864, 0.016852395609021187, -0.0291628148406744, 0.013778253458440304, -0.006553324870765209, -0.022709885612130165, -0.03173844888806343, 0.01989884488284588, -0.016367733478546143, 0.012490436434745789, -0.007456873543560505, 0.009873260743916035, 0.010787195526063442, 0.014761424623429775, -0.027653438970446587, -0.0002302145294379443, 0.014539864845573902, -0.015647664666175842, -0.018029432743787766, 0.057273220270872116, -0.010531016625463963, 0.02963363006711006, 0.021435916423797607, 0.033510927110910416, 0.0036765087861567736, 0.0060998196713626385, -0.013639777898788452, -0.02093740552663803, 0.002916627796366811, 0.0002674296556506306, 0.004168094601482153, -0.018029432743787766, -0.014262915588915348, -0.011223391629755497, 0.005985578056424856, 0.009443989023566246, -0.0006538612069562078, -0.0005590924411080778, -0.00812847726047039, -0.004486586898565292, 0.014262915588915348, -0.0016045779921114445, -0.02349919080734253, -0.010801042430102825, -0.053008195012807846, -0.008654681965708733, -0.019206469878554344, 0.011348018422722816, 0.003195308381691575, -0.008917784318327904, 0.008730842731893063, 0.006664104759693146, -0.02581172250211239, -0.016963176429271698, 0.03857911005616188, -0.04794001206755638, -0.027999626472592354, 0.00588864553719759, -0.03132302314043045, 0.018597180023789406, 0.025950197130441666, -0.0202034879475832, -0.002400808734819293, 0.004431196954101324, -0.0014609103091061115, 0.0018192140851169825, -0.02546553499996662, -0.005715551786124706, 0.011077992618083954, -0.011105687357485294, -0.020134251564741135, 0.019871149212121964, 0.03256929665803909, 0.04123782739043236, 0.015675360336899757, -0.022972986102104187, 0.00018769840244203806, 0.017835568636655807, 0.01866641826927662, -0.028553524985909462, 0.028221186250448227, 0.001203866209834814, 0.03060295432806015, 0.012054240331053734, 0.006508320569992065, 0.015259934589266777, 0.03268007934093475, 0.012026545591652393, 0.010219448246061802, 0.014235220849514008, -0.019926538690924644, -0.011638815514743328, -0.012435046955943108, 0.01722627878189087, -0.020494285970926285, 0.0010627949377521873, 0.0033441688865423203, 0.006757575552910566, -0.011348018422722816, -0.016783159226179123, -0.0022952216677367687, -0.005106262397021055, 0.01830638200044632, -0.0025894807185977697, 0.0018746040295809507, -0.025368602946400642, -0.02946745976805687, 0.005123571492731571, -0.0007975288899615407, 0.016852395609021187, 0.009263970889151096, -0.012940480373799801, 0.020535828545689583, -0.02222522161900997, 0.01856948435306549, 0.025133196264505386, -0.027030302211642265, 0.0006283298716880381, 0.017614008858799934, 0.01836177334189415, 0.016215411946177483, 0.013286666944622993, 0.0017421874217689037, 0.01177729107439518, -0.007103762589395046, 0.037360530346632004, 0.003136456711217761, -0.004715070594102144, 0.007477644830942154, -0.0011908841552212834, -0.02880278043448925, 0.025063958019018173, -0.009457835927605629, 0.011174924671649933, -0.018860282376408577, -0.026933370158076286, 0.0015093764523044229, -0.012206562794744968, 0.018375620245933533, -0.02201751060783863, 0.005701704416424036, 0.03292933106422424, -0.008585444651544094, 0.006227909121662378, 0.003596885595470667, 0.006027120165526867, 0.030270613729953766, 0.004375807009637356, 0.01339744683355093, 0.025285517796874046, 0.006885664537549019, 0.022183680906891823, -0.028692001476883888, 0.002972017740830779, -0.015204545110464096, -0.021532848477363586, -0.013120497576892376, -0.025894807651638985, -0.003596885595470667, 0.019414182752370834, -0.026254842057824135, 0.018444858491420746, 0.008066163398325443, 0.015176849439740181, 0.019400333985686302, 0.030741428956389427, 0.016215411946177483, -0.028221186250448227, -0.05203887075185776, 0.009603234939277172, 0.001980191096663475, -0.007809984963387251, -0.0032957028597593307, -0.05450372397899628, 0.011915765702724457, -0.003003174439072609, -0.014207525178790092, -0.014062127098441124, 0.018029432743787766, -0.0021567465737462044, 0.009762480854988098, -0.0026760275941342115, 0.018804892897605896, 0.027196472510695457, -0.004424273036420345, -0.012594292871654034, -0.0013709015911445022, 0.012026545591652393, -0.002528897952288389, -0.012850471772253513, -0.03179384022951126, 0.01568920724093914, -0.018583333119750023, -0.03522801771759987, -0.010177905671298504, 0.010240219533443451, -0.015619969926774502, 0.03148919343948364, 0.0005110589554533362, 0.010558711364865303, -0.017101651057600975, 0.0026033283211290836, 0.016755463555455208, -0.01676931045949459, 0.022336002439260483, 0.03514493256807327, 0.012684301473200321, 0.015966156497597694, -0.026088671758770943, -0.007713052444159985, 0.018126364797353745, 0.04007463902235031, 0.04057314619421959, -0.012649682350456715, -0.012476589530706406, -0.028359660878777504, 0.0021844415459781885, -0.030879903584718704, -0.017198583111166954, 0.012573521584272385, 0.021020490676164627, 0.01273276749998331, -0.023028377443552017, -0.010918746702373028, -0.0007533900206908584, 0.026711810380220413, 0.007415331434458494, -0.002338494872674346, -0.013383599929511547, -0.014318305067718029, -0.010524093173444271, 0.007387636229395866, -0.020909711718559265, 0.013639777898788452, -0.005788251291960478, 0.011348018422722816, -0.025340907275676727, 0.025797875598073006, 0.020494285970926285, -0.013210506178438663, -0.028068862855434418, -0.026254842057824135, -0.01237273309379816, -0.005968268495053053, 0.0052585843950510025, 0.005338207818567753, 0.005251660943031311, 0.001980191096663475, -0.014664492569863796, 0.012400427833199501, 0.016935480758547783, 0.010801042430102825, 0.011645739898085594, 0.0037457463331520557, -0.013168963603675365, -0.01789095811545849, -0.023236088454723358, 0.0031710753683000803, 0.02351303957402706, -0.008571596816182137, -0.03154458478093147, 0.013826719485223293, 0.020023470744490623, -0.024773159995675087, -0.0005435140337795019, 0.021989814937114716, 0.0025392835959792137, 0.0026967988815158606, 0.002516781445592642, -0.01938648708164692, -0.01964958943426609, -0.006494473200291395, -0.03611425682902336, 0.005556305404752493, -0.0026829512789845467, -0.011562654748558998, 0.0006806906894780695, -0.021560542285442352, -0.010503321886062622, -0.03478489816188812, 0.012926632538437843, -0.010704110376536846, -0.012504284270107746, -0.04877086356282234, -0.010634873062372208, 0.015301477164030075, 0.014719882979989052, 0.00547668244689703, -0.0026414089370518923, 0.0012886821059510112, 0.0034324468579143286, -0.014622949995100498, -0.02524397522211075, -0.0038184456061571836, -0.03085220977663994, 0.01919262297451496, 0.006404464598745108, -0.022862207144498825, -0.014512170106172562, -0.008530054241418839, 0.009499378502368927, -0.0007213677163235843, -0.01092567015439272, -0.007768442388623953, -0.008952402509748936, 0.015869224444031715, 0.019940387457609177, 0.014747577719390392, -0.003894606838002801, 0.007636891212314367, -0.027598049491643906, -0.02129743993282318, -0.027030302211642265, -0.006771422922611237, -0.0388837531208992, 0.012628911063075066, 0.005743246525526047, 0.020632760599255562, 0.012739690952003002, 0.02000962384045124, -0.023623818531632423, 0.013951347209513187, -0.0033649401739239693, 0.006664104759693146, -0.007318398915231228, 0.00843312218785286, 0.0006664104876108468, 0.005092414561659098, 0.022336002439260483, 0.01396519411355257, -0.017115497961640358, 0.0015102420002222061, -0.01784941554069519, 0.025036262348294258, 0.015924613922834396, -0.008273875340819359, 0.03148919343948364, 0.008336189202964306, 0.0019490342820063233, -0.04439505562186241, 0.009859412908554077, -0.014048279263079166, 0.013134344480931759, -0.0011960769770666957, -0.008073086850345135, -0.026877978816628456, -0.013162040151655674, -0.014733729884028435, 0.004496972542256117, -0.01355669368058443, 0.001865949365310371, -0.0036626611836254597, -0.006397540681064129, -0.007616119924932718, 0.005864412523806095, 0.0029339371249079704, -0.02597789280116558, -0.011431103572249413, 0.003718051128089428, 0.0212420504540205, 0.022917596623301506, -0.005223965737968683, 0.01629849709570408, -0.011888070963323116, -0.027071844786405563, -0.01273276749998331, -0.041902508586645126, -0.004573133774101734, 0.010807966813445091, 0.02849813550710678, 0.017032412812113762, 0.03533879667520523, -0.011230315081775188, 0.0022000200115144253, 0.039936162531375885, -0.011043373495340347, -0.006310993805527687, -0.010918746702373028, -0.015523036941885948, -0.02577017992734909, -0.0011830950388684869, -0.00727685634046793, -0.002025195397436619, 0.009859412908554077, -0.00188672065269202, -0.010690262541174889, 0.014235220849514008, -0.008626986294984818, 0.008516206406056881, -0.014387542381882668, -0.015190697275102139, -0.025700943544507027, 0.020397353917360306, -0.02169901877641678, 0.00239907787181437, -0.025188585743308067, -0.008987021632492542, 0.03154458478093147, 0.0042304084636271, -0.0031208782456815243, 0.014131364412605762, 0.042511794716119766, -0.01656159944832325, -0.028553524985909462, -0.0010653913486748934, 0.02067430317401886, 0.004026157781481743, -0.009734786115586758, -0.05339592322707176, 0.016880091279745102, 0.02201751060783863, 0.002222522161900997, 0.028830476105213165, -0.004742765333503485, 0.002565247705206275, -0.0014565829187631607, 0.01270507276058197, 0.004573133774101734, -0.009319361299276352, -0.00295124645344913, -0.0158830713480711, -0.009810946881771088, -0.004912397358566523, 0.011708053760230541, -0.0026569871697574854, -0.003480912884697318, 0.01974652148783207, -0.011354942806065083, 0.021283593028783798, -0.0338432677090168, -0.02180979773402214, 0.027445726096630096, 0.009900955483317375, 0.03198770433664322, 0.01183268055319786, 0.01384056732058525, 0.020646609365940094, 0.020480439066886902, -0.006373307667672634, 0.006487549282610416, 0.006906435824930668, -0.004188865888863802, 0.010399465449154377, 0.005933649837970734, 0.013265895657241344, -0.01861102692782879, 0.015107612125575542, 0.011008755303919315, -0.039991553872823715, -0.038496024906635284, 0.01789095811545849, 0.014747577719390392, 0.022876054048538208, -0.0309629887342453, -0.007796137128025293, -0.025687094777822495, -0.0002211271203123033, -0.011098763905465603, 0.02012040466070175, 0.007996926084160805, -0.009866337291896343, 0.006653719116002321, 0.030104445293545723, 0.005597847979515791, -0.0026050591841340065, -0.006414850242435932, 0.005452449433505535, 0.021214354783296585, 0.015661511570215225, -0.0072214663960039616, 0.013072031550109386, -0.008322342298924923, 0.027653438970446587, 0.005805560387670994, 0.005511301103979349, -0.0003154631413053721, -0.006369845476001501, 0.020549675449728966, 3.1670726457377896e-05, -0.0005153862875886261, 0.03071373514831066, -0.03226465359330177, -0.014318305067718029, 0.018389467149972916, 0.010108668357133865, -0.005521686747670174, -0.03287394344806671, -0.006584481801837683, 0.0018330615712329745, 0.006207137834280729, 0.024676227942109108, 0.019469572231173515, 0.0024890864733606577, -0.016187716275453568, 0.009298590011894703, -0.004188865888863802, -0.025950197130441666, -0.013535922393202782, -0.015439951792359352, 0.009741709567606449, -0.04669373854994774, 0.008183866739273071, 0.001936917775310576, -0.014034431427717209, 0.023734599351882935, 0.008440045639872551, -0.020854320377111435, 0.017614008858799934, 0.03866219520568848, -0.0071383812464773655, -0.012123477645218372, 0.00040092814015224576, -0.013591311872005463, -0.02577017992734909, 0.010884127579629421, -0.018167907372117043, -0.009194733574986458, 0.029328985139727592, -0.025645552203059196, -0.020715845748782158, -0.011015678755939007, -0.01743399165570736, -0.004441582597792149, 0.0028629687149077654, 0.004375807009637356, -0.001000481192022562, -0.007976154796779156, 0.0014513900969177485, -0.001772478804923594, 0.00632830336689949, -0.00912549626082182, 0.01422137301415205, 0.0014738922473043203, 0.018680265173316002, -0.014996832236647606, -0.002665641950443387, -0.006736804265528917, 0.01574459671974182, -0.021020490676164627, -0.060762789100408554, -0.009083953686058521, -0.016990870237350464, 0.0075884247198700905, 0.00235234247520566, 1.6362757378374226e-05, -0.016367733478546143, 0.0043619596399366856, -0.016367733478546143, 0.007713052444159985, 0.0032195416279137135, 0.012483512982726097, 0.001995769562199712, -0.005719013512134552, -0.004064238630235195, 0.010454855859279633, -0.03697279840707779, -0.02012040466070175, -0.014103669673204422, 0.0021307826973497868, -0.025590162724256516, -0.01398596540093422, -0.011375713162124157, 0.056026946753263474, -0.002451005857437849, -0.012033469043672085, -0.00936090387403965, -0.013189734891057014, -0.045807499438524246, -0.016644684597849846, -0.02309761382639408, 0.02247447706758976, -0.005213580094277859, 0.03337245434522629, -0.0003338543465360999, 0.01974652148783207, 0.0033458999823778868, 0.002044235821813345, -0.013535922393202782, 0.0007629101746715605, -0.00786537490785122, -0.004784307908266783, 0.0027210318949073553, 0.0031416495330631733, -0.037886735051870346, -0.00519280880689621, 0.007962306961417198, -0.020688151940703392, 0.003273200709372759, 0.023983852937817574, 0.0014929325552657247, 0.004351573996245861, 0.001603712560608983, 0.01545379962772131, 0.016783159226179123, 0.01809867098927498, 0.0255624670535326, -0.03547726944088936, -0.006082510109990835, 0.005736323073506355, -0.018223296850919724, -0.01881873980164528, -0.02313915640115738, 0.017600160092115402, 0.013951347209513187, 0.003544957609847188, -0.0012055971892550588, -0.007089915219694376, 0.008737766183912754, -0.016935480758547783, 0.03090759925544262, 0.0012384849833324552, -0.007581501267850399, 0.017170889303088188, -0.0005037024966441095, -0.019261859357357025, -0.024454668164253235, 0.008834699168801308, -0.022460630163550377, 0.014622949995100498, 0.0316830575466156, -0.046832215040922165, -0.010738728567957878, 0.0065914057195186615, 0.002613713964819908, -0.027431879192590714, -0.011278781108558178, -0.003316473914310336, -0.0019317249534651637, -0.019884996116161346, 0.02849813550710678, 0.04334264621138573, -0.02268218994140625, 0.01123723853379488, -0.004285798408091068, 0.015536884777247906, -0.0020338501781225204, -0.015384562313556671, 0.006989520974457264, -0.021837493404746056, 0.014027507975697517, 0.02309761382639408, -0.023623818531632423, -0.020757388323545456, 0.017835568636655807, 0.010115591809153557, -0.03924378752708435, 0.020591218024492264, 0.19907152652740479, -0.022723732516169548, 0.010177905671298504, 0.030159834772348404, 0.029965970665216446, 0.005580538883805275, 0.011015678755939007, 0.01417290698736906, 0.001807097578421235, -0.02012040466070175, 0.006210599560290575, -0.016478514298796654, -0.0026985297445207834, -0.0021896343678236008, 0.019981928169727325, -0.026725657284259796, -0.032015398144721985, -0.05519609898328781, -0.013438989408314228, 0.008924707770347595, -0.0032005012035369873, -0.023069920018315315, -0.00465621892362833, -0.022654494270682335, 0.02880278043448925, 0.0020044243428856134, -0.0024769699666649103, -0.006854508072137833, 0.018735654652118683, 0.013660549186170101, -0.01147264614701271, -0.005403983406722546, 0.013418218120932579, 0.0036834324710071087, -0.017558617517352104, 0.0033130121883004904, 0.008218485862016678, 0.006653719116002321, 0.005919802468270063, 0.010434084571897984, 0.031184548512101173, 0.0052585843950510025, 0.03683432564139366, -0.02669796161353588, 0.0045904433354735374, 0.018957214429974556, -0.01093259360641241, -0.0021567465737462044, -0.005275893956422806, -0.004929706454277039, -0.0331231988966465, 0.01928955502808094, 0.024676227942109108, 0.021685170009732246, -0.0248423982411623, 0.004406963940709829, -0.024980872869491577, -0.0076507385820150375, 0.02180979773402214, 0.0006690068985335529, -0.033870961517095566, -0.016727767884731293, -0.015066069550812244, 0.028830476105213165, -0.01246966514736414, 0.022142138332128525, -0.015163002535700798, -0.022252917289733887, 0.013002793304622173, -0.0012133864220231771, -0.02401154860854149, 0.010067125782370567, -0.0212420504540205, 0.00044831252307631075, -0.027071844786405563, -0.05004483088850975, 0.022432934492826462, 0.02010655589401722, 0.022308306768536568, 0.04298261180520058, 0.0025600548833608627, -0.030270613729953766, 0.003210886847227812, -0.020923558622598648, -0.023236088454723358, -0.03550496697425842, 0.017627855762839317, -0.011590349487960339, -0.008093858137726784, -0.011964231729507446, -0.017572466284036636, -0.006130976602435112, -0.005538996309041977, -0.0044208113104105, -0.004805079195648432, 0.004621599800884724, 0.0067125712521374226, 0.00435849791392684, 0.012331190519034863, -0.033040113747119904, -0.014678340405225754, 0.08125707507133484, 0.015439951792359352, 0.00959631148725748, 0.01396519411355257, 0.011112610809504986, 0.002101356629282236, 0.01542610488831997, -0.009187810122966766, -0.011451874859631062, 0.000480767572298646, -0.022959139198064804, 0.011368789710104465, 0.00542821642011404, -0.0005474085919559002, 0.01218579150736332, 0.004642371088266373, -0.0145814074203372, 0.025964045897126198, -0.017447838559746742, 0.014429084956645966, -0.012940480373799801, -0.017032412812113762, 0.002695067785680294, -0.016852395609021187, -0.047912318259477615, 0.004053852986544371, 0.010994907468557358, -0.030021360144019127, 0.009250123985111713, -0.01093259360641241, -0.047136858105659485, 0.015980003401637077, 0.011431103572249413, -0.005975192412734032, -0.003846140578389168, 0.006269451230764389, -0.0032628150656819344, -0.02041120082139969, 0.018486399203538895, 0.01188114657998085, 0.024814702570438385, -0.013792100362479687, -0.00456274813041091, 0.005338207818567753, -0.013508226722478867, 0.011846528388559818, 0.021020490676164627, 0.0026033283211290836, -0.004396578297019005, -0.00010656076483428478, -0.019123384729027748, 0.00933320913463831, 0.003981153480708599, 0.04500434547662735, 0.008668528869748116, -0.019428029656410217, -0.011306475847959518, 0.020480439066886902, 0.0005383212119340897, -0.04342573136091232, -0.0017058377852663398, 0.017101651057600975, -0.009374750778079033, -0.0065671722404658794, -0.007616119924932718, -0.17680476605892181, -0.005719013512134552, 0.007443026173859835, -0.03749900311231613, 0.016187716275453568, -0.009180886670947075, 0.011389560997486115, -0.00500240596011281, -0.014802967198193073, -0.008093858137726784, 0.01390980463474989, -0.004424273036420345, 0.0021203970536589622, -0.0024129252415150404, -0.010801042430102825, -0.003690356155857444, -0.017309363931417465, 0.0043792687356472015, 0.01300971768796444, 0.0080800112336874, 0.03902222961187363, -0.006882202811539173, 0.010641796514391899, -0.023637667298316956, -0.005400521215051413, 0.0023488805163651705, -0.0006850180798210204, 0.032375432550907135, 0.000562987057492137, -0.04409040883183479, -0.002925282344222069, -0.0037630556616932154, 0.021283593028783798, -0.001390807330608368, 0.016492361202836037, -0.0004976441850885749, 0.019511114805936813, -0.0022121365182101727, -0.028608916327357292, -0.002264064736664295, 0.018015585839748383, 0.02313915640115738, 0.01036484632641077, -0.017627855762839317, -0.01629849709570408, -0.01908184215426445, 0.01455371268093586, -0.0161600224673748, 0.015910767018795013, -0.013868262059986591, 0.022460630163550377, 0.00023086363216862082, 0.014401390217244625, 0.0033424380235373974, 0.011451874859631062, 0.011147229932248592, 0.011348018422722816, -0.01120954379439354, -0.01182575710117817, -0.00043078677845187485, -0.0037838267162442207, -0.007131457794457674, 0.006352536380290985, 0.022197527810931206, -0.01933109760284424, -0.021366678178310394, -0.013508226722478867, 0.01985730230808258, -0.02849813550710678, 0.01084950938820839, -0.01177729107439518, -0.0004712474183179438, -0.008571596816182137, -0.019995776936411858, 0.03049217350780964, 0.009547844529151917, -0.014082898385822773, 0.026670267805457115, 0.03226465359330177, -0.00340821361169219, -0.005082028917968273, 0.016603142023086548, 0.007346093654632568, 0.001196942524984479, 0.005348593462258577, -0.0011891532922163606, 0.016575446352362633, 0.022765275090932846, 1.2427582078089472e-05, -0.002502933843061328, 0.03821907564997673, -0.02118666097521782, -0.0017698823940008879, 0.010427160188555717, 0.021311288699507713, 0.016436971724033356, 0.002916627796366811, -0.015758443623781204, 0.01671392098069191, -0.009097801521420479, 0.0056290049105882645, 0.004424273036420345, 0.0024769699666649103, -0.009021639823913574, 0.04619522765278816, 0.012476589530706406, -0.003558805212378502, 0.025700943544507027, 0.042096372693777084, 0.012642758898437023, -0.033870961517095566, -0.013826719485223293, -0.003880759235471487, 0.00410231901332736, 0.014276762492954731, 0.015190697275102139, -0.012926632538437843, -0.024870093911886215, 0.010711033828556538, -0.004555824212729931, 0.03949304297566414, -0.02273757942020893, -0.013951347209513187, 0.020632760599255562, -0.003465334651991725, -0.03791442885994911, -0.12185791879892349, -0.006310993805527687, 0.020480439066886902, 0.017157040536403656, -0.0101017439737916, 0.03132302314043045, -0.0006240025395527482, 0.03976999223232269, -0.013805948197841644, 0.045225903391838074, -0.009215504862368107, -0.006719494704157114, 0.016326190903782845, -0.0057120900601148605, 0.004157708957791328, -0.01491374708712101, 0.025534773245453835, -0.010060202330350876, -0.014152135699987411, 0.04370268061757088, 0.0034826439805328846, -0.012441970407962799, 0.02448236383497715, -0.02277912199497223, -0.01033715158700943, -0.005663623567670584, -0.031156854704022408, 0.025784026831388474, 0.004715070594102144, 0.007166076451539993, 0.024703923612833023, -0.03575422242283821, 0.0031191473826766014, -0.02988288551568985, 0.01881873980164528, 0.008675453253090382, 0.00048552764928899705, -0.0255624670535326, 0.03353862091898918, 0.008827775716781616, -0.011382637545466423, -0.00394653482362628, 0.008516206406056881, -0.00784460362046957, -0.005542458035051823, -0.020494285970926285, -0.017087804153561592, 0.026614876464009285, 0.01578613929450512, -0.010870279744267464, -0.029910579323768616, 0.012213487178087234, -0.04337034001946449, 0.016063088551163673, 0.0070206779055297375, 0.023208394646644592, 0.02689182758331299, 0.017060108482837677, 0.015273782424628735, 0.01062102522701025, -0.024288497865200043, -0.019261859357357025, -0.005293203517794609, 0.012511207722127438, -0.01021252479404211, 0.005815946031361818, -0.0031970394775271416, -0.019580351188778877, 0.012054240331053734, -0.01595230959355831, -0.008523130789399147, 0.015343019738793373, -0.004614676348865032, -0.003480912884697318, -0.008924707770347595, -0.00794153567403555, -0.022502172738313675, -0.006349074654281139, -0.00010526256664888933, 0.010177905671298504, -0.031156854704022408, -0.014387542381882668, 0.016797006130218506, -0.0058090221136808395, -0.016949327662587166, 0.008356960490345955, 0.006972211413085461, -0.017475532367825508, 0.012808929197490215, -0.03749900311231613, -0.008086934685707092, 0.05134649574756622, -0.0006819888949394226, -0.015564579516649246, 0.0011294359574094415, -0.00407462427392602, -0.016672378405928612, -0.0014133094809949398, -0.003686894429847598, 0.01488605234771967, -0.011874223127961159, -0.009478607214987278, -0.06657873839139938, 0.02721031941473484, 0.02118666097521782, -0.021588237956166267, 0.015827681869268417, 0.022349849343299866, 0.016880091279745102, 0.007387636229395866, -0.021103575825691223, -0.00435849791392684, -0.025964045897126198, -0.005497453734278679, -0.02514704316854477, -0.008599291555583477, -0.008959326893091202, -0.021518999710679054, 0.004839697852730751, -0.013459760695695877, 0.007830755785107613, 0.00756072998046875, 0.0038149836473166943, -0.010344075970351696, 0.03589269518852234, 0.02175440825521946, -0.004535053391009569, -0.010773347690701485, -0.05358978733420372, 0.026268688961863518, -0.010794118978083134, -0.0194418765604496, -0.010967212729156017, -0.012116554193198681, -0.006861431524157524, -0.003766517387703061, 0.002546207280829549, 0.009797099977731705, 0.006037505809217691, 0.024551600217819214, 0.006283299066126347, 0.04242870956659317, -0.013868262059986591, -0.04325956106185913, -0.004060776438564062, -0.03378787636756897, 0.0007018946926109493, 0.0002853881160262972, 0.0037907506339251995, -0.01682470180094242, 0.019054146483540535, 0.00974863301962614, 0.023845378309488297, 0.008246180601418018, -0.006134438328444958, -0.03539418429136276, -0.01183268055319786, -0.013930575922131538, 0.010759499855339527, 0.010641796514391899, 0.014830662868916988, 0.005407445132732391, 0.04157016798853874, -0.0023973467759788036, 0.013335133902728558, -0.0019455724395811558, -0.0019628817681223154, 0.0019819221924990416, -0.01815406046807766, -0.0015543808694928885, 0.009714014828205109, -0.0015907305059954524, 0.012324267067015171, -0.0038184456061571836, 0.02093740552663803, 0.021879035979509354, 0.024703923612833023, -0.02762574329972267, 0.0010731805814430118, 0.005639390554279089, -0.012975098565220833, 0.02787499874830246, 0.023776141926646233, 0.01661698892712593, -0.013480531983077526, 0.01002558320760727, 0.018527941778302193, -0.003396097104996443, 0.0047116088680922985, 0.023831531405448914, -0.021062033250927925, 0.0006871817167848349, 0.018541790544986725, 0.006961825769394636, -0.0039292252622544765, 0.005757094360888004, -0.011749595403671265, 0.012220410630106926, -0.005037024617195129, 0.004123090300709009, 0.025493230670690536, 0.02036965824663639, 0.01917877420783043, -0.005088952835649252, -0.0011675165733322501, 0.0013033950235694647, -0.0324862115085125, 0.011313400231301785, -0.05414368957281113, -0.03436947241425514, 0.00931243784725666, 0.011583426035940647, 0.014117516577243805, 0.007602272555232048, 0.005971730221062899, 0.0026466017588973045, -0.02360997162759304, 0.025784026831388474, 0.010794118978083134, -0.02088201604783535, -0.030326005071401596, 0.026988759636878967, 0.026808742433786392, 0.010434084571897984, 0.04938015341758728, -0.016478514298796654, 0.038080599159002304, 0.0003704203700181097, 0.036751240491867065, -0.023679209873080254, 0.028304271399974823, 0.02365151420235634, -0.007934612222015858, -0.011998850852251053, 0.0034861057065427303, -0.02041120082139969, -0.005857488606125116, -0.010475627146661282, -0.016880091279745102, 0.02406693808734417, 0.02705799601972103, 0.08452508598566055, 0.013058183714747429, 0.0010281761642545462, -0.008440045639872551, 0.0034428322687745094, -0.0039499965496361256, 0.009180886670947075, 0.023997701704502106, -0.007519187405705452, 0.003160689724609256, 0.007117610424757004, -0.0048500834964215755, -0.010821813717484474, -0.020328115671873093, 0.006203675642609596, -0.006335226818919182, -0.0005863546975888312, -0.0010740460129454732, -0.018846435472369194, -0.007117610424757004, 0.04890933632850647, -0.013058183714747429, 0.02201751060783863, 0.004015772137790918, -0.028096558526158333, 0.0032489674631506205, 0.036031171679496765, 0.011361866258084774, -0.017309363931417465, -0.02448236383497715, 0.03932687267661095, 0.00787229835987091, -0.031239939853549004, -0.021117422729730606, -0.014899900183081627, -0.012892013415694237, 0.005781327374279499, -0.015869224444031715, 0.002992789028212428, 0.012012697756290436, 0.007436102256178856, -0.009575540199875832, -0.010441008023917675, 0.004213098902255297, 0.00915319100022316, -0.0004184538556728512, -0.014733729884028435, -0.0151214599609375, -0.01928955502808094], 'text': 'performance based on models trained with no more than 1/1,000th the compute of\\nGPT-4.\\n1 Introduction\\nThis technical report presents GPT-4, a large multimodal model capable of processing image and\\ntext inputs and producing text outputs. Such models are an important area of study as they have the\\npotential to be used in a wide range of applications, such as dialogue systems, text summarization,\\nand machine translation. As such, they have been the subject of substantial interest and progress in\\nrecent years [1–34].\\nOne of the main goals of developing such models is to improve their ability to understand and generate\\nnatural language text, particularly in more complex and nuanced scenarios. To test its capabilities\\nin such scenarios, GPT-4 was evaluated on a variety of exams originally designed for humans. In\\nthese evaluations it performs quite well and often outscores the vast majority of human test takers.', 'SimilarityScore': 0.8395394297757695}), 0.8395394297757695)\n", - "(Document(page_content='2 GPT-4 Observed Safety Challenges\\nGPT-4 demonstrates increased performance in areas such as reasoning, knowledge retention, and\\ncoding, compared to earlier models such as GPT-2[ 22] and GPT-3.[ 10] Many of these improvements\\nalso present new safety challenges, which we highlight in this section.\\nWe conducted a range of qualitative and quantitative evaluations of GPT-4. These evaluations\\nhelped us gain an understanding of GPT-4’s capabilities, limitations, and risks; prioritize our\\nmitigation efforts; and iteratively test and build safer versions of the model. Some of the specific\\nrisks we explored are:6\\n•Hallucinations\\n•Harmful content\\n•Harms of representation, allocation, and quality of service\\n•Disinformation and influence operations\\n•Proliferation of conventional and unconventional weapons\\n•Privacy\\n•Cybersecurity\\n•Potential for risky emergent behaviors\\n•Interactions with other systems\\n•Economic impacts\\n•Acceleration\\n•Overreliance', metadata={'id': '4a3bcc96-5927-477b-bc8c-f499ffceb310', 'embedding': [3.201593426638283e-05, -0.01149508636444807, -0.006550004705786705, -0.034101177006959915, -0.004657018929719925, 0.020493626594543457, -0.018833834677934647, 0.004362097475677729, -0.02326451987028122, -0.022660959511995316, 0.025678761303424835, 0.029574472457170486, -0.047050297260284424, -0.020493626594543457, 0.01711917482316494, 0.03264714404940605, 0.038161493837833405, -0.008086340501904488, -0.0072015756741166115, -0.0050582499243319035, 0.0031618347857147455, 0.01932765729725361, -0.02500661462545395, -0.015720011666417122, 0.0072015756741166115, 0.010905243456363678, 0.03187897428870201, -0.030479812994599342, 0.002822331851348281, 0.011693987064063549, 0.009903881698846817, -0.019492264837026596, -0.02755803056061268, -0.004879924934357405, -0.018532054498791695, -0.019890066236257553, 0.0030623844359070063, -0.020960014313459396, 0.00266629783436656, -0.000522542919497937, 0.0035699240397661924, 0.01788734272122383, 0.019135616719722748, 0.008662466891109943, 0.006735187955200672, 0.024869441986083984, 0.001978718675673008, -0.026789862662553787, -0.02666640654206276, 0.018230274319648743, 0.01909446343779564, 0.02075425535440445, -0.003881992306560278, 0.0019667160231620073, -0.003103536320850253, -0.012537600472569466, -0.001624641241505742, 0.00995189230889082, 0.016556765884160995, 0.01788734272122383, 0.01244157925248146, -0.005867569707334042, -0.017914777621626854, 0.040191650390625, -0.020493626594543457, -0.015665141865611076, -0.021357815712690353, 0.02067195251584053, -0.007908016443252563, -0.012222102843225002, 0.0011471081525087357, 0.020617082715034485, 0.00737990066409111, -0.01684482954442501, 0.03253740444779396, -0.01746210642158985, 0.0022444911301136017, -0.019231636077165604, -0.008545869961380959, 0.012777653522789478, 0.024704834446310997, -0.021975094452500343, 0.003437895094975829, 0.023895515128970146, 0.005078825633972883, 0.009300320409238338, 0.006419690325856209, 0.012084930203855038, -0.00858016312122345, -0.0016589344013482332, 0.009807860478758812, -0.007908016443252563, 0.013257757760584354, 0.02700933814048767, -0.008086340501904488, 0.02126179449260235, -0.008751628920435905, 0.01816168799996376, -0.0294098649173975, -0.021700749173760414, -0.011467652395367622, 0.002376520074903965, -0.035143692046403885, -0.012661056593060493, -0.013484093360602856, -0.008916236460208893, 0.01684482954442501, 0.007064403034746647, 0.006632308475673199, 0.015006712637841702, -0.004924505949020386, 0.025719914585351944, -0.008875085040926933, 0.005288014188408852, -0.02932756021618843, 0.0038134059868752956, 0.016104094684123993, 0.007798277772963047, -0.0022119125351309776, -0.016323572024703026, 0.02396410144865513, -0.01573372818529606, 0.018641794100403786, -0.0032338504679501057, 0.009732414968311787, -0.006159062031656504, 0.0047496105544269085, -0.015198754146695137, -0.005613799672573805, -0.015623990446329117, 0.002990368753671646, 0.008573304861783981, 0.0280930045992136, -0.019848914816975594, -0.044690921902656555, 0.033991437405347824, -0.04027395695447922, -0.01603550836443901, -0.0275854654610157, -0.01839488185942173, 0.002345656044781208, 0.02233174443244934, -0.006316810846328735, 0.008998540230095387, 0.003665945027023554, 0.01750325784087181, 0.0195882860571146, 0.05094600468873978, -0.0031223974656313658, 0.002926926128566265, -0.0031258268281817436, -0.014073937200009823, -0.01336749643087387, 0.01735236868262291, -0.022551219910383224, -0.016680222004652023, -0.008683042600750923, 0.01866922900080681, -0.02032901905477047, -0.03923144191503525, -0.00860073883086443, -0.007503356318920851, -0.0090602682903409, 0.010157651267945766, 0.0017309501999989152, 0.01777760498225689, 0.016131529584527016, 0.00958152487874031, 0.007791419047862291, -0.003028091276064515, -0.015116450376808643, 0.026693841442465782, -0.015212471596896648, 0.029382430016994476, 0.005860710982233286, 0.012709066271781921, 0.012084930203855038, -0.0072975968942046165, -0.016392158344388008, -0.010150793008506298, -0.008929953910410404, -0.0005645520868711174, 0.022894153371453285, -0.0003515054995659739, -0.006738617550581694, -0.03218075633049011, 0.03404630720615387, 0.014540324918925762, -0.028257612138986588, -0.006896366365253925, 0.017420955002307892, -0.00716728251427412, -0.03829866647720337, -0.008223514072597027, -0.6140955090522766, -0.019698023796081543, 0.01573372818529606, -0.02086399495601654, 0.010754353366792202, 0.025953108444809914, 0.0176678653806448, 0.018340013921260834, -0.004807909019291401, 0.05146726220846176, -0.006111051421612501, -0.009560949169099331, -0.01312058512121439, -0.025322113186120987, -0.0028789157513529062, -0.028778869658708572, 0.0029114943463355303, -0.030232900753617287, 0.01784619130194187, 0.023936666548252106, -0.013154878281056881, 0.014581476338207722, -0.0018792683258652687, -0.007338748779147863, -0.00879278127104044, 0.02569247968494892, 0.00022912156418897212, -0.010576028376817703, -0.011371631175279617, 0.011014982126653194, -0.00948550458997488, 0.0228804349899292, 0.002378234639763832, 0.017009437084197998, 0.03972526267170906, -0.028079288080334663, -0.024320749565958977, 0.015445665456354618, 0.01072691846638918, 0.04038369283080101, -0.024663683027029037, -0.01021937932819128, 0.023209650069475174, 0.03289405629038811, -0.0037448194343596697, 0.031247980892658234, 0.02364860288798809, -0.0008749057305976748, -0.020342737436294556, -0.0012954262783750892, -0.00013492237485479563, 0.006508852820843458, 0.015486817806959152, 0.009163147769868374, -0.006248224526643753, 0.006683748215436935, 0.031741801649332047, -0.011508803814649582, 0.0011573961237445474, -0.015212471596896648, -0.00807262398302555, 0.020685669034719467, -0.029848817735910416, -0.05152213200926781, -0.028449654579162598, 0.03393656760454178, -0.013168595731258392, 0.004015735816210508, 0.003223562613129616, -0.022578654810786247, 0.007715974003076553, 0.02886117249727249, -0.016200115904211998, -0.01028110645711422, -0.0003193556040059775, 0.030589550733566284, 0.03187897428870201, -0.0359392911195755, 0.008010895922780037, 0.02709164284169674, 0.01565142534673214, -0.019451113417744637, -0.0045095584355294704, 0.015706293284893036, 0.025761066004633904, -0.005685815587639809, -0.0009061982855200768, 0.019533416256308556, 0.024128708988428116, -0.008751628920435905, 0.015829749405384064, 0.017983363941311836, -0.0310285035520792, -0.032948922365903854, 0.02017812989652157, -0.006865502335131168, -0.011934040114283562, -0.007503356318920851, 0.0008311818819493055, -0.015788597986102104, -0.008628173731267452, -0.020575931295752525, 0.02303132601082325, 0.03245510160923004, 0.02422473020851612, 0.008244089782238007, -0.00490735936909914, -0.009691263549029827, -0.0028737715911120176, -0.01452660746872425, -0.017640432342886925, -0.002530839527025819, -0.007434769533574581, -0.014499172568321228, 0.009519797749817371, -0.03687206655740738, 0.02307247743010521, -0.01333320327103138, -0.0372561514377594, -0.02743457444012165, 0.030424943193793297, -0.0029012062586843967, 0.01189974695444107, -0.028422219678759575, 0.011145295575261116, 0.02315478026866913, 0.0253083948045969, -0.01956085115671158, 0.018710380420088768, 0.02233174443244934, -0.009732414968311787, -0.022784413769841194, 0.011028698645532131, 0.00556578952819109, 0.014855822548270226, 0.0019444255158305168, 0.019848914816975594, -0.0017369515262544155, 0.02315478026866913, -0.00811377540230751, -0.02628232166171074, -0.0018038232810795307, -0.0034121752250939608, 0.004790762439370155, -0.019067028537392616, -0.011069850996136665, -0.0069306595250964165, 0.002038731938228011, 0.0008067479357123375, -0.015898335725069046, -0.017901059240102768, -0.020918862894177437, 0.002997227245941758, 0.029492167755961418, -0.0040500289760529995, -0.021275512874126434, 0.008751628920435905, -0.031001068651676178, -0.01242100354284048, -0.03152232617139816, 0.0005645520868711174, 0.022976456210017204, 0.006213931366801262, 0.0006511425017379224, -0.013106867671012878, -0.00978042557835579, -0.016789959743618965, 0.030315205454826355, -0.0346224345266819, -0.023662321269512177, -0.009958750568330288, -0.04471835494041443, 0.025335829704999924, 0.028778869658708572, -0.00606647040694952, 0.018257709220051765, 0.0005752687575295568, -0.0007424482028000057, 0.005493773613125086, 0.001172828022390604, 0.007331890054047108, 0.0055452133528888226, -0.01141964178532362, -0.007421052549034357, 0.022743262350559235, 0.023401692509651184, 0.02673499286174774, 0.02128922939300537, -0.006858643610030413, 0.02864169515669346, 0.02071310393512249, 0.017187761142849922, -0.018340013921260834, 0.013264616951346397, -0.004262647125869989, 0.006508852820843458, 0.0033538767602294683, 0.007434769533574581, 0.009218016639351845, 0.0385730117559433, 0.029492167755961418, 0.0019804334733635187, 0.0024382479023188353, -0.02148127183318138, 0.01595320552587509, -0.004314086865633726, -0.008710477501153946, -0.012928543612360954, -0.0003549348039086908, 0.009320897050201893, -0.009355190210044384, -0.01588461920619011, -0.01743467152118683, -0.022660959511995316, 0.004739322699606419, 0.005706391762942076, -0.018655510619282722, 0.014581476338207722, -0.016858546063303947, -0.016858546063303947, 0.01688598096370697, 0.018998442217707634, 0.018463468179106712, 0.01769530028104782, -0.018230274319648743, 0.02886117249727249, -0.0011719707399606705, 0.01147451065480709, 0.02098744921386242, -0.023895515128970146, -0.010555452667176723, 0.01573372818529606, 0.02098744921386242, 0.014924408867955208, 0.01854577288031578, 0.006834638305008411, 0.020109543576836586, 0.0063511040061712265, 0.02241404727101326, -0.0044786944054067135, -0.0017643860774114728, -0.007558225188404322, -0.010390845127403736, -0.020644517615437508, 0.020589647814631462, -0.006498564966022968, 0.019300222396850586, -0.00402945326641202, -0.03602159768342972, 0.013394931331276894, 0.004969087429344654, 0.0009953606640920043, -0.03212588652968407, -0.00782571267336607, 0.004684453830122948, -0.001239699893631041, -0.005219427868723869, 0.00395057862624526, -0.0063065229915082455, 0.023909231647849083, 0.009746132418513298, 0.006762622855603695, 0.021152056753635406, 0.018998442217707634, 0.01526734046638012, -0.020150694996118546, -0.00954723171889782, -0.02171446569263935, -0.011282469145953655, -0.0024691116996109486, -0.012962836772203445, -0.01673508994281292, 0.01335377898067236, -0.021001167595386505, 0.005610370542854071, 0.012338699772953987, -0.0033401595428586006, 0.014718648977577686, 0.01310000941157341, 0.009519797749817371, -0.01023995503783226, -0.04400505870580673, 0.009375765919685364, 0.020466193556785583, -0.006361391860991716, 0.003991730511188507, -0.04436170682311058, -0.0034121752250939608, 0.005137124098837376, 0.011124719865620136, -0.017475824803113937, -0.0008011753088794649, -0.0039300029166042805, 0.015171320177614689, 0.0056069414131343365, -0.008381262421607971, 0.014499172568321228, -0.022043680772185326, 0.005017098039388657, -0.0195882860571146, -0.003290434367954731, 0.0027125936467200518, -0.022743262350559235, -0.0009859299752861261, 0.008902519941329956, -0.004910788964480162, -0.013806450180709362, -0.014101371169090271, -0.0017849620198830962, -0.03648798540234566, 0.035610079765319824, 0.0012782796984538436, 0.0014317418681457639, -0.016049226745963097, -0.0032887195702642202, 0.01170770451426506, -0.022057397291064262, -0.011714563705027103, 0.024677399545907974, 0.014005349949002266, 0.0004903930239379406, -0.013456658460199833, -0.00666660163551569, 0.02847708761692047, 0.050644226372241974, 0.03473217040300369, -0.015816032886505127, 0.01456775888800621, -0.003919715061783791, -0.0035939293447881937, -0.022043680772185326, -0.030424943193793297, 0.016090378165245056, 0.015185036696493626, 0.02179677039384842, -0.016584200784564018, 0.007078120484948158, -0.010137075558304787, 0.02128922939300537, 0.009211158379912376, -0.0002038303209701553, -0.029272690415382385, -0.012626763433218002, -0.024759704247117043, 0.020809125155210495, -0.014073937200009823, 0.014581476338207722, 0.00853901170194149, -3.365021984791383e-05, -0.005874428432434797, 0.02399153634905815, 0.025719914585351944, -0.017558127641677856, -0.014608911238610744, -0.013065716251730919, 0.005617229267954826, 0.00605618255212903, -0.0072221518494188786, -0.009259168989956379, 0.011769432574510574, 0.014444303698837757, -0.021769335493445396, 9.082344331545755e-05, 0.0008234659326262772, 0.0074484869837760925, 0.016680222004652023, 0.017366085201501846, -0.016940850764513016, 0.0034996229223906994, -0.0013194315833970904, 0.016392158344388008, 0.026831014081835747, -0.024211011826992035, -0.026337191462516785, 0.0019615720957517624, 0.01592577062547207, -0.030123163014650345, 0.005401181988418102, 0.02032901905477047, -0.0008003179682418704, -0.009142572060227394, 0.008943671360611916, -0.02696818672120571, -0.0058469935320317745, -0.01286681555211544, -0.02275697886943817, 0.00827152468264103, -0.008621315471827984, -0.02543185092508793, 0.01886126957833767, -0.015596555545926094, -0.014499172568321228, -0.03733845800161362, 0.01696828380227089, -0.0035664946772158146, -0.0042214952409267426, -0.021357815712690353, -0.009128854610025883, 0.01149508636444807, 0.005997884087264538, 0.011104144155979156, -0.0033933138474822044, 0.0036865209694951773, -0.005901862867176533, 0.002184478100389242, -0.035994160920381546, -0.009437493979930878, -0.02743457444012165, 0.028888607397675514, -0.01805195026099682, -0.008648749440908432, -0.01973917707800865, -0.00881335698068142, 0.011159013025462627, 0.009163147769868374, -0.012043777853250504, -0.006766051985323429, 0.004464976955205202, 0.012722783721983433, 0.022715827450156212, 0.009163147769868374, 0.01684482954442501, 0.015198754146695137, -0.016172681003808975, -0.004852490499615669, -0.030123163014650345, -0.016748808324337006, -0.04702286049723625, 0.01688598096370697, 0.007592518348246813, 0.002968078013509512, 0.01288739126175642, 0.02275697886943817, -0.024293316528201103, 0.0267761442810297, -0.011975191533565521, 0.005538354627788067, -0.0049313646741211414, 0.0036350812297314405, 0.0077571258880198, 0.0077502671629190445, 0.015939487144351006, -0.006111051421612501, -0.02422473020851612, -0.026954470202326775, -0.026981905102729797, 0.031741801649332047, 0.0028857742436230183, 0.006940947379916906, 0.024197295308113098, 0.025088919326663017, -0.009677546098828316, -0.06606245785951614, 0.015308492816984653, -0.019423678517341614, 0.02364860288798809, -0.0019272788194939494, -0.014924408867955208, 0.005836705677211285, -0.01045257318764925, -0.023621167987585068, 0.0118448780849576, -0.006429978646337986, 0.007798277772963047, -0.0316869355738163, 0.003923144191503525, 0.004269505850970745, -0.0009687833953648806, 0.034649867564439774, -0.03209845349192619, -0.017914777621626854, 0.011838018894195557, -0.0034258924424648285, 0.015377079136669636, 0.009842153638601303, 0.013161737471818924, -0.030342640355229378, -0.012853098101913929, -0.01626870222389698, -0.023936666548252106, 0.011344196274876595, 0.0021776193752884865, 0.0280930045992136, 0.019135616719722748, 0.03275688365101814, -0.016954567283391953, 0.0051336949691176414, 0.03473217040300369, 0.0012405571760609746, 0.001189117319881916, 0.011762574315071106, -0.007475921418517828, -0.023401692509651184, 0.006570580415427685, -0.008854509331285954, -0.015569121576845646, 0.015445665456354618, 0.01427969615906477, 0.0005894146743230522, 0.01785990782082081, -0.01408765371888876, 0.01004791259765625, -0.03706410899758339, -0.04115186259150505, -0.03547290340065956, 0.01939624361693859, -0.013051998801529408, -0.011172730475664139, -0.032976359128952026, -0.003926573321223259, 0.020274151116609573, 0.00077631272142753, 0.004423825070261955, -0.0011179589200764894, 0.028202742338180542, -0.0017275208374485373, -0.015171320177614689, 0.008497859351336956, 0.019190484657883644, -0.010624038986861706, -0.028202742338180542, -0.045239612460136414, -0.014951842837035656, 0.02477342076599598, 0.0067489054054021835, 0.037777408957481384, -0.0018947003409266472, -0.000686293060425669, -0.024197295308113098, 0.017750170081853867, -0.006622020620852709, -0.00037379609420895576, 0.011543096974492073, -0.026309756562113762, 0.013196030631661415, -0.01669393852353096, -0.0011393921449780464, 0.006364821456372738, -0.009807860478758812, 0.005737255327403545, -0.02283928357064724, 0.03838096931576729, -0.02875143475830555, -0.019876349717378616, 0.0077571258880198, 0.015610272996127605, 0.018710380420088768, 0.01641959324479103, 0.013086291961371899, 0.023278236389160156, 0.006128198001533747, 0.0008804783574305475, 0.015431948006153107, 0.030671855434775352, -0.013209748081862926, 0.007393618114292622, 0.011405924335122108, -0.01116587221622467, -0.010267389938235283, 0.0037928300444036722, 0.008888802491128445, -0.03478704020380974, -0.04757155105471611, 0.01595320552587509, 0.009204300120472908, 0.002568562049418688, -0.024992898106575012, -0.014115088619291782, -0.0322904959321022, -0.0038957095239311457, 0.0024399624671787024, 0.024951744824647903, 0.01565142534673214, -0.00286005437374115, -0.008854509331285954, 0.023950383067131042, -0.0022393472027033567, -0.002558274194598198, -0.008086340501904488, -0.005792124662548304, 0.006241365801542997, 0.011625400744378567, -0.0034653297625482082, -0.0006575724692083895, 0.0033778820652514696, 0.025747347623109818, -0.0019512841245159507, -0.012709066271781921, 0.019121898338198662, -0.020493626594543457, 0.025171222165226936, 4.68584694317542e-05, -0.01975289359688759, 0.030205465853214264, -0.012667914852499962, -0.017805039882659912, -0.006495135370641947, -0.011364772915840149, -0.0010605177376419306, -0.02562389336526394, -0.005850423127412796, 0.00723586929962039, 0.007818853482604027, 0.010877808555960655, 0.01839488185942173, 0.03700924292206764, -0.016872262582182884, 0.011522521264851093, 0.0006867217016406357, -0.033333007246255875, -3.190876668668352e-05, -0.00453013414517045, 0.034018874168395996, -0.04238641634583473, 0.012331841513514519, 0.00321670388802886, -0.01584346778690815, 0.011152154766023159, 0.010836657136678696, -0.016172681003808975, 0.008196079172194004, 0.023676037788391113, -0.007064403034746647, 0.00877220556139946, 0.003696809057146311, 0.011488228105008602, -0.0228804349899292, 0.00440667849034071, -0.019121898338198662, -0.012537600472569466, 0.016789959743618965, -0.01850462146103382, -0.01843603327870369, -0.030013425275683403, -0.020301586017012596, 0.021769335493445396, 0.011591107584536076, 0.0015226189279928803, 0.027023056522011757, -0.020809125155210495, -0.0018158259335905313, 0.006474559661000967, -0.009108278900384903, 0.006409402471035719, -0.003587070619687438, 0.021467555314302444, 0.01603550836443901, -0.007798277772963047, -0.0019444255158305168, -0.0228804349899292, 0.019080746918916702, -0.012962836772203445, -0.05212569236755371, -0.00667688949033618, 0.0021741900127381086, -0.003070957725867629, -0.008799639530479908, -0.015610272996127605, -0.006851784884929657, 0.0021124621853232384, -0.013264616951346397, 0.020932581275701523, 0.030150597915053368, 0.0018158259335905313, 0.0038922803942114115, -0.015445665456354618, 0.002902920823544264, 0.005366888828575611, -0.017132891342043877, -0.026762427762150764, -0.005706391762942076, -0.0099724680185318, -0.018490903079509735, -0.007839430123567581, 0.010397703386843204, 0.055280666798353195, 0.002047305228188634, -0.014361999928951263, -0.022304309532046318, -0.01956085115671158, -0.03223562613129616, -0.017338650301098824, -0.023278236389160156, 0.025335829704999924, 0.016817394644021988, 0.023950383067131042, 0.0029183528386056423, 0.017297498881816864, 0.016954567283391953, 0.0031978425104171038, -0.02979394793510437, 0.0030640990007668734, 0.002451965119689703, -0.021316664293408394, 0.005123406648635864, -0.0014025926357135177, -0.046967990696430206, -0.005654951557517052, 0.012613045983016491, -0.01172828022390604, -0.008888802491128445, 0.0181205365806818, -6.628235860262066e-05, -0.012770794332027435, -0.006916942074894905, -0.0063956850208342075, 0.022304309532046318, 0.000931918213609606, 0.008360686711966991, -0.054814279079437256, 0.010548594407737255, 0.006364821456372738, -0.010233096778392792, -0.019876349717378616, -0.013456658460199833, 0.015912054106593132, 0.0035973587073385715, -0.004194060806185007, 0.001626355922780931, -0.016488179564476013, 0.00641283206641674, -0.014608911238610744, 0.03053468093276024, 0.019313940778374672, 0.010630897246301174, 0.02218085341155529, 0.0029372142162173986, 0.00737990066409111, -0.016474461182951927, 0.012263255193829536, 0.01336749643087387, -0.0053737470880150795, 0.04060317203402519, -0.045075006783008575, 0.012338699772953987, 0.005970449186861515, 0.004735893569886684, -0.019766611978411674, -0.017009437084197998, 0.015898335725069046, -0.031083373352885246, -0.03423834964632988, 0.03357991948723793, 0.014211109839379787, -0.017997080460190773, 0.007496497593820095, -0.016488179564476013, 0.00225992314517498, 0.015980640426278114, -0.020548496395349503, -0.006063040811568499, -0.021206926554441452, 0.006584297865629196, -0.0025754207745194435, 0.006618591025471687, 0.01618639938533306, 0.00175581278745085, 0.015102732926607132, -0.01565142534673214, 0.01703687012195587, 0.1922615021467209, -0.009142572060227394, 0.0035699240397661924, 0.025953108444809914, 0.036268506199121475, 0.007791419047862291, 0.026254888623952866, 0.004108327440917492, -0.004958799574524164, -0.01650189608335495, 0.008518435060977936, -0.0028309051413089037, -0.0009953606640920043, 0.00021476128313224763, 0.013381213881075382, -0.011200165376067162, -0.0316869355738163, -0.022935304790735245, -0.00882707443088293, 0.01234555896371603, -0.013484093360602856, -0.017475824803113937, -0.009348331019282341, -0.016954567283391953, 0.021165775135159492, 0.004835343919694424, -0.016213834285736084, -0.0070506855845451355, 0.012318124063313007, 0.012578752823174, -0.018298860639333725, -0.01023995503783226, 0.003693379694595933, 0.00033907420584000647, -0.005819559097290039, -0.013593832030892372, 0.009087703190743923, -0.006625449750572443, -0.0029852245934307575, 0.0043278043158352375, 0.030013425275683403, 0.019300222396850586, 0.014115088619291782, -0.02318221516907215, -0.008484141901135445, 0.018696662038564682, 0.0032801462803035975, 0.003281861077994108, -0.030287770554423332, 0.009142572060227394, -0.0404934324324131, 0.012757076881825924, 0.03322327136993408, 0.009615818038582802, -0.026254888623952866, -0.00534288352355361, -0.020767973735928535, 0.0005221142782829702, 0.007263303734362125, -0.005850423127412796, -0.015623990446329117, -0.01928650587797165, -0.019108181819319725, 0.0330037921667099, -0.03179667145013809, 0.026364626362919807, 0.009766708128154278, 0.010562310926616192, 0.020932581275701523, -0.009046550840139389, -0.011810583993792534, 0.013731004670262337, -0.039780132472515106, 0.009266027249395847, -0.03772253915667534, -0.03980756923556328, 0.02001352235674858, 0.01924535445868969, 0.0004861063789576292, 0.0294098649173975, 0.004187202081084251, -0.011049275286495686, -0.001172828022390604, -0.019505983218550682, -0.011556814424693584, -0.03956065699458122, 0.026831014081835747, -0.013477235101163387, -0.016817394644021988, 0.017942212522029877, -0.030699288472533226, 0.00021090329391881824, -0.00607675826177001, 0.0029132089111953974, 0.00790115725249052, 0.02886117249727249, 0.03714641556143761, 0.007270162459462881, 0.005888145416975021, -0.04587060958147049, -0.021398968994617462, 0.09300320595502853, 0.0031686932779848576, 0.0008174646063707769, -0.0017360941274091601, 0.006772910710424185, 0.0033727381378412247, 0.02469111792743206, -1.2839863302360754e-05, -0.024595096707344055, -0.005438904277980328, -0.021206926554441452, 0.0020232999231666327, 0.01916304975748062, 0.013230323791503906, 0.020452475175261497, 0.0007613094639964402, 0.007537649478763342, 0.01236613467335701, -0.015870900824666023, 0.011234458535909653, -0.010555452667176723, 0.008840791881084442, 0.020274151116609573, -0.01735236868262291, -0.03818892687559128, -0.001005648635327816, 0.004430683795362711, -0.04331919178366661, -0.023744624108076096, 0.006604874040931463, -0.050068099051713943, 0.011618542484939098, 0.017571846023201942, -0.005826417822390795, -0.007352466229349375, -0.0034550416748970747, -0.00514055322855711, -0.007722832728177309, 0.018874987959861755, 0.011597966775298119, 0.007832570932805538, 0.004759898874908686, -0.008765346370637417, 0.007565083913505077, -0.015171320177614689, 0.006004742346704006, 0.008436131291091442, 0.0024399624671787024, -0.006896366365253925, -0.012228962033987045, -0.0030143738258630037, -0.008422414772212505, 0.01646074466407299, 0.029547037556767464, 0.017709018662571907, -0.018600640818476677, -0.02160472795367241, 0.022043680772185326, -0.0010605177376419306, -0.05421071872115135, 0.017763886600732803, 0.005702962167561054, -0.005123406648635864, -0.011035557836294174, -0.010562310926616192, -0.173935204744339, -0.01000676117837429, 0.00786686409264803, -0.04334662854671478, 0.017132891342043877, -0.010137075558304787, 0.035225994884967804, 0.000820465269498527, -0.03396400436758995, 0.007393618114292622, 0.03497908264398575, -0.006237936206161976, 0.0005619801231659949, -0.015610272996127605, -0.010768070816993713, -0.005068537779152393, -0.024320749565958977, 0.0065019940957427025, 0.02836734987795353, -0.003981442656368017, 0.03621364012360573, -0.022619806230068207, 0.012715925462543964, -0.00025677046505734324, 0.004694741684943438, -0.005788695067167282, -0.021632162854075432, 0.027064207941293716, -0.009752991609275341, -0.0385730117559433, -0.0011882600374519825, 0.005082254763692617, 0.03212588652968407, 0.002820617286488414, -0.0129079669713974, 0.008298958651721478, 0.01314116083085537, -0.004873066209256649, -0.04041112959384918, 0.006714612245559692, 0.02728368528187275, 0.03621364012360573, 0.011062991805374622, -0.009156289510428905, -0.03426578268408775, -0.012256396003067493, 0.008758488111197948, -0.03481447696685791, 0.006639167200773954, -0.00977356731891632, 0.026638971641659737, -0.016556765884160995, 0.01187917124480009, -0.0027588894590735435, 0.0027691773138940334, 0.009142572060227394, 0.004818197339773178, -0.018696662038564682, 0.0022444911301136017, -0.003628222504630685, -0.0055452133528888226, -0.015075298957526684, 0.020960014313459396, -0.007914874702692032, -0.004211207386106253, -0.023826928809285164, -0.004759898874908686, -0.003741390071809292, -0.028422219678759575, 0.016282420605421066, -0.025678761303424835, -0.003779112594202161, -0.014361999928951263, -0.017366085201501846, 0.03423834964632988, 0.02875143475830555, -0.01314116083085537, 0.006193355191498995, -0.0008761917124502361, 0.013196030631661415, 0.01835373044013977, 0.002527410164475441, -0.01615896448493004, -0.01580231450498104, -0.008010895922780037, -0.0038614163640886545, 0.004622725769877434, 0.033717092126607895, -0.007997178472578526, -0.02164587937295437, 0.02828504703938961, -0.02179677039384842, 0.013017705641686916, 0.00877220556139946, 0.021110905334353447, 0.008216654881834984, 0.020960014313459396, -0.009464927949011326, 0.013003988191485405, -0.02075425535440445, 0.017955929040908813, 0.0042523592710494995, -0.016858546063303947, -0.002601140644401312, 0.03308609873056412, -0.005613799672573805, 0.010054771788418293, 0.013724146410822868, 0.03193384408950806, -0.0009199155610986054, -0.01654304936528206, -0.011433359235525131, 0.024992898106575012, 0.005504061467945576, -0.01051430031657219, 0.012715925462543964, -0.020315302535891533, -0.020877711474895477, 0.010370269417762756, -0.008785923011600971, 0.06008172035217285, -0.023593734949827194, -0.02454022690653801, 0.011323620565235615, -0.00908084399998188, -0.03975269943475723, -0.1201634407043457, -0.03550034016370773, 0.029300125315785408, 0.022002529352903366, -0.005658381152898073, 0.022125983610749245, -0.011303044855594635, 0.032866619527339935, -0.01805195026099682, 0.04348380118608475, 0.012084930203855038, -0.024238446727395058, 0.022853000089526176, -0.012036919593811035, 0.005411469843238592, -0.005984166637063026, 0.03094620071351528, -0.023045042529702187, -0.015829749405384064, 0.04822998121380806, 0.01503414660692215, -0.000814035243820399, 0.01611781306564808, -0.01522618904709816, -0.01758556254208088, -0.015555404126644135, -0.0322904959321022, 0.012571893632411957, -0.016049226745963097, 0.013998491689562798, 0.016639068722724915, -0.0035322015173733234, -0.0018895562971010804, -0.019382527098059654, -0.012194668874144554, 0.011275609955191612, 0.007359324488788843, -0.0176678653806448, 0.017050588503479958, -0.005798983387649059, 0.004770186729729176, 0.0018449751660227776, 0.017420955002307892, 0.0034173191525042057, -0.029464732855558395, -0.015212471596896648, -0.02388179674744606, 0.023854361847043037, 0.010095923207700253, -0.008998540230095387, -0.029931120574474335, -0.006632308475673199, -0.027544312179088593, 0.0007137276115827262, 0.012256396003067493, 0.011906605213880539, 0.02311362884938717, 0.016666503623127937, 0.0012199812335893512, -0.007098696194589138, -0.018916139379143715, -0.016872262582182884, -0.0043723853304982185, -0.002184478100389242, -0.008785923011600971, 0.007914874702692032, -0.010178226977586746, -0.012276972644031048, 0.017023153603076935, -0.012084930203855038, 1.7628857676754706e-05, 0.008161786012351513, 0.0002308362309122458, 0.02079540677368641, -0.024828290566802025, -0.006172779481858015, 0.011124719865620136, -0.00475304014980793, 0.021069753915071487, -0.002633719239383936, -0.02585708722472191, -0.006680319085717201, 0.02071310393512249, -0.026145149022340775, 0.00303323520347476, 0.00860073883086443, 0.0030949630308896303, -0.019176768139004707, 0.012496449053287506, -0.018532054498791695, 0.0020627370104193687, 0.030589550733566284, -0.015788597986102104, -0.018367446959018707, 0.0036762331146746874, 0.004588432610034943, -0.01588461920619011, -0.024211011826992035, 0.008504718542098999, 0.013305768370628357, -0.014595193788409233, -0.007619953248649836, -0.04628212749958038, 0.024279598146677017, 0.007558225188404322, -0.010644614696502686, -0.004266076255589724, 0.020850276574492455, 0.02477342076599598, 0.001428312505595386, -0.023593734949827194, 0.011426500044763088, -0.01854577288031578, 0.004938223399221897, -0.0330037921667099, -0.029656775295734406, -0.01769530028104782, -0.02283928357064724, 0.0011359628988429904, 0.003254426410421729, 0.006450554355978966, 0.015980640426278114, -0.0022050540428608656, -0.009622677229344845, 0.006364821456372738, 0.023017607629299164, 0.024992898106575012, 0.015322210267186165, -0.036734893918037415, 0.022345460951328278, -0.023868080228567123, -0.019259070977568626, 0.02847708761692047, -0.0028806303162127733, -0.004722176119685173, 0.00042180658783763647, -0.010390845127403736, 0.006817491725087166, 0.00526400888338685, 0.012791370041668415, 0.026145149022340775, 0.0425235889852047, -0.015143885277211666, -0.03160462900996208, 0.01362126599997282, -0.037201281636953354, 0.015774879604578018, -0.02755803056061268, -0.009855871088802814, -0.004060316830873489, 0.02124807797372341, 0.012743360362946987, 0.026721276342868805, 0.0020627370104193687, -0.026309756562113762, -0.02670755796134472, -0.008881943300366402, -0.0010536591289564967, 0.007366183213889599, -0.0024588238447904587, 0.01603550836443901, -0.019300222396850586, 0.03942348435521126, 0.008449848741292953, 0.006529428996145725, 0.010925819166004658, 0.016940850764513016, 0.0008003179682418704, -0.02705049142241478, -0.016858546063303947, 0.024046404287219048, -0.00725644500926137, 0.001021937932819128, -0.012091788463294506, 0.021206926554441452, 0.02689960040152073, 0.008353828452527523, -0.022386612370610237, 0.008724194951355457, 0.024334467947483063, -0.026831014081835747, 0.003081245580688119, 0.025911955162882805, -0.0013185743009671569, -0.014608911238610744, -0.006237936206161976, 0.013148020021617413, 0.0006575724692083895, -0.022290591150522232, 0.0025857086293399334, -0.014348282478749752, 0.0007900299970060587, 0.017914777621626854, -0.0019375667907297611, -0.014691215008497238, 0.008189220912754536, 0.012064354494214058, 0.031165676191449165, -0.024142425507307053, 0.00556578952819109, 0.02360745146870613, 0.016515614464879036, 0.0010785217164084315, -0.004060316830873489, -0.013463517650961876, 0.002316506812348962, -0.01920420303940773, 0.018833834677934647, -0.04625469446182251, -0.038545578718185425, 0.03204358369112015, 0.03840840607881546, 0.016789959743618965, 0.01901216059923172, -0.007770843338221312, -0.001498613622970879, -0.009903881698846817, 0.006680319085717201, -0.0034190339501947165, -0.019190484657883644, -0.01935509219765663, 0.029931120574474335, 0.016817394644021988, 0.016872262582182884, 0.03525342792272568, -0.009183723479509354, 0.04208463802933693, -0.01522618904709816, 0.06430664658546448, -0.013847601599991322, 0.00716728251427412, 0.02507520094513893, -0.0009507794748060405, 0.005634375847876072, 0.009554090909659863, -0.004036311991512775, 0.0129079669713974, -0.040712907910346985, -0.022619806230068207, 0.011721421964466572, 0.020383888855576515, 0.0711652860045433, 0.029437297955155373, -0.0014077365631237626, -0.002091886242851615, -0.006841497030109167, -0.01375843957066536, 0.003038379130885005, 0.016104094684123993, 0.0008217512513510883, -0.026529233902692795, 0.02063080109655857, -0.015143885277211666, 0.01526734046638012, -0.02681729756295681, 0.008045189082622528, -0.022551219910383224, 0.011405924335122108, 0.016748808324337006, 0.00807948224246502, 0.0012379852123558521, 0.04812024533748627, -0.015706293284893036, 0.048092808574438095, 0.011906605213880539, -0.01595320552587509, -0.010205661877989769, 0.040795210748910904, -0.006131627596914768, -0.014039644040167332, -0.021879073232412338, 0.0051474119536578655, 0.026021694764494896, -0.049217626452445984, -0.02388179674744606, -0.015006712637841702, -0.014540324918925762, -0.0049210768193006516, -0.018888704478740692, 0.011392206884920597, -0.007811995223164558, -0.0013100008945912123, -0.0036419397220015526, 0.004557568579912186, -0.009855871088802814, -0.0027623188216239214, -0.007215293124318123, -0.026405777782201767, -0.006587727461010218, -0.005150841549038887], 'text': '2 GPT-4 Observed Safety Challenges\\nGPT-4 demonstrates increased performance in areas such as reasoning, knowledge retention, and\\ncoding, compared to earlier models such as GPT-2[ 22] and GPT-3.[ 10] Many of these improvements\\nalso present new safety challenges, which we highlight in this section.\\nWe conducted a range of qualitative and quantitative evaluations of GPT-4. These evaluations\\nhelped us gain an understanding of GPT-4’s capabilities, limitations, and risks; prioritize our\\nmitigation efforts; and iteratively test and build safer versions of the model. Some of the specific\\nrisks we explored are:6\\n•Hallucinations\\n•Harmful content\\n•Harms of representation, allocation, and quality of service\\n•Disinformation and influence operations\\n•Proliferation of conventional and unconventional weapons\\n•Privacy\\n•Cybersecurity\\n•Potential for risky emergent behaviors\\n•Interactions with other systems\\n•Economic impacts\\n•Acceleration\\n•Overreliance', 'SimilarityScore': 0.8297811051676229}), 0.8297811051676229)\n", - "(Document(page_content='GPT-4 Technical Report\\nOpenAI∗\\nAbstract\\nWe report the development of GPT-4, a large-scale, multimodal model which can\\naccept image and text inputs and produce text outputs. While less capable than\\nhumans in many real-world scenarios, GPT-4 exhibits human-level performance\\non various professional and academic benchmarks, including passing a simulated\\nbar exam with a score around the top 10% of test takers. GPT-4 is a Transformer-\\nbased model pre-trained to predict the next token in a document. The post-training\\nalignment process results in improved performance on measures of factuality and\\nadherence to desired behavior. A core component of this project was developing\\ninfrastructure and optimization methods that behave predictably across a wide\\nrange of scales. This allowed us to accurately predict some aspects of GPT-4’s\\nperformance based on models trained with no more than 1/1,000th the compute of\\nGPT-4.\\n1 Introduction', metadata={'id': '89def1ae-3e87-4a7b-8e00-3d5a340b83ae', 'embedding': [-0.007440360262989998, -0.016012800857424736, -0.015371742658317089, -0.010420596227049828, 0.025601385161280632, 0.012255111709237099, -0.01030466053634882, 0.014921639114618301, -0.016231033951044083, -0.01673569530248642, 0.031043555587530136, 0.037372294813394547, -0.03641752898693085, -0.01844063587486744, -0.0032530263997614384, 0.02834293060004711, 0.022627970203757286, -0.006424215622246265, -0.001998190302401781, -0.01643562503159046, 0.0033928314223885536, -0.00016473987489007413, -0.028670279309153557, -0.014471534639596939, -0.00011455068306531757, 0.002762003568932414, 0.027483640238642693, -0.037590526044368744, -0.0028267912566661835, -0.0037338195834308863, 0.01684481091797352, -0.005680861417204142, -0.0324893444776535, -0.0021345855202525854, -0.0017279571620747447, -0.010481974110007286, -0.0002007567381951958, -0.005595614667981863, 0.016612939536571503, -0.007269866298884153, 0.0028711196500808, 0.01304620411247015, 0.022764364257454872, -0.004381697159260511, -0.009397631511092186, 0.0026034440379589796, 0.020568402484059334, -0.040045641362667084, -0.009111201390624046, 0.01823604293167591, 0.021714121103286743, 0.030770765617489815, -0.004443074576556683, -0.0011968682520091534, -0.009281695820391178, -0.0032751907128840685, -0.01091843843460083, 0.027019895613193512, 0.009097562171518803, 0.00019191236060578376, -0.00019169924780726433, -0.025410432368516922, -0.01333263423293829, 0.03570827096700668, -0.009377172216773033, -0.007297145202755928, -0.04918412119150162, 0.03480806574225426, -0.004098676610738039, 0.000926635111682117, 0.021332215517759323, 0.013230337761342525, -0.0004607601440511644, -0.013298535719513893, 0.03268029913306236, -0.001023816759698093, -0.005772928241640329, -0.009629503823816776, -0.030170626938343048, 0.0024210154078900814, 0.024714816361665726, -0.007051633670926094, 0.016026441007852554, 0.029652323573827744, 0.00949992798268795, 0.001252278801985085, 0.01105483341962099, 0.003166074398905039, -0.022546133026480675, -0.007788167800754309, 0.01157313585281372, 0.005380792077630758, 0.02295531891286373, 0.03319859877228737, -0.006734514608979225, 0.007897283881902695, -0.00340306106954813, 0.036908552050590515, 0.0036724417004734278, -0.03554459661245346, -0.0012258521746844053, -0.0042828102596104145, -0.015426301397383213, -0.01597188226878643, -0.006315099541097879, 0.0072357673197984695, 0.012036879546940327, 0.010747944936156273, 0.014171465300023556, 0.017349474132061005, -0.017526788637042046, 0.03243478760123253, -0.005691091064363718, -0.013925953768193722, -0.020527483895421028, 0.0027091503143310547, 0.02459206059575081, -0.013428110629320145, -0.006642447784543037, 0.002199373207986355, 0.019613634794950485, -4.1344806959386915e-05, 0.01182546652853489, 0.0027227899990975857, 0.023978281766176224, 0.012296030297875404, -0.014144185930490494, -0.02005009911954403, 0.0013017220189794898, -0.004016839899122715, 0.018972577527165413, 0.014430616050958633, 0.011157129891216755, -0.004957966972142458, -0.03434431925415993, 0.020159216597676277, -0.02770187333226204, 0.007774528581649065, -0.039363663643598557, -0.01994098350405693, 0.019340844824910164, 0.028097419068217278, -0.022450655698776245, -0.0032240424770861864, -0.009765898808836937, 0.009459009394049644, 0.0211685411632061, 0.013530407100915909, 0.007310784421861172, 0.009718160144984722, 0.01261655893176794, -0.009349893778562546, -0.023978281766176224, 0.0222324226051569, 0.014116906560957432, 0.0002813151804730296, 0.005527416709810495, 0.016503823921084404, -0.02665162831544876, -0.02287348173558712, -0.009227138012647629, 0.017131241038441658, 0.002131175482645631, 0.004453304223716259, 0.013203058391809464, 0.029734160751104355, 0.02277800440788269, 0.019354483112692833, -0.004886359442025423, -0.015835486352443695, -0.021250378340482712, 0.026406116783618927, -0.05153011903166771, 0.009738619439303875, -0.0009445369942113757, 0.013012105599045753, 0.00963632296770811, -0.02097758650779724, -0.0024039659183472395, -0.007378982380032539, 0.007706330623477697, 0.02607876993715763, 0.0008524702279828489, 0.01078204344958067, -0.008183714002370834, -0.03508085384964943, 0.01957271620631218, -0.00013927232066635042, -0.004763603676110506, -0.0274154432117939, 0.028752116486430168, 0.0117231709882617, -0.003522407030686736, -0.015017115511000156, -0.6149788498878479, -0.020759355276823044, -0.01157995592802763, -0.02249157428741455, 0.02457842230796814, 0.002790987491607666, 0.0006905008922331035, 0.006212803069502115, -0.017076684162020683, 0.013394012115895748, -0.008770213462412357, -0.01852247305214405, -0.021141260862350464, -0.005217117723077536, 0.0054830885492265224, -0.03731773793697357, -0.0173085555434227, -0.016381068155169487, 0.010147805325686932, 0.012677936814725399, -0.018686147406697273, 0.012527902610599995, -0.007365342695266008, -0.001481593237258494, -0.021345853805541992, -0.004613569006323814, -0.00874293502420187, -0.01939540170133114, -0.005660402122884989, 0.0076995110139250755, -0.03358050808310509, 0.015589975751936436, 0.009322614409029484, 0.0012019829591736197, 0.040427546948194504, -0.008020039647817612, -0.034644391387701035, 0.016012800857424736, -0.006158244796097279, 0.02636519819498062, -0.01852247305214405, -0.023869166150689125, 0.0031814188696444035, 0.0056297131814062595, -0.011668612249195576, 0.029543207958340645, 0.013257617130875587, -0.0056058443151414394, -0.007508557755500078, -0.011259426362812519, -0.03224383294582367, 0.007774528581649065, 0.02371913194656372, 0.0023408832494169474, 0.003108106553554535, 0.01285525131970644, 0.04190061613917351, 0.0167220551520586, 0.0037883776240050793, -0.024987606331706047, 0.002731314627453685, 0.01185956597328186, -0.008108696900308132, -0.03120722994208336, -0.021673202514648438, 0.008592899888753891, 0.007338063791394234, 0.011395822279155254, -0.0007403703639283776, -0.045501451939344406, 0.002144815167412162, 0.024973968043923378, 0.010563811287283897, -0.008013220503926277, 0.0034132907167077065, 0.010502433404326439, 0.023773688822984695, -0.02022741362452507, 0.021427690982818604, 0.02024105377495289, 0.027483640238642693, -0.022737085819244385, 0.002279505366459489, 0.004726095125079155, 0.00765859242528677, 0.016762973740696907, -0.010413776151835918, 0.011975501663982868, 0.019218089058995247, 0.003897493937984109, 0.017431311309337616, 0.01775866001844406, -0.029243137687444687, -0.03576283156871796, 0.014717046171426773, 0.009684061631560326, -0.02787918597459793, -0.0021379953250288963, 0.0010357513092458248, -0.00949992798268795, -0.012125536799430847, -0.022450655698776245, 0.042009733617305756, 0.03404425084590912, 0.040700338780879974, 0.02365093305706978, -0.018304239958524704, 0.0019351074006408453, -0.00340306106954813, -0.01512623205780983, -0.01635378785431385, -0.001277852919884026, -0.019231727346777916, -0.013380372896790504, 0.020841192454099655, -0.022136947140097618, 0.008449684828519821, -0.0061343759298324585, -0.020541122183203697, -0.010461514815688133, 0.024182874709367752, -0.0016256606904789805, 0.007938202470541, 0.003774738172069192, 0.010897979140281677, 0.012602919712662697, 0.008626998402178288, -0.03221655637025833, -0.001570250140503049, 0.00802685972303152, -0.006809532176703215, -0.01665385812520981, 0.05739511549472809, -0.0061070965602993965, 0.02059568092226982, 0.02249157428741455, 0.047601938247680664, 0.006997075397521257, 0.010250101797282696, 0.0005685976357199252, -0.022928038612008095, 0.018386077135801315, -0.003399651264771819, 0.014048709534108639, -0.02731996588408947, -0.009738619439303875, -0.007460819557309151, 0.0009939803276211023, 0.006795892491936684, -0.017840497195720673, -0.0076995110139250755, 0.005162559449672699, -0.005813846830278635, 0.010536531917750835, 0.001232671900652349, -0.038436178117990494, -0.012930268421769142, -0.047220028936862946, -0.005524007137864828, -0.025083083659410477, 0.003877034643664956, 0.010393316857516766, -0.0017884825356304646, 0.0010391612304374576, 0.0011363427620381117, -0.013966872356832027, -0.006860680412501097, 0.04056394472718239, -0.04061850160360336, -0.032271113246679306, 0.0029018085915595293, -0.04634710028767586, 0.0069902557879686356, 0.00817007478326559, -0.0292158592492342, 0.0071130115538835526, 0.005878634750843048, -0.008238271810114384, 0.008415586315095425, -0.029461370781064034, -0.012664297595620155, 0.029815997928380966, -0.013769099488854408, -0.020868470892310143, 0.013994150795042515, 0.029843278229236603, 0.041464149951934814, 0.021032145246863365, -0.013059844262897968, 0.0018600900657474995, 0.0016120212385430932, 0.015044394880533218, -0.023951003327965736, 0.035871945321559906, -0.0013307060580700636, 0.023023515939712524, 0.011477659456431866, 0.0009487993665970862, 0.013741820119321346, 0.040427546948194504, 0.006298049818724394, 0.019627274945378304, 0.006267360877245665, -0.020159216597676277, -0.002550591016188264, -0.008981626480817795, 0.009895474649965763, -0.019231727346777916, 0.0006973206182010472, 0.003111516358330846, 0.008190534077584743, -0.011654973030090332, -0.024810293689370155, -0.005513777490705252, 0.010045508854091167, 0.014962557703256607, -0.008770213462412357, 0.013203058391809464, -0.01635378785431385, -0.023214468732476234, 0.005486498586833477, 0.004272580612450838, 0.0034439796581864357, 0.0033161090686917305, -0.015508138574659824, 0.03213471919298172, -0.014580650255084038, 0.008838411420583725, 0.023187190294265747, -0.020445646718144417, 0.007863185368478298, 0.01702212542295456, 0.016749335452914238, 0.011464019306004047, 0.02269616723060608, -0.00546262925490737, 0.019995542243123055, -0.002306784503161907, 0.03194376453757286, -0.010877519845962524, -0.011361722834408283, 0.0055205971002578735, -0.004125955980271101, -0.03750868886709213, 0.022764364257454872, -0.011920943856239319, 0.023555457592010498, -0.007440360262989998, -0.029925115406513214, -0.010366037487983704, -0.005762698594480753, 0.01332581415772438, -0.025219479575753212, 0.00029154482763260603, 0.016476543620228767, -0.006550381425768137, -0.006250311620533466, -0.001981140812858939, -0.009595404379069805, 0.0259832926094532, 0.010897979140281677, 0.016203753650188446, 0.01627195253968239, -0.005476268939673901, 0.013966872356832027, -0.02126401662826538, 0.0016572021413594484, -0.023282667621970177, -0.01015462540090084, -0.004095267038792372, -0.013346273452043533, -0.005813846830278635, 0.011873205192387104, -0.04222796484827995, 0.020827552303671837, 0.022164225578308105, 0.009118021465837955, 0.02040472812950611, 0.015071673318743706, 0.015726370736956596, -0.015617254190146923, -0.047792889177799225, 0.020254692062735558, 0.01777229830622673, -0.002690396038815379, -0.0022437016014009714, -0.04162782430648804, 0.0085997199639678, -0.005909323692321777, -0.010516072623431683, -0.013107581995427608, 0.009895474649965763, -0.00439874641597271, 0.008572440594434738, -0.0008916838560253382, 0.015999160706996918, 0.02967960387468338, -0.016694776713848114, -0.006887959316372871, -0.0002951678179670125, 0.01882254332304001, 0.008633818477392197, -0.011416281573474407, -0.012384687550365925, 0.01994098350405693, -0.022532492876052856, -0.029188580811023712, -0.003074007574468851, -0.007910924032330513, -0.03101627714931965, 0.025451350957155228, -0.0032445017714053392, 0.0043612378649413586, -0.010134166106581688, -0.0020118297543376684, 0.020350169390439987, -0.01853611320257187, -0.001953861676156521, 0.02298259735107422, 0.014757964760065079, 0.012684756889939308, -0.005762698594480753, -0.014116906560957432, 0.02618788555264473, 0.05717688426375389, 0.0466744489967823, -0.015194429084658623, -0.004797702189534903, -0.019259007647633553, 0.01063200831413269, -0.022900760173797607, -0.026215163990855217, -0.004221432376652956, 0.025178560987114906, 0.009718160144984722, -0.023814607411623, -0.009486288763582706, -0.0010604729177430272, 0.03169825300574303, 0.006298049818724394, -0.010113706812262535, -0.01627195253968239, -0.008449684828519821, -0.02676074579358101, -0.008401946164667606, -0.01684481091797352, 0.005595614667981863, 0.007378982380032539, 0.009465829469263554, -0.022818922996520996, 0.016872091218829155, 0.031916484236717224, -0.027279047295451164, -0.030116068199276924, -0.012746134772896767, -0.008265551179647446, -0.003445684676989913, 0.0037951974663883448, 0.0014219203731045127, -0.0048249815590679646, 0.008777033537626266, -0.017635904252529144, 0.012241472490131855, 0.01786777563393116, 0.0067583839409053326, 0.008592899888753891, -0.002182323718443513, -0.017349474132061005, 0.001904418459162116, 0.0017151701031252742, 0.0005801059887744486, 0.020472925156354904, -0.0044055660255253315, -0.029543207958340645, -0.0054251207038760185, 0.005977521184831858, -0.04026387259364128, -0.0008089942275546491, 0.019900064915418625, 0.021100342273712158, 0.0073448834009468555, 8.51404620334506e-05, -0.01834515854716301, -0.015371742658317089, -0.022246062755584717, -0.03685399144887924, -0.0011960157426074147, -0.013796377927064896, -0.007297145202755928, 0.0031848286744207144, -0.00963632296770811, -0.018099647015333176, -0.02534223534166813, 0.008947527036070824, -0.00936353299766779, -0.017731381580233574, -0.03944550082087517, -0.014812522567808628, 0.007644952740520239, 0.006686776410788298, 0.010747944936156273, -0.0077131506986916065, 0.015617254190146923, -0.006976616103202105, -0.005639942828565836, -0.022655248641967773, 0.002530131721869111, -0.030034231022000313, 0.019804587587714195, -0.00524439662694931, -0.012793873436748981, -0.022914400324225426, -0.006959566846489906, 0.00958858523517847, -0.0070925522595644, -0.00602866942062974, -0.0053023649379611015, -0.009029364213347435, 0.013168959878385067, 0.00864745769649744, 0.018099647015333176, -0.006502642761915922, 0.012316489592194557, -0.027006257325410843, -0.02504216507077217, -0.029052184894680977, -0.003339978400617838, -0.04501042887568474, 0.00822463259100914, 0.010509252548217773, 0.022832563146948814, 0.016312871128320694, 0.008572440594434738, -0.03505357727408409, 0.0180450901389122, -0.011689071543514729, -0.007767708506435156, -0.01512623205780983, -0.0009470944060012698, -0.004047528840601444, 0.0033791919704526663, 0.01844063587486744, -0.017417671158909798, -0.0292158592492342, -0.00808141753077507, -0.02230062149465084, 0.03879080340266228, 0.018167845904827118, -0.0005916143418289721, 0.029843278229236603, -0.0005204330664128065, -0.0005651877727359533, -0.048583984375, 0.00907710287719965, -0.0007812889525666833, 0.01426694169640541, -0.007815446704626083, -0.008415586315095425, -0.015385382808744907, -0.017076684162020683, -0.0015370037872344255, -0.009056643582880497, -0.018481554463505745, -0.010052328929305077, -0.026869861409068108, -0.005527416709810495, -0.01220055390149355, 0.008374667726457119, 0.011409461498260498, -0.025587746873497963, -0.011927763000130653, 0.014212383888661861, 0.012330129742622375, 0.01614919677376747, 0.010004590265452862, 0.012698396109044552, -0.009465829469263554, -0.018767984583973885, -0.014867080375552177, -0.032189276069402695, 0.009036184288561344, 0.0027500689029693604, 0.027961023151874542, 0.019886424764990807, 0.027865545824170113, -0.006642447784543037, 0.0029563666321337223, 0.04563784599304199, -0.0010042099747806787, 0.004531731829047203, -0.0045931097120046616, -0.01346902921795845, -0.03208015859127045, -0.003145615104585886, -0.013455389998853207, 0.0003631523286458105, 0.021332215517759323, 0.000726304657291621, -0.009227138012647629, 0.006703825667500496, -0.00802685972303152, 0.011982321739196777, -0.013107581995427608, -0.029434092342853546, -0.02391008473932743, 0.024619340896606445, -0.021291296929121017, -0.0049340976402163506, -0.030934439972043037, -0.01775866001844406, 0.03148002177476883, 0.006448084954172373, 0.014226023107767105, 0.014989836141467094, 0.04405566304922104, -0.01643562503159046, -0.01869978755712509, 0.007078912574797869, 0.026897139847278595, -0.0016827762592583895, 0.0050704930908977985, -0.0504935160279274, 0.006608349271118641, 0.029897835105657578, 0.001049390877597034, 0.02459206059575081, -0.016121916472911835, -0.0004187760059721768, 0.0024892131332308054, 0.00907028280198574, 0.007419900968670845, 0.008729294873774052, -0.002511377213522792, -0.010625189170241356, -0.009002085775136948, -0.01347584929317236, -0.0034951278939843178, -0.015399022027850151, -0.016490183770656586, 0.027715511620044708, -0.008551981300115585, 0.03829978406429291, -0.02088211104273796, -0.036253854632377625, 0.0146079296246171, 0.022573411464691162, 0.03543548285961151, 0.015399022027850151, 0.006509462837129831, 0.030388858169317245, 0.02883395366370678, 0.0012676232727244496, 0.016899369657039642, 0.005350103136152029, -0.007147110532969236, 0.01432149950414896, 0.011689071543514729, 0.007256226614117622, -0.0259832926094532, 0.017322195693850517, 0.0221233069896698, -0.026392478495836258, -0.05092998221516609, 0.026160607114434242, 0.016599299386143684, 0.018113287165760994, -0.016572020947933197, -0.0034013562835752964, -0.03742685168981552, 0.005796797573566437, 0.0026187885086983442, 0.009043004363775253, 0.0027381344698369503, -0.00873611494898796, -0.007754069287329912, 0.01586276665329933, 0.008006400428712368, -0.003917952999472618, 0.00030965980840846896, -0.00524439662694931, 0.02365093305706978, 0.008961167186498642, -0.01040695607662201, 0.006478773895651102, -0.0078086270950734615, 0.027865545824170113, -0.003576965071260929, 0.011273066513240337, 0.00949992798268795, -0.015958242118358612, 0.01395323220640421, -0.0028489555697888136, -0.0022573410533368587, 0.030743485316634178, -0.03944550082087517, -0.02571050263941288, -0.0034081758931279182, -0.008020039647817612, 0.002867709845304489, -0.024905769154429436, -0.0008831591694615781, 0.005568335298448801, -0.012507443316280842, 0.019627274945378304, 0.01824968308210373, 0.011313985101878643, -0.01700848527252674, 0.013373552821576595, -0.009158940054476261, -0.025096723809838295, -0.010679746977984905, -0.004235072061419487, 0.014075987972319126, -0.05136644467711449, -0.004371467512100935, 0.005558105651289225, -0.015371742658317089, 0.006877729669213295, -0.0006427625776268542, -0.017431311309337616, 0.007058453280478716, 0.04149143025279045, -0.016421986743807793, -0.014935278333723545, 0.014089628122746944, -0.004057758487761021, -0.020172854885458946, 0.0171994399279356, -0.0077131506986916065, 0.004074807744473219, 0.03428976237773895, -0.022737085819244385, -0.01664021797478199, 4.3875577830476686e-05, -0.0203228909522295, 0.006533331703394651, 0.005101182032376528, 0.006915238685905933, -0.01413054671138525, -0.007058453280478716, 0.003883854253217578, -0.006110506597906351, 0.004306679591536522, -0.014089628122746944, 0.0074676391668617725, 0.005295544862747192, 0.024933049455285072, -0.018004171550273895, 0.0047738333232700825, 0.0002124782040482387, 0.008333749137818813, -0.013093942776322365, -0.060068462044000626, -0.013162139803171158, -0.006714055314660072, 0.008981626480817795, 0.0022437016014009714, -0.004794292617589235, -0.014089628122746944, 0.007140290457755327, -0.02088211104273796, 0.008340568281710148, 0.017158521339297295, 0.00446694390848279, -0.0019709111656993628, 0.0006299755186773837, -0.002175504108890891, 0.0042998599819839, -0.033416833728551865, -0.028479324653744698, -0.015412661246955395, 0.009888654574751854, -0.009472649544477463, -0.02872483618557453, -0.01778593845665455, 0.05046623572707176, 0.0078086270950734615, -0.019927343353629112, -0.019436320289969444, -0.02099122665822506, -0.042691707611083984, -0.014594290405511856, -0.030361579731106758, 0.02636519819498062, 0.0044328453950583935, 0.03338955342769623, 0.0006005652830936015, 0.03311676159501076, 0.010291020385921001, 0.009752259589731693, -0.00685045076534152, 0.006714055314660072, -0.006621988490223885, 0.0032376819290220737, -0.014853441156446934, -0.0050704930908977985, -0.018454276025295258, -0.007474458776414394, 0.004975016228854656, -0.018590670078992844, -0.002608558861538768, 0.03025246411561966, 0.011989140883088112, -0.006318509113043547, 0.004657897166907787, 0.025110362097620964, 0.003128565615043044, 0.010045508854091167, 0.012752954848110676, -0.04479219391942024, -0.0021857337560504675, 0.00987501535564661, -0.0180450901389122, -0.013087122701108456, -0.0141851045191288, 0.006482183467596769, 0.015330824069678783, 0.00500911520794034, 0.006567430682480335, -0.01497619692236185, -0.0002487081801518798, -0.0035462761297822, 0.03458983078598976, -0.00645149452611804, -0.0026716417632997036, 0.012268751859664917, -0.005213707685470581, -0.01086388062685728, -0.014457895420491695, 0.02193235419690609, -0.010386496782302856, 0.008142795413732529, 0.032571181654930115, -0.04370103403925896, -0.004098676610738039, 0.00047226849710568786, 0.003921363037079573, -0.02695169858634472, -0.01664021797478199, 0.010475154034793377, -0.007781348191201687, -0.022750725969672203, 0.03641752898693085, 0.029461370781064034, -0.010536531917750835, 0.0032376819290220737, -0.007399441674351692, 0.01721307821571827, -0.008899789303541183, -0.013578145764768124, -0.0045283217914402485, -0.021004866808652878, -0.0016887434758245945, 0.02288712002336979, -0.011273066513240337, -0.011968681588768959, 0.01739039272069931, 0.028615720570087433, -0.032189276069402695, 0.01700848527252674, 0.20568402111530304, -0.032380230724811554, 0.005333053879439831, 0.021018505096435547, 0.023214468732476234, -0.007835905998945236, 0.01413054671138525, 0.01318941917270422, -0.004289630334824324, -0.012152815237641335, 0.003195058321580291, -0.002613673685118556, -0.004289630334824324, -0.004906818736344576, 0.014526092447340488, -0.021523168310523033, -0.030307020992040634, -0.03718134015798569, -0.005837716162204742, 0.006533331703394651, 0.0050568534061312675, -0.011873205192387104, -0.009561305865645409, -0.011845925822854042, 0.016599299386143684, 0.0014534617075696588, -0.010932078585028648, -0.007037993986159563, 0.02024105377495289, 0.01729491539299488, -0.02459206059575081, 0.0008797492482699454, 0.013509947806596756, 0.011539037339389324, -0.015371742658317089, -0.006386707071214914, 0.014389697462320328, 0.006918648257851601, 0.00571837043389678, 0.012084618210792542, 0.03374418243765831, 0.00831328984349966, 0.019722750410437584, -0.036335691809654236, 0.005749059375375509, 0.022559771314263344, -0.010822962038218975, -0.001524216728284955, -0.003002400044351816, 0.0023255387786775827, -0.02078663371503353, 0.0171994399279356, 0.02760639600455761, 0.021236738190054893, -0.022518852725625038, 0.007317604497075081, -0.019122611731290817, 0.014389697462320328, 0.007324424106627703, 0.006127555854618549, -0.023296305909752846, -0.01234376896172762, -0.016967566683888435, 0.03865440934896469, -0.016108278185129166, 0.021223098039627075, -0.011320804245769978, -0.0106388283893466, 0.016026441007852554, -0.005551286041736603, -0.01568545214831829, 0.0019709111656993628, -0.02326902747154236, -0.010079608298838139, -0.05324869975447655, -0.04367375373840332, 0.03423520550131798, 0.027456361800432205, 0.023978281766176224, 0.04100040718913078, 0.001798712182790041, -0.026719827204942703, 0.014676127582788467, -0.024605700746178627, -0.024714816361665726, -0.04124591872096062, -0.0023204239550977945, -0.006809532176703215, -0.011041194200515747, -0.020091017708182335, -0.007065273355692625, -0.008995265699923038, -0.0005084985168650746, 0.01072748564183712, 0.0003663490933831781, 0.008436045609414577, 0.005991160869598389, -0.0016120212385430932, 0.009383992291986942, -0.02967960387468338, -0.01483980193734169, 0.06885231286287308, 0.004262350965291262, 0.006311689503490925, 0.002550591016188264, 0.009254416450858116, -0.010243282653391361, 0.011252607218921185, -0.0027176751755177975, -0.0204320065677166, 0.0012377867242321372, -0.02153680846095085, 0.012486984021961689, 0.014171465300023556, 0.0035189969930797815, 0.022914400324225426, -0.0026989206671714783, 0.00522052776068449, 0.020636599510908127, -0.013005285523831844, -0.00921349786221981, -0.020281972363591194, -0.009029364213347435, 0.009145300835371017, -0.015221708454191685, -0.04631982371211052, -0.01213917601853609, 0.009479468688368797, -0.030934439972043037, 0.02062295936048031, 0.01597188226878643, -0.0423370823264122, 0.001209655194543302, 0.009206678718328476, -0.010543351992964745, -0.0031609595753252506, -0.01395323220640421, 0.0094453701749444, -0.016899369657039642, 0.017458589747548103, 0.009595404379069805, 0.024619340896606445, -0.013284895569086075, -0.010331938974559307, 0.011048014275729656, -0.02314627170562744, 0.01749950833618641, 0.006734514608979225, -0.004599929321557283, -0.017608625814318657, -0.024742096662521362, -0.009738619439303875, 0.014798883348703384, 0.02013193629682064, 0.029352255165576935, 0.0056433528661727905, -0.019218089058995247, -0.012541541829705238, 0.03172553330659866, -0.00808141753077507, -0.047629214823246, 0.00936353299766779, 0.01796325296163559, -0.008047319017350674, -0.017826857045292854, 0.0010766698978841305, -0.17305827140808105, -0.015712730586528778, 0.0016546447295695543, -0.035299088805913925, 0.021223098039627075, -0.0018975987331941724, 0.023964643478393555, 0.004057758487761021, -0.010263741947710514, 0.009950032457709312, 0.013114402070641518, -0.008626998402178288, -0.007263046223670244, 0.002284620190039277, -0.0060559483245015144, 0.006905009038746357, -0.010031869634985924, -0.00958858523517847, -0.004351008217781782, 0.004275990650057793, 0.03565371409058571, -0.03210743889212608, 0.0171994399279356, -0.0047226850874722, -0.0076858713291585445, -0.0038702148012816906, -0.01356450654566288, 0.045037705451250076, 0.006969796493649483, -0.0372631773352623, -0.007522197440266609, -0.007644952740520239, 0.030688928440213203, -0.008858870714902878, 0.004293039906769991, -0.013482669368386269, 0.028151975944638252, -0.009561305865645409, -0.03434431925415993, 0.01214599609375, 0.03156185895204544, 0.01882254332304001, 0.018836181610822678, -0.017922334372997284, -0.03186192736029625, -0.018208764493465424, 0.023882806301116943, -0.02022741362452507, 0.008824772201478481, -0.021304935216903687, 0.028206534683704376, -0.010509252548217773, 0.008558801375329494, -0.0013110991567373276, 0.009227138012647629, 0.004003200214356184, 0.010079608298838139, 0.007310784421861172, -0.0005749911651946604, -0.0040645780973136425, -0.008156435564160347, -0.010857060551643372, 0.010400136932730675, 0.019231727346777916, -0.01807236857712269, -0.021509528160095215, -0.022477934136986732, 0.013646343722939491, -0.023282667621970177, 0.014307860285043716, -0.015521777793765068, -0.0014594290405511856, -0.01441697683185339, -0.010761584155261517, 0.025642303749918938, 0.021113982424139977, -0.024442026391625404, 0.022477934136986732, 0.020582040771842003, -0.008476964198052883, -0.014512453228235245, 0.020650237798690796, -0.007140290457755327, 0.0028233814518898726, -0.004524911753833294, -0.008381486870348454, 0.011593595147132874, 0.016858451068401337, -0.009950032457709312, -0.0063662477768957615, 0.022668888792395592, -0.01483980193734169, -0.010250101797282696, 0.010291020385921001, 0.015467219986021519, 0.012514262460172176, 0.019436320289969444, -0.01634014956653118, 0.03194376453757286, -0.019136251881718636, 0.008674737066030502, 0.016694776713848114, -0.002419310389086604, -0.011607234366238117, 0.04397382587194443, 0.02618788555264473, -0.01794961281120777, 0.013503128662705421, 0.04075489565730095, 0.006253721658140421, -0.02711537294089794, -0.02185051701962948, 0.0010110297007486224, 0.02298259735107422, 0.0003407749754842371, 0.023569095879793167, -0.020854830741882324, -0.029652323573827744, 0.022464295849204063, -0.0025574106257408857, 0.03846345469355583, -0.009404451586306095, -0.03243478760123253, 0.023882806301116943, -0.00851106271147728, -0.028806673362851143, -0.11129850894212723, -0.012077798135578632, 0.023473620414733887, 0.033034924417734146, -0.004559010732918978, 0.015303545631468296, -0.0029819407500326633, 0.038217946887016296, -0.021141260862350464, 0.03639024868607521, -0.004337368533015251, -0.01994098350405693, 0.030225183814764023, -0.010038689710199833, 0.012609739787876606, -0.009138480760157108, 0.0171994399279356, -0.02333722449839115, -0.0055342367850244045, 0.037590526044368744, 0.004804522264748812, 0.003123450791463256, 0.030034231022000313, -0.02041836641728878, -0.01020918320864439, -0.010052328929305077, -0.03769964352250099, 0.024251073598861694, 0.003655392210930586, 0.009997771121561527, 0.017172159627079964, -0.0370449461042881, 0.015562696382403374, -0.02751091867685318, 0.02646067552268505, 0.02240973711013794, -0.011920943856239319, -0.03243478760123253, 0.019531797617673874, -0.0150307547301054, -0.010291020385921001, -0.01149129867553711, 0.019231727346777916, -0.017704101279377937, -0.005309184547513723, -0.017335833981633186, -0.010202364064753056, 0.023896444588899612, 0.003208698006346822, -0.023814607411623, -0.029379533603787422, 0.01531718485057354, -0.03221655637025833, 0.03063436970114708, 0.01920444890856743, 0.010086427442729473, 0.012309670448303223, 0.009649963118135929, 0.01995462365448475, 0.00035931621096096933, -0.024155596271157265, -0.010229642502963543, 0.003914543427526951, 0.021591365337371826, 0.0031149261631071568, 0.014444255270063877, -0.005650172475725412, -0.014935278333723545, 0.015930963680148125, -0.01441697683185339, 0.007747249212116003, 0.015767289325594902, 0.0003220206417609006, 0.007324424106627703, -0.0141851045191288, -0.01947723887860775, -0.008667916990816593, -0.012377867475152016, 0.007563116028904915, 0.013946413062512875, -0.027920104563236237, -0.03205288201570511, 0.018208764493465424, -0.012575640343129635, -0.019981902092695236, -0.0009189628763124347, 0.008981626480817795, -0.012793873436748981, 0.0167220551520586, -0.03268029913306236, 0.0070925522595644, 0.05215753987431526, -0.011777728796005249, -0.005145510192960501, -0.004340778570622206, -0.012111896649003029, -0.014457895420491695, 0.002236881759017706, -0.014689766801893711, 0.010318299755454063, -0.014062348753213882, -0.0012735904892906547, -0.07392621785402298, 0.02760639600455761, 0.020950308069586754, -0.02345998026430607, 0.01044105552136898, 0.008981626480817795, 0.014785243198275566, 0.0004692848597187549, -0.018972577527165413, -0.0053739724680781364, -0.024428386241197586, -0.002359637524932623, -0.0203228909522295, -0.014403336681425571, -0.0015318889636546373, -0.013721360825002193, 0.005595614667981863, -0.007433540187776089, 0.006659497506916523, -0.0012931972742080688, -0.0014415271580219269, 0.00017976465460378677, 0.025956014171242714, 0.011143490672111511, -0.011893664486706257, -0.004214612767100334, -0.04691996052861214, 0.028943069279193878, -0.00963632296770811, -0.01361224427819252, 0.005728600081056356, -0.013646343722939491, -0.02153680846095085, 0.014485173858702183, -0.005067083053290844, 0.01006596814841032, 0.0047022257931530476, 0.023814607411623, 0.003863394958898425, 0.0360901802778244, -0.013523587957024574, -0.03944550082087517, 0.005684271454811096, -0.022068748250603676, 0.014717046171426773, 0.003314404282718897, 0.00864745769649744, -0.02807013876736164, 0.02174140140414238, 0.020472925156354904, 0.024987606331706047, 0.01384411659091711, -0.017444951459765434, -0.028752116486430168, -0.0074539994820952415, -0.017799578607082367, 0.0229689572006464, 0.010536531917750835, 0.006506052799522877, -0.0022351769730448723, 0.04198245331645012, -0.0075903949327766895, 0.006417396012693644, 0.006437855307012796, -0.010952537879347801, 0.0022709807381033897, -0.01778593845665455, -0.005793387535959482, 0.011511757969856262, -0.005336463451385498, 0.006652677431702614, -0.015508138574659824, 0.019531797617673874, 0.02561502531170845, 0.020963948220014572, -0.007617673836648464, -0.005087542347609997, 0.010325118899345398, -0.017622264102101326, 0.037481412291526794, 0.019968261942267418, -0.0033808969892561436, -0.007269866298884153, -0.005489908158779144, 0.01681753247976303, -0.003326338715851307, 0.0012821152340620756, 0.01389185432344675, -0.028397487476468086, -2.7465524908620864e-05, 0.012589280493557453, 0.0041736941784620285, -0.004630618263036013, 0.023596376180648804, 0.004828391131013632, -0.0008916838560253382, -0.004490813240408897, 0.001849860418587923, 0.03393513336777687, 0.01361224427819252, 0.008531522005796432, -0.01526262704282999, -0.006103686988353729, -0.002124355873093009, -0.03308948501944542, 0.013312174938619137, -0.045501451939344406, -0.03731773793697357, 0.010904799215495586, 0.01681753247976303, 0.01749950833618641, 0.01318941917270422, 0.008374667726457119, 0.007010715082287788, -0.024237433448433876, 0.03287125006318092, -0.0011423100950196385, -0.022082388401031494, -0.0413823127746582, 0.022177865728735924, 0.029461370781064034, 0.01234376896172762, 0.0567404180765152, -0.012637018226087093, 0.02648795396089554, 0.00765859242528677, 0.031343623995780945, -0.016967566683888435, 0.030088789761066437, 0.02062295936048031, -0.014253302477300167, -0.007883644662797451, 0.0028830543160438538, -0.01842699572443962, -0.009043004363775253, -0.011443560011684895, -0.02135949395596981, 0.015821848064661026, 0.023391783237457275, 0.08963894844055176, 0.01739039272069931, -0.0019606815185397863, 0.008572440594434738, -0.003188238712027669, -0.012521082535386086, 0.004385106731206179, 0.02742908149957657, -0.005800207611173391, 0.006253721658140421, 0.009568125940859318, -0.0036758515052497387, -0.003542866325005889, -0.017035765573382378, 0.013946413062512875, -0.008981626480817795, 0.005316004157066345, -0.006724284961819649, -0.00874293502420187, -0.004630618263036013, 0.03958189859986305, -0.012943907640874386, 0.02685622125864029, 0.020200135186314583, -0.021400412544608116, -0.007972301915287971, 0.029925115406513214, 0.009602224454283714, -0.006980026140809059, -0.028506604954600334, 0.02940681204199791, 0.0019265826558694243, -0.043619196861982346, -0.022164225578308105, -0.007508557755500078, -0.0032155176158994436, -0.006188933737576008, -0.027292687445878983, 0.010106886737048626, -0.0016188409645110369, 0.014171465300023556, 0.007419900968670845, -0.0039691012352705, -0.0023204239550977945, -0.0018737295176833868, -0.0061343759298324585, -0.008163254708051682, -0.018304239958524704, -0.02231425978243351], 'text': 'GPT-4 Technical Report\\nOpenAI∗\\nAbstract\\nWe report the development of GPT-4, a large-scale, multimodal model which can\\naccept image and text inputs and produce text outputs. While less capable than\\nhumans in many real-world scenarios, GPT-4 exhibits human-level performance\\non various professional and academic benchmarks, including passing a simulated\\nbar exam with a score around the top 10% of test takers. GPT-4 is a Transformer-\\nbased model pre-trained to predict the next token in a document. The post-training\\nalignment process results in improved performance on measures of factuality and\\nadherence to desired behavior. A core component of this project was developing\\ninfrastructure and optimization methods that behave predictably across a wide\\nrange of scales. This allowed us to accurately predict some aspects of GPT-4’s\\nperformance based on models trained with no more than 1/1,000th the compute of\\nGPT-4.\\n1 Introduction', 'SimilarityScore': 0.8287368247866265}), 0.8287368247866265)\n", - "(Document(page_content='plan to refine these methods and register performance predictions across various capabilities before\\nlarge model training begins, and we hope this becomes a common goal in the field.\\n4 Capabilities\\nWe tested GPT-4 on a diverse set of benchmarks, including simulating exams that were originally\\ndesigned for humans.4We did no specific training for these exams. A minority of the problems in the\\nexams were seen by the model during training; for each exam we run a variant with these questions\\nremoved and report the lower score of the two. We believe the results to be representative. For further\\ndetails on contamination (methodology and per-exam statistics), see Appendix C.\\nExams were sourced from publicly-available materials. Exam questions included both multiple-\\nchoice and free-response questions; we designed separate prompts for each format, and images were\\nincluded in the input for questions which required it. The evaluation setup was designed based', metadata={'id': '4b369374-f24d-452c-a830-d9af481bc157', 'embedding': [-0.012286471202969551, -0.006482758093625307, 0.006040671374648809, -0.025973472744226456, -0.005934570450335741, 0.021644560620188713, 0.013184790499508381, -0.002574712270870805, -0.0237948689609766, -0.01914058066904545, 0.020838193595409393, 0.037319183349609375, -0.02171529456973076, -0.007844384759664536, 0.002850574441254139, 0.02864721231162548, 0.015151191502809525, -0.013708221726119518, -0.0005751546705141664, -0.019352782517671585, 0.008389035239815712, 0.007950485683977604, -0.032622452825307846, -0.016608309000730515, 0.002132625784724951, -0.0037241375539451838, 0.029566751793026924, -0.036045972257852554, 0.008466842584311962, 0.016438547521829605, 0.010617151856422424, -0.012229884043335915, -0.03287709504365921, -0.014330679550766945, -0.007893898524343967, -0.00890539214015007, -0.005895666778087616, -0.013177717104554176, 0.0314624197781086, -0.003886825405061245, 0.003653403604403138, 0.010702032595872879, 0.0025481872726231813, -0.005984084215015173, 0.014436780475080013, 0.010553491301834583, 0.015165338292717934, -0.008155613206326962, -0.01621219888329506, 0.006857647560536861, 0.023243146017193794, 0.018362509086728096, -0.006242262665182352, 0.008912465535104275, -0.0018302384996786714, 0.017372235655784607, -0.00907515361905098, 0.013531386852264404, 0.007278513628989458, -0.008693191222846508, -0.007794870994985104, 0.006033597979694605, -0.015335099771618843, 0.03151900693774223, -0.015094605274498463, -0.020272323861718178, -0.007462421897798777, 0.02462952956557274, 0.009867372922599316, -0.01019982248544693, 0.00740583473816514, 0.007596815936267376, 0.006924844346940517, -0.016084879636764526, 0.03644208237528801, -0.027147654443979263, -0.019678158685564995, -0.011388150975108147, -0.02397877722978592, 0.018857646733522415, 0.020852340385317802, -0.01222281064838171, -0.024954903870821, 0.021474799141287804, 0.006578248459845781, 0.009110520593822002, 0.02021573670208454, -0.007045092061161995, 0.006274092942476273, -0.009662244468927383, 0.025167105719447136, 0.016778070479631424, 0.02008841559290886, 0.02088063396513462, -0.010709105990827084, 0.014295312575995922, -0.011748893186450005, 0.035225458443164825, -0.009747125208377838, -0.02948187105357647, -0.017598584294319153, 0.012491598725318909, -0.028236955404281616, -0.010051281191408634, -0.016792217269539833, 0.007073385640978813, -0.0014217505231499672, -0.022111402824521065, -0.0008023871923796833, 0.004102563485503197, -0.0060265245847404, 0.026963746175169945, -0.014344826340675354, -0.015419980511069298, -0.013644561171531677, -0.004541113507002592, -0.0034854107070714235, -0.011338637210428715, 0.00981078576296568, 0.004282935056835413, 0.021106982603669167, -0.006351900286972523, 0.04538284242153168, -0.003989389631897211, 0.00740583473816514, -0.0024845267180353403, -0.025351013988256454, -0.016933685168623924, -0.0010353667894378304, -0.0030397875234484673, -0.0030698494520038366, 0.009209548123180866, 0.015363393351435661, -0.008954905904829502, -0.025365160778164864, 0.01864544488489628, -0.01747126318514347, 0.0075614494271576405, -0.04051635414361954, -0.02982139401137829, 0.009633950889110565, 0.017527850344777107, -0.0003656056069303304, 0.004024756606668234, -0.0160424392670393, 0.031038016080856323, 0.027487175539135933, 0.02041379176080227, 0.015023871324956417, -0.006454464513808489, -0.005527850706130266, -0.02719009481370449, -0.02021573670208454, 0.01993280090391636, -0.0017170642968267202, -0.02818036824464798, 0.006033597979694605, 0.01957913115620613, -0.017909811809659004, -0.019847920164465904, 0.0014606540789827704, 0.007172413170337677, 0.012088416144251823, 0.007497788872569799, -0.006613615434616804, 0.013892129063606262, 0.026482755318284035, 0.01261891983449459, -0.003257294185459614, -0.013114056549966335, -0.021800175309181213, 0.015900971367955208, -0.048636600375175476, 0.020852340385317802, 0.001419982174411416, 0.004576480481773615, 0.016735630109906197, -0.0003050397499464452, -0.011812553741037846, -0.012350130826234818, 0.0127886813133955, 0.01875861920416355, 0.012965516187250614, -0.005152961239218712, -0.002604774199426174, -0.02365340106189251, 0.010015914216637611, 0.006139698904007673, -0.011317417025566101, -0.0020088416058570147, 0.017032712697982788, 0.004403182305395603, -0.01716003380715847, 0.00921662151813507, -0.6220052242279053, -0.0234836395829916, -0.006302386522293091, -0.025832004845142365, 0.011232536286115646, 0.011692306026816368, 0.008488062769174576, 0.004831122234463692, -0.021474799141287804, 0.047052159905433655, -0.01748540997505188, -0.014217505231499672, -0.005220158491283655, -0.03584791719913483, -0.008792218752205372, -0.029340403154492378, -0.004551723599433899, -0.01537754014134407, 0.007370467763394117, 0.010574711486697197, -0.01636781357228756, 0.011685232631862164, 0.0008390803704969585, -0.00449159974232316, -0.016580015420913696, 0.018051279708743095, -0.005064544267952442, -0.009945180267095566, 0.013559680432081223, 0.0025977008044719696, -0.007087532430887222, 0.03355614095926285, 0.01765516959130764, -0.002748010214418173, 0.04306277260184288, -0.005923960357904434, -0.03632890805602074, 0.013630414381623268, -0.008551723323762417, 0.042496901005506516, -0.0336410216987133, -0.008035366423428059, -0.003405835246667266, 0.03969584032893181, -0.005828469526022673, 0.03853580355644226, 0.0025835540145635605, 0.0018320068484172225, -0.013170643709599972, -0.01280282810330391, -0.004505746532231569, -0.005874446593225002, 0.014528733678162098, 0.01045446377247572, 0.016551721841096878, -0.00915296096354723, 0.021927494555711746, -0.003453580429777503, -0.012045975774526596, -0.01714588701725006, 0.008488062769174576, 0.009938106872141361, 0.005874446593225002, -0.0589071549475193, -0.029057467356324196, 0.03870556503534317, -0.012696727178990841, 0.015561448410153389, 0.0012864719610661268, -0.03448982909321785, 0.0016259945696219802, 0.0077170636504888535, -0.01304332260042429, -0.02090892754495144, 0.008763925172388554, 0.014528733678162098, 0.024403180927038193, -0.02351193316280842, 0.002196286339312792, 0.01875861920416355, 0.011798406951129436, -0.008374888449907303, 0.008042439818382263, -0.005241378676146269, 0.006663129199296236, -0.001750662922859192, -0.01061007846146822, 0.01134571060538292, -0.0038337749429047108, 0.01700441911816597, 0.020343057811260223, 0.01959327794611454, -0.015193631872534752, -0.042808130383491516, 0.0007625994039699435, 0.010475683957338333, -0.02153138630092144, -0.009435896761715412, 0.004993810318410397, -0.012088416144251823, -0.020965514704585075, -0.0079434122890234, 0.03372590243816376, 0.01622634567320347, 0.04775949940085411, 0.012746240943670273, 0.0020548184402287006, -0.002703801728785038, 0.0005322722718119621, -0.01084350049495697, 0.0016648981254547834, -0.0003770998737309128, -0.015448274090886116, 0.006953137926757336, 0.010093721561133862, -0.030698493123054504, 0.02362510748207569, -0.001715296064503491, -0.02993456833064556, -0.03126436471939087, 0.0110061876475811, -0.017259061336517334, 0.01498143095523119, -0.018008839339017868, 0.016961978748440742, 0.01327674463391304, 0.004969053436070681, -0.022748008370399475, 0.0035508396103978157, 0.014797522686421871, -0.016891244798898697, -0.023738281801342964, 0.03415030613541603, -0.016099026426672935, 0.020159149542450905, 0.029425283893942833, 0.010093721561133862, -0.014351899735629559, 0.009174181148409843, -0.006786913610994816, -0.017046859487891197, 0.004272324964404106, -0.005998231004923582, -0.0031158262863755226, -0.023540226742625237, 0.011946948245167732, -0.00907515361905098, 0.0019717062823474407, 0.006341290194541216, -0.003900972194969654, -0.000811228936072439, 0.0009593279100954533, 0.0059274970553815365, 0.0056834653951227665, -0.010263482108712196, -0.02024403028190136, -0.006090185139328241, -0.04263836890459061, -0.010235188528895378, -0.03536692634224892, 0.007299733813852072, 0.016792217269539833, 0.0019434127025306225, -0.0013801943277940154, 0.01407603733241558, -0.011204242706298828, -0.013715295121073723, 0.03400883823633194, -0.015519008040428162, -0.034065425395965576, 0.011904507875442505, -0.03980901464819908, 0.020965514704585075, 0.028095487505197525, -0.024827582761645317, -0.00429354514926672, -0.011784260161221027, 0.00412378367036581, 0.0034252870827913284, -0.026921305805444717, 0.00623872596770525, 0.009428823366761208, -0.014684348367154598, -0.012081342749297619, 0.03347126021981239, 0.03949778527021408, 0.022762155160307884, 0.012442084960639477, -0.01432360615581274, 0.018716178834438324, 0.011748893186450005, -0.002923076506704092, -0.01813616044819355, 0.025308573618531227, 0.0012166223023086786, 0.020145002752542496, 0.013460652902722359, -0.011494251899421215, 0.012951369397342205, 0.034546416252851486, 0.020767459645867348, 0.007045092061161995, 0.009414676576852798, -0.025068078190088272, 0.007197170052677393, -0.0018231651047244668, 0.010914234444499016, -0.014839963056147099, -0.01166401244699955, -0.0005875331116840243, -0.004919539671391249, -0.01909814029932022, -0.022083109244704247, -0.013545533642172813, -0.011140582151710987, 0.010249335318803787, 0.015038018114864826, 0.011218389496207237, -0.023285584524273872, -0.00835366826504469, 0.015971705317497253, 0.0007953137974254787, 0.01636781357228756, 0.0013545533875003457, -0.02008841559290886, 0.019480103626847267, -0.01077983994036913, 0.013340405188500881, 0.0333297923207283, -0.009336869232356548, -0.0047356318682432175, 0.015632182359695435, 0.01779663749039173, 0.018687885254621506, 0.034885939210653305, -0.002988505410030484, 0.025167105719447136, -0.007147656287997961, 0.039412904530763626, 0.010984967462718487, 0.016396107152104378, -0.0039045088924467564, -0.008785145357251167, -0.014726788736879826, 0.010334216058254242, -0.007264366839081049, 0.023257290944457054, -0.0001488726556999609, -0.028576478362083435, 0.01450044009834528, -0.0007015914306975901, 0.003366931574419141, -0.009492483921349049, 0.008162686601281166, 0.01914058066904545, -0.008834658190608025, 0.012031828984618187, 0.0026967283338308334, 0.005046860780566931, 0.021149422973394394, 0.015745356678962708, 0.008410255424678326, 0.004017683211714029, -0.003766577923670411, 0.022776301950216293, -0.02864721231162548, -0.01059593167155981, -0.01990450732409954, -0.03550839424133301, -0.016240492463111877, -0.018093720078468323, -0.023200705647468567, -0.0008514588116668165, -0.0234836395829916, -0.0034005302004516125, 0.015462420880794525, -0.0002816091582644731, -0.0030309457797557116, 0.02545004151761532, 0.010327142663300037, -0.0286755058914423, -0.029595045372843742, 0.0042015910148620605, 0.008679044432938099, -0.008714411407709122, -0.006896550767123699, -0.026638370007276535, 0.0018673738231882453, -0.004350132308900356, 0.0024650748819112778, -0.007837311364710331, 0.008205126971006393, -0.030981428921222687, 0.0055809011682868, -0.009379309602081776, 0.0237948689609766, 0.03284880146384239, 0.0035826698876917362, 0.003717064158990979, -0.007098142523318529, -0.0012926611816510558, 0.006723253056406975, -0.02317241206765175, -0.03199999779462814, 0.019225461408495903, -0.011076921597123146, -0.03307515010237694, -0.026312993839383125, 0.0020265248604118824, -0.0012988504022359848, 0.02717594802379608, 0.022889476269483566, 0.004834658931940794, -0.008459769189357758, 0.005308575928211212, 0.01990450732409954, -0.02202652208507061, 0.008473915979266167, 0.02669495716691017, 0.02880282700061798, -0.009725905023515224, -0.02931210957467556, -0.0027108751237392426, 0.01621219888329506, 0.04422280564904213, 0.03465959057211876, -0.003329796250909567, -0.0006030061049386859, -0.020526964217424393, -0.010871794074773788, -0.02574712410569191, -0.03463129699230194, 0.028944293037056923, 0.01012201514095068, 0.012003535404801369, -0.010022987611591816, -0.0007259062258526683, -0.008403182029724121, 0.01929619535803795, 0.012675506994128227, -0.009832005947828293, -0.01928204856812954, -0.003036250825971365, 0.01496728416532278, 0.01892838068306446, -0.02137577161192894, 0.006008841097354889, 0.02107868902385235, 0.01909814029932022, -0.009096373803913593, 0.019494250416755676, 0.01846153661608696, -0.012102562934160233, -0.02059769816696644, -0.039582666009664536, 0.010397876612842083, -0.020017681643366814, 0.014670201577246189, -0.01327674463391304, 0.005955790635198355, 0.00769584346562624, 0.005425286944955587, 0.027458883821964264, 0.006578248459845781, 0.016466841101646423, -0.006100795231759548, 0.01351016666740179, -0.005170644726604223, -0.004721485078334808, -0.014040670357644558, 0.01416799146682024, 0.0076038893312215805, 0.0024102560710161924, -0.036838192492723465, 0.010037134401500225, -0.002864721231162548, -0.04125198349356651, -0.004862952511757612, 0.026440314948558807, 0.0032855875324457884, 0.01912643387913704, -0.004056586418300867, -0.020979661494493484, -0.011494251899421215, -0.020668432116508484, -0.020003534853458405, -0.0023236072156578302, -0.0042900084517896175, -0.023059237748384476, 0.009987620636820793, -0.036045972257852554, -0.016565868631005287, -0.03678160533308983, 0.009902739897370338, -0.01914058066904545, 0.00016202473489101976, -0.049457110464572906, 0.0034022985491901636, 0.015066311694681644, 0.009938106872141361, 0.015320952981710434, -0.004367815796285868, 0.00725022004917264, 0.008756851777434349, -0.023568520322442055, -0.038960207253694534, 0.0020176833495497704, -0.02655348926782608, 0.02622811309993267, 0.002862952882423997, -0.025860298424959183, -0.020625991746783257, -0.01730150170624256, 0.007264366839081049, 0.0018443852895870805, -0.009598583914339542, -0.0025676388759166002, -0.009534923359751701, 0.014542880468070507, 0.014698495157063007, 0.011925728060305119, 0.012852341867983341, 0.01864544488489628, -0.03468788415193558, 0.001015914953313768, -0.021022101864218712, 0.004463306162506342, -0.03027409128844738, 0.0045977006666362286, 0.00437135249376297, 0.0027108751237392426, 0.00940052978694439, 0.007172413170337677, -0.03630061447620392, 0.03078337386250496, -0.018970821052789688, 0.013906275853514671, 0.0030557026620954275, 0.024162685498595238, 0.0074482751078903675, -0.0037842614110559225, 0.03352784737944603, 0.00938638299703598, -0.023596813902258873, -0.013538460247218609, -0.014118477702140808, 0.036215733736753464, -0.0026472145691514015, -0.02378072217106819, 0.03516887500882149, 0.007879751734435558, -0.0028682579286396503, -0.06750839203596115, 0.00224579987116158, -0.02284703589975834, 0.026595929637551308, -0.005541997496038675, -0.01343235932290554, -0.003519009333103895, -0.015207778662443161, -0.013566753827035427, -0.0079434122890234, -0.013142350129783154, 0.026426168158650398, -0.012010608799755573, 0.0014456232311204076, 0.016113173216581345, 0.015674622729420662, 0.01843324303627014, -0.030641905963420868, -0.016155613586306572, 0.0064509278163313866, 0.004760388284921646, 0.012852341867983341, 0.0008956674719229341, 0.01126790326088667, -0.029425283893942833, -0.0060795750468969345, -0.028251102194190025, -0.04181785508990288, 0.024007070809602737, 0.016240492463111877, 0.023922190070152283, 0.015108752064406872, 0.015787797048687935, -0.020187443122267723, -0.0065004415810108185, 0.019168874248862267, -0.0006710874731652439, -0.008763925172388554, -0.0009389919578097761, -0.01843324303627014, -0.00039168872172012925, -0.008877098560333252, -0.005743589252233505, -0.01638196036219597, 0.009471263736486435, -0.004827585536986589, 0.0006940759485587478, 0.03299026936292648, -0.0015198937617242336, 0.007759504020214081, -0.014054817147552967, -0.033782489597797394, -0.030415557324886322, 0.03584791719913483, 0.014528733678162098, 0.01652342826128006, -0.037630412727594376, -0.003798407968133688, 0.012611846439540386, 0.0009778955718502402, 0.01649513468146324, -0.002838196000084281, 0.02154553309082985, -0.013389918953180313, -0.009301502257585526, -0.0019221925176680088, 0.013701148331165314, 0.0014491599285975099, -0.020286470651626587, -0.05217329040169716, -0.005128204356878996, 0.013248451054096222, -0.02042793668806553, 0.033273205161094666, -0.0015278513310477138, 0.017089299857616425, -0.02767108380794525, 0.005365163087844849, -0.003605658421292901, 0.003932802472263575, 0.00032051277230493724, -0.019211314618587494, -0.001948717748746276, -0.015207778662443161, 0.015900971367955208, 0.025704683735966682, -0.036187440156936646, 0.014316532760858536, -0.012519892305135727, 0.024827582761645317, -0.04043147340416908, -0.012413791380822659, 0.018362509086728096, 0.004010609816759825, 0.03151900693774223, 0.004438549280166626, 0.021616267040371895, 0.01799469254910946, 0.009259061887860298, -0.003305039368569851, 0.023851456120610237, 0.009089300408959389, 0.0064120241440832615, 0.011402297765016556, 0.028576478362083435, -0.013312111608684063, -0.02349778637290001, -0.009980547241866589, 0.026525195688009262, -0.038648977875709534, -0.05019274353981018, 0.02170114777982235, -0.002760388655588031, 0.022719714790582657, -0.009556143544614315, -0.013870908878743649, -0.04326082766056061, 0.007126436103135347, -0.00808488018810749, 0.009513703174889088, 0.0036675503943115473, -0.018503976985812187, -0.0051989383064210415, 0.02638372778892517, 0.012781607918441296, 0.020767459645867348, -0.008360741659998894, -0.01827762834727764, 0.012972589582204819, 0.016466841101646423, -0.007119362708181143, 0.0059522539377212524, 0.0037559678312391043, 0.022479219362139702, -0.0007639256655238569, 0.013312111608684063, 0.02525198645889759, -0.009902739897370338, 0.006022987887263298, 0.007504862267524004, -0.002233421429991722, 0.03216975927352905, -0.018037132918834686, -0.015915118157863617, -0.0015022102743387222, -0.007809017784893513, 0.0019168874714523554, -0.03171706199645996, -0.008113172836601734, 0.007766577415168285, 0.0017807248514145613, 0.011763039976358414, 0.038167987018823624, 0.008622457273304462, -0.03788505494594574, 0.005106984172016382, -0.0066560558043420315, -0.01351016666740179, -0.01214500330388546, -0.02351193316280842, 0.00017860298976302147, -0.04195932298898697, -0.008679044432938099, 0.010645445436239243, -0.004063659813255072, 0.019480103626847267, 0.006320070009678602, -0.026963746175169945, 0.006022987887263298, 0.02528028003871441, -0.007412908133119345, -0.010249335318803787, 0.0023854991886764765, 0.008792218752205372, -0.03836604207754135, 0.02041379176080227, -0.023059237748384476, -0.004799291957169771, 0.021743588149547577, -0.027232535183429718, -0.025520775467157364, -0.01832006871700287, 0.003503094194456935, 0.007349247578531504, 0.009669317863881588, -0.004024756606668234, 0.009365162812173367, -0.002728558611124754, 0.008091953583061695, -0.0012298849178478122, -0.006627762224525213, -0.0009000883437693119, 0.003897435497492552, 0.008290007710456848, 0.027006186544895172, -0.022125549614429474, 0.013913349248468876, 0.009973473846912384, -0.0006653403397649527, 0.00921662151813507, -0.045722365379333496, -0.013142350129783154, -0.01977718621492386, 0.017075153067708015, -0.016735630109906197, -0.01684880442917347, -0.006132625509053469, -0.0005884172860532999, -0.005170644726604223, 0.007342174183577299, 0.012717947363853455, 0.021163569763302803, 0.000557471183128655, -0.02523783966898918, 0.005775419529527426, 0.024049511179327965, -0.03389566391706467, -0.006702032871544361, -0.006302386522293091, -0.016749776899814606, -0.0233280248939991, 0.004286471754312515, -0.01092838030308485, 0.03129265829920769, 0.005623341538012028, -0.013241377659142017, -0.03485764563083649, -0.02996286191046238, -0.030330676585435867, -0.01036250963807106, -0.017740050330758095, 0.027473028749227524, 0.013057469390332699, 0.027572056278586388, -0.009442970156669617, 0.01271087396889925, -0.002749778563156724, 0.01764102280139923, -0.01084350049495697, 0.004587090574204922, -0.00818390678614378, -0.008721484802663326, 0.0025676388759166002, 0.008749778382480145, -0.02088063396513462, -0.001087533077225089, 0.008686117827892303, -0.009110520593822002, 0.010305922478437424, 0.019239608198404312, 0.025945179164409637, -0.004265251569449902, -0.004767461679875851, 0.011444738134741783, 0.013800174929201603, 0.011183022521436214, 0.01568876951932907, -0.033782489597797394, 0.018560564145445824, 0.015250219032168388, -0.023936336860060692, -0.023243146017193794, -0.02429000660777092, 0.017244914546608925, -0.0028063657227903605, 0.0005185675690881908, -0.0028717946261167526, -0.01360212080180645, -0.0007683465373702347, -0.02738814987242222, 0.025068078190088272, -0.008537576533854008, -0.00970468483865261, 0.023228999227285385, 0.003978779539465904, 0.002887709764763713, -0.017414676025509834, 0.008381961844861507, -0.004254641477018595, 0.012017682194709778, 0.025803711265325546, -0.0404597669839859, 0.002044208347797394, 0.0030167989898473024, 0.008749778382480145, -0.02329973131418228, -0.021290890872478485, 0.0022493365686386824, -0.013396992348134518, 0.0070769223384559155, 0.02655348926782608, 0.025336867198348045, -0.0005592395318672061, 0.016438547521829605, -0.009605657309293747, 0.017782490700483322, -0.009492483921349049, -0.015830237418413162, 0.005743589252233505, -0.012809901498258114, 0.008162686601281166, -0.006001767702400684, -0.014910697005689144, -0.007554376032203436, -0.00956321693956852, 0.025662243366241455, -0.0058390796184539795, 0.014472146518528461, 0.2013368457555771, -0.011678159236907959, 0.005421750247478485, 0.02755790948867798, 0.023752428591251373, 0.0006348363822326064, 0.01051812432706356, 0.0218567606061697, 0.005899203475564718, 0.006146772298961878, 0.007731210440397263, -0.011876214295625687, -0.021771881729364395, -0.0026949599850922823, 0.0015287355054169893, -0.022097256034612656, -0.03369760885834694, -0.03567815572023392, -0.016721483319997787, 0.0076675498858094215, 0.009527849964797497, -0.017740050330758095, 0.011748893186450005, -0.003320954507216811, 0.03092484176158905, 0.0067621567286551, -0.0027957556303590536, -0.002887709764763713, 0.02022988349199295, 0.024742702022194862, -0.01587267778813839, -0.006132625509053469, 0.004689654801040888, -0.011939874850213528, -0.0018779839156195521, 0.0016772765666246414, 0.011465958319604397, 0.014019450172781944, 0.005609194748103619, 0.005598584655672312, 0.03966754674911499, 0.013743587769567966, 0.013057469390332699, -0.016141466796398163, -0.006924844346940517, 0.015052164904773235, -0.01432360615581274, -0.016622455790638924, 0.0036215733271092176, -0.004190980922430754, -0.03188682347536087, 0.014953137375414371, 0.03194341063499451, 0.02641202136874199, -0.026426168158650398, 0.004176834132522345, -0.012831121683120728, -0.0023112287744879723, 0.019183021038770676, 0.009725905023515224, -0.016792217269539833, -0.009902739897370338, 0.0007466842653229833, 0.03369760885834694, -0.023115824908018112, 0.007126436103135347, -0.00996640045195818, -0.01621219888329506, 0.024162685498595238, 0.0015517239226028323, -0.022238723933696747, -0.01358797401189804, -0.014429707080125809, 0.016778070479631424, -0.027572056278586388, -0.03723430261015892, 0.02250751294195652, 0.03307515010237694, 0.02264898084104061, 0.041534919291734695, 0.006472148001194, -0.011536692269146442, -0.010801060125231743, -0.012286471202969551, -0.03811139985918999, -0.05005127564072609, 0.012937222607433796, -0.005520777311176062, -0.0117418197914958, -0.005977010820060968, -0.004353669006377459, -0.0030450925696641207, -0.018235187977552414, 0.0004434128641150892, 0.00858709029853344, 0.006850574165582657, 0.0011715295258909464, 0.00498320022597909, 0.011437664739787579, -0.05041909217834473, -0.019706452265381813, 0.07661890983581543, 0.0016525196842849255, -0.002231653081253171, 0.022903623059391975, 0.0018726788694038987, -0.012307691387832165, 0.0160424392670393, -0.0009442970040254295, -0.009987620636820793, 0.0023430590517818928, -0.03870556503534317, 0.009421749971807003, 0.013411139138042927, 0.010638372041285038, 0.01601414568722248, -0.0015862067230045795, -0.012739167548716068, 0.028562331572175026, -0.00198585307225585, 0.009053933434188366, -0.00036427934537641704, -0.005206011701375246, 0.012406717985868454, 0.02346949279308319, -0.03924314305186272, -0.020145002752542496, 0.005460653454065323, -0.012095489539206028, -0.009273208677768707, 0.0036357201170176268, -0.03777188062667847, 0.012088416144251823, -0.004562333691865206, 0.0030910694040358067, -0.008601237088441849, 0.004396109376102686, -0.01796639896929264, -0.021135276183485985, -0.008410255424678326, 0.023073384538292885, 0.012569406069815159, 0.004548186901956797, 0.018489830195903778, 0.0009098142036236823, 0.00485941581428051, 0.010015914216637611, 0.0241060983389616, 0.0033138811122626066, -0.02397877722978592, -0.03451812267303467, -0.010468610562384129, 0.0071794865652918816, 0.01384261529892683, 0.03777188062667847, 0.0004924844834022224, -0.0030786911956965923, -0.021630413830280304, 0.021474799141287804, -0.006546418182551861, -0.04855171963572502, -0.008452695794403553, 0.018079573288559914, 0.0035260827280580997, -0.024742702022194862, -0.002427939558401704, -0.18164454400539398, 0.007069848943501711, 0.021573826670646667, -0.04286471754312515, -0.002887709764763713, -0.010730326175689697, 0.0421573780477047, -0.01812201365828514, -0.0218567606061697, -0.0029832003638148308, 0.01650928147137165, -0.008148539811372757, -0.021149422973394394, -0.003964632749557495, -0.006129088811576366, -0.011352784000337124, -0.011939874850213528, -0.0022617150098085403, 0.010277628898620605, 6.855796073068632e-07, 0.021969934925436974, -0.03946949169039726, 0.006822280585765839, 0.0053510162979364395, -0.011296196840703487, -0.0033404063433408737, -0.0022245796862989664, 0.02380901575088501, -0.00018810784968081862, -0.05338991433382034, -0.004484526347368956, 0.015193631872534752, 0.026525195688009262, 0.007547302637249231, -0.02445976808667183, 0.0019274975638836622, 0.0007997346692718565, -0.011062774807214737, -0.04003536328673363, -0.0001731874217512086, 0.030076036229729652, 0.023243146017193794, 0.010164455510675907, -0.017527850344777107, -0.017570290714502335, -0.013658707961440086, 0.01864544488489628, -0.020838193595409393, 0.013941642828285694, -0.03451812267303467, 0.020442083477973938, -0.014118477702140808, 0.011770113371312618, 0.004244031384587288, 0.0038797520101070404, -0.006603005342185497, 0.007469495292752981, -0.0034376652911305428, 0.01312112994492054, 0.007299733813852072, -0.0029053932521492243, -0.006376657169312239, 0.02653934247791767, 0.016311226412653923, -0.02025817707180977, -0.039752427488565445, -0.00921662151813507, 0.017273208126425743, -0.020130855962634087, 0.01029884908348322, -0.012640140019357204, -0.015900971367955208, 0.004212201107293367, -0.01745711639523506, 0.04343058913946152, 0.01717418059706688, 0.006139698904007673, -0.003664013696834445, 0.021969934925436974, 0.0002628204820211977, 0.002095490461215377, 0.031207777559757233, -0.005283819045871496, -0.015900971367955208, -0.0012502209283411503, -0.01482581626623869, 0.009846152737736702, 0.01929619535803795, -0.018263481557369232, -0.013736514374613762, 0.0333297923207283, -0.015957558527588844, -0.01749955676496029, 0.0001873341971077025, 0.028944293037056923, 0.011381077580153942, 0.00533686950802803, -0.012689653784036636, 0.029538458213210106, -0.01568876951932907, 0.017273208126425743, 0.0058709098957479, -0.017372235655784607, 0.0019221925176680088, 0.0234836395829916, 0.013545533642172813, 0.00036980543518438935, 0.021290890872478485, 0.04453403502702713, -0.0009372236090712249, -0.027331562712788582, -0.013913349248468876, 0.017230767756700516, -0.008657824248075485, 0.010977894067764282, 0.02591688558459282, 0.006221042480319738, -0.017131740227341652, 0.006178602110594511, -0.0030309457797557116, 0.03822457417845726, -0.01523607224225998, -0.0033881517592817545, 0.021955788135528564, -0.009938106872141361, -0.024700261652469635, -0.13490360975265503, -0.023285584524273872, 0.0027533152606338263, 0.019635718315839767, 0.00034173292806372046, 0.02816622145473957, 0.003421750385314226, 0.045920420438051224, -0.0327073335647583, 0.04886294901371002, -0.013135276734828949, -0.013757734559476376, 0.025223692879080772, -0.007801944389939308, 0.006355436984449625, -0.00200176821090281, 0.01880105957388878, -0.02591688558459282, -0.021503092721104622, 0.03881873935461044, 0.0031034478452056646, -0.012901855632662773, 0.035876210778951645, -0.015646329149603844, -0.003025640733540058, -0.004452696070075035, -0.036045972257852554, 0.021404065191745758, 0.0048417323268949986, 0.007964632473886013, 0.0027656937018036842, -0.030387263745069504, -0.000163572040037252, -0.020767459645867348, 0.010277628898620605, 0.006369583774358034, -0.0064509278163313866, -0.03202829137444496, 0.039752427488565445, 0.007059238851070404, 0.001048629404976964, 0.00808488018810749, 0.00921662151813507, 0.000598585233092308, -0.017414676025509834, -0.0161980539560318, -0.018532270565629005, 0.02948187105357647, -0.004067196510732174, -0.030217504128813744, -0.016806364059448242, 0.010921306908130646, -0.028576478362083435, 0.009259061887860298, 0.002762157004326582, 0.006051281467080116, 0.018192747607827187, 0.0251105185598135, 0.02444562129676342, 0.001795755815692246, -0.028236955404281616, -0.020809900015592575, 0.00627055624499917, -0.0055950479581952095, -0.01440141350030899, 0.010907161049544811, -0.02250751294195652, -0.012965516187250614, 0.01076569315046072, -0.015066311694681644, 0.01587267778813839, 0.012951369397342205, -0.00594871724024415, 0.010277628898620605, -0.01666489616036415, 0.0020300615578889847, -0.008410255424678326, -0.013828468509018421, 0.0026083108969032764, -0.00761803612112999, -0.017358088865876198, -0.01911228708922863, 0.022465072572231293, -0.027232535183429718, -0.01084350049495697, 0.0006450043292716146, -0.0007679044501855969, -0.024190979078412056, -0.0026861182413995266, -0.03686648607254028, -0.001740052830427885, 0.0336410216987133, 0.011918654665350914, -0.023073384538292885, -0.014373119920492172, 0.005004420410841703, -0.009004419669508934, -0.004357205703854561, -0.011614499613642693, 0.02690715901553631, 0.0010893014259636402, -0.002603005850687623, -0.04855171963572502, 0.029085760936141014, 0.0032749774400144815, -0.01653757505118847, -2.2325373720377684e-05, 0.00466136122122407, 0.01973474584519863, -0.003343943040817976, -0.024643676355481148, -0.008608310483396053, -0.027289122343063354, 0.01798054575920105, -0.018659591674804688, -0.00874270498752594, -0.010206895880401134, -0.031320951879024506, 0.012251104228198528, -0.0016215736977756023, 0.003476568963378668, 0.026793984696269035, -0.01223695743829012, -0.005450043827295303, 0.03321661800146103, 0.01181962713599205, -0.017414676025509834, -0.006076038349419832, -0.034065425395965576, 0.02917064167559147, -0.007108752615749836, -0.029510164633393288, -0.0016171528259292245, 0.004017683211714029, -0.01407603733241558, 0.009145887568593025, -0.0017241376917809248, 0.003798407968133688, 0.0020282932091504335, 0.011183022521436214, 0.019635718315839767, 0.030076036229729652, -0.011069848202168941, -0.01686295121908188, -0.004053049720823765, -0.041365157812833786, 0.005294429138302803, -0.002877099672332406, -0.009117593988776207, -0.016735630109906197, 0.0037842614110559225, 0.008862951770424843, 0.0005822280654683709, 0.009690538048744202, -0.00575066264718771, -0.03468788415193558, -0.018221041187644005, -0.01465605478733778, -0.0074482751078903675, 0.009351016022264957, 0.033131737262010574, -0.00769584346562624, 0.042355433106422424, -0.010489830747246742, 0.006486294791102409, -0.0076038893312215805, 0.009513703174889088, 0.017612729221582413, -0.0026755081489682198, 0.00016014586435630918, -0.0032254639081656933, -0.01554730162024498, -0.004488063044846058, 0.0014022986870259047, 0.017103446647524834, 0.02090892754495144, 0.00850928295403719, -0.025195399299263954, 0.0039717061445117, 0.014882403425872326, -0.012470378540456295, 0.024360740557312965, 0.02670910395681858, 0.0036357201170176268, -0.022946063429117203, 0.0270486269146204, 0.02055525779724121, 0.0033492480870336294, 0.005301502533257008, 0.013418212532997131, -0.028081340715289116, 0.003106984542682767, 0.009711758233606815, 0.019069846719503403, -0.008954905904829502, 0.012159150093793869, 0.0019098140764981508, 0.019678158685564995, -0.006245799362659454, 0.007172413170337677, 0.047872673720121384, 0.023837309330701828, 0.001839080243371427, -0.0061043319292366505, -0.007218390237540007, -0.012335984036326408, -0.03429177403450012, -0.007879751734435558, -0.046401411294937134, -0.02948187105357647, -0.006956674624234438, 0.026751544326543808, 0.013340405188500881, 0.014783375896513462, 0.007950485683977604, 0.0008377541089430451, -0.0016268787439912558, 0.024502208456397057, 0.00315826665610075, -0.001591511769220233, -0.02786913886666298, 0.017541997134685516, 0.030415557324886322, 0.02253580652177334, 0.04538284242153168, 0.0008965516462922096, 0.034065425395965576, -0.010553491301834583, 0.02266312763094902, -0.02252165973186493, 0.006206895690411329, 0.033754196017980576, -0.007787797600030899, -0.00792926549911499, 0.01989036053419113, -0.017117593437433243, -0.015745356678962708, -0.030698493123054504, -0.02153138630092144, 0.011784260161221027, 0.02154553309082985, 0.07910874485969543, 0.028717946261167526, -0.004668434616178274, 0.006702032871544361, -0.011593279428780079, -0.003653403604403138, 0.00013118919741827995, 0.01164986565709114, -0.01599999889731407, -0.015306806191802025, 0.0016357203712686896, 0.008155613206326962, 0.02021573670208454, -0.026808131486177444, -0.006691422779113054, 0.009414676576852798, 0.007724137045443058, -0.000503536663018167, 0.002532272133976221, -0.0136799281463027, 0.0404597669839859, -0.027232535183429718, 0.028562331572175026, 0.015278512611985207, -0.013573827221989632, -0.01004420779645443, 0.025266133248806, 0.004431475885212421, -0.0035738281439989805, -0.0001597037771716714, 0.03075508028268814, 0.01523607224225998, -0.03129265829920769, -0.01183377392590046, -0.0022245796862989664, -0.0010778071591630578, 0.008601237088441849, -0.008304154500365257, 0.018334215506911278, -0.004916002973914146, 0.024205125868320465, -0.020343057811260223, -0.01697612553834915, -0.0028983198571950197, -0.011197169311344624, 0.003738284343853593, 0.008035366423428059, -0.020116709172725677, -0.017513703554868698], 'text': 'plan to refine these methods and register performance predictions across various capabilities before\\nlarge model training begins, and we hope this becomes a common goal in the field.\\n4 Capabilities\\nWe tested GPT-4 on a diverse set of benchmarks, including simulating exams that were originally\\ndesigned for humans.4We did no specific training for these exams. A minority of the problems in the\\nexams were seen by the model during training; for each exam we run a variant with these questions\\nremoved and report the lower score of the two. We believe the results to be representative. For further\\ndetails on contamination (methodology and per-exam statistics), see Appendix C.\\nExams were sourced from publicly-available materials. Exam questions included both multiple-\\nchoice and free-response questions; we designed separate prompts for each format, and images were\\nincluded in the input for questions which required it. The evaluation setup was designed based', 'SimilarityScore': 0.8285566330929187}), 0.8285566330929187)\n", - "(Document(page_content='overall GPT-4 training budget. When mixing in data from these math benchmarks, a portion of the\\ntraining data was held back, so each individual training example may or may not have been seen by\\nGPT-4 during training.\\nWe conducted contamination checking to verify the test set for GSM-8K is not included in the training\\nset (see Appendix D). We recommend interpreting the performance results reported for GPT-4\\nGSM-8K in Table 2 as something in-between true few-shot transfer and full benchmark-specific\\ntuning.\\nF Multilingual MMLU\\nWe translated all questions and answers from MMLU [ 49] using Azure Translate. We used an\\nexternal model to perform the translation, instead of relying on GPT-4 itself, in case the model had\\nunrepresentative performance for its own translations. We selected a range of languages that cover\\ndifferent geographic regions and scripts, we show an example question taken from the astronomy', metadata={'id': 'a7e86563-592a-4304-962b-25315f0d3f46', 'embedding': [-0.005677336826920509, -0.007514122407883406, 0.013824593275785446, -0.029277246445417404, -0.005625155754387379, 0.020705582574009895, 0.00037940230686217546, -0.006303513888269663, -0.042524367570877075, -0.03512156382203102, 0.025394950062036514, 0.05716298893094063, -0.031086202710866928, -0.008022021502256393, 0.0013036654563620687, 0.03437015414237976, 0.015584846027195454, 0.013984616845846176, 0.0026055914349853992, -0.02005157433450222, -0.01544569618999958, 0.008731688372790813, -0.029945168644189835, -0.005879105068743229, 0.007695017848163843, 0.0021742251701653004, 0.034147512167692184, -0.02949988842010498, -0.019981998950242996, 0.01817304454743862, 0.008279449306428432, -0.0009810103802010417, -0.037681933492422104, 0.006557463202625513, -0.01828436367213726, -0.017171161249279976, 0.0107215391471982, -0.01224523689597845, 0.03431449085474014, -0.0044841221533715725, 0.021777039393782616, 0.030780071392655373, 0.023029394447803497, -0.002196837216615677, 0.024727027863264084, 0.023432929068803787, 0.009830976836383343, -0.0059834676794707775, -0.009253502823412418, 0.015390035696327686, 6.017168198013678e-05, 0.020093319937586784, -0.016336258500814438, -0.0015419605188071728, -0.012878370471298695, 0.0027969232760369778, 0.007423674687743187, 0.0176721028983593, -0.009190885350108147, 0.006741837598383427, -0.007152331527322531, -0.0024316534399986267, -0.02152656950056553, 0.029945168644189835, -0.012537452392280102, -0.03918475657701492, -0.03659655898809433, 0.0030056489631533623, -0.00324220466427505, 0.008703858591616154, 0.0041466825641691685, 0.02422608807682991, 0.001679371576756239, -0.030112149193882942, 0.048285193741321564, -0.01651715487241745, 0.000572690914850682, -0.01476385910063982, -0.018534835427999496, 0.011431206949055195, 0.03228289633989334, -0.013748060911893845, -0.006275683641433716, 0.030724411830306053, 0.0056634219363331795, 0.020274216309189796, 0.014262917451560497, 0.001909839455038309, 0.004052755888551474, 0.007896785624325275, 0.016350174322724342, -0.00249601062387228, 0.023057224228978157, 0.01571008190512657, -0.0030352184548974037, -0.00639744009822607, -0.009858806617558002, 0.03192110359668732, -0.014833434484899044, -0.036095619201660156, -0.020260300487279892, 0.0032439441420137882, -0.019258417189121246, -0.008091596886515617, -0.038405515253543854, 0.017143331468105316, 0.018562665209174156, -0.010930265299975872, -0.012175661511719227, 0.0022629336453974247, -0.021610058844089508, 0.03219940513372421, -0.003892732784152031, -0.023168543353676796, -0.013087096624076366, 0.0011740815825760365, 0.018590494990348816, -0.005792135838419199, -0.01088156271725893, 0.005381642375141382, 0.017630357295274734, -0.012356556951999664, 0.026619473472237587, -0.00556253781542182, 0.03373005986213684, 0.02518622577190399, -0.006220023613423109, -0.0250609889626503, -0.03598429635167122, -0.004616315010935068, 0.0015637028263881803, 0.017922572791576385, 0.012760093435645103, 0.0013723709853366017, -0.02883196622133255, 0.02083081752061844, -0.024615708738565445, -0.007347141858190298, -0.025840232148766518, -0.008968244306743145, 0.0033309131395071745, 0.024504387751221657, 0.014200299978256226, -0.009601378813385963, 0.002789965830743313, 0.029444226995110512, 0.04068757966160774, 0.009156097657978535, 0.0056912521831691265, -0.02080298773944378, -0.008870839141309261, -0.029889509081840515, -0.004188427701592445, 0.01545961108058691, 0.015807487070560455, -0.02300156280398369, -0.004264960065484047, 0.01758861169219017, -0.016057956963777542, -0.0038301150780171156, -0.017171161249279976, 0.0189801175147295, 0.02558976039290428, 0.003910126630216837, 0.002181182848289609, 0.029750358313322067, 0.015403950586915016, 0.0005635591805912554, 0.0009149139514192939, -0.016405833885073662, -0.024267831817269325, 0.029138097539544106, -0.026563813909888268, 0.028177959844470024, 0.006439185235649347, 0.011758210137486458, 0.016725879162549973, -0.012711390852928162, -0.0068601155653595924, -0.006226981058716774, -0.0010749369394034147, 0.014680368825793266, 0.009928382001817226, -0.010102320462465286, -0.00798027589917183, -0.01834002509713173, 0.013560207560658455, 0.012808796018362045, -0.0072427792474627495, -0.019133182242512703, 0.017226820811629295, 0.012683560140430927, -0.002816056599840522, 0.009399610571563244, -0.6193863153457642, -0.01657281443476677, 0.014346407726407051, 0.00012599634646903723, 0.015890978276729584, 0.009448313154280186, 0.0013349743094295263, -0.010387578047811985, -0.040381450206041336, 0.07091104984283447, -0.005753869656473398, -0.0004900703788734972, -0.029026776552200317, -0.005350333638489246, 0.009065649472177029, -0.02540886588394642, -0.012725305743515491, -0.013706316240131855, 0.012947945855557919, 0.008564707823097706, -0.023419015109539032, 0.01177908293902874, 0.008738646283745766, 0.03094705194234848, -0.013434972614049911, 0.010443238541483879, -0.0024229565169662237, -0.019314076751470566, 0.008286407217383385, -0.00564950704574585, -0.03175412490963936, 0.011549483984708786, 0.0003996226005256176, 0.0004383237974252552, 0.03924041613936424, -0.017435546964406967, -0.03147582337260246, 0.014249002560973167, 0.010032745078206062, 0.008362939581274986, -0.03019564040005207, -0.012836625799536705, 0.009670954197645187, 0.025603676214814186, -0.007340184412896633, 0.02283458225429058, -0.0010540643706917763, 0.01069370936602354, 0.003986659459769726, -0.0064461431466042995, -0.010095362551510334, 0.010046659968793392, 0.009037819691002369, 0.002309896983206272, 0.00671400735154748, 0.009253502823412418, 0.012120001018047333, -0.009350907988846302, -0.004261481575667858, -0.006087830755859613, -0.014638623222708702, 0.014075064100325108, 0.0032300290185958147, -0.009851849637925625, -0.03150365501642227, -0.01692069135606289, -0.004445855971425772, 0.013720231130719185, 0.00841860007494688, -0.025784572586417198, 0.007354099303483963, 0.017839083448052406, 0.015111735090613365, -0.003367440076544881, -0.0009479622240178287, 0.014262917451560497, 0.015014329925179482, -0.025047075003385544, 0.013261034153401852, 0.01769993267953396, 0.007674145512282848, -0.01335844025015831, 0.015890978276729584, -0.001111463992856443, 0.015195225365459919, -0.013240162283182144, 0.007479334715753794, 0.014986500144004822, 0.0006270465673878789, 0.015626590698957443, 0.00738192955031991, 0.030613090842962265, -0.026925604790449142, -0.026243768632411957, 0.001408028299920261, 0.013845466077327728, -0.025659335777163506, -0.00018937501590698957, 0.02877630665898323, -0.03748712316155434, -0.012739220634102821, 0.000315914920065552, 0.05362857133150101, 0.023683400824666023, 0.02170746400952339, 0.014903009869158268, -0.02634117379784584, -0.014346407726407051, -0.012120001018047333, -0.013233204372227192, -0.0006096527795307338, 0.013365397229790688, 0.0019950689747929573, 0.0038301150780171156, 0.014012446627020836, -0.03656872734427452, 0.011166821233928204, 0.013233204372227192, -0.016016213223338127, -0.005604282952845097, -0.002468180377036333, -0.011751252226531506, 0.04344275966286659, -0.012419174425303936, 0.011639932170510292, 0.014429898001253605, 0.00999795738607645, -0.021262183785438538, 0.019773274660110474, 0.024991415441036224, -0.03320128843188286, -0.00814029946923256, 0.049537546932697296, -0.0231824591755867, 0.04912009462714195, 0.009336993098258972, 0.02572891116142273, -0.0052911946550011635, 0.013490633107721806, -0.023780805990099907, -0.04099371284246445, -0.0021011712960898876, 0.006348737515509129, -0.00981010403484106, -0.016030127182602882, -0.01753295212984085, -0.01657281443476677, -0.003750103758648038, 0.010575431399047375, -0.0035570324398577213, 0.0006470494554378092, -0.0028369291685521603, -0.007159288972616196, -0.0007114065228961408, -0.0013723709853366017, -0.023224204778671265, -0.010603261180222034, -0.03384138271212578, -0.022124916315078735, -0.03386921063065529, 0.007764593232423067, 0.01109724584966898, 0.011514697223901749, 0.017574697732925415, 0.009684869088232517, -6.560724432347342e-05, -0.011445121839642525, 0.016057956963777542, -0.011883445084095001, -0.025854147970676422, 0.0002359034406254068, -0.037765420973300934, 0.0075906552374362946, 0.011486866511404514, -0.023516420274972916, 1.8535271010478027e-05, -0.007388886995613575, -0.0026942999102175236, -0.019467143341898918, -0.029833849519491196, 0.007166246417909861, 0.01660064421594143, -0.025074904784560204, -0.008342067711055279, 0.04166163504123688, 0.02778833732008934, 0.02433740720152855, -0.0009001292637549341, -0.0068009765818715096, 0.008912583813071251, 0.012947945855557919, -0.005343375727534294, -0.029889509081840515, 0.019898509606719017, -0.0019133181776851416, 0.014652539044618607, 0.022097086533904076, -0.02112303301692009, 0.005232055671513081, 0.027635272592306137, 0.02315462939441204, 0.01657281443476677, 0.01646149344742298, -0.008015063591301441, 0.008056809194386005, -0.015000415034592152, 0.010749369859695435, -0.00250992551445961, 0.01611361838877201, 0.004390195477753878, 0.002487313700839877, -0.015501356683671474, -0.025144480168819427, -0.012377429753541946, -0.0010731975780799985, -0.006181756965816021, -0.003156975144520402, -0.007020138204097748, -0.02077515609562397, -0.0019550633151084185, 0.003146538743749261, 0.0107215391471982, 0.0140541922301054, 0.011528612114489079, -0.007841126061975956, 0.017825167626142502, -0.006567899603396654, 0.011173778213560581, 0.03715316206216812, -0.014172470197081566, -0.0037048798985779285, 0.001341931871138513, 0.029360737651586533, 0.015097820200026035, 0.026355087757110596, -0.006550505757331848, 0.023655571043491364, 0.014040276408195496, 0.029082436114549637, -2.145688449672889e-05, 0.02366948500275612, -0.0011984329903498292, -0.0004974627518095076, -0.02321028895676136, 0.021665720269083977, -0.010798072442412376, 0.023919956758618355, 0.0035413780715316534, -0.03253336623311043, 0.020329875871539116, -0.010700667276978493, 0.0027430024929344654, -0.022820668295025826, -0.005472090095281601, 0.005162480287253857, -0.020566431805491447, -0.0033813551999628544, 0.015334376133978367, 0.00011490780161693692, 0.020886477082967758, 0.013337567448616028, 0.011799954809248447, -0.00599042559042573, -0.012453962117433548, 0.018813136965036392, -0.015056074596941471, -1.521957619843306e-05, -0.004859828390181065, -0.027510037645697594, 0.007500207517296076, -0.011535569094121456, -0.022375386208295822, 0.003913605585694313, -0.030084319412708282, 0.02257019653916359, 0.011278141289949417, 0.0029569463804364204, 0.016503239050507545, 0.03239421546459198, 0.022375386208295822, -0.02958337776362896, -0.030418280512094498, 0.02382255159318447, 0.01003970205783844, -0.01973152905702591, -0.01022755540907383, -0.029221586883068085, -0.0016419749008491635, -0.014346407726407051, -0.007903743535280228, -0.025506271049380302, -0.0021846615709364414, -0.011264226399362087, -0.00025634115445427597, -0.01715724542737007, -0.004306705202907324, 0.02147090993821621, 0.005033766385167837, 0.00604260666295886, -0.00795244611799717, -0.010060574859380722, 0.0006609644624404609, -0.03937956690788269, -0.00884996633976698, 0.009009988978505135, -0.0021255225874483585, -0.022584112361073494, -0.021164778620004654, 0.0034074457362294197, -0.011347716674208641, 0.024212172254920006, -0.007875913754105568, -0.0026542942505329847, 0.008696900680661201, 0.013803721405565739, 0.0027116937562823296, -0.009858806617558002, 0.003529202425852418, 0.03612344712018967, 0.012147830799221992, -0.021512653678655624, -0.01906360685825348, -0.015097820200026035, 0.014304663054645061, 0.0359286367893219, 0.03094705194234848, -0.015376120805740356, -0.00949005875736475, -0.031086202710866928, 0.0032630772329866886, -0.024379152804613113, -0.03192110359668732, -0.005722560919821262, 0.006265247240662575, -0.007263651583343744, -0.0033639613538980484, 0.00930220540612936, -0.007009702268987894, 0.02961120754480362, 0.006623559631407261, -0.010192767716944218, -0.0208586473017931, -0.013977658934891224, 0.008244662545621395, 0.007451504934579134, -0.027106501162052155, 0.021025627851486206, 0.016266683116555214, 0.004758944269269705, -0.003341349307447672, 0.012913158163428307, -0.0015045638429000974, -0.014047234319150448, -0.02880413644015789, -0.02077515609562397, 0.0009749226155690849, 0.014875179156661034, 0.004560654982924461, -0.006091309245675802, 0.007555867545306683, 0.007201034110039473, -0.0072427792474627495, 0.012147830799221992, -0.010276257991790771, 0.019369738176465034, -0.0014863003743812442, -0.003078702837228775, -0.010714582167565823, 0.008801263757050037, -0.014916924759745598, 0.005033766385167837, 0.032060254365205765, 0.002301200060173869, -0.027913574129343033, -0.00046310998732224107, 0.005771263502538204, -0.014485558494925499, -0.011667761951684952, 0.025603676214814186, -0.0041118948720395565, -6.598773325094953e-05, 0.008578622713685036, -0.025478441268205643, -0.028720645233988762, -0.013671528548002243, -0.03286732733249664, 0.02575674094259739, -0.0017532951897010207, -0.03239421546459198, 0.010081447660923004, -0.019842850044369698, -0.011862573213875294, -0.02540886588394642, 0.012760093435645103, -0.030056489631533623, -0.021957935765385628, -0.04848000407218933, -0.013114926405251026, 0.01476385910063982, 0.02401736192405224, 0.03142016381025314, -0.0075906552374362946, -0.007771550677716732, 0.008307280018925667, -0.014165512286126614, -0.016364088281989098, 0.004546739626675844, -0.026090702041983604, -0.014889094047248363, -0.0022351036313921213, -0.009650081396102905, -0.0011653847759589553, 0.0020559474360197783, 0.01817304454743862, -0.013915041461586952, -0.0007492380100302398, -0.008863881230354309, -0.004491079598665237, 0.019870679825544357, 0.001178430044092238, 0.01726856641471386, -0.005162480287253857, 0.025283630937337875, -0.009135224856436253, -0.008989117108285427, -0.03437015414237976, -0.026229852810502052, -0.04283049702644348, 0.017783423885703087, 0.01831219531595707, 0.0044215042144060135, 0.016670219600200653, 0.017087670043110847, -0.00534685468301177, 0.007548910100013018, -0.020260300487279892, 0.0012949685333296657, 0.004602400120347738, 0.012913158163428307, -0.01943931356072426, 0.019842850044369698, 0.020580345764756203, -0.003348306752741337, -0.031253181397914886, -0.000644440355245024, -0.04383238032460213, 0.027370886877179146, 0.016906775534152985, 0.0038649027701467276, 0.033423930406570435, -0.0007918528281152248, -0.0015671815490350127, -0.050344619899988174, 0.003739667357876897, -0.00011577749683056027, 0.03921258822083473, 0.002102910540997982, 0.0016350173391401768, -0.0072149490006268024, -0.00029373783036135137, -0.02436523698270321, 0.005739954765886068, 0.00033570037339814007, 0.004216257482767105, -0.025812402367591858, 0.010839817114174366, 0.0005935634835623205, -0.0070270961150527, 0.007016659714281559, -0.026563813909888268, -0.02002374455332756, 0.011278141289949417, 0.003850987646728754, 0.023766890168190002, 0.017310312017798424, 0.013191459700465202, -0.0338970422744751, -0.021248267963528633, -0.019884593784809113, -0.016016213223338127, 0.004132767207920551, -0.014026361517608166, 0.0253532063215971, 0.02205534093081951, 0.04650406911969185, -0.015097820200026035, -0.009768359363079071, 0.015835316851735115, -0.011723422445356846, 0.0013341045705601573, -0.012746177613735199, 0.008035936392843723, -0.0027951840311288834, -0.010874604806303978, 0.0010714582167565823, -0.01408202201128006, 0.01625276915729046, -0.008195959031581879, -0.003115230007097125, 0.030640920624136925, -0.013448887504637241, 0.010116235353052616, -0.015863146632909775, -0.020524686202406883, -0.013441930525004864, 0.027370886877179146, -0.016266683116555214, -0.008766476064920425, -0.014471643604338169, -0.032060254365205765, 0.01002578716725111, -0.0022420610766857862, -0.0072358218021690845, 0.01342801470309496, 0.02848408930003643, -0.020037660375237465, -0.015988383442163467, -0.00792461633682251, 0.013581080362200737, -0.02277892269194126, -0.011215523816645145, -0.044082850217819214, -0.0011358152842149138, 0.01315667200833559, -0.020329875871539116, 0.018047809600830078, -0.0017445983830839396, -0.0060808733105659485, -0.018075639382004738, 0.005385120864957571, -0.016308428719639778, -0.014200299978256226, -0.012029553763568401, -0.03172629326581955, -0.011333800852298737, -0.012182618491351604, 0.009573548100888729, 0.013448887504637241, -0.003656177083030343, 0.017198991030454636, 0.004525867290794849, 0.02489400841295719, -0.024379152804613113, -0.003551814239472151, 0.024406982585787773, 0.011827785521745682, 0.05351724848151207, 0.005642549134790897, 0.007848083041608334, 0.041745126247406006, 0.0075976126827299595, -0.027370886877179146, 0.012440047226846218, 0.011145948432385921, 0.0028873709961771965, 0.02848408930003643, 0.033479589968919754, -0.026229852810502052, -0.011786039918661118, 0.019564548507332802, 0.01756078191101551, -0.01973152905702591, -0.0385446660220623, 0.02135958895087242, 0.019188841804862022, 0.020677750930190086, -0.01854874938726425, -0.006874030455946922, -0.0365130677819252, 0.01409593690186739, 0.000167632766533643, 0.00490505201742053, 0.007660230156034231, 0.007444547023624182, -0.0008357721962966025, 0.03142016381025314, 0.0012688778806477785, 0.004240608774125576, -0.008961286395788193, -0.0068009765818715096, 0.023405099287629128, -0.00220553413964808, -0.005736475810408592, 0.010763284750282764, 0.0033865731675177813, 0.03990833833813667, -0.011674719862639904, 0.015723997727036476, 0.007917658425867558, -0.012467877008020878, 0.013038394041359425, 0.0009984042262658477, 0.007312354166060686, 0.031308844685554504, -0.0324498750269413, 0.004522388335317373, 0.002577761420980096, -0.005381642375141382, -0.0009410046623088419, 0.002958685625344515, -0.009211757220327854, -0.017254650592803955, -0.012732262723147869, 0.015292630530893803, 0.012746177613735199, 0.003958829212933779, -0.015236970037221909, 0.016614560037851334, -0.001603708486072719, -0.02749612182378769, -0.019661953672766685, -0.013761975802481174, 0.030000830069184303, -0.04377672076225281, -0.014708198606967926, 0.003690964775159955, -0.013532377779483795, 0.025548016652464867, -0.01184865739196539, -0.025019245222210884, 0.0005809529684484005, 0.00982401892542839, -0.016475409269332886, -0.0023603388108313084, -4.606639777193777e-06, -0.008516005240380764, -0.020037660375237465, 0.006553984712809324, -0.016684135422110558, -0.005319024436175823, 0.019773274660110474, -0.028971116989850998, -0.014680368825793266, -0.016044043004512787, 0.005684294272214174, -0.001927233301103115, 0.01477777399122715, -0.015264800749719143, -0.0009870982030406594, -0.014569048769772053, 0.007030574604868889, -0.016711965203285217, 0.006265247240662575, -0.0024368716403841972, -0.007354099303483963, 0.01040149386972189, 0.018840966746211052, -0.011236395686864853, 0.0072358218021690845, 0.0009870982030406594, -0.0022559762001037598, -0.007263651583343744, -0.03100271336734295, 0.0012958382721990347, 0.006077394355088472, 0.017379887402057648, -0.02422608807682991, -0.006195672322064638, -0.015751827508211136, 0.00770893320441246, -0.012857498601078987, 0.007507164962589741, -0.00432409904897213, 0.00841860007494688, 0.022528452798724174, -0.0012123479973524809, 0.02147090993821621, 0.01616927795112133, -0.0486748144030571, -0.001636756700463593, -0.01159122958779335, 0.009051734581589699, -0.007472377270460129, 4.805037679034285e-05, 0.009030861780047417, 0.04146682471036911, -0.007841126061975956, -0.028233619406819344, -0.023419015109539032, -0.017018096521496773, -0.03242204710841179, -0.036874860525131226, -0.013789806514978409, 0.019856764003634453, 0.009830976836383343, 0.048257362097501755, -0.01113899052143097, 0.03573382645845413, 0.018938371911644936, 0.008202916942536831, -0.011069415137171745, 0.0013375834096223116, -0.017435546964406967, -0.008738646283745766, -0.004404110834002495, 0.0029395525343716145, -0.04850783571600914, -0.026675134897232056, 0.0009975344873964787, -0.019522802904248238, 0.01654498465359211, 0.021957935765385628, 0.015807487070560455, -0.02880413644015789, -0.007034053560346365, 0.008321194909512997, 0.013163628987967968, 0.006480930373072624, 0.025923721492290497, -0.032060254365205765, 0.007194076664745808, 0.006289598532021046, -0.013240162283182144, -0.021749209612607956, -0.009246544912457466, 0.02417042665183544, -0.019230587407946587, -0.00036722663207910955, -0.003649219637736678, -0.006515718065202236, 0.029917338863015175, -0.030390450730919838, 0.025979382917284966, 0.011034627445042133, -0.000648353947326541, 0.012940988875925541, 0.01619710773229599, 0.005948680453002453, -0.014304663054645061, 0.009893594309687614, -0.015209140256047249, -0.015014329925179482, 0.04163380339741707, -0.019216671586036682, -0.017602527514100075, -0.02048294059932232, -0.003341349307447672, -0.019828934222459793, -0.012989691458642483, -0.0035239842254668474, 0.0009262199746444821, -0.035483356565237045, 0.03225506469607353, 0.012140873819589615, -0.027816167101264, 0.0075697824358940125, -0.008759519085288048, 0.011236395686864853, -0.001740249921567738, -0.012620942667126656, -0.004000574350357056, -0.03400836139917374, -0.001925493823364377, -0.0020820379722863436, -0.0016915472224354744, -0.008355982601642609, -0.009552676230669022, 0.0009957951260730624, -0.015111735090613365, 0.013504547998309135, 0.1903577744960785, -0.03673570975661278, -0.00273082684725523, 0.03175412490963936, 0.014986500144004822, 0.005465132649987936, 0.015974467620253563, 0.01900794729590416, 0.006814891472458839, -0.0030908784829080105, -0.01587706245481968, -0.015334376133978367, 0.0009236108744516969, -0.0021342195104807615, 0.004407589323818684, -0.019578462466597557, -0.026299428194761276, -0.026939520612359047, -0.0063835252076387405, 0.012092171236872673, 0.0007192337070591748, -0.008474260568618774, 0.004365844186395407, -0.01476385910063982, 0.006668783724308014, 0.003844030201435089, -0.004623272456228733, 0.006021734327077866, 0.014235087670385838, 0.000670531066134572, -0.007361056748777628, -0.006282641086727381, 0.006185235921293497, -0.006094788201153278, -0.0038057637866586447, 0.0018820093246176839, 0.011278141289949417, -0.004585006274282932, 0.020496856421232224, 0.02034378983080387, 0.016823284327983856, 0.0028664986602962017, 0.0199959147721529, -0.0225980281829834, -0.011939105577766895, 0.027857912704348564, -0.010484984144568443, 0.015529186464846134, -0.010324960574507713, -0.004842434544116259, -0.015835316851735115, 0.004369323141872883, 0.026466408744454384, 0.016294512897729874, -0.011883445084095001, 0.0024281747173517942, -0.014847349375486374, -0.010192767716944218, 0.0029291161336004734, 0.004251045174896717, -0.022124916315078735, -0.005148565396666527, -0.02390604093670845, 0.037654101848602295, -0.02685602940618992, 0.015195225365459919, -0.004157118499279022, -0.009524845518171787, 0.017059840261936188, -0.00612957589328289, -0.02176312543451786, -0.002949988702312112, -0.008801263757050037, -0.004348450340330601, -0.03445364162325859, -0.038989946246147156, 0.020246384665369987, 0.029277246445417404, 0.02572891116142273, 0.036930520087480545, -0.011946063488721848, -0.026035042479634285, 0.004553697537630796, -0.028943287208676338, -0.025520186871290207, -0.04792340472340584, 0.001739380182698369, -0.02590980753302574, -0.00046267511788755655, -0.005969552788883448, -0.007124501280486584, -0.022389302030205727, -0.008780390955507755, 0.004177991300821304, -0.002911722520366311, 0.0055764527060091496, 0.025923721492290497, -0.015584846027195454, -0.010088404640555382, -0.04850783571600914, -0.025951553136110306, 0.05468611419200897, 0.00475198682397604, 0.013497590087354183, 0.02222232148051262, 0.011521654203534126, -0.009434398263692856, 0.008856924250721931, -0.00021927061607129872, -0.008175087161362171, 0.0034248395822942257, -0.03954654932022095, 0.018242619931697845, 0.0014471643371507525, -0.00694012688472867, 0.0044388980604708195, -0.0034561483189463615, 0.00884996633976698, 0.034815434366464615, -0.012732262723147869, 0.008286407217383385, -0.01596055179834366, 0.013149714097380638, 0.00692621199414134, 0.001561093726195395, -0.03142016381025314, -0.013476717285811901, 0.0015054335817694664, -0.023572079837322235, -0.01576574146747589, 0.019453227519989014, -0.07085539400577545, 0.0176721028983593, -0.02184661477804184, 0.002920419443398714, 0.012544410303235054, -0.0038822966162115335, -0.006112182047218084, -0.032700348645448685, -0.0031100118067115545, 0.033535249531269073, 0.006734880153089762, -0.0028369291685521603, -0.009023904800415039, 0.017463376745581627, -0.01646149344742298, 0.010686751455068588, 0.021136948838829994, -0.011612102389335632, -0.013191459700465202, -0.02387821115553379, -0.0007431501871906221, -0.006661826279014349, 0.00954571831971407, 0.04102154076099396, -0.002916940487921238, -0.013671528548002243, -0.015501356683671474, 0.018646156415343285, -0.004932882264256477, -0.03790457174181938, -0.0168093703687191, 0.016503239050507545, -0.007068841252475977, -0.012238278985023499, -0.0008657764992676675, -0.177889883518219, -0.001493257936090231, -0.0052911946550011635, -0.02179095521569252, 0.026327257975935936, -0.0009009989444166422, 0.030668752267956734, 0.004132767207920551, -0.02709258534014225, -0.011027670465409756, 0.0072427792474627495, -0.0035013724118471146, -0.0008057678933255374, 0.00022133612947072834, -0.027941403910517693, 0.0017524255672469735, -0.016531068831682205, 0.007778508123010397, 0.019842850044369698, 0.011758210137486458, 0.038934286683797836, -0.020538602024316788, 0.0026508152950555086, -0.007493250072002411, -0.003927520476281643, -0.0064357067458331585, -0.0008957808022387326, 0.029277246445417404, -0.004404110834002495, -0.03857249394059181, -0.011521654203534126, 0.008495132438838482, 0.01090939249843359, 0.009455271065235138, 0.0044215042144060135, 0.016558898612856865, 0.03153148293495178, -0.00039114311221055686, -0.025102734565734863, 0.011521654203534126, 0.034036193042993546, 0.02283458225429058, -0.015000415034592152, 0.004327578004449606, -0.029082436114549637, 0.014214214868843555, 0.0231824591755867, -0.018298279494047165, 0.01871572993695736, 0.013149714097380638, 0.02845625951886177, -0.009907509200274944, 0.010533686727285385, -0.0032717741560190916, -0.004529345780611038, 0.004254524130374193, -0.0013636740623041987, -0.013469760306179523, 0.00910043716430664, -0.011890402995049953, -0.01477777399122715, -0.011438163928687572, 0.016823284327983856, -0.0035083298571407795, -0.005141607951372862, -0.025367120280861855, -0.02051077038049698, 0.020900392904877663, -0.006901860702782869, 0.010290172882378101, -0.023544250056147575, -0.016294512897729874, 0.01003970205783844, -0.0075628249906003475, 0.033368270844221115, 0.017727762460708618, -0.005298152100294828, 0.009615293703973293, 0.024114767089486122, 0.01761644333600998, -0.01721290685236454, 0.03456496447324753, -0.007903743535280228, 0.0006840112619102001, -0.012767050415277481, -0.007034053560346365, 0.007065362296998501, 0.024351323023438454, -0.005127692595124245, -0.008168129250407219, 0.013038394041359425, 0.0028578017372637987, 0.011020712554454803, 0.013789806514978409, 0.04221823439002037, -0.001977675361558795, -0.011145948432385921, -0.012370471842586994, 0.01943931356072426, -0.016350174322724342, -0.003252641065046191, 0.01863224059343338, -0.020427281036973, -0.005259885918349028, 0.03192110359668732, 0.00951788853853941, -0.009497015736997128, 0.014652539044618607, 0.028205789625644684, -0.015835316851735115, -0.01622493751347065, -0.014360322616994381, 0.0029969520401209593, 0.02631334401667118, -0.005712124519050121, 0.028442345559597015, 0.004285832867026329, -0.025422781705856323, 0.021999681368470192, 0.0072358218021690845, 0.03818287327885628, -0.026925604790449142, -0.0026786455418914557, 0.015932722017169, -0.009657038375735283, -0.01657281443476677, -0.12167312204837799, -0.001468036905862391, 0.02379472181200981, 0.03219940513372421, -0.004397152923047543, 0.030362620949745178, -0.0003496154095046222, 0.04569699615240097, -0.012627900578081608, 0.04360973834991455, -0.025047075003385544, -0.017783423885703087, 0.008717773482203484, 0.0014567308826372027, -0.021665720269083977, -0.006313950289040804, 0.04419417306780815, -0.02543669566512108, -0.01335844025015831, 0.03793240338563919, 0.022403215989470482, 0.00021655282762367278, 0.01790865883231163, -0.026953434571623802, -0.006460058037191629, -0.014249002560973167, -0.05148565396666527, 0.028442345559597015, 0.012906201183795929, -0.008293365128338337, 0.01686502993106842, -0.035594675689935684, 0.010311045683920383, -0.031281013041734695, 0.0017072017071768641, -0.010985924862325191, -0.020580345764756203, -0.0037292311899363995, 0.01477777399122715, 0.008968244306743145, -0.005715603474527597, 0.01726856641471386, -0.006672262214124203, -0.02471311390399933, -0.00021253051818348467, -0.019077522680163383, -0.030585261061787605, 0.027245651930570602, -0.019077522680163383, -0.017950404435396194, -0.02294590324163437, -0.01111116074025631, -0.03718098998069763, 0.014499473385512829, 0.013970701955258846, 0.0210117120295763, -0.0029047648422420025, 0.008536878041923046, 0.014666453935205936, -0.011932147666811943, -0.02784399874508381, -0.01874356158077717, 0.0060982671566307545, -0.004306705202907324, 0.008411642163991928, 0.02634117379784584, -0.028720645233988762, -0.03306213766336441, 0.020274216309189796, -0.009817061945796013, -0.0047867740504443645, 0.029110265895724297, 0.002162049524486065, 0.012238278985023499, -0.030390450730919838, 0.0016863291384652257, -0.02147090993821621, -0.01854874938726425, 0.016308428719639778, -0.011145948432385921, -0.023836465552449226, -0.01909143663942814, 0.004825040698051453, -0.015223055146634579, -0.016670219600200653, 0.010978967882692814, -0.011201607994735241, -0.02433740720152855, 0.004338014405220747, -0.029416397213935852, -0.023432929068803787, 0.02613244764506817, -0.009218715131282806, -0.02353033609688282, -0.005117256660014391, 0.012168703600764275, 0.0048841796815395355, -0.008481217548251152, 0.032644689083099365, 0.0017863434040918946, -0.0072427792474627495, 0.013845466077327728, -0.06250636279582977, 0.016030127182602882, 0.013671528548002243, -0.012836625799536705, 0.0005583410384133458, 0.020329875871539116, 0.005746912211179733, 0.007681102957576513, -0.022639771923422813, 0.022124916315078735, -0.012120001018047333, 0.004338014405220747, -0.032060254365205765, -0.03239421546459198, -0.01338627003133297, -0.01764427311718464, -0.0015750087331980467, -0.003428318304941058, 0.015236970037221909, 0.025826316326856613, -0.01836785487830639, 0.00466501759365201, 0.038934286683797836, 0.0259376373142004, 0.02115086279809475, 0.01247483491897583, -0.03514939546585083, 0.0330343097448349, 0.0023185936734080315, -0.017393801361322403, 0.028581494465470314, -0.0044841221533715725, -0.015056074596941471, 0.0067731463350355625, -0.007458462379872799, -0.004021447151899338, 0.005927807651460171, 0.019161012023687363, 0.028066638857126236, 0.01962020806968212, -0.030752241611480713, -0.016739794984459877, 0.008815178647637367, -0.02567325159907341, -0.006133054383099079, 0.01200868096202612, 0.0034996329341083765, -0.00670009246096015, 0.011069415137171745, 0.014102894812822342, 0.02958337776362896, -0.0022385823540389538, 1.4431419913307764e-05, -0.025227969512343407, -0.010164937935769558, 0.010805029422044754, 0.00839077029377222, -0.0020142022985965014, 0.023864295333623886, 0.010436281561851501, 0.039824847131967545, 0.004929403308779001, 0.007764593232423067, -0.010818944312632084, 0.008975202217698097, 0.0007305396720767021, -0.022041425108909607, -0.012808796018362045, 0.003708358621224761, -0.016670219600200653, 0.0007044489611871541, -0.007416717242449522, 0.0172407366335392, 0.0168093703687191, 0.016725879162549973, -0.01018581073731184, -0.0005083338473923504, 0.0027777901850640774, -0.009865764528512955, 0.030112149193882942, 0.022180575877428055, 0.011173778213560581, -0.014158554375171661, -0.0009323077974840999, 0.016698049381375313, -0.016642389819025993, -0.0019985479302704334, 0.023627741262316704, -0.004630229901522398, 0.0113755464553833, 0.007416717242449522, 0.006393961608409882, 0.008599495515227318, 0.019314076751470566, -0.003562250640243292, 0.02952771820127964, 0.01686502993106842, -0.0037779337726533413, 0.024615708738565445, 0.03178195655345917, 0.008724731393158436, -0.027315227314829826, -0.011354673653841019, -0.012662687338888645, -0.028414513915777206, 0.005715603474527597, -0.05577148497104645, -0.03091922216117382, 0.01109724584966898, 0.02958337776362896, 0.004289311356842518, 0.01871572993695736, 0.00032526408904232085, 0.0035483355168253183, -0.008293365128338337, 0.01721290685236454, 0.011236395686864853, -0.0008192480891011655, -0.020469026640057564, 0.013274949975311756, 0.01854874938726425, 0.0282475333660841, 0.04628142714500427, -0.007479334715753794, 0.038349855691194534, -0.00544773880392313, 0.029054606333374977, -0.020427281036973, 0.03161497414112091, 0.025088820606470108, -0.006748795043677092, 0.02312679961323738, 0.01654498465359211, -0.003315258538350463, -0.0006518327281810343, -0.004571090918034315, -0.01628059893846512, 0.014499473385512829, 0.02717607654631138, 0.10163546353578568, 0.0208586473017931, -0.005058117676526308, -0.0032143746502697468, -0.00030982709722593427, -0.013309736736118793, 0.011292056180536747, 0.010985924862325191, -0.0048528709448874, -0.027649186551570892, 0.004376280587166548, -0.0018141735345125198, 0.012217406183481216, -0.02845625951886177, -0.006414833944290876, -0.013998531736433506, 0.013796763494610786, 0.0038822966162115335, -0.01866007037460804, -0.005117256660014391, 0.03386921063065529, 0.000449629791546613, 0.021512653678655624, 0.018242619931697845, 0.004529345780611038, -0.010631091892719269, 0.021456994116306305, 0.014582963660359383, -0.0033204767387360334, -0.01801997795701027, 0.011953020468354225, -0.0059660738334059715, -0.04355407878756523, -0.031281013041734695, 0.0007214079378172755, -0.028525834903120995, -0.005037244874984026, -0.018757475540041924, 0.0282475333660841, -0.004946797154843807, -0.0014019404770806432, 0.012544410303235054, -0.016378004103899002, -0.013768933713436127, 0.0033622218761593103, -0.021637888625264168, -0.010965052992105484, -0.02045511081814766, -0.024420898407697678], 'text': 'overall GPT-4 training budget. When mixing in data from these math benchmarks, a portion of the\\ntraining data was held back, so each individual training example may or may not have been seen by\\nGPT-4 during training.\\nWe conducted contamination checking to verify the test set for GSM-8K is not included in the training\\nset (see Appendix D). We recommend interpreting the performance results reported for GPT-4\\nGSM-8K in Table 2 as something in-between true few-shot transfer and full benchmark-specific\\ntuning.\\nF Multilingual MMLU\\nWe translated all questions and answers from MMLU [ 49] using Azure Translate. We used an\\nexternal model to perform the translation, instead of relying on GPT-4 itself, in case the model had\\nunrepresentative performance for its own translations. We selected a range of languages that cover\\ndifferent geographic regions and scripts, we show an example question taken from the astronomy', 'SimilarityScore': 0.8271415481962364}), 0.8271415481962364)\n" + "Result 1: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":0,\"id\":\"9d59c3ed-deac-48cb-9382-a8ab079334e5\"},\"page_content\":\"performance based on models trained with no more than 1/1,000th the compute of\\nGPT-4.\\n1 Introduction\\nThis technical report presents GPT-4, a large multimodal model capable of processing image and\\ntext inputs and producing text outputs. Such models are an important area of study as they have the\\npotential to be used in a wide range of applications, such as dialogue systems, text summarization,\\nand machine translation. As such, they have been the subject of substantial interest and progress in\\nrecent years [1–34].\\nOne of the main goals of developing such models is to improve their ability to understand and generate\\nnatural language text, particularly in more complex and nuanced scenarios. To test its capabilities\\nin such scenarios, GPT-4 was evaluated on a variety of exams originally designed for humans. In\\nthese evaluations it performs quite well and often outscores the vast majority of human test takers.\",\"type\":\"Document\"}\n", + "Score 1: 0.8394796122122777\n", + "\n", + "\n", + "Result 2: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":0,\"id\":\"cddfb7ac-d953-46f4-8a48-76655f116bcf\"},\"page_content\":\"GPT-4 Technical Report\\nOpenAI∗\\nAbstract\\nWe report the development of GPT-4, a large-scale, multimodal model which can\\naccept image and text inputs and produce text outputs. While less capable than\\nhumans in many real-world scenarios, GPT-4 exhibits human-level performance\\non various professional and academic benchmarks, including passing a simulated\\nbar exam with a score around the top 10% of test takers. GPT-4 is a Transformer-\\nbased model pre-trained to predict the next token in a document. The post-training\\nalignment process results in improved performance on measures of factuality and\\nadherence to desired behavior. A core component of this project was developing\\ninfrastructure and optimization methods that behave predictably across a wide\\nrange of scales. This allowed us to accurately predict some aspects of GPT-4’s\\nperformance based on models trained with no more than 1/1,000th the compute of\\nGPT-4.\\n1 Introduction\",\"type\":\"Document\"}\n", + "Score 2: 0.8286253517208215\n", + "\n", + "\n", + "Result 3: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":0,\"id\":\"ba814d15-2c12-40d2-8934-db58b393ecb8\"},\"page_content\":\"model capability results, as well as model safety improvements and results, in more detail in later\\nsections.\\nThis report also discusses a key challenge of the project, developing deep learning infrastructure and\\noptimization methods that behave predictably across a wide range of scales. This allowed us to make\\npredictions about the expected performance of GPT-4 (based on small runs trained in similar ways)\\nthat were tested against the final run to increase confidence in our training.\\nDespite its capabilities, GPT-4 has similar limitations to earlier GPT models [ 1,37,38]: it is not fully\\nreliable (e.g. can suffer from “hallucinations”), has a limited context window, and does not learn\\n∗Please cite this work as “OpenAI (2023)\\\". Full authorship contribution statements appear at the end of the\\ndocument. Correspondence regarding this technical report can be sent to gpt4-report@openai.comarXiv:2303.08774v6 [cs.CL] 4 Mar 2024\",\"type\":\"Document\"}\n", + "Score 3: 0.8215997601854081\n", + "\n", + "\n", + "Result 4: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":0,\"id\":\"dd040c08-6ae1-4c73-8b85-7f034d337891\"},\"page_content\":\"these evaluations it performs quite well and often outscores the vast majority of human test takers.\\nFor example, on a simulated bar exam, GPT-4 achieves a score that falls in the top 10% of test takers.\\nThis contrasts with GPT-3.5, which scores in the bottom 10%.\\nOn a suite of traditional NLP benchmarks, GPT-4 outperforms both previous large language models\\nand most state-of-the-art systems (which often have benchmark-specific training or hand-engineering).\\nOn the MMLU benchmark [ 35,36], an English-language suite of multiple-choice questions covering\\n57 subjects, GPT-4 not only outperforms existing models by a considerable margin in English, but\\nalso demonstrates strong performance in other languages. On translated variants of MMLU, GPT-4\\nsurpasses the English-language state-of-the-art in 24 of 26 languages considered. We discuss these\\nmodel capability results, as well as model safety improvements and results, in more detail in later\\nsections.\",\"type\":\"Document\"}\n", + "Score 4: 0.8209972517303962\n", + "\n", + "\n" ] } ], "source": [ "query = \"What were the compute requirements for training GPT 4\"\n", "\n", + "pre_filter = {\n", + " \"conditions\": [\n", + " {\"property\": \"metadata.page\", \"operator\": \"$eq\", \"value\": 0},\n", + " ],\n", + "}\n", + "\n", "results = vector_search.similarity_search_with_score(\n", " query=query,\n", " k=5,\n", + " pre_filter=pre_filter,\n", ")\n", "\n", "# Display results\n", - "for result in results:\n", - " print(result)" + "for i in range(0, len(results)):\n", + " print(f\"Result {i+1}: \", results[i][0].json())\n", + " print(f\"Score {i+1}: \", results[i][1])\n", + " print(\"\\n\")" ] }, + { + "cell_type": "markdown", + "id": "9fd7b4932ed5f42a", + "metadata": {}, + "source": "## Full Text Search" + }, { "cell_type": "code", - "execution_count": 44, - "id": "90f412e2ade96761", + "execution_count": 42, + "id": "a58a4cbcc160a0b0", + "metadata": { + "ExecuteTime": { + "end_time": "2024-12-13T21:44:49.510723Z", + "start_time": "2024-12-13T21:44:48.914182Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Result 1: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":0,\"id\":\"cddfb7ac-d953-46f4-8a48-76655f116bcf\"},\"page_content\":\"GPT-4 Technical Report\\nOpenAI∗\\nAbstract\\nWe report the development of GPT-4, a large-scale, multimodal model which can\\naccept image and text inputs and produce text outputs. While less capable than\\nhumans in many real-world scenarios, GPT-4 exhibits human-level performance\\non various professional and academic benchmarks, including passing a simulated\\nbar exam with a score around the top 10% of test takers. GPT-4 is a Transformer-\\nbased model pre-trained to predict the next token in a document. The post-training\\nalignment process results in improved performance on measures of factuality and\\nadherence to desired behavior. A core component of this project was developing\\ninfrastructure and optimization methods that behave predictably across a wide\\nrange of scales. This allowed us to accurately predict some aspects of GPT-4’s\\nperformance based on models trained with no more than 1/1,000th the compute of\\nGPT-4.\\n1 Introduction\",\"type\":\"Document\"}\n", + "\n", + "\n", + "Result 2: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":0,\"id\":\"9d59c3ed-deac-48cb-9382-a8ab079334e5\"},\"page_content\":\"performance based on models trained with no more than 1/1,000th the compute of\\nGPT-4.\\n1 Introduction\\nThis technical report presents GPT-4, a large multimodal model capable of processing image and\\ntext inputs and producing text outputs. Such models are an important area of study as they have the\\npotential to be used in a wide range of applications, such as dialogue systems, text summarization,\\nand machine translation. As such, they have been the subject of substantial interest and progress in\\nrecent years [1–34].\\nOne of the main goals of developing such models is to improve their ability to understand and generate\\nnatural language text, particularly in more complex and nuanced scenarios. To test its capabilities\\nin such scenarios, GPT-4 was evaluated on a variety of exams originally designed for humans. In\\nthese evaluations it performs quite well and often outscores the vast majority of human test takers.\",\"type\":\"Document\"}\n", + "\n", + "\n", + "Result 3: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":0,\"id\":\"dd040c08-6ae1-4c73-8b85-7f034d337891\"},\"page_content\":\"these evaluations it performs quite well and often outscores the vast majority of human test takers.\\nFor example, on a simulated bar exam, GPT-4 achieves a score that falls in the top 10% of test takers.\\nThis contrasts with GPT-3.5, which scores in the bottom 10%.\\nOn a suite of traditional NLP benchmarks, GPT-4 outperforms both previous large language models\\nand most state-of-the-art systems (which often have benchmark-specific training or hand-engineering).\\nOn the MMLU benchmark [ 35,36], an English-language suite of multiple-choice questions covering\\n57 subjects, GPT-4 not only outperforms existing models by a considerable margin in English, but\\nalso demonstrates strong performance in other languages. On translated variants of MMLU, GPT-4\\nsurpasses the English-language state-of-the-art in 24 of 26 languages considered. We discuss these\\nmodel capability results, as well as model safety improvements and results, in more detail in later\\nsections.\",\"type\":\"Document\"}\n", + "\n", + "\n", + "Result 4: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":0,\"id\":\"ba814d15-2c12-40d2-8934-db58b393ecb8\"},\"page_content\":\"model capability results, as well as model safety improvements and results, in more detail in later\\nsections.\\nThis report also discusses a key challenge of the project, developing deep learning infrastructure and\\noptimization methods that behave predictably across a wide range of scales. This allowed us to make\\npredictions about the expected performance of GPT-4 (based on small runs trained in similar ways)\\nthat were tested against the final run to increase confidence in our training.\\nDespite its capabilities, GPT-4 has similar limitations to earlier GPT models [ 1,37,38]: it is not fully\\nreliable (e.g. can suffer from “hallucinations”), has a limited context window, and does not learn\\n∗Please cite this work as “OpenAI (2023)\\\". Full authorship contribution statements appear at the end of the\\ndocument. Correspondence regarding this technical report can be sent to gpt4-report@openai.comarXiv:2303.08774v6 [cs.CL] 4 Mar 2024\",\"type\":\"Document\"}\n", + "\n", + "\n", + "Result 5: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":1,\"id\":\"9edf6760-a2d0-4a0b-a652-25fc89de1d34\"},\"page_content\":\"from experience. Care should be taken when using the outputs of GPT-4, particularly in contexts\\nwhere reliability is important.\\nGPT-4’s capabilities and limitations create significant and novel safety challenges, and we believe\\ncareful study of these challenges is an important area of research given the potential societal impact.\\nThis report includes an extensive system card (after the Appendix) describing some of the risks we\\nforesee around bias, disinformation, over-reliance, privacy, cybersecurity, proliferation, and more.\\nIt also describes interventions we made to mitigate potential harms from the deployment of GPT-4,\\nincluding adversarial testing with domain experts, and a model-assisted safety pipeline.\\n2 Scope and Limitations of this Technical Report\\nThis report focuses on the capabilities, limitations, and safety properties of GPT-4. GPT-4 is a\\nTransformer-style model [ 39] pre-trained to predict the next token in a document, using both publicly\",\"type\":\"Document\"}\n", + "\n", + "\n" + ] + } + ], + "source": [ + "from langchain_community.vectorstores.azure_cosmos_db_no_sql import CosmosDBQueryType\n", + "\n", + "query = \"What were the compute requirements for training GPT 4\"\n", + "pre_filter = {\n", + " \"conditions\": [\n", + " {\n", + " \"property\": \"text\",\n", + " \"operator\": \"$full_text_contains_any\",\n", + " \"value\": \"What were the compute requirements for training GPT 4\",\n", + " },\n", + " ],\n", + "}\n", + "results = vector_search.similarity_search_with_score(\n", + " query=query,\n", + " k=5,\n", + " query_type=CosmosDBQueryType.FULL_TEXT_SEARCH,\n", + " pre_filter=pre_filter,\n", + ")\n", + "\n", + "# Display results\n", + "for i in range(0, len(results)):\n", + " print(f\"Result {i+1}: \", results[i][0].json())\n", + " print(\"\\n\")" + ] + }, + { + "cell_type": "markdown", + "id": "2831548f1fb4eb90", + "metadata": {}, + "source": "## Full Text Search BM 25 Ranking" + }, + { + "cell_type": "code", + "execution_count": 47, + "id": "73dacc61ce8bf7a1", + "metadata": { + "ExecuteTime": { + "end_time": "2024-12-13T21:47:25.253222Z", + "start_time": "2024-12-13T21:47:23.680741Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Result 1: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":2,\"id\":\"f2746fd3-bbcb-4197-b2d5-ee7b355b6009\"},\"page_content\":\"the HumanEval dataset. A power law fit to the smaller models (excluding GPT-4) is shown as the dotted\\nline; this fit accurately predicts GPT-4’s performance. The x-axis is training compute normalized so that\\nGPT-4 is 1.\\n3\",\"type\":\"Document\"}\n", + "\n", + "\n", + "Result 2: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":1,\"id\":\"20153a6c-7c2c-4328-9b0e-e3502d7ac4dd\"},\"page_content\":\"safety considerations above against the scientific value of further transparency.\\n3 Predictable Scaling\\nA large focus of the GPT-4 project was building a deep learning stack that scales predictably. The\\nprimary reason is that for very large training runs like GPT-4, it is not feasible to do extensive\\nmodel-specific tuning. To address this, we developed infrastructure and optimization methods that\\nhave very predictable behavior across multiple scales. These improvements allowed us to reliably\\npredict some aspects of the performance of GPT-4 from smaller models trained using 1,000×–\\n10,000×less compute.\\n3.1 Loss Prediction\\nThe final loss of properly-trained large language models is thought to be well approximated by power\\nlaws in the amount of compute used to train the model [41, 42, 2, 14, 15].\\nTo verify the scalability of our optimization infrastructure, we predicted GPT-4’s final loss on our\",\"type\":\"Document\"}\n", + "\n", + "\n", + "Result 3: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":2,\"id\":\"6d88f369-4147-4530-9bfb-0ed008413211\"},\"page_content\":\"Observed\\nPrediction\\ngpt-4\\n100p 10n 1µ 100µ 0.01 1\\nCompute1.02.03.04.05.06.0Bits per wordOpenAI codebase next word predictionFigure 1. Performance of GPT-4 and smaller models. The metric is final loss on a dataset derived\\nfrom our internal codebase. This is a convenient, large dataset of code tokens which is not contained in\\nthe training set. We chose to look at loss because it tends to be less noisy than other measures across\\ndifferent amounts of training compute. A power law fit to the smaller models (excluding GPT-4) is\\nshown as the dotted line; this fit accurately predicts GPT-4’s final loss. The x-axis is training compute\\nnormalized so that GPT-4 is 1.\\nObserved\\nPrediction\\ngpt-4\\n1µ 10µ 100µ 0.001 0.01 0.1 1\\nCompute012345– Mean Log Pass RateCapability prediction on 23 coding problems\\nFigure 2. Performance of GPT-4 and smaller models. The metric is mean log pass rate on a subset of\\nthe HumanEval dataset. A power law fit to the smaller models (excluding GPT-4) is shown as the dotted\",\"type\":\"Document\"}\n", + "\n", + "\n", + "Result 4: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":1,\"id\":\"90e4bafe-55bb-406b-afba-a0143c810842\"},\"page_content\":\"which measures the ability to synthesize Python functions of varying complexity. We successfully\\npredicted the pass rate on a subset of the HumanEval dataset by extrapolating from models trained\\nwith at most 1,000×less compute (Figure 2).\\nFor an individual problem in HumanEval, performance may occasionally worsen with scale. Despite\\nthese challenges, we find an approximate power law relationship −EP[log(pass _rate(C))] = α∗C−k\\n2In addition to the accompanying system card, OpenAI will soon publish additional thoughts on the social\\nand economic implications of AI systems, including the need for effective regulation.\\n2\",\"type\":\"Document\"}\n", + "\n", + "\n", + "Result 5: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":71,\"id\":\"10ff5a7f-6638-4446-85b2-6e4314eca938\"},\"page_content\":\"Unsupervised Multitask Learners,” 2019.\\n[23]G. C. Bowker and S. L. Star, Sorting Things Out . MIT Press, Aug. 2000.\\n[24]L. Weidinger, J. Uesato, M. Rauh, C. Griffin, P.-S. Huang, J. Mellor, A. Glaese, M. Cheng,\\nB. Balle, A. Kasirzadeh, C. Biles, S. Brown, Z. Kenton, W. Hawkins, T. Stepleton, A. Birhane,\\nL. A. Hendricks, L. Rimell, W. Isaac, J. Haas, S. Legassick, G. Irving, and I. Gabriel, “Taxonomy\\nof Risks posed by Language Models,” in 2022 ACM Conference on Fairness, Accountability,\\nand Transparency , FAccT ’22, (New York, NY, USA), pp. 214–229, Association for Computing\\nMachinery, June 2022.\\n72\",\"type\":\"Document\"}\n", + "\n", + "\n" + ] + } + ], + "source": [ + "query = \"What were the compute requirements for training GPT 4\"\n", + "\n", + "results = vector_search.similarity_search_with_score(\n", + " query=query,\n", + " k=5,\n", + " query_type=CosmosDBQueryType.FULL_TEXT_RANK,\n", + ")\n", + "\n", + "# Display results\n", + "for i in range(0, len(results)):\n", + " print(f\"Result {i+1}: \", results[i][0].json())\n", + " print(\"\\n\")" + ] + }, + { + "cell_type": "markdown", + "id": "920e4eb20a141031", + "metadata": {}, + "source": "## Hybrid Search" + }, + { + "cell_type": "code", + "execution_count": 49, + "id": "5e1f7c17b02579b7", + "metadata": { + "ExecuteTime": { + "end_time": "2024-12-13T21:48:41.281808Z", + "start_time": "2024-12-13T21:48:32.232591Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Result 1: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":97,\"id\":\"36dfcd6c-d3cf-4e34-a5d6-cc4d63013cba\"},\"page_content\":\"Figure 11: Results on IF evaluations across GPT3.5, GPT3.5-Turbo, GPT-4-launch\\n98\",\"type\":\"Document\"}\n", + "Score 1: 0.8173275975778744\n", + "\n", + "\n", + "Result 2: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":7,\"id\":\"3d6e4715-4a38-40b1-89f1-e768bad5f9c8\"},\"page_content\":\"Preliminary results on a narrow set of academic vision benchmarks can be found in the GPT-4 blog\\npost [ 65]. We plan to release more information about GPT-4’s visual capabilities in follow-up work.\\n8\",\"type\":\"Document\"}\n", + "Score 2: 0.8176419674549597\n", + "\n", + "\n", + "Result 3: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":2,\"id\":\"f2746fd3-bbcb-4197-b2d5-ee7b355b6009\"},\"page_content\":\"the HumanEval dataset. A power law fit to the smaller models (excluding GPT-4) is shown as the dotted\\nline; this fit accurately predicts GPT-4’s performance. The x-axis is training compute normalized so that\\nGPT-4 is 1.\\n3\",\"type\":\"Document\"}\n", + "Score 3: 0.8053881702559759\n", + "\n", + "\n", + "Result 4: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":0,\"id\":\"9d59c3ed-deac-48cb-9382-a8ab079334e5\"},\"page_content\":\"performance based on models trained with no more than 1/1,000th the compute of\\nGPT-4.\\n1 Introduction\\nThis technical report presents GPT-4, a large multimodal model capable of processing image and\\ntext inputs and producing text outputs. Such models are an important area of study as they have the\\npotential to be used in a wide range of applications, such as dialogue systems, text summarization,\\nand machine translation. As such, they have been the subject of substantial interest and progress in\\nrecent years [1–34].\\nOne of the main goals of developing such models is to improve their ability to understand and generate\\nnatural language text, particularly in more complex and nuanced scenarios. To test its capabilities\\nin such scenarios, GPT-4 was evaluated on a variety of exams originally designed for humans. In\\nthese evaluations it performs quite well and often outscores the vast majority of human test takers.\",\"type\":\"Document\"}\n", + "Score 4: 0.8394796122122777\n", + "\n", + "\n", + "Result 5: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":1,\"id\":\"20153a6c-7c2c-4328-9b0e-e3502d7ac4dd\"},\"page_content\":\"safety considerations above against the scientific value of further transparency.\\n3 Predictable Scaling\\nA large focus of the GPT-4 project was building a deep learning stack that scales predictably. The\\nprimary reason is that for very large training runs like GPT-4, it is not feasible to do extensive\\nmodel-specific tuning. To address this, we developed infrastructure and optimization methods that\\nhave very predictable behavior across multiple scales. These improvements allowed us to reliably\\npredict some aspects of the performance of GPT-4 from smaller models trained using 1,000×–\\n10,000×less compute.\\n3.1 Loss Prediction\\nThe final loss of properly-trained large language models is thought to be well approximated by power\\nlaws in the amount of compute used to train the model [41, 42, 2, 14, 15].\\nTo verify the scalability of our optimization infrastructure, we predicted GPT-4’s final loss on our\",\"type\":\"Document\"}\n", + "Score 5: 0.8213247840132897\n", + "\n", + "\n" + ] + } + ], + "source": [ + "query = \"What were the compute requirements for training GPT 4\"\n", + "\n", + "results = vector_search.similarity_search_with_score(\n", + " query=query,\n", + " k=5,\n", + " query_type=CosmosDBQueryType.HYBRID,\n", + ")\n", + "\n", + "# Display results\n", + "for i in range(0, len(results)):\n", + " print(f\"Result {i+1}: \", results[i][0].json())\n", + " print(f\"Score {i+1}: \", results[i][1])\n", + " print(\"\\n\")" + ] + }, + { + "cell_type": "markdown", + "id": "1178119f59653565", + "metadata": {}, + "source": "## Hybrid Search with filtering" + }, + { + "cell_type": "code", + "execution_count": 51, + "id": "a644e56095c897fd", "metadata": { "ExecuteTime": { - "end_time": "2024-05-25T01:52:54.152486Z", - "start_time": "2024-05-25T01:52:54.140651Z" + "end_time": "2024-12-13T21:56:00.251680Z", + "start_time": "2024-12-13T21:55:55.253274Z" } }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Result 1: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":97,\"id\":\"36dfcd6c-d3cf-4e34-a5d6-cc4d63013cba\"},\"page_content\":\"Figure 11: Results on IF evaluations across GPT3.5, GPT3.5-Turbo, GPT-4-launch\\n98\",\"type\":\"Document\"}\n", + "Score 1: 0.8173275975778744\n", + "\n", + "\n", + "Result 2: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":7,\"id\":\"3d6e4715-4a38-40b1-89f1-e768bad5f9c8\"},\"page_content\":\"Preliminary results on a narrow set of academic vision benchmarks can be found in the GPT-4 blog\\npost [ 65]. We plan to release more information about GPT-4’s visual capabilities in follow-up work.\\n8\",\"type\":\"Document\"}\n", + "Score 2: 0.8176419674549597\n", + "\n", + "\n", + "Result 3: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":2,\"id\":\"f2746fd3-bbcb-4197-b2d5-ee7b355b6009\"},\"page_content\":\"the HumanEval dataset. A power law fit to the smaller models (excluding GPT-4) is shown as the dotted\\nline; this fit accurately predicts GPT-4’s performance. The x-axis is training compute normalized so that\\nGPT-4 is 1.\\n3\",\"type\":\"Document\"}\n", + "Score 3: 0.8053881702559759\n", + "\n", + "\n", + "Result 4: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":0,\"id\":\"9d59c3ed-deac-48cb-9382-a8ab079334e5\"},\"page_content\":\"performance based on models trained with no more than 1/1,000th the compute of\\nGPT-4.\\n1 Introduction\\nThis technical report presents GPT-4, a large multimodal model capable of processing image and\\ntext inputs and producing text outputs. Such models are an important area of study as they have the\\npotential to be used in a wide range of applications, such as dialogue systems, text summarization,\\nand machine translation. As such, they have been the subject of substantial interest and progress in\\nrecent years [1–34].\\nOne of the main goals of developing such models is to improve their ability to understand and generate\\nnatural language text, particularly in more complex and nuanced scenarios. To test its capabilities\\nin such scenarios, GPT-4 was evaluated on a variety of exams originally designed for humans. In\\nthese evaluations it performs quite well and often outscores the vast majority of human test takers.\",\"type\":\"Document\"}\n", + "Score 4: 0.8394796122122777\n", + "\n", + "\n", + "Result 5: {\"id\":null,\"metadata\":{\"source\":\"https://arxiv.org/pdf/2303.08774.pdf\",\"page\":1,\"id\":\"20153a6c-7c2c-4328-9b0e-e3502d7ac4dd\"},\"page_content\":\"safety considerations above against the scientific value of further transparency.\\n3 Predictable Scaling\\nA large focus of the GPT-4 project was building a deep learning stack that scales predictably. The\\nprimary reason is that for very large training runs like GPT-4, it is not feasible to do extensive\\nmodel-specific tuning. To address this, we developed infrastructure and optimization methods that\\nhave very predictable behavior across multiple scales. These improvements allowed us to reliably\\npredict some aspects of the performance of GPT-4 from smaller models trained using 1,000×–\\n10,000×less compute.\\n3.1 Loss Prediction\\nThe final loss of properly-trained large language models is thought to be well approximated by power\\nlaws in the amount of compute used to train the model [41, 42, 2, 14, 15].\\nTo verify the scalability of our optimization infrastructure, we predicted GPT-4’s final loss on our\",\"type\":\"Document\"}\n", + "Score 5: 0.8213247840132897\n", + "\n", + "\n" + ] + } + ], + "source": [ + "query = \"What were the compute requirements for training GPT 4\"\n", + "\n", + "pre_filter = {\n", + " \"conditions\": [\n", + " {\n", + " \"property\": \"text\",\n", + " \"operator\": \"$full_text_contains_any\",\n", + " \"value\": \"compute requirements\",\n", + " },\n", + " {\"property\": \"metadata.page\", \"operator\": \"$eq\", \"value\": 0},\n", + " ],\n", + " \"logical_operator\": \"$and\",\n", + "}\n", + "\n", + "results = vector_search.similarity_search_with_score(\n", + " query=query,\n", + " k=5,\n", + " query_type=CosmosDBQueryType.HYBRID,\n", + ")\n", + "\n", + "# Display results\n", + "for i in range(0, len(results)):\n", + " print(f\"Result {i+1}: \", results[i][0].json())\n", + " print(f\"Score {i+1}: \", results[i][1])\n", + " print(\"\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "de996623625ab622", + "metadata": {}, "outputs": [], "source": [] } diff --git a/docs/docs/integrations/vectorstores/falkordbvector.ipynb b/docs/docs/integrations/vectorstores/falkordbvector.ipynb new file mode 100644 index 0000000000000..ecc0aa68bf008 --- /dev/null +++ b/docs/docs/integrations/vectorstores/falkordbvector.ipynb @@ -0,0 +1,437 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# FalkorDBVectorStore\n", + "FalkorDB is an open-source graph database with integrated support for vector similarity search\n", + "\n", + "it supports:\n", + "- approximate nearest neighbor search\n", + "- Euclidean similarity & Cosine Similarity\n", + "- Hybrid search combining vector and keyword searches\n", + "\n", + "This notebook shows how to use the FalkorDB vector index (`FalkorDB`)\n", + "\n", + "See the installation instruction\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: falkordb in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (1.0.10)Note: you may need to restart the kernel to use updated packages.\n", + "\n", + "Requirement already satisfied: redis<6.0.0,>=5.0.1 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from falkordb) (5.2.0)\n", + "Requirement already satisfied: async-timeout>=4.0.3 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from redis<6.0.0,>=5.0.1->falkordb) (4.0.3)\n", + "Requirement already satisfied: tiktoken in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (0.8.0)\n", + "Requirement already satisfied: regex>=2022.1.18 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from tiktoken) (2024.11.6)\n", + "Requirement already satisfied: requests>=2.26.0 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from tiktoken) (2.32.3)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from requests>=2.26.0->tiktoken) (3.4.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from requests>=2.26.0->tiktoken) (3.10)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from requests>=2.26.0->tiktoken) (1.26.20)\n", + "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from requests>=2.26.0->tiktoken) (2024.8.30)\n", + "Note: you may need to restart the kernel to use updated packages.\n", + "Requirement already satisfied: langchain in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (0.3.9)Note: you may need to restart the kernel to use updated packages.\n", + "\n", + "Requirement already satisfied: langchain_huggingface in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (0.1.2)\n", + "Requirement already satisfied: PyYAML>=5.3 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from langchain) (6.0.2)\n", + "Requirement already satisfied: SQLAlchemy<3,>=1.4 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from langchain) (2.0.36)\n", + "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from langchain) (3.11.8)\n", + "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from langchain) (4.0.3)\n", + "Requirement already satisfied: langchain-core<0.4.0,>=0.3.21 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from langchain) (0.3.21)\n", + "Requirement already satisfied: langchain-text-splitters<0.4.0,>=0.3.0 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from langchain) (0.3.2)\n", + "Requirement already satisfied: langsmith<0.2.0,>=0.1.17 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from langchain) (0.1.147)\n", + "Requirement already satisfied: numpy<2,>=1.22.4 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from langchain) (1.26.4)\n", + "Requirement already satisfied: pydantic<3.0.0,>=2.7.4 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from langchain) (2.9.2)\n", + "Requirement already satisfied: requests<3,>=2 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from langchain) (2.32.3)\n", + "Requirement already satisfied: tenacity!=8.4.0,<10,>=8.1.0 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from langchain) (9.0.0)\n", + "Requirement already satisfied: huggingface-hub>=0.23.0 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from langchain_huggingface) (0.26.3)\n", + "Requirement already satisfied: sentence-transformers>=2.6.0 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from langchain_huggingface) (3.3.1)\n", + "Requirement already satisfied: tokenizers>=0.19.1 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from langchain_huggingface) (0.20.3)\n", + "Requirement already satisfied: transformers>=4.39.0 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from langchain_huggingface) (4.46.3)\n", + "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (2.4.4)\n", + "Requirement already satisfied: aiosignal>=1.1.2 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.3.1)\n", + "Requirement already satisfied: attrs>=17.3.0 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (24.2.0)\n", + "Requirement already satisfied: frozenlist>=1.1.1 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.5.0)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (6.1.0)\n", + "Requirement already satisfied: propcache>=0.2.0 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (0.2.0)\n", + "Requirement already satisfied: yarl<2.0,>=1.17.0 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.18.0)\n", + "Requirement already satisfied: filelock in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from huggingface-hub>=0.23.0->langchain_huggingface) (3.16.1)\n", + "Requirement already satisfied: fsspec>=2023.5.0 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from huggingface-hub>=0.23.0->langchain_huggingface) (2024.10.0)\n", + "Requirement already satisfied: packaging>=20.9 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from huggingface-hub>=0.23.0->langchain_huggingface) (24.2)\n", + "Requirement already satisfied: tqdm>=4.42.1 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from huggingface-hub>=0.23.0->langchain_huggingface) (4.67.1)\n", + "Requirement already satisfied: typing-extensions>=3.7.4.3 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from huggingface-hub>=0.23.0->langchain_huggingface) (4.12.2)\n", + "Requirement already satisfied: jsonpatch<2.0,>=1.33 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from langchain-core<0.4.0,>=0.3.21->langchain) (1.33)\n", + "Requirement already satisfied: httpx<1,>=0.23.0 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from langsmith<0.2.0,>=0.1.17->langchain) (0.27.2)\n", + "Requirement already satisfied: orjson<4.0.0,>=3.9.14 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from langsmith<0.2.0,>=0.1.17->langchain) (3.10.12)\n", + "Requirement already satisfied: requests-toolbelt<2.0.0,>=1.0.0 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from langsmith<0.2.0,>=0.1.17->langchain) (1.0.0)\n", + "Requirement already satisfied: annotated-types>=0.6.0 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from pydantic<3.0.0,>=2.7.4->langchain) (0.7.0)\n", + "Requirement already satisfied: pydantic-core==2.23.4 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from pydantic<3.0.0,>=2.7.4->langchain) (2.23.4)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from requests<3,>=2->langchain) (3.4.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from requests<3,>=2->langchain) (3.10)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from requests<3,>=2->langchain) (1.26.20)\n", + "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from requests<3,>=2->langchain) (2024.8.30)\n", + "Requirement already satisfied: torch>=1.11.0 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from sentence-transformers>=2.6.0->langchain_huggingface) (2.5.1)\n", + "Requirement already satisfied: scikit-learn in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from sentence-transformers>=2.6.0->langchain_huggingface) (1.5.2)\n", + "Requirement already satisfied: scipy in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from sentence-transformers>=2.6.0->langchain_huggingface) (1.13.1)\n", + "Requirement already satisfied: Pillow in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from sentence-transformers>=2.6.0->langchain_huggingface) (11.0.0)\n", + "Requirement already satisfied: greenlet!=0.4.17 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from SQLAlchemy<3,>=1.4->langchain) (3.1.1)\n", + "Requirement already satisfied: regex!=2019.12.17 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from transformers>=4.39.0->langchain_huggingface) (2024.11.6)\n", + "Requirement already satisfied: safetensors>=0.4.1 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from transformers>=4.39.0->langchain_huggingface) (0.4.5)\n", + "Requirement already satisfied: anyio in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from httpx<1,>=0.23.0->langsmith<0.2.0,>=0.1.17->langchain) (4.6.2.post1)\n", + "Requirement already satisfied: httpcore==1.* in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from httpx<1,>=0.23.0->langsmith<0.2.0,>=0.1.17->langchain) (1.0.7)\n", + "Requirement already satisfied: sniffio in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from httpx<1,>=0.23.0->langsmith<0.2.0,>=0.1.17->langchain) (1.3.1)\n", + "Requirement already satisfied: h11<0.15,>=0.13 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from httpcore==1.*->httpx<1,>=0.23.0->langsmith<0.2.0,>=0.1.17->langchain) (0.14.0)\n", + "Requirement already satisfied: jsonpointer>=1.9 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from jsonpatch<2.0,>=1.33->langchain-core<0.4.0,>=0.3.21->langchain) (3.0.0)\n", + "Requirement already satisfied: networkx in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from torch>=1.11.0->sentence-transformers>=2.6.0->langchain_huggingface) (3.2.1)\n", + "Requirement already satisfied: jinja2 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from torch>=1.11.0->sentence-transformers>=2.6.0->langchain_huggingface) (3.1.4)\n", + "Requirement already satisfied: sympy==1.13.1 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from torch>=1.11.0->sentence-transformers>=2.6.0->langchain_huggingface) (1.13.1)\n", + "Requirement already satisfied: mpmath<1.4,>=1.1.0 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from sympy==1.13.1->torch>=1.11.0->sentence-transformers>=2.6.0->langchain_huggingface) (1.3.0)\n", + "Requirement already satisfied: colorama in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from tqdm>=4.42.1->huggingface-hub>=0.23.0->langchain_huggingface) (0.4.6)\n", + "Requirement already satisfied: joblib>=1.2.0 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from scikit-learn->sentence-transformers>=2.6.0->langchain_huggingface) (1.4.2)\n", + "Requirement already satisfied: threadpoolctl>=3.1.0 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from scikit-learn->sentence-transformers>=2.6.0->langchain_huggingface) (3.5.0)\n", + "Requirement already satisfied: exceptiongroup>=1.0.2 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from anyio->httpx<1,>=0.23.0->langsmith<0.2.0,>=0.1.17->langchain) (1.2.2)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\dell\\desktop\\langchain\\.venv\\lib\\site-packages (from jinja2->torch>=1.11.0->sentence-transformers>=2.6.0->langchain_huggingface) (3.0.2)\n" + ] + } + ], + "source": [ + "# Pip install necessary package\n", + "%pip install --upgrade falkordb\n", + "%pip install --upgrade tiktoken\n", + "%pip install --upgrade langchain langchain_huggingface" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Credentials\n", + "We want to use `HuggingFace` so we have to get the HuggingFace API Key" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import getpass\n", + "import os\n", + "\n", + "if \"HUGGINGFACE_API_KEY\" not in os.environ:\n", + " os.environ[\"HUGGINGFACE_API_KEY\"] = getpass.getpass(\"HuggingFace API Key:\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you want to get automated tracing of your model calls you can also set your LangSmith API key by uncommenting below:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# os.environ[\"LANGSMITH_API_KEY\"] = getpass.getpass(\"Enter your LangSmith API key: \")\n", + "# os.environ[\"LANGSMITH_TRACING\"] = \"true\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Initialization" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "from langchain_community.vectorstores.falkordb_vector import FalkorDBVector\n", + "from langchain_core.documents import Document\n", + "from langchain_huggingface import HuggingFaceEmbeddings" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can use FalkorDBVector locally with docker. See installation instruction" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "host = \"localhost\"\n", + "port = 6379" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Or you can use FalkorDBVector with FalkorDB Cloud" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "# E.g\n", + "# host = \"r-6jissuruar.instance-zwb082gpf.hc-v8noonp0c.europe-west1.gcp.f2e0a955bb84.cloud\"\n", + "# port = 62471\n", + "# username = \"falkordb\" # SET ON FALKORDB CLOUD\n", + "# password = \"password\" # SET ON FALKORDB CLOUD" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "vector_store = FalkorDBVector(host=host, port=port, embedding=HuggingFaceEmbeddings())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Manage vector store" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Add items to vector store" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['1', '2', '3']" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from langchain_core.documents import Document\n", + "\n", + "document_1 = Document(page_content=\"foo\", metadata={\"source\": \"https://example.com\"})\n", + "\n", + "document_2 = Document(page_content=\"bar\", metadata={\"source\": \"https://example.com\"})\n", + "\n", + "document_3 = Document(page_content=\"baz\", metadata={\"source\": \"https://example.com\"})\n", + "\n", + "documents = [document_1, document_2, document_3]\n", + "\n", + "vector_store.add_documents(documents=documents, ids=[\"1\", \"2\", \"3\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Update items in vector store" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "updated_document = Document(\n", + " page_content=\"qux\", metadata={\"source\": \"https://another-example.com\"}\n", + ")\n", + "\n", + "vector_store.update_documents(document_id=\"1\", document=updated_document)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Delete items from vector store" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "vector_store.delete(ids=[\"3\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Query vector store\n", + "\n", + "Once your vector store has been created and the relevant documents have been added you will most likely wish to query it during the running of your chain or agent." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Query directly\n", + "\n", + "Performing a simple similarity search can be done as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "* qux [{'text': 'qux', 'id': '1', 'source': 'https://another-example.com'}]\n" + ] + } + ], + "source": [ + "results = vector_store.similarity_search(\n", + " query=\"thud\", k=1, filter={\"source\": \"https://another-example.com\"}\n", + ")\n", + "for doc in results:\n", + " print(f\"* {doc.page_content} [{doc.metadata}]\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you want to execute a similarity search and receive the corresponding scores you can run:" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "* [SIM=0.000001] bar [{'text': 'bar', 'id': '2', 'source': 'https://example.com'}]\n" + ] + } + ], + "source": [ + "results = vector_store.similarity_search_with_score(query=\"bar\")\n", + "for doc, score in results:\n", + " print(f\"* [SIM={score:3f}] {doc.page_content} [{doc.metadata}]\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Query by turning into retriever\n", + "You can also transform the vector store into a retriever for easier usage in your chains." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Document(metadata={'text': 'qux', 'id': '1', 'source': 'https://another-example.com'}, page_content='qux')]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "retriever = vector_store.as_retriever(search_type=\"mmr\", search_kwargs={\"k\": 1})\n", + "retriever.invoke(\"thud\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Usage for retrieval-augmented generation\n", + "For guides on how to use this vector store for retrieval-augmented generation (RAG), see the following sections:\n", + "- Tutorials: working with external knowledge\n", + "- How-to: Question and answer with RAG\n", + "- Retrieval conceptual docs\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## API reference\n", + "For detailed documentation of all `FalkorDBVector` features and configurations head to the API reference: https://python.langchain.com/api_reference/community/vectorstores/langchain_community.vectorstores.falkordb_vector.FalkorDBVector.html" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/docs/integrations/vectorstores/oceanbase.ipynb b/docs/docs/integrations/vectorstores/oceanbase.ipynb new file mode 100644 index 0000000000000..3ede92cb85355 --- /dev/null +++ b/docs/docs/integrations/vectorstores/oceanbase.ipynb @@ -0,0 +1,383 @@ +{ + "cells": [ + { + "cell_type": "raw", + "id": "1957f5cb", + "metadata": {}, + "source": [ + "---\n", + "sidebar_label: Oceanbase\n", + "---" + ] + }, + { + "cell_type": "markdown", + "id": "ef1f0986", + "metadata": {}, + "source": [ + "# OceanbaseVectorStore\n", + "\n", + "This notebook covers how to get started with the Oceanbase vector store." + ] + }, + { + "cell_type": "markdown", + "id": "36fdc060", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "To access Oceanbase vector stores you'll need to deploy a standalone OceanBase server:" + ] + }, + { + "cell_type": "raw", + "id": "a7b92118", + "metadata": {}, + "source": [ + "%docker run --name=ob433 -e MODE=mini -e OB_SERVER_IP=127.0.0.1 -p 2881:2881 -d quay.io/oceanbase/oceanbase-ce:4.3.3.1-101000012024102216" + ] + }, + { + "cell_type": "markdown", + "id": "5b687bdc", + "metadata": {}, + "source": [ + "And install the `langchain-oceanbase` integration package." + ] + }, + { + "cell_type": "raw", + "id": "64e28aa6", + "metadata": { + "vscode": { + "languageId": "raw" + } + }, + "source": [ + "%pip install -qU \"langchain-oceanbase\"" + ] + }, + { + "cell_type": "markdown", + "id": "ea850342", + "metadata": {}, + "source": [ + "Check the connection to OceanBase and set the memory usage ratio for vector data:" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "066bbc79", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from pyobvector import ObVecClient\n", + "\n", + "tmp_client = ObVecClient()\n", + "tmp_client.perform_raw_text_sql(\"ALTER SYSTEM ob_vector_memory_limit_percentage = 30\")" + ] + }, + { + "cell_type": "markdown", + "id": "93df377e", + "metadata": {}, + "source": [ + "## Initialization\n", + "\n", + "Configure the API key of the embedded model. Here we use `DashScopeEmbeddings` as an example. When deploying `Oceanbase` with a Docker image as described above, simply follow the script below to set the `host`, `port`, `user`, `password`, and `database name`. For other deployment methods, set these parameters according to the actual situation." + ] + }, + { + "cell_type": "raw", + "id": "ff29e3b7", + "metadata": {}, + "source": [ + "%pip install dashscope" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "dc37144c-208d-4ab3-9f3a-0407a69fe052", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "from langchain_community.embeddings import DashScopeEmbeddings\n", + "from langchain_oceanbase.vectorstores import OceanbaseVectorStore\n", + "\n", + "DASHSCOPE_API = os.environ.get(\"DASHSCOPE_API_KEY\", \"\")\n", + "connection_args = {\n", + " \"host\": \"127.0.0.1\",\n", + " \"port\": \"2881\",\n", + " \"user\": \"root@test\",\n", + " \"password\": \"\",\n", + " \"db_name\": \"test\",\n", + "}\n", + "\n", + "embeddings = DashScopeEmbeddings(\n", + " model=\"text-embedding-v1\", dashscope_api_key=DASHSCOPE_API\n", + ")\n", + "\n", + "vector_store = OceanbaseVectorStore(\n", + " embedding_function=embeddings,\n", + " table_name=\"langchain_vector\",\n", + " connection_args=connection_args,\n", + " vidx_metric_type=\"l2\",\n", + " drop_old=True,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "ac6071d4", + "metadata": {}, + "source": [ + "## Manage vector store\n", + "\n", + "### Add items to vector store\n", + "\n", + "- TODO: Edit and then run code cell to generate output" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "17f5efc0", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['1', '2', '3']" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from langchain_core.documents import Document\n", + "\n", + "document_1 = Document(page_content=\"foo\", metadata={\"source\": \"https://foo.com\"})\n", + "document_2 = Document(page_content=\"bar\", metadata={\"source\": \"https://bar.com\"})\n", + "document_3 = Document(page_content=\"baz\", metadata={\"source\": \"https://baz.com\"})\n", + "\n", + "documents = [document_1, document_2, document_3]\n", + "\n", + "vector_store.add_documents(documents=documents, ids=[\"1\", \"2\", \"3\"])" + ] + }, + { + "cell_type": "markdown", + "id": "c738c3e0", + "metadata": {}, + "source": [ + "### Update items in vector store" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "f0aa8b71", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['1']" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "updated_document = Document(\n", + " page_content=\"qux\", metadata={\"source\": \"https://another-example.com\"}\n", + ")\n", + "\n", + "vector_store.add_documents(documents=[updated_document], ids=[\"1\"])" + ] + }, + { + "cell_type": "markdown", + "id": "dcf1b905", + "metadata": {}, + "source": [ + "### Delete items from vector store" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "ef61e188", + "metadata": {}, + "outputs": [], + "source": [ + "vector_store.delete(ids=[\"3\"])" + ] + }, + { + "cell_type": "markdown", + "id": "c3620501", + "metadata": {}, + "source": [ + "## Query vector store\n", + "\n", + "Once your vector store has been created and the relevant documents have been added you will most likely wish to query it during the running of your chain or agent. \n", + "\n", + "### Query directly\n", + "\n", + "Performing a simple similarity search can be done as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "aa0a16fa", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "* bar [{'source': 'https://bar.com'}]\n" + ] + } + ], + "source": [ + "results = vector_store.similarity_search(\n", + " query=\"thud\", k=1, filter={\"source\": \"https://another-example.com\"}\n", + ")\n", + "for doc in results:\n", + " print(f\"* {doc.page_content} [{doc.metadata}]\")" + ] + }, + { + "cell_type": "markdown", + "id": "3ed9d733", + "metadata": {}, + "source": [ + "If you want to execute a similarity search and receive the corresponding scores you can run:" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "5efd2eaa", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "* [SIM=133.452299] bar [{'source': 'https://bar.com'}]\n" + ] + } + ], + "source": [ + "results = vector_store.similarity_search_with_score(\n", + " query=\"thud\", k=1, filter={\"source\": \"https://example.com\"}\n", + ")\n", + "for doc, score in results:\n", + " print(f\"* [SIM={score:3f}] {doc.page_content} [{doc.metadata}]\")" + ] + }, + { + "cell_type": "markdown", + "id": "0c235cdc", + "metadata": {}, + "source": [ + "### Query by turning into retriever\n", + "\n", + "You can also transform the vector store into a retriever for easier usage in your chains. " + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "f3460093", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Document(metadata={'source': 'https://bar.com'}, page_content='bar')]" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "retriever = vector_store.as_retriever(search_kwargs={\"k\": 1})\n", + "retriever.invoke(\"thud\")" + ] + }, + { + "cell_type": "markdown", + "id": "901c75dc", + "metadata": {}, + "source": [ + "## Usage for retrieval-augmented generation\n", + "\n", + "For guides on how to use this vector store for retrieval-augmented generation (RAG), see the following sections:\n", + "\n", + "- [Tutorials](/docs/tutorials/)\n", + "- [How-to: Question and answer with RAG](https://python.langchain.com/docs/how_to/#qa-with-rag)\n", + "- [Retrieval conceptual docs](https://python.langchain.com/docs/concepts/#retrieval)" + ] + }, + { + "cell_type": "markdown", + "id": "8a27244f", + "metadata": {}, + "source": [ + "## API reference\n", + "\n", + "For detailed documentation of all OceanbaseVectorStore features and configurations head to the API reference: https://python.langchain.com/docs/integrations/vectorstores/oceanbase" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docs/docs/integrations/vectorstores/tablestore.ipynb b/docs/docs/integrations/vectorstores/tablestore.ipynb new file mode 100644 index 0000000000000..0c82261d97f2d --- /dev/null +++ b/docs/docs/integrations/vectorstores/tablestore.ipynb @@ -0,0 +1,385 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "# TablestoreVectorStore\n", + "\n", + "> [Tablestore](https://www.aliyun.com/product/ots) is a fully managed NoSQL cloud database service that enables storage of a massive amount of structured\n", + "and semi-structured data.\n", + "\n", + "This notebook shows how to use functionality related to the `Tablestore` vector database.\n", + "\n", + "To use Tablestore, you must create an instance.\n", + "Here are the [creating instance instructions](https://help.aliyun.com/zh/tablestore/getting-started/manage-the-wide-column-model-in-the-tablestore-console)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": "## Setup" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install --upgrade --quiet langchain-community tablestore" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Initialization" + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "ExecuteTime": { + "end_time": "2024-08-20T11:10:04.469458Z", + "start_time": "2024-08-20T11:09:49.541150Z" + }, + "pycharm": { + "is_executing": true, + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "import getpass\n", + "import os\n", + "\n", + "os.environ[\"end_point\"] = getpass.getpass(\"Tablestore end_point:\")\n", + "os.environ[\"instance_name\"] = getpass.getpass(\"Tablestore instance_name:\")\n", + "os.environ[\"access_key_id\"] = getpass.getpass(\"Tablestore access_key_id:\")\n", + "os.environ[\"access_key_secret\"] = getpass.getpass(\"Tablestore access_key_secret:\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "Create vector store. " + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "ExecuteTime": { + "end_time": "2024-08-20T11:10:07.911086Z", + "start_time": "2024-08-20T11:10:07.351293Z" + } + }, + "outputs": [], + "source": [ + "import tablestore\n", + "from langchain_community.embeddings import FakeEmbeddings\n", + "from langchain_community.vectorstores import TablestoreVectorStore\n", + "from langchain_core.documents import Document\n", + "\n", + "test_embedding_dimension_size = 4\n", + "embeddings = FakeEmbeddings(size=test_embedding_dimension_size)\n", + "\n", + "store = TablestoreVectorStore(\n", + " embedding=embeddings,\n", + " endpoint=os.getenv(\"end_point\"),\n", + " instance_name=os.getenv(\"instance_name\"),\n", + " access_key_id=os.getenv(\"access_key_id\"),\n", + " access_key_secret=os.getenv(\"access_key_secret\"),\n", + " vector_dimension=test_embedding_dimension_size,\n", + " # metadata mapping is used to filter non-vector fields.\n", + " metadata_mappings=[\n", + " tablestore.FieldSchema(\n", + " \"type\", tablestore.FieldType.KEYWORD, index=True, enable_sort_and_agg=True\n", + " ),\n", + " tablestore.FieldSchema(\n", + " \"time\", tablestore.FieldType.LONG, index=True, enable_sort_and_agg=True\n", + " ),\n", + " ],\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Manage vector store" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "Create table and index." + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "ExecuteTime": { + "end_time": "2024-08-20T11:10:10.875422Z", + "start_time": "2024-08-20T11:10:10.566400Z" + } + }, + "outputs": [], + "source": [ + "store.create_table_if_not_exist()\n", + "store.create_search_index_if_not_exist()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "Add documents." + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "ExecuteTime": { + "end_time": "2024-08-20T11:10:14.974253Z", + "start_time": "2024-08-20T11:10:14.894190Z" + }, + "pycharm": { + "is_executing": true, + "name": "#%%\n" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['1', '2', '3', '4', '5']" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "store.add_documents(\n", + " [\n", + " Document(\n", + " id=\"1\", page_content=\"1 hello world\", metadata={\"type\": \"pc\", \"time\": 2000}\n", + " ),\n", + " Document(\n", + " id=\"2\", page_content=\"abc world\", metadata={\"type\": \"pc\", \"time\": 2009}\n", + " ),\n", + " Document(\n", + " id=\"3\", page_content=\"3 text world\", metadata={\"type\": \"sky\", \"time\": 2010}\n", + " ),\n", + " Document(\n", + " id=\"4\", page_content=\"hi world\", metadata={\"type\": \"sky\", \"time\": 2030}\n", + " ),\n", + " Document(\n", + " id=\"5\", page_content=\"hi world\", metadata={\"type\": \"sky\", \"time\": 2030}\n", + " ),\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": "Delete document." + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "ExecuteTime": { + "end_time": "2024-08-20T11:10:17.408739Z", + "start_time": "2024-08-20T11:10:17.269593Z" + }, + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "store.delete([\"3\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": "Get documents." + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "## Query vector store" + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "ExecuteTime": { + "end_time": "2024-08-20T11:10:19.379617Z", + "start_time": "2024-08-20T11:10:19.339970Z" + }, + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[Document(id='1', metadata={'embedding': '[1.3296732307905934, 0.0037521341868022385, 0.9821875819319514, 2.5644103644492393]', 'time': 2000, 'type': 'pc'}, page_content='1 hello world'),\n", + " None,\n", + " Document(id='5', metadata={'embedding': '[1.4558082172139821, -1.6441137122167426, -0.13113098640337423, -1.889685473174525]', 'time': 2030, 'type': 'sky'}, page_content='hi world')]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "store.get_by_ids([\"1\", \"3\", \"5\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "Similarity search." + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "ExecuteTime": { + "end_time": "2024-08-20T11:10:21.306717Z", + "start_time": "2024-08-20T11:10:21.284244Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[Document(id='1', metadata={'embedding': [1.3296732307905934, 0.0037521341868022385, 0.9821875819319514, 2.5644103644492393], 'time': 2000, 'type': 'pc'}, page_content='1 hello world'),\n", + " Document(id='4', metadata={'embedding': [-0.3310144199800685, 0.29250046478723635, -0.0646862290377582, -0.23664360156781225], 'time': 2030, 'type': 'sky'}, page_content='hi world')]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "store.similarity_search(query=\"hello world\", k=2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": "Similarity search with filters. " + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "ExecuteTime": { + "end_time": "2024-08-20T11:10:23.231425Z", + "start_time": "2024-08-20T11:10:23.213046Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[Document(id='5', metadata={'embedding': [1.4558082172139821, -1.6441137122167426, -0.13113098640337423, -1.889685473174525], 'time': 2030, 'type': 'sky'}, page_content='hi world'),\n", + " Document(id='4', metadata={'embedding': [-0.3310144199800685, 0.29250046478723635, -0.0646862290377582, -0.23664360156781225], 'time': 2030, 'type': 'sky'}, page_content='hi world')]" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "store.similarity_search(\n", + " query=\"hello world\",\n", + " k=10,\n", + " tablestore_filter_query=tablestore.BoolQuery(\n", + " must_queries=[tablestore.TermQuery(field_name=\"type\", column_value=\"sky\")],\n", + " should_queries=[tablestore.RangeQuery(field_name=\"time\", range_from=2020)],\n", + " must_not_queries=[tablestore.TermQuery(field_name=\"type\", column_value=\"pc\")],\n", + " ),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Usage for retrieval-augmented generation\n", + "\n", + "For guides on how to use this vector store for retrieval-augmented generation (RAG), see the following sections:\n", + "\n", + "- [Tutorials](/docs/tutorials/)\n", + "- [How-to: Question and answer with RAG](https://python.langchain.com/docs/how_to/#qa-with-rag)\n", + "- [Retrieval conceptual docs](https://python.langchain.com/docs/concepts/retrieval)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## API reference\n", + "\n", + "For detailed documentation of all `TablestoreVectorStore` features and configurations head to the API reference:\n", + " https://python.langchain.com/api_reference/community/vectorstores/langchain_community.vectorstores.tablestore.TablestoreVectorStore.html" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} diff --git a/docs/docs/introduction.mdx b/docs/docs/introduction.mdx index e0110afd973bb..185eeb4325bf8 100644 --- a/docs/docs/introduction.mdx +++ b/docs/docs/introduction.mdx @@ -8,9 +8,9 @@ sidebar_class_name: hidden **LangChain** is a framework for developing applications powered by large language models (LLMs). LangChain simplifies every stage of the LLM application lifecycle: -- **Development**: Build your applications using LangChain's open-source [building blocks](/docs/concepts/lcel), [components](/docs/concepts), and [third-party integrations](/docs/integrations/providers/). +- **Development**: Build your applications using LangChain's open-source [components](/docs/concepts) and [third-party integrations](/docs/integrations/providers/). Use [LangGraph](/docs/concepts/architecture/#langgraph) to build stateful agents with first-class streaming and human-in-the-loop support. -- **Productionization**: Use [LangSmith](https://docs.smith.langchain.com/) to inspect, monitor and evaluate your chains, so that you can continuously optimize and deploy with confidence. +- **Productionization**: Use [LangSmith](https://docs.smith.langchain.com/) to inspect, monitor and evaluate your applications, so that you can continuously optimize and deploy with confidence. - **Deployment**: Turn your LangGraph applications into production-ready APIs and Assistants with [LangGraph Platform](https://langchain-ai.github.io/langgraph/cloud/). import ThemedImage from '@theme/ThemedImage'; @@ -26,16 +26,18 @@ import useBaseUrl from '@docusaurus/useBaseUrl'; title="LangChain Framework Overview" /> -Concretely, the framework consists of the following open-source libraries: +LangChain implements a standard interface for large language models and related +technologies, such as embedding models and vector stores, and integrates with +hundreds of providers. See the [integrations](/docs/integrations/providers/) page for +more. -- **`langchain-core`**: Base abstractions and LangChain Expression Language. -- **Integration packages** (e.g. `langchain-openai`, `langchain-anthropic`, etc.): Important integrations have been split into lightweight packages that are co-maintained by the LangChain team and the integration developers. -- **`langchain`**: Chains, agents, and retrieval strategies that make up an application's cognitive architecture. -- **`langchain-community`**: Third-party integrations that are community maintained. -- **[LangGraph](https://langchain-ai.github.io/langgraph)**: Build robust and stateful multi-actor applications with LLMs by modeling steps as edges and nodes in a graph. Integrates smoothly with LangChain, but can be used without it. To learn more about LangGraph, check out our first LangChain Academy course, *Introduction to LangGraph*, available [here](https://academy.langchain.com/courses/intro-to-langgraph). -- **[LangGraph Platform](https://langchain-ai.github.io/langgraph/concepts/#langgraph-platform)**: Deploy LLM applications built with LangGraph to production. -- **[LangSmith](https://docs.smith.langchain.com)**: A developer platform that lets you debug, test, evaluate, and monitor LLM applications. +import ChatModelTabs from "@theme/ChatModelTabs"; + + +```python +model.invoke("Hello, world!") +``` :::note @@ -43,7 +45,20 @@ These docs focus on the Python LangChain library. [Head here](https://js.langcha ::: -## [Tutorials](/docs/tutorials) +## Architecture + +The LangChain framework consists of multiple open-source libraries. Read more in the +[Architecture](/docs/concepts/architecture/) page. + +- **`langchain-core`**: Base abstractions for chat models and other components. +- **Integration packages** (e.g. `langchain-openai`, `langchain-anthropic`, etc.): Important integrations have been split into lightweight packages that are co-maintained by the LangChain team and the integration developers. +- **`langchain`**: Chains, agents, and retrieval strategies that make up an application's cognitive architecture. +- **`langchain-community`**: Third-party integrations that are community maintained. +- **`langgraph`**: Orchestration framework for combining LangChain components into production-ready applications with persistence, streaming, and other key features. See [LangGraph documentation](https://langchain-ai.github.io/langgraph/). + +## Guides + +### [Tutorials](/docs/tutorials) If you're looking to build something specific or are more of a hands-on learner, check out our [tutorials section](/docs/tutorials). This is the best place to get started. @@ -58,7 +73,7 @@ These are the best ones to get started with: Explore the full list of LangChain tutorials [here](/docs/tutorials), and check out other [LangGraph tutorials here](https://langchain-ai.github.io/langgraph/tutorials/). To learn more about LangGraph, check out our first LangChain Academy course, *Introduction to LangGraph*, available [here](https://academy.langchain.com/courses/intro-to-langgraph). -## [How-to guides](/docs/how_to) +### [How-to guides](/docs/how_to) [Here](/docs/how_to) you’ll find short answers to “How do I….?” types of questions. These how-to guides don’t cover topics in depth – you’ll find that material in the [Tutorials](/docs/tutorials) and the [API Reference](https://python.langchain.com/api_reference/). @@ -67,20 +82,20 @@ However, these guides will help you quickly accomplish common tasks using [chat Check out [LangGraph-specific how-tos here](https://langchain-ai.github.io/langgraph/how-tos/). -## [Conceptual guide](/docs/concepts) +### [Conceptual guide](/docs/concepts) Introductions to all the key parts of LangChain you’ll need to know! [Here](/docs/concepts) you'll find high level explanations of all LangChain concepts. For a deeper dive into LangGraph concepts, check out [this page](https://langchain-ai.github.io/langgraph/concepts/). -## [Integrations](integrations/providers/index.mdx) +### [Integrations](integrations/providers/index.mdx) LangChain is part of a rich ecosystem of tools that integrate with our framework and build on top of it. If you're looking to get up and running quickly with [chat models](/docs/integrations/chat/), [vector stores](/docs/integrations/vectorstores/), or other LangChain components from a specific provider, check out our growing list of [integrations](/docs/integrations/providers/). -## [API reference](https://python.langchain.com/api_reference/) +### [API reference](https://python.langchain.com/api_reference/) Head to the reference section for full documentation of all classes and methods in the LangChain Python packages. ## Ecosystem diff --git a/docs/docs/tutorials/qa_chat_history.ipynb b/docs/docs/tutorials/qa_chat_history.ipynb index 2e0b09b2d1713..3acb3368a7bc3 100644 --- a/docs/docs/tutorials/qa_chat_history.ipynb +++ b/docs/docs/tutorials/qa_chat_history.ipynb @@ -51,8 +51,6 @@ "\n", "We will need to select three components from LangChain's suite of integrations.\n", "\n", - "A [chat model](/docs/integrations/chat/):\n", - "\n", "import ChatModelTabs from \"@theme/ChatModelTabs\";\n", "\n", "" @@ -78,8 +76,6 @@ "id": "da14773e-ac98-4a97-944b-4c6ec028d195", "metadata": {}, "source": [ - "An [embedding model](/docs/integrations/text_embedding/):\n", - "\n", "import EmbeddingTabs from \"@theme/EmbeddingTabs\";\n", "\n", "" @@ -105,8 +101,6 @@ "id": "22fdc314-b91d-4820-b0a8-873b5b6e76f5", "metadata": {}, "source": [ - "And a [vector store](/docs/integrations/vectorstores/):\n", - "\n", "import VectorStoreTabs from \"@theme/VectorStoreTabs\";\n", "\n", "" diff --git a/docs/docs/tutorials/rag.ipynb b/docs/docs/tutorials/rag.ipynb index 631b5537aa9bc..88232ceaa8b19 100644 --- a/docs/docs/tutorials/rag.ipynb +++ b/docs/docs/tutorials/rag.ipynb @@ -122,8 +122,6 @@ "\n", "We will need to select three components from LangChain's suite of integrations.\n", "\n", - "A [chat model](/docs/integrations/chat/):\n", - "\n", "import ChatModelTabs from \"@theme/ChatModelTabs\";\n", "\n", "" @@ -149,8 +147,6 @@ "id": "f1b78672-f21e-4827-843e-59514d18ca20", "metadata": {}, "source": [ - "An [embedding model](/docs/integrations/text_embedding/):\n", - "\n", "import EmbeddingTabs from \"@theme/EmbeddingTabs\";\n", "\n", "" @@ -176,8 +172,6 @@ "id": "859ffca8-055e-4f5a-95fe-55906ed1d63f", "metadata": {}, "source": [ - "And a [vector store](/docs/integrations/vectorstores/):\n", - "\n", "import VectorStoreTabs from \"@theme/VectorStoreTabs\";\n", "\n", "" @@ -712,7 +706,7 @@ "\n", "retrieved_docs = vector_store.similarity_search(question)\n", "docs_content = \"\\n\\n\".join(doc.page_content for doc in retrieved_docs)\n", - "prompt = prompt.invoke({\"question\": question, \"context\": formatted_docs})\n", + "prompt = prompt.invoke({\"question\": question, \"context\": docs_content})\n", "answer = llm.invoke(prompt)\n", "```\n", "\n", diff --git a/docs/src/theme/ChatModelTabs.js b/docs/src/theme/ChatModelTabs.js index c6aee8148568e..413af9ea9925a 100644 --- a/docs/src/theme/ChatModelTabs.js +++ b/docs/src/theme/ChatModelTabs.js @@ -3,7 +3,7 @@ import React, { useState } from "react"; import CodeBlock from "@theme-original/CodeBlock"; // Create a custom dropdown since Docusaurus's dropdown component isn't easily accessible -const CustomDropdown = ({ selectedOption, options, onSelect }) => { +export const CustomDropdown = ({ selectedOption, options, onSelect, modelType }) => { const [isOpen, setIsOpen] = React.useState(false); // Close dropdown when clicking outside @@ -18,13 +18,29 @@ const CustomDropdown = ({ selectedOption, options, onSelect }) => { return () => document.removeEventListener('click', handleClickOutside); }, [isOpen]); + // Determine the text and link based on the modelType + const getModelTextAndLink = () => { + switch (modelType) { + case 'chat': + return { text: 'chat model', link: '/docs/integrations/chat/' }; + case 'embeddings': + return { text: 'embeddings model', link: '/docs/integrations/text_embedding/' }; + case 'vectorstore': + return { text: 'vector store', link: '/docs/integrations/vectorstores/' }; + default: + return { text: 'chat model', link: '/docs/integrations/chat/' }; + } + }; + + const { text, link } = getModelTextAndLink(); + return (
- Select chat model: + Select {text}: