Merge branch 'main' into fix/bedrock_boto3_config

deepset-ai · Dec 3, 2024 · b231f62 · b231f62
2 parents 60a7a88 + 319b64b
commit b231f62
Show file tree

Hide file tree

Showing 122 changed files with 3,085 additions and 658 deletions.
diff --git a/.github/labeler.yml b/.github/labeler.yml
@@ -14,6 +14,11 @@ integration:astra:
       - any-glob-to-any-file: "integrations/astra/**/*"
       - any-glob-to-any-file: ".github/workflows/astra.yml"
 
+integration:azure-ai-search:
+  - changed-files:
+      - any-glob-to-any-file: "integrations/azure_ai_search/**/*"
+      - any-glob-to-any-file: ".github/workflows/azure_ai_search.yml"
+
 integration:chroma:
   - changed-files:
       - any-glob-to-any-file: "integrations/chroma/**/*"

diff --git a/.github/workflows/CI_pypi_release.yml b/.github/workflows/CI_pypi_release.yml
@@ -54,7 +54,7 @@ jobs:
         with:
           config: cliff.toml
           args: >
-            --include-path "${{ steps.pathfinder.outputs.project_path }}/*"
+            --include-path "${{ steps.pathfinder.outputs.project_path }}/**/*"
             --tag-pattern "${{ steps.pathfinder.outputs.project_path }}-v*"
 
       - name: Commit changelog

diff --git a/.github/workflows/pgvector.yml b/.github/workflows/pgvector.yml
@@ -33,7 +33,7 @@ jobs:
         python-version: ["3.9", "3.10", "3.11"]
     services:
       pgvector:
-        image: ankane/pgvector:latest
+        image: pgvector/pgvector:pg17
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres

diff --git a/README.md b/README.md
@@ -38,12 +38,12 @@ Please check out our [Contribution Guidelines](CONTRIBUTING.md) for all the deta
 | [google-ai-haystack](integrations/google_ai/)                                                                  | Generator           | [![PyPI - Version](https://img.shields.io/pypi/v/google-ai-haystack.svg)](https://pypi.org/project/google-ai-haystack)                                   | [![Test / google-ai](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/google_ai.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/google_ai.yml)                                  |
 | [google-vertex-haystack](integrations/google_vertex/)                                                          | Generator           | [![PyPI - Version](https://img.shields.io/pypi/v/google-vertex-haystack.svg)](https://pypi.org/project/google-vertex-haystack)                           | [![Test / google-vertex](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/google_vertex.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/google_vertex.yml)                      |
 | [instructor-embedders-haystack](integrations/instructor_embedders/)                                            | Embedder            | [![PyPI - Version](https://img.shields.io/pypi/v/instructor-embedders-haystack.svg)](https://pypi.org/project/instructor-embedders-haystack)             | [![Test / instructor-embedders](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/instructor_embedders.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/instructor_embedders.yml) |
-| [jina-haystack](integrations/jina/)                                                                            | Embedder, Ranker    | [![PyPI - Version](https://img.shields.io/pypi/v/jina-haystack.svg)](https://pypi.org/project/jina-haystack)                                             | [![Test / jina](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/jina.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/jina.yml)                                                 |
+| [jina-haystack](integrations/jina/)                                                                            | Connector, Embedder, Ranker    | [![PyPI - Version](https://img.shields.io/pypi/v/jina-haystack.svg)](https://pypi.org/project/jina-haystack)                                             | [![Test / jina](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/jina.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/jina.yml)                                                 |
 | [langfuse-haystack](integrations/langfuse/)                                                                    | Tracer              | [![PyPI - Version](https://img.shields.io/pypi/v/langfuse-haystack.svg?color=orange)](https://pypi.org/project/langfuse-haystack)                        | [![Test / langfuse](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/langfuse.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/langfuse.yml)                                     |
 | [llama-cpp-haystack](integrations/llama_cpp/)                                                                  | Generator           | [![PyPI - Version](https://img.shields.io/pypi/v/llama-cpp-haystack.svg?color=orange)](https://pypi.org/project/llama-cpp-haystack)                      | [![Test / llama-cpp](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/llama_cpp.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/llama_cpp.yml)                                  |
 | [mistral-haystack](integrations/mistral/)                                                                      | Embedder, Generator | [![PyPI - Version](https://img.shields.io/pypi/v/mistral-haystack.svg)](https://pypi.org/project/mistral-haystack)                                       | [![Test / mistral](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/mistral.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/mistral.yml)                                        |
 | [mongodb-atlas-haystack](integrations/mongodb_atlas/)                                                          | Document Store      | [![PyPI - Version](https://img.shields.io/pypi/v/mongodb-atlas-haystack.svg?color=orange)](https://pypi.org/project/mongodb-atlas-haystack)              | [![Test / mongodb-atlas](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/mongodb_atlas.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/mongodb_atlas.yml)                      |
-| [nvidia-haystack](integrations/nvidia/)                                                                        | Generator           | [![PyPI - Version](https://img.shields.io/pypi/v/nvidia-haystack.svg?color=orange)](https://pypi.org/project/nvidia-haystack)                            | [![Test / nvidia](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/nvidia.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/nvidia.yml)                                           |
+| [nvidia-haystack](integrations/nvidia/)                                                                        | Embedder, Generator, Ranker    | [![PyPI - Version](https://img.shields.io/pypi/v/nvidia-haystack.svg?color=orange)](https://pypi.org/project/nvidia-haystack)                            | [![Test / nvidia](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/nvidia.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/nvidia.yml)                                           |
 | [ollama-haystack](integrations/ollama/)                                                                        | Embedder, Generator | [![PyPI - Version](https://img.shields.io/pypi/v/ollama-haystack.svg?color=orange)](https://pypi.org/project/ollama-haystack)                            | [![Test / ollama](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/ollama.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/ollama.yml)                                           |
 | [opensearch-haystack](integrations/opensearch/)                                                                | Document Store      | [![PyPI - Version](https://img.shields.io/pypi/v/opensearch-haystack.svg)](https://pypi.org/project/opensearch-haystack)                                 | [![Test / opensearch](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/opensearch.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/opensearch.yml)                               |
 | [optimum-haystack](integrations/optimum/)                                                                      | Embedder            | [![PyPI - Version](https://img.shields.io/pypi/v/optimum-haystack.svg)](https://pypi.org/project/optimum-haystack)                                       | [![Test / optimum](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/optimum.yml/badge.svg)](https://github.com/deepset-ai/haystack-core-integrations/actions/workflows/optimum.yml)                                        |
@@ -85,3 +85,8 @@ GitHub. The GitHub Actions workflow will take care of the rest.
     git push --tags origin
     ```
 3. Wait for the CI to do its magic
+
+> [!IMPORTANT]  
+> When releasing a new integration version, always tag a commit that includes the changes for that integration 
+> (usually the PR merge commit). If you tag a commit that doesn't include changes for the integration being released, 
+> the generated changelog will be incorrect.
diff --git a/cliff.toml b/cliff.toml
@@ -19,11 +19,43 @@ body = """
     ## [unreleased]
 {% endif %}\
 {% for group, commits in commits | group_by(attribute="group") %}
+    {#
+        Skip the whole section if it contains only a single commit
+        and it's the commit that updated the changelog.
+        If we don't do this we get an empty section since we don't show
+        commits that update the changelog
+    #}\
+    {% if commits | length == 1 and commits[0].message == 'Update the changelog' %}\
+        {% continue %}\
+    {% endif %}\
     ### {{ group | striptags | trim | upper_first }}
-    {% for commit in commits %}
+    {% for commit in commits %}\
+        {#
+            Skip commits that update the changelog, they're not useful to the user
+        #}\
+        {% if commit.message == 'Update the changelog' %}\
+            {% continue %}\
+        {% endif %}
         - {% if commit.scope %}*({{ commit.scope }})* {% endif %}\
             {% if commit.breaking %}[**breaking**] {% endif %}\
-            {{ commit.message | upper_first }}\
+            {#
+                We first try to render the conventional commit message if present.
+                If it's not a conventional commit we get the PR title if present.
+                If the commit is neither conventional, nor has a PR title set
+                we fallback to whatever the commit message is.
+
+                We do this cause when merging PRs with multiple commits that don't
+                have a title following conventional commit guidelines we might get
+                a commit message that is multiple lines. That makes the changelog
+                look a bit funky so we handle it like so.
+            #}\
+            {% if commit.conventional %}\
+                {{ commit.message | upper_first }}\
+            {% elif commit.remote.pr_title %}\
+                {{ commit.remote.pr_title | upper_first }} (#{{ commit.remote.pr_number }})\
+            {% else %}\
+                {{ commit.message | upper_first }}\
+            {% endif %}\
     {% endfor %}
 {% endfor %}\n
 """
@@ -35,7 +67,7 @@ footer = """
 trim = true
 # postprocessors
 postprocessors = [
-  # { pattern = '<REPO>', replace = "https://github.com/orhun/git-cliff" }, # replace repository URL
+    # { pattern = '<REPO>', replace = "https://github.com/orhun/git-cliff" }, # replace repository URL
 ]
 
 [git]
@@ -47,24 +79,26 @@ filter_unconventional = false
 split_commits = false
 # regex for preprocessing the commit messages
 commit_preprocessors = [
-  # Replace issue numbers
-  #{ pattern = '\((\w+\s)?#([0-9]+)\)', replace = "([#${2}](<REPO>/issues/${2}))"},
-  # Check spelling of the commit with https://github.com/crate-ci/typos
-  # If the spelling is incorrect, it will be automatically fixed.
-  #{ pattern = '.*', replace_command = 'typos --write-changes -' },
+    # Replace issue numbers
+    #{ pattern = '\((\w+\s)?#([0-9]+)\)', replace = "([#${2}](<REPO>/issues/${2}))"},
+    # Check spelling of the commit with https://github.com/crate-ci/typos
+    # If the spelling is incorrect, it will be automatically fixed.
+    #{ pattern = '.*', replace_command = 'typos --write-changes -' },
 ]
 # regex for parsing and grouping commits
 commit_parsers = [
-  { message = "^feat", group = "<!-- 0 -->🚀 Features" },
-  { message = "^fix", group = "<!-- 1 -->🐛 Bug Fixes" },
-  { message = "^doc", group = "<!-- 3 -->📚 Documentation" },
-  { message = "^perf", group = "<!-- 4 -->⚡ Performance" },
-  { message = "^refactor", group = "<!-- 2 -->🚜 Refactor" },
-  { message = "^style", group = "<!-- 5 -->🎨 Styling" },
-  { message = "^test", group = "<!-- 6 -->🧪 Testing" },
-  { message = "^chore|^ci", group = "<!-- 7 -->⚙️ Miscellaneous Tasks" },
-  { body = ".*security", group = "<!-- 8 -->🛡️ Security" },
-  { message = "^revert", group = "<!-- 9 -->◀️ Revert" },
+    { message = "^feat", group = "<!-- 00 -->🚀 Features" },
+    { message = "^fix", group = "<!-- 01 -->🐛 Bug Fixes" },
+    { message = "^refactor", group = "<!-- 02 -->🚜 Refactor" },
+    { message = "^doc", group = "<!-- 03 -->📚 Documentation" },
+    { message = "^perf", group = "<!-- 04 -->⚡ Performance" },
+    { message = "^style", group = "<!-- 05 -->🎨 Styling" },
+    { message = "^test", group = "<!-- 06 -->🧪 Testing" },
+    { body = ".*security", group = "<!-- 07 -->🛡️ Security" },
+    { message = "^revert", group = "<!-- 08 -->◀️ Revert" },
+    { message = "^ci", group = "<!-- 09 -->⚙️ CI" },
+    { message = "^chore", group = "<!-- 10 -->🧹 Chores" },
+    { message = ".*", group = "<!-- 11 -->🌀 Miscellaneous" },
 ]
 # protect breaking changes from being skipped due to matching a skipping commit_parser
 protect_breaking_commits = false
@@ -82,3 +116,7 @@ topo_order = false
 sort_commits = "oldest"
 # limit the number of commits included in the changelog.
 # limit_commits = 42
+
+[remote.github]
+owner = "deepset-ai"
+repo = "haystack-core-integrations"
diff --git a/.../amazon_bedrock/src/haystack_integrations/components/embedders/amazon_bedrock/__init__.py b/.../amazon_bedrock/src/haystack_integrations/components/embedders/amazon_bedrock/__init__.py
@@ -4,4 +4,4 @@
 from .document_embedder import AmazonBedrockDocumentEmbedder
 from .text_embedder import AmazonBedrockTextEmbedder
 
-__all__ = ["AmazonBedrockTextEmbedder", "AmazonBedrockDocumentEmbedder"]
+__all__ = ["AmazonBedrockDocumentEmbedder", "AmazonBedrockTextEmbedder"]
diff --git a/...edrock/src/haystack_integrations/components/embedders/amazon_bedrock/document_embedder.py b/...edrock/src/haystack_integrations/components/embedders/amazon_bedrock/document_embedder.py
@@ -243,7 +243,7 @@ def run(self, documents: List[Document]):
             - `documents`: The `Document`s with the `embedding` field populated.
         :raises AmazonBedrockInferenceError: If the inference fails.
         """
-        if not isinstance(documents, list) or documents and not isinstance(documents[0], Document):
+        if not isinstance(documents, list) or (documents and not isinstance(documents[0], Document)):
             msg = (
                 "AmazonBedrockDocumentEmbedder expects a list of Documents as input."
                 "In case you want to embed a string, please use the AmazonBedrockTextEmbedder."

diff --git a/...amazon_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/__init__.py b/...amazon_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/__init__.py
@@ -4,4 +4,4 @@
 from .chat.chat_generator import AmazonBedrockChatGenerator
 from .generator import AmazonBedrockGenerator
 
-__all__ = ["AmazonBedrockGenerator", "AmazonBedrockChatGenerator"]
+__all__ = ["AmazonBedrockChatGenerator", "AmazonBedrockGenerator"]
diff --git a/...n_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/chat/adapters.py b/...n_bedrock/src/haystack_integrations/components/generators/amazon_bedrock/chat/adapters.py
@@ -212,6 +212,8 @@ def prepare_body(self, messages: List[ChatMessage], **inference_kwargs) -> Dict[
         stop_sequences = inference_kwargs.get("stop_sequences", []) + inference_kwargs.pop("stop_words", [])
         if stop_sequences:
             inference_kwargs["stop_sequences"] = stop_sequences
+        # pop stream kwarg from inference_kwargs as Anthropic does not support it (if provided)
+        inference_kwargs.pop("stream", None)
         params = self._get_params(inference_kwargs, default_params, self.ALLOWED_PARAMS)
         body = {**self.prepare_chat_messages(messages=messages), **params}
         return body
@@ -384,6 +386,10 @@ def prepare_body(self, messages: List[ChatMessage], **inference_kwargs) -> Dict[
         stop_words = inference_kwargs.pop("stop_words", [])
         if stop_words:
             inference_kwargs["stop"] = stop_words
+
+        # pop stream kwarg from inference_kwargs as Mistral does not support it (if provided)
+        inference_kwargs.pop("stream", None)
+
         params = self._get_params(inference_kwargs, default_params, self.ALLOWED_PARAMS)
         body = {"prompt": self.prepare_chat_messages(messages=messages), **params}
         return body

diff --git a/integrations/amazon_bedrock/tests/test_chat_generator.py b/integrations/amazon_bedrock/tests/test_chat_generator.py
@@ -17,7 +17,7 @@
 )
 
 KLASS = "haystack_integrations.components.generators.amazon_bedrock.chat.chat_generator.AmazonBedrockChatGenerator"
-MODELS_TO_TEST = ["anthropic.claude-3-sonnet-20240229-v1:0", "anthropic.claude-v2:1", "meta.llama2-13b-chat-v1"]
+MODELS_TO_TEST = ["anthropic.claude-3-sonnet-20240229-v1:0", "anthropic.claude-v2:1"]
 MODELS_TO_TEST_WITH_TOOLS = ["anthropic.claude-3-haiku-20240307-v1:0"]
 MISTRAL_MODELS = [
     "mistral.mistral-7b-instruct-v0:2",
@@ -248,10 +248,8 @@ def test_long_prompt_is_not_truncated_when_truncate_false(mock_boto3_session):
 
             generator.model_adapter.get_responses = MagicMock(
                 return_value=[
-                    ChatMessage(
+                    ChatMessage.from_assistant(
                         content="Some text",
-                        role=ChatRole.ASSISTANT,
-                        name=None,
                         meta={
                             "model": "claude-3-sonnet-20240229",
                             "index": 0,

diff --git a/integrations/anthropic/CHANGELOG.md b/integrations/anthropic/CHANGELOG.md
@@ -1,11 +1,29 @@
 # Changelog
 
+## [unreleased]
+
+### ⚙️ CI
+
+- Adopt uv as installer (#1142)
+
+### 🧹 Chores
+
+- Update ruff linting scripts and settings (#1105)
+
+### 🌀 Miscellaneous
+
+- Add AnthropicVertexChatGenerator component (#1192)
+
 ## [integrations/anthropic-v1.1.0] - 2024-09-20
 
 ### 🚀 Features
 
 - Add Anthropic prompt caching support, add example (#1006)
 
+### 🌀 Miscellaneous
+
+- Chore: Update Anthropic example, use ChatPromptBuilder properly (#978)
+
 ## [integrations/anthropic-v1.0.0] - 2024-08-12
 
 ### 🐛 Bug Fixes
@@ -20,25 +38,43 @@
 
 - Do not retry tests in `hatch run test` command (#954)
 
+
 ## [integrations/anthropic-v0.4.1] - 2024-07-17
 
-### ⚙️ Miscellaneous Tasks
+### 🧹 Chores
 
 - Update ruff invocation to include check parameter (#853)
 
+### 🌀 Miscellaneous
+
+- Ci: install `pytest-rerunfailures` where needed; add retry config to `test-cov` script (#845)
+- Add meta deprecration warning (#910)
+
 ## [integrations/anthropic-v0.4.0] - 2024-06-21
 
 ### 🚀 Features
 
 - Update Anthropic/Cohere for tools use (#790)
 - Update Anthropic default models, pydocs (#839)
 
-### ⚙️ Miscellaneous Tasks
+### ⚙️ CI
 
 - Retry tests to reduce flakyness (#836)
 
+### 🌀 Miscellaneous
+
+- Remove references to Python 3.7 (#601)
+- Chore: add license classifiers (#680)
+- Chore: change the pydoc renderer class (#718)
+- Docs: add missing api references (#728)
+
 ## [integrations/anthropic-v0.2.0] - 2024-03-15
 
+### 🌀 Miscellaneous
+
+- Docs: Replace amazon-bedrock with anthropic in readme (#584)
+- Chore: Use the correct sonnet model name (#587)
+
 ## [integrations/anthropic-v0.1.0] - 2024-03-15
 
 ### 🚀 Features

diff --git a/integrations/anthropic/pydoc/config.yml b/integrations/anthropic/pydoc/config.yml
@@ -4,6 +4,7 @@ loaders:
     modules: [
       "haystack_integrations.components.generators.anthropic.generator",
       "haystack_integrations.components.generators.anthropic.chat.chat_generator",
+      "haystack_integrations.components.generators.anthropic.chat.vertex_chat_generator",
     ]
     ignore_when_discovered: ["__init__"]
 processors:

diff --git a/integrations/anthropic/src/haystack_integrations/components/generators/anthropic/__init__.py b/integrations/anthropic/src/haystack_integrations/components/generators/anthropic/__init__.py
@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 from .chat.chat_generator import AnthropicChatGenerator
+from .chat.vertex_chat_generator import AnthropicVertexChatGenerator
 from .generator import AnthropicGenerator
 
-__all__ = ["AnthropicGenerator", "AnthropicChatGenerator"]
+__all__ = ["AnthropicChatGenerator", "AnthropicGenerator", "AnthropicVertexChatGenerator"]