feat(Qdrant): start to work on sparse vector integration

.github/workflows/qdrant.yml

@@ -58,7 +58,9 @@ jobs:
 
       - name: Run tests
         id: tests
-        run: hatch run cov
+        run: |
+          hatch run pip install git+https://github.com/deepset-ai/haystack.git #TODO: rm before merging
+          hatch run cov
 
       - name: Nightly - run unit tests with Haystack main branch
         if: github.event_name == 'schedule'

integrations/qdrant/pyproject.toml

@@ -24,7 +24,7 @@ classifiers = [
   "Programming Language :: Python :: Implementation :: CPython",
   "Programming Language :: Python :: Implementation :: PyPy",
 ]
-dependencies = ["haystack-ai>=2.0.0b6", "qdrant-client"]
+dependencies = ["haystack-ai", "qdrant-client"]
 
 [project.urls]
 Source = "https://github.com/deepset-ai/haystack-core-integrations"

integrations/qdrant/src/haystack_integrations/components/retrievers/qdrant/__init__.py

@@ -2,6 +2,6 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-from .retriever import QdrantEmbeddingRetriever
+from .retriever import QdrantEmbeddingRetriever, QdrantSparseRetriever
 
-__all__ = ("QdrantEmbeddingRetriever",)
+__all__ = ("QdrantEmbeddingRetriever", "QdrantSparseRetriever")

integrations/qdrant/src/haystack_integrations/components/retrievers/qdrant/retriever.py

@@ -1,13 +1,14 @@
 from typing import Any, Dict, List, Optional
 
 from haystack import Document, component, default_from_dict, default_to_dict
+from haystack.dataclasses.sparse_embedding import SparseEmbedding
 from haystack_integrations.document_stores.qdrant import QdrantDocumentStore
 
 
 @component
 class QdrantEmbeddingRetriever:
     """
-    A component for retrieving documents from an QdrantDocumentStore.
+    A component for retrieving documents from an QdrantDocumentStore using dense vectors.
 
     Usage example:
     ```python
@@ -120,3 +121,121 @@ def run(
         )
 
         return {"documents": docs}
+
+
+@component
+class QdrantSparseRetriever:
+    """
+    A component for retrieving documents from an QdrantDocumentStore using sparse vectors.
+
+    Usage example:
+    ```python
+    from haystack_integrations.components.retrievers.qdrant import QdrantSparseRetriever
+    from haystack_integrations.document_stores.qdrant import QdrantDocumentStore
+    from haystack.dataclasses.sparse_embedding import SparseEmbedding
+
+    document_store = QdrantDocumentStore(
+        ":memory:",
+        recreate_index=True,
+        return_sparse_embedding=True,
+        wait_result_from_api=True,
+    )
+    retriever = QdrantSparseRetriever(document_store=document_store)
+    sparse_embedding = SparseEmbedding(indices=[0, 1, 2, 3], values=[0.1, 0.8, 0.05, 0.33])
+    retriever.run(query_sparse_embedding=sparse_embedding)
+    ```
+    """
+
+    def __init__(
+        self,
+        document_store: QdrantDocumentStore,
+        filters: Optional[Dict[str, Any]] = None,
+        top_k: int = 10,
+        scale_score: bool = True,  # noqa: FBT001, FBT002
+        return_embedding: bool = False,  # noqa: FBT001, FBT002
+    ):
+        """
+        Create a QdrantSparseRetriever component.
+
+        :param document_store: An instance of QdrantDocumentStore.
+        :param filters: A dictionary with filters to narrow down the search space. Default is None.
+        :param top_k: The maximum number of documents to retrieve. Default is 10.
+        :param scale_score: Whether to scale the scores of the retrieved documents or not. Default is True.
+        :param return_embedding: Whether to return the sparse embedding of the retrieved Documents. Default is False.
+
+        :raises ValueError: If 'document_store' is not an instance of QdrantDocumentStore.
+        """
+
+        if not isinstance(document_store, QdrantDocumentStore):
+            msg = "document_store must be an instance of QdrantDocumentStore"
+            raise ValueError(msg)
+
+        self._document_store = document_store
+        self._filters = filters
+        self._top_k = top_k
+        self._scale_score = scale_score
+        self._return_embedding = return_embedding
+
+    def to_dict(self) -> Dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        d = default_to_dict(
+            self,
+            document_store=self._document_store,
+            filters=self._filters,
+            top_k=self._top_k,
+            scale_score=self._scale_score,
+            return_embedding=self._return_embedding,
+        )
+        d["init_parameters"]["document_store"] = self._document_store.to_dict()
+
+        return d
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Any]) -> "QdrantEmbeddingRetriever":
+        """
+        Deserializes the component from a dictionary.
+
+        :param data:
+            Dictionary to deserialize from.
+        :returns:
+            Deserialized component.
+        """
+        document_store = QdrantDocumentStore.from_dict(data["init_parameters"]["document_store"])
+        data["init_parameters"]["document_store"] = document_store
+        return default_from_dict(cls, data)
+
+    @component.output_types(documents=List[Document])
+    def run(
+        self,
+        query_sparse_embedding: SparseEmbedding,
+        filters: Optional[Dict[str, Any]] = None,
+        top_k: Optional[int] = None,
+        scale_score: Optional[bool] = None,
+        return_embedding: Optional[bool] = None,
+    ):
+        """
+        Run the Sparse Embedding Retriever on the given input data.
+
+        :param query_sparse_embedding: Sparse Embedding of the query.
+        :param filters: A dictionary with filters to narrow down the search space.
+        :param top_k: The maximum number of documents to return.
+        :param scale_score: Whether to scale the scores of the retrieved documents or not.
+        :param return_embedding: Whether to return the embedding of the retrieved Documents.
+        :returns:
+            The retrieved documents.
+
+        """
+        docs = self._document_store.query_by_sparse(
+            query_sparse_embedding=query_sparse_embedding,
+            filters=filters or self._filters,
+            top_k=top_k or self._top_k,
+            scale_score=scale_score or self._scale_score,
+            return_embedding=return_embedding or self._return_embedding,
+        )
+
+        return {"documents": docs}

integrations/qdrant/src/haystack_integrations/document_stores/qdrant/converters.py

@@ -7,6 +7,9 @@
 
 logger = logging.getLogger(__name__)
 
+DENSE_VECTORS_NAME = "text-dense"
+SPARSE_VECTORS_NAME = "text-sparse"
+
 
 class HaystackToQdrant:
     """A converter from Haystack to Qdrant types."""
@@ -18,23 +21,27 @@ def documents_to_batch(
         documents: List[Document],
         *,
         embedding_field: str,
+        use_sparse_embeddings: bool,
+        sparse_embedding_field: str,
     ) -> List[rest.PointStruct]:
         points = []
         for document in documents:
             payload = document.to_dict(flatten=False)
-            vector = payload.pop(embedding_field) or {}
-            _id = self.convert_id(payload.get("id"))
+            if use_sparse_embeddings:
+                vector = {}
+
+                dense_vector = payload.pop(embedding_field, None)
+                if dense_vector is not None:
+                    vector[DENSE_VECTORS_NAME] = dense_vector
 
-            # TODO: remove as soon as we introduce the support for sparse embeddings in Qdrant
-            if "sparse_embedding" in payload:
-                sparse_embedding = payload.pop("sparse_embedding", None)
-                if sparse_embedding:
-                    logger.warning(
-                        "Document %s has the `sparse_embedding` field set,"
-                        "but storing sparse embeddings in Qdrant is not currently supported."
-                        "The `sparse_embedding` field will be ignored.",
-                        payload["id"],
-                    )
+                sparse_vector = payload.pop(sparse_embedding_field, None)
+                if sparse_vector is not None:
+                    sparse_vector_instance = rest.SparseVector(**sparse_vector)
+                    vector[SPARSE_VECTORS_NAME] = sparse_vector_instance
+
+            if not use_sparse_embeddings:
+                vector = payload.pop(embedding_field) or {}
+            _id = self.convert_id(payload.get("id"))
 
             point = rest.PointStruct(
                 payload=payload,
@@ -58,13 +65,38 @@ def convert_id(self, _id: str) -> str:
 
 
 class QdrantToHaystack:
-    def __init__(self, content_field: str, name_field: str, embedding_field: str):
+    def __init__(
+        self,
+        content_field: str,
+        name_field: str,
+        embedding_field: str,
+        use_sparse_embeddings: bool,  # noqa: FBT001
+        sparse_embedding_field: str,
+    ):
         self.content_field = content_field
         self.name_field = name_field
         self.embedding_field = embedding_field
+        self.use_sparse_embeddings = use_sparse_embeddings
+        self.sparse_embedding_field = sparse_embedding_field
 
     def point_to_document(self, point: QdrantPoint) -> Document:
         payload = {**point.payload}
-        payload["embedding"] = point.vector if hasattr(point, "vector") else None
         payload["score"] = point.score if hasattr(point, "score") else None
+        if not self.use_sparse_embeddings:
+            payload["embedding"] = point.vector if hasattr(point, "vector") else None
+
+        if self.use_sparse_embeddings:
+            if hasattr(point, "vector") and point.vector is not None and DENSE_VECTORS_NAME in point.vector:
+                payload["embedding"] = point.vector[DENSE_VECTORS_NAME]
+            else:
+                payload["embedding"] = None
+
+            if hasattr(point, "vector") and point.vector is not None and SPARSE_VECTORS_NAME in point.vector:
+                parse_vector_dict = {
+                    "indices": point.vector[SPARSE_VECTORS_NAME].indices,
+                    "values": point.vector[SPARSE_VECTORS_NAME].values,
+                }
+                payload["sparse_embedding"] = parse_vector_dict
+            else:
+                payload["sparse_embedding"] = None
         return Document.from_dict(payload)