Skip to content

Commit

Permalink
feat(Qdrant): making some test progress
Browse files Browse the repository at this point in the history
  • Loading branch information
lambda-science committed Mar 14, 2024
1 parent 3824e8c commit 4253a1c
Show file tree
Hide file tree
Showing 3 changed files with 17 additions and 9 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,15 @@ def documents_to_batch(
points = []
for document in documents:
payload = document.to_dict(flatten=False)
dense_vector = payload.pop(embedding_field) or {}
vector = {}
if embedding_field in payload and payload[embedding_field] is not None:
dense_vector = payload.pop(embedding_field) or []
vector["text-dense"] = dense_vector
# TODO: Adapt to Haystack Modification of the Document Dataclass
sparse_vector = payload["meta"].pop("_sparse_vector") or {}
vector = {"text-dense": dense_vector, "text-sparse": sparse_vector}
if "_sparse_vector" in payload["meta"]:
sparse_vector = payload["meta"].pop("_sparse_vector", {"indices": [], "values": []})
sparse_vector_instance = rest.SparseVector(**sparse_vector)
vector["text-sparse"] = sparse_vector_instance
_id = self.convert_id(payload.get("id"))

point = rest.PointStruct(
Expand Down Expand Up @@ -61,7 +66,11 @@ def point_to_document(self, point: QdrantPoint) -> Document:
payload["score"] = point.score if hasattr(point, "score") else None
# TODO: Adapt to Haystack Modification of the Document Dataclass
if hasattr(point, "vector") and "text-dense" in point.vector:
payload["meta"]["_sparse_vector"] = point.vector["text-sparse"]
parse_vector_dict = {
"indices": point.vector["text-sparse"].indices,
"values": point.vector["text-sparse"].values
}
payload["meta"]["_sparse_vector"] = parse_vector_dict
else:
payload["meta"]["_sparse_vector"] = None
return Document.from_dict(payload)
Original file line number Diff line number Diff line change
Expand Up @@ -499,9 +499,8 @@ def _set_up_collection(
# Create Payload index if payload_fields_to_index is provided
self._create_payload_index(collection_name, payload_fields_to_index)
return

current_distance = collection_info.config.params.vectors.distance
current_vector_size = collection_info.config.params.vectors.size
current_distance = collection_info.config.params.vectors["text-dense"].distance
current_vector_size = collection_info.config.params.vectors["text-dense"].size

if current_distance != distance:
msg = (
Expand Down
4 changes: 2 additions & 2 deletions integrations/qdrant/tests/test_converters.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,11 @@ def test_point_to_document_reverts_proper_structure_from_record(
"test_field": 1,
},
},
vector=[1.0, 0.0, 0.0, 0.0],
vector={"text-dense":[1.0, 0.0, 0.0, 0.0], "text-sparse": {"indices": [7, 1024, 367], "values": [0.1, 0.98, 0.33]}},
)
document = qdrant_to_haystack.point_to_document(point)
assert "my-id" == document.id
assert "Lorem ipsum" == document.content
assert "text" == document.content_type
assert {"test_field": 1} == document.meta
assert {"test_field": 1, "_sparse_vector": {"indices": [7, 1024, 367], "values": [0.1, 0.98, 0.33]}} == document.meta
assert 0.0 == np.sum(np.array([1.0, 0.0, 0.0, 0.0]) - document.embedding)

0 comments on commit 4253a1c

Please sign in to comment.