Merge remote-tracking branch 'origin/feat/unstructured_meta_field' in…

…to feat/unstructured_meta_field
deepset-ai · Jan 22, 2024 · 3a80a94 · 3a80a94
2 parents 99a2847 + d06e2c3
commit 3a80a94
Show file tree

Hide file tree

Showing 22 changed files with 76 additions and 65 deletions.
diff --git a/integrations/elasticsearch/pyproject.toml b/integrations/elasticsearch/pyproject.toml
@@ -33,6 +33,9 @@ Documentation = "https://github.com/deepset-ai/haystack-core-integrations/tree/m
 Issues = "https://github.com/deepset-ai/haystack-core-integrations/issues"
 Source = "https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/elasticsearch"
 
+[tool.hatch.build.targets.wheel]
+packages = ["src/haystack_integrations"]
+
 [tool.hatch.version]
 source = "vcs"
 tag-pattern = 'integrations\/elasticsearch-v(?P<version>.*)'
@@ -70,7 +73,7 @@ dependencies = [
   "ruff>=0.0.243",
 ]
 [tool.hatch.envs.lint.scripts]
-typing = "mypy --install-types --non-interactive {args:src/elasticsearch_haystack tests}"
+typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}"
 style = [
   "ruff {args:.}",
   "black --check --diff {args:.}",
@@ -139,26 +142,23 @@ unfixable = [
 ]
 
 [tool.ruff.isort]
-known-first-party = ["elasticsearch_haystack"]
+known-first-party = ["src"]
 
 [tool.ruff.flake8-tidy-imports]
-ban-relative-imports = "all"
+ban-relative-imports = "parents"
 
 [tool.ruff.per-file-ignores]
 # Tests can use magic values, assertions, and relative imports
 "tests/**/*" = ["PLR2004", "S101", "TID252"]
 
 [tool.coverage.run]
-source_pkgs = ["elasticsearch_haystack", "tests"]
+source_pkgs = ["src", "tests"]
 branch = true
 parallel = true
-omit = [
-  "src/elasticsearch_haystack/__about__.py",
-]
 
 [tool.coverage.paths]
-elasticsearch_haystack = ["src/elasticsearch_haystack", "*/elasticsearch-haystack/src/elasticsearch_haystack"]
-tests = ["tests", "*/elasticsearch-haystack/tests"]
+elasticsearch_haystack = ["src/haystack_integrations", "*/elasticsearch/src/haystack_integrations"]
+tests = ["tests", "*/elasticsearch/src/tests"]
 
 [tool.coverage.report]
 exclude_lines = [
@@ -177,6 +177,7 @@ markers = [
 [[tool.mypy.overrides]]
 module = [
   "haystack.*",
+  "haystack_integrations.*",
   "pytest.*"
 ]
 ignore_missing_imports = true
diff --git a/...s/elasticsearch/src/haystack_integrations/components/retrievers/elasticsearch/__init__.py b/...s/elasticsearch/src/haystack_integrations/components/retrievers/elasticsearch/__init__.py
@@ -0,0 +1,7 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <[email protected]>
+#
+# SPDX-License-Identifier: Apache-2.0
+from .bm25_retriever import ElasticsearchBM25Retriever
+from .embedding_retriever import ElasticsearchEmbeddingRetriever
+
+__all__ = ["ElasticsearchBM25Retriever", "ElasticsearchEmbeddingRetriever"]
diff --git a/.../elasticsearch_haystack/bm25_retriever.py → ...etrievers/elasticsearch/bm25_retriever.py b/.../elasticsearch_haystack/bm25_retriever.py → ...etrievers/elasticsearch/bm25_retriever.py
@@ -5,8 +5,7 @@
 
 from haystack import component, default_from_dict, default_to_dict
 from haystack.dataclasses import Document
-
-from elasticsearch_haystack.document_store import ElasticsearchDocumentStore
+from haystack_integrations.document_stores.elasticsearch.document_store import ElasticsearchDocumentStore
 
 
 @component
@@ -19,8 +18,8 @@ class ElasticsearchBM25Retriever:
     Usage example:
     ```python
     from haystack import Document
-    from elasticsearch_haystack.document_store import ElasticsearchDocumentStore
-    from elasticsearch_haystack.bm25_retriever import ElasticsearchBM25Retriever
+    from haystack_integrations.document_stores.elasticsearch import ElasticsearchDocumentStore
+    from haystack_integrations.components.retrievers.elasticsearch import ElasticsearchBM25Retriever
 
     document_store = ElasticsearchDocumentStore(hosts="http://localhost:9200")
     retriever = ElasticsearchBM25Retriever(document_store=document_store)

diff --git a/...ticsearch_haystack/embedding_retriever.py → ...vers/elasticsearch/embedding_retriever.py b/...ticsearch_haystack/embedding_retriever.py → ...vers/elasticsearch/embedding_retriever.py
@@ -5,8 +5,7 @@
 
 from haystack import component, default_from_dict, default_to_dict
 from haystack.dataclasses import Document
-
-from elasticsearch_haystack.document_store import ElasticsearchDocumentStore
+from haystack_integrations.document_stores.elasticsearch.document_store import ElasticsearchDocumentStore
 
 
 @component

diff --git a/...ch/src/elasticsearch_haystack/__init__.py → ...document_stores/elasticsearch/__init__.py b/...ch/src/elasticsearch_haystack/__init__.py → ...document_stores/elasticsearch/__init__.py
@@ -1,6 +1,6 @@
 # SPDX-FileCopyrightText: 2023-present deepset GmbH <[email protected]>
 #
 # SPDX-License-Identifier: Apache-2.0
-from elasticsearch_haystack.document_store import ElasticsearchDocumentStore
+from .document_store import ElasticsearchDocumentStore
 
 __all__ = ["ElasticsearchDocumentStore"]
diff --git a/.../elasticsearch_haystack/document_store.py → ...nt_stores/elasticsearch/document_store.py b/.../elasticsearch_haystack/document_store.py → ...nt_stores/elasticsearch/document_store.py
@@ -8,14 +8,15 @@
 
 # There are no import stubs for elastic_transport and elasticsearch so mypy fails
 from elastic_transport import NodeConfig  # type: ignore[import-not-found]
-from elasticsearch import Elasticsearch, helpers  # type: ignore[import-not-found]
 from haystack import default_from_dict, default_to_dict
 from haystack.dataclasses import Document
 from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError
 from haystack.document_stores.types import DuplicatePolicy
 from haystack.utils.filters import convert
 
-from elasticsearch_haystack.filters import _normalize_filters
+from elasticsearch import Elasticsearch, helpers  # type: ignore[import-not-found]
+
+from .filters import _normalize_filters
 
 logger = logging.getLogger(__name__)
 

diff --git a/...rch/src/elasticsearch_haystack/filters.py → .../document_stores/elasticsearch/filters.py b/...rch/src/elasticsearch_haystack/filters.py → .../document_stores/elasticsearch/filters.py
diff --git a/integrations/elasticsearch/tests/test_bm25_retriever.py b/integrations/elasticsearch/tests/test_bm25_retriever.py
@@ -4,9 +4,8 @@
 from unittest.mock import Mock, patch
 
 from haystack.dataclasses import Document
-
-from elasticsearch_haystack.bm25_retriever import ElasticsearchBM25Retriever
-from elasticsearch_haystack.document_store import ElasticsearchDocumentStore
+from haystack_integrations.components.retrievers.elasticsearch import ElasticsearchBM25Retriever
+from haystack_integrations.document_stores.elasticsearch import ElasticsearchDocumentStore
 
 
 def test_init_default():
@@ -18,21 +17,21 @@ def test_init_default():
     assert not retriever._scale_score
 
 
-@patch("elasticsearch_haystack.document_store.Elasticsearch")
+@patch("haystack_integrations.document_stores.elasticsearch.document_store.Elasticsearch")
 def test_to_dict(_mock_elasticsearch_client):
     document_store = ElasticsearchDocumentStore(hosts="some fake host")
     retriever = ElasticsearchBM25Retriever(document_store=document_store)
     res = retriever.to_dict()
     assert res == {
-        "type": "elasticsearch_haystack.bm25_retriever.ElasticsearchBM25Retriever",
+        "type": "haystack_integrations.components.retrievers.elasticsearch.bm25_retriever.ElasticsearchBM25Retriever",
         "init_parameters": {
             "document_store": {
                 "init_parameters": {
                     "hosts": "some fake host",
                     "index": "default",
                     "embedding_similarity_function": "cosine",
                 },
-                "type": "elasticsearch_haystack.document_store.ElasticsearchDocumentStore",
+                "type": "haystack_integrations.document_stores.elasticsearch.document_store.ElasticsearchDocumentStore",
             },
             "filters": {},
             "fuzziness": "AUTO",
@@ -42,14 +41,14 @@ def test_to_dict(_mock_elasticsearch_client):
     }
 
 
-@patch("elasticsearch_haystack.document_store.Elasticsearch")
+@patch("haystack_integrations.document_stores.elasticsearch.document_store.Elasticsearch")
 def test_from_dict(_mock_elasticsearch_client):
     data = {
-        "type": "elasticsearch_haystack.bm25_retriever.ElasticsearchBM25Retriever",
+        "type": "haystack_integrations.components.retrievers.elasticsearch.bm25_retriever.ElasticsearchBM25Retriever",
         "init_parameters": {
             "document_store": {
                 "init_parameters": {"hosts": "some fake host", "index": "default"},
-                "type": "elasticsearch_haystack.document_store.ElasticsearchDocumentStore",
+                "type": "haystack_integrations.document_stores.elasticsearch.document_store.ElasticsearchDocumentStore",
             },
             "filters": {},
             "fuzziness": "AUTO",

diff --git a/integrations/elasticsearch/tests/test_document_store.py b/integrations/elasticsearch/tests/test_document_store.py
@@ -12,10 +12,10 @@
 from haystack.document_stores.errors import DocumentStoreError, DuplicateDocumentError
 from haystack.document_stores.types import DuplicatePolicy
 from haystack.testing.document_store import DocumentStoreBaseTests
-
-from elasticsearch_haystack.document_store import ElasticsearchDocumentStore
+from haystack_integrations.document_stores.elasticsearch import ElasticsearchDocumentStore
 
 
+@pytest.mark.integration
 class TestDocumentStore(DocumentStoreBaseTests):
     """
     Common test cases will be provided by `DocumentStoreBaseTests` but
@@ -67,23 +67,23 @@ def assert_documents_are_equal(self, received: List[Document], expected: List[Do
 
         super().assert_documents_are_equal(received, expected)
 
-    @patch("elasticsearch_haystack.document_store.Elasticsearch")
+    @patch("haystack_integrations.document_stores.elasticsearch.document_store.Elasticsearch")
     def test_to_dict(self, _mock_elasticsearch_client):
         document_store = ElasticsearchDocumentStore(hosts="some hosts")
         res = document_store.to_dict()
         assert res == {
-            "type": "elasticsearch_haystack.document_store.ElasticsearchDocumentStore",
+            "type": "haystack_integrations.document_stores.elasticsearch.document_store.ElasticsearchDocumentStore",
             "init_parameters": {
                 "hosts": "some hosts",
                 "index": "default",
                 "embedding_similarity_function": "cosine",
             },
         }
 
-    @patch("elasticsearch_haystack.document_store.Elasticsearch")
+    @patch("haystack_integrations.document_stores.elasticsearch.document_store.Elasticsearch")
     def test_from_dict(self, _mock_elasticsearch_client):
         data = {
-            "type": "elasticsearch_haystack.document_store.ElasticsearchDocumentStore",
+            "type": "haystack_integrations.document_stores.elasticsearch.document_store.ElasticsearchDocumentStore",
             "init_parameters": {
                 "hosts": "some hosts",
                 "index": "default",

diff --git a/integrations/elasticsearch/tests/test_embedding_retriever.py b/integrations/elasticsearch/tests/test_embedding_retriever.py
@@ -4,9 +4,8 @@
 from unittest.mock import Mock, patch
 
 from haystack.dataclasses import Document
-
-from elasticsearch_haystack.document_store import ElasticsearchDocumentStore
-from elasticsearch_haystack.embedding_retriever import ElasticsearchEmbeddingRetriever
+from haystack_integrations.components.retrievers.elasticsearch import ElasticsearchEmbeddingRetriever
+from haystack_integrations.document_stores.elasticsearch import ElasticsearchDocumentStore
 
 
 def test_init_default():
@@ -18,21 +17,22 @@ def test_init_default():
     assert retriever._num_candidates is None
 
 
-@patch("elasticsearch_haystack.document_store.Elasticsearch")
+@patch("haystack_integrations.document_stores.elasticsearch.document_store.Elasticsearch")
 def test_to_dict(_mock_elasticsearch_client):
     document_store = ElasticsearchDocumentStore(hosts="some fake host")
     retriever = ElasticsearchEmbeddingRetriever(document_store=document_store)
     res = retriever.to_dict()
+    t = "haystack_integrations.components.retrievers.elasticsearch.embedding_retriever.ElasticsearchEmbeddingRetriever"
     assert res == {
-        "type": "elasticsearch_haystack.embedding_retriever.ElasticsearchEmbeddingRetriever",
+        "type": t,
         "init_parameters": {
             "document_store": {
                 "init_parameters": {
                     "hosts": "some fake host",
                     "index": "default",
                     "embedding_similarity_function": "cosine",
                 },
-                "type": "elasticsearch_haystack.document_store.ElasticsearchDocumentStore",
+                "type": "haystack_integrations.document_stores.elasticsearch.document_store.ElasticsearchDocumentStore",
             },
             "filters": {},
             "top_k": 10,
@@ -41,14 +41,15 @@ def test_to_dict(_mock_elasticsearch_client):
     }
 
 
-@patch("elasticsearch_haystack.document_store.Elasticsearch")
+@patch("haystack_integrations.document_stores.elasticsearch.document_store.Elasticsearch")
 def test_from_dict(_mock_elasticsearch_client):
+    t = "haystack_integrations.components.retrievers.elasticsearch.embedding_retriever.ElasticsearchEmbeddingRetriever"
     data = {
-        "type": "elasticsearch_haystack.embedding_retriever.ElasticsearchEmbeddingRetriever",
+        "type": t,
         "init_parameters": {
             "document_store": {
                 "init_parameters": {"hosts": "some fake host", "index": "default"},
-                "type": "elasticsearch_haystack.document_store.ElasticsearchDocumentStore",
+                "type": "haystack_integrations.document_stores.elasticsearch.document_store.ElasticsearchDocumentStore",
             },
             "filters": {},
             "top_k": 10,

diff --git a/integrations/elasticsearch/tests/test_filters.py b/integrations/elasticsearch/tests/test_filters.py
@@ -3,8 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 import pytest
 from haystack.errors import FilterError
-
-from elasticsearch_haystack.filters import _normalize_filters, _normalize_ranges
+from haystack_integrations.document_stores.elasticsearch.filters import _normalize_filters, _normalize_ranges
 
 filters_data = [
     (

diff --git a/integrations/gradient/pyproject.toml b/integrations/gradient/pyproject.toml
@@ -37,6 +37,9 @@ Documentation = "https://github.com/deepset-ai/haystack-core-integrations/tree/m
 Issues = "https://github.com/deepset-ai/haystack-core-integrations/issues"
 Source = "https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/gradient"
 
+[tool.hatch.build.targets.wheel]
+packages = ["src/haystack_integrations"]
+
 [tool.hatch.version]
 source = "vcs"
 tag-pattern = 'integrations\/gradient-v(?P<version>.*)'
@@ -73,7 +76,7 @@ dependencies = [
   "ruff>=0.0.243",
 ]
 [tool.hatch.envs.lint.scripts]
-typing = "mypy --install-types --non-interactive {args:src/gradient_haystack tests}"
+typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}"
 style = [
   "ruff {args:.}",
   "black --check --diff {args:.}",
@@ -139,25 +142,22 @@ unfixable = [
 ]
 
 [tool.ruff.isort]
-known-first-party = ["gradient_haystack"]
+known-first-party = ["haystack_integrations"]
 
 [tool.ruff.flake8-tidy-imports]
-ban-relative-imports = "all"
+ban-relative-imports = "parents"
 
 [tool.ruff.per-file-ignores]
 # Tests can use magic values, assertions, and relative imports
 "tests/**/*" = ["PLR2004", "S101", "TID252"]
 
 [tool.coverage.run]
-source_pkgs = ["gradient_haystack", "tests"]
+source_pkgs = ["src", "tests"]
 branch = true
 parallel = true
-omit = [
-  "src/gradient_haystack/__about__.py",
-]
 
 [tool.coverage.paths]
-gradient_haystack = ["src/gradient_haystack", "*/gradient-haystack/src/gradient_haystack"]
+gradient_haystack = ["src"]
 tests = ["tests", "*/gradient-haystack/tests"]
 
 [tool.coverage.report]
@@ -171,6 +171,7 @@ exclude_lines = [
 module = [
   "gradientai.*",
   "haystack.*",
+  "haystack_integrations.*",
   "pytest.*",
   "numpy.*",
 ]

diff --git a/integrations/gradient/src/gradient_haystack/embedders/__init__.py b/integrations/gradient/src/gradient_haystack/embedders/__init__.py
diff --git a/integrations/gradient/src/gradient_haystack/generator/__init__.py b/integrations/gradient/src/gradient_haystack/generator/__init__.py
diff --git a/integrations/gradient/src/haystack_integrations/components/embedders/gradient/__init__.py b/integrations/gradient/src/haystack_integrations/components/embedders/gradient/__init__.py
@@ -0,0 +1,7 @@
+# SPDX-FileCopyrightText: 2023-present deepset GmbH <[email protected]>
+#
+# SPDX-License-Identifier: Apache-2.0
+from .gradient_document_embedder import GradientDocumentEmbedder
+from .gradient_text_embedder import GradientTextEmbedder
+
+__all__ = ["GradientDocumentEmbedder", "GradientTextEmbedder"]
diff --git a/...k/embedders/gradient_document_embedder.py → ...rs/gradient/gradient_document_embedder.py b/...k/embedders/gradient_document_embedder.py → ...rs/gradient/gradient_document_embedder.py
diff --git a/...stack/embedders/gradient_text_embedder.py → ...edders/gradient/gradient_text_embedder.py b/...stack/embedders/gradient_text_embedder.py → ...edders/gradient/gradient_text_embedder.py
diff --git a/...radient/src/gradient_haystack/__init__.py → ...omponents/generators/gradient/__init__.py b/...radient/src/gradient_haystack/__init__.py → ...omponents/generators/gradient/__init__.py
@@ -1,3 +1,6 @@
 # SPDX-FileCopyrightText: 2023-present deepset GmbH <[email protected]>
 #
 # SPDX-License-Identifier: Apache-2.0
+from .base import GradientGenerator
+
+__all__ = ["GradientGenerator"]
diff --git a/...t/src/gradient_haystack/generator/base.py → ...ns/components/generators/gradient/base.py b/...t/src/gradient_haystack/generator/base.py → ...ns/components/generators/gradient/base.py
diff --git a/integrations/gradient/tests/test_gradient_document_embedder.py b/integrations/gradient/tests/test_gradient_document_embedder.py
@@ -5,7 +5,7 @@
 from gradientai.openapi.client.models.generate_embedding_success import GenerateEmbeddingSuccess
 from haystack import Document
 
-from gradient_haystack.embedders.gradient_document_embedder import GradientDocumentEmbedder
+from haystack_integrations.components.embedders.gradient import GradientDocumentEmbedder
 
 access_token = "access_token"
 workspace_id = "workspace_id"
@@ -52,8 +52,9 @@ def test_init_from_params_precedence(self, monkeypatch):
     def test_to_dict(self):
         component = GradientDocumentEmbedder(access_token=access_token, workspace_id=workspace_id)
         data = component.to_dict()
+        t = "haystack_integrations.components.embedders.gradient.gradient_document_embedder.GradientDocumentEmbedder"
         assert data == {
-            "type": "gradient_haystack.embedders.gradient_document_embedder.GradientDocumentEmbedder",
+            "type": t,
             "init_parameters": {"workspace_id": workspace_id, "model": "bge-large"},
         }
 

diff --git a/integrations/gradient/tests/test_gradient_rag_pipelines.py b/integrations/gradient/tests/test_gradient_rag_pipelines.py
@@ -9,9 +9,8 @@
 from haystack.components.writers import DocumentWriter
 from haystack.document_stores.in_memory import InMemoryDocumentStore
 
-from gradient_haystack.embedders.gradient_document_embedder import GradientDocumentEmbedder
-from gradient_haystack.embedders.gradient_text_embedder import GradientTextEmbedder
-from gradient_haystack.generator.base import GradientGenerator
+from haystack_integrations.components.embedders.gradient import GradientDocumentEmbedder, GradientTextEmbedder
+from haystack_integrations.components.generators.gradient import GradientGenerator
 
 
 @pytest.mark.skipif(