From f8efcfe3126fd4066e708872386c06716f5415a4 Mon Sep 17 00:00:00 2001 From: Vladimir Blagojevic Date: Mon, 22 Jan 2024 10:59:37 +0100 Subject: [PATCH] chore: Cohere namespace change (#247) * Reorganize cohere integration project layout, adjust pyproject.toml * Update tests * Fix for tests * Remove workspace file * Remove file * Fix lint issues * More linting fixes * Add cohere suffix dirs * Remove idea folder --- integrations/cohere/pyproject.toml | 19 ++++++++++--------- .../components/embedders/cohere/__init__.py | 7 +++++++ .../embedders/cohere}/document_embedder.py | 4 ++-- .../embedders/cohere}/text_embedder.py | 4 ++-- .../components/embedders/cohere}/utils.py | 3 ++- .../components/generators/cohere}/__init__.py | 3 +++ .../generators/cohere/chat}/__init__.py | 3 +++ .../generators/cohere}/chat/chat_generator.py | 0 .../generators/cohere}/generator.py | 3 ++- .../tests/test_cohere_chat_generator.py | 13 ++++++------- .../cohere/tests/test_cohere_generators.py | 11 +++++------ .../cohere/tests/test_document_embedder.py | 7 +++---- .../cohere/tests/test_text_embedder.py | 7 +++---- 13 files changed, 48 insertions(+), 36 deletions(-) create mode 100644 integrations/cohere/src/haystack_integrations/components/embedders/cohere/__init__.py rename integrations/cohere/src/{cohere_haystack/embedders => haystack_integrations/components/embedders/cohere}/document_embedder.py (98%) rename integrations/cohere/src/{cohere_haystack/embedders => haystack_integrations/components/embedders/cohere}/text_embedder.py (98%) rename integrations/cohere/src/{cohere_haystack/embedders => haystack_integrations/components/embedders/cohere}/utils.py (99%) rename integrations/cohere/src/{cohere_haystack => haystack_integrations/components/generators/cohere}/__init__.py (61%) rename integrations/cohere/src/{cohere_haystack/embedders => haystack_integrations/components/generators/cohere/chat}/__init__.py (56%) rename integrations/cohere/src/{cohere_haystack => haystack_integrations/components/generators/cohere}/chat/chat_generator.py (100%) rename integrations/cohere/src/{cohere_haystack => haystack_integrations/components/generators/cohere}/generator.py (99%) diff --git a/integrations/cohere/pyproject.toml b/integrations/cohere/pyproject.toml index ae96f114d..42349d9fb 100644 --- a/integrations/cohere/pyproject.toml +++ b/integrations/cohere/pyproject.toml @@ -34,6 +34,9 @@ Documentation = "https://github.com/deepset-ai/haystack-core-integrations/tree/m Issues = "https://github.com/deepset-ai/haystack-core-integrations/issues" Source = "https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/cohere" +[tool.hatch.build.targets.wheel] +packages = ["src/haystack_integrations"] + [tool.hatch.version] source = "vcs" tag-pattern = 'integrations\/cohere-v(?P.*)' @@ -70,7 +73,7 @@ dependencies = [ "ruff>=0.0.243", ] [tool.hatch.envs.lint.scripts] -typing = "mypy --install-types --non-interactive {args:src/cohere_haystack tests}" +typing = "mypy --install-types --non-interactive --explicit-package-bases {args:src/ tests}" style = [ "ruff {args:.}", "black --check --diff {args:.}", @@ -133,26 +136,23 @@ unfixable = [ ] [tool.ruff.isort] -known-first-party = ["cohere_haystack"] +known-first-party = ["src"] [tool.ruff.flake8-tidy-imports] -ban-relative-imports = "all" +ban-relative-imports = "parents" [tool.ruff.per-file-ignores] # Tests can use magic values, assertions, and relative imports "tests/**/*" = ["PLR2004", "S101", "TID252"] [tool.coverage.run] -source_pkgs = ["cohere_haystack", "tests"] +source_pkgs = ["src", "tests"] branch = true parallel = true -omit = [ - "src/cohere_haystack/__about__.py", -] [tool.coverage.paths] -cohere_haystack = ["src/cohere_haystack", "*/cohere-haystack/src/cohere_haystack"] -tests = ["tests", "*/cohere-haystack/tests"] +cohere_haystack = ["src/haystack_integrations", "*/cohere/src/haystack_integrations"] +tests = ["tests", "*/cohere/tests"] [tool.coverage.report] exclude_lines = [ @@ -165,6 +165,7 @@ exclude_lines = [ module = [ "cohere.*", "haystack.*", + "haystack_integrations.*", "pytest.*", "numpy.*", ] diff --git a/integrations/cohere/src/haystack_integrations/components/embedders/cohere/__init__.py b/integrations/cohere/src/haystack_integrations/components/embedders/cohere/__init__.py new file mode 100644 index 000000000..73a863a73 --- /dev/null +++ b/integrations/cohere/src/haystack_integrations/components/embedders/cohere/__init__.py @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: 2023-present deepset GmbH +# +# SPDX-License-Identifier: Apache-2.0 +from .document_embedder import CohereDocumentEmbedder +from .text_embedder import CohereTextEmbedder + +__all__ = ["CohereDocumentEmbedder", "CohereTextEmbedder"] diff --git a/integrations/cohere/src/cohere_haystack/embedders/document_embedder.py b/integrations/cohere/src/haystack_integrations/components/embedders/cohere/document_embedder.py similarity index 98% rename from integrations/cohere/src/cohere_haystack/embedders/document_embedder.py rename to integrations/cohere/src/haystack_integrations/components/embedders/cohere/document_embedder.py index 151c4f794..69308ad19 100644 --- a/integrations/cohere/src/cohere_haystack/embedders/document_embedder.py +++ b/integrations/cohere/src/haystack_integrations/components/embedders/cohere/document_embedder.py @@ -5,10 +5,10 @@ import os from typing import Any, Dict, List, Optional -from cohere import COHERE_API_URL, AsyncClient, Client from haystack import Document, component, default_to_dict +from haystack_integrations.components.embedders.cohere.utils import get_async_response, get_response -from cohere_haystack.embedders.utils import get_async_response, get_response +from cohere import COHERE_API_URL, AsyncClient, Client @component diff --git a/integrations/cohere/src/cohere_haystack/embedders/text_embedder.py b/integrations/cohere/src/haystack_integrations/components/embedders/cohere/text_embedder.py similarity index 98% rename from integrations/cohere/src/cohere_haystack/embedders/text_embedder.py rename to integrations/cohere/src/haystack_integrations/components/embedders/cohere/text_embedder.py index bfef97dc3..2fa922004 100644 --- a/integrations/cohere/src/cohere_haystack/embedders/text_embedder.py +++ b/integrations/cohere/src/haystack_integrations/components/embedders/cohere/text_embedder.py @@ -5,10 +5,10 @@ import os from typing import Any, Dict, List, Optional -from cohere import COHERE_API_URL, AsyncClient, Client from haystack import component, default_to_dict +from haystack_integrations.components.embedders.cohere.utils import get_async_response, get_response -from cohere_haystack.embedders.utils import get_async_response, get_response +from cohere import COHERE_API_URL, AsyncClient, Client @component diff --git a/integrations/cohere/src/cohere_haystack/embedders/utils.py b/integrations/cohere/src/haystack_integrations/components/embedders/cohere/utils.py similarity index 99% rename from integrations/cohere/src/cohere_haystack/embedders/utils.py rename to integrations/cohere/src/haystack_integrations/components/embedders/cohere/utils.py index 1c1049852..7b9c90730 100644 --- a/integrations/cohere/src/cohere_haystack/embedders/utils.py +++ b/integrations/cohere/src/haystack_integrations/components/embedders/cohere/utils.py @@ -3,9 +3,10 @@ # SPDX-License-Identifier: Apache-2.0 from typing import Any, Dict, List, Tuple -from cohere import AsyncClient, Client, CohereError from tqdm import tqdm +from cohere import AsyncClient, Client, CohereError + async def get_async_response(cohere_async_client: AsyncClient, texts: List[str], model_name, input_type, truncate): all_embeddings: List[List[float]] = [] diff --git a/integrations/cohere/src/cohere_haystack/__init__.py b/integrations/cohere/src/haystack_integrations/components/generators/cohere/__init__.py similarity index 61% rename from integrations/cohere/src/cohere_haystack/__init__.py rename to integrations/cohere/src/haystack_integrations/components/generators/cohere/__init__.py index e873bc332..c36f982df 100644 --- a/integrations/cohere/src/cohere_haystack/__init__.py +++ b/integrations/cohere/src/haystack_integrations/components/generators/cohere/__init__.py @@ -1,3 +1,6 @@ # SPDX-FileCopyrightText: 2023-present deepset GmbH # # SPDX-License-Identifier: Apache-2.0 +from .generator import CohereGenerator + +__all__ = ["CohereGenerator"] diff --git a/integrations/cohere/src/cohere_haystack/embedders/__init__.py b/integrations/cohere/src/haystack_integrations/components/generators/cohere/chat/__init__.py similarity index 56% rename from integrations/cohere/src/cohere_haystack/embedders/__init__.py rename to integrations/cohere/src/haystack_integrations/components/generators/cohere/chat/__init__.py index e873bc332..dc14c9c1c 100644 --- a/integrations/cohere/src/cohere_haystack/embedders/__init__.py +++ b/integrations/cohere/src/haystack_integrations/components/generators/cohere/chat/__init__.py @@ -1,3 +1,6 @@ # SPDX-FileCopyrightText: 2023-present deepset GmbH # # SPDX-License-Identifier: Apache-2.0 +from .chat_generator import CohereChatGenerator + +__all__ = ["CohereChatGenerator"] diff --git a/integrations/cohere/src/cohere_haystack/chat/chat_generator.py b/integrations/cohere/src/haystack_integrations/components/generators/cohere/chat/chat_generator.py similarity index 100% rename from integrations/cohere/src/cohere_haystack/chat/chat_generator.py rename to integrations/cohere/src/haystack_integrations/components/generators/cohere/chat/chat_generator.py diff --git a/integrations/cohere/src/cohere_haystack/generator.py b/integrations/cohere/src/haystack_integrations/components/generators/cohere/generator.py similarity index 99% rename from integrations/cohere/src/cohere_haystack/generator.py rename to integrations/cohere/src/haystack_integrations/components/generators/cohere/generator.py index 9917f17ea..7bca3ed9f 100644 --- a/integrations/cohere/src/cohere_haystack/generator.py +++ b/integrations/cohere/src/haystack_integrations/components/generators/cohere/generator.py @@ -6,9 +6,10 @@ import sys from typing import Any, Callable, Dict, List, Optional, cast +from haystack import DeserializationError, component, default_from_dict, default_to_dict + from cohere import COHERE_API_URL, Client from cohere.responses import Generations -from haystack import DeserializationError, component, default_from_dict, default_to_dict logger = logging.getLogger(__name__) diff --git a/integrations/cohere/tests/test_cohere_chat_generator.py b/integrations/cohere/tests/test_cohere_chat_generator.py index e93db51fd..cc360f5c9 100644 --- a/integrations/cohere/tests/test_cohere_chat_generator.py +++ b/integrations/cohere/tests/test_cohere_chat_generator.py @@ -5,8 +5,7 @@ import pytest from haystack.components.generators.utils import default_streaming_callback from haystack.dataclasses import ChatMessage, ChatRole, StreamingChunk - -from cohere_haystack.chat.chat_generator import CohereChatGenerator +from haystack_integrations.components.generators.cohere.chat import CohereChatGenerator pytestmark = pytest.mark.chat_generators @@ -88,7 +87,7 @@ def test_to_dict_default(self): component = CohereChatGenerator(api_key="test-api-key") data = component.to_dict() assert data == { - "type": "cohere_haystack.chat.chat_generator.CohereChatGenerator", + "type": "haystack_integrations.components.generators.cohere.chat.chat_generator.CohereChatGenerator", "init_parameters": { "model": "command", "streaming_callback": None, @@ -108,7 +107,7 @@ def test_to_dict_with_parameters(self): ) data = component.to_dict() assert data == { - "type": "cohere_haystack.chat.chat_generator.CohereChatGenerator", + "type": "haystack_integrations.components.generators.cohere.chat.chat_generator.CohereChatGenerator", "init_parameters": { "model": "command-nightly", "streaming_callback": "haystack.components.generators.utils.default_streaming_callback", @@ -128,7 +127,7 @@ def test_to_dict_with_lambda_streaming_callback(self): ) data = component.to_dict() assert data == { - "type": "cohere_haystack.chat.chat_generator.CohereChatGenerator", + "type": "haystack_integrations.components.generators.cohere.chat.chat_generator.CohereChatGenerator", "init_parameters": { "model": "command", "api_base_url": "test-base-url", @@ -141,7 +140,7 @@ def test_to_dict_with_lambda_streaming_callback(self): def test_from_dict(self, monkeypatch): monkeypatch.setenv("COHERE_API_KEY", "fake-api-key") data = { - "type": "cohere_haystack.chat.chat_generator.CohereChatGenerator", + "type": "haystack_integrations.components.generators.cohere.chat.chat_generator.CohereChatGenerator", "init_parameters": { "model": "command", "api_base_url": "test-base-url", @@ -159,7 +158,7 @@ def test_from_dict(self, monkeypatch): def test_from_dict_fail_wo_env_var(self, monkeypatch): monkeypatch.delenv("COHERE_API_KEY", raising=False) data = { - "type": "cohere_haystack.chat.chat_generator.CohereChatGenerator", + "type": "haystack_integrations.components.generators.cohere.chat.chat_generator.CohereChatGenerator", "init_parameters": { "model": "command", "api_base_url": "test-base-url", diff --git a/integrations/cohere/tests/test_cohere_generators.py b/integrations/cohere/tests/test_cohere_generators.py index f22b38843..e2ce10405 100644 --- a/integrations/cohere/tests/test_cohere_generators.py +++ b/integrations/cohere/tests/test_cohere_generators.py @@ -5,8 +5,7 @@ import pytest from cohere import COHERE_API_URL - -from cohere_haystack.generator import CohereGenerator +from haystack_integrations.components.generators.cohere import CohereGenerator pytestmark = pytest.mark.generators @@ -48,7 +47,7 @@ def test_to_dict_default(self): component = CohereGenerator(api_key="test-api-key") data = component.to_dict() assert data == { - "type": "cohere_haystack.generator.CohereGenerator", + "type": "haystack_integrations.components.generators.cohere.generator.CohereGenerator", "init_parameters": { "model": "command", "streaming_callback": None, @@ -67,7 +66,7 @@ def test_to_dict_with_parameters(self): ) data = component.to_dict() assert data == { - "type": "cohere_haystack.generator.CohereGenerator", + "type": "haystack_integrations.components.generators.cohere.generator.CohereGenerator", "init_parameters": { "model": "command-light", "max_tokens": 10, @@ -88,7 +87,7 @@ def test_to_dict_with_lambda_streaming_callback(self): ) data = component.to_dict() assert data == { - "type": "cohere_haystack.generator.CohereGenerator", + "type": "haystack_integrations.components.generators.cohere.generator.CohereGenerator", "init_parameters": { "model": "command", "streaming_callback": "tests.test_cohere_generators.", @@ -101,7 +100,7 @@ def test_to_dict_with_lambda_streaming_callback(self): def test_from_dict(self, monkeypatch): monkeypatch.setenv("COHERE_API_KEY", "test-key") data = { - "type": "cohere_haystack.generator.CohereGenerator", + "type": "haystack_integrations.components.generators.cohere.generator.CohereGenerator", "init_parameters": { "model": "command", "max_tokens": 10, diff --git a/integrations/cohere/tests/test_document_embedder.py b/integrations/cohere/tests/test_document_embedder.py index c9770737e..efe8eb36a 100644 --- a/integrations/cohere/tests/test_document_embedder.py +++ b/integrations/cohere/tests/test_document_embedder.py @@ -6,8 +6,7 @@ import pytest from cohere import COHERE_API_URL from haystack import Document - -from cohere_haystack.embedders.document_embedder import CohereDocumentEmbedder +from haystack_integrations.components.embedders.cohere import CohereDocumentEmbedder pytestmark = pytest.mark.embedders @@ -60,7 +59,7 @@ def test_to_dict(self): embedder_component = CohereDocumentEmbedder(api_key="test-api-key") component_dict = embedder_component.to_dict() assert component_dict == { - "type": "cohere_haystack.embedders.document_embedder.CohereDocumentEmbedder", + "type": "haystack_integrations.components.embedders.cohere.document_embedder.CohereDocumentEmbedder", "init_parameters": { "model": "embed-english-v2.0", "input_type": "search_document", @@ -93,7 +92,7 @@ def test_to_dict_with_custom_init_parameters(self): ) component_dict = embedder_component.to_dict() assert component_dict == { - "type": "cohere_haystack.embedders.document_embedder.CohereDocumentEmbedder", + "type": "haystack_integrations.components.embedders.cohere.document_embedder.CohereDocumentEmbedder", "init_parameters": { "model": "embed-multilingual-v2.0", "input_type": "search_query", diff --git a/integrations/cohere/tests/test_text_embedder.py b/integrations/cohere/tests/test_text_embedder.py index 7e91b4812..657d8df83 100644 --- a/integrations/cohere/tests/test_text_embedder.py +++ b/integrations/cohere/tests/test_text_embedder.py @@ -5,8 +5,7 @@ import pytest from cohere import COHERE_API_URL - -from cohere_haystack.embedders.text_embedder import CohereTextEmbedder +from haystack_integrations.components.embedders.cohere import CohereTextEmbedder pytestmark = pytest.mark.embedders @@ -57,7 +56,7 @@ def test_to_dict(self): embedder_component = CohereTextEmbedder(api_key="test-api-key") component_dict = embedder_component.to_dict() assert component_dict == { - "type": "cohere_haystack.embedders.text_embedder.CohereTextEmbedder", + "type": "haystack_integrations.components.embedders.cohere.text_embedder.CohereTextEmbedder", "init_parameters": { "model": "embed-english-v2.0", "input_type": "search_query", @@ -85,7 +84,7 @@ def test_to_dict_with_custom_init_parameters(self): ) component_dict = embedder_component.to_dict() assert component_dict == { - "type": "cohere_haystack.embedders.text_embedder.CohereTextEmbedder", + "type": "haystack_integrations.components.embedders.cohere.text_embedder.CohereTextEmbedder", "init_parameters": { "model": "embed-multilingual-v2.0", "input_type": "classification",