Merge branch 'main' into amzn_chat

deepset-ai · Feb 7, 2024 · 199bdc6 · 199bdc6
2 parents 2a62b8c + c5480af
commit 199bdc6
Show file tree

Hide file tree

Showing 12 changed files with 143 additions and 32 deletions.
diff --git a/.github/workflows/google_ai.yml b/.github/workflows/google_ai.yml
@@ -53,5 +53,9 @@ jobs:
         if: matrix.python-version == '3.9' && runner.os == 'Linux'
         run: hatch run lint:all
 
+      - name: Generate docs
+        if: matrix.python-version == '3.9' && runner.os == 'Linux'
+        run: hatch run docs
+
       - name: Run tests
         run: hatch run cov
diff --git a/.github/workflows/google_vertex.yml b/.github/workflows/google_vertex.yml
@@ -52,5 +52,9 @@ jobs:
         if: matrix.python-version == '3.9' && runner.os == 'Linux'
         run: hatch run lint:all
 
+      - name: Generate docs
+        if: matrix.python-version == '3.9' && runner.os == 'Linux'
+        run: hatch run docs
+
       - name: Run tests
         run: hatch run cov
diff --git a/.github/workflows/unstructured.yml b/.github/workflows/unstructured.yml
@@ -27,18 +27,23 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         python-version: ["3.8", "3.9", "3.10", "3.11"]
-    services:
-      unstructured-api:
-        image: "quay.io/unstructured-io/unstructured-api:latest"
-        ports:
-          - 8000:8000
-        options: >-
-          --health-cmd "curl --fail http://localhost:8000/healthcheck || exit 1"
-          --health-interval 10s
-          --health-timeout 1s
-          --health-retries 10
 
     steps:
+      - name: Free up disk space
+        run: |
+          sudo docker image prune --all --force
+
+      - name: Run Unstructured API (docker)
+        run: |
+          docker run -d \
+            --name unstructured-api \
+            -p 8000:8000 \
+            --health-cmd "curl --fail http://localhost:8000/healthcheck || exit 1" \
+            --health-interval 10s \
+            --health-timeout 1s \
+            --health-retries 10 \
+            quay.io/unstructured-io/unstructured-api:latest       
+
       - uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python-version }}

diff --git a/.github/workflows/weaviate.yml b/.github/workflows/weaviate.yml
@@ -49,5 +49,9 @@ jobs:
       - name: Run Weaviate container
         run: docker-compose up -d
 
+      - name: Generate docs
+        if: matrix.python-version == '3.9' && runner.os == 'Linux'
+        run: hatch run docs
+
       - name: Run tests
         run: hatch run cov
diff --git a/integrations/cohere/tests/test_cohere_chat_generator.py b/integrations/cohere/tests/test_cohere_chat_generator.py
@@ -3,7 +3,7 @@
 
 import cohere
 import pytest
-from haystack.components.generators.utils import default_streaming_callback
+from haystack.components.generators.utils import print_streaming_chunk
 from haystack.dataclasses import ChatMessage, ChatRole, StreamingChunk
 from haystack_integrations.components.generators.cohere import CohereChatGenerator
 
@@ -72,13 +72,13 @@ def test_init_with_parameters(self):
         component = CohereChatGenerator(
             api_key="test-api-key",
             model="command-nightly",
-            streaming_callback=default_streaming_callback,
+            streaming_callback=print_streaming_chunk,
             api_base_url="test-base-url",
             generation_kwargs={"max_tokens": 10, "some_test_param": "test-params"},
         )
         assert component.api_key == "test-api-key"
         assert component.model == "command-nightly"
-        assert component.streaming_callback is default_streaming_callback
+        assert component.streaming_callback is print_streaming_chunk
         assert component.api_base_url == "test-base-url"
         assert component.generation_kwargs == {"max_tokens": 10, "some_test_param": "test-params"}
 
@@ -101,7 +101,7 @@ def test_to_dict_with_parameters(self):
         component = CohereChatGenerator(
             api_key="test-api-key",
             model="command-nightly",
-            streaming_callback=default_streaming_callback,
+            streaming_callback=print_streaming_chunk,
             api_base_url="test-base-url",
             generation_kwargs={"max_tokens": 10, "some_test_param": "test-params"},
         )
@@ -110,7 +110,7 @@ def test_to_dict_with_parameters(self):
             "type": "haystack_integrations.components.generators.cohere.chat.chat_generator.CohereChatGenerator",
             "init_parameters": {
                 "model": "command-nightly",
-                "streaming_callback": "haystack.components.generators.utils.default_streaming_callback",
+                "streaming_callback": "haystack.components.generators.utils.print_streaming_chunk",
                 "api_base_url": "test-base-url",
                 "generation_kwargs": {"max_tokens": 10, "some_test_param": "test-params"},
             },
@@ -144,13 +144,13 @@ def test_from_dict(self, monkeypatch):
             "init_parameters": {
                 "model": "command",
                 "api_base_url": "test-base-url",
-                "streaming_callback": "haystack.components.generators.utils.default_streaming_callback",
+                "streaming_callback": "haystack.components.generators.utils.print_streaming_chunk",
                 "generation_kwargs": {"max_tokens": 10, "some_test_param": "test-params"},
             },
         }
         component = CohereChatGenerator.from_dict(data)
         assert component.model == "command"
-        assert component.streaming_callback is default_streaming_callback
+        assert component.streaming_callback is print_streaming_chunk
         assert component.api_base_url == "test-base-url"
         assert component.generation_kwargs == {"max_tokens": 10, "some_test_param": "test-params"}
 
@@ -162,7 +162,7 @@ def test_from_dict_fail_wo_env_var(self, monkeypatch):
             "init_parameters": {
                 "model": "command",
                 "api_base_url": "test-base-url",
-                "streaming_callback": "haystack.components.generators.utils.default_streaming_callback",
+                "streaming_callback": "haystack.components.generators.utils.print_streaming_chunk",
                 "generation_kwargs": {"max_tokens": 10, "some_test_param": "test-params"},
             },
         }

diff --git a/integrations/cohere/tests/test_cohere_generators.py b/integrations/cohere/tests/test_cohere_generators.py
@@ -5,19 +5,12 @@
 
 import pytest
 from cohere import COHERE_API_URL
+from haystack.components.generators.utils import print_streaming_chunk
 from haystack_integrations.components.generators.cohere import CohereGenerator
 
 pytestmark = pytest.mark.generators
 
 
-def default_streaming_callback(chunk):
-    """
-    Default callback function for streaming responses from Cohere API.
-    Prints the tokens of the first completion to stdout as soon as they are received and returns the chunk unchanged.
-    """
-    print(chunk.text, flush=True, end="")  # noqa: T201
-
-
 class TestCohereGenerator:
     def test_init_default(self):
         component = CohereGenerator(api_key="test-api-key")
@@ -61,7 +54,7 @@ def test_to_dict_with_parameters(self):
             model="command-light",
             max_tokens=10,
             some_test_param="test-params",
-            streaming_callback=default_streaming_callback,
+            streaming_callback=print_streaming_chunk,
             api_base_url="test-base-url",
         )
         data = component.to_dict()
@@ -72,7 +65,7 @@ def test_to_dict_with_parameters(self):
                 "max_tokens": 10,
                 "some_test_param": "test-params",
                 "api_base_url": "test-base-url",
-                "streaming_callback": "tests.test_cohere_generators.default_streaming_callback",
+                "streaming_callback": "haystack.components.generators.utils.print_streaming_chunk",
             },
         }
 
@@ -106,13 +99,13 @@ def test_from_dict(self, monkeypatch):
                 "max_tokens": 10,
                 "some_test_param": "test-params",
                 "api_base_url": "test-base-url",
-                "streaming_callback": "tests.test_cohere_generators.default_streaming_callback",
+                "streaming_callback": "haystack.components.generators.utils.print_streaming_chunk",
             },
         }
         component: CohereGenerator = CohereGenerator.from_dict(data)
         assert component.api_key == "test-key"
         assert component.model == "command"
-        assert component.streaming_callback == default_streaming_callback
+        assert component.streaming_callback == print_streaming_chunk
         assert component.api_base_url == "test-base-url"
         assert component.model_parameters == {"max_tokens": 10, "some_test_param": "test-params"}
 

diff --git a/integrations/google_ai/pydoc/config.yml b/integrations/google_ai/pydoc/config.yml
@@ -0,0 +1,29 @@
+loaders:
+  - type: haystack_pydoc_tools.loaders.CustomPythonLoader
+    search_path: [../src]
+    modules: [
+      "haystack_integrations.components.generators.google_ai.gemini",
+      "haystack_integrations.components.generators.google_ai.chat.gemini",
+    ]
+    ignore_when_discovered: ["__init__"]
+processors:
+  - type: filter
+    expression:
+    documented_only: true
+    do_not_filter_modules: false
+    skip_empty_modules: true
+  - type: smart
+  - type: crossref
+renderer:
+  type: haystack_pydoc_tools.renderers.ReadmePreviewRenderer
+  excerpt: Google AI integration for Haystack
+  category_slug: haystack-integrations
+  title: Google AI
+  slug: integrations-google-ai
+  order: 60
+  markdown:
+    descriptive_class_title: false
+    descriptive_module_title: true
+    add_method_class_prefix: true
+    add_member_class_prefix: false
+    filename: _readme_google_ai.md
diff --git a/integrations/google_ai/pyproject.toml b/integrations/google_ai/pyproject.toml
@@ -49,6 +49,7 @@ git_describe_command = 'git describe --tags --match="integrations/google_ai-v[0-
 dependencies = [
   "coverage[toml]>=6.5",
   "pytest",
+  "haystack-pydoc-tools",
 ]
 [tool.hatch.envs.default.scripts]
 test = "pytest {args:tests}"
@@ -61,7 +62,9 @@ cov = [
   "test-cov",
   "cov-report",
 ]
-
+docs = [
+  "pydoc-markdown pydoc/config.yml"
+]
 [[tool.hatch.envs.all.matrix]]
 python = ["3.7", "3.8", "3.9", "3.10", "3.11"]
 

diff --git a/integrations/google_vertex/pydoc/config.yml b/integrations/google_vertex/pydoc/config.yml
@@ -0,0 +1,34 @@
+loaders:
+  - type: haystack_pydoc_tools.loaders.CustomPythonLoader
+    search_path: [../src]
+    modules: [
+      "haystack_integrations.components.generators.google_vertex.gemini",
+      "haystack_integrations.components.generators.google_vertex.captioner",
+      "haystack_integrations.components.generators.google_vertex.code_generator",
+      "haystack_integrations.components.generators.google_vertex.image_generator",
+      "haystack_integrations.components.generators.google_vertex.question_answering",
+      "haystack_integrations.components.generators.google_vertex.text_generator",
+      "haystack_integrations.components.generators.google_vertex.chat.gemini",
+    ]
+    ignore_when_discovered: ["__init__"]
+processors:
+  - type: filter
+    expression:
+    documented_only: true
+    do_not_filter_modules: false
+    skip_empty_modules: true
+  - type: smart
+  - type: crossref
+renderer:
+  type: haystack_pydoc_tools.renderers.ReadmePreviewRenderer
+  excerpt: Google Vertex integration for Haystack
+  category_slug: haystack-integrations
+  title: Google Vertex
+  slug: integrations-google-vertex
+  order: 70
+  markdown:
+    descriptive_class_title: false
+    descriptive_module_title: true
+    add_method_class_prefix: true
+    add_member_class_prefix: false
+    filename: _readme_google_vertex.md
diff --git a/integrations/google_vertex/pyproject.toml b/integrations/google_vertex/pyproject.toml
@@ -48,6 +48,7 @@ git_describe_command = 'git describe --tags --match="integrations/google_vertex-
 dependencies = [
   "coverage[toml]>=6.5",
   "pytest",
+  "haystack-pydoc-tools",
 ]
 [tool.hatch.envs.default.scripts]
 test = "pytest {args:tests}"
@@ -60,7 +61,9 @@ cov = [
   "test-cov",
   "cov-report",
 ]
-
+docs = [
+  "pydoc-markdown pydoc/config.yml"
+]
 [[tool.hatch.envs.all.matrix]]
 python = ["3.7", "3.8", "3.9", "3.10", "3.11"]
 

diff --git a/integrations/weaviate/pydoc/config.yml b/integrations/weaviate/pydoc/config.yml
@@ -0,0 +1,28 @@
+loaders:
+  - type: haystack_pydoc_tools.loaders.CustomPythonLoader
+    search_path: [../src]
+    modules: [
+      "haystack_integrations.document_stores.weaviate.document_store",
+    ]
+    ignore_when_discovered: ["__init__"]
+processors:
+  - type: filter
+    expression:
+    documented_only: true
+    do_not_filter_modules: false
+    skip_empty_modules: true
+  - type: smart
+  - type: crossref
+renderer:
+  type: haystack_pydoc_tools.renderers.ReadmePreviewRenderer
+  excerpt: Weaviate integration for Haystack
+  category_slug: haystack-integrations
+  title: Weaviate
+  slug: integrations-weaviate
+  order: 180
+  markdown:
+    descriptive_class_title: false
+    descriptive_module_title: true
+    add_method_class_prefix: true
+    add_member_class_prefix: false
+    filename: _readme_weaviate.md
diff --git a/integrations/weaviate/pyproject.toml b/integrations/weaviate/pyproject.toml
@@ -27,6 +27,7 @@ classifiers = [
 dependencies = [
   "haystack-ai",
   "weaviate-client==3.*",
+  "haystack-pydoc-tools",
 ]
 
 [project.urls]
@@ -62,6 +63,9 @@ cov = [
   "test-cov",
   "cov-report",
 ]
+docs = [
+  "pydoc-markdown pydoc/config.yml"
+]
 
 [[tool.hatch.envs.all.matrix]]
 python = ["3.8", "3.9", "3.10", "3.11", "3.12"]