diff --git a/.github/workflows/google_ai.yml b/.github/workflows/google_ai.yml index 46a871a76..6093df4a4 100644 --- a/.github/workflows/google_ai.yml +++ b/.github/workflows/google_ai.yml @@ -53,5 +53,9 @@ jobs: if: matrix.python-version == '3.9' && runner.os == 'Linux' run: hatch run lint:all + - name: Generate docs + if: matrix.python-version == '3.9' && runner.os == 'Linux' + run: hatch run docs + - name: Run tests run: hatch run cov diff --git a/.github/workflows/google_vertex.yml b/.github/workflows/google_vertex.yml index cf60d3229..6f6c6d0d9 100644 --- a/.github/workflows/google_vertex.yml +++ b/.github/workflows/google_vertex.yml @@ -52,5 +52,9 @@ jobs: if: matrix.python-version == '3.9' && runner.os == 'Linux' run: hatch run lint:all + - name: Generate docs + if: matrix.python-version == '3.9' && runner.os == 'Linux' + run: hatch run docs + - name: Run tests run: hatch run cov diff --git a/.github/workflows/unstructured.yml b/.github/workflows/unstructured.yml index 6338b06e8..77ebb10ca 100644 --- a/.github/workflows/unstructured.yml +++ b/.github/workflows/unstructured.yml @@ -27,18 +27,23 @@ jobs: matrix: os: [ubuntu-latest] python-version: ["3.8", "3.9", "3.10", "3.11"] - services: - unstructured-api: - image: "quay.io/unstructured-io/unstructured-api:latest" - ports: - - 8000:8000 - options: >- - --health-cmd "curl --fail http://localhost:8000/healthcheck || exit 1" - --health-interval 10s - --health-timeout 1s - --health-retries 10 steps: + - name: Free up disk space + run: | + sudo docker image prune --all --force + + - name: Run Unstructured API (docker) + run: | + docker run -d \ + --name unstructured-api \ + -p 8000:8000 \ + --health-cmd "curl --fail http://localhost:8000/healthcheck || exit 1" \ + --health-interval 10s \ + --health-timeout 1s \ + --health-retries 10 \ + quay.io/unstructured-io/unstructured-api:latest + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} diff --git a/.github/workflows/weaviate.yml b/.github/workflows/weaviate.yml index 03cbd45a5..051415336 100644 --- a/.github/workflows/weaviate.yml +++ b/.github/workflows/weaviate.yml @@ -49,5 +49,9 @@ jobs: - name: Run Weaviate container run: docker-compose up -d + - name: Generate docs + if: matrix.python-version == '3.9' && runner.os == 'Linux' + run: hatch run docs + - name: Run tests run: hatch run cov diff --git a/integrations/cohere/tests/test_cohere_chat_generator.py b/integrations/cohere/tests/test_cohere_chat_generator.py index edefc1a43..556535e10 100644 --- a/integrations/cohere/tests/test_cohere_chat_generator.py +++ b/integrations/cohere/tests/test_cohere_chat_generator.py @@ -3,7 +3,7 @@ import cohere import pytest -from haystack.components.generators.utils import default_streaming_callback +from haystack.components.generators.utils import print_streaming_chunk from haystack.dataclasses import ChatMessage, ChatRole, StreamingChunk from haystack_integrations.components.generators.cohere import CohereChatGenerator @@ -72,13 +72,13 @@ def test_init_with_parameters(self): component = CohereChatGenerator( api_key="test-api-key", model="command-nightly", - streaming_callback=default_streaming_callback, + streaming_callback=print_streaming_chunk, api_base_url="test-base-url", generation_kwargs={"max_tokens": 10, "some_test_param": "test-params"}, ) assert component.api_key == "test-api-key" assert component.model == "command-nightly" - assert component.streaming_callback is default_streaming_callback + assert component.streaming_callback is print_streaming_chunk assert component.api_base_url == "test-base-url" assert component.generation_kwargs == {"max_tokens": 10, "some_test_param": "test-params"} @@ -101,7 +101,7 @@ def test_to_dict_with_parameters(self): component = CohereChatGenerator( api_key="test-api-key", model="command-nightly", - streaming_callback=default_streaming_callback, + streaming_callback=print_streaming_chunk, api_base_url="test-base-url", generation_kwargs={"max_tokens": 10, "some_test_param": "test-params"}, ) @@ -110,7 +110,7 @@ def test_to_dict_with_parameters(self): "type": "haystack_integrations.components.generators.cohere.chat.chat_generator.CohereChatGenerator", "init_parameters": { "model": "command-nightly", - "streaming_callback": "haystack.components.generators.utils.default_streaming_callback", + "streaming_callback": "haystack.components.generators.utils.print_streaming_chunk", "api_base_url": "test-base-url", "generation_kwargs": {"max_tokens": 10, "some_test_param": "test-params"}, }, @@ -144,13 +144,13 @@ def test_from_dict(self, monkeypatch): "init_parameters": { "model": "command", "api_base_url": "test-base-url", - "streaming_callback": "haystack.components.generators.utils.default_streaming_callback", + "streaming_callback": "haystack.components.generators.utils.print_streaming_chunk", "generation_kwargs": {"max_tokens": 10, "some_test_param": "test-params"}, }, } component = CohereChatGenerator.from_dict(data) assert component.model == "command" - assert component.streaming_callback is default_streaming_callback + assert component.streaming_callback is print_streaming_chunk assert component.api_base_url == "test-base-url" assert component.generation_kwargs == {"max_tokens": 10, "some_test_param": "test-params"} @@ -162,7 +162,7 @@ def test_from_dict_fail_wo_env_var(self, monkeypatch): "init_parameters": { "model": "command", "api_base_url": "test-base-url", - "streaming_callback": "haystack.components.generators.utils.default_streaming_callback", + "streaming_callback": "haystack.components.generators.utils.print_streaming_chunk", "generation_kwargs": {"max_tokens": 10, "some_test_param": "test-params"}, }, } diff --git a/integrations/cohere/tests/test_cohere_generators.py b/integrations/cohere/tests/test_cohere_generators.py index 90d4d3e28..5b12374a7 100644 --- a/integrations/cohere/tests/test_cohere_generators.py +++ b/integrations/cohere/tests/test_cohere_generators.py @@ -5,19 +5,12 @@ import pytest from cohere import COHERE_API_URL +from haystack.components.generators.utils import print_streaming_chunk from haystack_integrations.components.generators.cohere import CohereGenerator pytestmark = pytest.mark.generators -def default_streaming_callback(chunk): - """ - Default callback function for streaming responses from Cohere API. - Prints the tokens of the first completion to stdout as soon as they are received and returns the chunk unchanged. - """ - print(chunk.text, flush=True, end="") # noqa: T201 - - class TestCohereGenerator: def test_init_default(self): component = CohereGenerator(api_key="test-api-key") @@ -61,7 +54,7 @@ def test_to_dict_with_parameters(self): model="command-light", max_tokens=10, some_test_param="test-params", - streaming_callback=default_streaming_callback, + streaming_callback=print_streaming_chunk, api_base_url="test-base-url", ) data = component.to_dict() @@ -72,7 +65,7 @@ def test_to_dict_with_parameters(self): "max_tokens": 10, "some_test_param": "test-params", "api_base_url": "test-base-url", - "streaming_callback": "tests.test_cohere_generators.default_streaming_callback", + "streaming_callback": "haystack.components.generators.utils.print_streaming_chunk", }, } @@ -106,13 +99,13 @@ def test_from_dict(self, monkeypatch): "max_tokens": 10, "some_test_param": "test-params", "api_base_url": "test-base-url", - "streaming_callback": "tests.test_cohere_generators.default_streaming_callback", + "streaming_callback": "haystack.components.generators.utils.print_streaming_chunk", }, } component: CohereGenerator = CohereGenerator.from_dict(data) assert component.api_key == "test-key" assert component.model == "command" - assert component.streaming_callback == default_streaming_callback + assert component.streaming_callback == print_streaming_chunk assert component.api_base_url == "test-base-url" assert component.model_parameters == {"max_tokens": 10, "some_test_param": "test-params"} diff --git a/integrations/google_ai/pydoc/config.yml b/integrations/google_ai/pydoc/config.yml new file mode 100644 index 000000000..dd0706c29 --- /dev/null +++ b/integrations/google_ai/pydoc/config.yml @@ -0,0 +1,29 @@ +loaders: + - type: haystack_pydoc_tools.loaders.CustomPythonLoader + search_path: [../src] + modules: [ + "haystack_integrations.components.generators.google_ai.gemini", + "haystack_integrations.components.generators.google_ai.chat.gemini", + ] + ignore_when_discovered: ["__init__"] +processors: + - type: filter + expression: + documented_only: true + do_not_filter_modules: false + skip_empty_modules: true + - type: smart + - type: crossref +renderer: + type: haystack_pydoc_tools.renderers.ReadmePreviewRenderer + excerpt: Google AI integration for Haystack + category_slug: haystack-integrations + title: Google AI + slug: integrations-google-ai + order: 60 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: _readme_google_ai.md \ No newline at end of file diff --git a/integrations/google_ai/pyproject.toml b/integrations/google_ai/pyproject.toml index 1127dc6bf..ced2310dd 100644 --- a/integrations/google_ai/pyproject.toml +++ b/integrations/google_ai/pyproject.toml @@ -49,6 +49,7 @@ git_describe_command = 'git describe --tags --match="integrations/google_ai-v[0- dependencies = [ "coverage[toml]>=6.5", "pytest", + "haystack-pydoc-tools", ] [tool.hatch.envs.default.scripts] test = "pytest {args:tests}" @@ -61,7 +62,9 @@ cov = [ "test-cov", "cov-report", ] - +docs = [ + "pydoc-markdown pydoc/config.yml" +] [[tool.hatch.envs.all.matrix]] python = ["3.7", "3.8", "3.9", "3.10", "3.11"] diff --git a/integrations/google_vertex/pydoc/config.yml b/integrations/google_vertex/pydoc/config.yml new file mode 100644 index 000000000..86d0f3b52 --- /dev/null +++ b/integrations/google_vertex/pydoc/config.yml @@ -0,0 +1,34 @@ +loaders: + - type: haystack_pydoc_tools.loaders.CustomPythonLoader + search_path: [../src] + modules: [ + "haystack_integrations.components.generators.google_vertex.gemini", + "haystack_integrations.components.generators.google_vertex.captioner", + "haystack_integrations.components.generators.google_vertex.code_generator", + "haystack_integrations.components.generators.google_vertex.image_generator", + "haystack_integrations.components.generators.google_vertex.question_answering", + "haystack_integrations.components.generators.google_vertex.text_generator", + "haystack_integrations.components.generators.google_vertex.chat.gemini", + ] + ignore_when_discovered: ["__init__"] +processors: + - type: filter + expression: + documented_only: true + do_not_filter_modules: false + skip_empty_modules: true + - type: smart + - type: crossref +renderer: + type: haystack_pydoc_tools.renderers.ReadmePreviewRenderer + excerpt: Google Vertex integration for Haystack + category_slug: haystack-integrations + title: Google Vertex + slug: integrations-google-vertex + order: 70 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: _readme_google_vertex.md \ No newline at end of file diff --git a/integrations/google_vertex/pyproject.toml b/integrations/google_vertex/pyproject.toml index ecd509f15..f846d5bc4 100644 --- a/integrations/google_vertex/pyproject.toml +++ b/integrations/google_vertex/pyproject.toml @@ -48,6 +48,7 @@ git_describe_command = 'git describe --tags --match="integrations/google_vertex- dependencies = [ "coverage[toml]>=6.5", "pytest", + "haystack-pydoc-tools", ] [tool.hatch.envs.default.scripts] test = "pytest {args:tests}" @@ -60,7 +61,9 @@ cov = [ "test-cov", "cov-report", ] - +docs = [ + "pydoc-markdown pydoc/config.yml" +] [[tool.hatch.envs.all.matrix]] python = ["3.7", "3.8", "3.9", "3.10", "3.11"] diff --git a/integrations/weaviate/pydoc/config.yml b/integrations/weaviate/pydoc/config.yml new file mode 100644 index 000000000..fa59e6874 --- /dev/null +++ b/integrations/weaviate/pydoc/config.yml @@ -0,0 +1,28 @@ +loaders: + - type: haystack_pydoc_tools.loaders.CustomPythonLoader + search_path: [../src] + modules: [ + "haystack_integrations.document_stores.weaviate.document_store", + ] + ignore_when_discovered: ["__init__"] +processors: + - type: filter + expression: + documented_only: true + do_not_filter_modules: false + skip_empty_modules: true + - type: smart + - type: crossref +renderer: + type: haystack_pydoc_tools.renderers.ReadmePreviewRenderer + excerpt: Weaviate integration for Haystack + category_slug: haystack-integrations + title: Weaviate + slug: integrations-weaviate + order: 180 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: _readme_weaviate.md diff --git a/integrations/weaviate/pyproject.toml b/integrations/weaviate/pyproject.toml index 50f1c157c..fb132516c 100644 --- a/integrations/weaviate/pyproject.toml +++ b/integrations/weaviate/pyproject.toml @@ -27,6 +27,7 @@ classifiers = [ dependencies = [ "haystack-ai", "weaviate-client==3.*", + "haystack-pydoc-tools", ] [project.urls] @@ -62,6 +63,9 @@ cov = [ "test-cov", "cov-report", ] +docs = [ + "pydoc-markdown pydoc/config.yml" +] [[tool.hatch.envs.all.matrix]] python = ["3.8", "3.9", "3.10", "3.11", "3.12"]