From 30164b8f759f1610b4002b45af2acf2ebcadbc0f Mon Sep 17 00:00:00 2001 From: Vladimir Blagojevic Date: Wed, 7 Feb 2024 13:11:01 +0100 Subject: [PATCH 1/5] feat: Generate weaviate API docs (#351) * Generate weaviate API docs * update order --------- Co-authored-by: Daria Fokina --- .github/workflows/weaviate.yml | 4 ++++ integrations/weaviate/pydoc/config.yml | 28 ++++++++++++++++++++++++++ integrations/weaviate/pyproject.toml | 4 ++++ 3 files changed, 36 insertions(+) create mode 100644 integrations/weaviate/pydoc/config.yml diff --git a/.github/workflows/weaviate.yml b/.github/workflows/weaviate.yml index 03cbd45a5..051415336 100644 --- a/.github/workflows/weaviate.yml +++ b/.github/workflows/weaviate.yml @@ -49,5 +49,9 @@ jobs: - name: Run Weaviate container run: docker-compose up -d + - name: Generate docs + if: matrix.python-version == '3.9' && runner.os == 'Linux' + run: hatch run docs + - name: Run tests run: hatch run cov diff --git a/integrations/weaviate/pydoc/config.yml b/integrations/weaviate/pydoc/config.yml new file mode 100644 index 000000000..fa59e6874 --- /dev/null +++ b/integrations/weaviate/pydoc/config.yml @@ -0,0 +1,28 @@ +loaders: + - type: haystack_pydoc_tools.loaders.CustomPythonLoader + search_path: [../src] + modules: [ + "haystack_integrations.document_stores.weaviate.document_store", + ] + ignore_when_discovered: ["__init__"] +processors: + - type: filter + expression: + documented_only: true + do_not_filter_modules: false + skip_empty_modules: true + - type: smart + - type: crossref +renderer: + type: haystack_pydoc_tools.renderers.ReadmePreviewRenderer + excerpt: Weaviate integration for Haystack + category_slug: haystack-integrations + title: Weaviate + slug: integrations-weaviate + order: 180 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: _readme_weaviate.md diff --git a/integrations/weaviate/pyproject.toml b/integrations/weaviate/pyproject.toml index 50f1c157c..fb132516c 100644 --- a/integrations/weaviate/pyproject.toml +++ b/integrations/weaviate/pyproject.toml @@ -27,6 +27,7 @@ classifiers = [ dependencies = [ "haystack-ai", "weaviate-client==3.*", + "haystack-pydoc-tools", ] [project.urls] @@ -62,6 +63,9 @@ cov = [ "test-cov", "cov-report", ] +docs = [ + "pydoc-markdown pydoc/config.yml" +] [[tool.hatch.envs.all.matrix]] python = ["3.8", "3.9", "3.10", "3.11", "3.12"] From 6365baeedce47031dba7ff727486518d8846efcd Mon Sep 17 00:00:00 2001 From: Massimiliano Pippi Date: Wed, 7 Feb 2024 14:59:52 +0100 Subject: [PATCH 2/5] update to latest haystack-ai version (#348) --- .../cohere/tests/test_cohere_chat_generator.py | 16 ++++++++-------- .../cohere/tests/test_cohere_generators.py | 17 +++++------------ 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/integrations/cohere/tests/test_cohere_chat_generator.py b/integrations/cohere/tests/test_cohere_chat_generator.py index edefc1a43..556535e10 100644 --- a/integrations/cohere/tests/test_cohere_chat_generator.py +++ b/integrations/cohere/tests/test_cohere_chat_generator.py @@ -3,7 +3,7 @@ import cohere import pytest -from haystack.components.generators.utils import default_streaming_callback +from haystack.components.generators.utils import print_streaming_chunk from haystack.dataclasses import ChatMessage, ChatRole, StreamingChunk from haystack_integrations.components.generators.cohere import CohereChatGenerator @@ -72,13 +72,13 @@ def test_init_with_parameters(self): component = CohereChatGenerator( api_key="test-api-key", model="command-nightly", - streaming_callback=default_streaming_callback, + streaming_callback=print_streaming_chunk, api_base_url="test-base-url", generation_kwargs={"max_tokens": 10, "some_test_param": "test-params"}, ) assert component.api_key == "test-api-key" assert component.model == "command-nightly" - assert component.streaming_callback is default_streaming_callback + assert component.streaming_callback is print_streaming_chunk assert component.api_base_url == "test-base-url" assert component.generation_kwargs == {"max_tokens": 10, "some_test_param": "test-params"} @@ -101,7 +101,7 @@ def test_to_dict_with_parameters(self): component = CohereChatGenerator( api_key="test-api-key", model="command-nightly", - streaming_callback=default_streaming_callback, + streaming_callback=print_streaming_chunk, api_base_url="test-base-url", generation_kwargs={"max_tokens": 10, "some_test_param": "test-params"}, ) @@ -110,7 +110,7 @@ def test_to_dict_with_parameters(self): "type": "haystack_integrations.components.generators.cohere.chat.chat_generator.CohereChatGenerator", "init_parameters": { "model": "command-nightly", - "streaming_callback": "haystack.components.generators.utils.default_streaming_callback", + "streaming_callback": "haystack.components.generators.utils.print_streaming_chunk", "api_base_url": "test-base-url", "generation_kwargs": {"max_tokens": 10, "some_test_param": "test-params"}, }, @@ -144,13 +144,13 @@ def test_from_dict(self, monkeypatch): "init_parameters": { "model": "command", "api_base_url": "test-base-url", - "streaming_callback": "haystack.components.generators.utils.default_streaming_callback", + "streaming_callback": "haystack.components.generators.utils.print_streaming_chunk", "generation_kwargs": {"max_tokens": 10, "some_test_param": "test-params"}, }, } component = CohereChatGenerator.from_dict(data) assert component.model == "command" - assert component.streaming_callback is default_streaming_callback + assert component.streaming_callback is print_streaming_chunk assert component.api_base_url == "test-base-url" assert component.generation_kwargs == {"max_tokens": 10, "some_test_param": "test-params"} @@ -162,7 +162,7 @@ def test_from_dict_fail_wo_env_var(self, monkeypatch): "init_parameters": { "model": "command", "api_base_url": "test-base-url", - "streaming_callback": "haystack.components.generators.utils.default_streaming_callback", + "streaming_callback": "haystack.components.generators.utils.print_streaming_chunk", "generation_kwargs": {"max_tokens": 10, "some_test_param": "test-params"}, }, } diff --git a/integrations/cohere/tests/test_cohere_generators.py b/integrations/cohere/tests/test_cohere_generators.py index 90d4d3e28..5b12374a7 100644 --- a/integrations/cohere/tests/test_cohere_generators.py +++ b/integrations/cohere/tests/test_cohere_generators.py @@ -5,19 +5,12 @@ import pytest from cohere import COHERE_API_URL +from haystack.components.generators.utils import print_streaming_chunk from haystack_integrations.components.generators.cohere import CohereGenerator pytestmark = pytest.mark.generators -def default_streaming_callback(chunk): - """ - Default callback function for streaming responses from Cohere API. - Prints the tokens of the first completion to stdout as soon as they are received and returns the chunk unchanged. - """ - print(chunk.text, flush=True, end="") # noqa: T201 - - class TestCohereGenerator: def test_init_default(self): component = CohereGenerator(api_key="test-api-key") @@ -61,7 +54,7 @@ def test_to_dict_with_parameters(self): model="command-light", max_tokens=10, some_test_param="test-params", - streaming_callback=default_streaming_callback, + streaming_callback=print_streaming_chunk, api_base_url="test-base-url", ) data = component.to_dict() @@ -72,7 +65,7 @@ def test_to_dict_with_parameters(self): "max_tokens": 10, "some_test_param": "test-params", "api_base_url": "test-base-url", - "streaming_callback": "tests.test_cohere_generators.default_streaming_callback", + "streaming_callback": "haystack.components.generators.utils.print_streaming_chunk", }, } @@ -106,13 +99,13 @@ def test_from_dict(self, monkeypatch): "max_tokens": 10, "some_test_param": "test-params", "api_base_url": "test-base-url", - "streaming_callback": "tests.test_cohere_generators.default_streaming_callback", + "streaming_callback": "haystack.components.generators.utils.print_streaming_chunk", }, } component: CohereGenerator = CohereGenerator.from_dict(data) assert component.api_key == "test-key" assert component.model == "command" - assert component.streaming_callback == default_streaming_callback + assert component.streaming_callback == print_streaming_chunk assert component.api_base_url == "test-base-url" assert component.model_parameters == {"max_tokens": 10, "some_test_param": "test-params"} From f86c758f8dedd09f2dfc3f0fd126691881b4aacb Mon Sep 17 00:00:00 2001 From: Daria Fokina Date: Wed, 7 Feb 2024 17:37:57 +0100 Subject: [PATCH 3/5] google_vertex: create api docs (#355) --- .github/workflows/google_vertex.yml | 4 +++ integrations/google_vertex/pydoc/config.yml | 34 +++++++++++++++++++++ integrations/google_vertex/pyproject.toml | 5 ++- 3 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 integrations/google_vertex/pydoc/config.yml diff --git a/.github/workflows/google_vertex.yml b/.github/workflows/google_vertex.yml index cf60d3229..6f6c6d0d9 100644 --- a/.github/workflows/google_vertex.yml +++ b/.github/workflows/google_vertex.yml @@ -52,5 +52,9 @@ jobs: if: matrix.python-version == '3.9' && runner.os == 'Linux' run: hatch run lint:all + - name: Generate docs + if: matrix.python-version == '3.9' && runner.os == 'Linux' + run: hatch run docs + - name: Run tests run: hatch run cov diff --git a/integrations/google_vertex/pydoc/config.yml b/integrations/google_vertex/pydoc/config.yml new file mode 100644 index 000000000..86d0f3b52 --- /dev/null +++ b/integrations/google_vertex/pydoc/config.yml @@ -0,0 +1,34 @@ +loaders: + - type: haystack_pydoc_tools.loaders.CustomPythonLoader + search_path: [../src] + modules: [ + "haystack_integrations.components.generators.google_vertex.gemini", + "haystack_integrations.components.generators.google_vertex.captioner", + "haystack_integrations.components.generators.google_vertex.code_generator", + "haystack_integrations.components.generators.google_vertex.image_generator", + "haystack_integrations.components.generators.google_vertex.question_answering", + "haystack_integrations.components.generators.google_vertex.text_generator", + "haystack_integrations.components.generators.google_vertex.chat.gemini", + ] + ignore_when_discovered: ["__init__"] +processors: + - type: filter + expression: + documented_only: true + do_not_filter_modules: false + skip_empty_modules: true + - type: smart + - type: crossref +renderer: + type: haystack_pydoc_tools.renderers.ReadmePreviewRenderer + excerpt: Google Vertex integration for Haystack + category_slug: haystack-integrations + title: Google Vertex + slug: integrations-google-vertex + order: 70 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: _readme_google_vertex.md \ No newline at end of file diff --git a/integrations/google_vertex/pyproject.toml b/integrations/google_vertex/pyproject.toml index ecd509f15..f846d5bc4 100644 --- a/integrations/google_vertex/pyproject.toml +++ b/integrations/google_vertex/pyproject.toml @@ -48,6 +48,7 @@ git_describe_command = 'git describe --tags --match="integrations/google_vertex- dependencies = [ "coverage[toml]>=6.5", "pytest", + "haystack-pydoc-tools", ] [tool.hatch.envs.default.scripts] test = "pytest {args:tests}" @@ -60,7 +61,9 @@ cov = [ "test-cov", "cov-report", ] - +docs = [ + "pydoc-markdown pydoc/config.yml" +] [[tool.hatch.envs.all.matrix]] python = ["3.7", "3.8", "3.9", "3.10", "3.11"] From 6b099b622c12e9c4cc399110421dbdd3a3afaccc Mon Sep 17 00:00:00 2001 From: Daria Fokina Date: Wed, 7 Feb 2024 17:38:11 +0100 Subject: [PATCH 4/5] google_ai: create api docs (#354) --- .github/workflows/google_ai.yml | 4 ++++ integrations/google_ai/pydoc/config.yml | 29 +++++++++++++++++++++++++ integrations/google_ai/pyproject.toml | 5 ++++- 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 integrations/google_ai/pydoc/config.yml diff --git a/.github/workflows/google_ai.yml b/.github/workflows/google_ai.yml index 46a871a76..6093df4a4 100644 --- a/.github/workflows/google_ai.yml +++ b/.github/workflows/google_ai.yml @@ -53,5 +53,9 @@ jobs: if: matrix.python-version == '3.9' && runner.os == 'Linux' run: hatch run lint:all + - name: Generate docs + if: matrix.python-version == '3.9' && runner.os == 'Linux' + run: hatch run docs + - name: Run tests run: hatch run cov diff --git a/integrations/google_ai/pydoc/config.yml b/integrations/google_ai/pydoc/config.yml new file mode 100644 index 000000000..dd0706c29 --- /dev/null +++ b/integrations/google_ai/pydoc/config.yml @@ -0,0 +1,29 @@ +loaders: + - type: haystack_pydoc_tools.loaders.CustomPythonLoader + search_path: [../src] + modules: [ + "haystack_integrations.components.generators.google_ai.gemini", + "haystack_integrations.components.generators.google_ai.chat.gemini", + ] + ignore_when_discovered: ["__init__"] +processors: + - type: filter + expression: + documented_only: true + do_not_filter_modules: false + skip_empty_modules: true + - type: smart + - type: crossref +renderer: + type: haystack_pydoc_tools.renderers.ReadmePreviewRenderer + excerpt: Google AI integration for Haystack + category_slug: haystack-integrations + title: Google AI + slug: integrations-google-ai + order: 60 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: _readme_google_ai.md \ No newline at end of file diff --git a/integrations/google_ai/pyproject.toml b/integrations/google_ai/pyproject.toml index 1127dc6bf..ced2310dd 100644 --- a/integrations/google_ai/pyproject.toml +++ b/integrations/google_ai/pyproject.toml @@ -49,6 +49,7 @@ git_describe_command = 'git describe --tags --match="integrations/google_ai-v[0- dependencies = [ "coverage[toml]>=6.5", "pytest", + "haystack-pydoc-tools", ] [tool.hatch.envs.default.scripts] test = "pytest {args:tests}" @@ -61,7 +62,9 @@ cov = [ "test-cov", "cov-report", ] - +docs = [ + "pydoc-markdown pydoc/config.yml" +] [[tool.hatch.envs.all.matrix]] python = ["3.7", "3.8", "3.9", "3.10", "3.11"] From c5480af3012aab44d8eab6ddb8cb54fce1cdc917 Mon Sep 17 00:00:00 2001 From: Stefano Fiorucci Date: Wed, 7 Feb 2024 20:44:52 +0100 Subject: [PATCH 5/5] try run docker container as a step in workflow (#349) --- .github/workflows/unstructured.yml | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/workflows/unstructured.yml b/.github/workflows/unstructured.yml index 6338b06e8..77ebb10ca 100644 --- a/.github/workflows/unstructured.yml +++ b/.github/workflows/unstructured.yml @@ -27,18 +27,23 @@ jobs: matrix: os: [ubuntu-latest] python-version: ["3.8", "3.9", "3.10", "3.11"] - services: - unstructured-api: - image: "quay.io/unstructured-io/unstructured-api:latest" - ports: - - 8000:8000 - options: >- - --health-cmd "curl --fail http://localhost:8000/healthcheck || exit 1" - --health-interval 10s - --health-timeout 1s - --health-retries 10 steps: + - name: Free up disk space + run: | + sudo docker image prune --all --force + + - name: Run Unstructured API (docker) + run: | + docker run -d \ + --name unstructured-api \ + -p 8000:8000 \ + --health-cmd "curl --fail http://localhost:8000/healthcheck || exit 1" \ + --health-interval 10s \ + --health-timeout 1s \ + --health-retries 10 \ + quay.io/unstructured-io/unstructured-api:latest + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }}