feat(core): user should be able to configure default types of default…

packages/ragbits-core/src/ragbits/core/cli.py

@@ -9,6 +9,7 @@
 from rich import print as pprint
 
 from ragbits.core.config import core_config
+from ragbits.core.llms.base import LLMType
 from ragbits.core.prompt.prompt import Prompt
 
 
@@ -38,7 +39,7 @@ def register(app: typer.Typer) -> None:
     @prompts_app.command()
     def lab(
         file_pattern: str = core_config.prompt_path_pattern,
-        llm_factory: str | None = core_config.default_llm_factory,
+        llm_factory: str | None = core_config.default_llm_factories[LLMType.TEXT],
     ) -> None:
         """
         Launches the interactive application for listing, rendering, and testing prompts
@@ -73,15 +74,16 @@ def render(prompt_path: str, payload: str | None = None) -> None:
 
     @prompts_app.command(name="exec")
     def execute(
-        prompt_path: str, payload: str | None = None, llm_factory: str | None = core_config.default_llm_factory
+        prompt_path: str,
+        payload: str | None = None,
+        llm_factory: str | None = core_config.default_llm_factories[LLMType.TEXT],
     ) -> None:
         """
         Executes a prompt using the specified prompt class and LLM factory.
 
         Raises:
             ValueError: If `llm_factory` is not provided.
         """
-
         from ragbits.core.llms.factory import get_llm_from_factory
 
         prompt = _render(prompt_path=prompt_path, payload=payload)

packages/ragbits-core/src/ragbits/core/config.py

@@ -1,5 +1,6 @@
 from pydantic import BaseModel
 
+from ragbits.core.llms.base import LLMType
 from ragbits.core.utils._pyproject import get_config_instance
 
 
@@ -11,8 +12,12 @@ class CoreConfig(BaseModel):
     # Pattern used to search for prompt files
     prompt_path_pattern: str = "**/prompt_*.py"
 
-    # Path to a function that returns an LLM object, e.g. "my_project.llms.get_llm"
-    default_llm_factory: str | None = None
+    # Path to a functions that returns LLM objects, e.g. "my_project.llms.get_llm"
+    default_llm_factories: dict[LLMType, str | None] = {
+        LLMType.TEXT: None,
+        LLMType.VISION: None,
+        LLMType.STRUCTURED_OUTPUT: None,
+    }
 
 
 core_config = get_config_instance(CoreConfig, subproject="core")

packages/ragbits-core/src/ragbits/core/llms/base.py

@@ -1,3 +1,4 @@
+import enum
 from abc import ABC, abstractmethod
 from functools import cached_property
 from typing import Generic, cast, overload
@@ -7,6 +8,16 @@
 from .clients.base import LLMClient, LLMClientOptions, LLMOptions
 
 
+class LLMType(enum.Enum):
+    """
+    Types of LLMs based on supported features
+    """
+
+    TEXT = "text"
+    VISION = "vision"
+    STRUCTURED_OUTPUT = "structured_output"
+
+
 class LLM(Generic[LLMClientOptions], ABC):
     """
     Abstract class for interaction with Large Language Model.

packages/ragbits-core/src/ragbits/core/llms/factory.py

@@ -1,7 +1,7 @@
 import importlib
 
 from ragbits.core.config import core_config
-from ragbits.core.llms.base import LLM
+from ragbits.core.llms.base import LLM, LLMType
 from ragbits.core.llms.litellm import LiteLLM
 
 
@@ -21,28 +21,34 @@ def get_llm_from_factory(factory_path: str) -> LLM:
     return function()
 
 
-def has_default_llm() -> bool:
+def has_default_llm(llm_type: LLMType = LLMType.TEXT) -> bool:
     """
     Check if the default LLM factory is set in the configuration.
 
     Returns:
         bool: Whether the default LLM factory is set.
     """
-    return core_config.default_llm_factory is not None
+    default_factory = core_config.default_llm_factories.get(llm_type, None)
+    return default_factory is not None
 
 
-def get_default_llm() -> LLM:
+def get_default_llm(llm_type: LLMType = LLMType.TEXT) -> LLM:
     """
     Get an instance of the default LLM using the factory function
     specified in the configuration.
 
+    Args:
+        llm_type: type of the LLM to get, defaults to text
+
     Returns:
         LLM: An instance of the default LLM.
 
     Raises:
-        ValueError: If the default LLM factory is not set.
+        ValueError: If the default LLM factory is not set or expected llm type is not defined in config
     """
-    factory = core_config.default_llm_factory
+    if llm_type not in core_config.default_llm_factories:
+        raise ValueError(f"Default LLM of type {llm_type} is not defined in pyproject.toml config.")
+    factory = core_config.default_llm_factories[llm_type]
     if factory is None:
         raise ValueError("Default LLM factory is not set")
 
@@ -58,3 +64,14 @@ def simple_litellm_factory() -> LLM:
         LLM: An instance of the LiteLLM.
     """
     return LiteLLM()
+
+
+def simple_litellm_vision_factory() -> LLM:
+    """
+    A basic LLM factory that creates an LiteLLM instance with the vision enabled model,
+    default options, and assumes that the API key is set in the environment.
+
+    Returns:
+        LLM: An instance of the LiteLLM.
+    """
+    return LiteLLM(model_name="gpt-4o-mini")

packages/ragbits-core/src/ragbits/core/prompt/lab/app.py

@@ -15,6 +15,7 @@
 
 from ragbits.core.config import core_config
 from ragbits.core.llms import LLM
+from ragbits.core.llms.base import LLMType
 from ragbits.core.llms.factory import get_llm_from_factory
 from ragbits.core.prompt import Prompt
 from ragbits.core.prompt.discovery import PromptDiscovery
@@ -137,7 +138,7 @@ def get_input_type_fields(obj: BaseModel | None) -> list[dict]:
 
 def lab_app(  # pylint: disable=missing-param-doc
     file_pattern: str = core_config.prompt_path_pattern,
-    llm_factory: str | None = core_config.default_llm_factory,
+    llm_factory: str | None = core_config.default_llm_factories[LLMType.TEXT],
 ) -> None:
     """
     Launches the interactive application for listing, rendering, and testing prompts

packages/ragbits-core/src/ragbits/core/utils/_pyproject.py

@@ -1,9 +1,12 @@
+import enum
 from pathlib import Path
 from typing import Any, TypeVar
 
 import tomli
 from pydantic import BaseModel
 
+from ragbits.core.llms.base import LLMType
+
 
 def find_pyproject(current_dir: Path | None = None) -> Path:
     """
@@ -64,7 +67,7 @@ def get_config_instance(
     model: type[ConfigModelT], subproject: str | None = None, current_dir: Path | None = None
 ) -> ConfigModelT:
     """
-    Creates an instace of pydantic model loaded with the configuration from pyproject.toml.
+    Creates an instance of pydantic model loaded with the configuration from pyproject.toml.
 
     Args:
         model (Type[BaseModel]): The pydantic model to instantiate.
@@ -81,4 +84,15 @@ def get_config_instance(
     config = get_ragbits_config(current_dir)
     if subproject:
         config = config.get(subproject, {})
+    if "default_llm_factories" in config:
+        config["default_llm_factories"] = {
+            _resolve_enum_member(k): v for k, v in config["default_llm_factories"].items()
+        }
     return model(**config)
+
+
+def _resolve_enum_member(enum_string: str) -> enum.Enum:
+    try:
+        return LLMType(enum_string)
+    except ValueError as err:
+        raise ValueError("Unsupported LLMType value provided in default_llm_factories in pyproject.toml") from err

packages/ragbits-core/tests/unit/llms/factory/test_get_default_llm.py

@@ -1,6 +1,7 @@
 import pytest
 
 from ragbits.core.config import core_config
+from ragbits.core.llms.base import LLMType
 from ragbits.core.llms.factory import get_default_llm
 from ragbits.core.llms.litellm import LiteLLM
 
@@ -9,7 +10,9 @@ def test_get_default_llm(monkeypatch: pytest.MonkeyPatch) -> None:
     """
     Test the get_llm_from_factory function.
     """
-    monkeypatch.setattr(core_config, "default_llm_factory", "factory.test_get_llm_from_factory.mock_llm_factory")
+    monkeypatch.setattr(
+        core_config, "default_llm_factories", {LLMType.TEXT: "factory.test_get_llm_from_factory.mock_llm_factory"}
+    )
 
     llm = get_default_llm()
     assert isinstance(llm, LiteLLM)

packages/ragbits-core/tests/unit/llms/factory/test_has_default_llm.py

@@ -1,14 +1,15 @@
 import pytest
 
 from ragbits.core.config import core_config
+from ragbits.core.llms.base import LLMType
 from ragbits.core.llms.factory import has_default_llm
 
 
 def test_has_default_llm(monkeypatch: pytest.MonkeyPatch) -> None:
     """
     Test the has_default_llm function when the default LLM factory is not set.
     """
-    monkeypatch.setattr(core_config, "default_llm_factory", None)
+    monkeypatch.setattr(core_config, "default_llm_factories", {})
 
     assert has_default_llm() is False
 
@@ -17,6 +18,6 @@ def test_has_default_llm_false(monkeypatch: pytest.MonkeyPatch) -> None:
     """
     Test the has_default_llm function when the default LLM factory is set.
     """
-    monkeypatch.setattr(core_config, "default_llm_factory", "my_project.llms.get_llm")
+    monkeypatch.setattr(core_config, "default_llm_factories", {LLMType.TEXT: "my_project.llms.get_llm"})
 
     assert has_default_llm() is True

packages/ragbits-core/tests/unit/utils/pyproject/test_get_instace.py

@@ -1,7 +1,10 @@
 from pathlib import Path
 
+import pytest
 from pydantic import BaseModel
 
+from ragbits.core.config import CoreConfig
+from ragbits.core.llms.base import LLMType
 from ragbits.core.utils._pyproject import get_config_instance
 
 projects_dir = Path(__file__).parent / "testprojects"
@@ -66,3 +69,30 @@ def test_get_config_instance_no_file():
     )
 
     assert config == OptionalHappyProjectConfig()
+
+
+def test_get_config_instance_factories():
+    """Test that default LLMs are loaded correctly"""
+    config = get_config_instance(
+        CoreConfig,
+        subproject="core",
+        current_dir=projects_dir / "factory_project",
+    )
+
+    assert config.default_llm_factories == {
+        LLMType.TEXT: "ragbits.core.llms.factory.simple_litellm_factory",
+        LLMType.VISION: "ragbits.core.llms.factory.simple_litellm_vision_factory",
+        LLMType.STRUCTURED_OUTPUT: "ragbits.core.llms.factory.simple_litellm_vision_factory",
+    }
+
+
+def test_get_config_instance_bad_factories():
+    """Test that non-existing LLM defined in pyproject raises error"""
+    with pytest.raises(ValueError) as err:
+        get_config_instance(
+            CoreConfig,
+            subproject="core",
+            current_dir=projects_dir / "bad_factory_project",
+        )
+
+    assert "Unsupported LLMType value provided in default_llm_factories in pyproject.toml" in str(err.value)

packages/ragbits-core/tests/unit/utils/pyproject/testprojects/bad_factory_project/pyproject.toml

@@ -0,0 +1,7 @@
+[project]
+name = "bad_factory_project"
+
+[tool.ragbits.core.default_llm_factories]
+non_existing = "ragbits.core.llms.factory.simple_litellm_factory"
+vision = "ragbits.core.llms.factory.simple_litellm_vision_factory"
+structured_output = "ragbits.core.llms.factory.simple_litellm_vision_factory"

packages/ragbits-core/tests/unit/utils/pyproject/testprojects/factory_project/pyproject.toml

@@ -0,0 +1,7 @@
+[project]
+name = "factory_project"
+
+[tool.ragbits.core.default_llm_factories]
+text = "ragbits.core.llms.factory.simple_litellm_factory"
+vision = "ragbits.core.llms.factory.simple_litellm_vision_factory"
+structured_output = "ragbits.core.llms.factory.simple_litellm_vision_factory"

packages/ragbits-document-search/src/ragbits/document_search/ingestion/providers/unstructured/images.py

@@ -1,11 +1,13 @@
+import warnings
 from pathlib import Path
 
 from PIL import Image
 from unstructured.chunking.basic import chunk_elements
 from unstructured.documents.elements import Element as UnstructuredElement
 from unstructured.documents.elements import ElementType
 
-from ragbits.core.llms.base import LLM
+from ragbits.core.llms.base import LLM, LLMType
+from ragbits.core.llms.factory import get_default_llm, has_default_llm
 from ragbits.core.llms.litellm import LiteLLM
 from ragbits.document_search.documents.document import DocumentMeta, DocumentType
 from ragbits.document_search.documents.element import Element, ImageElement
@@ -17,7 +19,7 @@
     to_text_element,
 )
 
-DEFAULT_LLM_IMAGE_SUMMARIZATION_MODEL = "gpt-4o-mini"
+DEFAULT_LLM_IMAGE_DESCRIPTION_MODEL = "gpt-4o-mini"
 
 
 class UnstructuredImageProvider(UnstructuredDefaultProvider):
@@ -53,7 +55,8 @@ def __init__(
             llm: llm to use
         """
         super().__init__(partition_kwargs, chunking_kwargs, api_key, api_server, use_api)
-        self.image_summarizer = ImageDescriber(llm or LiteLLM(DEFAULT_LLM_IMAGE_SUMMARIZATION_MODEL))
+        self.image_describer: ImageDescriber | None = None
+        self._llm = llm
 
     async def _chunk_and_convert(
         self, elements: list[UnstructuredElement], document_meta: DocumentMeta, document_path: Path
@@ -79,7 +82,18 @@ async def _to_image_element(
         )
 
         img_bytes = crop_and_convert_to_bytes(image, top_x, top_y, bottom_x, bottom_y)
-        image_description = await self.image_summarizer.get_image_description(img_bytes)
+        if self.image_describer is None:
+            if self._llm is not None:
+                llm_to_use = self._llm
+            elif has_default_llm(LLMType.VISION):
+                llm_to_use = get_default_llm(LLMType.VISION)
+            else:
+                warnings.warn(
+                    f"Vision LLM was not provided, setting default option to {DEFAULT_LLM_IMAGE_DESCRIPTION_MODEL}"
+                )
+                llm_to_use = LiteLLM(DEFAULT_LLM_IMAGE_DESCRIPTION_MODEL)
+            self.image_describer = ImageDescriber(llm_to_use)
+        image_description = await self.image_describer.get_image_description(img_bytes)
         return ImageElement(
             description=image_description,
             ocr_extracted_text=element.text,