Add cli for generating promptfoo configs

.pre-commit-config.yaml

@@ -44,7 +44,7 @@ repos:
           - id: mypy
             # You can add additional plugins for mypy below
             # such as types-python-dateutil
-            additional_dependencies: [pydantic>=2.8.2]
+            additional_dependencies: [pydantic>=2.8.2, types-pyyaml>=6.0.12]
             exclude: (/test_|setup.py|/tests/|docs/)
 
       # Sort imports alphabetically, and automatically separated into sections and by type.

packages/ragbits-dev-kit/README.md

@@ -1 +1,26 @@
 # Ragbits Development Kit
+
+## Promptfoo Integration
+
+Ragbits' `Prompt` abstraction can be seamlessly integrated with the `promptfoo` tool. After installing `promptfoo` as
+specified in the [promptfoo documentation](https://www.promptfoo.dev/docs/installation/), you can generate promptfoo
+configuration files for all the prompts discovered by our autodiscover mechanism by running the following command:
+
+```bash
+rbts prompts generate-promptfoo-configs
+```
+
+This command will generate a YAML files in the directory specified by `--target-path` (`promptfooconfigs` by
+default). The generated file should look like this:
+
+```yaml
+prompts:
+  - file:///path/to/your/prompt:PromptClass.to_promptfoo
+```
+
+You can then edit the generated file to add your custom `promptfoo` configurations. Once your `promptfoo` configuration
+file is ready, you can run `promptfoo` with the following command:
+
+```bash
+promptfoo -c /path/to/generated/promptfoo-config.yaml eval
+```

packages/ragbits-dev-kit/src/ragbits/dev_kit/cli.py

@@ -1,6 +1,7 @@
 import typer
 
 from .prompt_lab.app import lab_app
+from .promptfoo import generate_configs
 
 prompts_app = typer.Typer(no_args_is_help=True)
 
@@ -13,4 +14,5 @@ def register(app: typer.Typer) -> None:
         app: The Typer object to register the commands with.
     """
     prompts_app.command(name="lab")(lab_app)
+    prompts_app.command(name="generate-promptfoo-configs")(generate_configs)
     app.add_typer(prompts_app, name="prompts", help="Commands for managing prompts")

packages/ragbits-dev-kit/src/ragbits/dev_kit/promptfoo.py

@@ -0,0 +1,33 @@
+import os
+from pathlib import Path
+
+import yaml
+from rich.console import Console
+
+from ragbits.dev_kit.prompt_lab.discovery import PromptDiscovery
+from ragbits.dev_kit.prompt_lab.discovery.prompt_discovery import DEFAULT_FILE_PATTERN
+
+
+def generate_configs(
+    file_pattern: str = DEFAULT_FILE_PATTERN, root_path: Path = Path.cwd(), target_path: Path = Path("promptfooconfigs")
+) -> None:
+    """
+    Generates promptfoo configuration files for all discovered prompts.
+
+    Args:
+        file_pattern: The file pattern to search for Prompt objects. Defaults to "**/prompt_*.py"
+        root_path: The root path to search for Prompt objects. Defaults to the directory where the script is run.
+        target_path: The path to save the promptfoo configuration files. Defaults to "promptfooconfigs".
+    """
+    prompts = PromptDiscovery(file_pattern=file_pattern, root_path=root_path).discover()
+    Console().print(
+        f"Discovered {len(prompts)} prompts."
+        f" Saving promptfoo configuration files to [bold green]{target_path}[/] folder ..."
+    )
+
+    if not target_path.exists():
+        target_path.mkdir()
+    for prompt in prompts:
+        with open(target_path / f"{prompt.__qualname__}.yaml", "w", encoding="utf-8") as f:
+            prompt_path = f'file://{prompt.__module__.replace(".", os.sep)}.py:{prompt.__qualname__}.to_promptfoo'
+            yaml.dump({"prompts": [prompt_path]}, f)