diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b8e6b3bd5..7428c1b03 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -44,7 +44,7 @@ repos: - id: mypy # You can add additional plugins for mypy below # such as types-python-dateutil - additional_dependencies: [pydantic>=2.8.2] + additional_dependencies: [pydantic>=2.8.2, types-pyyaml>=6.0.12] exclude: (/test_|setup.py|/tests/|docs/) # Sort imports alphabetically, and automatically separated into sections and by type. diff --git a/packages/ragbits-dev-kit/README.md b/packages/ragbits-dev-kit/README.md index f27a26784..099abe680 100644 --- a/packages/ragbits-dev-kit/README.md +++ b/packages/ragbits-dev-kit/README.md @@ -1 +1,26 @@ # Ragbits Development Kit + +## Promptfoo Integration + +Ragbits' `Prompt` abstraction can be seamlessly integrated with the `promptfoo` tool. After installing `promptfoo` as +specified in the [promptfoo documentation](https://www.promptfoo.dev/docs/installation/), you can generate promptfoo +configuration files for all the prompts discovered by our autodiscover mechanism by running the following command: + +```bash +rbts prompts generate-promptfoo-configs +``` + +This command will generate a YAML files in the directory specified by `--target-path` (`promptfooconfigs` by +default). The generated file should look like this: + +```yaml +prompts: + - file:///path/to/your/prompt:PromptClass.to_promptfoo +``` + +You can then edit the generated file to add your custom `promptfoo` configurations. Once your `promptfoo` configuration +file is ready, you can run `promptfoo` with the following command: + +```bash +promptfoo -c /path/to/generated/promptfoo-config.yaml eval +``` \ No newline at end of file diff --git a/packages/ragbits-dev-kit/src/ragbits/dev_kit/cli.py b/packages/ragbits-dev-kit/src/ragbits/dev_kit/cli.py index 22475bb86..73f5267ff 100644 --- a/packages/ragbits-dev-kit/src/ragbits/dev_kit/cli.py +++ b/packages/ragbits-dev-kit/src/ragbits/dev_kit/cli.py @@ -1,6 +1,7 @@ import typer from .prompt_lab.app import lab_app +from .promptfoo import generate_configs prompts_app = typer.Typer(no_args_is_help=True) @@ -13,4 +14,5 @@ def register(app: typer.Typer) -> None: app: The Typer object to register the commands with. """ prompts_app.command(name="lab")(lab_app) + prompts_app.command(name="generate-promptfoo-configs")(generate_configs) app.add_typer(prompts_app, name="prompts", help="Commands for managing prompts") diff --git a/packages/ragbits-dev-kit/src/ragbits/dev_kit/promptfoo.py b/packages/ragbits-dev-kit/src/ragbits/dev_kit/promptfoo.py new file mode 100644 index 000000000..d73576212 --- /dev/null +++ b/packages/ragbits-dev-kit/src/ragbits/dev_kit/promptfoo.py @@ -0,0 +1,33 @@ +import os +from pathlib import Path + +import yaml +from rich.console import Console + +from ragbits.dev_kit.prompt_lab.discovery import PromptDiscovery +from ragbits.dev_kit.prompt_lab.discovery.prompt_discovery import DEFAULT_FILE_PATTERN + + +def generate_configs( + file_pattern: str = DEFAULT_FILE_PATTERN, root_path: Path = Path.cwd(), target_path: Path = Path("promptfooconfigs") +) -> None: + """ + Generates promptfoo configuration files for all discovered prompts. + + Args: + file_pattern: The file pattern to search for Prompt objects. Defaults to "**/prompt_*.py" + root_path: The root path to search for Prompt objects. Defaults to the directory where the script is run. + target_path: The path to save the promptfoo configuration files. Defaults to "promptfooconfigs". + """ + prompts = PromptDiscovery(file_pattern=file_pattern, root_path=root_path).discover() + Console().print( + f"Discovered {len(prompts)} prompts." + f" Saving promptfoo configuration files to [bold green]{target_path}[/] folder ..." + ) + + if not target_path.exists(): + target_path.mkdir() + for prompt in prompts: + with open(target_path / f"{prompt.__qualname__}.yaml", "w", encoding="utf-8") as f: + prompt_path = f'file://{prompt.__module__.replace(".", os.sep)}.py:{prompt.__qualname__}.to_promptfoo' + yaml.dump({"prompts": [prompt_path]}, f)