From 758cdf6669527ad05ed9c2f64d69e0f55d11f8cd Mon Sep 17 00:00:00 2001 From: William FH <13333726+hinthornw@users.noreply.github.com> Date: Mon, 25 Nov 2024 09:27:42 -0800 Subject: [PATCH] Improve support for pulling structured prompts w/ model info (#1210) --- python/langsmith/client.py | 32 +- python/pyproject.toml | 3 +- python/tests/unit_tests/test_client.py | 710 ++++++++++++++++++++++++- 3 files changed, 742 insertions(+), 3 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 9dd1442a1..b3b5fc993 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -5594,9 +5594,12 @@ def pull_prompt( Any: The prompt object in the specified format. """ try: + from langchain_core.language_models.base import BaseLanguageModel from langchain_core.load.load import loads + from langchain_core.output_parsers import BaseOutputParser from langchain_core.prompts import BasePromptTemplate - from langchain_core.runnables.base import RunnableSequence + from langchain_core.prompts.structured import StructuredPrompt + from langchain_core.runnables.base import RunnableBinding, RunnableSequence except ImportError: raise ImportError( "The client.pull_prompt function requires the langchain_core" @@ -5645,6 +5648,33 @@ def suppress_langchain_beta_warning(): "lc_hub_commit_hash": prompt_object.commit_hash, } ) + if ( + include_model + and isinstance(prompt, RunnableSequence) + and isinstance(prompt.first, StructuredPrompt) + # Make forward-compatible in case we let update the response type + and ( + len(prompt.steps) == 2 and not isinstance(prompt.last, BaseOutputParser) + ) + ): + if isinstance(prompt.last, RunnableBinding) and isinstance( + prompt.last.bound, BaseLanguageModel + ): + seq = cast(RunnableSequence, prompt.first | prompt.last.bound) + if len(seq.steps) == 3: # prompt | bound llm | output parser + rebound_llm = seq.steps[1] + prompt = RunnableSequence( + prompt.first, + rebound_llm.bind(**{**prompt.last.kwargs}), + seq.last, + ) + else: + prompt = seq # Not sure + + elif isinstance(prompt.last, BaseLanguageModel): + prompt: RunnableSequence = prompt.first | prompt.last # type: ignore[no-redef, assignment] + else: + pass return prompt diff --git a/python/pyproject.toml b/python/pyproject.toml index 12e89ba74..51dfb7e3f 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "langsmith" -version = "0.1.145" +version = "0.1.146" description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." authors = ["LangChain "] license = "MIT" @@ -106,6 +106,7 @@ convention = "google" "langsmith/client.py" = ["E501"] "langsmith/schemas.py" = ["E501"] "tests/evaluation/__init__.py" = ["E501"] +"tests/unit_tests/test_client.py" = ["E501"] "tests/*" = ["D", "UP"] "bench/*" = ["D", "UP", "T"] "docs/*" = ["T", "D"] diff --git a/python/tests/unit_tests/test_client.py b/python/tests/unit_tests/test_client.py index feec2c2f6..915830091 100644 --- a/python/tests/unit_tests/test_client.py +++ b/python/tests/unit_tests/test_client.py @@ -17,7 +17,7 @@ from datetime import datetime, timezone from enum import Enum from io import BytesIO -from typing import Dict, List, NamedTuple, Optional, Type, Union +from typing import Dict, List, Literal, NamedTuple, Optional, Type, Union from unittest import mock from unittest.mock import MagicMock, patch @@ -1254,3 +1254,711 @@ def test_parse_token_or_url(): invalid_url = "https://invalid.com/419dcab2-1d66-4b94-8901-0357ead390df" with pytest.raises(LangSmithUserError): _parse_token_or_url(invalid_url, api_url) + + +_PROMPT_COMMITS = [ + ( + True, + "tools", + { + "owner": "-", + "repo": "tweet-generator-example-with-tools", + "commit_hash": "b862ce708ffeb932331a9345ea2a2fe6a76d62cf83e9aab834c24bb12bd516c9", + "manifest": { + "lc": 1, + "type": "constructor", + "id": ["langchain", "schema", "runnable", "RunnableSequence"], + "kwargs": { + "first": { + "lc": 1, + "type": "constructor", + "id": ["langchain", "prompts", "chat", "ChatPromptTemplate"], + "kwargs": { + "input_variables": ["topic"], + "metadata": { + "lc_hub_owner": "-", + "lc_hub_repo": "tweet-generator-example", + "lc_hub_commit_hash": "c39837bd8d010da739d6d4adc7f2dca2f2461521661a393d37606f5c696109a5", + }, + "messages": [ + { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "chat", + "SystemMessagePromptTemplate", + ], + "kwargs": { + "prompt": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ], + "kwargs": { + "input_variables": [], + "template_format": "f-string", + "template": "Generate a tweet based on the provided topic.", + }, + } + }, + }, + { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "chat", + "HumanMessagePromptTemplate", + ], + "kwargs": { + "prompt": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ], + "kwargs": { + "input_variables": ["topic"], + "template_format": "f-string", + "template": "{topic}", + }, + } + }, + }, + ], + }, + "name": "StructuredPrompt", + }, + "last": { + "lc": 1, + "type": "constructor", + "id": ["langchain", "schema", "runnable", "RunnableBinding"], + "kwargs": { + "bound": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "chat_models", + "anthropic", + "ChatAnthropic", + ], + "kwargs": { + "temperature": 1, + "max_tokens": 1024, + "top_p": 1, + "top_k": -1, + "anthropic_api_key": { + "id": ["ANTHROPIC_API_KEY"], + "lc": 1, + "type": "secret", + }, + "model": "claude-3-5-sonnet-20240620", + }, + }, + "kwargs": { + "tools": [ + { + "type": "function", + "function": { + "name": "GenerateTweet", + "description": "Submit your tweet.", + "parameters": { + "properties": { + "tweet": { + "type": "string", + "description": "The generated tweet.", + } + }, + "required": ["tweet"], + "type": "object", + }, + }, + }, + { + "type": "function", + "function": { + "name": "SomethingElse", + "description": "", + "parameters": { + "properties": { + "aval": { + "type": "array", + "items": {"type": "string"}, + } + }, + "required": [], + "type": "object", + }, + }, + }, + ] + }, + }, + }, + }, + }, + "examples": [], + }, + ), + ( + True, + "structured", + { + "owner": "-", + "repo": "tweet-generator-example", + "commit_hash": "e8da7f9e80471ace9b96c4f8fd55a215020126521f1da8f66130604c101fc522", + "manifest": { + "lc": 1, + "type": "constructor", + "id": ["langchain", "schema", "runnable", "RunnableSequence"], + "kwargs": { + "first": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain_core", + "prompts", + "structured", + "StructuredPrompt", + ], + "kwargs": { + "input_variables": ["topic"], + "metadata": { + "lc_hub_owner": "langchain-ai", + "lc_hub_repo": "tweet-generator-example", + "lc_hub_commit_hash": "7c32ca78a2831b6b3a3904eb5704b48a0730e93f29afb0853cfaefc42dc09f9c", + }, + "messages": [ + { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "chat", + "SystemMessagePromptTemplate", + ], + "kwargs": { + "prompt": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ], + "kwargs": { + "input_variables": [], + "template_format": "f-string", + "template": "Generate a tweet about the given topic.", + }, + } + }, + }, + { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "chat", + "HumanMessagePromptTemplate", + ], + "kwargs": { + "prompt": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ], + "kwargs": { + "input_variables": ["topic"], + "template_format": "f-string", + "template": "{topic}", + }, + } + }, + }, + ], + "schema_": { + "title": "GenerateTweet", + "description": "Submit your tweet.", + "type": "object", + "properties": { + "tweet": { + "type": "string", + "description": "The generated tweet.", + } + }, + "required": ["tweet"], + }, + }, + "name": "StructuredPrompt", + }, + "last": { + "lc": 1, + "type": "constructor", + "id": ["langchain", "schema", "runnable", "RunnableBinding"], + "kwargs": { + "bound": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "chat_models", + "anthropic", + "ChatAnthropic", + ], + "kwargs": { + "temperature": 1, + "max_tokens": 1024, + "top_p": 1, + "top_k": -1, + "anthropic_api_key": { + "id": ["ANTHROPIC_API_KEY"], + "lc": 1, + "type": "secret", + }, + "model": "claude-3-5-sonnet-20240620", + }, + }, + "kwargs": {}, + }, + }, + }, + }, + "examples": [], + }, + ), + ( + True, + "none", + { + "owner": "-", + "repo": "tweet-generator-example-with-nothing", + "commit_hash": "06c657373bdfcadec0d4d0933416b2c11f1b283ef3d1ca5dfb35dd6ed28b9f78", + "manifest": { + "lc": 1, + "type": "constructor", + "id": ["langchain", "schema", "runnable", "RunnableSequence"], + "kwargs": { + "first": { + "lc": 1, + "type": "constructor", + "id": ["langchain", "prompts", "chat", "ChatPromptTemplate"], + "kwargs": { + "messages": [ + { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "chat", + "SystemMessagePromptTemplate", + ], + "kwargs": { + "prompt": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ], + "kwargs": { + "input_variables": [], + "template_format": "f-string", + "template": "Generate a tweet about the given topic.", + }, + } + }, + }, + { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "chat", + "HumanMessagePromptTemplate", + ], + "kwargs": { + "prompt": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ], + "kwargs": { + "input_variables": ["topic"], + "template_format": "f-string", + "template": "{topic}", + }, + } + }, + }, + ], + "input_variables": ["topic"], + }, + }, + "last": { + "lc": 1, + "type": "constructor", + "id": ["langchain", "schema", "runnable", "RunnableBinding"], + "kwargs": { + "bound": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "chat_models", + "openai", + "ChatOpenAI", + ], + "kwargs": { + "openai_api_key": { + "id": ["OPENAI_API_KEY"], + "lc": 1, + "type": "secret", + }, + "model": "gpt-4o-mini", + }, + }, + "kwargs": {}, + }, + }, + }, + }, + "examples": [], + }, + ), + ( + False, + "tools", + { + "owner": "-", + "repo": "tweet-generator-example-with-tools", + "commit_hash": "b862ce708ffeb932331a9345ea2a2fe6a76d62cf83e9aab834c24bb12bd516c9", + "manifest": { + "lc": 1, + "type": "constructor", + "id": ["langchain", "prompts", "chat", "ChatPromptTemplate"], + "kwargs": { + "input_variables": ["topic"], + "metadata": { + "lc_hub_owner": "-", + "lc_hub_repo": "tweet-generator-example", + "lc_hub_commit_hash": "c39837bd8d010da739d6d4adc7f2dca2f2461521661a393d37606f5c696109a5", + }, + "messages": [ + { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "chat", + "SystemMessagePromptTemplate", + ], + "kwargs": { + "prompt": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ], + "kwargs": { + "input_variables": [], + "template_format": "f-string", + "template": "Generate a tweet based on the provided topic.", + }, + } + }, + }, + { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "chat", + "HumanMessagePromptTemplate", + ], + "kwargs": { + "prompt": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ], + "kwargs": { + "input_variables": ["topic"], + "template_format": "f-string", + "template": "{topic}", + }, + } + }, + }, + ], + }, + "name": "StructuredPrompt", + }, + "examples": [], + }, + ), + ( + False, + "structured", + { + "owner": "-", + "repo": "tweet-generator-example", + "commit_hash": "e8da7f9e80471ace9b96c4f8fd55a215020126521f1da8f66130604c101fc522", + "manifest": { + "lc": 1, + "type": "constructor", + "id": ["langchain_core", "prompts", "structured", "StructuredPrompt"], + "kwargs": { + "input_variables": ["topic"], + "metadata": { + "lc_hub_owner": "langchain-ai", + "lc_hub_repo": "tweet-generator-example", + "lc_hub_commit_hash": "7c32ca78a2831b6b3a3904eb5704b48a0730e93f29afb0853cfaefc42dc09f9c", + }, + "messages": [ + { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "chat", + "SystemMessagePromptTemplate", + ], + "kwargs": { + "prompt": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ], + "kwargs": { + "input_variables": [], + "template_format": "f-string", + "template": "Generate a tweet about the given topic.", + }, + } + }, + }, + { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "chat", + "HumanMessagePromptTemplate", + ], + "kwargs": { + "prompt": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ], + "kwargs": { + "input_variables": ["topic"], + "template_format": "f-string", + "template": "{topic}", + }, + } + }, + }, + ], + "schema_": { + "title": "GenerateTweet", + "description": "Submit your tweet.", + "type": "object", + "properties": { + "tweet": { + "type": "string", + "description": "The generated tweet.", + } + }, + "required": ["tweet"], + }, + }, + "name": "StructuredPrompt", + }, + "examples": [], + }, + ), + ( + False, + "none", + { + "owner": "-", + "repo": "tweet-generator-example-with-nothing", + "commit_hash": "06c657373bdfcadec0d4d0933416b2c11f1b283ef3d1ca5dfb35dd6ed28b9f78", + "manifest": { + "lc": 1, + "type": "constructor", + "id": ["langchain", "prompts", "chat", "ChatPromptTemplate"], + "kwargs": { + "messages": [ + { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "chat", + "SystemMessagePromptTemplate", + ], + "kwargs": { + "prompt": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ], + "kwargs": { + "input_variables": [], + "template_format": "f-string", + "template": "Generate a tweet about the given topic.", + }, + } + }, + }, + { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "chat", + "HumanMessagePromptTemplate", + ], + "kwargs": { + "prompt": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ], + "kwargs": { + "input_variables": ["topic"], + "template_format": "f-string", + "template": "{topic}", + }, + } + }, + }, + ], + "input_variables": ["topic"], + }, + }, + "examples": [], + }, + ), +] + + +@pytest.mark.parametrize("include_model, manifest_type, manifest_data", _PROMPT_COMMITS) +def test_pull_prompt( + include_model: bool, + manifest_type: Literal["structured", "tool", "none"], + manifest_data: dict, +): + try: + from langchain_core.language_models.base import BaseLanguageModel + from langchain_core.output_parsers import JsonOutputKeyToolsParser + from langchain_core.prompts import ChatPromptTemplate + from langchain_core.prompts.structured import StructuredPrompt + from langchain_core.runnables import RunnableBinding, RunnableSequence + except ImportError: + pytest.skip("Skipping test that requires langchain") + # Create a mock session + mock_session = mock.Mock() + # prompt_commit = ls_schemas.PromptCommit(**manifest_data) + mock_session.request.side_effect = lambda method, url, **kwargs: mock.Mock( + json=lambda: manifest_data if "/commits/" in url else None + ) + + # Create a client with Info pre-created and version >= 0.6 + info = ls_schemas.LangSmithInfo(version="0.6.0") + client = Client( + api_url="http://localhost:1984", + api_key="fake_api_key", + session=mock_session, + info=info, + ) + with mock.patch.dict( + "os.environ", + { + "ANTHROPIC_API_KEY": "test_anthropic_key", + "OPENAI_API_KEY": "test_openai_key", + }, + ): + result = client.pull_prompt( + prompt_identifier=manifest_data["repo"], include_model=include_model + ) + expected_prompt_type = ( + StructuredPrompt if manifest_type == "structured" else ChatPromptTemplate + ) + if include_model: + assert isinstance(result, RunnableSequence) + assert isinstance(result.first, expected_prompt_type) + if manifest_type != "structured": + assert not isinstance(result.first, StructuredPrompt) + assert len(result.steps) == 2 + if manifest_type == "tool": + assert result.steps[1].kwargs.get("tools") + else: + assert len(result.steps) == 3 + assert isinstance(result.steps[1], RunnableBinding) + assert result.steps[1].kwargs.get("tools") + assert isinstance(result.steps[1].bound, BaseLanguageModel) + assert isinstance(result.steps[2], JsonOutputKeyToolsParser) + + else: + assert isinstance(result, expected_prompt_type) + if manifest_type != "structured": + assert not isinstance(result, StructuredPrompt)