From 723f98af9ab1a69420303b6f4766e884ea4ac8c3 Mon Sep 17 00:00:00 2001 From: William Fu-Hinthorn <13333726+hinthornw@users.noreply.github.com> Date: Wed, 13 Nov 2024 10:55:38 -0800 Subject: [PATCH] Test --- python/langsmith/client.py | 8 +- python/pyproject.toml | 1 + python/tests/unit_tests/test_client.py | 710 ++++++++++++++++++++++++- 3 files changed, 715 insertions(+), 4 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 53fed5bc5..c47ff0d1f 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -5617,17 +5617,19 @@ def suppress_langchain_beta_warning(): if isinstance(prompt.last, RunnableBinding) and isinstance( prompt.last.bound, BaseLanguageModel ): - seq: RunnableSequence = prompt.first | prompt.last.bound + seq = cast(RunnableSequence, prompt.first | prompt.last.bound) if len(seq.steps) == 3: # prompt | bound llm | output parser rebound_llm = seq.steps[1] prompt = RunnableSequence( - prompt.first, rebound_llm.bind(**{prompt.last.kwargs}), seq.last + prompt.first, + rebound_llm.bind(**{**prompt.last.kwargs}), + seq.last, ) else: prompt = seq # Not sure elif isinstance(prompt.last, BaseLanguageModel): - prompt: RunnableSequence = prompt.first | prompt.last + prompt: RunnableSequence = prompt.first | prompt.last # type: ignore[no-redef, assignment] else: pass diff --git a/python/pyproject.toml b/python/pyproject.toml index 81645c912..9f74929e4 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -106,6 +106,7 @@ convention = "google" "langsmith/client.py" = ["E501"] "langsmith/schemas.py" = ["E501"] "tests/evaluation/__init__.py" = ["E501"] +"tests/unit_tests/test_client.py" = ["E501"] "tests/*" = ["D", "UP"] "bench/*" = ["D", "UP", "T"] "docs/*" = ["T", "D"] diff --git a/python/tests/unit_tests/test_client.py b/python/tests/unit_tests/test_client.py index 5dc1bbe1e..8aa9f2fc8 100644 --- a/python/tests/unit_tests/test_client.py +++ b/python/tests/unit_tests/test_client.py @@ -17,7 +17,7 @@ from datetime import datetime, timezone from enum import Enum from io import BytesIO -from typing import Dict, List, NamedTuple, Optional, Type, Union +from typing import Dict, List, Literal, NamedTuple, Optional, Type, Union from unittest import mock from unittest.mock import MagicMock, patch @@ -1254,3 +1254,711 @@ def test_parse_token_or_url(): invalid_url = "https://invalid.com/419dcab2-1d66-4b94-8901-0357ead390df" with pytest.raises(LangSmithUserError): _parse_token_or_url(invalid_url, api_url) + + +_PROMPT_COMMITS = [ + ( + True, + "tools", + { + "owner": "-", + "repo": "tweet-generator-example-with-tools", + "commit_hash": "b862ce708ffeb932331a9345ea2a2fe6a76d62cf83e9aab834c24bb12bd516c9", + "manifest": { + "lc": 1, + "type": "constructor", + "id": ["langchain", "schema", "runnable", "RunnableSequence"], + "kwargs": { + "first": { + "lc": 1, + "type": "constructor", + "id": ["langchain", "prompts", "chat", "ChatPromptTemplate"], + "kwargs": { + "input_variables": ["topic"], + "metadata": { + "lc_hub_owner": "-", + "lc_hub_repo": "tweet-generator-example", + "lc_hub_commit_hash": "c39837bd8d010da739d6d4adc7f2dca2f2461521661a393d37606f5c696109a5", + }, + "messages": [ + { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "chat", + "SystemMessagePromptTemplate", + ], + "kwargs": { + "prompt": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ], + "kwargs": { + "input_variables": [], + "template_format": "f-string", + "template": "Generate a tweet based on the provided topic.", + }, + } + }, + }, + { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "chat", + "HumanMessagePromptTemplate", + ], + "kwargs": { + "prompt": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ], + "kwargs": { + "input_variables": ["topic"], + "template_format": "f-string", + "template": "{topic}", + }, + } + }, + }, + ], + }, + "name": "StructuredPrompt", + }, + "last": { + "lc": 1, + "type": "constructor", + "id": ["langchain", "schema", "runnable", "RunnableBinding"], + "kwargs": { + "bound": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "chat_models", + "anthropic", + "ChatAnthropic", + ], + "kwargs": { + "temperature": 1, + "max_tokens": 1024, + "top_p": 1, + "top_k": -1, + "anthropic_api_key": { + "id": ["ANTHROPIC_API_KEY"], + "lc": 1, + "type": "secret", + }, + "model": "claude-3-5-sonnet-20240620", + }, + }, + "kwargs": { + "tools": [ + { + "type": "function", + "function": { + "name": "GenerateTweet", + "description": "Submit your tweet.", + "parameters": { + "properties": { + "tweet": { + "type": "string", + "description": "The generated tweet.", + } + }, + "required": ["tweet"], + "type": "object", + }, + }, + }, + { + "type": "function", + "function": { + "name": "SomethingElse", + "description": "", + "parameters": { + "properties": { + "aval": { + "type": "array", + "items": {"type": "string"}, + } + }, + "required": [], + "type": "object", + }, + }, + }, + ] + }, + }, + }, + }, + }, + "examples": [], + }, + ), + ( + True, + "structured", + { + "owner": "-", + "repo": "tweet-generator-example", + "commit_hash": "e8da7f9e80471ace9b96c4f8fd55a215020126521f1da8f66130604c101fc522", + "manifest": { + "lc": 1, + "type": "constructor", + "id": ["langchain", "schema", "runnable", "RunnableSequence"], + "kwargs": { + "first": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain_core", + "prompts", + "structured", + "StructuredPrompt", + ], + "kwargs": { + "input_variables": ["topic"], + "metadata": { + "lc_hub_owner": "langchain-ai", + "lc_hub_repo": "tweet-generator-example", + "lc_hub_commit_hash": "7c32ca78a2831b6b3a3904eb5704b48a0730e93f29afb0853cfaefc42dc09f9c", + }, + "messages": [ + { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "chat", + "SystemMessagePromptTemplate", + ], + "kwargs": { + "prompt": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ], + "kwargs": { + "input_variables": [], + "template_format": "f-string", + "template": "Generate a tweet about the given topic.", + }, + } + }, + }, + { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "chat", + "HumanMessagePromptTemplate", + ], + "kwargs": { + "prompt": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ], + "kwargs": { + "input_variables": ["topic"], + "template_format": "f-string", + "template": "{topic}", + }, + } + }, + }, + ], + "schema_": { + "title": "GenerateTweet", + "description": "Submit your tweet.", + "type": "object", + "properties": { + "tweet": { + "type": "string", + "description": "The generated tweet.", + } + }, + "required": ["tweet"], + }, + }, + "name": "StructuredPrompt", + }, + "last": { + "lc": 1, + "type": "constructor", + "id": ["langchain", "schema", "runnable", "RunnableBinding"], + "kwargs": { + "bound": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "chat_models", + "anthropic", + "ChatAnthropic", + ], + "kwargs": { + "temperature": 1, + "max_tokens": 1024, + "top_p": 1, + "top_k": -1, + "anthropic_api_key": { + "id": ["ANTHROPIC_API_KEY"], + "lc": 1, + "type": "secret", + }, + "model": "claude-3-5-sonnet-20240620", + }, + }, + "kwargs": {}, + }, + }, + }, + }, + "examples": [], + }, + ), + ( + True, + "none", + { + "owner": "-", + "repo": "tweet-generator-example-with-nothing", + "commit_hash": "06c657373bdfcadec0d4d0933416b2c11f1b283ef3d1ca5dfb35dd6ed28b9f78", + "manifest": { + "lc": 1, + "type": "constructor", + "id": ["langchain", "schema", "runnable", "RunnableSequence"], + "kwargs": { + "first": { + "lc": 1, + "type": "constructor", + "id": ["langchain", "prompts", "chat", "ChatPromptTemplate"], + "kwargs": { + "messages": [ + { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "chat", + "SystemMessagePromptTemplate", + ], + "kwargs": { + "prompt": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ], + "kwargs": { + "input_variables": [], + "template_format": "f-string", + "template": "Generate a tweet about the given topic.", + }, + } + }, + }, + { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "chat", + "HumanMessagePromptTemplate", + ], + "kwargs": { + "prompt": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ], + "kwargs": { + "input_variables": ["topic"], + "template_format": "f-string", + "template": "{topic}", + }, + } + }, + }, + ], + "input_variables": ["topic"], + }, + }, + "last": { + "lc": 1, + "type": "constructor", + "id": ["langchain", "schema", "runnable", "RunnableBinding"], + "kwargs": { + "bound": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "chat_models", + "openai", + "ChatOpenAI", + ], + "kwargs": { + "openai_api_key": { + "id": ["OPENAI_API_KEY"], + "lc": 1, + "type": "secret", + }, + "model": "gpt-4o-mini", + }, + }, + "kwargs": {}, + }, + }, + }, + }, + "examples": [], + }, + ), + ( + False, + "tools", + { + "owner": "-", + "repo": "tweet-generator-example-with-tools", + "commit_hash": "b862ce708ffeb932331a9345ea2a2fe6a76d62cf83e9aab834c24bb12bd516c9", + "manifest": { + "lc": 1, + "type": "constructor", + "id": ["langchain", "prompts", "chat", "ChatPromptTemplate"], + "kwargs": { + "input_variables": ["topic"], + "metadata": { + "lc_hub_owner": "-", + "lc_hub_repo": "tweet-generator-example", + "lc_hub_commit_hash": "c39837bd8d010da739d6d4adc7f2dca2f2461521661a393d37606f5c696109a5", + }, + "messages": [ + { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "chat", + "SystemMessagePromptTemplate", + ], + "kwargs": { + "prompt": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ], + "kwargs": { + "input_variables": [], + "template_format": "f-string", + "template": "Generate a tweet based on the provided topic.", + }, + } + }, + }, + { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "chat", + "HumanMessagePromptTemplate", + ], + "kwargs": { + "prompt": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ], + "kwargs": { + "input_variables": ["topic"], + "template_format": "f-string", + "template": "{topic}", + }, + } + }, + }, + ], + }, + "name": "StructuredPrompt", + }, + "examples": [], + }, + ), + ( + False, + "structured", + { + "owner": "-", + "repo": "tweet-generator-example", + "commit_hash": "e8da7f9e80471ace9b96c4f8fd55a215020126521f1da8f66130604c101fc522", + "manifest": { + "lc": 1, + "type": "constructor", + "id": ["langchain_core", "prompts", "structured", "StructuredPrompt"], + "kwargs": { + "input_variables": ["topic"], + "metadata": { + "lc_hub_owner": "langchain-ai", + "lc_hub_repo": "tweet-generator-example", + "lc_hub_commit_hash": "7c32ca78a2831b6b3a3904eb5704b48a0730e93f29afb0853cfaefc42dc09f9c", + }, + "messages": [ + { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "chat", + "SystemMessagePromptTemplate", + ], + "kwargs": { + "prompt": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ], + "kwargs": { + "input_variables": [], + "template_format": "f-string", + "template": "Generate a tweet about the given topic.", + }, + } + }, + }, + { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "chat", + "HumanMessagePromptTemplate", + ], + "kwargs": { + "prompt": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ], + "kwargs": { + "input_variables": ["topic"], + "template_format": "f-string", + "template": "{topic}", + }, + } + }, + }, + ], + "schema_": { + "title": "GenerateTweet", + "description": "Submit your tweet.", + "type": "object", + "properties": { + "tweet": { + "type": "string", + "description": "The generated tweet.", + } + }, + "required": ["tweet"], + }, + }, + "name": "StructuredPrompt", + }, + "examples": [], + }, + ), + ( + False, + "none", + { + "owner": "-", + "repo": "tweet-generator-example-with-nothing", + "commit_hash": "06c657373bdfcadec0d4d0933416b2c11f1b283ef3d1ca5dfb35dd6ed28b9f78", + "manifest": { + "lc": 1, + "type": "constructor", + "id": ["langchain", "prompts", "chat", "ChatPromptTemplate"], + "kwargs": { + "messages": [ + { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "chat", + "SystemMessagePromptTemplate", + ], + "kwargs": { + "prompt": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ], + "kwargs": { + "input_variables": [], + "template_format": "f-string", + "template": "Generate a tweet about the given topic.", + }, + } + }, + }, + { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "chat", + "HumanMessagePromptTemplate", + ], + "kwargs": { + "prompt": { + "lc": 1, + "type": "constructor", + "id": [ + "langchain", + "prompts", + "prompt", + "PromptTemplate", + ], + "kwargs": { + "input_variables": ["topic"], + "template_format": "f-string", + "template": "{topic}", + }, + } + }, + }, + ], + "input_variables": ["topic"], + }, + }, + "examples": [], + }, + ), +] + + +@pytest.mark.parametrize("include_model, manifest_type, manifest_data", _PROMPT_COMMITS) +def test_pull_prompt( + include_model: bool, + manifest_type: Literal["structured", "tool", "none"], + manifest_data: dict, +): + try: + from langchain_core.language_models.base import BaseLanguageModel + from langchain_core.output_parsers import JsonOutputKeyToolsParser + from langchain_core.prompts import ChatPromptTemplate + from langchain_core.prompts.structured import StructuredPrompt + from langchain_core.runnables import RunnableBinding, RunnableSequence + except ImportError: + pytest.skip("Skipping test that requires langchain") + # Create a mock session + mock_session = mock.Mock() + # prompt_commit = ls_schemas.PromptCommit(**manifest_data) + mock_session.request.side_effect = lambda method, url, **kwargs: mock.Mock( + json=lambda: manifest_data if "/commits/" in url else None + ) + + # Create a client with Info pre-created and version >= 0.6 + info = ls_schemas.LangSmithInfo(version="0.6.0") + client = Client( + api_url="http://localhost:1984", + api_key="fake_api_key", + session=mock_session, + info=info, + ) + with mock.patch.dict( + "os.environ", + { + "ANTHROPIC_API_KEY": "test_anthropic_key", + "OPENAI_API_KEY": "test_openai_key", + }, + ): + result = client.pull_prompt( + prompt_identifier=manifest_data["repo"], include_model=include_model + ) + expected_prompt_type = ( + StructuredPrompt if manifest_type == "structured" else ChatPromptTemplate + ) + if include_model: + assert isinstance(result, RunnableSequence) + assert isinstance(result.first, expected_prompt_type) + if manifest_type != "structured": + assert not isinstance(result.first, StructuredPrompt) + assert len(result.steps) == 2 + if manifest_type == "tool": + assert result.steps[1].kwargs.get("tools") + else: + assert len(result.steps) == 3 + assert isinstance(result.steps[1], RunnableBinding) + assert result.steps[1].kwargs.get("tools") + assert isinstance(result.steps[1].bound, BaseLanguageModel) + assert isinstance(result.steps[2], JsonOutputKeyToolsParser) + + else: + assert isinstance(result, expected_prompt_type) + if manifest_type != "structured": + assert not isinstance(result, StructuredPrompt)