From 5471e88b566fac494f056caf11dc42a92ebbdd9f Mon Sep 17 00:00:00 2001 From: isaac hershenson Date: Mon, 9 Dec 2024 15:17:37 -0800 Subject: [PATCH 1/3] fmt --- python/langsmith/schemas.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/langsmith/schemas.py b/python/langsmith/schemas.py index 5b226a830..30a65a018 100644 --- a/python/langsmith/schemas.py +++ b/python/langsmith/schemas.py @@ -5,7 +5,6 @@ from datetime import datetime, timedelta, timezone from decimal import Decimal from enum import Enum -from pathlib import Path from typing import ( Any, Dict, @@ -64,7 +63,7 @@ def my_function(bar: int, my_val: Attachment): data: bytes -Attachments = Dict[str, Union[Tuple[str, bytes], Attachment, Tuple[str, Path]]] +Attachments = Dict[str, Union[Tuple[str, bytes], Attachment]] """Attachments associated with the run. Each entry is a tuple of (mime_type, bytes), or (mime_type, file_path)""" From 49246d06872857babfc607701abbea9ed996c5ec Mon Sep 17 00:00:00 2001 From: Bagatur Date: Mon, 9 Dec 2024 15:51:35 -0800 Subject: [PATCH 2/3] fmt --- python/langsmith/evaluation/_runner.py | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/python/langsmith/evaluation/_runner.py b/python/langsmith/evaluation/_runner.py index 805cfad03..51b464e68 100644 --- a/python/langsmith/evaluation/_runner.py +++ b/python/langsmith/evaluation/_runner.py @@ -1934,26 +1934,13 @@ def _include_attachments(target: Any) -> bool: "Target function must accept at most two " "arguments without default values: (inputs, attachments)." ) - else: - mismatches = [] - num_args = 0 - for i, (p, expected) in enumerate( - zip(positional_params, ("inputs", "attachments")) - ): - if p.name != expected: - mismatches.append((i, p.name)) - else: - num_args += 1 - - if mismatches: - msg = ( - "Target function is expected to have a first positional argument " - "'inputs' and optionally a second positional argument 'attachments'. " - "Received: " + ", ".join(f"'{p}' at index {i}" for i, p in mismatches) - ) + elif len(positional_no_default) == 2: + if [p.name for p in positional_no_default] != ["inputs", "attachments"]: + msg = "" raise ValueError(msg) - - return num_args == 2 + return True + else: + return [p.name for p in positional_params[:2]] == ["inputs", "attachments"] def _resolve_experiment( From b0921e06cd16ba0dbb78ad0dc7d3669273614900 Mon Sep 17 00:00:00 2001 From: isaac hershenson Date: Mon, 9 Dec 2024 15:57:54 -0800 Subject: [PATCH 3/3] tests --- python/langsmith/evaluation/_runner.py | 7 +++-- .../unit_tests/evaluation/test_runner.py | 26 +++++++++---------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/python/langsmith/evaluation/_runner.py b/python/langsmith/evaluation/_runner.py index 51b464e68..836aaabe4 100644 --- a/python/langsmith/evaluation/_runner.py +++ b/python/langsmith/evaluation/_runner.py @@ -1936,8 +1936,11 @@ def _include_attachments(target: Any) -> bool: ) elif len(positional_no_default) == 2: if [p.name for p in positional_no_default] != ["inputs", "attachments"]: - msg = "" - raise ValueError(msg) + raise ValueError( + "When passing 2 positional arguments, they must be named " + "'inputs' and 'attachments', respectively. Received: " + f"{[p.name for p in positional_no_default]}" + ) return True else: return [p.name for p in positional_params[:2]] == ["inputs", "attachments"] diff --git a/python/tests/unit_tests/evaluation/test_runner.py b/python/tests/unit_tests/evaluation/test_runner.py index 87ebe6042..04b269100 100644 --- a/python/tests/unit_tests/evaluation/test_runner.py +++ b/python/tests/unit_tests/evaluation/test_runner.py @@ -20,9 +20,6 @@ from langsmith import Client, aevaluate, evaluate from langsmith import schemas as ls_schemas -from langsmith.evaluation._arunner import ( - _include_attachments as a_include_attachments, -) from langsmith.evaluation._runner import _include_attachments from langsmith.evaluation.evaluator import ( _normalize_comparison_evaluator_func, @@ -738,17 +735,19 @@ async def async_extra_args(inputs, attachments, foo="bar"): ( lambda x, y: None, None, - "When target function has two positional arguments, they must be named " - "'inputs' and 'attachments', respectively. Received: 'x' at index 0,'y' " - "at index 1", + re.escape( + "When passing 2 positional arguments, they must be named 'inputs' and " + "'attachments', respectively. Received: ['x', 'y']" + ), False, ), ( lambda input, attachment: None, None, - "When target function has two positional arguments, they must be named " - "'inputs' and 'attachments', respectively. Received: 'input' at index 0," - "'attachment' at index 1", + re.escape( + "When passing 2 positional arguments, they must be named 'inputs' and " + "'attachments', respectively. Received: ['input', 'attachment']" + ), False, ), # Too many parameters @@ -756,8 +755,8 @@ async def async_extra_args(inputs, attachments, foo="bar"): lambda inputs, attachments, extra: None, None, re.escape( - "Target function must accept at most two positional arguments " - "(inputs, attachments)" + "Target function must accept at most two arguments without " + "default values: (inputs, attachments)." ), False, ), @@ -796,12 +795,11 @@ def test_include_attachments(target, expected, error_msg, is_async): expected = False error_msg = None - func = _include_attachments if not is_async else a_include_attachments if error_msg is not None: with pytest.raises(ValueError, match=error_msg): - func(target) + _include_attachments(target) else: - result = func(target) + result = _include_attachments(target) assert result == expected