Merge branch 'isaac/multipartstuff' into isaac/addattachmentsevaluator

python/langsmith/evaluation/_runner.py

@@ -1959,26 +1959,16 @@ def _include_attachments(
             "Target function must accept at most two "
             "arguments without default values: (inputs, attachments)."
         )
-    else:
-        mismatches = []
-        num_args = 0
-        for i, (p, expected) in enumerate(
-            zip(positional_params, ("inputs", "attachments"))
-        ):
-            if p.name != expected:
-                mismatches.append((i, p.name))
-            else:
-                num_args += 1
-
-        if mismatches:
-            msg = (
-                "Target function is expected to have a first positional argument "
-                "'inputs' and optionally a second positional argument 'attachments'. "
-                "Received: " + ", ".join(f"'{p}' at index {i}" for i, p in mismatches)
+    elif len(positional_no_default) == 2:
+        if [p.name for p in positional_no_default] != ["inputs", "attachments"]:
+            raise ValueError(
+                "When passing 2 positional arguments, they must be named "
+                "'inputs' and 'attachments', respectively. Received: "
+                f"{[p.name for p in positional_no_default]}"
             )
-            raise ValueError(msg)
-
-    return num_args == 2
+        return True
+    else:
+        return [p.name for p in positional_params[:2]] == ["inputs", "attachments"]
 
 
 def _resolve_experiment(

python/langsmith/schemas.py

@@ -5,7 +5,6 @@
 from datetime import datetime, timedelta, timezone
 from decimal import Decimal
 from enum import Enum
-from pathlib import Path
 from typing import (
     Any,
     Dict,
@@ -64,7 +63,7 @@ def my_function(bar: int, my_val: Attachment):
     data: bytes
 
 
-Attachments = Dict[str, Union[Tuple[str, bytes], Attachment, Tuple[str, Path]]]
+Attachments = Dict[str, Union[Tuple[str, bytes], Attachment]]
 """Attachments associated with the run. 
 Each entry is a tuple of (mime_type, bytes), or (mime_type, file_path)"""
 

python/tests/unit_tests/evaluation/test_runner.py

@@ -20,9 +20,6 @@
 
 from langsmith import Client, aevaluate, evaluate
 from langsmith import schemas as ls_schemas
-from langsmith.evaluation._arunner import (
-    _include_attachments as a_include_attachments,
-)
 from langsmith.evaluation._runner import _include_attachments
 from langsmith.evaluation.evaluator import (
     _normalize_comparison_evaluator_func,
@@ -757,26 +754,28 @@ async def async_extra_args(inputs, attachments, foo="bar"):
         (
             lambda x, y: None,
             None,
-            "When target function has two positional arguments, they must be named "
-            "'inputs' and 'attachments', respectively. Received: 'x' at index 0,'y' "
-            "at index 1",
+            re.escape(
+                "When passing 2 positional arguments, they must be named 'inputs' and "
+                "'attachments', respectively. Received: ['x', 'y']"
+            ),
             False,
         ),
         (
             lambda input, attachment: None,
             None,
-            "When target function has two positional arguments, they must be named "
-            "'inputs' and 'attachments', respectively. Received: 'input' at index 0,"
-            "'attachment' at index 1",
+            re.escape(
+                "When passing 2 positional arguments, they must be named 'inputs' and "
+                "'attachments', respectively. Received: ['input', 'attachment']"
+            ),
             False,
         ),
         # Too many parameters
         (
             lambda inputs, attachments, extra: None,
             None,
             re.escape(
-                "Target function must accept at most two positional arguments "
-                "(inputs, attachments)"
+                "Target function must accept at most two arguments without "
+                "default values: (inputs, attachments)."
             ),
             False,
         ),
@@ -815,12 +814,11 @@ def test_include_attachments(target, expected, error_msg, is_async):
         expected = False
         error_msg = None
 
-    func = _include_attachments if not is_async else a_include_attachments
     if error_msg is not None:
         with pytest.raises(ValueError, match=error_msg):
-            func(target)
+            _include_attachments(target)
     else:
-        result = func(target)
+        result = _include_attachments(target)
         assert result == expected