From c8a2b01e0625312884233c879316d1a3a4037b9a Mon Sep 17 00:00:00 2001 From: isaac hershenson Date: Mon, 9 Dec 2024 14:48:48 -0800 Subject: [PATCH 1/6] undo --- python/langsmith/_internal/_operations.py | 24 +++++++++-------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/python/langsmith/_internal/_operations.py b/python/langsmith/_internal/_operations.py index 3f82f5f9a..cc615d60d 100644 --- a/python/langsmith/_internal/_operations.py +++ b/python/langsmith/_internal/_operations.py @@ -214,7 +214,6 @@ def serialized_run_operation_to_multipart_parts_and_context( op: SerializedRunOperation, ) -> MultipartPartsAndContext: acc_parts: list[MultipartPart] = [] - valb: Union[bytes, Path] # this is main object, minus inputs/outputs/events/attachments acc_parts.append( ( @@ -257,22 +256,17 @@ def serialized_run_operation_to_multipart_parts_and_context( ) continue - if isinstance(valb, Path): - # TODO: actually deal with this case - # This is just for speed of getting something out - continue - else: - acc_parts.append( + acc_parts.append( + ( + f"attachment.{op.id}.{n}", ( - f"attachment.{op.id}.{n}", - ( - None, - valb, - content_type, - {"Content-Length": str(len(valb))}, - ), - ) + None, + valb, + content_type, + {"Content-Length": str(len(valb))}, + ), ) + ) return MultipartPartsAndContext( acc_parts, f"trace={op.trace_id},id={op.id}", From 8033b7e1dc85d5e31cb48e970ae5591730bc484e Mon Sep 17 00:00:00 2001 From: isaac hershenson Date: Mon, 9 Dec 2024 14:49:06 -0800 Subject: [PATCH 2/6] undo --- python/langsmith/_internal/_operations.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/langsmith/_internal/_operations.py b/python/langsmith/_internal/_operations.py index cc615d60d..c68c17499 100644 --- a/python/langsmith/_internal/_operations.py +++ b/python/langsmith/_internal/_operations.py @@ -3,7 +3,6 @@ import itertools import logging import uuid -from pathlib import Path from typing import Literal, Optional, Union, cast from langsmith import schemas as ls_schemas From 114a79d8cf121cba285bcd41c5c58adee0b96398 Mon Sep 17 00:00:00 2001 From: isaac hershenson Date: Mon, 9 Dec 2024 14:49:28 -0800 Subject: [PATCH 3/6] fix --- python/langsmith/_internal/_operations.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/langsmith/_internal/_operations.py b/python/langsmith/_internal/_operations.py index c68c17499..24c40efa0 100644 --- a/python/langsmith/_internal/_operations.py +++ b/python/langsmith/_internal/_operations.py @@ -213,6 +213,7 @@ def serialized_run_operation_to_multipart_parts_and_context( op: SerializedRunOperation, ) -> MultipartPartsAndContext: acc_parts: list[MultipartPart] = [] + # this is main object, minus inputs/outputs/events/attachments acc_parts.append( ( From b524f7235320974d833c0b3b63ef68cd3248bc8b Mon Sep 17 00:00:00 2001 From: isaac hershenson Date: Mon, 9 Dec 2024 14:49:52 -0800 Subject: [PATCH 4/6] fix --- python/langsmith/_internal/_operations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/langsmith/_internal/_operations.py b/python/langsmith/_internal/_operations.py index 24c40efa0..66decff0f 100644 --- a/python/langsmith/_internal/_operations.py +++ b/python/langsmith/_internal/_operations.py @@ -213,7 +213,7 @@ def serialized_run_operation_to_multipart_parts_and_context( op: SerializedRunOperation, ) -> MultipartPartsAndContext: acc_parts: list[MultipartPart] = [] - + # this is main object, minus inputs/outputs/events/attachments acc_parts.append( ( From 23187f172276b1b18c5b44a42abc4f5ca1019cc8 Mon Sep 17 00:00:00 2001 From: isaac hershenson Date: Mon, 9 Dec 2024 15:13:49 -0800 Subject: [PATCH 5/6] test fix --- python/langsmith/client.py | 6 +-- .../unit_tests/evaluation/test_runner.py | 48 ++++++++++++++----- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/python/langsmith/client.py b/python/langsmith/client.py index 82c25e8da..0f39aa9c0 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -3843,10 +3843,9 @@ def read_example( "presigned_url": value["presigned_url"], "reader": reader, } - del example["attachment_urls"] return ls_schemas.Example( - **example, + **{k: v for k, v in example.items() if k != "attachment_urls"}, attachments=attachments, _host_url=self._host_url, _tenant_id=self._get_optional_tenant_id(), @@ -3930,10 +3929,9 @@ def list_examples( "presigned_url": value["presigned_url"], "reader": reader, } - del example["attachment_urls"] yield ls_schemas.Example( - **example, + **{k: v for k, v in example.items() if k != "attachment_urls"}, attachments=attachments, _host_url=self._host_url, _tenant_id=self._get_optional_tenant_id(), diff --git a/python/tests/unit_tests/evaluation/test_runner.py b/python/tests/unit_tests/evaluation/test_runner.py index a1e2d79de..87ebe6042 100644 --- a/python/tests/unit_tests/evaluation/test_runner.py +++ b/python/tests/unit_tests/evaluation/test_runner.py @@ -11,7 +11,7 @@ import uuid from datetime import datetime, timezone from threading import Lock -from typing import Callable, List +from typing import Any, Callable, Dict, List, Tuple from unittest import mock from unittest.mock import MagicMock @@ -53,7 +53,9 @@ def request(self, verb: str, endpoint: str, *args, **kwargs): return res elif endpoint == "http://localhost:1984/examples": res = MagicMock() - res.json.return_value = [e.dict() for e in self.ds_examples] + res.json.return_value = [ + e.dict() if not isinstance(e, dict) else e for e in self.ds_examples + ] return res elif endpoint == "http://localhost:1984/sessions": res = {} # type: ignore @@ -143,14 +145,23 @@ def _wait_until(condition: Callable, timeout: int = 8): raise TimeoutError("Condition not met") -def _create_example(idx: int) -> ls_schemas.Example: +def _create_example(idx: int) -> Tuple[ls_schemas.Example, Dict[str, Any]]: + _id = uuid.uuid4() + _created_at = datetime.now(timezone.utc) return ls_schemas.Example( - id=uuid.uuid4(), + id=_id, inputs={"in": idx}, outputs={"answer": idx + 1}, dataset_id="00886375-eb2a-4038-9032-efff60309896", - created_at=datetime.now(timezone.utc), - ) + created_at=_created_at, + ), { + "id": _id, + "dataset_id": "00886375-eb2a-4038-9032-efff60309896", + "created_at": _created_at, + "inputs": {"in": idx}, + "outputs": {"answer": idx + 1}, + "attachment_urls": None, + } @pytest.mark.skipif(sys.version_info < (3, 9), reason="requires python3.9 or higher") @@ -166,10 +177,13 @@ def test_evaluate_results( SPLIT_SIZE = 3 NUM_REPETITIONS = 4 - ds_examples = [_create_example(i) for i in range(10)] + ds_example_responses = [_create_example(i) for i in range(10)] + ds_examples = [e[0] for e in ds_example_responses] dev_split = random.sample(ds_examples, SPLIT_SIZE) tenant_id = str(uuid.uuid4()) - fake_request = FakeRequest(ds_id, ds_name, ds_examples, tenant_id) + fake_request = FakeRequest( + ds_id, ds_name, [e[1] for e in ds_example_responses], tenant_id + ) session.request = fake_request.request client = Client( api_url="http://localhost:1984", @@ -393,7 +407,12 @@ def eval2(x, y, inputs): _normalize_evaluator_func(eval_) with pytest.raises(ValueError, match="Invalid evaluator function."): - evaluate((lambda x: x), data=ds_examples, evaluators=[eval_], client=client) + evaluate( + (lambda inputs: inputs), + data=ds_examples, + evaluators=[eval_], + client=client, + ) def test_evaluate_raises_for_async(): @@ -437,10 +456,13 @@ async def test_aevaluate_results( SPLIT_SIZE = 3 NUM_REPETITIONS = 4 - ds_examples = [_create_example(i) for i in range(10)] + ds_example_responses = [_create_example(i) for i in range(10)] + ds_examples = [e[0] for e in ds_example_responses] dev_split = random.sample(ds_examples, SPLIT_SIZE) tenant_id = str(uuid.uuid4()) - fake_request = FakeRequest(ds_id, ds_name, ds_examples, tenant_id) + fake_request = FakeRequest( + ds_id, ds_name, [e[1] for e in ds_example_responses], tenant_id + ) session.request = fake_request.request client = Client( api_url="http://localhost:1984", @@ -664,8 +686,8 @@ async def eval2(x, y, inputs): evaluators = [eval1, eval2] - async def atarget(x): - return x + async def atarget(inputs): + return inputs for eval_ in evaluators: with pytest.raises(ValueError, match="Invalid evaluator function."): From 5471e88b566fac494f056caf11dc42a92ebbdd9f Mon Sep 17 00:00:00 2001 From: isaac hershenson Date: Mon, 9 Dec 2024 15:17:37 -0800 Subject: [PATCH 6/6] fmt --- python/langsmith/schemas.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/langsmith/schemas.py b/python/langsmith/schemas.py index 5b226a830..30a65a018 100644 --- a/python/langsmith/schemas.py +++ b/python/langsmith/schemas.py @@ -5,7 +5,6 @@ from datetime import datetime, timedelta, timezone from decimal import Decimal from enum import Enum -from pathlib import Path from typing import ( Any, Dict, @@ -64,7 +63,7 @@ def my_function(bar: int, my_val: Attachment): data: bytes -Attachments = Dict[str, Union[Tuple[str, bytes], Attachment, Tuple[str, Path]]] +Attachments = Dict[str, Union[Tuple[str, bytes], Attachment]] """Attachments associated with the run. Each entry is a tuple of (mime_type, bytes), or (mime_type, file_path)"""