Skip to content

Commit

Permalink
small fix
Browse files Browse the repository at this point in the history
  • Loading branch information
isahers1 committed Dec 11, 2024
1 parent 1d7de25 commit 930b705
Show file tree
Hide file tree
Showing 2 changed files with 24 additions and 2 deletions.
4 changes: 4 additions & 0 deletions python/langsmith/evaluation/_arunner.py
Original file line number Diff line number Diff line change
Expand Up @@ -844,6 +844,10 @@ async def _arun_evaluators(
f" run {run.id}: {repr(e)}",
exc_info=True,
)
if example.attachments is not None:
for attachment in example.attachments:
reader = example.attachments[attachment]["reader"]
reader.seek(0)
return ExperimentResultRow(
run=run,
example=example,
Expand Down
22 changes: 20 additions & 2 deletions python/tests/integration_tests/test_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -1508,7 +1508,20 @@ async def target(
assert image_data.read() == b"fake image data for testing"
return {"answer": "test image"}

async def evaluator(
async def evaluator_1(
outputs: dict, reference_outputs: dict, attachments: dict
) -> Dict[str, Any]:
assert "image" in attachments
assert "presigned_url" in attachments["image"]
image_data = attachments["image"]["reader"]
assert image_data.read() == b"fake image data for testing"
return {
"score": float(
reference_outputs.get("answer") == outputs.get("answer") # type: ignore
)
}

async def evaluator_2(
outputs: dict, reference_outputs: dict, attachments: dict
) -> Dict[str, Any]:
assert "image" in attachments
Expand All @@ -1522,12 +1535,17 @@ async def evaluator(
}

results = await langchain_client.aevaluate(
target, data=dataset_name, evaluators=[evaluator], num_repetitions=2
target,
data=dataset_name,
evaluators=[evaluator_1, evaluator_2],
num_repetitions=2,
max_concurrency=3,
)

assert len(results) == 2
async for result in results:
assert result["evaluation_results"]["results"][0].score == 1.0
assert result["evaluation_results"]["results"][1].score == 1.0

langchain_client.delete_dataset(dataset_name=dataset_name)

Expand Down

0 comments on commit 930b705

Please sign in to comment.