Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

update examples multipart #1310

Merged
merged 17 commits into from
Dec 10, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 84 additions & 5 deletions python/langsmith/client.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""Client for interacting with the LangSmith API.

Check notice on line 1 in python/langsmith/client.py

View workflow job for this annotation

GitHub Actions / benchmark

Benchmark results

........... create_5_000_run_trees: Mean +- std dev: 747 ms +- 70 ms ........... WARNING: the benchmark result may be unstable * the standard deviation (152 ms) is 11% of the mean (1.43 sec) Try to rerun the benchmark with more runs, values and/or loops. Run 'python -m pyperf system tune' command to reduce the system jitter. Use pyperf stats, pyperf dump and pyperf hist to analyze results. Use --quiet option to hide these warnings. create_10_000_run_trees: Mean +- std dev: 1.43 sec +- 0.15 sec ........... WARNING: the benchmark result may be unstable * the standard deviation (181 ms) is 12% of the mean (1.45 sec) Try to rerun the benchmark with more runs, values and/or loops. Run 'python -m pyperf system tune' command to reduce the system jitter. Use pyperf stats, pyperf dump and pyperf hist to analyze results. Use --quiet option to hide these warnings. create_20_000_run_trees: Mean +- std dev: 1.45 sec +- 0.18 sec ........... dumps_class_nested_py_branch_and_leaf_200x400: Mean +- std dev: 701 us +- 5 us ........... dumps_class_nested_py_leaf_50x100: Mean +- std dev: 25.7 ms +- 0.2 ms ........... dumps_class_nested_py_leaf_100x200: Mean +- std dev: 105 ms +- 2 ms ........... dumps_dataclass_nested_50x100: Mean +- std dev: 26.0 ms +- 0.5 ms ........... WARNING: the benchmark result may be unstable * the standard deviation (18.0 ms) is 24% of the mean (75.9 ms) Try to rerun the benchmark with more runs, values and/or loops. Run 'python -m pyperf system tune' command to reduce the system jitter. Use pyperf stats, pyperf dump and pyperf hist to analyze results. Use --quiet option to hide these warnings. dumps_pydantic_nested_50x100: Mean +- std dev: 75.9 ms +- 18.0 ms ........... dumps_pydanticv1_nested_50x100: Mean +- std dev: 204 ms +- 2 ms

Check notice on line 1 in python/langsmith/client.py

View workflow job for this annotation

GitHub Actions / benchmark

Comparison against main

+-----------------------------------------------+----------+------------------------+ | Benchmark | main | changes | +===============================================+==========+========================+ | dumps_pydanticv1_nested_50x100 | 216 ms | 204 ms: 1.06x faster | +-----------------------------------------------+----------+------------------------+ | dumps_class_nested_py_branch_and_leaf_200x400 | 701 us | 701 us: 1.00x slower | +-----------------------------------------------+----------+------------------------+ | dumps_class_nested_py_leaf_100x200 | 103 ms | 105 ms: 1.02x slower | +-----------------------------------------------+----------+------------------------+ | dumps_class_nested_py_leaf_50x100 | 24.8 ms | 25.7 ms: 1.03x slower | +-----------------------------------------------+----------+------------------------+ | dumps_dataclass_nested_50x100 | 25.2 ms | 26.0 ms: 1.03x slower | +-----------------------------------------------+----------+------------------------+ | create_10_000_run_trees | 1.37 sec | 1.43 sec: 1.04x slower | +-----------------------------------------------+----------+------------------------+ | create_20_000_run_trees | 1.37 sec | 1.45 sec: 1.06x slower | +-----------------------------------------------+----------+------------------------+ | create_5_000_run_trees | 703 ms | 747 ms: 1.06x slower | +-----------------------------------------------+----------+------------------------+ | dumps_pydantic_nested_50x100 | 65.1 ms | 75.9 ms: 1.17x slower | +-----------------------------------------------+----------+------------------------+ | Geometric mean | (ref) | 1.04x slower | +-----------------------------------------------+----------+------------------------+

Use the client to customize API keys / workspace ocnnections, SSl certs,
etc. for tracing.
Expand Down Expand Up @@ -3464,6 +3464,7 @@
examples: Union[
List[ls_schemas.ExampleUploadWithAttachments]
| List[ls_schemas.ExampleUpsertWithAttachments]
| List[ls_schemas.ExampleUpdateWithAttachments],
],
include_dataset_id: bool = False,
) -> Tuple[Any, bytes]:
Expand All @@ -3477,21 +3478,29 @@
dataset_id = examples[0].dataset_id

for example in examples:
if not isinstance(
example, ls_schemas.ExampleUploadWithAttachments
) and not isinstance(example, ls_schemas.ExampleUpsertWithAttachments):
if (
not isinstance(example, ls_schemas.ExampleUploadWithAttachments)
and not isinstance(example, ls_schemas.ExampleUpsertWithAttachments)
and not isinstance(example, ls_schemas.ExampleUpdateWithAttachments)
):
raise ValueError(
"The examples must be of type ExampleUploadWithAttachments"
" or ExampleUpsertWithAttachments"
" or ExampleUpdateWithAttachments"
)
if example.id is not None:
example_id = str(example.id)
else:
example_id = str(uuid.uuid4())

if isinstance(example, ls_schemas.ExampleUpdateWithAttachments):
created_at = None
else:
created_at = example.created_at

example_body = {
**({"dataset_id": dataset_id} if include_dataset_id else {}),
"created_at": example.created_at,
**({"created_at": created_at} if created_at is not None else {}),
}
if example.metadata is not None:
example_body["metadata"] = example.metadata
Expand Down Expand Up @@ -3582,6 +3591,23 @@
)
)

if (
isinstance(example, ls_schemas.ExampleUpdateWithAttachments)
and example.attachments_operations
):
attachments_operationsb = _dumps_json(example.attachments_operations)
parts.append(
(
f"{example_id}.attachments_operations",
(
None,
attachments_operationsb,
"application/json",
{},
),
)
)

encoder = rqtb_multipart.MultipartEncoder(parts, boundary=BOUNDARY)
if encoder.len <= 20_000_000: # ~20 MB
data = encoder.to_string()
Expand All @@ -3590,6 +3616,38 @@

return encoder, data

def update_examples_multipart(
self,
*,
dataset_id: ID_TYPE,
updates: Optional[List[ls_schemas.ExampleUpdateWithAttachments]] = None,
) -> ls_schemas.UpsertExamplesResponse:
"""Upload examples."""
if not (self.info.instance_flags or {}).get(
"dataset_examples_multipart_enabled", False
):
raise ValueError(
"Your LangSmith version does not allow using the multipart examples endpoint, please update to the latest version."
)
if updates is None:
updates = []

encoder, data = self._prepate_multipart_data(updates, include_dataset_id=False)

response = self.request_with_retries(
"PATCH",
f"/v1/platform/datasets/{dataset_id}/examples",
request_kwargs={
"data": data,
"headers": {
**self._headers,
"Content-Type": encoder.content_type,
},
},
)
ls_utils.raise_for_status_with_text(response)
return response.json()

def upload_examples_multipart(
self,
*,
Expand Down Expand Up @@ -4072,6 +4130,7 @@
metadata: Optional[Dict] = None,
split: Optional[str | List[str]] = None,
dataset_id: Optional[ID_TYPE] = None,
attachments_operations: Optional[ls_schemas.AttachmentsOperations] = None,
) -> Dict[str, Any]:
"""Update a specific example.

Expand All @@ -4096,12 +4155,20 @@
Dict[str, Any]
The updated example.
"""
if attachments_operations is not None:
if not (self.info.instance_flags or {}).get(
"dataset_examples_multipart_enabled", False
):
raise ValueError(
"Your LangSmith version does not allow using the attachment operations, please update to the latest version."
)
example = dict(
inputs=inputs,
outputs=outputs,
dataset_id=dataset_id,
metadata=metadata,
split=split,
attachments_operations=attachments_operations,
)
response = self.request_with_retries(
"PATCH",
Expand All @@ -4121,6 +4188,9 @@
metadata: Optional[Sequence[Optional[Dict]]] = None,
splits: Optional[Sequence[Optional[str | List[str]]]] = None,
dataset_ids: Optional[Sequence[Optional[ID_TYPE]]] = None,
attachments_operations: Optional[
Sequence[Optional[ls_schemas.AttachmentsOperations]]
] = None,
) -> Dict[str, Any]:
"""Update multiple examples.

Expand All @@ -4145,12 +4215,20 @@
Dict[str, Any]
The response from the server (specifies the number of examples updated).
"""
if attachments_operations is not None:
if not (self.info.instance_flags or {}).get(
"dataset_examples_multipart_enabled", False
):
raise ValueError(
"Your LangSmith version does not allow using the attachment operations, please update to the latest version."
)
sequence_args = {
"inputs": inputs,
"outputs": outputs,
"metadata": metadata,
"splits": splits,
"dataset_ids": dataset_ids,
"attachments_operations": attachments_operations,
}
# Since inputs are required, we will check against them
examples_len = len(example_ids)
Expand All @@ -4168,8 +4246,9 @@
"dataset_id": dataset_id_,
"metadata": metadata_,
"split": split_,
"attachments_operations": attachments_operations_,
}
for id_, in_, out_, metadata_, split_, dataset_id_ in zip(
for id_, in_, out_, metadata_, split_, dataset_id_, attachments_operations_ in zip(
example_ids,
inputs or [None] * len(example_ids),
outputs or [None] * len(example_ids),
Expand Down
17 changes: 17 additions & 0 deletions python/langsmith/schemas.py
Original file line number Diff line number Diff line change
Expand Up @@ -183,12 +183,24 @@ class ExampleSearch(ExampleBase):
id: UUID


class AttachmentsOperations(BaseModel):
"""Operations to perform on attachments."""

rename: Dict[str, str] = Field(
default_factory=dict, description="Mapping of old attachment names to new names"
)
retain: List[str] = Field(
default_factory=list, description="List of attachment names to keep"
)


class ExampleUpdate(BaseModel):
"""Update class for Example."""

dataset_id: Optional[UUID] = None
inputs: Optional[Dict[str, Any]] = None
outputs: Optional[Dict[str, Any]] = None
attachments_operations: Optional[AttachmentsOperations] = None
metadata: Optional[Dict[str, Any]] = None
split: Optional[Union[str, List[str]]] = None

Expand All @@ -202,7 +214,12 @@ class ExampleUpdateWithAttachments(ExampleUpdate):
"""Example update with attachments."""

id: UUID
inputs: Dict[str, Any] = Field(default_factory=dict)
outputs: Optional[Dict[str, Any]] = Field(default=None)
metadata: Optional[Dict[str, Any]] = Field(default=None)
split: Optional[Union[str, List[str]]] = None
attachments: Optional[Attachments] = None
attachments_operations: Optional[AttachmentsOperations] = None


class DataType(str, Enum):
Expand Down
Loading
Loading