Skip to content

Commit

Permalink
use memoryview
Browse files Browse the repository at this point in the history
  • Loading branch information
angus-langchain committed Dec 6, 2024
1 parent 0ac6cef commit 3077860
Showing 1 changed file with 12 additions and 5 deletions.
17 changes: 12 additions & 5 deletions python/langsmith/_internal/_operations.py
Original file line number Diff line number Diff line change
Expand Up @@ -321,6 +321,17 @@ def _yield_and_reset_buffer(self) -> Iterator[bytes]:
self.buffer.seek(0)
self.buffer.truncate()

def _process_bytes(
self,
compressor: zstd.ZstdCompressor.stream_writer,
data: Union[bytes, bytearray],
) -> Iterator[bytes]:
with memoryview(data) as view:
for i in range(0, len(view), self.blocksize):
chunk = view[i:i + self.blocksize]
compressor.write(chunk)
yield from self._yield_and_reset_buffer()

def compress_multipart_stream(
self,
parts_and_contexts: MultipartPartsAndContext,
Expand Down Expand Up @@ -349,11 +360,7 @@ def compress_multipart_stream(

# Write part data in chunks
if isinstance(data, (bytes, bytearray)):
for i in range(0, len(data), self.blocksize):
chunk = data[i:i + self.blocksize]
compressor.write(chunk)
# After writing each chunk, yield compressed data
yield from self._yield_and_reset_buffer()
yield from self._process_bytes(compressor, data)
else:
# Handle other data types
compressor.write(str(data).encode())
Expand Down

0 comments on commit 3077860

Please sign in to comment.