Skip to content

Commit

Permalink
add compression to serializing numpy arrays
Browse files Browse the repository at this point in the history
  • Loading branch information
CPrescher committed Jul 24, 2024
1 parent 78bead5 commit 5a87808
Showing 1 changed file with 7 additions and 2 deletions.
9 changes: 7 additions & 2 deletions glassure/pattern.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from pydantic import PlainSerializer, PlainValidator, BaseModel
import numpy as np
import base64
import gzip

from dataclasses import dataclass

Expand Down Expand Up @@ -331,6 +332,7 @@ def validate(value):
if isinstance(value, str):
try:
v = base64.b64decode(value)
v = gzip.decompress(v)
numpy_array = np.load(io.BytesIO(v), allow_pickle=False)
return numpy_array
except Exception as e:
Expand All @@ -342,10 +344,13 @@ def validate(value):

def serialize(value):
if isinstance(value, np.ndarray):
# Save numpy array to bytes
# Save numpy array to compressed bytyes
with io.BytesIO() as buffer:
np.save(buffer, value, allow_pickle=False)
return base64.b64encode(buffer.getvalue()).decode("utf-8")
binary_data = buffer.getvalue()
compressed_data = gzip.compress(binary_data)
base_64_data = base64.b64encode(compressed_data).decode("utf-8")
return base_64_data
raise TypeError(f"Invalid type for numpy array: {type(value)}")


Expand Down

0 comments on commit 5a87808

Please sign in to comment.