Skip to content

Commit

Permalink
checkpoint-postgres: handle null chars in metadata (#1885)
Browse files Browse the repository at this point in the history
  • Loading branch information
vbarda authored Sep 27, 2024
1 parent 947c74c commit bf19dc7
Show file tree
Hide file tree
Showing 3 changed files with 21 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -229,7 +229,8 @@ def _load_metadata(self, metadata: dict[str, Any]) -> CheckpointMetadata:

def _dump_metadata(self, metadata: CheckpointMetadata) -> str:
serialized_metadata = self.jsonplus_serde.dumps(metadata)
return serialized_metadata.decode()
# NOTE: we're using JSON serializer (not msgpack), so we need to remove null characters before writing
return serialized_metadata.decode().replace("\\u0000", "")

def get_next_version(self, current: Optional[str], channel: ChannelProtocol) -> str:
if current is None:
Expand Down
10 changes: 10 additions & 0 deletions libs/checkpoint-postgres/tests/test_async.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,3 +101,13 @@ async def test_asearch(self) -> None:
} == {"", "inner"}

# TODO: test before and limit params

async def test_null_chars(self) -> None:
async with AsyncPostgresSaver.from_conn_string(DEFAULT_URI) as saver:
config = await saver.aput(
self.config_1, self.chkpnt_1, {"my_key": "\x00abc"}, {}
)
assert (await saver.aget_tuple(config)).metadata["my_key"] == "abc"
assert [c async for c in saver.alist(None, filter={"my_key": "abc"})][
0
].metadata["my_key"] == "abc"
9 changes: 9 additions & 0 deletions libs/checkpoint-postgres/tests/test_sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,3 +101,12 @@ def test_search(self) -> None:
} == {"", "inner"}

# TODO: test before and limit params

def test_null_chars(self) -> None:
with PostgresSaver.from_conn_string(DEFAULT_URI) as saver:
config = saver.put(self.config_1, self.chkpnt_1, {"my_key": "\x00abc"}, {})
assert saver.get_tuple(config).metadata["my_key"] == "abc"
assert (
list(saver.list(None, filter={"my_key": "abc"}))[0].metadata["my_key"]
== "abc"
)

0 comments on commit bf19dc7

Please sign in to comment.