From 68f88f7ae2f913ce5ba1e2068446f034a4ad2ec1 Mon Sep 17 00:00:00 2001 From: Joseph Perez Date: Fri, 4 Oct 2024 08:53:41 +0200 Subject: [PATCH 01/10] docs: update Rust/Python migration guide for serialization --- content/docs/migration_1.0/Python.md | 72 +++++++++++++++++----------- content/docs/migration_1.0/Rust.md | 38 +++++++++++---- 2 files changed, 73 insertions(+), 37 deletions(-) diff --git a/content/docs/migration_1.0/Python.md b/content/docs/migration_1.0/Python.md index eef5189f..76800c42 100644 --- a/content/docs/migration_1.0/Python.md +++ b/content/docs/migration_1.0/Python.md @@ -37,50 +37,68 @@ with zenoh.open(zenoh.Config()) as session: session.declare_subscriber("my/keyepxr", lambda s: print(s)) sleep(10) # subscriber stays in background and its callback can be called # `session.close()` will be called at the end of the block, and it will undeclare the subscriber -```¬ +``` -## ZBytes, encoding, and (de)serialization +## Value is gone, long live ZBytes -### Encoding +`Value` has been split into `ZBytes` and `Encoding`. `put` and other operations now requires a `ZBytes` payload, and builders accept an optional `Encoding` parameter. -`zenoh.Value` has been split into `zenoh.ZBytes` and `zenoh.Encoding`. Put and other operations now require a `ZBytes` payload, and accept an optional `Encoding`; the encoding is no longer automatically deduced from the payload type. +`ZBytes` is a raw bytes container. It can be created directly from raw bytes/strings using `ZBytes` constructor. Then bytes can be retrieved using `ZBytes.to_bytes` or `ZBytes.to_string`. Sample payload is now a `ZBytes` instead of `bytes`. -```python -session.put("my/keyexpr", 42) # default encoding `zenoh/bytes`session.put("my/keyexpr", 42, encoding=zenoh.Encoding.ZENOH_INT64) +- Zenoh 0.11.x + +```rust +sample = subscriber.recv() +my_string = sample.payload.decode("utf-8") ``` -Publishers can be declared with a default encoding, which will be used for each put operation. +- Zenoh 1.0.0 + +```rust +sample = subscriber.recv() +my_string = sample.payload.to_string() +``` + +You can look at a full set of examples in `examples/z_bytes.py`. + +### Serialization + +Zenoh does provide serialization for convenience as an extension in `zenoh.ext` module. Serialization is implemented for a bunch of standard types like `int`, `float`, `list`, `dict`, `tuple`, etc. and is used through functions `z_serialize`/`z_deserialize`. ```python -import json -publisher = session.declare_publisher("my/keyepxr", encoding=zenoh.Encoding.APPLICATION_JSON) -publisher.put(json.dumps({"key", "value"})) # default encoding from publisher `application/json` +input = b"raw bytes" +payload = ZBytes(input) +output = payload.to_bytes() ``` -### (De)serialization +`zenoh.ext` serialization doesn't pretend to cover every use cases, as it is just one available choice among other serialization format like JSON, Protobuf, CBOR, etc. In the end, Zenoh will just send and receive payload raw bytes independently of the serialization used. + +NOTE: ⚠️ Serialization of `bytes` is not the same as passing `bytes` to `ZBytes` constructor. + +## Encoding -Arbitrary types can be serialized to and deserialized from `ZBytes`. Default (de)serializers are provided for builtin types; `list`/`dict` are **no longer** serialized to JSON, they use instead the builtin serializer of Zenoh, which is compatible with other Zenoh bindings. +`Encoding` has been reworked. +Zenoh does not impose any encoding requirement on the user, nor does it operate on it. +It can be thought of as optional metadata, carried over by Zenoh in such a way that the end user’s application may perform different operations based on encoding. + +NOTE: ⚠️ The encoding is no longer automatically deduced from the payload type. ```python -payload = zenoh.ZBytes(42) -assert payload.deserialize(int) == 42# `ZBytes.deserialize` accepts generic `list`/`dict` typepayload = zenoh.ZBytes([0.5, 37.1]) -assert payload.deserialize(list[float]) == [0.5, 37.1] +session.put(json.dumps({"key", "value"}), encoding=Encoding.APPLICATION_JSON) ``` -(De)serializers can be registered for custom types: +Users can also define their own encoding scheme that does not need to be based on the pre-defined variants. ```python -from dataclasses import dataclass -import zenoh -@dataclassclass RGB: - red: int green: int blue: int@zenoh.serializerdef serialize_rgb(rgb: RGB) -> zenoh.ZBytes: - return zenoh.ZBytes(rgb.red | (rgb.green << 8) | (rgb.blue << 16)) -@zenoh.deserializerdef deserialize_rgb(payload: zenoh.ZBytes) -> RGB: - compact = payload.deserialize(int) - return RGB(compact & 255, (compact >> 8) & 255, (compact >> 16) & 255) -color = RGB(61, 67, 97) -assert zenoh.ZBytes(color).deserialize(RGB) == color -# types with a registered serializer can be used directly with `put`session.put("my/keyexpr", color) +encoding = Encoding("pointcloud/LAS") +``` + +Because encoding is now optional for `put`, `Publisher` can be declared with a default encoding, which will be used in every `Publisher.put`. + +```python +publisher = session.declare_publisher("my/keyepxr", encoding=Encoding.APPLICATION_JSON) +// default encoding from publisher `application/json` +publisher.put(json.dumps({"key", "value"})) ``` ## Handlers diff --git a/content/docs/migration_1.0/Rust.md b/content/docs/migration_1.0/Rust.md index 3e63865d..1ce1b052 100644 --- a/content/docs/migration_1.0/Rust.md +++ b/content/docs/migration_1.0/Rust.md @@ -142,22 +142,16 @@ However, the normal user would rarely need to call this method directly.* ## Value is gone, long live ZBytes -We have replaced `Value` with `ZBytes` and `Encoding` , and added a number of conversion implementations such that user structs can be serialized into `ZBytes`, sent via Zenoh, and de-serialized from `ZBytes` with ease. +`Value` has been split into `ZBytes` and `Encoding`. `put` and other operations now requires a `ZBytes` payload, and builders accept an optional `Encoding` parameter. The encoding is no longer automatically deduced from the payload type. -This is facilitated through the `ZSerde` struct, and implementations of the traits -`zenoh::bytes::Deserialize` and `zenoh::bytes::Serialize`. +`ZBytes` is a raw bytes container, which can also contain non-contiguous region of memory. It can be created directly from raw bytes/strings using `ZBytes::from`. Then bytes can be retrieved using `ZBytes::to_bytes`, which returns a `Cow<[u8]>`, as a copy may have to be done if the underlying bytes are not contiguous. -We provide implementations of Zenoh’s aforementioned `Deserialize` and `Serialize` traits for primitive Rust types, Rust’s `Vec`, the `Value` type exposed by `Serde`'s various libraries as well as an example of `Protobuf` ’s `prost::Message` type. - -You can look at a full set of examples in `examples/examples/z_bytes.rs`. - -NOTE: ⚠️ `ZSerde` is not the only serializer/deserializer users can make use of, nor a limitation to the types supported by Zenoh. Users are free to use whichever serializer/deserializer they wish! - Zenoh 0.11.x ```rust let sample = subscriber.recv_async().await.unwrap(); let value: Value = sample.value; -let the_string: String = value.try_into().unwrap(); +let raw_bytes: Vec = value.try_into().unwrap(); ``` - Zenoh 1.0.0 @@ -165,9 +159,25 @@ let the_string: String = value.try_into().unwrap(); ```rust let sample = subscriber.recv_async().await.unwrap(); let zbytes: ZBytes = sample.payload(); -let the_string: String = zbytes.deserialize::().unwrap(); +let raw_bytes: Cow<[u8]> = zbytes.as_bytes(); +``` + +You can look at a full set of examples in `examples/examples/z_bytes.rs`. + +### Serialization + +Zenoh does provide serialization for convenience as an extension in `zenoh-ext`. Serialization is implemented for a bunch of standard types like integers, floats, `Vec`, `HashMap`, etc. and is used through functions `z_serialize`/`z_deserialize`. + +```rust +let input: Vec = vec![0.0, 1.5, 42.0]; +let payload: ZBytes = z_serialize(&input); +let output: Vec = z_deserialize(&payload).unwrap(); ``` +`zenoh-ext` serialization doesn't pretend to cover every use cases, as it is just one available choice among other serialization format like JSON, Protobuf, CBOR, etc. In the end, Zenoh will just send and receive payload raw bytes independently of the serialization used. + +NOTE: ⚠️ Serialization of `Vec` is not the same as creating a `ZBytes` from a `Vec`: the resulting `ZBytes` are different, and serialization doesn't take ownership of the bytes. + ## Encoding `Encoding` has been reworked. @@ -206,6 +216,14 @@ Users can also define their own encoding scheme that does not need to be based o let encoding = Encoding::from("pointcloud/LAS"); ``` +Because encoding is now optional for `put`, `Publisher` can be declared with a default encoding, which will be used in every `Publisher::put`. + +```rust +let publisher = session.declare_publisher("my/keyepxr").encoding(Encoding::APPLICATION_JSON).await.unwrap(); +// default encoding from publisher `application/json` +publisher.put(serde_json::to_vec(json!({"key", "value"})).unwrap()).await.unwrap(); +``` + ## Attachment In Zenoh 0.11.x, the `AttachmentBuilder` was required to create an attachment. From 7a4704590bf74d2dd34948816212469d50bb610a Mon Sep 17 00:00:00 2001 From: Joseph Perez Date: Wed, 9 Oct 2024 16:46:54 +0200 Subject: [PATCH 02/10] Update content/docs/migration_1.0/Python.md Co-authored-by: oteffahi <70609372+oteffahi@users.noreply.github.com> --- content/docs/migration_1.0/Python.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/docs/migration_1.0/Python.md b/content/docs/migration_1.0/Python.md index 76800c42..4ea834cd 100644 --- a/content/docs/migration_1.0/Python.md +++ b/content/docs/migration_1.0/Python.md @@ -41,7 +41,7 @@ with zenoh.open(zenoh.Config()) as session: ## Value is gone, long live ZBytes -`Value` has been split into `ZBytes` and `Encoding`. `put` and other operations now requires a `ZBytes` payload, and builders accept an optional `Encoding` parameter. +`Value` has been split into `ZBytes` and `Encoding`. `put` and other operations now require a `ZBytes` payload, and builders accept an optional `Encoding` parameter. `ZBytes` is a raw bytes container. It can be created directly from raw bytes/strings using `ZBytes` constructor. Then bytes can be retrieved using `ZBytes.to_bytes` or `ZBytes.to_string`. Sample payload is now a `ZBytes` instead of `bytes`. From 5e81517eff11bd9c25abd28a9903e9ceee140d13 Mon Sep 17 00:00:00 2001 From: Joseph Perez Date: Wed, 9 Oct 2024 16:47:11 +0200 Subject: [PATCH 03/10] Update content/docs/migration_1.0/Rust.md Co-authored-by: oteffahi <70609372+oteffahi@users.noreply.github.com> --- content/docs/migration_1.0/Rust.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/docs/migration_1.0/Rust.md b/content/docs/migration_1.0/Rust.md index 1ce1b052..2626cd2d 100644 --- a/content/docs/migration_1.0/Rust.md +++ b/content/docs/migration_1.0/Rust.md @@ -142,7 +142,7 @@ However, the normal user would rarely need to call this method directly.* ## Value is gone, long live ZBytes -`Value` has been split into `ZBytes` and `Encoding`. `put` and other operations now requires a `ZBytes` payload, and builders accept an optional `Encoding` parameter. The encoding is no longer automatically deduced from the payload type. +`Value` has been split into `ZBytes` and `Encoding`. `put` and other operations now require a `ZBytes` payload, and builders accept an optional `Encoding` parameter. The encoding is no longer automatically deduced from the payload type. `ZBytes` is a raw bytes container, which can also contain non-contiguous region of memory. It can be created directly from raw bytes/strings using `ZBytes::from`. Then bytes can be retrieved using `ZBytes::to_bytes`, which returns a `Cow<[u8]>`, as a copy may have to be done if the underlying bytes are not contiguous. From 59454badb6051e6de5b9977c9b18074283d5c841 Mon Sep 17 00:00:00 2001 From: Joseph Perez Date: Wed, 9 Oct 2024 16:47:23 +0200 Subject: [PATCH 04/10] Update content/docs/migration_1.0/Rust.md Co-authored-by: oteffahi <70609372+oteffahi@users.noreply.github.com> --- content/docs/migration_1.0/Rust.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/docs/migration_1.0/Rust.md b/content/docs/migration_1.0/Rust.md index 2626cd2d..00891a7f 100644 --- a/content/docs/migration_1.0/Rust.md +++ b/content/docs/migration_1.0/Rust.md @@ -144,7 +144,7 @@ However, the normal user would rarely need to call this method directly.* `Value` has been split into `ZBytes` and `Encoding`. `put` and other operations now require a `ZBytes` payload, and builders accept an optional `Encoding` parameter. The encoding is no longer automatically deduced from the payload type. -`ZBytes` is a raw bytes container, which can also contain non-contiguous region of memory. It can be created directly from raw bytes/strings using `ZBytes::from`. Then bytes can be retrieved using `ZBytes::to_bytes`, which returns a `Cow<[u8]>`, as a copy may have to be done if the underlying bytes are not contiguous. +`ZBytes` is a raw bytes container, which can also contain non-contiguous regions of memory. It can be created directly from raw bytes/strings using `ZBytes::from`. The bytes can be retrieved using `ZBytes::to_bytes`, which returns a `Cow<[u8]>`, as a copy may have to be done if the underlying bytes are not contiguous. - Zenoh 0.11.x From 9936ddf6451af56b27b2f39f7394926ad306e1c2 Mon Sep 17 00:00:00 2001 From: Joseph Perez Date: Wed, 9 Oct 2024 16:47:39 +0200 Subject: [PATCH 05/10] Update content/docs/migration_1.0/Rust.md Co-authored-by: oteffahi <70609372+oteffahi@users.noreply.github.com> --- content/docs/migration_1.0/Rust.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/docs/migration_1.0/Rust.md b/content/docs/migration_1.0/Rust.md index 00891a7f..14843fe6 100644 --- a/content/docs/migration_1.0/Rust.md +++ b/content/docs/migration_1.0/Rust.md @@ -166,7 +166,7 @@ You can look at a full set of examples in `examples/examples/z_bytes.rs`. ### Serialization -Zenoh does provide serialization for convenience as an extension in `zenoh-ext`. Serialization is implemented for a bunch of standard types like integers, floats, `Vec`, `HashMap`, etc. and is used through functions `z_serialize`/`z_deserialize`. +Zenoh does provide serialization for convenience as an extension in the `zenoh-ext` crate. Serialization is implemented for a bunch of standard types like integers, floats, `Vec`, `HashMap`, etc. and is used through functions `z_serialize`/`z_deserialize`. ```rust let input: Vec = vec![0.0, 1.5, 42.0]; From 034fc482c8fd0a90d1dc34a63c31afa484377b49 Mon Sep 17 00:00:00 2001 From: Joseph Perez Date: Wed, 9 Oct 2024 17:15:09 +0200 Subject: [PATCH 06/10] Update content/docs/migration_1.0/Python.md Co-authored-by: oteffahi <70609372+oteffahi@users.noreply.github.com> --- content/docs/migration_1.0/Python.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/docs/migration_1.0/Python.md b/content/docs/migration_1.0/Python.md index 4ea834cd..b8d20b12 100644 --- a/content/docs/migration_1.0/Python.md +++ b/content/docs/migration_1.0/Python.md @@ -71,7 +71,7 @@ payload = ZBytes(input) output = payload.to_bytes() ``` -`zenoh.ext` serialization doesn't pretend to cover every use cases, as it is just one available choice among other serialization format like JSON, Protobuf, CBOR, etc. In the end, Zenoh will just send and receive payload raw bytes independently of the serialization used. +`zenoh.ext` serialization doesn't pretend to cover all use cases, as it is just one available choice among other serialization formats like JSON, Protobuf, CBOR, etc. In the end, Zenoh will just send and receive payload raw bytes independently of the serialization used. NOTE: ⚠️ Serialization of `bytes` is not the same as passing `bytes` to `ZBytes` constructor. From c518e1ca0abb51c23d9055d20aaf105075415fa9 Mon Sep 17 00:00:00 2001 From: Joseph Perez Date: Wed, 9 Oct 2024 17:15:18 +0200 Subject: [PATCH 07/10] Update content/docs/migration_1.0/Python.md Co-authored-by: oteffahi <70609372+oteffahi@users.noreply.github.com> --- content/docs/migration_1.0/Python.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/docs/migration_1.0/Python.md b/content/docs/migration_1.0/Python.md index b8d20b12..8c1a2117 100644 --- a/content/docs/migration_1.0/Python.md +++ b/content/docs/migration_1.0/Python.md @@ -61,7 +61,7 @@ my_string = sample.payload.to_string() You can look at a full set of examples in `examples/z_bytes.py`. -### Serialization +## Serialization Zenoh does provide serialization for convenience as an extension in `zenoh.ext` module. Serialization is implemented for a bunch of standard types like `int`, `float`, `list`, `dict`, `tuple`, etc. and is used through functions `z_serialize`/`z_deserialize`. From e4749b5beba2344e3426c66b9a92ebff33596606 Mon Sep 17 00:00:00 2001 From: Joseph Perez Date: Wed, 9 Oct 2024 17:15:38 +0200 Subject: [PATCH 08/10] Update content/docs/migration_1.0/Rust.md Co-authored-by: oteffahi <70609372+oteffahi@users.noreply.github.com> --- content/docs/migration_1.0/Rust.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/docs/migration_1.0/Rust.md b/content/docs/migration_1.0/Rust.md index 14843fe6..db74aa13 100644 --- a/content/docs/migration_1.0/Rust.md +++ b/content/docs/migration_1.0/Rust.md @@ -174,7 +174,7 @@ let payload: ZBytes = z_serialize(&input); let output: Vec = z_deserialize(&payload).unwrap(); ``` -`zenoh-ext` serialization doesn't pretend to cover every use cases, as it is just one available choice among other serialization format like JSON, Protobuf, CBOR, etc. In the end, Zenoh will just send and receive payload raw bytes independently of the serialization used. +`zenoh-ext` serialization doesn't pretend to cover all use cases, as it is just one available choice among other serialization formats like JSON, Protobuf, CBOR, etc. In the end, Zenoh will just send and receive payload raw bytes independently of the serialization used. NOTE: ⚠️ Serialization of `Vec` is not the same as creating a `ZBytes` from a `Vec`: the resulting `ZBytes` are different, and serialization doesn't take ownership of the bytes. From 0dfc440b42c966f7f4c48eb86e70228608239e3a Mon Sep 17 00:00:00 2001 From: Joseph Perez Date: Wed, 9 Oct 2024 17:15:45 +0200 Subject: [PATCH 09/10] Update content/docs/migration_1.0/Rust.md Co-authored-by: oteffahi <70609372+oteffahi@users.noreply.github.com> --- content/docs/migration_1.0/Rust.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/content/docs/migration_1.0/Rust.md b/content/docs/migration_1.0/Rust.md index db74aa13..b3d64283 100644 --- a/content/docs/migration_1.0/Rust.md +++ b/content/docs/migration_1.0/Rust.md @@ -164,7 +164,7 @@ let raw_bytes: Cow<[u8]> = zbytes.as_bytes(); You can look at a full set of examples in `examples/examples/z_bytes.rs`. -### Serialization +## Serialization Zenoh does provide serialization for convenience as an extension in the `zenoh-ext` crate. Serialization is implemented for a bunch of standard types like integers, floats, `Vec`, `HashMap`, etc. and is used through functions `z_serialize`/`z_deserialize`. From 805ccb3407b781b1c89fa40ee2707802516fd192 Mon Sep 17 00:00:00 2001 From: Joseph Perez Date: Wed, 9 Oct 2024 17:29:24 +0200 Subject: [PATCH 10/10] fix: add links to `z_bytes` example --- content/docs/migration_1.0/Python.md | 2 +- content/docs/migration_1.0/Rust.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/content/docs/migration_1.0/Python.md b/content/docs/migration_1.0/Python.md index 8c1a2117..e4aee639 100644 --- a/content/docs/migration_1.0/Python.md +++ b/content/docs/migration_1.0/Python.md @@ -59,7 +59,7 @@ sample = subscriber.recv() my_string = sample.payload.to_string() ``` -You can look at a full set of examples in `examples/z_bytes.py`. +You can look at a full set of examples in [`examples/z_bytes.py`](https://github.com/eclipse-zenoh/zenoh-python/blob/1.0.0-beta.4/examples/z_bytes.py). ## Serialization diff --git a/content/docs/migration_1.0/Rust.md b/content/docs/migration_1.0/Rust.md index b3d64283..1a9b9b83 100644 --- a/content/docs/migration_1.0/Rust.md +++ b/content/docs/migration_1.0/Rust.md @@ -162,7 +162,7 @@ let zbytes: ZBytes = sample.payload(); let raw_bytes: Cow<[u8]> = zbytes.as_bytes(); ``` -You can look at a full set of examples in `examples/examples/z_bytes.rs`. +You can look at a full set of examples in [`examples/examples/z_bytes.rs`](https://github.com/eclipse-zenoh/zenoh/blob/1.0.0-beta.4/examples/examples/z_bytes.rs). ## Serialization