From ca6943155426695b353dbca719fbf1bb5c4b2405 Mon Sep 17 00:00:00 2001 From: Denis Biryukov Date: Tue, 8 Oct 2024 16:55:12 +0200 Subject: [PATCH 1/4] serialization update for c/pico migration guide --- content/docs/migration_1.0/C_Pico.md | 254 ++++++++++++++++----------- 1 file changed, 147 insertions(+), 107 deletions(-) diff --git a/content/docs/migration_1.0/C_Pico.md b/content/docs/migration_1.0/C_Pico.md index e5b57f61..daf57002 100644 --- a/content/docs/migration_1.0/C_Pico.md +++ b/content/docs/migration_1.0/C_Pico.md @@ -48,7 +48,7 @@ Owned types support move semantics, which will consume the owned object and turn ### Moved types -Moved types are obtained when using `z_move` on an owned type object. They are consumed on use when passed to relevant functions. Any non-constructor function accepting a moved object (i.e. an object passed by owned pointer) becomes responsible for calling drop on it. The object is guaranteed to be in the null state upon such function return, even if it fails. +Moved types are obtained when using `z_move` on an owned type object. They are consumed on use when passed to relevant functions. Any non-constructor function accepting a moved object (i.e. an object passed by moved pointer) becomes responsible for calling drop on it. The object is guaranteed to be in the null state upon such function return, even if it fails. ### Loaned types @@ -127,111 +127,169 @@ if (z_declare_subscriber(&sub, z_loan(session), z_loan(ke), z_move(callback), NU ## Payload and Serialization -Zenoh 1.0.0 handles payload differently. Before one would pass the pointer to the buffer and its length, now everything must be serialized into `z_owned_bytes_t`. +Zenoh 1.0.0 handles payload differently. Before one would pass the pointer to the buffer and its length, now everything must be converted/serialized into `z_owned_bytes_t`. -To simplify serialization/deserialization we provide support for some primitive types like `uint8_t*` + length, (null-)terminated strings and arithmetic types. +Raw data in the form null-terminated strings or buffers (i.e. `uint8_t*` + length) can be converted directly into `z_owned_bytes_t` +by (optionaly) avoiding a copy as follows: - Zenoh 0.11.x ```c -char *value = "Some data to publish on Zenoh"; +int8_t send_string() { + char *value = "Some data to publish on Zenoh"; -if (z_put(z_loan(session), z_keyexpr(ke), (const uint8_t *)value, strlen(value), NULL) < 0) { - return -1; + if (z_put(z_loan(session), z_keyexpr(ke), (const uint8_t *)value, strlen(value), NULL) < 0) { + return -1; + } + return 0; +} + +int8_t send_buf() { + uint8_t *buf = (uint8_t*)z_malloc(16); + // fill the buf + if (z_put(z_loan(session), z_keyexpr(ke), buf, 16, NULL) < 0) { + return -1; + } + return 0; } ``` - Zenoh 1.0.0 ```c -char *value = "Some data to publish on Zenoh"; -z_owned_bytes_t payload; -z_bytes_serialize_from_str(&payload, value); +int8_t send_string() { + char *value = "Some data to publish on Zenoh"; + z_owned_bytes_t payload; + z_bytes_copy_from_str(&payload, value); // this copeies value to the payload -if (z_put(z_loan(session), z_loan(ke), z_move(payload), NULL) < 0) { - return -1; + if (z_put(z_loan(session), z_loan(ke), z_move(payload), NULL) < 0) { + return -1; + } + // z_bytes_to_string can be used to convert received z_loaned_bytes_t into z_owned_string_t + return 0; } -``` -To implement custom (de-)serialization functions, Zenoh 1.0.0 provides `z_bytes_iterator_t`, `z_bytes_reader_t` and `z_owned_bytes_wrtiter_t` types and corresponding functions. -Alternatively, it is always possible to perform serialization separately and only send/receive `uint8_t` arrays, by only calling trivial `z_bytes_serialize_from_slice` and `z_bytes_deserialize_into_slice` functions: +void receive_string(const z_loaned_bytes_t* payload) { + z_owned_string_t s; + z_bytes_to_string(payload, &s); -```c -void send_data(const z_loaned_publisher_t* pub, const uint8_t *data, size_t len) { - z_owned_bytes_t payload; - z_bytes_serialize_from_buf(&payload, data, len); - z_publisher_put(pub, z_move(payload), NULL); - // no need to drop the payload, since it is consumed by z_publisher_put + // do something with the string + // raw ptr can be accessed via z_string_data(z_loan(s)) + // data length can be accessed via z_string_len(z_loan(s)) + + // in the end string should be dropped since it contains a copy of the payload data + z_drop(z_move(s)); } -void receive_data(const z_loaned_bytes_t* payload) { - z_owned_slice_t slice; - z_bytes_deserialize_into_slice(payload, &slice); - // do something with serialized data - // raw ptr can be accessed via z_slice_data(z_loan(slice)) - // data length can be accessed via z_slice_len(z_loan(slice)) +int8_t void send_buf() { + uint8_t *buf = (uint8_t*)z_malloc(16); + // fill the buf + z_bytes_from_buf(&payload, buf, 16, my_custom_delete_function, NULL); // this moves buf into the payload + // my_custom_delete_function will be called to free the buf, once the corresponding data is send + // alternatively z_bytes_copy_from_buf(&payload, buf, 16) can be used, if coying the buf is required. + if (z_put(z_loan(session), z_loan(ke), z_move(payload), NULL) < 0) { + return -1; + } + // z_bytes_to_slice can be used to convert received z_loaned_bytes_t into z_owned_slice_t + return 0; +} - // in the end slice should be dropped since it contains a copy of the payload data - z_drop(z_move(slice)); +/// possible my_custom_delete_function implementation +void my_custom_delete_function(void *data, void* context) { + // perform delete of data by optionally using extra information in the context + free(data); } -``` -Note that it is no longer possible to access the underlying payload data pointer directly, since Zenoh cannot guarantee that the data is delivered as a single fragment. -So in order to get access to raw payload data one must use `z_bytes_reader_t` and related functions: +void receive_buf(const z_loaned_bytes_t* payload) { + z_owned_slice_t s; + z_bytes_to_slice(payload, &s); -```c -z_bytes_reader_t reader = z_bytes_get_reader(z_loan(payload)); -uint8_t data1[10] = {0}; -uint8_t data2[20] = {0}; + // do something with the string + // raw ptr can be accessed via z_slice_data(z_loan(s)) + // data length can be accessed via z_slice_len(z_loan(s)) + + // in the end string should be dropped since it contains a copy of the payload data + z_drop(z_move(s)); +} -z_bytes_reader_read(&reader, data1, 10); // copy first 10 payload bytes to data1 -z_bytes_reader_read(&reader, data2, 20); // copy next 20 payload bytes to data2 ``` -Note that all `z_bytes_serialize_from…` functions involve copying the data. -On the other hand, it is also possible to allow Zenoh to consume your data directly (which avoids the need to make an extra copy) using `z_bytes_from_buf` or `z_bytes_from_str`. -The user would need to provide a delete function to be called on data, when Zenoh finishes its processing: +The structured data can be serialized into `z_owned_bytes_t` by using provided serialization functionality. Zenoh provides +support for serializing arithmetic types, strings, sequences and tuples. + + +More comprehensive serialization/deserialization examples are provided in +https://github.com/eclipse-zenoh/zenoh-c/blob/main/examples/z_bytes.c and https://github.com/eclipse-zenoh/zenoh-pico/blob/main/examples/unix/c11/z_bytes.c. +To simplify serialization/deserialization we provide support for some primitive types like `uint8_t*` + length, null-terminated strings and arithmetic types. +Primitive types can be serialized directly into `z_owned_bytes_t`: ```c -void my_custom_delete_function(void *data, void* context) { - // perform delete of data by optionally using extra information in the context - free(data); -} + // Serialization + uint32_t input_u32 = 1234; + ze_serialize_uint32(&payload, input_u32); -void send_move_data(const z_loaned_publisher_t *publisher) { - uint8_t *my_data = malloc(10); - // fill my_data as necessary - z_owned_bytes_t b; - z_bytes_from_buf(&b, my_data, 10, my_custom_delete_function, NULL); - z_publisher_put(publisher, z_move(b)); -} + // Deserialization + uint32_t output_u32 = 0; + ze_deserialize_uint32(z_loan(payload), &output_u32); + // now output_u32 = 1234 + z_drop(z_move(payload)); -// an example of sending a data with more complex destructor -// a case of std::vector from c++ stl library +``` +while tuples and/or arrays require usage of `ze_owned_serializer_t` and `ze_deserializer_t`: +```c + typedef struct my_struct_t { + float f; + int32_t n; + } my_struct_t; -void delete_vector(void *data, void* context) { - std::vector *v = (std::vector *)context; - delete v; - // in this case data pointer is not used for destruction -} + ... -void send_move_vector(std::vector *v, const z_loaned_publisher_t *publisher) { - z_owned_bytes_t b; - z_bytes_from_buf(&b, v.data(), v.size(), delete_vector, (void*)v); - z_publisher_put(publisher, z_move(b)); -} + // Serialization + my_struct_t input_vec[] = {{1.5f, 1}, {2.4f, 2}, {-3.1f, 3}, {4.2f, 4}}; + ze_owned_serializer_t serializer; + ze_serializer_empty(&serializer); + ze_serializer_serialize_sequence_length(z_loan_mut(serializer), 4); + for (size_t i = 0; i < 4; ++i) { + ze_serializer_serialize_float(z_loan_mut(serializer), input_vec[i].f); + ze_serializer_serialize_int32(z_loan_mut(serializer), input_vec[i].n); + } + ze_serializer_finish(z_move(serializer), &payload); + + // Deserialization + my_struct_t output_vec[4] = {0}; + ze_deserializer_t deserializer = ze_deserializer_from_bytes(z_loan(payload)); + size_t num_elements = 0; + ze_deserializer_deserialize_sequence_length(&deserializer, &num_elements); + assert(num_elements == 4); + for (size_t i = 0; i < num_elements; ++i) { + ze_deserializer_deserialize_float(&deserializer, &output_vec[i].f); + ze_deserializer_deserialize_int32(&deserializer, &output_vec[i].n); + } + // now output_vec = {{1.5f, 1}, {2.4f, 2}, {-3.1f, 3}, {4.2f, 4}} + z_drop(z_move(payload)); ``` -Yet another alternative is to send the statically allocated constant data (hence that does not require to be deleted) without making an extra copy. This can be achieved using the `z_serialize_from_static_buf` or `z_serialize_from_static_str` functions: + +To implement custom (de-)serialization functions, Zenoh 1.0.0 provides `ze_owned_bytes_serializer`, `ze_bytes_deserializer_t` or lower-level `z_owned_bytes_wrtiter_t` and `ze_bytes_reader_t` types and corresponding functions. + +Note that it is no longer possible to access the underlying payload data pointer directly, since Zenoh cannot guarantee that the data is delivered as a single fragment. +So in order to get access to raw payload data one must use either `z_bytes_reader_t` or alternatively `z_bytes_slice_iterator_t` and their related functions: ```c -const char *my_constant_string = "my string"; +z_bytes_reader_t reader = z_bytes_get_reader(z_loan(payload)); +uint8_t data1[10] = {0}; +uint8_t data2[20] = {0}; + +z_bytes_reader_read(&reader, data1, 10); // copy first 10 payload bytes to data1 +z_bytes_reader_read(&reader, data2, 20); // copy next 20 payload bytes to data2 -void send_static_data(const z_loaned_publisher_t *publisher) { - z_owned_bytes_t b; - z_bytes_from_static_str(&b, my_constant_string); - z_publisher_put(publisher, z_move(b)); +// or +z_bytes_slice_iterator_t slice_iter = z_bytes_get_slice_iterator(z_bytes_loan(&payload)); +z_view_slice_t curr_slice; +while (z_bytes_slice_iterator_next(&slice_iter, &curr_slice)) { + // curr_slice provides a view on the corresponding fragment bytes. + // Note that there is no guarantee regarding each individual slice size } ``` @@ -428,7 +486,7 @@ void data_handler(const z_sample_t* sample, void* arg) { ``` -In 1.0.0, attachments were greatly simplified. They are now represented as `z_..._bytes_t` (i.e. the same type we use to represent serialized data) and can thus contain data in any format. +In 1.0.0, attachments were greatly simplified. They are now represented as `z_..._bytes_t` (i.e. the same type we use to represent payload data) and can thus contain data in any format. ```c // publish attachment @@ -437,39 +495,27 @@ typedef struct { char* value; } kv_pair_t; -typedef struct kv_it { - kv_pair_t *current; - kv_pair_t *end; -} kv_it; - -bool create_attachment_iter(z_owned_bytes_t* kv_pair, void* context) { - kv_it* it = (kv_it*)(context); - if (it->current == it->end) { - return false; - } - z_owned_bytes_t k, v; - // serialize as key-value pair - z_bytes_serialize_from_str(&k, it->current->key); - z_bytes_serialize_from_str(&v, it->current->value); - z_bytes_serialize_from_pair(kv_pair, z_move(k), z_move(v)); - it->current++; - return true; -}; - ... kv_pair_t attachment_kvs[2] = {; (kv_pair_t){.key = "index", .value = "1"}, (kv_pair_t){.key = "source", .value = "C"} } -kv_it it = { .begin = attachment_kvs, .end = attachment_kvs + 2 }; z_owned_bytes_t payload, attachment; -// serialzie key value pairs as attachment using z_bytes_serialize_from_iter -z_bytes_serialize_from_iter(&attachment, create_attachment_iter, (void*)&it); +// serialzie key value pairs as attachment + +ze_owned_serializer_t serializer; +ze_serializer_empty(&serializer); +ze_serializer_serialize_sequence_length(z_loan_mut(serializer), 2); +for (size_t i = 0; i < 2; ++i) { + ze_serializer_serialize_str(z_loan_mut(serializer), attachment_kvs[i].key); + ze_serializer_serialize_str(z_loan_mut(serializer), attachment_kvs[i].value); +} +ze_serializer_finish(z_move(serializer), &payload); options.attachment = &attachment; -z_bytes_serialize_from_str(&payload, "payload"); +z_bytes_copy_from_str(&payload, "payload"); z_publisher_put(z_loan(pub), z_move(payload), &options); // receive sample with attachment @@ -493,22 +539,16 @@ void data_handler(const z_loaned_sample_t *sample, void *arg) { } // read attachment key-value pairs using bytes_iterator - z_bytes_iterator_t iter = z_bytes_get_iterator(attachment); - z_owned_bytes_t kv; - while (z_bytes_iterator_next(&iter, &kv)) { - z_owned_bytes_t k, v; - z_owned_string_t key, value; - z_bytes_deserialize_into_pair(z_loan(kv), &k, &v); - - z_bytes_deserialize_into_string(z_loan(k), &key); - z_bytes_deserialize_into_string(z_loan(v), &value); - + ze_deserializer_t deserializer = ze_deserializer_from_bytes(z_loan(payload)); + size_t num_elements = 0; + ze_deserializer_deserialize_sequence_length(&deserializer, &num_elements); + z_owned_string_t key, value; + for (size_t i = 0; i < num_elements; ++i) { + ze_deserializer_deserialize_string(&deserializer, &key); + ze_deserializer_deserialize_string(&deserializer, &value); printf(" attachment: %.*s: '%.*s'\n", (int)z_string_len(z_loan(key)), z_string_data(z_loan(key)), (int)z_string_len(z_loan(value)), z_string_data(z_loan(value))); - z_drop(z_move(kv)); - z_drop(z_move(k)); - z_drop(z_move(v)); z_drop(z_move(key)); z_drop(z_move(value)); } @@ -700,7 +740,7 @@ In zenoh-pico, we recommend users follow the accessor pattern even though the st ## Usage of `z_bytes_clone` -In short, `z_bytes_t` is made of reference-counted data slices. In 1.0.0, we aligned the implementation of `z_bytes_clone` and made it perform a shallow copy for improved efficiency. +In short, `z_owned_bytes_t` is made of reference-counted data slices. In 1.0.0, we aligned the implementation of `z_bytes_clone` and made it perform a shallow copy for improved efficiency. - Zenoh 0.11.x From 70d8ffff0f10b6359faf9a30bee52dd402ee65ed Mon Sep 17 00:00:00 2001 From: Denis Biryukov Date: Tue, 8 Oct 2024 17:58:15 +0200 Subject: [PATCH 2/4] serialization update for cpp migration guide --- content/docs/migration_1.0/C++.md | 127 +++++++++++++++++------------- 1 file changed, 71 insertions(+), 56 deletions(-) diff --git a/content/docs/migration_1.0/C++.md b/content/docs/migration_1.0/C++.md index 745e55fa..861fc759 100644 --- a/content/docs/migration_1.0/C++.md +++ b/content/docs/migration_1.0/C++.md @@ -65,7 +65,7 @@ try { All returned and `std::move`'d-in objects are guaranteed to be left in an “empty” state in case of function call failure. -## Serialization and Deserialization +## Payload In version 0.11.0 it was only possible to send `std::string`/ `const char*` or `std::vector` / `uint8_t*` using the `BytesView` class: @@ -73,72 +73,87 @@ In version 0.11.0 it was only possible to send `std::string`/ `const char*` or ` publisher.put("my_payload"); ``` -In 1.0.0, the `BytesView` class is gone and we introduced the `Bytes` object which represents a serialized payload. - - +In 1.0.0, the `BytesView` class is gone and we introduced the `Bytes` object which represents a (serialized) payload. +Similarly to 0.11.0 it can be used to store raw bytes or strings: ```cpp -void publish_data(const Publisher& publisher, const MyData& data) { - publisher.put(Bytes::serialize(data)); +void publish_string(const Publisher& publisher, const std::string& data) { + publisher.put(Bytes(data)); +} + +void publish_string_without_copy(const Publisher& publisher, std::string&& data) { + publisher.put(Bytes(data)); } -void receive_data(const Sample &sample) { +void receive_string(const Sample &sample) { std::cout <<"Received: " - << sample.get_payload().deserialize() + << sample.get_payload().as_string() << "\n"; }; + +void publish_bytes(const Publisher& publisher, const std::vector& data) { + publisher.put(Bytes(data)); +} + +void publish_bytes_without_copy(const Publisher& publisher, std::vector&& data) { + publisher.put(Bytes(data)); +} + +void receive_bytes(const Sample &sample) { + std::vector = sample.get_payload().as_vector(); +}; ``` -We added a default `ZenohCodec`, which provides default serialization / deserialization for common numerical types, strings, and containers: +Additionaly `zenoh::ext` namespace provides support for serialization/deserialziation of typed data to/into `Bytes`: ```cpp - // stream of bytes serialization - std::vector data = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; - Bytes b = Bytes::serialize(data); - assert(b.deserialize>() == data); - - // arithmetic type serialization + // arithmetic types double pi = 3.1415926; - Bytes b = Bytes::serialize(pi); - assert(b.deserialize() == pi); + Bytes b = ext::serialize(pi); + assert(ext::deserialize(b) == pi); - // Composite types serialization - std::vector v = {0.1f, .2f, 0.3f}; - auto b = Bytes::serialize(v); - assert(b.deserialize() == v); + // Composite types + std::vector v = {0.1f, 0.2f, 0.3f}; + b = ext::serialize(v); + assert(ext::deserialize(b) == v); - std::map> m = { + std::unordered_map> m = { {"a", {0.5, 0.2}}, {"b", {-123.45, 0.4}}, {"abc", {3.1415926, -1.0} } }; - b = Bytes::serialize(m); - assert(b.deserialize() == m); - - // alternatively serialize via move - // the string keys will not be copied in this case, but rather move into Bytes - b_move = Bytes::serialize(std::move(m)); + b = ext::serialize(m); + assert(ext::deserialize(b) == m); ``` -Please note that this serialization functionality is only provided for prototyping and demonstration purposes and, as such, might be less efficient than custom-written serialization methods. - -Users can easily define their own serialization/deserialization functions by providing a custom codec: +Users can easily define serialization/deserialization for their own custom types by using `ext::Serializer` and `ext::Deserializer` classes. ```cpp -MyType my_data(...); -MyCodec my_codec(...); -Bytes b = Bytes::serialize(my_data, my_codec); -// or Bytes::serialize(my_data) if MyCodec is a stateless codec with an empty constructor -assert(b.deserialize>(my_codec) == my_data); -// or assert(b.deserialize>() == my_data); if MyCodec is a stateless with an empty constructor -``` +struct CustomStruct { + std::vector vd; + int32_t i; + std::string s; +}; -For finer control on serialization / deserialization implementation `Bytes::Iterator`, `Bytes::Writer` and `Bytes::Reader` classes are also introduced. +// One needs to implement __zenoh_serialize_with_serializer in the same namespace as CustomStruct +void __zenoh_serialize_with_serializer(ext::Serializer& serializer, const CustomStruct& s) { + serializer.serialize(s.vd); + serializer.serialize(s.i); + serializer.serialize(s.s); +} -A working example with custom-defined serialization / deserialization can be found here: +void serialize_custom() { + CustomStruct s = {{0.1, 0.2, -1000.55}, 32, "test"}; + Bytes b = ext::serialize(s); + CustomStruct s_out = ext::deserialize(b); + assert(s.vd == s_out.vd); + assert(s.i == s_out.i); + assert(s.s == s_out.s); +} +``` -https://github.com/eclipse-zenoh/zenoh-cpp/blob/dev/1.0.0/examples/simple/universal/z_simple.cxx +For lower level access to the `Bytes` content `Bytes::Reader`, `Bytes::Writer` and `Bytes::SliceIterator` classes can be used. ## Stream Handlers and Callbacks @@ -157,7 +172,7 @@ Reply reply(nullptr); for (recv(reply); reply.check(); recv(reply)) { auto sample = expect(reply.get()); std::cout << "Received ('" << sample.get_keyexpr().as_string_view() << "' : '" - << sample.get_payload().as_string_view() << "')\n"; + << sample.get_payload().as_string() << "')\n"; } // non-blocking @@ -169,7 +184,7 @@ for (bool call_success = recv(reply); !call_success || reply.check(); call_succe } auto sample = expect(reply.get()); std::cout << "\nReceived ('" << sample.get_keyexpr().as_string_view() << "' : '" - << sample.get_payload().as_string_view() << "')"; + << sample.get_payload().as_string() << "')"; } ``` @@ -194,7 +209,7 @@ auto replies = session.get( for (auto res = replies.recv(); std::has_alternative(res); res = replies.recv()) { const auto& sample = std::get(res).get_ok(); std::cout << "Received ('" << sample.get_keyexpr().as_string_view() << "' : '" - << sample.get_payload().deserialize() << "')\n"; + << sample.get_payload().as_string() << "')\n"; } // non-blocking while (true) { @@ -202,7 +217,7 @@ while (true) { if (std::has_alternative(res)) { const auto& sample = std::get(res).get_ok(); std::cout << "Received ('" << sample.get_keyexpr().as_string_view() << "' : '" - << sample.get_payload().deserialize() << "')\n"; + << sample.get_payload().as_string() << "')\n"; } else if (std::get(res) == channels::RecvError::Z_NODATA) { // try_recv is non-blocking call, so may fail to return a reply if the Fifo buffer is empty std::cout << "."; @@ -223,7 +238,7 @@ The same works for `Subscriber` and `Queryable`: auto data_callback = [](const Sample &sample) { std::cout << ">> [Subscriber] Received ('" << sample.get_keyexpr().as_string_view() - << "' : '" << sample.get_payload().deserialize() + << "' : '" << sample.get_payload().as_string() << "')\n"; }; @@ -245,7 +260,7 @@ for (auto res = messages.recv(); std::has_alternative(res); res = messag // it will return an empty sample and alive=false once subscriber gets disconnected const Sample& sample = std::get(res); std::cout << "Received ('" << sample.get_keyexpr().as_string_view() << "' : '" - << sample.get_payload().deserialize() << "')\n"; + << sample.get_payload().as_string() << "')\n"; } // non-blocking while (true) { @@ -253,7 +268,7 @@ while (true) { if (std::has_alternative(res)) { const auto& sample = std::get(res); std::cout << "Received ('" << sample.get_keyexpr().as_string_view() << "' : '" - << sample.get_payload().deserialize() << "')\n"; + << sample.get_payload().as_string() << "')\n"; } else if (std::get(res) == channels::RecvError::Z_NODATA) { // try_recv is non-blocking call, so may fail to return a sample if the Fifo buffer is empty std::cout << "."; @@ -281,8 +296,8 @@ options.set_attachment(attachment_map); pub.put(s, options); // subscriber callback function receiving message with attachment -data_handler(const Sample &sample) { - std::cout << ">> [Subscriber] Received " ('" +void data_handler(const Sample &sample) { + std::cout << ">> [Subscriber] Received \" ('" << sample.get_keyexpr().as_string_view() << "' : '" << sample.get_payload().as_string_view() @@ -317,8 +332,8 @@ std::unordered_map attachment_map = { {"index", "0"} }; pub.put( - Bytes::serialize("my_payload"), - {.encoding = Encoding("text/plain"), .attachment = std::move(attachment_map)} + Bytes("my_payload"), + {.encoding = Encoding("text/plain"), .attachment = ext::serialize(attachment_map)} ); @@ -327,13 +342,13 @@ void data_handler(const Sample &sample) { std::cout << ">> [Subscriber] Received ('" << sample.get_keyexpr().as_string_view() << "' : '" - << sample.get_payload().deserialize() + << sample.get_payload().as_string() << "')\n"; auto attachment = sample.get_attachment(); if (!attachment.has_value()) return; // we expect attachment in the form of key-value pairs - auto attachment_deserialized = attachment->get().deserialize>(); - for (auto&& [key, value]: attachment) { + auto attachment_deserialized = ext::deserialize>(attachment->get()); + for (auto&& [key, value]: attachment_deserialized) { std::cout << " attachment: " << key << ": '" << value << "'\n"; } }; @@ -359,5 +374,5 @@ session.get(keyexpr, "", {on_reply, on_done}, opts); In 1.0.0: ```cpp -session.get(keyexpr, "", on_reply, on_done, {.target = Z_QUERY_TARGET_ALL, .payload = Bytes::serialize(value)}); +session.get(keyexpr, "", on_reply, on_done, {.target = Z_QUERY_TARGET_ALL, .payload = ext::serialize(value)}); ``` \ No newline at end of file From bdaeba1aa80f82fd49e2241b8cef3900a82601d6 Mon Sep 17 00:00:00 2001 From: Denis Biryukov Date: Tue, 8 Oct 2024 18:12:22 +0200 Subject: [PATCH 3/4] format --- content/docs/migration_1.0/C++.md | 66 ++++---- content/docs/migration_1.0/C_Pico.md | 232 +++++++++++++-------------- 2 files changed, 149 insertions(+), 149 deletions(-) diff --git a/content/docs/migration_1.0/C++.md b/content/docs/migration_1.0/C++.md index 861fc759..bb9c7d9d 100644 --- a/content/docs/migration_1.0/C++.md +++ b/content/docs/migration_1.0/C++.md @@ -127,33 +127,33 @@ Additionaly `zenoh::ext` namespace provides support for serialization/deserialzi assert(ext::deserialize(b) == m); ``` -Users can easily define serialization/deserialization for their own custom types by using `ext::Serializer` and `ext::Deserializer` classes. +Users can easily define serialization/deserialization for their own custom types by using `ext::Serializer` and `ext::Deserializer` classes: ```cpp struct CustomStruct { - std::vector vd; - int32_t i; - std::string s; + std::vector vd; + int32_t i; + std::string s; }; // One needs to implement __zenoh_serialize_with_serializer in the same namespace as CustomStruct void __zenoh_serialize_with_serializer(ext::Serializer& serializer, const CustomStruct& s) { - serializer.serialize(s.vd); - serializer.serialize(s.i); - serializer.serialize(s.s); + serializer.serialize(s.vd); + serializer.serialize(s.i); + serializer.serialize(s.s); } void serialize_custom() { - CustomStruct s = {{0.1, 0.2, -1000.55}, 32, "test"}; - Bytes b = ext::serialize(s); - CustomStruct s_out = ext::deserialize(b); - assert(s.vd == s_out.vd); - assert(s.i == s_out.i); - assert(s.s == s_out.s); + CustomStruct s = {{0.1, 0.2, -1000.55}, 32, "test"}; + Bytes b = ext::serialize(s); + CustomStruct s_out = ext::deserialize(b); + assert(s.vd == s_out.vd); + assert(s.i == s_out.i); + assert(s.s == s_out.s); } ``` -For lower level access to the `Bytes` content `Bytes::Reader`, `Bytes::Writer` and `Bytes::SliceIterator` classes can be used. +For lower-level access to the `Bytes` content `Bytes::Reader`, `Bytes::Writer` and `Bytes::SliceIterator` classes can be used. ## Stream Handlers and Callbacks @@ -170,9 +170,9 @@ session.get(keyexpr, "", std::move(send), opts); Reply reply(nullptr); // blocking for (recv(reply); reply.check(); recv(reply)) { - auto sample = expect(reply.get()); - std::cout << "Received ('" << sample.get_keyexpr().as_string_view() << "' : '" - << sample.get_payload().as_string() << "')\n"; + auto sample = expect(reply.get()); + std::cout << "Received ('" << sample.get_keyexpr().as_string_view() << "' : '" + << sample.get_payload().as_string() << "')\n"; } // non-blocking @@ -256,11 +256,11 @@ auto subscriber = session.declare_subscriber(keyexpr, channels::FifoChannel(16)) const auto& messages = subscriber.handler(); //blocking for (auto res = messages.recv(); std::has_alternative(res); res = messages.recv()) { - // recv will block until there is at least one sample in the Fifo buffer - // it will return an empty sample and alive=false once subscriber gets disconnected - const Sample& sample = std::get(res); - std::cout << "Received ('" << sample.get_keyexpr().as_string_view() << "' : '" - << sample.get_payload().as_string() << "')\n"; + // recv will block until there is at least one sample in the Fifo buffer + // it will return an empty sample and alive=false once subscriber gets disconnected + const Sample& sample = std::get(res); + std::cout << "Received ('" << sample.get_keyexpr().as_string_view() << "' : '" + << sample.get_payload().as_string() << "')\n"; } // non-blocking while (true) { @@ -303,17 +303,17 @@ void data_handler(const Sample &sample) { << sample.get_payload().as_string_view() << "')\n"; if (sample.get_attachment().check()) { - // reads full attachment - sample.get_attachment().iterate([](const BytesView &key, const BytesView &value) -> bool { - std::cout << " attachment: " << key.as_string_view() << ": '" << value.as_string_view() << "'\n"; - return true; - }); - - // or read particular attachment item - auto index = sample.get_attachment().get("index"); - if (index != "") { - std::cout << " message number: " << index.as_string_view() << std::endl; - } + // reads full attachment + sample.get_attachment().iterate([](const BytesView &key, const BytesView &value) -> bool { + std::cout << " attachment: " << key.as_string_view() << ": '" << value.as_string_view() << "'\n"; + return true; + }); + + // or read particular attachment item + auto index = sample.get_attachment().get("index"); + if (index != "") { + std::cout << " message number: " << index.as_string_view() << std::endl; + } } }; ``` diff --git a/content/docs/migration_1.0/C_Pico.md b/content/docs/migration_1.0/C_Pico.md index daf57002..5840e7a8 100644 --- a/content/docs/migration_1.0/C_Pico.md +++ b/content/docs/migration_1.0/C_Pico.md @@ -129,8 +129,8 @@ if (z_declare_subscriber(&sub, z_loan(session), z_loan(ke), z_move(callback), NU Zenoh 1.0.0 handles payload differently. Before one would pass the pointer to the buffer and its length, now everything must be converted/serialized into `z_owned_bytes_t`. -Raw data in the form null-terminated strings or buffers (i.e. `uint8_t*` + length) can be converted directly into `z_owned_bytes_t` -by (optionaly) avoiding a copy as follows: +Raw data in the form of null-terminated strings or buffers (i.e. `uint8_t*` + length) can be converted directly into `z_owned_bytes_t` +as follows: - Zenoh 0.11.x @@ -225,49 +225,49 @@ https://github.com/eclipse-zenoh/zenoh-c/blob/main/examples/z_bytes.c and https: To simplify serialization/deserialization we provide support for some primitive types like `uint8_t*` + length, null-terminated strings and arithmetic types. Primitive types can be serialized directly into `z_owned_bytes_t`: ```c - // Serialization - uint32_t input_u32 = 1234; - ze_serialize_uint32(&payload, input_u32); +// Serialization +uint32_t input_u32 = 1234; +ze_serialize_uint32(&payload, input_u32); - // Deserialization - uint32_t output_u32 = 0; - ze_deserialize_uint32(z_loan(payload), &output_u32); - // now output_u32 = 1234 - z_drop(z_move(payload)); +// Deserialization +uint32_t output_u32 = 0; +ze_deserialize_uint32(z_loan(payload), &output_u32); +// now output_u32 = 1234 +z_drop(z_move(payload)); ``` while tuples and/or arrays require usage of `ze_owned_serializer_t` and `ze_deserializer_t`: ```c - typedef struct my_struct_t { - float f; - int32_t n; - } my_struct_t; - - ... - - // Serialization - my_struct_t input_vec[] = {{1.5f, 1}, {2.4f, 2}, {-3.1f, 3}, {4.2f, 4}}; - ze_owned_serializer_t serializer; - ze_serializer_empty(&serializer); - ze_serializer_serialize_sequence_length(z_loan_mut(serializer), 4); - for (size_t i = 0; i < 4; ++i) { - ze_serializer_serialize_float(z_loan_mut(serializer), input_vec[i].f); - ze_serializer_serialize_int32(z_loan_mut(serializer), input_vec[i].n); - } - ze_serializer_finish(z_move(serializer), &payload); +typedef struct my_struct_t { + float f; + int32_t n; +} my_struct_t; - // Deserialization - my_struct_t output_vec[4] = {0}; - ze_deserializer_t deserializer = ze_deserializer_from_bytes(z_loan(payload)); - size_t num_elements = 0; - ze_deserializer_deserialize_sequence_length(&deserializer, &num_elements); - assert(num_elements == 4); - for (size_t i = 0; i < num_elements; ++i) { - ze_deserializer_deserialize_float(&deserializer, &output_vec[i].f); - ze_deserializer_deserialize_int32(&deserializer, &output_vec[i].n); - } - // now output_vec = {{1.5f, 1}, {2.4f, 2}, {-3.1f, 3}, {4.2f, 4}} - z_drop(z_move(payload)); +... + +// Serialization +my_struct_t input_vec[] = {{1.5f, 1}, {2.4f, 2}, {-3.1f, 3}, {4.2f, 4}}; +ze_owned_serializer_t serializer; +ze_serializer_empty(&serializer); +ze_serializer_serialize_sequence_length(z_loan_mut(serializer), 4); +for (size_t i = 0; i < 4; ++i) { + ze_serializer_serialize_float(z_loan_mut(serializer), input_vec[i].f); + ze_serializer_serialize_int32(z_loan_mut(serializer), input_vec[i].n); +} +ze_serializer_finish(z_move(serializer), &payload); + +// Deserialization +my_struct_t output_vec[4] = {0}; +ze_deserializer_t deserializer = ze_deserializer_from_bytes(z_loan(payload)); +size_t num_elements = 0; +ze_deserializer_deserialize_sequence_length(&deserializer, &num_elements); +assert(num_elements == 4); +for (size_t i = 0; i < num_elements; ++i) { + ze_deserializer_deserialize_float(&deserializer, &output_vec[i].f); + ze_deserializer_deserialize_int32(&deserializer, &output_vec[i].n); +} +// now output_vec = {{1.5f, 1}, {2.4f, 2}, {-3.1f, 3}, {4.2f, 4}} +z_drop(z_move(payload)); ``` @@ -288,8 +288,8 @@ z_bytes_reader_read(&reader, data2, 20); // copy next 20 payload bytes to data2 z_bytes_slice_iterator_t slice_iter = z_bytes_get_slice_iterator(z_bytes_loan(&payload)); z_view_slice_t curr_slice; while (z_bytes_slice_iterator_next(&slice_iter, &curr_slice)) { - // curr_slice provides a view on the corresponding fragment bytes. - // Note that there is no guarantee regarding each individual slice size + // curr_slice provides a view on the corresponding fragment bytes. + // Note that there is no guarantee regarding each individual slice size } ``` @@ -308,13 +308,13 @@ z_owned_reply_channel_t channel = zc_reply_fifo_new(16); z_get(z_loan(session), z_keyexpr(keyexpr), "", z_move(channel.send), &opts); z_owned_reply_t reply = z_reply_null(); for (z_call(channel.recv, &reply); z_check(reply); z_call(channel.recv, &reply)) { - if (z_reply_is_ok(&reply)) { - z_sample_t sample = z_reply_ok(&reply); - // do something with sample and keystr - } else { - printf("Received an error\n"); - } - z_drop(z_move(reply)); + if (z_reply_is_ok(&reply)) { + z_sample_t sample = z_reply_ok(&reply); + // do something with sample and keystr + } else { + printf("Received an error\n"); + } + z_drop(z_move(reply)); } z_drop(z_move(channel)); @@ -323,17 +323,17 @@ z_owned_reply_channel_t channel = zc_reply_non_blocking_fifo_new(16); z_get(z_loan(s), keyexpr, "", z_move(channel.send), &opts); z_owned_reply_t reply = z_reply_null(); for (bool call_success = z_call(channel.recv, &reply); !call_success || z_check(reply); - call_success = z_call(channel.recv, &reply)) { - if (!call_success) { - continue; - } - if (z_reply_is_ok(z_loan(reply))) { - const z_loaned_sample_t *sample = z_reply_ok(&reply); - // do something with sample - } else { - printf("Received an error\n"); - } - z_drop(z_move(reply)); + call_success = z_call(channel.recv, &reply)) { + if (!call_success) { + continue; + } + if (z_reply_is_ok(z_loan(reply))) { + const z_loaned_sample_t *sample = z_reply_ok(&reply); + // do something with sample + } else { + printf("Received an error\n"); + } + z_drop(z_move(reply)); } z_drop(z_move(channel)); ``` @@ -357,33 +357,33 @@ z_owned_reply_t reply; // blocking while (z_recv(z_loan(handler), &reply) == Z_OK) { - // z_recv will block until there is at least one sample in the Fifo buffer - if (z_reply_is_ok(z_loan(reply))) { - const z_loaned_sample_t *sample = z_reply_ok(z_loan(reply)); - // do something with sample - } else { - printf("Received an error\n"); - } - z_drop(z_move(reply)); + // z_recv will block until there is at least one sample in the Fifo buffer + if (z_reply_is_ok(z_loan(reply))) { + const z_loaned_sample_t *sample = z_reply_ok(z_loan(reply)); + // do something with sample + } else { + printf("Received an error\n"); + } + z_drop(z_move(reply)); } // non-blocking while (true) { - z_result_t res = z_try_recv(z_loan(handler), &reply); - if (res == Z_CHANNEL_NODATA) { - // z_try_recv is non-blocking call, so will fail to return a reply if the Fifo buffer is empty - // do some other work or just sleep - } else if (res == Z_OK) { - if (z_reply_is_ok(z_loan(reply))) { - const z_loaned_sample_t *sample = z_reply_ok(z_loan(reply)); - // do something with sample - } else { - printf("Received an error\n"); - } - z_drop(z_move(reply)); - } else { // res == Z_CHANNEL_DISCONNECTED - break; // channel is closed - no more replies will arrive - } + z_result_t res = z_try_recv(z_loan(handler), &reply); + if (res == Z_CHANNEL_NODATA) { + // z_try_recv is non-blocking call, so will fail to return a reply if the Fifo buffer is empty + // do some other work or just sleep + } else if (res == Z_OK) { + if (z_reply_is_ok(z_loan(reply))) { + const z_loaned_sample_t *sample = z_reply_ok(z_loan(reply)); + // do something with sample + } else { + printf("Received an error\n"); + } + z_drop(z_move(reply)); + } else { // res == Z_CHANNEL_DISCONNECTED + break; // channel is closed - no more replies will arrive + } } ``` @@ -427,16 +427,16 @@ while (z_recv(z_loan(handler), &sample) == Z_OK) { // non-blocking while (true) { - z_result_t res = z_try_recv(z_loan(handler), &sample); - if (res == Z_CHANNEL_NODATA) { - // z_try_recv is non-blocking call, so will fail to return a sample if the Fifo buffer is empty - // do some other work or just sleep - } else if (res == Z_OK) { - // do something with sample - z_drop(z_move(sample)); - } else { // res == Z_CHANNEL_DISCONNECTED - break; // channel is closed - no more samples will be received - } + z_result_t res = z_try_recv(z_loan(handler), &sample); + if (res == Z_CHANNEL_NODATA) { + // z_try_recv is non-blocking call, so will fail to return a sample if the Fifo buffer is empty + // do some other work or just sleep + } else if (res == Z_OK) { + // do something with sample + z_drop(z_move(sample)); + } else { // res == Z_CHANNEL_DISCONNECTED + break; // channel is closed - no more samples will be received + } } ``` @@ -457,7 +457,7 @@ z_bytes_map_insert_by_alias(&map, _z_bytes_wrap((uint8_t *)"test", 2), _z_bytes_ options.attachment = z_bytes_map_as_attachment(&map); if (z_put(z_loan(s), z_keyexpr(keyexpr), (const uint8_t *)value, strlen(value), &options) < 0) { - return -1; + return -1; } z_bytes_map_drop(&map); @@ -465,23 +465,23 @@ z_bytes_map_drop(&map); // receive sample with attachment int8_t attachment_reader(z_bytes_t key, z_bytes_t val, void* ctx) { - printf(" attachment: %.*s: '%.*s'\n", (int)key.len, key.start, (int)val.len, val.start); - return 0; + printf(" attachment: %.*s: '%.*s'\n", (int)key.len, key.start, (int)val.len, val.start); + return 0; } void data_handler(const z_sample_t* sample, void* arg) { - // checks if attachment exists - if (z_check(sample->attachment)) { - // reads full attachment - z_attachment_iterate(sample->attachment, attachment_reader, NULL); - - // reads particular attachment item - z_bytes_t index = z_attachment_get(sample->attachment, z_bytes_from_str("index")); - if (z_check(index)) { - printf(" message number: %.*s\n", (int)index.len, index.start); - } + // checks if attachment exists + if (z_check(sample->attachment)) { + // reads full attachment + z_attachment_iterate(sample->attachment, attachment_reader, NULL); + + // reads particular attachment item + z_bytes_t index = z_attachment_get(sample->attachment, z_bytes_from_str("index")); + if (z_check(index)) { + printf(" message number: %.*s\n", (int)index.len, index.start); } - z_drop(z_move(keystr)); + } + z_drop(z_move(keystr)); } ``` @@ -544,13 +544,13 @@ void data_handler(const z_loaned_sample_t *sample, void *arg) { ze_deserializer_deserialize_sequence_length(&deserializer, &num_elements); z_owned_string_t key, value; for (size_t i = 0; i < num_elements; ++i) { - ze_deserializer_deserialize_string(&deserializer, &key); - ze_deserializer_deserialize_string(&deserializer, &value); - printf(" attachment: %.*s: '%.*s'\n", - (int)z_string_len(z_loan(key)), z_string_data(z_loan(key)), - (int)z_string_len(z_loan(value)), z_string_data(z_loan(value))); - z_drop(z_move(key)); - z_drop(z_move(value)); + ze_deserializer_deserialize_string(&deserializer, &key); + ze_deserializer_deserialize_string(&deserializer, &value); + printf(" attachment: %.*s: '%.*s'\n", + (int)z_string_len(z_loan(key)), z_string_data(z_loan(key)), + (int)z_string_len(z_loan(value)), z_string_data(z_loan(value))); + z_drop(z_move(key)); + z_drop(z_move(value)); } } ``` @@ -783,7 +783,7 @@ const size_t size = 1024 * 1024; const z_id_t id = z_info_zid(z_loan(s)); char idstr[33]; for (int i = 0; i < 16; i++) { - sprintf(idstr + 2 * i, "%02x", id.id[i]); + sprintf(idstr + 2 * i, "%02x", id.id[i]); } idstr[32] = 0; // create SHM manager @@ -840,7 +840,7 @@ z_buf_layout_alloc_result_t alloc; // allocation behavior and handle allocation failures automatically z_shm_provider_alloc_gc(&alloc, z_loan(provider), alloc_size, alignment); if (!z_check(alloc.buf)) { - printf("Failed to allocate an SHM buffer, even after GCing\n"); - exit(-1); + printf("Failed to allocate an SHM buffer, even after GCing\n"); + exit(-1); } ``` From 6ccccd0d2217ba627239ec950ec54fa891fd613d Mon Sep 17 00:00:00 2001 From: Denis Biryukov Date: Wed, 9 Oct 2024 10:37:39 +0200 Subject: [PATCH 4/4] review fixes --- content/docs/migration_1.0/C++.md | 6 +++--- content/docs/migration_1.0/C_Pico.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/content/docs/migration_1.0/C++.md b/content/docs/migration_1.0/C++.md index bb9c7d9d..8e3f678b 100644 --- a/content/docs/migration_1.0/C++.md +++ b/content/docs/migration_1.0/C++.md @@ -104,7 +104,7 @@ void receive_bytes(const Sample &sample) { }; ``` -Additionaly `zenoh::ext` namespace provides support for serialization/deserialziation of typed data to/into `Bytes`: +Additionally `zenoh::ext` namespace provides support for serialization/deserialziation of typed data to/into `Bytes`: ```cpp // arithmetic types @@ -115,7 +115,7 @@ Additionaly `zenoh::ext` namespace provides support for serialization/deserialzi // Composite types std::vector v = {0.1f, 0.2f, 0.3f}; b = ext::serialize(v); - assert(ext::deserialize(b) == v); + assert(ext::deserialize>(b) == v); std::unordered_map> m = { {"a", {0.5, 0.2}}, @@ -124,7 +124,7 @@ Additionaly `zenoh::ext` namespace provides support for serialization/deserialzi }; b = ext::serialize(m); - assert(ext::deserialize(b) == m); + assert(ext::deserialize>>(b) == m); ``` Users can easily define serialization/deserialization for their own custom types by using `ext::Serializer` and `ext::Deserializer` classes: diff --git a/content/docs/migration_1.0/C_Pico.md b/content/docs/migration_1.0/C_Pico.md index 5840e7a8..7ac48c56 100644 --- a/content/docs/migration_1.0/C_Pico.md +++ b/content/docs/migration_1.0/C_Pico.md @@ -271,7 +271,7 @@ z_drop(z_move(payload)); ``` -To implement custom (de-)serialization functions, Zenoh 1.0.0 provides `ze_owned_bytes_serializer`, `ze_bytes_deserializer_t` or lower-level `z_owned_bytes_wrtiter_t` and `ze_bytes_reader_t` types and corresponding functions. +To implement custom (de-)serialization, Zenoh 1.0.0 provides `ze_owned_bytes_serializer`, `ze_bytes_deserializer_t` or lower-level `z_owned_bytes_wrtiter_t` and `z_bytes_reader_t` types and corresponding functions. Note that it is no longer possible to access the underlying payload data pointer directly, since Zenoh cannot guarantee that the data is delivered as a single fragment. So in order to get access to raw payload data one must use either `z_bytes_reader_t` or alternatively `z_bytes_slice_iterator_t` and their related functions: