From 98850b1cc99c782fa4f56176a65b121bae095151 Mon Sep 17 00:00:00 2001 From: Yoshiki Matsuda Date: Mon, 1 Apr 2024 09:33:00 +0900 Subject: [PATCH] WIP --- README.md | 334 +++++++++++++++++++++----------------- include/cpp_yyjson.hpp | 360 ++++++++++++++++++++++++++++------------- test/bench_write.cpp | 71 +++++++- test/test.cpp | 94 ++++++++--- 4 files changed, 572 insertions(+), 287 deletions(-) diff --git a/README.md b/README.md index 8687652..8773f8c 100644 --- a/README.md +++ b/README.md @@ -4,49 +4,50 @@ Ultra-fast and intuitive C++ JSON reader/writer with yyjson backend. [![CI](https://github.com/yosh-matsuda/cpp-yyjson/actions/workflows/tests.yml/badge.svg)](https://github.com/yosh-matsuda/cpp-yyjson/actions/workflows/tests.yml) -1. [Features](#features) -2. [Requirements](#requirements) -3. [Overview](#overview) - 1. [JSON Reader](#json-reader) - 2. [JSON Writer](#json-writer) - 3. [Serialization and Deserialization](#serialization-and-deserialization) -4. [Installation](#installation) - 1. [Using CMake](#using-cmake) -5. [Benchmark](#benchmark) - 1. [Read performance](#read-performance) - 2. [Write performance](#write-performance) -6. [Reference](#reference) - 1. [Namespaces](#namespaces) - 2. [Immutable JSON classes](#immutable-json-classes) - 3. [Mutable JSON classes](#mutable-json-classes) - 4. [Serialize and deserialize JSON](#serialize-and-deserialize-json) - 5. [Performance best practices](#performance-best-practices) - 6. [Misc](#misc) -7. [Author](#author) +1. [Features](#features) +2. [Requirements](#requirements) +3. [Overview](#overview) + 1. [JSON Reader](#json-reader) + 2. [JSON Writer](#json-writer) + 3. [Serialization and Deserialization](#serialization-and-deserialization) +4. [Installation](#installation) + 1. [Using CMake](#using-cmake) +5. [Benchmark](#benchmark) + 1. [Read performance](#read-performance) + 2. [Write performance](#write-performance) +6. [Reference](#reference) + 1. [Namespaces](#namespaces) + 2. [Immutable JSON classes](#immutable-json-classes) + 3. [Mutable JSON classes](#mutable-json-classes) + 4. [Allocator classes](#allocator-classes) + 5. [Serialize and deserialize JSON](#serialize-and-deserialize-json) + 6. [Performance best practices](#performance-best-practices) + 7. [Misc](#misc) +7. [Author](#author) ## Features -- Header-only -- C++20 range adaption -- STL-like accessors -- Intuitive JSON construction -- Mutual transformation of JSON and C++ classes with - - compile-time reflection of struct/class field name - - pre-defined STL casters - - user-defined casters in two ways -- Minimum overhead compared to yyjson -- Object lifetime safety +* Header-only +* C++20 range adaption +* STL-like accessors +* Intuitive JSON construction +* Mutual transformation of JSON and C++ classes with + * compile-time reflection of struct/class field name + * pre-defined STL casters + * user-defined casters in two ways +* Minimum overhead compared to yyjson +* Object lifetime safety ## Requirements -- C++20 compiler with range supports - - LLVM >= 16 - - GCC >= 12 - - clang-cl >= 17 (Windows) - - Visual Studio >= 2022 version 17.5 -- [yyjson](https://github.com/ibireme/yyjson) -- [{fmt}](https://github.com/fmtlib/fmt) -- [Nameof C++](https://github.com/Neargye/nameof) +* C++20 compiler with range supports + * LLVM >= 16 + * GCC >= 12 + * clang-cl >= 17 (Windows) + * Visual Studio >= 2022 version 17.5 +* [yyjson](https://github.com/ibireme/yyjson) +* [{fmt}](https://github.com/fmtlib/fmt) +* [Nameof C++](https://github.com/Neargye/nameof) ## Overview @@ -164,10 +165,10 @@ auto init_obj = object{{"id", 1}, As shown above, cpp-yyjson provides conversion between JSON value/array/object classes and C++ ranges and container types recursively. In addition to that, the following additional JSON casters are available (see the [reference](#serialize-and-deserialize-json) in detail): -* Pre-defined STL casters (e.g., `std::optional`, `std::variant`, `std::tuple` ([C++23 tuple-like](https://wg21.link/P2165R4))). -* Conversion using compile-time reflection of struct/class if it is available. -* Registration of field names with `VISITABLE_STRUCT` macro. -* User-defined casters. +* Pre-defined STL casters (e.g., `std::optional`, `std::variant`, `std::tuple` ([C++23 tuple-like](https://wg21.link/P2165R4))). +* Conversion using compile-time reflection of struct/class if it is available. +* Registration of field names with `VISITABLE_STRUCT` macro. +* User-defined casters. #### Pre-defined STL casters @@ -231,7 +232,7 @@ auto deserialized = cast(serialized); // -> X{.a = 1, .b = std::nullopt, .c = "default"} ``` -#### User-define caster (see the [reference](#serialize-and-deserialize-json) in detail): +#### User-define caster (see the [reference](#serialize-and-deserialize-json) in detail) ```cpp template <> @@ -250,7 +251,6 @@ auto serialized = value(visitable); // -> "1 null x" ``` - ## Installation To use cpp-yyjson, the dependent packages are required to be installed. It is convenient to use [vcpkg](https://github.com/microsoft/vcpkg) to install the packages: @@ -355,17 +355,23 @@ The `yyjson` namespace includes the following function and classes. Typically, y | -------------- | ---------------------------------------------------------------------------------------------------------- | | `yyjson::read` | Read a JSON string and return an *immutable* JSON document class which is alias of `yyjson::reader::value` | -| Type | | +| JSON Type | | | ---------------- | -------------------------------------------------------------- | | `yyjson::value` | *Mutable* JSON value class; alias of `yyjson::writer::value` | | `yyjson::array` | *Mutable* JSON array class; alias of `yyjson::writer::array` | | `yyjson::object` | *Mutable* JSON object class; alias of `yyjson::writer::object` | +| Allocator class | | +| ------------------------------ | ------------------------------------------------------------------ | +| `yyjson::dynamic_allocator` | Dynamic memory allocator wrapper, available for yyjson >= v0.8.0 | +| `yyjson::pool_allocator` | Fixed size memory allocator wrapper on the heap | +| `yyjson::stack_pool_allocator` | Fixed size memory allocator wrapper on the stack | + In internal namespaces, the cpp-yyjson provides JSON value, array, object, reference, and iterator classes. Each internal `yyjson::reader` and `yyjson::writer` namespace defines *immutable* and *mutable* JSON classes, respectively. Although you rarely need to be aware of the classes provided in the internal namespaces, the *reference* classes are noted here. The JSON value, array, and object classes have the corresponding *reference* (only for *mutable* classes) and *const reference* versions of them as shown later, such as `value_ref`, `const_value_ref`, `array_ref`, `const_array_ref`, and so on. The *reference* classes have member functions with almost the same signature as the normal versions. The difference between a normal `value` class and a `[const_]value_ref` class is whether they have their data ownership or not. The *(const) reference* JSON classes appear in return types of member functions of the JSON classes. This is to maximize performance by avoiding copying. -> **Note** +> [!NOTE] > The *reference* class refers to data in the owner, so its lifetime should be noted. ### Immutable JSON classes @@ -391,105 +397,15 @@ enum class yyjson::ReadFlag : yyjson_read_flag AllowComments = YYJSON_READ_ALLOW_COMMENTS, AllowInfAndNan = YYJSON_READ_ALLOW_INF_AND_NAN, NumberAsRaw = YYJSON_READ_NUMBER_AS_RAW, - AllowInvalidUnicode = YYJSON_READ_ALLOW_INVALID_UNICODE + AllowInvalidUnicode = YYJSON_READ_ALLOW_INVALID_UNICODE, + BignumAsRaw = YYJSON_READ_BIGNUM_AS_RAW // for yyjson >= v0.7.0 }; ``` -The `read` function takes a JSON string and returns an immutable JSON value. The function argument may optionally specify a string length and an allocator to be described later. Otherwise, the length of the JSON string is determined and the yyjson default allocator is used. +The `read` function takes a JSON string and returns an immutable JSON value. The function argument may optionally specify a string length and an allocator to be described [later](#allocator-classes). Otherwise, the length of the JSON string is determined and the yyjson default allocator is used. If the read option has the [`ReadInsitu`](https://ibireme.github.io/yyjson/doc/doxygen/html/md_doc__a_p_i.html#autotoc_md34) flag, You must specify the JSON string as writable (`std::string&` or `char*`) and its length. This writable string must be padded at least [`YYJSON_PADDING_SIZE`](https://ibireme.github.io/yyjson/doc/doxygen/html/yyjson_8h.html#abbe8e69f634b1a5a78c1dae08b88e0ef) bytes to the end. The length of the JSON string should be unpadded. -This library provides the following two special allocators. Both are pool allocators and are useful for tasks that parse multiple JSON strings with a single buffer. You must be careful not to destroy the allocator or perform any modification operations on the allocator buffer while an instance of the JSON class is in use. - -##### `yyjson::reader::pool_allocator` - -This pool_allocator is based on `std::vector` and has a buffer on the heap. The `read` function will reallocate the buffer if it is not large enough for the given JSON string. - -**Constructor** - -```cpp -// Default constructors -pool_allocator() = default; -pool_allocator(const pool_allocator&) = default; -pool_allocator(pool_allocator&&) noexcept = default; - -// Allocate specified bytes of buffer -explicit pool_allocator(std::size_t size_byte); -// Allocate a buffer large enough to read the specified JSON string with read flags -explicit pool_allocator(std::string_view json, ReadFlag flag = ReadFlag::NoFlag); -``` - -**Member function** - -```cpp -// Return the buffer size -std::size_t size() const; -// Resize the buffer to the specified bytes -void resize(std::size_t); -// Allocate the specified bytes of buffer if not large enough -void allocate(std::size_t); -// Allocate a buffer large enough to read the specified JSON string with read flags -void allocate(std::string_view json, ReadFlag = ReadFlag::NoFlag); -// Deallocate the buffer -void deallocate(); -// Shrink the buffer capacity to the size -void shrink_to_fit(); -// Check if the buffer is large enough to read the specified JSON string with read flags -bool check_capacity(std::string_view json, ReadFlag = ReadFlag::NoFlag) const; -``` - -##### `yyjson::reader::stack_pool_allocator` - -This allocator is based on `std::array` and has a fixed buffer on the stack. The `check_capacity` function should be called to check if the buffer size is sufficient before using it because the `read` function cannot increase the size of the `stack_alloctor` buffer. - -**Constructor** - -```cpp -// Default constructors -stack_pool_allocator() = default; -stack_pool_allocator(const stack_pool_allocator&) = default; -stack_pool_allocator(stack_pool_allocator&&) noexcept = default; -``` - -**Member function** - -```cpp -// Return the buffer size -constexpr std::size_t size() const; -// Check if the buffer is large enough to read the specified JSON string with read flags -bool check_capacity(std::string_view json, ReadFlag = ReadFlag::NoFlag) const -``` - -**Example** - -See also [Overview](#json-reader). - -```cpp -using namespace yyjson; - -auto json_str = R"( -{ - "id": 1, - "pi": 3.141592, - "emoji": "🫠", - "array": [0, 1, 2, 3, 4,], - "currency": { - "USD": 129.66, - "EUR": 140.35, - "GBP": 158.72, - }, - "success": true, -})"; - -// Read JSON string with the default allocator -auto val = read(json_str, ReadFlag::AllowTrailingCommas); - -// Read JSON string with heap allocator and ReadInsitu flag -auto json_str_insitu = fmt::format("{}{}", json_obj_str, std::string(YYJSON_PADDING_SIZE, '\0')); // Padded -auto heap_alloc = reader::pool_allocator(); // Heap allocator can automatically expand the buffer -auto val = read(json_str_insitu, json_str.size(), heap_alloc, ReadFlag::ReadInsitu); -``` - #### `yyjson::reader::value` The immutable JSON value class is returned from `yyjson::read` function. @@ -552,7 +468,8 @@ enum class yyjson::WriteFlag : yyjson_write_flag EscapeSlashes = YYJSON_WRITE_ESCAPE_SLASHES, AllowInfAndNan = YYJSON_WRITE_ALLOW_INF_AND_NAN, InfAndNanAsNull = YYJSON_WRITE_INF_AND_NAN_AS_NULL, - AllowInvalidUnicode = YYJSON_WRITE_ALLOW_INVALID_UNICODE + AllowInvalidUnicode = YYJSON_WRITE_ALLOW_INVALID_UNICODE, + PrettyTwoSpaces = YYJSON_WRITE_PRETTY_TWO_SPACES // for yyjson >= v0.7.0 }; ``` @@ -1113,7 +1030,7 @@ The `yyjson::object` is also designed to be implemented like STL map containers Iterator classes are not exposed. They are tentatively described as `obejct_iter` and `const_object_iter` in the above. -> **Note** +> [!NOTE] > The `emplace` member functions do **NOT** check for key duplication. **Example** @@ -1145,6 +1062,133 @@ std::cout << obj.write() << std::endl; // {"key0":{"a":0,"b":1},"key1":{"c":2,"d":3},"key2":{"e":4,"f":5},"key3":{"e":6,"f":7}} ``` +### Allocator classes + +This library provides the following memory allocator wrappers for reading and writing JSON strings. They are useful to improve performance for tasks that parse or stringify multiple JSON strings to avoid multiple memory allocations and deallocations. + +#### `yyjson::dynamic_allocator` + +This is a smart pointer wrapper for the dynamic allocator of yyjson. The lifetime of the allocator's internal smart pointer will be tied to JSON objects and strings created by `read`/`write` (member) functions, respectively. This allocator is for general purposes since it can automatically expand the buffer. + +> [!NOTE] +> This allocator is only available for yyjson >= v0.8.0. + +**Constructor** + +```cpp +// Default constructors +dynamic_allocator() = default; +dynamic_allocator(const dynamic_allocator&) = default; +dynamic_allocator(dynamic_allocator&&) noexcept = default; +``` + +**Member function** + +```cpp +// Reset to a new dynamic allocator +void reset() const; +``` + +**Example** + +```cpp +using namespace yyjson; + +auto alc = dynamic_allocator(); + +// Read JSON string using dynamic allocator +auto read_json(const string& json) +{ + return read(json, alc); +} + +// Create JSON string using dynamic allocator +auto write_json(const auto& json) +{ + return json.write(alc); +} +``` + +#### `yyjson::pool_allocator` + +This is a smart pointer wrapper for the fixed-size memory allocator of yyjson. The lifetime of the allocator's internal smart pointer will be tied to JSON objects and strings created by `read`/`write` (member) functions, respectively. This allocator is useful when the required memory size is known; for reading JSON strings. Since the buffer size is fixed, the `read` function using the single buffer would be faster than the dynamic allocator. + +If the free space of the buffer is not large enough, the `read` or `write` (member) function will throw an exception. The `reset` function re-creates the buffer with the specified size and the `reserve` function can be used to expand the buffer size to be required. + +**Constructor** + +```cpp +// Default constructors +pool_allocator() = default; +pool_allocator(const pool_allocator&) = default; +pool_allocator(pool_allocator&&) noexcept = default; + +// Allocate specified bytes of buffer +explicit pool_allocator(std::size_t size_byte); +// Allocate a buffer large enough to read the specified JSON string with read flags +explicit pool_allocator(std::string_view json, ReadFlag flag = ReadFlag::NoFlag); +``` + +**Member function** + +```cpp +// Return the buffer size +std::size_t size() const; +// Reset to a new fixed size allocator +void reset(std::size_t size_byte = 0); +void reset(std::string_view json, ReadFlag flag = ReadFlag::NoFlag); +// Expand the buffer to required size +void reserve(std::size_t size_byte); +void reserve(std::string_view json, ReadFlag flag = ReadFlag::NoFlag); +// Check if the buffer, when empty, is large enough to read the specified JSON string with read flags +bool check_capacity(std::string_view json, ReadFlag = ReadFlag::NoFlag) const; +``` + +**Example** + +```cpp +using namespace yyjson; + +// Create a single buffer allocator +auto alc = pool_allocator(); + +// Read JSON string using pool allocator multiple times +for (const auto& json : json_strings) +{ + // Expand the buffer required to read JSON string + alc.reserve(json); + auto val = read(json, alc); + // ... +} +``` + +##### `yyjson::stack_pool_allocator` + +This allocator is a fixed-size and stack-allocated buffer. The buffer size is specified by the template parameter and thus the buffer size cannot be changed. The required buffer size must be known in advance. + +> [!NOTE] +> The lifetime of `stack_pool_allocator` is not managed automatically unlike other allocators. Not to destroy the allocator while an instance of the JSON object/string is in use. + +**Constructor** + +```cpp +// Default constructors +stack_pool_allocator() = default; +stack_pool_allocator(const stack_pool_allocator&) = default; +stack_pool_allocator(stack_pool_allocator&&) noexcept = default; +``` + +**Member function** + +```cpp +// Return the buffer size +constexpr std::size_t size() const; +// Reset to a new fixed size allocator +void reset(); +// Check if the buffer, when empty, is large enough to read the specified JSON string with read flags +bool check_capacity(std::string_view json, ReadFlag = ReadFlag::NoFlag) const +``` + ### Serialize and deserialize JSON In the cpp-yyjson, conversion from C++ objects to JSON values is very flexible and vice versa. @@ -1185,10 +1229,10 @@ In addition to the above, the following four methods are provided for converting Several casters from/to STL classes to JSON are pre-defined in the library for convenience. Currently, available STL types are as follows: -- `std::optional` -- `std::variant` -- `std::tuple`-like -- `std::vector` (specialized) +* `std::optional` +* `std::variant` +* `std::tuple`-like +* `std::vector` (specialized) For example, if you receive elements of multiple types, casts from/to `std::optional` or `std::variant` are useful. @@ -1324,7 +1368,7 @@ struct yyjson::caster }; ``` -The `from_json` template function has a JSON value as an argument template and must return type `X`. +The `from_json` template function has a JSON value as an argument template and must return type `X`. For the `to_json` function, there are two ways. The first way is to define a *translator* for a *value_constructible* type and return it. The return type must be `value_constructible`: diff --git a/include/cpp_yyjson.hpp b/include/cpp_yyjson.hpp index 0bd72e1..3f673df 100644 --- a/include/cpp_yyjson.hpp +++ b/include/cpp_yyjson.hpp @@ -46,7 +46,10 @@ namespace yyjson AllowComments = YYJSON_READ_ALLOW_COMMENTS, AllowInfAndNan = YYJSON_READ_ALLOW_INF_AND_NAN, NumberAsRaw = YYJSON_READ_NUMBER_AS_RAW, - AllowInvalidUnicode = YYJSON_READ_ALLOW_INVALID_UNICODE + AllowInvalidUnicode = YYJSON_READ_ALLOW_INVALID_UNICODE, +#if YYJSON_VERSION_HEX >= 0x000700 + BignumAsRaw = YYJSON_READ_BIGNUM_AS_RAW +#endif }; enum class WriteFlag : yyjson_write_flag @@ -57,7 +60,10 @@ namespace yyjson EscapeSlashes = YYJSON_WRITE_ESCAPE_SLASHES, AllowInfAndNan = YYJSON_WRITE_ALLOW_INF_AND_NAN, InfAndNanAsNull = YYJSON_WRITE_INF_AND_NAN_AS_NULL, - AllowInvalidUnicode = YYJSON_WRITE_ALLOW_INVALID_UNICODE + AllowInvalidUnicode = YYJSON_WRITE_ALLOW_INVALID_UNICODE, +#if YYJSON_VERSION_HEX >= 0x000700 + PrettyTwoSpaces = YYJSON_WRITE_PRETTY_TWO_SPACES +#endif }; // equivalent to C++23 std::to_underlying @@ -116,13 +122,25 @@ namespace yyjson class json_string : public std::string_view { - std::shared_ptr str_ptr_; + std::shared_ptr str_ptr_ = {}; public: json_string(char* ptr, std::size_t len) : std::string_view(ptr, len), str_ptr_(std::shared_ptr(ptr, [](auto* p) { std::free(p); })) { } + json_string(char* ptr, std::size_t len, yyjson_alc* alc) + : std::string_view(ptr, len), + str_ptr_(std::shared_ptr(ptr, [alc = alc](auto* p) mutable { alc->free(alc->ctx, p); })) + { + } + json_string(char* ptr, std::size_t len, const std::shared_ptr& alc) + : std::string_view(ptr, len), str_ptr_(std::shared_ptr(ptr, [alc = alc](auto* p) mutable { + alc->free(alc->ctx, p); + alc.reset(); + })) + { + } }; class key_string : public std::string_view @@ -137,11 +155,145 @@ namespace yyjson explicit operator const char*() { return base::data(); } }; + template > + concept yyjson_allocator = + (std::same_as, std::remove_cvref_t().ptr())>>) || + (std::same_as().ptr())>>) || + std::same_as; + +#if YYJSON_VERSION_HEX >= 0x000800 + class dynamic_allocator + { + std::shared_ptr alc_ = {yyjson_alc_dyn_new(), [](auto* ptr) { yyjson_alc_dyn_free(ptr); }}; + + public: + auto& ptr() & { return alc_; } + [[nodiscard]] const auto& ptr() const& { return alc_; } + auto ptr() && { return std::move(alc_); } + void reset() + { + alc_ = {yyjson_alc_dyn_new(), [](auto* ptr) { yyjson_alc_dyn_free(ptr); }}; + } + }; +#endif + + class pool_allocator + { + using char_like = char; + auto init() + { + buf_ = nullptr; + size_ = 0; + return std::shared_ptr(); + } + auto init(std::size_t size) + { + if (size == 0) return init(); + buf_ = std::allocator().allocate(size); + std::ranges::uninitialized_default_construct(buf_, buf_ + size); + size_ = size; + + // allocate memory pool + auto alc = new yyjson_alc; + yyjson_alc_pool_init(alc, buf_, size); + return std::shared_ptr(alc, [b = buf_, s = size_](auto* a) { + std::allocator().deallocate(b, s); + delete a; + }); + } + char_like* buf_ = nullptr; + std::size_t size_ = 0; + std::shared_ptr alc_ = {}; + + public: + pool_allocator() = default; + pool_allocator(const pool_allocator&) = default; + pool_allocator(pool_allocator&&) noexcept = default; + explicit pool_allocator(std::size_t size_byte) : alc_(init(size_byte)) {} + explicit pool_allocator(std::string_view json, ReadFlag flag = ReadFlag::NoFlag) + : alc_(init(yyjson_read_max_memory_usage(json.size(), to_underlying(flag)))) + { + } + auto& ptr() & { return alc_; } + [[nodiscard]] const auto& ptr() const& { return alc_; } + auto ptr() && { return std::move(alc_); } + [[nodiscard]] constexpr auto size() const noexcept { return size_; } + void reset(std::size_t size_byte = 0) { alc_ = init(size_byte); } + void reset(std::string_view json, ReadFlag flag = ReadFlag::NoFlag) + { + alc_ = init(yyjson_read_max_memory_usage(json.size(), to_underlying(flag))); + } + void reserve(std::size_t size_byte) + { + if (size_byte > size_) alc_ = init(size_byte); + } + void reserve(std::string_view json, ReadFlag flag = ReadFlag::NoFlag) + { + reserve(yyjson_read_max_memory_usage(json.size(), to_underlying(flag))); + } + [[nodiscard]] bool check_capacity(std::string_view json, ReadFlag flag = ReadFlag::NoFlag) const + { + return size() >= yyjson_read_max_memory_usage(json.size(), to_underlying(flag)); + } + + pool_allocator& operator=(const pool_allocator& r) = default; + pool_allocator& operator=(pool_allocator&& r) noexcept = default; + }; + + template + class stack_pool_allocator + { + using char_like = char; + std::array buf_; + yyjson_alc alc_ = init(); + yyjson_alc init() + { + yyjson_alc alc; + yyjson_alc_pool_init(&alc, buf_.data(), buf_.size()); + return alc; + } + + public: + auto* ptr() & noexcept { return &alc_; } + const auto* ptr() const& noexcept { return &alc_; } + stack_pool_allocator() = default; + stack_pool_allocator(const stack_pool_allocator&) = default; + stack_pool_allocator(stack_pool_allocator&&) noexcept = default; + stack_pool_allocator& operator=(const stack_pool_allocator&) + { + alc_ = init(); + return *this; + } + [[nodiscard]] constexpr auto size() const noexcept { return buf_.size(); } + void reset() noexcept { alc_ = init(); } + [[nodiscard]] bool check_capacity(std::string_view json, ReadFlag flag = ReadFlag::NoFlag) const + { + return size() >= yyjson_read_max_memory_usage(json.size(), to_underlying(flag)); + } + }; + namespace detail { template inline constexpr bool false_v = false; + template + auto* get_allocator_pointer(Alloc& alc) noexcept + { + if constexpr (std::same_as) + { + return &alc; + } + else if constexpr (std::is_pointer_v) + { + return alc.ptr(); + } + else + { + return alc.ptr().get(); + } + } + template constexpr auto proxy(T&& value) { @@ -219,6 +371,11 @@ namespace yyjson class object; using const_key_value_ref_pair = std::pair; + // for backward compatibility + using pool_allocator = yyjson::pool_allocator; + template + using stack_pool_allocator = yyjson::stack_pool_allocator; + namespace detail { using namespace yyjson::detail; // NOLINT @@ -1348,6 +1505,33 @@ namespace yyjson } throw write_error(fmt::format("write JSON error: {}", err.msg)); } + template + [[nodiscard]] auto write(Alloc& alc, WriteFlag write_flag = WriteFlag::NoFlag) const + { + const auto write_func = [this](Args&&... args) { + const auto write_doc = + doc_.ptrs->self != nullptr && val_ == yyjson_mut_doc_get_root(doc_.ptrs->self); + return write_doc ? yyjson_mut_write_opts(doc_.ptrs->self, std::forward(args)...) + : yyjson_mut_val_write_opts(val_, std::forward(args)...); + }; + + auto err = yyjson_write_err(); + auto len = static_cast(0); + auto result = write_func(to_underlying(write_flag), detail::get_allocator_pointer(alc), &len, &err); + + if constexpr (std::same_as, std::remove_cvref_t>) + { + if (result != nullptr) [[likely]] + return json_string(result, len, alc.ptr()); + } + else + { + if (result != nullptr) [[likely]] + return json_string(result, len, detail::get_allocator_pointer(alc)); + } + + throw write_error(fmt::format("write JSON error: {}", err.msg)); + } }; template @@ -2887,6 +3071,25 @@ namespace yyjson } throw write_error(fmt::format("write JSON error: {}", err.msg)); } + template + [[nodiscard]] auto write(Alloc& alc, const WriteFlag write_flag = WriteFlag::NoFlag) const + { + auto err = yyjson_write_err(); + auto len = static_cast(0); + auto result = yyjson_val_write_opts(val_, to_underlying(write_flag), detail::get_allocator_pointer(alc), + &len, &err); + if constexpr (std::same_as, std::remove_cvref_t>) + { + if (result != nullptr) [[likely]] + return json_string(result, len, alc.ptr()); + } + else + { + if (result != nullptr) [[likely]] + return json_string(result, len, detail::get_allocator_pointer(alc)); + } + throw write_error(fmt::format("write JSON error: {}", err.msg)); + } }; class const_value_ref : public abstract_value_ref @@ -3267,101 +3470,9 @@ namespace yyjson return *this; } - template - concept yyjson_allocator = (std::same_as().get())>>) || - std::same_as; - template value read(char*, std::size_t, Alloc&, ReadFlag = ReadFlag::NoFlag); - class pool_allocator - { - template - friend class stack_pool_allocator; - - using char_like = char; - yyjson_alc init_allocator() - { - yyjson_alc alc; - yyjson_alc_pool_init(&alc, buf_.data(), buf_.size()); - return alc; - } - std::vector buf_; - yyjson_alc alc_ = init_allocator(); - - public: - pool_allocator() = default; - pool_allocator(const pool_allocator&) = default; - pool_allocator(pool_allocator&&) noexcept = default; - explicit pool_allocator(std::size_t size_byte) : buf_(size_byte) {} - explicit pool_allocator(std::string_view json, ReadFlag flag = ReadFlag::NoFlag) - : buf_(yyjson_read_max_memory_usage(json.size(), to_underlying(flag))) - { - } - auto& get() & { return alc_; } - [[nodiscard]] const auto& get() const& { return alc_; } - auto get() && { return std::move(alc_); } - [[nodiscard]] constexpr auto begin() noexcept { return buf_.begin(); } - [[nodiscard]] constexpr auto end() noexcept { return buf_.end(); } - [[nodiscard]] constexpr auto size() const noexcept { return buf_.size(); } - void resize(std::size_t req) - { - buf_.resize(req); - alc_ = init_allocator(); - } - void allocate(std::size_t req_size_byte) - { - if (req_size_byte > size()) resize(req_size_byte); - } - void allocate(std::string_view json, ReadFlag flag = ReadFlag::NoFlag) - { - allocate(yyjson_read_max_memory_usage(json.size(), to_underlying(flag))); - } - void deallocate() - { - buf_.clear(); - buf_.shrink_to_fit(); - alc_ = init_allocator(); - } - void shrink_to_fit() - { - buf_.shrink_to_fit(); - alc_ = init_allocator(); - } - [[nodiscard]] bool check_capacity(std::string_view json, ReadFlag flag = ReadFlag::NoFlag) const - { - return size() >= yyjson_read_max_memory_usage(json.size(), to_underlying(flag)); - } - }; - - template - class stack_pool_allocator - { - std::array buf_; - yyjson_alc alc_ = init(); - yyjson_alc init() - { - yyjson_alc alc; - yyjson_alc_pool_init(&alc, buf_.data(), buf_.size()); - return alc; - } - - public: - auto& get() & { return alc_; } - const auto& get() const& { return alc_; } - auto get() && { return std::move(alc_); } - stack_pool_allocator() = default; - stack_pool_allocator(const stack_pool_allocator&) = default; - stack_pool_allocator(stack_pool_allocator&&) noexcept = default; - [[nodiscard]] constexpr auto begin() noexcept { return buf_.begin(); } - [[nodiscard]] constexpr auto end() noexcept { return buf_.end(); } - [[nodiscard]] constexpr auto size() const noexcept { return buf_.size(); } - [[nodiscard]] bool check_capacity(std::string_view json, ReadFlag flag = ReadFlag::NoFlag) const - { - return size() >= yyjson_read_max_memory_usage(json.size(), to_underlying(flag)); - } - }; - class value final : public const_value_ref { using base = const_value_ref; @@ -3372,6 +3483,13 @@ namespace yyjson doc_(std::shared_ptr(doc, [](auto* ptr) { yyjson_doc_free(ptr); })) { } + value(yyjson_doc* doc, const std::shared_ptr& alc) + : base(yyjson_doc_get_root(doc)), doc_(std::shared_ptr(doc, [alc = alc](auto* ptr) mutable { + yyjson_doc_free(ptr); + alc.reset(); + })) + { + } public: value(const value&) = default; @@ -3391,6 +3509,26 @@ namespace yyjson throw write_error(fmt::format("write JSON error: {}", err.msg)); } + template + [[nodiscard]] auto write(Alloc& alc, const WriteFlag write_flag = WriteFlag::NoFlag) const + { + auto err = yyjson_write_err(); + auto len = static_cast(0); + auto result = yyjson_write_opts(doc_.get(), to_underlying(write_flag), + detail::get_allocator_pointer(alc), &len, &err); + if constexpr (std::same_as, std::remove_cvref_t>) + { + if (result != nullptr) [[likely]] + return json_string(result, len, alc.ptr()); + } + else + { + if (result != nullptr) [[likely]] + return json_string(result, len, detail::get_allocator_pointer(alc)); + } + throw write_error(fmt::format("write JSON error: {}", err.msg)); + } + template friend value read(char*, std::size_t, Alloc&, ReadFlag); friend value read(char* str, std::size_t len, ReadFlag); @@ -3474,27 +3612,29 @@ namespace yyjson { auto err = yyjson_read_err(); yyjson_doc* result; - if constexpr (std::same_as) + + if constexpr (requires { alc.allocate({str, len}, read_flag); }) { - result = yyjson_read_opts(str, len, to_underlying(read_flag), &alc, &err); + alc.allocate({str, len}, read_flag); } - else + else if constexpr (requires { alc.check_capacity({str, len}, read_flag); }) { - if constexpr (requires { alc.allocate({str, len}, read_flag); }) + if (!alc.check_capacity({str, len}, read_flag)) { - alc.allocate({str, len}, read_flag); + throw std::runtime_error( + fmt::format("Insufficient capacity in the pool allocator for {}", NAMEOF_TYPE(Alloc))); } - else - { - if (!alc.check_capacity({str, len}, read_flag)) - { - throw std::runtime_error( - fmt::format("Insufficient capacity in the pool allocator for {}", NAMEOF_TYPE(Alloc))); - } - } - result = yyjson_read_opts(str, len, to_underlying(read_flag), &alc.get(), &err); } - if (result != nullptr) return value(result); + result = yyjson_read_opts(str, len, to_underlying(read_flag), detail::get_allocator_pointer(alc), &err); + + if constexpr (std::same_as, std::remove_cvref_t>) + { + if (result != nullptr) return value(result, alc.ptr()); + } + else + { + if (result != nullptr) return value(result); + } throw read_error(fmt::format("read JSON error: {} at position: {}", err.msg, err.pos)); } template diff --git a/test/bench_write.cpp b/test/bench_write.cpp index 5bd61d9..7dcf87e 100644 --- a/test/bench_write.cpp +++ b/test/bench_write.cpp @@ -34,6 +34,26 @@ constexpr auto json_size_obj_int64 = 15777781; constexpr auto json_size_obj_double = 17777781; constexpr auto json_size_obj_string = 17777781; +void write_c_yyjson_array_int64(benchmark::State& state) +{ + std::iota(vec_int64.begin(), vec_int64.end(), 0); + for (auto _ : state) + { + yyjson_mut_doc* doc = yyjson_mut_doc_new(NULL); + auto root = yyjson_mut_arr_with_sint64(doc, vec_int64.data(), vec_int64.size()); + yyjson_mut_doc_set_root(doc, root); + const char* json = yyjson_mut_write(doc, 0, NULL); + auto result = std::string_view(json); + if (result.size() != json_size_arr_int64) + { + state.SkipWithError("Invalid JSON string"); + break; + } + free(const_cast(static_cast(json))); + yyjson_mut_doc_free(doc); + } +} + void write_cpp_yyjson_array_int64(benchmark::State& state) { using namespace yyjson; @@ -50,23 +70,20 @@ void write_cpp_yyjson_array_int64(benchmark::State& state) } } -void write_c_yyjson_array_int64(benchmark::State& state) +void write_cpp_yyjson_single_array_int64(benchmark::State& state) { + using namespace yyjson; std::iota(vec_int64.begin(), vec_int64.end(), 0); + auto alc = dynamic_allocator(); for (auto _ : state) { - yyjson_mut_doc* doc = yyjson_mut_doc_new(NULL); - auto root = yyjson_mut_arr_with_sint64(doc, vec_int64.data(), vec_int64.size()); - yyjson_mut_doc_set_root(doc, root); - const char* json = yyjson_mut_write(doc, 0, NULL); - auto result = std::string_view(json); + auto array = yyjson::array(vec_int64); + auto result = array.write(alc); if (result.size() != json_size_arr_int64) { state.SkipWithError("Invalid JSON string"); break; } - free(const_cast(static_cast(json))); - yyjson_mut_doc_free(doc); } } @@ -144,6 +161,23 @@ void write_cpp_yyjson_array_double(benchmark::State& state) } } +void write_cpp_yyjson_single_array_double(benchmark::State& state) +{ + using namespace yyjson; + std::iota(vec_double.begin(), vec_double.end(), 0); + auto alc = dynamic_allocator(); + for (auto _ : state) + { + auto array = yyjson::array(vec_double); + auto result = array.write(alc); + if (result.size() != json_size_arr_double) + { + state.SkipWithError("Invalid JSON string"); + break; + } + } +} + void write_rapidjson_array_double(benchmark::State& state) { using namespace rapidjson; @@ -222,6 +256,24 @@ void write_cpp_yyjson_array_string(benchmark::State& state) } } +void write_cpp_yyjson_single_array_string(benchmark::State& state) +{ + using namespace yyjson; + std::iota(vec_int64.begin(), vec_int64.end(), 0); + auto alc = dynamic_allocator(); + std::ranges::transform(vec_int64, vec_string.begin(), [](const auto n) { return fmt::format("{}", n); }); + for (auto _ : state) + { + auto array = yyjson::array(vec_string); + auto result = array.write(alc); + if (result.size() != json_size_arr_string) + { + state.SkipWithError("Invalid JSON string"); + break; + } + } +} + void write_rapidjson_array_string(benchmark::State& state) { using namespace rapidjson; @@ -1000,14 +1052,17 @@ void write_nlohmann_object_string_copy(benchmark::State& state) } BENCHMARK(write_cpp_yyjson_array_int64)->Unit(benchmark::kMillisecond); +BENCHMARK(write_cpp_yyjson_single_array_int64)->Unit(benchmark::kMillisecond); BENCHMARK(write_c_yyjson_array_int64)->Unit(benchmark::kMillisecond); BENCHMARK(write_rapidjson_array_int64)->Unit(benchmark::kMillisecond); BENCHMARK(write_nlohmann_array_int64)->Unit(benchmark::kMillisecond); BENCHMARK(write_cpp_yyjson_array_double)->Unit(benchmark::kMillisecond); +BENCHMARK(write_cpp_yyjson_single_array_double)->Unit(benchmark::kMillisecond); BENCHMARK(write_c_yyjson_array_double)->Unit(benchmark::kMillisecond); BENCHMARK(write_rapidjson_array_double)->Unit(benchmark::kMillisecond); BENCHMARK(write_nlohmann_array_double)->Unit(benchmark::kMillisecond); BENCHMARK(write_cpp_yyjson_array_string)->Unit(benchmark::kMillisecond); +BENCHMARK(write_cpp_yyjson_single_array_string)->Unit(benchmark::kMillisecond); BENCHMARK(write_c_yyjson_array_string)->Unit(benchmark::kMillisecond); BENCHMARK(write_rapidjson_array_string)->Unit(benchmark::kMillisecond); BENCHMARK(write_cpp_yyjson_array_string_copy)->Unit(benchmark::kMillisecond); diff --git a/test/test.cpp b/test/test.cpp index 1ff8170..8507c13 100644 --- a/test/test.cpp +++ b/test/test.cpp @@ -2506,45 +2506,91 @@ TEST(Reader, Allocator) using namespace std::literals::string_literals; using namespace std::string_view_literals; - auto h_alloc = reader::pool_allocator(); + auto json_str = "[1,2,3,4,5]"sv; + + auto h_alloc = reader::pool_allocator(json_str); + auto d_alloc = dynamic_allocator(); auto s_alloc_s = reader::stack_pool_allocator<10>(); auto s_alloc_l = reader::stack_pool_allocator<512>(); - auto result = std::string(); - auto json_str = "[1,2,3,4,5]"sv; - EXPECT_NO_THROW(result = read(json_str, h_alloc).write()); - EXPECT_THROW(result = read(json_str, s_alloc_s).write(), std::runtime_error); - EXPECT_EQ(json_str, result); - EXPECT_NO_THROW(result = read(json_str, s_alloc_l).write()); - EXPECT_EQ(json_str, result); + { + auto json = read(json_str, h_alloc); + auto result = json.write(); + EXPECT_EQ(json_str, result); + EXPECT_EQ(2, h_alloc.ptr().use_count()); + } + EXPECT_EQ(1, h_alloc.ptr().use_count()); + + { + auto json = read(json_str, d_alloc); + auto result = json.write(); + EXPECT_EQ(json_str, result); + EXPECT_EQ(2, d_alloc.ptr().use_count()); + } + EXPECT_EQ(1, d_alloc.ptr().use_count()); + + { + EXPECT_THROW(read(json_str, s_alloc_s).write(), std::runtime_error); + } + { + auto json = read(json_str, s_alloc_l); + auto result = json.write(); + EXPECT_EQ(json_str, result); + } + + { + auto json = read(json_str, h_alloc); + EXPECT_EQ(2, h_alloc.ptr().use_count()); + h_alloc.reset(json_str); + EXPECT_EQ(1, h_alloc.ptr().use_count()); + auto result = json.write(h_alloc); + EXPECT_EQ(json_str, result); + EXPECT_EQ(2, h_alloc.ptr().use_count()); + } + EXPECT_EQ(1, h_alloc.ptr().use_count()); + + { + auto json = read(json_str, d_alloc); + auto result = json.write(d_alloc); + EXPECT_EQ(json_str, result); + EXPECT_EQ(3, d_alloc.ptr().use_count()); + } + EXPECT_EQ(1, d_alloc.ptr().use_count()); + + { + auto json = read(json_str, s_alloc_l); + auto result = json.write(s_alloc_l); + EXPECT_EQ(json_str, result); + } auto str_insitu = std::string(json_str) + " "s; - EXPECT_NO_THROW(result = read(str_insitu, json_str.size(), h_alloc, yyjson::ReadFlag::ReadInsitu).write()); + h_alloc.reserve(json_str, ReadFlag::ReadInsitu); + auto result = read(str_insitu, json_str.size(), h_alloc, yyjson::ReadFlag::ReadInsitu).write(); EXPECT_EQ(json_str, result); auto json_obj_str = R"( - { - "id": 1, - "pi": 3.141592, - "emoji": "🫠", - "array": [0, 1, 2, 3, 4,], - "currency": { - "USD": 129.66, - "EUR": 140.35, - "GBP": 158.72, - }, - "success": true, - })"sv; + { + "id": 1, + "pi": 3.141592, + "emoji": "🫠", + "array": [0, 1, 2, 3, 4,], + "currency": { + "USD": 129.66, + "EUR": 140.35, + "GBP": 158.72, + }, + "success": true, + })"sv; { auto json_str_insitu = fmt::format("{}{}", json_obj_str, std::string(YYJSON_PADDING_SIZE, '\0')); EXPECT_EQ(json_str_insitu.size(), json_obj_str.size() + YYJSON_PADDING_SIZE); + h_alloc.reserve(json_obj_str, yyjson::ReadFlag::ReadInsitu | yyjson::ReadFlag::AllowTrailingCommas); auto val = read(json_str_insitu, json_obj_str.size(), h_alloc, yyjson::ReadFlag::ReadInsitu | yyjson::ReadFlag::AllowTrailingCommas); - EXPECT_TRUE(h_alloc.size() > 0); - fmt::print("{}\n", val); + EXPECT_EQ(2, h_alloc.ptr().use_count()); } - h_alloc.deallocate(); + EXPECT_EQ(1, h_alloc.ptr().use_count()); } TEST(Reader, Constructor)