From 0b6d283e009320d3fed3d254e8a65059742d8141 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Mon, 20 Jul 2026 23:27:51 +0000 Subject: [PATCH 1/6] Devirtualize binary_writer via a value-type output sink to_cbor/to_msgpack/to_ubjson/to_bjdata/to_bson wrote every byte through output_adapter_t, a shared_ptr whose write_character/write_characters are virtual. Unlike the lexer (templated on a concrete InputAdapterType), the binary writer never got that treatment, so binary output paid a vtable lookup per byte and a make_shared per call. Template binary_writer on an OutputSinkType and give it two concrete, non-virtual sinks: - output_vector_sink: appends straight into a std::vector (push_back / insert), used by the vector-returning to_* convenience functions. No vtable, no shared_ptr; the writes inline. - output_adapter_sink: forwards to a type-erased output_adapter_t, so the existing to_*(j, output_adapter) overloads (streams, strings, custom adapters) keep working exactly as before -- one virtual call each, unchanged. binary_writer keeps a convenience constructor taking output_adapter_t (building the default output_adapter_sink), so the adapter overloads are untouched; only the convenience functions switch to the vector sink. The friend declaration and the basic_json binary_writer alias gain the new (defaulted) template parameter. Output is byte-for-byte identical: verified across ~3000 randomized values plus curated edge cases (all scalar widths, strings with invalid UTF-8, binary, nested arrays/objects) for CBOR, MessagePack, UBJSON (both size/type settings), BJData, and BSON, plus the output_adapter path, in C++11/17/20. Warning-clean under clang -Weverything and the gcc pedantic set; clang-tidy clean on the changed headers; make check-amalgamation clean. Throughput (g++ -O3, vs develop): scalar-dense binary output such as integer arrays ~1.4x; many small to_cbor calls ~1.04x (DOM traversal bound); string/blob-heavy output unchanged (already bulk-bound). No workload regressed. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01XAYM1qhSA2FDaDcGfPW3fG Signed-off-by: Niels Lohmann --- .../nlohmann/detail/output/binary_writer.hpp | 303 +++++++++--------- .../detail/output/output_adapters.hpp | 62 ++++ include/nlohmann/json.hpp | 20 +- 3 files changed, 233 insertions(+), 152 deletions(-) diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 4bd1732576d..00181c5dd28 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -42,7 +42,7 @@ enum class bjdata_version_t /*! @brief serialization to CBOR and MessagePack values */ -template +template> class binary_writer { using string_t = typename BasicJsonType::string_t; @@ -53,12 +53,25 @@ class binary_writer /*! @brief create a binary writer + @param[in] sink output sink to write to (a value-type sink such as + output_vector_sink, or output_adapter_sink wrapping a + type-erased output adapter) + */ + explicit binary_writer(OutputSinkType sink) : oa(std::move(sink)) + {} + + /*! + @brief create a binary writer from a type-erased output adapter + + Convenience constructor for the default (output_adapter_sink) sink so the + `output_adapter`-based overloads keep constructing the writer directly from + an adapter. Only participates in overload resolution when the sink can be + built from an adapter. + @param[in] adapter output adapter to write to */ - explicit binary_writer(output_adapter_t adapter) : oa(std::move(adapter)) - { - JSON_ASSERT(oa); - } + explicit binary_writer(output_adapter_t adapter) : oa(OutputSinkType(std::move(adapter))) + {} /*! @param[in] j JSON value to serialize @@ -99,15 +112,15 @@ class binary_writer { case value_t::null: { - oa->write_character(to_char_type(0xF6)); + oa.write_character(to_char_type(0xF6)); break; } case value_t::boolean: { - oa->write_character(j.m_data.m_value.boolean - ? to_char_type(0xF5) - : to_char_type(0xF4)); + oa.write_character(j.m_data.m_value.boolean + ? to_char_type(0xF5) + : to_char_type(0xF4)); break; } @@ -124,22 +137,22 @@ class binary_writer } else if (j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x18)); + oa.write_character(to_char_type(0x18)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x19)); + oa.write_character(to_char_type(0x19)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x1A)); + oa.write_character(to_char_type(0x1A)); write_number(static_cast(j.m_data.m_value.number_integer)); } else { - oa->write_character(to_char_type(0x1B)); + oa.write_character(to_char_type(0x1B)); write_number(static_cast(j.m_data.m_value.number_integer)); } } @@ -154,22 +167,22 @@ class binary_writer } else if (positive_number <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x38)); + oa.write_character(to_char_type(0x38)); write_number(static_cast(positive_number)); } else if (positive_number <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x39)); + oa.write_character(to_char_type(0x39)); write_number(static_cast(positive_number)); } else if (positive_number <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x3A)); + oa.write_character(to_char_type(0x3A)); write_number(static_cast(positive_number)); } else { - oa->write_character(to_char_type(0x3B)); + oa.write_character(to_char_type(0x3B)); write_number(static_cast(positive_number)); } } @@ -184,22 +197,22 @@ class binary_writer } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x18)); + oa.write_character(to_char_type(0x18)); write_number(static_cast(j.m_data.m_value.number_unsigned)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x19)); + oa.write_character(to_char_type(0x19)); write_number(static_cast(j.m_data.m_value.number_unsigned)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x1A)); + oa.write_character(to_char_type(0x1A)); write_number(static_cast(j.m_data.m_value.number_unsigned)); } else { - oa->write_character(to_char_type(0x1B)); + oa.write_character(to_char_type(0x1B)); write_number(static_cast(j.m_data.m_value.number_unsigned)); } break; @@ -210,16 +223,16 @@ class binary_writer if (std::isnan(j.m_data.m_value.number_float)) { // NaN is 0xf97e00 in CBOR - oa->write_character(to_char_type(0xF9)); - oa->write_character(to_char_type(0x7E)); - oa->write_character(to_char_type(0x00)); + oa.write_character(to_char_type(0xF9)); + oa.write_character(to_char_type(0x7E)); + oa.write_character(to_char_type(0x00)); } else if (std::isinf(j.m_data.m_value.number_float)) { // Infinity is 0xf97c00, -Infinity is 0xf9fc00 - oa->write_character(to_char_type(0xf9)); - oa->write_character(j.m_data.m_value.number_float > 0 ? to_char_type(0x7C) : to_char_type(0xFC)); - oa->write_character(to_char_type(0x00)); + oa.write_character(to_char_type(0xf9)); + oa.write_character(j.m_data.m_value.number_float > 0 ? to_char_type(0x7C) : to_char_type(0xFC)); + oa.write_character(to_char_type(0x00)); } else { @@ -238,29 +251,29 @@ class binary_writer } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x78)); + oa.write_character(to_char_type(0x78)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x79)); + oa.write_character(to_char_type(0x79)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x7A)); + oa.write_character(to_char_type(0x7A)); write_number(static_cast(N)); } // LCOV_EXCL_START else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x7B)); + oa.write_character(to_char_type(0x7B)); write_number(static_cast(N)); } // LCOV_EXCL_STOP // step 2: write the string - oa->write_characters( + oa.write_characters( reinterpret_cast(j.m_data.m_value.string->data()), j.m_data.m_value.string->size()); break; @@ -276,23 +289,23 @@ class binary_writer } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x98)); + oa.write_character(to_char_type(0x98)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x99)); + oa.write_character(to_char_type(0x99)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x9A)); + oa.write_character(to_char_type(0x9A)); write_number(static_cast(N)); } // LCOV_EXCL_START else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x9B)); + oa.write_character(to_char_type(0x9B)); write_number(static_cast(N)); } // LCOV_EXCL_STOP @@ -339,31 +352,31 @@ class binary_writer } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x58)); + oa.write_character(to_char_type(0x58)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x59)); + oa.write_character(to_char_type(0x59)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x5A)); + oa.write_character(to_char_type(0x5A)); write_number(static_cast(N)); } // LCOV_EXCL_START else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x5B)); + oa.write_character(to_char_type(0x5B)); write_number(static_cast(N)); } // LCOV_EXCL_STOP // step 2: write each element - oa->write_characters( - reinterpret_cast(j.m_data.m_value.binary->data()), - N); + oa.write_characters( + reinterpret_cast(j.m_data.m_value.binary->data()), + N); break; } @@ -378,23 +391,23 @@ class binary_writer } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0xB8)); + oa.write_character(to_char_type(0xB8)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0xB9)); + oa.write_character(to_char_type(0xB9)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0xBA)); + oa.write_character(to_char_type(0xBA)); write_number(static_cast(N)); } // LCOV_EXCL_START else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0xBB)); + oa.write_character(to_char_type(0xBB)); write_number(static_cast(N)); } // LCOV_EXCL_STOP @@ -423,15 +436,15 @@ class binary_writer { case value_t::null: // nil { - oa->write_character(to_char_type(0xC0)); + oa.write_character(to_char_type(0xC0)); break; } case value_t::boolean: // true and false { - oa->write_character(j.m_data.m_value.boolean - ? to_char_type(0xC3) - : to_char_type(0xC2)); + oa.write_character(j.m_data.m_value.boolean + ? to_char_type(0xC3) + : to_char_type(0xC2)); break; } @@ -450,25 +463,25 @@ class binary_writer else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 8 - oa->write_character(to_char_type(0xCC)); + oa.write_character(to_char_type(0xCC)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 16 - oa->write_character(to_char_type(0xCD)); + oa.write_character(to_char_type(0xCD)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 32 - oa->write_character(to_char_type(0xCE)); + oa.write_character(to_char_type(0xCE)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 64 - oa->write_character(to_char_type(0xCF)); + oa.write_character(to_char_type(0xCF)); write_number(static_cast(j.m_data.m_value.number_integer)); } } @@ -483,28 +496,28 @@ class binary_writer j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { // int 8 - oa->write_character(to_char_type(0xD0)); + oa.write_character(to_char_type(0xD0)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { // int 16 - oa->write_character(to_char_type(0xD1)); + oa.write_character(to_char_type(0xD1)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { // int 32 - oa->write_character(to_char_type(0xD2)); + oa.write_character(to_char_type(0xD2)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { // int 64 - oa->write_character(to_char_type(0xD3)); + oa.write_character(to_char_type(0xD3)); write_number(static_cast(j.m_data.m_value.number_integer)); } } @@ -521,25 +534,25 @@ class binary_writer else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 8 - oa->write_character(to_char_type(0xCC)); + oa.write_character(to_char_type(0xCC)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 16 - oa->write_character(to_char_type(0xCD)); + oa.write_character(to_char_type(0xCD)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 32 - oa->write_character(to_char_type(0xCE)); + oa.write_character(to_char_type(0xCE)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 64 - oa->write_character(to_char_type(0xCF)); + oa.write_character(to_char_type(0xCF)); write_number(static_cast(j.m_data.m_value.number_integer)); } break; @@ -563,24 +576,24 @@ class binary_writer else if (N <= (std::numeric_limits::max)()) { // str 8 - oa->write_character(to_char_type(0xD9)); + oa.write_character(to_char_type(0xD9)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { // str 16 - oa->write_character(to_char_type(0xDA)); + oa.write_character(to_char_type(0xDA)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { // str 32 - oa->write_character(to_char_type(0xDB)); + oa.write_character(to_char_type(0xDB)); write_number(static_cast(N)); } // step 2: write the string - oa->write_characters( + oa.write_characters( reinterpret_cast(j.m_data.m_value.string->data()), j.m_data.m_value.string->size()); break; @@ -598,13 +611,13 @@ class binary_writer else if (N <= (std::numeric_limits::max)()) { // array 16 - oa->write_character(to_char_type(0xDC)); + oa.write_character(to_char_type(0xDC)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { // array 32 - oa->write_character(to_char_type(0xDD)); + oa.write_character(to_char_type(0xDD)); write_number(static_cast(N)); } @@ -660,7 +673,7 @@ class binary_writer fixed = false; } - oa->write_character(to_char_type(output_type)); + oa.write_character(to_char_type(output_type)); if (!fixed) { write_number(static_cast(N)); @@ -672,7 +685,7 @@ class binary_writer ? 0xC8 // ext 16 : 0xC5; // bin 16 - oa->write_character(to_char_type(output_type)); + oa.write_character(to_char_type(output_type)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) @@ -681,7 +694,7 @@ class binary_writer ? 0xC9 // ext 32 : 0xC6; // bin 32 - oa->write_character(to_char_type(output_type)); + oa.write_character(to_char_type(output_type)); write_number(static_cast(N)); } @@ -692,9 +705,9 @@ class binary_writer } // step 2: write the byte string - oa->write_characters( - reinterpret_cast(j.m_data.m_value.binary->data()), - N); + oa.write_characters( + reinterpret_cast(j.m_data.m_value.binary->data()), + N); break; } @@ -711,13 +724,13 @@ class binary_writer else if (N <= (std::numeric_limits::max)()) { // map 16 - oa->write_character(to_char_type(0xDE)); + oa.write_character(to_char_type(0xDE)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { // map 32 - oa->write_character(to_char_type(0xDF)); + oa.write_character(to_char_type(0xDF)); write_number(static_cast(N)); } @@ -756,7 +769,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('Z')); + oa.write_character(to_char_type('Z')); } break; } @@ -765,9 +778,9 @@ class binary_writer { if (add_prefix) { - oa->write_character(j.m_data.m_value.boolean - ? to_char_type('T') - : to_char_type('F')); + oa.write_character(j.m_data.m_value.boolean + ? to_char_type('T') + : to_char_type('F')); } break; } @@ -794,10 +807,10 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('S')); + oa.write_character(to_char_type('S')); } write_number_with_ubjson_prefix(j.m_data.m_value.string->size(), true, use_bjdata); - oa->write_characters( + oa.write_characters( reinterpret_cast(j.m_data.m_value.string->data()), j.m_data.m_value.string->size()); break; @@ -807,7 +820,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('[')); + oa.write_character(to_char_type('[')); } bool prefix_required = true; @@ -839,14 +852,14 @@ class binary_writer && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) { prefix_required = false; - oa->write_character(to_char_type('$')); - oa->write_character(first_prefix); + oa.write_character(to_char_type('$')); + oa.write_character(first_prefix); } } if (use_count) { - oa->write_character(to_char_type('#')); + oa.write_character(to_char_type('#')); write_number_with_ubjson_prefix(j.m_data.m_value.array->size(), true, use_bjdata); } @@ -857,7 +870,7 @@ class binary_writer if (!use_count) { - oa->write_character(to_char_type(']')); + oa.write_character(to_char_type(']')); } break; @@ -867,7 +880,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('[')); + oa.write_character(to_char_type('[')); } if (use_type && (bjdata_draft3 || !j.m_data.m_value.binary->empty())) @@ -876,36 +889,36 @@ class binary_writer { JSON_THROW(other_error::create(502, "use_type requires use_size = true", &j)); } - oa->write_character(to_char_type('$')); - oa->write_character(bjdata_draft3 ? 'B' : 'U'); + oa.write_character(to_char_type('$')); + oa.write_character(bjdata_draft3 ? 'B' : 'U'); } if (use_count) { - oa->write_character(to_char_type('#')); + oa.write_character(to_char_type('#')); write_number_with_ubjson_prefix(j.m_data.m_value.binary->size(), true, use_bjdata); } if (use_type) { - oa->write_characters( - reinterpret_cast(j.m_data.m_value.binary->data()), - j.m_data.m_value.binary->size()); + oa.write_characters( + reinterpret_cast(j.m_data.m_value.binary->data()), + j.m_data.m_value.binary->size()); } else { for (size_t i = 0; i < j.m_data.m_value.binary->size(); ++i) { - oa->write_character(to_char_type(bjdata_draft3 ? 'B' : 'U')); + oa.write_character(to_char_type(bjdata_draft3 ? 'B' : 'U')); // the cast is needed for binary types whose value type // is not an integer (e.g., std::byte) - oa->write_character(to_char_type(static_cast(j.m_data.m_value.binary->data()[i]))); + oa.write_character(to_char_type(static_cast(j.m_data.m_value.binary->data()[i]))); } } if (!use_count) { - oa->write_character(to_char_type(']')); + oa.write_character(to_char_type(']')); } break; @@ -923,7 +936,7 @@ class binary_writer if (add_prefix) { - oa->write_character(to_char_type('{')); + oa.write_character(to_char_type('{')); } bool prefix_required = true; @@ -945,21 +958,21 @@ class binary_writer if (same_prefix && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) { prefix_required = false; - oa->write_character(to_char_type('$')); - oa->write_character(first_prefix); + oa.write_character(to_char_type('$')); + oa.write_character(first_prefix); } } if (use_count) { - oa->write_character(to_char_type('#')); + oa.write_character(to_char_type('#')); write_number_with_ubjson_prefix(j.m_data.m_value.object->size(), true, use_bjdata); } for (const auto& el : *j.m_data.m_value.object) { write_number_with_ubjson_prefix(el.first.size(), true, use_bjdata); - oa->write_characters( + oa.write_characters( reinterpret_cast(el.first.data()), el.first.size()); write_ubjson(el.second, use_count, use_type, prefix_required, use_bjdata, bjdata_version); @@ -967,7 +980,7 @@ class binary_writer if (!use_count) { - oa->write_character(to_char_type('}')); + oa.write_character(to_char_type('}')); } break; @@ -1021,13 +1034,13 @@ class binary_writer void write_bson_entry_header(const string_t& name, const std::uint8_t element_type) { - oa->write_character(to_char_type(element_type)); - oa->write_characters( + oa.write_character(to_char_type(element_type)); + oa.write_characters( reinterpret_cast(name.data()), name.size()); // the terminating null byte is written explicitly rather than taken // from the buffer, so that string_t::data() need not be null-terminated - oa->write_character(to_char_type(0x00)); + oa.write_character(to_char_type(0x00)); } /*! @@ -1037,7 +1050,7 @@ class binary_writer const bool value) { write_bson_entry_header(name, 0x08); - oa->write_character(value ? to_char_type(0x01) : to_char_type(0x00)); + oa.write_character(value ? to_char_type(0x01) : to_char_type(0x00)); } /*! @@ -1067,12 +1080,12 @@ class binary_writer write_bson_entry_header(name, 0x02); write_number(to_bson_length(value.size() + 1ul), true); - oa->write_characters( + oa.write_characters( reinterpret_cast(value.data()), value.size()); // the terminating null byte is written explicitly rather than taken // from the buffer, so that string_t::data() need not be null-terminated - oa->write_character(to_char_type(0x00)); + oa.write_character(to_char_type(0x00)); } /*! @@ -1201,7 +1214,7 @@ class binary_writer write_bson_element(string_t(key.data(), key.size()), el); } - oa->write_character(to_char_type(0x00)); + oa.write_character(to_char_type(0x00)); } /*! @@ -1215,7 +1228,7 @@ class binary_writer write_number(to_bson_length(value.size()), true); write_number(value.has_subtype() ? static_cast(value.subtype()) : static_cast(0x00)); - oa->write_characters(reinterpret_cast(value.data()), value.size()); + oa.write_characters(reinterpret_cast(value.data()), value.size()); } /*! @@ -1341,7 +1354,7 @@ class binary_writer write_bson_element(el.first, el.second); } - oa->write_character(to_char_type(0x00)); + oa.write_character(to_char_type(0x00)); } ////////// @@ -1385,7 +1398,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(get_ubjson_float_prefix(n)); + oa.write_character(get_ubjson_float_prefix(n)); } write_number(n, use_bjdata); } @@ -1401,7 +1414,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('i')); // int8 + oa.write_character(to_char_type('i')); // int8 } write_number(static_cast(n), use_bjdata); } @@ -1409,7 +1422,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('U')); // uint8 + oa.write_character(to_char_type('U')); // uint8 } write_number(static_cast(n), use_bjdata); } @@ -1417,7 +1430,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('I')); // int16 + oa.write_character(to_char_type('I')); // int16 } write_number(static_cast(n), use_bjdata); } @@ -1425,7 +1438,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('u')); // uint16 - bjdata only + oa.write_character(to_char_type('u')); // uint16 - bjdata only } write_number(static_cast(n), use_bjdata); } @@ -1433,7 +1446,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('l')); // int32 + oa.write_character(to_char_type('l')); // int32 } write_number(static_cast(n), use_bjdata); } @@ -1441,7 +1454,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('m')); // uint32 - bjdata only + oa.write_character(to_char_type('m')); // uint32 - bjdata only } write_number(static_cast(n), use_bjdata); } @@ -1449,7 +1462,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('L')); // int64 + oa.write_character(to_char_type('L')); // int64 } write_number(static_cast(n), use_bjdata); } @@ -1457,7 +1470,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('M')); // uint64 - bjdata only + oa.write_character(to_char_type('M')); // uint64 - bjdata only } write_number(static_cast(n), use_bjdata); } @@ -1465,14 +1478,14 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('H')); // high-precision number + oa.write_character(to_char_type('H')); // high-precision number } const auto number = BasicJsonType(n).dump(); write_number_with_ubjson_prefix(number.size(), true, use_bjdata); for (std::size_t i = 0; i < number.size(); ++i) { - oa->write_character(to_char_type(static_cast(number[i]))); + oa.write_character(to_char_type(static_cast(number[i]))); } } } @@ -1489,7 +1502,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('i')); // int8 + oa.write_character(to_char_type('i')); // int8 } write_number(static_cast(n), use_bjdata); } @@ -1497,7 +1510,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('U')); // uint8 + oa.write_character(to_char_type('U')); // uint8 } write_number(static_cast(n), use_bjdata); } @@ -1505,7 +1518,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('I')); // int16 + oa.write_character(to_char_type('I')); // int16 } write_number(static_cast(n), use_bjdata); } @@ -1513,7 +1526,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('u')); // uint16 - bjdata only + oa.write_character(to_char_type('u')); // uint16 - bjdata only } write_number(static_cast(n), use_bjdata); } @@ -1521,7 +1534,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('l')); // int32 + oa.write_character(to_char_type('l')); // int32 } write_number(static_cast(n), use_bjdata); } @@ -1529,7 +1542,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('m')); // uint32 - bjdata only + oa.write_character(to_char_type('m')); // uint32 - bjdata only } write_number(static_cast(n), use_bjdata); } @@ -1537,7 +1550,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('L')); // int64 + oa.write_character(to_char_type('L')); // int64 } write_number(static_cast(n), use_bjdata); } @@ -1546,14 +1559,14 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('H')); // high-precision number + oa.write_character(to_char_type('H')); // high-precision number } const auto number = BasicJsonType(n).dump(); write_number_with_ubjson_prefix(number.size(), true, use_bjdata); for (std::size_t i = 0; i < number.size(); ++i) { - oa->write_character(to_char_type(static_cast(number[i]))); + oa.write_character(to_char_type(static_cast(number[i]))); } } // LCOV_EXCL_STOP @@ -1845,10 +1858,10 @@ class binary_writer } } - oa->write_character('['); - oa->write_character('$'); - oa->write_character(dtype); - oa->write_character('#'); + oa.write_character('['); + oa.write_character('$'); + oa.write_character(dtype); + oa.write_character('#'); key = "_ArraySize_"; write_ubjson(value.at(key), use_count, use_type, true, true, bjdata_version); @@ -1958,7 +1971,7 @@ class binary_writer std::reverse(vec.begin(), vec.end()); } - oa->write_characters(vec.data(), sizeof(NumberType)); + oa.write_characters(vec.data(), sizeof(NumberType)); } void write_compact_float(const number_float_t n, detail::input_format_t format) @@ -1971,16 +1984,16 @@ class binary_writer static_cast(n) <= static_cast((std::numeric_limits::max)()) && static_cast(static_cast(n)) == static_cast(n)))) { - oa->write_character(format == detail::input_format_t::cbor - ? get_cbor_float_prefix(static_cast(n)) - : get_msgpack_float_prefix(static_cast(n))); + oa.write_character(format == detail::input_format_t::cbor + ? get_cbor_float_prefix(static_cast(n)) + : get_msgpack_float_prefix(static_cast(n))); write_number(static_cast(n)); } else { - oa->write_character(format == detail::input_format_t::cbor - ? get_cbor_float_prefix(n) - : get_msgpack_float_prefix(n)); + oa.write_character(format == detail::input_format_t::cbor + ? get_cbor_float_prefix(n) + : get_msgpack_float_prefix(n)); write_number(n); } #ifdef __GNUC__ @@ -2047,7 +2060,7 @@ class binary_writer const bool is_little_endian = little_endianness(); /// the output - output_adapter_t oa = nullptr; + OutputSinkType oa; }; } // namespace detail diff --git a/include/nlohmann/detail/output/output_adapters.hpp b/include/nlohmann/detail/output/output_adapters.hpp index 94763e64a72..e2390e4e55c 100644 --- a/include/nlohmann/detail/output/output_adapters.hpp +++ b/include/nlohmann/detail/output/output_adapters.hpp @@ -118,6 +118,68 @@ class output_string_adapter : public output_adapter_protocol StringType& str; }; +/// @brief non-virtual output sink writing into a std::vector +/// +/// Unlike output_vector_adapter, this sink is not part of the virtual +/// output_adapter_protocol hierarchy: it is passed to binary_writer by value as +/// a template parameter, so write_character()/write_characters() are ordinary +/// (inlinable) calls with no vtable lookup and no shared_ptr. It is used for the +/// common `to_cbor`/`to_msgpack`/... into a std::vector. +template> +class output_vector_sink +{ + public: + explicit output_vector_sink(std::vector& vec) noexcept + : v(vec) + {} + + void write_character(CharType c) + { + v.push_back(c); + } + + JSON_HEDLEY_NON_NULL(2) + void write_characters(const CharType* s, std::size_t length) + { + v.insert(v.end(), s, s + length); + } + + private: + std::vector& v; +}; + +/// @brief output sink forwarding to a type-erased output adapter +/// +/// Wraps the polymorphic output_adapter_t so the same binary_writer template can +/// also target arbitrary adapters (output streams, strings, user-provided +/// adapters) via the `output_adapter`-based overloads. Each write still goes +/// through one virtual call, exactly as before; only the concrete sinks above +/// avoid it. +template +class output_adapter_sink +{ + public: + explicit output_adapter_sink(output_adapter_t adapter) + : oa(std::move(adapter)) + { + JSON_ASSERT(oa); + } + + void write_character(CharType c) + { + oa->write_character(c); + } + + JSON_HEDLEY_NON_NULL(2) + void write_characters(const CharType* s, std::size_t length) + { + oa->write_characters(s, length); + } + + private: + output_adapter_t oa = nullptr; +}; + template> class output_adapter { diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 1aafbf78a78..8c6df33ef98 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -140,7 +140,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec friend ::nlohmann::detail::serializer; template friend class ::nlohmann::detail::iter_impl; - template + template friend class ::nlohmann::detail::binary_writer; template friend class ::nlohmann::detail::binary_reader; @@ -187,7 +187,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec template using binary_reader = ::nlohmann::detail::binary_reader; - template using binary_writer = ::nlohmann::detail::binary_writer; + template> + using binary_writer = ::nlohmann::detail::binary_writer; JSON_PRIVATE_UNLESS_TESTED: using serializer = ::nlohmann::detail::serializer; @@ -4438,7 +4439,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static std::vector to_cbor(const basic_json& j) { std::vector result; - to_cbor(j, result); + binary_writer>( + detail::output_vector_sink(result)).write_cbor(j); return result; } @@ -4461,7 +4463,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static std::vector to_msgpack(const basic_json& j) { std::vector result; - to_msgpack(j, result); + binary_writer>( + detail::output_vector_sink(result)).write_msgpack(j); return result; } @@ -4486,7 +4489,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool use_type = false) { std::vector result; - to_ubjson(j, result, use_size, use_type); + binary_writer>( + detail::output_vector_sink(result)).write_ubjson(j, use_size, use_type); return result; } @@ -4514,7 +4518,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bjdata_version_t version = bjdata_version_t::draft2) { std::vector result; - to_bjdata(j, result, use_size, use_type, version); + binary_writer>( + detail::output_vector_sink(result)).write_ubjson(j, use_size, use_type, true, true, version); return result; } @@ -4541,7 +4546,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static std::vector to_bson(const basic_json& j) { std::vector result; - to_bson(j, result); + binary_writer>( + detail::output_vector_sink(result)).write_bson(j); return result; } From de894c408b10778e05bfc6f7cee701a635fcd9bf Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 22 Jul 2026 08:47:24 +0000 Subject: [PATCH 2/6] Fix CI failures from binary_writer output-sink change Four CI jobs failed on the initial commit; all are addressed here without changing any output (binary encodings remain byte-for-byte identical to develop across the differential corpus): 1. ci_test_gcc / cuda (-Werror=duplicated-branches): for number_float_t == float, static_cast(n) is the identity, so write_compact_float's two branches are intentionally identical. Once the concrete vector sink is inlined, GCC constant-folds and diagnoses this (the type-erased path hid it behind a non-inlined virtual call). Silence -Wduplicated-branches for GCC (clang has no such warning) alongside the existing -Wfloat-equal pragma. 2. ci_static_analysis_clang (UBSan nonnull-attribute): binary_writer passes a null pointer with length 0 for empty strings/binary. output_vector_sink / output_adapter_sink declared write_characters JSON_HEDLEY_NON_NULL, so the sanitizer flagged the (harmless) zero-length call once the sink was called directly rather than through the attribute-free virtual base. Drop the attribute from both sinks, matching the pre-existing behavior. 3. ci_cpplint (build/include_what_you_use): output_adapter_sink uses std::move; add #include . 4. ci_cuda_example (nvcc 11.8): NVCC's front end rejects the default template argument on the binary_writer alias template. Revert the alias to its original single-parameter form (relying on binary_writer's own defaulted OutputSinkType) and spell out the full type in the vector-sink convenience functions. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01XAYM1qhSA2FDaDcGfPW3fG Signed-off-by: Niels Lohmann --- include/nlohmann/detail/output/binary_writer.hpp | 7 +++++++ include/nlohmann/detail/output/output_adapters.hpp | 9 +++++++-- include/nlohmann/json.hpp | 13 ++++++------- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 00181c5dd28..65adae2e6cf 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -1979,6 +1979,13 @@ class binary_writer #ifdef __GNUC__ JSON_HEDLEY_DIAGNOSTIC_PUSH JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") +#endif + // When number_float_t is float, static_cast(n) is the identity and + // both branches below are intentionally identical (the "compact" float + // representation is the value itself). Only GCC diagnoses this, and only + // when the sink calls are inlined; clang has no such warning. +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic ignored "-Wduplicated-branches" #endif if (!std::isfinite(n) || ((static_cast(n) >= static_cast(std::numeric_limits::lowest()) && static_cast(n) <= static_cast((std::numeric_limits::max)()) && diff --git a/include/nlohmann/detail/output/output_adapters.hpp b/include/nlohmann/detail/output/output_adapters.hpp index e2390e4e55c..497bde76843 100644 --- a/include/nlohmann/detail/output/output_adapters.hpp +++ b/include/nlohmann/detail/output/output_adapters.hpp @@ -13,6 +13,7 @@ #include // back_inserter #include // shared_ptr, make_shared #include // basic_string +#include // move #include // vector #ifndef JSON_NO_IO @@ -138,7 +139,10 @@ class output_vector_sink v.push_back(c); } - JSON_HEDLEY_NON_NULL(2) + // no JSON_HEDLEY_NON_NULL here: binary_writer legitimately passes a null + // pointer with length 0 for empty strings/binary values. Appending an empty + // range is a no-op; the type-erased path tolerates this via the (unattributed) + // virtual base, and the concrete sink must do the same. void write_characters(const CharType* s, std::size_t length) { v.insert(v.end(), s, s + length); @@ -170,7 +174,8 @@ class output_adapter_sink oa->write_character(c); } - JSON_HEDLEY_NON_NULL(2) + // no JSON_HEDLEY_NON_NULL: forwards (null, 0) for empty payloads, exactly as + // the type-erased path already did before this sink existed void write_characters(const CharType* s, std::size_t length) { oa->write_characters(s, length); diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 8c6df33ef98..3cf4f484283 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -187,8 +187,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec template using binary_reader = ::nlohmann::detail::binary_reader; - template> - using binary_writer = ::nlohmann::detail::binary_writer; + template using binary_writer = ::nlohmann::detail::binary_writer; JSON_PRIVATE_UNLESS_TESTED: using serializer = ::nlohmann::detail::serializer; @@ -4439,7 +4438,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static std::vector to_cbor(const basic_json& j) { std::vector result; - binary_writer>( + detail::binary_writer>( detail::output_vector_sink(result)).write_cbor(j); return result; } @@ -4463,7 +4462,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static std::vector to_msgpack(const basic_json& j) { std::vector result; - binary_writer>( + detail::binary_writer>( detail::output_vector_sink(result)).write_msgpack(j); return result; } @@ -4489,7 +4488,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool use_type = false) { std::vector result; - binary_writer>( + detail::binary_writer>( detail::output_vector_sink(result)).write_ubjson(j, use_size, use_type); return result; } @@ -4518,7 +4517,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bjdata_version_t version = bjdata_version_t::draft2) { std::vector result; - binary_writer>( + detail::binary_writer>( detail::output_vector_sink(result)).write_ubjson(j, use_size, use_type, true, true, version); return result; } @@ -4546,7 +4545,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static std::vector to_bson(const basic_json& j) { std::vector result; - binary_writer>( + detail::binary_writer>( detail::output_vector_sink(result)).write_bson(j); return result; } From bd722bd83aa412c1977d8f7cf06094b9c05c9fc2 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 22 Jul 2026 10:20:33 +0000 Subject: [PATCH 3/6] Encode big-endian numbers with a byte swap instead of std::reverse write_number() reordered multi-byte numbers for the big-endian formats (CBOR/MessagePack/UBJSON) with std::reverse over the byte array. GCC lowered only some sizes to a bswap; clang kept a scalar byte shuffle (0 bswap instructions in the CBOR number path). Replace the reverse with size-dispatched __builtin_bswap16/32/64 helpers (portable shift fallback for other compilers; std::reverse retained for exotic sizes such as a long double number_float_t). Codegen: the CBOR number path now emits bswap on both compilers (gcc 2 -> 16, clang 0 -> 4). Output is byte-for-byte identical to the previous implementation across the binary differential corpus. Throughput (isolated vs the std::reverse version, best of 9): CBOR int64 array gcc +7% clang +10% CBOR uint16 array gcc +27% clang flat Modest but consistent on number-dense encodings; negligible on string/blob-heavy output, as expected. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01XAYM1qhSA2FDaDcGfPW3fG Signed-off-by: Niels Lohmann --- .../nlohmann/detail/output/binary_writer.hpp | 67 ++++++++++++++++++- single_include/nlohmann/json.hpp | 67 ++++++++++++++++++- 2 files changed, 132 insertions(+), 2 deletions(-) diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 65adae2e6cf..188c6d4c4a9 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -1957,6 +1957,71 @@ class binary_writer On the other hand, BSON and BJData use little endian and should reorder on big endian systems. */ + // single-instruction byte swaps (compilers lower these to bswap/rev/movbe); + // used to emit big-endian numbers without a per-byte std::reverse loop + static std::uint16_t byte_swap(std::uint16_t x) noexcept + { +#if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap16(x); +#else + return static_cast((x >> 8) | (x << 8)); +#endif + } + + static std::uint32_t byte_swap(std::uint32_t x) noexcept + { +#if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap32(x); +#else + return ((x & 0x000000FFu) << 24) | ((x & 0x0000FF00u) << 8) + | ((x & 0x00FF0000u) >> 8) | ((x & 0xFF000000u) >> 24); +#endif + } + + static std::uint64_t byte_swap(std::uint64_t x) noexcept + { +#if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap64(x); +#else + x = ((x & 0x00000000FFFFFFFFull) << 32) | ((x & 0xFFFFFFFF00000000ull) >> 32); + x = ((x & 0x0000FFFF0000FFFFull) << 16) | ((x & 0xFFFF0000FFFF0000ull) >> 16); + x = ((x & 0x00FF00FF00FF00FFull) << 8) | ((x & 0xFF00FF00FF00FF00ull) >> 8); + return x; +#endif + } + + // reverse the bytes of a fixed-size buffer; a single byte_swap() for the + // common 2/4/8-byte number payloads, std::reverse for any other size + static void reverse_bytes(std::array& a) noexcept + { + std::uint16_t v{}; + std::memcpy(&v, a.data(), sizeof(v)); + v = byte_swap(v); + std::memcpy(a.data(), &v, sizeof(v)); + } + + static void reverse_bytes(std::array& a) noexcept + { + std::uint32_t v{}; + std::memcpy(&v, a.data(), sizeof(v)); + v = byte_swap(v); + std::memcpy(a.data(), &v, sizeof(v)); + } + + static void reverse_bytes(std::array& a) noexcept + { + std::uint64_t v{}; + std::memcpy(&v, a.data(), sizeof(v)); + v = byte_swap(v); + std::memcpy(a.data(), &v, sizeof(v)); + } + + template + static void reverse_bytes(std::array& a) noexcept + { + std::reverse(a.begin(), a.end()); + } + template void write_number(const NumberType n, const bool OutputIsLittleEndian = false) { @@ -1968,7 +2033,7 @@ class binary_writer if (is_little_endian != OutputIsLittleEndian) { // reverse byte order prior to conversion if necessary - std::reverse(vec.begin(), vec.end()); + reverse_bytes(vec); } oa.write_characters(vec.data(), sizeof(NumberType)); diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 213236b5108..db2de554a1b 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -20621,6 +20621,71 @@ class binary_writer On the other hand, BSON and BJData use little endian and should reorder on big endian systems. */ + // single-instruction byte swaps (compilers lower these to bswap/rev/movbe); + // used to emit big-endian numbers without a per-byte std::reverse loop + static std::uint16_t byte_swap(std::uint16_t x) noexcept + { +#if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap16(x); +#else + return static_cast((x >> 8) | (x << 8)); +#endif + } + + static std::uint32_t byte_swap(std::uint32_t x) noexcept + { +#if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap32(x); +#else + return ((x & 0x000000FFu) << 24) | ((x & 0x0000FF00u) << 8) + | ((x & 0x00FF0000u) >> 8) | ((x & 0xFF000000u) >> 24); +#endif + } + + static std::uint64_t byte_swap(std::uint64_t x) noexcept + { +#if defined(__GNUC__) || defined(__clang__) + return __builtin_bswap64(x); +#else + x = ((x & 0x00000000FFFFFFFFull) << 32) | ((x & 0xFFFFFFFF00000000ull) >> 32); + x = ((x & 0x0000FFFF0000FFFFull) << 16) | ((x & 0xFFFF0000FFFF0000ull) >> 16); + x = ((x & 0x00FF00FF00FF00FFull) << 8) | ((x & 0xFF00FF00FF00FF00ull) >> 8); + return x; +#endif + } + + // reverse the bytes of a fixed-size buffer; a single byte_swap() for the + // common 2/4/8-byte number payloads, std::reverse for any other size + static void reverse_bytes(std::array& a) noexcept + { + std::uint16_t v{}; + std::memcpy(&v, a.data(), sizeof(v)); + v = byte_swap(v); + std::memcpy(a.data(), &v, sizeof(v)); + } + + static void reverse_bytes(std::array& a) noexcept + { + std::uint32_t v{}; + std::memcpy(&v, a.data(), sizeof(v)); + v = byte_swap(v); + std::memcpy(a.data(), &v, sizeof(v)); + } + + static void reverse_bytes(std::array& a) noexcept + { + std::uint64_t v{}; + std::memcpy(&v, a.data(), sizeof(v)); + v = byte_swap(v); + std::memcpy(a.data(), &v, sizeof(v)); + } + + template + static void reverse_bytes(std::array& a) noexcept + { + std::reverse(a.begin(), a.end()); + } + template void write_number(const NumberType n, const bool OutputIsLittleEndian = false) { @@ -20632,7 +20697,7 @@ class binary_writer if (is_little_endian != OutputIsLittleEndian) { // reverse byte order prior to conversion if necessary - std::reverse(vec.begin(), vec.end()); + reverse_bytes(vec); } oa->write_characters(vec.data(), sizeof(NumberType)); From d72913da2fc518db3d97e37519cb047b887fe8c7 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 22 Jul 2026 10:59:50 +0000 Subject: [PATCH 4/6] Reserve output capacity up front for binary serialization The vector-returning to_cbor/to_msgpack/to_ubjson/to_bjdata/to_bson grew the output buffer purely by geometric reallocation. Reserving an estimate up front avoids the early reallocations, which is the dominant per-byte cost for array/object-heavy output. The estimate (binary_reserve_hint) is deliberately conservative and safe against untrusted input: it consults only the top-level element count (O(1), no walk of the DOM), guards the multiplication against overflow, and clamps the result to a fixed 1 MiB ceiling, so a large or hostile DOM can never force an oversized allocation here. The buffer still grows geometrically past the hint, so an underestimate only costs a few later reallocations; scalars/strings/binary are written in one shot and get no hint. Reserving capacity does not change the bytes produced. Throughput (g++/clang -O3, vs the previous commit): cbor int array +10% / +13% cbor object array +20% / +38% Output is byte-for-byte identical to develop across the binary differential corpus. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01XAYM1qhSA2FDaDcGfPW3fG Signed-off-by: Niels Lohmann --- .../nlohmann/detail/output/binary_writer.hpp | 29 +++++++++++++++++++ include/nlohmann/json.hpp | 5 ++++ 2 files changed, 34 insertions(+) diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 188c6d4c4a9..cb3aadd85bc 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -39,6 +39,35 @@ enum class bjdata_version_t // binary writer // /////////////////// +/*! +@brief conservative capacity hint for binary serialization into a std::vector + +Returns an approximate number of bytes to reserve up front so that serializing +an array/object of many elements does not repeatedly reallocate the output +buffer. Only the top-level element count is consulted (O(1), no walk of the +DOM), and the result is clamped to a fixed ceiling: a large or untrusted DOM can +therefore never trigger an oversized allocation here, and the multiplication +cannot overflow. The buffer still grows geometrically beyond the hint, so a hint +that is too small only costs a few later reallocations. A single scalar, string, +or binary value is written in one shot and needs no hint. +*/ +template +std::size_t binary_reserve_hint(const BasicJsonType& j) +{ + constexpr std::size_t max_hint = static_cast(1) << 20; // 1 MiB + if (j.is_array() || j.is_object()) + { + const std::size_t elements = j.size(); + // guard the multiplication against overflow and cap the reservation + if (elements > max_hint / 4) + { + return max_hint; + } + return (elements * 4) + 2; + } + return 0; +} + /*! @brief serialization to CBOR and MessagePack values */ diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 3cf4f484283..cca2f59f2d4 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -4438,6 +4438,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static std::vector to_cbor(const basic_json& j) { std::vector result; + result.reserve(detail::binary_reserve_hint(j)); detail::binary_writer>( detail::output_vector_sink(result)).write_cbor(j); return result; @@ -4462,6 +4463,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static std::vector to_msgpack(const basic_json& j) { std::vector result; + result.reserve(detail::binary_reserve_hint(j)); detail::binary_writer>( detail::output_vector_sink(result)).write_msgpack(j); return result; @@ -4488,6 +4490,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool use_type = false) { std::vector result; + result.reserve(detail::binary_reserve_hint(j)); detail::binary_writer>( detail::output_vector_sink(result)).write_ubjson(j, use_size, use_type); return result; @@ -4517,6 +4520,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bjdata_version_t version = bjdata_version_t::draft2) { std::vector result; + result.reserve(detail::binary_reserve_hint(j)); detail::binary_writer>( detail::output_vector_sink(result)).write_ubjson(j, use_size, use_type, true, true, version); return result; @@ -4545,6 +4549,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static std::vector to_bson(const basic_json& j) { std::vector result; + result.reserve(detail::binary_reserve_hint(j)); detail::binary_writer>( detail::output_vector_sink(result)).write_bson(j); return result; From 7db5b116cc0fc7bb48e0d62096ec82156a018ac3 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 19 Aug 2026 20:32:52 +0200 Subject: [PATCH 5/6] Address review findings on the binary writer output sinks - binary_reserve_hint(): the 4-bytes-per-element estimate over-reserved by up to 4x for arrays of small scalars (CBOR encodes 0..23 in one byte), and the returned vector kept that capacity. Make the hint a strict lower bound on the encoded size instead, which also removes the 1 MiB clamp whose branch no test could reach (the largest container in the suite has 65793 elements). - Guard the -Wduplicated-branches pragma with __GNUC__ >= 7. The warning does not exist before GCC 7, so naming it made GCC 4.8/4.9/5/6 - which the CI matrix still builds - warn under -Wpragmas on every including translation unit, breaking downstream -Werror builds. - Constrain the adapter constructor of binary_writer with the enable_if its documentation already claimed, so a writer over some other sink type is no longer advertised as constructible from an output adapter. - Let output_vector_adapter wrap output_vector_sink rather than duplicating the append logic, so the type-erased and templated paths share one implementation. - Collapse the three copies of the memcpy/byte_swap/memcpy dance into a single byte_swap_buffer() helper, and add the MSVC _byteswap_* intrinsics so MSVC no longer falls back to the scalar shuffle this change exists to eliminate. - Add a vector_writer() helper for the five vector-returning to_* overloads instead of spelling out the writer type at each call site, and drop a dead default member initializer on output_adapter_sink. - New tests: the vector sink and the adapter sink must produce identical bytes for every format (the two to_* overloads no longer delegate to each other and could otherwise drift), and binary_reserve_hint() must never exceed the size actually written. Signed-off-by: Niels Lohmann --- .../nlohmann/detail/output/binary_writer.hpp | 98 +++++---- .../detail/output/output_adapters.hpp | 81 +++---- include/nlohmann/json.hpp | 23 +- tests/src/unit-binary_writer_sinks.cpp | 198 ++++++++++++++++++ 4 files changed, 317 insertions(+), 83 deletions(-) create mode 100644 tests/src/unit-binary_writer_sinks.cpp diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index cb3aadd85bc..c197b42211d 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -16,9 +16,14 @@ #include // memcpy #include // numeric_limits #include // string +#include // enable_if, is_constructible #include // move #include // vector +#ifdef _MSC_VER + #include // _byteswap_ushort, _byteswap_ulong, _byteswap_uint64 +#endif + #include #include #include @@ -40,31 +45,33 @@ enum class bjdata_version_t /////////////////// /*! -@brief conservative capacity hint for binary serialization into a std::vector - -Returns an approximate number of bytes to reserve up front so that serializing -an array/object of many elements does not repeatedly reallocate the output -buffer. Only the top-level element count is consulted (O(1), no walk of the -DOM), and the result is clamped to a fixed ceiling: a large or untrusted DOM can -therefore never trigger an oversized allocation here, and the multiplication -cannot overflow. The buffer still grows geometrically beyond the hint, so a hint -that is too small only costs a few later reallocations. A single scalar, string, -or binary value is written in one shot and needs no hint. +@brief capacity hint for binary serialization into a std::vector + +Returns a *lower* bound on the number of bytes the serialization will produce, +so that writing an array/object of many elements does not start reallocating +from an empty buffer. Every array element occupies at least one byte in every +supported binary format, and every object entry at least two (a key of at least +one byte plus a value of at least one), plus one byte for the container header, +so the hint can never exceed the final size and the returned vector is never +left holding capacity the caller did not ask for. The buffer still grows +geometrically past the hint, so under-reserving only costs a few later +reallocations. Only the top-level element count is consulted (O(1), no walk of +the DOM); a single scalar, string, or binary value is written in one shot and +needs no hint. */ template std::size_t binary_reserve_hint(const BasicJsonType& j) { - constexpr std::size_t max_hint = static_cast(1) << 20; // 1 MiB - if (j.is_array() || j.is_object()) + if (j.is_array()) { - const std::size_t elements = j.size(); - // guard the multiplication against overflow and cap the reservation - if (elements > max_hint / 4) - { - return max_hint; - } - return (elements * 4) + 2; + return j.size() + 1; + } + + if (j.is_object()) + { + return (j.size() * 2) + 1; } + return 0; } @@ -94,12 +101,15 @@ class binary_writer Convenience constructor for the default (output_adapter_sink) sink so the `output_adapter`-based overloads keep constructing the writer directly from - an adapter. Only participates in overload resolution when the sink can be - built from an adapter. + an adapter. Constrained to sinks that can actually be built from an adapter, + so that a writer over some other sink type is not advertised as constructible + from one. @param[in] adapter output adapter to write to */ - explicit binary_writer(output_adapter_t adapter) : oa(OutputSinkType(std::move(adapter))) + template < typename SinkType = OutputSinkType, + typename std::enable_if < std::is_constructible>::value, int >::type = 0 > + explicit binary_writer(output_adapter_t adapter) : oa(SinkType(std::move(adapter))) {} /*! @@ -1992,6 +2002,8 @@ class binary_writer { #if defined(__GNUC__) || defined(__clang__) return __builtin_bswap16(x); +#elif defined(_MSC_VER) + return _byteswap_ushort(x); #else return static_cast((x >> 8) | (x << 8)); #endif @@ -2001,6 +2013,8 @@ class binary_writer { #if defined(__GNUC__) || defined(__clang__) return __builtin_bswap32(x); +#elif defined(_MSC_VER) + return _byteswap_ulong(x); #else return ((x & 0x000000FFu) << 24) | ((x & 0x0000FF00u) << 8) | ((x & 0x00FF0000u) >> 8) | ((x & 0xFF000000u) >> 24); @@ -2011,6 +2025,8 @@ class binary_writer { #if defined(__GNUC__) || defined(__clang__) return __builtin_bswap64(x); +#elif defined(_MSC_VER) + return _byteswap_uint64(x); #else x = ((x & 0x00000000FFFFFFFFull) << 32) | ((x & 0xFFFFFFFF00000000ull) >> 32); x = ((x & 0x0000FFFF0000FFFFull) << 16) | ((x & 0xFFFF0000FFFF0000ull) >> 16); @@ -2019,30 +2035,40 @@ class binary_writer #endif } - // reverse the bytes of a fixed-size buffer; a single byte_swap() for the - // common 2/4/8-byte number payloads, std::reverse for any other size - static void reverse_bytes(std::array& a) noexcept + /*! + @brief reverse the bytes of a buffer by byte-swapping it as UIntType + + Loading the buffer into an unsigned integer of the same width and swapping + that is what lets the compiler emit a single bswap/rev/movbe; reversing the + buffer element by element does not reliably get there (clang keeps a scalar + shuffle). The two memcpy calls are the only portable way to reinterpret the + bytes and are folded away by every optimizer. + */ + template + static void byte_swap_buffer(std::array& a) noexcept { - std::uint16_t v{}; + static_assert(sizeof(UIntType) == N, "swap width must match the buffer size"); + UIntType v{}; std::memcpy(&v, a.data(), sizeof(v)); v = byte_swap(v); std::memcpy(a.data(), &v, sizeof(v)); } + // reverse the bytes of a fixed-size buffer; a single byte_swap() for the + // common 2/4/8-byte number payloads, std::reverse for any other size + static void reverse_bytes(std::array& a) noexcept + { + byte_swap_buffer(a); + } + static void reverse_bytes(std::array& a) noexcept { - std::uint32_t v{}; - std::memcpy(&v, a.data(), sizeof(v)); - v = byte_swap(v); - std::memcpy(a.data(), &v, sizeof(v)); + byte_swap_buffer(a); } static void reverse_bytes(std::array& a) noexcept { - std::uint64_t v{}; - std::memcpy(&v, a.data(), sizeof(v)); - v = byte_swap(v); - std::memcpy(a.data(), &v, sizeof(v)); + byte_swap_buffer(a); } template @@ -2078,7 +2104,9 @@ class binary_writer // both branches below are intentionally identical (the "compact" float // representation is the value itself). Only GCC diagnoses this, and only // when the sink calls are inlined; clang has no such warning. -#if defined(__GNUC__) && !defined(__clang__) + // (-Wduplicated-branches only exists from GCC 7 on; naming it on an older + // GCC would itself warn under -Wpragmas) +#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 7) #pragma GCC diagnostic ignored "-Wduplicated-branches" #endif if (!std::isfinite(n) || ((static_cast(n) >= static_cast(std::numeric_limits::lowest()) && diff --git a/include/nlohmann/detail/output/output_adapters.hpp b/include/nlohmann/detail/output/output_adapters.hpp index 497bde76843..7eb73121c92 100644 --- a/include/nlohmann/detail/output/output_adapters.hpp +++ b/include/nlohmann/detail/output/output_adapters.hpp @@ -45,28 +45,66 @@ template struct output_adapter_protocol template using output_adapter_t = std::shared_ptr>; +/// @brief non-virtual output sink writing into a std::vector +/// +/// This sink is not part of the virtual output_adapter_protocol hierarchy: it is +/// passed to binary_writer by value as a template parameter, so +/// write_character()/write_characters() are ordinary (inlinable) calls with no +/// vtable lookup and no shared_ptr. It is used for the common +/// `to_cbor`/`to_msgpack`/... into a std::vector. output_vector_adapter below +/// wraps this same sink to provide the virtual interface. +template> +class output_vector_sink +{ + public: + explicit output_vector_sink(std::vector& vec) noexcept + : v(vec) + {} + + void write_character(CharType c) + { + v.push_back(c); + } + + // no JSON_HEDLEY_NON_NULL here: binary_writer legitimately passes a null + // pointer with length 0 for empty strings/binary values. Appending an empty + // range is a no-op; the type-erased path tolerates this via the (unattributed) + // virtual base, and the concrete sink must do the same. + void write_characters(const CharType* s, std::size_t length) + { + v.insert(v.end(), s, s + length); + } + + private: + std::vector& v; +}; + /// output adapter for byte vectors +/// +/// The appending itself lives in output_vector_sink; this class only adds the +/// virtual output_adapter_protocol interface on top of it, so both the +/// type-erased and the templated path share one implementation. template> class output_vector_adapter : public output_adapter_protocol { public: explicit output_vector_adapter(std::vector& vec) noexcept - : v(vec) + : sink(vec) {} void write_character(CharType c) override { - v.push_back(c); + sink.write_character(c); } JSON_HEDLEY_NON_NULL(2) void write_characters(const CharType* s, std::size_t length) override { - v.insert(v.end(), s, s + length); + sink.write_characters(s, length); } private: - std::vector& v; + output_vector_sink sink; }; #ifndef JSON_NO_IO @@ -119,39 +157,6 @@ class output_string_adapter : public output_adapter_protocol StringType& str; }; -/// @brief non-virtual output sink writing into a std::vector -/// -/// Unlike output_vector_adapter, this sink is not part of the virtual -/// output_adapter_protocol hierarchy: it is passed to binary_writer by value as -/// a template parameter, so write_character()/write_characters() are ordinary -/// (inlinable) calls with no vtable lookup and no shared_ptr. It is used for the -/// common `to_cbor`/`to_msgpack`/... into a std::vector. -template> -class output_vector_sink -{ - public: - explicit output_vector_sink(std::vector& vec) noexcept - : v(vec) - {} - - void write_character(CharType c) - { - v.push_back(c); - } - - // no JSON_HEDLEY_NON_NULL here: binary_writer legitimately passes a null - // pointer with length 0 for empty strings/binary values. Appending an empty - // range is a no-op; the type-erased path tolerates this via the (unattributed) - // virtual base, and the concrete sink must do the same. - void write_characters(const CharType* s, std::size_t length) - { - v.insert(v.end(), s, s + length); - } - - private: - std::vector& v; -}; - /// @brief output sink forwarding to a type-erased output adapter /// /// Wraps the polymorphic output_adapter_t so the same binary_writer template can @@ -182,7 +187,7 @@ class output_adapter_sink } private: - output_adapter_t oa = nullptr; + output_adapter_t oa; }; template> diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index cca2f59f2d4..58009b9caa2 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -188,6 +188,14 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec template using binary_reader = ::nlohmann::detail::binary_reader; template using binary_writer = ::nlohmann::detail::binary_writer; + // binary_writer over a concrete (non-virtual) sink appending into a std::vector, + // used by the vector-returning to_* overloads + template using vector_binary_writer = + ::nlohmann::detail::binary_writer>; + template static vector_binary_writer vector_writer(std::vector& v) + { + return vector_binary_writer(::nlohmann::detail::output_vector_sink(v)); + } JSON_PRIVATE_UNLESS_TESTED: using serializer = ::nlohmann::detail::serializer; @@ -4439,8 +4447,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec { std::vector result; result.reserve(detail::binary_reserve_hint(j)); - detail::binary_writer>( - detail::output_vector_sink(result)).write_cbor(j); + vector_writer(result).write_cbor(j); return result; } @@ -4464,8 +4471,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec { std::vector result; result.reserve(detail::binary_reserve_hint(j)); - detail::binary_writer>( - detail::output_vector_sink(result)).write_msgpack(j); + vector_writer(result).write_msgpack(j); return result; } @@ -4491,8 +4497,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec { std::vector result; result.reserve(detail::binary_reserve_hint(j)); - detail::binary_writer>( - detail::output_vector_sink(result)).write_ubjson(j, use_size, use_type); + vector_writer(result).write_ubjson(j, use_size, use_type); return result; } @@ -4521,8 +4526,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec { std::vector result; result.reserve(detail::binary_reserve_hint(j)); - detail::binary_writer>( - detail::output_vector_sink(result)).write_ubjson(j, use_size, use_type, true, true, version); + vector_writer(result).write_ubjson(j, use_size, use_type, true, true, version); return result; } @@ -4550,8 +4554,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec { std::vector result; result.reserve(detail::binary_reserve_hint(j)); - detail::binary_writer>( - detail::output_vector_sink(result)).write_bson(j); + vector_writer(result).write_bson(j); return result; } diff --git a/tests/src/unit-binary_writer_sinks.cpp b/tests/src/unit-binary_writer_sinks.cpp new file mode 100644 index 00000000000..f60e1bf514d --- /dev/null +++ b/tests/src/unit-binary_writer_sinks.cpp @@ -0,0 +1,198 @@ +// __ _____ _____ _____ +// __| | __| | | | JSON for Modern C++ (supporting code) +// | | |__ | | | | | | version 3.12.0 +// |_____|_____|_____|_|___| https://github.com/nlohmann/json +// +// SPDX-FileCopyrightText: 2013-2026 Niels Lohmann +// SPDX-License-Identifier: MIT + +#include "doctest_compatibility.h" + +#include +using nlohmann::json; + +#include +#include +#include + +namespace +{ + +// a spread of values exercising every writer path: scalars of each width, the +// float paths, strings, binary, and containers big enough to reallocate +std::vector test_values() +{ + json big_array = json::array(); + for (int i = 0; i < 5000; ++i) + { + big_array.push_back(i); + } + + json big_object = json::object(); + for (int i = 0; i < 1000; ++i) + { + big_object[std::to_string(i)] = i; + } + + return + { + json(nullptr), json(true), json(false), + json(0), json(-1), json(255), json(-129), json(65535), json(-32769), + json(4294967295U), json(-2147483649LL), json(18446744073709551615ULL), + json(0.0), json(-0.5), json(3.1415926535897932), + json(""), json("hello"), json(std::string(1000, 'x')), + json::binary({0x00, 0x01, 0x02}, 42), + json::array(), json::object(), + json::array({1, 2, 3}), json({{"a", 1}, {"b", nullptr}}), + json({{"nested", {{"deep", json::array({1, "two", 3.0, nullptr})}}}}), + big_array, big_object + }; +} + +// values to_bson() accepts: the document must be an object +std::vector bson_values() +{ + json big_object = json::object(); + for (int i = 0; i < 1000; ++i) + { + big_object[std::to_string(i)] = i; + } + + return + { + json::object(), + json({{"a", 1}, {"b", nullptr}, {"c", true}, {"d", 2.5}, {"e", "text"}}), + json({{"arr", json::array({1, 2, 3})}, {"obj", {{"k", "v"}}}}), + big_object + }; +} + +} // namespace + +// The vector-returning to_*(j) overloads write through the non-virtual +// output_vector_sink, while to_*(j, adapter) goes through output_adapter_sink. +// The two are separate code paths that must stay byte-for-byte identical; these +// checks fail if either overload is ever changed without the other. +TEST_CASE("binary writer output sinks") +{ + SECTION("vector sink and adapter sink agree") + { + // note: no SUBCASE inside these loops - doctest keys subcases by + // name/file/line, so a subcase in a loop body would only ever run for + // the first iteration + for (const auto& j : test_values()) + { + CAPTURE(j.dump(-1, ' ', false, json::error_handler_t::replace)); + + std::vector cbor; + json::to_cbor(j, cbor); + CHECK(json::to_cbor(j) == cbor); + + std::vector msgpack; + json::to_msgpack(j, msgpack); + CHECK(json::to_msgpack(j) == msgpack); + + for (const bool use_size : + { + false, true + }) + { + for (const bool use_type : + { + false, true + }) + { + if (use_type && !use_size) + { + continue; // not a supported combination + } + CAPTURE(use_size); + CAPTURE(use_type); + std::vector ubjson; + json::to_ubjson(j, ubjson, use_size, use_type); + CHECK(json::to_ubjson(j, use_size, use_type) == ubjson); + } + } + + for (const auto version : + { + json::bjdata_version_t::draft2, json::bjdata_version_t::draft3 + }) + { + std::vector bjdata; + json::to_bjdata(j, bjdata, false, false, version); + CHECK(json::to_bjdata(j, false, false, version) == bjdata); + } + } + + for (const auto& j : bson_values()) + { + CAPTURE(j.dump()); + std::vector bson; + json::to_bson(j, bson); + CHECK(json::to_bson(j) == bson); + } + } + + SECTION("the char adapter produces the same bytes") + { + for (const auto& j : test_values()) + { + CAPTURE(j.dump(-1, ' ', false, json::error_handler_t::replace)); + + const std::vector expected = json::to_cbor(j); + std::vector as_char; + json::to_cbor(j, as_char); + + REQUIRE(as_char.size() == expected.size()); + std::vector as_bytes; + as_bytes.reserve(as_char.size()); + for (const char c : as_char) + { + as_bytes.push_back(static_cast(c)); + } + CHECK(as_bytes == expected); + } + } +} + +// binary_reserve_hint() is documented as a *lower* bound on the serialized size, +// so that reserving it up front can never leave the returned vector holding +// capacity beyond what the value actually needs. +TEST_CASE("binary_reserve_hint never over-reserves") +{ + for (const auto& j : test_values()) + { + CAPTURE(j.dump(-1, ' ', false, json::error_handler_t::replace)); + + const std::size_t hint = nlohmann::detail::binary_reserve_hint(j); + + CHECK(hint <= json::to_cbor(j).size()); + CHECK(hint <= json::to_msgpack(j).size()); + CHECK(hint <= json::to_ubjson(j).size()); + CHECK(hint <= json::to_ubjson(j, true, true).size()); + CHECK(hint <= json::to_bjdata(j).size()); + } + + for (const auto& j : bson_values()) + { + CAPTURE(j.dump()); + CHECK(nlohmann::detail::binary_reserve_hint(j) <= json::to_bson(j).size()); + } + + SECTION("scalars get no hint") + { + CHECK(nlohmann::detail::binary_reserve_hint(json(nullptr)) == 0); + CHECK(nlohmann::detail::binary_reserve_hint(json(42)) == 0); + CHECK(nlohmann::detail::binary_reserve_hint(json("a string")) == 0); + CHECK(nlohmann::detail::binary_reserve_hint(json::binary({0x01})) == 0); + } + + SECTION("containers are hinted from their element count") + { + CHECK(nlohmann::detail::binary_reserve_hint(json::array()) == 1); + CHECK(nlohmann::detail::binary_reserve_hint(json::array({1, 2, 3})) == 4); + CHECK(nlohmann::detail::binary_reserve_hint(json::object()) == 1); + CHECK(nlohmann::detail::binary_reserve_hint(json({{"a", 1}, {"b", 2}})) == 5); + } +} From 4d0ccb4c998b1d03f5f6a0c655f5ef4baa60550f Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 11 Sep 2026 06:27:28 +0000 Subject: [PATCH 6/6] Route the -Wduplicated-branches pragma through Hedley Match #5485, which moved the binary writer's hand-rolled diagnostic pragmas onto JSON_HEDLEY_PRAGMA (merged into develop while this branch was open). The devirtualization's -Wduplicated-branches suppression in write_compact_float was the one raw '#pragma GCC diagnostic' left; it now uses JSON_HEDLEY_PRAGMA like the adjacent -Wfloat-equal line, still guarded to GCC >= 7 and non-clang (the warning exists only there). Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01XAYM1qhSA2FDaDcGfPW3fG Signed-off-by: Niels Lohmann --- .../nlohmann/detail/output/binary_writer.hpp | 26 +- single_include/nlohmann/json.hpp | 520 ++++++++++++------ 2 files changed, 354 insertions(+), 192 deletions(-) diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index c197b42211d..06458e8bf5c 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -313,8 +313,8 @@ class binary_writer // step 2: write the string oa.write_characters( - reinterpret_cast(j.m_data.m_value.string->data()), - j.m_data.m_value.string->size()); + reinterpret_cast(j.m_data.m_value.string->data()), + j.m_data.m_value.string->size()); break; } @@ -633,8 +633,8 @@ class binary_writer // step 2: write the string oa.write_characters( - reinterpret_cast(j.m_data.m_value.string->data()), - j.m_data.m_value.string->size()); + reinterpret_cast(j.m_data.m_value.string->data()), + j.m_data.m_value.string->size()); break; } @@ -850,8 +850,8 @@ class binary_writer } write_number_with_ubjson_prefix(j.m_data.m_value.string->size(), true, use_bjdata); oa.write_characters( - reinterpret_cast(j.m_data.m_value.string->data()), - j.m_data.m_value.string->size()); + reinterpret_cast(j.m_data.m_value.string->data()), + j.m_data.m_value.string->size()); break; } @@ -1012,8 +1012,8 @@ class binary_writer { write_number_with_ubjson_prefix(el.first.size(), true, use_bjdata); oa.write_characters( - reinterpret_cast(el.first.data()), - el.first.size()); + reinterpret_cast(el.first.data()), + el.first.size()); write_ubjson(el.second, use_count, use_type, prefix_required, use_bjdata, bjdata_version); } @@ -1075,8 +1075,8 @@ class binary_writer { oa.write_character(to_char_type(element_type)); oa.write_characters( - reinterpret_cast(name.data()), - name.size()); + reinterpret_cast(name.data()), + name.size()); // the terminating null byte is written explicitly rather than taken // from the buffer, so that string_t::data() need not be null-terminated oa.write_character(to_char_type(0x00)); @@ -1120,8 +1120,8 @@ class binary_writer write_number(to_bson_length(value.size() + 1ul), true); oa.write_characters( - reinterpret_cast(value.data()), - value.size()); + reinterpret_cast(value.data()), + value.size()); // the terminating null byte is written explicitly rather than taken // from the buffer, so that string_t::data() need not be null-terminated oa.write_character(to_char_type(0x00)); @@ -2107,7 +2107,7 @@ class binary_writer // (-Wduplicated-branches only exists from GCC 7 on; naming it on an older // GCC would itself warn under -Wpragmas) #if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 7) -#pragma GCC diagnostic ignored "-Wduplicated-branches" + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wduplicated-branches") #endif if (!std::isfinite(n) || ((static_cast(n) >= static_cast(std::numeric_limits::lowest()) && static_cast(n) <= static_cast((std::numeric_limits::max)()) && diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index db2de554a1b..021e296aa98 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -18541,9 +18541,14 @@ NLOHMANN_JSON_NAMESPACE_END #include // memcpy #include // numeric_limits #include // string +#include // enable_if, is_constructible #include // move #include // vector +#ifdef _MSC_VER + #include // _byteswap_ushort, _byteswap_ulong, _byteswap_uint64 +#endif + // #include // #include @@ -18564,6 +18569,7 @@ NLOHMANN_JSON_NAMESPACE_END #include // back_inserter #include // shared_ptr, make_shared #include // basic_string +#include // move #include // vector #ifndef JSON_NO_IO @@ -18596,28 +18602,66 @@ template struct output_adapter_protocol template using output_adapter_t = std::shared_ptr>; +/// @brief non-virtual output sink writing into a std::vector +/// +/// This sink is not part of the virtual output_adapter_protocol hierarchy: it is +/// passed to binary_writer by value as a template parameter, so +/// write_character()/write_characters() are ordinary (inlinable) calls with no +/// vtable lookup and no shared_ptr. It is used for the common +/// `to_cbor`/`to_msgpack`/... into a std::vector. output_vector_adapter below +/// wraps this same sink to provide the virtual interface. +template> +class output_vector_sink +{ + public: + explicit output_vector_sink(std::vector& vec) noexcept + : v(vec) + {} + + void write_character(CharType c) + { + v.push_back(c); + } + + // no JSON_HEDLEY_NON_NULL here: binary_writer legitimately passes a null + // pointer with length 0 for empty strings/binary values. Appending an empty + // range is a no-op; the type-erased path tolerates this via the (unattributed) + // virtual base, and the concrete sink must do the same. + void write_characters(const CharType* s, std::size_t length) + { + v.insert(v.end(), s, s + length); + } + + private: + std::vector& v; +}; + /// output adapter for byte vectors +/// +/// The appending itself lives in output_vector_sink; this class only adds the +/// virtual output_adapter_protocol interface on top of it, so both the +/// type-erased and the templated path share one implementation. template> class output_vector_adapter : public output_adapter_protocol { public: explicit output_vector_adapter(std::vector& vec) noexcept - : v(vec) + : sink(vec) {} void write_character(CharType c) override { - v.push_back(c); + sink.write_character(c); } JSON_HEDLEY_NON_NULL(2) void write_characters(const CharType* s, std::size_t length) override { - v.insert(v.end(), s, s + length); + sink.write_characters(s, length); } private: - std::vector& v; + output_vector_sink sink; }; #ifndef JSON_NO_IO @@ -18670,6 +18714,39 @@ class output_string_adapter : public output_adapter_protocol StringType& str; }; +/// @brief output sink forwarding to a type-erased output adapter +/// +/// Wraps the polymorphic output_adapter_t so the same binary_writer template can +/// also target arbitrary adapters (output streams, strings, user-provided +/// adapters) via the `output_adapter`-based overloads. Each write still goes +/// through one virtual call, exactly as before; only the concrete sinks above +/// avoid it. +template +class output_adapter_sink +{ + public: + explicit output_adapter_sink(output_adapter_t adapter) + : oa(std::move(adapter)) + { + JSON_ASSERT(oa); + } + + void write_character(CharType c) + { + oa->write_character(c); + } + + // no JSON_HEDLEY_NON_NULL: forwards (null, 0) for empty payloads, exactly as + // the type-erased path already did before this sink existed + void write_characters(const CharType* s, std::size_t length) + { + oa->write_characters(s, length); + } + + private: + output_adapter_t oa; +}; + template> class output_adapter { @@ -18716,10 +18793,41 @@ enum class bjdata_version_t // binary writer // /////////////////// +/*! +@brief capacity hint for binary serialization into a std::vector + +Returns a *lower* bound on the number of bytes the serialization will produce, +so that writing an array/object of many elements does not start reallocating +from an empty buffer. Every array element occupies at least one byte in every +supported binary format, and every object entry at least two (a key of at least +one byte plus a value of at least one), plus one byte for the container header, +so the hint can never exceed the final size and the returned vector is never +left holding capacity the caller did not ask for. The buffer still grows +geometrically past the hint, so under-reserving only costs a few later +reallocations. Only the top-level element count is consulted (O(1), no walk of +the DOM); a single scalar, string, or binary value is written in one shot and +needs no hint. +*/ +template +std::size_t binary_reserve_hint(const BasicJsonType& j) +{ + if (j.is_array()) + { + return j.size() + 1; + } + + if (j.is_object()) + { + return (j.size() * 2) + 1; + } + + return 0; +} + /*! @brief serialization to CBOR and MessagePack values */ -template +template> class binary_writer { using string_t = typename BasicJsonType::string_t; @@ -18730,12 +18838,28 @@ class binary_writer /*! @brief create a binary writer + @param[in] sink output sink to write to (a value-type sink such as + output_vector_sink, or output_adapter_sink wrapping a + type-erased output adapter) + */ + explicit binary_writer(OutputSinkType sink) : oa(std::move(sink)) + {} + + /*! + @brief create a binary writer from a type-erased output adapter + + Convenience constructor for the default (output_adapter_sink) sink so the + `output_adapter`-based overloads keep constructing the writer directly from + an adapter. Constrained to sinks that can actually be built from an adapter, + so that a writer over some other sink type is not advertised as constructible + from one. + @param[in] adapter output adapter to write to */ - explicit binary_writer(output_adapter_t adapter) : oa(std::move(adapter)) - { - JSON_ASSERT(oa); - } + template < typename SinkType = OutputSinkType, + typename std::enable_if < std::is_constructible>::value, int >::type = 0 > + explicit binary_writer(output_adapter_t adapter) : oa(SinkType(std::move(adapter))) + {} /*! @param[in] j JSON value to serialize @@ -18776,15 +18900,15 @@ class binary_writer { case value_t::null: { - oa->write_character(to_char_type(0xF6)); + oa.write_character(to_char_type(0xF6)); break; } case value_t::boolean: { - oa->write_character(j.m_data.m_value.boolean - ? to_char_type(0xF5) - : to_char_type(0xF4)); + oa.write_character(j.m_data.m_value.boolean + ? to_char_type(0xF5) + : to_char_type(0xF4)); break; } @@ -18801,22 +18925,22 @@ class binary_writer } else if (j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x18)); + oa.write_character(to_char_type(0x18)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x19)); + oa.write_character(to_char_type(0x19)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x1A)); + oa.write_character(to_char_type(0x1A)); write_number(static_cast(j.m_data.m_value.number_integer)); } else { - oa->write_character(to_char_type(0x1B)); + oa.write_character(to_char_type(0x1B)); write_number(static_cast(j.m_data.m_value.number_integer)); } } @@ -18831,22 +18955,22 @@ class binary_writer } else if (positive_number <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x38)); + oa.write_character(to_char_type(0x38)); write_number(static_cast(positive_number)); } else if (positive_number <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x39)); + oa.write_character(to_char_type(0x39)); write_number(static_cast(positive_number)); } else if (positive_number <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x3A)); + oa.write_character(to_char_type(0x3A)); write_number(static_cast(positive_number)); } else { - oa->write_character(to_char_type(0x3B)); + oa.write_character(to_char_type(0x3B)); write_number(static_cast(positive_number)); } } @@ -18861,22 +18985,22 @@ class binary_writer } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x18)); + oa.write_character(to_char_type(0x18)); write_number(static_cast(j.m_data.m_value.number_unsigned)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x19)); + oa.write_character(to_char_type(0x19)); write_number(static_cast(j.m_data.m_value.number_unsigned)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x1A)); + oa.write_character(to_char_type(0x1A)); write_number(static_cast(j.m_data.m_value.number_unsigned)); } else { - oa->write_character(to_char_type(0x1B)); + oa.write_character(to_char_type(0x1B)); write_number(static_cast(j.m_data.m_value.number_unsigned)); } break; @@ -18887,16 +19011,16 @@ class binary_writer if (std::isnan(j.m_data.m_value.number_float)) { // NaN is 0xf97e00 in CBOR - oa->write_character(to_char_type(0xF9)); - oa->write_character(to_char_type(0x7E)); - oa->write_character(to_char_type(0x00)); + oa.write_character(to_char_type(0xF9)); + oa.write_character(to_char_type(0x7E)); + oa.write_character(to_char_type(0x00)); } else if (std::isinf(j.m_data.m_value.number_float)) { // Infinity is 0xf97c00, -Infinity is 0xf9fc00 - oa->write_character(to_char_type(0xf9)); - oa->write_character(j.m_data.m_value.number_float > 0 ? to_char_type(0x7C) : to_char_type(0xFC)); - oa->write_character(to_char_type(0x00)); + oa.write_character(to_char_type(0xf9)); + oa.write_character(j.m_data.m_value.number_float > 0 ? to_char_type(0x7C) : to_char_type(0xFC)); + oa.write_character(to_char_type(0x00)); } else { @@ -18915,31 +19039,31 @@ class binary_writer } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x78)); + oa.write_character(to_char_type(0x78)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x79)); + oa.write_character(to_char_type(0x79)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x7A)); + oa.write_character(to_char_type(0x7A)); write_number(static_cast(N)); } // LCOV_EXCL_START else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x7B)); + oa.write_character(to_char_type(0x7B)); write_number(static_cast(N)); } // LCOV_EXCL_STOP // step 2: write the string - oa->write_characters( - reinterpret_cast(j.m_data.m_value.string->data()), - j.m_data.m_value.string->size()); + oa.write_characters( + reinterpret_cast(j.m_data.m_value.string->data()), + j.m_data.m_value.string->size()); break; } @@ -18953,23 +19077,23 @@ class binary_writer } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x98)); + oa.write_character(to_char_type(0x98)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x99)); + oa.write_character(to_char_type(0x99)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x9A)); + oa.write_character(to_char_type(0x9A)); write_number(static_cast(N)); } // LCOV_EXCL_START else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x9B)); + oa.write_character(to_char_type(0x9B)); write_number(static_cast(N)); } // LCOV_EXCL_STOP @@ -19016,31 +19140,31 @@ class binary_writer } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x58)); + oa.write_character(to_char_type(0x58)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x59)); + oa.write_character(to_char_type(0x59)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x5A)); + oa.write_character(to_char_type(0x5A)); write_number(static_cast(N)); } // LCOV_EXCL_START else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0x5B)); + oa.write_character(to_char_type(0x5B)); write_number(static_cast(N)); } // LCOV_EXCL_STOP // step 2: write each element - oa->write_characters( - reinterpret_cast(j.m_data.m_value.binary->data()), - N); + oa.write_characters( + reinterpret_cast(j.m_data.m_value.binary->data()), + N); break; } @@ -19055,23 +19179,23 @@ class binary_writer } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0xB8)); + oa.write_character(to_char_type(0xB8)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0xB9)); + oa.write_character(to_char_type(0xB9)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0xBA)); + oa.write_character(to_char_type(0xBA)); write_number(static_cast(N)); } // LCOV_EXCL_START else if (N <= (std::numeric_limits::max)()) { - oa->write_character(to_char_type(0xBB)); + oa.write_character(to_char_type(0xBB)); write_number(static_cast(N)); } // LCOV_EXCL_STOP @@ -19100,15 +19224,15 @@ class binary_writer { case value_t::null: // nil { - oa->write_character(to_char_type(0xC0)); + oa.write_character(to_char_type(0xC0)); break; } case value_t::boolean: // true and false { - oa->write_character(j.m_data.m_value.boolean - ? to_char_type(0xC3) - : to_char_type(0xC2)); + oa.write_character(j.m_data.m_value.boolean + ? to_char_type(0xC3) + : to_char_type(0xC2)); break; } @@ -19127,25 +19251,25 @@ class binary_writer else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 8 - oa->write_character(to_char_type(0xCC)); + oa.write_character(to_char_type(0xCC)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 16 - oa->write_character(to_char_type(0xCD)); + oa.write_character(to_char_type(0xCD)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 32 - oa->write_character(to_char_type(0xCE)); + oa.write_character(to_char_type(0xCE)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 64 - oa->write_character(to_char_type(0xCF)); + oa.write_character(to_char_type(0xCF)); write_number(static_cast(j.m_data.m_value.number_integer)); } } @@ -19160,28 +19284,28 @@ class binary_writer j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { // int 8 - oa->write_character(to_char_type(0xD0)); + oa.write_character(to_char_type(0xD0)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { // int 16 - oa->write_character(to_char_type(0xD1)); + oa.write_character(to_char_type(0xD1)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { // int 32 - oa->write_character(to_char_type(0xD2)); + oa.write_character(to_char_type(0xD2)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_integer >= (std::numeric_limits::min)() && j.m_data.m_value.number_integer <= (std::numeric_limits::max)()) { // int 64 - oa->write_character(to_char_type(0xD3)); + oa.write_character(to_char_type(0xD3)); write_number(static_cast(j.m_data.m_value.number_integer)); } } @@ -19198,25 +19322,25 @@ class binary_writer else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 8 - oa->write_character(to_char_type(0xCC)); + oa.write_character(to_char_type(0xCC)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 16 - oa->write_character(to_char_type(0xCD)); + oa.write_character(to_char_type(0xCD)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 32 - oa->write_character(to_char_type(0xCE)); + oa.write_character(to_char_type(0xCE)); write_number(static_cast(j.m_data.m_value.number_integer)); } else if (j.m_data.m_value.number_unsigned <= (std::numeric_limits::max)()) { // uint 64 - oa->write_character(to_char_type(0xCF)); + oa.write_character(to_char_type(0xCF)); write_number(static_cast(j.m_data.m_value.number_integer)); } break; @@ -19240,26 +19364,26 @@ class binary_writer else if (N <= (std::numeric_limits::max)()) { // str 8 - oa->write_character(to_char_type(0xD9)); + oa.write_character(to_char_type(0xD9)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { // str 16 - oa->write_character(to_char_type(0xDA)); + oa.write_character(to_char_type(0xDA)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { // str 32 - oa->write_character(to_char_type(0xDB)); + oa.write_character(to_char_type(0xDB)); write_number(static_cast(N)); } // step 2: write the string - oa->write_characters( - reinterpret_cast(j.m_data.m_value.string->data()), - j.m_data.m_value.string->size()); + oa.write_characters( + reinterpret_cast(j.m_data.m_value.string->data()), + j.m_data.m_value.string->size()); break; } @@ -19275,13 +19399,13 @@ class binary_writer else if (N <= (std::numeric_limits::max)()) { // array 16 - oa->write_character(to_char_type(0xDC)); + oa.write_character(to_char_type(0xDC)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { // array 32 - oa->write_character(to_char_type(0xDD)); + oa.write_character(to_char_type(0xDD)); write_number(static_cast(N)); } @@ -19337,7 +19461,7 @@ class binary_writer fixed = false; } - oa->write_character(to_char_type(output_type)); + oa.write_character(to_char_type(output_type)); if (!fixed) { write_number(static_cast(N)); @@ -19349,7 +19473,7 @@ class binary_writer ? 0xC8 // ext 16 : 0xC5; // bin 16 - oa->write_character(to_char_type(output_type)); + oa.write_character(to_char_type(output_type)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) @@ -19358,7 +19482,7 @@ class binary_writer ? 0xC9 // ext 32 : 0xC6; // bin 32 - oa->write_character(to_char_type(output_type)); + oa.write_character(to_char_type(output_type)); write_number(static_cast(N)); } @@ -19369,9 +19493,9 @@ class binary_writer } // step 2: write the byte string - oa->write_characters( - reinterpret_cast(j.m_data.m_value.binary->data()), - N); + oa.write_characters( + reinterpret_cast(j.m_data.m_value.binary->data()), + N); break; } @@ -19388,13 +19512,13 @@ class binary_writer else if (N <= (std::numeric_limits::max)()) { // map 16 - oa->write_character(to_char_type(0xDE)); + oa.write_character(to_char_type(0xDE)); write_number(static_cast(N)); } else if (N <= (std::numeric_limits::max)()) { // map 32 - oa->write_character(to_char_type(0xDF)); + oa.write_character(to_char_type(0xDF)); write_number(static_cast(N)); } @@ -19433,7 +19557,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('Z')); + oa.write_character(to_char_type('Z')); } break; } @@ -19442,9 +19566,9 @@ class binary_writer { if (add_prefix) { - oa->write_character(j.m_data.m_value.boolean - ? to_char_type('T') - : to_char_type('F')); + oa.write_character(j.m_data.m_value.boolean + ? to_char_type('T') + : to_char_type('F')); } break; } @@ -19471,12 +19595,12 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('S')); + oa.write_character(to_char_type('S')); } write_number_with_ubjson_prefix(j.m_data.m_value.string->size(), true, use_bjdata); - oa->write_characters( - reinterpret_cast(j.m_data.m_value.string->data()), - j.m_data.m_value.string->size()); + oa.write_characters( + reinterpret_cast(j.m_data.m_value.string->data()), + j.m_data.m_value.string->size()); break; } @@ -19484,7 +19608,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('[')); + oa.write_character(to_char_type('[')); } bool prefix_required = true; @@ -19516,14 +19640,14 @@ class binary_writer && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) { prefix_required = false; - oa->write_character(to_char_type('$')); - oa->write_character(first_prefix); + oa.write_character(to_char_type('$')); + oa.write_character(first_prefix); } } if (use_count) { - oa->write_character(to_char_type('#')); + oa.write_character(to_char_type('#')); write_number_with_ubjson_prefix(j.m_data.m_value.array->size(), true, use_bjdata); } @@ -19534,7 +19658,7 @@ class binary_writer if (!use_count) { - oa->write_character(to_char_type(']')); + oa.write_character(to_char_type(']')); } break; @@ -19544,7 +19668,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('[')); + oa.write_character(to_char_type('[')); } if (use_type && (bjdata_draft3 || !j.m_data.m_value.binary->empty())) @@ -19553,36 +19677,36 @@ class binary_writer { JSON_THROW(other_error::create(502, "use_type requires use_size = true", &j)); } - oa->write_character(to_char_type('$')); - oa->write_character(bjdata_draft3 ? 'B' : 'U'); + oa.write_character(to_char_type('$')); + oa.write_character(bjdata_draft3 ? 'B' : 'U'); } if (use_count) { - oa->write_character(to_char_type('#')); + oa.write_character(to_char_type('#')); write_number_with_ubjson_prefix(j.m_data.m_value.binary->size(), true, use_bjdata); } if (use_type) { - oa->write_characters( - reinterpret_cast(j.m_data.m_value.binary->data()), - j.m_data.m_value.binary->size()); + oa.write_characters( + reinterpret_cast(j.m_data.m_value.binary->data()), + j.m_data.m_value.binary->size()); } else { for (size_t i = 0; i < j.m_data.m_value.binary->size(); ++i) { - oa->write_character(to_char_type(bjdata_draft3 ? 'B' : 'U')); + oa.write_character(to_char_type(bjdata_draft3 ? 'B' : 'U')); // the cast is needed for binary types whose value type // is not an integer (e.g., std::byte) - oa->write_character(to_char_type(static_cast(j.m_data.m_value.binary->data()[i]))); + oa.write_character(to_char_type(static_cast(j.m_data.m_value.binary->data()[i]))); } } if (!use_count) { - oa->write_character(to_char_type(']')); + oa.write_character(to_char_type(']')); } break; @@ -19600,7 +19724,7 @@ class binary_writer if (add_prefix) { - oa->write_character(to_char_type('{')); + oa.write_character(to_char_type('{')); } bool prefix_required = true; @@ -19622,29 +19746,29 @@ class binary_writer if (same_prefix && !(use_bjdata && std::find(bjdx.begin(), bjdx.end(), first_prefix) != bjdx.end())) { prefix_required = false; - oa->write_character(to_char_type('$')); - oa->write_character(first_prefix); + oa.write_character(to_char_type('$')); + oa.write_character(first_prefix); } } if (use_count) { - oa->write_character(to_char_type('#')); + oa.write_character(to_char_type('#')); write_number_with_ubjson_prefix(j.m_data.m_value.object->size(), true, use_bjdata); } for (const auto& el : *j.m_data.m_value.object) { write_number_with_ubjson_prefix(el.first.size(), true, use_bjdata); - oa->write_characters( - reinterpret_cast(el.first.data()), - el.first.size()); + oa.write_characters( + reinterpret_cast(el.first.data()), + el.first.size()); write_ubjson(el.second, use_count, use_type, prefix_required, use_bjdata, bjdata_version); } if (!use_count) { - oa->write_character(to_char_type('}')); + oa.write_character(to_char_type('}')); } break; @@ -19698,13 +19822,13 @@ class binary_writer void write_bson_entry_header(const string_t& name, const std::uint8_t element_type) { - oa->write_character(to_char_type(element_type)); - oa->write_characters( - reinterpret_cast(name.data()), - name.size()); + oa.write_character(to_char_type(element_type)); + oa.write_characters( + reinterpret_cast(name.data()), + name.size()); // the terminating null byte is written explicitly rather than taken // from the buffer, so that string_t::data() need not be null-terminated - oa->write_character(to_char_type(0x00)); + oa.write_character(to_char_type(0x00)); } /*! @@ -19714,7 +19838,7 @@ class binary_writer const bool value) { write_bson_entry_header(name, 0x08); - oa->write_character(value ? to_char_type(0x01) : to_char_type(0x00)); + oa.write_character(value ? to_char_type(0x01) : to_char_type(0x00)); } /*! @@ -19744,12 +19868,12 @@ class binary_writer write_bson_entry_header(name, 0x02); write_number(to_bson_length(value.size() + 1ul), true); - oa->write_characters( - reinterpret_cast(value.data()), - value.size()); + oa.write_characters( + reinterpret_cast(value.data()), + value.size()); // the terminating null byte is written explicitly rather than taken // from the buffer, so that string_t::data() need not be null-terminated - oa->write_character(to_char_type(0x00)); + oa.write_character(to_char_type(0x00)); } /*! @@ -19878,7 +20002,7 @@ class binary_writer write_bson_element(string_t(key.data(), key.size()), el); } - oa->write_character(to_char_type(0x00)); + oa.write_character(to_char_type(0x00)); } /*! @@ -19892,7 +20016,7 @@ class binary_writer write_number(to_bson_length(value.size()), true); write_number(value.has_subtype() ? static_cast(value.subtype()) : static_cast(0x00)); - oa->write_characters(reinterpret_cast(value.data()), value.size()); + oa.write_characters(reinterpret_cast(value.data()), value.size()); } /*! @@ -20018,7 +20142,7 @@ class binary_writer write_bson_element(el.first, el.second); } - oa->write_character(to_char_type(0x00)); + oa.write_character(to_char_type(0x00)); } ////////// @@ -20062,7 +20186,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(get_ubjson_float_prefix(n)); + oa.write_character(get_ubjson_float_prefix(n)); } write_number(n, use_bjdata); } @@ -20078,7 +20202,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('i')); // int8 + oa.write_character(to_char_type('i')); // int8 } write_number(static_cast(n), use_bjdata); } @@ -20086,7 +20210,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('U')); // uint8 + oa.write_character(to_char_type('U')); // uint8 } write_number(static_cast(n), use_bjdata); } @@ -20094,7 +20218,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('I')); // int16 + oa.write_character(to_char_type('I')); // int16 } write_number(static_cast(n), use_bjdata); } @@ -20102,7 +20226,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('u')); // uint16 - bjdata only + oa.write_character(to_char_type('u')); // uint16 - bjdata only } write_number(static_cast(n), use_bjdata); } @@ -20110,7 +20234,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('l')); // int32 + oa.write_character(to_char_type('l')); // int32 } write_number(static_cast(n), use_bjdata); } @@ -20118,7 +20242,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('m')); // uint32 - bjdata only + oa.write_character(to_char_type('m')); // uint32 - bjdata only } write_number(static_cast(n), use_bjdata); } @@ -20126,7 +20250,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('L')); // int64 + oa.write_character(to_char_type('L')); // int64 } write_number(static_cast(n), use_bjdata); } @@ -20134,7 +20258,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('M')); // uint64 - bjdata only + oa.write_character(to_char_type('M')); // uint64 - bjdata only } write_number(static_cast(n), use_bjdata); } @@ -20142,14 +20266,14 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('H')); // high-precision number + oa.write_character(to_char_type('H')); // high-precision number } const auto number = BasicJsonType(n).dump(); write_number_with_ubjson_prefix(number.size(), true, use_bjdata); for (std::size_t i = 0; i < number.size(); ++i) { - oa->write_character(to_char_type(static_cast(number[i]))); + oa.write_character(to_char_type(static_cast(number[i]))); } } } @@ -20166,7 +20290,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('i')); // int8 + oa.write_character(to_char_type('i')); // int8 } write_number(static_cast(n), use_bjdata); } @@ -20174,7 +20298,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('U')); // uint8 + oa.write_character(to_char_type('U')); // uint8 } write_number(static_cast(n), use_bjdata); } @@ -20182,7 +20306,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('I')); // int16 + oa.write_character(to_char_type('I')); // int16 } write_number(static_cast(n), use_bjdata); } @@ -20190,7 +20314,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('u')); // uint16 - bjdata only + oa.write_character(to_char_type('u')); // uint16 - bjdata only } write_number(static_cast(n), use_bjdata); } @@ -20198,7 +20322,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('l')); // int32 + oa.write_character(to_char_type('l')); // int32 } write_number(static_cast(n), use_bjdata); } @@ -20206,7 +20330,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('m')); // uint32 - bjdata only + oa.write_character(to_char_type('m')); // uint32 - bjdata only } write_number(static_cast(n), use_bjdata); } @@ -20214,7 +20338,7 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('L')); // int64 + oa.write_character(to_char_type('L')); // int64 } write_number(static_cast(n), use_bjdata); } @@ -20223,14 +20347,14 @@ class binary_writer { if (add_prefix) { - oa->write_character(to_char_type('H')); // high-precision number + oa.write_character(to_char_type('H')); // high-precision number } const auto number = BasicJsonType(n).dump(); write_number_with_ubjson_prefix(number.size(), true, use_bjdata); for (std::size_t i = 0; i < number.size(); ++i) { - oa->write_character(to_char_type(static_cast(number[i]))); + oa.write_character(to_char_type(static_cast(number[i]))); } } // LCOV_EXCL_STOP @@ -20522,10 +20646,10 @@ class binary_writer } } - oa->write_character('['); - oa->write_character('$'); - oa->write_character(dtype); - oa->write_character('#'); + oa.write_character('['); + oa.write_character('$'); + oa.write_character(dtype); + oa.write_character('#'); key = "_ArraySize_"; write_ubjson(value.at(key), use_count, use_type, true, true, bjdata_version); @@ -20627,6 +20751,8 @@ class binary_writer { #if defined(__GNUC__) || defined(__clang__) return __builtin_bswap16(x); +#elif defined(_MSC_VER) + return _byteswap_ushort(x); #else return static_cast((x >> 8) | (x << 8)); #endif @@ -20636,6 +20762,8 @@ class binary_writer { #if defined(__GNUC__) || defined(__clang__) return __builtin_bswap32(x); +#elif defined(_MSC_VER) + return _byteswap_ulong(x); #else return ((x & 0x000000FFu) << 24) | ((x & 0x0000FF00u) << 8) | ((x & 0x00FF0000u) >> 8) | ((x & 0xFF000000u) >> 24); @@ -20646,6 +20774,8 @@ class binary_writer { #if defined(__GNUC__) || defined(__clang__) return __builtin_bswap64(x); +#elif defined(_MSC_VER) + return _byteswap_uint64(x); #else x = ((x & 0x00000000FFFFFFFFull) << 32) | ((x & 0xFFFFFFFF00000000ull) >> 32); x = ((x & 0x0000FFFF0000FFFFull) << 16) | ((x & 0xFFFF0000FFFF0000ull) >> 16); @@ -20654,30 +20784,40 @@ class binary_writer #endif } - // reverse the bytes of a fixed-size buffer; a single byte_swap() for the - // common 2/4/8-byte number payloads, std::reverse for any other size - static void reverse_bytes(std::array& a) noexcept + /*! + @brief reverse the bytes of a buffer by byte-swapping it as UIntType + + Loading the buffer into an unsigned integer of the same width and swapping + that is what lets the compiler emit a single bswap/rev/movbe; reversing the + buffer element by element does not reliably get there (clang keeps a scalar + shuffle). The two memcpy calls are the only portable way to reinterpret the + bytes and are folded away by every optimizer. + */ + template + static void byte_swap_buffer(std::array& a) noexcept { - std::uint16_t v{}; + static_assert(sizeof(UIntType) == N, "swap width must match the buffer size"); + UIntType v{}; std::memcpy(&v, a.data(), sizeof(v)); v = byte_swap(v); std::memcpy(a.data(), &v, sizeof(v)); } + // reverse the bytes of a fixed-size buffer; a single byte_swap() for the + // common 2/4/8-byte number payloads, std::reverse for any other size + static void reverse_bytes(std::array& a) noexcept + { + byte_swap_buffer(a); + } + static void reverse_bytes(std::array& a) noexcept { - std::uint32_t v{}; - std::memcpy(&v, a.data(), sizeof(v)); - v = byte_swap(v); - std::memcpy(a.data(), &v, sizeof(v)); + byte_swap_buffer(a); } static void reverse_bytes(std::array& a) noexcept { - std::uint64_t v{}; - std::memcpy(&v, a.data(), sizeof(v)); - v = byte_swap(v); - std::memcpy(a.data(), &v, sizeof(v)); + byte_swap_buffer(a); } template @@ -20700,7 +20840,7 @@ class binary_writer reverse_bytes(vec); } - oa->write_characters(vec.data(), sizeof(NumberType)); + oa.write_characters(vec.data(), sizeof(NumberType)); } void write_compact_float(const number_float_t n, detail::input_format_t format) @@ -20708,21 +20848,30 @@ class binary_writer #ifdef __GNUC__ JSON_HEDLEY_DIAGNOSTIC_PUSH JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal") +#endif + // When number_float_t is float, static_cast(n) is the identity and + // both branches below are intentionally identical (the "compact" float + // representation is the value itself). Only GCC diagnoses this, and only + // when the sink calls are inlined; clang has no such warning. + // (-Wduplicated-branches only exists from GCC 7 on; naming it on an older + // GCC would itself warn under -Wpragmas) +#if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 7) + JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wduplicated-branches") #endif if (!std::isfinite(n) || ((static_cast(n) >= static_cast(std::numeric_limits::lowest()) && static_cast(n) <= static_cast((std::numeric_limits::max)()) && static_cast(static_cast(n)) == static_cast(n)))) { - oa->write_character(format == detail::input_format_t::cbor - ? get_cbor_float_prefix(static_cast(n)) - : get_msgpack_float_prefix(static_cast(n))); + oa.write_character(format == detail::input_format_t::cbor + ? get_cbor_float_prefix(static_cast(n)) + : get_msgpack_float_prefix(static_cast(n))); write_number(static_cast(n)); } else { - oa->write_character(format == detail::input_format_t::cbor - ? get_cbor_float_prefix(n) - : get_msgpack_float_prefix(n)); + oa.write_character(format == detail::input_format_t::cbor + ? get_cbor_float_prefix(n) + : get_msgpack_float_prefix(n)); write_number(n); } #ifdef __GNUC__ @@ -20789,7 +20938,7 @@ class binary_writer const bool is_little_endian = little_endianness(); /// the output - output_adapter_t oa = nullptr; + OutputSinkType oa; }; } // namespace detail @@ -24130,7 +24279,7 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec friend ::nlohmann::detail::serializer; template friend class ::nlohmann::detail::iter_impl; - template + template friend class ::nlohmann::detail::binary_writer; template friend class ::nlohmann::detail::binary_reader; @@ -24178,6 +24327,14 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec template using binary_reader = ::nlohmann::detail::binary_reader; template using binary_writer = ::nlohmann::detail::binary_writer; + // binary_writer over a concrete (non-virtual) sink appending into a std::vector, + // used by the vector-returning to_* overloads + template using vector_binary_writer = + ::nlohmann::detail::binary_writer>; + template static vector_binary_writer vector_writer(std::vector& v) + { + return vector_binary_writer(::nlohmann::detail::output_vector_sink(v)); + } JSON_PRIVATE_UNLESS_TESTED: using serializer = ::nlohmann::detail::serializer; @@ -28428,7 +28585,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static std::vector to_cbor(const basic_json& j) { std::vector result; - to_cbor(j, result); + result.reserve(detail::binary_reserve_hint(j)); + vector_writer(result).write_cbor(j); return result; } @@ -28451,7 +28609,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static std::vector to_msgpack(const basic_json& j) { std::vector result; - to_msgpack(j, result); + result.reserve(detail::binary_reserve_hint(j)); + vector_writer(result).write_msgpack(j); return result; } @@ -28476,7 +28635,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bool use_type = false) { std::vector result; - to_ubjson(j, result, use_size, use_type); + result.reserve(detail::binary_reserve_hint(j)); + vector_writer(result).write_ubjson(j, use_size, use_type); return result; } @@ -28504,7 +28664,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec const bjdata_version_t version = bjdata_version_t::draft2) { std::vector result; - to_bjdata(j, result, use_size, use_type, version); + result.reserve(detail::binary_reserve_hint(j)); + vector_writer(result).write_ubjson(j, use_size, use_type, true, true, version); return result; } @@ -28531,7 +28692,8 @@ class basic_json // NOLINT(cppcoreguidelines-special-member-functions,hicpp-spec static std::vector to_bson(const basic_json& j) { std::vector result; - to_bson(j, result); + result.reserve(detail::binary_reserve_hint(j)); + vector_writer(result).write_bson(j); return result; }