Skip to content

Round-trip BJData ND-array annotations exactly (single precision, key order) #5707

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nlohmann merged 1 commit into from
Oct 7, 2026
Merged

Round-trip BJData ND-array annotations exactly (single precision, key order) #5707

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions docs/mkdocs/docs/features/binary_formats/bjdata.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,8 +141,14 @@ The library uses the following mapping from JSON values types to BJData types ac
parsed back as a regular array,
- every entry of `"_ArraySize_"` is a positive integer, and their product is representable as a `std::size_t`,
- `"_ArrayData_"` is an array holding exactly that many elements, and
- every element of `"_ArrayData_"` is a number of the kind named by `"_ArrayType_"` (a floating-point number for
`single` and `double`, an integer otherwise).
- every element of `"_ArrayData_"` is a number of the kind named by `"_ArrayType_"`: for the integer types, a
value that fits the named width; for `double`, any value; for `single`, a value that survives narrowing to
`float` and back without change (for instance, `0.1` does not, since it is not exactly representable as
`float`).

An annotated object is always read back with its keys in the order shown above, `"_ArrayType_"`, `"_ArraySize_"`,
`"_ArrayData_"`, regardless of the order the ND-array's header stores them in on the wire. This matters for
`ordered_json`, whose comparison takes key order into account.

The current version of this library does not yet support automatic detection of and conversion from a nested JSON
array input to a BJData ND-array.
Expand Down
57 changes: 39 additions & 18 deletions include/nlohmann/detail/input/binary_reader.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2716,10 +2716,15 @@ class binary_reader
is_ndarray can only return `true` when its initial value
is `false`
@param[in] prefix type marker if already read, otherwise set to 0
@param[in] ndarray_dtype the element type marker of the enclosing bjdata ndarray if
already known (it precedes the dimension vector read here),
otherwise 0; used to emit the "_ArrayType_" annotation key
before "_ArraySize_" if a dimension vector turns out to
describe an ndarray

@return whether size determination completed
*/
bool get_ubjson_size_value(std::size_t& result, bool& is_ndarray, char_int_type prefix = 0)
bool get_ubjson_size_value(std::size_t& result, bool& is_ndarray, char_int_type prefix = 0, char_int_type ndarray_dtype = 0)
{
if (prefix == 0)
{
Expand Down Expand Up @@ -2832,8 +2837,34 @@ class binary_reader
}
}

if (JSON_HEDLEY_UNLIKELY(!sax->start_object(3)))
{
return false;
}

// the element type precedes the dimension vector (see get_ubjson_size_type)
// and is passed down as ndarray_dtype; emit it here so the annotation keys
// follow the documented _ArrayType_, _ArraySize_, _ArrayData_ order
if (ndarray_dtype != 0)
{
const char* type_name = bjd_type_name(ndarray_dtype);
if (JSON_HEDLEY_UNLIKELY(type_name == nullptr))
{
auto last_token = get_token_string();
return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,
exception_message("invalid byte: 0x" + last_token, "type"), nullptr));
}

string_t type_key = "_ArrayType_";
string_t type = type_name; // sax->string() takes a reference
if (JSON_HEDLEY_UNLIKELY(!sax->key(type_key) || !sax->string(type)))
{
return false;
}
}

string_t key = "_ArraySize_";
if (JSON_HEDLEY_UNLIKELY(!sax->start_object(3) || !sax->key(key) || !sax->start_array(dim.size())))
if (JSON_HEDLEY_UNLIKELY(!sax->key(key) || !sax->start_array(dim.size())))
{
return false;
}
Expand Down Expand Up @@ -2941,7 +2972,7 @@ class binary_reader
exception_message(concat("expected '#' after type information; last byte: 0x", last_token), "size"), nullptr));
}

const bool is_error = get_ubjson_size_value(result.first, is_ndarray);
const bool is_error = get_ubjson_size_value(result.first, is_ndarray, 0, result.second);
// an ndarray was read here only if the flag flipped; when it was
// seeded true, get_ubjson_size_value() already rejected the nested
// dimension vector
Expand Down Expand Up @@ -3134,27 +3165,17 @@ class binary_reader
if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0)
{
size_and_type.second &= ~(static_cast<char_int_type>(1) << 8); // use bit 8 to indicate ndarray, here we remove the bit to restore the type marker
const char* type_name = bjd_type_name(size_and_type.second);
string_t key = "_ArrayType_";
if (JSON_HEDLEY_UNLIKELY(type_name == nullptr))
{
auto last_token = get_token_string();
return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,
exception_message("invalid byte: 0x" + last_token, "type"), nullptr));
}

string_t type = type_name; // sax->string() takes a reference
if (JSON_HEDLEY_UNLIKELY(!sax->key(key) || !sax->string(type)))
{
return false;
}

// the "_ArrayType_" and "_ArraySize_" annotation keys were already emitted by
// get_ubjson_size_value() (the type marker is known before the dimension vector
// that determines size_and_type.first is read, so it is emitted first there to
// match the documented _ArrayType_, _ArraySize_, _ArrayData_ key order)
if (size_and_type.second == 'C' || size_and_type.second == 'B')
{
size_and_type.second = 'U';
}

key = "_ArrayData_";
string_t key = "_ArrayData_";
if (JSON_HEDLEY_UNLIKELY(!sax->key(key) || !sax->start_array(size_and_type.first) ))
{
return false;
Expand Down
23 changes: 19 additions & 4 deletions include/nlohmann/detail/output/binary_writer.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1839,16 +1839,31 @@ class binary_writer

/*!
@brief validate (dry_run) or write one BJData ND-array element of dtype 'd' (single precision)
@return whether @a el's value fits a float without overflow; always true when @a dry_run is false
@return whether @a el's value survives narrowing to float and back without any change
(so the ND-array round-trips exactly), or is infinite or NaN; always true when
@a dry_run is false
*/
bool write_bjdata_ndarray_float_element(const BasicJsonType& el, const bool dry_run)
{
const auto dval = el.template get<double>();
if (dry_run)
{
return !std::isfinite(dval) ||
(dval >= static_cast<double>(std::numeric_limits<float>::lowest()) &&
dval <= static_cast<double>((std::numeric_limits<float>::max)()));
#ifdef __GNUC__
JSON_HEDLEY_DIAGNOSTIC_PUSH
JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal")
#endif
// a value that would be rounded (rather than exactly represented) by the
// narrowing to float is treated like an out-of-range integer element; this
// is the same criterion write_compact_float() uses for CBOR/MessagePack
const bool in_range = std::isnan(dval) ||
(dval >= static_cast<double>(std::numeric_limits<float>::lowest()) &&
dval <= static_cast<double>((std::numeric_limits<float>::max)()) &&
static_cast<double>(static_cast<float>(dval)) == dval) ||
std::isinf(dval);
#ifdef __GNUC__
JSON_HEDLEY_DIAGNOSTIC_POP
#endif
return in_range;
}
write_number(static_cast<float>(dval), true);
return true;
Expand Down
80 changes: 58 additions & 22 deletions single_include/nlohmann/json.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16412,10 +16412,15 @@ class binary_reader
is_ndarray can only return `true` when its initial value
is `false`
@param[in] prefix type marker if already read, otherwise set to 0
@param[in] ndarray_dtype the element type marker of the enclosing bjdata ndarray if
already known (it precedes the dimension vector read here),
otherwise 0; used to emit the "_ArrayType_" annotation key
before "_ArraySize_" if a dimension vector turns out to
describe an ndarray

@return whether size determination completed
*/
bool get_ubjson_size_value(std::size_t& result, bool& is_ndarray, char_int_type prefix = 0)
bool get_ubjson_size_value(std::size_t& result, bool& is_ndarray, char_int_type prefix = 0, char_int_type ndarray_dtype = 0)
{
if (prefix == 0)
{
Expand Down Expand Up @@ -16528,8 +16533,34 @@ class binary_reader
}
}

if (JSON_HEDLEY_UNLIKELY(!sax->start_object(3)))
{
return false;
}

// the element type precedes the dimension vector (see get_ubjson_size_type)
// and is passed down as ndarray_dtype; emit it here so the annotation keys
// follow the documented _ArrayType_, _ArraySize_, _ArrayData_ order
if (ndarray_dtype != 0)
{
const char* type_name = bjd_type_name(ndarray_dtype);
if (JSON_HEDLEY_UNLIKELY(type_name == nullptr))
{
auto last_token = get_token_string();
return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,
exception_message("invalid byte: 0x" + last_token, "type"), nullptr));
}

string_t type_key = "_ArrayType_";
string_t type = type_name; // sax->string() takes a reference
if (JSON_HEDLEY_UNLIKELY(!sax->key(type_key) || !sax->string(type)))
{
return false;
}
}

string_t key = "_ArraySize_";
if (JSON_HEDLEY_UNLIKELY(!sax->start_object(3) || !sax->key(key) || !sax->start_array(dim.size())))
if (JSON_HEDLEY_UNLIKELY(!sax->key(key) || !sax->start_array(dim.size())))
{
return false;
}
Expand Down Expand Up @@ -16637,7 +16668,7 @@ class binary_reader
exception_message(concat("expected '#' after type information; last byte: 0x", last_token), "size"), nullptr));
}

const bool is_error = get_ubjson_size_value(result.first, is_ndarray);
const bool is_error = get_ubjson_size_value(result.first, is_ndarray, 0, result.second);
// an ndarray was read here only if the flag flipped; when it was
// seeded true, get_ubjson_size_value() already rejected the nested
// dimension vector
Expand Down Expand Up @@ -16830,27 +16861,17 @@ class binary_reader
if (input_format == input_format_t::bjdata && size_and_type.first != npos && (size_and_type.second & (1 << 8)) != 0)
{
size_and_type.second &= ~(static_cast<char_int_type>(1) << 8); // use bit 8 to indicate ndarray, here we remove the bit to restore the type marker
const char* type_name = bjd_type_name(size_and_type.second);
string_t key = "_ArrayType_";
if (JSON_HEDLEY_UNLIKELY(type_name == nullptr))
{
auto last_token = get_token_string();
return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read,
exception_message("invalid byte: 0x" + last_token, "type"), nullptr));
}

string_t type = type_name; // sax->string() takes a reference
if (JSON_HEDLEY_UNLIKELY(!sax->key(key) || !sax->string(type)))
{
return false;
}

// the "_ArrayType_" and "_ArraySize_" annotation keys were already emitted by
// get_ubjson_size_value() (the type marker is known before the dimension vector
// that determines size_and_type.first is read, so it is emitted first there to
// match the documented _ArrayType_, _ArraySize_, _ArrayData_ key order)
if (size_and_type.second == 'C' || size_and_type.second == 'B')
{
size_and_type.second = 'U';
}

key = "_ArrayData_";
string_t key = "_ArrayData_";
if (JSON_HEDLEY_UNLIKELY(!sax->key(key) || !sax->start_array(size_and_type.first) ))
{
return false;
Expand Down Expand Up @@ -23310,16 +23331,31 @@ class binary_writer

/*!
@brief validate (dry_run) or write one BJData ND-array element of dtype 'd' (single precision)
@return whether @a el's value fits a float without overflow; always true when @a dry_run is false
@return whether @a el's value survives narrowing to float and back without any change
(so the ND-array round-trips exactly), or is infinite or NaN; always true when
@a dry_run is false
*/
bool write_bjdata_ndarray_float_element(const BasicJsonType& el, const bool dry_run)
{
const auto dval = el.template get<double>();
if (dry_run)
{
return !std::isfinite(dval) ||
(dval >= static_cast<double>(std::numeric_limits<float>::lowest()) &&
dval <= static_cast<double>((std::numeric_limits<float>::max)()));
#ifdef __GNUC__
JSON_HEDLEY_DIAGNOSTIC_PUSH
JSON_HEDLEY_PRAGMA(GCC diagnostic ignored "-Wfloat-equal")
#endif
// a value that would be rounded (rather than exactly represented) by the
// narrowing to float is treated like an out-of-range integer element; this
// is the same criterion write_compact_float() uses for CBOR/MessagePack
const bool in_range = std::isnan(dval) ||
(dval >= static_cast<double>(std::numeric_limits<float>::lowest()) &&
dval <= static_cast<double>((std::numeric_limits<float>::max)()) &&
static_cast<double>(static_cast<float>(dval)) == dval) ||
std::isinf(dval);
#ifdef __GNUC__
JSON_HEDLEY_DIAGNOSTIC_POP
#endif
return in_range;
}
write_number(static_cast<float>(dval), true);
return true;
Expand Down
Loading
Loading