Reject MessagePack/BSON binary subtypes that don't fit their wire format

Both formats store byte_container_with_subtype's subtype (a uint64_t)
in a single byte. The writers cast to std::int8_t/std::uint8_t without
a range check, so subtypes above 255 were silently truncated modulo
256 instead of raising an error. Throw out_of_range.413 instead when
the subtype exceeds the representable range of 0-255.

Signed-off-by: Niels Lohmann <mail@nlohmann.me>

Author: nlohmann

docs/mkdocs/docs/home/exceptions.md

@@ -933,6 +933,17 @@ BSON stores the length of documents, arrays, strings, and binary values in a sig
     [`to_bson`](../api/basic_json/to_bson.md) produced documents with negative length prefixes that
     [`from_bson`](../api/basic_json/from_bson.md) rejected.
 
+### json.exception.out_of_range.413
+
+MessagePack's ext type and BSON's binary subtype are each stored in a single byte. This exception is thrown when serializing a
+[`byte_container_with_subtype`](../api/byte_container_with_subtype/index.md) whose subtype exceeds 255.
+
+!!! failure "Example message"
+
+    ```
+    [json.exception.out_of_range.413] subtype 70000 is too large for the MessagePack ext type (max 255)
+    ```
+
 ## Further exceptions
 
 This exception is thrown in case of errors that cannot be classified with the

include/nlohmann/detail/output/binary_writer.hpp

@@ -688,6 +688,11 @@ class binary_writer
                 // step 1.5: if this is an ext type, write the subtype
                 if (use_ext)
                 {
+                    if (JSON_HEDLEY_UNLIKELY(j.m_data.m_value.binary->subtype() > (std::numeric_limits<std::uint8_t>::max)()))
+                    {
+                        JSON_THROW(out_of_range::create(413, concat("subtype ", std::to_string(j.m_data.m_value.binary->subtype()), " is too large for the MessagePack ext type (max 255)"), &j));
+                    }
+
                     write_number(static_cast<std::int8_t>(j.m_data.m_value.binary->subtype()));
                 }
 
@@ -1187,6 +1192,12 @@ class binary_writer
         write_bson_entry_header(name, 0x05);
 
         write_number<std::int32_t>(to_bson_length(value.size()), true);
+
+        if (value.has_subtype() && JSON_HEDLEY_UNLIKELY(value.subtype() > (std::numeric_limits<std::uint8_t>::max)()))
+        {
+            JSON_THROW(out_of_range::create(413, concat("subtype ", std::to_string(value.subtype()), " is too large for the BSON binary subtype (max 255)"), nullptr));
+        }
+
         write_number(value.has_subtype() ? static_cast<std::uint8_t>(value.subtype()) : static_cast<std::uint8_t>(0x00));
 
         oa->write_characters(reinterpret_cast<const CharType*>(value.data()), value.size());

single_include/nlohmann/json.hpp

@@ -17775,6 +17775,11 @@ class binary_writer
                 // step 1.5: if this is an ext type, write the subtype
                 if (use_ext)
                 {
+                    if (JSON_HEDLEY_UNLIKELY(j.m_data.m_value.binary->subtype() > (std::numeric_limits<std::uint8_t>::max)()))
+                    {
+                        JSON_THROW(out_of_range::create(413, concat("subtype ", std::to_string(j.m_data.m_value.binary->subtype()), " is too large for the MessagePack ext type (max 255)"), &j));
+                    }
+
                     write_number(static_cast<std::int8_t>(j.m_data.m_value.binary->subtype()));
                 }
 
@@ -18274,6 +18279,12 @@ class binary_writer
         write_bson_entry_header(name, 0x05);
 
         write_number<std::int32_t>(to_bson_length(value.size()), true);
+
+        if (value.has_subtype() && JSON_HEDLEY_UNLIKELY(value.subtype() > (std::numeric_limits<std::uint8_t>::max)()))
+        {
+            JSON_THROW(out_of_range::create(413, concat("subtype ", std::to_string(value.subtype()), " is too large for the BSON binary subtype (max 255)"), nullptr));
+        }
+
         write_number(value.has_subtype() ? static_cast<std::uint8_t>(value.subtype()) : static_cast<std::uint8_t>(0x00));
 
         oa->write_characters(reinterpret_cast<const CharType*>(value.data()), value.size());

tests/src/unit-regression2.cpp

@@ -1668,4 +1668,30 @@ TEST_CASE("regression test - excessive binary container size honors allow_except
     CHECK(json::from_cbor(std::vector<std::uint8_t> {0x9b, 0, 0, 0, 0, 0, 0, 0, 0x02}, true, false).is_discarded());
 }
 
+TEST_CASE("regression test - MessagePack/BSON writers reject binary subtypes that don't fit their wire format")
+{
+    // MessagePack: subtype 0-255 must still round-trip correctly (regression guard, pre-existing behavior)
+    CHECK(json::from_msgpack(json::to_msgpack(json::binary({1, 2}, 0))).get_binary().subtype() == 0);
+    CHECK(json::from_msgpack(json::to_msgpack(json::binary({1, 2}, 200))).get_binary().subtype() == 200);
+    CHECK(json::from_msgpack(json::to_msgpack(json::binary({1, 2}, 255))).get_binary().subtype() == 255);
+
+    // MessagePack: subtype > 255 must now throw instead of silently truncating
+    CHECK_THROWS_AS(json::to_msgpack(json::binary({1, 2}, 256)), json::out_of_range);
+    CHECK_THROWS_AS(json::to_msgpack(json::binary({1, 2}, 70000)), json::out_of_range);
+    CHECK_THROWS_WITH_AS(json::to_msgpack(json::binary({1, 2}, 70000)), "[json.exception.out_of_range.413] subtype 70000 is too large for the MessagePack ext type (max 255)", json::out_of_range);
+
+    // BSON: same pattern
+    json doc255 = {{"b", json::binary({1, 2}, 255)}};
+    CHECK(json::from_bson(json::to_bson(doc255))["b"].get_binary().subtype() == 255);
+    CHECK_THROWS_AS(json::to_bson(json{{"b", json::binary({1, 2}, 256)}}), json::out_of_range);
+    CHECK_THROWS_AS(json::to_bson(json{{"b", json::binary({1, 2}, 300)}}), json::out_of_range);
+    CHECK_THROWS_WITH_AS(json::to_bson(json{{"b", json::binary({1, 2}, 300)}}), "[json.exception.out_of_range.413] subtype 300 is too large for the BSON binary subtype (max 255)", json::out_of_range);
+
+    // CBOR must remain unaffected (full 64-bit subtype range already supported correctly) - regression guard
+    CHECK(json::from_cbor(json::to_cbor(json::binary({1, 2}, 70000)), true, true, json::cbor_tag_handler_t::store).get_binary().subtype() == 70000);
+
+    // a binary value with NO subtype at all must be completely unaffected by this change
+    CHECK(json::from_msgpack(json::to_msgpack(json::binary({1, 2}))).get_binary().has_subtype() == false);
+}
+
 DOCTEST_CLANG_SUPPRESS_WARNING_POP