From b76e682a24200a0fba80788d81f54e6895b5ed72 Mon Sep 17 00:00:00 2001 From: mrabine Date: Wed, 14 Oct 2026 18:42:21 +0200 Subject: [PATCH] Json dom stack --- core/include/join/utils.hpp | 2 + data/include/join/json.hpp | 45 +++-- data/include/join/pack.hpp | 20 +- data/include/join/sax.hpp | 260 +++++++++++++------------ data/include/join/view.hpp | 104 ++++++++++ data/tests/json_canonicalizer_test.cpp | 22 +++ data/tests/json_reader_test.cpp | 10 + data/tests/json_writer_test.cpp | 27 +++ data/tests/pack_reader_test.cpp | 18 ++ 9 files changed, 361 insertions(+), 147 deletions(-) diff --git a/core/include/join/utils.hpp b/core/include/join/utils.hpp index 4eecc28b..51004a80 100644 --- a/core/include/join/utils.hpp +++ b/core/include/join/utils.hpp @@ -43,6 +43,8 @@ #include #include +#define JOIN_ALWAYS_INLINE inline __attribute__ ((always_inline)) + #define JOIN_LIKELY(x) __builtin_expect (!!(x), 1) #define JOIN_UNLIKELY(x) __builtin_expect (!!(x), 0) diff --git a/data/include/join/json.hpp b/data/include/join/json.hpp index 06761478..ad4fbbf5 100644 --- a/data/include/join/json.hpp +++ b/data/include/join/json.hpp @@ -202,7 +202,7 @@ namespace join virtual int setNull () override { array (); - append4 ("null"); + append ("null", 4); _first = false; return 0; } @@ -217,11 +217,11 @@ namespace join array (); if (value) { - append4 ("true"); + append ("true", 4); } else { - append5 ("false"); + append ("false", 5); } _first = false; return 0; @@ -300,22 +300,22 @@ namespace join { if (neg) { - append4 ("-NaN"); + append ("-NaN", 4); } else { - append3 ("NaN"); + append ("NaN", 3); } } else { if (neg) { - append4 ("-Inf"); + append ("-Inf", 4); } else { - append3 ("Inf"); + append ("Inf", 3); } } _first = false; @@ -576,7 +576,7 @@ namespace join else { char escapeSeq[2] = {'\\', static_cast (esc)}; - append2 (escapeSeq); + append (escapeSeq, 2); } ++cur; @@ -729,7 +729,7 @@ namespace join } else { - append4 ("null"); + append ("null", 4); } _first = false; return 0; @@ -1115,6 +1115,8 @@ namespace join template int read (ViewType& document) { + _depth = 0; + if (skipWhitespaces (document) != 0) { return -1; @@ -1455,8 +1457,7 @@ namespace join } else if (JOIN_LIKELY (isDigit (view.peek ()))) { - u = view.get () - '0'; - ++digits; + digits = view.readDigits (u, 18); while (JOIN_LIKELY (isDigit (view.peek ()))) { @@ -1488,11 +1489,7 @@ namespace join if (isDouble) { - while (JOIN_LIKELY (isDigit (view.peek ()))) - { - u = (u * 10) + (view.get () - '0'); - ++digits; - } + digits += view.readDigits (u); } int64_t frac = 0; @@ -1500,15 +1497,17 @@ namespace join { isDouble = true; - while (JOIN_LIKELY (isDigit (view.peek ()))) + if (JOIN_UNLIKELY (digits == 0)) { - u = (u * 10) + (view.get () - '0'); - if (JOIN_LIKELY (u || digits)) + while (view.getIf ('0')) { - ++digits; + --frac; } - --frac; } + + size_t count = view.readDigits (u); + digits += count; + frac -= static_cast (count); } int64_t exponent = 0; @@ -1767,7 +1766,7 @@ namespace join * @return 0 on success, -1 otherwise. */ template - int readArray (ViewType& document) + JOIN_ALWAYS_INLINE int readArray (ViewType& document) { if (JOIN_UNLIKELY (startArray () == -1)) { @@ -1829,7 +1828,7 @@ namespace join * @return 0 on success, -1 otherwise. */ template - int readObject (ViewType& document) + JOIN_ALWAYS_INLINE int readObject (ViewType& document) { if (JOIN_UNLIKELY (startObject () == -1)) { diff --git a/data/include/join/pack.hpp b/data/include/join/pack.hpp index 75dd3227..f1b02c0b 100644 --- a/data/include/join/pack.hpp +++ b/data/include/join/pack.hpp @@ -551,6 +551,8 @@ namespace join template int read (ViewType& document) { + _depth = 0; + if (JOIN_LIKELY (readValue (document) == 0)) { if (JOIN_LIKELY (document.peek () == std::char_traits::eof ())) @@ -572,7 +574,14 @@ namespace join template int readValue (ViewType& document) { - uint8_t head = static_cast (document.peek ()); + int ch = document.peek (); + if (JOIN_UNLIKELY (ch == std::char_traits::eof ())) + { + join::lastError = make_error_code (SaxErrc::InvalidValue); + return -1; + } + + uint8_t head = static_cast (ch); try { @@ -755,7 +764,14 @@ namespace join template int readKey (ViewType& document) { - uint8_t head = static_cast (document.peek ()); + int ch = document.peek (); + if (JOIN_UNLIKELY (ch == std::char_traits::eof ())) + { + join::lastError = make_error_code (SaxErrc::InvalidValue); + return -1; + } + + uint8_t head = static_cast (ch); try { diff --git a/data/include/join/sax.hpp b/data/include/join/sax.hpp index deba4d96..d2190c13 100644 --- a/data/include/join/sax.hpp +++ b/data/include/join/sax.hpp @@ -33,13 +33,16 @@ // C++. #include +#include #include #include #include #include +#include #include // C. +#include #include namespace join @@ -288,7 +291,21 @@ namespace join */ virtual int serialize (const Value& value) { - return setValue (value); + _capacity = sizeof (_buffer); + int result; + try + { + result = setValue (value); + } + catch (...) + { + flush (); + _capacity = 0; + throw; + } + flush (); + _capacity = 0; + return result; } protected: @@ -342,7 +359,7 @@ namespace join { if (startArray (array.size ()) == -1) { - return -1; + return -1; // LCOV_EXCL_LINE } for (auto const& element : array) @@ -365,7 +382,7 @@ namespace join { if (startObject (object.size ()) == -1) { - return -1; + return -1; // LCOV_EXCL_LINE } for (auto const& member : object) @@ -390,68 +407,62 @@ namespace join */ inline void append (char data) noexcept { - _out->sputc (data); - } - - /** - * @brief append 2-character literal to output stream in batch. - * @param data data to append to output stream. - */ - inline void append2 (const char* data) noexcept - { - _out->sputc (data[0]); - _out->sputc (data[1]); - } - - /** - * @brief append 3-character literal to output stream in batch. - * @param data data to append to output stream. - */ - inline void append3 (const char* data) noexcept - { - _out->sputc (data[0]); - _out->sputc (data[1]); - _out->sputc (data[2]); - } - - /** - * @brief append 4-character literal to output stream in batch. - * @param data data to append to output stream. - */ - inline void append4 (const char* data) noexcept - { - _out->sputc (data[0]); - _out->sputc (data[1]); - _out->sputc (data[2]); - _out->sputc (data[3]); + if (JOIN_UNLIKELY (_size == _capacity)) + { + flush (); + if (_capacity == 0) + { + _out->sputc (data); + return; + } + } + _buffer[_size++] = data; } /** - * @brief append 5-character literal to output stream in batch. + * @brief append data to output stream in batch. * @param data data to append to output stream. + * @param size data size. */ - inline void append5 (const char* data) noexcept + inline void append (const char* data, uint32_t size) noexcept { - _out->sputc (data[0]); - _out->sputc (data[1]); - _out->sputc (data[2]); - _out->sputc (data[3]); - _out->sputc (data[4]); + if (JOIN_UNLIKELY (size > _capacity - _size)) + { + flush (); + if (size >= _capacity) + { + _out->sputn (data, size); + return; + } + } + std::memcpy (_buffer + _size, data, size); + _size += size; } /** - * @brief append characters to output stream in batch. - * @param data data to append to output stream. - * @param size data size. + * @brief flush buffered data to output stream. */ - inline void append (const char* data, uint32_t size) noexcept + inline void flush () noexcept { - _out->sputn (data, size); + if (_size) + { + _out->sputn (_buffer, _size); + _size = 0; + } } protected: /// underlying output stream. std::streambuf* _out; + + /// output buffer. + char _buffer[1024]; + + /// output buffer size. + size_t _size = 0; + + /// output buffer capacity, 0 outside of serialize. + size_t _capacity = 0; }; /** @@ -466,6 +477,7 @@ namespace join */ StreamReader (Value& root) : SaxHandler () + , _stack (stack ()) , _root (root) { } @@ -573,7 +585,7 @@ namespace join */ virtual int setNull () override { - return setValue (Value (in_place_index_t{}, nullptr)); + return emplaceValue (nullptr); } /** @@ -583,7 +595,7 @@ namespace join */ virtual int setBool (bool value) override { - return setValue (Value (in_place_index_t{}, value)); + return emplaceValue (value); } /** @@ -593,7 +605,7 @@ namespace join */ virtual int setInt (int32_t value) override { - return setValue (Value (in_place_index_t{}, value)); + return emplaceValue (value); } /** @@ -603,7 +615,7 @@ namespace join */ virtual int setUint (uint32_t value) override { - return setValue (Value (in_place_index_t{}, value)); + return emplaceValue (value); } /** @@ -613,7 +625,7 @@ namespace join */ virtual int setInt64 (int64_t value) override { - return setValue (Value (in_place_index_t{}, value)); + return emplaceValue (value); } /** @@ -623,7 +635,7 @@ namespace join */ virtual int setUint64 (uint64_t value) override { - return setValue (Value (in_place_index_t{}, value)); + return emplaceValue (value); } /** @@ -633,7 +645,7 @@ namespace join */ virtual int setDouble (double value) override { - return setValue (Value (in_place_index_t{}, value)); + return emplaceValue (value); } /** @@ -643,42 +655,27 @@ namespace join */ virtual int setString (const std::string& value) override { - return setValue (Value (in_place_index_t{}, value)); + return emplaceValue (value); } /** * @brief start array. - * @param size array size. + * @param size array size (ignored). * @return 0 on success, -1 otherwise. */ - virtual int startArray (uint32_t size = 0) override + virtual int startArray ([[maybe_unused]] uint32_t size = 0) override { - Array array; - array.reserve (size ? size : 8); - - if (JOIN_UNLIKELY (_stack.empty ())) + if (JOIN_UNLIKELY (_depth == 0)) { - _root = std::move (array); - _stack.push (&_root); - return 0; + _stack.clear (); } - - if (JOIN_UNLIKELY (_stack.size () >= _maxdepth)) + else if (JOIN_UNLIKELY (_depth >= _maxdepth)) { join::lastError = make_error_code (SaxErrc::StackOverflow); return -1; } - Value::Ptr parent = _stack.top (); - - if (parent->index () == Value::ObjectValue) - { - _stack.push (&parent->insert ({std::move (_curkey), std::move (array)})); - } - else - { - _stack.push (&parent->pushBack (std::move (array))); - } + _starts[_depth++] = _stack.size (); return 0; } @@ -689,47 +686,44 @@ namespace join */ virtual int stopArray () override { - if (JOIN_LIKELY (_stack.size ())) + if (JOIN_UNLIKELY (_depth == 0)) { - _stack.pop (); + return 0; // LCOV_EXCL_LINE + } + + auto first = _stack.begin () + _starts[--_depth]; + Array array (std::make_move_iterator (first), std::make_move_iterator (_stack.end ())); + _stack.erase (first, _stack.end ()); + + if (_depth == 0) + { + _root = std::move (array); + return 0; } + _stack.emplace_back (in_place_index_t{}, std::move (array)); + return 0; } /** * @brief start object. - * @param size object size. + * @param size object size (ignored). * @return 0 on success, -1 otherwise. */ - virtual int startObject (uint32_t size = 0) override + virtual int startObject ([[maybe_unused]] uint32_t size = 0) override { - Object object; - object.reserve (size ? size : 8); - - if (JOIN_UNLIKELY (_stack.empty ())) + if (JOIN_UNLIKELY (_depth == 0)) { - _root = std::move (object); - _stack.push (&_root); - return 0; + _stack.clear (); } - - if (JOIN_UNLIKELY (_stack.size () >= _maxdepth)) + else if (JOIN_UNLIKELY (_depth >= _maxdepth)) { join::lastError = make_error_code (SaxErrc::StackOverflow); return -1; } - Value::Ptr parent = _stack.top (); - - if (parent->index () == Value::ObjectValue) - { - _stack.push (&parent->insert ({std::move (_curkey), std::move (object)})); - } - else - { - _stack.push (&parent->pushBack (std::move (object))); - } + _starts[_depth++] = _stack.size (); return 0; } @@ -741,7 +735,7 @@ namespace join */ virtual int setKey (const Value& key) override { - _curkey = key; + _stack.emplace_back (key); return 0; } @@ -751,49 +745,71 @@ namespace join */ virtual int stopObject () override { - if (JOIN_LIKELY (_stack.size ())) + if (JOIN_UNLIKELY (_depth == 0)) + { + return 0; // LCOV_EXCL_LINE + } + + auto first = _stack.begin () + _starts[--_depth]; + Object object; + object.reserve ((_stack.end () - first) / 2); + for (auto it = first; it != _stack.end (); it += 2) + { + object.emplace_back (std::move (it[0]), std::move (it[1])); + } + _stack.erase (first, _stack.end ()); + + if (_depth == 0) { - _stack.pop (); + _root = std::move (object); + return 0; } + _stack.emplace_back (in_place_index_t{}, std::move (object)); + return 0; } /** - * @brief set value. - * @param value value to set. + * @brief construct a value in place in the current container. + * @param args arguments forwarded to the value constructor. * @return 0 on success, -1 otherwise. */ - virtual int setValue (Value&& value) + template + int emplaceValue (Args&&... args) { - if (JOIN_UNLIKELY (_stack.empty ())) + if (JOIN_UNLIKELY (_depth == 0)) { join::lastError = make_error_code (SaxErrc::InvalidParent); return -1; } - Value::Ptr parent = _stack.top (); - - if (parent->index () == Value::ObjectValue) - { - parent->insert ({std::move (_curkey), std::move (value)}); - } - else - { - parent->pushBack (std::move (value)); - } + _stack.emplace_back (in_place_index_t{}, std::forward (args)...); return 0; } + /** + * @brief get the value stack of the calling thread. + * @return value stack. + */ + static std::vector& stack () noexcept + { + thread_local std::vector values; + return values; + } + /// max stack depth. static constexpr size_t _maxdepth = 19; - /// stack. - std::stack _stack; + /// start of each open container in _stack. + size_t _starts[_maxdepth]; + + /// number of open containers. + size_t _depth = 0; - /// current key. - Value _curkey; + /// stack of the values of the open containers. + std::vector& _stack; /// root. Value& _root; diff --git a/data/include/join/view.hpp b/data/include/join/view.hpp index 2ef1b3bb..3069793b 100644 --- a/data/include/join/view.hpp +++ b/data/include/join/view.hpp @@ -32,6 +32,7 @@ #include #include #include +#include // C. #include @@ -210,6 +211,40 @@ namespace join return false; } + /** + * @brief read consecutive digits. + * @param value value the digits are appended to. + * @param max maximum number of digits to read. + * @return number of digits read. + */ + inline size_t readDigits (uint64_t& value, size_t max = std::numeric_limits::max ()) noexcept + { + const char* cur = _cur; + const char* end = (static_cast (_end - _cur) > max) ? _cur + max : _end; + uint64_t word; + + while ((end - cur >= 8) && + (std::memcpy (&word, cur, 8), + ((word & 0xF0F0F0F0F0F0F0F0ULL) | + (((word + 0x0606060606060606ULL) & 0xF0F0F0F0F0F0F0F0ULL) >> 4)) == 0x3333333333333333ULL)) + { + word = ((word & 0x0F0F0F0F0F0F0F0FULL) * 2561) >> 8; + word = ((word & 0x00FF00FF00FF00FFULL) * 6553601) >> 16; + word = ((word & 0x0000FFFF0000FFFFULL) * 42949672960001ULL) >> 32; + value = (value * 100000000) + word; + cur += 8; + } + + while ((cur < end) && (static_cast (*cur - '0') < 10)) + { + value = (value * 10) + (*cur++ - '0'); + } + + size_t count = cur - _cur; + _cur = cur; + return count; + } + /** * @brief read characters. * @param buf output buffer. @@ -235,6 +270,24 @@ namespace join const char* cur = _cur; const char* end = _end; + while (end - cur >= 8) + { + uint64_t word, quote, backslash; + std::memcpy (&word, cur, 8); + quote = word ^ 0x2222222222222222ULL; + backslash = word ^ 0x5C5C5C5C5C5C5C5CULL; + uint64_t mask = + (((quote - 0x0101010101010101ULL) & ~quote) | ((backslash - 0x0101010101010101ULL) & ~backslash) | + ((word - 0x2020202020202020ULL) & ~word)) & + 0x8080808080808080ULL; + if (mask) + { + cur += __builtin_ctzll (mask) >> 3; + break; + } + cur += 8; + } + while (cur < end && !details::escapedLookup.data[static_cast (*cur)]) { ++cur; @@ -256,6 +309,12 @@ namespace join while (cur < end && details::whitespaceLookup.data[static_cast (*cur)]) { ++cur; + + uint64_t word; + while ((end - cur >= 8) && (std::memcpy (&word, cur, 8), word == 0x2020202020202020ULL)) + { + cur += 8; + } } _cur = cur; @@ -466,6 +525,23 @@ namespace join return false; } + /** + * @brief read consecutive digits. + * @param value value the digits are appended to. + * @param max maximum number of digits to read. + * @return number of digits read. + */ + inline size_t readDigits (uint64_t& value, size_t max = std::numeric_limits::max ()) noexcept + { + size_t count = 0; + while ((count < max) && (static_cast (peek () - '0') < 10)) + { + value = (value * 10) + (get () - '0'); + ++count; + } + return count; + } + /** * @brief read characters. * @param buf output buffer. @@ -700,6 +776,17 @@ namespace join return _view.getIfNoCase (expected); } + /** + * @brief read consecutive digits. + * @param value value the digits are appended to. + * @param max maximum number of digits to read. + * @return number of digits read. + */ + inline size_t readDigits (uint64_t& value, size_t max = std::numeric_limits::max ()) noexcept + { + return _view.readDigits (value, max); + } + /** * @brief get snapshot. * @param out destination. @@ -808,6 +895,23 @@ namespace join return false; } + /** + * @brief read consecutive digits. + * @param value value the digits are appended to. + * @param max maximum number of digits to read. + * @return number of digits read. + */ + inline size_t readDigits (uint64_t& value, size_t max = std::numeric_limits::max ()) noexcept + { + size_t count = 0; + while ((count < max) && (static_cast (peek () - '0') < 10)) + { + value = (value * 10) + (get () - '0'); + ++count; + } + return count; + } + /** * @brief get snapshot. * @param out destination. diff --git a/data/tests/json_canonicalizer_test.cpp b/data/tests/json_canonicalizer_test.cpp index 40725977..eacc02c2 100644 --- a/data/tests/json_canonicalizer_test.cpp +++ b/data/tests/json_canonicalizer_test.cpp @@ -195,6 +195,28 @@ TEST (JsonCanonicalizer, invalid) out.str (""); ASSERT_EQ (writer.serialize (value), -1); ASSERT_EQ (join::lastError, JsonErrc::InvalidEncoding); + + value = Array{"\xFF"}; + out.str (""); + ASSERT_EQ (writer.serialize (value), -1); + ASSERT_EQ (join::lastError, JsonErrc::InvalidEncoding); +} + +/** + * @brief Test serialize method. + */ +TEST (JsonCanonicalizer, serialize) +{ + Value value = Object{{Value (1), "One"}, {"a", "Ascii"}}; + + std::stringstream out; + JsonCanonicalizer writer (out); + EXPECT_THROW (writer.serialize (value), std::bad_cast); + EXPECT_EQ (out.str (), "{"); + + out.str (""); + EXPECT_EQ (writer.setNull (), 0); + EXPECT_EQ (out.str (), "null"); } /** diff --git a/data/tests/json_reader_test.cpp b/data/tests/json_reader_test.cpp index 9e3f7b98..2293766d 100644 --- a/data/tests/json_reader_test.cpp +++ b/data/tests/json_reader_test.cpp @@ -75,6 +75,16 @@ TEST (JsonReader, deserialize) ASSERT_EQ (value.deserialize (&data[0], &data[sizeof (data)]), -1); ASSERT_EQ (join::lastError, SaxErrc::ExtraData); + + JsonReader reader (value); + ASSERT_EQ (reader.deserialize (std::string ("[1, [2, ")), -1); + ASSERT_TRUE (value.isArray ()); + ASSERT_TRUE (value.empty ()); + + ASSERT_EQ (reader.deserialize (std::string ("[3]")), 0) << join::lastError.message (); + ASSERT_TRUE (value.isArray ()); + ASSERT_EQ (value.size (), 1); + ASSERT_EQ (value[0], 3); } /** diff --git a/data/tests/json_writer_test.cpp b/data/tests/json_writer_test.cpp index 7028f220..b6cf329f 100644 --- a/data/tests/json_writer_test.cpp +++ b/data/tests/json_writer_test.cpp @@ -33,6 +33,9 @@ using join::JsonWriter; using join::Value; +using join::Array; +using join::Object; +using join::SaxErrc; /** * @brief Test setNull method. @@ -280,6 +283,30 @@ TEST (JsonWriter, serialize) stream.str (""); EXPECT_EQ (jsonWriter.serialize (value), 0) << join::lastError.message (); EXPECT_EQ (stream.str (), "[null,true,1,2,3,4,1.23456789e-13,\"test\"]"); + + value = Array{}; + std::string expected = "["; + for (int i = 0; i < 1000; ++i) + { + value.pushBack (int32_t (1)); + expected += "1,"; + } + value.pushBack (std::string (2000, 'a')); + expected += "\"" + std::string (2000, 'a') + "\"]"; + + stream.str (""); + EXPECT_EQ (jsonWriter.serialize (value), 0) << join::lastError.message (); + EXPECT_EQ (stream.str (), expected); + + value = Object{{Value (1), "One"}}; + stream.str (""); + EXPECT_EQ (jsonWriter.serialize (value), -1); + EXPECT_EQ (join::lastError, SaxErrc::InvalidKey); + + value = Object{{"a", Object{{Value (1), "One"}}}}; + stream.str (""); + EXPECT_EQ (jsonWriter.serialize (value), -1); + EXPECT_EQ (join::lastError, SaxErrc::InvalidKey); } /** diff --git a/data/tests/pack_reader_test.cpp b/data/tests/pack_reader_test.cpp index 7e9c8c9a..cc9efd42 100644 --- a/data/tests/pack_reader_test.cpp +++ b/data/tests/pack_reader_test.cpp @@ -73,6 +73,16 @@ TEST (PackReader, deserialize) ASSERT_EQ (value.deserialize (&data[0], &data[sizeof (data)]), -1); ASSERT_EQ (join::lastError, SaxErrc::ExtraData); + + PackReader reader (value); + ASSERT_EQ (reader.deserialize (std::string ({'\x92', '\x01'})), -1); + ASSERT_TRUE (value.isArray ()); + ASSERT_TRUE (value.empty ()); + + ASSERT_EQ (reader.deserialize (std::string ({'\x91', '\x03'})), 0) << join::lastError.message (); + ASSERT_TRUE (value.isArray ()); + ASSERT_EQ (value.size (), 1); + ASSERT_EQ (value[0], 3); } /** @@ -627,6 +637,14 @@ TEST (PackReader, fail) stream.clear (); stream.str (std::string ({'\xd7'})); ASSERT_NE (value.deserialize (stream), 0); + + stream.clear (); + stream.str (std::string ({'\x92', '\x01'})); + ASSERT_NE (value.deserialize (stream), 0); + + stream.clear (); + stream.str (std::string ({'\x81'})); + ASSERT_NE (value.deserialize (stream), 0); } /**