diff --git a/deps/simdjson/simdjson.cpp b/deps/simdjson/simdjson.cpp index 08b0de352947..6db84f295573 100644 --- a/deps/simdjson/simdjson.cpp +++ b/deps/simdjson/simdjson.cpp @@ -1,4 +1,4 @@ -/* auto-generated on 2026-10-29 18:49:29 -0400. version 5.1.0 Do not edit! */ +/* auto-generated on 2026-10-31 19:14:12 -0400. version 5.1.1 Do not edit! */ /* including simdjson.cpp: */ /* begin file simdjson.cpp */ #define SIMDJSON_SRC_SIMDJSON_CPP diff --git a/deps/simdjson/simdjson.h b/deps/simdjson/simdjson.h index 9d79c053f092..04dbe98eb574 100644 --- a/deps/simdjson/simdjson.h +++ b/deps/simdjson/simdjson.h @@ -1,4 +1,4 @@ -/* auto-generated on 2026-10-29 18:49:29 -0400. version 5.1.0 Do not edit! */ +/* auto-generated on 2026-10-31 19:14:12 -0400. version 5.1.1 Do not edit! */ /* including simdjson.h: */ /* begin file simdjson.h */ #ifndef SIMDJSON_H @@ -2715,7 +2715,7 @@ namespace std { #define SIMDJSON_SIMDJSON_VERSION_H /** The version of simdjson being used (major.minor.revision) */ -#define SIMDJSON_VERSION "5.1.0" +#define SIMDJSON_VERSION "5.1.1" namespace simdjson { enum { @@ -2730,7 +2730,7 @@ enum { /** * The revision (major.minor.REVISION) of simdjson being used. */ - SIMDJSON_VERSION_REVISION = 0 + SIMDJSON_VERSION_REVISION = 1 }; } // namespace simdjson @@ -8405,6 +8405,8 @@ inline constexpr bool enable_view + namespace simdjson { /** @@ -8492,6 +8494,12 @@ template class base_formatter { buffer[size++] = c; } + /** Drops the space characters at the end of the buffer, if any. **/ + simdjson_inline void trim_trailing_spaces() { + while (size > 0 && buffer[size - 1] == ' ') + size--; + } + simdjson_inline void append(const char *begin, const char *end) { const size_t new_size = size + (end - begin); if (capacity < new_size) @@ -8501,6 +8509,26 @@ template class base_formatter { size = new_size; } + simdjson_inline void append_n(char c, size_t count) { + const size_t new_size = size + count; + if (capacity < new_size) { + grow(new_size < capacity * 2 ? capacity * 2 : new_size); + } + std::memset(buffer + size, c, count); + size = new_size; + } + + /** Makes room for n more bytes and returns where they go. **/ + simdjson_inline char *reserve(size_t n) { + if (capacity - size < n) { + grow(size + n < capacity * 2 ? capacity * 2 : size + n); + } + return buffer + size; + } + + /** Ends the content at end, which reserve() returned plus at most n. **/ + simdjson_inline void set_end(const char *end) { size = size_t(end - buffer); } + std::string_view str() const { return std::string_view(buffer, size); } private: @@ -8540,6 +8568,8 @@ class mini_formatter : public base_formatter { class pretty_formatter : public base_formatter { public: + explicit pretty_formatter(int step = 4) : indent_step(step) {} + simdjson_inline void print_newline(); simdjson_inline void print_indents(size_t depth); @@ -8564,6 +8594,9 @@ template class string_builder { public: /** Construct an initially empty builder, would print the empty string **/ string_builder() = default; + /** Construct a builder whose formatter uses the given indentation step **/ + explicit string_builder(int indent_step) : format(indent_step) {} + /** Append an element to the builder (to be printed) **/ inline void append(simdjson::dom::element value); /** Append an array to the builder (to be printed) **/ @@ -8705,19 +8738,19 @@ template std::string minify(simdjson_result x) { * cout << prettify(doc) << endl; * */ -template std::string prettify(T x) { - simdjson::internal::string_builder sb; +template std::string prettify(T x, int indent_step = 4) { + simdjson::internal::string_builder sb(indent_step); sb.append(x); std::string_view answer = sb.str(); return std::string(answer.data(), answer.size()); } #if SIMDJSON_EXCEPTIONS -template std::string prettify(simdjson_result x) { +template std::string prettify(simdjson_result x, int indent_step = 4) { if (x.error()) { throw simdjson_error(x.error()); } - return prettify(x.value()); + return prettify(x.value(), indent_step); } #endif @@ -10058,7 +10091,16 @@ inline bool object::iterator::key_equals_case_insensitive(std::string_view o) co // See For case-insensitive string comparisons, avoid char-by-char functions // https://lemire.me/blog/2020/04/30/for-case-insensitive-string-comparisons-avoid-char-by-char-functions/ // Note that it might be worth rolling our own strncasecmp function, with vectorization. - return (simdjson_strncasecmp(o.data(), key_c_str(), len) == 0); + // + // strncasecmp stops at a null character, and a JSON key may contain some: + // when it does, we resume the comparison right after the null character. + const char *key = key_c_str(); + size_t i = 0; + while (simdjson_strncasecmp(o.data() + i, key + i, len - i) == 0) { + const void *null_char = std::memchr(key + i, '\0', len - i); + if (null_char == nullptr) { return true; } + i = size_t(static_cast(null_char) - key) + 1; + } } return false; } @@ -11252,8 +11294,15 @@ inline void document_stream::start() noexcept { #ifdef SIMDJSON_THREADS_ENABLED if (use_thread && next_batch_start() < len) { // Kick off the first thread if needed + // load_from_stage1_thread() swaps *parser with stage1_thread_parser, which + // then parses every other batch: give it the same settings. + stage1_thread_parser.set_max_capacity(parser->max_capacity()); + stage1_thread_parser.number_as_string(parser->number_as_string()); + error = stage1_thread_parser.allocate(batch_size, parser->max_depth()); + if (error) { return; } error = stage1_thread_parser.ensure_capacity(batch_size); if (error) { return; } + stage1_thread_parser.implementation->_number_as_string = parser->number_as_string(); worker->start_thread(); start_stage1_thread(); if (error) { return; } @@ -11890,6 +11939,777 @@ simdjson_really_inline char *write_uint_jeaiii(char *b, uint64_t n) noexcept { #endif // SIMDJSON_INTERNAL_JEAIII_H /* end file simdjson/internal/jeaiii.h */ +/* including simdjson/internal/json_escape.h: #include "simdjson/internal/json_escape.h" */ +/* begin file simdjson/internal/json_escape.h */ +#ifndef SIMDJSON_INTERNAL_JSON_ESCAPE_H +#define SIMDJSON_INTERNAL_JSON_ESCAPE_H + +// Escaping of JSON strings, shared by the builder and the DOM serializer. + +/* skipped duplicate #include "simdjson/base.h" */ +#include +#include +#include +#include +#ifdef _MSC_VER +#include +#endif + +/* + * Empirically, we have found that an inlined optimization is important for + * performance. The following macros are not ideal. We should find a better + * way to inline the code. + */ + +#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \ + (defined(_M_AMD64) || defined(_M_X64) || \ + (defined(_M_IX86_FP) && _M_IX86_FP == 2)) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2 +#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1 +#endif +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON +#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1 +#endif +#endif +#if defined(__loongarch_sx) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX +#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1 +#endif +#endif +#if defined(__loongarch_asx) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_LASX +#define SIMDJSON_EXPERIMENTAL_HAS_LASX 1 +#endif +#endif +#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \ + defined(__riscv_vector) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV +#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 +#endif +#endif +#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) +#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 +#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 +#endif +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_NEON +#include +#ifdef _MSC_VER +#include +#endif +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 +#include +#if defined(__AVX2__) +#include +#endif +#ifdef _MSC_VER +#include +#endif +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_LSX +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_LASX +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_RVV +#include +#endif +#if SIMDJSON_EXPERIMENTAL_HAS_PPC64 +#include +#ifdef bool +#undef bool +#endif +#ifdef vector +#undef vector +#endif +#endif + +namespace simdjson { +namespace internal { + +// The index of the lowest set bit of a non-zero value. The escape code runs +// outside any implementation, so it cannot use their trailing_zeroes. +simdjson_inline int trailing_zeroes(uint64_t input_num) noexcept { +#ifdef _MSC_VER + unsigned long ret; +#if defined(_M_X64) || defined(_M_ARM64) + _BitScanForward64(&ret, input_num); +#else + if (!_BitScanForward(&ret, (unsigned long)input_num)) { + _BitScanForward(&ret, (unsigned long)(input_num >> 32)); + ret += 32; + } +#endif + return int(ret); +#else + return __builtin_ctzll(input_num); +#endif +} + +// The tables are static members of a class template rather than namespace-scope +// statics: a static object would be a different object in every translation +// unit, and inline functions that index it would break the one-definition rule. +template struct escape_tables { + // 1 for the bytes that must be escaped: '"', '\\' and control characters. + static constexpr uint8_t quotable[256] = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + // The escape sequence of each control character. It is const rather than + // constexpr: MSVC rejects a constexpr char[32][7] member of a class template + // initialized from string literals (C2131). + static const char control[32][7]; +}; + +#if !SIMDJSON_CPLUSPLUS17 +template constexpr uint8_t escape_tables::quotable[256]; +#endif +template const char escape_tables::control[32][7] = { + "\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", "\\u0005", "\\u0006", + "\\u0007", "\\b", "\\t", "\\n", "\\u000b", "\\f", "\\r", + "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012", "\\u0013", "\\u0014", + "\\u0015", "\\u0016", "\\u0017", "\\u0018", "\\u0019", "\\u001a", "\\u001b", + "\\u001c", "\\u001d", "\\u001e", "\\u001f"}; + +// All Unicode characters may be placed within the quotation marks, except for +// the characters that MUST be escaped: quotation mark, reverse solidus, and the +// control characters (U+0000 through U+001F). There are two-character sequence +// escape representations of some popular characters: +// \", \\, \b, \f, \n, \r, \t. +simdjson_inline void escape_json_char(char c, char *&out) { + if (c == '"') { + memcpy(out, "\\\"", 2); + out += 2; + } else if (c == '\\') { + memcpy(out, "\\\\", 2); + out += 2; + } else { + const char *v = escape_tables<>::control[uint8_t(c)]; + const size_t n = v[1] == 'u' ? 6 : 2; + memcpy(out, v, n); + out += n; + } +} + +/** + +A possible SWAR implementation of has_json_escapable_byte. It is not used +because it is slower than the current implementation. It is kept here for +reference (to show that we tried it). + +inline bool has_json_escapable_byte(uint64_t x) { + uint64_t is_ascii = 0x8080808080808080ULL & ~x; + uint64_t xor2 = x ^ 0x0202020202020202ULL; + uint64_t lt32_or_eq34 = xor2 - 0x2121212121212121ULL; + uint64_t sub92 = x ^ 0x5C5C5C5C5C5C5C5CULL; + uint64_t eq92 = (sub92 - 0x0101010101010101ULL); + return ((lt32_or_eq34 | eq92) & is_ascii) != 0; +} + +**/ + +/* + * Every variant of the escaper lives in its own namespace (escape_scalar, + * escape_sse2, escape_avx2, ...), whose content does not depend on compiler + * flags. Callers use escape_best, defined at the end of this file. + */ + +// Scalar fallback for finding next quotable character +SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t +find_next_json_quotable_character_scalar(const std::string_view view, + size_t location) noexcept { + for (auto pos = view.begin() + location; pos != view.end(); ++pos) { + if (escape_tables<>::quotable[static_cast(*pos)]) { + return pos - view.begin(); + } + } + return size_t(view.size()); +} + +// Writes the escaped version of input to out, returning the number of bytes +// written. Finder::find(input, location) returns the position of the next +// quotable character at or after location, or input.size(). +template +inline size_t write_string_escaped_with(const std::string_view input, char *out) { + size_t mysize = input.size(); + + // Use SIMD position finder directly - it returns mysize if no escape needed + size_t location = Finder::find(input, 0); + if (location == mysize) { + // Fast path: no escaping needed + memcpy(out, input.data(), input.size()); + return input.size(); + } + + const char *const initout = out; + memcpy(out, input.data(), location); + out += location; + escape_json_char(input[location], out); + location += 1; + while (location < mysize) { + size_t newlocation = Finder::find(input, location); + memcpy(out, input.data() + location, newlocation - location); + out += newlocation - location; + location = newlocation; + if (location == mysize) { + break; + } + escape_json_char(input[location], out); + location += 1; + } + return size_t(out - initout); +} + +namespace escape_scalar { +struct finder { + static simdjson_inline size_t find(const std::string_view view, + size_t location) noexcept { + return find_next_json_quotable_character_scalar(view, location); + } +}; + +inline size_t write_string_escaped(const std::string_view input, char *out) { + return write_string_escaped_with(input, out); +} +} // namespace escape_scalar + +// SIMD-accelerated position finding that directly locates the first quotable +// character, combining detection and position extraction in a single pass to +// minimize redundant work. +#if SIMDJSON_EXPERIMENTAL_HAS_LASX +namespace escape_lasx { +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + // SIMD constants for characters requiring escape + __m256i v34 = __lasx_xvreplgr2vr_b(34); // '"' + __m256i v92 = __lasx_xvreplgr2vr_b(92); // '\\' + __m256i v32 = __lasx_xvreplgr2vr_b(32); // control char threshold + + while (remaining >= 32) { + __m256i word = __lasx_xvld(ptr, 0); + + // Check for quotable characters: '"', '\\', or control chars (< 32) + __m256i needs_escape = __lasx_xvseq_b(word, v34); + needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvseq_b(word, v92)); + needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvslt_bu(word, v32)); + + if (!__lasx_xbz_v(needs_escape)) { + // Found a quotable character - locate it via the four 64-bit lanes + uint64_t lane0 = __lasx_xvpickve2gr_du(needs_escape, 0); + uint64_t lane1 = __lasx_xvpickve2gr_du(needs_escape, 1); + uint64_t lane2 = __lasx_xvpickve2gr_du(needs_escape, 2); + uint64_t lane3 = __lasx_xvpickve2gr_du(needs_escape, 3); + size_t offset = ptr - reinterpret_cast(view.data()); + if (lane0 != 0) { + return offset + trailing_zeroes(lane0) / 8; + } else if (lane1 != 0) { + return offset + 8 + trailing_zeroes(lane1) / 8; + } else if (lane2 != 0) { + return offset + 16 + trailing_zeroes(lane2) / 8; + } else { + return offset + 24 + trailing_zeroes(lane3) / 8; + } + } + ptr += 32; + remaining -= 32; + } + size_t current = len - remaining; + return find_next_json_quotable_character_scalar(view, current); +} +struct finder { + static simdjson_inline size_t find(const std::string_view view, + size_t location) noexcept { + return find_next_json_quotable_character(view, location); + } +}; + +inline size_t write_string_escaped(const std::string_view input, char *out) { + return write_string_escaped_with(input, out); +} +} // namespace escape_lasx +#endif + +#if SIMDJSON_EXPERIMENTAL_HAS_LSX +namespace escape_lsx { +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + //SIMD constants for characters requiring escape + __m128i v34 = __lsx_vreplgr2vr_b(34); // '"' + __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\' + __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold + + while (remaining >= 16){ + __m128i word = __lsx_vld(ptr, 0); + + //Check for the quotable characters: '"', '\\', or control char (<32) + __m128i needs_escape = __lsx_vseq_b(word, v34); + needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92)); + needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32)); + + if (!__lsx_bz_v(needs_escape)){ + + //Found quotable character - extract exact byte position + uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0); + uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1); + size_t offset = ptr - reinterpret_cast(view.data()); + if ( lo != 0) { + return offset + trailing_zeroes(lo) / 8; + } else { + return offset + 8 + trailing_zeroes(hi) / 8; + } + } + ptr += 16; + remaining -= 16; + } + size_t current = len - remaining; + return find_next_json_quotable_character_scalar(view, current); +} +struct finder { + static simdjson_inline size_t find(const std::string_view view, + size_t location) noexcept { + return find_next_json_quotable_character(view, location); + } +}; + +inline size_t write_string_escaped(const std::string_view input, char *out) { + return write_string_escaped_with(input, out); +} +} // namespace escape_lsx +#endif + +#if SIMDJSON_EXPERIMENTAL_HAS_RVV +namespace escape_rvv { +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + while (remaining > 0) { + size_t vl = __riscv_vsetvl_e8m1(remaining); + vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl); + + // Check for quotable characters: '"', '\\', or control chars (< 32) + vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl); + needs_escape = __riscv_vmor(needs_escape, + __riscv_vmseq(word, (uint8_t)92, vl), vl); + needs_escape = __riscv_vmor(needs_escape, + __riscv_vmsltu(word, (uint8_t)32, vl), vl); + + long first = __riscv_vfirst(needs_escape, vl); + if (first >= 0) { + size_t offset = ptr - reinterpret_cast(view.data()); + return offset + first; + } + ptr += vl; + remaining -= vl; + } + + return len; +} +struct finder { + static simdjson_inline size_t find(const std::string_view view, + size_t location) noexcept { + return find_next_json_quotable_character(view, location); + } +}; + +inline size_t write_string_escaped(const std::string_view input, char *out) { + return write_string_escaped_with(input, out); +} +} // namespace escape_rvv +#endif + +#if SIMDJSON_EXPERIMENTAL_HAS_PPC64 +namespace escape_ppc64 { +simdjson_inline size_t +find_next_json_quotable_character(const std::string_view view, + size_t location) noexcept { + const size_t len = view.size(); + const uint8_t *ptr = + reinterpret_cast(view.data()) + location; + size_t remaining = len - location; + + // SIMD constants for characters requiring escape + __vector unsigned char v34 = vec_splats((unsigned char)34); // '"' + __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\' + __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold + + // Bitmask for vec_vbpermq to extract one bit per byte + const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50, + 0x48, 0x40, 0x38, 0x30, 0x28, 0x20, + 0x18, 0x10, 0x08, 0x00}; + + while (remaining >= 16) { + __vector unsigned char word = + vec_vsx_ld(0, reinterpret_cast(ptr)); + + // Check for quotable characters: '"', '\\', or control chars (< 32) + __vector unsigned char needs_escape = + (__vector unsigned char)vec_cmpeq(word, v34); + needs_escape = vec_or(needs_escape, + (__vector unsigned char)vec_cmpeq(word, v92)); + needs_escape = vec_or(needs_escape, + (__vector unsigned char)vec_cmplt(word, v32)); + + __vector unsigned long long result = + (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask); +#ifdef __LITTLE_ENDIAN__ + unsigned int mask = static_cast(result[1]); +#else + unsigned int mask = static_cast(result[0]); +#endif + if (mask != 0) { + size_t offset = ptr - reinterpret_cast(view.data()); + return offset + __builtin_ctz(mask); + } + ptr += 16; + remaining -= 16; + } + + // Scalar fallback for remaining bytes + size_t current = len - remaining; + return find_next_json_quotable_character_scalar(view, current); +} +struct finder { + static simdjson_inline size_t find(const std::string_view view, + size_t location) noexcept { + return find_next_json_quotable_character(view, location); + } +}; + +inline size_t write_string_escaped(const std::string_view input, char *out) { + return write_string_escaped_with(input, out); +} +} // namespace escape_ppc64 +#endif + +#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 || SIMDJSON_EXPERIMENTAL_HAS_NEON +// Block escaping: write_string_escaped scans the input itself, 16 bytes at a +// time. The two instruction sets are never available together. +#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 +namespace escape_sse2 { +#else +namespace escape_neon { +#endif + +#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 + +using escape_vector = __m128i; +// Mask bits that each input byte contributes to escape_bitmask(). +static constexpr unsigned escape_mask_bits = 1; + +simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept { + return _mm_loadu_si128(reinterpret_cast(p)); +} + +simdjson_inline void escape_store16(char *out, escape_vector v) noexcept { + _mm_storeu_si128(reinterpret_cast<__m128i *>(out), v); +} + +// Builds a vector whose bytes 0..7 come from a and bytes 8..15 from b. +simdjson_inline escape_vector escape_load8x2(const uint8_t *a, + const uint8_t *b) noexcept { + return _mm_unpacklo_epi64( + _mm_loadl_epi64(reinterpret_cast(a)), + _mm_loadl_epi64(reinterpret_cast(b))); +} + +// Builds a vector whose bytes 0..3 come from a and bytes 4..7 from b. The +// upper half is unspecified; callers only look at the low eight bits. +simdjson_inline escape_vector escape_load4x2(const uint8_t *a, + const uint8_t *b) noexcept { + int32_t a32, b32; + memcpy(&a32, a, 4); + memcpy(&b32, b, 4); + return _mm_unpacklo_epi32(_mm_cvtsi32_si128(a32), _mm_cvtsi32_si128(b32)); +} + +// Sets every bit of byte k when byte k of v is a quotable character ('"', +// '\\' or a control character). +simdjson_inline escape_vector escape_flags(escape_vector v) noexcept { + const __m128i v34 = _mm_set1_epi8(34); // '"' + const __m128i v92 = _mm_set1_epi8(92); // '\\' + const __m128i v31 = _mm_set1_epi8(31); // for control char detection + __m128i needs_escape = _mm_cmpeq_epi8(v, v34); + needs_escape = _mm_or_si128(needs_escape, _mm_cmpeq_epi8(v, v92)); + return _mm_or_si128( + needs_escape, _mm_cmpeq_epi8(_mm_subs_epu8(v, v31), _mm_setzero_si128())); +} + +// True when any byte needs escaping. Kept separate from escape_bitmask +// because some instruction sets can answer it without leaving the vector +// register file; on SSE2 the compiler folds the two together. +simdjson_inline bool escape_any(escape_vector flags) noexcept { + return _mm_movemask_epi8(flags) != 0; +} + +// A 16-bit mask with bit k set when byte k needed escaping. +simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept { + return uint64_t(uint32_t(_mm_movemask_epi8(flags))); +} + +#else // SIMDJSON_EXPERIMENTAL_HAS_NEON + +using escape_vector = uint8x16_t; +static constexpr unsigned escape_mask_bits = 4; + +simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept { + return vld1q_u8(p); +} + +simdjson_inline void escape_store16(char *out, escape_vector v) noexcept { + vst1q_u8(reinterpret_cast(out), v); +} + +simdjson_inline escape_vector escape_load8x2(const uint8_t *a, + const uint8_t *b) noexcept { + uint64_t a64, b64; + memcpy(&a64, a, 8); + memcpy(&b64, b, 8); + return vreinterpretq_u8_u64(vsetq_lane_u64(b64, vdupq_n_u64(a64), 1)); +} + +simdjson_inline escape_vector escape_load4x2(const uint8_t *a, + const uint8_t *b) noexcept { + uint32_t a32, b32; + memcpy(&a32, a, 4); + memcpy(&b32, b, 4); + return vreinterpretq_u8_u32(vsetq_lane_u32(b32, vdupq_n_u32(a32), 1)); +} + +simdjson_inline escape_vector escape_flags(escape_vector v) noexcept { + uint8x16_t needs_escape = vceqq_u8(v, vdupq_n_u8(34)); // '"' + needs_escape = vorrq_u8(needs_escape, vceqq_u8(v, vdupq_n_u8(92))); // '\\' + return vorrq_u8(needs_escape, vcltq_u8(v, vdupq_n_u8(32))); +} + +simdjson_inline bool escape_any(escape_vector flags) noexcept { + return vmaxvq_u32(vreinterpretq_u32_u8(flags)) != 0; +} + +// Four bits per byte rather than one: escape_block and the tail paths scale +// their shifts by escape_mask_bits to match. +simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept { + uint8x8_t narrowed = vshrn_n_u16(vreinterpretq_u16_u8(flags), 4); + return vget_lane_u64(vreinterpret_u64_u8(narrowed), 0); +} + +#endif // instruction set selection +// The escape bitmask of a 16-byte block. +simdjson_inline uint64_t escape_mask(escape_vector v) noexcept { + return escape_bitmask(escape_flags(v)); +} + +// Copies n bytes with n < 16, using overlapping loads and stores. It never +// reads more than n bytes from src, nor writes more than n bytes to dst. +simdjson_inline void copy_lt16(char *dst, const uint8_t *src, + size_t n) noexcept { + if (n >= 8) { + memcpy(dst, src, 8); + memcpy(dst + n - 8, src + n - 8, 8); + } else if (n >= 4) { + memcpy(dst, src, 4); + memcpy(dst + n - 4, src + n - 4, 4); + } else if (n > 0) { + dst[0] = char(src[0]); + dst[n >> 1] = char(src[n >> 1]); + dst[n - 1] = char(src[n - 1]); + } +} + +// Escapes the bytes of src in the range [i, blockend), given that m is the +// (non-zero) escape mask for that range: the escape_mask_bits-wide lane of m +// at byte k is non-zero when src[i + k] requires escaping. Returns the updated +// output pointer. +simdjson_never_inline char *escape_block(const uint8_t *src, char *out, + size_t i, size_t blockend, + uint64_t m) noexcept { + constexpr uint64_t lane = (uint64_t(1) << escape_mask_bits) - 1; + + size_t pos = i; // first byte not yet copied + while (m) { + const size_t tz = trailing_zeroes(m); + const size_t next = i + tz / escape_mask_bits; + // Copy the run of safe bytes that precedes this escape. + copy_lt16(out, src + pos, next - pos); + out += next - pos; + escape_json_char(char(src[next]), out); + pos = next + 1; + m &= ~(lane << tz); + } + // Copy whatever follows the last escape. + copy_lt16(out, src + pos, blockend - pos); + return out + (blockend - pos); +} + +// Escapes src[i, len) to out, returning the updated output pointer. The input +// must be the whole string: when len >= 16, the tail reloads its last 16 bytes. +simdjson_really_inline char *escape_from(const uint8_t *src, size_t len, + size_t i, char *out) noexcept { + while (i + 16 <= len) { + escape_vector word = escape_load16(src + i); + escape_vector flags = escape_flags(word); + if (simdjson_likely(!escape_any(flags))) { + escape_store16(out, word); + out += 16; + } else { + out = escape_block(src, out, i, i + 16, escape_bitmask(flags)); + } + i += 16; + } + if (i < len) { + const size_t rem = len - i; + uint64_t m; + if (len >= 16) { + // The last 16 bytes of the input are in bounds. Bit k of that block's + // mask belongs to input position len - 16 + k, so shift it down to align + // bit 0 with position i. + m = escape_mask(escape_load16(src + len - 16)) >> + (escape_mask_bits * (16 - rem)); + } else if (len >= 8) { + // Here i == 0 and rem == len. Two overlapping 8-byte loads cover + // [0, 8) and [len - 8, len), which is the whole input since len < 16. + uint64_t mm = escape_mask(escape_load8x2(src, src + len - 8)); + constexpr uint64_t low8 = (uint64_t(1) << (escape_mask_bits * 8)) - 1; + m = (mm & low8) | + ((mm >> (escape_mask_bits * 8)) << (escape_mask_bits * (len - 8))); + } else if (len >= 4) { + // Same idea with two overlapping 4-byte loads. + uint64_t mm = escape_mask(escape_load4x2(src, src + len - 4)); + constexpr uint64_t low4 = (uint64_t(1) << (escape_mask_bits * 4)) - 1; + m = (mm & low4) | (((mm >> (escape_mask_bits * 4)) & low4) + << (escape_mask_bits * (len - 4))); + } else { + // Fewer than 4 bytes: at most three table lookups, no need for SIMD. + for (size_t k = 0; k < len; k++) { + uint8_t c = src[k]; + if (escape_tables<>::quotable[c]) { + escape_json_char(char(c), out); + } else { + *out++ = char(c); + } + } + return out; + } + if (m == 0) { + copy_lt16(out, src + i, rem); + out += rem; + } else { + out = escape_block(src, out, i, len, m); + } + } + return out; +} + +// Writes the escaped version of input to out, returning the number of bytes +// written. +simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out) { + const uint8_t *src = reinterpret_cast(input.data()); + return size_t(escape_from(src, input.size(), 0, out) - out); +} + +#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 +} // namespace escape_sse2 +#else +} // namespace escape_neon +#endif +#endif // SIMDJSON_EXPERIMENTAL_HAS_SSE2 || SIMDJSON_EXPERIMENTAL_HAS_NEON + +#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 && defined(__AVX2__) +// SSE2 with a 32-byte main loop. It has its own namespace because it is +// compiled only when __AVX2__ is defined. +namespace escape_avx2 { +simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out) { + using namespace escape_sse2; + const size_t len = input.size(); + const uint8_t *src = reinterpret_cast(input.data()); + const char *const initout = out; + + size_t i = 0; + while (i + 32 <= len) { + const __m256i word = _mm256_loadu_si256(reinterpret_cast(src + i)); + const __m256i flags = _mm256_or_si256( + _mm256_or_si256(_mm256_cmpeq_epi8(word, _mm256_set1_epi8(34)), // '"' + _mm256_cmpeq_epi8(word, _mm256_set1_epi8(92))), // '\\' + _mm256_cmpeq_epi8(_mm256_subs_epu8(word, _mm256_set1_epi8(31)), + _mm256_setzero_si256())); // control + const uint32_t mask = uint32_t(_mm256_movemask_epi8(flags)); + if (simdjson_likely(mask == 0)) { + _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), word); + out += 32; + } else { + for (size_t half = 0; half < 32; half += 16) { + const uint64_t m = (mask >> half) & 0xFFFF; + if (m == 0) { + escape_store16(out, escape_load16(src + i + half)); + out += 16; + } else { + out = escape_block(src, out, i + half, i + half + 16, m); + } + } + } + i += 32; + } + return size_t(escape_from(src, len, i, out) - initout); +} +} // namespace escape_avx2 +#endif + +// The variant that the builder and the DOM serializer use: the best one that +// the compiler flags allow. Translation units compiled with different flags +// may select different variants for the same inline functions; see +// "Mixing compiler flags" in doc/implementation-selection.md. +#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 && defined(__AVX2__) +namespace escape_best = escape_avx2; +#elif SIMDJSON_EXPERIMENTAL_HAS_SSE2 +namespace escape_best = escape_sse2; +#elif SIMDJSON_EXPERIMENTAL_HAS_NEON +namespace escape_best = escape_neon; +#elif SIMDJSON_EXPERIMENTAL_HAS_LASX +namespace escape_best = escape_lasx; +#elif SIMDJSON_EXPERIMENTAL_HAS_LSX +namespace escape_best = escape_lsx; +#elif SIMDJSON_EXPERIMENTAL_HAS_RVV +namespace escape_best = escape_rvv; +#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64 +namespace escape_best = escape_ppc64; +#else +namespace escape_best = escape_scalar; +#endif + +} // namespace internal +} // namespace simdjson + +#endif // SIMDJSON_INTERNAL_JSON_ESCAPE_H +/* end file simdjson/internal/json_escape.h */ /* skipped duplicate #include "simdjson/internal/tape_type.h" */ /* skipped duplicate #include "simdjson/dom/array-inl.h" */ @@ -11962,20 +12782,6 @@ operator<<(std::ostream &out, } // namespace dom -/*** - * Number utility functions - **/ -namespace { -/**@private - * Escape sequence like \b or \u0001 - * We expect that most compilers will use 8 bytes for this data structure. - **/ -struct escape_sequence { - uint8_t length; - const char - string[7]; // technically, we only ever need 6 characters, we pad to 8 -}; -} // anonymous namespace namespace internal { /*** @@ -12091,86 +12897,14 @@ template simdjson_inline void base_formatter::string(std::string_view unescaped) { one_char('\"'); - size_t i = 0; - // Fast path for the case where we have no control character, no ", and no - // backslash. This should include most keys. - // - // We would like to use 'bool' but some compilers take offense to bitwise - // operation with bool types. - constexpr static char needs_escaping[] = { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - for (; i + 8 <= unescaped.length(); i += 8) { - // Poor's man vectorization. This could get much faster if we used SIMD. - // - // It is not the case that replacing '|' with '||' would be neutral - // performance-wise. - if (needs_escaping[uint8_t(unescaped[i])] | - needs_escaping[uint8_t(unescaped[i + 1])] | - needs_escaping[uint8_t(unescaped[i + 2])] | - needs_escaping[uint8_t(unescaped[i + 3])] | - needs_escaping[uint8_t(unescaped[i + 4])] | - needs_escaping[uint8_t(unescaped[i + 5])] | - needs_escaping[uint8_t(unescaped[i + 6])] | - needs_escaping[uint8_t(unescaped[i + 7])]) { - break; - } - } - for (; i < unescaped.length(); i++) { - if (needs_escaping[uint8_t(unescaped[i])]) { - break; - } + // Each input byte becomes at most 6 output bytes. Escaping the string in + // pieces bounds the room reserved ahead of a long string. + constexpr size_t piece = 4096; + for (size_t i = 0; i < unescaped.size(); i += piece) { + const std::string_view part = unescaped.substr(i, piece); + char *out = buffer.reserve(6 * part.size()); + buffer.set_end(out + escape_best::write_string_escaped(part, out)); } - // The following is also possible and omits a 256-byte table, but it is - // slower: for (; (i < unescaped.length()) && (uint8_t(unescaped[i]) > 0x1F) - // && (unescaped[i] != '\"') && (unescaped[i] != '\\'); i++) {} - - // At least for long strings, the following should be fast. We could - // do better by integrating the checks and the insertion. - chars(unescaped.data(), unescaped.data() + i); - // We caught a control character if we enter this loop (slow). - // Note that we are do not restart from the beginning, but rather we continue - // from the point where we encountered something that requires escaping. - for (; i < unescaped.length(); i++) { - switch (unescaped[i]) { - case '\"': { - const char *s = "\\\""; - chars(s, s + 2); - } break; - case '\\': { - const char *s = "\\\\"; - chars(s, s + 2); - } break; - default: - if (uint8_t(unescaped[i]) <= 0x1F) { - // If packed, this uses 8 * 32 bytes. - // Note that we expect most compilers to embed this code in the data - // section. - constexpr static escape_sequence escaped[32] = { - {6, "\\u0000"}, {6, "\\u0001"}, {6, "\\u0002"}, {6, "\\u0003"}, - {6, "\\u0004"}, {6, "\\u0005"}, {6, "\\u0006"}, {6, "\\u0007"}, - {2, "\\b"}, {2, "\\t"}, {2, "\\n"}, {6, "\\u000b"}, - {2, "\\f"}, {2, "\\r"}, {6, "\\u000e"}, {6, "\\u000f"}, - {6, "\\u0010"}, {6, "\\u0011"}, {6, "\\u0012"}, {6, "\\u0013"}, - {6, "\\u0014"}, {6, "\\u0015"}, {6, "\\u0016"}, {6, "\\u0017"}, - {6, "\\u0018"}, {6, "\\u0019"}, {6, "\\u001a"}, {6, "\\u001b"}, - {6, "\\u001c"}, {6, "\\u001d"}, {6, "\\u001e"}, {6, "\\u001f"}}; - auto u = escaped[uint8_t(unescaped[i])]; - chars(u.string, u.string + u.length); - } else { - one_char(unescaped[i]); - } - } // switch - } // for one_char('\"'); } @@ -12195,12 +12929,10 @@ simdjson_inline void mini_formatter::print_space() { return; } simdjson_inline void pretty_formatter::print_newline() { one_char('\n'); } simdjson_inline void pretty_formatter::print_indents(size_t depth) { - if (this->indent_step <= 0) { + if (indent_step <= 0) { return; } - for (size_t i = 0; i < this->indent_step * depth; i++) { - one_char(' '); - } + buffer.append_n(' ', depth * static_cast(indent_step)); } simdjson_inline void pretty_formatter::print_space() { one_char(' '); } @@ -12790,7 +13522,7 @@ class fractured_string_builder { /** Row for a uniform scalar array: writes elem inline, then pads to width so every row lines up. */ void format_table_scalar_row(const dom::element& elem, const element_metrics& row_metrics, - size_t width, size_t depth, table_column_type column_type); + size_t width, table_column_type column_type); /** Shared per-column writer: recurses if the column has children, * otherwise writes a plain padded value or blank. */ @@ -12803,7 +13535,7 @@ class fractured_string_builder { /** Writes a single aligned leaf value */ void format_table_leaf_value(const dom::element& elem, const element_metrics& vm, size_t width, table_column_type column_type, bool needs_comma, - bool add_comma_space, size_t depth); + bool add_comma_space); /** Whether, for a column of the given type, the comma goes right after the value */ bool comma_goes_before_padding(table_column_type column_type) const; @@ -12963,7 +13695,7 @@ inline element_metrics structure_analyzer::analyze_array(const dom::array& arr, for (dom::element child : arr) { if (!first) { - metrics.estimated_inline_len += 2; // ", " + metrics.estimated_inline_len += current_opts_->comma_padding ? 2 : 1; // ", " or "," } first = false; @@ -13012,12 +13744,13 @@ inline element_metrics structure_analyzer::analyze_object(const dom::object& obj for (dom::key_value_pair field : obj) { if (!first) { - metrics.estimated_inline_len += 2; // ", " + metrics.estimated_inline_len += current_opts_->comma_padding ? 2 : 1; // ", " or "," } first = false; - // Key length: quotes + key + colon + space - metrics.estimated_inline_len += estimate_string_length(field.key) + 2; + // Key length: quotes + key + colon (+ space) + metrics.estimated_inline_len += estimate_string_length(field.key) + + (current_opts_->colon_padding ? 2 : 1); element_metrics child_metrics = analyze_element(field.value, depth + 1); metrics.estimated_inline_len += child_metrics.estimated_inline_len; @@ -13411,6 +14144,10 @@ simdjson_inline void fractured_formatter::print_newline() { if (current_layout_ == layout_mode::single_line) { return; // No newlines in inline mode } + // Like the reference FracturedJson implementation, do not end a line with + // white space: the padding that aligns a value in a column is only needed + // when something follows it on the same line. + buffer.trim_trailing_spaces(); one_char('\n'); current_line_length_ = 0; } @@ -13688,7 +14425,7 @@ inline void fractured_string_builder::format_array_compact_multiline_aligned( const element_metrics& row_metrics = child_metrics_at(metrics.children, child_idx); if (columns.empty()) { - format_table_scalar_row(elem, row_metrics, metrics.table_row_width_full, depth + 1, + format_table_scalar_row(elem, row_metrics, metrics.table_row_width_full, metrics.scalar_column_type); } else { format_table_row(elem, row_metrics, columns, depth + 1); @@ -13761,7 +14498,7 @@ inline void fractured_string_builder::format_table_row_columns( } else { const element_metrics& vm = child_metrics_at(value_metrics[col_idx]); format_table_leaf_value(values[col_idx], vm, column.width, column.type, needs_comma, - /*add_comma_space=*/true, depth); + /*add_comma_space=*/true); } if (!is_last_col && !needs_comma) { @@ -13879,7 +14616,7 @@ inline bool fractured_string_builder::comma_goes_before_padding(table_column_typ inline void fractured_string_builder::format_table_leaf_value( const dom::element& elem, const element_metrics& vm, size_t width, - table_column_type column_type, bool needs_comma, bool add_comma_space, size_t depth) { + table_column_type column_type, bool needs_comma, bool add_comma_space) { bool comma_before_pad = needs_comma && comma_goes_before_padding(column_type); bool comma_after_pad = needs_comma && !comma_before_pad; @@ -13901,10 +14638,8 @@ inline void fractured_string_builder::format_table_leaf_value( format_.one_char(' '); } - { - scoped_single_line_mode single_line(format_); - format_element(elem, vm, depth); - } + // The cell shares its line with the rest of the row. + format_element_inline(elem, vm); if (comma_before_pad) { format_.comma(); @@ -13921,10 +14656,10 @@ inline void fractured_string_builder::format_table_leaf_value( } inline void fractured_string_builder::format_table_scalar_row( - const dom::element& elem, const element_metrics& row_metrics, size_t width, size_t depth, + const dom::element& elem, const element_metrics& row_metrics, size_t width, table_column_type column_type) { format_table_leaf_value(elem, row_metrics, width, column_type, - /*needs_comma=*/false, /*add_comma_space=*/false, depth); + /*needs_comma=*/false, /*add_comma_space=*/false); } inline void fractured_string_builder::format_array_as_table(const dom::array& arr, @@ -13952,7 +14687,7 @@ inline void fractured_string_builder::format_array_as_table(const dom::array& ar const element_metrics& row_metrics = child_metrics_at(metrics.children, child_idx); if (columns.empty()) { - format_table_scalar_row(elem, row_metrics, metrics.table_row_width, depth + 1, + format_table_scalar_row(elem, row_metrics, metrics.table_row_width, metrics.scalar_column_type); } else { format_table_row(elem, row_metrics, columns, depth + 1); @@ -44090,6 +44825,7 @@ consteval const char* get_json_key_name() { #endif // SIMDJSON_ANNOTATIONS_H /* end file simdjson/annotations.h */ /* skipped duplicate #include "simdjson/internal/jeaiii.h" */ +/* skipped duplicate #include "simdjson/internal/json_escape.h" */ #endif // SIMDJSON_GENERIC_BUILDER_DEPENDENCIES_H /* end file simdjson/generic/builder/dependencies.h */ @@ -44964,6 +45700,7 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ namespace simdjson { @@ -45000,6 +45737,9 @@ struct has_custom_serialization : std::false_type {}; namespace arm64 { namespace builder { + +using simdjson::internal::escape_best::write_string_escaped; + /** * A builder for JSON strings representing documents. This is a low-level * builder that is not meant to be used directly by end-users. Though it @@ -45346,6 +46086,7 @@ simdjson_warn_unused error_code to_json(const Z &z, std::string &s, size_t initi /* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ /* amalgamation skipped (editor-only): #include "simdjson/concepts.h" */ /* amalgamation skipped (editor-only): #include "simdjson/annotations.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ #if SIMDJSON_STATIC_REFLECTION @@ -45371,7 +46112,6 @@ namespace internal { using simdjson::internal::write_uint_jeaiii; simdjson_inline char *write_double(char *p, double v) noexcept; } // namespace internal -simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out); SIMDJSON_PUSH_DISABLE_WARNINGS SIMDJSON_DISABLE_GCC_WARNING(-Warray-bounds) @@ -46392,619 +47132,13 @@ simdjson_warn_unused simdjson_result extract_fractured_json( /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_INL_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/jeaiii.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ -/* - * Empirically, we have found that an inlined optimization is important for - * performance. The following macros are not ideal. We should find a better - * way to inline the code. - */ - -#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \ - (defined(_M_AMD64) || defined(_M_X64) || \ - (defined(_M_IX86_FP) && _M_IX86_FP == 2)) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2 -#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1 -#endif -#endif - -#if defined(__aarch64__) || defined(_M_ARM64) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON -#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1 -#endif -#endif -#if defined(__loongarch_sx) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX -#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1 -#endif -#endif -#if defined(__loongarch_asx) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_LASX -#define SIMDJSON_EXPERIMENTAL_HAS_LASX 1 -#endif -#endif -#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \ - defined(__riscv_vector) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV -#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 -#endif -#endif -#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 -#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_NEON -#include -#ifdef _MSC_VER -#include -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 -#include -#if defined(__AVX2__) -#include -#endif -#ifdef _MSC_VER -#include -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_LSX -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_LASX -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_RVV -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_PPC64 -#include -#ifdef bool -#undef bool -#endif -#ifdef vector -#undef vector -#endif -#endif - - namespace simdjson { namespace arm64 { namespace builder { -static SIMDJSON_CONSTEXPR_LAMBDA std::array - json_quotable_character = { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - -/** - -A possible SWAR implementation of has_json_escapable_byte. It is not used -because it is slower than the current implementation. It is kept here for -reference (to show that we tried it). - -inline bool has_json_escapable_byte(uint64_t x) { - uint64_t is_ascii = 0x8080808080808080ULL & ~x; - uint64_t xor2 = x ^ 0x0202020202020202ULL; - uint64_t lt32_or_eq34 = xor2 - 0x2121212121212121ULL; - uint64_t sub92 = x ^ 0x5C5C5C5C5C5C5C5CULL; - uint64_t eq92 = (sub92 - 0x0101010101010101ULL); - return ((lt32_or_eq34 | eq92) & is_ascii) != 0; -} - -**/ - -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 || SIMDJSON_EXPERIMENTAL_HAS_NEON -#define SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE 1 -#endif - -// With block escaping (SSE2/NEON), write_string_escaped scans the input itself. -#if !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE -// Scalar fallback for finding next quotable character -SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t -find_next_json_quotable_character_scalar(const std::string_view view, - size_t location) noexcept { - for (auto pos = view.begin() + location; pos != view.end(); ++pos) { - if (json_quotable_character[static_cast(*pos)]) { - return pos - view.begin(); - } - } - return size_t(view.size()); -} - -// SIMD-accelerated position finding that directly locates the first quotable -// character, combining detection and position extraction in a single pass to -// minimize redundant work. -#if SIMDJSON_EXPERIMENTAL_HAS_LASX -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - // SIMD constants for characters requiring escape - __m256i v34 = __lasx_xvreplgr2vr_b(34); // '"' - __m256i v92 = __lasx_xvreplgr2vr_b(92); // '\\' - __m256i v32 = __lasx_xvreplgr2vr_b(32); // control char threshold - - while (remaining >= 32) { - __m256i word = __lasx_xvld(ptr, 0); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - __m256i needs_escape = __lasx_xvseq_b(word, v34); - needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvseq_b(word, v92)); - needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvslt_bu(word, v32)); - - if (!__lasx_xbz_v(needs_escape)) { - // Found a quotable character - locate it via the four 64-bit lanes - uint64_t lane0 = __lasx_xvpickve2gr_du(needs_escape, 0); - uint64_t lane1 = __lasx_xvpickve2gr_du(needs_escape, 1); - uint64_t lane2 = __lasx_xvpickve2gr_du(needs_escape, 2); - uint64_t lane3 = __lasx_xvpickve2gr_du(needs_escape, 3); - size_t offset = ptr - reinterpret_cast(view.data()); - if (lane0 != 0) { - return offset + trailing_zeroes(lane0) / 8; - } else if (lane1 != 0) { - return offset + 8 + trailing_zeroes(lane1) / 8; - } else if (lane2 != 0) { - return offset + 16 + trailing_zeroes(lane2) / 8; - } else { - return offset + 24 + trailing_zeroes(lane3) / 8; - } - } - ptr += 32; - remaining -= 32; - } - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -#elif SIMDJSON_EXPERIMENTAL_HAS_LSX -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - //SIMD constants for characters requiring escape - __m128i v34 = __lsx_vreplgr2vr_b(34); // '"' - __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\' - __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold - - while (remaining >= 16){ - __m128i word = __lsx_vld(ptr, 0); - - //Check for the quotable characters: '"', '\\', or control char (<32) - __m128i needs_escape = __lsx_vseq_b(word, v34); - needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92)); - needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32)); - - if (!__lsx_bz_v(needs_escape)){ - - //Found quotable character - extract exact byte position - uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0); - uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1); - size_t offset = ptr - reinterpret_cast(view.data()); - if ( lo != 0) { - return offset + trailing_zeroes(lo) / 8; - } else { - return offset + 8 + trailing_zeroes(hi) / 8; - } - } - ptr += 16; - remaining -= 16; - } - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -#elif SIMDJSON_EXPERIMENTAL_HAS_RVV -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - while (remaining > 0) { - size_t vl = __riscv_vsetvl_e8m1(remaining); - vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl); - needs_escape = __riscv_vmor(needs_escape, - __riscv_vmseq(word, (uint8_t)92, vl), vl); - needs_escape = __riscv_vmor(needs_escape, - __riscv_vmsltu(word, (uint8_t)32, vl), vl); - - long first = __riscv_vfirst(needs_escape, vl); - if (first >= 0) { - size_t offset = ptr - reinterpret_cast(view.data()); - return offset + first; - } - ptr += vl; - remaining -= vl; - } - - return len; -} -#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64 -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - // SIMD constants for characters requiring escape - __vector unsigned char v34 = vec_splats((unsigned char)34); // '"' - __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\' - __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold - - // Bitmask for vec_vbpermq to extract one bit per byte - const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50, - 0x48, 0x40, 0x38, 0x30, 0x28, 0x20, - 0x18, 0x10, 0x08, 0x00}; - - while (remaining >= 16) { - __vector unsigned char word = - vec_vsx_ld(0, reinterpret_cast(ptr)); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - __vector unsigned char needs_escape = - (__vector unsigned char)vec_cmpeq(word, v34); - needs_escape = vec_or(needs_escape, - (__vector unsigned char)vec_cmpeq(word, v92)); - needs_escape = vec_or(needs_escape, - (__vector unsigned char)vec_cmplt(word, v32)); - - __vector unsigned long long result = - (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask); -#ifdef __LITTLE_ENDIAN__ - unsigned int mask = static_cast(result[1]); -#else - unsigned int mask = static_cast(result[0]); -#endif - if (mask != 0) { - size_t offset = ptr - reinterpret_cast(view.data()); - return offset + __builtin_ctz(mask); - } - ptr += 16; - remaining -= 16; - } - - // Scalar fallback for remaining bytes - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -#else -SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - return find_next_json_quotable_character_scalar(view, location); -} -#endif -#endif // !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - -SIMDJSON_CONSTEXPR_LAMBDA static std::string_view control_chars[] = { - "\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", "\\u0005", "\\u0006", - "\\u0007", "\\b", "\\t", "\\n", "\\u000b", "\\f", "\\r", - "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012", "\\u0013", "\\u0014", - "\\u0015", "\\u0016", "\\u0017", "\\u0018", "\\u0019", "\\u001a", "\\u001b", - "\\u001c", "\\u001d", "\\u001e", "\\u001f"}; - -// All Unicode characters may be placed within the quotation marks, except for -// the characters that MUST be escaped: quotation mark, reverse solidus, and the -// control characters (U+0000 through U+001F). There are two-character sequence -// escape representations of some popular characters: -// \", \\, \b, \f, \n, \r, \t. -SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline void escape_json_char(char c, char *&out) { - if (c == '"') { - memcpy(out, "\\\"", 2); - out += 2; - } else if (c == '\\') { - memcpy(out, "\\\\", 2); - out += 2; - } else { - std::string_view v = control_chars[uint8_t(c)]; - memcpy(out, v.data(), v.size()); - out += v.size(); - } -} - -#if SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 - -using escape_vector = __m128i; -// Mask bits that each input byte contributes to escape_bitmask(). -static constexpr unsigned escape_mask_bits = 1; - -simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept { - return _mm_loadu_si128(reinterpret_cast(p)); -} - -simdjson_inline void escape_store16(char *out, escape_vector v) noexcept { - _mm_storeu_si128(reinterpret_cast<__m128i *>(out), v); -} - -// Builds a vector whose bytes 0..7 come from a and bytes 8..15 from b. -simdjson_inline escape_vector escape_load8x2(const uint8_t *a, - const uint8_t *b) noexcept { - return _mm_unpacklo_epi64( - _mm_loadl_epi64(reinterpret_cast(a)), - _mm_loadl_epi64(reinterpret_cast(b))); -} - -// Builds a vector whose bytes 0..3 come from a and bytes 4..7 from b. The -// upper half is unspecified; callers only look at the low eight bits. -simdjson_inline escape_vector escape_load4x2(const uint8_t *a, - const uint8_t *b) noexcept { - int32_t a32, b32; - memcpy(&a32, a, 4); - memcpy(&b32, b, 4); - return _mm_unpacklo_epi32(_mm_cvtsi32_si128(a32), _mm_cvtsi32_si128(b32)); -} - -// Sets every bit of byte k when byte k of v is a quotable character ('"', -// '\\' or a control character). -simdjson_inline escape_vector escape_flags(escape_vector v) noexcept { - const __m128i v34 = _mm_set1_epi8(34); // '"' - const __m128i v92 = _mm_set1_epi8(92); // '\\' - const __m128i v31 = _mm_set1_epi8(31); // for control char detection - __m128i needs_escape = _mm_cmpeq_epi8(v, v34); - needs_escape = _mm_or_si128(needs_escape, _mm_cmpeq_epi8(v, v92)); - return _mm_or_si128( - needs_escape, _mm_cmpeq_epi8(_mm_subs_epu8(v, v31), _mm_setzero_si128())); -} - -// True when any byte needs escaping. Kept separate from escape_bitmask -// because some instruction sets can answer it without leaving the vector -// register file; on SSE2 the compiler folds the two together. -simdjson_inline bool escape_any(escape_vector flags) noexcept { - return _mm_movemask_epi8(flags) != 0; -} - -// A 16-bit mask with bit k set when byte k needed escaping. -simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept { - return uint64_t(uint32_t(_mm_movemask_epi8(flags))); -} - -#else // SIMDJSON_EXPERIMENTAL_HAS_NEON - -using escape_vector = uint8x16_t; -static constexpr unsigned escape_mask_bits = 4; - -simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept { - return vld1q_u8(p); -} - -simdjson_inline void escape_store16(char *out, escape_vector v) noexcept { - vst1q_u8(reinterpret_cast(out), v); -} - -simdjson_inline escape_vector escape_load8x2(const uint8_t *a, - const uint8_t *b) noexcept { - uint64_t a64, b64; - memcpy(&a64, a, 8); - memcpy(&b64, b, 8); - return vreinterpretq_u8_u64(vsetq_lane_u64(b64, vdupq_n_u64(a64), 1)); -} - -simdjson_inline escape_vector escape_load4x2(const uint8_t *a, - const uint8_t *b) noexcept { - uint32_t a32, b32; - memcpy(&a32, a, 4); - memcpy(&b32, b, 4); - return vreinterpretq_u8_u32(vsetq_lane_u32(b32, vdupq_n_u32(a32), 1)); -} - -simdjson_inline escape_vector escape_flags(escape_vector v) noexcept { - uint8x16_t needs_escape = vceqq_u8(v, vdupq_n_u8(34)); // '"' - needs_escape = vorrq_u8(needs_escape, vceqq_u8(v, vdupq_n_u8(92))); // '\\' - return vorrq_u8(needs_escape, vcltq_u8(v, vdupq_n_u8(32))); -} - -simdjson_inline bool escape_any(escape_vector flags) noexcept { - return vmaxvq_u32(vreinterpretq_u32_u8(flags)) != 0; -} - -// Four bits per byte rather than one: escape_block and the tail paths scale -// their shifts by escape_mask_bits to match. -simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept { - uint8x8_t narrowed = vshrn_n_u16(vreinterpretq_u16_u8(flags), 4); - return vget_lane_u64(vreinterpret_u64_u8(narrowed), 0); -} - -#endif // instruction set selection - -// The escape bitmask of a 16-byte block. -simdjson_inline uint64_t escape_mask(escape_vector v) noexcept { - return escape_bitmask(escape_flags(v)); -} - -// Copies n bytes with n < 16, using overlapping loads and stores. It never -// reads more than n bytes from src, nor writes more than n bytes to dst. -simdjson_inline void copy_lt16(char *dst, const uint8_t *src, - size_t n) noexcept { - if (n >= 8) { - memcpy(dst, src, 8); - memcpy(dst + n - 8, src + n - 8, 8); - } else if (n >= 4) { - memcpy(dst, src, 4); - memcpy(dst + n - 4, src + n - 4, 4); - } else if (n > 0) { - dst[0] = char(src[0]); - dst[n >> 1] = char(src[n >> 1]); - dst[n - 1] = char(src[n - 1]); - } -} - -// Escapes the bytes of src in the range [i, blockend), given that m is the -// (non-zero) escape mask for that range: the escape_mask_bits-wide lane of m -// at byte k is non-zero when src[i + k] requires escaping. Returns the updated -// output pointer. -simdjson_never_inline char *escape_block(const uint8_t *src, char *out, - size_t i, size_t blockend, - uint64_t m) noexcept { - constexpr uint64_t lane = (uint64_t(1) << escape_mask_bits) - 1; - - size_t pos = i; // first byte not yet copied - while (m) { - const size_t tz = trailing_zeroes(m); - const size_t next = i + tz / escape_mask_bits; - // Copy the run of safe bytes that precedes this escape. - copy_lt16(out, src + pos, next - pos); - out += next - pos; - escape_json_char(char(src[next]), out); - pos = next + 1; - m &= ~(lane << tz); - } - // Copy whatever follows the last escape. - copy_lt16(out, src + pos, blockend - pos); - return out + (blockend - pos); -} - -// Writes the escaped version of input to out, returning the number of bytes -// written. -simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out) { - const size_t len = input.size(); - const uint8_t *src = reinterpret_cast(input.data()); - const char *const initout = out; - - size_t i = 0; -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 && defined(__AVX2__) - while (i + 32 <= len) { - const __m256i word = _mm256_loadu_si256(reinterpret_cast(src + i)); - const __m256i flags = _mm256_or_si256( - _mm256_or_si256(_mm256_cmpeq_epi8(word, _mm256_set1_epi8(34)), // '"' - _mm256_cmpeq_epi8(word, _mm256_set1_epi8(92))), // '\\' - _mm256_cmpeq_epi8(_mm256_subs_epu8(word, _mm256_set1_epi8(31)), - _mm256_setzero_si256())); // control - const uint32_t mask = uint32_t(_mm256_movemask_epi8(flags)); - if (simdjson_likely(mask == 0)) { - _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), word); - out += 32; - } else { - for (size_t half = 0; half < 32; half += 16) { - const uint64_t m = (mask >> half) & 0xFFFF; - if (m == 0) { - escape_store16(out, escape_load16(src + i + half)); - out += 16; - } else { - out = escape_block(src, out, i + half, i + half + 16, m); - } - } - } - i += 32; - } -#endif - while (i + 16 <= len) { - escape_vector word = escape_load16(src + i); - escape_vector flags = escape_flags(word); - if (simdjson_likely(!escape_any(flags))) { - escape_store16(out, word); - out += 16; - } else { - out = escape_block(src, out, i, i + 16, escape_bitmask(flags)); - } - i += 16; - } - if (i < len) { - const size_t rem = len - i; - uint64_t m; - if (len >= 16) { - // The last 16 bytes of the input are in bounds. Bit k of that block's - // mask belongs to input position len - 16 + k, so shift it down to align - // bit 0 with position i. - m = escape_mask(escape_load16(src + len - 16)) >> - (escape_mask_bits * (16 - rem)); - } else if (len >= 8) { - // Here i == 0 and rem == len. Two overlapping 8-byte loads cover - // [0, 8) and [len - 8, len), which is the whole input since len < 16. - uint64_t mm = escape_mask(escape_load8x2(src, src + len - 8)); - constexpr uint64_t low8 = (uint64_t(1) << (escape_mask_bits * 8)) - 1; - m = (mm & low8) | - ((mm >> (escape_mask_bits * 8)) << (escape_mask_bits * (len - 8))); - } else if (len >= 4) { - // Same idea with two overlapping 4-byte loads. - uint64_t mm = escape_mask(escape_load4x2(src, src + len - 4)); - constexpr uint64_t low4 = (uint64_t(1) << (escape_mask_bits * 4)) - 1; - m = (mm & low4) | (((mm >> (escape_mask_bits * 4)) & low4) - << (escape_mask_bits * (len - 4))); - } else { - // Fewer than 4 bytes: at most three table lookups, no need for SIMD. - for (size_t k = 0; k < len; k++) { - uint8_t c = src[k]; - if (json_quotable_character[c]) { - escape_json_char(char(c), out); - } else { - *out++ = char(c); - } - } - return size_t(out - initout); - } - if (m == 0) { - copy_lt16(out, src + i, rem); - out += rem; - } else { - out = escape_block(src, out, i, len, m); - } - } - return size_t(out - initout); -} - -#else // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - -// Writes the escaped version of input to out, returning the number of bytes -// written. Uses SIMD position finding to locate quotable characters efficiently. -inline size_t write_string_escaped(const std::string_view input, char *out) { - size_t mysize = input.size(); - - // Use SIMD position finder directly - it returns mysize if no escape needed - size_t location = find_next_json_quotable_character(input, 0); - if (location == mysize) { - // Fast path: no escaping needed - memcpy(out, input.data(), input.size()); - return input.size(); - } - - const char *const initout = out; - memcpy(out, input.data(), location); - out += location; - escape_json_char(input[location], out); - location += 1; - while (location < mysize) { - size_t newlocation = find_next_json_quotable_character(input, location); - memcpy(out, input.data() + location, newlocation - location); - out += newlocation - location; - location = newlocation; - if (location == mysize) { - break; - } - escape_json_char(input[location], out); - location += 1; - } - return size_t(out - initout); -} - -#endif // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE -#undef SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - namespace internal { // Sets the size of s to n without initializing the new bytes when the @@ -47859,6 +47993,7 @@ simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ namespace simdjson { @@ -47895,6 +48030,9 @@ struct has_custom_serialization : std::false_type {}; namespace fallback { namespace builder { + +using simdjson::internal::escape_best::write_string_escaped; + /** * A builder for JSON strings representing documents. This is a low-level * builder that is not meant to be used directly by end-users. Though it @@ -48241,6 +48379,7 @@ simdjson_warn_unused error_code to_json(const Z &z, std::string &s, size_t initi /* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ /* amalgamation skipped (editor-only): #include "simdjson/concepts.h" */ /* amalgamation skipped (editor-only): #include "simdjson/annotations.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ #if SIMDJSON_STATIC_REFLECTION @@ -48266,7 +48405,6 @@ namespace internal { using simdjson::internal::write_uint_jeaiii; simdjson_inline char *write_double(char *p, double v) noexcept; } // namespace internal -simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out); SIMDJSON_PUSH_DISABLE_WARNINGS SIMDJSON_DISABLE_GCC_WARNING(-Warray-bounds) @@ -49287,619 +49425,13 @@ simdjson_warn_unused simdjson_result extract_fractured_json( /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_INL_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/jeaiii.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ -/* - * Empirically, we have found that an inlined optimization is important for - * performance. The following macros are not ideal. We should find a better - * way to inline the code. - */ - -#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \ - (defined(_M_AMD64) || defined(_M_X64) || \ - (defined(_M_IX86_FP) && _M_IX86_FP == 2)) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2 -#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1 -#endif -#endif - -#if defined(__aarch64__) || defined(_M_ARM64) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON -#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1 -#endif -#endif -#if defined(__loongarch_sx) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX -#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1 -#endif -#endif -#if defined(__loongarch_asx) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_LASX -#define SIMDJSON_EXPERIMENTAL_HAS_LASX 1 -#endif -#endif -#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \ - defined(__riscv_vector) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV -#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 -#endif -#endif -#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 -#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_NEON -#include -#ifdef _MSC_VER -#include -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 -#include -#if defined(__AVX2__) -#include -#endif -#ifdef _MSC_VER -#include -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_LSX -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_LASX -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_RVV -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_PPC64 -#include -#ifdef bool -#undef bool -#endif -#ifdef vector -#undef vector -#endif -#endif - - namespace simdjson { namespace fallback { namespace builder { -static SIMDJSON_CONSTEXPR_LAMBDA std::array - json_quotable_character = { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - -/** - -A possible SWAR implementation of has_json_escapable_byte. It is not used -because it is slower than the current implementation. It is kept here for -reference (to show that we tried it). - -inline bool has_json_escapable_byte(uint64_t x) { - uint64_t is_ascii = 0x8080808080808080ULL & ~x; - uint64_t xor2 = x ^ 0x0202020202020202ULL; - uint64_t lt32_or_eq34 = xor2 - 0x2121212121212121ULL; - uint64_t sub92 = x ^ 0x5C5C5C5C5C5C5C5CULL; - uint64_t eq92 = (sub92 - 0x0101010101010101ULL); - return ((lt32_or_eq34 | eq92) & is_ascii) != 0; -} - -**/ - -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 || SIMDJSON_EXPERIMENTAL_HAS_NEON -#define SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE 1 -#endif - -// With block escaping (SSE2/NEON), write_string_escaped scans the input itself. -#if !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE -// Scalar fallback for finding next quotable character -SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t -find_next_json_quotable_character_scalar(const std::string_view view, - size_t location) noexcept { - for (auto pos = view.begin() + location; pos != view.end(); ++pos) { - if (json_quotable_character[static_cast(*pos)]) { - return pos - view.begin(); - } - } - return size_t(view.size()); -} - -// SIMD-accelerated position finding that directly locates the first quotable -// character, combining detection and position extraction in a single pass to -// minimize redundant work. -#if SIMDJSON_EXPERIMENTAL_HAS_LASX -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - // SIMD constants for characters requiring escape - __m256i v34 = __lasx_xvreplgr2vr_b(34); // '"' - __m256i v92 = __lasx_xvreplgr2vr_b(92); // '\\' - __m256i v32 = __lasx_xvreplgr2vr_b(32); // control char threshold - - while (remaining >= 32) { - __m256i word = __lasx_xvld(ptr, 0); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - __m256i needs_escape = __lasx_xvseq_b(word, v34); - needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvseq_b(word, v92)); - needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvslt_bu(word, v32)); - - if (!__lasx_xbz_v(needs_escape)) { - // Found a quotable character - locate it via the four 64-bit lanes - uint64_t lane0 = __lasx_xvpickve2gr_du(needs_escape, 0); - uint64_t lane1 = __lasx_xvpickve2gr_du(needs_escape, 1); - uint64_t lane2 = __lasx_xvpickve2gr_du(needs_escape, 2); - uint64_t lane3 = __lasx_xvpickve2gr_du(needs_escape, 3); - size_t offset = ptr - reinterpret_cast(view.data()); - if (lane0 != 0) { - return offset + trailing_zeroes(lane0) / 8; - } else if (lane1 != 0) { - return offset + 8 + trailing_zeroes(lane1) / 8; - } else if (lane2 != 0) { - return offset + 16 + trailing_zeroes(lane2) / 8; - } else { - return offset + 24 + trailing_zeroes(lane3) / 8; - } - } - ptr += 32; - remaining -= 32; - } - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -#elif SIMDJSON_EXPERIMENTAL_HAS_LSX -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - //SIMD constants for characters requiring escape - __m128i v34 = __lsx_vreplgr2vr_b(34); // '"' - __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\' - __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold - - while (remaining >= 16){ - __m128i word = __lsx_vld(ptr, 0); - - //Check for the quotable characters: '"', '\\', or control char (<32) - __m128i needs_escape = __lsx_vseq_b(word, v34); - needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92)); - needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32)); - - if (!__lsx_bz_v(needs_escape)){ - - //Found quotable character - extract exact byte position - uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0); - uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1); - size_t offset = ptr - reinterpret_cast(view.data()); - if ( lo != 0) { - return offset + trailing_zeroes(lo) / 8; - } else { - return offset + 8 + trailing_zeroes(hi) / 8; - } - } - ptr += 16; - remaining -= 16; - } - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -#elif SIMDJSON_EXPERIMENTAL_HAS_RVV -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - while (remaining > 0) { - size_t vl = __riscv_vsetvl_e8m1(remaining); - vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl); - needs_escape = __riscv_vmor(needs_escape, - __riscv_vmseq(word, (uint8_t)92, vl), vl); - needs_escape = __riscv_vmor(needs_escape, - __riscv_vmsltu(word, (uint8_t)32, vl), vl); - - long first = __riscv_vfirst(needs_escape, vl); - if (first >= 0) { - size_t offset = ptr - reinterpret_cast(view.data()); - return offset + first; - } - ptr += vl; - remaining -= vl; - } - - return len; -} -#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64 -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - // SIMD constants for characters requiring escape - __vector unsigned char v34 = vec_splats((unsigned char)34); // '"' - __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\' - __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold - - // Bitmask for vec_vbpermq to extract one bit per byte - const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50, - 0x48, 0x40, 0x38, 0x30, 0x28, 0x20, - 0x18, 0x10, 0x08, 0x00}; - - while (remaining >= 16) { - __vector unsigned char word = - vec_vsx_ld(0, reinterpret_cast(ptr)); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - __vector unsigned char needs_escape = - (__vector unsigned char)vec_cmpeq(word, v34); - needs_escape = vec_or(needs_escape, - (__vector unsigned char)vec_cmpeq(word, v92)); - needs_escape = vec_or(needs_escape, - (__vector unsigned char)vec_cmplt(word, v32)); - - __vector unsigned long long result = - (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask); -#ifdef __LITTLE_ENDIAN__ - unsigned int mask = static_cast(result[1]); -#else - unsigned int mask = static_cast(result[0]); -#endif - if (mask != 0) { - size_t offset = ptr - reinterpret_cast(view.data()); - return offset + __builtin_ctz(mask); - } - ptr += 16; - remaining -= 16; - } - - // Scalar fallback for remaining bytes - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -#else -SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - return find_next_json_quotable_character_scalar(view, location); -} -#endif -#endif // !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - -SIMDJSON_CONSTEXPR_LAMBDA static std::string_view control_chars[] = { - "\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", "\\u0005", "\\u0006", - "\\u0007", "\\b", "\\t", "\\n", "\\u000b", "\\f", "\\r", - "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012", "\\u0013", "\\u0014", - "\\u0015", "\\u0016", "\\u0017", "\\u0018", "\\u0019", "\\u001a", "\\u001b", - "\\u001c", "\\u001d", "\\u001e", "\\u001f"}; - -// All Unicode characters may be placed within the quotation marks, except for -// the characters that MUST be escaped: quotation mark, reverse solidus, and the -// control characters (U+0000 through U+001F). There are two-character sequence -// escape representations of some popular characters: -// \", \\, \b, \f, \n, \r, \t. -SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline void escape_json_char(char c, char *&out) { - if (c == '"') { - memcpy(out, "\\\"", 2); - out += 2; - } else if (c == '\\') { - memcpy(out, "\\\\", 2); - out += 2; - } else { - std::string_view v = control_chars[uint8_t(c)]; - memcpy(out, v.data(), v.size()); - out += v.size(); - } -} - -#if SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 - -using escape_vector = __m128i; -// Mask bits that each input byte contributes to escape_bitmask(). -static constexpr unsigned escape_mask_bits = 1; - -simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept { - return _mm_loadu_si128(reinterpret_cast(p)); -} - -simdjson_inline void escape_store16(char *out, escape_vector v) noexcept { - _mm_storeu_si128(reinterpret_cast<__m128i *>(out), v); -} - -// Builds a vector whose bytes 0..7 come from a and bytes 8..15 from b. -simdjson_inline escape_vector escape_load8x2(const uint8_t *a, - const uint8_t *b) noexcept { - return _mm_unpacklo_epi64( - _mm_loadl_epi64(reinterpret_cast(a)), - _mm_loadl_epi64(reinterpret_cast(b))); -} - -// Builds a vector whose bytes 0..3 come from a and bytes 4..7 from b. The -// upper half is unspecified; callers only look at the low eight bits. -simdjson_inline escape_vector escape_load4x2(const uint8_t *a, - const uint8_t *b) noexcept { - int32_t a32, b32; - memcpy(&a32, a, 4); - memcpy(&b32, b, 4); - return _mm_unpacklo_epi32(_mm_cvtsi32_si128(a32), _mm_cvtsi32_si128(b32)); -} - -// Sets every bit of byte k when byte k of v is a quotable character ('"', -// '\\' or a control character). -simdjson_inline escape_vector escape_flags(escape_vector v) noexcept { - const __m128i v34 = _mm_set1_epi8(34); // '"' - const __m128i v92 = _mm_set1_epi8(92); // '\\' - const __m128i v31 = _mm_set1_epi8(31); // for control char detection - __m128i needs_escape = _mm_cmpeq_epi8(v, v34); - needs_escape = _mm_or_si128(needs_escape, _mm_cmpeq_epi8(v, v92)); - return _mm_or_si128( - needs_escape, _mm_cmpeq_epi8(_mm_subs_epu8(v, v31), _mm_setzero_si128())); -} - -// True when any byte needs escaping. Kept separate from escape_bitmask -// because some instruction sets can answer it without leaving the vector -// register file; on SSE2 the compiler folds the two together. -simdjson_inline bool escape_any(escape_vector flags) noexcept { - return _mm_movemask_epi8(flags) != 0; -} - -// A 16-bit mask with bit k set when byte k needed escaping. -simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept { - return uint64_t(uint32_t(_mm_movemask_epi8(flags))); -} - -#else // SIMDJSON_EXPERIMENTAL_HAS_NEON - -using escape_vector = uint8x16_t; -static constexpr unsigned escape_mask_bits = 4; - -simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept { - return vld1q_u8(p); -} - -simdjson_inline void escape_store16(char *out, escape_vector v) noexcept { - vst1q_u8(reinterpret_cast(out), v); -} - -simdjson_inline escape_vector escape_load8x2(const uint8_t *a, - const uint8_t *b) noexcept { - uint64_t a64, b64; - memcpy(&a64, a, 8); - memcpy(&b64, b, 8); - return vreinterpretq_u8_u64(vsetq_lane_u64(b64, vdupq_n_u64(a64), 1)); -} - -simdjson_inline escape_vector escape_load4x2(const uint8_t *a, - const uint8_t *b) noexcept { - uint32_t a32, b32; - memcpy(&a32, a, 4); - memcpy(&b32, b, 4); - return vreinterpretq_u8_u32(vsetq_lane_u32(b32, vdupq_n_u32(a32), 1)); -} - -simdjson_inline escape_vector escape_flags(escape_vector v) noexcept { - uint8x16_t needs_escape = vceqq_u8(v, vdupq_n_u8(34)); // '"' - needs_escape = vorrq_u8(needs_escape, vceqq_u8(v, vdupq_n_u8(92))); // '\\' - return vorrq_u8(needs_escape, vcltq_u8(v, vdupq_n_u8(32))); -} - -simdjson_inline bool escape_any(escape_vector flags) noexcept { - return vmaxvq_u32(vreinterpretq_u32_u8(flags)) != 0; -} - -// Four bits per byte rather than one: escape_block and the tail paths scale -// their shifts by escape_mask_bits to match. -simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept { - uint8x8_t narrowed = vshrn_n_u16(vreinterpretq_u16_u8(flags), 4); - return vget_lane_u64(vreinterpret_u64_u8(narrowed), 0); -} - -#endif // instruction set selection - -// The escape bitmask of a 16-byte block. -simdjson_inline uint64_t escape_mask(escape_vector v) noexcept { - return escape_bitmask(escape_flags(v)); -} - -// Copies n bytes with n < 16, using overlapping loads and stores. It never -// reads more than n bytes from src, nor writes more than n bytes to dst. -simdjson_inline void copy_lt16(char *dst, const uint8_t *src, - size_t n) noexcept { - if (n >= 8) { - memcpy(dst, src, 8); - memcpy(dst + n - 8, src + n - 8, 8); - } else if (n >= 4) { - memcpy(dst, src, 4); - memcpy(dst + n - 4, src + n - 4, 4); - } else if (n > 0) { - dst[0] = char(src[0]); - dst[n >> 1] = char(src[n >> 1]); - dst[n - 1] = char(src[n - 1]); - } -} - -// Escapes the bytes of src in the range [i, blockend), given that m is the -// (non-zero) escape mask for that range: the escape_mask_bits-wide lane of m -// at byte k is non-zero when src[i + k] requires escaping. Returns the updated -// output pointer. -simdjson_never_inline char *escape_block(const uint8_t *src, char *out, - size_t i, size_t blockend, - uint64_t m) noexcept { - constexpr uint64_t lane = (uint64_t(1) << escape_mask_bits) - 1; - - size_t pos = i; // first byte not yet copied - while (m) { - const size_t tz = trailing_zeroes(m); - const size_t next = i + tz / escape_mask_bits; - // Copy the run of safe bytes that precedes this escape. - copy_lt16(out, src + pos, next - pos); - out += next - pos; - escape_json_char(char(src[next]), out); - pos = next + 1; - m &= ~(lane << tz); - } - // Copy whatever follows the last escape. - copy_lt16(out, src + pos, blockend - pos); - return out + (blockend - pos); -} - -// Writes the escaped version of input to out, returning the number of bytes -// written. -simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out) { - const size_t len = input.size(); - const uint8_t *src = reinterpret_cast(input.data()); - const char *const initout = out; - - size_t i = 0; -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 && defined(__AVX2__) - while (i + 32 <= len) { - const __m256i word = _mm256_loadu_si256(reinterpret_cast(src + i)); - const __m256i flags = _mm256_or_si256( - _mm256_or_si256(_mm256_cmpeq_epi8(word, _mm256_set1_epi8(34)), // '"' - _mm256_cmpeq_epi8(word, _mm256_set1_epi8(92))), // '\\' - _mm256_cmpeq_epi8(_mm256_subs_epu8(word, _mm256_set1_epi8(31)), - _mm256_setzero_si256())); // control - const uint32_t mask = uint32_t(_mm256_movemask_epi8(flags)); - if (simdjson_likely(mask == 0)) { - _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), word); - out += 32; - } else { - for (size_t half = 0; half < 32; half += 16) { - const uint64_t m = (mask >> half) & 0xFFFF; - if (m == 0) { - escape_store16(out, escape_load16(src + i + half)); - out += 16; - } else { - out = escape_block(src, out, i + half, i + half + 16, m); - } - } - } - i += 32; - } -#endif - while (i + 16 <= len) { - escape_vector word = escape_load16(src + i); - escape_vector flags = escape_flags(word); - if (simdjson_likely(!escape_any(flags))) { - escape_store16(out, word); - out += 16; - } else { - out = escape_block(src, out, i, i + 16, escape_bitmask(flags)); - } - i += 16; - } - if (i < len) { - const size_t rem = len - i; - uint64_t m; - if (len >= 16) { - // The last 16 bytes of the input are in bounds. Bit k of that block's - // mask belongs to input position len - 16 + k, so shift it down to align - // bit 0 with position i. - m = escape_mask(escape_load16(src + len - 16)) >> - (escape_mask_bits * (16 - rem)); - } else if (len >= 8) { - // Here i == 0 and rem == len. Two overlapping 8-byte loads cover - // [0, 8) and [len - 8, len), which is the whole input since len < 16. - uint64_t mm = escape_mask(escape_load8x2(src, src + len - 8)); - constexpr uint64_t low8 = (uint64_t(1) << (escape_mask_bits * 8)) - 1; - m = (mm & low8) | - ((mm >> (escape_mask_bits * 8)) << (escape_mask_bits * (len - 8))); - } else if (len >= 4) { - // Same idea with two overlapping 4-byte loads. - uint64_t mm = escape_mask(escape_load4x2(src, src + len - 4)); - constexpr uint64_t low4 = (uint64_t(1) << (escape_mask_bits * 4)) - 1; - m = (mm & low4) | (((mm >> (escape_mask_bits * 4)) & low4) - << (escape_mask_bits * (len - 4))); - } else { - // Fewer than 4 bytes: at most three table lookups, no need for SIMD. - for (size_t k = 0; k < len; k++) { - uint8_t c = src[k]; - if (json_quotable_character[c]) { - escape_json_char(char(c), out); - } else { - *out++ = char(c); - } - } - return size_t(out - initout); - } - if (m == 0) { - copy_lt16(out, src + i, rem); - out += rem; - } else { - out = escape_block(src, out, i, len, m); - } - } - return size_t(out - initout); -} - -#else // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - -// Writes the escaped version of input to out, returning the number of bytes -// written. Uses SIMD position finding to locate quotable characters efficiently. -inline size_t write_string_escaped(const std::string_view input, char *out) { - size_t mysize = input.size(); - - // Use SIMD position finder directly - it returns mysize if no escape needed - size_t location = find_next_json_quotable_character(input, 0); - if (location == mysize) { - // Fast path: no escaping needed - memcpy(out, input.data(), input.size()); - return input.size(); - } - - const char *const initout = out; - memcpy(out, input.data(), location); - out += location; - escape_json_char(input[location], out); - location += 1; - while (location < mysize) { - size_t newlocation = find_next_json_quotable_character(input, location); - memcpy(out, input.data() + location, newlocation - location); - out += newlocation - location; - location = newlocation; - if (location == mysize) { - break; - } - escape_json_char(input[location], out); - location += 1; - } - return size_t(out - initout); -} - -#endif // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE -#undef SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - namespace internal { // Sets the size of s to n without initializing the new bytes when the @@ -51224,6 +50756,7 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ namespace simdjson { @@ -51260,6 +50793,9 @@ struct has_custom_serialization : std::false_type {}; namespace haswell { namespace builder { + +using simdjson::internal::escape_best::write_string_escaped; + /** * A builder for JSON strings representing documents. This is a low-level * builder that is not meant to be used directly by end-users. Though it @@ -51606,6 +51142,7 @@ simdjson_warn_unused error_code to_json(const Z &z, std::string &s, size_t initi /* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ /* amalgamation skipped (editor-only): #include "simdjson/concepts.h" */ /* amalgamation skipped (editor-only): #include "simdjson/annotations.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ #if SIMDJSON_STATIC_REFLECTION @@ -51631,7 +51168,6 @@ namespace internal { using simdjson::internal::write_uint_jeaiii; simdjson_inline char *write_double(char *p, double v) noexcept; } // namespace internal -simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out); SIMDJSON_PUSH_DISABLE_WARNINGS SIMDJSON_DISABLE_GCC_WARNING(-Warray-bounds) @@ -52652,619 +52188,13 @@ simdjson_warn_unused simdjson_result extract_fractured_json( /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_INL_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/jeaiii.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ -/* - * Empirically, we have found that an inlined optimization is important for - * performance. The following macros are not ideal. We should find a better - * way to inline the code. - */ - -#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \ - (defined(_M_AMD64) || defined(_M_X64) || \ - (defined(_M_IX86_FP) && _M_IX86_FP == 2)) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2 -#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1 -#endif -#endif - -#if defined(__aarch64__) || defined(_M_ARM64) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON -#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1 -#endif -#endif -#if defined(__loongarch_sx) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX -#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1 -#endif -#endif -#if defined(__loongarch_asx) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_LASX -#define SIMDJSON_EXPERIMENTAL_HAS_LASX 1 -#endif -#endif -#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \ - defined(__riscv_vector) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV -#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 -#endif -#endif -#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 -#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_NEON -#include -#ifdef _MSC_VER -#include -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 -#include -#if defined(__AVX2__) -#include -#endif -#ifdef _MSC_VER -#include -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_LSX -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_LASX -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_RVV -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_PPC64 -#include -#ifdef bool -#undef bool -#endif -#ifdef vector -#undef vector -#endif -#endif - - namespace simdjson { namespace haswell { namespace builder { -static SIMDJSON_CONSTEXPR_LAMBDA std::array - json_quotable_character = { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - -/** - -A possible SWAR implementation of has_json_escapable_byte. It is not used -because it is slower than the current implementation. It is kept here for -reference (to show that we tried it). - -inline bool has_json_escapable_byte(uint64_t x) { - uint64_t is_ascii = 0x8080808080808080ULL & ~x; - uint64_t xor2 = x ^ 0x0202020202020202ULL; - uint64_t lt32_or_eq34 = xor2 - 0x2121212121212121ULL; - uint64_t sub92 = x ^ 0x5C5C5C5C5C5C5C5CULL; - uint64_t eq92 = (sub92 - 0x0101010101010101ULL); - return ((lt32_or_eq34 | eq92) & is_ascii) != 0; -} - -**/ - -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 || SIMDJSON_EXPERIMENTAL_HAS_NEON -#define SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE 1 -#endif - -// With block escaping (SSE2/NEON), write_string_escaped scans the input itself. -#if !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE -// Scalar fallback for finding next quotable character -SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t -find_next_json_quotable_character_scalar(const std::string_view view, - size_t location) noexcept { - for (auto pos = view.begin() + location; pos != view.end(); ++pos) { - if (json_quotable_character[static_cast(*pos)]) { - return pos - view.begin(); - } - } - return size_t(view.size()); -} - -// SIMD-accelerated position finding that directly locates the first quotable -// character, combining detection and position extraction in a single pass to -// minimize redundant work. -#if SIMDJSON_EXPERIMENTAL_HAS_LASX -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - // SIMD constants for characters requiring escape - __m256i v34 = __lasx_xvreplgr2vr_b(34); // '"' - __m256i v92 = __lasx_xvreplgr2vr_b(92); // '\\' - __m256i v32 = __lasx_xvreplgr2vr_b(32); // control char threshold - - while (remaining >= 32) { - __m256i word = __lasx_xvld(ptr, 0); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - __m256i needs_escape = __lasx_xvseq_b(word, v34); - needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvseq_b(word, v92)); - needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvslt_bu(word, v32)); - - if (!__lasx_xbz_v(needs_escape)) { - // Found a quotable character - locate it via the four 64-bit lanes - uint64_t lane0 = __lasx_xvpickve2gr_du(needs_escape, 0); - uint64_t lane1 = __lasx_xvpickve2gr_du(needs_escape, 1); - uint64_t lane2 = __lasx_xvpickve2gr_du(needs_escape, 2); - uint64_t lane3 = __lasx_xvpickve2gr_du(needs_escape, 3); - size_t offset = ptr - reinterpret_cast(view.data()); - if (lane0 != 0) { - return offset + trailing_zeroes(lane0) / 8; - } else if (lane1 != 0) { - return offset + 8 + trailing_zeroes(lane1) / 8; - } else if (lane2 != 0) { - return offset + 16 + trailing_zeroes(lane2) / 8; - } else { - return offset + 24 + trailing_zeroes(lane3) / 8; - } - } - ptr += 32; - remaining -= 32; - } - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -#elif SIMDJSON_EXPERIMENTAL_HAS_LSX -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - //SIMD constants for characters requiring escape - __m128i v34 = __lsx_vreplgr2vr_b(34); // '"' - __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\' - __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold - - while (remaining >= 16){ - __m128i word = __lsx_vld(ptr, 0); - - //Check for the quotable characters: '"', '\\', or control char (<32) - __m128i needs_escape = __lsx_vseq_b(word, v34); - needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92)); - needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32)); - - if (!__lsx_bz_v(needs_escape)){ - - //Found quotable character - extract exact byte position - uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0); - uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1); - size_t offset = ptr - reinterpret_cast(view.data()); - if ( lo != 0) { - return offset + trailing_zeroes(lo) / 8; - } else { - return offset + 8 + trailing_zeroes(hi) / 8; - } - } - ptr += 16; - remaining -= 16; - } - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -#elif SIMDJSON_EXPERIMENTAL_HAS_RVV -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - while (remaining > 0) { - size_t vl = __riscv_vsetvl_e8m1(remaining); - vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl); - needs_escape = __riscv_vmor(needs_escape, - __riscv_vmseq(word, (uint8_t)92, vl), vl); - needs_escape = __riscv_vmor(needs_escape, - __riscv_vmsltu(word, (uint8_t)32, vl), vl); - - long first = __riscv_vfirst(needs_escape, vl); - if (first >= 0) { - size_t offset = ptr - reinterpret_cast(view.data()); - return offset + first; - } - ptr += vl; - remaining -= vl; - } - - return len; -} -#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64 -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - // SIMD constants for characters requiring escape - __vector unsigned char v34 = vec_splats((unsigned char)34); // '"' - __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\' - __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold - - // Bitmask for vec_vbpermq to extract one bit per byte - const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50, - 0x48, 0x40, 0x38, 0x30, 0x28, 0x20, - 0x18, 0x10, 0x08, 0x00}; - - while (remaining >= 16) { - __vector unsigned char word = - vec_vsx_ld(0, reinterpret_cast(ptr)); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - __vector unsigned char needs_escape = - (__vector unsigned char)vec_cmpeq(word, v34); - needs_escape = vec_or(needs_escape, - (__vector unsigned char)vec_cmpeq(word, v92)); - needs_escape = vec_or(needs_escape, - (__vector unsigned char)vec_cmplt(word, v32)); - - __vector unsigned long long result = - (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask); -#ifdef __LITTLE_ENDIAN__ - unsigned int mask = static_cast(result[1]); -#else - unsigned int mask = static_cast(result[0]); -#endif - if (mask != 0) { - size_t offset = ptr - reinterpret_cast(view.data()); - return offset + __builtin_ctz(mask); - } - ptr += 16; - remaining -= 16; - } - - // Scalar fallback for remaining bytes - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -#else -SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - return find_next_json_quotable_character_scalar(view, location); -} -#endif -#endif // !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - -SIMDJSON_CONSTEXPR_LAMBDA static std::string_view control_chars[] = { - "\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", "\\u0005", "\\u0006", - "\\u0007", "\\b", "\\t", "\\n", "\\u000b", "\\f", "\\r", - "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012", "\\u0013", "\\u0014", - "\\u0015", "\\u0016", "\\u0017", "\\u0018", "\\u0019", "\\u001a", "\\u001b", - "\\u001c", "\\u001d", "\\u001e", "\\u001f"}; - -// All Unicode characters may be placed within the quotation marks, except for -// the characters that MUST be escaped: quotation mark, reverse solidus, and the -// control characters (U+0000 through U+001F). There are two-character sequence -// escape representations of some popular characters: -// \", \\, \b, \f, \n, \r, \t. -SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline void escape_json_char(char c, char *&out) { - if (c == '"') { - memcpy(out, "\\\"", 2); - out += 2; - } else if (c == '\\') { - memcpy(out, "\\\\", 2); - out += 2; - } else { - std::string_view v = control_chars[uint8_t(c)]; - memcpy(out, v.data(), v.size()); - out += v.size(); - } -} - -#if SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 - -using escape_vector = __m128i; -// Mask bits that each input byte contributes to escape_bitmask(). -static constexpr unsigned escape_mask_bits = 1; - -simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept { - return _mm_loadu_si128(reinterpret_cast(p)); -} - -simdjson_inline void escape_store16(char *out, escape_vector v) noexcept { - _mm_storeu_si128(reinterpret_cast<__m128i *>(out), v); -} - -// Builds a vector whose bytes 0..7 come from a and bytes 8..15 from b. -simdjson_inline escape_vector escape_load8x2(const uint8_t *a, - const uint8_t *b) noexcept { - return _mm_unpacklo_epi64( - _mm_loadl_epi64(reinterpret_cast(a)), - _mm_loadl_epi64(reinterpret_cast(b))); -} - -// Builds a vector whose bytes 0..3 come from a and bytes 4..7 from b. The -// upper half is unspecified; callers only look at the low eight bits. -simdjson_inline escape_vector escape_load4x2(const uint8_t *a, - const uint8_t *b) noexcept { - int32_t a32, b32; - memcpy(&a32, a, 4); - memcpy(&b32, b, 4); - return _mm_unpacklo_epi32(_mm_cvtsi32_si128(a32), _mm_cvtsi32_si128(b32)); -} - -// Sets every bit of byte k when byte k of v is a quotable character ('"', -// '\\' or a control character). -simdjson_inline escape_vector escape_flags(escape_vector v) noexcept { - const __m128i v34 = _mm_set1_epi8(34); // '"' - const __m128i v92 = _mm_set1_epi8(92); // '\\' - const __m128i v31 = _mm_set1_epi8(31); // for control char detection - __m128i needs_escape = _mm_cmpeq_epi8(v, v34); - needs_escape = _mm_or_si128(needs_escape, _mm_cmpeq_epi8(v, v92)); - return _mm_or_si128( - needs_escape, _mm_cmpeq_epi8(_mm_subs_epu8(v, v31), _mm_setzero_si128())); -} - -// True when any byte needs escaping. Kept separate from escape_bitmask -// because some instruction sets can answer it without leaving the vector -// register file; on SSE2 the compiler folds the two together. -simdjson_inline bool escape_any(escape_vector flags) noexcept { - return _mm_movemask_epi8(flags) != 0; -} - -// A 16-bit mask with bit k set when byte k needed escaping. -simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept { - return uint64_t(uint32_t(_mm_movemask_epi8(flags))); -} - -#else // SIMDJSON_EXPERIMENTAL_HAS_NEON - -using escape_vector = uint8x16_t; -static constexpr unsigned escape_mask_bits = 4; - -simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept { - return vld1q_u8(p); -} - -simdjson_inline void escape_store16(char *out, escape_vector v) noexcept { - vst1q_u8(reinterpret_cast(out), v); -} - -simdjson_inline escape_vector escape_load8x2(const uint8_t *a, - const uint8_t *b) noexcept { - uint64_t a64, b64; - memcpy(&a64, a, 8); - memcpy(&b64, b, 8); - return vreinterpretq_u8_u64(vsetq_lane_u64(b64, vdupq_n_u64(a64), 1)); -} - -simdjson_inline escape_vector escape_load4x2(const uint8_t *a, - const uint8_t *b) noexcept { - uint32_t a32, b32; - memcpy(&a32, a, 4); - memcpy(&b32, b, 4); - return vreinterpretq_u8_u32(vsetq_lane_u32(b32, vdupq_n_u32(a32), 1)); -} - -simdjson_inline escape_vector escape_flags(escape_vector v) noexcept { - uint8x16_t needs_escape = vceqq_u8(v, vdupq_n_u8(34)); // '"' - needs_escape = vorrq_u8(needs_escape, vceqq_u8(v, vdupq_n_u8(92))); // '\\' - return vorrq_u8(needs_escape, vcltq_u8(v, vdupq_n_u8(32))); -} - -simdjson_inline bool escape_any(escape_vector flags) noexcept { - return vmaxvq_u32(vreinterpretq_u32_u8(flags)) != 0; -} - -// Four bits per byte rather than one: escape_block and the tail paths scale -// their shifts by escape_mask_bits to match. -simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept { - uint8x8_t narrowed = vshrn_n_u16(vreinterpretq_u16_u8(flags), 4); - return vget_lane_u64(vreinterpret_u64_u8(narrowed), 0); -} - -#endif // instruction set selection - -// The escape bitmask of a 16-byte block. -simdjson_inline uint64_t escape_mask(escape_vector v) noexcept { - return escape_bitmask(escape_flags(v)); -} - -// Copies n bytes with n < 16, using overlapping loads and stores. It never -// reads more than n bytes from src, nor writes more than n bytes to dst. -simdjson_inline void copy_lt16(char *dst, const uint8_t *src, - size_t n) noexcept { - if (n >= 8) { - memcpy(dst, src, 8); - memcpy(dst + n - 8, src + n - 8, 8); - } else if (n >= 4) { - memcpy(dst, src, 4); - memcpy(dst + n - 4, src + n - 4, 4); - } else if (n > 0) { - dst[0] = char(src[0]); - dst[n >> 1] = char(src[n >> 1]); - dst[n - 1] = char(src[n - 1]); - } -} - -// Escapes the bytes of src in the range [i, blockend), given that m is the -// (non-zero) escape mask for that range: the escape_mask_bits-wide lane of m -// at byte k is non-zero when src[i + k] requires escaping. Returns the updated -// output pointer. -simdjson_never_inline char *escape_block(const uint8_t *src, char *out, - size_t i, size_t blockend, - uint64_t m) noexcept { - constexpr uint64_t lane = (uint64_t(1) << escape_mask_bits) - 1; - - size_t pos = i; // first byte not yet copied - while (m) { - const size_t tz = trailing_zeroes(m); - const size_t next = i + tz / escape_mask_bits; - // Copy the run of safe bytes that precedes this escape. - copy_lt16(out, src + pos, next - pos); - out += next - pos; - escape_json_char(char(src[next]), out); - pos = next + 1; - m &= ~(lane << tz); - } - // Copy whatever follows the last escape. - copy_lt16(out, src + pos, blockend - pos); - return out + (blockend - pos); -} - -// Writes the escaped version of input to out, returning the number of bytes -// written. -simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out) { - const size_t len = input.size(); - const uint8_t *src = reinterpret_cast(input.data()); - const char *const initout = out; - - size_t i = 0; -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 && defined(__AVX2__) - while (i + 32 <= len) { - const __m256i word = _mm256_loadu_si256(reinterpret_cast(src + i)); - const __m256i flags = _mm256_or_si256( - _mm256_or_si256(_mm256_cmpeq_epi8(word, _mm256_set1_epi8(34)), // '"' - _mm256_cmpeq_epi8(word, _mm256_set1_epi8(92))), // '\\' - _mm256_cmpeq_epi8(_mm256_subs_epu8(word, _mm256_set1_epi8(31)), - _mm256_setzero_si256())); // control - const uint32_t mask = uint32_t(_mm256_movemask_epi8(flags)); - if (simdjson_likely(mask == 0)) { - _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), word); - out += 32; - } else { - for (size_t half = 0; half < 32; half += 16) { - const uint64_t m = (mask >> half) & 0xFFFF; - if (m == 0) { - escape_store16(out, escape_load16(src + i + half)); - out += 16; - } else { - out = escape_block(src, out, i + half, i + half + 16, m); - } - } - } - i += 32; - } -#endif - while (i + 16 <= len) { - escape_vector word = escape_load16(src + i); - escape_vector flags = escape_flags(word); - if (simdjson_likely(!escape_any(flags))) { - escape_store16(out, word); - out += 16; - } else { - out = escape_block(src, out, i, i + 16, escape_bitmask(flags)); - } - i += 16; - } - if (i < len) { - const size_t rem = len - i; - uint64_t m; - if (len >= 16) { - // The last 16 bytes of the input are in bounds. Bit k of that block's - // mask belongs to input position len - 16 + k, so shift it down to align - // bit 0 with position i. - m = escape_mask(escape_load16(src + len - 16)) >> - (escape_mask_bits * (16 - rem)); - } else if (len >= 8) { - // Here i == 0 and rem == len. Two overlapping 8-byte loads cover - // [0, 8) and [len - 8, len), which is the whole input since len < 16. - uint64_t mm = escape_mask(escape_load8x2(src, src + len - 8)); - constexpr uint64_t low8 = (uint64_t(1) << (escape_mask_bits * 8)) - 1; - m = (mm & low8) | - ((mm >> (escape_mask_bits * 8)) << (escape_mask_bits * (len - 8))); - } else if (len >= 4) { - // Same idea with two overlapping 4-byte loads. - uint64_t mm = escape_mask(escape_load4x2(src, src + len - 4)); - constexpr uint64_t low4 = (uint64_t(1) << (escape_mask_bits * 4)) - 1; - m = (mm & low4) | (((mm >> (escape_mask_bits * 4)) & low4) - << (escape_mask_bits * (len - 4))); - } else { - // Fewer than 4 bytes: at most three table lookups, no need for SIMD. - for (size_t k = 0; k < len; k++) { - uint8_t c = src[k]; - if (json_quotable_character[c]) { - escape_json_char(char(c), out); - } else { - *out++ = char(c); - } - } - return size_t(out - initout); - } - if (m == 0) { - copy_lt16(out, src + i, rem); - out += rem; - } else { - out = escape_block(src, out, i, len, m); - } - } - return size_t(out - initout); -} - -#else // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - -// Writes the escaped version of input to out, returning the number of bytes -// written. Uses SIMD position finding to locate quotable characters efficiently. -inline size_t write_string_escaped(const std::string_view input, char *out) { - size_t mysize = input.size(); - - // Use SIMD position finder directly - it returns mysize if no escape needed - size_t location = find_next_json_quotable_character(input, 0); - if (location == mysize) { - // Fast path: no escaping needed - memcpy(out, input.data(), input.size()); - return input.size(); - } - - const char *const initout = out; - memcpy(out, input.data(), location); - out += location; - escape_json_char(input[location], out); - location += 1; - while (location < mysize) { - size_t newlocation = find_next_json_quotable_character(input, location); - memcpy(out, input.data() + location, newlocation - location); - out += newlocation - location; - location = newlocation; - if (location == mysize) { - break; - } - escape_json_char(input[location], out); - location += 1; - } - return size_t(out - initout); -} - -#endif // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE -#undef SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - namespace internal { // Sets the size of s to n without initializing the new bytes when the @@ -54585,6 +53515,7 @@ simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ namespace simdjson { @@ -54621,6 +53552,9 @@ struct has_custom_serialization : std::false_type {}; namespace icelake { namespace builder { + +using simdjson::internal::escape_best::write_string_escaped; + /** * A builder for JSON strings representing documents. This is a low-level * builder that is not meant to be used directly by end-users. Though it @@ -54967,6 +53901,7 @@ simdjson_warn_unused error_code to_json(const Z &z, std::string &s, size_t initi /* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ /* amalgamation skipped (editor-only): #include "simdjson/concepts.h" */ /* amalgamation skipped (editor-only): #include "simdjson/annotations.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ #if SIMDJSON_STATIC_REFLECTION @@ -54992,7 +53927,6 @@ namespace internal { using simdjson::internal::write_uint_jeaiii; simdjson_inline char *write_double(char *p, double v) noexcept; } // namespace internal -simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out); SIMDJSON_PUSH_DISABLE_WARNINGS SIMDJSON_DISABLE_GCC_WARNING(-Warray-bounds) @@ -56013,619 +54947,13 @@ simdjson_warn_unused simdjson_result extract_fractured_json( /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_INL_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/jeaiii.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ -/* - * Empirically, we have found that an inlined optimization is important for - * performance. The following macros are not ideal. We should find a better - * way to inline the code. - */ - -#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \ - (defined(_M_AMD64) || defined(_M_X64) || \ - (defined(_M_IX86_FP) && _M_IX86_FP == 2)) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2 -#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1 -#endif -#endif - -#if defined(__aarch64__) || defined(_M_ARM64) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON -#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1 -#endif -#endif -#if defined(__loongarch_sx) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX -#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1 -#endif -#endif -#if defined(__loongarch_asx) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_LASX -#define SIMDJSON_EXPERIMENTAL_HAS_LASX 1 -#endif -#endif -#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \ - defined(__riscv_vector) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV -#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 -#endif -#endif -#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 -#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_NEON -#include -#ifdef _MSC_VER -#include -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 -#include -#if defined(__AVX2__) -#include -#endif -#ifdef _MSC_VER -#include -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_LSX -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_LASX -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_RVV -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_PPC64 -#include -#ifdef bool -#undef bool -#endif -#ifdef vector -#undef vector -#endif -#endif - - namespace simdjson { namespace icelake { namespace builder { -static SIMDJSON_CONSTEXPR_LAMBDA std::array - json_quotable_character = { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - -/** - -A possible SWAR implementation of has_json_escapable_byte. It is not used -because it is slower than the current implementation. It is kept here for -reference (to show that we tried it). - -inline bool has_json_escapable_byte(uint64_t x) { - uint64_t is_ascii = 0x8080808080808080ULL & ~x; - uint64_t xor2 = x ^ 0x0202020202020202ULL; - uint64_t lt32_or_eq34 = xor2 - 0x2121212121212121ULL; - uint64_t sub92 = x ^ 0x5C5C5C5C5C5C5C5CULL; - uint64_t eq92 = (sub92 - 0x0101010101010101ULL); - return ((lt32_or_eq34 | eq92) & is_ascii) != 0; -} - -**/ - -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 || SIMDJSON_EXPERIMENTAL_HAS_NEON -#define SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE 1 -#endif - -// With block escaping (SSE2/NEON), write_string_escaped scans the input itself. -#if !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE -// Scalar fallback for finding next quotable character -SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t -find_next_json_quotable_character_scalar(const std::string_view view, - size_t location) noexcept { - for (auto pos = view.begin() + location; pos != view.end(); ++pos) { - if (json_quotable_character[static_cast(*pos)]) { - return pos - view.begin(); - } - } - return size_t(view.size()); -} - -// SIMD-accelerated position finding that directly locates the first quotable -// character, combining detection and position extraction in a single pass to -// minimize redundant work. -#if SIMDJSON_EXPERIMENTAL_HAS_LASX -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - // SIMD constants for characters requiring escape - __m256i v34 = __lasx_xvreplgr2vr_b(34); // '"' - __m256i v92 = __lasx_xvreplgr2vr_b(92); // '\\' - __m256i v32 = __lasx_xvreplgr2vr_b(32); // control char threshold - - while (remaining >= 32) { - __m256i word = __lasx_xvld(ptr, 0); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - __m256i needs_escape = __lasx_xvseq_b(word, v34); - needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvseq_b(word, v92)); - needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvslt_bu(word, v32)); - - if (!__lasx_xbz_v(needs_escape)) { - // Found a quotable character - locate it via the four 64-bit lanes - uint64_t lane0 = __lasx_xvpickve2gr_du(needs_escape, 0); - uint64_t lane1 = __lasx_xvpickve2gr_du(needs_escape, 1); - uint64_t lane2 = __lasx_xvpickve2gr_du(needs_escape, 2); - uint64_t lane3 = __lasx_xvpickve2gr_du(needs_escape, 3); - size_t offset = ptr - reinterpret_cast(view.data()); - if (lane0 != 0) { - return offset + trailing_zeroes(lane0) / 8; - } else if (lane1 != 0) { - return offset + 8 + trailing_zeroes(lane1) / 8; - } else if (lane2 != 0) { - return offset + 16 + trailing_zeroes(lane2) / 8; - } else { - return offset + 24 + trailing_zeroes(lane3) / 8; - } - } - ptr += 32; - remaining -= 32; - } - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -#elif SIMDJSON_EXPERIMENTAL_HAS_LSX -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - //SIMD constants for characters requiring escape - __m128i v34 = __lsx_vreplgr2vr_b(34); // '"' - __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\' - __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold - - while (remaining >= 16){ - __m128i word = __lsx_vld(ptr, 0); - - //Check for the quotable characters: '"', '\\', or control char (<32) - __m128i needs_escape = __lsx_vseq_b(word, v34); - needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92)); - needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32)); - - if (!__lsx_bz_v(needs_escape)){ - - //Found quotable character - extract exact byte position - uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0); - uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1); - size_t offset = ptr - reinterpret_cast(view.data()); - if ( lo != 0) { - return offset + trailing_zeroes(lo) / 8; - } else { - return offset + 8 + trailing_zeroes(hi) / 8; - } - } - ptr += 16; - remaining -= 16; - } - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -#elif SIMDJSON_EXPERIMENTAL_HAS_RVV -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - while (remaining > 0) { - size_t vl = __riscv_vsetvl_e8m1(remaining); - vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl); - needs_escape = __riscv_vmor(needs_escape, - __riscv_vmseq(word, (uint8_t)92, vl), vl); - needs_escape = __riscv_vmor(needs_escape, - __riscv_vmsltu(word, (uint8_t)32, vl), vl); - - long first = __riscv_vfirst(needs_escape, vl); - if (first >= 0) { - size_t offset = ptr - reinterpret_cast(view.data()); - return offset + first; - } - ptr += vl; - remaining -= vl; - } - - return len; -} -#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64 -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - // SIMD constants for characters requiring escape - __vector unsigned char v34 = vec_splats((unsigned char)34); // '"' - __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\' - __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold - - // Bitmask for vec_vbpermq to extract one bit per byte - const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50, - 0x48, 0x40, 0x38, 0x30, 0x28, 0x20, - 0x18, 0x10, 0x08, 0x00}; - - while (remaining >= 16) { - __vector unsigned char word = - vec_vsx_ld(0, reinterpret_cast(ptr)); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - __vector unsigned char needs_escape = - (__vector unsigned char)vec_cmpeq(word, v34); - needs_escape = vec_or(needs_escape, - (__vector unsigned char)vec_cmpeq(word, v92)); - needs_escape = vec_or(needs_escape, - (__vector unsigned char)vec_cmplt(word, v32)); - - __vector unsigned long long result = - (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask); -#ifdef __LITTLE_ENDIAN__ - unsigned int mask = static_cast(result[1]); -#else - unsigned int mask = static_cast(result[0]); -#endif - if (mask != 0) { - size_t offset = ptr - reinterpret_cast(view.data()); - return offset + __builtin_ctz(mask); - } - ptr += 16; - remaining -= 16; - } - - // Scalar fallback for remaining bytes - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -#else -SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - return find_next_json_quotable_character_scalar(view, location); -} -#endif -#endif // !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - -SIMDJSON_CONSTEXPR_LAMBDA static std::string_view control_chars[] = { - "\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", "\\u0005", "\\u0006", - "\\u0007", "\\b", "\\t", "\\n", "\\u000b", "\\f", "\\r", - "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012", "\\u0013", "\\u0014", - "\\u0015", "\\u0016", "\\u0017", "\\u0018", "\\u0019", "\\u001a", "\\u001b", - "\\u001c", "\\u001d", "\\u001e", "\\u001f"}; - -// All Unicode characters may be placed within the quotation marks, except for -// the characters that MUST be escaped: quotation mark, reverse solidus, and the -// control characters (U+0000 through U+001F). There are two-character sequence -// escape representations of some popular characters: -// \", \\, \b, \f, \n, \r, \t. -SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline void escape_json_char(char c, char *&out) { - if (c == '"') { - memcpy(out, "\\\"", 2); - out += 2; - } else if (c == '\\') { - memcpy(out, "\\\\", 2); - out += 2; - } else { - std::string_view v = control_chars[uint8_t(c)]; - memcpy(out, v.data(), v.size()); - out += v.size(); - } -} - -#if SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 - -using escape_vector = __m128i; -// Mask bits that each input byte contributes to escape_bitmask(). -static constexpr unsigned escape_mask_bits = 1; - -simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept { - return _mm_loadu_si128(reinterpret_cast(p)); -} - -simdjson_inline void escape_store16(char *out, escape_vector v) noexcept { - _mm_storeu_si128(reinterpret_cast<__m128i *>(out), v); -} - -// Builds a vector whose bytes 0..7 come from a and bytes 8..15 from b. -simdjson_inline escape_vector escape_load8x2(const uint8_t *a, - const uint8_t *b) noexcept { - return _mm_unpacklo_epi64( - _mm_loadl_epi64(reinterpret_cast(a)), - _mm_loadl_epi64(reinterpret_cast(b))); -} - -// Builds a vector whose bytes 0..3 come from a and bytes 4..7 from b. The -// upper half is unspecified; callers only look at the low eight bits. -simdjson_inline escape_vector escape_load4x2(const uint8_t *a, - const uint8_t *b) noexcept { - int32_t a32, b32; - memcpy(&a32, a, 4); - memcpy(&b32, b, 4); - return _mm_unpacklo_epi32(_mm_cvtsi32_si128(a32), _mm_cvtsi32_si128(b32)); -} - -// Sets every bit of byte k when byte k of v is a quotable character ('"', -// '\\' or a control character). -simdjson_inline escape_vector escape_flags(escape_vector v) noexcept { - const __m128i v34 = _mm_set1_epi8(34); // '"' - const __m128i v92 = _mm_set1_epi8(92); // '\\' - const __m128i v31 = _mm_set1_epi8(31); // for control char detection - __m128i needs_escape = _mm_cmpeq_epi8(v, v34); - needs_escape = _mm_or_si128(needs_escape, _mm_cmpeq_epi8(v, v92)); - return _mm_or_si128( - needs_escape, _mm_cmpeq_epi8(_mm_subs_epu8(v, v31), _mm_setzero_si128())); -} - -// True when any byte needs escaping. Kept separate from escape_bitmask -// because some instruction sets can answer it without leaving the vector -// register file; on SSE2 the compiler folds the two together. -simdjson_inline bool escape_any(escape_vector flags) noexcept { - return _mm_movemask_epi8(flags) != 0; -} - -// A 16-bit mask with bit k set when byte k needed escaping. -simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept { - return uint64_t(uint32_t(_mm_movemask_epi8(flags))); -} - -#else // SIMDJSON_EXPERIMENTAL_HAS_NEON - -using escape_vector = uint8x16_t; -static constexpr unsigned escape_mask_bits = 4; - -simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept { - return vld1q_u8(p); -} - -simdjson_inline void escape_store16(char *out, escape_vector v) noexcept { - vst1q_u8(reinterpret_cast(out), v); -} - -simdjson_inline escape_vector escape_load8x2(const uint8_t *a, - const uint8_t *b) noexcept { - uint64_t a64, b64; - memcpy(&a64, a, 8); - memcpy(&b64, b, 8); - return vreinterpretq_u8_u64(vsetq_lane_u64(b64, vdupq_n_u64(a64), 1)); -} - -simdjson_inline escape_vector escape_load4x2(const uint8_t *a, - const uint8_t *b) noexcept { - uint32_t a32, b32; - memcpy(&a32, a, 4); - memcpy(&b32, b, 4); - return vreinterpretq_u8_u32(vsetq_lane_u32(b32, vdupq_n_u32(a32), 1)); -} - -simdjson_inline escape_vector escape_flags(escape_vector v) noexcept { - uint8x16_t needs_escape = vceqq_u8(v, vdupq_n_u8(34)); // '"' - needs_escape = vorrq_u8(needs_escape, vceqq_u8(v, vdupq_n_u8(92))); // '\\' - return vorrq_u8(needs_escape, vcltq_u8(v, vdupq_n_u8(32))); -} - -simdjson_inline bool escape_any(escape_vector flags) noexcept { - return vmaxvq_u32(vreinterpretq_u32_u8(flags)) != 0; -} - -// Four bits per byte rather than one: escape_block and the tail paths scale -// their shifts by escape_mask_bits to match. -simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept { - uint8x8_t narrowed = vshrn_n_u16(vreinterpretq_u16_u8(flags), 4); - return vget_lane_u64(vreinterpret_u64_u8(narrowed), 0); -} - -#endif // instruction set selection - -// The escape bitmask of a 16-byte block. -simdjson_inline uint64_t escape_mask(escape_vector v) noexcept { - return escape_bitmask(escape_flags(v)); -} - -// Copies n bytes with n < 16, using overlapping loads and stores. It never -// reads more than n bytes from src, nor writes more than n bytes to dst. -simdjson_inline void copy_lt16(char *dst, const uint8_t *src, - size_t n) noexcept { - if (n >= 8) { - memcpy(dst, src, 8); - memcpy(dst + n - 8, src + n - 8, 8); - } else if (n >= 4) { - memcpy(dst, src, 4); - memcpy(dst + n - 4, src + n - 4, 4); - } else if (n > 0) { - dst[0] = char(src[0]); - dst[n >> 1] = char(src[n >> 1]); - dst[n - 1] = char(src[n - 1]); - } -} - -// Escapes the bytes of src in the range [i, blockend), given that m is the -// (non-zero) escape mask for that range: the escape_mask_bits-wide lane of m -// at byte k is non-zero when src[i + k] requires escaping. Returns the updated -// output pointer. -simdjson_never_inline char *escape_block(const uint8_t *src, char *out, - size_t i, size_t blockend, - uint64_t m) noexcept { - constexpr uint64_t lane = (uint64_t(1) << escape_mask_bits) - 1; - - size_t pos = i; // first byte not yet copied - while (m) { - const size_t tz = trailing_zeroes(m); - const size_t next = i + tz / escape_mask_bits; - // Copy the run of safe bytes that precedes this escape. - copy_lt16(out, src + pos, next - pos); - out += next - pos; - escape_json_char(char(src[next]), out); - pos = next + 1; - m &= ~(lane << tz); - } - // Copy whatever follows the last escape. - copy_lt16(out, src + pos, blockend - pos); - return out + (blockend - pos); -} - -// Writes the escaped version of input to out, returning the number of bytes -// written. -simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out) { - const size_t len = input.size(); - const uint8_t *src = reinterpret_cast(input.data()); - const char *const initout = out; - - size_t i = 0; -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 && defined(__AVX2__) - while (i + 32 <= len) { - const __m256i word = _mm256_loadu_si256(reinterpret_cast(src + i)); - const __m256i flags = _mm256_or_si256( - _mm256_or_si256(_mm256_cmpeq_epi8(word, _mm256_set1_epi8(34)), // '"' - _mm256_cmpeq_epi8(word, _mm256_set1_epi8(92))), // '\\' - _mm256_cmpeq_epi8(_mm256_subs_epu8(word, _mm256_set1_epi8(31)), - _mm256_setzero_si256())); // control - const uint32_t mask = uint32_t(_mm256_movemask_epi8(flags)); - if (simdjson_likely(mask == 0)) { - _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), word); - out += 32; - } else { - for (size_t half = 0; half < 32; half += 16) { - const uint64_t m = (mask >> half) & 0xFFFF; - if (m == 0) { - escape_store16(out, escape_load16(src + i + half)); - out += 16; - } else { - out = escape_block(src, out, i + half, i + half + 16, m); - } - } - } - i += 32; - } -#endif - while (i + 16 <= len) { - escape_vector word = escape_load16(src + i); - escape_vector flags = escape_flags(word); - if (simdjson_likely(!escape_any(flags))) { - escape_store16(out, word); - out += 16; - } else { - out = escape_block(src, out, i, i + 16, escape_bitmask(flags)); - } - i += 16; - } - if (i < len) { - const size_t rem = len - i; - uint64_t m; - if (len >= 16) { - // The last 16 bytes of the input are in bounds. Bit k of that block's - // mask belongs to input position len - 16 + k, so shift it down to align - // bit 0 with position i. - m = escape_mask(escape_load16(src + len - 16)) >> - (escape_mask_bits * (16 - rem)); - } else if (len >= 8) { - // Here i == 0 and rem == len. Two overlapping 8-byte loads cover - // [0, 8) and [len - 8, len), which is the whole input since len < 16. - uint64_t mm = escape_mask(escape_load8x2(src, src + len - 8)); - constexpr uint64_t low8 = (uint64_t(1) << (escape_mask_bits * 8)) - 1; - m = (mm & low8) | - ((mm >> (escape_mask_bits * 8)) << (escape_mask_bits * (len - 8))); - } else if (len >= 4) { - // Same idea with two overlapping 4-byte loads. - uint64_t mm = escape_mask(escape_load4x2(src, src + len - 4)); - constexpr uint64_t low4 = (uint64_t(1) << (escape_mask_bits * 4)) - 1; - m = (mm & low4) | (((mm >> (escape_mask_bits * 4)) & low4) - << (escape_mask_bits * (len - 4))); - } else { - // Fewer than 4 bytes: at most three table lookups, no need for SIMD. - for (size_t k = 0; k < len; k++) { - uint8_t c = src[k]; - if (json_quotable_character[c]) { - escape_json_char(char(c), out); - } else { - *out++ = char(c); - } - } - return size_t(out - initout); - } - if (m == 0) { - copy_lt16(out, src + i, rem); - out += rem; - } else { - out = escape_block(src, out, i, len, m); - } - } - return size_t(out - initout); -} - -#else // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - -// Writes the escaped version of input to out, returning the number of bytes -// written. Uses SIMD position finding to locate quotable characters efficiently. -inline size_t write_string_escaped(const std::string_view input, char *out) { - size_t mysize = input.size(); - - // Use SIMD position finder directly - it returns mysize if no escape needed - size_t location = find_next_json_quotable_character(input, 0); - if (location == mysize) { - // Fast path: no escaping needed - memcpy(out, input.data(), input.size()); - return input.size(); - } - - const char *const initout = out; - memcpy(out, input.data(), location); - out += location; - escape_json_char(input[location], out); - location += 1; - while (location < mysize) { - size_t newlocation = find_next_json_quotable_character(input, location); - memcpy(out, input.data() + location, newlocation - location); - out += newlocation - location; - location = newlocation; - if (location == mysize) { - break; - } - escape_json_char(input[location], out); - location += 1; - } - return size_t(out - initout); -} - -#endif // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE -#undef SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - namespace internal { // Sets the size of s to n without initializing the new bytes when the @@ -58064,6 +56392,7 @@ backslash_and_quote::copy_and_find(const uint8_t *src, uint8_t *dst) { /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ namespace simdjson { @@ -58100,6 +56429,9 @@ struct has_custom_serialization : std::false_type {}; namespace ppc64 { namespace builder { + +using simdjson::internal::escape_best::write_string_escaped; + /** * A builder for JSON strings representing documents. This is a low-level * builder that is not meant to be used directly by end-users. Though it @@ -58446,6 +56778,7 @@ simdjson_warn_unused error_code to_json(const Z &z, std::string &s, size_t initi /* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ /* amalgamation skipped (editor-only): #include "simdjson/concepts.h" */ /* amalgamation skipped (editor-only): #include "simdjson/annotations.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ #if SIMDJSON_STATIC_REFLECTION @@ -58471,7 +56804,6 @@ namespace internal { using simdjson::internal::write_uint_jeaiii; simdjson_inline char *write_double(char *p, double v) noexcept; } // namespace internal -simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out); SIMDJSON_PUSH_DISABLE_WARNINGS SIMDJSON_DISABLE_GCC_WARNING(-Warray-bounds) @@ -59492,619 +57824,13 @@ simdjson_warn_unused simdjson_result extract_fractured_json( /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_INL_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/jeaiii.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ -/* - * Empirically, we have found that an inlined optimization is important for - * performance. The following macros are not ideal. We should find a better - * way to inline the code. - */ - -#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \ - (defined(_M_AMD64) || defined(_M_X64) || \ - (defined(_M_IX86_FP) && _M_IX86_FP == 2)) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2 -#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1 -#endif -#endif - -#if defined(__aarch64__) || defined(_M_ARM64) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON -#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1 -#endif -#endif -#if defined(__loongarch_sx) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX -#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1 -#endif -#endif -#if defined(__loongarch_asx) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_LASX -#define SIMDJSON_EXPERIMENTAL_HAS_LASX 1 -#endif -#endif -#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \ - defined(__riscv_vector) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV -#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 -#endif -#endif -#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 -#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_NEON -#include -#ifdef _MSC_VER -#include -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 -#include -#if defined(__AVX2__) -#include -#endif -#ifdef _MSC_VER -#include -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_LSX -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_LASX -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_RVV -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_PPC64 -#include -#ifdef bool -#undef bool -#endif -#ifdef vector -#undef vector -#endif -#endif - - namespace simdjson { namespace ppc64 { namespace builder { -static SIMDJSON_CONSTEXPR_LAMBDA std::array - json_quotable_character = { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - -/** - -A possible SWAR implementation of has_json_escapable_byte. It is not used -because it is slower than the current implementation. It is kept here for -reference (to show that we tried it). - -inline bool has_json_escapable_byte(uint64_t x) { - uint64_t is_ascii = 0x8080808080808080ULL & ~x; - uint64_t xor2 = x ^ 0x0202020202020202ULL; - uint64_t lt32_or_eq34 = xor2 - 0x2121212121212121ULL; - uint64_t sub92 = x ^ 0x5C5C5C5C5C5C5C5CULL; - uint64_t eq92 = (sub92 - 0x0101010101010101ULL); - return ((lt32_or_eq34 | eq92) & is_ascii) != 0; -} - -**/ - -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 || SIMDJSON_EXPERIMENTAL_HAS_NEON -#define SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE 1 -#endif - -// With block escaping (SSE2/NEON), write_string_escaped scans the input itself. -#if !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE -// Scalar fallback for finding next quotable character -SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t -find_next_json_quotable_character_scalar(const std::string_view view, - size_t location) noexcept { - for (auto pos = view.begin() + location; pos != view.end(); ++pos) { - if (json_quotable_character[static_cast(*pos)]) { - return pos - view.begin(); - } - } - return size_t(view.size()); -} - -// SIMD-accelerated position finding that directly locates the first quotable -// character, combining detection and position extraction in a single pass to -// minimize redundant work. -#if SIMDJSON_EXPERIMENTAL_HAS_LASX -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - // SIMD constants for characters requiring escape - __m256i v34 = __lasx_xvreplgr2vr_b(34); // '"' - __m256i v92 = __lasx_xvreplgr2vr_b(92); // '\\' - __m256i v32 = __lasx_xvreplgr2vr_b(32); // control char threshold - - while (remaining >= 32) { - __m256i word = __lasx_xvld(ptr, 0); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - __m256i needs_escape = __lasx_xvseq_b(word, v34); - needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvseq_b(word, v92)); - needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvslt_bu(word, v32)); - - if (!__lasx_xbz_v(needs_escape)) { - // Found a quotable character - locate it via the four 64-bit lanes - uint64_t lane0 = __lasx_xvpickve2gr_du(needs_escape, 0); - uint64_t lane1 = __lasx_xvpickve2gr_du(needs_escape, 1); - uint64_t lane2 = __lasx_xvpickve2gr_du(needs_escape, 2); - uint64_t lane3 = __lasx_xvpickve2gr_du(needs_escape, 3); - size_t offset = ptr - reinterpret_cast(view.data()); - if (lane0 != 0) { - return offset + trailing_zeroes(lane0) / 8; - } else if (lane1 != 0) { - return offset + 8 + trailing_zeroes(lane1) / 8; - } else if (lane2 != 0) { - return offset + 16 + trailing_zeroes(lane2) / 8; - } else { - return offset + 24 + trailing_zeroes(lane3) / 8; - } - } - ptr += 32; - remaining -= 32; - } - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -#elif SIMDJSON_EXPERIMENTAL_HAS_LSX -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - //SIMD constants for characters requiring escape - __m128i v34 = __lsx_vreplgr2vr_b(34); // '"' - __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\' - __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold - - while (remaining >= 16){ - __m128i word = __lsx_vld(ptr, 0); - - //Check for the quotable characters: '"', '\\', or control char (<32) - __m128i needs_escape = __lsx_vseq_b(word, v34); - needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92)); - needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32)); - - if (!__lsx_bz_v(needs_escape)){ - - //Found quotable character - extract exact byte position - uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0); - uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1); - size_t offset = ptr - reinterpret_cast(view.data()); - if ( lo != 0) { - return offset + trailing_zeroes(lo) / 8; - } else { - return offset + 8 + trailing_zeroes(hi) / 8; - } - } - ptr += 16; - remaining -= 16; - } - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -#elif SIMDJSON_EXPERIMENTAL_HAS_RVV -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - while (remaining > 0) { - size_t vl = __riscv_vsetvl_e8m1(remaining); - vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl); - needs_escape = __riscv_vmor(needs_escape, - __riscv_vmseq(word, (uint8_t)92, vl), vl); - needs_escape = __riscv_vmor(needs_escape, - __riscv_vmsltu(word, (uint8_t)32, vl), vl); - - long first = __riscv_vfirst(needs_escape, vl); - if (first >= 0) { - size_t offset = ptr - reinterpret_cast(view.data()); - return offset + first; - } - ptr += vl; - remaining -= vl; - } - - return len; -} -#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64 -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - // SIMD constants for characters requiring escape - __vector unsigned char v34 = vec_splats((unsigned char)34); // '"' - __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\' - __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold - - // Bitmask for vec_vbpermq to extract one bit per byte - const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50, - 0x48, 0x40, 0x38, 0x30, 0x28, 0x20, - 0x18, 0x10, 0x08, 0x00}; - - while (remaining >= 16) { - __vector unsigned char word = - vec_vsx_ld(0, reinterpret_cast(ptr)); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - __vector unsigned char needs_escape = - (__vector unsigned char)vec_cmpeq(word, v34); - needs_escape = vec_or(needs_escape, - (__vector unsigned char)vec_cmpeq(word, v92)); - needs_escape = vec_or(needs_escape, - (__vector unsigned char)vec_cmplt(word, v32)); - - __vector unsigned long long result = - (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask); -#ifdef __LITTLE_ENDIAN__ - unsigned int mask = static_cast(result[1]); -#else - unsigned int mask = static_cast(result[0]); -#endif - if (mask != 0) { - size_t offset = ptr - reinterpret_cast(view.data()); - return offset + __builtin_ctz(mask); - } - ptr += 16; - remaining -= 16; - } - - // Scalar fallback for remaining bytes - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -#else -SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - return find_next_json_quotable_character_scalar(view, location); -} -#endif -#endif // !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - -SIMDJSON_CONSTEXPR_LAMBDA static std::string_view control_chars[] = { - "\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", "\\u0005", "\\u0006", - "\\u0007", "\\b", "\\t", "\\n", "\\u000b", "\\f", "\\r", - "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012", "\\u0013", "\\u0014", - "\\u0015", "\\u0016", "\\u0017", "\\u0018", "\\u0019", "\\u001a", "\\u001b", - "\\u001c", "\\u001d", "\\u001e", "\\u001f"}; - -// All Unicode characters may be placed within the quotation marks, except for -// the characters that MUST be escaped: quotation mark, reverse solidus, and the -// control characters (U+0000 through U+001F). There are two-character sequence -// escape representations of some popular characters: -// \", \\, \b, \f, \n, \r, \t. -SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline void escape_json_char(char c, char *&out) { - if (c == '"') { - memcpy(out, "\\\"", 2); - out += 2; - } else if (c == '\\') { - memcpy(out, "\\\\", 2); - out += 2; - } else { - std::string_view v = control_chars[uint8_t(c)]; - memcpy(out, v.data(), v.size()); - out += v.size(); - } -} - -#if SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 - -using escape_vector = __m128i; -// Mask bits that each input byte contributes to escape_bitmask(). -static constexpr unsigned escape_mask_bits = 1; - -simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept { - return _mm_loadu_si128(reinterpret_cast(p)); -} - -simdjson_inline void escape_store16(char *out, escape_vector v) noexcept { - _mm_storeu_si128(reinterpret_cast<__m128i *>(out), v); -} - -// Builds a vector whose bytes 0..7 come from a and bytes 8..15 from b. -simdjson_inline escape_vector escape_load8x2(const uint8_t *a, - const uint8_t *b) noexcept { - return _mm_unpacklo_epi64( - _mm_loadl_epi64(reinterpret_cast(a)), - _mm_loadl_epi64(reinterpret_cast(b))); -} - -// Builds a vector whose bytes 0..3 come from a and bytes 4..7 from b. The -// upper half is unspecified; callers only look at the low eight bits. -simdjson_inline escape_vector escape_load4x2(const uint8_t *a, - const uint8_t *b) noexcept { - int32_t a32, b32; - memcpy(&a32, a, 4); - memcpy(&b32, b, 4); - return _mm_unpacklo_epi32(_mm_cvtsi32_si128(a32), _mm_cvtsi32_si128(b32)); -} - -// Sets every bit of byte k when byte k of v is a quotable character ('"', -// '\\' or a control character). -simdjson_inline escape_vector escape_flags(escape_vector v) noexcept { - const __m128i v34 = _mm_set1_epi8(34); // '"' - const __m128i v92 = _mm_set1_epi8(92); // '\\' - const __m128i v31 = _mm_set1_epi8(31); // for control char detection - __m128i needs_escape = _mm_cmpeq_epi8(v, v34); - needs_escape = _mm_or_si128(needs_escape, _mm_cmpeq_epi8(v, v92)); - return _mm_or_si128( - needs_escape, _mm_cmpeq_epi8(_mm_subs_epu8(v, v31), _mm_setzero_si128())); -} - -// True when any byte needs escaping. Kept separate from escape_bitmask -// because some instruction sets can answer it without leaving the vector -// register file; on SSE2 the compiler folds the two together. -simdjson_inline bool escape_any(escape_vector flags) noexcept { - return _mm_movemask_epi8(flags) != 0; -} - -// A 16-bit mask with bit k set when byte k needed escaping. -simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept { - return uint64_t(uint32_t(_mm_movemask_epi8(flags))); -} - -#else // SIMDJSON_EXPERIMENTAL_HAS_NEON - -using escape_vector = uint8x16_t; -static constexpr unsigned escape_mask_bits = 4; - -simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept { - return vld1q_u8(p); -} - -simdjson_inline void escape_store16(char *out, escape_vector v) noexcept { - vst1q_u8(reinterpret_cast(out), v); -} - -simdjson_inline escape_vector escape_load8x2(const uint8_t *a, - const uint8_t *b) noexcept { - uint64_t a64, b64; - memcpy(&a64, a, 8); - memcpy(&b64, b, 8); - return vreinterpretq_u8_u64(vsetq_lane_u64(b64, vdupq_n_u64(a64), 1)); -} - -simdjson_inline escape_vector escape_load4x2(const uint8_t *a, - const uint8_t *b) noexcept { - uint32_t a32, b32; - memcpy(&a32, a, 4); - memcpy(&b32, b, 4); - return vreinterpretq_u8_u32(vsetq_lane_u32(b32, vdupq_n_u32(a32), 1)); -} - -simdjson_inline escape_vector escape_flags(escape_vector v) noexcept { - uint8x16_t needs_escape = vceqq_u8(v, vdupq_n_u8(34)); // '"' - needs_escape = vorrq_u8(needs_escape, vceqq_u8(v, vdupq_n_u8(92))); // '\\' - return vorrq_u8(needs_escape, vcltq_u8(v, vdupq_n_u8(32))); -} - -simdjson_inline bool escape_any(escape_vector flags) noexcept { - return vmaxvq_u32(vreinterpretq_u32_u8(flags)) != 0; -} - -// Four bits per byte rather than one: escape_block and the tail paths scale -// their shifts by escape_mask_bits to match. -simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept { - uint8x8_t narrowed = vshrn_n_u16(vreinterpretq_u16_u8(flags), 4); - return vget_lane_u64(vreinterpret_u64_u8(narrowed), 0); -} - -#endif // instruction set selection - -// The escape bitmask of a 16-byte block. -simdjson_inline uint64_t escape_mask(escape_vector v) noexcept { - return escape_bitmask(escape_flags(v)); -} - -// Copies n bytes with n < 16, using overlapping loads and stores. It never -// reads more than n bytes from src, nor writes more than n bytes to dst. -simdjson_inline void copy_lt16(char *dst, const uint8_t *src, - size_t n) noexcept { - if (n >= 8) { - memcpy(dst, src, 8); - memcpy(dst + n - 8, src + n - 8, 8); - } else if (n >= 4) { - memcpy(dst, src, 4); - memcpy(dst + n - 4, src + n - 4, 4); - } else if (n > 0) { - dst[0] = char(src[0]); - dst[n >> 1] = char(src[n >> 1]); - dst[n - 1] = char(src[n - 1]); - } -} - -// Escapes the bytes of src in the range [i, blockend), given that m is the -// (non-zero) escape mask for that range: the escape_mask_bits-wide lane of m -// at byte k is non-zero when src[i + k] requires escaping. Returns the updated -// output pointer. -simdjson_never_inline char *escape_block(const uint8_t *src, char *out, - size_t i, size_t blockend, - uint64_t m) noexcept { - constexpr uint64_t lane = (uint64_t(1) << escape_mask_bits) - 1; - - size_t pos = i; // first byte not yet copied - while (m) { - const size_t tz = trailing_zeroes(m); - const size_t next = i + tz / escape_mask_bits; - // Copy the run of safe bytes that precedes this escape. - copy_lt16(out, src + pos, next - pos); - out += next - pos; - escape_json_char(char(src[next]), out); - pos = next + 1; - m &= ~(lane << tz); - } - // Copy whatever follows the last escape. - copy_lt16(out, src + pos, blockend - pos); - return out + (blockend - pos); -} - -// Writes the escaped version of input to out, returning the number of bytes -// written. -simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out) { - const size_t len = input.size(); - const uint8_t *src = reinterpret_cast(input.data()); - const char *const initout = out; - - size_t i = 0; -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 && defined(__AVX2__) - while (i + 32 <= len) { - const __m256i word = _mm256_loadu_si256(reinterpret_cast(src + i)); - const __m256i flags = _mm256_or_si256( - _mm256_or_si256(_mm256_cmpeq_epi8(word, _mm256_set1_epi8(34)), // '"' - _mm256_cmpeq_epi8(word, _mm256_set1_epi8(92))), // '\\' - _mm256_cmpeq_epi8(_mm256_subs_epu8(word, _mm256_set1_epi8(31)), - _mm256_setzero_si256())); // control - const uint32_t mask = uint32_t(_mm256_movemask_epi8(flags)); - if (simdjson_likely(mask == 0)) { - _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), word); - out += 32; - } else { - for (size_t half = 0; half < 32; half += 16) { - const uint64_t m = (mask >> half) & 0xFFFF; - if (m == 0) { - escape_store16(out, escape_load16(src + i + half)); - out += 16; - } else { - out = escape_block(src, out, i + half, i + half + 16, m); - } - } - } - i += 32; - } -#endif - while (i + 16 <= len) { - escape_vector word = escape_load16(src + i); - escape_vector flags = escape_flags(word); - if (simdjson_likely(!escape_any(flags))) { - escape_store16(out, word); - out += 16; - } else { - out = escape_block(src, out, i, i + 16, escape_bitmask(flags)); - } - i += 16; - } - if (i < len) { - const size_t rem = len - i; - uint64_t m; - if (len >= 16) { - // The last 16 bytes of the input are in bounds. Bit k of that block's - // mask belongs to input position len - 16 + k, so shift it down to align - // bit 0 with position i. - m = escape_mask(escape_load16(src + len - 16)) >> - (escape_mask_bits * (16 - rem)); - } else if (len >= 8) { - // Here i == 0 and rem == len. Two overlapping 8-byte loads cover - // [0, 8) and [len - 8, len), which is the whole input since len < 16. - uint64_t mm = escape_mask(escape_load8x2(src, src + len - 8)); - constexpr uint64_t low8 = (uint64_t(1) << (escape_mask_bits * 8)) - 1; - m = (mm & low8) | - ((mm >> (escape_mask_bits * 8)) << (escape_mask_bits * (len - 8))); - } else if (len >= 4) { - // Same idea with two overlapping 4-byte loads. - uint64_t mm = escape_mask(escape_load4x2(src, src + len - 4)); - constexpr uint64_t low4 = (uint64_t(1) << (escape_mask_bits * 4)) - 1; - m = (mm & low4) | (((mm >> (escape_mask_bits * 4)) & low4) - << (escape_mask_bits * (len - 4))); - } else { - // Fewer than 4 bytes: at most three table lookups, no need for SIMD. - for (size_t k = 0; k < len; k++) { - uint8_t c = src[k]; - if (json_quotable_character[c]) { - escape_json_char(char(c), out); - } else { - *out++ = char(c); - } - } - return size_t(out - initout); - } - if (m == 0) { - copy_lt16(out, src + i, rem); - out += rem; - } else { - out = escape_block(src, out, i, len, m); - } - } - return size_t(out - initout); -} - -#else // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - -// Writes the escaped version of input to out, returning the number of bytes -// written. Uses SIMD position finding to locate quotable characters efficiently. -inline size_t write_string_escaped(const std::string_view input, char *out) { - size_t mysize = input.size(); - - // Use SIMD position finder directly - it returns mysize if no escape needed - size_t location = find_next_json_quotable_character(input, 0); - if (location == mysize) { - // Fast path: no escaping needed - memcpy(out, input.data(), input.size()); - return input.size(); - } - - const char *const initout = out; - memcpy(out, input.data(), location); - out += location; - escape_json_char(input[location], out); - location += 1; - while (location < mysize) { - size_t newlocation = find_next_json_quotable_character(input, location); - memcpy(out, input.data() + location, newlocation - location); - out += newlocation - location; - location = newlocation; - if (location == mysize) { - break; - } - escape_json_char(input[location], out); - location += 1; - } - return size_t(out - initout); -} - -#endif // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE -#undef SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - namespace internal { // Sets the size of s to n without initializing the new bytes when the @@ -61851,6 +59577,7 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ namespace simdjson { @@ -61887,6 +59614,9 @@ struct has_custom_serialization : std::false_type {}; namespace westmere { namespace builder { + +using simdjson::internal::escape_best::write_string_escaped; + /** * A builder for JSON strings representing documents. This is a low-level * builder that is not meant to be used directly by end-users. Though it @@ -62233,6 +59963,7 @@ simdjson_warn_unused error_code to_json(const Z &z, std::string &s, size_t initi /* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ /* amalgamation skipped (editor-only): #include "simdjson/concepts.h" */ /* amalgamation skipped (editor-only): #include "simdjson/annotations.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ #if SIMDJSON_STATIC_REFLECTION @@ -62258,7 +59989,6 @@ namespace internal { using simdjson::internal::write_uint_jeaiii; simdjson_inline char *write_double(char *p, double v) noexcept; } // namespace internal -simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out); SIMDJSON_PUSH_DISABLE_WARNINGS SIMDJSON_DISABLE_GCC_WARNING(-Warray-bounds) @@ -63279,619 +61009,13 @@ simdjson_warn_unused simdjson_result extract_fractured_json( /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_INL_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/jeaiii.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ -/* - * Empirically, we have found that an inlined optimization is important for - * performance. The following macros are not ideal. We should find a better - * way to inline the code. - */ - -#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \ - (defined(_M_AMD64) || defined(_M_X64) || \ - (defined(_M_IX86_FP) && _M_IX86_FP == 2)) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2 -#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1 -#endif -#endif - -#if defined(__aarch64__) || defined(_M_ARM64) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON -#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1 -#endif -#endif -#if defined(__loongarch_sx) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX -#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1 -#endif -#endif -#if defined(__loongarch_asx) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_LASX -#define SIMDJSON_EXPERIMENTAL_HAS_LASX 1 -#endif -#endif -#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \ - defined(__riscv_vector) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV -#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 -#endif -#endif -#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 -#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_NEON -#include -#ifdef _MSC_VER -#include -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 -#include -#if defined(__AVX2__) -#include -#endif -#ifdef _MSC_VER -#include -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_LSX -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_LASX -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_RVV -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_PPC64 -#include -#ifdef bool -#undef bool -#endif -#ifdef vector -#undef vector -#endif -#endif - - namespace simdjson { namespace westmere { namespace builder { -static SIMDJSON_CONSTEXPR_LAMBDA std::array - json_quotable_character = { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - -/** - -A possible SWAR implementation of has_json_escapable_byte. It is not used -because it is slower than the current implementation. It is kept here for -reference (to show that we tried it). - -inline bool has_json_escapable_byte(uint64_t x) { - uint64_t is_ascii = 0x8080808080808080ULL & ~x; - uint64_t xor2 = x ^ 0x0202020202020202ULL; - uint64_t lt32_or_eq34 = xor2 - 0x2121212121212121ULL; - uint64_t sub92 = x ^ 0x5C5C5C5C5C5C5C5CULL; - uint64_t eq92 = (sub92 - 0x0101010101010101ULL); - return ((lt32_or_eq34 | eq92) & is_ascii) != 0; -} - -**/ - -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 || SIMDJSON_EXPERIMENTAL_HAS_NEON -#define SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE 1 -#endif - -// With block escaping (SSE2/NEON), write_string_escaped scans the input itself. -#if !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE -// Scalar fallback for finding next quotable character -SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t -find_next_json_quotable_character_scalar(const std::string_view view, - size_t location) noexcept { - for (auto pos = view.begin() + location; pos != view.end(); ++pos) { - if (json_quotable_character[static_cast(*pos)]) { - return pos - view.begin(); - } - } - return size_t(view.size()); -} - -// SIMD-accelerated position finding that directly locates the first quotable -// character, combining detection and position extraction in a single pass to -// minimize redundant work. -#if SIMDJSON_EXPERIMENTAL_HAS_LASX -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - // SIMD constants for characters requiring escape - __m256i v34 = __lasx_xvreplgr2vr_b(34); // '"' - __m256i v92 = __lasx_xvreplgr2vr_b(92); // '\\' - __m256i v32 = __lasx_xvreplgr2vr_b(32); // control char threshold - - while (remaining >= 32) { - __m256i word = __lasx_xvld(ptr, 0); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - __m256i needs_escape = __lasx_xvseq_b(word, v34); - needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvseq_b(word, v92)); - needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvslt_bu(word, v32)); - - if (!__lasx_xbz_v(needs_escape)) { - // Found a quotable character - locate it via the four 64-bit lanes - uint64_t lane0 = __lasx_xvpickve2gr_du(needs_escape, 0); - uint64_t lane1 = __lasx_xvpickve2gr_du(needs_escape, 1); - uint64_t lane2 = __lasx_xvpickve2gr_du(needs_escape, 2); - uint64_t lane3 = __lasx_xvpickve2gr_du(needs_escape, 3); - size_t offset = ptr - reinterpret_cast(view.data()); - if (lane0 != 0) { - return offset + trailing_zeroes(lane0) / 8; - } else if (lane1 != 0) { - return offset + 8 + trailing_zeroes(lane1) / 8; - } else if (lane2 != 0) { - return offset + 16 + trailing_zeroes(lane2) / 8; - } else { - return offset + 24 + trailing_zeroes(lane3) / 8; - } - } - ptr += 32; - remaining -= 32; - } - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -#elif SIMDJSON_EXPERIMENTAL_HAS_LSX -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - //SIMD constants for characters requiring escape - __m128i v34 = __lsx_vreplgr2vr_b(34); // '"' - __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\' - __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold - - while (remaining >= 16){ - __m128i word = __lsx_vld(ptr, 0); - - //Check for the quotable characters: '"', '\\', or control char (<32) - __m128i needs_escape = __lsx_vseq_b(word, v34); - needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92)); - needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32)); - - if (!__lsx_bz_v(needs_escape)){ - - //Found quotable character - extract exact byte position - uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0); - uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1); - size_t offset = ptr - reinterpret_cast(view.data()); - if ( lo != 0) { - return offset + trailing_zeroes(lo) / 8; - } else { - return offset + 8 + trailing_zeroes(hi) / 8; - } - } - ptr += 16; - remaining -= 16; - } - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -#elif SIMDJSON_EXPERIMENTAL_HAS_RVV -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - while (remaining > 0) { - size_t vl = __riscv_vsetvl_e8m1(remaining); - vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl); - needs_escape = __riscv_vmor(needs_escape, - __riscv_vmseq(word, (uint8_t)92, vl), vl); - needs_escape = __riscv_vmor(needs_escape, - __riscv_vmsltu(word, (uint8_t)32, vl), vl); - - long first = __riscv_vfirst(needs_escape, vl); - if (first >= 0) { - size_t offset = ptr - reinterpret_cast(view.data()); - return offset + first; - } - ptr += vl; - remaining -= vl; - } - - return len; -} -#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64 -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - // SIMD constants for characters requiring escape - __vector unsigned char v34 = vec_splats((unsigned char)34); // '"' - __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\' - __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold - - // Bitmask for vec_vbpermq to extract one bit per byte - const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50, - 0x48, 0x40, 0x38, 0x30, 0x28, 0x20, - 0x18, 0x10, 0x08, 0x00}; - - while (remaining >= 16) { - __vector unsigned char word = - vec_vsx_ld(0, reinterpret_cast(ptr)); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - __vector unsigned char needs_escape = - (__vector unsigned char)vec_cmpeq(word, v34); - needs_escape = vec_or(needs_escape, - (__vector unsigned char)vec_cmpeq(word, v92)); - needs_escape = vec_or(needs_escape, - (__vector unsigned char)vec_cmplt(word, v32)); - - __vector unsigned long long result = - (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask); -#ifdef __LITTLE_ENDIAN__ - unsigned int mask = static_cast(result[1]); -#else - unsigned int mask = static_cast(result[0]); -#endif - if (mask != 0) { - size_t offset = ptr - reinterpret_cast(view.data()); - return offset + __builtin_ctz(mask); - } - ptr += 16; - remaining -= 16; - } - - // Scalar fallback for remaining bytes - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -#else -SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - return find_next_json_quotable_character_scalar(view, location); -} -#endif -#endif // !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - -SIMDJSON_CONSTEXPR_LAMBDA static std::string_view control_chars[] = { - "\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", "\\u0005", "\\u0006", - "\\u0007", "\\b", "\\t", "\\n", "\\u000b", "\\f", "\\r", - "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012", "\\u0013", "\\u0014", - "\\u0015", "\\u0016", "\\u0017", "\\u0018", "\\u0019", "\\u001a", "\\u001b", - "\\u001c", "\\u001d", "\\u001e", "\\u001f"}; - -// All Unicode characters may be placed within the quotation marks, except for -// the characters that MUST be escaped: quotation mark, reverse solidus, and the -// control characters (U+0000 through U+001F). There are two-character sequence -// escape representations of some popular characters: -// \", \\, \b, \f, \n, \r, \t. -SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline void escape_json_char(char c, char *&out) { - if (c == '"') { - memcpy(out, "\\\"", 2); - out += 2; - } else if (c == '\\') { - memcpy(out, "\\\\", 2); - out += 2; - } else { - std::string_view v = control_chars[uint8_t(c)]; - memcpy(out, v.data(), v.size()); - out += v.size(); - } -} - -#if SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 - -using escape_vector = __m128i; -// Mask bits that each input byte contributes to escape_bitmask(). -static constexpr unsigned escape_mask_bits = 1; - -simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept { - return _mm_loadu_si128(reinterpret_cast(p)); -} - -simdjson_inline void escape_store16(char *out, escape_vector v) noexcept { - _mm_storeu_si128(reinterpret_cast<__m128i *>(out), v); -} - -// Builds a vector whose bytes 0..7 come from a and bytes 8..15 from b. -simdjson_inline escape_vector escape_load8x2(const uint8_t *a, - const uint8_t *b) noexcept { - return _mm_unpacklo_epi64( - _mm_loadl_epi64(reinterpret_cast(a)), - _mm_loadl_epi64(reinterpret_cast(b))); -} - -// Builds a vector whose bytes 0..3 come from a and bytes 4..7 from b. The -// upper half is unspecified; callers only look at the low eight bits. -simdjson_inline escape_vector escape_load4x2(const uint8_t *a, - const uint8_t *b) noexcept { - int32_t a32, b32; - memcpy(&a32, a, 4); - memcpy(&b32, b, 4); - return _mm_unpacklo_epi32(_mm_cvtsi32_si128(a32), _mm_cvtsi32_si128(b32)); -} - -// Sets every bit of byte k when byte k of v is a quotable character ('"', -// '\\' or a control character). -simdjson_inline escape_vector escape_flags(escape_vector v) noexcept { - const __m128i v34 = _mm_set1_epi8(34); // '"' - const __m128i v92 = _mm_set1_epi8(92); // '\\' - const __m128i v31 = _mm_set1_epi8(31); // for control char detection - __m128i needs_escape = _mm_cmpeq_epi8(v, v34); - needs_escape = _mm_or_si128(needs_escape, _mm_cmpeq_epi8(v, v92)); - return _mm_or_si128( - needs_escape, _mm_cmpeq_epi8(_mm_subs_epu8(v, v31), _mm_setzero_si128())); -} - -// True when any byte needs escaping. Kept separate from escape_bitmask -// because some instruction sets can answer it without leaving the vector -// register file; on SSE2 the compiler folds the two together. -simdjson_inline bool escape_any(escape_vector flags) noexcept { - return _mm_movemask_epi8(flags) != 0; -} - -// A 16-bit mask with bit k set when byte k needed escaping. -simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept { - return uint64_t(uint32_t(_mm_movemask_epi8(flags))); -} - -#else // SIMDJSON_EXPERIMENTAL_HAS_NEON - -using escape_vector = uint8x16_t; -static constexpr unsigned escape_mask_bits = 4; - -simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept { - return vld1q_u8(p); -} - -simdjson_inline void escape_store16(char *out, escape_vector v) noexcept { - vst1q_u8(reinterpret_cast(out), v); -} - -simdjson_inline escape_vector escape_load8x2(const uint8_t *a, - const uint8_t *b) noexcept { - uint64_t a64, b64; - memcpy(&a64, a, 8); - memcpy(&b64, b, 8); - return vreinterpretq_u8_u64(vsetq_lane_u64(b64, vdupq_n_u64(a64), 1)); -} - -simdjson_inline escape_vector escape_load4x2(const uint8_t *a, - const uint8_t *b) noexcept { - uint32_t a32, b32; - memcpy(&a32, a, 4); - memcpy(&b32, b, 4); - return vreinterpretq_u8_u32(vsetq_lane_u32(b32, vdupq_n_u32(a32), 1)); -} - -simdjson_inline escape_vector escape_flags(escape_vector v) noexcept { - uint8x16_t needs_escape = vceqq_u8(v, vdupq_n_u8(34)); // '"' - needs_escape = vorrq_u8(needs_escape, vceqq_u8(v, vdupq_n_u8(92))); // '\\' - return vorrq_u8(needs_escape, vcltq_u8(v, vdupq_n_u8(32))); -} - -simdjson_inline bool escape_any(escape_vector flags) noexcept { - return vmaxvq_u32(vreinterpretq_u32_u8(flags)) != 0; -} - -// Four bits per byte rather than one: escape_block and the tail paths scale -// their shifts by escape_mask_bits to match. -simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept { - uint8x8_t narrowed = vshrn_n_u16(vreinterpretq_u16_u8(flags), 4); - return vget_lane_u64(vreinterpret_u64_u8(narrowed), 0); -} - -#endif // instruction set selection - -// The escape bitmask of a 16-byte block. -simdjson_inline uint64_t escape_mask(escape_vector v) noexcept { - return escape_bitmask(escape_flags(v)); -} - -// Copies n bytes with n < 16, using overlapping loads and stores. It never -// reads more than n bytes from src, nor writes more than n bytes to dst. -simdjson_inline void copy_lt16(char *dst, const uint8_t *src, - size_t n) noexcept { - if (n >= 8) { - memcpy(dst, src, 8); - memcpy(dst + n - 8, src + n - 8, 8); - } else if (n >= 4) { - memcpy(dst, src, 4); - memcpy(dst + n - 4, src + n - 4, 4); - } else if (n > 0) { - dst[0] = char(src[0]); - dst[n >> 1] = char(src[n >> 1]); - dst[n - 1] = char(src[n - 1]); - } -} - -// Escapes the bytes of src in the range [i, blockend), given that m is the -// (non-zero) escape mask for that range: the escape_mask_bits-wide lane of m -// at byte k is non-zero when src[i + k] requires escaping. Returns the updated -// output pointer. -simdjson_never_inline char *escape_block(const uint8_t *src, char *out, - size_t i, size_t blockend, - uint64_t m) noexcept { - constexpr uint64_t lane = (uint64_t(1) << escape_mask_bits) - 1; - - size_t pos = i; // first byte not yet copied - while (m) { - const size_t tz = trailing_zeroes(m); - const size_t next = i + tz / escape_mask_bits; - // Copy the run of safe bytes that precedes this escape. - copy_lt16(out, src + pos, next - pos); - out += next - pos; - escape_json_char(char(src[next]), out); - pos = next + 1; - m &= ~(lane << tz); - } - // Copy whatever follows the last escape. - copy_lt16(out, src + pos, blockend - pos); - return out + (blockend - pos); -} - -// Writes the escaped version of input to out, returning the number of bytes -// written. -simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out) { - const size_t len = input.size(); - const uint8_t *src = reinterpret_cast(input.data()); - const char *const initout = out; - - size_t i = 0; -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 && defined(__AVX2__) - while (i + 32 <= len) { - const __m256i word = _mm256_loadu_si256(reinterpret_cast(src + i)); - const __m256i flags = _mm256_or_si256( - _mm256_or_si256(_mm256_cmpeq_epi8(word, _mm256_set1_epi8(34)), // '"' - _mm256_cmpeq_epi8(word, _mm256_set1_epi8(92))), // '\\' - _mm256_cmpeq_epi8(_mm256_subs_epu8(word, _mm256_set1_epi8(31)), - _mm256_setzero_si256())); // control - const uint32_t mask = uint32_t(_mm256_movemask_epi8(flags)); - if (simdjson_likely(mask == 0)) { - _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), word); - out += 32; - } else { - for (size_t half = 0; half < 32; half += 16) { - const uint64_t m = (mask >> half) & 0xFFFF; - if (m == 0) { - escape_store16(out, escape_load16(src + i + half)); - out += 16; - } else { - out = escape_block(src, out, i + half, i + half + 16, m); - } - } - } - i += 32; - } -#endif - while (i + 16 <= len) { - escape_vector word = escape_load16(src + i); - escape_vector flags = escape_flags(word); - if (simdjson_likely(!escape_any(flags))) { - escape_store16(out, word); - out += 16; - } else { - out = escape_block(src, out, i, i + 16, escape_bitmask(flags)); - } - i += 16; - } - if (i < len) { - const size_t rem = len - i; - uint64_t m; - if (len >= 16) { - // The last 16 bytes of the input are in bounds. Bit k of that block's - // mask belongs to input position len - 16 + k, so shift it down to align - // bit 0 with position i. - m = escape_mask(escape_load16(src + len - 16)) >> - (escape_mask_bits * (16 - rem)); - } else if (len >= 8) { - // Here i == 0 and rem == len. Two overlapping 8-byte loads cover - // [0, 8) and [len - 8, len), which is the whole input since len < 16. - uint64_t mm = escape_mask(escape_load8x2(src, src + len - 8)); - constexpr uint64_t low8 = (uint64_t(1) << (escape_mask_bits * 8)) - 1; - m = (mm & low8) | - ((mm >> (escape_mask_bits * 8)) << (escape_mask_bits * (len - 8))); - } else if (len >= 4) { - // Same idea with two overlapping 4-byte loads. - uint64_t mm = escape_mask(escape_load4x2(src, src + len - 4)); - constexpr uint64_t low4 = (uint64_t(1) << (escape_mask_bits * 4)) - 1; - m = (mm & low4) | (((mm >> (escape_mask_bits * 4)) & low4) - << (escape_mask_bits * (len - 4))); - } else { - // Fewer than 4 bytes: at most three table lookups, no need for SIMD. - for (size_t k = 0; k < len; k++) { - uint8_t c = src[k]; - if (json_quotable_character[c]) { - escape_json_char(char(c), out); - } else { - *out++ = char(c); - } - } - return size_t(out - initout); - } - if (m == 0) { - copy_lt16(out, src + i, rem); - out += rem; - } else { - out = escape_block(src, out, i, len, m); - } - } - return size_t(out - initout); -} - -#else // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - -// Writes the escaped version of input to out, returning the number of bytes -// written. Uses SIMD position finding to locate quotable characters efficiently. -inline size_t write_string_escaped(const std::string_view input, char *out) { - size_t mysize = input.size(); - - // Use SIMD position finder directly - it returns mysize if no escape needed - size_t location = find_next_json_quotable_character(input, 0); - if (location == mysize) { - // Fast path: no escaping needed - memcpy(out, input.data(), input.size()); - return input.size(); - } - - const char *const initout = out; - memcpy(out, input.data(), location); - out += location; - escape_json_char(input[location], out); - location += 1; - while (location < mysize) { - size_t newlocation = find_next_json_quotable_character(input, location); - memcpy(out, input.data() + location, newlocation - location); - out += newlocation - location; - location = newlocation; - if (location == mysize) { - break; - } - escape_json_char(input[location], out); - location += 1; - } - return size_t(out - initout); -} - -#endif // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE -#undef SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - namespace internal { // Sets the size of s to n without initializing the new bytes when the @@ -65128,6 +62252,7 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ namespace simdjson { @@ -65164,6 +62289,9 @@ struct has_custom_serialization : std::false_type {}; namespace lsx { namespace builder { + +using simdjson::internal::escape_best::write_string_escaped; + /** * A builder for JSON strings representing documents. This is a low-level * builder that is not meant to be used directly by end-users. Though it @@ -65510,6 +62638,7 @@ simdjson_warn_unused error_code to_json(const Z &z, std::string &s, size_t initi /* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ /* amalgamation skipped (editor-only): #include "simdjson/concepts.h" */ /* amalgamation skipped (editor-only): #include "simdjson/annotations.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ #if SIMDJSON_STATIC_REFLECTION @@ -65535,7 +62664,6 @@ namespace internal { using simdjson::internal::write_uint_jeaiii; simdjson_inline char *write_double(char *p, double v) noexcept; } // namespace internal -simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out); SIMDJSON_PUSH_DISABLE_WARNINGS SIMDJSON_DISABLE_GCC_WARNING(-Warray-bounds) @@ -66556,619 +63684,13 @@ simdjson_warn_unused simdjson_result extract_fractured_json( /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_INL_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/jeaiii.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ -/* - * Empirically, we have found that an inlined optimization is important for - * performance. The following macros are not ideal. We should find a better - * way to inline the code. - */ - -#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \ - (defined(_M_AMD64) || defined(_M_X64) || \ - (defined(_M_IX86_FP) && _M_IX86_FP == 2)) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2 -#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1 -#endif -#endif - -#if defined(__aarch64__) || defined(_M_ARM64) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON -#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1 -#endif -#endif -#if defined(__loongarch_sx) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX -#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1 -#endif -#endif -#if defined(__loongarch_asx) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_LASX -#define SIMDJSON_EXPERIMENTAL_HAS_LASX 1 -#endif -#endif -#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \ - defined(__riscv_vector) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV -#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 -#endif -#endif -#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 -#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_NEON -#include -#ifdef _MSC_VER -#include -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 -#include -#if defined(__AVX2__) -#include -#endif -#ifdef _MSC_VER -#include -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_LSX -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_LASX -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_RVV -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_PPC64 -#include -#ifdef bool -#undef bool -#endif -#ifdef vector -#undef vector -#endif -#endif - - namespace simdjson { namespace lsx { namespace builder { -static SIMDJSON_CONSTEXPR_LAMBDA std::array - json_quotable_character = { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - -/** - -A possible SWAR implementation of has_json_escapable_byte. It is not used -because it is slower than the current implementation. It is kept here for -reference (to show that we tried it). - -inline bool has_json_escapable_byte(uint64_t x) { - uint64_t is_ascii = 0x8080808080808080ULL & ~x; - uint64_t xor2 = x ^ 0x0202020202020202ULL; - uint64_t lt32_or_eq34 = xor2 - 0x2121212121212121ULL; - uint64_t sub92 = x ^ 0x5C5C5C5C5C5C5C5CULL; - uint64_t eq92 = (sub92 - 0x0101010101010101ULL); - return ((lt32_or_eq34 | eq92) & is_ascii) != 0; -} - -**/ - -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 || SIMDJSON_EXPERIMENTAL_HAS_NEON -#define SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE 1 -#endif - -// With block escaping (SSE2/NEON), write_string_escaped scans the input itself. -#if !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE -// Scalar fallback for finding next quotable character -SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t -find_next_json_quotable_character_scalar(const std::string_view view, - size_t location) noexcept { - for (auto pos = view.begin() + location; pos != view.end(); ++pos) { - if (json_quotable_character[static_cast(*pos)]) { - return pos - view.begin(); - } - } - return size_t(view.size()); -} - -// SIMD-accelerated position finding that directly locates the first quotable -// character, combining detection and position extraction in a single pass to -// minimize redundant work. -#if SIMDJSON_EXPERIMENTAL_HAS_LASX -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - // SIMD constants for characters requiring escape - __m256i v34 = __lasx_xvreplgr2vr_b(34); // '"' - __m256i v92 = __lasx_xvreplgr2vr_b(92); // '\\' - __m256i v32 = __lasx_xvreplgr2vr_b(32); // control char threshold - - while (remaining >= 32) { - __m256i word = __lasx_xvld(ptr, 0); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - __m256i needs_escape = __lasx_xvseq_b(word, v34); - needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvseq_b(word, v92)); - needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvslt_bu(word, v32)); - - if (!__lasx_xbz_v(needs_escape)) { - // Found a quotable character - locate it via the four 64-bit lanes - uint64_t lane0 = __lasx_xvpickve2gr_du(needs_escape, 0); - uint64_t lane1 = __lasx_xvpickve2gr_du(needs_escape, 1); - uint64_t lane2 = __lasx_xvpickve2gr_du(needs_escape, 2); - uint64_t lane3 = __lasx_xvpickve2gr_du(needs_escape, 3); - size_t offset = ptr - reinterpret_cast(view.data()); - if (lane0 != 0) { - return offset + trailing_zeroes(lane0) / 8; - } else if (lane1 != 0) { - return offset + 8 + trailing_zeroes(lane1) / 8; - } else if (lane2 != 0) { - return offset + 16 + trailing_zeroes(lane2) / 8; - } else { - return offset + 24 + trailing_zeroes(lane3) / 8; - } - } - ptr += 32; - remaining -= 32; - } - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -#elif SIMDJSON_EXPERIMENTAL_HAS_LSX -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - //SIMD constants for characters requiring escape - __m128i v34 = __lsx_vreplgr2vr_b(34); // '"' - __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\' - __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold - - while (remaining >= 16){ - __m128i word = __lsx_vld(ptr, 0); - - //Check for the quotable characters: '"', '\\', or control char (<32) - __m128i needs_escape = __lsx_vseq_b(word, v34); - needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92)); - needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32)); - - if (!__lsx_bz_v(needs_escape)){ - - //Found quotable character - extract exact byte position - uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0); - uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1); - size_t offset = ptr - reinterpret_cast(view.data()); - if ( lo != 0) { - return offset + trailing_zeroes(lo) / 8; - } else { - return offset + 8 + trailing_zeroes(hi) / 8; - } - } - ptr += 16; - remaining -= 16; - } - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -#elif SIMDJSON_EXPERIMENTAL_HAS_RVV -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - while (remaining > 0) { - size_t vl = __riscv_vsetvl_e8m1(remaining); - vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl); - needs_escape = __riscv_vmor(needs_escape, - __riscv_vmseq(word, (uint8_t)92, vl), vl); - needs_escape = __riscv_vmor(needs_escape, - __riscv_vmsltu(word, (uint8_t)32, vl), vl); - - long first = __riscv_vfirst(needs_escape, vl); - if (first >= 0) { - size_t offset = ptr - reinterpret_cast(view.data()); - return offset + first; - } - ptr += vl; - remaining -= vl; - } - - return len; -} -#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64 -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - // SIMD constants for characters requiring escape - __vector unsigned char v34 = vec_splats((unsigned char)34); // '"' - __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\' - __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold - - // Bitmask for vec_vbpermq to extract one bit per byte - const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50, - 0x48, 0x40, 0x38, 0x30, 0x28, 0x20, - 0x18, 0x10, 0x08, 0x00}; - - while (remaining >= 16) { - __vector unsigned char word = - vec_vsx_ld(0, reinterpret_cast(ptr)); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - __vector unsigned char needs_escape = - (__vector unsigned char)vec_cmpeq(word, v34); - needs_escape = vec_or(needs_escape, - (__vector unsigned char)vec_cmpeq(word, v92)); - needs_escape = vec_or(needs_escape, - (__vector unsigned char)vec_cmplt(word, v32)); - - __vector unsigned long long result = - (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask); -#ifdef __LITTLE_ENDIAN__ - unsigned int mask = static_cast(result[1]); -#else - unsigned int mask = static_cast(result[0]); -#endif - if (mask != 0) { - size_t offset = ptr - reinterpret_cast(view.data()); - return offset + __builtin_ctz(mask); - } - ptr += 16; - remaining -= 16; - } - - // Scalar fallback for remaining bytes - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -#else -SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - return find_next_json_quotable_character_scalar(view, location); -} -#endif -#endif // !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - -SIMDJSON_CONSTEXPR_LAMBDA static std::string_view control_chars[] = { - "\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", "\\u0005", "\\u0006", - "\\u0007", "\\b", "\\t", "\\n", "\\u000b", "\\f", "\\r", - "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012", "\\u0013", "\\u0014", - "\\u0015", "\\u0016", "\\u0017", "\\u0018", "\\u0019", "\\u001a", "\\u001b", - "\\u001c", "\\u001d", "\\u001e", "\\u001f"}; - -// All Unicode characters may be placed within the quotation marks, except for -// the characters that MUST be escaped: quotation mark, reverse solidus, and the -// control characters (U+0000 through U+001F). There are two-character sequence -// escape representations of some popular characters: -// \", \\, \b, \f, \n, \r, \t. -SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline void escape_json_char(char c, char *&out) { - if (c == '"') { - memcpy(out, "\\\"", 2); - out += 2; - } else if (c == '\\') { - memcpy(out, "\\\\", 2); - out += 2; - } else { - std::string_view v = control_chars[uint8_t(c)]; - memcpy(out, v.data(), v.size()); - out += v.size(); - } -} - -#if SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 - -using escape_vector = __m128i; -// Mask bits that each input byte contributes to escape_bitmask(). -static constexpr unsigned escape_mask_bits = 1; - -simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept { - return _mm_loadu_si128(reinterpret_cast(p)); -} - -simdjson_inline void escape_store16(char *out, escape_vector v) noexcept { - _mm_storeu_si128(reinterpret_cast<__m128i *>(out), v); -} - -// Builds a vector whose bytes 0..7 come from a and bytes 8..15 from b. -simdjson_inline escape_vector escape_load8x2(const uint8_t *a, - const uint8_t *b) noexcept { - return _mm_unpacklo_epi64( - _mm_loadl_epi64(reinterpret_cast(a)), - _mm_loadl_epi64(reinterpret_cast(b))); -} - -// Builds a vector whose bytes 0..3 come from a and bytes 4..7 from b. The -// upper half is unspecified; callers only look at the low eight bits. -simdjson_inline escape_vector escape_load4x2(const uint8_t *a, - const uint8_t *b) noexcept { - int32_t a32, b32; - memcpy(&a32, a, 4); - memcpy(&b32, b, 4); - return _mm_unpacklo_epi32(_mm_cvtsi32_si128(a32), _mm_cvtsi32_si128(b32)); -} - -// Sets every bit of byte k when byte k of v is a quotable character ('"', -// '\\' or a control character). -simdjson_inline escape_vector escape_flags(escape_vector v) noexcept { - const __m128i v34 = _mm_set1_epi8(34); // '"' - const __m128i v92 = _mm_set1_epi8(92); // '\\' - const __m128i v31 = _mm_set1_epi8(31); // for control char detection - __m128i needs_escape = _mm_cmpeq_epi8(v, v34); - needs_escape = _mm_or_si128(needs_escape, _mm_cmpeq_epi8(v, v92)); - return _mm_or_si128( - needs_escape, _mm_cmpeq_epi8(_mm_subs_epu8(v, v31), _mm_setzero_si128())); -} - -// True when any byte needs escaping. Kept separate from escape_bitmask -// because some instruction sets can answer it without leaving the vector -// register file; on SSE2 the compiler folds the two together. -simdjson_inline bool escape_any(escape_vector flags) noexcept { - return _mm_movemask_epi8(flags) != 0; -} - -// A 16-bit mask with bit k set when byte k needed escaping. -simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept { - return uint64_t(uint32_t(_mm_movemask_epi8(flags))); -} - -#else // SIMDJSON_EXPERIMENTAL_HAS_NEON - -using escape_vector = uint8x16_t; -static constexpr unsigned escape_mask_bits = 4; - -simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept { - return vld1q_u8(p); -} - -simdjson_inline void escape_store16(char *out, escape_vector v) noexcept { - vst1q_u8(reinterpret_cast(out), v); -} - -simdjson_inline escape_vector escape_load8x2(const uint8_t *a, - const uint8_t *b) noexcept { - uint64_t a64, b64; - memcpy(&a64, a, 8); - memcpy(&b64, b, 8); - return vreinterpretq_u8_u64(vsetq_lane_u64(b64, vdupq_n_u64(a64), 1)); -} - -simdjson_inline escape_vector escape_load4x2(const uint8_t *a, - const uint8_t *b) noexcept { - uint32_t a32, b32; - memcpy(&a32, a, 4); - memcpy(&b32, b, 4); - return vreinterpretq_u8_u32(vsetq_lane_u32(b32, vdupq_n_u32(a32), 1)); -} - -simdjson_inline escape_vector escape_flags(escape_vector v) noexcept { - uint8x16_t needs_escape = vceqq_u8(v, vdupq_n_u8(34)); // '"' - needs_escape = vorrq_u8(needs_escape, vceqq_u8(v, vdupq_n_u8(92))); // '\\' - return vorrq_u8(needs_escape, vcltq_u8(v, vdupq_n_u8(32))); -} - -simdjson_inline bool escape_any(escape_vector flags) noexcept { - return vmaxvq_u32(vreinterpretq_u32_u8(flags)) != 0; -} - -// Four bits per byte rather than one: escape_block and the tail paths scale -// their shifts by escape_mask_bits to match. -simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept { - uint8x8_t narrowed = vshrn_n_u16(vreinterpretq_u16_u8(flags), 4); - return vget_lane_u64(vreinterpret_u64_u8(narrowed), 0); -} - -#endif // instruction set selection - -// The escape bitmask of a 16-byte block. -simdjson_inline uint64_t escape_mask(escape_vector v) noexcept { - return escape_bitmask(escape_flags(v)); -} - -// Copies n bytes with n < 16, using overlapping loads and stores. It never -// reads more than n bytes from src, nor writes more than n bytes to dst. -simdjson_inline void copy_lt16(char *dst, const uint8_t *src, - size_t n) noexcept { - if (n >= 8) { - memcpy(dst, src, 8); - memcpy(dst + n - 8, src + n - 8, 8); - } else if (n >= 4) { - memcpy(dst, src, 4); - memcpy(dst + n - 4, src + n - 4, 4); - } else if (n > 0) { - dst[0] = char(src[0]); - dst[n >> 1] = char(src[n >> 1]); - dst[n - 1] = char(src[n - 1]); - } -} - -// Escapes the bytes of src in the range [i, blockend), given that m is the -// (non-zero) escape mask for that range: the escape_mask_bits-wide lane of m -// at byte k is non-zero when src[i + k] requires escaping. Returns the updated -// output pointer. -simdjson_never_inline char *escape_block(const uint8_t *src, char *out, - size_t i, size_t blockend, - uint64_t m) noexcept { - constexpr uint64_t lane = (uint64_t(1) << escape_mask_bits) - 1; - - size_t pos = i; // first byte not yet copied - while (m) { - const size_t tz = trailing_zeroes(m); - const size_t next = i + tz / escape_mask_bits; - // Copy the run of safe bytes that precedes this escape. - copy_lt16(out, src + pos, next - pos); - out += next - pos; - escape_json_char(char(src[next]), out); - pos = next + 1; - m &= ~(lane << tz); - } - // Copy whatever follows the last escape. - copy_lt16(out, src + pos, blockend - pos); - return out + (blockend - pos); -} - -// Writes the escaped version of input to out, returning the number of bytes -// written. -simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out) { - const size_t len = input.size(); - const uint8_t *src = reinterpret_cast(input.data()); - const char *const initout = out; - - size_t i = 0; -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 && defined(__AVX2__) - while (i + 32 <= len) { - const __m256i word = _mm256_loadu_si256(reinterpret_cast(src + i)); - const __m256i flags = _mm256_or_si256( - _mm256_or_si256(_mm256_cmpeq_epi8(word, _mm256_set1_epi8(34)), // '"' - _mm256_cmpeq_epi8(word, _mm256_set1_epi8(92))), // '\\' - _mm256_cmpeq_epi8(_mm256_subs_epu8(word, _mm256_set1_epi8(31)), - _mm256_setzero_si256())); // control - const uint32_t mask = uint32_t(_mm256_movemask_epi8(flags)); - if (simdjson_likely(mask == 0)) { - _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), word); - out += 32; - } else { - for (size_t half = 0; half < 32; half += 16) { - const uint64_t m = (mask >> half) & 0xFFFF; - if (m == 0) { - escape_store16(out, escape_load16(src + i + half)); - out += 16; - } else { - out = escape_block(src, out, i + half, i + half + 16, m); - } - } - } - i += 32; - } -#endif - while (i + 16 <= len) { - escape_vector word = escape_load16(src + i); - escape_vector flags = escape_flags(word); - if (simdjson_likely(!escape_any(flags))) { - escape_store16(out, word); - out += 16; - } else { - out = escape_block(src, out, i, i + 16, escape_bitmask(flags)); - } - i += 16; - } - if (i < len) { - const size_t rem = len - i; - uint64_t m; - if (len >= 16) { - // The last 16 bytes of the input are in bounds. Bit k of that block's - // mask belongs to input position len - 16 + k, so shift it down to align - // bit 0 with position i. - m = escape_mask(escape_load16(src + len - 16)) >> - (escape_mask_bits * (16 - rem)); - } else if (len >= 8) { - // Here i == 0 and rem == len. Two overlapping 8-byte loads cover - // [0, 8) and [len - 8, len), which is the whole input since len < 16. - uint64_t mm = escape_mask(escape_load8x2(src, src + len - 8)); - constexpr uint64_t low8 = (uint64_t(1) << (escape_mask_bits * 8)) - 1; - m = (mm & low8) | - ((mm >> (escape_mask_bits * 8)) << (escape_mask_bits * (len - 8))); - } else if (len >= 4) { - // Same idea with two overlapping 4-byte loads. - uint64_t mm = escape_mask(escape_load4x2(src, src + len - 4)); - constexpr uint64_t low4 = (uint64_t(1) << (escape_mask_bits * 4)) - 1; - m = (mm & low4) | (((mm >> (escape_mask_bits * 4)) & low4) - << (escape_mask_bits * (len - 4))); - } else { - // Fewer than 4 bytes: at most three table lookups, no need for SIMD. - for (size_t k = 0; k < len; k++) { - uint8_t c = src[k]; - if (json_quotable_character[c]) { - escape_json_char(char(c), out); - } else { - *out++ = char(c); - } - } - return size_t(out - initout); - } - if (m == 0) { - copy_lt16(out, src + i, rem); - out += rem; - } else { - out = escape_block(src, out, i, len, m); - } - } - return size_t(out - initout); -} - -#else // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - -// Writes the escaped version of input to out, returning the number of bytes -// written. Uses SIMD position finding to locate quotable characters efficiently. -inline size_t write_string_escaped(const std::string_view input, char *out) { - size_t mysize = input.size(); - - // Use SIMD position finder directly - it returns mysize if no escape needed - size_t location = find_next_json_quotable_character(input, 0); - if (location == mysize) { - // Fast path: no escaping needed - memcpy(out, input.data(), input.size()); - return input.size(); - } - - const char *const initout = out; - memcpy(out, input.data(), location); - out += location; - escape_json_char(input[location], out); - location += 1; - while (location < mysize) { - size_t newlocation = find_next_json_quotable_character(input, location); - memcpy(out, input.data() + location, newlocation - location); - out += newlocation - location; - location = newlocation; - if (location == mysize) { - break; - } - escape_json_char(input[location], out); - location += 1; - } - return size_t(out - initout); -} - -#endif // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE -#undef SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - namespace internal { // Sets the size of s to n without initializing the new bytes when the @@ -68428,6 +64950,7 @@ simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uin /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ namespace simdjson { @@ -68464,6 +64987,9 @@ struct has_custom_serialization : std::false_type {}; namespace lasx { namespace builder { + +using simdjson::internal::escape_best::write_string_escaped; + /** * A builder for JSON strings representing documents. This is a low-level * builder that is not meant to be used directly by end-users. Though it @@ -68810,6 +65336,7 @@ simdjson_warn_unused error_code to_json(const Z &z, std::string &s, size_t initi /* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ /* amalgamation skipped (editor-only): #include "simdjson/concepts.h" */ /* amalgamation skipped (editor-only): #include "simdjson/annotations.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ #if SIMDJSON_STATIC_REFLECTION @@ -68835,7 +65362,6 @@ namespace internal { using simdjson::internal::write_uint_jeaiii; simdjson_inline char *write_double(char *p, double v) noexcept; } // namespace internal -simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out); SIMDJSON_PUSH_DISABLE_WARNINGS SIMDJSON_DISABLE_GCC_WARNING(-Warray-bounds) @@ -69856,619 +66382,13 @@ simdjson_warn_unused simdjson_result extract_fractured_json( /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_INL_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/jeaiii.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ -/* - * Empirically, we have found that an inlined optimization is important for - * performance. The following macros are not ideal. We should find a better - * way to inline the code. - */ - -#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \ - (defined(_M_AMD64) || defined(_M_X64) || \ - (defined(_M_IX86_FP) && _M_IX86_FP == 2)) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2 -#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1 -#endif -#endif - -#if defined(__aarch64__) || defined(_M_ARM64) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON -#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1 -#endif -#endif -#if defined(__loongarch_sx) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX -#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1 -#endif -#endif -#if defined(__loongarch_asx) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_LASX -#define SIMDJSON_EXPERIMENTAL_HAS_LASX 1 -#endif -#endif -#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \ - defined(__riscv_vector) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV -#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 -#endif -#endif -#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 -#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_NEON -#include -#ifdef _MSC_VER -#include -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 -#include -#if defined(__AVX2__) -#include -#endif -#ifdef _MSC_VER -#include -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_LSX -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_LASX -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_RVV -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_PPC64 -#include -#ifdef bool -#undef bool -#endif -#ifdef vector -#undef vector -#endif -#endif - - namespace simdjson { namespace lasx { namespace builder { -static SIMDJSON_CONSTEXPR_LAMBDA std::array - json_quotable_character = { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - -/** - -A possible SWAR implementation of has_json_escapable_byte. It is not used -because it is slower than the current implementation. It is kept here for -reference (to show that we tried it). - -inline bool has_json_escapable_byte(uint64_t x) { - uint64_t is_ascii = 0x8080808080808080ULL & ~x; - uint64_t xor2 = x ^ 0x0202020202020202ULL; - uint64_t lt32_or_eq34 = xor2 - 0x2121212121212121ULL; - uint64_t sub92 = x ^ 0x5C5C5C5C5C5C5C5CULL; - uint64_t eq92 = (sub92 - 0x0101010101010101ULL); - return ((lt32_or_eq34 | eq92) & is_ascii) != 0; -} - -**/ - -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 || SIMDJSON_EXPERIMENTAL_HAS_NEON -#define SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE 1 -#endif - -// With block escaping (SSE2/NEON), write_string_escaped scans the input itself. -#if !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE -// Scalar fallback for finding next quotable character -SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t -find_next_json_quotable_character_scalar(const std::string_view view, - size_t location) noexcept { - for (auto pos = view.begin() + location; pos != view.end(); ++pos) { - if (json_quotable_character[static_cast(*pos)]) { - return pos - view.begin(); - } - } - return size_t(view.size()); -} - -// SIMD-accelerated position finding that directly locates the first quotable -// character, combining detection and position extraction in a single pass to -// minimize redundant work. -#if SIMDJSON_EXPERIMENTAL_HAS_LASX -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - // SIMD constants for characters requiring escape - __m256i v34 = __lasx_xvreplgr2vr_b(34); // '"' - __m256i v92 = __lasx_xvreplgr2vr_b(92); // '\\' - __m256i v32 = __lasx_xvreplgr2vr_b(32); // control char threshold - - while (remaining >= 32) { - __m256i word = __lasx_xvld(ptr, 0); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - __m256i needs_escape = __lasx_xvseq_b(word, v34); - needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvseq_b(word, v92)); - needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvslt_bu(word, v32)); - - if (!__lasx_xbz_v(needs_escape)) { - // Found a quotable character - locate it via the four 64-bit lanes - uint64_t lane0 = __lasx_xvpickve2gr_du(needs_escape, 0); - uint64_t lane1 = __lasx_xvpickve2gr_du(needs_escape, 1); - uint64_t lane2 = __lasx_xvpickve2gr_du(needs_escape, 2); - uint64_t lane3 = __lasx_xvpickve2gr_du(needs_escape, 3); - size_t offset = ptr - reinterpret_cast(view.data()); - if (lane0 != 0) { - return offset + trailing_zeroes(lane0) / 8; - } else if (lane1 != 0) { - return offset + 8 + trailing_zeroes(lane1) / 8; - } else if (lane2 != 0) { - return offset + 16 + trailing_zeroes(lane2) / 8; - } else { - return offset + 24 + trailing_zeroes(lane3) / 8; - } - } - ptr += 32; - remaining -= 32; - } - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -#elif SIMDJSON_EXPERIMENTAL_HAS_LSX -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - //SIMD constants for characters requiring escape - __m128i v34 = __lsx_vreplgr2vr_b(34); // '"' - __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\' - __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold - - while (remaining >= 16){ - __m128i word = __lsx_vld(ptr, 0); - - //Check for the quotable characters: '"', '\\', or control char (<32) - __m128i needs_escape = __lsx_vseq_b(word, v34); - needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92)); - needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32)); - - if (!__lsx_bz_v(needs_escape)){ - - //Found quotable character - extract exact byte position - uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0); - uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1); - size_t offset = ptr - reinterpret_cast(view.data()); - if ( lo != 0) { - return offset + trailing_zeroes(lo) / 8; - } else { - return offset + 8 + trailing_zeroes(hi) / 8; - } - } - ptr += 16; - remaining -= 16; - } - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -#elif SIMDJSON_EXPERIMENTAL_HAS_RVV -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - while (remaining > 0) { - size_t vl = __riscv_vsetvl_e8m1(remaining); - vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl); - needs_escape = __riscv_vmor(needs_escape, - __riscv_vmseq(word, (uint8_t)92, vl), vl); - needs_escape = __riscv_vmor(needs_escape, - __riscv_vmsltu(word, (uint8_t)32, vl), vl); - - long first = __riscv_vfirst(needs_escape, vl); - if (first >= 0) { - size_t offset = ptr - reinterpret_cast(view.data()); - return offset + first; - } - ptr += vl; - remaining -= vl; - } - - return len; -} -#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64 -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - // SIMD constants for characters requiring escape - __vector unsigned char v34 = vec_splats((unsigned char)34); // '"' - __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\' - __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold - - // Bitmask for vec_vbpermq to extract one bit per byte - const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50, - 0x48, 0x40, 0x38, 0x30, 0x28, 0x20, - 0x18, 0x10, 0x08, 0x00}; - - while (remaining >= 16) { - __vector unsigned char word = - vec_vsx_ld(0, reinterpret_cast(ptr)); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - __vector unsigned char needs_escape = - (__vector unsigned char)vec_cmpeq(word, v34); - needs_escape = vec_or(needs_escape, - (__vector unsigned char)vec_cmpeq(word, v92)); - needs_escape = vec_or(needs_escape, - (__vector unsigned char)vec_cmplt(word, v32)); - - __vector unsigned long long result = - (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask); -#ifdef __LITTLE_ENDIAN__ - unsigned int mask = static_cast(result[1]); -#else - unsigned int mask = static_cast(result[0]); -#endif - if (mask != 0) { - size_t offset = ptr - reinterpret_cast(view.data()); - return offset + __builtin_ctz(mask); - } - ptr += 16; - remaining -= 16; - } - - // Scalar fallback for remaining bytes - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -#else -SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - return find_next_json_quotable_character_scalar(view, location); -} -#endif -#endif // !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - -SIMDJSON_CONSTEXPR_LAMBDA static std::string_view control_chars[] = { - "\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", "\\u0005", "\\u0006", - "\\u0007", "\\b", "\\t", "\\n", "\\u000b", "\\f", "\\r", - "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012", "\\u0013", "\\u0014", - "\\u0015", "\\u0016", "\\u0017", "\\u0018", "\\u0019", "\\u001a", "\\u001b", - "\\u001c", "\\u001d", "\\u001e", "\\u001f"}; - -// All Unicode characters may be placed within the quotation marks, except for -// the characters that MUST be escaped: quotation mark, reverse solidus, and the -// control characters (U+0000 through U+001F). There are two-character sequence -// escape representations of some popular characters: -// \", \\, \b, \f, \n, \r, \t. -SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline void escape_json_char(char c, char *&out) { - if (c == '"') { - memcpy(out, "\\\"", 2); - out += 2; - } else if (c == '\\') { - memcpy(out, "\\\\", 2); - out += 2; - } else { - std::string_view v = control_chars[uint8_t(c)]; - memcpy(out, v.data(), v.size()); - out += v.size(); - } -} - -#if SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 - -using escape_vector = __m128i; -// Mask bits that each input byte contributes to escape_bitmask(). -static constexpr unsigned escape_mask_bits = 1; - -simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept { - return _mm_loadu_si128(reinterpret_cast(p)); -} - -simdjson_inline void escape_store16(char *out, escape_vector v) noexcept { - _mm_storeu_si128(reinterpret_cast<__m128i *>(out), v); -} - -// Builds a vector whose bytes 0..7 come from a and bytes 8..15 from b. -simdjson_inline escape_vector escape_load8x2(const uint8_t *a, - const uint8_t *b) noexcept { - return _mm_unpacklo_epi64( - _mm_loadl_epi64(reinterpret_cast(a)), - _mm_loadl_epi64(reinterpret_cast(b))); -} - -// Builds a vector whose bytes 0..3 come from a and bytes 4..7 from b. The -// upper half is unspecified; callers only look at the low eight bits. -simdjson_inline escape_vector escape_load4x2(const uint8_t *a, - const uint8_t *b) noexcept { - int32_t a32, b32; - memcpy(&a32, a, 4); - memcpy(&b32, b, 4); - return _mm_unpacklo_epi32(_mm_cvtsi32_si128(a32), _mm_cvtsi32_si128(b32)); -} - -// Sets every bit of byte k when byte k of v is a quotable character ('"', -// '\\' or a control character). -simdjson_inline escape_vector escape_flags(escape_vector v) noexcept { - const __m128i v34 = _mm_set1_epi8(34); // '"' - const __m128i v92 = _mm_set1_epi8(92); // '\\' - const __m128i v31 = _mm_set1_epi8(31); // for control char detection - __m128i needs_escape = _mm_cmpeq_epi8(v, v34); - needs_escape = _mm_or_si128(needs_escape, _mm_cmpeq_epi8(v, v92)); - return _mm_or_si128( - needs_escape, _mm_cmpeq_epi8(_mm_subs_epu8(v, v31), _mm_setzero_si128())); -} - -// True when any byte needs escaping. Kept separate from escape_bitmask -// because some instruction sets can answer it without leaving the vector -// register file; on SSE2 the compiler folds the two together. -simdjson_inline bool escape_any(escape_vector flags) noexcept { - return _mm_movemask_epi8(flags) != 0; -} - -// A 16-bit mask with bit k set when byte k needed escaping. -simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept { - return uint64_t(uint32_t(_mm_movemask_epi8(flags))); -} - -#else // SIMDJSON_EXPERIMENTAL_HAS_NEON - -using escape_vector = uint8x16_t; -static constexpr unsigned escape_mask_bits = 4; - -simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept { - return vld1q_u8(p); -} - -simdjson_inline void escape_store16(char *out, escape_vector v) noexcept { - vst1q_u8(reinterpret_cast(out), v); -} - -simdjson_inline escape_vector escape_load8x2(const uint8_t *a, - const uint8_t *b) noexcept { - uint64_t a64, b64; - memcpy(&a64, a, 8); - memcpy(&b64, b, 8); - return vreinterpretq_u8_u64(vsetq_lane_u64(b64, vdupq_n_u64(a64), 1)); -} - -simdjson_inline escape_vector escape_load4x2(const uint8_t *a, - const uint8_t *b) noexcept { - uint32_t a32, b32; - memcpy(&a32, a, 4); - memcpy(&b32, b, 4); - return vreinterpretq_u8_u32(vsetq_lane_u32(b32, vdupq_n_u32(a32), 1)); -} - -simdjson_inline escape_vector escape_flags(escape_vector v) noexcept { - uint8x16_t needs_escape = vceqq_u8(v, vdupq_n_u8(34)); // '"' - needs_escape = vorrq_u8(needs_escape, vceqq_u8(v, vdupq_n_u8(92))); // '\\' - return vorrq_u8(needs_escape, vcltq_u8(v, vdupq_n_u8(32))); -} - -simdjson_inline bool escape_any(escape_vector flags) noexcept { - return vmaxvq_u32(vreinterpretq_u32_u8(flags)) != 0; -} - -// Four bits per byte rather than one: escape_block and the tail paths scale -// their shifts by escape_mask_bits to match. -simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept { - uint8x8_t narrowed = vshrn_n_u16(vreinterpretq_u16_u8(flags), 4); - return vget_lane_u64(vreinterpret_u64_u8(narrowed), 0); -} - -#endif // instruction set selection - -// The escape bitmask of a 16-byte block. -simdjson_inline uint64_t escape_mask(escape_vector v) noexcept { - return escape_bitmask(escape_flags(v)); -} - -// Copies n bytes with n < 16, using overlapping loads and stores. It never -// reads more than n bytes from src, nor writes more than n bytes to dst. -simdjson_inline void copy_lt16(char *dst, const uint8_t *src, - size_t n) noexcept { - if (n >= 8) { - memcpy(dst, src, 8); - memcpy(dst + n - 8, src + n - 8, 8); - } else if (n >= 4) { - memcpy(dst, src, 4); - memcpy(dst + n - 4, src + n - 4, 4); - } else if (n > 0) { - dst[0] = char(src[0]); - dst[n >> 1] = char(src[n >> 1]); - dst[n - 1] = char(src[n - 1]); - } -} - -// Escapes the bytes of src in the range [i, blockend), given that m is the -// (non-zero) escape mask for that range: the escape_mask_bits-wide lane of m -// at byte k is non-zero when src[i + k] requires escaping. Returns the updated -// output pointer. -simdjson_never_inline char *escape_block(const uint8_t *src, char *out, - size_t i, size_t blockend, - uint64_t m) noexcept { - constexpr uint64_t lane = (uint64_t(1) << escape_mask_bits) - 1; - - size_t pos = i; // first byte not yet copied - while (m) { - const size_t tz = trailing_zeroes(m); - const size_t next = i + tz / escape_mask_bits; - // Copy the run of safe bytes that precedes this escape. - copy_lt16(out, src + pos, next - pos); - out += next - pos; - escape_json_char(char(src[next]), out); - pos = next + 1; - m &= ~(lane << tz); - } - // Copy whatever follows the last escape. - copy_lt16(out, src + pos, blockend - pos); - return out + (blockend - pos); -} - -// Writes the escaped version of input to out, returning the number of bytes -// written. -simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out) { - const size_t len = input.size(); - const uint8_t *src = reinterpret_cast(input.data()); - const char *const initout = out; - - size_t i = 0; -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 && defined(__AVX2__) - while (i + 32 <= len) { - const __m256i word = _mm256_loadu_si256(reinterpret_cast(src + i)); - const __m256i flags = _mm256_or_si256( - _mm256_or_si256(_mm256_cmpeq_epi8(word, _mm256_set1_epi8(34)), // '"' - _mm256_cmpeq_epi8(word, _mm256_set1_epi8(92))), // '\\' - _mm256_cmpeq_epi8(_mm256_subs_epu8(word, _mm256_set1_epi8(31)), - _mm256_setzero_si256())); // control - const uint32_t mask = uint32_t(_mm256_movemask_epi8(flags)); - if (simdjson_likely(mask == 0)) { - _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), word); - out += 32; - } else { - for (size_t half = 0; half < 32; half += 16) { - const uint64_t m = (mask >> half) & 0xFFFF; - if (m == 0) { - escape_store16(out, escape_load16(src + i + half)); - out += 16; - } else { - out = escape_block(src, out, i + half, i + half + 16, m); - } - } - } - i += 32; - } -#endif - while (i + 16 <= len) { - escape_vector word = escape_load16(src + i); - escape_vector flags = escape_flags(word); - if (simdjson_likely(!escape_any(flags))) { - escape_store16(out, word); - out += 16; - } else { - out = escape_block(src, out, i, i + 16, escape_bitmask(flags)); - } - i += 16; - } - if (i < len) { - const size_t rem = len - i; - uint64_t m; - if (len >= 16) { - // The last 16 bytes of the input are in bounds. Bit k of that block's - // mask belongs to input position len - 16 + k, so shift it down to align - // bit 0 with position i. - m = escape_mask(escape_load16(src + len - 16)) >> - (escape_mask_bits * (16 - rem)); - } else if (len >= 8) { - // Here i == 0 and rem == len. Two overlapping 8-byte loads cover - // [0, 8) and [len - 8, len), which is the whole input since len < 16. - uint64_t mm = escape_mask(escape_load8x2(src, src + len - 8)); - constexpr uint64_t low8 = (uint64_t(1) << (escape_mask_bits * 8)) - 1; - m = (mm & low8) | - ((mm >> (escape_mask_bits * 8)) << (escape_mask_bits * (len - 8))); - } else if (len >= 4) { - // Same idea with two overlapping 4-byte loads. - uint64_t mm = escape_mask(escape_load4x2(src, src + len - 4)); - constexpr uint64_t low4 = (uint64_t(1) << (escape_mask_bits * 4)) - 1; - m = (mm & low4) | (((mm >> (escape_mask_bits * 4)) & low4) - << (escape_mask_bits * (len - 4))); - } else { - // Fewer than 4 bytes: at most three table lookups, no need for SIMD. - for (size_t k = 0; k < len; k++) { - uint8_t c = src[k]; - if (json_quotable_character[c]) { - escape_json_char(char(c), out); - } else { - *out++ = char(c); - } - } - return size_t(out - initout); - } - if (m == 0) { - copy_lt16(out, src + i, rem); - out += rem; - } else { - out = escape_block(src, out, i, len, m); - } - } - return size_t(out - initout); -} - -#else // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - -// Writes the escaped version of input to out, returning the number of bytes -// written. Uses SIMD position finding to locate quotable characters efficiently. -inline size_t write_string_escaped(const std::string_view input, char *out) { - size_t mysize = input.size(); - - // Use SIMD position finder directly - it returns mysize if no escape needed - size_t location = find_next_json_quotable_character(input, 0); - if (location == mysize) { - // Fast path: no escaping needed - memcpy(out, input.data(), input.size()); - return input.size(); - } - - const char *const initout = out; - memcpy(out, input.data(), location); - out += location; - escape_json_char(input[location], out); - location += 1; - while (location < mysize) { - size_t newlocation = find_next_json_quotable_character(input, location); - memcpy(out, input.data() + location, newlocation - location); - out += newlocation - location; - location = newlocation; - if (location == mysize) { - break; - } - escape_json_char(input[location], out); - location += 1; - } - return size_t(out - initout); -} - -#endif // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE -#undef SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - namespace internal { // Sets the size of s to n without initializing the new bytes when the @@ -71738,6 +67658,7 @@ simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ namespace simdjson { @@ -71774,6 +67695,9 @@ struct has_custom_serialization : std::false_type {}; namespace rvv_vls { namespace builder { + +using simdjson::internal::escape_best::write_string_escaped; + /** * A builder for JSON strings representing documents. This is a low-level * builder that is not meant to be used directly by end-users. Though it @@ -72120,6 +68044,7 @@ simdjson_warn_unused error_code to_json(const Z &z, std::string &s, size_t initi /* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ /* amalgamation skipped (editor-only): #include "simdjson/concepts.h" */ /* amalgamation skipped (editor-only): #include "simdjson/annotations.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ #if SIMDJSON_STATIC_REFLECTION @@ -72145,7 +68070,6 @@ namespace internal { using simdjson::internal::write_uint_jeaiii; simdjson_inline char *write_double(char *p, double v) noexcept; } // namespace internal -simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out); SIMDJSON_PUSH_DISABLE_WARNINGS SIMDJSON_DISABLE_GCC_WARNING(-Warray-bounds) @@ -73166,619 +69090,13 @@ simdjson_warn_unused simdjson_result extract_fractured_json( /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_STRING_BUILDER_INL_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/builder/json_string_builder.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/jeaiii.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/json_escape.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ -/* - * Empirically, we have found that an inlined optimization is important for - * performance. The following macros are not ideal. We should find a better - * way to inline the code. - */ - -#if defined(__SSE2__) || defined(__x86_64__) || defined(__x86_64) || \ - (defined(_M_AMD64) || defined(_M_X64) || \ - (defined(_M_IX86_FP) && _M_IX86_FP == 2)) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_SSE2 -#define SIMDJSON_EXPERIMENTAL_HAS_SSE2 1 -#endif -#endif - -#if defined(__aarch64__) || defined(_M_ARM64) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_NEON -#define SIMDJSON_EXPERIMENTAL_HAS_NEON 1 -#endif -#endif -#if defined(__loongarch_sx) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_LSX -#define SIMDJSON_EXPERIMENTAL_HAS_LSX 1 -#endif -#endif -#if defined(__loongarch_asx) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_LASX -#define SIMDJSON_EXPERIMENTAL_HAS_LASX 1 -#endif -#endif -#if defined(__riscv_v_intrinsic) && __riscv_v_intrinsic >= 11000 && \ - defined(__riscv_vector) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_RVV -#define SIMDJSON_EXPERIMENTAL_HAS_RVV 1 -#endif -#endif -#if (defined(__PPC64__) || defined(_M_PPC64)) && defined(__ALTIVEC__) && defined(__POWER8_VECTOR__) -#ifndef SIMDJSON_EXPERIMENTAL_HAS_PPC64 -#define SIMDJSON_EXPERIMENTAL_HAS_PPC64 1 -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_NEON -#include -#ifdef _MSC_VER -#include -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 -#include -#if defined(__AVX2__) -#include -#endif -#ifdef _MSC_VER -#include -#endif -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_LSX -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_LASX -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_RVV -#include -#endif -#if SIMDJSON_EXPERIMENTAL_HAS_PPC64 -#include -#ifdef bool -#undef bool -#endif -#ifdef vector -#undef vector -#endif -#endif - - namespace simdjson { namespace rvv_vls { namespace builder { -static SIMDJSON_CONSTEXPR_LAMBDA std::array - json_quotable_character = { - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; - -/** - -A possible SWAR implementation of has_json_escapable_byte. It is not used -because it is slower than the current implementation. It is kept here for -reference (to show that we tried it). - -inline bool has_json_escapable_byte(uint64_t x) { - uint64_t is_ascii = 0x8080808080808080ULL & ~x; - uint64_t xor2 = x ^ 0x0202020202020202ULL; - uint64_t lt32_or_eq34 = xor2 - 0x2121212121212121ULL; - uint64_t sub92 = x ^ 0x5C5C5C5C5C5C5C5CULL; - uint64_t eq92 = (sub92 - 0x0101010101010101ULL); - return ((lt32_or_eq34 | eq92) & is_ascii) != 0; -} - -**/ - -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 || SIMDJSON_EXPERIMENTAL_HAS_NEON -#define SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE 1 -#endif - -// With block escaping (SSE2/NEON), write_string_escaped scans the input itself. -#if !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE -// Scalar fallback for finding next quotable character -SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t -find_next_json_quotable_character_scalar(const std::string_view view, - size_t location) noexcept { - for (auto pos = view.begin() + location; pos != view.end(); ++pos) { - if (json_quotable_character[static_cast(*pos)]) { - return pos - view.begin(); - } - } - return size_t(view.size()); -} - -// SIMD-accelerated position finding that directly locates the first quotable -// character, combining detection and position extraction in a single pass to -// minimize redundant work. -#if SIMDJSON_EXPERIMENTAL_HAS_LASX -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - // SIMD constants for characters requiring escape - __m256i v34 = __lasx_xvreplgr2vr_b(34); // '"' - __m256i v92 = __lasx_xvreplgr2vr_b(92); // '\\' - __m256i v32 = __lasx_xvreplgr2vr_b(32); // control char threshold - - while (remaining >= 32) { - __m256i word = __lasx_xvld(ptr, 0); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - __m256i needs_escape = __lasx_xvseq_b(word, v34); - needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvseq_b(word, v92)); - needs_escape = __lasx_xvor_v(needs_escape, __lasx_xvslt_bu(word, v32)); - - if (!__lasx_xbz_v(needs_escape)) { - // Found a quotable character - locate it via the four 64-bit lanes - uint64_t lane0 = __lasx_xvpickve2gr_du(needs_escape, 0); - uint64_t lane1 = __lasx_xvpickve2gr_du(needs_escape, 1); - uint64_t lane2 = __lasx_xvpickve2gr_du(needs_escape, 2); - uint64_t lane3 = __lasx_xvpickve2gr_du(needs_escape, 3); - size_t offset = ptr - reinterpret_cast(view.data()); - if (lane0 != 0) { - return offset + trailing_zeroes(lane0) / 8; - } else if (lane1 != 0) { - return offset + 8 + trailing_zeroes(lane1) / 8; - } else if (lane2 != 0) { - return offset + 16 + trailing_zeroes(lane2) / 8; - } else { - return offset + 24 + trailing_zeroes(lane3) / 8; - } - } - ptr += 32; - remaining -= 32; - } - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -#elif SIMDJSON_EXPERIMENTAL_HAS_LSX -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - //SIMD constants for characters requiring escape - __m128i v34 = __lsx_vreplgr2vr_b(34); // '"' - __m128i v92 = __lsx_vreplgr2vr_b(92); // '\\' - __m128i v32 = __lsx_vreplgr2vr_b(32); // control char threshold - - while (remaining >= 16){ - __m128i word = __lsx_vld(ptr, 0); - - //Check for the quotable characters: '"', '\\', or control char (<32) - __m128i needs_escape = __lsx_vseq_b(word, v34); - needs_escape = __lsx_vor_v(needs_escape, __lsx_vseq_b(word, v92)); - needs_escape = __lsx_vor_v(needs_escape, __lsx_vslt_bu(word, v32)); - - if (!__lsx_bz_v(needs_escape)){ - - //Found quotable character - extract exact byte position - uint64_t lo = __lsx_vpickve2gr_du(needs_escape,0); - uint64_t hi = __lsx_vpickve2gr_du(needs_escape,1); - size_t offset = ptr - reinterpret_cast(view.data()); - if ( lo != 0) { - return offset + trailing_zeroes(lo) / 8; - } else { - return offset + 8 + trailing_zeroes(hi) / 8; - } - } - ptr += 16; - remaining -= 16; - } - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -#elif SIMDJSON_EXPERIMENTAL_HAS_RVV -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - while (remaining > 0) { - size_t vl = __riscv_vsetvl_e8m1(remaining); - vuint8m1_t word = __riscv_vle8_v_u8m1(ptr, vl); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - vbool8_t needs_escape = __riscv_vmseq(word, (uint8_t)34, vl); - needs_escape = __riscv_vmor(needs_escape, - __riscv_vmseq(word, (uint8_t)92, vl), vl); - needs_escape = __riscv_vmor(needs_escape, - __riscv_vmsltu(word, (uint8_t)32, vl), vl); - - long first = __riscv_vfirst(needs_escape, vl); - if (first >= 0) { - size_t offset = ptr - reinterpret_cast(view.data()); - return offset + first; - } - ptr += vl; - remaining -= vl; - } - - return len; -} -#elif SIMDJSON_EXPERIMENTAL_HAS_PPC64 -simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - const size_t len = view.size(); - const uint8_t *ptr = - reinterpret_cast(view.data()) + location; - size_t remaining = len - location; - - // SIMD constants for characters requiring escape - __vector unsigned char v34 = vec_splats((unsigned char)34); // '"' - __vector unsigned char v92 = vec_splats((unsigned char)92); // '\\' - __vector unsigned char v32 = vec_splats((unsigned char)32); // control char threshold - - // Bitmask for vec_vbpermq to extract one bit per byte - const __vector unsigned char perm_mask = {0x78, 0x70, 0x68, 0x60, 0x58, 0x50, - 0x48, 0x40, 0x38, 0x30, 0x28, 0x20, - 0x18, 0x10, 0x08, 0x00}; - - while (remaining >= 16) { - __vector unsigned char word = - vec_vsx_ld(0, reinterpret_cast(ptr)); - - // Check for quotable characters: '"', '\\', or control chars (< 32) - __vector unsigned char needs_escape = - (__vector unsigned char)vec_cmpeq(word, v34); - needs_escape = vec_or(needs_escape, - (__vector unsigned char)vec_cmpeq(word, v92)); - needs_escape = vec_or(needs_escape, - (__vector unsigned char)vec_cmplt(word, v32)); - - __vector unsigned long long result = - (__vector unsigned long long)vec_vbpermq(needs_escape, perm_mask); -#ifdef __LITTLE_ENDIAN__ - unsigned int mask = static_cast(result[1]); -#else - unsigned int mask = static_cast(result[0]); -#endif - if (mask != 0) { - size_t offset = ptr - reinterpret_cast(view.data()); - return offset + __builtin_ctz(mask); - } - ptr += 16; - remaining -= 16; - } - - // Scalar fallback for remaining bytes - size_t current = len - remaining; - return find_next_json_quotable_character_scalar(view, current); -} -#else -SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline size_t -find_next_json_quotable_character(const std::string_view view, - size_t location) noexcept { - return find_next_json_quotable_character_scalar(view, location); -} -#endif -#endif // !SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - -SIMDJSON_CONSTEXPR_LAMBDA static std::string_view control_chars[] = { - "\\u0000", "\\u0001", "\\u0002", "\\u0003", "\\u0004", "\\u0005", "\\u0006", - "\\u0007", "\\b", "\\t", "\\n", "\\u000b", "\\f", "\\r", - "\\u000e", "\\u000f", "\\u0010", "\\u0011", "\\u0012", "\\u0013", "\\u0014", - "\\u0015", "\\u0016", "\\u0017", "\\u0018", "\\u0019", "\\u001a", "\\u001b", - "\\u001c", "\\u001d", "\\u001e", "\\u001f"}; - -// All Unicode characters may be placed within the quotation marks, except for -// the characters that MUST be escaped: quotation mark, reverse solidus, and the -// control characters (U+0000 through U+001F). There are two-character sequence -// escape representations of some popular characters: -// \", \\, \b, \f, \n, \r, \t. -SIMDJSON_CONSTEXPR_LAMBDA simdjson_inline void escape_json_char(char c, char *&out) { - if (c == '"') { - memcpy(out, "\\\"", 2); - out += 2; - } else if (c == '\\') { - memcpy(out, "\\\\", 2); - out += 2; - } else { - std::string_view v = control_chars[uint8_t(c)]; - memcpy(out, v.data(), v.size()); - out += v.size(); - } -} - -#if SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 - -using escape_vector = __m128i; -// Mask bits that each input byte contributes to escape_bitmask(). -static constexpr unsigned escape_mask_bits = 1; - -simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept { - return _mm_loadu_si128(reinterpret_cast(p)); -} - -simdjson_inline void escape_store16(char *out, escape_vector v) noexcept { - _mm_storeu_si128(reinterpret_cast<__m128i *>(out), v); -} - -// Builds a vector whose bytes 0..7 come from a and bytes 8..15 from b. -simdjson_inline escape_vector escape_load8x2(const uint8_t *a, - const uint8_t *b) noexcept { - return _mm_unpacklo_epi64( - _mm_loadl_epi64(reinterpret_cast(a)), - _mm_loadl_epi64(reinterpret_cast(b))); -} - -// Builds a vector whose bytes 0..3 come from a and bytes 4..7 from b. The -// upper half is unspecified; callers only look at the low eight bits. -simdjson_inline escape_vector escape_load4x2(const uint8_t *a, - const uint8_t *b) noexcept { - int32_t a32, b32; - memcpy(&a32, a, 4); - memcpy(&b32, b, 4); - return _mm_unpacklo_epi32(_mm_cvtsi32_si128(a32), _mm_cvtsi32_si128(b32)); -} - -// Sets every bit of byte k when byte k of v is a quotable character ('"', -// '\\' or a control character). -simdjson_inline escape_vector escape_flags(escape_vector v) noexcept { - const __m128i v34 = _mm_set1_epi8(34); // '"' - const __m128i v92 = _mm_set1_epi8(92); // '\\' - const __m128i v31 = _mm_set1_epi8(31); // for control char detection - __m128i needs_escape = _mm_cmpeq_epi8(v, v34); - needs_escape = _mm_or_si128(needs_escape, _mm_cmpeq_epi8(v, v92)); - return _mm_or_si128( - needs_escape, _mm_cmpeq_epi8(_mm_subs_epu8(v, v31), _mm_setzero_si128())); -} - -// True when any byte needs escaping. Kept separate from escape_bitmask -// because some instruction sets can answer it without leaving the vector -// register file; on SSE2 the compiler folds the two together. -simdjson_inline bool escape_any(escape_vector flags) noexcept { - return _mm_movemask_epi8(flags) != 0; -} - -// A 16-bit mask with bit k set when byte k needed escaping. -simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept { - return uint64_t(uint32_t(_mm_movemask_epi8(flags))); -} - -#else // SIMDJSON_EXPERIMENTAL_HAS_NEON - -using escape_vector = uint8x16_t; -static constexpr unsigned escape_mask_bits = 4; - -simdjson_inline escape_vector escape_load16(const uint8_t *p) noexcept { - return vld1q_u8(p); -} - -simdjson_inline void escape_store16(char *out, escape_vector v) noexcept { - vst1q_u8(reinterpret_cast(out), v); -} - -simdjson_inline escape_vector escape_load8x2(const uint8_t *a, - const uint8_t *b) noexcept { - uint64_t a64, b64; - memcpy(&a64, a, 8); - memcpy(&b64, b, 8); - return vreinterpretq_u8_u64(vsetq_lane_u64(b64, vdupq_n_u64(a64), 1)); -} - -simdjson_inline escape_vector escape_load4x2(const uint8_t *a, - const uint8_t *b) noexcept { - uint32_t a32, b32; - memcpy(&a32, a, 4); - memcpy(&b32, b, 4); - return vreinterpretq_u8_u32(vsetq_lane_u32(b32, vdupq_n_u32(a32), 1)); -} - -simdjson_inline escape_vector escape_flags(escape_vector v) noexcept { - uint8x16_t needs_escape = vceqq_u8(v, vdupq_n_u8(34)); // '"' - needs_escape = vorrq_u8(needs_escape, vceqq_u8(v, vdupq_n_u8(92))); // '\\' - return vorrq_u8(needs_escape, vcltq_u8(v, vdupq_n_u8(32))); -} - -simdjson_inline bool escape_any(escape_vector flags) noexcept { - return vmaxvq_u32(vreinterpretq_u32_u8(flags)) != 0; -} - -// Four bits per byte rather than one: escape_block and the tail paths scale -// their shifts by escape_mask_bits to match. -simdjson_inline uint64_t escape_bitmask(escape_vector flags) noexcept { - uint8x8_t narrowed = vshrn_n_u16(vreinterpretq_u16_u8(flags), 4); - return vget_lane_u64(vreinterpret_u64_u8(narrowed), 0); -} - -#endif // instruction set selection - -// The escape bitmask of a 16-byte block. -simdjson_inline uint64_t escape_mask(escape_vector v) noexcept { - return escape_bitmask(escape_flags(v)); -} - -// Copies n bytes with n < 16, using overlapping loads and stores. It never -// reads more than n bytes from src, nor writes more than n bytes to dst. -simdjson_inline void copy_lt16(char *dst, const uint8_t *src, - size_t n) noexcept { - if (n >= 8) { - memcpy(dst, src, 8); - memcpy(dst + n - 8, src + n - 8, 8); - } else if (n >= 4) { - memcpy(dst, src, 4); - memcpy(dst + n - 4, src + n - 4, 4); - } else if (n > 0) { - dst[0] = char(src[0]); - dst[n >> 1] = char(src[n >> 1]); - dst[n - 1] = char(src[n - 1]); - } -} - -// Escapes the bytes of src in the range [i, blockend), given that m is the -// (non-zero) escape mask for that range: the escape_mask_bits-wide lane of m -// at byte k is non-zero when src[i + k] requires escaping. Returns the updated -// output pointer. -simdjson_never_inline char *escape_block(const uint8_t *src, char *out, - size_t i, size_t blockend, - uint64_t m) noexcept { - constexpr uint64_t lane = (uint64_t(1) << escape_mask_bits) - 1; - - size_t pos = i; // first byte not yet copied - while (m) { - const size_t tz = trailing_zeroes(m); - const size_t next = i + tz / escape_mask_bits; - // Copy the run of safe bytes that precedes this escape. - copy_lt16(out, src + pos, next - pos); - out += next - pos; - escape_json_char(char(src[next]), out); - pos = next + 1; - m &= ~(lane << tz); - } - // Copy whatever follows the last escape. - copy_lt16(out, src + pos, blockend - pos); - return out + (blockend - pos); -} - -// Writes the escaped version of input to out, returning the number of bytes -// written. -simdjson_really_inline size_t write_string_escaped(const std::string_view input, char *out) { - const size_t len = input.size(); - const uint8_t *src = reinterpret_cast(input.data()); - const char *const initout = out; - - size_t i = 0; -#if SIMDJSON_EXPERIMENTAL_HAS_SSE2 && defined(__AVX2__) - while (i + 32 <= len) { - const __m256i word = _mm256_loadu_si256(reinterpret_cast(src + i)); - const __m256i flags = _mm256_or_si256( - _mm256_or_si256(_mm256_cmpeq_epi8(word, _mm256_set1_epi8(34)), // '"' - _mm256_cmpeq_epi8(word, _mm256_set1_epi8(92))), // '\\' - _mm256_cmpeq_epi8(_mm256_subs_epu8(word, _mm256_set1_epi8(31)), - _mm256_setzero_si256())); // control - const uint32_t mask = uint32_t(_mm256_movemask_epi8(flags)); - if (simdjson_likely(mask == 0)) { - _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), word); - out += 32; - } else { - for (size_t half = 0; half < 32; half += 16) { - const uint64_t m = (mask >> half) & 0xFFFF; - if (m == 0) { - escape_store16(out, escape_load16(src + i + half)); - out += 16; - } else { - out = escape_block(src, out, i + half, i + half + 16, m); - } - } - } - i += 32; - } -#endif - while (i + 16 <= len) { - escape_vector word = escape_load16(src + i); - escape_vector flags = escape_flags(word); - if (simdjson_likely(!escape_any(flags))) { - escape_store16(out, word); - out += 16; - } else { - out = escape_block(src, out, i, i + 16, escape_bitmask(flags)); - } - i += 16; - } - if (i < len) { - const size_t rem = len - i; - uint64_t m; - if (len >= 16) { - // The last 16 bytes of the input are in bounds. Bit k of that block's - // mask belongs to input position len - 16 + k, so shift it down to align - // bit 0 with position i. - m = escape_mask(escape_load16(src + len - 16)) >> - (escape_mask_bits * (16 - rem)); - } else if (len >= 8) { - // Here i == 0 and rem == len. Two overlapping 8-byte loads cover - // [0, 8) and [len - 8, len), which is the whole input since len < 16. - uint64_t mm = escape_mask(escape_load8x2(src, src + len - 8)); - constexpr uint64_t low8 = (uint64_t(1) << (escape_mask_bits * 8)) - 1; - m = (mm & low8) | - ((mm >> (escape_mask_bits * 8)) << (escape_mask_bits * (len - 8))); - } else if (len >= 4) { - // Same idea with two overlapping 4-byte loads. - uint64_t mm = escape_mask(escape_load4x2(src, src + len - 4)); - constexpr uint64_t low4 = (uint64_t(1) << (escape_mask_bits * 4)) - 1; - m = (mm & low4) | (((mm >> (escape_mask_bits * 4)) & low4) - << (escape_mask_bits * (len - 4))); - } else { - // Fewer than 4 bytes: at most three table lookups, no need for SIMD. - for (size_t k = 0; k < len; k++) { - uint8_t c = src[k]; - if (json_quotable_character[c]) { - escape_json_char(char(c), out); - } else { - *out++ = char(c); - } - } - return size_t(out - initout); - } - if (m == 0) { - copy_lt16(out, src + i, rem); - out += rem; - } else { - out = escape_block(src, out, i, len, m); - } - } - return size_t(out - initout); -} - -#else // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - -// Writes the escaped version of input to out, returning the number of bytes -// written. Uses SIMD position finding to locate quotable characters efficiently. -inline size_t write_string_escaped(const std::string_view input, char *out) { - size_t mysize = input.size(); - - // Use SIMD position finder directly - it returns mysize if no escape needed - size_t location = find_next_json_quotable_character(input, 0); - if (location == mysize) { - // Fast path: no escaping needed - memcpy(out, input.data(), input.size()); - return input.size(); - } - - const char *const initout = out; - memcpy(out, input.data(), location); - out += location; - escape_json_char(input[location], out); - location += 1; - while (location < mysize) { - size_t newlocation = find_next_json_quotable_character(input, location); - memcpy(out, input.data() + location, newlocation - location); - out += newlocation - location; - location = newlocation; - if (location == mysize) { - break; - } - escape_json_char(input[location], out); - location += 1; - } - return size_t(out - initout); -} - -#endif // SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE -#undef SIMDJSON_BUILDER_HAS_BLOCK_ESCAPE - namespace internal { // Sets the size of s to n without initializing the new bytes when the @@ -76022,8 +71340,13 @@ class value { * Create a new invalid value. * * Exists so you can declare a variable and later assign to it before use. + * + * It is user-provided (not = default) on purpose: value-initialization (std::pair, T{}) + * then skips zeroing the whole object, which led GCC 15 and 16 to keep + * simdjson_result on the stack when iterating over objects, causing + * store-forwarding stalls (up to 2x slower). */ - simdjson_inline value() noexcept = default; + simdjson_inline value() noexcept {} /** * Get this value as the given type. @@ -87306,7 +82629,7 @@ simdjson_inline document_stream::iterator document_stream::end() noexcept { inline void document_stream::start() noexcept { if (error) { return; } - error = parser->allocate(batch_size); + error = parser->allocate(batch_size, parser->max_depth()); if (error) { return; } // Always run the first stage 1 parse immediately batch_start = 0; @@ -87332,7 +82655,10 @@ inline void document_stream::start() noexcept { worker.reset(new(std::nothrow) stage1_worker()); if (worker.get() == nullptr) { error = MEMALLOC; return; } } - error = stage1_thread_parser.allocate(batch_size); + // load_from_stage1_thread() swaps *parser with stage1_thread_parser, which + // then parses every other batch: give it the same settings. + stage1_thread_parser.set_max_capacity(parser->max_capacity()); + error = stage1_thread_parser.allocate(batch_size, parser->max_depth()); if (error) { return; } worker->start_thread(); start_stage1_thread(); @@ -93507,8 +88833,13 @@ class value { * Create a new invalid value. * * Exists so you can declare a variable and later assign to it before use. + * + * It is user-provided (not = default) on purpose: value-initialization (std::pair, T{}) + * then skips zeroing the whole object, which led GCC 15 and 16 to keep + * simdjson_result on the stack when iterating over objects, causing + * store-forwarding stalls (up to 2x slower). */ - simdjson_inline value() noexcept = default; + simdjson_inline value() noexcept {} /** * Get this value as the given type. @@ -104791,7 +100122,7 @@ simdjson_inline document_stream::iterator document_stream::end() noexcept { inline void document_stream::start() noexcept { if (error) { return; } - error = parser->allocate(batch_size); + error = parser->allocate(batch_size, parser->max_depth()); if (error) { return; } // Always run the first stage 1 parse immediately batch_start = 0; @@ -104817,7 +100148,10 @@ inline void document_stream::start() noexcept { worker.reset(new(std::nothrow) stage1_worker()); if (worker.get() == nullptr) { error = MEMALLOC; return; } } - error = stage1_thread_parser.allocate(batch_size); + // load_from_stage1_thread() swaps *parser with stage1_thread_parser, which + // then parses every other batch: give it the same settings. + stage1_thread_parser.set_max_capacity(parser->max_capacity()); + error = stage1_thread_parser.allocate(batch_size, parser->max_depth()); if (error) { return; } worker->start_thread(); start_stage1_thread(); @@ -111462,8 +106796,13 @@ class value { * Create a new invalid value. * * Exists so you can declare a variable and later assign to it before use. + * + * It is user-provided (not = default) on purpose: value-initialization (std::pair, T{}) + * then skips zeroing the whole object, which led GCC 15 and 16 to keep + * simdjson_result on the stack when iterating over objects, causing + * store-forwarding stalls (up to 2x slower). */ - simdjson_inline value() noexcept = default; + simdjson_inline value() noexcept {} /** * Get this value as the given type. @@ -122746,7 +118085,7 @@ simdjson_inline document_stream::iterator document_stream::end() noexcept { inline void document_stream::start() noexcept { if (error) { return; } - error = parser->allocate(batch_size); + error = parser->allocate(batch_size, parser->max_depth()); if (error) { return; } // Always run the first stage 1 parse immediately batch_start = 0; @@ -122772,7 +118111,10 @@ inline void document_stream::start() noexcept { worker.reset(new(std::nothrow) stage1_worker()); if (worker.get() == nullptr) { error = MEMALLOC; return; } } - error = stage1_thread_parser.allocate(batch_size); + // load_from_stage1_thread() swaps *parser with stage1_thread_parser, which + // then parses every other batch: give it the same settings. + stage1_thread_parser.set_max_capacity(parser->max_capacity()); + error = stage1_thread_parser.allocate(batch_size, parser->max_depth()); if (error) { return; } worker->start_thread(); start_stage1_thread(); @@ -129413,8 +124755,13 @@ class value { * Create a new invalid value. * * Exists so you can declare a variable and later assign to it before use. + * + * It is user-provided (not = default) on purpose: value-initialization (std::pair, T{}) + * then skips zeroing the whole object, which led GCC 15 and 16 to keep + * simdjson_result on the stack when iterating over objects, causing + * store-forwarding stalls (up to 2x slower). */ - simdjson_inline value() noexcept = default; + simdjson_inline value() noexcept {} /** * Get this value as the given type. @@ -140697,7 +136044,7 @@ simdjson_inline document_stream::iterator document_stream::end() noexcept { inline void document_stream::start() noexcept { if (error) { return; } - error = parser->allocate(batch_size); + error = parser->allocate(batch_size, parser->max_depth()); if (error) { return; } // Always run the first stage 1 parse immediately batch_start = 0; @@ -140723,7 +136070,10 @@ inline void document_stream::start() noexcept { worker.reset(new(std::nothrow) stage1_worker()); if (worker.get() == nullptr) { error = MEMALLOC; return; } } - error = stage1_thread_parser.allocate(batch_size); + // load_from_stage1_thread() swaps *parser with stage1_thread_parser, which + // then parses every other batch: give it the same settings. + stage1_thread_parser.set_max_capacity(parser->max_capacity()); + error = stage1_thread_parser.allocate(batch_size, parser->max_depth()); if (error) { return; } worker->start_thread(); start_stage1_thread(); @@ -147482,8 +142832,13 @@ class value { * Create a new invalid value. * * Exists so you can declare a variable and later assign to it before use. + * + * It is user-provided (not = default) on purpose: value-initialization (std::pair, T{}) + * then skips zeroing the whole object, which led GCC 15 and 16 to keep + * simdjson_result on the stack when iterating over objects, causing + * store-forwarding stalls (up to 2x slower). */ - simdjson_inline value() noexcept = default; + simdjson_inline value() noexcept {} /** * Get this value as the given type. @@ -158766,7 +154121,7 @@ simdjson_inline document_stream::iterator document_stream::end() noexcept { inline void document_stream::start() noexcept { if (error) { return; } - error = parser->allocate(batch_size); + error = parser->allocate(batch_size, parser->max_depth()); if (error) { return; } // Always run the first stage 1 parse immediately batch_start = 0; @@ -158792,7 +154147,10 @@ inline void document_stream::start() noexcept { worker.reset(new(std::nothrow) stage1_worker()); if (worker.get() == nullptr) { error = MEMALLOC; return; } } - error = stage1_thread_parser.allocate(batch_size); + // load_from_stage1_thread() swaps *parser with stage1_thread_parser, which + // then parses every other batch: give it the same settings. + stage1_thread_parser.set_max_capacity(parser->max_capacity()); + error = stage1_thread_parser.allocate(batch_size, parser->max_depth()); if (error) { return; } worker->start_thread(); start_stage1_thread(); @@ -165859,8 +161217,13 @@ class value { * Create a new invalid value. * * Exists so you can declare a variable and later assign to it before use. + * + * It is user-provided (not = default) on purpose: value-initialization (std::pair, T{}) + * then skips zeroing the whole object, which led GCC 15 and 16 to keep + * simdjson_result on the stack when iterating over objects, causing + * store-forwarding stalls (up to 2x slower). */ - simdjson_inline value() noexcept = default; + simdjson_inline value() noexcept {} /** * Get this value as the given type. @@ -177143,7 +172506,7 @@ simdjson_inline document_stream::iterator document_stream::end() noexcept { inline void document_stream::start() noexcept { if (error) { return; } - error = parser->allocate(batch_size); + error = parser->allocate(batch_size, parser->max_depth()); if (error) { return; } // Always run the first stage 1 parse immediately batch_start = 0; @@ -177169,7 +172532,10 @@ inline void document_stream::start() noexcept { worker.reset(new(std::nothrow) stage1_worker()); if (worker.get() == nullptr) { error = MEMALLOC; return; } } - error = stage1_thread_parser.allocate(batch_size); + // load_from_stage1_thread() swaps *parser with stage1_thread_parser, which + // then parses every other batch: give it the same settings. + stage1_thread_parser.set_max_capacity(parser->max_capacity()); + error = stage1_thread_parser.allocate(batch_size, parser->max_depth()); if (error) { return; } worker->start_thread(); start_stage1_thread(); @@ -183726,8 +179092,13 @@ class value { * Create a new invalid value. * * Exists so you can declare a variable and later assign to it before use. + * + * It is user-provided (not = default) on purpose: value-initialization (std::pair, T{}) + * then skips zeroing the whole object, which led GCC 15 and 16 to keep + * simdjson_result on the stack when iterating over objects, causing + * store-forwarding stalls (up to 2x slower). */ - simdjson_inline value() noexcept = default; + simdjson_inline value() noexcept {} /** * Get this value as the given type. @@ -195010,7 +190381,7 @@ simdjson_inline document_stream::iterator document_stream::end() noexcept { inline void document_stream::start() noexcept { if (error) { return; } - error = parser->allocate(batch_size); + error = parser->allocate(batch_size, parser->max_depth()); if (error) { return; } // Always run the first stage 1 parse immediately batch_start = 0; @@ -195036,7 +190407,10 @@ inline void document_stream::start() noexcept { worker.reset(new(std::nothrow) stage1_worker()); if (worker.get() == nullptr) { error = MEMALLOC; return; } } - error = stage1_thread_parser.allocate(batch_size); + // load_from_stage1_thread() swaps *parser with stage1_thread_parser, which + // then parses every other batch: give it the same settings. + stage1_thread_parser.set_max_capacity(parser->max_capacity()); + error = stage1_thread_parser.allocate(batch_size, parser->max_depth()); if (error) { return; } worker->start_thread(); start_stage1_thread(); @@ -201616,8 +196990,13 @@ class value { * Create a new invalid value. * * Exists so you can declare a variable and later assign to it before use. + * + * It is user-provided (not = default) on purpose: value-initialization (std::pair, T{}) + * then skips zeroing the whole object, which led GCC 15 and 16 to keep + * simdjson_result on the stack when iterating over objects, causing + * store-forwarding stalls (up to 2x slower). */ - simdjson_inline value() noexcept = default; + simdjson_inline value() noexcept {} /** * Get this value as the given type. @@ -212900,7 +208279,7 @@ simdjson_inline document_stream::iterator document_stream::end() noexcept { inline void document_stream::start() noexcept { if (error) { return; } - error = parser->allocate(batch_size); + error = parser->allocate(batch_size, parser->max_depth()); if (error) { return; } // Always run the first stage 1 parse immediately batch_start = 0; @@ -212926,7 +208305,10 @@ inline void document_stream::start() noexcept { worker.reset(new(std::nothrow) stage1_worker()); if (worker.get() == nullptr) { error = MEMALLOC; return; } } - error = stage1_thread_parser.allocate(batch_size); + // load_from_stage1_thread() swaps *parser with stage1_thread_parser, which + // then parses every other batch: give it the same settings. + stage1_thread_parser.set_max_capacity(parser->max_capacity()); + error = stage1_thread_parser.allocate(batch_size, parser->max_depth()); if (error) { return; } worker->start_thread(); start_stage1_thread(); @@ -219516,8 +214898,13 @@ class value { * Create a new invalid value. * * Exists so you can declare a variable and later assign to it before use. + * + * It is user-provided (not = default) on purpose: value-initialization (std::pair, T{}) + * then skips zeroing the whole object, which led GCC 15 and 16 to keep + * simdjson_result on the stack when iterating over objects, causing + * store-forwarding stalls (up to 2x slower). */ - simdjson_inline value() noexcept = default; + simdjson_inline value() noexcept {} /** * Get this value as the given type. @@ -230800,7 +226187,7 @@ simdjson_inline document_stream::iterator document_stream::end() noexcept { inline void document_stream::start() noexcept { if (error) { return; } - error = parser->allocate(batch_size); + error = parser->allocate(batch_size, parser->max_depth()); if (error) { return; } // Always run the first stage 1 parse immediately batch_start = 0; @@ -230826,7 +226213,10 @@ inline void document_stream::start() noexcept { worker.reset(new(std::nothrow) stage1_worker()); if (worker.get() == nullptr) { error = MEMALLOC; return; } } - error = stage1_thread_parser.allocate(batch_size); + // load_from_stage1_thread() swaps *parser with stage1_thread_parser, which + // then parses every other batch: give it the same settings. + stage1_thread_parser.set_max_capacity(parser->max_capacity()); + error = stage1_thread_parser.allocate(batch_size, parser->max_depth()); if (error) { return; } worker->start_thread(); start_stage1_thread();