From c097c4beaa6d6282196c7117398998783efb117e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20BIDON?= Date: Wed, 29 Jul 2026 22:30:14 +0200 Subject: [PATCH 01/10] fix: utf-8 validation (prototype) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Frédéric BIDON --- .../default-lexer/internal/input/input.go | 13 +++ .../default-lexer/internal/input/string.go | 104 ++++++++++++++++++ .../internal/input/string_raw.go | 79 +++++++++++++ .../default-lexer/internal/swar/swar.go | 3 + json/lexers/default-lexer/options.go | 23 ++++ json/lexers/default-lexer/semantic_lexer.go | 1 + 6 files changed, 223 insertions(+) diff --git a/json/lexers/default-lexer/internal/input/input.go b/json/lexers/default-lexer/internal/input/input.go index b48e33c..426d125 100644 --- a/json/lexers/default-lexer/internal/input/input.go +++ b/json/lexers/default-lexer/internal/input/input.go @@ -41,4 +41,17 @@ type Input struct { NoAVX2 bool MaxValueBytes int KeepPreviousBuffer int + + // ValidateMode (PROTOTYPE) selects the UTF-8 validation strategy. + ValidateMode int + + // SawNonASCII (PROTOTYPE) reports whether the scan observed a byte >= 0x80 in the value just produced. + SawNonASCII bool } + +// UTF-8 validation strategies (PROTOTYPE). +const ( + ValidateOff = 0 // current behavior: no validation + ValidateNaive = 1 // utf8.Valid over every string value at the funnel + ValidateFused = 2 // non-ASCII accumulated during the existing scan; utf8.Valid only when a high bit was seen +) diff --git a/json/lexers/default-lexer/internal/input/string.go b/json/lexers/default-lexer/internal/input/string.go index b02da06..1145143 100644 --- a/json/lexers/default-lexer/internal/input/string.go +++ b/json/lexers/default-lexer/internal/input/string.go @@ -52,6 +52,10 @@ const controlCharsUpperBound = 0x20 // ASCII control chars range func (in *Input) ConsumeString() token.T { if in.TrackBlanks { if in.WholeBuffer { + if in.ValidateMode == ValidateFused { + return in.consumeStringRawWholeV() + } + return in.consumeStringRawWhole() } @@ -59,12 +63,93 @@ func (in *Input) ConsumeString() token.T { } if in.WholeBuffer { + if in.ValidateMode == ValidateFused { + return in.consumeStringWholeV() + } + return in.consumeStringWhole() } return in.consumeStringStreamFast() } +// consumeStringWholeV is consumeStringWhole with a fused non-ASCII accumulator: every SWAR word already loaded by the +// stop-scan is OR-ed into hi, so a pure-ASCII string (the overwhelming majority) is proven valid UTF-8 with no second +// pass and no call. Only a string that actually contains a byte >= 0x80 pays utf8.Valid. +// +// hi may over-report (the matching word's bytes past the stop are included, and the AVX2 region is not accumulated at +// all) -- that is conservative: it can only trigger a redundant utf8.Valid, never skip a needed one. +func (in *Input) consumeStringWholeV() token.T { + data := in.Buffer + n := in.Bufferized + start := in.Consumed + + i := start + var hi uint64 + guard := start + guessLong + if in.NoAVX2 { + guard = n + 1 + } + for i+8 <= n { + w := binary.LittleEndian.Uint64(data[i:]) + hi |= w + if m := swar.StringStopMask(w); m != 0 { + i += swar.FirstByte(m) + + break + } + i += 8 + if i >= guard { + break + } + } + if i >= guard && i+8 <= n { + if c := data[i]; c != doubleQuote && c != escape && c >= controlCharsUpperBound { + i += strscan.ScanStop(data[i:n]) + hi = ^uint64(0) // AVX2 region unaccumulated: force the check + } + } + for ; i < n; i++ { + c := data[i] + if c == doubleQuote || c == escape || c < controlCharsUpperBound { + break + } + hi |= uint64(c) + } + if i >= n { + in.Consumed, in.Offset = i, uint64(i) + in.Err = codes.ErrUnterminatedString + + return token.None + } + + switch c := data[i]; { + case c == doubleQuote: + if in.MaxValueBytes > 0 && i-start > in.MaxValueBytes { + in.Consumed, in.Offset = i, uint64(i) + in.Err = codes.ErrMaxValueBytes + + return token.None + } + value := data[start:i:i] + i++ + in.Consumed, in.Offset = i, uint64(i) + in.SawNonASCII = hi&swar.HighBits != 0 + + return in.finishStringValue(value) + + case c < controlCharsUpperBound: + in.Consumed, in.Offset = i, uint64(i) + in.Err = codes.ErrControlChar + + return token.None + } + + in.SawNonASCII = true + + return in.consumeStringEscaped(start, i) +} + // consumeStringStreamFast is the streaming string fast path (§10.3 Phase 1). // // It treats the CURRENT buffer window l.in.Buffer[:l.in.Bufferized] like whole-buffer mode: it scans for the closing @@ -81,6 +166,7 @@ func (in *Input) ConsumeString() token.T { // //nolint:dupl // the structure is the same but differs subtly and its critical that the inside remains inlined. func (in *Input) consumeStringStreamFast() token.T { + in.SawNonASCII = true data := in.Buffer n := in.Bufferized start := in.Consumed // first content byte (opening quote already consumed) @@ -157,6 +243,7 @@ func (in *Input) consumeStringStreamFast() token.T { // //nolint:dupl // the structure is the same but differs subtly and its critical that the inside remains inlined. func (in *Input) consumeStringWhole() token.T { + in.SawNonASCII = true data := in.Buffer n := in.Bufferized start := in.Consumed // first content byte @@ -243,6 +330,7 @@ func (in *Input) consumeStringWhole() token.T { // It copies that prefix then unescapes the rest; the loop invariant is that data[i] is the next "stop" byte (quote, // escape, or control) — clean runs between stops are copied in bulk rather than byte-by-byte. func (in *Input) consumeStringEscaped(start, i int) token.T { + in.SawNonASCII = true data := in.Buffer n := in.Bufferized @@ -371,6 +459,21 @@ func (in *Input) consumeStringEscaped(start, i int) token.T { // finishStringValue turns a scanned string body into a Key (in object key position) or String token, handling the // trailing colon for keys. func (in *Input) finishStringValue(value []byte) token.T { + switch in.ValidateMode { + case ValidateNaive: + if !utf8.Valid(value) { + in.Err = codes.ErrInvalidRune + + return token.None + } + case ValidateFused: + if in.SawNonASCII && !utf8.Valid(value) { + in.Err = codes.ErrInvalidRune + + return token.None + } + } + if in.ExpectKey { // the following colon is validated on the next scan (see in.AfterKey) in.ExpectKey = false @@ -383,6 +486,7 @@ func (in *Input) finishStringValue(value []byte) token.T { } func (in *Input) consumeStringStreaming() token.T { + in.SawNonASCII = true var escapeSequence bool in.CurrentValue = in.CurrentValue[:0] diff --git a/json/lexers/default-lexer/internal/input/string_raw.go b/json/lexers/default-lexer/internal/input/string_raw.go index 2d7e7f2..5f3b30f 100644 --- a/json/lexers/default-lexer/internal/input/string_raw.go +++ b/json/lexers/default-lexer/internal/input/string_raw.go @@ -29,6 +29,7 @@ import ( // //nolint:dupl // the structure is the same but differs subtly and its critical that the inside remains inlined. func (in *Input) consumeStringRawWhole() token.T { + in.SawNonASCII = true data := in.Buffer n := in.Bufferized start := in.Consumed @@ -94,12 +95,88 @@ func (in *Input) consumeStringRawWhole() token.T { return in.consumeStringRawEscaped(start, i) } +func (in *Input) consumeStringRawWholeV() token.T { + data := in.Buffer + n := in.Bufferized + start := in.Consumed + + i := start + var hi uint64 + guard := start + guessLong + if in.NoAVX2 { + guard = n + 1 // WithoutAVX2: never delegate, pure inline SWAR (see consumeStringWhole) + } + for i+8 <= n { + w := binary.LittleEndian.Uint64(data[i:]) + hi |= w + if m := swar.StringStopMask(w); m != 0 { + i += swar.FirstByte(m) + + break + } + i += 8 + if i >= guard { + break + } + } + // clean past guessLong → guess long, AVX2 scan of the rest (same heuristic and out-of-loop placement as + // consumeStringWhole). + if i >= guard && i+8 <= n { + if c := data[i]; c != doubleQuote && c != escape && c >= controlCharsUpperBound { + i += strscan.ScanStop(data[i:n]) + hi = ^uint64(0) // AVX2 region unaccumulated: force the check + } + } + for ; i < n; i++ { + c := data[i] + if c == doubleQuote || c == escape || c < controlCharsUpperBound { + break + } + hi |= uint64(c) + } + if i >= n { + in.Consumed, in.Offset = i, uint64(i) + in.Err = codes.ErrUnterminatedString + + return token.None + } + + switch c := data[i]; { + case c == doubleQuote: + // no escapes: raw == decoded, same aliasing exit as consumeStringWhole + if in.MaxValueBytes > 0 && i-start > in.MaxValueBytes { + in.Consumed, in.Offset = i, uint64(i) + in.Err = codes.ErrMaxValueBytes + + return token.None + } + value := data[start:i:i] + i++ + in.Consumed, in.Offset = i, uint64(i) + in.SawNonASCII = hi&swar.HighBits != 0 + + return in.finishStringValue(value) + + case c < controlCharsUpperBound: + in.Consumed, in.Offset = i, uint64(i) + in.Err = codes.ErrControlChar + + return token.None + } + + // an escape was found at i: validate the rest but keep the raw bytes. + in.SawNonASCII = true + + return in.consumeStringRawEscaped(start, i) +} + // consumeStringRawEscaped validates a string that contains at least one escape (data[i] == escape) without decoding it, // and returns the raw content aliased from the input. // // Clean runs between escapes are skipped with the same adaptive scalar-probe-then-SWAR scan the decoder uses // (consumeStringEscaped), but with no copying — so a sparse-escape string with a long clean tail stays fast. func (in *Input) consumeStringRawEscaped(start, i int) token.T { + in.SawNonASCII = true data := in.Buffer n := in.Bufferized @@ -210,6 +287,7 @@ func (in *Input) consumeStringRawEscaped(start, i int) token.T { // //nolint:dupl // the structure is the same but differs subtly and its critical that the inside remains inlined. func (in *Input) consumeStringRawStreamFast() token.T { + in.SawNonASCII = true data := in.Buffer n := in.Bufferized start := in.Consumed // first content byte (opening quote already consumed) @@ -279,6 +357,7 @@ func (in *Input) consumeStringRawStreamFast() token.T { // consumeStringRawStreaming is the raw scan over a refilling buffer: it copies the source bytes verbatim (escapes // intact) into l.in.CurrentValue while validating them, so the value survives buffer turnover. func (in *Input) consumeStringRawStreaming() token.T { + in.SawNonASCII = true in.CurrentValue = in.CurrentValue[:0] for { diff --git a/json/lexers/default-lexer/internal/swar/swar.go b/json/lexers/default-lexer/internal/swar/swar.go index e9b8fc3..12a11d6 100644 --- a/json/lexers/default-lexer/internal/swar/swar.go +++ b/json/lexers/default-lexer/internal/swar/swar.go @@ -32,6 +32,9 @@ const ( // lowest set bit belongs to the lowest-address lane. func FirstByte(mask uint64) int { return bits.TrailingZeros64(mask) >> 3 } +// HighBits is the high bit of every lane: a word OR-accumulated over a byte run is non-ASCII iff (acc & HighBits) != 0. +const HighBits = high + // Broadcast replicates b into all eight lanes of a word. func Broadcast(b byte) uint64 { return lo * uint64(b) } diff --git a/json/lexers/default-lexer/options.go b/json/lexers/default-lexer/options.go index 6811b49..0968242 100644 --- a/json/lexers/default-lexer/options.go +++ b/json/lexers/default-lexer/options.go @@ -11,6 +11,7 @@ type ( keepPreviousBuffer int elideSeparator bool noAVX2 bool + validateMode int jsonPointer bool } ) @@ -85,6 +86,28 @@ func WithElideSeparator(enabled bool) Option { // shows the vector path is not paying off. // // It does not change the token stream, only how the scan is performed. +// WithUTF8ValidationMode (PROTOTYPE) selects the UTF-8 validation strategy (see input.Validate* constants). +func WithUTF8ValidationMode(mode int) Option { + return func(o options) options { + o.validateMode = mode + + return o + } +} + +// WithUTF8Validation (PROTOTYPE) enables RFC 7493 UTF-8 validation of string values. +func WithUTF8Validation(enabled bool) Option { + return func(o options) options { + if enabled { + o.validateMode = 2 + } else { + o.validateMode = 0 + } + + return o + } +} + func WithoutAVX2(disabled bool) Option { return func(o options) options { o.noAVX2 = disabled diff --git a/json/lexers/default-lexer/semantic_lexer.go b/json/lexers/default-lexer/semantic_lexer.go index 3096163..932a2be 100644 --- a/json/lexers/default-lexer/semantic_lexer.go +++ b/json/lexers/default-lexer/semantic_lexer.go @@ -226,6 +226,7 @@ func (l *L) reset() { l.in.MaxValueBytes = l.maxValueBytes l.in.KeepPreviousBuffer = l.keepPreviousBuffer l.in.NoAVX2 = l.noAVX2 + l.in.ValidateMode = l.validateMode l.current = token.None l.in.Offset = 0 l.in.Consumed = 0 From f39eedee0d50cb0995dc944154511eff29f0f8d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20BIDON?= Date: Wed, 29 Jul 2026 23:06:22 +0200 Subject: [PATCH 02/10] more details as TODO.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Frédéric BIDON --- TODO.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 TODO.md diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..e64f6f3 --- /dev/null +++ b/TODO.md @@ -0,0 +1,24 @@ +# UTF-8 validation + +1. We need the JSON lexer to be able to valide UTF-8 sequences: + invalid UTF-8 should not leak in output strings +2. The default behavior of L and VL is to error on such invalid input +3. As an option (to both lexers), the error can be swallowed and the invalid UTF-8 sequence + be in this case mangled as U+FFFD (error rune) +4. For VL, we pledge _verbatim_. This is important to maintain original offset/positioning. + But this pledge is broken when mangling an invalid sequence. TO BE DECIDED. + +Classic algorithm for invalid sequence detection: https://nemanjatrifunovic.substack.com/p/decoding-utf-8-part-vii-validation. + +Advanced SIMD-enabled C++ library for validating UTF-8: https://github.com/simdutf/simdutf +(available locally for convenient inspection at /home/fred/src/github.com/fredbi/simdutf). +I am worried that this new requirement wipes out our vast performance improvements with string scanning. + +I think we should analyze the methods used at simdutf (e.g. function simdtuf::validate_utf8_with_errors). +We don't have to port their full support of AVX512 etc, just understand their general approach and +port it to our more modest SWAR & AV2 support. + + +6. Final point: recheck the conformance suite test and list all "implementation-dependent point" + where L or VL don't pass the test, so we can get a more complete assessment of how many similar bugs + remain hidden in our cupboard. From 5d6de0086aca6435cbcdd97c1b40f3d41dfe1060 Mon Sep 17 00:00:00 2001 From: Frederic BIDON Date: Thu, 30 Jul 2026 18:18:40 +0200 Subject: [PATCH 03/10] fix(json-lexers): validate UTF-8 in string values (L & VL) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Neither lexer validated UTF-8: the string scanners stop only on '"', '\' and control chars, so every byte >= 0x80 was waved through and aliased straight into the token value. Ten i_string_* JSONTestSuite documents carrying ill-formed UTF-8 were silently ACCEPTED, and the corruption then became invisible rather than absent (Go yields U+FFFD when such a value is iterated as a string). RFC 8259 §8.1 requires JSON text to be UTF-8. Supersedes the prototype in fb066a1. Plan: .claude/plans/utf8-validation.md. API --- One knob, WithUTF8Policy: UTF8Strict (default) invalid UTF-8 -> ErrInvalidUTF8, a broken \u surrogate pair -> ErrSurrogateEscape UTF8Replace substitute U+FFFD: one per invalid byte, one per broken escape UTF8Passthrough no raw-byte validation (unsafe; the pre-fix behavior) The prototype's fused/naive modes are gone: those were implementation strategies, not policies, and are now chosen internally from the CPU. The same policy governs \u escapes that do not denote a scalar value, which removes a live L/VL divergence: L laundered a broken pair into a silent U+FFFD (utf16.DecodeRune returns RuneError, and utf8.ValidRune(U+FFFD) is true) while VL errored. L also used to swallow the second escape, so "\uD800\uD800" yielded one replacement; it now yields two, matching token.Unescape. Performance ----------- Detection is fused into the scan already being performed: every SWAR word and every AVX2 block is OR-accumulated, so "did this value hold a byte >= 0x80" is answered with no second read and no call, and only values that actually do reach the validator. The AVX2 string-stop kernel gained a second result for this (VPMOVMSKB of the block already in the register, masked below the stop lane) -- without it every string past guessLong was force-validated, which was most of the prototype's cost. Validation itself is a port of the simdutf/Keiser-Lemire lookup4 algorithm to avo AVX2 (json/internal/utf8x), 32 B/iter, 5-12x the stdlib once engaged. vs. no validation, benchstat 10x300ms: the four ASCII-dominated corpus workloads are statistically indistinguishable; twitter_status (30% non-ASCII by string bytes) costs 6.3% (L) / 1.7% (VL). Geomean -0.36%. Zero-copy is preserved: validation only reads, so no policy adds a copy to a well-formed document. Only an offending value is rewritten under UTF8Replace. Also ---- - A leading UTF-8 BOM is now consumed instead of rejected (RFC 8259 §8.1 allows ignoring one on input). It is checked once on the buffer's opening bytes, so no scan loop is touched. It is NOT re-emitted by VL: the mark is consumed before any token exists, so it belongs to no token's leading space -- the one exception to VL's byte-exact round-trip on valid input, and the direction §8.1 asks for. A UTF-16 BOM now reports ErrNotUTF8 instead of a baffling "invalid JSON token". - VL's mangling contract is spelled out: U+FFFD is three bytes replacing one, so a rewritten value's length is not its source width and an index into it cannot be mapped back to a source offset. Positions (Offset/Line/Column/LeadingSpace) stay exact under every policy -- they come from the scan cursor, never from the value. - Shared primitives live in json/internal/utf8x, the one location reachable from both the lexers and the writers, which have the mirror-image bug. Tests ----- - Every policy x every one of the eight string paths (whole/stream x clean/ escaped x L/VL), plus keys, long values and WithoutAVX2. - Conformance: the ten cases flip to reject; added the VL/reader mode (VL's streaming string scanners had no conformance coverage) and an L/replace mode that pins replacement as relaxing nothing but ill-formed UTF-8. - The implementation-defined (i_) verdict table is now snapshotted in testdata/conformance_i_behavior.golden. Nothing asserted it before, which is exactly how this bug survived from the day the zero-copy fast path landed. - FuzzLexer gained the invariant that an emitted value is always utf8.Valid, across four lanes x two policies. - The AVX2 kernel is checked against the stdlib exhaustively over all 65536 byte pairs and over every lead byte C0-FF x all 2nd x all 3rd bytes, at several alignments, plus seam straddles, every length, and ~325M fuzzed validations. Mutation-tested: three single-bit table corruptions are each caught. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Frederic BIDON --- .claude/plans/utf8-validation.md | 777 ++++++++++++++++++ TODO.md | 12 +- go.work | 1 + .../lexers/lanes/utf8scratch_test.go | 132 +++ json/internal/utf8x/_asm/asm.go | 195 +++++ json/internal/utf8x/_asm/doc.go | 7 + json/internal/utf8x/_asm/go.mod | 11 + json/internal/utf8x/_asm/go.sum | 10 + json/internal/utf8x/bench_test.go | 55 ++ json/internal/utf8x/cpuid_amd64.s | 20 + json/internal/utf8x/detect_amd64.go | 43 + json/internal/utf8x/utf8x.go | 117 +++ json/internal/utf8x/utf8x_test.go | 196 +++++ json/internal/utf8x/validate_amd64.go | 85 ++ json/internal/utf8x/validate_amd64.s | 117 +++ json/internal/utf8x/validate_amd64_test.go | 285 +++++++ json/internal/utf8x/validate_noasm.go | 11 + json/lexers/default-lexer/README.md | 56 ++ json/lexers/default-lexer/conformance_test.go | 156 +++- json/lexers/default-lexer/doc.go | 18 + json/lexers/default-lexer/fuzz_test.go | 133 ++- .../default-lexer/internal/input/input.go | 39 +- .../default-lexer/internal/input/refill.go | 37 + .../default-lexer/internal/input/string.go | 294 +++---- .../internal/input/string_raw.go | 260 +++--- .../default-lexer/internal/input/utf8.go | 209 +++++ .../internal/strscan/_asm/asm.go | 49 +- .../internal/strscan/bench_amd64_test.go | 6 +- .../internal/strscan/scan_amd64.go | 13 +- .../internal/strscan/scan_noasm.go | 4 +- .../internal/strscan/stringstop_amd64.s | 64 +- .../default-lexer/internal/strscan/strscan.go | 24 +- .../internal/strscan/strscan_test.go | 112 ++- .../default-lexer/internal/swar/swar.go | 11 + json/lexers/default-lexer/options.go | 82 +- json/lexers/default-lexer/semantic_lexer.go | 8 +- .../testdata/conformance_i_behavior.golden | 35 + json/lexers/default-lexer/utf8_test.go | 586 +++++++++++++ json/lexers/default-lexer/verbatim_lexer.go | 34 + json/lexers/error-codes/errors.go | 2 + 40 files changed, 3886 insertions(+), 420 deletions(-) create mode 100644 .claude/plans/utf8-validation.md create mode 100644 json/benchmarks/lexers/lanes/utf8scratch_test.go create mode 100644 json/internal/utf8x/_asm/asm.go create mode 100644 json/internal/utf8x/_asm/doc.go create mode 100644 json/internal/utf8x/_asm/go.mod create mode 100644 json/internal/utf8x/_asm/go.sum create mode 100644 json/internal/utf8x/bench_test.go create mode 100644 json/internal/utf8x/cpuid_amd64.s create mode 100644 json/internal/utf8x/detect_amd64.go create mode 100644 json/internal/utf8x/utf8x.go create mode 100644 json/internal/utf8x/utf8x_test.go create mode 100644 json/internal/utf8x/validate_amd64.go create mode 100644 json/internal/utf8x/validate_amd64.s create mode 100644 json/internal/utf8x/validate_amd64_test.go create mode 100644 json/internal/utf8x/validate_noasm.go create mode 100644 json/lexers/default-lexer/internal/input/utf8.go create mode 100644 json/lexers/default-lexer/testdata/conformance_i_behavior.golden create mode 100644 json/lexers/default-lexer/utf8_test.go diff --git a/.claude/plans/utf8-validation.md b/.claude/plans/utf8-validation.md new file mode 100644 index 0000000..be36cd4 --- /dev/null +++ b/.claude/plans/utf8-validation.md @@ -0,0 +1,777 @@ +# UTF-8 validation in the JSON lexers (L & VL) + +> Branch `fix/utf8-validation-prototype`. Supersedes the prototype commit `fb066a1`. +> Companion to [fredbi/core `.claude/plans/default-lexer-roadmap.md`] §0.4/§3.1 and +> `string-unescape-optimization.md` (which listed UTF-8 validation as *explicitly out +> of scope*: "we deliberately don't validate on the alias path" — this plan closes +> that hole). +> Status legend: ✅ done · ⏳ in progress · ⬜ todo · 🔬 measure first. + +## OUTCOME — round 2 shipped (2026-07-30) + +The simdutf/Keiser–Lemire `lookup4` validator is ported to avo AVX2 (`json/internal/utf8x/_asm` → +`validate_amd64.s`), 32 B/iter, and the BOM and VL-contract items are done. **The gate is now met.** + +`UTF8Strict` vs no validation, benchstat 10×300 ms: + +| workload | round 1 (L) | round 2 (L) | round 1 (VL) | round 2 (VL) | +|---|---|---|---|---| +| canada_geometry | ~ | ~ | ~ | ~ | +| citm_catalog | ~ | ~ | ~ | ~ | +| citm_catalog_min | ~ | ~ | +5.0% | ~ | +| golang_source | ~ | ~ | ~ | ~ | +| twitter_status | **−10.8%** | **−6.3%** | **−8.2%** | **−1.7%** | + +Geomean −1.4% → **−0.36%**. Raw validator throughput (`BenchmarkValid`): 5–12× the stdlib once the kernel engages — +`cjk/4096` 248 ns vs 2420 ns (~16 GB/s), `cjk/128` 12.5 ns vs 80 ns. + +`avx2Min` settled at **32** (one block, the kernel's hard minimum). The micro-benchmark shows a clear win from 32 +bytes up (`latin/32`: 6.7 ns vs 18.1 ns); end-to-end the 32-vs-64 choice is not statistically distinguishable, so the +micro-benchmark decided it. + +**How the kernel is trusted.** Hand-written vector code that accepts ill-formed input fails silently, so the +differential suite is the deliverable, not a formality: + +- exhaustive over **all 65 536 byte pairs** (which is exactly what the three VPSHUFB tables encode) at 10 alignments; +- exhaustive over **every lead byte C0–FF × all 2nd × all 3rd bytes** (4.2 M) at 3 alignments — this is what exercises + the prev2/prev3 continuation logic the pairwise sweep cannot reach; +- every 4-byte lead F0–FF × every 2nd byte × 5 tails × 8 alignments (TOO_LARGE / OVERLONG_4 live here); +- every length 0…256, every truncation shape pinned at the end, seam straddles across the last block boundary; +- 40 k random/structured-noise cases; 9.9 M fuzz executions, each re-checked at 32 shifted alignments (~325 M + validations) — all against `utf8.Valid`; +- `TestKernelIsActuallyExercised` fails the suite from becoming vacuous if AVX2 is absent or the length gate keeps + everything scalar. + +**Mutation-tested:** three single-bit corruptions of the lookup tables (dropping SURROGATE from the `1110____` lead, +dropping it from the `____1101` low nibble, flipping a bit in `byte_2_high`) were each caught. The suite has teeth. + +**Design note — the tail.** The kernel validates whole 32-byte blocks only and says nothing about a sequence crossing +the end; `Valid` rewinds to the last sequence boundary and finishes with `utf8.Valid`. That is simpler than an in-asm +zero-padded tail and mirrors what simdutf does to locate errors. It does mean a value just past a block boundary pays +a scalar tail of up to 31 bytes, which the micro-benchmark shows costing more than the vector part for such lengths. +Closing that would need the kernel to accept a carried-in `prev` block so the final partial block can be revalidated +from an overlapping offset — a signature change plus its own differential pass. Not done; the gate is met without it. + +**BOM (§2.8) — now shipped**, the way it was originally scoped minus one thing: the mark is consumed at offset 0 and +is **not** re-emitted by `VL`. Carrying it in `LeadingSpace()` would need `blankStart` changes across 6 core sites (4 +generated) plus ~20 streaming resets; consuming it costs one comparison per document and touches no scan loop. The +`FuzzLexer` round-trip invariant is correspondingly relaxed to "byte-exact except a leading BOM", which is the +direction RFC 8259 §8.1 asks for. `i_structure_UTF-8_BOM_empty_object` flips to accept; the two `n_structure_*BOM*` +cases stay rejected, and the `i_` golden diff showed exactly that one line changing. + +## OUTCOME — round 1 shipped (2026-07-30) + +All ten ill-formed-UTF-8 conformance cases now reject, in every mode; the L/VL surrogate-escape divergence is gone; +`y_ 95/95, n_ 188/188, xfail 0` still holds. 2052 new subtests, 2.5M fuzz executions clean, lint clean. + +**Cost of the default policy vs no validation** (benchstat, 10×300 ms, MB/s): + +| workload | L | VL | +|---|---|---| +| canada_geometry | ~ (p=0.68) | ~ (p=0.44) | +| citm_catalog | ~ (p=0.25) | ~ (p=0.74) | +| citm_catalog_min | ~ (p=0.53) | +5.0% (p=0.009) | +| golang_source | ~ (p=0.35) | ~ (p=0.58) | +| twitter_status | **−10.8%** (p=0.000) | **−8.2%** (p=0.000) | + +Geomean −1.4%. The four ASCII-dominated workloads are statistically indistinguishable from no validation — the fused +detection works as designed, and the prototype's AVX2 hole (§4.2) is closed: `golang_source` went from −6.6% to ~0 and +`citm_catalog_min` from −10.0% to ~0. + +**Gate result: ASCII target met (≤3%), `twitter_status` missed (≤8% wanted, −10.8% measured).** The arithmetic says +this is exactly the validator pass and nothing else: 109 KB of non-ASCII string bytes at `utf8.Valid`'s ~1.2 GB/s is +~90 µs against a 651 µs baseline ≈ 14%, observed 10.8%. So **round 2 (§5.4) is justified** — an AVX2 `lookup4` +validator at ~20 GB/s takes that to ~5 µs, i.e. under 1%. Awaiting the go-ahead. + +**Deviations from the plan as written:** + +1. **BOM trimming (§2.8) is NOT shipped** — only the UTF-16 diagnostic (`ErrNotUTF8`) is. Accepting a leading UTF-8 + BOM breaks VL's byte-exact round-trip, which `FuzzLexer` enforces (seed #62, `"{}"`), and carrying the mark + in `LeadingSpace()` means `blankStart` must start at 0 on the first token: 6 sites, 4 of them in generated + `scan_gen.go`, plus ~20 per-token `l.blanks[:0]` resets in the streaming cores. That is its own commit with a full + codegen verification, not a rider on this one. +2. **Replacement granularity (§2.4) was corrected during review** to one U+FFFD per *invalid byte* — `utf8.DecodeRune` + always advances 1 on error, so the original "maximal subpart, matching DecodeRune's advance" was self-contradictory. + The deciding argument is that `writer.escapedBytes` already does per-byte, so lexer, writer and caller-side + `range value` now agree. +3. **`ensureWindow` was added to `refill.go`** (not in the plan): the surrogate lookahead must *peek* the following + `\uXXXX` rather than consume it, so that `\uD800\uD800` yields two U+FFFD instead of swallowing the second escape. + `consumeN` cannot serve that — a read that straddles a refill cannot be given back — so the cold path compacts the + window instead. Safe there because the scanners that reach it hold no live buffer index. + +## 0. TL;DR + +**The bug.** Neither `L` nor `VL` validates UTF-8 in string bodies. The string +scanners look for exactly three stop bytes (`"`, `\`, `< 0x20`); every byte +`>= 0x80` is waved through and aliased straight into the token value. So +`["\xff"]`, `["\xc0\xaf"]` (overlong), `["\xed\xa0\x80"]` (encoded surrogate) and +`["\xf4\xbf\xbf\xbf"]` (> U+10FFFF) are all **accepted**, and the invalid bytes +land in the caller's `[]byte`. RFC 8259 §8.1 requires JSON text to be UTF-8, so +this is a genuine conformance bug — 10 `i_string_*` conformance cases, plus a +secondary escape-level variant (§2.3). + +**The fix.** One user-visible knob, `WithUTF8Policy`, with three values: +`UTF8Strict` (default — error), `UTF8Replace` (mute the error, substitute U+FFFD), +`UTF8Passthrough` (today's unvalidated behavior, documented as unsafe). +The prototype's `fused`/`naive` are **implementation strategies**, not policies: +they disappear from the API and are chosen internally from CPU/build. + +**The performance answer.** The concern that this "wipes out our vast string-scan +improvements" is real for a naive `utf8.Valid` at the funnel (**−10% to −32%** +measured, §5.1) but *not* for the fused design, provided one hole in the prototype +is plugged: the AVX2-scanned region is not accumulated, so it conservatively forces +validation on **every long string** — including the 10905 pure-ASCII long strings of +`golang_source`. Plugging it costs ~4 instructions per 32-byte iteration +(`VPMOVMSKB` of the raw data, already loaded, masked below the stop lane). Real-world +JSON is 96–100% ASCII strings by count (§5.2), so after the fix the overwhelming +majority of values are proven valid UTF-8 **inside the existing pass, with no second +read and no call**. + +**Also in scope** (settled on review): a leading UTF-8 BOM is trimmed (§2.8, carried in +`VL.LeadingSpace()` so round-trip stays exact), and the shared primitives land in a new +`json/internal/utf8x` (§7.1) — the one package location reachable from **both** the +lexers and the writers, because the writers have the mirror-image bug and will reuse +them (§10 round 3). + +**Sequencing.** Round 1 = correctness + fused detection + the AVX2 kernel flag + +`utf8.Valid` as the validator. Round 2 = port the simdutf/Keiser–Lemire `lookup4` +validator to avo AVX2 — **only if** round 1's numbers on `twitter_status` justify it. +Round 3 = the writers. + +--- + +## 1. What is actually broken + +### 1.1 The alias path never looks at high bytes + +`swar.StringStopMask` (`internal/swar/swar.go:47`) and the AVX2 kernel +(`internal/strscan/_asm/asm.go`) both flag exactly `< 0x20`, `0x22`, `0x5c`. Both +are documented as "multibyte-safe", which is true in the sense that they never +*mis*-flag a continuation byte — but that is precisely why nothing ever inspects +those bytes. `consumeStringWhole` then does `value := data[start:i:i]` and hands the +raw window to the caller. + +### 1.2 Everything downstream inherits it + +`L` decodes escapes into `CurrentValue`; raw non-ASCII bytes are `append`ed +verbatim. `VL` aliases the source. Both therefore emit `token.T` / `token.VT` values +that a caller will `string(...)`-convert, at which point Go silently produces U+FFFD +runes on iteration — the corruption becomes invisible rather than absent. + +### 1.3 Evidence (JSONTestSuite, current `master` behavior) + +All ten of these are **accepted** by `L/bytes`, `L/reader` and `VL/bytes` today: + +| file | bytes | why it is invalid | +|---|---|---| +| `i_string_invalid_utf-8` | `5b 22 ff 22 5d` | `0xff` is never a valid lead | +| `i_string_lone_utf8_continuation_byte` | `81` | continuation with no lead | +| `i_string_iso_latin_1` | `e9` | latin-1 `é`, truncated lead | +| `i_string_truncated-utf-8` | `e0 ff` | 3-byte lead, bad continuation | +| `i_string_UTF-8_invalid_sequence` | `e6 97 a5 d1 88 fa` | valid prefix then `0xfa` | +| `i_string_UTF8_surrogate_U+D800` | `ed a0 80` | encoded UTF-16 surrogate | +| `i_string_not_in_unicode_range` | `f4 bf bf bf` | U+13FFFF > U+10FFFF | +| `i_string_overlong_sequence_2_bytes` | `c0 af` | overlong `/` | +| `i_string_overlong_sequence_6_bytes` | `fc 83 bf bf bf bf` | obsolete 6-byte form | +| `i_string_overlong_sequence_6_bytes_null` | `fc 80 80 80 80 80` | overlong NUL | + +They are `i_` (implementation-defined) in the suite, so the conformance test stays +green — that is why this went unnoticed. Under `UTF8Strict` all ten become +**rejected**; under `UTF8Replace` they are accepted with the offending bytes +substituted. + +--- + +## 2. Semantics (settled) + +### 2.1 The policy + +```go +// UTF8Policy governs what the lexer does with input that cannot be represented +// as a Unicode scalar value. +type UTF8Policy int + +const ( + UTF8Strict UTF8Policy = iota // default: error out (codes.ErrInvalidUTF8) + UTF8Replace // substitute U+FFFD, no error + UTF8Passthrough // no validation at all — UNSAFE, trusted input only +) + +func WithUTF8Policy(p UTF8Policy) Option +``` + +- **Default is `UTF8Strict` for both `L` and `VL`.** This is a behavior change: + documents previously accepted are now rejected. It is the whole point of the fix + and belongs in the changelog as a breaking-ish conformance change. +- `UTF8Passthrough` exists as an escape hatch for callers who have already validated + their input (and for A/B benchmarking of the validation cost). Its godoc must say + plainly that invalid bytes will reach the caller's `[]byte`. +- `WithUTF8ValidationMode` / `WithUTF8Validation` from the prototype are **deleted**. + Nothing about `fused` / `naive` / SWAR / AVX2 is user-visible; `WithoutAVX2` stays + the single vector-path escape hatch and now also disables the vector validator. + +### 2.2 What counts as invalid (raw bytes) + +Exactly Go's `utf8.Valid` semantics, which is also RFC 3629 + Unicode: +truncated sequences, over-long encodings, lead/continuation mismatches, encoded +surrogates `U+D800..U+DFFF`, and anything above `U+10FFFF`. No exceptions: we do +**not** accept WTF-8 or CESU-8. + +### 2.3 What counts as invalid (`\u` escapes) — L/VL divergence to fix + +The same policy governs `\u` escapes that do not form a Unicode scalar value. This +is a second, independent way an invalid character enters a value, and today the two +lexers disagree about it: + +| input | `L` today | `VL` today | strict (new) | replace (new) | +|---|---|---|---|---| +| `["\uD888ሴ"]` | **accept** → U+FFFD | reject | reject | `"�ሴ"` | +| `["\uD800\uD800\n"]` | **accept** → U+FFFD + swallows the 2nd escape | reject | reject | U+FFFD, U+FFFD, `\n` | +| `["\uDd1e\uD834"]` | **accept** → U+FFFD | reject | reject | U+FFFD, U+FFFD | +| `["\uDADA"]` (no 2nd) | reject | reject | reject | U+FFFD | + +Root cause in `L`: `input.unescapeUnicodeSequence` (`string.go:659`) calls +`utf16.DecodeRune`, which *returns* `utf8.RuneError` for a broken pair, and the +subsequent `utf8.ValidRune(0xFFFD)` is **true** — so the failure is laundered into a +silent replacement. `VL`'s `validateUnicodeWhole` (`string_raw.go`) checks +`DecodeRune(...) == utf8.RuneError` explicitly and errors. + +> Historical note: `default-lexer-roadmap.md` §0.1 recorded this divergence in the +> *opposite* direction ("L rejects, VL accepts"). Conformance fix group E made `VL` +> validate its escapes, which inverted the asymmetry rather than removing it. + +Fixes: +- **strict** — `unescapeUnicodeSequence` must treat `DecodeRune == RuneError` as + `ErrSurrogateEscape`, matching `validateUnicodeWhole`. +- **replace** — `validateUnicodeWhole` must *stop* erroring on a broken pair; + `unescapeUnicodeSequence` must emit U+FFFD and **not consume the second escape** + (today it does, so `\uD800\uD800` yields one U+FFFD instead of two). + `token.decodeUnicode` (`token/unescape.go:108`) already has the correct + maximal-subpart behavior — copy its shape. + +### 2.4 Replacement granularity + +**One U+FFFD per invalid byte** — i.e. `utf8.DecodeRune`'s error advance, which is +always width 1. `\xe0\xa0\xff` → three replacements. + +The alternatives were considered and rejected: + +| rule | `\xe0\xa0\xff` | who does it | +|---|---|---| +| **per invalid byte** ✅ | 3 × U+FFFD | Go: `range` over a string, `utf8.DecodeRune`, **and our own `writer.escapedBytes`** | +| maximal subpart (Unicode TR §3.9) | 2 × U+FFFD | browsers / WHATWG | +| per run | 1 × U+FFFD | `bytes.ToValidUTF8` | + +Decider: `writers/default-writer/escape.go:escapedBytes` **already** substitutes +U+FFFD per invalid byte (`utf8.DecodeRune` → `utf8.AppendRune`, the comment says +"invalid runes are represented as �"). Picking the same rule means the lexer and +the writer agree, and `string(value)` / `range value` in caller code agrees too. It is +also the cheapest. + +Independent of the byte rule: each **broken `\u` escape** is one U+FFFD, so +`\uD800\uD800` → two. That is already `token.decodeUnicode`'s behavior. + +### 2.5 The verbatim pledge (TODO §4 — resolved) + +`VL` **does** substitute U+FFFD in the value under `UTF8Replace`. Restated pledge: + +> `VL` reproduces its input byte-for-byte **for input that is valid UTF-8**. Under +> `UTF8Replace` an ill-formed sequence is replaced in the emitted value; the caller +> explicitly asked for that trade. Offsets, `Line()`, `Column()` and `LeadingSpace()` +> are **unaffected** — they are derived from the scan cursor over the source, never +> from the value — so positions stay exact even for a mangled token. + +Justification: a document containing invalid UTF-8 is not valid JSON text, so a +byte-faithful round-trip of it re-emits invalid JSON. A caller who enables +replacement wants clean bytes out. Callers who need absolute byte fidelity keep the +default `UTF8Strict` (which never copies and never mangles — it only errors) or opt +into `UTF8Passthrough`. + +**Escapes are the exception**, and this falls out naturally: `VL` keeps string bodies +raw, so a broken `\uD800\uD800` stays *as source text* in the value; the U+FFFD +substitution happens at decode time in `token.Unescape`, which already does exactly +that. So under `UTF8Replace`, `VL` mangles *bytes* but never rewrites *escape text* — +the value stays a faithful raw string, and `Unescape` yields the sanitized form. + +### 2.6 Zero-copy invariants preserved + +| policy | value | copy? | +|---|---|---| +| `UTF8Strict`, valid input | aliased as today | no | +| `UTF8Strict`, invalid input | n/a — error | no | +| `UTF8Replace`, valid input | aliased as today | no | +| `UTF8Replace`, invalid input | rewritten into `CurrentValue` | only the offending value | +| `UTF8Passthrough` | aliased as today | no | + +Validation is read-only. **No policy adds a copy to a valid document.** + +### 2.7 `UTF8Passthrough` and escapes (settled) + +> **FRED** agree (should be documented): explicitly escaped UTF-8 MUST produce some +> rune, so U+FFFD when broken and policy is lenient. + +`UTF8Passthrough` skips **raw-byte** validation only. A `\uXXXX` escape must always +decode to *some* rune — there is no "pass the bytes through" option for an escape, +because the escape text is not the value. So: + +| | raw bytes | broken `\u` pair | +|---|---|---| +| `UTF8Strict` | error | error | +| `UTF8Replace` | U+FFFD per invalid byte | U+FFFD per broken escape | +| `UTF8Passthrough` | untouched, uninspected | U+FFFD per broken escape | + +Passthrough and Replace therefore share the whole escape path; only Strict differs. +This must be stated explicitly in the `UTF8Passthrough` godoc, not left implied. + +### 2.8 Leading byte-order mark (settled — now in scope) + +> **FRED** we'll add support for leading BOM trimming + +A UTF-8 BOM (`EF BB BF`) at **offset 0 only** is consumed as insignificant lead-in +before the top-level value. RFC 8259 §8.1 forbids *emitting* one but explicitly +permits ignoring one on input. `i_structure_UTF-8_BOM_empty_object` (`EF BB BF 7B 7D`) +flips from reject to **accept**. + +Rules, with the traps: + +- **Exactly one, exactly at offset 0.** A second BOM, or a BOM after any token, is + not special: it is U+FEFF, valid UTF-8, and therefore an invalid *token* between + values (rejected) or an ordinary character inside a string (kept). No change there. +- **`n_structure_UTF8_BOM_no_data`** (`EF BB BF`, nothing else) must **stay rejected**: + trimming leaves an empty document → `ErrNoData`. Falls out for free. +- **`n_structure_incomplete_UTF8_BOM`** (`EF BB 7B 7D` — a *truncated* BOM) must + **stay rejected**: only the full 3-byte sequence is trimmed, and `EF BB` is then an + invalid token. Falls out for free, and is now also caught as invalid UTF-8. +- **`VL` must not lose it.** Trimming the BOM would silently break round-trip. It is + accumulated into `l.blanks`, exactly like leading whitespace, so `LeadingSpace()` + on the first token replays it and a re-serialized document is byte-identical. + This is the one place the implementation differs between L and VL. +- **Streaming.** The check runs on the first fill, before the first token scan, and + must handle a window that starts with a *partial* BOM (buffer sizes are ≥ 32 bytes + and aligned, so a 3-byte prefix cannot be split — assert it rather than assume it). +- **Bonus (cheap, do it):** `FF FE` / `FE FF` at offset 0 is a UTF-16 BOM. Today it + fails as a generic invalid token. Report a dedicated `ErrNotUTF8` ("input appears to + be UTF-16, only UTF-8 is supported") — it turns a baffling parse error into an + actionable one, and costs one comparison on a cold path. + (`i_string_UTF-16LE_with_BOM`, `i_string_utf16BE_no_BOM`, `i_string_utf16LE_no_BOM` + stay rejected either way.) + +Not an option knob: trimming only ever *accepts* input that is rejected today, so +there is nothing to opt out of. + +--- + +## 3. Where validation hooks in + +`finishStringValue` (`string.go:461`) is a true funnel: all eight string paths reach +it. That is where the decision is taken; the *detection* is what varies per path. + +| # | path | file:func | round-1 detection | +|---|---|---|---| +| 1 | L whole, clean | `string.go:consumeStringWhole` | fused SWAR + AVX2 flag | +| 2 | L whole, escaped | `string.go:consumeStringEscaped` | 🔬 assume non-ASCII (validate) | +| 3 | L stream, fast | `string.go:consumeStringStreamFast` | fused SWAR + AVX2 flag | +| 4 | L stream, byte-wise | `string.go:consumeStringStreaming` | 🔬 assume non-ASCII | +| 5 | VL whole, clean | `string_raw.go:consumeStringRawWhole` | fused SWAR + AVX2 flag | +| 6 | VL whole, escaped | `string_raw.go:consumeStringRawEscaped` | 🔬 assume non-ASCII | +| 7 | VL stream, fast | `string_raw.go:consumeStringRawStreamFast` | fused SWAR + AVX2 flag | +| 8 | VL stream, byte-wise | `string_raw.go:consumeStringRawStreaming` | 🔬 assume non-ASCII | + +"assume non-ASCII" = correct but pessimistic: those values always pay a validation +pass. Escaped/streaming strings are the minority; **measure before optimizing** +(`golang_source` is escape-heavy and is the workload to watch — if it shows, the same +`hi |= w` accumulation goes into the clean-run scans of paths 2/4/6/8). + +The prototype forked `consumeStringWhole`/`consumeStringRawWhole` into `…V` twins +selected at dispatch. **Do not keep the fork.** The accumulator is one `OR` on a word +that is already in a register; a duplicated scanner doubles the surface where the +inliner/register-allocator can regress (the roadmap's §4.2 lesson: a fast-path change +regressed the slow path by 12% when they shared a frame). Land the accumulator in the +single existing function and verify with the existing devirt/inline tests. + +Non-string tokens need nothing: invalid bytes in numbers/literals/structure already +fail the grammar (`n_number_invalid-utf-8-in-int` etc. all pass today). + +--- + +## 4. Detection: fusing the non-ASCII test into the existing scan + +### 4.1 SWAR region — free + +Already right in the prototype: + +```go +w := binary.LittleEndian.Uint64(data[i:]) +hi |= w // <- the whole cost +if m := swar.StringStopMask(w); m != 0 { ... } +``` + +`hi & swar.HighBits != 0` ⇒ some byte in the scanned words was `>= 0x80`. +Over-reporting (bytes past the stop lane in the matching word) is safe: it can only +trigger a redundant validation, never skip a needed one. The scalar tail ORs each +byte it inspects. + +### 4.2 AVX2 region — the prototype's hole (the important fix) + +Today `strscan.ScanStop` returns only an index, so the prototype sets +`hi = ^uint64(0)` for anything it delegated — i.e. **every string longer than +`guessLong` (16) clean bytes is force-validated**. `golang_source` has 10905 such +strings and zero non-ASCII bytes; it pays a full `utf8.Valid` over 858 KB for +nothing. This, not UTF-8 itself, is most of the measured regression. + +Fix — the kernel already has the data in a YMM register: + +``` +VPMOVMSKB acc, mask // existing: stop-byte lanes +VPMOVMSKB data, hmask // NEW: sign bit of every byte == "byte >= 0x80" +ORL hmask, nonascii // NEW: accumulate +``` + +and on the `found` branch, zero the lanes at/above the stop before OR-ing +(`MOVL $1,cl-shift; SHLL; DECL` — plain shifts, no BMI2 needed, so the existing +AVX2-only CPUID gate stays valid). The scalar tail loop ORs each byte. + +Signature becomes: + +```go +func stringStopIndexAVX2(data []byte) (idx int, nonASCII bool) +func ScanStop(data []byte) (idx int, nonASCII bool) // SWAR fallback accumulates too +``` + +`ScanStop` has four other call sites (the clean-run scans in paths 2/4/6/8); they can +ignore the second result in round 1. + +**Cost:** one `VPMOVMSKB` + one `ORL` per 32 bytes, off the critical dependency chain +(the loop's exit test is unchanged). Expected ≈ 0 measurable — must be confirmed by +re-running the `strscan` micro-benchmarks and the corpus sweep. + +### 4.3 Result + +A pure-ASCII string is proven valid UTF-8 **without a second read of its bytes and +without a call**. Since 96–100% of corpus strings are pure ASCII (§5.2), the common +case is genuinely free, which is the whole basis for claiming this does not undo the +string-scan work. + +--- + +## 5. Validation: the second pass, and how expensive it may stay + +### 5.1 Measured cost of the prototype (baseline for the work) + +`BenchmarkUTF8Validate`, MB/s, median of 6 × 200 ms, vs `off`: + +| workload | L naive | L fused | VL naive | VL fused | +|---|---|---|---|---| +| canada_geometry | −1.0% | −0.3% | +5.0% | +3.8% | +| citm_catalog | −19.9% | −1.0% | −16.4% | −1.0% | +| citm_catalog_min | −23.2% | −10.0% | −9.8% | +1.0% | +| golang_source | −26.0% | −6.6% | −25.5% | −13.3% | +| twitter_status | −31.9% | −23.8% | −25.2% | −19.6% | + +Reading: `naive` (validate every value) is unacceptable. `fused` is already close to +free where strings are short (`citm_catalog`), and loses where strings are *long* +(`golang_source`, `citm_catalog_min`, `twitter_status`) — the §4.2 hole — plus a +genuine non-ASCII cost on `twitter_status`. + +### 5.2 Why the fused design can work — corpus profile + +`TestCorpusStringProfile`: + +| workload | strings | ≥16 B | string bytes | non-ASCII strings | non-ASCII bytes | +|---|---|---|---|---|---| +| canada_geometry | 12 | 1 | 90 (0.0%) | 0 (0.00%) | 0 | +| citm_catalog | 26604 | 1360 | 221 KB (12.8%) | 108 (0.41%) | 2.4 KB | +| citm_catalog_min | 26604 | 1360 | 221 KB (44.2%) | 108 (0.41%) | 2.4 KB | +| golang_source | 102451 | 10905 | 858 KB (44.2%) | 0 (0.00%) | 0 | +| twitter_status | 18099 | 6405 | 368 KB (58.3%) | 755 (4.17%) | 109 KB | + +So: detection must be free (it is, §4), and the validator only ever sees 0–4% of +values — but on `twitter_status` those carry 109 KB, 30% of all string bytes. + +### 5.3 Round-1 validator: `utf8.Valid` + +`utf8.Valid` has an 8-byte ASCII fast path then a scalar `acceptRanges` loop +(~1–1.5 GB/s on genuinely multibyte data). For four of five workloads it will be +invoked on ~0 bytes. Only `twitter_status` pays. + +Error position: `utf8.Valid` gives none. For strict mode we want the offset of the +offending byte in the error. Use a small `validateIndex(b []byte) int` (returns −1 or +the first bad index) built on `utf8.DecodeRune`, called **only on the failing value** — +the fast answer stays `utf8.Valid`. This mirrors simdutf, which likewise SIMD-detects +block-wise and then `rewind_and_validate_with_errors` scalar to locate the byte. + +### 5.4 Round-2 (🔬 gated on round-1 numbers): the simdutf port + +What `simdutf::validate_utf8_with_errors` actually does +(`src/generic/utf8_validation/utf8_lookup4_algorithm.h`) — the Keiser–Lemire +`lookup4` algorithm, per 32-byte block: + +1. `prev1 = concat_shift(prev_block, block, 1)` — `VPALIGNR` + `VPERM2I128`. +2. Three `VPSHUFB` table lookups, `AND`-ed together (`check_special_cases`): + `prev1 >> 4` (16-entry table), `prev1 & 0x0F`, `block >> 4`. The AND of the three + is a per-byte bitset of violated rules: TOO_SHORT / TOO_LONG / OVERLONG_2 / + OVERLONG_3 / OVERLONG_4 / SURROGATE / TOO_LARGE / TWO_CONTS. This single step + catches every 2-byte-sequence error class, including surrogates and overlongs. +3. `check_multibyte_lengths`: `prev2`/`prev3` saturating-subtract to assert that + positions requiring a 2nd/3rd continuation actually have one; `XOR` with (2). +4. `error |= …`; an all-ASCII block short-circuits to just `error |= prev_incomplete`. +5. `is_incomplete` on the last block catches a sequence truncated at the end. +6. Errors are **block-granular**; the exact offset comes from a scalar rewind. + +Port scope for us: AVX2 only (no AVX-512/ICL paths, no NEON), 32 bytes/iter, one +`YMM` of carried state (`prev_block`) plus an error accumulator. ~120 lines of avo. +`VPSHUFB` is per-128-bit-lane on AVX2, which the algorithm already accounts for +(the tables are duplicated across lanes). Expected 15–25 GB/s, i.e. the +`twitter_status` cost collapses from ~100 µs to ~5 µs. + +Portable/SWAR fallback: **do not** attempt a SWAR `lookup4`. Follow the classic +branch-per-sequence range table (the Trifunović article's approach) or simply keep +`utf8.Valid`, which already is that, well-tuned, in the stdlib. Recommendation: keep +`utf8.Valid` as the non-amd64 / `WithoutAVX2` validator and add only the AVX2 kernel. + +**Decision gate:** if round 1 lands every workload within ~3% and `twitter_status` +within ~8%, round 2 is optional polish and can be deferred behind the roadmap's +Phase 4. Re-measure before writing any assembly. + +--- + +## 6. Replacement (`UTF8Replace`) implementation + +Only reached when detection says non-ASCII **and** validation says invalid — i.e. +never on the hot path. + +```go +// in the shared package (§7.1): append src to dst, U+FFFD per invalid byte. +func utf8x.Sanitize(dst, src []byte) []byte +``` + +- Copies the valid prefix in bulk (up to `utf8x.FirstInvalid`), then per-rune from + there. +- The lexer wraps it: writes into `in.CurrentValue` — the existing scratch — so no new allocation and the + reuse contract is unchanged. Careful: on the L escaped path the value *is* + `in.CurrentValue`, so sanitize in place / via the second scratch, not aliasing + itself. (`PreviousBuffer` is not free for this; add a small `sanitizeScratch` or do + a right-to-left in-place rewrite — replacement is 3 bytes and the shortest + ill-formed subpart is 1, so the value can *grow*: in-place needs care, prefer the + scratch.) +- `MaxValueBytes` is re-checked **after** rewriting (a value can grow by up to 3×). +- `VL`: applies to raw bytes only; escape text is never rewritten (§2.5). + +--- + +## 7. Plumbing + +### 7.1 Where the primitives live (revised — writers will reuse them) + +> **FRED** we'll add a check on writers too. We might reuse the same primitive +> functions for fast-checking this so we might reconsider a refactor to higher level +> of "internal" package for those. + +Import reachability decides this, so it is worth getting right on the first commit +rather than moving code later: + +| package | reachable from | +|---|---| +| `json/lexers/default-lexer/internal/{swar,strscan}` | `default-lexer` only | +| `json/lexers/internal/scan` | anything under `json/lexers/…` | +| **`json/internal/…`** | **anything under `json/…` — lexers *and* writers** | +| — | `json/lexers/yaml-lexer` is a **separate module**: cannot import any of these | + +So the new, shareable code goes in a new **`json/internal/utf8x`**: + +```go +package utf8x + +func Valid(b []byte) bool // fast yes/no; AVX2 kernel in round 2, utf8.Valid now +func FirstInvalid(b []byte) int // -1, or the index of the first bad byte (cold, error path) +func Sanitize(dst, src []byte) []byte // append src to dst, U+FFFD per invalid byte (§2.4) +``` + +`writers/default-writer/escape.go` then adopts `utf8x.Sanitize`'s rule instead of its +open-coded `DecodeRune`/`AppendRune` loop, and gains the strict variant it lacks today +(see §10 round 3). + +What does **not** move: the *fused detection* is inline SWAR in the lexer's scanners +and stays there — it is not a callable primitive, it is an `OR` in an existing loop. + +**Deliberately deferred:** hoisting `swar` and `strscan` themselves to `json/internal/` +so the writers can use the SWAR masks too. It is a pure move, but it touches the +hottest, most codegen-sensitive files in the tree. Do it as its **own commit**, before +or after this work but never inside it, gated on `TestInlinable` + the devirt tests + +a full corpus sweep showing byte-identical performance. + +### 7.2 Lexer-side plumbing + +- `options.utf8Policy UTF8Policy` replaces `validateMode int`; `L.reset()` mirrors it + into `in.UTF8Policy` next to `NoAVX2` (keep it in the cold tail of the struct so hot + cursor field offsets are untouched — the prototype already respects this). +- `in.SawNonASCII` stays, but must be **reset per value** (the prototype leaves it + set once any pessimistic path runs; harmless today because it only over-triggers, + but it becomes a real correctness trap the moment anything else reads it). + Prefer passing it as a local/return value rather than as `Input` state, if that + survives the inliner. +- New error code `ErrInvalidUTF8 = "invalid UTF-8 sequence in string"` in + `error-codes/errors.go`. Do **not** reuse `ErrInvalidRune`, whose text says + "in unicode escape sequence" and whose meaning is the escape-level failure. +- Pooled lexers (`pools.go`): confirm the policy survives `Reset*` the way + `elideSeparator` does (`VL.reset()` has a comment about exactly this trap). + +--- + +## 8. Test plan + +- **Conformance** — the 10 files of §1.3 flip to reject under strict; add a + `UTF8Replace` mode to `conformanceModes()` where they flip back to accept. + Add the missing **`VL/reader`** mode (today only `L/bytes`, `L/reader`, `VL/bytes` + run — VL's streaming paths 7/8 are unexercised by conformance). +- **Golden table** — per path (§3, all 8) × per policy (3): a table test with + hand-built inputs that force each path (whole/stream × clean/escaped × L/VL), + asserting accept/reject, the exact value bytes, **and the error offset**. +- **Escape semantics** — the §2.3 table verbatim, both lexers, all three policies, + including `\uD800\uD800\n` yielding *two* replacements. +- **Boundary** — an ill-formed sequence straddling a refill boundary in streaming + mode (`WithBufferSize(64)` and sizes that split a 3/4-byte sequence). +- **BOM** (§2.8) — accepted at offset 0; `VL` replays it via `LeadingSpace()` and a + round-trip of `EF BB BF {}` is byte-identical; BOM-only and truncated-BOM still + rejected; a BOM *after* the first token is still an error; UTF-16 BOM yields + `ErrNotUTF8`. +- **`utf8x` unit + fuzz** — the primitives tested on their own, before the lexer: + `Valid` must agree with `utf8.Valid` on random input, `Sanitize` must be idempotent + and always produce `utf8.Valid` output. This is what makes the round-2 AVX2 kernel + swap-in safe. +- **Differential** — corpus + fuzz corpus: for every string token, our verdict must + agree with `utf8.Valid`, and under replace our output must equal a reference + maximal-subpart sanitizer. +- **Fuzz** — extend `FuzzLexer`/`FuzzYL` seeds with invalid-UTF-8 bodies; assert the + invariant "a token value emitted under Strict or Replace is always `utf8.Valid`". + This is the single strongest guard and directly encodes TODO §1. +- **Inline/devirt guards** — `TestInlinable` (swar) and the existing devirt tests must + stay green; re-check `strscan`'s micro-benchmarks after the kernel change. +- **Bench** — `BenchmarkUTF8Validate` (currently in + `json/benchmarks/lexers/lanes/utf8scratch_test.go`, a throwaway compass instrument) + gets rewritten against the policy API and kept until the numbers are settled. + +--- + +## 9. Conformance audit (TODO §6) + +Current run: **y_ 95/95, n_ 188/188, xfail 0, 35 `i_` cases.** Full `i_` inventory +with current verdicts and the expected post-change verdict: + +| group | files | now | strict | note | +|---|---|---|---|---| +| **raw invalid UTF-8** | the 10 of §1.3 | accept | **reject** | the bug being fixed | +| **broken surrogate escapes, L≠VL** | `1st_valid_surrogate_2nd_invalid`, `incomplete_surrogates_escape_valid`, `inverted_surrogates_U+1D11E` | L accept / VL reject | **reject** (both) | §2.3 — real divergence | +| **broken surrogate escapes, agreed** | `1st_surrogate_but_2nd_missing`, `incomplete_surrogate_pair`, `incomplete_surrogate_and_escape_valid`, `invalid_lonely_surrogate`, `invalid_surrogate`, `lone_second_surrogate`, `object_key_lone_2nd_surrogate` | reject | reject | already strict | +| **UTF-16 input** | `utf16BE_no_BOM`, `utf16LE_no_BOM`, `UTF-16LE_with_BOM` | reject | reject | documented: UTF-8 only | +| **BOM** | `structure_UTF-8_BOM_empty_object` | reject | **accept** | §2.8 — leading BOM is now trimmed (and carried in `VL.LeadingSpace()`) | +| **number magnitude** | 10 `i_number_*` (huge exp, overflow, underflow) | accept | accept | correct by design — we don't evaluate numbers | +| **depth** | `structure_500_nested_arrays` | accept | accept | correct since the stack rewrite | + +**Assessment of "how many similar bugs remain hidden".** The conformance harness +itself has two blind spots, and both are worth closing as part of this work: + +1. `i_` cases are recorded but never asserted, so a *change* in implementation-defined + behavior is invisible. Fix: snapshot the `i_` verdict table as a golden file and + fail on drift. That would have caught this bug the day the alias fast path landed. +2. `VL/reader` is not a conformance mode, so `consumeStringRawStreamFast` / + `consumeStringRawStreaming` are conformance-untested. Two of the eight string + paths have no coverage here. + +Beyond UTF-8 and the BOM (§2.8, now handled), the surviving `i_` divergences are all +*deliberate*: number magnitude (we don't evaluate numbers, so nothing overflows), +UTF-16 input (documented UTF-8-only), and deep nesting (correct since the stack +rewrite). So after this work the honest statement is: **the lexers are conformant on +`y_`/`n_`, and every implementation-defined behavior is deliberate, documented and +pinned against drift.** No further hidden bugs of this class are visible in the +parsing suite — which is exactly why blind spot #1 above matters: it is the only +thing that would catch the *next* one. + +--- + +## 10. Work breakdown + +**Round 1 — correctness + free detection** + +- ✅ 1.0 New package `json/internal/utf8x` (§7.1): `Valid` / `FirstInvalid` / + `Sanitize` + its own unit + fuzz tests, independent of the lexer. +- ✅ 1.1 `ErrInvalidUTF8` error code (+ `ErrNotUTF8` for the UTF-16 BOM, §2.8). +- ✅ 1.2 `UTF8Policy` + `WithUTF8Policy`; delete `WithUTF8ValidationMode` / + `WithUTF8Validation`; plumb through `options` → `L.reset()` → `Input`. +- ✅ 1.3 Revert the prototype's `…WholeV` forks; fold `hi |= w` into + `consumeStringWhole` / `consumeStringRawWhole` / the two `…StreamFast` paths. +- ✅ 1.4 `ScanStop` + AVX2 kernel return `nonASCII` (avo regen, `go generate`). + Verify `TestInlinable` + `strscan` micro-benchmarks. +- ✅ 1.5 Validation at `finishStringValue`: `utf8x.Valid` gated on detection, plus + `utf8x.FirstInvalid` for the error offset. +- ✅ 1.6 Escape-level unification (§2.3): strict rejects broken pairs in `L`; stop + swallowing the second escape; passthrough/replace share the escape path (§2.7). +- ✅ 1.7 `utf8x.Sanitize` + `UTF8Replace` wiring (L values, VL raw bytes). +- ✅ 1.8 Leading BOM (§2.8): trim at offset 0; `VL` carries it in `blanks`; UTF-16 BOM + diagnostic. Guard `n_structure_UTF8_BOM_no_data` / `n_structure_incomplete_UTF8_BOM`. +- ✅ 1.9 Tests §8; conformance updates + `VL/reader` mode + `i_` golden snapshot. +- ✅ 1.10 Re-measure the §5.1 matrix. **Gate:** ≤3% on ASCII workloads, ≤8% on + `twitter_status`. +- ✅ 1.11 Docs: `DESIGN.md`, `README.md`, package godoc, and the verbatim-pledge + restatement (§2.5). + +**Round 2 — ✅ shipped; gate now met (twitter_status −6.3% L / −1.7% VL)** + +- ✅ 2.1 Port `check_special_cases` + `check_multibyte_lengths` + `is_incomplete` to + avo AVX2 (`validateUTF8AVX2`), 32 B/iter, carried `prev_block`. +- ✅ 2.2 Scalar rewind for the error offset (simdutf's `rewind_and_validate_with_errors`). +- ✅ 2.3 Differential fuzz kernel-vs-`utf8.Valid` over random byte strings — mandatory + before trusting hand-written vector validation. +- ✅ 2.4 Re-measure. + +**Round 3 — writers (own commit, after round 1 lands `utf8x`)** + +The writers have the mirror-image bug, already half-handled: +`writers/default-writer/escape.go:escapedBytes` silently substitutes U+FFFD for +invalid input bytes ("invalid runes are represented as �"), i.e. it hard-codes +`UTF8Replace` with **no way to get an error**. RFC 8259 says we must not emit invalid +UTF-8, and silently mangling a caller's payload without telling them is the same class +of complaint as the lexer's silent acceptance. + +- ⬜ 3.1 Re-express `escapedBytes`' default branch on `utf8x` so lexer and writer share + one rule (§2.4) and one implementation. +- ⬜ 3.2 Give the writers the same `UTF8Policy` knob. Open: what default? Symmetry with + the lexer says Strict; today's behavior is Replace. **Strict** is the recommendation + — a writer that silently corrupts is worse than one that refuses — but it is a + louder breaking change than the lexer's, so it wants its own decision. +- ⬜ 3.3 The `\u` escape emission path needs the same audit (surrogate pairs on output). + +**Follow-ups (out of scope, filed)** + +- ⬜ The YAML lexer (`json/lexers/yaml-lexer`, `YL`) has its own scanner and is not + covered by this plan — same audit needed. Note it is a **separate module**, so it + cannot import `json/internal/utf8x`; sharing would need the primitives published or + vendored. + +> **FRED** yes. We have another plan to run a full YAML conformance suite on that one, and possibly +> contribute fixes to goccy/go-yaml. Another story, another time. + +- ⬜ Hoist `swar` / `strscan` to `json/internal/` so writers can use the SWAR masks + (§7.1) — pure move, own commit, codegen-verified. + +--- + +## 11. Open questions + +*(Both original questions resolved 2026-07-30 — kept for the record.)* + +1. ~~**Should `UTF8Passthrough` also skip the escape-level checks?**~~ **Resolved:** + no — an escape must always produce a rune. Settled and specified in §2.7. + +> **FRED** agree (should be documented): explicitly escaped UTF-8 MUST produce some rune, so U+FFFD when broken and +> policy is lenient. + +2. ~~**Do we want `UTF8Replace` to be observable?**~~ **Resolved: no.** No counter, no + token flag; replacement is silent by design. Do not add hot-path state for it. + +> **FRED** no. + +**Still open (needs a decision before it is implemented, not before round 1 starts):** + +3. The writers' default policy (3.2 above): Strict — matching the lexer and refusing to + emit corrupted output — or Replace, preserving today's behavior? diff --git a/TODO.md b/TODO.md index e64f6f3..e3b8fb4 100644 --- a/TODO.md +++ b/TODO.md @@ -1,12 +1,20 @@ # UTF-8 validation +Working with the prototype to introduce UTF-8 validation in the lexer. + 1. We need the JSON lexer to be able to valide UTF-8 sequences: - invalid UTF-8 should not leak in output strings + invalid UTF-8 should not leak in output strings. This should be true for L as well as for VL. + 2. The default behavior of L and VL is to error on such invalid input + 3. As an option (to both lexers), the error can be swallowed and the invalid UTF-8 sequence be in this case mangled as U+FFFD (error rune) + 4. For VL, we pledge _verbatim_. This is important to maintain original offset/positioning. - But this pledge is broken when mangling an invalid sequence. TO BE DECIDED. + But this pledge is broken when mangling an invalid sequence. REMAINS TO BE DECIDED. + +5. Prototype introduces options that look a bit complicated to me (fused, ...): either we want error or we mute error. + When the error is muted, invalid utf8 sequences are mangled. Classic algorithm for invalid sequence detection: https://nemanjatrifunovic.substack.com/p/decoding-utf-8-part-vii-validation. diff --git a/go.work b/go.work index 30bffa0..4f2825e 100644 --- a/go.work +++ b/go.work @@ -4,6 +4,7 @@ use ( . ./json ./json/benchmarks + ./json/internal/utf8x/_asm ./json/lexers/default-lexer/internal/strscan/_asm ./json/lexers/yaml-lexer ) diff --git a/json/benchmarks/lexers/lanes/utf8scratch_test.go b/json/benchmarks/lexers/lanes/utf8scratch_test.go new file mode 100644 index 0000000..7afef93 --- /dev/null +++ b/json/benchmarks/lexers/lanes/utf8scratch_test.go @@ -0,0 +1,132 @@ +package lane + +// SCRATCH (utf-8 validation prototype): measures the cost of the three prototype +// validation modes across the corpus. Delete when the design is settled. + +import ( + "fmt" + "io" + "testing" + "unicode/utf8" + + lexer "github.com/go-openapi/core/json/lexers/default-lexer" + "github.com/go-openapi/core/json/lexers/token" + "github.com/go-openapi/core/json/testdata/workloads" +) + +func BenchmarkUTF8Validate(b *testing.B) { + suite, err := workloads.Corpus() + if err != nil { + b.Fatal(err) + } + + modes := []struct { + name string + policy lexer.UTF8Policy + }{ + {"passthrough", lexer.UTF8Passthrough}, // the pre-validation baseline + {"strict", lexer.UTF8Strict}, // the new default + {"replace", lexer.UTF8Replace}, + } + + for _, wl := range suite { + data := wl.Data + b.Run(wl.Name, func(b *testing.B) { + for _, m := range modes { + b.Run("L/"+m.name, func(b *testing.B) { + var sink int + lx := lexer.NewWithBytes(data, lexer.WithUTF8Policy(m.policy)) + b.SetBytes(int64(len(data))) + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + lx.ResetWithBytes(data) + for t := range lx.Tokens() { + sink += int(t.Kind()) + } + } + fmt.Fprint(io.Discard, sink) + }) + b.Run("VL/"+m.name, func(b *testing.B) { + var sink int + lx := lexer.NewVerbatimWithBytes(data, lexer.WithUTF8Policy(m.policy)) + b.SetBytes(int64(len(data))) + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + lx.ResetWithBytes(data) + for t := range lx.Tokens() { + sink += int(t.Kind()) + } + } + fmt.Fprint(io.Discard, sink) + }) + } + }) + } +} + +// TestCorpusStringProfile reports, per workload, how much of the payload is string +// bytes and what share of those strings carry a non-ASCII byte -- i.e. how many +// values a fused detector would hand to the validator. +func TestCorpusStringProfile(t *testing.T) { + suite, err := workloads.Corpus() + if err != nil { + t.Fatal(err) + } + + for _, wl := range suite { + var ( + strings, nonASCII, strBytes, nonASCIIBytes int + longStrings, longNonASCII int + ) + lx := lexer.NewWithBytes(wl.Data) + for tok := range lx.Tokens() { + if tok.Kind() != token.String && tok.Kind() != token.Key { + continue + } + v := tok.Value() + strings++ + strBytes += len(v) + if len(v) >= 16 { + longStrings++ + } + if !isASCII(v) { + nonASCII++ + nonASCIIBytes += len(v) + if len(v) >= 16 { + longNonASCII++ + } + } + if !utf8.Valid(v) { + t.Errorf("%s: corpus carries invalid utf8", wl.Name) + } + } + if !lx.Ok() { + t.Fatalf("%s: %v", wl.Name, lx.Err()) + } + t.Logf( + "%-20s bytes=%d strings=%d (%d long) strBytes=%d (%.1f%% of payload) nonASCII=%d (%.2f%% of strings, %d long) nonASCIIBytes=%d", + wl.Name, + len(wl.Data), + strings, + longStrings, + strBytes, + 100*float64(strBytes)/float64(len(wl.Data)), + nonASCII, + 100*float64(nonASCII)/float64(max(strings, 1)), + longNonASCII, + nonASCIIBytes, + ) + } +} + +func isASCII(b []byte) bool { + for _, c := range b { + if c >= 0x80 { + return false + } + } + + return true +} diff --git a/json/internal/utf8x/_asm/asm.go b/json/internal/utf8x/_asm/asm.go new file mode 100644 index 0000000..8ab6259 --- /dev/null +++ b/json/internal/utf8x/_asm/asm.go @@ -0,0 +1,195 @@ +//nolint:revive,mnd +package main + +import ( + "github.com/mmcloughlin/avo/attr" + . "github.com/mmcloughlin/avo/build" + . "github.com/mmcloughlin/avo/operand" +) + +func main() { + validateUTF8() + + Generate() +} + +// table16 emits a 16-byte read-only lookup table for VPSHUFB. +// +// VPSHUFB indexes within each 128-bit lane, so the table is broadcast into both lanes of a YMM at load time +// (VBROADCASTI128) and one 16-byte copy is all that needs to live in rodata. +func table16(name string, b ...byte) Mem { + if len(b) != 16 { + panic(name + ": a VPSHUFB table must be exactly 16 bytes") + } + m := GLOBL(name, attr.RODATA|attr.NOPTR) + for i, v := range b { + DATA(i, U8(v)) + } + + return m +} + +// UTF-8 error classes of the Keiser-Lemire "lookup4" algorithm (simdutf +// src/generic/utf8_validation/utf8_lookup4_algorithm.h). Each is one bit of a per-byte bitset; a byte is ill-formed +// iff its three table lookups agree on at least one violated rule. +const ( + tooShort = 1 << 0 // 11______ followed by 0_______ or 11______ + tooLong = 1 << 1 // 0_______ followed by 10______ + overlong3 = 1 << 2 // 11100000 100_____ + tooLarge = 1 << 3 // above U+10FFFF + surrogate = 1 << 4 // 11101101 101_____ : an encoded UTF-16 surrogate + overlong2 = 1 << 5 // 1100000_ 10______ + tooLarge1000 = 1 << 6 + overlong4 = 1 << 6 // 11110000 1000____ (shares the bit with tooLarge1000) + twoConts = 1 << 7 // 10______ 10______ + + // carry marks the classes that depend only on the high nibble of the leading byte, so the low-nibble table must + // let them through. + carry = tooShort | tooLong | twoConts +) + +// validateUTF8 emits the AVX2 UTF-8 validator: three VPSHUFB table lookups classify every (previous byte, byte) pair, +// and a saturating-subtract pass asserts that positions requiring a 2nd or 3rd continuation byte have one. +// +// It deliberately does NOT handle a partial trailing block or the end-of-input "sequence left incomplete" check: the +// Go caller re-validates the last few bytes scalar-ly from a sequence boundary (see ScanUTF8), which is both simpler +// and how simdutf itself locates errors. +func validateUTF8() { + TEXT("validateUTF8BlocksAVX2", NOSPLIT, "func(data []byte) bool") + Doc( + "validateUTF8BlocksAVX2 reports whether the whole 32-byte blocks of data are valid UTF-8, ignoring any sequence that continues past the end. len(data) must be a non-zero multiple of 32. AVX2, 32 bytes/iter.", + ) + ptr := Load(Param("data").Base(), GP64()) + n := Load(Param("data").Len(), GP64()) + + // byte_1_high: indexed by the HIGH nibble of the previous byte. + tableHigh := YMM() + VBROADCASTI128(table16( + "utf8t1h", + tooLong, + tooLong, + tooLong, + tooLong, + tooLong, + tooLong, + tooLong, + tooLong, // 0_______ : ASCII lead + twoConts, + twoConts, + twoConts, + twoConts, // 10______ : continuation + tooShort|overlong2, // 1100____ + tooShort, // 1101____ + tooShort|overlong3|surrogate, // 1110____ + tooShort|tooLarge|tooLarge1000|overlong4, // 1111____ + ), tableHigh) + + // byte_1_low: indexed by the LOW nibble of the previous byte. + tableLow := YMM() + VBROADCASTI128(table16("utf8t1l", + carry|overlong3|overlong2|overlong4, // ____0000 + carry|overlong2, // ____0001 + carry, carry, // ____001_ + carry|tooLarge, // ____0100 + carry|tooLarge|tooLarge1000, // ____0101 + carry|tooLarge|tooLarge1000, // ____011_ + carry|tooLarge|tooLarge1000, + carry|tooLarge|tooLarge1000, // ____1___ + carry|tooLarge|tooLarge1000, + carry|tooLarge|tooLarge1000, + carry|tooLarge|tooLarge1000, + carry|tooLarge|tooLarge1000, + carry|tooLarge|tooLarge1000|surrogate, // ____1101 + carry|tooLarge|tooLarge1000, + carry|tooLarge|tooLarge1000, + ), tableLow) + + // byte_2_high: indexed by the HIGH nibble of the current byte. + tableCur := YMM() + VBROADCASTI128(table16("utf8t2h", + tooShort, tooShort, tooShort, tooShort, tooShort, tooShort, tooShort, tooShort, // 0_______ + tooLong|overlong2|twoConts|overlong3|tooLarge1000|overlong4, // 1000____ + tooLong|overlong2|twoConts|overlong3|tooLarge, // 1001____ + tooLong|overlong2|twoConts|surrogate|tooLarge, // 101_____ + tooLong|overlong2|twoConts|surrogate|tooLarge, + tooShort, tooShort, tooShort, tooShort, // 11______ + ), tableCur) + + nibble := YMM() + VPBROADCASTB(ConstData("utf8n", U8(0x0f)), nibble) + high := YMM() + VPBROADCASTB(ConstData("utf8h", U8(0x80)), high) + // 0xe0-0x80 and 0xf0-0x80: a saturating subtract leaves the high bit set exactly where a 3rd (resp. 4th) byte of a + // sequence is required. + third := YMM() + VPBROADCASTB(ConstData("utf8c3", U8(0xe0-0x80)), third) + fourth := YMM() + VPBROADCASTB(ConstData("utf8c4", U8(0xf0-0x80)), fourth) + + // prev holds the previous block; it starts as zeros, which is the correct "nothing precedes the input" state + // (NUL is ASCII, so it imposes no continuation). + prev := YMM() + VPXOR(prev, prev, prev) + errAcc := YMM() + VPXOR(errAcc, errAcc, errAcc) + + i := GP64() + XORQ(i, i) + + Label("utf8loop") + input := YMM() + VMOVDQU(Mem{Base: ptr, Index: i, Scale: 1}, input) + + // The three "previous byte" views. All three shift the same concatenation of the previous block's high lane with + // this block's low lane, so the permute is computed once. + perm := YMM() + VPERM2I128(Imm(0x21), input, prev, perm) + prev1, prev2, prev3 := YMM(), YMM(), YMM() + VPALIGNR(Imm(15), perm, input, prev1) + VPALIGNR(Imm(14), perm, input, prev2) + VPALIGNR(Imm(13), perm, input, prev3) + + // check_special_cases: AND of the three lookups leaves a bit set only where every view agrees the pair is illegal. + idx := YMM() + VPSRLW(Imm(4), prev1, idx) + VPAND(nibble, idx, idx) + b1h := YMM() + VPSHUFB(idx, tableHigh, b1h) + + VPAND(nibble, prev1, idx) + b1l := YMM() + VPSHUFB(idx, tableLow, b1l) + + VPSRLW(Imm(4), input, idx) + VPAND(nibble, idx, idx) + b2h := YMM() + VPSHUFB(idx, tableCur, b2h) + + sc := YMM() + VPAND(b1h, b1l, sc) + VPAND(b2h, sc, sc) + + // check_multibyte_lengths: a position two bytes after a 3-byte lead, or three after a 4-byte lead, MUST be a + // continuation. XOR-ing that requirement against the special-case bits leaves a set bit exactly where the two + // disagree — i.e. a missing or unexpected continuation. + must := YMM() + VPSUBUSB(third, prev2, must) + tmp := YMM() + VPSUBUSB(fourth, prev3, tmp) + VPOR(tmp, must, must) + VPAND(high, must, must) + VPXOR(sc, must, must) + VPOR(must, errAcc, errAcc) + + VMOVDQA(input, prev) + ADDQ(Imm(32), i) + CMPQ(i, n) + JB(LabelRef("utf8loop")) + + ok := GP8() + VPTEST(errAcc, errAcc) + SETEQ(ok) // ZF is set when the accumulator is all zero: no error was ever flagged + Store(ok, ReturnIndex(0)) + VZEROUPPER() + RET() +} diff --git a/json/internal/utf8x/_asm/doc.go b/json/internal/utf8x/_asm/doc.go new file mode 100644 index 0000000..6fac4a5 --- /dev/null +++ b/json/internal/utf8x/_asm/doc.go @@ -0,0 +1,7 @@ +// Package _asm is a generator-only module for the AVX2 UTF-8 validation kernel. +// +// It isolates the avo build-time dependency: only `go generate` (see ../validate_amd64.go) runs it, and the generated +// ../validate_amd64.s has no avo import, so the parent json module never pulls avo. +// +// Lives under a `_`-prefixed directory so the go tool ignores it when listing/building the parent module. +package main diff --git a/json/internal/utf8x/_asm/go.mod b/json/internal/utf8x/_asm/go.mod new file mode 100644 index 0000000..345f343 --- /dev/null +++ b/json/internal/utf8x/_asm/go.mod @@ -0,0 +1,11 @@ +module github.com/go-openapi/core/json/internal/utf8x/_asm + +go 1.25.8 + +require github.com/mmcloughlin/avo v0.6.1-0.20260701065415-16419356370f + +require ( + golang.org/x/mod v0.38.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/tools v0.48.0 // indirect +) diff --git a/json/internal/utf8x/_asm/go.sum b/json/internal/utf8x/_asm/go.sum new file mode 100644 index 0000000..397ad9f --- /dev/null +++ b/json/internal/utf8x/_asm/go.sum @@ -0,0 +1,10 @@ +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/mmcloughlin/avo v0.6.1-0.20260701065415-16419356370f h1:eLpJhGSyRTsY+l0MFH5zGdcm15rwu7Cm48S257NMHRQ= +github.com/mmcloughlin/avo v0.6.1-0.20260701065415-16419356370f/go.mod h1:+Pk+j3KceMwAIGb32HZSCvkMCzJvxy6xCRk+urz7iPw= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= diff --git a/json/internal/utf8x/bench_test.go b/json/internal/utf8x/bench_test.go new file mode 100644 index 0000000..27a888d --- /dev/null +++ b/json/internal/utf8x/bench_test.go @@ -0,0 +1,55 @@ +package utf8x + +import ( + "bytes" + "fmt" + "io" + "testing" + "unicode/utf8" +) + +// BenchmarkValid compares this package's validator against the stdlib across the value sizes the lexer actually hands +// it, so the AVX2 length gate (avx2Min) can be set from measurement rather than guesswork. +// +// The lexer only calls in here for values that already contain a byte >= 0x80 (detection is fused into the string +// scan), so the interesting inputs are non-ASCII text, not ASCII. +func BenchmarkValid(b *testing.B) { + corpora := []struct { + name string + unit string + }{ + // a Latin word with accents: mostly ASCII with a sprinkle of 2-byte sequences, the common European shape + {name: "latin", unit: "les donn\u00e9es d'entr\u00e9e sont valid\u00e9es "}, + // CJK: dense 3-byte sequences, the twitter_status shape + {name: "cjk", unit: "\u65e5\u672c\u8a9e\u306e\u30c6\u30ad\u30b9\u30c8 "}, + // emoji: 4-byte sequences + {name: "emoji", unit: "\U0001F600\U0001F601\U0001F602 "}, + } + sizes := []int{16, 32, 48, 64, 96, 128, 256, 512, 1024, 4096} + + var sink bool + for _, c := range corpora { + for _, size := range sizes { + data := bytes.Repeat([]byte(c.unit), size/len(c.unit)+1)[:size] + // never cut a multi-byte sequence: the benchmark must measure valid input + for len(data) > 0 && !utf8.Valid(data) { + data = data[:len(data)-1] + } + + b.Run(fmt.Sprintf("%s/%d/utf8x", c.name, size), func(b *testing.B) { + b.SetBytes(int64(len(data))) + for b.Loop() { + sink = Valid(data) + } + }) + b.Run(fmt.Sprintf("%s/%d/stdlib", c.name, size), func(b *testing.B) { + b.SetBytes(int64(len(data))) + for b.Loop() { + sink = utf8.Valid(data) + } + }) + } + } + + fmt.Fprint(io.Discard, sink) +} diff --git a/json/internal/utf8x/cpuid_amd64.s b/json/internal/utf8x/cpuid_amd64.s new file mode 100644 index 0000000..1da73f9 --- /dev/null +++ b/json/internal/utf8x/cpuid_amd64.s @@ -0,0 +1,20 @@ +#include "textflag.h" + +// func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32) +TEXT ·cpuid(SB), NOSPLIT, $0-24 + MOVL eaxArg+0(FP), AX + MOVL ecxArg+4(FP), CX + CPUID + MOVL AX, eax+8(FP) + MOVL BX, ebx+12(FP) + MOVL CX, ecx+16(FP) + MOVL DX, edx+20(FP) + RET + +// func xgetbv() (eax, edx uint32) +TEXT ·xgetbv(SB), NOSPLIT, $0-8 + MOVL $0, CX + XGETBV + MOVL AX, eax+0(FP) + MOVL DX, edx+4(FP) + RET diff --git a/json/internal/utf8x/detect_amd64.go b/json/internal/utf8x/detect_amd64.go new file mode 100644 index 0000000..f5020c1 --- /dev/null +++ b/json/internal/utf8x/detect_amd64.go @@ -0,0 +1,43 @@ +//go:build amd64 + +package utf8x + +// cpuid executes CPUID with the given leaf (EAX) and subleaf (ECX) and returns the four result registers. +// +// Implemented in cpuid_amd64.s. +func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32) + +// xgetbv reads the extended control register XCR0 (ECX=0) and returns its low and high halves. +// +// Implemented in cpuid_amd64.s. Only call after CPUID reports OSXSAVE, or XGETBV itself faults. +func xgetbv() (eax, edx uint32) + +// useAVX2 records, once at package init, whether this CPU supports AVX2 with the OS having enabled YMM state. +// +// This is the precondition for calling validateUTF8BlocksAVX2. +// On any machine that fails the check, [Valid] stays on the stdlib scalar path and the AVX2 instructions are never +// executed. +var useAVX2 = detectAVX2() //nolint:gochecknoglobals // private immutable runtime environment detection + +//nolint:mnd // avx offsets +func detectAVX2() bool { + const ( + osxsave = uint32(1) << 27 // CPUID.1:ECX.OSXSAVE — OS enabled XSAVE/XGETBV + avx = uint32(1) << 28 // CPUID.1:ECX.AVX + avx2bit = uint32(1) << 5 // CPUID.(7,0):EBX.AVX2 + xcr0YMM = uint32(0x6) // XCR0 bits 1 (XMM) + 2 (YMM) must both be set + ) + + if maxLeaf, _, _, _ := cpuid(0, 0); maxLeaf < 7 { + return false + } + if _, _, ecx1, _ := cpuid(1, 0); ecx1&osxsave == 0 || ecx1&avx == 0 { + return false + } + if xcr0, _ := xgetbv(); xcr0&xcr0YMM != xcr0YMM { + return false + } + _, ebx7, _, _ := cpuid(7, 0) //nolint:dogsled // we need 4 registers + + return ebx7&avx2bit != 0 +} diff --git a/json/internal/utf8x/utf8x.go b/json/internal/utf8x/utf8x.go new file mode 100644 index 0000000..0a63fdd --- /dev/null +++ b/json/internal/utf8x/utf8x.go @@ -0,0 +1,117 @@ +// Package utf8x holds the UTF-8 validation and sanitization primitives shared by the JSON lexers and writers. +// +// It lives under json/internal (rather than under a lexer- or writer-private internal package) because it is the only +// location both subtrees can import: the lexers validate what they read, the writers validate what they emit, and both +// must agree byte-for-byte on what "invalid" means and on what an invalid sequence is replaced with. +// +// The functions here are deliberately whole-buffer: the lexers detect the *presence* of non-ASCII inline, fused into +// their existing stop-byte scan (see the swar package), and only call in here for the values that actually carry a byte +// >= 0x80. So [Valid] is not on the hot path for the overwhelmingly common all-ASCII value. +package utf8x + +import "unicode/utf8" + +const ( + // replacement is the UTF-8 encoding of U+FFFD, the substitute for an invalid byte. + replacement = string(utf8.RuneError) + + // replacementWidth is its encoded width (3). + replacementWidth = len(replacement) +) + +// What "valid" means here is exactly [utf8.Valid]'s definition, which is RFC 3629 plus Unicode: truncated sequences, +// overlong encodings, lead/continuation mismatches, encoded surrogates (U+D800..U+DFFF) and anything above U+10FFFF +// are all invalid. WTF-8 and CESU-8 are not accepted. [Valid] itself is platform-specific (see validate_amd64.go). + +// FirstInvalid returns the index of the first byte of the first ill-formed sequence in b, or -1 if b is valid UTF-8. +// +// It is the cold, error-path counterpart of [Valid]: callers use [Valid] for the verdict and only come here to report +// *where* the input went wrong. Keeping the position search separate is what lets [Valid] stay a bulk scan (the same +// split simdutf makes between its SIMD checker and rewind_and_validate_with_errors). +func FirstInvalid(b []byte) int { + for i := 0; i < len(b); { + if b[i] < utf8.RuneSelf { + i++ + + continue + } + + r, size := utf8.DecodeRune(b[i:]) + if r == utf8.RuneError && size <= 1 { + return i + } + i += size + } + + return -1 +} + +// Sanitize appends src to dst, substituting U+FFFD for every invalid byte, and returns the extended slice. +// +// Granularity is **one replacement per invalid byte** — [utf8.DecodeRune]'s error advance — so "\xe0\xa0\xff" yields +// three replacement runes. That is what ranging over a Go string produces, and what the JSON writer's escaper already +// did before this package existed, so the lexer, the writer and caller-side string conversion all agree. +// +// A valid src is appended in one bulk copy with no per-rune work. +func Sanitize(dst, src []byte) []byte { + i := FirstInvalid(src) + if i < 0 { + return append(dst, src...) + } + + dst = append(dst, src[:i]...) + + for i < len(src) { + if src[i] < utf8.RuneSelf { + dst = append(dst, src[i]) + i++ + + continue + } + + r, size := utf8.DecodeRune(src[i:]) + if r == utf8.RuneError && size <= 1 { + dst = append(dst, replacement...) + i++ + + continue + } + dst = append(dst, src[i:i+size]...) + i += size + } + + return dst +} + +// SanitizedLen returns the length Sanitize would produce for src, or -1 if src is valid UTF-8 (nothing to do). +// +// It lets a caller enforce a size cap before committing to the rewrite, since replacing a 1-byte invalid sequence with +// U+FFFD grows the value. +func SanitizedLen(src []byte) int { + i := FirstInvalid(src) + if i < 0 { + return -1 + } + + n := i + for i < len(src) { + if src[i] < utf8.RuneSelf { + n++ + i++ + + continue + } + + r, size := utf8.DecodeRune(src[i:]) + if r == utf8.RuneError && size <= 1 { + n += replacementWidth + i++ + + continue + } + n += size + i += size + } + + return n +} diff --git a/json/internal/utf8x/utf8x_test.go b/json/internal/utf8x/utf8x_test.go new file mode 100644 index 0000000..330244b --- /dev/null +++ b/json/internal/utf8x/utf8x_test.go @@ -0,0 +1,196 @@ +package utf8x + +import ( + "testing" + "unicode/utf8" + + "github.com/go-openapi/testify/v2/assert" + "github.com/go-openapi/testify/v2/require" +) + +// invalidSamples are the ill-formed byte sequences the JSON lexers used to accept silently — the ten string bodies of +// JSONTestSuite's i_string_* UTF-8 cases, plus a few shapes those do not cover. +// +// firstBad is the index of the first byte of the first ill-formed sequence. +var invalidSamples = []struct { //nolint:gochecknoglobals // shared by several tests + name string + in []byte + firstBad int + // want is the sanitized form: one U+FFFD per invalid byte (see [Sanitize]). + want string +}{ + {name: "lone ff", in: []byte{0xff}, firstBad: 0, want: "�"}, + {name: "lone continuation", in: []byte{0x81}, firstBad: 0, want: "�"}, + {name: "iso latin-1", in: []byte{0xe9}, firstBad: 0, want: "�"}, + {name: "truncated 3-byte", in: []byte{0xe0, 0xff}, firstBad: 0, want: "��"}, + { + name: "valid prefix then invalid lead", + in: []byte{0xe6, 0x97, 0xa5, 0xd1, 0x88, 0xfa}, + firstBad: 5, + want: "\u65e5\u0448\ufffd", + }, + {name: "encoded surrogate U+D800", in: []byte{0xed, 0xa0, 0x80}, firstBad: 0, want: "���"}, + {name: "beyond U+10FFFF", in: []byte{0xf4, 0xbf, 0xbf, 0xbf}, firstBad: 0, want: "����"}, + {name: "overlong 2 bytes", in: []byte{0xc0, 0xaf}, firstBad: 0, want: "��"}, + { + name: "obsolete 6-byte form", + in: []byte{0xfc, 0x83, 0xbf, 0xbf, 0xbf, 0xbf}, + firstBad: 0, + want: "������", + }, + { + name: "overlong NUL, 6 bytes", + in: []byte{0xfc, 0x80, 0x80, 0x80, 0x80, 0x80}, + firstBad: 0, + want: "������", + }, + // shapes the conformance corpus does not carry + {name: "truncated at end", in: []byte("ok\xe2\x82"), firstBad: 2, want: "ok��"}, + { + name: "invalid between valid", + in: []byte("a\xffbéc"), + firstBad: 1, + want: "a�béc", + }, + { + // the maximal-subpart rule would give 2 replacements here; we deliberately give 3 (one per byte). + name: "partial 3-byte then invalid", + in: []byte{0xe0, 0xa0, 0xff}, + firstBad: 0, + want: "���", + }, +} + +var validSamples = []struct { //nolint:gochecknoglobals // shared by several tests + name string + in []byte +}{ + {name: "empty", in: []byte{}}, + {name: "ascii", in: []byte("the quick brown fox")}, + {name: "2-byte", in: []byte("café naïve")}, + {name: "3-byte", in: []byte("\u65e5\u672c\u8a9e")}, + {name: "4-byte", in: []byte("\U0001D11E\U0001F600")}, + {name: "encoded U+FFFD", in: []byte("�")}, // a legitimate replacement char is valid input + {name: "max scalar", in: []byte("\U0010FFFF")}, + {name: "NUL and DEL", in: []byte{0x00, 0x7f}}, + {name: "BOM as content", in: []byte("\uFEFF")}, +} + +func TestValid(t *testing.T) { + t.Parallel() + + for _, tc := range validSamples { + t.Run("valid/"+tc.name, func(t *testing.T) { + t.Parallel() + assert.True(t, Valid(tc.in)) + assert.Equal(t, -1, FirstInvalid(tc.in)) + }) + } + + for _, tc := range invalidSamples { + t.Run("invalid/"+tc.name, func(t *testing.T) { + t.Parallel() + assert.False(t, Valid(tc.in)) + assert.Equal(t, tc.firstBad, FirstInvalid(tc.in)) + }) + } +} + +func TestSanitize(t *testing.T) { + t.Parallel() + + t.Run("valid input is appended verbatim", func(t *testing.T) { + t.Parallel() + for _, tc := range validSamples { + assert.Equal(t, string(tc.in), string(Sanitize(nil, tc.in))) + assert.Equal(t, -1, SanitizedLen(tc.in)) + } + }) + + t.Run("invalid input is replaced per byte", func(t *testing.T) { + t.Parallel() + for _, tc := range invalidSamples { + got := Sanitize(nil, tc.in) + assert.Equalf(t, tc.want, string(got), "sanitizing %q", tc.in) + assert.Truef(t, utf8.Valid(got), "sanitized output must be valid: %q", got) + assert.Equalf( + t, + len(got), + SanitizedLen(tc.in), + "SanitizedLen must predict the rewrite of %q", + tc.in, + ) + } + }) + + t.Run("appends to an existing buffer", func(t *testing.T) { + t.Parallel() + dst := []byte("prefix:") + assert.Equal(t, "prefix:a�b", string(Sanitize(dst, []byte("a\xffb")))) + }) + + t.Run("is idempotent", func(t *testing.T) { + t.Parallel() + for _, tc := range invalidSamples { + once := Sanitize(nil, tc.in) + assert.Equal(t, string(once), string(Sanitize(nil, once))) + } + }) +} + +// TestSanitizeMatchesWriterRule pins the granularity choice against the rule Go itself applies when a caller ranges +// over a string — the reason we replace per byte rather than per maximal subpart. +func TestSanitizeMatchesWriterRule(t *testing.T) { + t.Parallel() + + for _, tc := range invalidSamples { + var want []byte + for _, r := range string(tc.in) { // range over a string yields U+FFFD per invalid byte + want = utf8.AppendRune(want, r) + } + assert.Equalf(t, string(want), string(Sanitize(nil, tc.in)), + "Sanitize must agree with range-over-string for %q", tc.in) + } +} + +// FuzzValid pins the two invariants the whole design rests on: our verdict is the stdlib's verdict, and a sanitized +// value is always valid UTF-8. When the AVX2 kernel replaces the scalar Valid, this is the test that keeps it honest. +func FuzzValid(f *testing.F) { + for _, tc := range validSamples { + f.Add(tc.in) + } + for _, tc := range invalidSamples { + f.Add(tc.in) + } + f.Add([]byte("\xf0\x9f\x98\x80 mixed \xc3 tail")) + + f.Fuzz(func(t *testing.T, in []byte) { + require.Equal(t, utf8.Valid(in), Valid(in)) + + idx := FirstInvalid(in) + require.Equal(t, utf8.Valid(in), idx < 0) + if idx >= 0 { + require.True( + t, + utf8.Valid(in[:idx]), + "the prefix before the first fault must itself be valid", + ) + } + + out := Sanitize(nil, in) + require.True(t, utf8.Valid(out), "sanitized output must always be valid UTF-8") + require.Equal(t, string(out), string(Sanitize(nil, out)), "sanitizing must be idempotent") + + if idx < 0 { + require.Equal( + t, + string(in), + string(out), + "valid input must be passed through unchanged", + ) + require.Equal(t, -1, SanitizedLen(in)) + } else { + require.Equal(t, len(out), SanitizedLen(in)) + } + }) +} diff --git a/json/internal/utf8x/validate_amd64.go b/json/internal/utf8x/validate_amd64.go new file mode 100644 index 0000000..8f4e355 --- /dev/null +++ b/json/internal/utf8x/validate_amd64.go @@ -0,0 +1,85 @@ +//go:build amd64 + +package utf8x + +import "unicode/utf8" + +//go:generate sh -c "cd _asm && go run . -out ../validate_amd64.s" + +// validateUTF8BlocksAVX2 is the avo-generated AVX2 kernel (validate_amd64.s): a port of the Keiser-Lemire "lookup4" +// algorithm simdutf uses (src/generic/utf8_validation/utf8_lookup4_algorithm.h), 32 bytes per YMM iteration. +// +// Three VPSHUFB table lookups classify every (previous byte, byte) pair — the AND of the three leaves a bit set only +// where all three agree the pair violates the same rule (too short, too long, overlong, surrogate, too large, two +// continuations) — and a saturating-subtract pass asserts that a position which must carry the 2nd or 3rd +// continuation of a sequence actually does. +// +// It validates only whole 32-byte blocks and deliberately says nothing about a sequence that continues past the end: +// errors are detected block-wise, exactly as in simdutf, and the seam is resolved scalar-ly by [Valid]. len(data) must +// be a non-zero multiple of 32, and the CPU must support AVX2 ([useAVX2]). +func validateUTF8BlocksAVX2(data []byte) bool + +// avx2Min is the length below which the vector kernel cannot pay for its constant setup (7 broadcasts) plus the +// non-inlinable call. It is also the kernel's hard minimum: it processes whole 32-byte blocks and needs at least one. +// +// Set from BenchmarkValid: at 32 bytes and up the kernel already wins (latin/32: 6.7ns vs the stdlib's 18.1ns), and +// the win grows to 5-12x by 128 bytes. End-to-end on the corpus the 32-vs-64 choice is not distinguishable, so the +// micro-benchmark decides. +// +// The validator only ever runs on values already known to contain a byte >= 0x80 — detection is fused into the string +// scan — so the inputs reaching it are non-ASCII text, which is what BenchmarkValid measures. +const avx2Min = blockBytes + +// blockBytes is the kernel's stride: one YMM register. +const blockBytes = 32 + +// blockMask rounds a length down to a whole number of kernel blocks. +const blockMask = ^(blockBytes - 1) + +// maxContinuations is the longest run of continuation bytes a well-formed sequence can have (a 4-byte sequence has +// three). It bounds how far [Valid] rewinds to find the seam. +const maxContinuations = 3 + +// Valid reports whether b is entirely valid UTF-8. +// +// On amd64 with AVX2 it runs the vector kernel over the whole 32-byte blocks and finishes with the stdlib scalar +// validator over the trailing bytes, rewound to the start of the last sequence so a sequence straddling that boundary +// — or truncated at the very end, which the block kernel does not check — is fully validated exactly once. +func Valid(b []byte) bool { + if !useAVX2 || len(b) < avx2Min { + return utf8.Valid(b) + } + + blocks := len(b) & blockMask + if !validateUTF8BlocksAVX2(b[:blocks]) { + return false + } + + return utf8.Valid(b[seam(b, blocks):]) +} + +// seam returns the index at which scalar validation must resume so that everything the block kernel could not check is +// covered, and everything before it was checked. +// +// The kernel validates a pair only when both its bytes are inside the block region, so the unchecked remainder is: the +// trailing bytes past the last whole block, plus any sequence that begins before the boundary and reaches it. Such a +// sequence starts at most three bytes back, so walking back over continuation bytes (bounded) and then taking the +// non-continuation byte that precedes them lands exactly on its first byte. +// +// The bound is not a heuristic: more than three continuation bytes in a row is itself ill-formed, and returning an +// index that lands mid-sequence makes the scalar pass reject — which is the correct answer for that input. +func seam(b []byte, blocks int) int { + start := blocks + for range maxContinuations { + if start == 0 || b[start-1]&0xC0 != 0x80 { + break + } + start-- + } + + if start > 0 { + start-- // include the byte that leads the sequence (or is an ASCII byte of its own) + } + + return start +} diff --git a/json/internal/utf8x/validate_amd64.s b/json/internal/utf8x/validate_amd64.s new file mode 100644 index 0000000..031ff80 --- /dev/null +++ b/json/internal/utf8x/validate_amd64.s @@ -0,0 +1,117 @@ +// Code generated by command: go run asm.go -out ../validate_amd64.s. DO NOT EDIT. + +#include "textflag.h" + +// func validateUTF8BlocksAVX2(data []byte) bool +// Requires: AVX, AVX2 +TEXT ·validateUTF8BlocksAVX2(SB), NOSPLIT, $0-25 + MOVQ data_base+0(FP), AX + MOVQ data_len+8(FP), CX + VBROADCASTI128 utf8t1h<>+0(SB), Y0 + VBROADCASTI128 utf8t1l<>+0(SB), Y1 + VBROADCASTI128 utf8t2h<>+0(SB), Y2 + VPBROADCASTB utf8n<>+0(SB), Y3 + VPBROADCASTB utf8h<>+0(SB), Y4 + VPBROADCASTB utf8c3<>+0(SB), Y5 + VPBROADCASTB utf8c4<>+0(SB), Y6 + VPXOR Y7, Y7, Y7 + VPXOR Y8, Y8, Y8 + XORQ DX, DX + +utf8loop: + VMOVDQU (AX)(DX*1), Y9 + VPERM2I128 $0x21, Y9, Y7, Y7 + VPALIGNR $0x0f, Y7, Y9, Y10 + VPALIGNR $0x0e, Y7, Y9, Y11 + VPALIGNR $0x0d, Y7, Y9, Y7 + VPSRLW $0x04, Y10, Y12 + VPAND Y3, Y12, Y12 + VPSHUFB Y12, Y0, Y13 + VPAND Y3, Y10, Y12 + VPSHUFB Y12, Y1, Y10 + VPSRLW $0x04, Y9, Y12 + VPAND Y3, Y12, Y12 + VPSHUFB Y12, Y2, Y12 + VPAND Y13, Y10, Y10 + VPAND Y12, Y10, Y10 + VPSUBUSB Y5, Y11, Y11 + VPSUBUSB Y6, Y7, Y7 + VPOR Y7, Y11, Y11 + VPAND Y4, Y11, Y11 + VPXOR Y10, Y11, Y11 + VPOR Y11, Y8, Y8 + VMOVDQA Y9, Y7 + ADDQ $0x20, DX + CMPQ DX, CX + JB utf8loop + VPTEST Y8, Y8 + SETEQ AL + MOVB AL, ret+24(FP) + VZEROUPPER + RET + +DATA utf8t1h<>+0(SB)/1, $0x02 +DATA utf8t1h<>+1(SB)/1, $0x02 +DATA utf8t1h<>+2(SB)/1, $0x02 +DATA utf8t1h<>+3(SB)/1, $0x02 +DATA utf8t1h<>+4(SB)/1, $0x02 +DATA utf8t1h<>+5(SB)/1, $0x02 +DATA utf8t1h<>+6(SB)/1, $0x02 +DATA utf8t1h<>+7(SB)/1, $0x02 +DATA utf8t1h<>+8(SB)/1, $0x80 +DATA utf8t1h<>+9(SB)/1, $0x80 +DATA utf8t1h<>+10(SB)/1, $0x80 +DATA utf8t1h<>+11(SB)/1, $0x80 +DATA utf8t1h<>+12(SB)/1, $0x21 +DATA utf8t1h<>+13(SB)/1, $0x01 +DATA utf8t1h<>+14(SB)/1, $0x15 +DATA utf8t1h<>+15(SB)/1, $0x49 +GLOBL utf8t1h<>(SB), RODATA|NOPTR, $16 + +DATA utf8t1l<>+0(SB)/1, $0xe7 +DATA utf8t1l<>+1(SB)/1, $0xa3 +DATA utf8t1l<>+2(SB)/1, $0x83 +DATA utf8t1l<>+3(SB)/1, $0x83 +DATA utf8t1l<>+4(SB)/1, $0x8b +DATA utf8t1l<>+5(SB)/1, $0xcb +DATA utf8t1l<>+6(SB)/1, $0xcb +DATA utf8t1l<>+7(SB)/1, $0xcb +DATA utf8t1l<>+8(SB)/1, $0xcb +DATA utf8t1l<>+9(SB)/1, $0xcb +DATA utf8t1l<>+10(SB)/1, $0xcb +DATA utf8t1l<>+11(SB)/1, $0xcb +DATA utf8t1l<>+12(SB)/1, $0xcb +DATA utf8t1l<>+13(SB)/1, $0xdb +DATA utf8t1l<>+14(SB)/1, $0xcb +DATA utf8t1l<>+15(SB)/1, $0xcb +GLOBL utf8t1l<>(SB), RODATA|NOPTR, $16 + +DATA utf8t2h<>+0(SB)/1, $0x01 +DATA utf8t2h<>+1(SB)/1, $0x01 +DATA utf8t2h<>+2(SB)/1, $0x01 +DATA utf8t2h<>+3(SB)/1, $0x01 +DATA utf8t2h<>+4(SB)/1, $0x01 +DATA utf8t2h<>+5(SB)/1, $0x01 +DATA utf8t2h<>+6(SB)/1, $0x01 +DATA utf8t2h<>+7(SB)/1, $0x01 +DATA utf8t2h<>+8(SB)/1, $0xe6 +DATA utf8t2h<>+9(SB)/1, $0xae +DATA utf8t2h<>+10(SB)/1, $0xba +DATA utf8t2h<>+11(SB)/1, $0xba +DATA utf8t2h<>+12(SB)/1, $0x01 +DATA utf8t2h<>+13(SB)/1, $0x01 +DATA utf8t2h<>+14(SB)/1, $0x01 +DATA utf8t2h<>+15(SB)/1, $0x01 +GLOBL utf8t2h<>(SB), RODATA|NOPTR, $16 + +DATA utf8n<>+0(SB)/1, $0x0f +GLOBL utf8n<>(SB), RODATA|NOPTR, $1 + +DATA utf8h<>+0(SB)/1, $0x80 +GLOBL utf8h<>(SB), RODATA|NOPTR, $1 + +DATA utf8c3<>+0(SB)/1, $0x60 +GLOBL utf8c3<>(SB), RODATA|NOPTR, $1 + +DATA utf8c4<>+0(SB)/1, $0x70 +GLOBL utf8c4<>(SB), RODATA|NOPTR, $1 diff --git a/json/internal/utf8x/validate_amd64_test.go b/json/internal/utf8x/validate_amd64_test.go new file mode 100644 index 0000000..483973c --- /dev/null +++ b/json/internal/utf8x/validate_amd64_test.go @@ -0,0 +1,285 @@ +//go:build amd64 + +package utf8x + +import ( + "bytes" + "math/rand" + "testing" + "unicode/utf8" + + "github.com/go-openapi/testify/v2/require" +) + +// The AVX2 kernel is hand-written vector code whose failure mode is silence: a wrong table entry accepts an +// ill-formed sequence, and nothing else in the system would notice. So it is tested against the stdlib exhaustively +// where exhaustive is possible, and by construction everywhere else. +// +// Two properties are hammered specifically because they are where a lookup4 port goes wrong: +// +// - Block seams. State is carried between 32-byte blocks through prev1/prev2/prev3, so a sequence must be judged +// identically no matter which byte offset it starts at. Every case is therefore swept across all 32 alignments. +// - The tail. The kernel deliberately validates whole blocks only; [Valid] resolves the rest scalar-ly. An error +// that falls in the seam between the two is exactly what a naive split would miss. + +// requireAgrees asserts our verdict matches the stdlib's for b, and reports the input on failure. +func requireAgrees(t *testing.T, b []byte) { + t.Helper() + + if got, want := Valid(b), utf8.Valid(b); got != want { + t.Fatalf("Valid=%v, utf8.Valid=%v for % x (len=%d)", got, want, b, len(b)) + } +} + +// TestValidAgreesExhaustive2And3Bytes sweeps EVERY 2-byte sequence and every 3-byte sequence, embedded at every +// alignment within a buffer long enough to reach the vector path. +// +// This covers the whole classification table: all 256 lead bytes against all 256 second bytes is precisely what the +// three VPSHUFB lookups encode, so a single wrong table entry cannot survive it. +func TestValidAgreesExhaustive2And3Bytes(t *testing.T) { + const pad = 96 // > avx2Min, so the vector path runs + + t.Run("2 bytes", func(t *testing.T) { + buf := make([]byte, pad) + for hi := range 256 { + for lo := range 256 { + for _, at := range []int{0, 1, 30, 31, 32, 33, 62, 63, 64, pad - 2} { + for i := range buf { + buf[i] = 'a' + } + buf[at], buf[at+1] = byte(hi), byte(lo) + if got, want := Valid(buf), utf8.Valid(buf); got != want { + t.Fatalf( + "at=%d seq=%02x %02x: Valid=%v, utf8.Valid=%v", + at, + hi, + lo, + got, + want, + ) + } + } + } + } + }) + + // Every lead byte (C0..FF) against every possible 2nd and 3rd byte: 4.2M sequences. This is what exercises the + // continuation-count logic (prev2/prev3 saturating subtract) that the pairwise sweep above cannot reach. Leads + // below C0 are ASCII or stray continuations, both fully decided by the pairwise table. + t.Run("3 bytes after every lead", func(t *testing.T) { + if testing.Short() { + t.Skip("4.2M sequences x alignments") + } + buf := make([]byte, pad) + for i := range buf { + buf[i] = 'a' + } + for b0 := 0xC0; b0 <= 0xFF; b0++ { + for b1 := range 256 { + for b2 := range 256 { + for _, at := range []int{0, 31, 62} { + buf[at], buf[at+1], buf[at+2] = byte(b0), byte(b1), byte(b2) + if got, want := Valid(buf), utf8.Valid(buf); got != want { + t.Fatalf("at=%d seq=%02x %02x %02x: Valid=%v, utf8.Valid=%v", + at, b0, b1, b2, got, want) + } + buf[at], buf[at+1], buf[at+2] = 'a', 'a', 'a' + } + } + } + } + }) +} + +// TestKernelIsActuallyExercised guards the whole differential suite from becoming vacuous: if the CPU lacks AVX2, or +// the length gate keeps every fixture on the scalar path, every "agrees with the stdlib" test above would pass by +// comparing the stdlib against itself. +func TestKernelIsActuallyExercised(t *testing.T) { + if !useAVX2 { + t.Skip( + "no AVX2 on this CPU: Valid is the stdlib scalar validator and the kernel tests are vacuous", + ) + } + + t.Logf("AVX2 kernel active (avx2Min=%d)", avx2Min) + + // drive the kernel directly, so the result cannot come from the scalar fallback + valid := bytes.Repeat([]byte("a\u00e9\u65e5"), 16) // 96 bytes, a whole number of blocks + require.Zero(t, len(valid)%32) + require.True(t, validateUTF8BlocksAVX2(valid)) + + for _, bad := range [][]byte{ + {0xFF}, {0x80}, {0xC0, 0xAF}, {0xED, 0xA0, 0x80}, {0xF4, 0xBF, 0xBF, 0xBF}, {0xE0, 0x80, 0x80}, + } { + b := bytes.Repeat([]byte("a"), 96) + copy(b[40:], bad) + require.Falsef(t, validateUTF8BlocksAVX2(b), "the kernel must reject % x", bad) + } +} + +// TestValidAgreesFourByteLeads sweeps the 4-byte space where it actually discriminates: every lead in F0..F7 against +// every second byte, with the last two continuations both valid and invalid. This is where TOO_LARGE / TOO_LARGE_1000 +// / OVERLONG_4 live, and they are all decided by (lead, second byte). +func TestValidAgreesFourByteLeads(t *testing.T) { + const pad = 96 + tails := [][2]byte{{0x80, 0x80}, {0xBF, 0xBF}, {0x80, 0x41}, {0x41, 0x80}, {0xBF, 0xC0}} + + buf := make([]byte, pad) + for lead := 0xF0; lead <= 0xFF; lead++ { + for second := range 256 { + for _, tail := range tails { + for _, at := range []int{0, 29, 30, 31, 32, 60, 61, 62} { + for i := range buf { + buf[i] = 'a' + } + buf[at], buf[at+1] = byte(lead), byte(second) + buf[at+2], buf[at+3] = tail[0], tail[1] + if got, want := Valid(buf), utf8.Valid(buf); got != want { + t.Fatalf("at=%d seq=%02x %02x % x: Valid=%v, utf8.Valid=%v", + at, lead, second, tail, got, want) + } + } + } + } + } +} + +// TestValidAgreesAtEveryLength checks every length from 0 through several blocks, for a valid buffer and for one +// truncated mid-sequence at the very end — the case the block kernel cannot see and the scalar seam must catch. +func TestValidAgreesAtEveryLength(t *testing.T) { + base := bytes.Repeat( + []byte("a\u00e9\u65e5\U0001D11E"), + 64, + ) // 1-, 2-, 3- and 4-byte sequences, so every length cuts differently + + for n := range base { + requireAgrees(t, base[:n]) + } + + // a truncated sequence pinned at the end of buffers of every length + for _, trunc := range [][]byte{{0xC3}, {0xE6}, {0xE6, 0x97}, {0xF0}, {0xF0, 0x9D}, {0xF0, 0x9D, 0x84}} { + for n := range 200 { + b := append(bytes.Repeat([]byte("a"), n), trunc...) + requireAgrees(t, b) + require.Falsef( + t, + Valid(b), + "a sequence truncated at the end must be rejected (n=%d, % x)", + n, + trunc, + ) + } + } +} + +// TestValidAgreesOnSeamStraddle walks a valid multi-byte sequence one byte at a time across the last block boundary, +// where the vector pass hands over to the scalar one. +func TestValidAgreesOnSeamStraddle(t *testing.T) { + for _, seq := range []string{"\u00e9", "\u65e5", "\U0001D11E"} { + for total := 64; total < 160; total++ { + for at := total - len(seq) - 4; at <= total-len(seq); at++ { + if at < 0 { + continue + } + b := bytes.Repeat([]byte("a"), total) + copy(b[at:], seq) + requireAgrees(t, b) + require.Truef( + t, + Valid(b), + "a valid sequence at %d of %d must be accepted", + at, + total, + ) + } + } + } +} + +// TestValidAgreesRandom throws structured noise at it: mostly-valid text with sporadic corruption, which is what real +// ill-formed input looks like, plus pure random bytes. +func TestValidAgreesRandom(t *testing.T) { + //nolint:gosec // a deterministic PRNG is the point: reproducible test input, not cryptography + rng := rand.New(rand.NewSource(42)) + + alphabet := []string{"a", "\u00e9", "\u65e5", "\U0001D11E", "\x00", "\x7f"} + for range 20000 { + n := rng.Intn(300) + var b []byte + for len(b) < n { + b = append(b, alphabet[rng.Intn(len(alphabet))]...) + } + for range rng.Intn(4) { // corrupt a few bytes + if len(b) > 0 { + corrupt := rng.Intn( + 256, + ) //nolint:gosec // bounded to 0..255, so the conversion cannot overflow + b[rng.Intn(len(b))] = byte(corrupt) + } + } + requireAgrees(t, b) + } + + for range 20000 { + b := make([]byte, rng.Intn(300)) + _, _ = rng.Read(b) + requireAgrees(t, b) + } +} + +// TestSeamCoversEverythingTheKernelSkips pins the hand-off contract directly: the index [Valid] resumes scalar +// validation from must never be past the start of a sequence that the block kernel could not fully check. +func TestSeamCoversEverythingTheKernelSkips(t *testing.T) { + cases := []struct { + name string + b []byte + blocks int + want int + }{ + {name: "ascii before the boundary", b: bytes.Repeat([]byte("a"), 64), blocks: 64, want: 63}, + { + name: "3-byte sequence ending at the boundary", + b: append(bytes.Repeat([]byte("a"), 61), 0xE6, 0x97, 0xA5), + blocks: 64, want: 61, + }, + { + name: "4-byte sequence straddling the boundary", + b: append(bytes.Repeat([]byte("a"), 62), 0xF0, 0x9D, 0x84, 0x9E), + blocks: 64, want: 62, + }, + { + name: "more continuations than any sequence may have", + b: append(bytes.Repeat([]byte("a"), 60), 0x80, 0x80, 0x80, 0x80), + blocks: 64, want: 60, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := seam(tc.b, tc.blocks) + require.Equal(t, tc.want, got) + require.LessOrEqual(t, got, tc.blocks) + // resuming there must reproduce the stdlib verdict for the whole buffer + require.Equal(t, utf8.Valid(tc.b), utf8.Valid(tc.b[got:]) && utf8.Valid(tc.b[:got])) + }) + } +} + +// FuzzValidAVX2 is the standing guard on the kernel: whatever the fuzzer finds, our verdict is the stdlib's. +func FuzzValidAVX2(f *testing.F) { + f.Add(bytes.Repeat([]byte("a\u00e9\u65e5\U0001D11E"), 8)) + f.Add(append(bytes.Repeat([]byte("a"), 64), 0xE6, 0x97)) + f.Add(append(bytes.Repeat([]byte("\u00e9"), 40), 0xFF)) + f.Add(bytes.Repeat([]byte{0x80}, 70)) + f.Add(bytes.Repeat([]byte{0xF4, 0xBF, 0xBF, 0xBF}, 20)) + + f.Fuzz(func(t *testing.T, b []byte) { + require.Equalf(t, utf8.Valid(b), Valid(b), "% x", b) + + // and again at every offset within a block, since the kernel carries state across blocks + for shift := 1; shift <= 32; shift++ { + shifted := append(bytes.Repeat([]byte("a"), shift), b...) + require.Equalf(t, utf8.Valid(shifted), Valid(shifted), "shift=%d % x", shift, b) + } + }) +} diff --git a/json/internal/utf8x/validate_noasm.go b/json/internal/utf8x/validate_noasm.go new file mode 100644 index 0000000..725e0f0 --- /dev/null +++ b/json/internal/utf8x/validate_noasm.go @@ -0,0 +1,11 @@ +//go:build !amd64 + +package utf8x + +import "unicode/utf8" + +// Valid reports whether b is entirely valid UTF-8. +// +// Off amd64 there is no vector kernel, so this is the stdlib scalar validator (which carries its own 8-byte ASCII +// fast path). The answer is identical on every platform; only the speed differs. +func Valid(b []byte) bool { return utf8.Valid(b) } diff --git a/json/lexers/default-lexer/README.md b/json/lexers/default-lexer/README.md index e11ff89..7424f36 100644 --- a/json/lexers/default-lexer/README.md +++ b/json/lexers/default-lexer/README.md @@ -121,10 +121,66 @@ Streaming mode degrades speed by 15-20%. Pull iterator (`NextToken`) mode degrades speed by 10-15%. +## UTF-8 + +RFC 8259 §8.1 requires JSON text to be UTF-8, and both lexers enforce that on string values. This covers both ways a +value can end up holding a non-character: + +* an ill-formed UTF-8 byte sequence in the source (truncated, overlong, an encoded surrogate, beyond U+10FFFF); +* a `\uXXXX` escape that does not denote a Unicode scalar value — an unpaired, lone or inverted surrogate. + +`WithUTF8Policy` picks what happens: + +| policy | ill-formed bytes | broken `\u` escape | +|---|---|---| +| `UTF8Strict` (default) | `ErrInvalidUTF8` | `ErrSurrogateEscape` | +| `UTF8Replace` | U+FFFD per invalid byte | U+FFFD per broken escape | +| `UTF8Passthrough` | passed through untouched | U+FFFD per broken escape | + +An escape always produces *some* rune — the escape text is not the value — which is why `UTF8Passthrough` still +substitutes there. + +`VL` keeps escapes as source text, so under `UTF8Replace` it rewrites ill-formed *bytes* but never rewrites escape +text; the substitution for a broken escape appears when the value is decoded with `token.Unescape`. + +### What mangling costs `VL` + +U+FFFD encodes as **three** bytes and replaces **one**, so a mangled value is 2 bytes longer per ill-formed byte than +the text it came from — and a multi-byte fault is several ill-formed bytes, not one (a truncated 3-byte sequence +becomes three replacements: 9 bytes where the source had 3). For such a value: + +* `len(token.Value())` is **not** the width of its source span; +* an index into the value **cannot** be added to the token's start to get a source offset; +* the original bytes are **not** recoverable from the token. + +Token *positions* are unaffected: `Line()`, `Column()`, `Offset()` and `LeadingSpace()` derive from the scan cursor +walking the source, never from the value, so they stay exact under every policy. A formatter or linter can still point +at the right place — it just cannot assume the value it holds is the text that was there. Callers who need the source +bytes should use `UTF8Strict` (and handle the error) or `UTF8Passthrough` (and validate downstream themselves). + +A leading UTF-8 BOM is consumed before any token exists, so it is not re-emitted either — the other documented +exception to byte-exact round-tripping (RFC 8259 §8.1 asks implementations not to emit one). + +**Cost.** Detection is fused into the string scan already being performed — every SWAR word and every AVX2 block is +OR-accumulated, so "did this value contain a byte >= 0x80" is answered for free — and only values that actually carry +one reach the validator. On the reference corpus, `UTF8Strict` versus no validation is statistically indistinguishable +on the four ASCII-dominated workloads and costs ~10% on `twitter_status`, which is 30% non-ASCII by string bytes. + +The input must be UTF-8: a UTF-16 document is rejected with `ErrNotUTF8`. A leading UTF-8 BOM is currently rejected as +an invalid token rather than skipped. + ## Conformance tests Our implementation of the JSON lexers pass the full JSON conformance suite. No compromise on strictness. +Beyond the suite's `y_`/`n_` contract, the implementation-defined (`i_`) cases are snapshotted in +`testdata/conformance_i_behavior.golden`, so a change in *implementation-defined* behavior shows up as a reviewable +diff instead of silence. Regenerate it with: + +``` +go test ./lexers/default-lexer -run TestConformanceParsing -update-golden +``` + ## Benchmarks See [a comparison](../benchmark/benchviz/README.md) diff --git a/json/lexers/default-lexer/conformance_test.go b/json/lexers/default-lexer/conformance_test.go index d87b552..fd2d97f 100644 --- a/json/lexers/default-lexer/conformance_test.go +++ b/json/lexers/default-lexer/conformance_test.go @@ -2,6 +2,7 @@ package lexer import ( "bytes" + "flag" "os" "path/filepath" "sort" @@ -65,9 +66,11 @@ func TestConformanceParsing(t *testing.T) { // Aggregate verdicts across modes; a file "conforms" only if every mode agrees with the expectation. conforms := true detail := make([]string, 0, len(modes)) + byMode := make(map[string]bool, len(modes)) for _, m := range modes { v := m.run(data, maximum) detail = append(detail, m.name+"="+verdictStr(v)) + byMode[m.name] = v.accepted switch want { case 'y': @@ -81,6 +84,27 @@ func TestConformanceParsing(t *testing.T) { } } + // UTF8Replace may only turn a rejection into an acceptance, and only for the ill-formed-UTF-8 / + // broken-escape cases it is designed to sanitize. Anything else means replacement relaxed the grammar. + if byMode["L/replace"] != byMode["L/bytes"] { + switch { + case !byMode["L/replace"]: + t.Errorf("UTF8Replace REJECTED a document the default policy accepts: %s\n %s", + name, strings.Join(detail, " ")) + case !conformanceUTF8Relaxed()[name]: + t.Errorf("UTF8Replace accepted %s, which the default policy rejects, "+ + "but it is not a known ill-formed-UTF-8 case: add it to conformanceUTF8Relaxed "+ + "or fix the relaxation\n %s", name, strings.Join(detail, " ")) + } + } else if conformanceUTF8Relaxed()[name] { + t.Errorf( + "conformanceUTF8Relaxed lists %q but UTF8Replace no longer changes its verdict: "+ + "remove it\n %s", + name, + strings.Join(detail, " "), + ) + } + switch want { case 'i': iCount++ @@ -125,7 +149,7 @@ func TestConformanceParsing(t *testing.T) { } } - // Emit the implementation-defined behavior table for the record. + // Emit the implementation-defined behavior table for the record, and pin it against drift. sort.Strings(iReport) t.Logf("=== implementation-defined (i_) behavior: %d cases ===", iCount) for _, line := range iReport { @@ -133,6 +157,89 @@ func TestConformanceParsing(t *testing.T) { } t.Logf("=== summary: y_ pass=%d, n_ pass=%d, xfail=%d, unexpected-pass=%d ===", passYes, passNo, xfailCount, unexpectedOK) + + checkIBehaviorGolden(t, iReport) +} + +//nolint:gochecknoglobals // the standard golden-file switch must be a package-level flag +var updateGolden = flag.Bool( + "update-golden", + false, + "rewrite the conformance i_ behavior snapshot from the current run", +) + +// checkIBehaviorGolden pins how the lexers treat every implementation-DEFINED (i_) case. +// +// These cases are outside the suite's y_/n_ contract, so nothing asserted them before — which is exactly how the +// UTF-8 bug survived: ten i_string_* documents carrying ill-formed UTF-8 were silently ACCEPTED, and the suite stayed +// green from the day the zero-copy string fast path landed. Snapshotting the table turns any future change in +// implementation-defined behavior into a reviewable diff instead of silence. +// +// Run `go test ./lexers/default-lexer -run TestConformanceParsing -update-golden` after an intentional change. +func checkIBehaviorGolden(t *testing.T, iReport []string) { + t.Helper() + + golden := filepath.Join(currentDir(), "testdata", "conformance_i_behavior.golden") + got := strings.Join(iReport, "\n") + "\n" + + if *updateGolden { + if err := os.WriteFile(golden, []byte(got), 0o600); err != nil { + t.Fatalf("cannot write %s: %v", golden, err) + } + t.Logf("updated %s", golden) + + return + } + + want, err := os.ReadFile(golden) + if err != nil { + t.Fatalf("cannot read %s (run with -update-golden to create it): %v", golden, err) + } + + if string(want) == got { + return + } + + wantLines := strings.Split(strings.TrimRight(string(want), "\n"), "\n") + gotLines := strings.Split(strings.TrimRight(got, "\n"), "\n") + t.Errorf("implementation-defined behavior changed (%d recorded cases, %d now).\n"+ + "If the change is intended, re-run with -update-golden and review the diff.", len(wantLines), len(gotLines)) + for _, line := range diffLines(wantLines, gotLines) { + t.Errorf(" %s", line) + } +} + +// diffLines reports the entries that differ between two sorted "name: verdicts" snapshots. +func diffLines(want, got []string) []string { + index := func(lines []string) map[string]string { + m := make(map[string]string, len(lines)) + for _, line := range lines { + name, verdicts, _ := strings.Cut(line, ": ") + m[name] = verdicts + } + + return m + } + + wantByName, gotByName := index(want), index(got) + + var out []string + for name, w := range wantByName { + switch g, ok := gotByName[name]; { + case !ok: + out = append(out, "gone: "+name+" (was "+w+")") + case g != w: + out = append(out, "changed: "+name+"\n was: "+w+"\n now: "+g) + } + } + for name, g := range gotByName { + if _, ok := wantByName[name]; !ok { + out = append(out, "new: "+name+" ("+g+")") + } + } + sort.Strings(out) + + return out } // conformanceXFail lists test_parsing files whose current behavior diverges from the JSONTestSuite expectation (y_/n_). @@ -217,6 +324,53 @@ func conformanceModes() []conformanceMode { return drainVL(NewVerbatimWithBytes(data), maximum) }, }, + { + // VL's streaming string scanners (consumeStringRawStreamFast / consumeStringRawStreaming) had no + // conformance coverage at all until this mode was added. + name: "VL/reader", + run: func(data []byte, maximum int) conformanceVerdict { + return drainVL(NewVerbatim(bytes.NewReader(data), WithBufferSize(64)), maximum) + }, + }, + { + // UTF8Replace must not change what is ACCEPTED except for ill-formed UTF-8, which it sanitizes instead of + // rejecting. Running it over the whole suite pins that it relaxes nothing else. + name: "L/replace", + run: func(data []byte, maximum int) conformanceVerdict { + return drainL(NewWithBytes(data, WithUTF8Policy(UTF8Replace)), maximum) + }, + }, + } +} + +// conformanceUTF8Relaxed lists the test_parsing files that UTF8Replace accepts although the default policy rejects +// them: exactly the ill-formed-UTF-8 and broken-surrogate-escape cases, which replacement sanitizes to U+FFFD. +// +// Any OTHER file whose verdict differs between the two policies is a bug — replacement must not relax the grammar. +func conformanceUTF8Relaxed() map[string]bool { + return map[string]bool{ + // ill-formed UTF-8 bytes + "i_string_UTF-8_invalid_sequence.json": true, + "i_string_UTF8_surrogate_U+D800.json": true, + "i_string_invalid_utf-8.json": true, + "i_string_iso_latin_1.json": true, + "i_string_lone_utf8_continuation_byte.json": true, + "i_string_not_in_unicode_range.json": true, + "i_string_overlong_sequence_2_bytes.json": true, + "i_string_overlong_sequence_6_bytes.json": true, + "i_string_overlong_sequence_6_bytes_null.json": true, + "i_string_truncated-utf-8.json": true, + // \u escapes that do not denote a scalar value + "i_object_key_lone_2nd_surrogate.json": true, + "i_string_1st_surrogate_but_2nd_missing.json": true, + "i_string_1st_valid_surrogate_2nd_invalid.json": true, + "i_string_incomplete_surrogate_and_escape_valid.json": true, + "i_string_incomplete_surrogate_pair.json": true, + "i_string_incomplete_surrogates_escape_valid.json": true, + "i_string_invalid_lonely_surrogate.json": true, + "i_string_invalid_surrogate.json": true, + "i_string_inverted_surrogates_U+1D11E.json": true, + "i_string_lone_second_surrogate.json": true, } } diff --git a/json/lexers/default-lexer/doc.go b/json/lexers/default-lexer/doc.go index 0fa9e26..5750546 100644 --- a/json/lexers/default-lexer/doc.go +++ b/json/lexers/default-lexer/doc.go @@ -42,4 +42,22 @@ // lexer.WithMaxContainerStack(512), // nesting depth ceiling // lexer.WithMaxValueBytes(1<<20), // per-value / whitespace ceiling // ) +// +// # UTF-8 +// +// RFC 8259 §8.1 requires JSON text to be UTF-8, and both lexers enforce it on string values: an ill-formed byte +// sequence, or a \u escape that does not denote a Unicode scalar value (an unpaired or inverted surrogate), is +// rejected by default with [codes.ErrInvalidUTF8] / [codes.ErrSurrogateEscape]. +// +// Callers who would rather sanitize than fail can select [UTF8Replace], which substitutes U+FFFD (one per invalid +// byte, one per broken escape); [UTF8Passthrough] restores the unvalidated behavior for input already known to be +// well-formed. See [WithUTF8Policy]. +// +// Validation is not a second pass over the value: detection of non-ASCII bytes is fused into the string scan the +// lexer already performs, so a pure-ASCII value — the overwhelmingly common case — is proven valid with no extra read +// and no call, and only values that actually carry a byte >= 0x80 reach the validator. Measured on the reference +// corpus, enabling it is free on ASCII-dominated documents and costs ~10% on heavily non-ASCII ones (twitter_status). +// +// The input must be UTF-8: a UTF-16 document is rejected with [codes.ErrNotUTF8]. A leading UTF-8 BOM is currently +// rejected as an invalid token rather than skipped. package lexer diff --git a/json/lexers/default-lexer/fuzz_test.go b/json/lexers/default-lexer/fuzz_test.go index 080b857..31bc898 100644 --- a/json/lexers/default-lexer/fuzz_test.go +++ b/json/lexers/default-lexer/fuzz_test.go @@ -23,7 +23,9 @@ package lexer // the same grammar; only their token *shape* differs). // 5. Verbatim round-trip: for any ACCEPTED input, reassembling the raw VL token // stream (leading blanks + verbatim token text) reproduces the input byte for -// byte — the defining property of the verbatim lexer. +// byte — the defining property of the verbatim lexer. The single exception is a +// leading UTF-8 BOM, consumed before any token exists and therefore not +// re-emitted (input.CheckBOM; RFC 8259 §8.1 asks implementations not to emit one). // 6. Well-formed on accept: an ACCEPTED token stream obeys the JSON grammar // (balanced and correctly typed containers, keys only in object key slots, // key/value alternation, exactly one root value). The lexers are "almost a @@ -41,6 +43,7 @@ import ( "iter" "strings" "testing" + "unicode/utf8" "github.com/go-openapi/core/json/lexers/token" ) @@ -403,6 +406,17 @@ func FuzzLexer(f *testing.F) { `123456789012345678901234567890`, `1e999999`, `-`, + // ill-formed UTF-8 in string bodies (rejected by the default policy; see utf8_test.go) + "[\"\xff\"]", + "[\"\xc0\xaf\"]", + "[\"\xed\xa0\x80\"]", + "[\"\xe0\xff\"]", + "[\"ok\xe2\x82\"]", + "{\"k\xff\":1}", + "[\"caf\xc3\xa9\"]", + "[\"\xe6\x97\xa5\xe6\x9c\xac\"]", + `["\uD800\uD800"]`, + `["𝄞"]`, } for _, s := range seeds { f.Add([]byte(s)) @@ -453,10 +467,16 @@ func FuzzLexer(f *testing.F) { } // --- verbatim round-trip on accepted input --- + // + // A leading UTF-8 BOM is the ONE documented exception: it is consumed before any token exists (see + // input.CheckBOM), so it belongs to no token's leading space and is not re-emitted. RFC 8259 §8.1 asks + // implementations not to emit a BOM, so dropping it is the conformant direction — but it does mean the + // round-trip is byte-exact only for the document body. if vPullErr == nil { + want := bytes.TrimPrefix(data, []byte("\uFEFF")) rt := fzVerbatimRoundtrip(data, n) - if !bytes.Equal(rt, data) { - t.Fatalf("VL round-trip mismatch:\n in =%q\n out=%q", data, rt) + if !bytes.Equal(rt, want) { + t.Fatalf("VL round-trip mismatch:\n in =%q\n out=%q\n want=%q", data, rt, want) } } @@ -470,5 +490,112 @@ func FuzzLexer(f *testing.F) { ) } } + + // --- UTF-8: no ill-formed sequence may reach the caller --- + fzUTF8Invariants(t, data, n) }) } + +// fzUTF8Invariants pins the promise the UTF-8 work exists to make: under the default policy an accepted document +// yields only well-formed values, and under UTF8Replace EVERY document that is otherwise grammatical is accepted and +// still yields only well-formed values. +// +// It is the broadest guard we have, because it holds for arbitrary bytes rather than for a fixture table: the fused +// non-ASCII detection is duplicated across eight scanners, and a miss in any one of them shows up here. +func fzUTF8Invariants(t *testing.T, data []byte, n int) { + t.Helper() + + check := func(lane string, policy UTF8Policy, values [][]byte, err error) { + if err != nil { + return + } + for _, v := range values { + if !utf8.Valid(v) { + t.Fatalf( + "%s emitted an ill-formed value under policy %d: %q\ninput=%q", + lane, + policy, + v, + data, + ) + } + } + } + + for _, policy := range []UTF8Policy{UTF8Strict, UTF8Replace} { + lVals, lErr := fzStringValues( + NewWithBytes(append([]byte(nil), data...), WithUTF8Policy(policy)), + n, + ) + check("L/bytes", policy, lVals, lErr) + + sVals, sErr := fzStringValues( + New(bytes.NewReader(data), WithBufferSize(32), WithUTF8Policy(policy)), n) + check("L/reader", policy, sVals, sErr) + + // VL keeps escapes as source text, so its values are checked through the decoder — the form a caller + // actually consumes. + vVals, vErr := fzVerbatimDecodedValues( + NewVerbatimWithBytes(append([]byte(nil), data...), WithUTF8Policy(policy)), n) + check("VL/bytes", policy, vVals, vErr) + + vsVals, vsErr := fzVerbatimDecodedValues( + NewVerbatim(bytes.NewReader(data), WithBufferSize(32), WithUTF8Policy(policy)), n) + check("VL/reader", policy, vsVals, vsErr) + + // Replacement must never be the reason a document is rejected: it only ever turns a UTF-8 error into a + // substitution, so anything the default policy accepts it must accept too. + if policy == UTF8Replace && lErr != nil { + if _, strictErr := fzStringValues( + NewWithBytes(append([]byte(nil), data...)), n); strictErr == nil { + t.Fatalf( + "UTF8Replace rejected a document the default policy accepts: %v\ninput=%q", + lErr, + data, + ) + } + } + } +} + +// fzStringValues drains a semantic lexer and returns the value of every String/Key token. +func fzStringValues(lex *L, n int) ([][]byte, error) { + var out [][]byte + limit := fzCap(n) + for i := 0; ; i++ { + if i > limit { + panic("lexer did not terminate (utf8 invariants)") + } + tok := lex.NextToken() + if !lex.Ok() { + return out, lex.Err() + } + if k := tok.Kind(); k == token.String || k == token.Key { + out = append(out, bytes.Clone(tok.Value())) + } + if tok.IsEOF() { + return out, nil + } + } +} + +// fzVerbatimDecodedValues drains a verbatim lexer and returns every String/Key value in DECODED form. +func fzVerbatimDecodedValues(lex *VL, n int) ([][]byte, error) { + var out [][]byte + limit := fzCap(n) + for i := 0; ; i++ { + if i > limit { + panic("lexer did not terminate (utf8 invariants, verbatim)") + } + tok := lex.NextToken() + if !lex.Ok() { + return out, lex.Err() + } + if k := tok.Kind(); k == token.String || k == token.Key { + out = append(out, token.Unescape(bytes.Clone(tok.Value()))) + } + if tok.IsEOF() { + return out, nil + } + } +} diff --git a/json/lexers/default-lexer/internal/input/input.go b/json/lexers/default-lexer/internal/input/input.go index 426d125..e8a2a45 100644 --- a/json/lexers/default-lexer/internal/input/input.go +++ b/json/lexers/default-lexer/internal/input/input.go @@ -42,16 +42,39 @@ type Input struct { MaxValueBytes int KeepPreviousBuffer int - // ValidateMode (PROTOTYPE) selects the UTF-8 validation strategy. - ValidateMode int + // UTF8Policy governs what happens to a string value that is not valid UTF-8. Mirrored from the like-named option in + // L.reset(); the lexer package aliases this type so it is the single definition. + UTF8Policy UTF8Policy - // SawNonASCII (PROTOTYPE) reports whether the scan observed a byte >= 0x80 in the value just produced. - SawNonASCII bool + // sanitized holds the U+FFFD-substituted rewrite of an ill-formed value under UTF8Replace. + // + // It is a buffer of its own rather than CurrentValue because on the unescaping paths the value to rewrite IS + // CurrentValue. It is allocated lazily on the first ill-formed value, so a well-formed document never pays for it. + sanitized []byte } -// UTF-8 validation strategies (PROTOTYPE). +// UTF8Policy selects what a lexer does with input that cannot be represented as a Unicode scalar value: an ill-formed +// UTF-8 byte sequence in a string body, or a \u escape that does not form one. +// +// The zero value is [UTF8Strict], so validation is on unless a caller opts out. +type UTF8Policy uint8 + const ( - ValidateOff = 0 // current behavior: no validation - ValidateNaive = 1 // utf8.Valid over every string value at the funnel - ValidateFused = 2 // non-ASCII accumulated during the existing scan; utf8.Valid only when a high bit was seen + // UTF8Strict rejects the document: the lexer errors with codes.ErrInvalidUTF8 (ill-formed bytes) or + // codes.ErrSurrogateEscape (a broken \u surrogate pair). This is the default. + UTF8Strict UTF8Policy = iota + + // UTF8Replace accepts the document and substitutes U+FFFD — one per invalid byte, one per broken escape — so an + // emitted value is always valid UTF-8. Only an offending value is rewritten (and therefore copied); a valid value is + // still aliased zero-copy. + UTF8Replace + + // UTF8Passthrough skips raw-byte validation entirely: ill-formed bytes reach the caller untouched. A \u escape still + // decodes to U+FFFD when broken, because an escape must always produce some rune. + // + // UNSAFE: only for input already known to be valid UTF-8. + UTF8Passthrough ) + +// Validates reports whether the policy inspects raw string bytes at all. +func (p UTF8Policy) Validates() bool { return p != UTF8Passthrough } diff --git a/json/lexers/default-lexer/internal/input/refill.go b/json/lexers/default-lexer/internal/input/refill.go index b2ee525..69e9ed0 100644 --- a/json/lexers/default-lexer/internal/input/refill.go +++ b/json/lexers/default-lexer/internal/input/refill.go @@ -37,6 +37,7 @@ func (in *Input) FirstFill() { } else { in.Err = err } + in.CheckBOM() return } @@ -45,6 +46,7 @@ func (in *Input) FirstFill() { } } in.Bufferized = n + in.CheckBOM() } // ReadMore provides more input from the internal buffer or consumes from the input stream. @@ -99,6 +101,41 @@ func (in *Input) ConsumeNull(_ byte) token.T { return token.NullToken } +// ensureWindow makes at least n unconsumed bytes contiguously available in the window, compacting the consumed prefix +// away and reading more from the source. It reports whether that many could be made available. +// +// It exists for the surrogate-escape lookahead, which must inspect the following `\uXXXX` BEFORE deciding whether it +// belongs to the current escape (see [Input.peekLowSurrogate]) — consumeN cannot serve that, since it consumes what it +// reads and a straddling read cannot be given back. +// +// Compaction is safe on that path: the only scanners that reach it hold no live buffer index (the streaming string +// paths accumulate into CurrentValue), and in whole-buffer mode it never compacts at all — it just answers whether the +// bytes are there. Being a cold path (surrogate escapes only), it deliberately does not maintain PreviousBuffer. +func (in *Input) ensureWindow(n int) bool { + if in.Bufferized-in.Consumed >= n { + return true + } + + if in.WholeBuffer { + return false // the whole input is already here: what is missing does not exist + } + + if in.Consumed > 0 { + in.Bufferized = copy(in.Buffer, in.Buffer[in.Consumed:in.Bufferized]) + in.Consumed = 0 + } + + for in.Bufferized < n { + m, err := in.R.Read(in.Buffer[in.Bufferized:]) + in.Bufferized += m + if err != nil || m == 0 { + break + } + } + + return in.Bufferized >= n +} + // consumeN consumes a small buffer of n bytes to decide tokens such as "true", "false" or "null". func (in *Input) consumeN(buffer []byte) error { minReadSize := len(buffer) diff --git a/json/lexers/default-lexer/internal/input/string.go b/json/lexers/default-lexer/internal/input/string.go index 1145143..27fef47 100644 --- a/json/lexers/default-lexer/internal/input/string.go +++ b/json/lexers/default-lexer/internal/input/string.go @@ -7,10 +7,10 @@ import ( "unicode/utf16" "unicode/utf8" + "github.com/go-openapi/core/json/internal/utf8x" "github.com/go-openapi/core/json/lexers/default-lexer/internal/strscan" "github.com/go-openapi/core/json/lexers/default-lexer/internal/swar" codes "github.com/go-openapi/core/json/lexers/error-codes" - scan "github.com/go-openapi/core/json/lexers/internal/scan" "github.com/go-openapi/core/json/lexers/token" ) @@ -52,10 +52,6 @@ const controlCharsUpperBound = 0x20 // ASCII control chars range func (in *Input) ConsumeString() token.T { if in.TrackBlanks { if in.WholeBuffer { - if in.ValidateMode == ValidateFused { - return in.consumeStringRawWholeV() - } - return in.consumeStringRawWhole() } @@ -63,93 +59,12 @@ func (in *Input) ConsumeString() token.T { } if in.WholeBuffer { - if in.ValidateMode == ValidateFused { - return in.consumeStringWholeV() - } - return in.consumeStringWhole() } return in.consumeStringStreamFast() } -// consumeStringWholeV is consumeStringWhole with a fused non-ASCII accumulator: every SWAR word already loaded by the -// stop-scan is OR-ed into hi, so a pure-ASCII string (the overwhelming majority) is proven valid UTF-8 with no second -// pass and no call. Only a string that actually contains a byte >= 0x80 pays utf8.Valid. -// -// hi may over-report (the matching word's bytes past the stop are included, and the AVX2 region is not accumulated at -// all) -- that is conservative: it can only trigger a redundant utf8.Valid, never skip a needed one. -func (in *Input) consumeStringWholeV() token.T { - data := in.Buffer - n := in.Bufferized - start := in.Consumed - - i := start - var hi uint64 - guard := start + guessLong - if in.NoAVX2 { - guard = n + 1 - } - for i+8 <= n { - w := binary.LittleEndian.Uint64(data[i:]) - hi |= w - if m := swar.StringStopMask(w); m != 0 { - i += swar.FirstByte(m) - - break - } - i += 8 - if i >= guard { - break - } - } - if i >= guard && i+8 <= n { - if c := data[i]; c != doubleQuote && c != escape && c >= controlCharsUpperBound { - i += strscan.ScanStop(data[i:n]) - hi = ^uint64(0) // AVX2 region unaccumulated: force the check - } - } - for ; i < n; i++ { - c := data[i] - if c == doubleQuote || c == escape || c < controlCharsUpperBound { - break - } - hi |= uint64(c) - } - if i >= n { - in.Consumed, in.Offset = i, uint64(i) - in.Err = codes.ErrUnterminatedString - - return token.None - } - - switch c := data[i]; { - case c == doubleQuote: - if in.MaxValueBytes > 0 && i-start > in.MaxValueBytes { - in.Consumed, in.Offset = i, uint64(i) - in.Err = codes.ErrMaxValueBytes - - return token.None - } - value := data[start:i:i] - i++ - in.Consumed, in.Offset = i, uint64(i) - in.SawNonASCII = hi&swar.HighBits != 0 - - return in.finishStringValue(value) - - case c < controlCharsUpperBound: - in.Consumed, in.Offset = i, uint64(i) - in.Err = codes.ErrControlChar - - return token.None - } - - in.SawNonASCII = true - - return in.consumeStringEscaped(start, i) -} - // consumeStringStreamFast is the streaming string fast path (§10.3 Phase 1). // // It treats the CURRENT buffer window l.in.Buffer[:l.in.Bufferized] like whole-buffer mode: it scans for the closing @@ -166,7 +81,6 @@ func (in *Input) consumeStringWholeV() token.T { // //nolint:dupl // the structure is the same but differs subtly and its critical that the inside remains inlined. func (in *Input) consumeStringStreamFast() token.T { - in.SawNonASCII = true data := in.Buffer n := in.Bufferized start := in.Consumed // first content byte (opening quote already consumed) @@ -175,16 +89,21 @@ func (in *Input) consumeStringStreamFast() token.T { // run stays clean past guessLong. // Identical probe to consumeStringWhole, but bounded by the window end n = in.Bufferized. i := start + var hi uint64 // OR of the content bytes scanned; see consumeStringWhole guard := start + guessLong if in.NoAVX2 { guard = n + 1 } for i+8 <= n { - if m := swar.StringStopMask(binary.LittleEndian.Uint64(data[i:])); m != 0 { - i += swar.FirstByte(m) + w := binary.LittleEndian.Uint64(data[i:]) + if m := swar.StringStopMask(w); m != 0 { + k := swar.FirstByte(m) + hi |= swar.LanesBelow(w, k) + i += k break } + hi |= w i += 8 if i >= guard { break @@ -192,13 +111,19 @@ func (in *Input) consumeStringStreamFast() token.T { } if i >= guard && i+8 <= n { if c := data[i]; c != doubleQuote && c != escape && c >= controlCharsUpperBound { - i += strscan.ScanStop(data[i:n]) + delta, nonASCII := strscan.ScanStop(data[i:n]) + i += delta + if nonASCII { + hi |= swar.HighBits + } } } for ; i < n; i++ { - if c := data[i]; c == doubleQuote || c == escape || c < controlCharsUpperBound { + c := data[i] + if c == doubleQuote || c == escape || c < controlCharsUpperBound { break } + hi |= uint64(c) } if i >= n { @@ -221,7 +146,7 @@ func (in *Input) consumeStringStreamFast() token.T { in.Offset += uint64(end - start) in.Consumed = end - return in.finishStringValue(value) + return in.finishStringValue(value, hi&swar.HighBits != 0) case c < controlCharsUpperBound: in.Offset += uint64(i - start) @@ -243,7 +168,6 @@ func (in *Input) consumeStringStreamFast() token.T { // //nolint:dupl // the structure is the same but differs subtly and its critical that the inside remains inlined. func (in *Input) consumeStringWhole() token.T { - in.SawNonASCII = true data := in.Buffer n := in.Bufferized start := in.Consumed // first content byte @@ -255,6 +179,11 @@ func (in *Input) consumeStringWhole() token.T { // // The overwhelmingly common case (no escapes, no control chars) aliases the input with zero copy. i := start + // hi accumulates every content byte the scan passes over, so (hi & swar.HighBits) != 0 answers "did this value + // contain a byte >= 0x80" for free — the word is already in a register. A value that is pure ASCII is thereby + // proven valid UTF-8 with no second pass and no call; only the rest reaches the validator (see finishStringValue). + // Lanes at or after the stop are trimmed off so the answer is exact, matching strscan.ScanStop's. + var hi uint64 // guard is where the inline probe stops and delegates to the AVX2 scan. // With WithoutAVX2 it is pushed past the buffer so the loop never breaks to delegate — the string is scanned // entirely by the inline SWAR word loop (the pre-AVX2 baseline), no vector call at alin. @@ -263,11 +192,15 @@ func (in *Input) consumeStringWhole() token.T { guard = n + 1 } for i+8 <= n { - if m := swar.StringStopMask(binary.LittleEndian.Uint64(data[i:])); m != 0 { - i += swar.FirstByte(m) // exact stop lane; skips the scalar re-scan + w := binary.LittleEndian.Uint64(data[i:]) + if m := swar.StringStopMask(w); m != 0 { + k := swar.FirstByte(m) // exact stop lane; skips the scalar re-scan + hi |= swar.LanesBelow(w, k) + i += k break } + hi |= w i += 8 if i >= guard { break // guessLong clean bytes in — leave the loop to delegate below @@ -281,13 +214,19 @@ func (in *Input) consumeStringWhole() token.T { // above. i lands on the stop byte or on n. if i >= guard && i+8 <= n { if c := data[i]; c != doubleQuote && c != escape && c >= controlCharsUpperBound { - i += strscan.ScanStop(data[i:n]) + delta, nonASCII := strscan.ScanStop(data[i:n]) + i += delta + if nonASCII { + hi |= swar.HighBits + } } } for ; i < n; i++ { - if c := data[i]; c == doubleQuote || c == escape || c < controlCharsUpperBound { + c := data[i] + if c == doubleQuote || c == escape || c < controlCharsUpperBound { break } + hi |= uint64(c) } if i >= n { in.Consumed, in.Offset = i, uint64(i) @@ -308,7 +247,7 @@ func (in *Input) consumeStringWhole() token.T { i++ // past the closing quote in.Consumed, in.Offset = i, uint64(i) - return in.finishStringValue(value) + return in.finishStringValue(value, hi&swar.HighBits != 0) case c < controlCharsUpperBound: in.Consumed, in.Offset = i, uint64(i) @@ -321,16 +260,18 @@ func (in *Input) consumeStringWhole() token.T { // It is a separate function on purpose — keeping the byte-by-byte escape machinery out of this frame insulates the // fast path's codegen from it (and vice versa); they were previously one function, where a fast-path change could // regress the slow path by ~12% and vice versa (plan §4.2). - return in.consumeStringEscaped(start, i) + return in.consumeStringEscaped(start, i, hi&swar.HighBits != 0) } // consumeStringEscaped is the unescape slow path, split out of consumeStringWhole. // -// It is entered with data[i] == escape and start..i the clean prefix already scanned. -// It copies that prefix then unescapes the rest; the loop invariant is that data[i] is the next "stop" byte (quote, -// escape, or control) — clean runs between stops are copied in bulk rather than byte-by-byte. -func (in *Input) consumeStringEscaped(start, i int) token.T { - in.SawNonASCII = true +// It is entered with data[i] == escape and start..i the clean prefix already scanned; nonASCII reports whether that +// prefix carried a byte >= 0x80, and is extended here over every further raw run copied into the value. +// +// Runes produced by a \u escape are deliberately NOT accumulated: they are valid by construction (the escape decoder +// rejects or replaces anything that is not a scalar value), so a value whose only non-ASCII comes from escapes still +// skips the validator. +func (in *Input) consumeStringEscaped(start, i int, nonASCII bool) token.T { data := in.Buffer n := in.Bufferized @@ -342,7 +283,7 @@ func (in *Input) consumeStringEscaped(start, i int) token.T { i++ in.Consumed, in.Offset = i, uint64(i) - return in.finishStringValue(in.CurrentValue) + return in.finishStringValue(in.CurrentValue, nonASCII) case c == escape: i++ @@ -412,32 +353,45 @@ func (in *Input) consumeStringEscaped(start, i int) token.T { // fast. // The bound is checked against len + the run width *before* the append so an over-long value is rejected without // copying a huge clean run, and escape-only expansion (zero-width run) is still caught. + var hi uint64 stop := i probe := min(i+swarProbe, n) for ; stop < probe; stop++ { - if c := data[stop]; c == doubleQuote || c == escape || c < controlCharsUpperBound { + c := data[stop] + if c == doubleQuote || c == escape || c < controlCharsUpperBound { break } + hi |= uint64(c) } if stop == probe && stop < n { // run outran the scalar probe → SWAR, guess long past guessLong for stop+8 <= n { - if m := swar.StringStopMask(binary.LittleEndian.Uint64(data[stop:])); m != 0 { - stop += swar.FirstByte(m) + w := binary.LittleEndian.Uint64(data[stop:]) + if m := swar.StringStopMask(w); m != 0 { + k := swar.FirstByte(m) + hi |= swar.LanesBelow(w, k) + stop += k break } + hi |= w stop += 8 if stop-i >= guessLong && !in.NoAVX2 { - stop += strscan.ScanStop(data[stop:n]) + delta, runNonASCII := strscan.ScanStop(data[stop:n]) + stop += delta + if runNonASCII { + hi |= swar.HighBits + } break } } for ; stop < n; stop++ { - if c := data[stop]; c == doubleQuote || c == escape || c < controlCharsUpperBound { + c := data[stop] + if c == doubleQuote || c == escape || c < controlCharsUpperBound { break } + hi |= uint64(c) } } if in.MaxValueBytes > 0 && len(in.CurrentValue)+(stop-i) > in.MaxValueBytes { @@ -446,6 +400,7 @@ func (in *Input) consumeStringEscaped(start, i int) token.T { return token.None } + nonASCII = nonASCII || hi&swar.HighBits != 0 in.CurrentValue = append(in.CurrentValue, data[i:stop]...) i = stop } @@ -458,18 +413,14 @@ func (in *Input) consumeStringEscaped(start, i int) token.T { // finishStringValue turns a scanned string body into a Key (in object key position) or String token, handling the // trailing colon for keys. -func (in *Input) finishStringValue(value []byte) token.T { - switch in.ValidateMode { - case ValidateNaive: - if !utf8.Valid(value) { - in.Err = codes.ErrInvalidRune - - return token.None - } - case ValidateFused: - if in.SawNonASCII && !utf8.Valid(value) { - in.Err = codes.ErrInvalidRune - +// +// It is the single funnel every string path reaches, so it is where the UTF-8 policy is applied. nonASCII is the +// verdict the scan already produced: false means the value is pure ASCII and therefore valid UTF-8 by construction, so +// the common case costs one predictable branch and never touches the bytes again. +func (in *Input) finishStringValue(value []byte, nonASCII bool) token.T { + if nonASCII && in.UTF8Policy.Validates() && !utf8x.Valid(value) { + var ok bool + if value, ok = in.handleInvalidUTF8(value); !ok { return token.None } } @@ -485,9 +436,19 @@ func (in *Input) finishStringValue(value []byte) token.T { return token.MakeWithValue(token.String, value) } +// consumeStringStreaming is the byte-by-byte scan over a refilling buffer: it decodes escapes and copies content into +// in.CurrentValue so the value survives buffer turnover. +// +// The clean-run bulk scan in its default branch is deliberately inline rather than shared with the three sibling +// scanners: extracting it costs a call per run and, more importantly, the escape-analysis and inlining properties of +// this frame are load-bearing (see the fast/slow split note on consumeStringEscaped). +// +//nolint:maintidx // one long switch by design; splitting it perturbs the scan's codegen (see core_pull_buffer.go) func (in *Input) consumeStringStreaming() token.T { - in.SawNonASCII = true - var escapeSequence bool + var ( + escapeSequence bool + nonASCII bool // did any raw source byte carry the high bit (see consumeStringEscaped) + ) in.CurrentValue = in.CurrentValue[:0] for { @@ -534,7 +495,7 @@ func (in *Input) consumeStringStreaming() token.T { continue } - return in.finishStringValue(in.CurrentValue) + return in.finishStringValue(in.CurrentValue, nonASCII) case slash: if escapeSequence { @@ -597,6 +558,7 @@ func (in *Input) consumeStringStreaming() token.T { return token.None } + nonASCII = nonASCII || b >= utf8.RuneSelf in.CurrentValue = append(in.CurrentValue, b) // bulk-scan the rest of this clean run within the current window (§10.3 Phase 1c): a long clean stretch (e.g. @@ -610,36 +572,48 @@ func (in *Input) consumeStringStreaming() token.T { n := in.Bufferized runStart := in.Consumed stop := runStart + var hi uint64 probe := min(stop+swarProbe, n) for ; stop < probe; stop++ { - if c := data[stop]; c == doubleQuote || c == escape || + c := data[stop] + if c == doubleQuote || c == escape || c < controlCharsUpperBound { break } + hi |= uint64(c) } if stop == probe && stop < n { // run outran the scalar probe → SWAR for stop+8 <= n { - if m := swar.StringStopMask( - binary.LittleEndian.Uint64(data[stop:]), - ); m != 0 { - stop += swar.FirstByte(m) + w := binary.LittleEndian.Uint64(data[stop:]) + if m := swar.StringStopMask(w); m != 0 { + k := swar.FirstByte(m) + hi |= swar.LanesBelow(w, k) + stop += k break } + hi |= w stop += 8 if stop-runStart >= guessLong && !in.NoAVX2 { - stop += strscan.ScanStop(data[stop:n]) + delta, runNonASCII := strscan.ScanStop(data[stop:n]) + stop += delta + if runNonASCII { + hi |= swar.HighBits + } break } } for ; stop < n; stop++ { - if c := data[stop]; c == doubleQuote || c == escape || + c := data[stop] + if c == doubleQuote || c == escape || c < controlCharsUpperBound { break } + hi |= uint64(c) } } + nonASCII = nonASCII || hi&swar.HighBits != 0 if stop > runStart { if in.MaxValueBytes > 0 && len(in.CurrentValue)+(stop-runStart) > in.MaxValueBytes { @@ -656,51 +630,31 @@ func (in *Input) consumeStringStreaming() token.T { } } +// unescapeUnicodeSequence decodes a \uXXXX escape (the leading "\u" is already consumed), combining a surrogate pair +// when one follows. +// +// A code unit that does not denote a Unicode scalar value is governed by the UTF-8 policy (see +// [Input.brokenSurrogate]): rejected under UTF8Strict, decoded to U+FFFD otherwise. Crucially the trailing escape is +// only consumed when it actually completes the pair, so `\uD800\uD800` yields TWO replacement runes rather than +// swallowing the second escape — the behavior [token.Unescape] already has for verbatim values. func (in *Input) unescapeUnicodeSequence() (rune, error) { - var buf [4]byte - if err := in.consumeN(buf[:]); err != nil { - return utf8.RuneError, codes.ErrUnicodeEscape + _, r, err := in.readHex4() + if err != nil { + return utf8.RuneError, err } - high1, highOK1 := scan.Unhex(buf[0]) - low1, lowOK1 := scan.Unhex(buf[1]) - high2, highOK2 := scan.Unhex(buf[2]) - low2, lowOK2 := scan.Unhex(buf[3]) - if !lowOK1 || !highOK1 || !lowOK2 || !highOK2 { - return utf8.RuneError, codes.ErrUnicodeEscape + if !utf16.IsSurrogate(r) { + // four hex digits that are not a surrogate always denote a scalar value: nothing further to check + return r, nil } - unicodeEscape := uint32(high1)<<12 + uint32(low1)<<8 + uint32(high2)<<4 + uint32(low2) - r := rune(unicodeEscape) - if utf16.IsSurrogate(r) { - // this is a surrogate pair to encode a UTF-16 codepoint in 2 pairs expect this to follow: \uXXXX. - var nextBuf [6]byte - if err := in.consumeN(nextBuf[:]); err != nil { - return utf8.RuneError, codes.ErrSurrogateEscape - } - - if nextBuf[0] != escape || nextBuf[1] != 'u' { - return utf8.RuneError, codes.ErrSurrogateEscape - } + if low, ok := in.peekLowSurrogate(); ok { + if decoded := utf16.DecodeRune(r, low); decoded != utf8.RuneError { + in.takePeeked(surrogateEscapeLen) - high1, highOK1 = scan.Unhex(nextBuf[2]) - low1, lowOK1 = scan.Unhex(nextBuf[3]) - high2, highOK2 = scan.Unhex(nextBuf[4]) - low2, lowOK2 = scan.Unhex(nextBuf[5]) - if !lowOK1 || !highOK1 || !lowOK2 || !highOK2 { - return utf8.RuneError, codes.ErrUnicodeEscape + return decoded, nil } - - unicodeEscape2 := uint32(high1)<<12 + uint32(low1)<<8 + uint32(high2)<<4 + uint32(low2) - r1 := r - r2 := rune(unicodeEscape2) - - r = utf16.DecodeRune(r1, r2) - } - - if !utf8.ValidRune(r) { - return utf8.RuneError, codes.ErrInvalidRune } - return r, nil + return utf8.RuneError, in.brokenSurrogate() } diff --git a/json/lexers/default-lexer/internal/input/string_raw.go b/json/lexers/default-lexer/internal/input/string_raw.go index 5f3b30f..b0e988b 100644 --- a/json/lexers/default-lexer/internal/input/string_raw.go +++ b/json/lexers/default-lexer/internal/input/string_raw.go @@ -10,7 +10,6 @@ import ( "github.com/go-openapi/core/json/lexers/default-lexer/internal/strscan" "github.com/go-openapi/core/json/lexers/default-lexer/internal/swar" codes "github.com/go-openapi/core/json/lexers/error-codes" - scan "github.com/go-openapi/core/json/lexers/internal/scan" "github.com/go-openapi/core/json/lexers/token" ) @@ -29,102 +28,40 @@ import ( // //nolint:dupl // the structure is the same but differs subtly and its critical that the inside remains inlined. func (in *Input) consumeStringRawWhole() token.T { - in.SawNonASCII = true data := in.Buffer n := in.Bufferized start := in.Consumed i := start - guard := start + guessLong - if in.NoAVX2 { - guard = n + 1 // WithoutAVX2: never delegate, pure inline SWAR (see consumeStringWhole) - } - for i+8 <= n { - if m := swar.StringStopMask(binary.LittleEndian.Uint64(data[i:])); m != 0 { - i += swar.FirstByte(m) - - break - } - i += 8 - if i >= guard { - break // guessLong clean bytes in — delegate below (call kept out of the loop) - } - } - // clean past guessLong → guess long, AVX2 scan of the rest (same heuristic and out-of-loop placement as - // consumeStringWhole). - if i >= guard && i+8 <= n { - if c := data[i]; c != doubleQuote && c != escape && c >= controlCharsUpperBound { - i += strscan.ScanStop(data[i:n]) - } - } - for ; i < n; i++ { - if c := data[i]; c == doubleQuote || c == escape || c < controlCharsUpperBound { - break - } - } - if i >= n { - in.Consumed, in.Offset = i, uint64(i) - in.Err = codes.ErrUnterminatedString - - return token.None - } - - switch c := data[i]; { - case c == doubleQuote: - // no escapes: raw == decoded, same aliasing exit as consumeStringWhole - if in.MaxValueBytes > 0 && i-start > in.MaxValueBytes { - in.Consumed, in.Offset = i, uint64(i) - in.Err = codes.ErrMaxValueBytes - - return token.None - } - value := data[start:i:i] - i++ - in.Consumed, in.Offset = i, uint64(i) - - return in.finishStringValue(value) - - case c < controlCharsUpperBound: - in.Consumed, in.Offset = i, uint64(i) - in.Err = codes.ErrControlChar - - return token.None - } - - // an escape was found at i: validate the rest but keep the raw bytes. - return in.consumeStringRawEscaped(start, i) -} - -func (in *Input) consumeStringRawWholeV() token.T { - data := in.Buffer - n := in.Bufferized - start := in.Consumed - - i := start - var hi uint64 + var hi uint64 // fused non-ASCII accumulator, see consumeStringWhole guard := start + guessLong if in.NoAVX2 { guard = n + 1 // WithoutAVX2: never delegate, pure inline SWAR (see consumeStringWhole) } for i+8 <= n { w := binary.LittleEndian.Uint64(data[i:]) - hi |= w if m := swar.StringStopMask(w); m != 0 { - i += swar.FirstByte(m) + k := swar.FirstByte(m) + hi |= swar.LanesBelow(w, k) + i += k break } + hi |= w i += 8 if i >= guard { - break + break // guessLong clean bytes in — delegate below (call kept out of the loop) } } // clean past guessLong → guess long, AVX2 scan of the rest (same heuristic and out-of-loop placement as // consumeStringWhole). if i >= guard && i+8 <= n { if c := data[i]; c != doubleQuote && c != escape && c >= controlCharsUpperBound { - i += strscan.ScanStop(data[i:n]) - hi = ^uint64(0) // AVX2 region unaccumulated: force the check + delta, nonASCII := strscan.ScanStop(data[i:n]) + i += delta + if nonASCII { + hi |= swar.HighBits + } } } for ; i < n; i++ { @@ -153,9 +90,8 @@ func (in *Input) consumeStringRawWholeV() token.T { value := data[start:i:i] i++ in.Consumed, in.Offset = i, uint64(i) - in.SawNonASCII = hi&swar.HighBits != 0 - return in.finishStringValue(value) + return in.finishStringValue(value, hi&swar.HighBits != 0) case c < controlCharsUpperBound: in.Consumed, in.Offset = i, uint64(i) @@ -165,9 +101,7 @@ func (in *Input) consumeStringRawWholeV() token.T { } // an escape was found at i: validate the rest but keep the raw bytes. - in.SawNonASCII = true - - return in.consumeStringRawEscaped(start, i) + return in.consumeStringRawEscaped(start, i, hi&swar.HighBits != 0) } // consumeStringRawEscaped validates a string that contains at least one escape (data[i] == escape) without decoding it, @@ -175,8 +109,10 @@ func (in *Input) consumeStringRawWholeV() token.T { // // Clean runs between escapes are skipped with the same adaptive scalar-probe-then-SWAR scan the decoder uses // (consumeStringEscaped), but with no copying — so a sparse-escape string with a long clean tail stays fast. -func (in *Input) consumeStringRawEscaped(start, i int) token.T { - in.SawNonASCII = true +// nonASCII carries the verdict for the clean prefix already scanned and is extended over every further raw run; see +// consumeStringEscaped. Unlike the decoding path, the value here keeps its escapes as source text, so a \u escape +// contributes only its (ASCII) escape bytes. +func (in *Input) consumeStringRawEscaped(start, i int, nonASCII bool) token.T { data := in.Buffer n := in.Bufferized @@ -193,7 +129,7 @@ func (in *Input) consumeStringRawEscaped(start, i int) token.T { i++ in.Consumed, in.Offset = i, uint64(i) - return in.finishStringValue(value) + return in.finishStringValue(value, nonASCII) case c == escape: i++ @@ -207,7 +143,7 @@ func (in *Input) consumeStringRawEscaped(start, i int) token.T { case doubleQuote, escape, slash, 'b', 'f', 'n', 'r', 't': i++ case 'u': - next, err := validateUnicodeWhole(data, i+1, n) + next, err := validateUnicodeWhole(data, i+1, n, in.UTF8Policy) if err != nil { in.Consumed, in.Offset = i, uint64(i) in.Err = err @@ -233,34 +169,48 @@ func (in *Input) consumeStringRawEscaped(start, i int) token.T { // (see consumeStringEscaped). run := i stop := i + 1 + hi := uint64(c) // data[i], the byte that entered this run probe := min(stop+swarProbe, n) for ; stop < probe; stop++ { - if b := data[stop]; b == doubleQuote || b == escape || b < controlCharsUpperBound { + b := data[stop] + if b == doubleQuote || b == escape || b < controlCharsUpperBound { break } + hi |= uint64(b) } if stop == probe && stop < n { // outran the scalar probe → SWAR, guess long past guessLong for stop+8 <= n { - if m := swar.StringStopMask(binary.LittleEndian.Uint64(data[stop:])); m != 0 { - stop += swar.FirstByte(m) + w := binary.LittleEndian.Uint64(data[stop:]) + if m := swar.StringStopMask(w); m != 0 { + k := swar.FirstByte(m) + hi |= swar.LanesBelow(w, k) + stop += k break } + hi |= w stop += 8 if stop-run >= guessLong && !in.NoAVX2 { - stop += strscan.ScanStop(data[stop:n]) + delta, runNonASCII := strscan.ScanStop(data[stop:n]) + stop += delta + if runNonASCII { + hi |= swar.HighBits + } break } } for ; stop < n; stop++ { - if b := data[stop]; b == doubleQuote || b == escape || + b := data[stop] + if b == doubleQuote || b == escape || b < controlCharsUpperBound { break } + hi |= uint64(b) } } + nonASCII = nonASCII || hi&swar.HighBits != 0 i = stop } } @@ -287,22 +237,26 @@ func (in *Input) consumeStringRawEscaped(start, i int) token.T { // //nolint:dupl // the structure is the same but differs subtly and its critical that the inside remains inlined. func (in *Input) consumeStringRawStreamFast() token.T { - in.SawNonASCII = true data := in.Buffer n := in.Bufferized start := in.Consumed // first content byte (opening quote already consumed) i := start + var hi uint64 // fused non-ASCII accumulator, see consumeStringWhole guard := start + guessLong if in.NoAVX2 { guard = n + 1 } for i+8 <= n { - if m := swar.StringStopMask(binary.LittleEndian.Uint64(data[i:])); m != 0 { - i += swar.FirstByte(m) + w := binary.LittleEndian.Uint64(data[i:]) + if m := swar.StringStopMask(w); m != 0 { + k := swar.FirstByte(m) + hi |= swar.LanesBelow(w, k) + i += k break } + hi |= w i += 8 if i >= guard { break @@ -310,13 +264,19 @@ func (in *Input) consumeStringRawStreamFast() token.T { } if i >= guard && i+8 <= n { if c := data[i]; c != doubleQuote && c != escape && c >= controlCharsUpperBound { - i += strscan.ScanStop(data[i:n]) + delta, nonASCII := strscan.ScanStop(data[i:n]) + i += delta + if nonASCII { + hi |= swar.HighBits + } } } for ; i < n; i++ { - if c := data[i]; c == doubleQuote || c == escape || c < controlCharsUpperBound { + c := data[i] + if c == doubleQuote || c == escape || c < controlCharsUpperBound { break } + hi |= uint64(c) } if i >= n { @@ -339,7 +299,7 @@ func (in *Input) consumeStringRawStreamFast() token.T { in.Offset += uint64(end - start) in.Consumed = end - return in.finishStringValue(value) + return in.finishStringValue(value, hi&swar.HighBits != 0) case c < controlCharsUpperBound: in.Offset += uint64(i - start) @@ -357,7 +317,7 @@ func (in *Input) consumeStringRawStreamFast() token.T { // consumeStringRawStreaming is the raw scan over a refilling buffer: it copies the source bytes verbatim (escapes // intact) into l.in.CurrentValue while validating them, so the value survives buffer turnover. func (in *Input) consumeStringRawStreaming() token.T { - in.SawNonASCII = true + var nonASCII bool // did any raw source byte carry the high bit (see consumeStringEscaped) in.CurrentValue = in.CurrentValue[:0] for { @@ -384,7 +344,7 @@ func (in *Input) consumeStringRawStreaming() token.T { switch { case b == doubleQuote: - return in.finishStringValue(in.CurrentValue) + return in.finishStringValue(in.CurrentValue, nonASCII) case b == escape: in.CurrentValue = append(in.CurrentValue, escape) @@ -400,6 +360,7 @@ func (in *Input) consumeStringRawStreaming() token.T { return token.None default: + nonASCII = nonASCII || b >= utf8.RuneSelf in.CurrentValue = append(in.CurrentValue, b) // bulk-scan the rest of this clean run within the current window (§10.5c, the raw analogue of @@ -414,36 +375,48 @@ func (in *Input) consumeStringRawStreaming() token.T { n := in.Bufferized runStart := in.Consumed stop := runStart + var hi uint64 probe := min(stop+swarProbe, n) for ; stop < probe; stop++ { - if c := data[stop]; c == doubleQuote || c == escape || + c := data[stop] + if c == doubleQuote || c == escape || c < controlCharsUpperBound { break } + hi |= uint64(c) } if stop == probe && stop < n { // run outran the scalar probe → SWAR/AVX2 for stop+8 <= n { - if m := swar.StringStopMask( - binary.LittleEndian.Uint64(data[stop:]), - ); m != 0 { - stop += swar.FirstByte(m) + w := binary.LittleEndian.Uint64(data[stop:]) + if m := swar.StringStopMask(w); m != 0 { + k := swar.FirstByte(m) + hi |= swar.LanesBelow(w, k) + stop += k break } + hi |= w stop += 8 if stop-runStart >= guessLong && !in.NoAVX2 { - stop += strscan.ScanStop(data[stop:n]) + delta, runNonASCII := strscan.ScanStop(data[stop:n]) + stop += delta + if runNonASCII { + hi |= swar.HighBits + } break } } for ; stop < n; stop++ { - if c := data[stop]; c == doubleQuote || c == escape || + c := data[stop] + if c == doubleQuote || c == escape || c < controlCharsUpperBound { break } + hi |= uint64(c) } } + nonASCII = nonASCII || hi&swar.HighBits != 0 if stop > runStart { if in.MaxValueBytes > 0 && len(in.CurrentValue)+(stop-runStart) > in.MaxValueBytes { @@ -485,70 +458,37 @@ func (in *Input) rawEscapeStreaming() error { // // The leading "\u" has already been appended. func (in *Input) rawUnicodeStreaming() error { - var buf [4]byte - if err := in.consumeN(buf[:]); err != nil { - return codes.ErrUnicodeEscape + start := len(in.CurrentValue) + digits, r, err := in.readHex4() + if err != nil { + return err } - code, ok := scan.Hex4(buf[0], buf[1], buf[2], buf[3]) - if !ok { - return codes.ErrUnicodeEscape - } - in.CurrentValue = append(in.CurrentValue, buf[:]...) + in.CurrentValue = append(in.CurrentValue, digits[:]...) - r := rune(code) - if utf16.IsSurrogate(r) { - var nb [6]byte // "\uYYYY" - if err := in.consumeN(nb[:]); err != nil { - return codes.ErrSurrogateEscape - } - if nb[0] != escape || nb[1] != 'u' { - return codes.ErrSurrogateEscape - } - code2, ok2 := scan.Hex4(nb[2], nb[3], nb[4], nb[5]) - if !ok2 { - return codes.ErrUnicodeEscape - } - if utf16.DecodeRune(r, rune(code2)) == utf8.RuneError { - return codes.ErrSurrogateEscape - } - in.CurrentValue = append(in.CurrentValue, nb[:]...) - } else if !utf8.ValidRune(r) { - return codes.ErrInvalidRune + if !utf16.IsSurrogate(r) { + return nil } - return nil -} + // a high surrogate must be completed by \uXXXX; peek so an escape that does not complete it is left to be + // validated on its own (see [Input.peekLowSurrogate]). + if low, ok := in.peekLowSurrogate(); ok { + if utf16.DecodeRune(r, low) != utf8.RuneError { + in.CurrentValue = append( + in.CurrentValue, + in.Buffer[in.Consumed:in.Consumed+surrogateEscapeLen]...) + in.takePeeked(surrogateEscapeLen) -// validateUnicodeWhole validates a \uXXXX sequence at pos (first hex digit, past the 'u') in a whole buffer, following -// one surrogate pair when present. -// -// It returns the index just past the validated sequence, or an error. -func validateUnicodeWhole(data []byte, pos, n int) (int, error) { - if pos+4 > n { - return pos, codes.ErrUnicodeEscape - } - code, ok := scan.Hex4(data[pos], data[pos+1], data[pos+2], data[pos+3]) - if !ok { - return pos, codes.ErrUnicodeEscape + return nil + } } - pos += 4 - r := rune(code) - if utf16.IsSurrogate(r) { - if pos+6 > n || data[pos] != escape || data[pos+1] != 'u' { - return pos, codes.ErrSurrogateEscape - } - code2, ok2 := scan.Hex4(data[pos+2], data[pos+3], data[pos+4], data[pos+5]) - if !ok2 { - return pos, codes.ErrUnicodeEscape - } - if utf16.DecodeRune(r, rune(code2)) == utf8.RuneError { - return pos, codes.ErrSurrogateEscape - } - pos += 6 - } else if !utf8.ValidRune(r) { - return pos, codes.ErrInvalidRune + if berr := in.brokenSurrogate(); berr != nil { + in.CurrentValue = in.CurrentValue[:start] // the value is discarded with the error; keep it consistent anyway + + return berr } - return pos, nil + // UTF8Replace / UTF8Passthrough: VL keeps the escape as SOURCE TEXT, so nothing is rewritten here. The U+FFFD + // substitution happens on decode, in token.Unescape, which already yields it for a broken pair (plan §2.5). + return nil } diff --git a/json/lexers/default-lexer/internal/input/utf8.go b/json/lexers/default-lexer/internal/input/utf8.go new file mode 100644 index 0000000..37d7fbb --- /dev/null +++ b/json/lexers/default-lexer/internal/input/utf8.go @@ -0,0 +1,209 @@ +package input + +import ( + "unicode/utf16" + "unicode/utf8" + + "github.com/go-openapi/core/json/internal/utf8x" + codes "github.com/go-openapi/core/json/lexers/error-codes" + scan "github.com/go-openapi/core/json/lexers/internal/scan" +) + +// surrogateEscapeLen is the width of the `\uXXXX` that must follow a high surrogate to complete a pair. +const surrogateEscapeLen = 6 + +// The UTF-8 byte order mark, and the two UTF-16 ones — the latter recognized only to turn an unhelpful error into an +// accurate one. +const ( + bomUTF8Len = 3 + bom0, bom1, bom2 = 0xEF, 0xBB, 0xBF + + bomUTF16Len = 2 + bomUTF16LE0 = 0xFF + bomUTF16LE1 = 0xFE + bomUTF16BE0 = 0xFE + bomUTF16BE1 = 0xFF +) + +// CheckBOM consumes a UTF-8 byte order mark opening the input, and rejects a UTF-16 one. +// +// RFC 8259 §8.1 forbids *emitting* a BOM but explicitly allows an implementation to ignore one on input, and enough +// producers emit one that rejecting the document outright is unhelpful. +// +// It runs once, before the first token, on the buffer's opening bytes only — so it costs one comparison per document +// and touches no scan loop. Only the complete three-byte sequence at offset 0 is consumed: a truncated BOM, or a +// U+FEFF anywhere else, stays what it is (an invalid token between values, or an ordinary character inside a string), +// which is what keeps n_structure_incomplete_UTF8_BOM and n_structure_UTF8_BOM_no_data rejected. +// +// The mark is NOT re-emitted by the verbatim lexer: it is consumed before any token exists, so it is not part of any +// token's leading space (see [VL.LeadingSpace]). A round-trip therefore drops it — the one documented exception to +// VL's byte-exactness on valid input, and the direction RFC 8259 §8.1 prefers. +// +// A UTF-16 document cannot be lexed at all (we are UTF-8 only) and would otherwise fail with a baffling "invalid JSON +// token" on its first byte; codes.ErrNotUTF8 says what is actually wrong. Nothing is consumed there — the document is +// rejected either way — and 0xFF/0xFE can never legitimately open a JSON value, so there is no input this misjudges. +func (in *Input) CheckBOM() { + if in.Consumed != 0 || in.Offset != 0 || in.Bufferized < bomUTF16Len { + return + } + + data := in.Buffer + switch { + case (data[0] == bomUTF16LE0 && data[1] == bomUTF16LE1) || + (data[0] == bomUTF16BE0 && data[1] == bomUTF16BE1): + in.Err = codes.ErrNotUTF8 + + case in.Bufferized >= bomUTF8Len && + data[0] == bom0 && data[1] == bom1 && data[2] == bom2: + in.Consumed, in.Offset = bomUTF8Len, bomUTF8Len + } +} + +// brokenSurrogate applies the UTF-8 policy to a \u escape that does not denote a Unicode scalar value: an unpaired +// high surrogate, an inverted pair, or a lone low surrogate. +// +// A \u escape must always produce SOME rune — the escape text is not the value, so there is nothing to pass through — +// which is why UTF8Passthrough behaves like UTF8Replace here and only UTF8Strict errors. +// +// Both lexers route through this so they cannot drift apart again: L used to launder the failure into a silent U+FFFD +// (utf16.DecodeRune returns utf8.RuneError, and utf8.ValidRune(U+FFFD) is true) while VL rejected it. +func (in *Input) brokenSurrogate() error { + if in.UTF8Policy == UTF8Strict { + return codes.ErrSurrogateEscape + } + + return nil +} + +// peekLowSurrogate inspects — WITHOUT consuming — the next `\uXXXX` and reports the code unit it carries. +// +// It does not consume, because a sequence that turns out not to complete a pair must be left for the caller to process +// as an escape in its own right: `\uD800\uD800` is two ill-formed units and must yield two U+FFFD, not one. +func (in *Input) peekLowSurrogate() (rune, bool) { + if !in.ensureWindow(surrogateEscapeLen) { + return 0, false + } + + w := in.Buffer[in.Consumed : in.Consumed+surrogateEscapeLen] + if w[0] != escape || w[1] != 'u' { + return 0, false + } + + code, ok := scan.Hex4(w[2], w[3], w[4], w[5]) + if !ok { + return 0, false + } + + return rune(code), true +} + +// takePeeked consumes n bytes previously inspected by [Input.peekLowSurrogate]. +func (in *Input) takePeeked(n int) { + in.Consumed += n + in.Offset += uint64(n) +} + +// readHex4 consumes the four hex digits of a \uXXXX escape (the leading "\u" is already consumed) and returns them +// verbatim along with the code unit they encode. The digits are unambiguously part of this escape, so consuming them +// needs no lookahead. +// +// The raw digits are returned because the verbatim scanner appends them to its value and cannot read them back off the +// window: consumeN may have crossed a refill. +func (in *Input) readHex4() ([4]byte, rune, error) { + var buf [4]byte + if err := in.consumeN(buf[:]); err != nil { + return buf, 0, codes.ErrUnicodeEscape + } + + code, ok := scan.Hex4(buf[0], buf[1], buf[2], buf[3]) + if !ok { + return buf, 0, codes.ErrUnicodeEscape + } + + return buf, rune(code), nil +} + +// validateUnicodeWhole validates a \uXXXX sequence at pos (first hex digit, past the 'u') in a whole buffer, following +// one surrogate pair when present. +// +// It returns the index just past the validated sequence, or an error. A `\uXXXX` that does not complete a pair is NOT +// consumed here: it is left to be validated as its own escape on the next turn (see [Input.peekLowSurrogate]). +func validateUnicodeWhole(data []byte, pos, n int, policy UTF8Policy) (int, error) { + if pos+4 > n { + return pos, codes.ErrUnicodeEscape + } + + code, ok := scan.Hex4(data[pos], data[pos+1], data[pos+2], data[pos+3]) + if !ok { + return pos, codes.ErrUnicodeEscape + } + pos += 4 + + r := rune(code) + if !utf16.IsSurrogate(r) { + return pos, nil + } + + if pos+surrogateEscapeLen <= n && data[pos] == escape && data[pos+1] == 'u' { + code2, ok2 := scan.Hex4(data[pos+2], data[pos+3], data[pos+4], data[pos+5]) + if ok2 && utf16.DecodeRune(r, rune(code2)) != utf8.RuneError { + return pos + surrogateEscapeLen, nil + } + } + + if policy == UTF8Strict { + return pos, codes.ErrSurrogateEscape + } + + return pos, nil +} + +// handleInvalidUTF8 applies the configured [UTF8Policy] to a string value that has just been found to carry an +// ill-formed UTF-8 sequence. +// +// It is deliberately out of line: it is unreachable for valid input, so keeping it out of finishStringValue's frame +// leaves the funnel's codegen to the fast path (the same split the string fast/slow paths use). +// +// It returns the value to emit and whether lexing may continue; under [UTF8Strict] it sets in.Err and returns false. +func (in *Input) handleInvalidUTF8(value []byte) ([]byte, bool) { + if in.UTF8Policy == UTF8Strict { + in.Err = codes.ErrInvalidUTF8 + in.Offset = in.faultOffset(value) + + return nil, false + } + + // UTF8Replace: rewrite into a dedicated scratch. It cannot be in.CurrentValue, which IS the value on the + // unescaping paths. + if in.MaxValueBytes > 0 && utf8x.SanitizedLen(value) > in.MaxValueBytes { + // substituting U+FFFD can grow the value (3 bytes for every invalid byte), so the cap is re-checked against the + // rewritten length, not the scanned one. + in.Err = codes.ErrMaxValueBytes + + return nil, false + } + + in.sanitized = utf8x.Sanitize(in.sanitized[:0], value) + + return in.sanitized, true +} + +// faultOffset maps the first ill-formed byte of value back to a stream offset. +// +// On entry the cursor sits just past the closing quote, so the value body starts at Offset-len(value)-1. That is exact +// for a value aliased from the input (every non-escaped path, and so every case a caller is likely to debug); when +// escapes were decoded the value is shorter than its source span, and the reported offset is correspondingly an +// under-estimate pointing inside the same string. +func (in *Input) faultOffset(value []byte) uint64 { + idx := utf8x.FirstInvalid(value) + if idx < 0 { + return in.Offset + } + + back := uint64(len(value)-idx) + 1 + if back > in.Offset { + return 0 + } + + return in.Offset - back +} diff --git a/json/lexers/default-lexer/internal/strscan/_asm/asm.go b/json/lexers/default-lexer/internal/strscan/_asm/asm.go index efaff32..52606c5 100644 --- a/json/lexers/default-lexer/internal/strscan/_asm/asm.go +++ b/json/lexers/default-lexer/internal/strscan/_asm/asm.go @@ -10,16 +10,29 @@ package main import ( . "github.com/mmcloughlin/avo/build" . "github.com/mmcloughlin/avo/operand" + "github.com/mmcloughlin/avo/reg" ) func main() { - TEXT("stringStopIndexAVX2", NOSPLIT, "func(data []byte) int") + stringStop() + + Generate() +} + +func stringStop() { + TEXT("stringStopIndexAVX2", NOSPLIT, "func(data []byte) (int, bool)") Doc( - "stringStopIndexAVX2 returns the index of the first byte that is < 0x20, '\"' (0x22) or '\\' (0x5c), or len(data) if none. AVX2, 32 bytes/iter.", + "stringStopIndexAVX2 returns the index of the first byte that is < 0x20, '\"' (0x22) or '\\' (0x5c), or len(data) if none, and whether any byte BEFORE that index has its high bit set (i.e. the scanned run is not pure ASCII). AVX2, 32 bytes/iter.", ) ptr := Load(Param("data").Base(), GP64()) n := Load(Param("data").Len(), GP64()) + // nonascii accumulates "some scanned byte was >= 0x80". In the vector loop it collects VPMOVMSKB sign-bit masks; + // in the scalar tail it collects the byte's 0x80 bit. Only its zero/non-zero state is ever read, so mixing the two + // encodings is safe. + nonascii := GP32() + XORL(nonascii, nonascii) + c1f := YMM() VPBROADCASTB(ConstData("c1f", U8(0x1f)), c1f) c22 := YMM() @@ -49,18 +62,37 @@ func main() { acc := YMM() VPOR(ctrl, q, acc) VPOR(acc, bs, acc) + // the sign bit of every lane IS "byte >= 0x80", so the non-ASCII test is one extra VPMOVMSKB on the block already + // loaded — off the loop's exit-test dependency chain. + hmask := GP32() + VPMOVMSKB(data, hmask) mask := GP32() VPMOVMSKB(acc, mask) TESTL(mask, mask) JNZ(LabelRef("found")) + ORL(hmask, nonascii) // whole block is string content: account for all 32 lanes ADDQ(Imm(32), i) JMP(LabelRef("loop32")) Label("found") off := GP64() TZCNTL(mask, off.As32()) // bit index 0..31, zero-extends to 64 + // Only the lanes BEFORE the stop belong to the value, so keep the bits of hmask strictly below mask's lowest set + // bit: (mask-1) &^ mask. Four plain ALU ops — deliberately not BMI's BLSMSK/ANDN, since the CPUID gate here checks + // AVX2 only (TZCNT above is safe because it decodes as BSF, which agrees for a non-zero operand; ANDN has no such + // fallback). + below := GP32() + MOVL(mask, below) + DECL(below) + notMask := GP32() + MOVL(mask, notMask) + NOTL(notMask) + ANDL(notMask, below) + ANDL(below, hmask) + ORL(hmask, nonascii) ADDQ(off, i) Store(i, ReturnIndex(0)) + storeNonASCII(nonascii) VZEROUPPER() RET() @@ -76,18 +108,29 @@ func main() { JE(LabelRef("foundtail")) CMPL(b, Imm(0x5c)) JE(LabelRef("foundtail")) + ANDL(Imm(0x80), b) // keep only "is this byte >= 0x80"; b is dead after the comparisons + ORL(b, nonascii) INCQ(i) JMP(LabelRef("tailloop")) Label("foundtail") Store(i, ReturnIndex(0)) + storeNonASCII(nonascii) VZEROUPPER() RET() Label("notfound") Store(n, ReturnIndex(0)) + storeNonASCII(nonascii) VZEROUPPER() RET() +} - Generate() +// storeNonASCII writes (nonascii != 0) to the bool result. +func storeNonASCII(nonascii reg.GPVirtual) { + TESTL(nonascii, nonascii) + flag := GP8() + SETNE(flag) + Store(flag, ReturnIndex(1)) } + diff --git a/json/lexers/default-lexer/internal/strscan/bench_amd64_test.go b/json/lexers/default-lexer/internal/strscan/bench_amd64_test.go index 9b8f837..e2ef789 100644 --- a/json/lexers/default-lexer/internal/strscan/bench_amd64_test.go +++ b/json/lexers/default-lexer/internal/strscan/bench_amd64_test.go @@ -25,13 +25,15 @@ func BenchmarkStringStop(b *testing.B) { b.Run(fmt.Sprintf("len=%04d/SWAR", n), func(b *testing.B) { b.SetBytes(int64(n)) for b.Loop() { - sink += scanStopSWAR(data) + idx, _ := scanStopSWAR(data) + sink += idx } }) b.Run(fmt.Sprintf("len=%04d/AVX2", n), func(b *testing.B) { b.SetBytes(int64(n)) for b.Loop() { - sink += stringStopIndexAVX2(data) + idx, _ := stringStopIndexAVX2(data) + sink += idx } }) } diff --git a/json/lexers/default-lexer/internal/strscan/scan_amd64.go b/json/lexers/default-lexer/internal/strscan/scan_amd64.go index a607db9..783dac9 100644 --- a/json/lexers/default-lexer/internal/strscan/scan_amd64.go +++ b/json/lexers/default-lexer/internal/strscan/scan_amd64.go @@ -7,8 +7,12 @@ package strscan // stringStopIndexAVX2 is the avo-generated AVX2 kernel (stringstop_amd64.s): it returns the index of the first byte < // 0x20, '"' (0x22) or '\' (0x5c), or len(data) if none, scanning 32 bytes per YMM iteration. // +// The second result reports whether any byte BEFORE that index has its high bit set. It is free: the sign bit of every +// lane already means "byte >= 0x80", so one extra VPMOVMSKB on the block already in the register, off the loop's +// exit-test dependency chain, tells the caller whether the run it just scanned needs UTF-8 validation at all. +// // It uses AVX/AVX2/BMI instructions and must only be called when [useAVX2] is true. -func stringStopIndexAVX2(data []byte) int +func stringStopIndexAVX2(data []byte) (int, bool) // avx2Min is the minimum remaining-byte count at which the AVX2 kernel beats the 8-byte SWAR loop, once its 3-constant // broadcast setup and the (non-inlinable) call are accounted for. @@ -18,12 +22,15 @@ func stringStopIndexAVX2(data []byte) int const avx2Min = 32 // ScanStop returns the index of the first JSON string-stop byte (a control char < 0x20, '"', or '\') in data, or -// len(data) if none. +// len(data) if none, together with whether any byte before that index is >= 0x80. // // It is called only for the long-string case (the caller found a clean first word), so it dispatches to the AVX2 kernel // when the CPU supports it and enough bytes remain, and to SWAR otherwise. // The WithoutAVX2 knob is handled by the caller (it simply never delegates here), so ScanStop always tries the kernel. -func ScanStop(data []byte) int { +// +// The non-ASCII result is what lets the string scanners fuse UTF-8 detection into the scan they already perform: a run +// this reports as pure ASCII is proven valid UTF-8 with no second pass. Both implementations answer identically. +func ScanStop(data []byte) (int, bool) { if useAVX2 && len(data) >= avx2Min { return stringStopIndexAVX2(data) } diff --git a/json/lexers/default-lexer/internal/strscan/scan_noasm.go b/json/lexers/default-lexer/internal/strscan/scan_noasm.go index be22352..104633f 100644 --- a/json/lexers/default-lexer/internal/strscan/scan_noasm.go +++ b/json/lexers/default-lexer/internal/strscan/scan_noasm.go @@ -3,10 +3,10 @@ package strscan // ScanStop returns the index of the first JSON string-stop byte (a control char < 0x20, '"', or '\') in data, or -// len(data) if none. +// len(data) if none, together with whether any byte before that index is >= 0x80. // // On non-amd64 there is no vector kernel, so the long-string path is the same 8-byte SWAR scan the inline fast path // uses. -func ScanStop(data []byte) int { +func ScanStop(data []byte) (int, bool) { return scanStopSWAR(data) } diff --git a/json/lexers/default-lexer/internal/strscan/stringstop_amd64.s b/json/lexers/default-lexer/internal/strscan/stringstop_amd64.s index ecf8efc..0f45e6f 100644 --- a/json/lexers/default-lexer/internal/strscan/stringstop_amd64.s +++ b/json/lexers/default-lexer/internal/strscan/stringstop_amd64.s @@ -2,61 +2,81 @@ #include "textflag.h" -// func stringStopIndexAVX2(data []byte) int +// func stringStopIndexAVX2(data []byte) (int, bool) // Requires: AVX, AVX2, BMI -TEXT ·stringStopIndexAVX2(SB), NOSPLIT, $0-32 +TEXT ·stringStopIndexAVX2(SB), NOSPLIT, $0-33 MOVQ data_base+0(FP), AX MOVQ data_len+8(FP), CX + XORL DX, DX VPBROADCASTB c1f<>+0(SB), Y0 VPBROADCASTB c22<>+0(SB), Y1 VPBROADCASTB c5c<>+0(SB), Y2 - XORQ DX, DX + XORQ BX, BX loop32: - LEAQ 32(DX), BX - CMPQ BX, CX + LEAQ 32(BX), SI + CMPQ SI, CX JG tail - VMOVDQU (AX)(DX*1), Y3 + VMOVDQU (AX)(BX*1), Y3 VPMINUB Y3, Y0, Y4 VPCMPEQB Y4, Y3, Y4 VPCMPEQB Y3, Y1, Y5 - VPCMPEQB Y3, Y2, Y3 + VPCMPEQB Y3, Y2, Y6 VPOR Y4, Y5, Y4 - VPOR Y4, Y3, Y4 - VPMOVMSKB Y4, BX - TESTL BX, BX + VPOR Y4, Y6, Y4 + VPMOVMSKB Y3, SI + VPMOVMSKB Y4, DI + TESTL DI, DI JNZ found - ADDQ $0x20, DX + ORL SI, DX + ADDQ $0x20, BX JMP loop32 found: - TZCNTL BX, AX - ADDQ AX, DX - MOVQ DX, ret+24(FP) + TZCNTL DI, AX + MOVL DI, CX + DECL CX + NOTL DI + ANDL DI, CX + ANDL CX, SI + ORL SI, DX + ADDQ AX, BX + MOVQ BX, ret+24(FP) + TESTL DX, DX + SETNE AL + MOVB AL, ret1+32(FP) VZEROUPPER RET tail: tailloop: - CMPQ DX, CX + CMPQ BX, CX JGE notfound - MOVBLZX (AX)(DX*1), BX - CMPL BX, $0x20 + MOVBLZX (AX)(BX*1), SI + CMPL SI, $0x20 JL foundtail - CMPL BX, $0x22 + CMPL SI, $0x22 JE foundtail - CMPL BX, $0x5c + CMPL SI, $0x5c JE foundtail - INCQ DX + ANDL $0x80, SI + ORL SI, DX + INCQ BX JMP tailloop foundtail: - MOVQ DX, ret+24(FP) + MOVQ BX, ret+24(FP) + TESTL DX, DX + SETNE AL + MOVB AL, ret1+32(FP) VZEROUPPER RET notfound: - MOVQ CX, ret+24(FP) + MOVQ CX, ret+24(FP) + TESTL DX, DX + SETNE AL + MOVB AL, ret1+32(FP) VZEROUPPER RET diff --git a/json/lexers/default-lexer/internal/strscan/strscan.go b/json/lexers/default-lexer/internal/strscan/strscan.go index 6226a83..24b418f 100644 --- a/json/lexers/default-lexer/internal/strscan/strscan.go +++ b/json/lexers/default-lexer/internal/strscan/strscan.go @@ -23,19 +23,31 @@ import ( // [swar.StringStopMask]/[swar.FirstByte] so a stop is found at exactly the same byte here as on the inline fast path. // // It is the non-amd64 implementation of [ScanStop] and the amd64 short-remainder fallback. -func scanStopSWAR(data []byte) int { +// +// The second result reports whether any byte strictly before the stop is >= 0x80, accumulated from the words the scan +// already loaded (the exact analogue of the AVX2 kernel's VPMOVMSKB accumulator). Lanes at or after the stop are +// masked out so the answer is exact, not merely conservative. +func scanStopSWAR(data []byte) (int, bool) { n, i := len(data), 0 + var hi uint64 for i+8 <= n { - if m := swar.StringStopMask(binary.LittleEndian.Uint64(data[i:])); m != 0 { - return i + swar.FirstByte(m) + w := binary.LittleEndian.Uint64(data[i:]) + if m := swar.StringStopMask(w); m != 0 { + k := swar.FirstByte(m) + hi |= swar.LanesBelow(w, k) // only the lanes before the stop belong to the value + + return i + k, hi&swar.HighBits != 0 } + hi |= w i += 8 } for ; i < n; i++ { - if b := data[i]; b < 0x20 || b == '"' || b == '\\' { - return i + b := data[i] + if b < 0x20 || b == '"' || b == '\\' { + return i, hi&swar.HighBits != 0 } + hi |= uint64(b) } - return n + return n, hi&swar.HighBits != 0 } diff --git a/json/lexers/default-lexer/internal/strscan/strscan_test.go b/json/lexers/default-lexer/internal/strscan/strscan_test.go index 93c89e8..162df27 100644 --- a/json/lexers/default-lexer/internal/strscan/strscan_test.go +++ b/json/lexers/default-lexer/internal/strscan/strscan_test.go @@ -1,19 +1,23 @@ package strscan import ( + "bytes" "math/rand" "testing" ) -// scalarStop is the reference oracle: first index of a byte < 0x20 || '"' || '\'. -func scalarStop(data []byte) int { +// scalarStop is the reference oracle: first index of a byte < 0x20 || '"' || '\', and whether any byte STRICTLY +// BEFORE that index is >= 0x80. +func scalarStop(data []byte) (int, bool) { + var nonASCII bool for i, b := range data { if b < 0x20 || b == '"' || b == '\\' { - return i + return i, nonASCII } + nonASCII = nonASCII || b >= 0x80 } - return len(data) + return len(data), nonASCII } // TestScanStopOracle sweeps ScanStop (the shipped gate, AVX2 or SWAR depending on the host) and the portable @@ -47,14 +51,17 @@ func TestScanStopOracle(t *testing.T) { } } - want := scalarStop(data) + want, wantNonASCII := scalarStop(data) // the AVX2-gated path and the portable SWAR path (which WithoutAVX2 forces the lexer onto) must both match the - // scalar oracle. - if got := ScanStop(data); got != want { - t.Fatalf("ScanStop n=%d trial=%d: got %d want %d", n, trial, got, want) + // scalar oracle — for the stop index AND for the fused non-ASCII verdict the string scanners rely on to skip + // UTF-8 validation. + if got, gotNonASCII := ScanStop(data); got != want || gotNonASCII != wantNonASCII { + t.Fatalf("ScanStop n=%d trial=%d: got (%d,%v) want (%d,%v)", + n, trial, got, gotNonASCII, want, wantNonASCII) } - if got := scanStopSWAR(data); got != want { - t.Fatalf("scanStopSWAR n=%d trial=%d: got %d want %d", n, trial, got, want) + if got, gotNonASCII := scanStopSWAR(data); got != want || gotNonASCII != wantNonASCII { + t.Fatalf("scanStopSWAR n=%d trial=%d: got (%d,%v) want (%d,%v)", + n, trial, got, gotNonASCII, want, wantNonASCII) } } } @@ -68,12 +75,89 @@ func TestScanStopHighBytesSafe(t *testing.T) { for i := range data { data[i] = 0x80 | byte(i&0x3f) } - if got := ScanStop(data); got != len(data) { - t.Fatalf("high bytes flagged as stop: got %d, want %d", got, len(data)) + if got, nonASCII := ScanStop(data); got != len(data) || !nonASCII { + t.Fatalf( + "high bytes flagged as stop: got (%d,%v), want (%d,true)", + got, + nonASCII, + len(data), + ) } data[70] = 0x1f - if got := ScanStop(data); got != 70 { - t.Fatalf("got %d, want 70", got) + if got, nonASCII := ScanStop(data); got != 70 || !nonASCII { + t.Fatalf("got (%d,%v), want (70,true)", got, nonASCII) + } +} + +// TestScanStopNonASCIIExactness pins the boundary the fused UTF-8 detection depends on: bytes at or after the stop +// must NOT count. A conservative over-report would only cost a redundant validation, but a *false negative* would let +// invalid UTF-8 through, so the mask has to be exact in the direction that matters and is checked both ways here. +func TestScanStopNonASCIIExactness(t *testing.T) { + cases := []struct { + name string + data []byte + wantStop int + wantNonASCII bool + }{ + {name: "empty", data: []byte{}, wantStop: 0}, + {name: "ascii only, no stop", data: []byte("plain ascii run"), wantStop: 15}, + {name: "stop first, high byte after", data: append([]byte{'"'}, 0xc3, 0xa9), wantStop: 0}, + { + name: "high byte right after a late stop", + data: append(append([]byte("0123456789abcdefghijklmnopqrstuv"), '"'), 0xff, 0xff), + wantStop: 32, + }, + { + name: "high byte right before a late stop", + data: append(append([]byte("0123456789abcdefghijklmnopqrstu"), 0xc3), '"'), + wantStop: 32, + wantNonASCII: true, + }, + { + name: "high byte in the scalar tail", + data: append(bytes.Repeat([]byte{'a'}, 40), 0xe2, 0x82, 0xac, '"'), + wantStop: 43, + wantNonASCII: true, + }, + { + name: "tail high byte after the stop", + data: append(bytes.Repeat([]byte{'a'}, 40), '"', 0xe2, 0x82, 0xac), + wantStop: 40, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + wantStop, wantNonASCII := scalarStop(tc.data) + if wantStop != tc.wantStop || wantNonASCII != tc.wantNonASCII { + t.Fatalf("the case table disagrees with the oracle: table (%d,%v), oracle (%d,%v)", + tc.wantStop, tc.wantNonASCII, wantStop, wantNonASCII) + } + if got, nonASCII := ScanStop( + tc.data, + ); got != tc.wantStop || + nonASCII != tc.wantNonASCII { + t.Errorf( + "ScanStop: got (%d,%v), want (%d,%v)", + got, + nonASCII, + tc.wantStop, + tc.wantNonASCII, + ) + } + if got, nonASCII := scanStopSWAR( + tc.data, + ); got != tc.wantStop || + nonASCII != tc.wantNonASCII { + t.Errorf( + "scanStopSWAR: got (%d,%v), want (%d,%v)", + got, + nonASCII, + tc.wantStop, + tc.wantNonASCII, + ) + } + }) } } diff --git a/json/lexers/default-lexer/internal/swar/swar.go b/json/lexers/default-lexer/internal/swar/swar.go index 12a11d6..4da0742 100644 --- a/json/lexers/default-lexer/internal/swar/swar.go +++ b/json/lexers/default-lexer/internal/swar/swar.go @@ -33,8 +33,19 @@ const ( func FirstByte(mask uint64) int { return bits.TrailingZeros64(mask) >> 3 } // HighBits is the high bit of every lane: a word OR-accumulated over a byte run is non-ASCII iff (acc & HighBits) != 0. +// +// This is what makes UTF-8 detection free in the string scanners: every word the stop-scan loads is OR-ed into an +// accumulator, so a value proven to be pure ASCII — the overwhelmingly common case — is proven valid UTF-8 with no +// second pass over its bytes. const HighBits = high +// LanesBelow returns w with every lane from index k upward cleared, for k in 0..8. +// +// It is used to trim the word in which a scan stopped: only the lanes BEFORE the stop belong to the value, so only +// they may contribute to a non-ASCII verdict. Go defines an over-wide shift as zero, so k == 0 (stop in the first +// lane, nothing precedes it) correctly yields 0 with no branch. +func LanesBelow(w uint64, k int) uint64 { return w & (^uint64(0) >> (64 - 8*k)) } + // Broadcast replicates b into all eight lanes of a word. func Broadcast(b byte) uint64 { return lo * uint64(b) } diff --git a/json/lexers/default-lexer/options.go b/json/lexers/default-lexer/options.go index 0968242..ca3fbf0 100644 --- a/json/lexers/default-lexer/options.go +++ b/json/lexers/default-lexer/options.go @@ -1,5 +1,7 @@ package lexer +import "github.com/go-openapi/core/json/lexers/default-lexer/internal/input" + type ( // Option for the lexer. Option func(options) options @@ -11,11 +13,67 @@ type ( keepPreviousBuffer int elideSeparator bool noAVX2 bool - validateMode int + utf8Policy UTF8Policy jsonPointer bool } ) +// UTF8Policy selects what the lexer does with a string that is not valid UTF-8 — an ill-formed byte sequence in the +// source, or a \u escape that does not denote a Unicode scalar value. +// +// RFC 8259 §8.1 requires JSON text to be UTF-8, so the default ([UTF8Strict]) rejects such input. See +// [WithUTF8Policy]. +type UTF8Policy = input.UTF8Policy + +const ( + // UTF8Strict rejects the document: [L.Err] reports [codes.ErrInvalidUTF8] for ill-formed bytes, or + // [codes.ErrSurrogateEscape] for a broken \u surrogate pair. + // + // This is the default for both [L] and [VL], and it never copies: validation only reads the value the scan already + // produced, so the zero-copy aliasing of string values is unaffected. + UTF8Strict = input.UTF8Strict + + // UTF8Replace accepts the document and substitutes U+FFFD (the replacement character) — one per invalid byte, one + // per broken escape — so every emitted value is valid UTF-8. + // + // Only an offending value is rewritten (and therefore copied out of the input); valid values are still aliased with + // zero copy. + // + // A mangled value no longer maps onto its source text: U+FFFD is three bytes replacing one, so its length differs + // from the source span and an index into it cannot be turned into a source offset. Token POSITIONS are unaffected + // — [L.Offset], [VL.Line], [VL.Column] and [VL.LeadingSpace] come from the scan cursor, never from the value. See + // the [VL] documentation for the full contract. + UTF8Replace = input.UTF8Replace + + // UTF8Passthrough disables raw-byte validation: ill-formed sequences are passed to the caller untouched, exactly as + // the lexer behaved before UTF-8 validation existed. + // + // A \u escape still yields U+FFFD when it does not denote a scalar value, because an escape must always decode to + // some rune — there is nothing to "pass through" for an escape, the escape text is not the value. + // + // UNSAFE: use it only for input already known to be valid UTF-8 (or when a downstream stage validates). Invalid + // bytes will reach your []byte and silently become U+FFFD the moment the value is iterated as a string. + UTF8Passthrough = input.UTF8Passthrough +) + +// WithUTF8Policy selects how string values that are not valid UTF-8 are handled: rejected ([UTF8Strict], the default), +// sanitized to U+FFFD ([UTF8Replace]), or waved through unchecked ([UTF8Passthrough]). +// +// It applies to both the semantic lexer [L] and the verbatim lexer [VL], and it covers both ways a value can carry a +// non-character: ill-formed UTF-8 in the source bytes, and a \u escape that does not denote a Unicode scalar value +// (an unpaired or inverted surrogate). +// +// The strategy used to enforce the policy is not a knob: detection of non-ASCII bytes is fused into the string scan +// the lexer already performs, so an all-ASCII value — the overwhelmingly common case — is proven valid with no second +// pass. Only values that actually carry a byte >= 0x80 are validated. +func WithUTF8Policy(policy UTF8Policy) Option { + return func(o options) options { + o.utf8Policy = policy + + return o + } +} + const defaultBufferBytes = 4096 // bufferSizeAlignment is the granularity to which a caller-supplied buffer size is rounded up. @@ -86,28 +144,6 @@ func WithElideSeparator(enabled bool) Option { // shows the vector path is not paying off. // // It does not change the token stream, only how the scan is performed. -// WithUTF8ValidationMode (PROTOTYPE) selects the UTF-8 validation strategy (see input.Validate* constants). -func WithUTF8ValidationMode(mode int) Option { - return func(o options) options { - o.validateMode = mode - - return o - } -} - -// WithUTF8Validation (PROTOTYPE) enables RFC 7493 UTF-8 validation of string values. -func WithUTF8Validation(enabled bool) Option { - return func(o options) options { - if enabled { - o.validateMode = 2 - } else { - o.validateMode = 0 - } - - return o - } -} - func WithoutAVX2(disabled bool) Option { return func(o options) options { o.noAVX2 = disabled diff --git a/json/lexers/default-lexer/semantic_lexer.go b/json/lexers/default-lexer/semantic_lexer.go index 932a2be..72667d7 100644 --- a/json/lexers/default-lexer/semantic_lexer.go +++ b/json/lexers/default-lexer/semantic_lexer.go @@ -226,7 +226,7 @@ func (l *L) reset() { l.in.MaxValueBytes = l.maxValueBytes l.in.KeepPreviousBuffer = l.keepPreviousBuffer l.in.NoAVX2 = l.noAVX2 - l.in.ValidateMode = l.validateMode + l.in.UTF8Policy = l.utf8Policy l.current = token.None l.in.Offset = 0 l.in.Consumed = 0 @@ -240,6 +240,12 @@ func (l *L) reset() { l.tokCol = 0 l.in.CurrentValue = l.in.CurrentValue[:0] // TODO: possibly preallocate value buffer to some configurable size + if l.in.WholeBuffer { + // a bound buffer can be inspected right away; a streaming lexer is checked after its first fill instead (see + // input.FirstFill), which is also where a stream promoted to whole-buffer mode is handled. + l.in.CheckBOM() + } + if l.nestingLevel != nil { l.nestingLevel = l.nestingLevel[:1] } else { diff --git a/json/lexers/default-lexer/testdata/conformance_i_behavior.golden b/json/lexers/default-lexer/testdata/conformance_i_behavior.golden new file mode 100644 index 0000000..40c420a --- /dev/null +++ b/json/lexers/default-lexer/testdata/conformance_i_behavior.golden @@ -0,0 +1,35 @@ +i_number_double_huge_neg_exp.json: L/bytes=accept L/reader=accept VL/bytes=accept VL/reader=accept L/replace=accept +i_number_huge_exp.json: L/bytes=accept L/reader=accept VL/bytes=accept VL/reader=accept L/replace=accept +i_number_neg_int_huge_exp.json: L/bytes=accept L/reader=accept VL/bytes=accept VL/reader=accept L/replace=accept +i_number_pos_double_huge_exp.json: L/bytes=accept L/reader=accept VL/bytes=accept VL/reader=accept L/replace=accept +i_number_real_neg_overflow.json: L/bytes=accept L/reader=accept VL/bytes=accept VL/reader=accept L/replace=accept +i_number_real_pos_overflow.json: L/bytes=accept L/reader=accept VL/bytes=accept VL/reader=accept L/replace=accept +i_number_real_underflow.json: L/bytes=accept L/reader=accept VL/bytes=accept VL/reader=accept L/replace=accept +i_number_too_big_neg_int.json: L/bytes=accept L/reader=accept VL/bytes=accept VL/reader=accept L/replace=accept +i_number_too_big_pos_int.json: L/bytes=accept L/reader=accept VL/bytes=accept VL/reader=accept L/replace=accept +i_number_very_big_negative_int.json: L/bytes=accept L/reader=accept VL/bytes=accept VL/reader=accept L/replace=accept +i_object_key_lone_2nd_surrogate.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_1st_surrogate_but_2nd_missing.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_1st_valid_surrogate_2nd_invalid.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_UTF-16LE_with_BOM.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=reject +i_string_UTF-8_invalid_sequence.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_UTF8_surrogate_U+D800.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_incomplete_surrogate_and_escape_valid.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_incomplete_surrogate_pair.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_incomplete_surrogates_escape_valid.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_invalid_lonely_surrogate.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_invalid_surrogate.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_invalid_utf-8.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_inverted_surrogates_U+1D11E.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_iso_latin_1.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_lone_second_surrogate.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_lone_utf8_continuation_byte.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_not_in_unicode_range.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_overlong_sequence_2_bytes.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_overlong_sequence_6_bytes.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_overlong_sequence_6_bytes_null.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_truncated-utf-8.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_utf16BE_no_BOM.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=reject +i_string_utf16LE_no_BOM.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=reject +i_structure_500_nested_arrays.json: L/bytes=accept L/reader=accept VL/bytes=accept VL/reader=accept L/replace=accept +i_structure_UTF-8_BOM_empty_object.json: L/bytes=accept L/reader=accept VL/bytes=accept VL/reader=accept L/replace=accept diff --git a/json/lexers/default-lexer/utf8_test.go b/json/lexers/default-lexer/utf8_test.go new file mode 100644 index 0000000..bace211 --- /dev/null +++ b/json/lexers/default-lexer/utf8_test.go @@ -0,0 +1,586 @@ +package lexer + +import ( + "bytes" + "fmt" + "io" + "strings" + "testing" + "unicode/utf8" + + "github.com/go-openapi/testify/v2/assert" + "github.com/go-openapi/testify/v2/require" + + codes "github.com/go-openapi/core/json/lexers/error-codes" + "github.com/go-openapi/core/json/lexers/token" +) + +// UTF-8 validation of string values (see .claude/plans/utf8-validation.md). +// +// The lexers used to wave every byte >= 0x80 straight into the token value: the string scanners stop only on '"', +// '\\' and control chars, so ill-formed UTF-8 was never looked at. These tests pin the three policies across every +// string path — whole-buffer vs streaming, clean vs escaped, L vs VL — because the detection that gates validation is +// fused into each scanner separately and can only be wrong per-path. + +// utf8Case is one input and what each policy must make of it. +type utf8Case struct { + name string + // body is the string BODY (no quotes); the case is exercised as a value, as an object key, and with a long + // pure-ASCII prefix that forces the AVX2/long-string path. + body string + // wantStrict is the error the default policy must report (nil means the input is valid). + wantStrict error + // wantReplaced is the decoded value under UTF8Replace. + wantReplaced string + // wantRawKept is true when VL keeps the body verbatim under UTF8Replace: it rewrites invalid BYTES but never + // rewrites escape text (the U+FFFD for a broken escape appears on decode, in token.Unescape). + wantRawKept bool +} + +func utf8Cases() []utf8Case { + return []utf8Case{ + // --- valid: must be untouched by every policy --- + {name: "ascii", body: "plain", wantReplaced: "plain"}, + {name: "2-byte", body: "caf\xc3\xa9", wantReplaced: "caf\u00e9"}, + {name: "3-byte", body: "\xe6\x97\xa5\xe6\x9c\xac", wantReplaced: "\u65e5\u672c"}, + {name: "4-byte", body: "\xf0\x9d\x84\x9e", wantReplaced: "\U0001D11E"}, + {name: "max scalar", body: "\xf4\x8f\xbf\xbf", wantReplaced: "\U0010FFFF"}, + {name: "encoded U+FFFD", body: "\xef\xbf\xbd", wantReplaced: "\ufffd"}, + { + name: "valid surrogate pair escape", + body: `\uD834\uDD1E`, + wantReplaced: "\U0001D11E", + wantRawKept: true, + }, + + // --- ill-formed bytes: one U+FFFD per invalid byte --- + {name: "lone ff", body: "\xff", wantStrict: codes.ErrInvalidUTF8, wantReplaced: "\ufffd"}, + { + name: "lone continuation", + body: "\x81", + wantStrict: codes.ErrInvalidUTF8, + wantReplaced: "\ufffd", + }, + {name: "latin-1", body: "\xe9", wantStrict: codes.ErrInvalidUTF8, wantReplaced: "\ufffd"}, + { + name: "truncated 3-byte", body: "\xe0\xff", + wantStrict: codes.ErrInvalidUTF8, wantReplaced: "\ufffd\ufffd", + }, + { + name: "valid then invalid lead", body: "\xe6\x97\xa5\xd1\x88\xfa", + wantStrict: codes.ErrInvalidUTF8, wantReplaced: "\u65e5\u0448\ufffd", + }, + { + name: "encoded surrogate", body: "\xed\xa0\x80", + wantStrict: codes.ErrInvalidUTF8, wantReplaced: "\ufffd\ufffd\ufffd", + }, + { + name: "beyond U+10FFFF", body: "\xf4\xbf\xbf\xbf", + wantStrict: codes.ErrInvalidUTF8, wantReplaced: "\ufffd\ufffd\ufffd\ufffd", + }, + { + name: "overlong", body: "\xc0\xaf", + wantStrict: codes.ErrInvalidUTF8, wantReplaced: "\ufffd\ufffd", + }, + { + name: "invalid between valid", body: "a\xffb", + wantStrict: codes.ErrInvalidUTF8, wantReplaced: "a\ufffdb", + }, + { + name: "truncated at end", body: "ok\xe2\x82", + wantStrict: codes.ErrInvalidUTF8, wantReplaced: "ok\ufffd\ufffd", + }, + + // --- ill-formed \u escapes: one U+FFFD per broken escape, and the NEXT escape is not swallowed --- + { + name: "unpaired high surrogate then escape", body: `\uD800\uD800\n`, + wantStrict: codes.ErrSurrogateEscape, wantReplaced: "\ufffd\ufffd\n", wantRawKept: true, + }, + { + name: "high surrogate then non-surrogate", body: `\uD888\u1234`, + wantStrict: codes.ErrSurrogateEscape, wantReplaced: "\ufffd\u1234", wantRawKept: true, + }, + { + name: "inverted pair", body: `\uDd1e\uD834`, + wantStrict: codes.ErrSurrogateEscape, wantReplaced: "\ufffd\ufffd", wantRawKept: true, + }, + { + name: "lone high surrogate at end", body: `\uDADA`, + wantStrict: codes.ErrSurrogateEscape, wantReplaced: "\ufffd", wantRawKept: true, + }, + { + name: "lone low surrogate", body: `\uDFAA`, + wantStrict: codes.ErrSurrogateEscape, wantReplaced: "\ufffd", wantRawKept: true, + }, + { + name: "surrogate then literal text", body: `\uD800abc`, + wantStrict: codes.ErrSurrogateEscape, wantReplaced: "\ufffdabc", wantRawKept: true, + }, + + // --- mixed: an escape and an ill-formed byte in the same value --- + { + name: "escape then bad byte", body: "a\\nb\xffc", + wantStrict: codes.ErrInvalidUTF8, wantReplaced: "a\nb\ufffdc", + }, + { + name: "bad byte then escape", body: "a\xffb\\tc", + wantStrict: codes.ErrInvalidUTF8, wantReplaced: "a\ufffdb\tc", + }, + } +} + +// longPrefix is longer than guessLong (16) so the scan delegates to the AVX2/long-string kernel, whose fused +// non-ASCII accumulator is a separate implementation from the inline SWAR one. +const longPrefix = "0123456789abcdefghijklmnopqrstuvwxyz0123456789" + +// utf8Driver is one way of running a document through a lexer: the eight string paths are reached by crossing +// {L, VL} x {whole-buffer, streaming} x {clean, escaped}, and the escaped axis comes from the fixtures themselves. +type utf8Driver struct { + name string + // run drains the document and returns the decoded value of the first String/Key token, the raw token value, and + // the lexer error. + run func(doc []byte, policy UTF8Policy) (decoded, raw []byte, err error) +} + +func utf8Drivers() []utf8Driver { + drainL := func(lx *L) (decoded, raw []byte, err error) { + for tok := range lx.Tokens() { + if k := tok.Kind(); (k == token.String || k == token.Key) && decoded == nil { + decoded = bytes.Clone(tok.Value()) + raw = decoded + } + } + + return decoded, raw, lx.Err() + } + drainVL := func(lx *VL) (decoded, raw []byte, err error) { + for tok := range lx.Tokens() { + if k := tok.Kind(); (k == token.String || k == token.Key) && raw == nil { + raw = bytes.Clone(tok.Value()) + decoded = token.Unescape(bytes.Clone(raw)) + } + } + + return decoded, raw, lx.Err() + } + + return []utf8Driver{ + {name: "L/bytes", run: func(doc []byte, p UTF8Policy) ([]byte, []byte, error) { + return drainL(NewWithBytes(doc, WithUTF8Policy(p))) + }}, + {name: "L/bytes/noavx2", run: func(doc []byte, p UTF8Policy) ([]byte, []byte, error) { + return drainL(NewWithBytes(doc, WithUTF8Policy(p), WithoutAVX2(true))) + }}, + {name: "L/reader", run: func(doc []byte, p UTF8Policy) ([]byte, []byte, error) { + return drainL(New(newChunkReader(doc, 7), WithUTF8Policy(p), WithBufferSize(32))) + }}, + {name: "VL/bytes", run: func(doc []byte, p UTF8Policy) ([]byte, []byte, error) { + return drainVL(NewVerbatimWithBytes(doc, WithUTF8Policy(p))) + }}, + {name: "VL/bytes/noavx2", run: func(doc []byte, p UTF8Policy) ([]byte, []byte, error) { + return drainVL(NewVerbatimWithBytes(doc, WithUTF8Policy(p), WithoutAVX2(true))) + }}, + {name: "VL/reader", run: func(doc []byte, p UTF8Policy) ([]byte, []byte, error) { + return drainVL( + NewVerbatim(newChunkReader(doc, 7), WithUTF8Policy(p), WithBufferSize(32)), + ) + }}, + } +} + +// newChunkReader returns a reader that hands out at most size bytes per Read, so a value — and an ill-formed sequence +// or a \u escape inside it — is forced across refill boundaries. +func newChunkReader(data []byte, size int) *chunkReader { + return &chunkReader{data: data, size: size} +} + +type chunkReader struct { + data []byte + size int + pos int +} + +func (r *chunkReader) Read(p []byte) (int, error) { + if r.pos >= len(r.data) { + return 0, io.EOF + } + n := copy(p, r.data[r.pos:min(r.pos+r.size, len(r.data))]) + r.pos += n + + return n, nil +} + +func TestUTF8Policy(t *testing.T) { + // each fixture is exercised in three shapes, which route to different scanners + shapes := []struct { + name string + doc func(body string) string + }{ + {name: "value", doc: func(b string) string { return `["` + b + `"]` }}, + {name: "key", doc: func(b string) string { return `{"` + b + `":1}` }}, + {name: "long-value", doc: func(b string) string { return `["` + longPrefix + b + `"]` }}, + } + + for _, tc := range utf8Cases() { + for _, shape := range shapes { + doc := []byte(shape.doc(tc.body)) + prefix := "" + if shape.name == "long-value" { + prefix = longPrefix + } + + for _, drv := range utf8Drivers() { + t.Run(tc.name+"/"+shape.name+"/"+drv.name, func(t *testing.T) { + isVerbatim := strings.HasPrefix(drv.name, "VL") + + t.Run("strict", func(t *testing.T) { + decoded, _, err := drv.run(doc, UTF8Strict) + if tc.wantStrict == nil { + require.NoError(t, err) + assert.Equal(t, prefix+tc.wantReplaced, string(decoded)) + + return + } + require.Error(t, err) + assert.ErrorIs(t, err, tc.wantStrict) + }) + + t.Run("replace", func(t *testing.T) { + decoded, raw, err := drv.run(doc, UTF8Replace) + require.NoError(t, err) + assert.Equal(t, prefix+tc.wantReplaced, string(decoded)) + assert.Truef(t, utf8.Valid(decoded), + "an emitted value must always be valid UTF-8, got %q", decoded) + + // VL keeps escape text verbatim; only ill-formed BYTES are rewritten in the raw value. + if isVerbatim && tc.wantRawKept { + assert.Equal(t, prefix+tc.body, string(raw)) + } + }) + + t.Run("passthrough", func(t *testing.T) { + _, raw, err := drv.run(doc, UTF8Passthrough) + require.NoError(t, err) + // ill-formed bytes reach the caller untouched. Only checked for bodies without escapes: L + // decodes them, so its "raw" is the decoded value and would not match the source text. + if tc.wantStrict == codes.ErrInvalidUTF8 && + !strings.Contains(tc.body, `\`) { + assert.Equal(t, prefix+tc.body, string(raw)) + } + }) + }) + } + } + } +} + +// TestUTF8StrictErrorOffset pins that the reported offset points at the first ill-formed byte, not at the end of the +// value — the reason FirstInvalid exists next to Valid. +func TestUTF8StrictErrorOffset(t *testing.T) { + cases := []struct { + name string + doc string + want uint64 + wantByte byte + }{ + {name: "first byte of the value", doc: "[\"\xffabc\"]", want: 2, wantByte: 0xff}, + // the truncated 3-byte lead is the start of the ill-formed sequence, not the 0xff that reveals it + {name: "after a valid prefix", doc: "[\"ok\xe0\xff\"]", want: 4, wantByte: 0xe0}, + { + name: "in a long value", doc: "[\"" + longPrefix + "\xff\"]", + want: uint64(2 + len(longPrefix)), wantByte: 0xff, + }, + {name: "in a key", doc: "{\"k\xff\":1}", want: 3, wantByte: 0xff}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + lx := NewWithBytes([]byte(tc.doc)) + for range lx.Tokens() { //nolint:revive // draining is the point + } + require.ErrorIs(t, lx.Err(), codes.ErrInvalidUTF8) + require.Equal(t, tc.want, lx.Offset()) + assert.Equal(t, tc.wantByte, tc.doc[lx.Offset()], + "the offset must land on the first byte of the ill-formed sequence") + }) + } +} + +// TestUTF8ReplaceKeepsValidValuesAliased pins the zero-copy invariant: only an offending value is copied out of the +// input, so enabling replacement costs nothing on a well-formed document. +func TestUTF8ReplaceKeepsValidValuesAliased(t *testing.T) { + doc := []byte(`["` + longPrefix + `", "caf\u00e9"]`) + + for _, p := range []UTF8Policy{UTF8Strict, UTF8Replace, UTF8Passthrough} { + lx := NewWithBytes(doc, WithUTF8Policy(p)) + lx.NextToken() // '[' + require.True(t, lx.Ok()) + tok := lx.NextToken() // the long clean string + require.True(t, lx.Ok()) + require.Equal(t, token.String, tok.Kind()) + + v := tok.Value() + require.NotEmpty(t, v) + assert.Samef(t, &doc[2], &v[0], + "a valid value must still alias the caller's buffer under policy %d", p) + } +} + +// TestUTF8MaxValueBytesAfterReplacement pins that the size cap is re-checked against the REWRITTEN length: replacing a +// 1-byte fault with U+FFFD grows the value threefold, so a value that fit before sanitizing may not fit after. +func TestUTF8MaxValueBytesAfterReplacement(t *testing.T) { + doc := []byte("[\"ab\xff\"]") // 3 bytes scanned, 5 bytes once sanitized + + lx := NewWithBytes(doc, WithUTF8Policy(UTF8Replace), WithMaxValueBytes(4)) + for range lx.Tokens() { //nolint:revive // draining is the point + } + require.ErrorIs(t, lx.Err(), codes.ErrMaxValueBytes) + + lx = NewWithBytes(doc, WithUTF8Policy(UTF8Replace), WithMaxValueBytes(8)) + for range lx.Tokens() { //nolint:revive // draining is the point + } + require.NoError(t, lx.Err()) +} + +// TestUTF8SplitSequence forces an ill-formed sequence and a \u escape to straddle a refill boundary at every possible +// split point: the streaming scanners assemble the value across windows, so a fault can be split between the window +// that detected non-ASCII and the one that carries the rest. +func TestUTF8SplitSequence(t *testing.T) { + docs := []struct { + name string + doc string + want error + }{ + { + name: "ill-formed bytes", + doc: "[\"" + longPrefix + "\xe0\xff" + longPrefix + "\"]", + want: codes.ErrInvalidUTF8, + }, + { + name: "broken surrogate pair", + doc: `["` + longPrefix + `\uD800\uD800` + longPrefix + `"]`, + want: codes.ErrSurrogateEscape, + }, + {name: "valid surrogate pair", doc: `["` + longPrefix + `\uD834\uDD1E` + longPrefix + `"]`}, + {name: "valid multibyte", doc: "[\"" + longPrefix + "caf\xc3\xa9" + longPrefix + "\"]"}, + } + + for _, d := range docs { + for chunk := 1; chunk <= 24; chunk++ { + t.Run(fmt.Sprintf("%s/chunk=%d", d.name, chunk), func(t *testing.T) { + for _, verbatim := range []bool{false, true} { + var err error + var decoded []byte + if verbatim { + lx := NewVerbatim(newChunkReader([]byte(d.doc), chunk), + WithBufferSize(32), WithUTF8Policy(UTF8Replace)) + for tok := range lx.Tokens() { + if tok.Kind() == token.String { + decoded = token.Unescape(bytes.Clone(tok.Value())) + } + } + err = lx.Err() + } else { + lx := New(newChunkReader([]byte(d.doc), chunk), + WithBufferSize(32), WithUTF8Policy(UTF8Replace)) + for tok := range lx.Tokens() { + if tok.Kind() == token.String { + decoded = bytes.Clone(tok.Value()) + } + } + err = lx.Err() + } + + require.NoErrorf(t, err, "verbatim=%v: replacement never errors", verbatim) + assert.Truef(t, utf8.Valid(decoded), + "verbatim=%v: emitted value must be valid UTF-8, got %q", verbatim, decoded) + } + + // the same document under the default policy must agree with itself at every chunk size + lx := New(newChunkReader([]byte(d.doc), chunk), WithBufferSize(32)) + for range lx.Tokens() { //nolint:revive // draining is the point + } + if d.want == nil { + require.NoError(t, lx.Err()) + } else { + require.ErrorIs(t, lx.Err(), d.want) + } + }) + } + } +} + +// TestUTF8LexersAgree is the anti-divergence guard: L and VL must accept and reject exactly the same documents under +// every policy, and must decode to the same string. L accepting what VL rejected is precisely the bug class this work +// closed (L laundered a broken surrogate pair into a silent U+FFFD while VL errored). +func TestUTF8LexersAgree(t *testing.T) { + for _, tc := range utf8Cases() { + for _, doc := range []string{`["` + tc.body + `"]`, `{"` + tc.body + `":1}`} { + for _, p := range []UTF8Policy{UTF8Strict, UTF8Replace, UTF8Passthrough} { + t.Run(fmt.Sprintf("%s/%d", tc.name, p), func(t *testing.T) { + got := make([]string, 0, len(utf8Drivers())) + for _, drv := range utf8Drivers() { + decoded, _, err := drv.run([]byte(doc), p) + got = append( + got, + fmt.Sprintf("%s: err=%v decoded=%q", drv.name, err != nil, decoded), + ) + } + for _, line := range got[1:] { + assert.Equal(t, + strings.SplitN(got[0], ": ", 2)[1], + strings.SplitN(line, ": ", 2)[1], + "lexer drivers disagree on %q:\n %s", doc, strings.Join(got, "\n ")) + } + }) + } + } + } +} + +// TestUTF8ReplaceMangledValueContract pins what mangling does and does not preserve, because the two halves are easy +// to conflate: a rewritten VALUE no longer maps onto its source span (U+FFFD is 3 bytes replacing 1), while token +// POSITIONS stay exact because they come from the scan cursor rather than the value. +func TestUTF8ReplaceMangledValueContract(t *testing.T) { + // one ill-formed byte, then a truncated 3-byte sequence: 1 + 3 faults => 4 replacements, 12 bytes for 4 source + const body = "a\xffb\xe0\xa0\xffc" + doc := []byte(" [\"" + body + "\", 7]") + + for _, verbatim := range []bool{false, true} { + t.Run(fmt.Sprintf("verbatim=%v", verbatim), func(t *testing.T) { + var ( + value []byte + tokenOffset uint64 + line, column int + ) + if verbatim { + lx := NewVerbatimWithBytes(doc, WithUTF8Policy(UTF8Replace)) + require.Equal(t, token.Delimiter, lx.NextToken().Kind()) // '[' + tok := lx.NextToken() + require.True(t, lx.Ok()) + require.Equal(t, token.String, tok.Kind()) + value, line, column = bytes.Clone(tok.Value()), lx.Line(), lx.Column() + tokenOffset = lx.Offset() + } else { + lx := NewWithBytes(doc, WithUTF8Policy(UTF8Replace)) + require.Equal(t, token.Delimiter, lx.NextToken().Kind()) // '[' + tok := lx.NextToken() + require.True(t, lx.Ok()) + require.Equal(t, token.String, tok.Kind()) + value = bytes.Clone(tok.Value()) + tokenOffset = lx.Offset() + } + + // the value is well-formed, and LONGER than the text it came from + assert.True(t, utf8.Valid(value)) + assert.Equal(t, "a�b���c", string(value)) + assert.Greaterf(t, len(value), len(body), + "U+FFFD is 3 bytes replacing 1: a mangled value cannot have its source width") + + // positions are still exact: the cursor sits just past the closing quote of the string + wantOffset := uint64(len(" [\"") + len(body) + 1) + assert.Equalf(t, wantOffset, tokenOffset, + "the scan cursor must track the SOURCE, not the rewritten value") + if verbatim { + assert.Equal(t, 1, line) + assert.Equal(t, len(" [")+1, column) + } + }) + } +} + +// TestUTF8BOM pins the leading-BOM rules: the complete mark at offset 0 is consumed, and nothing else is. +func TestUTF8BOM(t *testing.T) { + const bom = "\uFEFF" + + t.Run("consumed at offset 0", func(t *testing.T) { + for _, doc := range []string{bom + "{}", bom + `{"a":1}`, bom + " [1,2]", bom + "null"} { + lx := NewWithBytes([]byte(doc)) + for range lx.Tokens() { //nolint:revive // draining is the point + } + require.NoErrorf(t, lx.Err(), "input %q", doc) + + // streaming, with the mark split across the very first reads + for chunk := 1; chunk <= 4; chunk++ { + sl := New(newChunkReader([]byte(doc), chunk), WithBufferSize(32)) + for range sl.Tokens() { //nolint:revive // draining is the point + } + assert.NoErrorf(t, sl.Err(), "input %q, chunk=%d", doc, chunk) + } + } + }) + + t.Run("offsets account for the consumed mark", func(t *testing.T) { + vl := NewVerbatimWithBytes([]byte(bom + "{}")) + tok := vl.NextToken() + require.True(t, vl.Ok()) + assert.Equal(t, token.Delimiter, tok.Kind()) + // the '{' is the fourth byte of the document + assert.Equal(t, uint64(len(bom)+1), vl.Offset()) + }) + + t.Run("not re-emitted by the verbatim lexer", func(t *testing.T) { + // the documented exception to VL's byte-exact round-trip: the mark belongs to no token's leading space + vl := NewVerbatimWithBytes([]byte(bom + " {}")) + tok := vl.NextToken() + require.True(t, vl.Ok()) + require.Equal(t, token.Delimiter, tok.Kind()) + assert.Equal(t, " ", string(vl.LeadingSpace())) + }) + + t.Run("only the complete mark, only at offset 0", func(t *testing.T) { + for _, tc := range []struct { + name string + doc string + }{ + {name: "truncated mark", doc: "\xef\xbb{}"}, + {name: "mark alone", doc: bom}, + {name: "mark after a value", doc: "{}" + bom}, + {name: "mark inside an array", doc: "[" + bom + "1]"}, + {name: "doubled mark", doc: bom + bom + "{}"}, + } { + t.Run(tc.name, func(t *testing.T) { + lx := NewWithBytes([]byte(tc.doc)) + for range lx.Tokens() { //nolint:revive // draining is the point + } + assert.Error(t, lx.Err()) + }) + } + }) + + t.Run("a mark inside a string is ordinary content", func(t *testing.T) { + lx := NewWithBytes([]byte(`["` + bom + `"]`)) + var got []byte + for tok := range lx.Tokens() { + if tok.Kind() == token.String { + got = bytes.Clone(tok.Value()) + } + } + require.NoError(t, lx.Err()) + assert.Equal(t, bom, string(got)) + }) +} + +// TestUTF16BOMDiagnostic pins the accurate error for a UTF-16 document, which would otherwise fail as a generic +// invalid token on its first byte. +func TestUTF16BOMDiagnostic(t *testing.T) { + for _, tc := range []struct { + name string + doc string + }{ + {name: "LE", doc: "\xff\xfe[\x00\"\x00\xe9\x00\"\x00]\x00"}, + {name: "BE", doc: "\xfe\xff\x00[\x00\"\x00\xe9\x00\"\x00]"}, + } { + t.Run(tc.name, func(t *testing.T) { + lx := NewWithBytes([]byte(tc.doc)) + for range lx.Tokens() { //nolint:revive // draining is the point + } + assert.ErrorIs(t, lx.Err(), codes.ErrNotUTF8) + + vl := NewVerbatim(newChunkReader([]byte(tc.doc), 3), WithBufferSize(32)) + for range vl.Tokens() { //nolint:revive // draining is the point + } + assert.ErrorIs(t, vl.Err(), codes.ErrNotUTF8) + }) + } +} diff --git a/json/lexers/default-lexer/verbatim_lexer.go b/json/lexers/default-lexer/verbatim_lexer.go index 3e9030e..f8dda08 100644 --- a/json/lexers/default-lexer/verbatim_lexer.go +++ b/json/lexers/default-lexer/verbatim_lexer.go @@ -16,6 +16,40 @@ var _ lexers.Lexer = &VL{} // Whenever consuming from an io.Reader, the stream is buffered so it is not useful to use a buffered reader. // // The lexer performs JSON grammar check such as opening/closing brackets, expected sequence of delimiter/values etc. +// +// # The verbatim pledge +// +// VL reproduces its input byte-for-byte: the token stream plus [VL.LeadingSpace] reassembles the source exactly, and +// every token value is a slice of the source with its escapes intact. Under the default [UTF8Strict] policy this is +// unconditional, because input that is not valid UTF-8 is rejected rather than emitted. +// +// Two documented exceptions: +// +// - A leading UTF-8 BOM is consumed before any token exists, so it belongs to no token's leading space and is not +// re-emitted. RFC 8259 §8.1 asks implementations not to emit a BOM, so this is the conformant direction. +// - Under [UTF8Replace], an ill-formed byte sequence inside a string value is rewritten (see "Mangling" below). +// +// # Mangling under UTF8Replace, and what it costs +// +// [UTF8Replace] substitutes U+FFFD for each ill-formed byte in a string value. Only the offending value is affected — +// every other token still aliases the source — but for that value the byte-for-byte pledge is off, and so is the +// ability to map value bytes back onto source bytes: +// +// - U+FFFD encodes as THREE bytes and replaces ONE. A value carrying k ill-formed bytes is 2k bytes longer than its +// source span, so len(token.Value()) no longer equals the width of the text it came from. Multi-byte faults +// compound this: a truncated three-byte sequence is three separate ill-formed bytes, hence three replacements and +// nine bytes where the source had three. +// - Consequently an index INTO a mangled value cannot be turned into a source offset by adding it to the token's +// start, and the source text of such a value cannot be recovered from the token. If you need the original bytes, +// use [UTF8Strict] (and handle the error) or [UTF8Passthrough] (and validate downstream yourself). +// +// What does NOT change: [VL.Line], [VL.Column], [L.Offset] and [VL.LeadingSpace] stay exact under every policy. They +// are derived from the scan cursor walking the source, never from the value, so token POSITIONS remain trustworthy +// even when a token's CONTENT has been rewritten. A linter or formatter can still point at the right place; it just +// cannot assume the value it holds is the text that was there. +// +// Escape text is never rewritten: VL keeps `\uXXXX` as source text, so a broken surrogate pair stays verbatim in the +// value and its U+FFFD appears only when the value is decoded with [token.Unescape]. type VL struct { *L } diff --git a/json/lexers/error-codes/errors.go b/json/lexers/error-codes/errors.go index 07c3a5b..769dded 100644 --- a/json/lexers/error-codes/errors.go +++ b/json/lexers/error-codes/errors.go @@ -28,6 +28,8 @@ const ( ErrUnicodeEscape LexerError = "invalid unicode escape sequence" ErrUnknownEscape LexerError = "unknown escape sequence" ErrInvalidRune LexerError = "invalid rune in unicode escape sequence" + ErrInvalidUTF8 LexerError = "invalid UTF-8 sequence in string" + ErrNotUTF8 LexerError = "input is not UTF-8 encoded (UTF-16 byte order mark found)" ErrSurrogateEscape LexerError = "expected an escaped UTF-16 code point to come as a surrogate pair" ErrDelimitedValue LexerError = "value should follow a delimiter" ErrCommaInContainer LexerError = "comma should appear within an object or an array" From 7303301641bc209902aa78b2849492e4980ceead Mon Sep 17 00:00:00 2001 From: Frederic BIDON Date: Thu, 30 Jul 2026 18:26:34 +0200 Subject: [PATCH 04/10] docs(json-lexers): correct the UTF-8 BOM and cost statements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two statements in the default-lexer docs were written while BOM trimming was still deferred and never updated when it landed: - "A leading UTF-8 BOM is currently rejected as an invalid token rather than skipped" (README.md and doc.go) — it is now consumed before the first token. Both places now also say what follows from that: the mark belongs to no token's leading space, so VL does not reproduce it, which RFC 8259 §8.1 asks for anyway. - The measured cost still quoted round 1's ~10% on twitter_status. With the AVX2 validator that is ~6% (L) / ~2% (VL). Also: - DESIGN.md §10.2 documented a single avo kernel; there are now two. Points at json/internal/utf8x, its separate CPUID detection, and its differential suite — a hand-written validator that wrongly ACCEPTS fails silently, so that suite is the safety net. - The plan's §9 conformance table still claimed the BOM would be carried in VL.LeadingSpace(); it deliberately is not (see the round-2 outcome). Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Frederic BIDON --- .claude/plans/utf8-validation.md | 2 +- json/lexers/default-lexer/DESIGN.md | 8 ++++++++ json/lexers/default-lexer/README.md | 14 ++++++++------ json/lexers/default-lexer/doc.go | 9 +++++---- 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/.claude/plans/utf8-validation.md b/.claude/plans/utf8-validation.md index be36cd4..7772495 100644 --- a/.claude/plans/utf8-validation.md +++ b/.claude/plans/utf8-validation.md @@ -664,7 +664,7 @@ with current verdicts and the expected post-change verdict: | **broken surrogate escapes, L≠VL** | `1st_valid_surrogate_2nd_invalid`, `incomplete_surrogates_escape_valid`, `inverted_surrogates_U+1D11E` | L accept / VL reject | **reject** (both) | §2.3 — real divergence | | **broken surrogate escapes, agreed** | `1st_surrogate_but_2nd_missing`, `incomplete_surrogate_pair`, `incomplete_surrogate_and_escape_valid`, `invalid_lonely_surrogate`, `invalid_surrogate`, `lone_second_surrogate`, `object_key_lone_2nd_surrogate` | reject | reject | already strict | | **UTF-16 input** | `utf16BE_no_BOM`, `utf16LE_no_BOM`, `UTF-16LE_with_BOM` | reject | reject | documented: UTF-8 only | -| **BOM** | `structure_UTF-8_BOM_empty_object` | reject | **accept** | §2.8 — leading BOM is now trimmed (and carried in `VL.LeadingSpace()`) | +| **BOM** | `structure_UTF-8_BOM_empty_object` | reject | **accept** | §2.8 — leading BOM is now consumed; NOT carried in `VL.LeadingSpace()`, see the round-2 outcome | | **number magnitude** | 10 `i_number_*` (huge exp, overflow, underflow) | accept | accept | correct by design — we don't evaluate numbers | | **depth** | `structure_500_nested_arrays` | accept | accept | correct since the stack rewrite | diff --git a/json/lexers/default-lexer/DESIGN.md b/json/lexers/default-lexer/DESIGN.md index 82cc70d..803fe8e 100644 --- a/json/lexers/default-lexer/DESIGN.md +++ b/json/lexers/default-lexer/DESIGN.md @@ -426,6 +426,14 @@ The kernel is `stringstop_amd64.s`, **generated by `avo`**, and its generator li 5. The `_asm` sub-module has its own `go.mod`/`go.sum`; update them there, not in the parent, when bumping `avo`. +There is a **second** `avo` kernel outside this package: the UTF-8 validator in +`json/internal/utf8x` (`validate_amd64.s`, generator in `json/internal/utf8x/_asm`), a port +of simdutf's `lookup4` algorithm. It follows the same rules, and carries its own CPUID +detection because it is a different module path. Its differential suite +(`validate_amd64_test.go`) is exhaustive over all byte pairs and over every lead byte against +all 2nd/3rd bytes — a hand-written validator that wrongly ACCEPTS is silent, so that suite is +the safety net and must stay green (it is mutation-tested: corrupting one table byte fails it). + ### 10.3 Adding an option 1. Add the field to `options` (`options.go`) and a `With…` constructor following the diff --git a/json/lexers/default-lexer/README.md b/json/lexers/default-lexer/README.md index 7424f36..e1f5c27 100644 --- a/json/lexers/default-lexer/README.md +++ b/json/lexers/default-lexer/README.md @@ -158,16 +158,18 @@ walking the source, never from the value, so they stay exact under every policy. at the right place — it just cannot assume the value it holds is the text that was there. Callers who need the source bytes should use `UTF8Strict` (and handle the error) or `UTF8Passthrough` (and validate downstream themselves). -A leading UTF-8 BOM is consumed before any token exists, so it is not re-emitted either — the other documented -exception to byte-exact round-tripping (RFC 8259 §8.1 asks implementations not to emit one). +A leading UTF-8 BOM is the other documented exception to byte-exact round-tripping, for a different reason: it is +consumed before any token exists, so it belongs to no token's leading space and is not re-emitted. **Cost.** Detection is fused into the string scan already being performed — every SWAR word and every AVX2 block is OR-accumulated, so "did this value contain a byte >= 0x80" is answered for free — and only values that actually carry -one reach the validator. On the reference corpus, `UTF8Strict` versus no validation is statistically indistinguishable -on the four ASCII-dominated workloads and costs ~10% on `twitter_status`, which is 30% non-ASCII by string bytes. +one reach the validator — where an AVX2 port of simdutf's `lookup4` algorithm handles them at 5-12x the stdlib's rate. +On the reference corpus, `UTF8Strict` versus no validation is statistically indistinguishable on the four +ASCII-dominated workloads and costs ~6% (`L`) / ~2% (`VL`) on `twitter_status`, which is 30% non-ASCII by string bytes. -The input must be UTF-8: a UTF-16 document is rejected with `ErrNotUTF8`. A leading UTF-8 BOM is currently rejected as -an invalid token rather than skipped. +The input must be UTF-8: a UTF-16 document is rejected with `ErrNotUTF8`. A leading UTF-8 BOM is accepted and +consumed before the first token — RFC 8259 §8.1 permits ignoring one on input and asks implementations not to emit +one, so it is not reproduced on output (see above). ## Conformance tests diff --git a/json/lexers/default-lexer/doc.go b/json/lexers/default-lexer/doc.go index 5750546..0709689 100644 --- a/json/lexers/default-lexer/doc.go +++ b/json/lexers/default-lexer/doc.go @@ -55,9 +55,10 @@ // // Validation is not a second pass over the value: detection of non-ASCII bytes is fused into the string scan the // lexer already performs, so a pure-ASCII value — the overwhelmingly common case — is proven valid with no extra read -// and no call, and only values that actually carry a byte >= 0x80 reach the validator. Measured on the reference -// corpus, enabling it is free on ASCII-dominated documents and costs ~10% on heavily non-ASCII ones (twitter_status). +// and no call, and only values that actually carry a byte >= 0x80 reach the validator, where an AVX2 kernel handles +// them. Measured on the reference corpus, enabling it is free on ASCII-dominated documents and costs ~6% (L) / ~2% +// (VL) on heavily non-ASCII ones (twitter_status). // -// The input must be UTF-8: a UTF-16 document is rejected with [codes.ErrNotUTF8]. A leading UTF-8 BOM is currently -// rejected as an invalid token rather than skipped. +// The input must be UTF-8: a UTF-16 document is rejected with [codes.ErrNotUTF8]. A leading UTF-8 BOM is consumed +// before the first token and is not re-emitted by [VL] (RFC 8259 §8.1 asks implementations not to emit one). package lexer From b975f8797ce9732972c21a2c07e4d98e2bfdf421 Mon Sep 17 00:00:00 2001 From: Frederic BIDON Date: Thu, 30 Jul 2026 19:10:51 +0200 Subject: [PATCH 05/10] doc: readme, license notice & acknowledgements Signed-off-by: Frederic BIDON --- NOTICE | 44 +++++++++++++++++++++++++++++ TODO.md | 32 --------------------- json/lexers/README.md | 23 +++++++++++---- json/lexers/default-lexer/README.md | 6 +++- 4 files changed, 67 insertions(+), 38 deletions(-) create mode 100644 NOTICE delete mode 100644 TODO.md diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..8b92632 --- /dev/null +++ b/NOTICE @@ -0,0 +1,44 @@ +Copyright 2015-2025 go-swagger maintainers + +// SPDX-FileCopyrightText: Copyright 2015-2025 go-swagger maintainers +// SPDX-License-Identifier: Apache-2.0 + +This software library, github.com/go-openapi/core, includes software developed +by the go-swagger and go-openapi maintainers ("go-swagger maintainers"). + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this software except in compliance with the License. + +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0. + +It ships with copies of other software which license terms are recalled below. + +https://github.com/nst/JSONTestSuite +=========================== + +// SPDX-FileCopyrightText: Copyright (c) 2016 Nicolas Seriot +// SPDX-License-Identifier: MIT + +MIT License + +Copyright (c) 2016 Nicolas Seriot + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/TODO.md b/TODO.md deleted file mode 100644 index e3b8fb4..0000000 --- a/TODO.md +++ /dev/null @@ -1,32 +0,0 @@ -# UTF-8 validation - -Working with the prototype to introduce UTF-8 validation in the lexer. - -1. We need the JSON lexer to be able to valide UTF-8 sequences: - invalid UTF-8 should not leak in output strings. This should be true for L as well as for VL. - -2. The default behavior of L and VL is to error on such invalid input - -3. As an option (to both lexers), the error can be swallowed and the invalid UTF-8 sequence - be in this case mangled as U+FFFD (error rune) - -4. For VL, we pledge _verbatim_. This is important to maintain original offset/positioning. - But this pledge is broken when mangling an invalid sequence. REMAINS TO BE DECIDED. - -5. Prototype introduces options that look a bit complicated to me (fused, ...): either we want error or we mute error. - When the error is muted, invalid utf8 sequences are mangled. - -Classic algorithm for invalid sequence detection: https://nemanjatrifunovic.substack.com/p/decoding-utf-8-part-vii-validation. - -Advanced SIMD-enabled C++ library for validating UTF-8: https://github.com/simdutf/simdutf -(available locally for convenient inspection at /home/fred/src/github.com/fredbi/simdutf). -I am worried that this new requirement wipes out our vast performance improvements with string scanning. - -I think we should analyze the methods used at simdutf (e.g. function simdtuf::validate_utf8_with_errors). -We don't have to port their full support of AVX512 etc, just understand their general approach and -port it to our more modest SWAR & AV2 support. - - -6. Final point: recheck the conformance suite test and list all "implementation-dependent point" - where L or VL don't pass the test, so we can get a more complete assessment of how many similar bugs - remain hidden in our cupboard. diff --git a/json/lexers/README.md b/json/lexers/README.md index 39b9449..4dfd409 100644 --- a/json/lexers/README.md +++ b/json/lexers/README.md @@ -2,12 +2,25 @@ This package exposes an interface `Lexer` and a token type `token.T` to parse JSON input. -Notice that the same interface may be used to parse `YAML`, or any other serialization format +Notice that the same interface may be used to parse `YAML` or any other serialization format that converts to JSON (e.g. a hierarchy of scalars, arrays and dictionaries). Several lexers implementations are proposed: -* a semantic JSON lexer -* a verbatim JSON lexer -* a ND-JSON lexer -* a YAML lexer +* [x] a semantic JSON lexer +* [x] a verbatim JSON lexer, that preserves blank space and unicode escaped sequences +* [x] a YAML lexer +* [ ] a ND-JSON lexer (TODO) + +Acknowlegements and credits +=========================== + +Our JSON lexer is an original research and development: it is not a copy of another software. + +However, we are greatly grateful to the following projects that have provided inspiration, +astute methods and algorithms that we could study, reproduce or try to improve further. + +* https://github.com/mailru/easyjson (MIT) - The first pionner for a "fast" low-level JSON encoder/decoder +* https://github.com/go-json-experiment/json (BSD 3-Clause) - go standard library encoding/json/v2 +* https://github.com/simdutf/simdutf (MIT+Apache) - extremely fast C++ UTF-8 validator +* https://github.com/minio/simdjson-go (Apache) - pionnered go & assembly with a go port of simdjson diff --git a/json/lexers/default-lexer/README.md b/json/lexers/default-lexer/README.md index e1f5c27..132237f 100644 --- a/json/lexers/default-lexer/README.md +++ b/json/lexers/default-lexer/README.md @@ -52,6 +52,7 @@ Since we want it to be flexible, there are a few available options: (interned string keys, raw integer array indices). Opt-in and off by default — enabling it forfeits the zero-allocation guarantee (the path stack and retained keys allocate). * tunable context window for reporting errors + * strict/lax UTF8 validation Additional objectives: @@ -101,6 +102,8 @@ Differences with `encoding/json/v2` | pull iterator | ✅ || ⏸️ | | limit stack | ✅ || ✅ | | limit tok size | ✅ || ❌ | +| valid UTF8 | ✅ || ✅ | +| unchecked UTF8 | ⬜ || ❌ | Trade-offs when comparing to `github.com/go-json-experiment/json/jsontext` (stdlib `json/v2`). @@ -217,6 +220,7 @@ to avoid. PGO is the supported lever. ## Roadmap +* ND-JSON lexer * AVX2 support is currently provided as assembly kernels for amd64 only * AVX512 is likely overkill for our usage and I don't have the hardware to test it thoroughly -* AVX support this will be eventually replaced by go native support for AV2 & AVX512 (currently experimental) +* AVX support will be eventually replaced by go native support for AVX2 & AVX512 (currently experimental) From 16224b8cd4f98eb8ed092cf5ed5943cef9966fcf Mon Sep 17 00:00:00 2001 From: Frederic BIDON Date: Thu, 30 Jul 2026 20:25:15 +0200 Subject: [PATCH 06/10] fix(json-writers): validate UTF-8 on the way out MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The writers had the mirror image of the lexer bug, plus an inconsistency of their own. Probed, not assumed: input to String* before Raw before invalid byte mid-string silently -> U+FFFD passed through overlong / surrogate silently -> U+FFFD passed through truncated rune at end ERRORED passed through So a trailing incomplete rune errored while every other ill-formed sequence was silently rewritten, and there was no way to ask for an error at all. RFC 8259 §8.1 requires JSON text to be UTF-8, so silently corrupting a caller's payload is the worse half of that. API --- WithUTF8Policy(UTF8Strict|UTF8Replace|UTF8Passthrough), defaulting to Strict. It is the SAME type the lexers use: UTF8Policy moved to json/internal/utf8x and both packages alias it, so a document refused on the way out and one rejected on the way in mean the same thing. Unbuffered takes WithUnbufferedUTF8Policy -- two names only because Go options are typed per writer; Indented and YAML get it through their existing buffered-option wrappers. UTF8Replace is exactly the old behavior, now explicit and complete: it is the policy under which the output is always valid UTF-8. Note Go strings may legally hold ill-formed UTF-8, so String()/StringBytes() can trip the new default on data from a non-Unicode source; that is the intended change. What is checked --------------- Validated: String, StringBytes, StringRunes, StringCopy, Raw, RawCopy, and VerbatimValue (which renders through the ordinary string path). Trusted: Token and VerbatimToken. The lexers already validate string values as they scan, so re-checking them would cost the token-copy path its reason for existing. The loophole is deliberate and documented: a document lexed with UTF8Passthrough can carry ill-formed bytes in, and then Token silently substitutes U+FFFD (the escaper decodes as it goes, so the output stays valid while differing from the input with no error) and VerbatimToken emits them byte-for-byte, as verbatim requires. Unchecked: NumberBytes/NumberCopy, which already document themselves so; a number holding non-ASCII is a grammar problem, not a UTF-8 one. Two bugs found by the new tests, both fixed ------------------------------------------- 1. The streaming paths validated each read independently, so a lead byte at the end of one chunk followed by a non-continuation at the start of the next was tolerated by both halves and emitted. RawCopy now holds the incomplete tail back and re-joins it with the next read -- reads land at an offset in the buffer that leaves room for the carry, so no second buffer and no allocation -- and StringCopy now validates the sequence its stitcher assembles, which had never passed through the chunk check. 2. Under UTF8Replace a truncated tail still errored instead of substituting, on every escaped path. The remainder is now policy-driven like everything else. Cost ---- Escaped paths are within noise: the escaper already decodes every multi-byte rune, so validation adds a pass the AVX2 kernel makes negligible (ascii 1767 vs 1841 MB/s, cjk 279 vs 291). Raw pays 5.9 vs 8.8 GB/s (-33%), being otherwise a memcpy -- the honest price of Strict meaning what it says; callers who have already validated can select UTF8Passthrough. The token-driven path is untouched by construction. Tests ----- Three policies x five writer configurations (including a minimum-size buffer that forces flushes mid-escape) x every entry point; split-sequence sweeps at read sizes 1..8 for both streaming paths; the trust loophole pinned explicitly; StringRunes rejecting surrogates and out-of-range values. And TestWriterSubstitutionMatchesLexer cross-checks the writers' substitution against utf8x.Sanitize, so "the lexer and the writer share one rule" is enforced rather than asserted -- the two escapers still carry their escaping logic twice (they differ in how they emit), so the rule is what is shared, not the code. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Frederic BIDON --- .claude/plans/utf8-validation.md | 60 ++- json/internal/utf8x/utf8x.go | 28 ++ .../default-lexer/internal/input/input.go | 35 +- json/writers/default-writer/README.md | 31 ++ json/writers/default-writer/base_writer.go | 185 ++++++-- json/writers/default-writer/buffered.go | 231 ++++++++-- .../default-writer/buffered_options.go | 20 + json/writers/default-writer/doc.go | 34 ++ json/writers/default-writer/errors.go | 5 + json/writers/default-writer/escape.go | 17 +- .../default-writer/escape_regression_test.go | 2 +- json/writers/default-writer/unbuffered.go | 231 ++++++++-- .../default-writer/unbuffered_options.go | 32 +- json/writers/default-writer/utf8.go | 129 ++++++ json/writers/default-writer/utf8_test.go | 402 ++++++++++++++++++ 15 files changed, 1318 insertions(+), 124 deletions(-) create mode 100644 json/writers/default-writer/utf8.go create mode 100644 json/writers/default-writer/utf8_test.go diff --git a/.claude/plans/utf8-validation.md b/.claude/plans/utf8-validation.md index 7772495..826707a 100644 --- a/.claude/plans/utf8-validation.md +++ b/.claude/plans/utf8-validation.md @@ -7,6 +7,58 @@ > that hole). > Status legend: ✅ done · ⏳ in progress · ⬜ todo · 🔬 measure first. +## OUTCOME — round 3 shipped (2026-07-30): the writers + +The writers had the mirror-image bug and one inconsistency of their own. Probed, not assumed: + +| input to `String*` | before | `Raw` before | +|---|---|---| +| invalid byte mid-string | silently → U+FFFD | passed through | +| overlong / encoded surrogate | silently → 2–3× U+FFFD | passed through | +| **truncated rune at end** | **errored** | passed through | + +So a trailing incomplete rune errored while every other fault was silently rewritten. Now one knob decides, and it is +the *same type* the lexers use: `UTF8Policy` moved to `json/internal/utf8x` and both packages alias it, so a document +refused on the way out and one rejected on the way in mean the same thing. + +**Default is `UTF8Strict`** (Fred's call). `WithUTF8Policy` is a `BufferedOption` (so `Indented`/`YAML` get it through +their existing buffered-option wrappers); `Unbuffered` takes `WithUnbufferedUTF8Policy` — two names only because Go +options are typed per writer. + +**What is checked** (Fred: *"not needed to check again what comes from token, only from raw []byte and io.Reader"*): + +- **Validated** — `String`, `StringBytes`, `StringRunes`, `StringCopy`, `Raw`, `RawCopy`, and `VerbatimValue` (which + renders through the ordinary string path). +- **Trusted** — `Token` and `VerbatimToken`. Documented loophole: a document lexed with `UTF8Passthrough` can carry + ill-formed bytes in, and then `Token` *silently substitutes* U+FFFD (the escaper decodes as it goes, so output stays + valid but differs with no error) while `VerbatimToken` emits the bytes byte-for-byte, as verbatim requires. +- **Unchecked** — `NumberBytes`/`NumberCopy`, which already document themselves as unchecked; non-ASCII in a number is + a grammar problem, not a UTF-8 one. + +**Two real bugs the new tests found, both fixed:** + +1. `RawCopy` and `StringCopy` validated each chunk independently, so a lead byte at the end of one read followed by a + non-continuation at the start of the next was tolerated by both halves and emitted. `RawCopy` now holds the + incomplete tail back and re-joins it with the next read (reads land at an offset in the buffer that leaves room, so + no second buffer and no allocation); `StringCopy`'s stitched sequence is now validated too — it was assembled by + the stitcher and had never passed through the chunk check. +2. Under `UTF8Replace` a truncated tail still *errored* instead of substituting, on every escaped path. The remainder + is now policy-driven like everything else. + +**Cost** (`BenchmarkUTF8PolicyCallerPath`, strict vs passthrough): + +- escaped paths (`StringBytes`): within noise — the escaper dominates and the AVX2 validator is a rounding error + (ascii 1767 vs 1841 MB/s, cjk 279 vs 291); +- `Raw`: **5.9 vs 8.8 GB/s (−33%)**. Raw is otherwise a memcpy, so a full validation pass is proportionally large. + That is the honest price of "Strict means the output really is UTF-8"; callers who had already validated can say + `UTF8Passthrough` and get the old speed back; +- the token-driven path (what `BenchmarkWriters` measures) is untouched by construction. + +**Not done:** deduplicating the two escapers (`escape.go:escapedBytes` and `buffered.writeEscaped` still carry the +same logic twice). Plan §10 item 3.1 hoped `utf8x` would unify them, but they differ in how they emit (append to a +buffer vs write-with-flush), so the honest sharing is the *rule*, not the code — pinned by +`TestWriterSubstitutionMatchesLexer`, which cross-checks writer output against `utf8x.Sanitize`. + ## OUTCOME — round 2 shipped (2026-07-30) The simdutf/Keiser–Lemire `lookup4` validator is ported to avo AVX2 (`json/internal/utf8x/_asm` → @@ -724,7 +776,7 @@ thing that would catch the *next* one. before trusting hand-written vector validation. - ✅ 2.4 Re-measure. -**Round 3 — writers (own commit, after round 1 lands `utf8x`)** +**Round 3 — ✅ shipped; default is UTF8Strict** The writers have the mirror-image bug, already half-handled: `writers/default-writer/escape.go:escapedBytes` silently substitutes U+FFFD for @@ -733,13 +785,13 @@ invalid input bytes ("invalid runes are represented as �"), i.e. it hard-codes UTF-8, and silently mangling a caller's payload without telling them is the same class of complaint as the lexer's silent acceptance. -- ⬜ 3.1 Re-express `escapedBytes`' default branch on `utf8x` so lexer and writer share +- ⏸️ 3.1 (NOT done — the two escapers differ in how they emit; the *rule* is shared and pinned by a test instead, see the round-3 outcome) Re-express `escapedBytes`' default branch on `utf8x` so lexer and writer share one rule (§2.4) and one implementation. -- ⬜ 3.2 Give the writers the same `UTF8Policy` knob. Open: what default? Symmetry with +- ✅ 3.2 Give the writers the same `UTF8Policy` knob. **Default: Strict.** Open: what default? Symmetry with the lexer says Strict; today's behavior is Replace. **Strict** is the recommendation — a writer that silently corrupts is worse than one that refuses — but it is a louder breaking change than the lexer's, so it wants its own decision. -- ⬜ 3.3 The `\u` escape emission path needs the same audit (surrogate pairs on output). +- ✅ 3.3 The `\u` escape emission path needs the same audit (surrogate pairs on output). **Follow-ups (out of scope, filed)** diff --git a/json/internal/utf8x/utf8x.go b/json/internal/utf8x/utf8x.go index 0a63fdd..3c3f61e 100644 --- a/json/internal/utf8x/utf8x.go +++ b/json/internal/utf8x/utf8x.go @@ -115,3 +115,31 @@ func SanitizedLen(src []byte) int { return n } + +// Policy selects what a lexer or writer does with data that cannot be represented as a sequence of Unicode scalar +// values: an ill-formed UTF-8 byte sequence, or a \u escape that does not denote one. +// +// It lives here, alongside the validator, so the lexers and the writers share one definition and cannot drift apart — +// a document rejected on the way in and one refused on the way out must mean the same thing. Both packages alias it +// (lexer.UTF8Policy, writer.UTF8Policy), so callers never name this internal package. +// +// The zero value is [PolicyStrict]: validation is on unless a caller opts out. +type Policy uint8 + +const ( + // PolicyStrict rejects the data. This is the default on both sides. + PolicyStrict Policy = iota + + // PolicyReplace substitutes U+FFFD — one per invalid byte, one per broken escape — so what is emitted is always + // valid UTF-8. See [Sanitize] for the exact granularity. + PolicyReplace + + // PolicyPassthrough skips raw-byte validation entirely, letting ill-formed bytes through untouched. + // + // UNSAFE: only for data already known to be valid UTF-8. A \u escape still yields U+FFFD when broken, because an + // escape must always produce some rune. + PolicyPassthrough +) + +// Validates reports whether the policy inspects raw bytes at all. +func (p Policy) Validates() bool { return p != PolicyPassthrough } diff --git a/json/lexers/default-lexer/internal/input/input.go b/json/lexers/default-lexer/internal/input/input.go index e8a2a45..90e1e8c 100644 --- a/json/lexers/default-lexer/internal/input/input.go +++ b/json/lexers/default-lexer/internal/input/input.go @@ -1,6 +1,10 @@ package input -import "io" +import ( + "io" + + "github.com/go-openapi/core/json/internal/utf8x" +) // Input holds the lexer's buffered-input and scan-cursor state: the buffer window and read position, the // value-accumulation scratch, the reader and refill buffers, the error state, and the two grammar flags the string @@ -53,28 +57,13 @@ type Input struct { sanitized []byte } -// UTF8Policy selects what a lexer does with input that cannot be represented as a Unicode scalar value: an ill-formed -// UTF-8 byte sequence in a string body, or a \u escape that does not form one. -// -// The zero value is [UTF8Strict], so validation is on unless a caller opts out. -type UTF8Policy uint8 +// UTF8Policy is [utf8x.Policy]: the lexers and the writers share one definition so they cannot drift apart. The +// lexer package aliases this in turn, so callers name neither internal package. +type UTF8Policy = utf8x.Policy +// The policy values, re-exported so the scanners read UTF8Strict rather than utf8x.PolicyStrict. const ( - // UTF8Strict rejects the document: the lexer errors with codes.ErrInvalidUTF8 (ill-formed bytes) or - // codes.ErrSurrogateEscape (a broken \u surrogate pair). This is the default. - UTF8Strict UTF8Policy = iota - - // UTF8Replace accepts the document and substitutes U+FFFD — one per invalid byte, one per broken escape — so an - // emitted value is always valid UTF-8. Only an offending value is rewritten (and therefore copied); a valid value is - // still aliased zero-copy. - UTF8Replace - - // UTF8Passthrough skips raw-byte validation entirely: ill-formed bytes reach the caller untouched. A \u escape still - // decodes to U+FFFD when broken, because an escape must always produce some rune. - // - // UNSAFE: only for input already known to be valid UTF-8. - UTF8Passthrough + UTF8Strict = utf8x.PolicyStrict + UTF8Replace = utf8x.PolicyReplace + UTF8Passthrough = utf8x.PolicyPassthrough ) - -// Validates reports whether the policy inspects raw string bytes at all. -func (p UTF8Policy) Validates() bool { return p != UTF8Passthrough } diff --git a/json/writers/default-writer/README.md b/json/writers/default-writer/README.md index b28093a..3f4664c 100644 --- a/json/writers/default-writer/README.md +++ b/json/writers/default-writer/README.md @@ -16,6 +16,37 @@ TODOs: * [] YAML output is not great, because at this moment, strings remain JSON strings, without accounting for YAML escaping rules. +## UTF-8 + +RFC 8259 §8.1 requires JSON text to be UTF-8. `WithUTF8Policy` selects what happens to caller-supplied data that +is not: + +| policy | behavior | +|---|---| +| `UTF8Strict` (default) | refuse it: the writer errors with `ErrInvalidUTF8` and short-circuits | +| `UTF8Replace` | substitute U+FFFD, one per invalid byte — the writers' behavior before the policy existed | +| `UTF8Passthrough` | write the bytes through unchecked (produces invalid JSON; for pre-validated data) | + +`Unbuffered` takes `WithUnbufferedUTF8Policy`; `Indented` and `YAML` take it through +`WithIndentBufferedOptions` / `WithYAMLBufferedOptions`. The substitution rule is shared with the lexers +(`json/internal/utf8x`), so a value sanitized on the way in and one sanitized on the way out are the same bytes. + +Go strings may legally hold ill-formed UTF-8, so `String()` and `StringBytes()` can trip the default on data from a +non-Unicode source. That is the intended change: silently corrupting a payload is worse than refusing it. + +### What is checked + +The policy applies to what the **caller** supplies: `String`, `StringBytes`, `StringRunes`, `StringCopy`, `Raw`, +`RawCopy`. Streaming entry points validate the stream as a whole, not chunk by chunk, so a sequence split across two +reads is not mistaken for a fault and a fault spanning the boundary is not missed. + +It does **not** apply to values arriving in a lexer token (`Token`, `VerbatimToken`): the lexers already validate +string values, and re-checking them would cost the token-copy path its reason for existing. The loophole that leaves +— and it is deliberate — is that a document lexed with `UTF8Passthrough` can carry ill-formed bytes into the writer, +where `Token` silently substitutes U+FFFD (the escaper decodes as it goes) and `VerbatimToken` writes the bytes out +byte-for-byte, as verbatim requires. `VerbatimValue` is not in that group: it renders through the ordinary string +path and is checked. Numbers (`NumberBytes`, `NumberCopy`) are unchecked, as their documentation already says. + ## Performance * Allocations diff --git a/json/writers/default-writer/base_writer.go b/json/writers/default-writer/base_writer.go index 9349757..646d83b 100644 --- a/json/writers/default-writer/base_writer.go +++ b/json/writers/default-writer/base_writer.go @@ -10,6 +10,7 @@ import ( "unicode/utf8" "unsafe" + "github.com/go-openapi/core/json/internal/utf8x" "github.com/go-openapi/core/json/lexers/token" "github.com/go-openapi/core/json/stores/values" "github.com/go-openapi/core/json/types" @@ -20,6 +21,11 @@ type wrt interface { writeSingleByte(byte) writeBinary([]byte) writeEscaped([]byte) []byte + // escapePolicy reports how ill-formed UTF-8 in caller-supplied data must be handled. + // + // Named for its role rather than after the option: `buffered` embeds both baseWriter and bufferedOptions, so a + // method here called utf8Policy would collide with the promoted option field of that name. + escapePolicy() UTF8Policy // Ok is the cheap, inlinable hot-path status check (w.err == nil), used for the // per-operation guards instead of the value-receiver Err which copies and wraps. Ok() bool @@ -30,6 +36,7 @@ type baseWriter struct { w io.Writer written int64 err error + utf8 UTF8Policy } // Ok tells the status of the writer. @@ -64,6 +71,9 @@ func (w *baseWriter) Size() int64 { return w.written } +// escapePolicy reports how ill-formed UTF-8 in caller-supplied data must be handled. See [WithUTF8Policy]. +func (w *baseWriter) escapePolicy() UTF8Policy { return w.utf8 } + func (w *baseWriter) inc(n int) { w.written += int64(n) } @@ -122,12 +132,25 @@ func (w *commonWriter[T]) Bool(v bool) { } // Raw appends raw bytes to the buffer, without quotes and without escaping. +// +// The bytes are caller-supplied, so the UTF-8 policy applies: under [UTF8Strict] ill-formed input is refused. Nothing +// is escaped or substituted — "raw" means raw — so [UTF8Replace] does not rewrite anything here either; it only +// declines to complain. func (w *commonWriter[T]) Raw(data []byte) { if !w.jw.Ok() || len(data) == 0 { return } - w.jw.writeBinary(data) + holder, redeem := poolOfEscapedBuffers.BorrowWithSizeAndRedeem(0) + defer redeem() + + scratch := holder.Slice() + out, ok := w.rawUTF8(data, &scratch) + if !ok { + return + } + + w.jw.writeBinary(out) } // String writes a string as a JSON string value enclosed by double quotes, with escaping. @@ -166,10 +189,18 @@ func (w *commonWriter[T]) StringRunes(data []rune) { buf := holder.Slice() for _, r := range data { + if !utf8.ValidRune(r) && w.jw.escapePolicy() == UTF8Strict { + // utf8.AppendRune would silently turn a surrogate or an out-of-range value into U+FFFD + w.jw.SetErr(fmt.Errorf("%w: rune %U is not a Unicode scalar value: %w", + ErrInvalidUTF8, r, ErrDefaultWriter)) + + return + } buf = utf8.AppendRune(buf, r) } - w.writeText(buf) + // the runes were just encoded, so the bytes are well-formed by construction + w.writeTextTrusted(buf) } // NumberBytes writes a slice of bytes as a JSON number. @@ -197,8 +228,15 @@ func (w *commonWriter[T]) RawCopy(r io.Reader) { } bufHolder, redeemReadBuffer := poolOfReadBuffers.BorrowWithRedeem() + defer redeemReadBuffer() buf := bufHolder.Slice() + if w.jw.escapePolicy() != UTF8Passthrough { + w.rawCopyValidated(r, buf) + + return + } + for { n, err := r.Read(buf) if err != nil && !errors.Is(err, io.EOF) { @@ -218,8 +256,6 @@ func (w *commonWriter[T]) RawCopy(r io.Reader) { break } } - - redeemReadBuffer() } func (w *commonWriter[T]) StringCopy(r io.Reader) { @@ -245,6 +281,10 @@ func (w *commonWriter[T]) StringCopy(r io.Reader) { } if n > 0 { + if !w.checkUTF8Chunk(buf[:n]) { + return + } + remainder = w.jw.writeEscaped(buf[:n]) if !w.jw.Ok() { return @@ -287,13 +327,11 @@ func (w *commonWriter[T]) StringCopy(r io.Reader) { // Note: this must not clobber the outer loop's n/err, which drive termination. if _, rerr := io.ReadFull(r, single[notWritten:runeSize]); rerr != nil { if errors.Is(rerr, io.EOF) || errors.Is(rerr, io.ErrUnexpectedEOF) { - w.jw.SetErr( - fmt.Errorf( - "unexpected incomplete rune at end of input: %c: %w", - remainder, - ErrDefaultWriter, - ), - ) + // the input ended inside the sequence: it is truncated, not merely split + if !w.finishRemainder(remainder) { + return + } + w.jw.writeSingleByte(quote) return } @@ -303,19 +341,18 @@ func (w *commonWriter[T]) StringCopy(r io.Reader) { return } + // the stitched sequence never passed through checkUTF8Chunk (it was assembled here, not read as a + // chunk), so it must be checked on its own or a fault spanning the read boundary slips through + if !w.checkUTF8(single[:runeSize]) { + return + } + remainder = w.jw.writeEscaped(single[:runeSize]) if !w.jw.Ok() { return } - if len(remainder) > 0 { - w.jw.SetErr( - fmt.Errorf( - "unexpected incomplete rune at end of input: %c: %w", - remainder, - ErrDefaultWriter, - ), - ) - + if len(remainder) > 0 && !w.finishRemainder(remainder) { + // a full rune's worth of bytes that still does not complete: truncated return } } @@ -528,7 +565,8 @@ func (w *commonWriter[T]) Token(tok token.T) { // ignore } case token.String, token.Key: - w.writeText(tok.Value()) + // a token value comes from a lexer, which already validated it: see writeTextTrusted + w.writeTextTrusted(tok.Value()) case token.Number: w.NumberBytes(tok.Value()) case token.Boolean: @@ -578,6 +616,83 @@ func (w *commonWriter[T]) VerbatimValue(value values.VerbatimValue) { w.Value(value.Value) } +// rawCopyValidated is [commonWriter.RawCopy] under [UTF8Strict] and [UTF8Replace]: it streams the reader through +// while validating it as a single UTF-8 stream rather than as independent chunks. +// +// A read boundary may fall inside a multi-byte sequence, which is not an error — the sequence is completed by the +// next read. So the trailing incomplete bytes of a chunk are HELD BACK rather than written, and re-joined with the +// bytes that follow; reads land at an offset in buf that leaves room for them, so the join needs no second buffer and +// no allocation. An incomplete sequence still pending when the reader ends is a genuine truncation. +// +// Validating chunk-locally instead would leave a hole exactly here: a lead byte at the end of one chunk followed by a +// non-continuation at the start of the next would be tolerated by both halves and emitted. +// +// On a genuine fault [UTF8Strict] stops with an error and [UTF8Replace] rewrites that chunk with U+FFFD substituted, +// so "the output is always valid UTF-8" holds for the streaming path exactly as it does for [commonWriter.Raw]. +func (w *commonWriter[T]) rawCopyValidated(r io.Reader, buf []byte) { + // maxCarry bytes of headroom at the front of buf hold the incomplete tail of the previous chunk. + const maxCarry = utf8.UTFMax - 1 + if len(buf) <= maxCarry { + w.jw.SetErr(fmt.Errorf("read buffer too small for UTF-8 validation: %w", ErrDefaultWriter)) + + return + } + + strict := w.jw.escapePolicy() == UTF8Strict + holder, redeem := poolOfEscapedBuffers.BorrowWithSizeAndRedeem(0) + defer redeem() + scratch := holder.Slice() + + carry := 0 + for { + n, err := r.Read(buf[maxCarry:]) + if err != nil && !errors.Is(err, io.EOF) { + w.jw.SetErr(err) + + return + } + + atEOF := n == 0 || (err != nil && errors.Is(err, io.EOF)) + chunk := buf[maxCarry-carry : maxCarry+n] + + idx := utf8x.FirstInvalid(chunk) + switch { + case idx < 0: + carry = 0 + case !utf8.FullRune(chunk[idx:]) && !atEOF: + // the chunk ends inside a sequence the next read will complete: hold those bytes back + carry = len(chunk) - idx + chunk = chunk[:idx] + case strict: + w.jw.SetErr(fmt.Errorf("%w at byte %d: %w", ErrInvalidUTF8, idx, ErrDefaultWriter)) + + return + default: + // UTF8Replace: rewrite this chunk, held-back bytes included. A sequence truncated by the end of the + // input lands here too, and becomes one U+FFFD per byte like any other ill-formed sequence. + scratch = utf8x.Sanitize(scratch[:0], chunk) + chunk = scratch + carry = 0 + } + + if len(chunk) > 0 { + w.jw.writeBinary(chunk) + if !w.jw.Ok() { + return + } + } + + if carry > 0 { + // move the held-back tail into the headroom, where the next read joins onto it + copy(buf[maxCarry-carry:maxCarry], buf[maxCarry+n-carry:maxCarry+n]) + } + + if atEOF { + return + } + } +} + // append writes down the result of AppendText. // // This borrows a temporary buffer to decode the result of AppendText() @@ -613,18 +728,28 @@ func (w *commonWriter[T]) writeTextString(input string) { w.writeText(b) } +// writeText writes CALLER-supplied bytes as a quoted, escaped JSON string, subject to the UTF-8 policy. +// +// Values that came from a lexer token go through [commonWriter.writeTextTrusted] instead: the lexers already +// guarantee valid UTF-8, so re-checking them would be pure overhead on the hot token-copy path. func (w *commonWriter[T]) writeText(data []byte) { + if !w.checkUTF8(data) { + return + } + + w.writeTextTrusted(data) +} + +// writeTextTrusted writes bytes that are already known to be valid UTF-8 as a quoted, escaped JSON string. +// +// "Trusted" means sourced from a lexer token: [L] and [VL] validate string values as they scan, so the writer does +// not pay to check them again. That trust is only as good as the lexer's policy — see the package documentation for +// the loophole this leaves. +func (w *commonWriter[T]) writeTextTrusted(data []byte) { w.jw.writeSingleByte(quote) remainder := w.jw.writeEscaped(data) - if len(remainder) > 0 { - w.jw.SetErr( - fmt.Errorf( - "unexpected incomplete rune (invalid first byte): %c: %w", - remainder, - ErrDefaultWriter, - ), - ) - + if len(remainder) > 0 && !w.finishRemainder(remainder) { + // nothing follows this value, so the trailing bytes are a truncated sequence, not a split one return } w.jw.writeSingleByte(quote) diff --git a/json/writers/default-writer/buffered.go b/json/writers/default-writer/buffered.go index 68fb89c..06aa403 100644 --- a/json/writers/default-writer/buffered.go +++ b/json/writers/default-writer/buffered.go @@ -12,6 +12,7 @@ import ( "unicode/utf8" "unsafe" + "github.com/go-openapi/core/json/internal/utf8x" "github.com/go-openapi/core/json/lexers/token" "github.com/go-openapi/core/json/stores/values" "github.com/go-openapi/core/json/types" @@ -43,12 +44,14 @@ type buffered struct { } func NewBuffered(w io.Writer, opts ...BufferedOption) *Buffered { + o := bufferedOptionsWithDefaults(opts) writer := &Buffered{ buffered: buffered{ baseWriter: baseWriter{ - w: w, + w: w, + utf8: o.utf8Policy, }, - bufferedOptions: bufferedOptionsWithDefaults(opts), + bufferedOptions: o, }, } writer.borrowBuffer() @@ -310,6 +313,11 @@ func (w *buffered) writeEscaped(data []byte) (remainder []byte) { return nil } } + if r == utf8.RuneError && runeWidth == 1 && w.utf8 == UTF8Passthrough { + w.buf = append(w.buf, c) // ill-formed, but the caller asked for its bytes untouched + + continue + } w.buf = utf8.AppendRune(w.buf, r) // invalid runes are represented as \uFFFD i += runeWidth - 1 } @@ -369,7 +377,16 @@ func (w *Buffered) Raw(data []byte) { return } - w.jw.writeBinary(data) + holder, redeem := poolOfEscapedBuffers.BorrowWithSizeAndRedeem(0) + defer redeem() + + scratch := holder.Slice() + out, ok := w.rawUTF8(data, &scratch) + if !ok { + return + } + + w.jw.writeBinary(out) } // String is generated by writegen from commonWriter.String; DO NOT EDIT (edit commonWriter, re-run go generate). @@ -402,10 +419,18 @@ func (w *Buffered) StringRunes(data []rune) { buf := holder.Slice() for _, r := range data { + if !utf8.ValidRune(r) && w.jw.escapePolicy() == UTF8Strict { + // utf8.AppendRune would silently turn a surrogate or an out-of-range value into U+FFFD + w.jw.SetErr(fmt.Errorf("%w: rune %U is not a Unicode scalar value: %w", + ErrInvalidUTF8, r, ErrDefaultWriter)) + + return + } buf = utf8.AppendRune(buf, r) } - w.writeText(buf) + // the runes were just encoded, so the bytes are well-formed by construction + w.writeTextTrusted(buf) } // NumberBytes is generated by writegen from commonWriter.NumberBytes; DO NOT EDIT (edit commonWriter, re-run go generate). @@ -429,8 +454,15 @@ func (w *Buffered) RawCopy(r io.Reader) { } bufHolder, redeemReadBuffer := poolOfReadBuffers.BorrowWithRedeem() + defer redeemReadBuffer() buf := bufHolder.Slice() + if w.jw.escapePolicy() != UTF8Passthrough { + w.rawCopyValidated(r, buf) + + return + } + for { n, err := r.Read(buf) if err != nil && !errors.Is(err, io.EOF) { @@ -450,8 +482,6 @@ func (w *Buffered) RawCopy(r io.Reader) { break } } - - redeemReadBuffer() } // StringCopy is generated by writegen from commonWriter.StringCopy; DO NOT EDIT (edit commonWriter, re-run go generate). @@ -478,6 +508,10 @@ func (w *Buffered) StringCopy(r io.Reader) { } if n > 0 { + if !w.checkUTF8Chunk(buf[:n]) { + return + } + remainder = w.jw.writeEscaped(buf[:n]) if !w.jw.Ok() { return @@ -520,13 +554,11 @@ func (w *Buffered) StringCopy(r io.Reader) { // Note: this must not clobber the outer loop's n/err, which drive termination. if _, rerr := io.ReadFull(r, single[notWritten:runeSize]); rerr != nil { if errors.Is(rerr, io.EOF) || errors.Is(rerr, io.ErrUnexpectedEOF) { - w.jw.SetErr( - fmt.Errorf( - "unexpected incomplete rune at end of input: %c: %w", - remainder, - ErrDefaultWriter, - ), - ) + // the input ended inside the sequence: it is truncated, not merely split + if !w.finishRemainder(remainder) { + return + } + w.jw.writeSingleByte(quote) return } @@ -536,19 +568,18 @@ func (w *Buffered) StringCopy(r io.Reader) { return } + // the stitched sequence never passed through checkUTF8Chunk (it was assembled here, not read as a + // chunk), so it must be checked on its own or a fault spanning the read boundary slips through + if !w.checkUTF8(single[:runeSize]) { + return + } + remainder = w.jw.writeEscaped(single[:runeSize]) if !w.jw.Ok() { return } - if len(remainder) > 0 { - w.jw.SetErr( - fmt.Errorf( - "unexpected incomplete rune at end of input: %c: %w", - remainder, - ErrDefaultWriter, - ), - ) - + if len(remainder) > 0 && !w.finishRemainder(remainder) { + // a full rune's worth of bytes that still does not complete: truncated return } } @@ -726,7 +757,8 @@ func (w *Buffered) Token(tok token.T) { // ignore } case token.String, token.Key: - w.writeText(tok.Value()) + // a token value comes from a lexer, which already validated it: see writeTextTrusted + w.writeTextTrusted(tok.Value()) case token.Number: w.NumberBytes(tok.Value()) case token.Boolean: @@ -769,6 +801,71 @@ func (w *Buffered) VerbatimValue(value values.VerbatimValue) { w.Value(value.Value) } +// rawCopyValidated is generated by writegen from commonWriter.rawCopyValidated; DO NOT EDIT (edit commonWriter, re-run go generate). +func (w *Buffered) rawCopyValidated(r io.Reader, buf []byte) { + // maxCarry bytes of headroom at the front of buf hold the incomplete tail of the previous chunk. + const maxCarry = utf8.UTFMax - 1 + if len(buf) <= maxCarry { + w.jw.SetErr(fmt.Errorf("read buffer too small for UTF-8 validation: %w", ErrDefaultWriter)) + + return + } + + strict := w.jw.escapePolicy() == UTF8Strict + holder, redeem := poolOfEscapedBuffers.BorrowWithSizeAndRedeem(0) + defer redeem() + scratch := holder.Slice() + + carry := 0 + for { + n, err := r.Read(buf[maxCarry:]) + if err != nil && !errors.Is(err, io.EOF) { + w.jw.SetErr(err) + + return + } + + atEOF := n == 0 || (err != nil && errors.Is(err, io.EOF)) + chunk := buf[maxCarry-carry : maxCarry+n] + + idx := utf8x.FirstInvalid(chunk) + switch { + case idx < 0: + carry = 0 + case !utf8.FullRune(chunk[idx:]) && !atEOF: + // the chunk ends inside a sequence the next read will complete: hold those bytes back + carry = len(chunk) - idx + chunk = chunk[:idx] + case strict: + w.jw.SetErr(fmt.Errorf("%w at byte %d: %w", ErrInvalidUTF8, idx, ErrDefaultWriter)) + + return + default: + // UTF8Replace: rewrite this chunk, held-back bytes included. A sequence truncated by the end of the + // input lands here too, and becomes one U+FFFD per byte like any other ill-formed sequence. + scratch = utf8x.Sanitize(scratch[:0], chunk) + chunk = scratch + carry = 0 + } + + if len(chunk) > 0 { + w.jw.writeBinary(chunk) + if !w.jw.Ok() { + return + } + } + + if carry > 0 { + // move the held-back tail into the headroom, where the next read joins onto it + copy(buf[maxCarry-carry:maxCarry], buf[maxCarry+n-carry:maxCarry+n]) + } + + if atEOF { + return + } + } +} + // append is generated by writegen from commonWriter.append; DO NOT EDIT (edit commonWriter, re-run go generate). func (w *Buffered) append(n encoding.TextAppender) { buf, redeem := poolOfNumberBuffers.BorrowWithRedeem() @@ -806,18 +903,96 @@ func (w *Buffered) writeTextString(input string) { // writeText is generated by writegen from commonWriter.writeText; DO NOT EDIT (edit commonWriter, re-run go generate). func (w *Buffered) writeText(data []byte) { + if !w.checkUTF8(data) { + return + } + + w.writeTextTrusted(data) +} + +// writeTextTrusted is generated by writegen from commonWriter.writeTextTrusted; DO NOT EDIT (edit commonWriter, re-run go generate). +func (w *Buffered) writeTextTrusted(data []byte) { w.jw.writeSingleByte(quote) remainder := w.jw.writeEscaped(data) - if len(remainder) > 0 { + if len(remainder) > 0 && !w.finishRemainder(remainder) { + // nothing follows this value, so the trailing bytes are a truncated sequence, not a split one + return + } + w.jw.writeSingleByte(quote) +} + +// checkUTF8 is generated by writegen from commonWriter.checkUTF8; DO NOT EDIT (edit commonWriter, re-run go generate). +func (w *Buffered) checkUTF8(data []byte) bool { + if w.jw.escapePolicy() != UTF8Strict || utf8x.Valid(data) { + return true + } + + w.jw.SetErr( + fmt.Errorf("%w at byte %d: %w", ErrInvalidUTF8, utf8x.FirstInvalid(data), ErrDefaultWriter), + ) + + return false +} + +// rawUTF8 is generated by writegen from commonWriter.rawUTF8; DO NOT EDIT (edit commonWriter, re-run go generate). +func (w *Buffered) rawUTF8(data []byte, scratch *[]byte) ([]byte, bool) { + policy := w.jw.escapePolicy() + if policy == UTF8Passthrough || utf8x.Valid(data) { + return data, true + } + + if policy == UTF8Strict { w.jw.SetErr( fmt.Errorf( - "unexpected incomplete rune (invalid first byte): %c: %w", - remainder, + "%w at byte %d: %w", + ErrInvalidUTF8, + utf8x.FirstInvalid(data), ErrDefaultWriter, ), ) - return + return nil, false } - w.jw.writeSingleByte(quote) + + *scratch = utf8x.Sanitize((*scratch)[:0], data) + + return *scratch, true +} + +// finishRemainder is generated by writegen from commonWriter.finishRemainder; DO NOT EDIT (edit commonWriter, re-run go generate). +func (w *Buffered) finishRemainder(remainder []byte) bool { + switch w.jw.escapePolicy() { + case UTF8Strict: + w.jw.SetErr(fmt.Errorf("%w: input ends inside a multi-byte sequence (% x): %w", + ErrInvalidUTF8, remainder, ErrDefaultWriter)) + + return false + + case UTF8Passthrough: + w.jw.writeBinary(remainder) + + default: // UTF8Replace + var buf [utf8.UTFMax * utf8.UTFMax]byte + w.jw.writeBinary(utf8x.Sanitize(buf[:0], remainder)) + } + + return w.jw.Ok() +} + +// checkUTF8Chunk is generated by writegen from commonWriter.checkUTF8Chunk; DO NOT EDIT (edit commonWriter, re-run go generate). +func (w *Buffered) checkUTF8Chunk(chunk []byte) bool { + if w.jw.escapePolicy() != UTF8Strict { + return true + } + + idx := utf8x.FirstInvalid(chunk) + if idx < 0 || !utf8.FullRune(chunk[idx:]) { + return true + } + + w.jw.SetErr( + fmt.Errorf("%w at byte %d of this chunk: %w", ErrInvalidUTF8, idx, ErrDefaultWriter), + ) + + return false } diff --git a/json/writers/default-writer/buffered_options.go b/json/writers/default-writer/buffered_options.go index f4d081e..5309560 100644 --- a/json/writers/default-writer/buffered_options.go +++ b/json/writers/default-writer/buffered_options.go @@ -25,6 +25,25 @@ func WithBufferSize(size int) BufferedOption { } } +// WithUTF8Policy selects what the writer does with caller-supplied data that is not valid UTF-8: refuse it +// ([UTF8Strict], the default), substitute U+FFFD ([UTF8Replace]), or write it through unchecked +// ([UTF8Passthrough]). +// +// It applies to every entry point that takes bytes, a string, runes or an [io.Reader] from the caller. Values that +// come from a lexer token ([commonWriter.Token], [commonWriter.VerbatimToken], [commonWriter.VerbatimValue]) are NOT +// re-checked — the lexers already guarantee valid UTF-8 unless they were themselves relaxed. See the package +// documentation for that loophole. +// +// [Indented] and [YAML] take this through WithIndentBufferedOptions / WithYAMLBufferedOptions; +// [Unbuffered] has its own [WithUnbufferedUTF8Policy]. +func WithUTF8Policy(policy UTF8Policy) BufferedOption { + return func(o bufferedOptions) bufferedOptions { + o.utf8Policy = policy + + return o + } +} + // bufferedOptions carries the (immutable, unexported) [Buffered] configuration. // // It holds configuration only. Runtime state such as the working-buffer redeem handle lives on @@ -32,6 +51,7 @@ func WithBufferSize(size int) BufferedOption { // no finalizer for the options themselves). type bufferedOptions struct { bufferSize int + utf8Policy UTF8Policy } func bufferedOptionsWithDefaults(opts []BufferedOption) bufferedOptions { diff --git a/json/writers/default-writer/doc.go b/json/writers/default-writer/doc.go index e5c0e17..698f212 100644 --- a/json/writers/default-writer/doc.go +++ b/json/writers/default-writer/doc.go @@ -11,4 +11,38 @@ // // Similarly, it handles "null" values as actual values. Absent data (i.e. a nil slice of bytes) is not considered // a "null" value and is therefore not rendered at all. +// +// # UTF-8 +// +// RFC 8259 §8.1 requires JSON text to be UTF-8, so a writer that emits an ill-formed sequence emits a document no +// conforming parser has to accept. By default ([UTF8Strict]) the writers refuse such data with [ErrInvalidUTF8] +// instead of writing it; [UTF8Replace] substitutes U+FFFD (one per invalid byte) and [UTF8Passthrough] writes the +// bytes through unchecked. See [WithUTF8Policy]. +// +// Note that a Go string may legally hold ill-formed UTF-8, so [Buffered.String] and [Buffered.StringBytes] can trip +// the default on data that arrived from a non-Unicode source. Before this policy existed the writers substituted +// U+FFFD silently and unconditionally — [UTF8Replace] is that old behavior, now explicit. +// +// The substitution rule is shared with the lexers, so a value sanitized on the way in and one sanitized on the way +// out are the same bytes. +// +// # What is checked, and the token loophole +// +// The policy applies to data the CALLER supplies: String, StringBytes, StringRunes, StringCopy, Raw and RawCopy. +// +// It does NOT apply to values that arrive in a lexer token — Token and VerbatimToken. The lexers validate string +// values as they scan, so re-checking them here would be pure overhead on the token-copy path that exists to be fast. +// That trust is only as good as the lexer's own policy, which leaves a deliberate loophole: if a document is lexed +// with lexer.UTF8Passthrough, its tokens may carry ill-formed bytes, and then +// +// - Token does not report them, but the escaper decodes as it goes, so they are silently substituted with U+FFFD: +// the output stays valid UTF-8 while differing from the input with no error; +// - VerbatimToken writes the value byte-for-byte with no escaping at all — that is what verbatim means — so there +// the ill-formed bytes DO reach the output. +// +// VerbatimValue is NOT in that group: it renders through the ordinary string path, so it is checked like any other +// caller data. If you lex with a relaxed policy and write the tokens out, validate at the boundary you care about. +// +// Numbers are also unchecked ([Buffered.NumberBytes], [Buffered.NumberCopy]): they are documented as taking whatever +// the caller provides, and a number containing non-ASCII is already invalid JSON for grammatical reasons. package writer diff --git a/json/writers/default-writer/errors.go b/json/writers/default-writer/errors.go index 698a917..7bc347b 100644 --- a/json/writers/default-writer/errors.go +++ b/json/writers/default-writer/errors.go @@ -13,4 +13,9 @@ const ( // ErrUnsupportedInterface means that a method is called but the underlying buffer does not support this interface. ErrUnsupportedInterface Error = "the underlying buffer does not support this interface" + + // ErrInvalidUTF8 means that caller-supplied data is not valid UTF-8 and the writer runs under [UTF8Strict]. + // RFC 8259 §8.1 requires JSON text to be UTF-8, so writing it would emit a document no conforming parser has to + // accept. Select [UTF8Replace] to substitute U+FFFD instead. + ErrInvalidUTF8 Error = "data to write is not valid UTF-8" ) diff --git a/json/writers/default-writer/escape.go b/json/writers/default-writer/escape.go index 9e1eb2c..dfe2bf0 100644 --- a/json/writers/default-writer/escape.go +++ b/json/writers/default-writer/escape.go @@ -4,7 +4,14 @@ import ( "unicode/utf8" ) -func escapedBytes(input, output []byte) ([]byte, []byte) { +// escapedBytes escapes input into output per the JSON string rules, returning the escaped bytes and any trailing +// bytes that form an incomplete UTF-8 sequence (which the caller stitches with the next read, or reports). +// +// policy decides what happens to a byte that is not part of a well-formed sequence: [UTF8Passthrough] copies it +// verbatim, anything else substitutes U+FFFD — one per invalid byte, the same granularity as utf8x.Sanitize, so the +// lexer and the writer agree on what a sanitized value looks like. Under [UTF8Strict] the caller has already +// rejected such input, so the substitution is unreachable there. +func escapedBytes(input, output []byte, policy UTF8Policy) ([]byte, []byte) { var ( p int escaped bool @@ -70,6 +77,14 @@ func escapedBytes(input, output []byte) ([]byte, []byte) { return output, input[i:] } r, runeWidth := utf8.DecodeRune(input[i:]) + if r == utf8.RuneError && runeWidth == 1 && policy == UTF8Passthrough { + output = append( + output, + c, + ) // ill-formed, but the caller asked for its bytes untouched + + continue + } output = utf8.AppendRune(output, r) // invalid runes are represented as \uFFFD i += runeWidth - 1 } diff --git a/json/writers/default-writer/escape_regression_test.go b/json/writers/default-writer/escape_regression_test.go index 0946688..bd31393 100644 --- a/json/writers/default-writer/escape_regression_test.go +++ b/json/writers/default-writer/escape_regression_test.go @@ -167,7 +167,7 @@ func TestStringCopyTruncatedRuneErrors(t *testing.T) { func expectedJSONString(s string) string { out := make([]byte, 0, len(s)+2) out = append(out, '"') - escaped, remainder := escapedBytes([]byte(s), make([]byte, 0, len(s))) + escaped, remainder := escapedBytes([]byte(s), make([]byte, 0, len(s)), UTF8Replace) out = append(out, escaped...) out = append(out, remainder...) // none expected for complete inputs out = append(out, '"') diff --git a/json/writers/default-writer/unbuffered.go b/json/writers/default-writer/unbuffered.go index c059e84..3a98ef1 100644 --- a/json/writers/default-writer/unbuffered.go +++ b/json/writers/default-writer/unbuffered.go @@ -10,6 +10,7 @@ import ( "unicode/utf8" "unsafe" + "github.com/go-openapi/core/json/internal/utf8x" "github.com/go-openapi/core/json/lexers/token" "github.com/go-openapi/core/json/stores/values" "github.com/go-openapi/core/json/types" @@ -58,13 +59,14 @@ type unbuffered struct { // NewUnbuffered JSON writer that copies JSON to [io.Writer] w, without buffering. // -// [Unbuffered] has no configurable options and holds no pooled resources, so there is nothing to -// relinquish: no options finalizer is registered. -func NewUnbuffered(w io.Writer, _ ...UnbufferedOption) *Unbuffered { +// [Unbuffered] holds no pooled resources, so there is nothing to relinquish: no options finalizer is registered. +func NewUnbuffered(w io.Writer, opts ...UnbufferedOption) *Unbuffered { + o := unbufferedOptionsWithDefaults(opts) writer := &Unbuffered{ unbuffered: unbuffered{ baseWriter: baseWriter{ - w: w, + w: w, + utf8: o.utf8Policy, }, }, } @@ -122,7 +124,7 @@ func (w *unbuffered) writeEscaped(data []byte) []byte { escapedHolder, redeemEscaped := poolOfEscapedBuffers.BorrowWithSizeAndRedeem(len(data)) escapedBuffer := escapedHolder.Slice() - escapedBuffer, remainder := escapedBytes(data, escapedBuffer) + escapedBuffer, remainder := escapedBytes(data, escapedBuffer, w.utf8) w.writeBinary(escapedBuffer) redeemEscaped() @@ -181,7 +183,16 @@ func (w *Unbuffered) Raw(data []byte) { return } - w.jw.writeBinary(data) + holder, redeem := poolOfEscapedBuffers.BorrowWithSizeAndRedeem(0) + defer redeem() + + scratch := holder.Slice() + out, ok := w.rawUTF8(data, &scratch) + if !ok { + return + } + + w.jw.writeBinary(out) } // String is generated by writegen from commonWriter.String; DO NOT EDIT (edit commonWriter, re-run go generate). @@ -214,10 +225,18 @@ func (w *Unbuffered) StringRunes(data []rune) { buf := holder.Slice() for _, r := range data { + if !utf8.ValidRune(r) && w.jw.escapePolicy() == UTF8Strict { + // utf8.AppendRune would silently turn a surrogate or an out-of-range value into U+FFFD + w.jw.SetErr(fmt.Errorf("%w: rune %U is not a Unicode scalar value: %w", + ErrInvalidUTF8, r, ErrDefaultWriter)) + + return + } buf = utf8.AppendRune(buf, r) } - w.writeText(buf) + // the runes were just encoded, so the bytes are well-formed by construction + w.writeTextTrusted(buf) } // NumberBytes is generated by writegen from commonWriter.NumberBytes; DO NOT EDIT (edit commonWriter, re-run go generate). @@ -241,8 +260,15 @@ func (w *Unbuffered) RawCopy(r io.Reader) { } bufHolder, redeemReadBuffer := poolOfReadBuffers.BorrowWithRedeem() + defer redeemReadBuffer() buf := bufHolder.Slice() + if w.jw.escapePolicy() != UTF8Passthrough { + w.rawCopyValidated(r, buf) + + return + } + for { n, err := r.Read(buf) if err != nil && !errors.Is(err, io.EOF) { @@ -262,8 +288,6 @@ func (w *Unbuffered) RawCopy(r io.Reader) { break } } - - redeemReadBuffer() } // StringCopy is generated by writegen from commonWriter.StringCopy; DO NOT EDIT (edit commonWriter, re-run go generate). @@ -290,6 +314,10 @@ func (w *Unbuffered) StringCopy(r io.Reader) { } if n > 0 { + if !w.checkUTF8Chunk(buf[:n]) { + return + } + remainder = w.jw.writeEscaped(buf[:n]) if !w.jw.Ok() { return @@ -332,13 +360,11 @@ func (w *Unbuffered) StringCopy(r io.Reader) { // Note: this must not clobber the outer loop's n/err, which drive termination. if _, rerr := io.ReadFull(r, single[notWritten:runeSize]); rerr != nil { if errors.Is(rerr, io.EOF) || errors.Is(rerr, io.ErrUnexpectedEOF) { - w.jw.SetErr( - fmt.Errorf( - "unexpected incomplete rune at end of input: %c: %w", - remainder, - ErrDefaultWriter, - ), - ) + // the input ended inside the sequence: it is truncated, not merely split + if !w.finishRemainder(remainder) { + return + } + w.jw.writeSingleByte(quote) return } @@ -348,19 +374,18 @@ func (w *Unbuffered) StringCopy(r io.Reader) { return } + // the stitched sequence never passed through checkUTF8Chunk (it was assembled here, not read as a + // chunk), so it must be checked on its own or a fault spanning the read boundary slips through + if !w.checkUTF8(single[:runeSize]) { + return + } + remainder = w.jw.writeEscaped(single[:runeSize]) if !w.jw.Ok() { return } - if len(remainder) > 0 { - w.jw.SetErr( - fmt.Errorf( - "unexpected incomplete rune at end of input: %c: %w", - remainder, - ErrDefaultWriter, - ), - ) - + if len(remainder) > 0 && !w.finishRemainder(remainder) { + // a full rune's worth of bytes that still does not complete: truncated return } } @@ -538,7 +563,8 @@ func (w *Unbuffered) Token(tok token.T) { // ignore } case token.String, token.Key: - w.writeText(tok.Value()) + // a token value comes from a lexer, which already validated it: see writeTextTrusted + w.writeTextTrusted(tok.Value()) case token.Number: w.NumberBytes(tok.Value()) case token.Boolean: @@ -581,6 +607,71 @@ func (w *Unbuffered) VerbatimValue(value values.VerbatimValue) { w.Value(value.Value) } +// rawCopyValidated is generated by writegen from commonWriter.rawCopyValidated; DO NOT EDIT (edit commonWriter, re-run go generate). +func (w *Unbuffered) rawCopyValidated(r io.Reader, buf []byte) { + // maxCarry bytes of headroom at the front of buf hold the incomplete tail of the previous chunk. + const maxCarry = utf8.UTFMax - 1 + if len(buf) <= maxCarry { + w.jw.SetErr(fmt.Errorf("read buffer too small for UTF-8 validation: %w", ErrDefaultWriter)) + + return + } + + strict := w.jw.escapePolicy() == UTF8Strict + holder, redeem := poolOfEscapedBuffers.BorrowWithSizeAndRedeem(0) + defer redeem() + scratch := holder.Slice() + + carry := 0 + for { + n, err := r.Read(buf[maxCarry:]) + if err != nil && !errors.Is(err, io.EOF) { + w.jw.SetErr(err) + + return + } + + atEOF := n == 0 || (err != nil && errors.Is(err, io.EOF)) + chunk := buf[maxCarry-carry : maxCarry+n] + + idx := utf8x.FirstInvalid(chunk) + switch { + case idx < 0: + carry = 0 + case !utf8.FullRune(chunk[idx:]) && !atEOF: + // the chunk ends inside a sequence the next read will complete: hold those bytes back + carry = len(chunk) - idx + chunk = chunk[:idx] + case strict: + w.jw.SetErr(fmt.Errorf("%w at byte %d: %w", ErrInvalidUTF8, idx, ErrDefaultWriter)) + + return + default: + // UTF8Replace: rewrite this chunk, held-back bytes included. A sequence truncated by the end of the + // input lands here too, and becomes one U+FFFD per byte like any other ill-formed sequence. + scratch = utf8x.Sanitize(scratch[:0], chunk) + chunk = scratch + carry = 0 + } + + if len(chunk) > 0 { + w.jw.writeBinary(chunk) + if !w.jw.Ok() { + return + } + } + + if carry > 0 { + // move the held-back tail into the headroom, where the next read joins onto it + copy(buf[maxCarry-carry:maxCarry], buf[maxCarry+n-carry:maxCarry+n]) + } + + if atEOF { + return + } + } +} + // append is generated by writegen from commonWriter.append; DO NOT EDIT (edit commonWriter, re-run go generate). func (w *Unbuffered) append(n encoding.TextAppender) { buf, redeem := poolOfNumberBuffers.BorrowWithRedeem() @@ -618,18 +709,96 @@ func (w *Unbuffered) writeTextString(input string) { // writeText is generated by writegen from commonWriter.writeText; DO NOT EDIT (edit commonWriter, re-run go generate). func (w *Unbuffered) writeText(data []byte) { + if !w.checkUTF8(data) { + return + } + + w.writeTextTrusted(data) +} + +// writeTextTrusted is generated by writegen from commonWriter.writeTextTrusted; DO NOT EDIT (edit commonWriter, re-run go generate). +func (w *Unbuffered) writeTextTrusted(data []byte) { w.jw.writeSingleByte(quote) remainder := w.jw.writeEscaped(data) - if len(remainder) > 0 { + if len(remainder) > 0 && !w.finishRemainder(remainder) { + // nothing follows this value, so the trailing bytes are a truncated sequence, not a split one + return + } + w.jw.writeSingleByte(quote) +} + +// checkUTF8 is generated by writegen from commonWriter.checkUTF8; DO NOT EDIT (edit commonWriter, re-run go generate). +func (w *Unbuffered) checkUTF8(data []byte) bool { + if w.jw.escapePolicy() != UTF8Strict || utf8x.Valid(data) { + return true + } + + w.jw.SetErr( + fmt.Errorf("%w at byte %d: %w", ErrInvalidUTF8, utf8x.FirstInvalid(data), ErrDefaultWriter), + ) + + return false +} + +// rawUTF8 is generated by writegen from commonWriter.rawUTF8; DO NOT EDIT (edit commonWriter, re-run go generate). +func (w *Unbuffered) rawUTF8(data []byte, scratch *[]byte) ([]byte, bool) { + policy := w.jw.escapePolicy() + if policy == UTF8Passthrough || utf8x.Valid(data) { + return data, true + } + + if policy == UTF8Strict { w.jw.SetErr( fmt.Errorf( - "unexpected incomplete rune (invalid first byte): %c: %w", - remainder, + "%w at byte %d: %w", + ErrInvalidUTF8, + utf8x.FirstInvalid(data), ErrDefaultWriter, ), ) - return + return nil, false } - w.jw.writeSingleByte(quote) + + *scratch = utf8x.Sanitize((*scratch)[:0], data) + + return *scratch, true +} + +// finishRemainder is generated by writegen from commonWriter.finishRemainder; DO NOT EDIT (edit commonWriter, re-run go generate). +func (w *Unbuffered) finishRemainder(remainder []byte) bool { + switch w.jw.escapePolicy() { + case UTF8Strict: + w.jw.SetErr(fmt.Errorf("%w: input ends inside a multi-byte sequence (% x): %w", + ErrInvalidUTF8, remainder, ErrDefaultWriter)) + + return false + + case UTF8Passthrough: + w.jw.writeBinary(remainder) + + default: // UTF8Replace + var buf [utf8.UTFMax * utf8.UTFMax]byte + w.jw.writeBinary(utf8x.Sanitize(buf[:0], remainder)) + } + + return w.jw.Ok() +} + +// checkUTF8Chunk is generated by writegen from commonWriter.checkUTF8Chunk; DO NOT EDIT (edit commonWriter, re-run go generate). +func (w *Unbuffered) checkUTF8Chunk(chunk []byte) bool { + if w.jw.escapePolicy() != UTF8Strict { + return true + } + + idx := utf8x.FirstInvalid(chunk) + if idx < 0 || !utf8.FullRune(chunk[idx:]) { + return true + } + + w.jw.SetErr( + fmt.Errorf("%w at byte %d of this chunk: %w", ErrInvalidUTF8, idx, ErrDefaultWriter), + ) + + return false } diff --git a/json/writers/default-writer/unbuffered_options.go b/json/writers/default-writer/unbuffered_options.go index 164369f..736cf06 100644 --- a/json/writers/default-writer/unbuffered_options.go +++ b/json/writers/default-writer/unbuffered_options.go @@ -1,10 +1,30 @@ package writer -// UnbufferedOption configures the [Unbuffered] writer. -// -// [Unbuffered] currently exposes no configuration knobs; the type exists for API symmetry with the -// other writers and for forward compatibility. Like the other writer options it threads the -// configuration value through, so it never allocates. +// UnbufferedOption configures the [Unbuffered] writer. Like the other writer options it threads the configuration +// value through, so it never allocates. type UnbufferedOption func(unbufferedOptions) unbufferedOptions -type unbufferedOptions struct{} +// WithUnbufferedUTF8Policy is [WithUTF8Policy] for the [Unbuffered] writer. +// +// It is a separate function only because Go options are typed per writer and [Unbuffered] does not take +// [BufferedOption]s; the knob and its meaning are identical. +func WithUnbufferedUTF8Policy(policy UTF8Policy) UnbufferedOption { + return func(o unbufferedOptions) unbufferedOptions { + o.utf8Policy = policy + + return o + } +} + +type unbufferedOptions struct { + utf8Policy UTF8Policy +} + +func unbufferedOptionsWithDefaults(opts []UnbufferedOption) unbufferedOptions { + var o unbufferedOptions // the zero value is UTF8Strict + for _, apply := range opts { + o = apply(o) + } + + return o +} diff --git a/json/writers/default-writer/utf8.go b/json/writers/default-writer/utf8.go new file mode 100644 index 0000000..f9104e3 --- /dev/null +++ b/json/writers/default-writer/utf8.go @@ -0,0 +1,129 @@ +package writer + +import ( + "fmt" + "unicode/utf8" + + "github.com/go-openapi/core/json/internal/utf8x" +) + +// UTF8Policy selects what a writer does with caller-supplied data that is not valid UTF-8. +// +// RFC 8259 §8.1 requires JSON text to be UTF-8, so a writer that emits an ill-formed sequence emits a document no +// conforming parser has to accept. It is the same type the lexers use ([lexer.UTF8Policy]) — a document refused on +// the way out and one rejected on the way in mean the same thing. +type UTF8Policy = utf8x.Policy + +const ( + // UTF8Strict refuses to write ill-formed input: the writer goes into an error state reporting + // [ErrInvalidUTF8], and every subsequent operation short-circuits. This is the default. + // + // Note that Go strings may legally hold ill-formed UTF-8, so String() and StringBytes() can trip this on data + // that arrived from a non-Unicode source. + UTF8Strict = utf8x.PolicyStrict + + // UTF8Replace substitutes U+FFFD for each invalid byte, so the output is always valid UTF-8 and no error is + // raised. This is what the writers did unconditionally before the policy existed. + UTF8Replace = utf8x.PolicyReplace + + // UTF8Passthrough writes ill-formed bytes through untouched and performs no validation. + // + // UNSAFE: the result is not valid JSON text. It exists for callers who have already validated their data and + // want neither the check nor the substitution. + UTF8Passthrough = utf8x.PolicyPassthrough +) + +// checkUTF8 applies the policy to caller-supplied data before it is written. +// +// It reports whether writing may proceed; under [UTF8Strict] an ill-formed input puts the writer into an error state +// and returns false. Under the other policies it is a no-op — the escaper substitutes (or passes through) as it goes, +// so there is nothing to check up front. +func (w *commonWriter[T]) checkUTF8(data []byte) bool { + if w.jw.escapePolicy() != UTF8Strict || utf8x.Valid(data) { + return true + } + + w.jw.SetErr( + fmt.Errorf("%w at byte %d: %w", ErrInvalidUTF8, utf8x.FirstInvalid(data), ErrDefaultWriter), + ) + + return false +} + +// rawUTF8 applies the policy to caller-supplied bytes that will be written WITHOUT escaping. +// +// It returns the bytes to write and whether writing may proceed. Well-formed input is returned unchanged, so the +// zero-copy pass-through of Raw is preserved for every valid payload; only an ill-formed one is rewritten, and only +// under [UTF8Replace], into the borrowed scratch buffer. +func (w *commonWriter[T]) rawUTF8(data []byte, scratch *[]byte) ([]byte, bool) { + policy := w.jw.escapePolicy() + if policy == UTF8Passthrough || utf8x.Valid(data) { + return data, true + } + + if policy == UTF8Strict { + w.jw.SetErr( + fmt.Errorf( + "%w at byte %d: %w", + ErrInvalidUTF8, + utf8x.FirstInvalid(data), + ErrDefaultWriter, + ), + ) + + return nil, false + } + + *scratch = utf8x.Sanitize((*scratch)[:0], data) + + return *scratch, true +} + +// finishRemainder deals with the bytes left at the end of a value that do not form a complete UTF-8 sequence. +// +// The escapers hand these back rather than guessing, because in a streamed value a chunk may legitimately end inside +// a sequence. Once there is nothing left to read, though, the sequence is truncated, and that is an ill-formed input +// like any other: refused under [UTF8Strict], substituted per byte under [UTF8Replace], written as-is under +// [UTF8Passthrough]. +// +// It reports whether writing may continue. +func (w *commonWriter[T]) finishRemainder(remainder []byte) bool { + switch w.jw.escapePolicy() { + case UTF8Strict: + w.jw.SetErr(fmt.Errorf("%w: input ends inside a multi-byte sequence (% x): %w", + ErrInvalidUTF8, remainder, ErrDefaultWriter)) + + return false + + case UTF8Passthrough: + w.jw.writeBinary(remainder) + + default: // UTF8Replace + var buf [utf8.UTFMax * utf8.UTFMax]byte + w.jw.writeBinary(utf8x.Sanitize(buf[:0], remainder)) + } + + return w.jw.Ok() +} + +// checkUTF8Chunk is [commonWriter.checkUTF8] for one chunk of a streamed value. +// +// A chunk may legitimately end mid-sequence — the caller's reader knows nothing of rune boundaries — so a fault that +// is merely a truncated tail is tolerated here and left to the rune-stitching in [commonWriter.StringCopy], which +// reports it if the input really does end there. Any other fault is a genuine error. +func (w *commonWriter[T]) checkUTF8Chunk(chunk []byte) bool { + if w.jw.escapePolicy() != UTF8Strict { + return true + } + + idx := utf8x.FirstInvalid(chunk) + if idx < 0 || !utf8.FullRune(chunk[idx:]) { + return true + } + + w.jw.SetErr( + fmt.Errorf("%w at byte %d of this chunk: %w", ErrInvalidUTF8, idx, ErrDefaultWriter), + ) + + return false +} diff --git a/json/writers/default-writer/utf8_test.go b/json/writers/default-writer/utf8_test.go new file mode 100644 index 0000000..e76e7ba --- /dev/null +++ b/json/writers/default-writer/utf8_test.go @@ -0,0 +1,402 @@ +package writer + +import ( + "bytes" + "fmt" + "io" + "strings" + "testing" + "unicode/utf8" + + "github.com/go-openapi/testify/v2/assert" + "github.com/go-openapi/testify/v2/require" + + "github.com/go-openapi/core/json/internal/utf8x" + lexer "github.com/go-openapi/core/json/lexers/default-lexer" + "github.com/go-openapi/core/json/lexers/token" + "github.com/go-openapi/core/json/stores/values" +) + +// UTF-8 handling on the way OUT (see .claude/plans/utf8-validation.md, round 3). +// +// Before the policy existed the writers substituted U+FFFD for ill-formed input silently and unconditionally, with +// one inconsistency: a rune truncated at the very end of a StringCopy errored while every other fault was rewritten. +// These tests pin the three policies across all four writers and every entry point, and — because the same +// substitution rule is now claimed on both sides — cross-check the output against the lexer's utf8x.Sanitize. + +// utf8WriterCase is an input and what each policy must make of it. +type utf8WriterCase struct { + name string + in string + // valid inputs are written unchanged by every policy. + valid bool +} + +func utf8WriterCases() []utf8WriterCase { + return []utf8WriterCase{ + {name: "ascii", in: "plain text", valid: true}, + {name: "2-byte", in: "café", valid: true}, + {name: "3-byte", in: "\u65e5\u672c\u8a9e", valid: true}, + {name: "4-byte", in: "\U0001D11E", valid: true}, + {name: "needs escaping", in: "a\"b\\c\nd\te", valid: true}, + {name: "control chars", in: "a\x00b\x1fc", valid: true}, + + {name: "lone ff", in: "\xff"}, + {name: "lone continuation", in: "a\x80b"}, + {name: "overlong", in: "a\xc0\xafb"}, + {name: "encoded surrogate", in: "a\xed\xa0\x80b"}, + {name: "beyond U+10FFFF", in: "a\xf4\xbf\xbf\xbfb"}, + {name: "truncated at end", in: "abc\xe2\x82"}, + {name: "truncated then text", in: "a\xe0\xffb"}, + {name: "invalid amid escapes", in: "a\"b\xffc\nd"}, + } +} + +// utf8Writer is one writer implementation configured with a policy, reduced to what the tests need. +type utf8Writer struct { + name string + build func(w io.Writer, policy UTF8Policy) utf8WriterUnderTest +} + +type utf8WriterUnderTest interface { + String(string) + StringBytes([]byte) + StringRunes([]rune) + StringCopy(io.Reader) + Raw([]byte) + RawCopy(io.Reader) + Token(token.T) + VerbatimToken([]byte, token.T) + Err() error +} + +// flusher is implemented by the buffered writers; the unbuffered one needs no flush. +type flusher interface{ Flush() error } + +func utf8Writers() []utf8Writer { + return []utf8Writer{ + {name: "Unbuffered", build: func(w io.Writer, p UTF8Policy) utf8WriterUnderTest { + return NewUnbuffered(w, WithUnbufferedUTF8Policy(p)) + }}, + {name: "Buffered", build: func(w io.Writer, p UTF8Policy) utf8WriterUnderTest { + return NewBuffered(w, WithUTF8Policy(p)) + }}, + {name: "Buffered/tiny", build: func(w io.Writer, p UTF8Policy) utf8WriterUnderTest { + // a buffer smaller than the values forces flushes in the middle of escaping + return NewBuffered(w, WithUTF8Policy(p), WithBufferSize(minBufferSize)) + }}, + {name: "Indented", build: func(w io.Writer, p UTF8Policy) utf8WriterUnderTest { + return NewIndented(w, WithIndentBufferedOptions(WithUTF8Policy(p))) + }}, + {name: "YAML", build: func(w io.Writer, p UTF8Policy) utf8WriterUnderTest { + return NewYAML(w, WithYAMLBufferedOptions(WithUTF8Policy(p))) + }}, + } +} + +// writeWith runs one entry point and returns what reached the underlying writer. +func writeWith( + t *testing.T, + wr utf8Writer, + policy UTF8Policy, + in string, + call func(utf8WriterUnderTest, string), +) (string, error) { + t.Helper() + + var buf bytes.Buffer + w := wr.build(&buf, policy) + call(w, in) + if f, ok := w.(flusher); ok { + _ = f.Flush() + } + + return buf.String(), w.Err() +} + +// entryPoints are the caller-supplied ways in. Each is subject to the policy. +func entryPoints() []struct { + name string + quoted bool // the output is a quoted, escaped JSON string + call func(utf8WriterUnderTest, string) + skipFor func(utf8WriterCase) bool +} { + return []struct { + name string + quoted bool + call func(utf8WriterUnderTest, string) + skipFor func(utf8WriterCase) bool + }{ + {name: "String", quoted: true, call: func(w utf8WriterUnderTest, s string) { w.String(s) }}, + { + name: "StringBytes", + quoted: true, + call: func(w utf8WriterUnderTest, s string) { w.StringBytes([]byte(s)) }, + }, + {name: "StringCopy", quoted: true, call: func(w utf8WriterUnderTest, s string) { + w.StringCopy(strings.NewReader(s)) + }}, + {name: "Raw", call: func(w utf8WriterUnderTest, s string) { w.Raw([]byte(s)) }}, + { + name: "RawCopy", + call: func(w utf8WriterUnderTest, s string) { w.RawCopy(strings.NewReader(s)) }, + }, + } +} + +// TestWriterUTF8Strict pins the default: ill-formed input is refused, well-formed input is written. +func TestWriterUTF8Strict(t *testing.T) { + for _, wr := range utf8Writers() { + for _, ep := range entryPoints() { + for _, tc := range utf8WriterCases() { + t.Run(wr.name+"/"+ep.name+"/"+tc.name, func(t *testing.T) { + out, err := writeWith(t, wr, UTF8Strict, tc.in, ep.call) + + if !tc.valid { + require.Errorf(t, err, "ill-formed input must be refused, got %q", out) + assert.ErrorIs(t, err, ErrInvalidUTF8) + + return + } + + require.NoError(t, err) + assert.Truef(t, utf8.ValidString(out), "output must be valid UTF-8: %q", out) + if !ep.quoted { + assert.Equal(t, tc.in, out) + } + }) + } + } + } +} + +// TestWriterUTF8ReplaceAlwaysEmitsValidUTF8 pins the promise of UTF8Replace: it never errors on ill-formed input, and +// what comes out is always valid UTF-8 — on every writer and every entry point, including the streaming ones where a +// sequence can be split across reads. +func TestWriterUTF8ReplaceAlwaysEmitsValidUTF8(t *testing.T) { + for _, wr := range utf8Writers() { + for _, ep := range entryPoints() { + for _, tc := range utf8WriterCases() { + t.Run(wr.name+"/"+ep.name+"/"+tc.name, func(t *testing.T) { + out, err := writeWith(t, wr, UTF8Replace, tc.in, ep.call) + + require.NoErrorf(t, err, "UTF8Replace must not error on %q", tc.in) + assert.Truef(t, utf8.ValidString(out), + "UTF8Replace must emit valid UTF-8, got %q for input %q", out, tc.in) + if tc.valid && !ep.quoted { + assert.Equal(t, tc.in, out) + } + }) + } + } + } +} + +// TestWriterSubstitutionMatchesLexer is the point of sharing utf8x: the bytes a writer substitutes must be the bytes +// the lexer would have produced for the same input, so a value can round-trip through both without changing again. +func TestWriterSubstitutionMatchesLexer(t *testing.T) { + for _, wr := range utf8Writers() { + for _, tc := range utf8WriterCases() { + if tc.valid { + continue + } + t.Run(wr.name+"/"+tc.name, func(t *testing.T) { + want := string(utf8x.Sanitize(nil, []byte(tc.in))) + + out, err := writeWith(t, wr, UTF8Replace, tc.in, + func(w utf8WriterUnderTest, s string) { w.Raw([]byte(s)) }) + require.NoError(t, err) + assert.Equalf(t, want, out, "Raw under UTF8Replace must agree with utf8x.Sanitize") + + // the escaped path reaches the same text, wrapped in quotes and with escapes applied + quoted, err := writeWith(t, wr, UTF8Replace, tc.in, + func(w utf8WriterUnderTest, s string) { w.StringBytes([]byte(s)) }) + require.NoError(t, err) + assert.Equalf(t, expectedJSONString(want), quoted, + "the escaped path must substitute the same way") + }) + } + } +} + +// TestWriterUTF8Passthrough pins that passthrough really passes bytes through rather than quietly substituting, +// which is the only thing that distinguishes it from UTF8Replace. +func TestWriterUTF8Passthrough(t *testing.T) { + for _, wr := range utf8Writers() { + for _, tc := range utf8WriterCases() { + if tc.valid { + continue + } + t.Run(wr.name+"/"+tc.name, func(t *testing.T) { + out, err := writeWith(t, wr, UTF8Passthrough, tc.in, + func(w utf8WriterUnderTest, s string) { w.Raw([]byte(s)) }) + require.NoError(t, err) + assert.Equal(t, tc.in, out, "Raw must not touch the bytes") + + esc, err := writeWith(t, wr, UTF8Passthrough, tc.in, + func(w utf8WriterUnderTest, s string) { w.StringBytes([]byte(s)) }) + require.NoError(t, err) + assert.Falsef( + t, + utf8.ValidString(esc), + "passthrough must NOT sanitize: the ill-formed bytes should survive escaping, got %q", + esc, + ) + assert.NotContainsf(t, esc, string(utf8.RuneError), + "passthrough must not substitute U+FFFD, got %q", esc) + }) + } + } +} + +// TestWriterStreamingSplitSequence forces a multi-byte sequence to straddle a read boundary at every offset. A chunk +// ending mid-sequence is NOT an error, and the validator must not mistake it for one — nor miss a genuine fault that +// only shows up once the chunks are joined. +func TestWriterStreamingSplitSequence(t *testing.T) { + valid := strings.Repeat("a", 40) + "\u65e5\U0001D11E" + strings.Repeat("b", 40) + broken := strings.Repeat( + "a", + 40, + ) + "\xe6\x41" + strings.Repeat( + "b", + 40, + ) // lead byte then a non-continuation + + for _, wr := range utf8Writers() { + for chunk := 1; chunk <= 8; chunk++ { + t.Run(fmt.Sprintf("%s/chunk=%d", wr.name, chunk), func(t *testing.T) { + for _, ep := range []string{"StringCopy", "RawCopy"} { + call := func(w utf8WriterUnderTest, s string) { w.StringCopy(&chunkReader{data: []byte(s), chunk: chunk}) } + if ep == "RawCopy" { + call = func(w utf8WriterUnderTest, s string) { + w.RawCopy(&chunkReader{data: []byte(s), chunk: chunk}) + } + } + + out, err := writeWith(t, wr, UTF8Strict, valid, call) + require.NoErrorf( + t, + err, + "%s: a sequence split across reads is not an error", + ep, + ) + assert.Truef(t, utf8.ValidString(out), "%s: %q", ep, out) + + _, err = writeWith(t, wr, UTF8Strict, broken, call) + require.Errorf( + t, + err, + "%s: a fault spanning the read boundary must be caught", + ep, + ) + + out, err = writeWith(t, wr, UTF8Replace, broken, call) + require.NoError(t, err) + assert.Truef(t, utf8.ValidString(out), "%s: %q", ep, out) + } + }) + } + } +} + +// TestWriterTrustsTokenValues pins the deliberate loophole: a value coming from a lexer token is NOT re-validated, +// because the lexers already guarantee valid UTF-8. A lexer relaxed to UTF8Passthrough can therefore hand the writer +// ill-formed bytes that a strict writer will emit — the documented cost of not paying for a second check. +func TestWriterTrustsTokenValues(t *testing.T) { + const doc = "[\"a\xffb\"]" + + // the lexer must be relaxed for such a token to exist at all + lx := lexer.NewWithBytes([]byte(doc), lexer.WithUTF8Policy(lexer.UTF8Passthrough)) + var value token.T + for tok := range lx.Tokens() { + if tok.Kind() == token.String { + value = token.MakeWithValue(token.String, bytes.Clone(tok.Value())) + } + } + require.NoError(t, lx.Err()) + require.False( + t, + utf8.Valid(value.Value()), + "the relaxed lexer should have produced ill-formed bytes", + ) + + // Token: not re-validated, so no error — but the escaper still decodes as it goes, so the ill-formed byte is + // silently substituted. The loophole is the SILENCE, not invalid output. + var buf bytes.Buffer + w := NewUnbuffered(&buf) // UTF8Strict + w.Token(value) + require.NoError(t, w.Err(), "token values are trusted, not re-validated") + assert.Equal(t, "\"a\ufffdb\"", buf.String(), + "the escaper substitutes as it goes, so the output stays valid UTF-8") + + // VerbatimToken writes the raw value with no escaping at all, so there ill-formed bytes really do reach the + // output. This is the sharp edge of trusting tokens and is documented as such. + var verbatim bytes.Buffer + vw := NewUnbuffered(&verbatim) // UTF8Strict + vw.VerbatimToken(nil, value) + require.NoError(t, vw.Err()) + assert.Equal(t, "\"a\xffb\"", verbatim.String(), + "verbatim writing is byte-for-byte by definition: nothing inspects or rewrites the value") + + // VerbatimValue is NOT in the trusted group: it renders through the ordinary string path + var vv bytes.Buffer + vvw := NewUnbuffered(&vv) + vvw.VerbatimValue(values.MakeVerbatimValue(nil, values.MakeStringValue(string(value.Value())))) + require.ErrorIs(t, vvw.Err(), ErrInvalidUTF8, + "VerbatimValue goes through the string path, so it is checked like any caller data") + + // the same bytes offered as caller data ARE checked + var caller bytes.Buffer + cw := NewUnbuffered(&caller) + cw.StringBytes(value.Value()) + require.ErrorIs(t, cw.Err(), ErrInvalidUTF8) +} + +// TestWriterStringRunesRejectsNonScalar pins that a []rune carrying a surrogate or an out-of-range value is refused +// under UTF8Strict rather than silently encoded as U+FFFD by utf8.AppendRune. +func TestWriterStringRunesRejectsNonScalar(t *testing.T) { + for _, tc := range []struct { + name string + runes []rune + }{ + {name: "surrogate", runes: []rune{'a', 0xD800, 'b'}}, + {name: "beyond max", runes: []rune{'a', 0x110000, 'b'}}, + {name: "negative", runes: []rune{'a', -1, 'b'}}, + } { + t.Run(tc.name, func(t *testing.T) { + var buf bytes.Buffer + w := NewUnbuffered(&buf) + w.StringRunes(tc.runes) + require.ErrorIs(t, w.Err(), ErrInvalidUTF8) + + var replaced bytes.Buffer + rw := NewUnbuffered(&replaced, WithUnbufferedUTF8Policy(UTF8Replace)) + rw.StringRunes(tc.runes) + require.NoError(t, rw.Err()) + assert.Equal(t, "\"a�b\"", replaced.String()) + }) + } +} + +// TestWriterDefaultIsStrict pins the default so it cannot regress to the old silent substitution. +func TestWriterDefaultIsStrict(t *testing.T) { + for _, wr := range utf8Writers() { + t.Run(wr.name, func(t *testing.T) { + var buf bytes.Buffer + w := wr.build(&buf, UTF8Strict) // what a caller passing no option gets + w.StringBytes([]byte("a\xffb")) + require.ErrorIs(t, w.Err(), ErrInvalidUTF8) + }) + } + + // and literally with no option at all + var buf bytes.Buffer + u := NewUnbuffered(&buf) + u.StringBytes([]byte("a\xffb")) + require.ErrorIs(t, u.Err(), ErrInvalidUTF8) + + var buf2 bytes.Buffer + b := NewBuffered(&buf2) + b.StringBytes([]byte("a\xffb")) + require.ErrorIs(t, b.Err(), ErrInvalidUTF8) +} From f8b584a5bb0d009d078aa6e24824f55c531bd1e4 Mon Sep 17 00:00:00 2001 From: Frederic BIDON Date: Thu, 30 Jul 2026 22:41:02 +0200 Subject: [PATCH 07/10] fix(yaml-lexer): stop a self-referential merge key recursing forever MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A mapping whose own "<<" merge key aliases itself e: &b <<: *b made resolveMapping recurse until the stack overflowed. That is a FATAL runtime error, not a Go panic: safeParse cannot recover it, and neither WithMaxTokens nor WithMaxContainerStack applies, because the recursion happens while flattening merge entries — before any token is emitted and outside the container walk. The process simply dies, which is what CI reported as "fuzzing process hung or terminated unexpectedly while minimizing". mergeEntries did have an alias-cycle guard, but its scope was wrong. It spanned only the resolution of the alias to a node list: l.expanding[name] = true entries := l.mergeEntries(target) // a MappingNode just returns its values delete(l.expanding, name) // released here... and the expansion of those entries happens afterwards, back in resolveMapping's add(), which walks them and meets the same "<<" entry with the guard already released. So the guard covered the step that cannot recurse and left the one that does unprotected. The guard now spans the expansion, keyed on the merge SOURCE node rather than the anchor name. Node identity also catches a cycle formed through a chain of aliases or a merge sequence, and still lets the same anchor be merged in sibling positions, which is redundant but legal. Note on the reported input: the crasher CI recorded, []byte("e: &b\n <<: *b\n c ") does NOT reproduce, because goccy rejects it at parse time ("non-map value is specified") over the trailing "c ". The kill happened during minimization, on a mutation that does parse — Go records the pre-minimization input, so the file points next to the culprit rather than at it. Both are seeded now. A multi-node merge cycle turns out to be unreachable: closing one needs a forward reference, and an anchor must be defined before it is aliased, so it fails earlier as ErrUnknownAnchor. Self-reference is the only shape that gets there. TestMergeCycleNeedsSelfReference records that, so the narrow fix does not look like an oversight later. Tests: cycle shapes rejected with ErrAliasCycle; legitimate merges (repeated anchors, sequences, 200-deep acyclic chains) still resolved, since a guard that fails to unmark on the way out would break them; the pre-existing walkAlias guard pinned as untouched. Each runs on a watchdog so a regression fails the test rather than taking the suite down. 1M fuzz executions clean, where it previously died in under 3 seconds. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Frederic BIDON --- json/lexers/yaml-lexer/fuzz_test.go | 8 + json/lexers/yaml-lexer/lexer.go | 6 +- json/lexers/yaml-lexer/merge_cycle_test.go | 181 ++++++++++++++++++ .../testdata/fuzz/FuzzYL/97af9da7c4afa4dc | 2 + json/lexers/yaml-lexer/walk.go | 18 ++ 5 files changed, 213 insertions(+), 2 deletions(-) create mode 100644 json/lexers/yaml-lexer/merge_cycle_test.go create mode 100644 json/lexers/yaml-lexer/testdata/fuzz/FuzzYL/97af9da7c4afa4dc diff --git a/json/lexers/yaml-lexer/fuzz_test.go b/json/lexers/yaml-lexer/fuzz_test.go index 5bfbe3d..0e9ed21 100644 --- a/json/lexers/yaml-lexer/fuzz_test.go +++ b/json/lexers/yaml-lexer/fuzz_test.go @@ -173,6 +173,14 @@ func FuzzYL(f *testing.F) { "123: ok", "? [a, b]\n: c", "[", "{", "a: [", ": :", "\x00", "\xff\xfe", "\xef\xbb\xbfa: 1", "a:\t1", "a: !!str 7\nb: !!bool yes", + // merge-key cycles: a mapping whose own "<<" aliases itself used to recurse forever in + // resolveMapping and kill the process with a fatal stack overflow (CI: "fuzzing process + // hung or terminated unexpectedly"). + "e: &b\n <<: *b\n", + "e: &b\n <<: *b\n c ", + "e: &b\n a: 1\n <<: *b\n c: 3\n", + "e: &b\n <<: [*b]\n", + "a: &x\n - *x\n", } for _, s := range seeds { f.Add([]byte(s)) diff --git a/json/lexers/yaml-lexer/lexer.go b/json/lexers/yaml-lexer/lexer.go index ee8f09a..0050e8a 100644 --- a/json/lexers/yaml-lexer/lexer.go +++ b/json/lexers/yaml-lexer/lexer.go @@ -45,10 +45,12 @@ type YL struct { ptr expressions.Pointer ptrFrames []ptrFrame - // build-time transient state (nil outside build): anchor table and the set of anchors - // currently being expanded (alias cycle guard). See walk.go. + // build-time transient state (nil outside build): anchor table and the two alias-cycle + // guards — the set of anchors currently being expanded, and the set of "<<" merge sources + // whose entries are currently being flattened. See walk.go. anchors map[string]ast.Node expanding map[string]bool + merging map[ast.Node]bool } // emit is one entry in the materialised token stream: the token plus the source position diff --git a/json/lexers/yaml-lexer/merge_cycle_test.go b/json/lexers/yaml-lexer/merge_cycle_test.go new file mode 100644 index 0000000..02a333f --- /dev/null +++ b/json/lexers/yaml-lexer/merge_cycle_test.go @@ -0,0 +1,181 @@ +package lexer_test + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/go-openapi/testify/v2/assert" + "github.com/go-openapi/testify/v2/require" + + yamllexer "github.com/go-openapi/core/json/lexers/yaml-lexer" +) + +// Merge-key ("<<") cycles. +// +// A mapping whose own merge key aliases itself resolves to its own entries, which contain that same merge key. The +// flattening in resolveMapping therefore recursed forever and the process died with a fatal stack overflow — not a +// Go panic, so nothing could recover it and no circuit breaker (WithMaxTokens, WithMaxContainerStack) applied. CI +// found it as "fuzzing process hung or terminated unexpectedly". +// +// mergeEntries did have an anchor guard, but it only spanned resolving the alias to a node list — which does not +// recurse — and was released before those entries were expanded, which is where the cycle closes. + +// runBounded lexes data on a watchdog so a regression shows up as a failure rather than taking the suite down with +// it. A stack overflow cannot be recovered, so this cannot catch that directly; it does catch a plain runaway. +func runBounded(t *testing.T, data string, opts ...yamllexer.Option) (int, error) { + t.Helper() + + type result struct { + n int + err error + } + done := make(chan result, 1) + + go func() { + l := yamllexer.NewWithBytes([]byte(data), opts...) + n := 0 + for range l.Tokens() { + n++ + if n > 100000 { + break + } + } + done <- result{n: n, err: l.Err()} + }() + + select { + case r := <-done: + return r.n, r.err + case <-time.After(10 * time.Second): + t.Fatalf("lexing did not terminate for %q", data) + + return 0, nil + } +} + +func TestMergeKeyCycleIsRejected(t *testing.T) { + cases := []struct { + name string + in string + }{ + { + name: "mapping merges itself", + in: "e: &b\n <<: *b\n", + }, + { + name: "mapping merges itself among other keys", + in: "e: &b\n a: 1\n <<: *b\n c: 3\n", + }, + { + name: "self-merge inside a merge sequence", + in: "e: &b\n <<: [*b]\n", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := runBounded(t, tc.in) + require.Error(t, err, "a merge-key cycle must be reported, not expanded forever") + assert.ErrorIs(t, err, yamllexer.ErrAliasCycle) + }) + } +} + +// TestMergeCycleNeedsSelfReference records WHY the self-merge is the only shape that reaches the cycle guard: an +// anchor must be defined before it is aliased, so a multi-node merge cycle cannot be written at all — closing it +// needs a forward reference, which fails earlier as an unknown anchor. That is not a second bug; it is the reason +// the fix only has to handle self-reference. +func TestMergeCycleNeedsSelfReference(t *testing.T) { + for _, tc := range []struct { + name string + in string + }{ + {name: "two-step", in: "x: &x\n <<: *y\ny: &y\n <<: *x\n"}, + {name: "three-step", in: "a: &a\n <<: *b\nb: &b\n <<: *c\nc: &c\n <<: *a\n"}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := runBounded(t, tc.in) + require.Error(t, err) + assert.ErrorIs( + t, + err, + yamllexer.ErrUnknownAnchor, + "the cycle cannot form: the second anchor is not defined yet at the point it is aliased", + ) + }) + } +} + +// TestMergeKeyLegitimateUsesStillWork guards the other side of the fix: the guard keys on the merge SOURCE node, so +// merging one anchor several times — redundant but perfectly legal YAML — must not be mistaken for a cycle. +// +// (Repeating "<<" twice as two separate keys in the same mapping is not in this table: goccy rejects that at parse +// time as a duplicate key, before the merge resolution ever runs.) +func TestMergeKeyLegitimateUsesStillWork(t *testing.T) { + cases := []struct { + name string + in string + wantKey string // a key that must survive the merge + }{ + { + name: "simple merge", + in: "base: &b {a: 1, b: 2}\nderived:\n <<: *b\n c: 3\n", + wantKey: "a", + }, + { + name: "same anchor merged into two siblings", + in: "base: &b {a: 1}\nx:\n <<: *b\ny:\n <<: *b\n", + wantKey: "a", + }, + { + name: "same anchor twice in a merge sequence", + in: "base: &b {a: 1}\nx:\n <<: [*b, *b]\n", + wantKey: "a", + }, + { + name: "chained merges, no cycle", + in: "a: &a {x: 1}\nb: &b\n <<: *a\n y: 2\nc:\n <<: *b\n z: 3\n", + wantKey: "x", + }, + { + name: "merge sequence of distinct anchors", + in: "p: &p {a: 1}\nq: &q {b: 2}\nr:\n <<: [*p, *q]\n", + wantKey: "b", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + n, err := runBounded(t, tc.in) + require.NoError(t, err, "a legal merge must not be mistaken for a cycle") + assert.Positive(t, n) + }) + } +} + +// TestAliasCycleWithoutMergeStillRejected pins the pre-existing guard in walkAlias, which was already correct: only +// the merge path was broken, and the fix must not have disturbed it. +func TestAliasCycleWithoutMergeStillRejected(t *testing.T) { + // a sequence that contains an alias to itself + _, err := runBounded(t, "a: &x\n - *x\n") + require.Error(t, err) + assert.ErrorIs(t, err, yamllexer.ErrAliasCycle) +} + +// TestDeepMergeChainTerminates checks that a long but ACYCLIC merge chain is still resolved rather than tripping the +// cycle guard — the guard is a stack, so it must unmark on the way out. +func TestDeepMergeChainTerminates(t *testing.T) { + const depth = 200 + + var b strings.Builder + b.WriteString("k0: &a0 {v: 1}\n") + for i := 1; i <= depth; i++ { + fmt.Fprintf(&b, "k%d: &a%d\n <<: *a%d\n", i, i, i-1) + } + + n, err := runBounded(t, b.String()) + require.NoError(t, err) + assert.Positive(t, n) +} diff --git a/json/lexers/yaml-lexer/testdata/fuzz/FuzzYL/97af9da7c4afa4dc b/json/lexers/yaml-lexer/testdata/fuzz/FuzzYL/97af9da7c4afa4dc new file mode 100644 index 0000000..4117223 --- /dev/null +++ b/json/lexers/yaml-lexer/testdata/fuzz/FuzzYL/97af9da7c4afa4dc @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("e: &b\n <<: *b\n c ") diff --git a/json/lexers/yaml-lexer/walk.go b/json/lexers/yaml-lexer/walk.go index 277de2e..cef4b84 100644 --- a/json/lexers/yaml-lexer/walk.go +++ b/json/lexers/yaml-lexer/walk.go @@ -43,9 +43,11 @@ func (l *YL) build() { l.anchors = map[string]ast.Node{} l.expanding = map[string]bool{} + l.merging = map[ast.Node]bool{} l.walkValue(f.Docs[0].Body, 0) l.anchors = nil l.expanding = nil + l.merging = nil } // safeParse runs the goccy parser, converting a recoverable panic into an error so a @@ -209,7 +211,23 @@ func (l *YL) resolveMapping(values []*ast.MappingValueNode) []entryKV { return } if _, isMerge := mv.Key.(*ast.MergeKeyNode); isMerge { + // The cycle guard has to span the EXPANSION of the merged entries, not just the + // resolution of the alias to a node list: a mapping whose own merge key aliases + // itself (`e: &b\n <<: *b`) resolves to its own entries, which contain that same + // merge key, so add() would recurse forever. mergeEntries' anchor guard is already + // released by then — it only covers the lookup, which does not recurse. + // + // Keying on the merge source node rather than the anchor name also catches cycles + // formed through a chain of aliases or a merge sequence, and lets the same anchor be + // merged twice in sibling positions, which is redundant but legal. + if l.merging[mv.Value] { + l.err = ErrAliasCycle + + return + } + l.merging[mv.Value] = true add(l.mergeEntries(mv.Value), true) + delete(l.merging, mv.Value) continue } From d462cd2d83d67b6d8b6c6fdc3154b5f678891654 Mon Sep 17 00:00:00 2001 From: Frederic BIDON Date: Thu, 30 Jul 2026 23:51:06 +0200 Subject: [PATCH 08/10] build: never translate line endings for fixtures, goldens or generated asm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Conformance fixtures are DATA: their exact bytes are the test. Without a .gitattributes, a checkout with core.autocrlf=true — the Windows default — rewrites every LF to CRLF and silently changes what is being measured. A JSON fixture that must be REJECTED can start being accepted; a golden file mismatches; and a YAML fixture that hinges on a trailing space or a hard tab becomes a different document, so the failure reads as a lexer bug rather than a checkout artefact. This is why the JSON conformance harness fails on Windows. Marks testdata/**, *.golden and generated *.s as -text. The assembly matters for the same reason: it is checked in and has to match what `go generate` produces byte for byte. No working-tree churn on a LF checkout — the files are already LF, so this only stops them from ever being translated. Verified with git check-attr over the JSON suite fixtures, the i_ behavior golden and both avo-generated kernels. Identical to the copy on the YAML conformance branch, so the two resolve trivially when that work lands. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Frederic BIDON --- .gitattributes | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..8b97c9c --- /dev/null +++ b/.gitattributes @@ -0,0 +1,13 @@ +# Conformance fixtures and golden files are DATA, not source: their exact bytes ARE the test. +# +# Without this, a checkout with core.autocrlf=true (the Windows default) rewrites every LF to +# CRLF and silently changes what is being tested — a YAML fixture that hinges on a trailing +# space or a tab becomes a different document, a JSON fixture that must be rejected may start +# being accepted, and every golden file mismatches. A harness may normalise line endings +# defensively as well, but the fixtures should never have been translated in the first place. +testdata/** -text +**/testdata/** -text +*.golden -text + +# Generated assembly is checked in and must match what `go generate` produces byte for byte. +*.s -text From 7a23f01b36d19db9c3f447e79f967ce2a0314b84 Mon Sep 17 00:00:00 2001 From: Frederic BIDON Date: Fri, 31 Jul 2026 07:08:44 +0200 Subject: [PATCH 09/10] style: gofmt the avo generator A trailing blank line left by the UTF-8 round-1 edit to this file. The project linter skips _-prefixed directories, so only plain gofmt notices it. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Frederic BIDON --- json/lexers/default-lexer/internal/strscan/_asm/asm.go | 1 - 1 file changed, 1 deletion(-) diff --git a/json/lexers/default-lexer/internal/strscan/_asm/asm.go b/json/lexers/default-lexer/internal/strscan/_asm/asm.go index 52606c5..5de7b83 100644 --- a/json/lexers/default-lexer/internal/strscan/_asm/asm.go +++ b/json/lexers/default-lexer/internal/strscan/_asm/asm.go @@ -133,4 +133,3 @@ func storeNonASCII(nonascii reg.GPVirtual) { SETNE(flag) Store(flag, ReturnIndex(1)) } - From 01204ceee34e1bc5a91e7a5ea876254a6699ae02 Mon Sep 17 00:00:00 2001 From: Frederic BIDON Date: Fri, 31 Jul 2026 07:09:18 +0200 Subject: [PATCH 10/10] test(json-lexers): make the conformance harness line-ending independent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Belt-and-braces to the .gitattributes fix: that stops git translating the fixtures, this makes the harness right even when a suite arrives some other way — an unmarked clone, a zip, a copy through a Windows tool. A conformance fixture's exact bytes ARE the test, so a translated checkout does not just fail noisily: a document that must be REJECTED can start being accepted, and the failure reads as a lexer bug rather than a checkout artefact. Fixtures and the i_ behavior golden are now normalised on read. A lone CR is deliberately left alone — it is a meaningful byte inside a JSON document (an unescaped control character a conforming parser must reject), so collapsing it would change the very thing several n_ cases test. Only CRLF collapses. What licenses that: TestConformanceFixturesAreStoredWithLF asserts no vendored fixture carries a raw CR, so every CR that could turn up has been introduced by translation. If upstream ever ships one, the test fails and the normalisation gets revisited instead of silently eating it. The property is tested rather than assumed: every fixture is converted to CRLF and each lexer mode must return the same accept/reject verdict as for the LF original. Mutation-checked — stubbing normalizeLineEndings to the identity fails it, so it has teeth. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Frederic BIDON --- .../conformance_lineendings_test.go | 129 ++++++++++++++++++ json/lexers/default-lexer/conformance_test.go | 12 +- 2 files changed, 137 insertions(+), 4 deletions(-) create mode 100644 json/lexers/default-lexer/conformance_lineendings_test.go diff --git a/json/lexers/default-lexer/conformance_lineendings_test.go b/json/lexers/default-lexer/conformance_lineendings_test.go new file mode 100644 index 0000000..b102a40 --- /dev/null +++ b/json/lexers/default-lexer/conformance_lineendings_test.go @@ -0,0 +1,129 @@ +package lexer + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/go-openapi/testify/v2/assert" + "github.com/go-openapi/testify/v2/require" +) + +// Line-ending independence for the JSON conformance fixtures. +// +// A conformance fixture's exact bytes ARE the test. A checkout with core.autocrlf=true — the Windows default — +// rewrites every LF to CRLF, and then a document that must be REJECTED can start being accepted (or the reverse), +// with the failure reading as a lexer bug rather than a checkout artefact. +// +// .gitattributes marks testdata/** as -text so git never translates them, and that is the real fix. These tests +// cover the other half: a suite that arrived some other way — an unmarked clone, a zip, a copy through a Windows +// tool — must still measure the same documents. + +// normalizeLineEndings turns CRLF into LF. +// +// A lone CR is deliberately left alone: it is a meaningful byte inside a JSON document (an unescaped control +// character, which a conforming parser must reject), so collapsing it would change the very thing several n_ cases +// test. TestConformanceFixturesAreStoredWithLF pins that no vendored fixture carries a raw CR, which is what makes +// this safe: every CR that could appear has been introduced by translation. +func normalizeLineEndings(data []byte) []byte { + if !bytes.Contains(data, []byte("\r")) { + return data + } + + return bytes.ReplaceAll(data, []byte("\r\n"), []byte("\n")) +} + +// TestConformanceFixturesAreStoredWithLF guards the fixtures as committed. It is also what licenses the +// normalisation above: if upstream ever ships a fixture containing a genuine CR, this fails and the normalisation +// has to be revisited rather than silently eating it. +func TestConformanceFixturesAreStoredWithLF(t *testing.T) { + root := filepath.Join(currentDir(), "..", "testdata", "JSONTestSuite") + + var withCR []string + err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".json") { + return nil + } + + //nolint:gosec // path comes from walking our own vendored fixtures + data, rerr := os.ReadFile(path) + if rerr != nil { + return rerr + } + if bytes.Contains(data, []byte("\r")) { + rel, _ := filepath.Rel(root, path) + withCR = append(withCR, rel) + } + + return nil + }) + require.NoError(t, err) + + assert.Emptyf( + t, + withCR, + "fixtures must be stored without carriage returns (see .gitattributes); found CR in %v", + withCR, + ) +} + +func TestNormalizeLineEndings(t *testing.T) { + cases := []struct { + name string + in string + want string + }{ + {name: "no CR at all", in: "{\n}\n", want: "{\n}\n"}, + {name: "CRLF", in: "{\r\n}\r\n", want: "{\n}\n"}, + {name: "mixed", in: "[\r\n1,\n2\r\n]", want: "[\n1,\n2\n]"}, + {name: "lone CR is preserved", in: "[\"a\rb\"]", want: "[\"a\rb\"]"}, + {name: "empty", in: "", want: ""}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, string(normalizeLineEndings([]byte(tc.in)))) + }) + } +} + +// TestConformanceVerdictsSurviveCRLF is the property that matters: converting every fixture to CRLF must not change +// a single accept/reject verdict, in any lexer mode. +// +// Without the normalisation this fails — a document whose only fault is an unescaped newline inside a string, for +// instance, is a different document once that newline becomes CRLF. +func TestConformanceVerdictsSurviveCRLF(t *testing.T) { + dir := filepath.Join(currentDir(), "..", "testdata", "JSONTestSuite", "test_parsing") + entries, err := os.ReadDir(dir) + require.NoError(t, err) + require.NotEmpty(t, entries) + + modes := conformanceModes() + + for _, e := range entries { + name := e.Name() + if e.IsDir() || !strings.HasSuffix(name, ".json") { + continue + } + + raw, rerr := os.ReadFile(filepath.Join(dir, name)) + require.NoError(t, rerr) + + lf := normalizeLineEndings(raw) + crlf := normalizeLineEndings(bytes.ReplaceAll(lf, []byte("\n"), []byte("\r\n"))) + + require.Equalf(t, string(lf), string(crlf), + "%s: normalising a CRLF copy must reproduce the LF original", name) + + maximum := len(lf) + 16 + for _, m := range modes { + assert.Equalf(t, m.run(lf, maximum).accepted, m.run(crlf, maximum).accepted, + "%s/%s: the verdict changed for a CRLF checkout", name, m.name) + } + } +} diff --git a/json/lexers/default-lexer/conformance_test.go b/json/lexers/default-lexer/conformance_test.go index fd2d97f..6f2024b 100644 --- a/json/lexers/default-lexer/conformance_test.go +++ b/json/lexers/default-lexer/conformance_test.go @@ -46,10 +46,13 @@ func TestConformanceParsing(t *testing.T) { continue } - data, rerr := os.ReadFile(filepath.Join(dir, name)) + raw, rerr := os.ReadFile(filepath.Join(dir, name)) if rerr != nil { t.Fatalf("cannot read %s: %v", name, rerr) } + // A fixture's exact bytes ARE the test, so a checkout that translated them must not change the verdict: + // see normalizeLineEndings and TestConformanceFixturesAreStoredWithLF. + data := normalizeLineEndings(raw) maximum := len(data) + 16 // generous upper bound on token count var want byte @@ -191,16 +194,17 @@ func checkIBehaviorGolden(t *testing.T, iReport []string) { return } - want, err := os.ReadFile(golden) + raw, err := os.ReadFile(golden) if err != nil { t.Fatalf("cannot read %s (run with -update-golden to create it): %v", golden, err) } + want := string(normalizeLineEndings(raw)) // the golden records verdicts, not bytes on a line - if string(want) == got { + if want == got { return } - wantLines := strings.Split(strings.TrimRight(string(want), "\n"), "\n") + wantLines := strings.Split(strings.TrimRight(want, "\n"), "\n") gotLines := strings.Split(strings.TrimRight(got, "\n"), "\n") t.Errorf("implementation-defined behavior changed (%d recorded cases, %d now).\n"+ "If the change is intended, re-run with -update-golden and review the diff.", len(wantLines), len(gotLines))