From 9f2c5b5a26ee9855487702d4cdda0e4d4520f812 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sat, 15 Aug 2026 18:09:20 +0100 Subject: [PATCH 1/3] Add JSON content detection --- README.md | 18 ++- benchmark_test.go | 9 ++ fuzz_test.go | 22 ++- json.go | 332 ++++++++++++++++++++++++++++++++++++++++++++++ json_test.go | 177 ++++++++++++++++++++++++ magic.go | 6 + magic_test.go | 1 + 7 files changed, 558 insertions(+), 7 deletions(-) create mode 100644 json.go create mode 100644 json_test.go diff --git a/README.md b/README.md index 8512010..8814cb8 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ The format registry contains: - ZIP, TAR, native PHAR, ar, gzip, bzip2, xz, zstd, PDF, CFBF, PNG, JPEG, and GIF - ELF, Mach-O (thin and universal), PE/COFF, and WebAssembly -- plain text, HTML, XML, and SVG +- plain text, JSON, HTML, XML, and SVG Detection uses bytes only. ZIP-based package types such as JAR, wheel, and NuGet remain `zip`, and compressed payloads are not opened. A `CA FE BA BE` @@ -71,6 +71,11 @@ carriage return, and escape are the permitted C0 controls. Other C0 controls classify the input as binary. Invalid UTF-8 without a NUL is unknown with `ReasonInvalidText`; callers that need Latin-1 can apply their own fallback. +JSON detection validates the complete input, including arrays and scalar +top-level values. Surrounding JSON whitespace is accepted. A bounded prefix +that contains valid or incomplete JSON syntax reports JSON with +`ReasonNeedMore` because later bytes can complete or invalidate the value. + HTML, XML, and SVG signatures supply format metadata before the shared text rules run. The metadata remains present if malformed or control-bearing input is classified as unknown or binary. @@ -78,10 +83,10 @@ is classified as unknown or binary. ## Performance The detector performs no allocations for the supplied fixtures. On an Apple -M1 Pro with Go 1.26.5, a 4 KiB text input takes about 1.5 microseconds, the -mixed 4 KiB fixture corpus averages about 0.77 microseconds per call, and a +M1 Pro with Go 1.26.6, a 4 KiB text input takes about 1.5 microseconds, the +mixed 4 KiB fixture corpus averages about 2.1 microseconds per call, and a 1 MiB text input takes about 0.35 milliseconds. Importing and calling the -package adds 16,640 bytes to a stripped minimal binary. +package adds about 20 KiB to a stripped minimal binary. Run the package benchmarks on the target machine: @@ -91,8 +96,9 @@ go test -run '^$' -bench . -benchmem Fixed signatures inspect at most 512 bytes, while native PHAR detection searches for the end of the PHP stub and validates the manifest and stored -payload bounds. Text validation is linear in the supplied byte count and uses -fixed auxiliary memory. +payload bounds. JSON parsing and text validation are linear in the supplied +byte count. JSON parsing uses stack space proportional to nesting depth; text +validation uses fixed auxiliary memory. ## Provenance diff --git a/benchmark_test.go b/benchmark_test.go index 3dbf311..91151bd 100644 --- a/benchmark_test.go +++ b/benchmark_test.go @@ -8,11 +8,13 @@ import ( var benchmarkResult Result var benchmarkText1MiB = bytes.Repeat([]byte{'x'}, 1<<20) +var benchmarkJSON4KiB = jsonFixture(4096) var benchmarkCorpus = [][]byte{ padFixture([]byte("package magic\n\nfunc Detect(data []byte) Result { return Result{} }\n")), padFixture([]byte("\xef\xbb\xbfUnicode text: héllo, 世界\n")), padFixture([]byte("")), + benchmarkJSON4KiB, padFixture([]byte("PK\x03\x04")), padFixture([]byte("\x89PNG\r\n\x1a\n")), padFixture([]byte{0xff}), @@ -72,3 +74,10 @@ func padFixture(prefix []byte) []byte { } return append(bytes.Clone(prefix), bytes.Repeat([]byte{'x'}, 4096-len(prefix))...) } + +func jsonFixture(size int) []byte { + data := bytes.Repeat([]byte{'x'}, size) + data[0] = '"' + data[len(data)-1] = '"' + return data +} diff --git a/fuzz_test.go b/fuzz_test.go index 90be752..f461661 100644 --- a/fuzz_test.go +++ b/fuzz_test.go @@ -1,6 +1,10 @@ package magic -import "testing" +import ( + "encoding/json" + "testing" + "unicode/utf8" +) func FuzzDetect(f *testing.F) { seeds := [][]byte{ @@ -26,6 +30,12 @@ func FuzzDetect(f *testing.F) { []byte(""), []byte(""), []byte(""), + []byte(`{"schemaVersion":2}`), + []byte(`[1,"two",false,null]`), + []byte(`"value"`), + []byte(`1e+2`), + []byte(`{"truncated"`), + []byte(`1e+`), } for _, seed := range seeds { f.Add(seed) @@ -40,10 +50,17 @@ func FuzzDetect(f *testing.F) { t.Fatalf("Detect is not deterministic: %#v then %#v", first, second) } assertResultInvariants(t, first, false, len(data)) + if got, expect := first.Format == FormatJSON, json.Valid(data) && utf8.Valid(data); got != expect { + t.Fatalf("Detect JSON match = %v, want %v for %x", got, expect, data) + } prefix := DetectPrefix(data) if len(data) > 0 { expectedPrefix := first + if parseJSON(data) == jsonIncomplete { + expectedPrefix.Format = FormatJSON + expectedPrefix.MIME = mimeJSON + } if prefix.Reason == ReasonNeedMore { expectedPrefix.Reason = ReasonNeedMore } @@ -68,6 +85,9 @@ func FuzzDetectPrefix(f *testing.F) { []byte("\x89PNG\r\n"), []byte("\x89PNG\r\n\x1a\n"), []byte(""), + []byte(`{"schemaVersion":2}`), + []byte(`{"truncated"`), + []byte(`1e+`), } for _, seed := range seeds { f.Add(seed) diff --git a/json.go b/json.go new file mode 100644 index 0000000..e1e04c9 --- /dev/null +++ b/json.go @@ -0,0 +1,332 @@ +package magic + +import "unicode/utf8" + +type jsonParseResult uint8 + +const ( + jsonInvalid jsonParseResult = iota + jsonIncomplete + jsonComplete + + jsonControlLimit = 0x20 + jsonMaximumDepth = 10000 +) + +type jsonParser struct { + data []byte + offset int + depth int +} + +func isJSON(data []byte, prefix bool) bool { + result := parseJSON(data) + return result == jsonComplete || prefix && result == jsonIncomplete +} + +func parseJSON(data []byte) jsonParseResult { + parser := jsonParser{data: data} + parser.skipWhitespace() + if parser.offset == len(parser.data) { + return jsonInvalid + } + + result := parser.parseValue() + if result != jsonComplete { + return result + } + parser.skipWhitespace() + if parser.offset != len(parser.data) { + return jsonInvalid + } + return jsonComplete +} + +func (parser *jsonParser) parseValue() jsonParseResult { + if parser.offset == len(parser.data) { + return jsonIncomplete + } + + switch parser.data[parser.offset] { + case '{': + if parser.depth == jsonMaximumDepth { + return jsonInvalid + } + parser.depth++ + result := parser.parseObject() + parser.depth-- + return result + case '[': + if parser.depth == jsonMaximumDepth { + return jsonInvalid + } + parser.depth++ + result := parser.parseArray() + parser.depth-- + return result + case '"': + return parser.parseString() + case 't': + return parser.parseLiteral("true") + case 'f': + return parser.parseLiteral("false") + case 'n': + return parser.parseLiteral("null") + case '-': + return parser.parseNumber() + default: + if parser.data[parser.offset] >= '0' && parser.data[parser.offset] <= '9' { + return parser.parseNumber() + } + return jsonInvalid + } +} + +func (parser *jsonParser) parseObject() jsonParseResult { + parser.offset++ + parser.skipWhitespace() + if parser.offset == len(parser.data) { + return jsonIncomplete + } + if parser.data[parser.offset] == '}' { + parser.offset++ + return jsonComplete + } + + for { + if parser.data[parser.offset] != '"' { + return jsonInvalid + } + if result := parser.parseString(); result != jsonComplete { + return result + } + + parser.skipWhitespace() + if parser.offset == len(parser.data) { + return jsonIncomplete + } + if parser.data[parser.offset] != ':' { + return jsonInvalid + } + parser.offset++ + parser.skipWhitespace() + + if result := parser.parseValue(); result != jsonComplete { + return result + } + parser.skipWhitespace() + if parser.offset == len(parser.data) { + return jsonIncomplete + } + + switch parser.data[parser.offset] { + case '}': + parser.offset++ + return jsonComplete + case ',': + parser.offset++ + parser.skipWhitespace() + if parser.offset == len(parser.data) { + return jsonIncomplete + } + default: + return jsonInvalid + } + } +} + +func (parser *jsonParser) parseArray() jsonParseResult { + parser.offset++ + parser.skipWhitespace() + if parser.offset == len(parser.data) { + return jsonIncomplete + } + if parser.data[parser.offset] == ']' { + parser.offset++ + return jsonComplete + } + + for { + if result := parser.parseValue(); result != jsonComplete { + return result + } + parser.skipWhitespace() + if parser.offset == len(parser.data) { + return jsonIncomplete + } + + switch parser.data[parser.offset] { + case ']': + parser.offset++ + return jsonComplete + case ',': + parser.offset++ + parser.skipWhitespace() + if parser.offset == len(parser.data) { + return jsonIncomplete + } + default: + return jsonInvalid + } + } +} + +func (parser *jsonParser) parseString() jsonParseResult { + parser.offset++ + for parser.offset < len(parser.data) { + value := parser.data[parser.offset] + switch { + case value == '"': + parser.offset++ + return jsonComplete + case value == '\\': + parser.offset++ + if parser.offset == len(parser.data) { + return jsonIncomplete + } + escape := parser.data[parser.offset] + parser.offset++ + switch escape { + case '"', '\\', '/', 'b', 'f', 'n', 'r', 't': + case 'u': + for range 4 { + if parser.offset == len(parser.data) { + return jsonIncomplete + } + if !isHexadecimal(parser.data[parser.offset]) { + return jsonInvalid + } + parser.offset++ + } + default: + return jsonInvalid + } + case value < jsonControlLimit: + return jsonInvalid + case value < utf8.RuneSelf: + parser.offset++ + default: + remaining := parser.data[parser.offset:] + if !utf8.FullRune(remaining) { + return jsonIncomplete + } + runeValue, size := utf8.DecodeRune(remaining) + if runeValue == utf8.RuneError && size == 1 { + return jsonInvalid + } + parser.offset += size + } + } + return jsonIncomplete +} + +func (parser *jsonParser) parseLiteral(literal string) jsonParseResult { + for index := range len(literal) { + if parser.offset == len(parser.data) { + return jsonIncomplete + } + if parser.data[parser.offset] != literal[index] { + return jsonInvalid + } + parser.offset++ + } + return jsonComplete +} + +func (parser *jsonParser) parseNumber() jsonParseResult { + if parser.data[parser.offset] == '-' { + parser.offset++ + } + + if result := parser.parseInteger(); result != jsonComplete { + return result + } + + if parser.offset < len(parser.data) && parser.data[parser.offset] == '.' { + parser.offset++ + if result := parser.parseDigits(); result != jsonComplete { + return result + } + } + + if parser.offset < len(parser.data) && + (parser.data[parser.offset] == 'e' || parser.data[parser.offset] == 'E') { + parser.offset++ + if parser.offset == len(parser.data) { + return jsonIncomplete + } + if parser.data[parser.offset] == '+' || parser.data[parser.offset] == '-' { + parser.offset++ + } + if result := parser.parseDigits(); result != jsonComplete { + return result + } + } + + return jsonComplete +} + +func (parser *jsonParser) parseInteger() jsonParseResult { + if parser.offset == len(parser.data) { + return jsonIncomplete + } + + value := parser.data[parser.offset] + if value == '0' { + parser.offset++ + if parser.offset < len(parser.data) && isDigit(parser.data[parser.offset]) { + return jsonInvalid + } + return jsonComplete + } + if value < '1' || value > '9' { + return jsonInvalid + } + + parser.offset++ + parser.skipDigits() + return jsonComplete +} + +func (parser *jsonParser) parseDigits() jsonParseResult { + if parser.offset == len(parser.data) { + return jsonIncomplete + } + if !isDigit(parser.data[parser.offset]) { + return jsonInvalid + } + + parser.skipDigits() + return jsonComplete +} + +func (parser *jsonParser) skipDigits() { + for parser.offset < len(parser.data) && isDigit(parser.data[parser.offset]) { + parser.offset++ + } +} + +func (parser *jsonParser) skipWhitespace() { + for parser.offset < len(parser.data) && isJSONWhitespace(parser.data[parser.offset]) { + parser.offset++ + } +} + +func isJSONWhitespace(value byte) bool { + switch value { + case ' ', '\t', '\n', '\r': + return true + default: + return false + } +} + +func isHexadecimal(value byte) bool { + return value >= '0' && value <= '9' || + value >= 'a' && value <= 'f' || + value >= 'A' && value <= 'F' +} + +func isDigit(value byte) bool { + return value >= '0' && value <= '9' +} diff --git a/json_test.go b/json_test.go new file mode 100644 index 0000000..24918a9 --- /dev/null +++ b/json_test.go @@ -0,0 +1,177 @@ +package magic + +import ( + "strings" + "testing" +) + +func TestJSONDetection(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + }{ + {name: "object", input: `{"schemaVersion": 2}`}, + {name: "array", input: `[1, "two", false, null]`}, + {name: "string", input: `"hello"`}, + {name: "number", input: `-12.5e+2`}, + {name: "true", input: `true`}, + {name: "false", input: `false`}, + {name: "null", input: `null`}, + {name: "surrounding whitespace", input: " \t\r\n{\"key\": \"value\"}\n"}, + {name: "Unicode", input: `{"message":"héllo, 世界","escaped":"\uD834\uDD1E"}`}, + {name: "escaped characters", input: `["\b\f\n\r\t\/\\\""]`}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + assertResult(t, Detect([]byte(test.input)), Result{ + Kind: KindText, + MIME: mimeJSON, + Format: FormatJSON, + Encoding: encodingUTF8, + }) + }) + } +} + +func TestInvalidJSONRetainsExistingClassification(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + }{ + {name: "mismatched delimiters", input: `{]`}, + {name: "whitespace only", input: " \t\r\n"}, + {name: "missing value", input: `{"key":}`}, + {name: "single quoted key", input: `{'key': 1}`}, + {name: "trailing comma", input: `[1,]`}, + {name: "unterminated string", input: `"value`}, + {name: "truncated literal", input: `tru`}, + {name: "leading zero", input: `01`}, + {name: "truncated fraction", input: `1.`}, + {name: "truncated exponent", input: `1e+`}, + {name: "invalid escape", input: `"\x"`}, + {name: "unescaped control", input: "\"value\tvalue\""}, + {name: "leading form feed", input: "\f{}"}, + {name: "trailing form feed", input: "{}\f"}, + {name: "trailing content", input: `{}x`}, + {name: "second value", input: `true false`}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + assertResult(t, Detect([]byte(test.input)), Result{ + Kind: KindText, + MIME: mimeText, + Format: FormatText, + Encoding: encodingUTF8, + }) + }) + } +} + +func TestInvalidUTF8JSONRetainsUnknownClassification(t *testing.T) { + t.Parallel() + + tests := [][]byte{ + {'"', 0xff, '"'}, + {'{', '}', 0xff}, + } + for _, input := range tests { + assertResult(t, Detect(input), Result{ + Kind: KindUnknown, + Reason: ReasonInvalidText, + }) + } +} + +func TestJSONNestingDepth(t *testing.T) { + t.Parallel() + + input := strings.Repeat("[", jsonMaximumDepth) + "0" + + strings.Repeat("]", jsonMaximumDepth) + if got := Detect([]byte(input)); got.Format != FormatJSON { + t.Fatalf("Detect() = %#v, want JSON at maximum nesting depth", got) + } + + input = "[" + input + "]" + if got := Detect([]byte(input)); got.Format == FormatJSON { + t.Fatalf("Detect() = %#v, want nesting depth limit to reject JSON", got) + } +} + +func TestJSONPrefixDetection(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + }{ + {name: "complete object", input: `{}`}, + {name: "truncated object", input: `{`}, + {name: "truncated object key", input: `{"key`}, + {name: "truncated object value", input: `{"key":`}, + {name: "truncated array", input: `[`}, + {name: "truncated array value", input: `[1,`}, + {name: "truncated string", input: `"value`}, + {name: "truncated escape", input: `"value\`}, + {name: "truncated Unicode escape", input: `"value\u12`}, + {name: "truncated negative number", input: `-`}, + {name: "truncated fraction", input: `1.`}, + {name: "truncated exponent", input: `1e`}, + {name: "truncated signed exponent", input: `1e+`}, + {name: "truncated true", input: `tru`}, + {name: "truncated false", input: `fals`}, + {name: "truncated null", input: `nul`}, + {name: "leading whitespace", input: " \n{"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + assertResult(t, DetectPrefix([]byte(test.input)), Result{ + Kind: KindText, + MIME: mimeJSON, + Format: FormatJSON, + Encoding: encodingUTF8, + Reason: ReasonNeedMore, + }) + }) + } +} + +func TestInvalidJSONPrefixRetainsExistingClassification(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + }{ + {name: "plain text", input: `package`}, + {name: "whitespace only", input: " \t\r\n"}, + {name: "mismatched delimiters", input: `{]`}, + {name: "trailing comma", input: `[1,]`}, + {name: "leading zero", input: `01`}, + {name: "invalid literal", input: `truex`}, + {name: "invalid escape", input: `"\x"`}, + {name: "trailing content", input: `{}x`}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + assertResult(t, DetectPrefix([]byte(test.input)), Result{ + Kind: KindText, + MIME: mimeText, + Format: FormatText, + Encoding: encodingUTF8, + Reason: ReasonNeedMore, + }) + }) + } +} diff --git a/magic.go b/magic.go index f9abaeb..cbed39a 100644 --- a/magic.go +++ b/magic.go @@ -46,6 +46,7 @@ const ( FormatHTML = "html" FormatXML = "xml" FormatSVG = "svg" + FormatJSON = "json" FormatZIP = "zip" FormatTAR = "tar" FormatPHAR = "phar" @@ -70,6 +71,7 @@ const ( mimeHTML = "text/html" mimeXML = "text/xml" mimeSVG = "image/svg+xml" + mimeJSON = "application/json" mimeZIP = "application/zip" mimeTAR = "application/x-tar" mimePHAR = "application/x-phar" @@ -120,6 +122,10 @@ func detect(data []byte, prefix bool) Result { } format, mime = textFormat(data) + if format == "" && isJSON(data, prefix) { + format = FormatJSON + mime = mimeJSON + } result := classifyText(data) if format != "" { result.Format = format diff --git a/magic_test.go b/magic_test.go index f832414..b828f03 100644 --- a/magic_test.go +++ b/magic_test.go @@ -133,6 +133,7 @@ func TestDetectAllocations(t *testing.T) { inputs := [][]byte{ make([]byte, 4096), []byte("package magic\n"), + []byte(`{"schemaVersion":2}`), []byte("\xff\xfeh\x00i\x00"), []byte("\x89PNG\r\n\x1a\n"), makeNativePHAR(pharTestStub, "", nil, pharTestEntry{name: "file", content: []byte("data")}), From 608da0b03dea8f25392c5ea1432199c4168b9fc1 Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sat, 15 Aug 2026 18:54:03 +0100 Subject: [PATCH 2/3] Avoid recursive JSON parsing --- README.md | 4 +- json.go | 286 ++++++++++++++++++++++++++++++++------------------- json_test.go | 40 +++++-- 3 files changed, 215 insertions(+), 115 deletions(-) diff --git a/README.md b/README.md index 8814cb8..6f54b74 100644 --- a/README.md +++ b/README.md @@ -97,8 +97,8 @@ go test -run '^$' -bench . -benchmem Fixed signatures inspect at most 512 bytes, while native PHAR detection searches for the end of the PHP stub and validates the manifest and stored payload bounds. JSON parsing and text validation are linear in the supplied -byte count. JSON parsing uses stack space proportional to nesting depth; text -validation uses fixed auxiliary memory. +byte count. JSON parsing uses auxiliary memory proportional to nesting depth; +text validation uses fixed auxiliary memory. ## Provenance diff --git a/json.go b/json.go index e1e04c9..d373dea 100644 --- a/json.go +++ b/json.go @@ -10,13 +10,36 @@ const ( jsonComplete jsonControlLimit = 0x20 + jsonInlineDepth = 64 jsonMaximumDepth = 10000 ) +type jsonContainer uint8 + +const ( + jsonArray jsonContainer = iota + jsonObject +) + +type jsonExpectation uint8 + +const ( + jsonExpectValue jsonExpectation = iota + jsonExpectArrayValueOrEnd + jsonExpectObjectKeyOrEnd + jsonExpectObjectKey + jsonExpectObjectColon + jsonExpectArrayCommaOrEnd + jsonExpectObjectCommaOrEnd + jsonExpectDocumentEnd +) + type jsonParser struct { - data []byte - offset int - depth int + data []byte + offset int + depth int + containers [jsonInlineDepth]jsonContainer + extraContainers []jsonContainer } func isJSON(data []byte, prefix bool) bool { @@ -30,40 +53,140 @@ func parseJSON(data []byte) jsonParseResult { if parser.offset == len(parser.data) { return jsonInvalid } + return parser.parse() +} - result := parser.parseValue() - if result != jsonComplete { - return result - } - parser.skipWhitespace() - if parser.offset != len(parser.data) { - return jsonInvalid +func (parser *jsonParser) parse() jsonParseResult { + expectation := jsonExpectValue + for { + parser.skipWhitespace() + if parser.offset == len(parser.data) { + if expectation == jsonExpectDocumentEnd { + return jsonComplete + } + return jsonIncomplete + } + + next, result := parser.parseExpectation(expectation) + if result != jsonComplete { + return result + } + expectation = next } - return jsonComplete } -func (parser *jsonParser) parseValue() jsonParseResult { - if parser.offset == len(parser.data) { - return jsonIncomplete +func (parser *jsonParser) parseExpectation( + expectation jsonExpectation, +) (jsonExpectation, jsonParseResult) { + switch expectation { + case jsonExpectValue: + return parser.parseExpectedValue() + case jsonExpectArrayValueOrEnd: + return parser.parseExpectedArrayValueOrEnd() + case jsonExpectObjectKeyOrEnd: + return parser.parseExpectedObjectKeyOrEnd() + case jsonExpectObjectKey: + return parser.parseExpectedObjectKey() + case jsonExpectObjectColon: + return parser.parseExpectedObjectColon() + case jsonExpectArrayCommaOrEnd: + return parser.parseExpectedArrayCommaOrEnd() + case jsonExpectObjectCommaOrEnd: + return parser.parseExpectedObjectCommaOrEnd() + default: + return expectation, jsonInvalid } +} +func (parser *jsonParser) parseExpectedValue() (jsonExpectation, jsonParseResult) { switch parser.data[parser.offset] { case '{': - if parser.depth == jsonMaximumDepth { - return jsonInvalid - } - parser.depth++ - result := parser.parseObject() - parser.depth-- - return result + return jsonExpectObjectKeyOrEnd, parser.openContainer(jsonObject) case '[': - if parser.depth == jsonMaximumDepth { - return jsonInvalid - } - parser.depth++ - result := parser.parseArray() - parser.depth-- - return result + return jsonExpectArrayValueOrEnd, parser.openContainer(jsonArray) + default: + result := parser.parseScalar() + return parser.expectAfterValue(), result + } +} + +func (parser *jsonParser) parseExpectedArrayValueOrEnd() ( + jsonExpectation, + jsonParseResult, +) { + if parser.data[parser.offset] != ']' { + return jsonExpectValue, jsonComplete + } + parser.closeContainer() + return parser.expectAfterValue(), jsonComplete +} + +func (parser *jsonParser) parseExpectedObjectKeyOrEnd() ( + jsonExpectation, + jsonParseResult, +) { + if parser.data[parser.offset] != '}' { + return jsonExpectObjectKey, jsonComplete + } + parser.closeContainer() + return parser.expectAfterValue(), jsonComplete +} + +func (parser *jsonParser) parseExpectedObjectKey() ( + jsonExpectation, + jsonParseResult, +) { + if parser.data[parser.offset] != '"' { + return jsonExpectObjectColon, jsonInvalid + } + return jsonExpectObjectColon, parser.parseString() +} + +func (parser *jsonParser) parseExpectedObjectColon() ( + jsonExpectation, + jsonParseResult, +) { + if parser.data[parser.offset] != ':' { + return jsonExpectValue, jsonInvalid + } + parser.offset++ + return jsonExpectValue, jsonComplete +} + +func (parser *jsonParser) parseExpectedArrayCommaOrEnd() ( + jsonExpectation, + jsonParseResult, +) { + switch parser.data[parser.offset] { + case ']': + parser.closeContainer() + return parser.expectAfterValue(), jsonComplete + case ',': + parser.offset++ + return jsonExpectValue, jsonComplete + default: + return jsonExpectValue, jsonInvalid + } +} + +func (parser *jsonParser) parseExpectedObjectCommaOrEnd() ( + jsonExpectation, + jsonParseResult, +) { + switch parser.data[parser.offset] { + case '}': + parser.closeContainer() + return parser.expectAfterValue(), jsonComplete + case ',': + parser.offset++ + return jsonExpectObjectKey, jsonComplete + default: + return jsonExpectObjectKey, jsonInvalid + } +} + +func (parser *jsonParser) parseScalar() jsonParseResult { + switch parser.data[parser.offset] { case '"': return parser.parseString() case 't': @@ -75,100 +198,51 @@ func (parser *jsonParser) parseValue() jsonParseResult { case '-': return parser.parseNumber() default: - if parser.data[parser.offset] >= '0' && parser.data[parser.offset] <= '9' { + if isDigit(parser.data[parser.offset]) { return parser.parseNumber() } return jsonInvalid } } -func (parser *jsonParser) parseObject() jsonParseResult { - parser.offset++ - parser.skipWhitespace() - if parser.offset == len(parser.data) { - return jsonIncomplete +func (parser *jsonParser) openContainer(container jsonContainer) jsonParseResult { + if parser.depth == jsonMaximumDepth { + return jsonInvalid } - if parser.data[parser.offset] == '}' { - parser.offset++ - return jsonComplete + if parser.depth < len(parser.containers) { + parser.containers[parser.depth] = container + } else { + parser.extraContainers = append(parser.extraContainers, container) } + parser.depth++ + parser.offset++ + return jsonComplete +} - for { - if parser.data[parser.offset] != '"' { - return jsonInvalid - } - if result := parser.parseString(); result != jsonComplete { - return result - } - - parser.skipWhitespace() - if parser.offset == len(parser.data) { - return jsonIncomplete - } - if parser.data[parser.offset] != ':' { - return jsonInvalid - } - parser.offset++ - parser.skipWhitespace() - - if result := parser.parseValue(); result != jsonComplete { - return result - } - parser.skipWhitespace() - if parser.offset == len(parser.data) { - return jsonIncomplete - } - - switch parser.data[parser.offset] { - case '}': - parser.offset++ - return jsonComplete - case ',': - parser.offset++ - parser.skipWhitespace() - if parser.offset == len(parser.data) { - return jsonIncomplete - } - default: - return jsonInvalid - } +func (parser *jsonParser) closeContainer() { + parser.depth-- + if parser.depth >= len(parser.containers) { + parser.extraContainers = parser.extraContainers[:len(parser.extraContainers)-1] } + parser.offset++ } -func (parser *jsonParser) parseArray() jsonParseResult { - parser.offset++ - parser.skipWhitespace() - if parser.offset == len(parser.data) { - return jsonIncomplete +func (parser *jsonParser) expectAfterValue() jsonExpectation { + if parser.depth == 0 { + return jsonExpectDocumentEnd } - if parser.data[parser.offset] == ']' { - parser.offset++ - return jsonComplete + if parser.currentContainer() == jsonArray { + return jsonExpectArrayCommaOrEnd } + return jsonExpectObjectCommaOrEnd +} - for { - if result := parser.parseValue(); result != jsonComplete { - return result - } - parser.skipWhitespace() - if parser.offset == len(parser.data) { - return jsonIncomplete - } - - switch parser.data[parser.offset] { - case ']': - parser.offset++ - return jsonComplete - case ',': - parser.offset++ - parser.skipWhitespace() - if parser.offset == len(parser.data) { - return jsonIncomplete - } - default: - return jsonInvalid - } +func (parser *jsonParser) currentContainer() jsonContainer { + index := parser.depth - 1 + if index < len(parser.containers) { + return parser.containers[index] } + return parser.extraContainers[index-len(parser.containers)] } func (parser *jsonParser) parseString() jsonParseResult { diff --git a/json_test.go b/json_test.go index 24918a9..c687ad3 100644 --- a/json_test.go +++ b/json_test.go @@ -93,15 +93,41 @@ func TestInvalidUTF8JSONRetainsUnknownClassification(t *testing.T) { func TestJSONNestingDepth(t *testing.T) { t.Parallel() - input := strings.Repeat("[", jsonMaximumDepth) + "0" + - strings.Repeat("]", jsonMaximumDepth) - if got := Detect([]byte(input)); got.Format != FormatJSON { - t.Fatalf("Detect() = %#v, want JSON at maximum nesting depth", got) + tests := []struct { + name string + open string + close string + }{ + {name: "arrays", open: "[", close: "]"}, + {name: "objects", open: `{"value":`, close: "}"}, } - input = "[" + input + "]" - if got := Detect([]byte(input)); got.Format == FormatJSON { - t.Fatalf("Detect() = %#v, want nesting depth limit to reject JSON", got) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + input := strings.Repeat(test.open, jsonMaximumDepth) + "0" + + strings.Repeat(test.close, jsonMaximumDepth) + if got := Detect([]byte(input)); got.Format != FormatJSON { + t.Fatalf("Detect() = %#v, want JSON at maximum nesting depth", got) + } + + input = test.open + input + test.close + if got := Detect([]byte(input)); got.Format == FormatJSON { + t.Fatalf("Detect() = %#v, want nesting depth limit to reject JSON", got) + } + }) + } +} + +func TestJSONInlineNestingDoesNotAllocate(t *testing.T) { + input := []byte(strings.Repeat("[", jsonInlineDepth) + "0" + + strings.Repeat("]", jsonInlineDepth)) + + if allocations := testing.AllocsPerRun(1000, func() { + Detect(input) + }); allocations != 0 { + t.Fatalf("Detect allocated %.2f times for inline JSON nesting", allocations) } } From 71749339ca6ebc4b6c5a4897a592feb1d593810d Mon Sep 17 00:00:00 2001 From: Andrew Nesbitt Date: Sun, 16 Aug 2026 08:48:33 +0100 Subject: [PATCH 3/3] Fix JSON fuzz oracle --- fuzz_test.go | 38 +++++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/fuzz_test.go b/fuzz_test.go index f461661..0a69e8d 100644 --- a/fuzz_test.go +++ b/fuzz_test.go @@ -1,7 +1,9 @@ package magic import ( + "bytes" "encoding/json" + "fmt" "testing" "unicode/utf8" ) @@ -42,6 +44,7 @@ func FuzzDetect(f *testing.F) { } f.Add(makeTAR(f)) f.Add(makeNativePHAR(pharTestStub, "", nil, pharTestEntry{name: "file", content: []byte("data")})) + f.Add(makeJSONTARCollision(f)) f.Fuzz(func(t *testing.T, data []byte) { first := Detect(data) @@ -50,8 +53,10 @@ func FuzzDetect(f *testing.F) { t.Fatalf("Detect is not deterministic: %#v then %#v", first, second) } assertResultInvariants(t, first, false, len(data)) - if got, expect := first.Format == FormatJSON, json.Valid(data) && utf8.Valid(data); got != expect { - t.Fatalf("Detect JSON match = %v, want %v for %x", got, expect, data) + binary, _ := binaryFormat(data) + expectJSON := binary == "" && json.Valid(data) && utf8.Valid(data) + if got := first.Format == FormatJSON; got != expectJSON { + t.Fatalf("Detect JSON match = %v, want %v for %x", got, expectJSON, data) } prefix := DetectPrefix(data) @@ -69,12 +74,39 @@ func FuzzDetect(f *testing.F) { } } - if format, _ := binaryFormat(data); format != "" && prefix != first { + if binary != "" && prefix != first { t.Fatalf("terminal binary signature changed for prefix: %#v, complete: %#v", prefix, first) } }) } +func makeJSONTARCollision(t testing.TB) []byte { + t.Helper() + + data := bytes.Repeat([]byte{'a'}, sniffLength) + data[0] = '"' + data[len(data)-1] = '"' + copy(data[tarMagicOffset:tarMagicEnd], "ustar ") + + checksum := 0 + for index, value := range data { + if index >= tarChecksumFrom && index < tarChecksumTo { + checksum += ' ' + } else { + checksum += int(value) + } + } + copy(data[tarChecksumFrom:tarChecksumTo], fmt.Sprintf("%06o ", checksum)) + + if !json.Valid(data) { + t.Fatal("JSON/TAR fixture is not valid JSON") + } + if format, _ := binaryFormat(data); format != FormatTAR { + t.Fatalf("JSON/TAR fixture format = %q, want %q", format, FormatTAR) + } + return data +} + func FuzzDetectPrefix(f *testing.F) { seeds := [][]byte{ nil,