From e50374ee41d45b778f542b61085e356b7b5f71a3 Mon Sep 17 00:00:00 2001 From: korya <148461+korya@users.noreply.github.com> Date: Tue, 11 Aug 2026 07:05:28 -0400 Subject: [PATCH] feat(compress): Decode brotli and zstd response bodies A body arriving as br or zstd could not be asserted on at all: the three body assertions refused by name while the status and header assertions carried on. Both are what CDNs actually serve, so testing the payload behind content negotiation -- the ordinary case -- was the one thing the tool could not do. Neither coding is in the standard library, so each costs a dependency, and the count was cut to three deliberately when viper went (#54). Measured at release flags rather than guessed: brotli adds 233KB and zstd 186KB to a 7.38MB binary, 5.7% together, and both modules bring nothing transitively. klauspost/compress is a large repository but only its zstd package links, so the cost is the decoder rather than the library. That is a smaller price than a CDN-facing HTTP assertion tool that cannot read CDN responses. A build tag was the alternative and fits this project badly: released binaries are how the tool is used, so a tag either ships releases that cannot decode the thing the change is about, or doubles the artifacts and makes the user choose. The list of supported codings in the failure is now derived from the decoder map instead of being spelled out, so the next coding cannot be added while the message still claims otherwise. The test suite used br throughout as its stand-in for "an encoding with no decoder here", which is no longer true of it. That role moves to compress (LZW): RFC 9110 still registers it, effectively nothing serves it, and unlike br and zstd it is not a plausible candidate for support later. Brotli also gains a corrupt-stream fixture, a failure only reachable now that the bytes get as far as a decoder. Verified against streams from the reference implementations -- brotli 1.2.0 and zstd 1.5.7 -- rather than only against Go's own writers, and brotli end to end against jsdelivr and the npm registry. Closes #77 Closes #78 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01CrknafJSP5hF8u865cbnqX --- README.md | 10 +++---- assertions_test.go | 8 +++--- compression_test.go | 14 +++++----- e2e_compression_test.go | 60 ++++++++++++++++++++++++++++++++++++----- e2e_jq_test.go | 4 +-- e2e_server_test.go | 40 ++++++++++++++++++++++++--- go.mod | 2 ++ go.sum | 6 +++++ jq_test.go | 6 ++--- main.go | 47 +++++++++++++++++++++++++++----- 10 files changed, 158 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 81ad960..4cb452e 100644 --- a/README.md +++ b/README.md @@ -377,15 +377,15 @@ itself, and leaves the rest of the run intact: ```console $ http-assert --assert-body '"status":"success"' https://cdn.example.com/ -- body: response is br-encoded and was not decoded: no decoder for "br"; gzip and deflate are supported +- body: response is compress-encoded and was not decoded: no decoder for "compress"; br, deflate, gzip, zstd are supported $ http-assert --assert-ok https://cdn.example.com/ [+] PASSED 38ms ``` -Brotli (`br`) and zstd are the encodings this covers in practice. Neither is -supported yet: [#77](https://github.com/korya/http-assert/issues/77) tracks -brotli and [#78](https://github.com/korya/http-assert/issues/78) tracks zstd. +gzip, deflate, brotli (`br`) and zstd are decoded. Brotli and zstd each cost a +dependency, which is why they are named here rather than assumed: they are what +CDNs actually serve, and a body assertion against one is the ordinary case. A header that claims an encoding the body does not have is treated the same way. Reading those bytes as plain text would be its own silent corruption. @@ -604,7 +604,7 @@ a tool whose job is checking. This is the complete list: | `-d @file` | reads the file | sends the literal string `@file` | | `-d` repeated | values joined with `&` | rejected, exit `71` | | `--retry` | transport errors and a fixed set of transient statuses, exponential backoff | any failed attempt, assertion failures included, fixed delay | -| Response decompression | opt-in via `--compressed` | always: gzip and deflate are decoded before assertions run | +| Response decompression | opt-in via `--compressed` | always: gzip, deflate, br and zstd are decoded before assertions run | | Pointing at a backend | `--resolve host:port:addr` takes an address | `--maphost 'host:port=dst[:port]'` takes a hostname or an address | ## License diff --git a/assertions_test.go b/assertions_test.go index dd32cad..679a9f4 100644 --- a/assertions_test.go +++ b/assertions_test.go @@ -729,8 +729,8 @@ func Test_AssertionCheckSeparatesFailureFromError(t *testing.T) { t.Run("an undecodable body is an error, not a Failure", func(t *testing.T) { res := &httpResponse{ - Encoding: "br", - DecodeErr: errors.New(`no decoder for "br"`), + Encoding: "compress", + DecodeErr: errors.New(`no decoder for "compress"`), } for name, a := range map[string]Assertion{ @@ -755,8 +755,8 @@ func Test_AssertionCheckSeparatesFailureFromError(t *testing.T) { t.Run("a status assertion is unaffected by an undecodable body", func(t *testing.T) { res := &httpResponse{ Response: &http.Response{StatusCode: 200, Status: "200 OK"}, - Encoding: "br", - DecodeErr: errors.New(`no decoder for "br"`), + Encoding: "compress", + DecodeErr: errors.New(`no decoder for "compress"`), } f, err := AssertStatusOK().Check(res) diff --git a/compression_test.go b/compression_test.go index 283804e..33dda65 100644 --- a/compression_test.go +++ b/compression_test.go @@ -145,10 +145,10 @@ func Test_decodeBody(t *testing.T) { }, { Name: "an encoding with no decoder here", - Enc: "br", - Body: []byte{0x1b, 0x13, 0x00}, + Enc: "compress", + Body: []byte("payload"), WantErr: true, - WantEncoding: "br", + WantEncoding: "compress", }, { // The header claims gzip and the bytes are not. Silently asserting @@ -246,11 +246,11 @@ func Test_bodyOf(t *testing.T) { }) t.Run("an undecoded body is refused, and says why", func(t *testing.T) { - res := encoded("br", []byte{0x1b, 0x13, 0x00}) + res := encoded("compress", []byte("payload")) res.decodeBody() _, err := bodyOf(res) - checkErrMatch(t, "bodyOf", err, `^body: response is br-encoded and was not decoded: `) + checkErrMatch(t, "bodyOf", err, `^body: response is compress-encoded and was not decoded: `) }) } @@ -275,10 +275,10 @@ func Test_bodyAssertionsRefuseAnEncodedBody(t *testing.T) { for name, a := range assertions { t.Run(name, func(t *testing.T) { - res := encoded("br", []byte{0x1b, 0x13, 0x00}) + res := encoded("compress", []byte("payload")) res.decodeBody() - checkErrMatch(t, name, check(a, res), `^body: response is br-encoded and was not decoded: `) + checkErrMatch(t, name, check(a, res), `^body: response is compress-encoded and was not decoded: `) }) } } diff --git a/e2e_compression_test.go b/e2e_compression_test.go index 5d6db7a..45ba398 100644 --- a/e2e_compression_test.go +++ b/e2e_compression_test.go @@ -111,28 +111,74 @@ func TestE2ECompressionHeadersSurvive(t *testing.T) { }) } +// TestE2ECompressionBrotliZstd covers the two codings that cost a dependency. +// +// Both are what CDNs actually serve, so a body assertion against one is the +// ordinary case rather than an exotic one (#77, #78). +func TestE2ECompressionBrotliZstd(t *testing.T) { + for path, coding := range map[string]string{"/brotli": "br", "/zstd": "zstd"} { + t.Run(coding+" is decoded before the body assertions run", func(t *testing.T) { + assertExit(t, run(t, nil, "--assert-body", `"status":"success"`, url(path)), exitOK) + assertExit(t, run(t, nil, "--assert-body-eq", `{"status":"success"}`, url(path)), exitOK) + }) + + t.Run(coding+" decodes for --assert-jq too", func(t *testing.T) { + assertExit(t, run(t, nil, "--assert-jq", `.status == "success"`, url(path)), exitOK) + }) + + // Decoding must not rewrite what the response said about itself: the + // point of decoding by hand is to assert on the payload and on the + // Content-Encoding at the same time. + t.Run(coding+" is still reported as the encoding that arrived", func(t *testing.T) { + r := run(t, nil, + "--assert-header-eq", "Content-Encoding: "+coding, + "--assert-body", `"status":"success"`, url(path)) + assertExit(t, r, exitOK) + }) + } + + // A decoder makes this failure reachable for br at all; without one the + // body never got as far as being malformed. + t.Run("a body that claims br and is not br fails the body assertions", func(t *testing.T) { + r := run(t, nil, "--assert-body", `"status":"success"`, url("/brotli-corrupt")) + assertExit(t, r, exitAssertFail) + assertContains(t, r, "body: response is br-encoded and was not decoded") + }) + + t.Run("a status check is unaffected by a corrupt br body", func(t *testing.T) { + assertExit(t, run(t, nil, "--assert-ok", url("/brotli-corrupt")), exitOK) + }) +} + // TestE2ECompressionUndecodable covers an encoding with no decoder here. // // The response still has a status and headers worth asserting on, so only the // body assertions refuse. Failing the whole run would be simpler and would -// break a status check against any CDN serving brotli. +// break a status check against any CDN serving a coding this does not know. func TestE2ECompressionUndecodable(t *testing.T) { t.Run("a status check is unaffected", func(t *testing.T) { - assertExit(t, run(t, nil, "--assert-ok", url("/brotli")), exitOK) + assertExit(t, run(t, nil, "--assert-ok", url("/unsupported")), exitOK) }) t.Run("a header check is unaffected", func(t *testing.T) { - r := run(t, nil, "--assert-header-eq", "Content-Encoding: br", url("/brotli")) + r := run(t, nil, "--assert-header-eq", "Content-Encoding: compress", url("/unsupported")) assertExit(t, r, exitOK) }) t.Run("a body check refuses, and names the encoding", func(t *testing.T) { - r := run(t, nil, "--assert-body", `"status":"success"`, url("/brotli")) + r := run(t, nil, "--assert-body", `"status":"success"`, url("/unsupported")) assertExit(t, r, exitAssertFail) - assertContains(t, r, "body: response is br-encoded and was not decoded") - assertContains(t, r, "no decoder for \"br\"") + assertContains(t, r, "body: response is compress-encoded and was not decoded") + assertContains(t, r, "no decoder for \"compress\"") // The old failure was a bare hex dump with nothing explaining it. - assertContains(t, r, "<< Payload is br-encoded and was not decoded >>") + assertContains(t, r, "<< Payload is compress-encoded and was not decoded >>") + }) + + // The list in the failure is derived from the decoder map, so a coding + // added without updating the message is not possible. + t.Run("it names what it does support", func(t *testing.T) { + r := run(t, nil, "--assert-body", `"status":"success"`, url("/unsupported")) + assertContains(t, r, "br, deflate, gzip, zstd are supported") }) // A header claiming an encoding the body does not have. Treating those diff --git a/e2e_jq_test.go b/e2e_jq_test.go index 52f426a..058e6f1 100644 --- a/e2e_jq_test.go +++ b/e2e_jq_test.go @@ -177,9 +177,9 @@ func TestE2EAssertJQBodyProblems(t *testing.T) { // An encoding nothing can decode reports the encoding, not invalid JSON. t.Run("a body still encoded", func(t *testing.T) { - r := run(t, nil, "--assert-jq", `. == 1`, url("/brotli")) + r := run(t, nil, "--assert-jq", `. == 1`, url("/unsupported")) assertExit(t, r, exitAssertFail) - assertContains(t, r, "body: response is br-encoded") + assertContains(t, r, "body: response is compress-encoded") }) // A query yielding nothing has checked nothing, so it must not pass. diff --git a/e2e_server_test.go b/e2e_server_test.go index 4c594cd..5c1ca3a 100644 --- a/e2e_server_test.go +++ b/e2e_server_test.go @@ -1,6 +1,7 @@ package main_test import ( + "bytes" "compress/flate" "compress/gzip" "compress/zlib" @@ -17,6 +18,9 @@ import ( "sync/atomic" "testing" "time" + + "github.com/andybalholm/brotli" + "github.com/klauspost/compress/zstd" ) // The e2e suite talks to two real servers started once per test run: a plain @@ -267,11 +271,39 @@ func testHandler() http.Handler { _ = zw.Close() }) - // An encoding with no decoder here. Brotli is the realistic case; the bytes - // are not real brotli because nothing in the suite could produce them, and - // the CLI never gets far enough to care. mux.HandleFunc("/brotli", func(w http.ResponseWriter, _ *http.Request) { - write(w, http.StatusOK, []byte{0x1b, 0x13, 0x00, 0x00, 0xa4, 0xb0, 0xb2}, + var buf bytes.Buffer + bw := brotli.NewWriter(&buf) + _, _ = bw.Write([]byte(`{"status":"success"}`)) + _ = bw.Close() + write(w, http.StatusOK, buf.Bytes(), http.Header{"Content-Encoding": {"br"}}) + }) + + mux.HandleFunc("/zstd", func(w http.ResponseWriter, _ *http.Request) { + var buf bytes.Buffer + zw, err := zstd.NewWriter(&buf) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + _, _ = zw.Write([]byte(`{"status":"success"}`)) + _ = zw.Close() + write(w, http.StatusOK, buf.Bytes(), http.Header{"Content-Encoding": {"zstd"}}) + }) + + // An encoding with no decoder here, and the one place the suite needs a + // coding that will stay unsupported. RFC 9110 still registers `compress` + // (LZW), and effectively nothing serves it -- unlike br and zstd, which + // were the realistic examples right up until they were implemented. + mux.HandleFunc("/unsupported", func(w http.ResponseWriter, _ *http.Request) { + write(w, http.StatusOK, []byte(`{"status":"success"}`), + http.Header{"Content-Encoding": {"compress"}}) + }) + + // Claims brotli and is not brotli. gzip-corrupt covers the same class, but + // only since br gained a decoder can a br body fail this way at all. + mux.HandleFunc("/brotli-corrupt", func(w http.ResponseWriter, _ *http.Request) { + write(w, http.StatusOK, []byte(`{"status":"success"}`), http.Header{"Content-Encoding": {"br"}}) }) diff --git a/go.mod b/go.mod index 9fd1be7..550f4d7 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,9 @@ go 1.26 toolchain go1.26.5 require ( + github.com/andybalholm/brotli v1.2.2 github.com/itchyny/gojq v0.12.19 + github.com/klauspost/compress v1.19.2 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 ) diff --git a/go.sum b/go.sum index 3de9430..327a92a 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= +github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= @@ -5,11 +7,15 @@ github.com/itchyny/gojq v0.12.19 h1:ttXA0XCLEMoaLOz5lSeFOZ6u6Q3QxmG46vfgI4O0DEs= github.com/itchyny/gojq v0.12.19/go.mod h1:5galtVPDywX8SPSOrqjGxkBeDhSxEW1gSxoy7tn1iZY= github.com/itchyny/timefmt-go v0.1.8 h1:1YEo1JvfXeAHKdjelbYr/uCuhkybaHCeTkH8Bo791OI= github.com/itchyny/timefmt-go v0.1.8/go.mod h1:5E46Q+zj7vbTgWY8o5YkMeYb4I6GeWLFnetPy5oBrAI= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/jq_test.go b/jq_test.go index dc04d49..ad7b6ea 100644 --- a/jq_test.go +++ b/jq_test.go @@ -286,10 +286,10 @@ func Test_decodeJSON_parsesOnce(t *testing.T) { // the reader would otherwise go looking for a malformed payload (#27). t.Run("an undecoded body reports the encoding", func(t *testing.T) { enc := jqResponse(jqDoc) - enc.Encoding = "br" - enc.DecodeErr = errors.New("no decoder for \"br\"") + enc.Encoding = "compress" + enc.DecodeErr = errors.New("no decoder for \"compress\"") _, err := enc.decodeJSON() - checkErrMatch(t, "decodeJSON", err, `^body: response is br-encoded`) + checkErrMatch(t, "decodeJSON", err, `^body: response is compress-encoded`) }) } diff --git a/main.go b/main.go index 2b336e1..20bb34b 100644 --- a/main.go +++ b/main.go @@ -76,9 +76,9 @@ // # Compression // // A compressed body is decoded before the assertions run, so --assert-body and -// friends always see the payload. gzip and deflate are understood; an encoding -// nothing here can remove fails the body assertions by name and leaves every -// other assertion alone. +// friends always see the payload. gzip, deflate, br and zstd are understood; an +// encoding nothing here can remove fails the body assertions by name and leaves +// every other assertion alone. // // The request advertises no Accept-Encoding of its own, and the response // headers are reported exactly as they arrived. net/http would decode only @@ -107,6 +107,8 @@ import ( "strings" "time" + "github.com/andybalholm/brotli" + "github.com/klauspost/compress/zstd" "github.com/spf13/cobra" "github.com/spf13/pflag" ) @@ -187,9 +189,10 @@ Retries: Compression: A compressed body is decoded before the assertions run, so --assert-body and - the other body assertions always see the payload. gzip and deflate are - understood; an encoding with no decoder here fails the body assertions by - name and leaves --assert-ok, --assert-status and --assert-header* alone. + the other body assertions always see the payload. gzip, deflate, br (brotli) + and zstd are understood; an encoding with no decoder here fails the body + assertions by name and leaves --assert-ok, --assert-status and + --assert-header* alone. Nothing is advertised in Accept-Encoding unless -H says so, and the response headers are reported exactly as they arrived -- so a body can be asserted on @@ -1239,6 +1242,15 @@ func (r *httpResponse) decodeJSON() (any, error) { var decoders = map[string]func([]byte) ([]byte, error){ "gzip": decodeGzip, "deflate": decodeDeflate, + "br": decodeBrotli, + "zstd": decodeZstd, +} + +// supportedCodings names the decoders in a stable order, so the failure for an +// encoding with no decoder can say what it does have without drifting from the +// map as it grows. +func supportedCodings() string { + return strings.Join(slices.Sorted(maps.Keys(decoders)), ", ") } // decodeBody removes the Content-Encoding from BodyBytes, leaving every header @@ -1263,7 +1275,7 @@ func (r *httpResponse) decodeBody() { default: decode, ok := decoders[enc] if !ok { - r.DecodeErr = fmt.Errorf("no decoder for %q; gzip and deflate are supported", r.Encoding) + r.DecodeErr = fmt.Errorf("no decoder for %q; %s are supported", r.Encoding, supportedCodings()) return } @@ -1276,6 +1288,27 @@ func (r *httpResponse) decodeBody() { } } +// decodeBrotli removes a brotli coding. There is no brotli in the standard +// library, which is the whole reason this took a dependency; andybalholm/brotli +// is pure Go and brings nothing else with it. +func decodeBrotli(b []byte) ([]byte, error) { + return io.ReadAll(brotli.NewReader(bytes.NewReader(b))) +} + +// decodeZstd removes a zstd coding (RFC 8878). +// +// klauspost/compress is a large repository, but only the zstd package links +// into the binary, so the cost is the decoder rather than the library. +func decodeZstd(b []byte) ([]byte, error) { + zr, err := zstd.NewReader(bytes.NewReader(b)) + if err != nil { + return nil, err + } + defer zr.Close() + + return io.ReadAll(zr) +} + func decodeGzip(b []byte) ([]byte, error) { zr, err := gzip.NewReader(bytes.NewReader(b)) if err != nil {