diff --git a/README.md b/README.md index 81ad960..4cb452e 100644 --- a/README.md +++ b/README.md @@ -377,15 +377,15 @@ itself, and leaves the rest of the run intact: ```console $ http-assert --assert-body '"status":"success"' https://cdn.example.com/ -- body: response is br-encoded and was not decoded: no decoder for "br"; gzip and deflate are supported +- body: response is compress-encoded and was not decoded: no decoder for "compress"; br, deflate, gzip, zstd are supported $ http-assert --assert-ok https://cdn.example.com/ [+] PASSED 38ms ``` -Brotli (`br`) and zstd are the encodings this covers in practice. Neither is -supported yet: [#77](https://github.com/korya/http-assert/issues/77) tracks -brotli and [#78](https://github.com/korya/http-assert/issues/78) tracks zstd. +gzip, deflate, brotli (`br`) and zstd are decoded. Brotli and zstd each cost a +dependency, which is why they are named here rather than assumed: they are what +CDNs actually serve, and a body assertion against one is the ordinary case. A header that claims an encoding the body does not have is treated the same way. Reading those bytes as plain text would be its own silent corruption. @@ -604,7 +604,7 @@ a tool whose job is checking. This is the complete list: | `-d @file` | reads the file | sends the literal string `@file` | | `-d` repeated | values joined with `&` | rejected, exit `71` | | `--retry` | transport errors and a fixed set of transient statuses, exponential backoff | any failed attempt, assertion failures included, fixed delay | -| Response decompression | opt-in via `--compressed` | always: gzip and deflate are decoded before assertions run | +| Response decompression | opt-in via `--compressed` | always: gzip, deflate, br and zstd are decoded before assertions run | | Pointing at a backend | `--resolve host:port:addr` takes an address | `--maphost 'host:port=dst[:port]'` takes a hostname or an address | ## License diff --git a/assertions_test.go b/assertions_test.go index dd32cad..679a9f4 100644 --- a/assertions_test.go +++ b/assertions_test.go @@ -729,8 +729,8 @@ func Test_AssertionCheckSeparatesFailureFromError(t *testing.T) { t.Run("an undecodable body is an error, not a Failure", func(t *testing.T) { res := &httpResponse{ - Encoding: "br", - DecodeErr: errors.New(`no decoder for "br"`), + Encoding: "compress", + DecodeErr: errors.New(`no decoder for "compress"`), } for name, a := range map[string]Assertion{ @@ -755,8 +755,8 @@ func Test_AssertionCheckSeparatesFailureFromError(t *testing.T) { t.Run("a status assertion is unaffected by an undecodable body", func(t *testing.T) { res := &httpResponse{ Response: &http.Response{StatusCode: 200, Status: "200 OK"}, - Encoding: "br", - DecodeErr: errors.New(`no decoder for "br"`), + Encoding: "compress", + DecodeErr: errors.New(`no decoder for "compress"`), } f, err := AssertStatusOK().Check(res) diff --git a/compression_test.go b/compression_test.go index 283804e..33dda65 100644 --- a/compression_test.go +++ b/compression_test.go @@ -145,10 +145,10 @@ func Test_decodeBody(t *testing.T) { }, { Name: "an encoding with no decoder here", - Enc: "br", - Body: []byte{0x1b, 0x13, 0x00}, + Enc: "compress", + Body: []byte("payload"), WantErr: true, - WantEncoding: "br", + WantEncoding: "compress", }, { // The header claims gzip and the bytes are not. Silently asserting @@ -246,11 +246,11 @@ func Test_bodyOf(t *testing.T) { }) t.Run("an undecoded body is refused, and says why", func(t *testing.T) { - res := encoded("br", []byte{0x1b, 0x13, 0x00}) + res := encoded("compress", []byte("payload")) res.decodeBody() _, err := bodyOf(res) - checkErrMatch(t, "bodyOf", err, `^body: response is br-encoded and was not decoded: `) + checkErrMatch(t, "bodyOf", err, `^body: response is compress-encoded and was not decoded: `) }) } @@ -275,10 +275,10 @@ func Test_bodyAssertionsRefuseAnEncodedBody(t *testing.T) { for name, a := range assertions { t.Run(name, func(t *testing.T) { - res := encoded("br", []byte{0x1b, 0x13, 0x00}) + res := encoded("compress", []byte("payload")) res.decodeBody() - checkErrMatch(t, name, check(a, res), `^body: response is br-encoded and was not decoded: `) + checkErrMatch(t, name, check(a, res), `^body: response is compress-encoded and was not decoded: `) }) } } diff --git a/e2e_compression_test.go b/e2e_compression_test.go index 5d6db7a..45ba398 100644 --- a/e2e_compression_test.go +++ b/e2e_compression_test.go @@ -111,28 +111,74 @@ func TestE2ECompressionHeadersSurvive(t *testing.T) { }) } +// TestE2ECompressionBrotliZstd covers the two codings that cost a dependency. +// +// Both are what CDNs actually serve, so a body assertion against one is the +// ordinary case rather than an exotic one (#77, #78). +func TestE2ECompressionBrotliZstd(t *testing.T) { + for path, coding := range map[string]string{"/brotli": "br", "/zstd": "zstd"} { + t.Run(coding+" is decoded before the body assertions run", func(t *testing.T) { + assertExit(t, run(t, nil, "--assert-body", `"status":"success"`, url(path)), exitOK) + assertExit(t, run(t, nil, "--assert-body-eq", `{"status":"success"}`, url(path)), exitOK) + }) + + t.Run(coding+" decodes for --assert-jq too", func(t *testing.T) { + assertExit(t, run(t, nil, "--assert-jq", `.status == "success"`, url(path)), exitOK) + }) + + // Decoding must not rewrite what the response said about itself: the + // point of decoding by hand is to assert on the payload and on the + // Content-Encoding at the same time. + t.Run(coding+" is still reported as the encoding that arrived", func(t *testing.T) { + r := run(t, nil, + "--assert-header-eq", "Content-Encoding: "+coding, + "--assert-body", `"status":"success"`, url(path)) + assertExit(t, r, exitOK) + }) + } + + // A decoder makes this failure reachable for br at all; without one the + // body never got as far as being malformed. + t.Run("a body that claims br and is not br fails the body assertions", func(t *testing.T) { + r := run(t, nil, "--assert-body", `"status":"success"`, url("/brotli-corrupt")) + assertExit(t, r, exitAssertFail) + assertContains(t, r, "body: response is br-encoded and was not decoded") + }) + + t.Run("a status check is unaffected by a corrupt br body", func(t *testing.T) { + assertExit(t, run(t, nil, "--assert-ok", url("/brotli-corrupt")), exitOK) + }) +} + // TestE2ECompressionUndecodable covers an encoding with no decoder here. // // The response still has a status and headers worth asserting on, so only the // body assertions refuse. Failing the whole run would be simpler and would -// break a status check against any CDN serving brotli. +// break a status check against any CDN serving a coding this does not know. func TestE2ECompressionUndecodable(t *testing.T) { t.Run("a status check is unaffected", func(t *testing.T) { - assertExit(t, run(t, nil, "--assert-ok", url("/brotli")), exitOK) + assertExit(t, run(t, nil, "--assert-ok", url("/unsupported")), exitOK) }) t.Run("a header check is unaffected", func(t *testing.T) { - r := run(t, nil, "--assert-header-eq", "Content-Encoding: br", url("/brotli")) + r := run(t, nil, "--assert-header-eq", "Content-Encoding: compress", url("/unsupported")) assertExit(t, r, exitOK) }) t.Run("a body check refuses, and names the encoding", func(t *testing.T) { - r := run(t, nil, "--assert-body", `"status":"success"`, url("/brotli")) + r := run(t, nil, "--assert-body", `"status":"success"`, url("/unsupported")) assertExit(t, r, exitAssertFail) - assertContains(t, r, "body: response is br-encoded and was not decoded") - assertContains(t, r, "no decoder for \"br\"") + assertContains(t, r, "body: response is compress-encoded and was not decoded") + assertContains(t, r, "no decoder for \"compress\"") // The old failure was a bare hex dump with nothing explaining it. - assertContains(t, r, "<< Payload is br-encoded and was not decoded >>") + assertContains(t, r, "<< Payload is compress-encoded and was not decoded >>") + }) + + // The list in the failure is derived from the decoder map, so a coding + // added without updating the message is not possible. + t.Run("it names what it does support", func(t *testing.T) { + r := run(t, nil, "--assert-body", `"status":"success"`, url("/unsupported")) + assertContains(t, r, "br, deflate, gzip, zstd are supported") }) // A header claiming an encoding the body does not have. Treating those diff --git a/e2e_jq_test.go b/e2e_jq_test.go index 52f426a..058e6f1 100644 --- a/e2e_jq_test.go +++ b/e2e_jq_test.go @@ -177,9 +177,9 @@ func TestE2EAssertJQBodyProblems(t *testing.T) { // An encoding nothing can decode reports the encoding, not invalid JSON. t.Run("a body still encoded", func(t *testing.T) { - r := run(t, nil, "--assert-jq", `. == 1`, url("/brotli")) + r := run(t, nil, "--assert-jq", `. == 1`, url("/unsupported")) assertExit(t, r, exitAssertFail) - assertContains(t, r, "body: response is br-encoded") + assertContains(t, r, "body: response is compress-encoded") }) // A query yielding nothing has checked nothing, so it must not pass. diff --git a/e2e_server_test.go b/e2e_server_test.go index 4c594cd..5c1ca3a 100644 --- a/e2e_server_test.go +++ b/e2e_server_test.go @@ -1,6 +1,7 @@ package main_test import ( + "bytes" "compress/flate" "compress/gzip" "compress/zlib" @@ -17,6 +18,9 @@ import ( "sync/atomic" "testing" "time" + + "github.com/andybalholm/brotli" + "github.com/klauspost/compress/zstd" ) // The e2e suite talks to two real servers started once per test run: a plain @@ -267,11 +271,39 @@ func testHandler() http.Handler { _ = zw.Close() }) - // An encoding with no decoder here. Brotli is the realistic case; the bytes - // are not real brotli because nothing in the suite could produce them, and - // the CLI never gets far enough to care. mux.HandleFunc("/brotli", func(w http.ResponseWriter, _ *http.Request) { - write(w, http.StatusOK, []byte{0x1b, 0x13, 0x00, 0x00, 0xa4, 0xb0, 0xb2}, + var buf bytes.Buffer + bw := brotli.NewWriter(&buf) + _, _ = bw.Write([]byte(`{"status":"success"}`)) + _ = bw.Close() + write(w, http.StatusOK, buf.Bytes(), http.Header{"Content-Encoding": {"br"}}) + }) + + mux.HandleFunc("/zstd", func(w http.ResponseWriter, _ *http.Request) { + var buf bytes.Buffer + zw, err := zstd.NewWriter(&buf) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + _, _ = zw.Write([]byte(`{"status":"success"}`)) + _ = zw.Close() + write(w, http.StatusOK, buf.Bytes(), http.Header{"Content-Encoding": {"zstd"}}) + }) + + // An encoding with no decoder here, and the one place the suite needs a + // coding that will stay unsupported. RFC 9110 still registers `compress` + // (LZW), and effectively nothing serves it -- unlike br and zstd, which + // were the realistic examples right up until they were implemented. + mux.HandleFunc("/unsupported", func(w http.ResponseWriter, _ *http.Request) { + write(w, http.StatusOK, []byte(`{"status":"success"}`), + http.Header{"Content-Encoding": {"compress"}}) + }) + + // Claims brotli and is not brotli. gzip-corrupt covers the same class, but + // only since br gained a decoder can a br body fail this way at all. + mux.HandleFunc("/brotli-corrupt", func(w http.ResponseWriter, _ *http.Request) { + write(w, http.StatusOK, []byte(`{"status":"success"}`), http.Header{"Content-Encoding": {"br"}}) }) diff --git a/go.mod b/go.mod index 9fd1be7..550f4d7 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,9 @@ go 1.26 toolchain go1.26.5 require ( + github.com/andybalholm/brotli v1.2.2 github.com/itchyny/gojq v0.12.19 + github.com/klauspost/compress v1.19.2 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 ) diff --git a/go.sum b/go.sum index 3de9430..327a92a 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= +github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= @@ -5,11 +7,15 @@ github.com/itchyny/gojq v0.12.19 h1:ttXA0XCLEMoaLOz5lSeFOZ6u6Q3QxmG46vfgI4O0DEs= github.com/itchyny/gojq v0.12.19/go.mod h1:5galtVPDywX8SPSOrqjGxkBeDhSxEW1gSxoy7tn1iZY= github.com/itchyny/timefmt-go v0.1.8 h1:1YEo1JvfXeAHKdjelbYr/uCuhkybaHCeTkH8Bo791OI= github.com/itchyny/timefmt-go v0.1.8/go.mod h1:5E46Q+zj7vbTgWY8o5YkMeYb4I6GeWLFnetPy5oBrAI= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/jq_test.go b/jq_test.go index dc04d49..ad7b6ea 100644 --- a/jq_test.go +++ b/jq_test.go @@ -286,10 +286,10 @@ func Test_decodeJSON_parsesOnce(t *testing.T) { // the reader would otherwise go looking for a malformed payload (#27). t.Run("an undecoded body reports the encoding", func(t *testing.T) { enc := jqResponse(jqDoc) - enc.Encoding = "br" - enc.DecodeErr = errors.New("no decoder for \"br\"") + enc.Encoding = "compress" + enc.DecodeErr = errors.New("no decoder for \"compress\"") _, err := enc.decodeJSON() - checkErrMatch(t, "decodeJSON", err, `^body: response is br-encoded`) + checkErrMatch(t, "decodeJSON", err, `^body: response is compress-encoded`) }) } diff --git a/main.go b/main.go index 2b336e1..20bb34b 100644 --- a/main.go +++ b/main.go @@ -76,9 +76,9 @@ // # Compression // // A compressed body is decoded before the assertions run, so --assert-body and -// friends always see the payload. gzip and deflate are understood; an encoding -// nothing here can remove fails the body assertions by name and leaves every -// other assertion alone. +// friends always see the payload. gzip, deflate, br and zstd are understood; an +// encoding nothing here can remove fails the body assertions by name and leaves +// every other assertion alone. // // The request advertises no Accept-Encoding of its own, and the response // headers are reported exactly as they arrived. net/http would decode only @@ -107,6 +107,8 @@ import ( "strings" "time" + "github.com/andybalholm/brotli" + "github.com/klauspost/compress/zstd" "github.com/spf13/cobra" "github.com/spf13/pflag" ) @@ -187,9 +189,10 @@ Retries: Compression: A compressed body is decoded before the assertions run, so --assert-body and - the other body assertions always see the payload. gzip and deflate are - understood; an encoding with no decoder here fails the body assertions by - name and leaves --assert-ok, --assert-status and --assert-header* alone. + the other body assertions always see the payload. gzip, deflate, br (brotli) + and zstd are understood; an encoding with no decoder here fails the body + assertions by name and leaves --assert-ok, --assert-status and + --assert-header* alone. Nothing is advertised in Accept-Encoding unless -H says so, and the response headers are reported exactly as they arrived -- so a body can be asserted on @@ -1239,6 +1242,15 @@ func (r *httpResponse) decodeJSON() (any, error) { var decoders = map[string]func([]byte) ([]byte, error){ "gzip": decodeGzip, "deflate": decodeDeflate, + "br": decodeBrotli, + "zstd": decodeZstd, +} + +// supportedCodings names the decoders in a stable order, so the failure for an +// encoding with no decoder can say what it does have without drifting from the +// map as it grows. +func supportedCodings() string { + return strings.Join(slices.Sorted(maps.Keys(decoders)), ", ") } // decodeBody removes the Content-Encoding from BodyBytes, leaving every header @@ -1263,7 +1275,7 @@ func (r *httpResponse) decodeBody() { default: decode, ok := decoders[enc] if !ok { - r.DecodeErr = fmt.Errorf("no decoder for %q; gzip and deflate are supported", r.Encoding) + r.DecodeErr = fmt.Errorf("no decoder for %q; %s are supported", r.Encoding, supportedCodings()) return } @@ -1276,6 +1288,27 @@ func (r *httpResponse) decodeBody() { } } +// decodeBrotli removes a brotli coding. There is no brotli in the standard +// library, which is the whole reason this took a dependency; andybalholm/brotli +// is pure Go and brings nothing else with it. +func decodeBrotli(b []byte) ([]byte, error) { + return io.ReadAll(brotli.NewReader(bytes.NewReader(b))) +} + +// decodeZstd removes a zstd coding (RFC 8878). +// +// klauspost/compress is a large repository, but only the zstd package links +// into the binary, so the cost is the decoder rather than the library. +func decodeZstd(b []byte) ([]byte, error) { + zr, err := zstd.NewReader(bytes.NewReader(b)) + if err != nil { + return nil, err + } + defer zr.Close() + + return io.ReadAll(zr) +} + func decodeGzip(b []byte) ([]byte, error) { zr, err := gzip.NewReader(bytes.NewReader(b)) if err != nil {