Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
115 changes: 68 additions & 47 deletions http-client.carp
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,7 @@ apart.
pos Int
error (Maybe ClientError)
chunked Bool
chunk-left Int
done Bool
remaining (Maybe Long)
status-code Int
Expand Down Expand Up @@ -295,6 +296,13 @@ apart.
(set-pos! s 0)
true))))

(hidden fill-or-fail!)
(private fill-or-fail!)
; false once the body has ended, failing it with `why` unless a read failed
(defn fill-or-fail! [s why]
(or (fill! s)
(do (when (Maybe.nothing? (error s)) (fail-parse! s @why)) false)))

(hidden skip-trailers!)
(private skip-trailers!)
; consumes as much of the trailer section as is already buffered
Expand Down Expand Up @@ -354,57 +362,69 @@ apart.

(hidden poll-chunked)
(private poll-chunked)
; chunk-left: -1 before a size line, 0 before a chunk's CRLF, else bytes owed
(defn poll-chunked [s]
(let-do [result (Maybe.Nothing)
running true]
running true
len (String.length (buf s))]
(while running
(let [len (String.length (buf s))
p @(pos s)
eol (String.find-crlf (buf s) p len)]
(if (= eol -1)
(unless-do (fill! s)
(when (Maybe.nothing? (error s))
(fail-parse! s
(if (< p len)
@"malformed chunked body: unterminated chunk size line"
@"malformed chunked body: missing terminating zero-size chunk")))
(set! running false))
(let [line &(String.byte-slice (buf s) p eol)]
(match (parse-size-line line)
(Maybe.Nothing)
(let [p @(pos s)
left @(chunk-left s)]
(cond
(> left 0)
(if (< p len)
(let-do [n (min left (- len p))]
(set! result
(Maybe.Just (String.byte-slice (buf s) p (+ p n))))
(set-pos! s (+ p n))
(set-chunk-left! s (- left n))
(set! running false))
(do
(set! running
(fill-or-fail! s
"malformed chunked body: truncated chunk data"))
(set! len (String.length (buf s)))))
(= left 0)
(cond
(< (- len p) 2)
(do
(fail-parse! s
(fmt
"malformed chunked body: invalid chunk size '%s'"
line))
(set! running false))
(Maybe.Just size)
(if (= size 0)
(do
(set-pos! s (+ eol 2))
(skip-trailers! s)
(set-done! s true)
(set! running false))
(let [start (+ eol 2)
avail (- len start)]
; compared without adding, so a huge size cannot overflow
(if (> size (- avail 2))
(unless-do (fill! s)
(when (Maybe.nothing? (error s))
(fail-parse! s
@"malformed chunked body: truncated chunk data"))
(set! running false))
(let-do [end (+ start size)]
(if (crlf-at? (buf s) end)
(do
(set! result
(Maybe.Just (String.byte-slice (buf s)
start
end)))
(set-pos! s (+ end 2)))
(fail-parse! s
@"malformed chunked body: chunk data missing CRLF"))
(set! running false))))))))))
(set! running
(fill-or-fail! s
"malformed chunked body: truncated chunk data"))
(set! len (String.length (buf s))))
(crlf-at? (buf s) p)
(do (set-pos! s (+ p 2)) (set-chunk-left! s -1))
(do
(fail-parse! s
@"malformed chunked body: chunk data missing CRLF")
(set! running false)))
(let [eol (String.find-crlf (buf s) p len)]
(if (= eol -1)
(do
(set! running
(fill-or-fail! s
(if (< p len)
"malformed chunked body: unterminated chunk size line"
"malformed chunked body: missing terminating zero-size chunk")))
(set! len (String.length (buf s))))
(let [line &(String.byte-slice (buf s) p eol)]
(match (parse-size-line line)
(Maybe.Nothing)
(do
(fail-parse! s
(fmt
"malformed chunked body: invalid chunk size '%s'"
line))
(set! running false))
(Maybe.Just size)
(do
(set-pos! s (+ eol 2))
(if (= size 0)
(do
(skip-trailers! s)
(set-done! s true)
(set! running false))
(set-chunk-left! s size))))))))))
result))

(doc poll "returns the next chunk of decoded body data, or `Nothing` when the
Expand Down Expand Up @@ -708,6 +728,7 @@ to follow. Used by `request`, `request-stream`, and convenience methods.")
0
(Maybe.Nothing)
is-chunked
-1
false
left
code
Expand Down
45 changes: 45 additions & 0 deletions test/http-client.carp
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,23 @@
(ResponseStream.close stream)
(fmt "[%s] %s" &body &ended)))))

; The data the first poll of a stream hands out.
(defn first-poll [url]
(match (Client.request-stream "GET"
url
(the (Map String (Array String)) {})
"")
(Result.Error e) (ClientError.message &e)
(Result.Success stream)
(let-do [first (Maybe.from (ResponseStream.poll &stream) @"nothing")]
(ResponseStream.close stream)
first)))

(defn numbered-lines [n]
(let-do [sb (StringBuf.create)]
(for [i 0 n] (StringBuf.append-str &sb &(fmt "%07d\n" i)))
(StringBuf.into-string sb)))

; The held routes keep the connection open for 10 s after the body.
(defn without-waiting [f]
(let-do [t0 (System.time)
Expand Down Expand Up @@ -918,6 +935,16 @@
&(status-and-body (Client.get "http://127.0.0.1:8791/chunked-missing-crlf"))
"chunk data not followed by CRLF is an error")

(assert-equal test
"malformed chunked body: truncated chunk data"
&(status-and-body (Client.get "http://127.0.0.1:8791/chunked-cr-close"))
"a close between the CR and the LF after chunk data is an error")

(assert-equal test
"200 [hello]"
&(status-and-body (Client.get "http://127.0.0.1:8791/chunked-split"))
"a size line and a CRLF that arrive in pieces are read whole")

(assert-equal test
"malformed chunked body: truncated chunk data"
&(status-and-body (Client.get "http://127.0.0.1:8791/chunked-oversize"))
Expand Down Expand Up @@ -970,6 +997,17 @@
&(stream-error "http://127.0.0.1:8791/chunked-truncated")
"a stream that dies mid-chunk carries the reason")

(assert-equal test
"hello "
&(without-waiting
&(fn [] (first-poll "http://127.0.0.1:8791/chunked-held-half")))
"the part of a chunk that has arrived is handed out without waiting for the rest")

(assert-true test
(= &(numbered-lines 131072)
&(drained-body "http://127.0.0.1:8791/chunked-large"))
"a 1 MiB chunk is decoded whole")

(assert-equal test
"ok"
&(drained-body "http://127.0.0.1:8791/get")
Expand Down Expand Up @@ -1053,6 +1091,13 @@
&(stream-error-kind "http://127.0.0.1:8791/close-short")
"a body that ends before Content-Length is a Receive error")

(assert-equal test
"receive"
&(error-kind
(let [cfg (RequestConfig.init 0 1 10)]
(Client.get-with-config "http://127.0.0.1:8791/chunked-held-half" &cfg)))
"a read timeout mid-chunk stays a Receive error")

(assert-equal test
"413 [payload too large]"
&(status-and-body
Expand Down
21 changes: 21 additions & 0 deletions test/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,16 @@ def _route(self):
return self._chunked_raw(b"5\r\nhello\r\n")
if path == "/chunked-missing-crlf":
return self._chunked_raw(b"5\r\nhelloXX0\r\n\r\n")
if path == "/chunked-cr-close":
return self._chunked_raw(b"5\r\nhello\r")
# one chunk whose size line and CRLF each arrive in pieces
if path == "/chunked-split":
self.connection.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self._chunked_raw(b"5")
for piece in (b"\r\nhello", b"\r", b"\n0\r\n\r\n"):
time.sleep(0.05)
self.wfile.write(piece)
return
if path == "/chunked-leading-space":
return self._chunked_raw(b" 5\r\nhello\r\n0\r\n\r\n")
if path == "/chunked-trailing-space":
Expand All @@ -245,6 +255,17 @@ def _route(self):
if path == "/chunked-oversize":
return self._chunked_raw(b"1000001\r\nhello")

# one chunk whose second half arrives 10 s after the first
if path == "/chunked-held-half":
self._chunked_raw(b"b\r\nhello ")
time.sleep(10)
self.wfile.write(b"world\r\n0\r\n\r\n")
return
# one 1 MiB chunk of numbered lines
if path == "/chunked-large":
body = b"".join(b"%07d\n" % i for i in range(131072))
return self._chunked_raw(b"%x\r\n%s\r\n0\r\n\r\n" % (len(body), body))

# a well-formed body with a chunk extension and a trailer section
if path == "/chunked-ext":
return self._chunked_raw(b"5;name=value\r\nhello\r\n0\r\n\r\n")
Expand Down
Loading