From 5aeaa26a9ba556b9f9e15f84508d51dc267d5bb3 Mon Sep 17 00:00:00 2001 From: Mohammad Nejati Date: Thu, 6 Aug 2026 23:31:37 +0330 Subject: [PATCH 1/3] eliminate the per-message buffer compaction --- include/boost/burl/detail/circular_buffer.hpp | 15 + include/boost/burl/head_parser.hpp | 202 ++++++------ src/detail/circular_buffer.cpp | 84 ++++- src/detail/parser.cpp | 77 ++--- src/head_parser.cpp | 63 ++-- test/unit/detail/parser.cpp | 142 ++++++++ test/unit/head_parser.cpp | 304 +++++++++++------- test/unit/request_head.cpp | 6 +- test/unit/response_head.cpp | 11 +- 9 files changed, 589 insertions(+), 315 deletions(-) diff --git a/include/boost/burl/detail/circular_buffer.hpp b/include/boost/burl/detail/circular_buffer.hpp index 031d53a..178654d 100644 --- a/include/boost/burl/detail/circular_buffer.hpp +++ b/include/boost/burl/detail/circular_buffer.hpp @@ -35,6 +35,9 @@ struct circular_buffer bool full() const noexcept; + bool + wrapped() const noexcept; + std::size_t size() const noexcept; @@ -52,6 +55,18 @@ struct circular_buffer void consume(std::size_t n) noexcept; + + void + reset(char* p) noexcept; + + void + shed(std::size_t n) noexcept; + + void + slide(char* p) noexcept; + + char* + linearize(char* floor) noexcept; }; } // namespace detail diff --git a/include/boost/burl/head_parser.hpp b/include/boost/burl/head_parser.hpp index e273ab6..c8984ae 100644 --- a/include/boost/burl/head_parser.hpp +++ b/include/boost/burl/head_parser.hpp @@ -14,7 +14,6 @@ #include #include -#include #include #include @@ -50,28 +49,36 @@ struct header_limits directly in a supplied buffer. Received bytes remain in place throughout parsing. + The caller owns the buffer and the fill + cursor. Bytes are placed at the parse base, + the address the header is built at, which is + initially the start of the buffer. Their + running total is handed to @ref parse, which + resumes where the previous call left off. While more input is required, @ref parse reports @ref http::error::need_data. Once the terminating empty line is parsed, the header can be inspected through @ref message_head, - ref request_head, or @ref response_head. Bytes - following the header belong to the payload and - are returned by @ref leftovers. + @ref request_head, or @ref response_head. + Bytes beyond @ref message_head_base::buffer + belong to the payload and are left untouched. Space for the field lookup table is reserved - at the end of the buffer. Use @ref - bytes_needed to determine the buffer size + at the end of the buffer, beginning at + @ref ceiling. Nothing may be written there. Use + @ref bytes_needed to determine the buffer size required for a given set of limits. If the - remaining writable space cannot accommodate the - rest of the header, @ref parse fails with + header cannot complete below @ref ceiling, + @ref parse fails with @ref http::error::in_place_overflow. Obsolete folded field values (obs-fold) are unfolded in place by replacing each folding - CRLF with spaces. This is the only - modification made to received bytes. - Consequently, bytes already consumed by the - parser must not be overwritten. The buffer must + CRLF with spaces; this is the only + modification made to received bytes. Because + the header is built from the bytes where they + lie, bytes already consumed by the parser + must not be overwritten, and the buffer must remain valid for the lifetime of the parser. @see @@ -112,7 +119,8 @@ class head_parser /** Constructor. Default constructed parsers behave as if - constructed with a zero-size buffer. + constructed with `is_request == true` and + a zero-size buffer. */ head_parser() noexcept : head_parser(true, nullptr, 0) @@ -152,62 +160,73 @@ class head_parser /** Re-arm the parser for a new header. Clears the parsed header and parse state, - re-arming the parser over the same buffer - and limits. Nothing outside the parser - itself is touched. + re-arming the parser to build the next + header at `base`. The limits and @ref + ceiling are retained; the usable capacity + becomes the distance from `base` to + @ref ceiling. Bytes belonging to the next message may - already be present in the buffer. Move - those bytes to the front of the buffer and - pass their count here. The parser treats - them as though they had been reported - through - @ref commit. - - @param leftovers The number of received - bytes at the front of the buffer. + already be present at `base`; they are + reported to the next @ref parse like any + others. Building the header where its + bytes already lie avoids moving them. + + @par Preconditions + `base` lies within the buffer supplied at + construction and is not greater than + @ref ceiling. + + @param base The address to build the + header at. */ BOOST_BURL_DECL void - reset(std::size_t leftovers = 0) noexcept; + reset(char* base) noexcept; - /** Return the buffer region available for - receiving bytes. + /** Continue building the header at a lower address. - The returned region starts immediately - after the received bytes and ends before - the space reserved for the field lookup - table. + The caller has already relocated the + received bytes to `base`; parsing continues + from there, gaining room to receive more + input. Parsing may resume normally, whether + the header is complete or not. - The region may be empty. In that case, - a @ref parse operation that requires - additional buffer space fails with @ref - http::error::in_place_overflow instead of - requesting more input. + The field lookup table does not move. + Pointers and views into the header + obtained beforehand are invalidated. - @par Complexity - Constant. + @par Preconditions + `base` is not greater than the address the + header was built at, and the received bytes + have been moved to `base`. + + @param base The address the received + bytes were moved to. */ BOOST_BURL_DECL - capy::mutable_buffer - prepare() noexcept; + void + rebase(char* base) noexcept; - /** Report bytes received into the buffer. + /** Return the end of the writable region. - The bytes become visible to the next call - to @ref parse. + Nothing may be written at or beyond the + returned address; it is where the field + lookup table is reserved. The address is + fixed by the buffer supplied at + construction and does not move with + @ref reset or @ref rebase. - @par Preconditions - @code - n <= this->prepare().size() - @endcode + A buffer too small to hold the table + reports the parse base itself, leaving no + room to receive anything. - @param n The number of bytes received at - the front of @ref prepare. + @par Complexity + Constant. */ BOOST_BURL_DECL - void - commit(std::size_t n) noexcept; + char* + ceiling() const noexcept; /** Parse the received bytes. @@ -215,58 +234,37 @@ class head_parser It continues until the header is complete, more data is required, or an error occurs. + @par Preconditions + `n` bytes are readable at the parse base, and + `n` is not less than the count passed to the + previous call, nor greater than the distance + from the parse base to @ref ceiling. + @par Complexity - Linear in the size of @ref leftovers. + Linear in the number of bytes not yet parsed. + + @param n The total number of bytes received at + the parse base. @param ec Set to: + - Zero if the header completed and its + payload framing is valid. - @ref http::error::need_data if more input is - required and @ref prepare still provides space. - - @ref http::error::in_place_overflow if more input - is required but no writable space remains. - - A syntax or limit error otherwise. + required and room remains below @ref ceiling. + - @ref http::error::in_place_overflow if more + input is required but no room remains. + - A syntax, framing, or limit error otherwise. */ BOOST_BURL_DECL void - parse(system::error_code& ec) noexcept; - - /** Return true if any bytes were received. - - This becomes true when the first byte of the - message arrives. - - @par Complexity - Constant. - */ - bool - got_some() const noexcept - { - return in_size_ != 0; - } - - /** Return the received but unconsumed bytes. - - Once @ref parse succeeds this holds the - payload bytes which followed the header into - the buffer, beginning at the end of - @ref message_head_base::buffer. - - @par Complexity - Constant. - */ - capy::mutable_buffer - leftovers() noexcept - { - auto& h = h_(); - return { - h.buf_ + h.size_, - in_size_ - - (std::size_t(h.prefix_) + h.size_) }; - } + parse( + std::size_t n, + system::error_code& ec) noexcept; - /** Returns the limits supplied at construction. + /** Return the limits enforced by the parser. - The returned value may differ only in that - `max_size` is capped at + These are the limits supplied at + construction, with `max_size` capped at @ref fields_base::max_buffer_size. */ header_limits const& @@ -278,8 +276,9 @@ class head_parser /** Return the parsed header. Returns the parts of the header common to - requests and responses. The header is empty - until @ref parse succeeds. + requests and responses. Until @ref parse + succeeds, the header holds only the parts + parsed so far. */ class message_head_base const& message_head() const noexcept @@ -289,8 +288,8 @@ class head_parser /** Return the parsed header. - The header is empty until @ref parse - succeeds. + Until @ref parse succeeds, the header + holds only the parts parsed so far. @par Preconditions The parser was constructed with @@ -305,8 +304,8 @@ class head_parser /** Return the parsed header. - The header is empty until @ref parse - succeeds. + Until @ref parse succeeds, the header + holds only the parts parsed so far. @par Preconditions The parser was constructed with @@ -382,7 +381,6 @@ class head_parser header_limits limits_; bool is_req_ = false; state st_ = state::start_line; - std::size_t in_size_ = 0; burl::message_head_base& h_() noexcept diff --git a/src/detail/circular_buffer.cpp b/src/detail/circular_buffer.cpp index d0ee7ea..a237e6f 100644 --- a/src/detail/circular_buffer.cpp +++ b/src/detail/circular_buffer.cpp @@ -11,6 +11,11 @@ #include "util.hpp" +#include + +#include +#include + namespace boost { namespace burl @@ -32,6 +37,13 @@ full() const noexcept return len == cap; } +bool +circular_buffer:: +wrapped() const noexcept +{ + return pos + len > cap; +} + std::size_t circular_buffer:: size() const noexcept @@ -43,7 +55,7 @@ std::array circular_buffer:: data() const noexcept { - if(pos + len <= cap) + if(!wrapped()) return { { { ptr + pos, len }, { ptr, 0 } } }; return { { { ptr + pos, cap - pos }, { ptr, len - (cap - pos) } } }; @@ -53,7 +65,7 @@ capy::const_buffer circular_buffer:: first(std::size_t n) const noexcept { - auto const k = (pos + len <= cap) ? len : cap - pos; + auto const k = wrapped() ? cap - pos : len; return { ptr + pos, clamp(k, n) }; } @@ -92,6 +104,74 @@ consume(std::size_t n) noexcept len -= n; } +void +circular_buffer:: +reset(char* p) noexcept +{ + BOOST_ASSERT(p <= ptr + cap); + cap = static_cast((ptr + cap) - p); + ptr = p; + pos = 0; + len = 0; +} + +void +circular_buffer:: +shed(std::size_t n) noexcept +{ + BOOST_ASSERT(pos == 0); + BOOST_ASSERT(n <= len); + ptr += n; + cap -= n; + len -= n; +} + +void +circular_buffer:: +slide(char* p) noexcept +{ + BOOST_ASSERT(pos == 0); + BOOST_ASSERT(p <= ptr); + std::memmove(p, ptr, len); + cap += static_cast(ptr - p); + ptr = p; +} + +char* +circular_buffer:: +linearize(char* floor) noexcept +{ + BOOST_ASSERT(floor <= ptr); + char* p = floor; + if(len != 0 && !wrapped()) + { + p = ptr + pos; + } + else if(len != 0) + { + auto const bufs = data(); + auto const* a = static_cast(bufs[0].data()); + auto an = bufs[0].size(); + auto const* b = static_cast(bufs[1].data()); + auto const bn = bufs[1].size(); + char* base = floor; + do + { + auto* bp = (std::min)(base + an, const_cast(a) - bn); + b = static_cast(std::memmove(bp, b, bn)); + auto chunk_a = static_cast(b - base); + std::memcpy(base, a, chunk_a); + an -= chunk_a; + base += chunk_a; + a += chunk_a; + } while(an); + } + cap = static_cast((ptr + cap) - p); + ptr = p; + pos = 0; + return p; +} + } // namespace detail } // namespace burl } // namespace boost diff --git a/src/detail/parser.cpp b/src/detail/parser.cpp index 67c9d1c..65585a7 100644 --- a/src/detail/parser.cpp +++ b/src/detail/parser.cpp @@ -236,32 +236,6 @@ collect( return dest.first(n); } -void -move_leftovers( - char* base, - std::array const& bufs) noexcept -{ - auto const* a = static_cast(bufs[0].data()); - auto an = bufs[0].size(); - auto const* b = static_cast(bufs[1].data()); - auto const bn = bufs[1].size(); - if(bn == 0) - { - std::memmove(base, a, an); - return; - } - do - { - auto* bp = (std::min)(base + an, const_cast(a) - bn); - b = static_cast(std::memmove(bp, b, bn)); - auto chunk_a = static_cast(b - base); - std::memcpy(base, a, chunk_a); - an -= chunk_a; - base += chunk_a; - a += chunk_a; - } while(an); -} - auto prefix( auto buf, @@ -312,7 +286,8 @@ parser( buf_ = std::make_unique_for_overwrite( h_cap + cfg.dec_buffer); hp_ = { is_req_, buf_.get(), h_cap, cfg.hdr_limits }; - in_ = { buf_.get(), 0 }; + in_ = { buf_.get(), static_cast( + hp_.ceiling() - buf_.get()) }; out_ = { buf_.get() + h_cap, cfg.dec_buffer }; } @@ -391,9 +366,8 @@ start(bool head) if(payload_sized() && got_body_) in_.consume(payload_rem()); - move_leftovers(buf_.get(), in_.data()); - hp_.reset(in_.size()); // pass leftovers - in_ = { buf_.get(), 0 }; + hp_.reset( + in_.linearize(buf_.get())); dec_ = nullptr; chunk_rem_ = 0; @@ -414,8 +388,8 @@ void parser:: reset(capy::any_read_stream stream) noexcept { - hp_.reset(); - in_ = { buf_.get(), 0 }; + hp_.reset(buf_.get()); + in_.reset(buf_.get()); stream_ = std::move(stream); dec_ = nullptr; @@ -598,39 +572,35 @@ read_header() for(;;) { system::error_code ec; - hp_.parse(ec); + hp_.parse(in_.size(), ec); if(ec) { + if(ec == in_place_overflow && in_.ptr != buf_.get()) + { + in_.slide(buf_.get()); + hp_.rebase(buf_.get()); + continue; + } if(ec != need_more_input) co_return { ec }; if(eof_) { - if(!hp_.got_some()) + if(in_.empty()) co_return { end_of_stream }; co_return { incomplete }; } - auto [rec, n] = co_await stream_.read_some(hp_.prepare()); - hp_.commit(n); - if(rec == capy::cond::eof) - eof_ = true; - else if(rec) - co_return { rec }; + if(auto [fec] = co_await refill(); fec) + co_return { fec }; continue; } - // TODO: resize out_ based on payload and decoder - auto const leftovers = hp_.leftovers(); - in_ = { - static_cast(leftovers.data()), - leftovers.size() + hp_.prepare().size(), - 0, - leftovers.size() }; - auto const& h = hp_.message_head(); got_header_ = true; payload_ = head_ ? payload::none : h.payload(); payload_size_ = h.content_length().value_or(0); + auto const head_size = h.buffer().size(); + switch(payload_) { case payload::error: @@ -639,7 +609,7 @@ read_header() got_body_ = true; break; case payload::size: - if(payload_rem() <= in_.size()) + if(payload_rem() <= in_.size() - head_size) got_body_ = true; break; case payload::chunked: @@ -649,6 +619,15 @@ read_header() got_body_ = true; break; } + + if(!got_body_) + { + in_.slide(buf_.get()); + hp_.rebase(buf_.get()); + } + + in_.shed(head_size); + co_return {}; } } diff --git a/src/head_parser.cpp b/src/head_parser.cpp index d2f6754..1bd90fb 100644 --- a/src/head_parser.cpp +++ b/src/head_parser.cpp @@ -469,7 +469,6 @@ head_parser(head_parser&& other) noexcept : limits_(other.limits_) , is_req_(other.is_req_) , st_(other.st_) - , in_size_(other.in_size_) { if(is_req_) ::new(static_cast(&s_.req)) @@ -485,10 +484,9 @@ operator=(head_parser&& other) noexcept { if(this == &other) return *this; - limits_ = other.limits_; - in_size_ = other.in_size_; - is_req_ = other.is_req_; - st_ = other.st_; + limits_ = other.limits_; + is_req_ = other.is_req_; + st_ = other.st_; if(is_req_) ::new(static_cast(&s_.req)) class request_head_base(other.s_.req); @@ -498,53 +496,54 @@ operator=(head_parser&& other) noexcept return *this; } -void +char* head_parser:: -reset(std::size_t leftovers) noexcept +ceiling() const noexcept { - auto& h = h_(); + auto const& h = h_(); + auto const reserve = + message_head_base::table_space_(limits_.max_fields); auto const cap = h.capacity_in_bytes(); - if(is_req_) - ::new(static_cast(&s_.req)) - class request_head_base(h.base_(), cap); - else - ::new(static_cast(&s_.res)) - class response_head_base(h.base_(), cap); - in_size_ = 0; - st_ = state::start_line; - commit(leftovers); + return h.base_() + (cap > reserve ? cap - reserve : 0); } -capy::mutable_buffer +void head_parser:: -prepare() noexcept +reset(char* base) noexcept { auto& h = h_(); - auto const reserve = - message_head_base::table_space_(limits_.max_fields); - auto const cap = h.capacity_in_bytes(); - if(reserve + in_size_ >= cap) - return { h.base_() + in_size_, 0 }; - return { h.base_() + in_size_, cap - reserve - in_size_ }; + BOOST_ASSERT(base <= ceiling()); + auto const cap = static_cast(h.end_ - base); + if(is_req_) + ::new(static_cast(&s_.req)) + class request_head_base(base, cap); + else + ::new(static_cast(&s_.res)) + class response_head_base(base, cap); + st_ = state::start_line; } void head_parser:: -commit(std::size_t n) noexcept +rebase(char* base) noexcept { - BOOST_ASSERT(n <= prepare().size()); - in_size_ += n; + auto& h = h_(); + BOOST_ASSERT(base <= h.base_()); + h.buf_ = base + h.prefix_; } void head_parser:: -parse(system::error_code& ec) noexcept +parse( + std::size_t n, + system::error_code& ec) noexcept { ec.clear(); auto const& h = h_(); char const* it = h.buf_ + h.size_; - auto* const end = it + - (in_size_ - h.prefix_ - h.size_); + BOOST_ASSERT(n >= std::size_t(h.prefix_) + h.size_); + BOOST_ASSERT(h.base_() + n <= ceiling()); + auto* const end = h.base_() + n; switch(st_) { @@ -588,7 +587,7 @@ parse(system::error_code& ec) noexcept } } - if(ec == error::need_data && prepare().size() == 0) + if(ec == error::need_data && end >= ceiling()) ec = error::in_place_overflow; } diff --git a/test/unit/detail/parser.cpp b/test/unit/detail/parser.cpp index 7b09d82..e315a07 100644 --- a/test/unit/detail/parser.cpp +++ b/test/unit/detail/parser.cpp @@ -2683,6 +2683,145 @@ class parser_test }()); } + void + testStartParksCompleteMessage() + { + capy::test::read_stream server; + capy::any_read_stream stream(&server); + test_parser pr({}, &stream); + + capy::test::run_blocking()([&]() -> capy::task<> + { + server.provide( + "HTTP/1.1 200 OK\r\n" + "Content-Length: 5\r\n" + "\r\n" + "hello" + "HTTP/1.1 200 OK\r\n" + "Content-Length: 3\r\n" + "\r\n" + "bye"); + + pr.start(); + char const* first = nullptr; + { + auto [ec, body] = co_await pr.read_body(); + BOOST_TEST(!ec); + BOOST_TEST(body == "hello"); + first = body.data(); + } + pr.consume(5); + + pr.start(); + + // the octets carried over are reported even though + // the header has not completed yet + BOOST_TEST(pr.has_buffered_data()); + { + auto [ec, body] = co_await pr.read_body(); + BOOST_TEST(!ec); + BOOST_TEST(body == "bye"); + + // the whole message was already buffered, so the + // header was parsed where its octets lay; nothing + // was moved back to the front of the buffer + BOOST_TEST(body.data() > first + 5); + } + }()); + } + + void + testStartCompactsIncompleteMessage() + { + capy::test::read_stream server; + capy::any_read_stream stream(&server); + test_parser pr({}, &stream); + + capy::test::run_blocking()([&]() -> capy::task<> + { + server.provide( + "HTTP/1.1 200 OK\r\n" + "Content-Length: 5\r\n" + "\r\n" + "hello" + "HTTP/1.1 200 OK\r\n" + "Content-Length: 3\r\n" + "\r\n"); + + pr.start(); + char const* first = nullptr; + { + auto [ec, body] = co_await pr.read_body(); + BOOST_TEST(!ec); + BOOST_TEST(body == "hello"); + first = body.data(); + } + pr.consume(5); + + pr.start(); + { + auto [ec] = co_await pr.read_header(); + BOOST_TEST(!ec); + } + server.provide("bye"); + { + auto [ec, body] = co_await pr.read_body(); + BOOST_TEST(!ec); + BOOST_TEST(body == "bye"); + + // the body had not arrived, so the header was + // moved back to the front to recover the window; + // both headers are the same length + BOOST_TEST(body.data() == first); + } + }()); + } + + void + testStartRetriesOverflowAtParkedBase() + { + // sized so that once the first message is parked, the run + // above the parked base is too short for the second + // header. Parsing must fall back to the front of the + // buffer rather than fail with in_place_overflow. + parser::config cfg; + cfg.hdr_limits.max_fields = 1; + cfg.hdr_limits.max_size = 39; + cfg.in_buffer = 36; + + capy::test::read_stream server; + capy::any_read_stream stream(&server); + test_parser pr(cfg, &stream); + + capy::test::run_blocking()([&]() -> capy::task<> + { + server.provide( + "HTTP/1.1 200 OK\r\n" // 39 octets + "Content-Length: 20\r\n" + "\r\n" + "01234567890123456789" // 20 octets + "HTTP/1.1 200 OK\r\n" // 38 octets + "Content-Length: 3\r\n" + "\r\n" + "bye"); + + pr.start(); + { + auto [ec, body] = co_await pr.read_body(); + BOOST_TEST(!ec); + BOOST_TEST(body == "01234567890123456789"); + } + pr.consume(20); + + pr.start(); + { + auto [ec, body] = co_await pr.read_body(); + BOOST_TEST(!ec); + BOOST_TEST(body == "bye"); + } + }()); + } + void testStartSkipsUndeliveredRemainder() { @@ -2795,6 +2934,9 @@ class parser_test testMoveAssign(); testReset(); testStartPipelined(); + testStartParksCompleteMessage(); + testStartCompactsIncompleteMessage(); + testStartRetriesOverflowAtParkedBase(); testStartSkipsUndeliveredRemainder(); } }; diff --git a/test/unit/head_parser.cpp b/test/unit/head_parser.cpp index c9ca97f..e4bfc7c 100644 --- a/test/unit/head_parser.cpp +++ b/test/unit/head_parser.cpp @@ -50,23 +50,57 @@ class head_parser_test head_parser::bytes_needed( { .max_size = 1, .max_fields = 0 }); - // copy s into the parser's prepare region, - // commit it, and parse + // the parse base, which the parser exposes as + // the start of the header it is building + static + char* + base_of(head_parser const& pr) noexcept + { + return const_cast( + pr.message_head().buffer().data()); + } + + // the bytes past the header, which the caller + // locates from the header and its own total + static + std::string_view + leftovers( + head_parser const& pr, + std::size_t size) noexcept + { + auto const h = pr.message_head().buffer(); + return { h.data() + h.size(), size - h.size() }; + } + + // place s at the parse base past the `size` + // bytes already there, then parse the new total static system::error_code feed( head_parser& pr, + std::size_t& size, std::string_view s) { - auto const mb = pr.prepare(); - BOOST_ASSERT(mb.size() >= s.size()); - std::memcpy(mb.data(), s.data(), s.size()); - pr.commit(s.size()); + auto* const base = base_of(pr); + BOOST_ASSERT(base + size + s.size() <= pr.ceiling()); + std::memcpy(base + size, s.data(), s.size()); + size += s.size(); system::error_code ec; - pr.parse(ec); + pr.parse(size, ec); return ec; } + // one shot into a parser with nothing in it yet + static + system::error_code + feed( + head_parser& pr, + std::string_view s) + { + std::size_t size = 0; + return feed(pr, size, s); + } + public: void testRequest() @@ -83,16 +117,12 @@ class head_parser_test std::string const body = "BODYBYTES"; head_parser pr(true, buf_, sizeof(buf_)); - auto const mb = pr.prepare(); - BOOST_TEST(mb.data() == buf_); - std::memcpy(mb.data(), msg.data(), msg.size()); - std::memcpy( - static_cast(mb.data()) + msg.size(), - body.data(), - body.size()); - pr.commit(msg.size() + body.size()); + BOOST_TEST(base_of(pr) == buf_); + std::memcpy(buf_, msg.data(), msg.size()); + std::memcpy(buf_ + msg.size(), body.data(), body.size()); + auto const fed = msg.size() + body.size(); system::error_code ec; - pr.parse(ec); + pr.parse(fed, ec); BOOST_TEST(!ec); auto const& h = pr.request_head(); @@ -132,22 +162,17 @@ class head_parser_test // untouched and reported as leftovers BOOST_TEST_EQ( std::string_view(buf_ + msg.size(), body.size()), body); - BOOST_TEST(pr.leftovers().data() == buf_ + msg.size()); - BOOST_TEST_EQ(pr.leftovers().size(), body.size()); + BOOST_TEST_EQ(leftovers(pr, fed), body); // the table reserve is withheld from the // writable region, before and after // completion alike auto const reserve = table_space( pr.limits().max_fields); - BOOST_TEST( - pr.prepare().data() == buf_ + msg.size() + body.size()); - BOOST_TEST_EQ( - pr.prepare().size(), - sizeof(buf_) - reserve - msg.size() - body.size()); + BOOST_TEST(pr.ceiling() == buf_ + sizeof(buf_) - reserve); // parsing again is a no-op success - pr.parse(ec); + pr.parse(fed, ec); BOOST_TEST(!ec); } @@ -162,14 +187,14 @@ class head_parser_test head_parser pr(true, buf_, sizeof(buf_)); system::error_code ec; + std::size_t n = 0; for(char const c : msg) { - pr.parse(ec); + pr.parse(n, ec); BOOST_TEST(ec == http::error::need_data); - *static_cast(pr.prepare().data()) = c; - pr.commit(1); + buf_[n++] = c; } - pr.parse(ec); + pr.parse(n, ec); BOOST_TEST(!ec); auto const& h = pr.request_head(); @@ -393,18 +418,17 @@ class head_parser_test head_parser pr(true, buf_, sizeof(buf_)); system::error_code ec; - // bytes are appended through prepare - // and committed once; those already - // parsed are resolved in place and - // never rewritten + // bytes are appended one at a time; + // those already parsed are resolved in + // place and never rewritten + std::size_t n = 0; for(char const c : msg) { - pr.parse(ec); + pr.parse(n, ec); BOOST_TEST(ec == http::error::need_data); - *static_cast(pr.prepare().data()) = c; - pr.commit(1); + buf_[n++] = c; } - pr.parse(ec); + pr.parse(n, ec); BOOST_TEST(!ec); BOOST_TEST_EQ(pr.request_head().at("X"), "1 continued"); } @@ -568,10 +592,11 @@ class head_parser_test header_limits const& limits = {}) { head_parser pr(is_request, buf_, sizeof(buf_), limits); - system::error_code ec = feed(pr, msg); + std::size_t n = 0; + system::error_code ec = feed(pr, n, msg); BOOST_TEST(ec == e); // errors are sticky - pr.parse(ec); + pr.parse(n, ec); BOOST_TEST(ec == e); }; @@ -866,13 +891,11 @@ class head_parser_test system::error_code ec; for(std::size_t n = 0;; ++n) { - pr.parse(ec); + pr.parse(n, ec); if(ec != http::error::need_data || n == s.size()) break; - *static_cast( - pr.prepare().data()) = s[n]; - pr.commit(1); + buf_[n] = s[n]; } if(e == http::error::success) BOOST_TEST(! ec); @@ -958,13 +981,12 @@ class head_parser_test alignas(4) char tiny[64]; head_parser pr(true, tiny, sizeof(tiny)); - BOOST_TEST_EQ( - pr.prepare().size(), 0u); // default max_fields = 100 + BOOST_TEST(pr.ceiling() == tiny); // default max_fields = 100 system::error_code ec; - pr.parse(ec); + pr.parse(0, ec); BOOST_TEST(ec == http::error::in_place_overflow); // the error is derived again on request - pr.parse(ec); + pr.parse(0, ec); BOOST_TEST(ec == http::error::in_place_overflow); // with fitting limits the same buffer works @@ -1034,16 +1056,17 @@ class head_parser_test header_limits const lim{ .max_fields = 4 }; head_parser pr(true, buf, sizeof(buf), lim); system::error_code ec; - std::memcpy(pr.prepare().data(), msg.data(), 21); - pr.commit(21); - pr.parse(ec); + std::size_t n = 0; + std::memcpy(buf, msg.data(), 21); + n = 21; + pr.parse(n, ec); BOOST_TEST(ec == http::error::need_data); - // the new object continues from the same - // committed bytes + // the new object continues over the same + // bytes head_parser pr2(std::move(pr)); - BOOST_TEST(pr2.prepare().data() == buf + 21); - BOOST_TEST(! feed(pr2, msg.substr(21))); + BOOST_TEST(base_of(pr2) == buf); + BOOST_TEST(! feed(pr2, n, msg.substr(21))); auto const& h = pr2.request_head(); BOOST_TEST_EQ(h.target(), "/index"); BOOST_TEST_EQ(h.at(http::field::host), "example.com"); @@ -1066,8 +1089,8 @@ class head_parser_test // default construction: zero-size buffer, // no room to receive anything head_parser pr3; - BOOST_TEST_EQ(pr3.prepare().size(), 0u); - pr3.parse(ec); + BOOST_TEST(pr3.ceiling() == base_of(pr3)); + pr3.parse(0, ec); BOOST_TEST(ec == http::error::in_place_overflow); } @@ -1085,9 +1108,9 @@ class head_parser_test BOOST_TEST(! feed(pr, msg1)); BOOST_TEST_EQ(pr.request_head().target(), "/a"); - pr.reset(); - BOOST_TEST(! pr.got_some()); - BOOST_TEST(pr.prepare().data() == buf); + pr.reset(buf); + BOOST_TEST(base_of(pr) == buf); + BOOST_TEST_EQ(pr.message_head().buffer().size(), 0u); std::string_view const msg2 = "POST /bb HTTP/1.1\r\nHost: b\r\nX: y\r\n\r\n"; BOOST_TEST(! feed(pr, msg2)); @@ -1115,26 +1138,22 @@ class head_parser_test "POST /b HTTP/1.1\r\n\r\n"; head_parser pr(true, buf, sizeof(buf), { .max_fields = 4 }); - BOOST_TEST(! pr.got_some()); auto const both = std::string(msg1) + std::string(msg2); - BOOST_TEST(! feed(pr, both)); - BOOST_TEST(pr.got_some()); - auto const lo = pr.leftovers(); + std::size_t n = 0; + BOOST_TEST(! feed(pr, n, both)); + auto const lo = leftovers(pr, n); BOOST_TEST(lo.data() == buf + msg1.size()); - BOOST_TEST_EQ( - std::string_view( - static_cast(lo.data()), lo.size()), - msg2); + BOOST_TEST_EQ(lo, msg2); std::memmove(buf, lo.data(), lo.size()); - pr.reset(lo.size()); - BOOST_TEST(pr.got_some()); + pr.reset(buf); + n = lo.size(); system::error_code ec; - pr.parse(ec); + pr.parse(n, ec); BOOST_TEST(! ec); BOOST_TEST_EQ(pr.request_head().target(), "/b"); - BOOST_TEST_EQ(pr.leftovers().size(), 0u); + BOOST_TEST_EQ(leftovers(pr, n).size(), 0u); } void @@ -1151,8 +1170,8 @@ class head_parser_test head_parser pr(true, raw + 1, 41, lim); // raw + 42 aligns down to raw + 40: // 39 usable bytes, 12 reserved - BOOST_TEST_EQ(pr.prepare().size(), 27u); - BOOST_TEST(pr.prepare().data() == raw + 1); + BOOST_TEST(base_of(pr) == raw + 1); + BOOST_TEST_EQ(pr.ceiling() - (raw + 1), 27); BOOST_TEST(! feed(pr, msg)); BOOST_TEST_EQ( pr.request_head().at(http::field::host), "x"); @@ -1165,9 +1184,9 @@ class head_parser_test for(std::size_t n = 0; n <= 12; ++n) { head_parser pr(true, raw, n, lim); - BOOST_TEST_EQ(pr.prepare().size(), 0u); + BOOST_TEST(pr.ceiling() == raw); system::error_code ec; - pr.parse(ec); + pr.parse(0, ec); BOOST_TEST(ec == http::error::in_place_overflow); } } @@ -1177,20 +1196,21 @@ class head_parser_test { alignas(4) char raw[36]; head_parser pr(true, raw, sizeof(raw), lim); - BOOST_TEST_EQ(pr.prepare().size(), 24u); - auto ec = feed(pr, msg.substr(0, 24)); + BOOST_TEST_EQ(pr.ceiling() - raw, 24); + std::size_t n = 0; + auto ec = feed(pr, n, msg.substr(0, 24)); BOOST_TEST(ec == http::error::in_place_overflow); // and stays so - pr.parse(ec); + pr.parse(n, ec); BOOST_TEST(ec == http::error::in_place_overflow); } } - // Feed `s` through prepare()/commit()/parse() in - // chunks of at most `chunk` bytes (0 for as much as - // prepare() offers), stopping once the parser stops - // asking for bytes or the caller has none left to - // give. Reports how much was committed. + // Feed `s` to the parse base in chunks of at most + // `chunk` bytes (0 for as much as fits below the + // table), stopping once the parser stops asking for + // bytes or the caller has none left to give. Reports + // how much was placed. static system::error_code drive( @@ -1199,23 +1219,24 @@ class head_parser_test std::size_t chunk, std::size_t& fed) { + auto* const base = base_of(pr); system::error_code ec; fed = 0; for(;;) { - pr.parse(ec); + pr.parse(fed, ec); if(ec != http::error::need_data) break; - auto const mb = pr.prepare(); - if(mb.size() == 0 || fed == s.size()) + auto const room = static_cast( + pr.ceiling() - (base + fed)); + if(room == 0 || fed == s.size()) break; auto n = s.size() - fed; if(chunk != 0 && n > chunk) n = chunk; - if(n > mb.size()) - n = mb.size(); - std::memcpy(mb.data(), s.data() + fed, n); - pr.commit(n); + if(n > room) + n = room; + std::memcpy(base + fed, s.data() + fed, n); fed += n; } return ec; @@ -1283,10 +1304,9 @@ class head_parser_test // the region offered is always inside // the buffer - auto const mb0 = pr.prepare(); - BOOST_TEST(mb0.size() <= n); - BOOST_TEST( - static_cast(mb0.data()) >= raw + off); + BOOST_TEST(base_of(pr) == raw + off); + BOOST_TEST(pr.ceiling() >= raw + off); + BOOST_TEST(pr.ceiling() <= raw + off + n); std::size_t fed = 0; auto const ec = drive(pr, wire, chunk, fed); @@ -1323,21 +1343,16 @@ class head_parser_test // and the payload which followed it in // is byte for byte where it landed auto const got = fed - h.size(); - BOOST_TEST_EQ(pr.leftovers().size(), got); - BOOST_TEST( - pr.leftovers().data() == - raw + off + h.size()); + auto const lo = leftovers(pr, fed); + BOOST_TEST_EQ(lo.size(), got); + BOOST_TEST(lo.data() == raw + off + h.size()); BOOST_TEST_EQ( - std::string_view( - raw + off + h.size(), got), - std::string_view(pay).substr(0, got)); + lo, std::string_view(pay).substr(0, got)); // leftovers and the still-writable // region together stay in the buffer BOOST_TEST( - static_cast(pr.leftovers().data()) + - pr.leftovers().size() + - pr.prepare().size() <= raw + off + n); + lo.data() + lo.size() <= pr.ceiling()); } } } @@ -1583,13 +1598,57 @@ class head_parser_test head_parser pr(true, owned.get(), cap, lim); std::size_t fed = 0; BOOST_TEST(! drive(pr, wire, 0, fed)); - BOOST_TEST_EQ(pr.prepare().size(), 0u); + BOOST_TEST(base_of(pr) + fed == pr.ceiling()); - auto const lo = pr.leftovers(); + auto const lo = leftovers(pr, fed); BOOST_TEST_EQ(lo.size(), carry); std::memmove(owned.get(), lo.data(), lo.size()); - pr.reset(lo.size()); - BOOST_TEST_EQ(pr.prepare().size(), one.size()); + pr.reset(owned.get()); + BOOST_TEST_EQ( + pr.ceiling() - (owned.get() + lo.size()), one.size()); + } + + void + testRebase() + { + std::string_view const msg = + "GET /a HTTP/1.1\r\nHost: x\r\n\r\n"; + std::string_view const carry = "GET /b"; + + // a header built away from the front, moved down + // once it is complete + head_parser pr(true, buf_, sizeof(buf_)); + pr.reset(buf_ + 64); + std::size_t n = 0; + BOOST_TEST( + ! feed(pr, n, std::string(msg) + std::string(carry))); + BOOST_TEST(pr.message_head().buffer().data() == buf_ + 64); + + std::memmove(buf_, buf_ + 64, n); + pr.rebase(buf_); + BOOST_TEST(pr.message_head().buffer().data() == buf_); + BOOST_TEST_EQ(pr.message_head().buffer(), msg); + BOOST_TEST_EQ(pr.request_head().target(), "/a"); + BOOST_TEST_EQ(pr.request_head().at(http::field::host), "x"); + BOOST_TEST_EQ(leftovers(pr, n), carry); + + // rebasing where it already is is harmless + pr.rebase(buf_); + BOOST_TEST_EQ(pr.request_head().target(), "/a"); + + // a rebase in the middle of a parse keeps the + // progress made so far + head_parser pr2(true, buf_, sizeof(buf_)); + pr2.reset(buf_ + 64); + std::size_t n2 = 0; + BOOST_TEST( + feed(pr2, n2, "GET /c HTTP/1.1\r\nHo") == + http::error::need_data); + std::memmove(buf_, buf_ + 64, n2); + pr2.rebase(buf_); + BOOST_TEST(! feed(pr2, n2, "st: y\r\n\r\n")); + BOOST_TEST_EQ(pr2.request_head().target(), "/c"); + BOOST_TEST_EQ(pr2.request_head().at(http::field::host), "y"); } void @@ -1610,25 +1669,30 @@ class head_parser_test auto owned = std::unique_ptr(new char[cap]); head_parser pr(true, owned.get(), cap, lim); - std::size_t fed = 0; + // bytes taken off the wire, and how many of + // them are sitting at the parse base + std::size_t sent = 0; + std::size_t held = 0; int done = 0; for(int round = 0; round < 4; ++round) { system::error_code ec; for(;;) { - pr.parse(ec); + pr.parse(held, ec); if(ec != http::error::need_data) break; - auto const mb = pr.prepare(); - if(mb.size() == 0 || fed == wire.size()) + auto* const at = base_of(pr) + held; + auto const room = static_cast( + pr.ceiling() - at); + if(room == 0 || sent == wire.size()) break; - auto n = wire.size() - fed; - if(n > mb.size()) - n = mb.size(); - std::memcpy(mb.data(), wire.data() + fed, n); - pr.commit(n); - fed += n; + auto n = wire.size() - sent; + if(n > room) + n = room; + std::memcpy(at, wire.data() + sent, n); + sent += n; + held += n; } if(! BOOST_TEST(! ec)) break; @@ -1636,9 +1700,10 @@ class head_parser_test BOOST_TEST_EQ(pr.request_head().at(http::field::host), "x"); ++done; - auto const lo = pr.leftovers(); + auto const lo = leftovers(pr, held); std::memmove(owned.get(), lo.data(), lo.size()); - pr.reset(lo.size()); + pr.reset(owned.get()); + held = lo.size(); } BOOST_TEST_EQ(done, 4); } @@ -1668,6 +1733,7 @@ class head_parser_test testLimitSweep(); testFieldLineAtUint16Max(); testResetFullBuffer(); + testRebase(); testPipelineRounds(); } }; diff --git a/test/unit/request_head.cpp b/test/unit/request_head.cpp index 03cb012..2ee2a74 100644 --- a/test/unit/request_head.cpp +++ b/test/unit/request_head.cpp @@ -1059,11 +1059,9 @@ class request_head_test auto const parse = [&](head_parser& pr) { - std::memcpy( - pr.prepare().data(), msg.data(), msg.size()); - pr.commit(msg.size()); + std::memcpy(buf, msg.data(), msg.size()); system::error_code ec; - pr.parse(ec); + pr.parse(msg.size(), ec); BOOST_TEST(!ec); }; diff --git a/test/unit/response_head.cpp b/test/unit/response_head.cpp index 3f34959..87da62e 100644 --- a/test/unit/response_head.cpp +++ b/test/unit/response_head.cpp @@ -473,11 +473,9 @@ class response_head_test auto const parse = [&](head_parser& pr) { - std::memcpy( - pr.prepare().data(), msg.data(), msg.size()); - pr.commit(msg.size()); + std::memcpy(buf, msg.data(), msg.size()); system::error_code ec; - pr.parse(ec); + pr.parse(msg.size(), ec); BOOST_TEST(!ec); }; @@ -524,10 +522,9 @@ class response_head_test "Transfer-Encoding: chunked\r\n" "\r\n"; head_parser pr(false, buf, sizeof(buf)); - std::memcpy(pr.prepare().data(), msg.data(), msg.size()); - pr.commit(msg.size()); + std::memcpy(buf, msg.data(), msg.size()); system::error_code ec; - pr.parse(ec); + pr.parse(msg.size(), ec); BOOST_TEST(!ec); response_head_base const& base = pr.response_head(); From fbf378086107661261537b9ebb24fe4e7e52c30f Mon Sep 17 00:00:00 2001 From: Mohammad Nejati Date: Fri, 7 Aug 2026 18:04:28 +0330 Subject: [PATCH 2/3] update compression docs for built-in decoders --- .../ROOT/pages/2.guide/2m.compression.adoc | 54 +++++-------------- 1 file changed, 14 insertions(+), 40 deletions(-) diff --git a/doc/modules/ROOT/pages/2.guide/2m.compression.adoc b/doc/modules/ROOT/pages/2.guide/2m.compression.adoc index ded0bff..bdbc467 100644 --- a/doc/modules/ROOT/pages/2.guide/2m.compression.adoc +++ b/doc/modules/ROOT/pages/2.guide/2m.compression.adoc @@ -10,49 +10,22 @@ = Compression Burl can advertise the content codings it accepts and transparently decode a -compressed response body, so `gzip`, `deflate`, and `br` responses arrive -already decompressed. Decoding depends on the relevant service being installed -in the system context. +compressed response body, so `gzip`, `deflate`, `br`, and `zstd` responses +arrive already decompressed. Decoding for each coding is compiled into the +library when it is built with the corresponding compression library. -== Installing the Decode Services +== Build Support -Decoding is provided by Boost.Http and must be registered in the system context -once, at startup, before any request. The zlib service handles `gzip` and -`deflate`; the brotli service handles `br`: - -[source,cpp] ----- -#include -#include - -int main() -{ - http::zlib::install_inflate_service(capy::get_system_context()); - http::brotli::install_decode_service(capy::get_system_context()); - - // ... run the client ... -} ----- - -These are available only when Boost.Http is built with zlib and brotli support. -Guarding the calls with the feature macros keeps the program portable across -builds that lack them: - -[source,cpp] ----- -#ifdef BOOST_HTTP_HAS_ZLIB - http::zlib::install_inflate_service(capy::get_system_context()); -#endif -#ifdef BOOST_HTTP_HAS_BROTLI - http::brotli::install_decode_service(capy::get_system_context()); -#endif ----- +The build detects zlib, Brotli, and zstd and enables the codings they provide: +zlib handles `gzip` and `deflate`, Brotli handles `br`, and zstd handles +`zstd`. The macros `BOOST_BURL_HAS_ZLIB`, `BOOST_BURL_HAS_BROTLI`, and +`BOOST_BURL_HAS_ZSTD` indicate which codings the library was built with. == How It Works -With a service installed and the corresponding setting enabled, the client adds -that coding to the `Accept-Encoding` request header and decodes a response -encoded with it. +Every supported coding is enabled by default: the client adds it to the +`Accept-Encoding` request header and decodes a response encoded with it. The +configuration controls each coding individually. [source,cpp] ---- @@ -60,14 +33,15 @@ burl::client::config cfg; cfg.gzip = true; cfg.deflate = true; cfg.brotli = false; // do not advertise or decode br +cfg.zstd = false; // do not advertise or decode zstd burl::client client(co_await capy::this_coro::executor, tls_ctx, cfg); ---- [NOTE] ==== -A coding whose decode service is not installed is disabled regardless of the -setting, since the client cannot honor what it advertises. +A coding whose decoder was not compiled into the library is disabled regardless +of the setting, since the client cannot honor what it advertises. ==== The decoding is transparent: the body you read through From fabae2c835476318dddf7c99c2fa37a6664d7a7a Mon Sep 17 00:00:00 2001 From: Mohammad Nejati Date: Fri, 7 Aug 2026 17:35:19 +0330 Subject: [PATCH 3/3] parser is sans-io --- include/boost/burl/client.hpp | 6 +- include/boost/burl/detail/connection_pool.hpp | 15 +- include/boost/burl/detail/parser.hpp | 207 ---- include/boost/burl/detail/request_parser.hpp | 60 -- include/boost/burl/detail/response_parser.hpp | 60 -- include/boost/burl/message_reader.hpp | 401 +++++++ include/boost/burl/parser.hpp | 547 ++++++++++ include/boost/burl/request_parser.hpp | 91 ++ include/boost/burl/response.hpp | 11 +- include/boost/burl/response_parser.hpp | 95 ++ include/boost/burl/test/response_factory.hpp | 8 +- src/client.cpp | 20 +- src/detail/can_reuse_conn.hpp | 2 +- src/detail/decoders.hpp | 2 +- src/detail/drain_body.cpp | 46 - src/detail/drain_body.hpp | 33 +- src/detail/http_tunnel.cpp | 8 +- src/{detail => }/parser.cpp | 622 ++++++----- src/response.cpp | 15 +- test/unit/detail/can_reuse_conn.cpp | 9 +- test/unit/detail/circular_buffer.cpp | 304 ++++++ test/unit/detail/connection_pool.cpp | 18 +- test/unit/detail/decoders.cpp | 1 + test/unit/detail/drain_body.cpp | 11 +- test/unit/{detail => }/parser.cpp | 994 +++++++++++++++++- test/unit/{detail => }/request_parser.cpp | 18 +- test/unit/{detail => }/response_parser.cpp | 22 +- 27 files changed, 2877 insertions(+), 749 deletions(-) delete mode 100644 include/boost/burl/detail/parser.hpp delete mode 100644 include/boost/burl/detail/request_parser.hpp delete mode 100644 include/boost/burl/detail/response_parser.hpp create mode 100644 include/boost/burl/message_reader.hpp create mode 100644 include/boost/burl/parser.hpp create mode 100644 include/boost/burl/request_parser.hpp create mode 100644 include/boost/burl/response_parser.hpp delete mode 100644 src/detail/drain_body.cpp rename src/{detail => }/parser.cpp (64%) create mode 100644 test/unit/detail/circular_buffer.cpp rename test/unit/{detail => }/parser.cpp (75%) rename test/unit/{detail => }/request_parser.cpp (76%) rename test/unit/{detail => }/response_parser.cpp (79%) diff --git a/include/boost/burl/client.hpp b/include/boost/burl/client.hpp index 010c338..52d4202 100644 --- a/include/boost/burl/client.hpp +++ b/include/boost/burl/client.hpp @@ -335,9 +335,9 @@ class client /** Constructor. Constructs a client with the provided - configuration. Content codings whose decode - service is not installed in the system - context are disabled, regardless of the + configuration. Content codings whose + decoder was not compiled into the library + are disabled, regardless of the configuration. @param exec The executor used to perform diff --git a/include/boost/burl/detail/connection_pool.hpp b/include/boost/burl/detail/connection_pool.hpp index 63228c6..c2a1f2d 100644 --- a/include/boost/burl/detail/connection_pool.hpp +++ b/include/boost/burl/detail/connection_pool.hpp @@ -14,7 +14,6 @@ #include #include #include -#include #include #include @@ -94,10 +93,18 @@ class pooled_connection public: pooled_connection() = default; - capy::any_stream - stream() noexcept + template + capy::io_task + read_some(MB buffers) + { + return conn_->read_some(std::move(buffers)); + } + + template + capy::io_task + write_some(CB buffers) { - return conn_.get(); + return conn_->write_some(std::move(buffers)); } explicit diff --git a/include/boost/burl/detail/parser.hpp b/include/boost/burl/detail/parser.hpp deleted file mode 100644 index 32ee112..0000000 --- a/include/boost/burl/detail/parser.hpp +++ /dev/null @@ -1,207 +0,0 @@ -// -// Copyright (c) 2026 Mohammad Nejati -// -// Distributed under the Boost Software License, Version 1.0. (See accompanying -// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -// -// Official repository: https://github.com/cppalliance/burl -// - -#ifndef BOOST_BURL_DETAIL_PARSER_HPP -#define BOOST_BURL_DETAIL_PARSER_HPP - -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include - -namespace boost -{ -namespace burl -{ -namespace detail -{ - -class parser -{ -public: - struct decoder - { - struct result - { - std::size_t consumed; - std::size_t produced; - std::error_code ec; - }; - - virtual ~decoder() = default; - - virtual result - process( - capy::mutable_buffer out, - capy::const_buffer in, - bool eof) = 0; - }; - - struct config - { - header_limits hdr_limits; - - std::size_t in_buffer = 64 * 1024; - std::size_t dec_buffer = 8 * 1024; - std::uint64_t body_limit = std::uint64_t(-1); - }; - - bool - got_header() const noexcept; - - bool - got_body() const noexcept; - - bool - has_buffered_data() const noexcept; - - capy::io_task<> - read_header(); - - void - reset(capy::any_read_stream stream) noexcept; - - void - set_decoder(decoder* dec) noexcept; - - void - set_body_limit(std::uint64_t n) noexcept; - - capy::io_task - read_body(); - - template - capy::io_task - read_some(Buffers buffers); - - template - capy::io_task - read(Buffers buffers); - - capy::io_task> - pull(std::span dest); - - void - consume(std::size_t n) noexcept; - -protected: - parser() = default; - - parser( - config const& cfg, - bool is_request, - capy::any_read_stream stream = {}); - - parser(parser&& other) noexcept = default; - - parser& - operator=(parser&& other) noexcept = default; - - parser(const parser&) = delete; - - parser& - operator=(const parser&) = delete; - - ~parser() = default; - - void - start(bool head); - - burl::response_head_base const& - get_response() const; - - burl::request_head_base const& - get_request() const; - -private: - struct chunk_fn; - - std::size_t - raw_limit_rem() const noexcept; - - std::size_t - dec_limit_rem() const noexcept; - - bool - payload_sized() const noexcept; - - std::size_t - payload_rem() const noexcept; - - capy::io_task<> - refill(); - - std::error_code - walk_chunks(chunk_fn f, bool dry = false); - - std::error_code - flatten_chunks(); - - capy::io_task - decode_some( - std::span buffers); - - capy::io_task - do_read_some( - std::span buffers); - - capy::any_read_stream stream_; - std::unique_ptr buf_; - head_parser hp_; - decoder * dec_ = nullptr; - circular_buffer in_; - circular_buffer out_; - std::uint64_t chunk_rem_ = 0; - std::uint64_t transferred_ = 0; - std::uint64_t decoded_ = 0; - std::uint64_t body_limit_ = 0; - std::uint64_t payload_size_ = 0; - std::error_code dec_err_; - http::payload payload_ = http::payload::none; - bool is_req_ = true; - bool head_ = false; - bool started_ = false; - bool got_header_ = false; - bool got_body_ = false; - bool mid_chunk_ = false; - bool fin_chunk_ = false; - bool eof_ = false; -}; - -template -capy::io_task -parser:: -read_some(Buffers buffers) -{ - capy::buffer_param bp(buffers); - co_return co_await do_read_some(bp.data()); -} - -template -capy::io_task -parser:: -read(Buffers buffers) -{ - return capy::read(*this, std::move(buffers)); -} - -} // namespace detail -} // namespace burl -} // namespace boost - -#endif diff --git a/include/boost/burl/detail/request_parser.hpp b/include/boost/burl/detail/request_parser.hpp deleted file mode 100644 index 7f2b94c..0000000 --- a/include/boost/burl/detail/request_parser.hpp +++ /dev/null @@ -1,60 +0,0 @@ -// -// Copyright (c) 2026 Mohammad Nejati -// -// Distributed under the Boost Software License, Version 1.0. (See accompanying -// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -// -// Official repository: https://github.com/cppalliance/burl -// - -#ifndef BOOST_BURL_DETAIL_REQUEST_PARSER_HPP -#define BOOST_BURL_DETAIL_REQUEST_PARSER_HPP - -#include - -#include - -namespace boost -{ -namespace burl -{ -namespace detail -{ - -class request_parser - : public parser -{ -public: - request_parser() = default; - - explicit - request_parser( - config const& cfg, - capy::any_read_stream stream = {}) - : parser(cfg, true, std::move(stream)) - { - } - - request_parser(request_parser&&) noexcept = default; - - request_parser& - operator=(request_parser&&) noexcept = default; - - void - start() - { - parser::start(false); - } - - burl::request_head_base const& - get() const - { - return get_request(); - } -}; - -} // namespace detail -} // namespace burl -} // namespace boost - -#endif diff --git a/include/boost/burl/detail/response_parser.hpp b/include/boost/burl/detail/response_parser.hpp deleted file mode 100644 index 5177a9f..0000000 --- a/include/boost/burl/detail/response_parser.hpp +++ /dev/null @@ -1,60 +0,0 @@ -// -// Copyright (c) 2026 Mohammad Nejati -// -// Distributed under the Boost Software License, Version 1.0. (See accompanying -// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -// -// Official repository: https://github.com/cppalliance/burl -// - -#ifndef BOOST_BURL_DETAIL_RESPONSE_PARSER_HPP -#define BOOST_BURL_DETAIL_RESPONSE_PARSER_HPP - -#include - -#include - -namespace boost -{ -namespace burl -{ -namespace detail -{ - -class response_parser - : public parser -{ -public: - response_parser() = default; - - explicit - response_parser( - config const& cfg, - capy::any_read_stream stream = {}) - : parser(cfg, false, std::move(stream)) - { - } - - response_parser(response_parser&&) noexcept = default; - - response_parser& - operator=(response_parser&&) noexcept = default; - - void - start(bool head = false) - { - parser::start(head); - } - - burl::response_head_base const& - get() const - { - return get_response(); - } -}; - -} // namespace detail -} // namespace burl -} // namespace boost - -#endif diff --git a/include/boost/burl/message_reader.hpp b/include/boost/burl/message_reader.hpp new file mode 100644 index 0000000..94ffbfd --- /dev/null +++ b/include/boost/burl/message_reader.hpp @@ -0,0 +1,401 @@ +// +// Copyright (c) 2026 Mohammad Nejati +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// Official repository: https://github.com/cppalliance/burl +// + +#ifndef BOOST_BURL_MESSAGE_READER_HPP +#define BOOST_BURL_MESSAGE_READER_HPP + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace boost +{ +namespace burl +{ + +/** Drives a @ref parser over a stream. + + A reader binds a stream to a parser. It holds + only pointers to both, which must outlive it. + + @par Example + @code + response_parser pr( cfg ); + message_reader reader( &sock, &pr ); + + pr.start(); + + if(auto [ec] = co_await reader.read_header(); ec) + co_return { ec }; + + auto const& head = pr.get(); + std::cout + << head.status_int() << " " << head.reason() << "\n" + << head.value_or( http::field::content_type, "" ) << "\n"; + + capy::const_buffer bufs[ 8 ]; + for(;;) + { + auto [ec, data] = co_await reader.pull( bufs ); + if(ec == capy::cond::eof) + break; + if(ec) + co_return { ec }; + write_to_file( data ); + reader.consume( capy::buffer_size( data )); + } + @endcode + + The loop above never copies the body: @ref pull + hands out descriptors into the parser's own + buffer. Use @ref read_some instead when the + octets have to land in memory of your choosing. + + Every operation drives @ref parser::parse_header + first, so reading a body without having read the + header explicitly works as expected. This serves + the caller who has no interest in the header: + anything decided from it, installing a decoder + above all, needs @ref read_header called + explicitly, because @ref parser::set_decoder + requires a parsed header and an untouched body. + + This type satisfies @ref capy::ReadStream, @ref + http::ReadSource, and @ref http::BufferSource, + all over the octets of the message body. + + @tparam S A type satisfying @ref capy::ReadStream. + + @see @ref parser. +*/ +template +class message_reader +{ + S* s_; + parser* p_; + +public: + /** Constructor. + + @par Preconditions + Neither pointer is null, and both objects + outlive the reader. + + @param stream The stream to read from. + + @param pr The parser to drive. + */ + message_reader(S* stream, parser* pr) noexcept + : s_(stream) + , p_(pr) + { + BOOST_ASSERT(s_ != nullptr); + BOOST_ASSERT(p_ != nullptr); + } + + /** Asynchronously parse the message header. + + Reads from the stream until the header is + complete or an error occurs. Has no effect + once @ref parser::got_header returns true. + + @return An awaitable yielding `(error_code)`. + */ + capy::io_task<> + read_header() + { + return read_header_(*s_, *p_); + } + + /** Asynchronously read the complete body in place. + + Reads the remainder of the body into the + parser's buffer and returns a view of the + whole body, without copying. Fails with + @ref http::error::in_place_overflow if the + body does not fit. + + @return An awaitable yielding + `(error_code,std::string_view)`. + + @see @ref parser::body. + */ + capy::io_task + read_body() + { + return read_body_(*s_, *p_); + } + + /** Asynchronously read body octets. + + Copies into `buffers`, or lets an installed + decoder write into them directly. Yields + `capy::error::eof` once the body is + complete. + + @param buffers The destination. + + @return An awaitable yielding + `(error_code,std::size_t)`. + + @see @ref parser::read_some. + */ + template + capy::io_task + read_some(MB buffers) + { + return read_some_(*s_, *p_, std::move(buffers)); + } + + /** Asynchronously fill a buffer sequence with body octets. + + Reads until `buffers` is full, the body is + complete, or an error occurs. A body shorter + than `buffers` yields `capy::error::eof` + alongside the octets transferred. + + @param buffers The destination. + + @return An awaitable yielding + `(error_code,std::size_t)`. + */ + template + capy::io_task + read(MB buffers) + { + return read_(*s_, *p_, std::move(buffers)); + } + + /** Asynchronously borrow body octets. + + Fills `dest` with descriptors referring to + the parser's own buffers, which @ref consume + then releases. Yields `capy::error::eof` + once the body is complete. + + @param dest The descriptors to fill. + + @return An awaitable yielding + `(error_code,std::span)`. + + @see @ref consume, @ref parser::pull. + */ + capy::io_task> + pull(std::span dest) + { + return pull_(*s_, *p_, dest); + } + + /** Release body octets returned by @ref pull. + + @param n The number of octets to release. + + @see @ref pull. + */ + void + consume(std::size_t n) noexcept + { + p_->consume(n); + } + +private: + static capy::io_task<> + refill_(S& stream, parser& pr); + + static capy::io_task<> + read_header_(S& stream, parser& pr); + + static capy::io_task + read_body_(S& stream, parser& pr); + + static capy::io_task> + pull_( + S& stream, + parser& pr, + std::span dest); + + template + static capy::io_task + read_some_( + S& stream, + parser& pr, + MB buffers); + + template + static capy::io_task + read_( + S& stream, + parser& pr, + MB buffers); +}; + +//------------------------------------------------ + +template +capy::io_task<> +message_reader:: +refill_(S& stream, parser& pr) +{ + auto [ec, n] = co_await stream.read_some(pr.prepare()); + pr.commit(n); + if(ec == capy::cond::eof) + { + pr.commit_eof(); + co_return {}; + } + if(ec) + co_return { ec }; + co_return {}; +} + +template +capy::io_task<> +message_reader:: +read_header_(S& stream, parser& pr) +{ + for(;;) + { + system::error_code ec; + pr.parse_header(ec); + if(!ec) + co_return {}; + if(ec != http::error::need_data) + co_return { std::error_code(ec) }; + if(auto [rec] = co_await refill_(stream, pr); rec) + co_return { rec }; + } +} + +template +capy::io_task +message_reader:: +read_body_(S& stream, parser& pr) +{ + if(!pr.got_header()) + if(auto [ec] = co_await read_header_(stream, pr); ec) + co_return { ec, {} }; + + for(;;) + { + system::error_code ec; + auto const sv = pr.body(ec); + if(ec != http::error::need_data) + co_return { std::error_code(ec), sv }; + if(auto [rec] = co_await refill_(stream, pr); rec) + co_return { rec, {} }; + } +} + +template +template +capy::io_task +message_reader:: +read_some_( + S& stream, + parser& pr, + MB buffers) +{ + if(!pr.got_header()) + if(auto [ec] = co_await read_header_(stream, pr); ec) + co_return { ec, 0 }; + + capy::buffer_param bp(buffers); + + for(;;) + { + system::error_code ec; + auto const n = pr.read_some(bp.data(), ec); + if(ec != http::error::need_data) + co_return { std::error_code(ec), n }; + + if(auto const lim = pr.direct_capacity(); lim != 0) + { + auto const mbs = bp.data(); + auto [rec, rn] = co_await stream.read_some( + capy::buffer_slice(mbs, 0, lim)); + pr.commit_direct(rn); + if(rec == capy::cond::eof) + pr.commit_eof(); + else if(rec) + co_return { rec, rn }; + if(rn != 0) + co_return { {}, rn }; + continue; + } + + if(auto [rec] = co_await refill_(stream, pr); rec) + co_return { rec, 0 }; + } +} + +template +template +capy::io_task +message_reader:: +read_( + S& stream, + parser& pr, + MB buffers) +{ + auto const total_size = capy::buffer_size(buffers); + capy::consuming_buffers dest(buffers); + std::size_t total = 0; + + while(total < total_size) + { + auto [ec, n] = co_await read_some_(stream, pr, dest.data()); + dest.consume(n); + total += n; + if(ec && total < total_size) + co_return { ec, total }; + } + + co_return { {}, total }; +} + +template +capy::io_task> +message_reader:: +pull_( + S& stream, + parser& pr, + std::span dest) +{ + if(!pr.got_header()) + if(auto [ec] = co_await read_header_(stream, pr); ec) + co_return { ec, {} }; + + for(;;) + { + system::error_code ec; + auto const bufs = pr.pull(dest, ec); + if(ec != http::error::need_data) + co_return { std::error_code(ec), bufs }; + if(auto [rec] = co_await refill_(stream, pr); rec) + co_return { rec, {} }; + } +} + +} // namespace burl +} // namespace boost + +#endif diff --git a/include/boost/burl/parser.hpp b/include/boost/burl/parser.hpp new file mode 100644 index 0000000..7e3bc56 --- /dev/null +++ b/include/boost/burl/parser.hpp @@ -0,0 +1,547 @@ +// +// Copyright (c) 2026 Mohammad Nejati +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// Official repository: https://github.com/cppalliance/burl +// + +#ifndef BOOST_BURL_PARSER_HPP +#define BOOST_BURL_PARSER_HPP + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace boost +{ +namespace burl +{ + +/** A parser for HTTP/1 messages. + + The parser performs no I/O. Received bytes are + handed to it through @ref prepare and @ref + commit, and each parsing operation reports @ref + http::error::need_data when it requires more. + Driving the parser over a stream is the job of + @ref message_reader. + + The parser uses a single block of memory + allocated during construction and never exceeds + it. The space is reused across messages, one at + a time, and holds: + + @li raw octets received from the stream, + @li the message header, with O(1) access to the + start line, + @li all or part of the message body, and + @li decoded output when a @ref decoder is + installed. + + @par Operations + + The body can be retrieved three ways, which + differ in where the octets end up: + + @li @ref body returns the whole body in place, + without copying, + @li @ref read_some copies into caller-supplied + memory, or lets an installed decoder write + into it directly, and + @li @ref pull borrows the parser's own buffers, + which @ref consume then releases. + + @par Errors + + Every parsing operation reports through an + `error_code` out parameter: + + @li @ref http::error::need_data — fill @ref + prepare, call @ref commit, and try again. + Reported only while @ref prepare has room. + @li @ref http::error::in_place_overflow — more + input is required but no writable space + remains. + @li @ref http::error::incomplete — more input is + required but @ref commit_eof was called. + @li @ref http::error::end_of_stream — the stream + closed cleanly before the message began. + + An operation reports either transferred octets + or an error, never both. + + @see + @ref message_reader, + @ref request_parser, + @ref response_parser. +*/ +class parser +{ +public: + /** A content decoder. + + Installed with @ref set_decoder before body + parsing begins, a decoder transforms the + payload octets as they arrive. + */ + struct decoder + { + /// The outcome of a call to @ref process. + struct result + { + /// The number of input octets consumed. + std::size_t consumed; + + /// The number of output octets produced. + std::size_t produced; + + /** The error, if any. + + Set to `capy::error::eof` once the + decoder has produced the complete + output. + */ + std::error_code ec; + }; + + /// Destructor. + virtual ~decoder() = default; + + /** Transform payload octets. + + @param out The destination for decoded + output. + + @param in The octets to decode. + + @param eof True when `in` ends the + payload. + + @return The octets consumed and + produced, and the error if any. + */ + virtual result + process( + capy::mutable_buffer out, + capy::const_buffer in, + bool eof) = 0; + }; + + /// Settings which apply for the life of the parser. + struct config + { + /// The limits enforced while parsing a header. + header_limits hdr_limits; + + /// The space reserved for buffering received octets. + std::size_t in_buffer = 64 * 1024; + + /// The space reserved for decoded output. + std::size_t dec_buffer = 8 * 1024; + + /// The default maximum body size. + std::uint64_t body_limit = std::uint64_t(-1); + }; + + //-------------------------------------------- + // + // Observers + // + //-------------------------------------------- + + /** Return true if the header has been parsed. + */ + BOOST_BURL_DECL + bool + got_header() const noexcept; + + /** Return true if the entire message has arrived. + */ + BOOST_BURL_DECL + bool + got_body() const noexcept; + + /** Return true if octets are buffered past the message. + + Returns true when the buffer holds octets + which lie beyond the current message, such + as the start of a pipelined message. Returns + false before the message is complete, and + false for a payload which is delimited by + the end of the stream. + + @see @ref buffered_data. + */ + BOOST_BURL_DECL + bool + has_buffered_data() const noexcept; + + /** Return the unconsumed octets in the buffer. + + The returned octets are raw: no message + framing is applied. After a message whose + body has been read to completion they are + the octets which follow it, which is how the + remainder of a tunnel is recovered following + a CONNECT request. + + Note that the framing of a response to + CONNECT cannot be determined from the + response alone, so the parser reports a + payload which continues to the end of the + stream and @ref has_buffered_data returns + false. A caller which knows it issued a + CONNECT should use this function directly. + + If the body of a sized payload has only + partially been read, the unread remainder is + included. + + @par Preconditions + `this->got_header() == true` + + @see @ref has_buffered_data. + */ + BOOST_BURL_DECL + std::array + buffered_data() const noexcept; + + //-------------------------------------------- + // + // Modifiers + // + //-------------------------------------------- + + /** Prepare for a new stream. + + Discards all parsing state and any buffered + octets. + */ + BOOST_BURL_DECL + void + reset() noexcept; + + /** Install a content decoder. + + The decoder must remain valid until the + message has been parsed. Passing `nullptr` + removes a previously installed decoder. + + @par Preconditions + `this->got_header() == true` and no body + octet has been parsed. + + @param dec The decoder to install. + */ + BOOST_BURL_DECL + void + set_decoder(decoder* dec) noexcept; + + /** Set the maximum body size. + + Overrides @ref config::body_limit. The limit + is sticky: it applies to every subsequent + message until changed, and is not restored + by @ref start or @ref reset. + + @param n The body size limit in octets. + */ + BOOST_BURL_DECL + void + set_body_limit(std::uint64_t n) noexcept; + + /** Return the buffer region for receiving octets. + + The second region is empty unless the buffer + has wrapped. Report octets written into it + with @ref commit. + + The region may be empty; in that case an + operation which requires more input fails + with @ref http::error::in_place_overflow + rather than asking for it. + + @see @ref commit, @ref commit_eof. + */ + BOOST_BURL_DECL + std::array + prepare() noexcept; + + /** Report octets received into the buffer. + + @par Preconditions + `n <= capy::buffer_size( this->prepare() )` + + @par Postconditions + Regions returned by @ref prepare are + invalidated. + + @param n The number of octets received. + + @see @ref prepare. + */ + BOOST_BURL_DECL + void + commit(std::size_t n) noexcept; + + /** Report the end of the stream. + + Call this when the stream has closed and no + further octets will arrive. + + @par Postconditions + Regions returned by @ref prepare are + invalidated. + + @see @ref prepare. + */ + BOOST_BURL_DECL + void + commit_eof() noexcept; + + /** Return the octets which may be received directly. + + Returns the number of octets which may be + read from the stream straight into + caller-supplied memory, bypassing the + parser's buffer entirely, or zero when that + is not permitted. Report octets received + this way with @ref commit_direct. + + This is zero unless the header has been + parsed, no decoder is installed, the payload + has a known size or is delimited by the end + of the stream, the buffer holds no payload + octets, the stream has not ended, and the + body limit has not been reached. + + @see @ref commit_direct. + */ + BOOST_BURL_DECL + std::size_t + direct_capacity() const noexcept; + + /** Report octets received into caller memory. + + @par Preconditions + `n <= this->direct_capacity()` + + @param n The number of octets received. + + @see @ref direct_capacity. + */ + BOOST_BURL_DECL + void + commit_direct(std::size_t n) noexcept; + + //-------------------------------------------- + // + // Parsing + // + //-------------------------------------------- + + /** Parse the message header. + + Returns as soon as the header is complete, + so that @ref set_decoder and @ref + set_body_limit can be called before any body + octet is parsed. Has no effect once @ref + got_header returns true. + + @par Preconditions + @ref start has been called. + + @param ec Set to the error, if any occurred. + */ + BOOST_BURL_DECL + void + parse_header(system::error_code& ec); + + /** Return the complete body in place. + + Reads the remainder of the body into the + parser's own buffer and returns a view of + the whole body, without copying. A chunked + payload is coalesced in place. + + @par Preconditions + @li `this->got_header() == true` + @li No octet of the body has been retrieved + by @ref read_some or @ref pull. + + @param ec Set to the error, if any occurred. + Set to @ref http::error::in_place_overflow if + the body does not fit in the buffer. + + @return A view of the body, valid until the + parser is modified. + */ + BOOST_BURL_DECL + std::string_view + body(system::error_code& ec); + + /** Copy body octets into caller-supplied memory. + + When a decoder is installed, it writes its + output into `buffers` directly. + + @par Preconditions + `this->got_header() == true` + + @param buffers The destination. + + @param ec Set to the error, if any occurred. + Set to `capy::error::eof` once the body is + complete. + + @return The number of octets written. + */ + BOOST_BURL_DECL + std::size_t + read_some( + std::span buffers, + system::error_code& ec); + + /** Return available body octets in place. + + Fills `dest` with descriptors referring to + the parser's own buffers. Release them with + @ref consume. + + @par Preconditions + `this->got_header() == true` + + @param dest The descriptors to fill. + + @param ec Set to the error, if any occurred. + Set to `capy::error::eof` once the body is + complete. + + @return The filled prefix of `dest`, valid + until the parser is modified. + + @see @ref consume. + */ + BOOST_BURL_DECL + std::span + pull( + std::span dest, + system::error_code& ec); + + /** Release body octets returned by @ref pull. + + @par Preconditions + `n` does not exceed the octets returned by + the last call to @ref pull. + + @param n The number of octets to release. + + @see @ref pull. + */ + BOOST_BURL_DECL + void + consume(std::size_t n) noexcept; + +protected: + parser() = default; + + BOOST_BURL_DECL + parser( + config const& cfg, + bool is_request); + + parser(parser&& other) noexcept = default; + + parser& + operator=(parser&& other) noexcept = default; + + parser(const parser&) = delete; + + parser& + operator=(const parser&) = delete; + + ~parser() = default; + + BOOST_BURL_DECL + void + start(bool head); + + BOOST_BURL_DECL + burl::response_head_base const& + get_response() const; + + BOOST_BURL_DECL + burl::request_head_base const& + get_request() const; + +private: + struct chunk_fn; + + std::error_code + need_more() const noexcept; + + std::size_t + raw_limit_rem() const noexcept; + + std::size_t + dec_limit_rem() const noexcept; + + bool + payload_sized() const noexcept; + + std::size_t + payload_rem() const noexcept; + + std::error_code + walk_chunks(chunk_fn f, bool dry = false); + + std::error_code + flatten_chunks(); + + std::size_t + decode_some( + std::span buffers, + system::error_code& ec); + + std::unique_ptr buf_; + head_parser hp_; + decoder * dec_ = nullptr; + detail::circular_buffer in_; + detail::circular_buffer out_; + std::uint64_t chunk_rem_ = 0; + std::uint64_t transferred_ = 0; + std::uint64_t decoded_ = 0; + std::uint64_t body_limit_ = 0; + std::uint64_t payload_size_ = 0; + std::error_code dec_err_; + http::payload payload_ = http::payload::none; + bool is_req_ : 1 = true; + bool head_ : 1 = false; + bool started_ : 1 = false; + bool got_header_ : 1 = false; + bool got_body_ : 1 = false; + bool mid_chunk_ : 1 = false; + bool fin_chunk_ : 1 = false; + bool eof_ : 1 = false; +}; + +} // namespace burl +} // namespace boost + +#endif diff --git a/include/boost/burl/request_parser.hpp b/include/boost/burl/request_parser.hpp new file mode 100644 index 0000000..5083ee8 --- /dev/null +++ b/include/boost/burl/request_parser.hpp @@ -0,0 +1,91 @@ +// +// Copyright (c) 2026 Mohammad Nejati +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// Official repository: https://github.com/cppalliance/burl +// + +#ifndef BOOST_BURL_REQUEST_PARSER_HPP +#define BOOST_BURL_REQUEST_PARSER_HPP + +#include +#include + +namespace boost +{ +namespace burl +{ + +/** A parser for HTTP/1 requests. + + @see @ref parser, @ref message_reader. +*/ +class request_parser + : public parser +{ +public: + /** Constructor. + + A default-constructed parser behaves as if + constructed with a zero-size buffer, and is + intended only as a target for assignment. + */ + request_parser() = default; + + /** Constructor. + + @param cfg The settings to apply for the + life of the parser. + */ + explicit + request_parser(config const& cfg) + : parser(cfg, true) + { + } + + /// Move constructor. + request_parser(request_parser&&) noexcept = default; + + /// Move assignment. + request_parser& + operator=(request_parser&&) noexcept = default; + + /** Prepare for a new message. + + Any octets already received which belong to + the new message are retained. + + This does not drain: octets of a previous + body which have not been received are not + skipped. Reaching @ref got_body is the + caller's responsibility. + + @par Preconditions + Either this is the first message in the + stream, or the previous message has arrived + in full. + */ + void + start() + { + parser::start(false); + } + + /** Return the parsed header. + + The header is empty until @ref parse_header + succeeds. + */ + burl::request_head_base const& + get() const + { + return get_request(); + } +}; + +} // namespace burl +} // namespace boost + +#endif diff --git a/include/boost/burl/response.hpp b/include/boost/burl/response.hpp index fd4e695..38813db 100644 --- a/include/boost/burl/response.hpp +++ b/include/boost/burl/response.hpp @@ -13,8 +13,9 @@ #include #include #include -#include +#include #include +#include #include #include #include @@ -81,16 +82,16 @@ class response urls::url url_; detail::pooled_connection conn_; - detail::response_parser parser_; - std::unique_ptr decoder_; + response_parser parser_; + std::unique_ptr decoder_; std::optional deadline_; BOOST_BURL_DECL response( urls::url url, detail::pooled_connection conn, - detail::response_parser parser, - std::unique_ptr dec, + response_parser parser, + std::unique_ptr dec, std::optional deadline); public: diff --git a/include/boost/burl/response_parser.hpp b/include/boost/burl/response_parser.hpp new file mode 100644 index 0000000..3d03c54 --- /dev/null +++ b/include/boost/burl/response_parser.hpp @@ -0,0 +1,95 @@ +// +// Copyright (c) 2026 Mohammad Nejati +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// Official repository: https://github.com/cppalliance/burl +// + +#ifndef BOOST_BURL_RESPONSE_PARSER_HPP +#define BOOST_BURL_RESPONSE_PARSER_HPP + +#include +#include + +namespace boost +{ +namespace burl +{ + +/** A parser for HTTP/1 responses. + + @see @ref parser, @ref message_reader. +*/ +class response_parser + : public parser +{ +public: + /** Constructor. + + A default-constructed parser behaves as if + constructed with a zero-size buffer, and is + intended only as a target for assignment. + */ + response_parser() = default; + + /** Constructor. + + @param cfg The settings to apply for the + life of the parser. + */ + explicit + response_parser(config const& cfg) + : parser(cfg, false) + { + } + + /// Move constructor. + response_parser(response_parser&&) noexcept = default; + + /// Move assignment. + response_parser& + operator=(response_parser&&) noexcept = default; + + /** Prepare for a new message. + + Any octets already received which belong to + the new message are retained. + + This does not drain: octets of a previous + body which have not been received are not + skipped. Reaching @ref got_body is the + caller's responsibility. + + @par Preconditions + Either this is the first message in the + stream, or the previous message has arrived + in full. + + @param head True if the response answers a + HEAD request, which states the size of a + representation without sending one. + */ + void + start(bool head = false) + { + parser::start(head); + } + + /** Return the parsed header. + + The header is empty until @ref parse_header + succeeds. + */ + burl::response_head_base const& + get() const + { + return get_response(); + } +}; + +} // namespace burl +} // namespace boost + +#endif diff --git a/include/boost/burl/test/response_factory.hpp b/include/boost/burl/test/response_factory.hpp index 922e2d7..e706dba 100644 --- a/include/boost/burl/test/response_factory.hpp +++ b/include/boost/burl/test/response_factory.hpp @@ -11,7 +11,8 @@ #define BOOST_BURL_TEST_RESPONSE_FACTORY_HPP #include -#include +#include +#include #include #include #include @@ -269,11 +270,12 @@ class response_factory {}, {}); - burl::detail::response_parser parser({}, conn.stream()); + response_parser parser(response_parser::config{}); parser.start(); capy::test::run_blocking()([&]() -> capy::task<> { - if(auto [ec] = co_await parser.read_header(); ec) + if(auto [ec] = co_await message_reader{ + &conn, &parser }.read_header(); ec) throw system::system_error(ec); }()); diff --git a/src/client.cpp b/src/client.cpp index 8a3a1a8..a682560 100644 --- a/src/client.cpp +++ b/src/client.cpp @@ -22,9 +22,11 @@ #include #include +#include #include #include -#include +#include +#include #include #include @@ -220,14 +222,13 @@ client::execute_impl( if(auto_decode) set_accept_encoding(head, config_); - detail::response_parser parser( + response_parser parser( { .hdr_limits = {}, .in_buffer = config_.response_inplace_buffer, .dec_buffer = config_.response_inplace_buffer, .body_limit = config_.response_body_limit - }, - {}); + }); detail::serializer sr({}); auto url = request.url; @@ -260,7 +261,7 @@ client::execute_impl( // TODO: expect100timeout - auto stream = conn.stream(); + capy::any_write_stream stream(&conn); sr.reset(&stream, &head); if(request.body.has_value()) { @@ -274,10 +275,11 @@ client::execute_impl( co_return { wec, {} }; } - parser.reset(std::move(stream)); + parser.reset(); parser.start(is_head); - auto [rec] = co_await parser.read_header(); + auto [rec] = co_await message_reader{ + &conn, &parser }.read_header(); if(rec) co_return { rec, {} }; @@ -302,7 +304,7 @@ client::execute_impl( if(status_int >= 400) ec = std::error_code(status_int, burl_category()); - std::unique_ptr dec; + std::unique_ptr dec; if(auto_decode && !is_head) { dec = detail::make_decoder( @@ -319,7 +321,7 @@ client::execute_impl( // Read and discard small bodies so the connection can be reused auto [dec, drained] = co_await corosio::timeout( - detail::drain_body(parser, 3), + detail::drain_body(conn, parser, 3), std::chrono::seconds(2)); if(drained && detail::can_reuse_conn(parser)) conn.return_to_pool(); diff --git a/src/detail/can_reuse_conn.hpp b/src/detail/can_reuse_conn.hpp index 486bfb1..e5aa1b3 100644 --- a/src/detail/can_reuse_conn.hpp +++ b/src/detail/can_reuse_conn.hpp @@ -10,7 +10,7 @@ #ifndef BOOST_BURL_SRC_DETAIL_CAN_REUSE_CONN_HPP #define BOOST_BURL_SRC_DETAIL_CAN_REUSE_CONN_HPP -#include +#include namespace boost { diff --git a/src/detail/decoders.hpp b/src/detail/decoders.hpp index 72c0b5e..4502c54 100644 --- a/src/detail/decoders.hpp +++ b/src/detail/decoders.hpp @@ -10,7 +10,7 @@ #ifndef BOOST_BURL_SRC_DETAIL_DECODERS_HPP #define BOOST_BURL_SRC_DETAIL_DECODERS_HPP -#include +#include #include diff --git a/src/detail/drain_body.cpp b/src/detail/drain_body.cpp deleted file mode 100644 index 5a17842..0000000 --- a/src/detail/drain_body.cpp +++ /dev/null @@ -1,46 +0,0 @@ -// -// Copyright (c) 2026 Mohammad Nejati -// -// Distributed under the Boost Software License, Version 1.0. (See accompanying -// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) -// -// Official repository: https://github.com/cppalliance/burl -// - -#include "drain_body.hpp" - -#include - -namespace boost -{ -namespace burl -{ -namespace detail -{ - -capy::io_task -drain_body( - response_parser& parser, - std::size_t attempts) -{ - while(!parser.got_body()) - { - if(attempts-- == 0) - co_return { {}, false }; - - capy::const_buffer arr[8]; - auto [ec, bufs] = co_await parser.pull(arr); - if(ec) - { - if(ec == capy::cond::eof) - break; - co_return { ec, false }; - } - parser.consume(capy::buffer_size(bufs)); - } - co_return { {}, true }; -} - -} // namespace detail -} // namespace burl -} // namespace boost diff --git a/src/detail/drain_body.hpp b/src/detail/drain_body.hpp index 2543f0e..89ae0ca 100644 --- a/src/detail/drain_body.hpp +++ b/src/detail/drain_body.hpp @@ -10,10 +10,15 @@ #ifndef BOOST_BURL_SRC_DETAIL_DRAIN_BODY_HPP #define BOOST_BURL_SRC_DETAIL_DRAIN_BODY_HPP -#include +#include +#include + +#include +#include +#include #include -#include +#include namespace boost { @@ -22,10 +27,32 @@ namespace burl namespace detail { +template capy::io_task drain_body( + S& stream, response_parser& parser, - std::size_t attempts); + std::size_t attempts) +{ + message_reader reader{ &stream, &parser }; + + while(!parser.got_body()) + { + if(attempts-- == 0) + co_return { {}, false }; + + capy::const_buffer arr[8]; + auto [ec, bufs] = co_await reader.pull(arr); + if(ec) + { + if(ec == capy::cond::eof) + break; + co_return { ec, false }; + } + parser.consume(capy::buffer_size(bufs)); + } + co_return { {}, true }; +} } // namespace detail } // namespace burl diff --git a/src/detail/http_tunnel.cpp b/src/detail/http_tunnel.cpp index 028fde9..47fb151 100644 --- a/src/detail/http_tunnel.cpp +++ b/src/detail/http_tunnel.cpp @@ -10,7 +10,8 @@ #include "http_tunnel.hpp" #include -#include +#include +#include #include #include "base64.hpp" @@ -54,9 +55,10 @@ open_http_tunnel( ec) co_return ec; - detail::response_parser parser({}, &stream); + response_parser parser(response_parser::config{}); parser.start(); - if(auto [ec] = co_await parser.read_header(); ec) + if(auto [ec] = co_await message_reader{ + &stream, &parser }.read_header(); ec) co_return { error::proxy_connect_failed }; auto status = parser.get().status(); diff --git a/src/detail/parser.cpp b/src/parser.cpp similarity index 64% rename from src/detail/parser.cpp rename to src/parser.cpp index 65585a7..db1523a 100644 --- a/src/detail/parser.cpp +++ b/src/parser.cpp @@ -7,29 +7,32 @@ // Official repository: https://github.com/cppalliance/burl // -#include +#include -#include "util.hpp" +#include + +#include "detail/util.hpp" #include -#include +#include #include #include -#include +#include #include +#include #include #include #include +#include #include -#include namespace boost { namespace burl { -namespace detail -{ + +using detail::clamp; using http::condition::need_more_input; using http::error::bad_payload; @@ -165,10 +168,9 @@ parse_chunk_header( chained_sequence& cs, std::uint64_t& size) noexcept { - for(auto const start = cs.size();;) + auto const start = cs.size(); + while(!cs.empty()) { - if(cs.empty()) - return need_data; auto const n = urls::grammar::hexdig_value(cs.value()); if(n < 0) { @@ -183,15 +185,14 @@ parse_chunk_header( size = (size << 4) | static_cast(n); cs.next(); } + return need_data; } std::error_code skip_trailer(chained_sequence& cs) noexcept { - for(;;) + while(!cs.empty()) { - if(cs.empty()) - return need_data; if(cs.value() == '\r') { if(!cs.next()) @@ -205,6 +206,7 @@ skip_trailer(chained_sequence& cs) noexcept if(auto ec = skip_to_eol(cs); ec) return ec; } + return need_data; } std::error_code @@ -275,10 +277,8 @@ struct parser::chunk_fn parser:: parser( config const& cfg, - bool is_request, - capy::any_read_stream stream) - : stream_(std::move(stream)) - , body_limit_(cfg.body_limit) + bool is_request) + : body_limit_(cfg.body_limit) , is_req_(is_request) { auto const h_cap = head_parser::bytes_needed( @@ -322,6 +322,24 @@ has_buffered_data() const noexcept } } +std::array +parser:: +buffered_data() const noexcept +{ + return in_.data(); +} + +std::error_code +parser:: +need_more() const noexcept +{ + if(eof_) + return incomplete; + if(in_.full()) + return in_place_overflow; + return need_data; +} + std::size_t parser:: raw_limit_rem() const noexcept @@ -386,51 +404,68 @@ start(bool head) void parser:: -reset(capy::any_read_stream stream) noexcept +reset() noexcept { hp_.reset(buf_.get()); in_.reset(buf_.get()); - stream_ = std::move(stream); - dec_ = nullptr; - chunk_rem_ = 0; - transferred_ = 0; - decoded_ = 0; - payload_size_ = 0; - dec_err_ = {}; - payload_ = payload::none; - head_ = false; - started_ = false; - got_header_ = false; - got_body_ = false; - mid_chunk_ = false; - fin_chunk_ = false; - eof_ = false; + payload_ = payload::none; + started_ = false; + got_header_ = false; + got_body_ = false; + eof_ = false; } -capy::io_task<> -parser::refill() +std::array +parser:: +prepare() noexcept +{ + return in_.prepare(); +} + +void +parser:: +commit(std::size_t n) noexcept { - if(eof_) - co_return { incomplete }; - if(in_.full()) - co_return { in_place_overflow }; - auto [ec, n] = co_await stream_.read_some(in_.prepare()); in_.commit(n); if(payload_sized() && payload_rem() <= in_.size()) got_body_ = true; - if(ec) +} + +void +parser:: +commit_eof() noexcept +{ + eof_ = true; + if(got_header_ && payload_ == payload::to_eof) + got_body_ = true; +} + +std::size_t +parser:: +direct_capacity() const noexcept +{ + if(!got_header_ || dec_ || eof_ || !in_.empty()) + return 0; + + switch(payload_) { - if(ec == capy::cond::eof) - { - eof_ = true; - if(payload_ == payload::to_eof) - got_body_ = true; - co_return {}; - } - co_return ec; + case payload::size: + return clamp(payload_rem(), raw_limit_rem()); + case payload::to_eof: + return raw_limit_rem(); + default: + return 0; } - co_return {}; +} + +void +parser:: +commit_direct(std::size_t n) noexcept +{ + transferred_ += n; + if(payload_sized() && payload_rem() == 0) + got_body_ = true; } std::error_code @@ -560,76 +595,78 @@ flatten_chunks() } } -capy::io_task<> +void parser:: -read_header() +parse_header(system::error_code& ec) { BOOST_ASSERT(started_); + ec = {}; + if(got_header_) - co_return {}; + return; for(;;) { - system::error_code ec; hp_.parse(in_.size(), ec); - if(ec) + if(ec == in_place_overflow && in_.ptr != buf_.get()) { - if(ec == in_place_overflow && in_.ptr != buf_.get()) - { - in_.slide(buf_.get()); - hp_.rebase(buf_.get()); - continue; - } - if(ec != need_more_input) - co_return { ec }; - if(eof_) - { - if(in_.empty()) - co_return { end_of_stream }; - co_return { incomplete }; - } - if(auto [fec] = co_await refill(); fec) - co_return { fec }; + in_.slide(buf_.get()); + hp_.rebase(buf_.get()); continue; } - - auto const& h = hp_.message_head(); - got_header_ = true; - payload_ = head_ ? payload::none : h.payload(); - payload_size_ = h.content_length().value_or(0); - - auto const head_size = h.buffer().size(); - - switch(payload_) + break; + } + if(ec) + { + if(ec != need_more_input) + return; + if(eof_) { - case payload::error: - co_return { bad_payload }; - case payload::none: - got_body_ = true; - break; - case payload::size: - if(payload_rem() <= in_.size() - head_size) - got_body_ = true; - break; - case payload::chunked: - break; - case payload::to_eof: - if(eof_) - got_body_ = true; - break; + if(in_.empty()) + ec = end_of_stream; + else + ec = incomplete; + return; } + ec = need_data; + return; + } - if(!got_body_) - { - in_.slide(buf_.get()); - hp_.rebase(buf_.get()); - } + auto const& h = hp_.message_head(); + got_header_ = true; + payload_ = head_ ? payload::none : h.payload(); + payload_size_ = h.content_length().value_or(0); - in_.shed(head_size); + auto const head_size = h.buffer().size(); - co_return {}; + switch(payload_) + { + case payload::error: + ec = bad_payload; + return; + case payload::none: + got_body_ = true; + break; + case payload::size: + if(payload_rem() <= in_.size() - head_size) + got_body_ = true; + break; + case payload::chunked: + break; + case payload::to_eof: + if(eof_) + got_body_ = true; + break; } + + if(!got_body_) + { + in_.slide(buf_.get()); + hp_.rebase(buf_.get()); + } + + in_.shed(head_size); } void @@ -647,56 +684,74 @@ set_body_limit(std::uint64_t n) noexcept body_limit_ = n; } -capy::io_task +std::string_view parser:: -read_body() +body(system::error_code& ec) { - if(auto [ec] = co_await read_header(); ec) - co_return { ec, {} }; + BOOST_ASSERT(got_header_); + + ec = {}; if(dec_) { if(decoded_ != out_.size()) - co_return { incomplete, {} }; + { + ec = incomplete; + return {}; + } for(;;) { if(out_.full()) - co_return { in_place_overflow, {} }; - auto [ec, n] = co_await decode_some(out_.prepare()); + { + ec = in_place_overflow; + return {}; + } + auto pb = out_.prepare(); + auto const n = decode_some(pb, ec); out_.commit(n); if(ec) { if(ec == capy::cond::eof) - co_return { {}, { out_.ptr, out_.len } }; - co_return { ec, {} }; + { + ec = {}; + return { out_.ptr, out_.len }; + } + return {}; } } } if(transferred_ != 0) - co_return { incomplete, {} }; + { + ec = incomplete; + return {}; + } switch(payload_) { case payload::error: case payload::none: { - co_return { {}, {} }; + return {}; } case payload::chunked: { for(;;) { if(chunk_rem_ > raw_limit_rem()) - co_return { body_too_large, {} }; + { + ec = body_too_large; + return {}; + } if(fin_chunk_) - co_return { {}, { in_.ptr, clamp(chunk_rem_) } }; - if(auto ec = flatten_chunks(); ec) + return { in_.ptr, clamp(chunk_rem_) }; + if(auto fec = flatten_chunks(); fec) { - if(ec != need_more_input) - co_return { ec, {} }; - if(auto [fec] = co_await refill(); fec) - co_return { fec, {} }; + if(fec != need_more_input) + ec = fec; + else + ec = need_more(); + return {}; } } } @@ -704,28 +759,30 @@ read_body() { auto const rem = payload_rem(); if(rem > raw_limit_rem()) - co_return { body_too_large, {} }; - for(;;) { - if(got_body_) - co_return { {}, { in_.ptr, clamp(in_.len, rem) } }; - if(auto [fec] = co_await refill(); fec) - co_return { fec, {} }; + ec = body_too_large; + return {}; } + if(got_body_) + return { in_.ptr, clamp(in_.len, rem) }; + ec = need_more(); + return {}; } case payload::to_eof: { - for(;;) + if(in_.size() > raw_limit_rem()) { - if(in_.size() > raw_limit_rem()) - co_return { body_too_large, {} }; - if(got_body_) - co_return { {}, { in_.ptr, in_.len } }; - if(auto [fec] = co_await refill(); fec) - co_return { fec, {} }; + ec = body_too_large; + return {}; } + if(got_body_) + return { in_.ptr, in_.len }; + ec = need_more(); + return {}; } } + + return {}; } burl::response_head_base const& @@ -742,13 +799,16 @@ get_request() const return hp_.request_head(); } -capy::io_task +std::size_t parser:: decode_some( - std::span buffers) + std::span buffers, + system::error_code& ec) { + ec = {}; + if(capy::buffer_empty(buffers)) - co_return { {}, 0 }; + return 0; auto outbufs = capy::consuming_buffers(buffers); std::size_t prod = 0; @@ -802,20 +862,19 @@ decode_some( case payload::error: case payload::none: { - co_return { capy::error::eof, 0 }; + ec = capy::error::eof; + return 0; } case payload::chunked: { - for(;;) - { - auto ec = walk_chunks(decode); - if(prod != 0) - co_return { {}, prod }; - if(ec != need_more_input) - co_return { ec, 0 }; - if(auto [fec] = co_await refill(); fec) - co_return { fec, 0 }; - } + auto const wec = walk_chunks(decode); + if(prod != 0) + return prod; + if(wec != need_more_input) + ec = wec; + else + ec = need_more(); + return 0; } case payload::size: case payload::to_eof: @@ -826,28 +885,34 @@ decode_some( auto const in = in_.first(rem); if(in.size() == 0 && !got_body_) { - if(auto [fec] = co_await refill(); fec) - co_return { fec, 0 }; - continue; + ec = need_more(); + return 0; } - auto [ec, cons] = decode(in, got_body_ && in.size() == rem); + auto [dec_ec, cons] = decode(in, got_body_ && in.size() == rem); in_.consume(cons); if(prod != 0) - co_return { {}, prod }; - if(ec) - co_return { ec, 0 }; + return prod; + if(dec_ec) + { + ec = dec_ec; + return 0; + } } } } + + return 0; } -capy::io_task +std::size_t parser:: -do_read_some( - std::span buffers) +read_some( + std::span buffers, + system::error_code& ec) { - if(auto [ec] = co_await read_header(); ec) - co_return { ec, 0 }; + BOOST_ASSERT(got_header_); + + ec = {}; if(dec_) { @@ -855,9 +920,9 @@ do_read_some( { auto const n = capy::buffer_copy(buffers, out_.data()); out_.consume(n); - co_return { {}, n }; + return n; } - co_return co_await decode_some(buffers); + return decode_some(buffers, ec); } auto copy = [&](std::size_t at_most) @@ -874,106 +939,107 @@ do_read_some( case payload::error: case payload::none: { - co_return { capy::error::eof, 0 }; + ec = capy::error::eof; + return 0; } case payload::chunked: { - for(;;) + std::size_t read = 0; + std::size_t lim = raw_limit_rem(); + auto outbufs = capy::consuming_buffers(buffers); + auto const wec = walk_chunks( + [&](capy::const_buffer b, bool) + -> capy::io_result { - std::size_t read = 0; - std::size_t lim = raw_limit_rem(); - auto outbufs = capy::consuming_buffers(buffers); - auto ec = walk_chunks( - [&](capy::const_buffer b, bool) - -> capy::io_result - { - auto const take = clamp(b.size(), lim); - lim -= take; - auto const n = capy::buffer_copy(outbufs.data(), b, take); - read += n; - outbufs.consume(n); - if(take < b.size()) - return { body_too_large, n }; - return { {}, n }; - - }); - if(read != 0) - co_return { {}, read }; - if(ec == need_more_input) - { - if(auto [fec] = co_await refill(); fec) - co_return { fec, 0 }; - continue; - } - else if(ec) - co_return { ec, 0 }; - BOOST_ASSERT(got_body_); - co_return { capy::error::eof, 0 }; + auto const take = clamp(b.size(), lim); + lim -= take; + auto const n = capy::buffer_copy(outbufs.data(), b, take); + read += n; + outbufs.consume(n); + if(take < b.size()) + return { body_too_large, n }; + return { {}, n }; + + }); + if(read != 0) + return read; + if(wec == need_more_input) + { + ec = need_more(); + return 0; } + if(wec) + { + ec = wec; + return 0; + } + BOOST_ASSERT(got_body_); + ec = capy::error::eof; + return 0; } case payload::size: { auto const rem = payload_rem(); if(rem == 0) - co_return { capy::error::eof, 0 }; + { + ec = capy::error::eof; + return 0; + } auto const lim = raw_limit_rem(); if(lim == 0) - co_return { body_too_large, 0 }; - if(!in_.empty()) - co_return { {}, copy(clamp(rem, lim)) }; - if(eof_) - co_return { incomplete, 0 }; - auto [ec, n] = co_await stream_.read_some( - capy::buffer_slice(buffers, 0, clamp(rem, lim))); - transferred_ += n; - if(n == rem) - got_body_ = true; - if(ec == capy::cond::eof) { - eof_ = true; - if(n != rem) - co_return { incomplete, n }; + ec = body_too_large; + return 0; } - co_return { ec, n }; + if(!in_.empty()) + return copy(clamp(rem, lim)); + ec = need_more(); + return 0; } case payload::to_eof: { - if(eof_) - co_return { capy::error::eof, 0 }; auto const lim = raw_limit_rem(); if(lim == 0) - co_return { body_too_large, 0 }; + { + ec = body_too_large; + return 0; + } if(!in_.empty()) - co_return { {}, copy(lim) }; - auto [ec, n] = co_await stream_.read_some( - capy::buffer_slice(buffers, 0, lim)); - transferred_ += n; - if(ec == capy::cond::eof) + return copy(lim); + if(eof_) { - eof_ = true; - got_body_ = true; + ec = capy::error::eof; + return 0; } - co_return { ec, n }; + ec = need_more(); + return 0; } } + + return 0; } -capy::io_task> +std::span parser:: -pull(std::span dest) +pull( + std::span dest, + system::error_code& ec) { - if(auto [ec] = co_await read_header(); ec) - co_return { ec, {} }; + BOOST_ASSERT(got_header_); + + ec = {}; if(dec_) { if(!out_.empty()) - co_return { {}, collect(dest, out_.data()) }; - auto [ec, n] = co_await decode_some(out_.prepare()); + return collect(dest, out_.data()); + auto pb = out_.prepare(); + auto const n = decode_some(pb, ec); out_.commit(n); if(ec && n == 0) - co_return { ec, {} }; - co_return { {}, collect(dest, out_.data()) }; + return {}; + ec = {}; + return collect(dest, out_.data()); } switch(payload_) @@ -981,75 +1047,84 @@ pull(std::span dest) case payload::error: case payload::none: { - co_return { capy::error::eof, {} }; + ec = capy::error::eof; + return {}; } case payload::chunked: { - for(;;) + std::size_t n = 0; + std::size_t lim = raw_limit_rem(); + if(lim == 0) { - std::size_t n = 0; - std::size_t lim = raw_limit_rem(); - if(lim == 0) - co_return { body_too_large, {} }; - auto ec = walk_chunks( - [&](capy::const_buffer b, bool last) - -> capy::io_result - { - if(last && b.size() == 0) - return { capy::error::eof, 0 }; - auto const take = clamp(b.size(), lim); - if(take == 0 || n == dest.size()) - return { {}, 0 }; - lim -= take; - dest[n++] = { b.data(), take }; - return { {}, take }; - }, - true); - if(n != 0) - co_return { {}, dest.first(n) }; - if(ec != need_more_input) - { - if(ec == capy::error::eof) - consume(0); // chunk trailer - co_return { ec, {} }; - } - if(auto [fec] = co_await refill(); fec) - co_return { fec, {} }; + ec = body_too_large; + return {}; } + auto const wec = walk_chunks( + [&](capy::const_buffer b, bool last) + -> capy::io_result + { + if(last && b.size() == 0) + return { capy::error::eof, 0 }; + auto const take = clamp(b.size(), lim); + if(take == 0 || n == dest.size()) + return { {}, 0 }; + lim -= take; + dest[n++] = { b.data(), take }; + return { {}, take }; + }, + true); + if(n != 0) + return dest.first(n); + if(wec != need_more_input) + { + if(wec == capy::error::eof) + consume(0); // chunk trailer + ec = wec; + return {}; + } + ec = need_more(); + return {}; } case payload::size: { auto const rem = payload_rem(); auto const lim = raw_limit_rem(); if(rem == 0) - co_return { capy::error::eof, {} }; + { + ec = capy::error::eof; + return {}; + } if(lim == 0) - co_return { body_too_large, {} }; - for(;;) { - if(!in_.empty()) - co_return { {}, collect( - dest, in_.data(), clamp(rem, lim)) }; - if(auto [fec] = co_await refill(); fec) - co_return { fec, {} }; + ec = body_too_large; + return {}; } + if(!in_.empty()) + return collect(dest, in_.data(), clamp(rem, lim)); + ec = need_more(); + return {}; } case payload::to_eof: { auto const lim = raw_limit_rem(); if(lim == 0) - co_return { body_too_large, {} }; - for(;;) { - if(!in_.empty()) - co_return { {}, collect(dest, in_.data(), lim) }; - if(eof_) - co_return { capy::error::eof, {} }; - if(auto [fec] = co_await refill(); fec) - co_return { fec, {} }; + ec = body_too_large; + return {}; } + if(!in_.empty()) + return collect(dest, in_.data(), lim); + if(eof_) + { + ec = capy::error::eof; + return {}; + } + ec = need_more(); + return {}; } } + + return {}; } void @@ -1078,6 +1153,5 @@ consume(std::size_t n) noexcept } } -} // namespace detail } // namespace burl } // namespace boost diff --git a/src/response.cpp b/src/response.cpp index db0f2c5..a1f11cf 100644 --- a/src/response.cpp +++ b/src/response.cpp @@ -25,8 +25,8 @@ namespace burl response::response( urls::url url, detail::pooled_connection conn, - detail::response_parser parser, - std::unique_ptr dec, + response_parser parser, + std::unique_ptr dec, std::optional deadline) : url_(std::move(url)) , conn_(std::move(conn)) @@ -72,8 +72,9 @@ response::try_as_view() & { if(deadline_) co_return co_await corosio::timeout( - parser_.read_body(), *deadline_ - clock::now()); - co_return co_await parser_.read_body(); + message_reader{ &conn_, &parser_ }.read_body(), + *deadline_ - clock::now()); + co_return co_await message_reader{ &conn_, &parser_ }.read_body(); } capy::task @@ -90,13 +91,15 @@ response::as_view() & http::any_buffer_source response::as_buffer_source() & { - return http::any_buffer_source(&parser_); + return http::any_buffer_source( + message_reader{ &conn_, &parser_ }); } http::any_read_source response::as_read_source() & { - return http::any_read_source(&parser_); + return http::any_read_source( + message_reader{ &conn_, &parser_ }); } } // namespace burl diff --git a/test/unit/detail/can_reuse_conn.cpp b/test/unit/detail/can_reuse_conn.cpp index 7928a0c..d616497 100644 --- a/test/unit/detail/can_reuse_conn.cpp +++ b/test/unit/detail/can_reuse_conn.cpp @@ -10,6 +10,8 @@ // Test that header file is self-contained. #include "src/detail/can_reuse_conn.hpp" +#include + #include "test_suite.hpp" #include @@ -33,12 +35,13 @@ class can_reuse_conn_test auto [client, server] = capy::test::make_stream_pair(); server.provide(response); - response_parser parser({}, capy::any_read_stream(&client)); + response_parser parser(response_parser::config{}); bool result = false; capy::test::run_blocking()([&]() -> capy::task<> { parser.start(); - if(auto [rec] = co_await parser.read_header(); rec) + if(auto [rec] = co_await message_reader{ + &client, &parser }.read_header(); rec) co_return; result = can_reuse_conn(parser); }()); @@ -90,7 +93,7 @@ class can_reuse_conn_test void testNoHeader() { - response_parser parser({}, {}); + response_parser parser(response_parser::config{}); BOOST_TEST(!can_reuse_conn(parser)); } diff --git a/test/unit/detail/circular_buffer.cpp b/test/unit/detail/circular_buffer.cpp new file mode 100644 index 0000000..6e183e1 --- /dev/null +++ b/test/unit/detail/circular_buffer.cpp @@ -0,0 +1,304 @@ +// +// Copyright (c) 2026 Mohammad Nejati +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// Official repository: https://github.com/cppalliance/burl +// + +// Test that header file is self-contained. +#include + +#include +#include + +#include +#include + +#include "test_suite.hpp" + +namespace boost +{ +namespace burl +{ +namespace detail +{ + +class circular_buffer_test +{ + static + std::string + str(std::array const& bufs) + { + std::string s; + for(auto b : bufs) + s.append(static_cast(b.data()), b.size()); + return s; + } + + static + std::string + str(capy::const_buffer b) + { + return { static_cast(b.data()), b.size() }; + } + + // Writes through prepare/commit, like a stream read would. + static + void + put(circular_buffer& cb, std::string_view s) + { + auto const n = capy::buffer_copy( + cb.prepare(), capy::make_buffer(s.data(), s.size())); + BOOST_TEST_EQ(n, s.size()); + cb.commit(n); + } + +public: + void + testEmpty() + { + char store[8]; + circular_buffer cb{ store, sizeof(store) }; + + BOOST_TEST(cb.empty()); + BOOST_TEST(!cb.full()); + BOOST_TEST(!cb.wrapped()); + BOOST_TEST_EQ(cb.size(), 0); + BOOST_TEST(str(cb.data()).empty()); + BOOST_TEST_EQ(cb.first(5).size(), 0); + + auto const pb = cb.prepare(); + BOOST_TEST_EQ(pb[0].size(), sizeof(store)); + BOOST_TEST_EQ(pb[1].size(), 0); + } + + void + testFillAndDrain() + { + char store[8]; + circular_buffer cb{ store, sizeof(store) }; + + put(cb, "hello"); + BOOST_TEST(!cb.empty()); + BOOST_TEST_EQ(cb.size(), 5); + BOOST_TEST(str(cb.data()) == "hello"); + + // first clamps to the smaller of n and what is contiguous + BOOST_TEST(str(cb.first(3)) == "hel"); + BOOST_TEST(str(cb.first(100)) == "hello"); + + cb.consume(2); + BOOST_TEST(str(cb.data()) == "llo"); + + // over-commit saturates at capacity + cb.commit(100); + BOOST_TEST(cb.full()); + BOOST_TEST_EQ(cb.size(), sizeof(store)); + + // over-consume saturates at empty + cb.consume(100); + BOOST_TEST(cb.empty()); + BOOST_TEST_EQ(cb.size(), 0); + } + + void + testWrapAround() + { + char store[8]; + circular_buffer cb{ store, sizeof(store) }; + + put(cb, "abcdefgh"); + BOOST_TEST(cb.full()); + cb.consume(6); + + // the free region is contiguous at the front + put(cb, "XYZ"); + BOOST_TEST(cb.wrapped()); + BOOST_TEST_EQ(cb.size(), 5); + BOOST_TEST(str(cb.data()) == "ghXYZ"); + + // first serves only the tail segment of a wrapped buffer + BOOST_TEST(str(cb.first(100)) == "gh"); + BOOST_TEST(str(cb.first(1)) == "g"); + + // consuming past the end wraps the read position + cb.consume(4); + BOOST_TEST(!cb.wrapped()); + BOOST_TEST(str(cb.data()) == "Z"); + cb.consume(1); + BOOST_TEST(cb.empty()); + } + + void + testReset() + { + char store[8]; + circular_buffer cb{ store, sizeof(store) }; + put(cb, "abcdefgh"); + + // rebasing to an interior pointer discards the contents + // and shrinks the capacity to what lies above it + cb.reset(store + 2); + BOOST_TEST(cb.empty()); + BOOST_TEST_EQ(cb.ptr, store + 2); + BOOST_TEST_EQ(cb.cap, sizeof(store) - 2); + + put(cb, "XYZ"); + BOOST_TEST(str(cb.data()) == "XYZ"); + } + + void + testShedAndSlide() + { + char store[8]; + circular_buffer cb{ store, sizeof(store) }; + put(cb, "hello"); + + // shed gives up the leading octets and the region they + // occupy + cb.shed(2); + BOOST_TEST_EQ(cb.ptr, store + 2); + BOOST_TEST_EQ(cb.cap, sizeof(store) - 2); + BOOST_TEST(str(cb.data()) == "llo"); + + // slide moves the contents down and reclaims the region + cb.slide(store); + BOOST_TEST_EQ(cb.ptr, store); + BOOST_TEST_EQ(cb.cap, sizeof(store)); + BOOST_TEST(str(cb.data()) == "llo"); + } + + void + testLinearizeEmpty() + { + char store[8]; + circular_buffer cb{ store + 4, 4 }; + + // an empty buffer rebases to the floor and reclaims the + // whole region + auto* p = cb.linearize(store); + BOOST_TEST_EQ(p, store); + BOOST_TEST_EQ(cb.ptr, store); + BOOST_TEST_EQ(cb.cap, sizeof(store)); + BOOST_TEST(cb.empty()); + } + + void + testLinearizeStraight() + { + char store[8]; + circular_buffer cb{ store, sizeof(store) }; + put(cb, "abcde"); + cb.consume(2); + + // contiguous contents stay put: only the base moves up + // to them + auto* p = cb.linearize(store); + BOOST_TEST_EQ(p, store + 2); + BOOST_TEST_EQ(cb.ptr, store + 2); + BOOST_TEST_EQ(cb.cap, sizeof(store) - 2); + BOOST_TEST_EQ(cb.pos, 0); + BOOST_TEST(str(cb.data()) == "cde"); + } + + void + testLinearizeWrapped() + { + char store[8]; + circular_buffer cb{ store, sizeof(store) }; + put(cb, "abcdefgh"); + cb.consume(6); + put(cb, "XYZ"); + BOOST_TEST(cb.wrapped()); + + // wrapped contents are rotated down to the floor + auto* p = cb.linearize(store); + BOOST_TEST_EQ(p, store); + BOOST_TEST_EQ(cb.ptr, store); + BOOST_TEST_EQ(cb.cap, sizeof(store)); + BOOST_TEST_EQ(cb.pos, 0); + BOOST_TEST(!cb.wrapped()); + BOOST_TEST(str(cb.data()) == "ghXYZ"); + BOOST_TEST(str(cb.first(100)) == "ghXYZ"); + } + + void + testLinearizeWrappedOverlap() + { + // the two segments overlap both source and destination; + // the rotation has to proceed in more than one step + char store[8]; + circular_buffer cb{ store, sizeof(store) }; + put(cb, "abcdefgh"); + cb.consume(5); + put(cb, "XYZ"); + BOOST_TEST(cb.wrapped()); + BOOST_TEST_EQ(cb.size(), 6); + + auto* p = cb.linearize(store); + BOOST_TEST_EQ(p, store); + BOOST_TEST(str(cb.data()) == "fghXYZ"); + } + + void + testLinearizeWrappedLongFront() + { + // the wrapped-around segment is longer than the tail one + char store[8]; + circular_buffer cb{ store, sizeof(store) }; + put(cb, "abcdefgh"); + cb.consume(7); + put(cb, "UVWXY"); + BOOST_TEST(cb.wrapped()); + BOOST_TEST_EQ(cb.size(), 6); + + auto* p = cb.linearize(store); + BOOST_TEST_EQ(p, store); + BOOST_TEST(str(cb.data()) == "hUVWXY"); + } + + void + testLinearizeWrappedBelowBase() + { + // room below the buffer: the rotation lands at the floor + // and the capacity grows by the reclaimed region + char store[12]; + circular_buffer cb{ store + 4, 8 }; + put(cb, "abcdefgh"); + cb.consume(6); + put(cb, "XYZ"); + BOOST_TEST(cb.wrapped()); + + auto* p = cb.linearize(store); + BOOST_TEST_EQ(p, store); + BOOST_TEST_EQ(cb.ptr, store); + BOOST_TEST_EQ(cb.cap, sizeof(store)); + BOOST_TEST(str(cb.data()) == "ghXYZ"); + } + + void + run() + { + testEmpty(); + testFillAndDrain(); + testWrapAround(); + testReset(); + testShedAndSlide(); + testLinearizeEmpty(); + testLinearizeStraight(); + testLinearizeWrapped(); + testLinearizeWrappedOverlap(); + testLinearizeWrappedLongFront(); + testLinearizeWrappedBelowBase(); + } +}; + +TEST_SUITE( + circular_buffer_test, + "boost.burl.detail.circular_buffer"); + +} // namespace detail +} // namespace burl +} // namespace boost diff --git a/test/unit/detail/connection_pool.cpp b/test/unit/detail/connection_pool.cpp index 48e0bab..fb16290 100644 --- a/test/unit/detail/connection_pool.cpp +++ b/test/unit/detail/connection_pool.cpp @@ -135,7 +135,7 @@ class connection_pool_test } static capy::task<> - ping(capy::any_stream s) + ping(pooled_connection& s) { auto [wec, wn] = co_await capy::write( s, make_buffer("ping", 4)); @@ -328,7 +328,7 @@ class connection_pool_test // The connection is still fully usable. net.server(0).provide("hello"); char buf[8]; - auto [rec, n] = co_await pc.stream().read_some( + auto [rec, n] = co_await pc.read_some( capy::mutable_buffer(buf, sizeof(buf))); BOOST_TEST(!rec); BOOST_TEST_EQ(std::string_view(buf, n), "hello"); @@ -395,12 +395,12 @@ class connection_pool_test char buf[1] = {}; - auto [rec, n1] = co_await pc.stream().read_some( + auto [rec, n1] = co_await pc.read_some( make_buffer(buf)); BOOST_TEST(rec == capy::error::timeout); BOOST_TEST_EQ(n1, 0); - auto [wec, n2] = co_await pc.stream().write_some( + auto [wec, n2] = co_await pc.write_some( make_buffer(buf)); BOOST_TEST(wec == capy::error::timeout); BOOST_TEST_EQ(n2, 0); @@ -434,7 +434,7 @@ class connection_pool_test { auto [aec, pc] = co_await pool->acquire(server.url("http")); BOOST_TEST(!aec); - co_await ping(pc.stream()); + co_await ping(pc); pool->release(std::move(pc)); } }; @@ -511,7 +511,7 @@ class connection_pool_test { auto [aec, pc] = co_await pool->acquire(server.url("https")); BOOST_TEST(!aec); - co_await ping(pc.stream()); + co_await ping(pc); pool->release(std::move(pc)); } }; @@ -618,7 +618,7 @@ class connection_pool_test { auto [aec, pc] = co_await pool->acquire("http://example.com"); BOOST_TEST(!aec); - co_await ping(pc.stream()); + co_await ping(pc); pool->release(std::move(pc)); } }; @@ -698,7 +698,7 @@ class connection_pool_test { auto [aec, pc] = co_await pool->acquire("http://127.0.0.1"); BOOST_TEST(!aec); - co_await ping(pc.stream()); + co_await ping(pc); pool->release(std::move(pc)); } }; @@ -752,7 +752,7 @@ class connection_pool_test { auto [aec, pc] = co_await pool->acquire("http://example.com"); BOOST_TEST(!aec); - co_await ping(pc.stream()); + co_await ping(pc); pool->release(std::move(pc)); } }; diff --git a/test/unit/detail/decoders.cpp b/test/unit/detail/decoders.cpp index 74015c2..4ee6a29 100644 --- a/test/unit/detail/decoders.cpp +++ b/test/unit/detail/decoders.cpp @@ -12,6 +12,7 @@ #include +#include #include #include "test_suite.hpp" diff --git a/test/unit/detail/drain_body.cpp b/test/unit/detail/drain_body.cpp index b2d90d9..1ca094b 100644 --- a/test/unit/detail/drain_body.cpp +++ b/test/unit/detail/drain_body.cpp @@ -10,6 +10,8 @@ // Test that header file is self-contained. #include "src/detail/drain_body.hpp" +#include + #include "test_suite.hpp" #include @@ -39,7 +41,7 @@ class drain_body_test std::size_t max_read_size = std::size_t(-1)) { result rs; - response_parser pr({}, {}); + response_parser pr(response_parser::config{}); capy::test::fuse f; auto r = f.armed([&](capy::test::fuse&) -> capy::task<> { @@ -48,15 +50,16 @@ class drain_body_test server.provide(msg); server.close(); - pr.reset(&client); + pr.reset(); pr.start(); - if(auto [rec] = co_await pr.read_header(); rec) + if(auto [rec] = co_await message_reader{ + &client, &pr }.read_header(); rec) co_return; BOOST_TEST(pr.got_header()); - auto [dec, drained] = co_await drain_body(pr, attempts); + auto [dec, drained] = co_await drain_body(client, pr, attempts); if(dec) BOOST_TEST(!drained); rs = { dec, drained, pr.got_body() }; diff --git a/test/unit/detail/parser.cpp b/test/unit/parser.cpp similarity index 75% rename from test/unit/detail/parser.cpp rename to test/unit/parser.cpp index e315a07..6532303 100644 --- a/test/unit/detail/parser.cpp +++ b/test/unit/parser.cpp @@ -8,17 +8,21 @@ // // Test that header file is self-contained. -#include +#include #include +#include #include +#include #include #include #include +#include #include #include +#include #include "test_suite.hpp" @@ -26,17 +30,19 @@ namespace boost { namespace burl { -namespace detail -{ // parser has protected members (it is a base for the request/response -// parsers); this shim exposes them so the base can be exercised directly. +// parsers) and performs no I/O. This shim exposes the protected members +// and binds a stream, so that the sans-io base can be exercised through +// the same spellings a @ref message_reader offers. struct test_parser : parser { test_parser( config const& cfg, - capy::any_read_stream* stream = nullptr) - : parser(cfg, false, stream) + capy::any_read_stream* stream = nullptr, + bool is_request = false) + : parser(cfg, is_request) + , stream_(stream) { } @@ -51,6 +57,59 @@ struct test_parser : parser { return get_response(); } + + auto + read_header() + { + return reader().read_header(); + } + + auto + read_body() + { + return reader().read_body(); + } + + template + auto + read_some(MB buffers) + { + return reader().read_some(std::move(buffers)); + } + + template + auto + read(MB buffers) + { + return reader().read(std::move(buffers)); + } + + auto + pull(std::span dest) + { + return reader().pull(dest); + } + + // The stream is bound at the reader now, so rebinding is the shim's + // job rather than the parser's. + void + reset(capy::any_read_stream* stream) noexcept + { + parser::reset(); + stream_ = stream; + } + + using parser::consume; + using parser::reset; + +private: + message_reader + reader() noexcept + { + return { stream_, this }; + } + + capy::any_read_stream* stream_; }; class parser_test @@ -132,6 +191,71 @@ class parser_test std::size_t trailer_pos_ = 0; }; + // A stream that reports eof alongside the octets which complete the + // transfer, rather than on a following empty read. Real transports do + // this; capy::test::read_stream never does, so the case that a read + // delivers data and a contingency at once needs its own double. + class eager_eof_stream + { + std::string data_; + std::size_t pos_ = 0; + std::size_t max_read_size_; + + public: + explicit + eager_eof_stream( + std::string data, + std::size_t max_read_size = std::size_t(-1)) + : data_(std::move(data)) + , max_read_size_(max_read_size) + { + } + + template + auto + read_some(MB buffers) + { + struct awaitable + { + eager_eof_stream* self_; + MB buffers_; + + bool + await_ready() const noexcept + { + return true; + } + + void + await_suspend( + std::coroutine_handle<>, + capy::io_env const*) const noexcept + { + } + + capy::io_result + await_resume() + { + auto avail = self_->data_.size() - self_->pos_; + if(avail == 0) + return { capy::error::eof, 0 }; + if(avail > self_->max_read_size_) + avail = self_->max_read_size_; + auto const n = capy::buffer_copy( + buffers_, + capy::make_buffer( + self_->data_.data() + self_->pos_, avail)); + self_->pos_ += n; + // eof accompanies the last octets + if(self_->pos_ == self_->data_.size()) + return { capy::error::eof, n }; + return { {}, n }; + } + }; + return awaitable{ this, buffers }; + } + }; + // A stream that reports a scripted error a number of times before // delegating to the underlying test stream; models a timed-out or // hard-failed read at the transport. @@ -328,6 +452,31 @@ class parser_test }()); } + void + testRequestBadTransferEncoding() + { + capy::test::read_stream server; + capy::any_read_stream stream(&server); + test_parser pr({}, &stream, true); + + capy::test::run_blocking()([&]() -> capy::task<> + { + // a request whose Transfer-Encoding does not end in + // chunked has no defined length; unlike a response it + // has no to-eof fallback, so the header is rejected + server.provide( + "PUT / HTTP/1.1\r\n" + "Transfer-Encoding: gzip\r\n" + "\r\n"); + + pr.start(); + auto [ec] = co_await pr.read_header(); + BOOST_TEST(ec == http::error::bad_transfer_encoding); + BOOST_TEST(!pr.got_header()); + BOOST_TEST(!pr.got_body()); + }()); + } + void testEndOfStream() { @@ -736,6 +885,32 @@ class parser_test }()); } + void + testSizedPullBodyLimitZero() + { + // a zero budget fails the pull before any delivery + parser::config cfg; + cfg.body_limit = 0; + capy::test::read_stream server; + capy::any_read_stream stream(&server); + test_parser pr(cfg, &stream); + + capy::test::run_blocking()([&]() -> capy::task<> + { + server.provide( + "HTTP/1.1 200 OK\r\n" + "Content-Length: 5\r\n" + "\r\n" + "hello"); + + pr.start(); + capy::const_buffer arr[2]; + auto [ec, bufs] = co_await pr.pull(arr); + BOOST_TEST(ec == http::error::body_too_large); + BOOST_TEST_EQ(bufs.size(), 0); + }()); + } + void testSizedMixedStreamThenView() { @@ -1079,6 +1254,50 @@ class parser_test }()); } + void + testChunkedPullSmallDest() + { + capy::test::read_stream server; + capy::any_read_stream stream(&server); + test_parser pr({}, &stream); + + capy::test::run_blocking()([&]() -> capy::task<> + { + server.provide( + "HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n" + "3\r\nabc\r\n" + "3\r\ndef\r\n" + "0\r\n\r\n"); + + pr.start(); + // a single descriptor cannot span chunks: delivery + // stops at the boundary and resumes on the next pull + capy::const_buffer arr[1]; + { + auto [ec, bufs] = co_await pr.pull(arr); + BOOST_TEST(!ec); + BOOST_TEST_EQ(bufs.size(), 1); + BOOST_TEST_EQ(bufs[0].size(), 3); + } + pr.consume(3); + { + auto [ec, bufs] = co_await pr.pull(arr); + BOOST_TEST(!ec); + BOOST_TEST_EQ(bufs.size(), 1); + BOOST_TEST_EQ(bufs[0].size(), 3); + } + pr.consume(3); + { + auto [ec, bufs] = co_await pr.pull(arr); + BOOST_TEST(ec == capy::cond::eof); + BOOST_TEST_EQ(bufs.size(), 0); + } + BOOST_TEST(pr.got_body()); + }()); + } + void testChunkedBadFraming() { @@ -1140,6 +1359,114 @@ class parser_test }()); } + void + testChunkedBadChunkExtension() + { + capy::test::read_stream server; + capy::any_read_stream stream(&server); + test_parser pr({}, &stream); + + capy::test::run_blocking()([&]() -> capy::task<> + { + // a lone CR inside a chunk extension + server.provide( + "HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n" + "5;ext\rZ\r\nhello\r\n" + "0\r\n\r\n"); + + pr.start(); + char buf[16]; + auto [ec, n] = co_await pr.read_some(capy::make_buffer(buf)); + BOOST_TEST(ec == http::error::bad_payload); + BOOST_TEST_EQ(n, 0); + }()); + } + + void + testChunkedSizeOverflow() + { + capy::test::read_stream server; + capy::any_read_stream stream(&server); + test_parser pr({}, &stream); + + capy::test::run_blocking()([&]() -> capy::task<> + { + // a chunk size that does not fit in 64 bits + server.provide( + "HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n" + "FFFFFFFFFFFFFFFFF\r\nhello\r\n" + "0\r\n\r\n"); + + pr.start(); + char buf[16]; + auto [ec, n] = co_await pr.read_some(capy::make_buffer(buf)); + BOOST_TEST(ec == http::error::bad_payload); + BOOST_TEST_EQ(n, 0); + }()); + } + + void + testChunkedBadChunkTerminator() + { + // the CRLF closing the chunk data is malformed in two + // ways: no CR at all, and a CR followed by the wrong octet + for(std::string_view tail : { "XY", "\rX" }) + { + capy::test::read_stream server; + capy::any_read_stream stream(&server); + test_parser pr({}, &stream); + + capy::test::run_blocking()([&]() -> capy::task<> + { + server.provide( + "HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n" + "5\r\nhello" + std::string(tail)); + + pr.start(); + char buf[16]; + // the valid chunk data is delivered first + auto [ec, n] = co_await pr.read_some(capy::make_buffer(buf)); + BOOST_TEST(!ec); + BOOST_TEST_EQ(n, 5); + + auto [ec2, n2] = co_await pr.read_some(capy::make_buffer(buf)); + BOOST_TEST(ec2 == http::error::bad_payload); + BOOST_TEST_EQ(n2, 0); + }()); + } + } + + void + testChunkedBadTrailer() + { + capy::test::read_stream server; + capy::any_read_stream stream(&server); + test_parser pr({}, &stream); + + capy::test::run_blocking()([&]() -> capy::task<> + { + // a lone CR inside the trailer section + server.provide( + "HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n" + "0\r\n\rX"); + + pr.start(); + char buf[16]; + auto [ec, n] = co_await pr.read_some(capy::make_buffer(buf)); + BOOST_TEST(ec == http::error::bad_payload); + BOOST_TEST_EQ(n, 0); + BOOST_TEST(!pr.got_body()); + }()); + } + void testChunkedPullDrainThenError() { @@ -1255,7 +1582,31 @@ class parser_test } void - testChunkedBodyLimit() + testChunkedReadBodyChunkLargerThanBuffer() + { + capy::test::read_stream server; + capy::any_read_stream stream(&server); + test_parser pr({}, &stream); + + capy::test::run_blocking()([&]() -> capy::task<> + { + // the chunk declares more octets than in_ can ever + // hold: assembling a contiguous body cannot succeed + server.provide( + "HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n" + "FFFFFF\r\nhello"); + + pr.start(); + auto [ec, body] = co_await pr.read_body(); + BOOST_TEST(ec == http::error::in_place_overflow); + BOOST_TEST(body.empty()); + }()); + } + + void + testChunkedBodyLimitViaReadBody() { parser::config cfg; cfg.body_limit = 4; @@ -1273,20 +1624,16 @@ class parser_test "0\r\n\r\n"); pr.start(); - char buf[16]; - // the in-limit prefix is delivered first - auto [ec, n] = co_await pr.read_some(capy::make_buffer(buf)); - BOOST_TEST(!ec); - BOOST_TEST_EQ(n, 4); - - auto [ec2, n2] = co_await pr.read_some(capy::make_buffer(buf)); - BOOST_TEST(ec2 == http::error::body_too_large); - BOOST_TEST_EQ(n2, 0); + // the body can never be returned whole; read_body + // yields an empty view alongside the error + auto [ec, body] = co_await pr.read_body(); + BOOST_TEST(ec == http::error::body_too_large); + BOOST_TEST(body.empty()); }()); } void - testChunkedBodyLimitViaConsume() + testChunkedBodyLimit() { parser::config cfg; cfg.body_limit = 4; @@ -1304,11 +1651,42 @@ class parser_test "0\r\n\r\n"); pr.start(); - capy::const_buffer arr[2]; - // delivery is clamped at the limit: the caller never - // observes body octets past it - auto [ec, bufs] = co_await pr.pull(arr); - BOOST_TEST(!ec); + char buf[16]; + // the in-limit prefix is delivered first + auto [ec, n] = co_await pr.read_some(capy::make_buffer(buf)); + BOOST_TEST(!ec); + BOOST_TEST_EQ(n, 4); + + auto [ec2, n2] = co_await pr.read_some(capy::make_buffer(buf)); + BOOST_TEST(ec2 == http::error::body_too_large); + BOOST_TEST_EQ(n2, 0); + }()); + } + + void + testChunkedBodyLimitViaConsume() + { + parser::config cfg; + cfg.body_limit = 4; + capy::test::read_stream server; + capy::any_read_stream stream(&server); + test_parser pr(cfg, &stream); + + capy::test::run_blocking()([&]() -> capy::task<> + { + server.provide( + "HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n" + "a\r\n0123456789\r\n" + "0\r\n\r\n"); + + pr.start(); + capy::const_buffer arr[2]; + // delivery is clamped at the limit: the caller never + // observes body octets past it + auto [ec, bufs] = co_await pr.pull(arr); + BOOST_TEST(!ec); BOOST_TEST_EQ(capy::buffer_size(bufs), 4); pr.consume(4); @@ -1579,6 +1957,54 @@ class parser_test // //-------------------------------------------- + void + testToEofBodyLimitViaReadBody() + { + parser::config cfg; + cfg.body_limit = 4; + capy::test::read_stream server; + capy::any_read_stream stream(&server); + test_parser pr(cfg, &stream); + + capy::test::run_blocking()([&]() -> capy::task<> + { + server.provide( + "HTTP/1.1 200 OK\r\n" + "\r\n" + "hello"); + + pr.start(); + auto [ec, body] = co_await pr.read_body(); + BOOST_TEST(ec == http::error::body_too_large); + BOOST_TEST(body.empty()); + }()); + } + + void + testToEofPullBodyLimitZero() + { + // a zero budget fails the pull before any delivery + parser::config cfg; + cfg.body_limit = 0; + capy::test::read_stream server; + capy::any_read_stream stream(&server); + test_parser pr(cfg, &stream); + + capy::test::run_blocking()([&]() -> capy::task<> + { + server.provide( + "HTTP/1.1 200 OK\r\n" + "\r\n" + "hello"); + + pr.start(); + capy::const_buffer arr[2]; + auto [ec, bufs] = co_await pr.pull(arr); + BOOST_TEST(ec == http::error::body_too_large); + BOOST_TEST_EQ(bufs.size(), 0); + }()); + } + void testSetBodyLimitEnforced() { @@ -2089,6 +2515,183 @@ class parser_test }()); } + void + testDecoderNoPayload() + { + capy::test::read_stream server; + capy::any_read_stream stream(&server); + test_parser pr({}, &stream); + test_decoder dec; + + capy::test::run_blocking()([&]() -> capy::task<> + { + server.provide( + "HTTP/1.1 200 OK\r\n" + "Content-Length: 0\r\n" + "\r\n"); + + pr.start(); + auto [hec] = co_await pr.read_header(); + BOOST_TEST(!hec); + pr.set_decoder(&dec); + + // a bodiless message ends the decode path without + // ever invoking the decoder + char buf[8]; + auto [ec, n] = co_await pr.read_some(capy::make_buffer(buf)); + BOOST_TEST(ec == capy::cond::eof); + BOOST_TEST_EQ(n, 0); + BOOST_TEST(!dec.finished); + }()); + } + + void + testDecoderReadSomeEmptyBuffer() + { + capy::test::read_stream server; + capy::any_read_stream stream(&server); + test_parser pr({}, &stream); + test_decoder dec; + + capy::test::run_blocking()([&]() -> capy::task<> + { + server.provide( + "HTTP/1.1 200 OK\r\n" + "Content-Length: 5\r\n" + "\r\n" + "hello"); + + pr.start(); + auto [hec] = co_await pr.read_header(); + BOOST_TEST(!hec); + pr.set_decoder(&dec); + + // an empty destination transfers nothing and is + // not an error + { + auto [ec, n] = co_await pr.read_some( + capy::mutable_buffer{}); + BOOST_TEST(!ec); + BOOST_TEST_EQ(n, 0); + } + char buf[16]; + { + auto [ec, n] = co_await pr.read_some(capy::make_buffer(buf)); + BOOST_TEST(!ec); + BOOST_TEST_EQ(n, 5); + BOOST_TEST(std::string_view(buf, n) == decoded("hello")); + } + }()); + } + + void + testDecoderBodyAfterStreaming() + { + capy::test::read_stream server; + capy::any_read_stream stream(&server); + test_parser pr({}, &stream); + test_decoder dec; + + capy::test::run_blocking()([&]() -> capy::task<> + { + server.provide( + "HTTP/1.1 200 OK\r\n" + "Content-Length: 5\r\n" + "\r\n" + "hello"); + + pr.start(); + auto [hec] = co_await pr.read_header(); + BOOST_TEST(!hec); + pr.set_decoder(&dec); + + char buf[3]; + { + auto [ec, n] = co_await pr.read_some(capy::make_buffer(buf)); + BOOST_TEST(!ec); + BOOST_TEST_EQ(n, 3); + } + // read_body cannot reconstruct a body whose leading + // octets were already streamed out + auto [ec, body] = co_await pr.read_body(); + BOOST_TEST(ec == http::error::incomplete); + BOOST_TEST(body.empty()); + }()); + } + + void + testDecoderReadBodyHardError() + { + capy::test::read_stream server; + capy::any_read_stream stream(&server); + test_parser pr({}, &stream); + test_decoder dec; + dec.fail_ec = capy::error::test_failure; + dec.fail_at = 3; + + capy::test::run_blocking()([&]() -> capy::task<> + { + server.provide( + "HTTP/1.1 200 OK\r\n" + "Content-Length: 5\r\n" + "\r\n" + "hello"); + + pr.start(); + auto [hec] = co_await pr.read_header(); + BOOST_TEST(!hec); + pr.set_decoder(&dec); + + // the in-place body cannot be completed + auto [ec, body] = co_await pr.read_body(); + BOOST_TEST(ec == capy::error::test_failure); + BOOST_TEST(body.empty()); + }()); + } + + void + testDecoderPullTwiceWithoutConsume() + { + capy::test::read_stream server; + capy::any_read_stream stream(&server); + test_parser pr({}, &stream); + test_decoder dec; + + capy::test::run_blocking()([&]() -> capy::task<> + { + server.provide( + "HTTP/1.1 200 OK\r\n" + "Content-Length: 5\r\n" + "\r\n" + "hello"); + + pr.start(); + auto [hec] = co_await pr.read_header(); + BOOST_TEST(!hec); + pr.set_decoder(&dec); + + capy::const_buffer arr[2]; + { + auto [ec, bufs] = co_await pr.pull(arr); + BOOST_TEST(!ec); + BOOST_TEST_EQ(capy::buffer_size(bufs), 5); + } + // without a consume the same octets are served again, + // straight from the decoded buffer + { + auto [ec, bufs] = co_await pr.pull(arr); + BOOST_TEST(!ec); + BOOST_TEST_EQ(capy::buffer_size(bufs), 5); + } + pr.consume(5); + { + auto [ec, bufs] = co_await pr.pull(arr); + BOOST_TEST(ec == capy::cond::eof); + BOOST_TEST_EQ(bufs.size(), 0); + } + }()); + } + void testDecoderChunked() { @@ -2166,6 +2769,41 @@ class parser_test }()); } + void + testDecoderChunkedIncomplete() + { + capy::test::read_stream server; + capy::any_read_stream stream(&server); + test_parser pr({}, &stream); + test_decoder dec; + + capy::test::run_blocking()([&]() -> capy::task<> + { + // the stream ends mid-chunk + server.provide( + "HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n" + "5\r\nab"); + + pr.start(); + auto [hec] = co_await pr.read_header(); + BOOST_TEST(!hec); + pr.set_decoder(&dec); + + char buf[16]; + // the cleanly decoded bytes are delivered first + auto [ec, n] = co_await pr.read_some(capy::make_buffer(buf)); + BOOST_TEST(!ec); + BOOST_TEST_EQ(n, 2); + BOOST_TEST(std::string_view(buf, n) == decoded("ab")); + + auto [ec2, n2] = co_await pr.read_some(capy::make_buffer(buf)); + BOOST_TEST(ec2 == http::error::incomplete); + BOOST_TEST_EQ(n2, 0); + }()); + } + void testDecoderToEof() { @@ -2858,6 +3496,290 @@ class parser_test }()); } + //-------------------------------------------- + // + // sans-io surface + // + //-------------------------------------------- + + void + testDirectCapacity() + { + // The pass-through budget is what lets a body be received straight + // into caller memory. It must be offered exactly when that is safe. + { + capy::test::read_stream server; + capy::any_read_stream stream(&server); + test_parser pr({}, &stream); + + capy::test::run_blocking()([&]() -> capy::task<> + { + server.provide( + "HTTP/1.1 200 OK\r\n" + "Content-Length: 5\r\n" + "\r\n"); + + pr.start(); + auto [ec] = co_await pr.read_header(); + BOOST_TEST(!ec); + // nothing buffered, so the whole payload may go direct + BOOST_TEST_EQ(pr.direct_capacity(), 5u); + + // a lowered limit clamps the budget + pr.set_body_limit(3); + BOOST_TEST_EQ(pr.direct_capacity(), 3u); + pr.set_body_limit(std::uint64_t(-1)); + }()); + } + { + // octets which arrived with the header must be drained first + capy::test::read_stream server; + capy::any_read_stream stream(&server); + test_parser pr({}, &stream); + + capy::test::run_blocking()([&]() -> capy::task<> + { + server.provide( + "HTTP/1.1 200 OK\r\n" + "Content-Length: 5\r\n" + "\r\n" + "he"); + + pr.start(); + auto [ec] = co_await pr.read_header(); + BOOST_TEST(!ec); + BOOST_TEST_EQ(pr.direct_capacity(), 0u); + + char buf[8]; + auto [rec, n] = co_await pr.read_some( + capy::make_buffer(buf)); + BOOST_TEST(!rec); + BOOST_TEST_EQ(n, 2); + + // buffer drained: the remainder may go direct + BOOST_TEST_EQ(pr.direct_capacity(), 3u); + }()); + } + { + // a decoder has to see the octets, so nothing may bypass it + capy::test::read_stream server; + capy::any_read_stream stream(&server); + test_parser pr({}, &stream); + test_decoder dec; + + capy::test::run_blocking()([&]() -> capy::task<> + { + server.provide( + "HTTP/1.1 200 OK\r\n" + "Content-Length: 5\r\n" + "\r\n"); + + pr.start(); + auto [ec] = co_await pr.read_header(); + BOOST_TEST(!ec); + pr.set_decoder(&dec); + BOOST_TEST_EQ(pr.direct_capacity(), 0u); + }()); + } + { + // chunked framing has to be walked, so it cannot go direct + capy::test::read_stream server; + capy::any_read_stream stream(&server); + test_parser pr({}, &stream); + + capy::test::run_blocking()([&]() -> capy::task<> + { + server.provide( + "HTTP/1.1 200 OK\r\n" + "Transfer-Encoding: chunked\r\n" + "\r\n"); + + pr.start(); + auto [ec] = co_await pr.read_header(); + BOOST_TEST(!ec); + BOOST_TEST_EQ(pr.direct_capacity(), 0u); + }()); + } + } + + void + testDirectReadBypassesBuffer() + { + // With a buffer far smaller than the body, a single read_some can + // only deliver more than the buffer holds by reading straight into + // the caller's memory. + capy::test::read_stream server; + capy::any_read_stream stream(&server); + auto const body = make_body(4096); + test_parser pr( + { + .hdr_limits = { .max_size = 128, .max_fields = 4 }, + .in_buffer = 64 + }, + &stream); + + capy::test::run_blocking()([&]() -> capy::task<> + { + server.provide( + "HTTP/1.1 200 OK\r\n" + "Content-Length: 4096\r\n" + "\r\n"); + + pr.start(); + auto [ec] = co_await pr.read_header(); + BOOST_TEST(!ec); + + // the body arrives only after the header, so none of it is + // buffered and all of it is eligible for the direct path + server.provide(body); + + std::string dest(4096, '\0'); + auto [rec, n] = co_await pr.read_some( + capy::mutable_buffer(dest.data(), dest.size())); + BOOST_TEST(!rec); + BOOST_TEST_EQ(n, 4096); + BOOST_TEST(std::string_view(dest.data(), n) == body); + BOOST_TEST(pr.got_body()); + }()); + } + + void + testBufferedData() + { + { + // pipelined: the next message is visible once this one is done + capy::test::read_stream server; + capy::any_read_stream stream(&server); + test_parser pr({}, &stream); + + capy::test::run_blocking()([&]() -> capy::task<> + { + server.provide( + "HTTP/1.1 200 OK\r\n" + "Content-Length: 5\r\n" + "\r\n" + "hello" + "HTTP/1.1 204 No Content\r\n" + "\r\n"); + + pr.start(); + auto [ec] = co_await pr.read_header(); + BOOST_TEST(!ec); + + char buf[8]; + auto [rec, n] = co_await pr.read_some( + capy::make_buffer(buf)); + BOOST_TEST(!rec); + BOOST_TEST_EQ(n, 5); + + BOOST_TEST(pr.has_buffered_data()); + auto const bufs = pr.buffered_data(); + BOOST_TEST_EQ(capy::buffer_size(bufs), 27u); + BOOST_TEST(std::string_view( + static_cast(bufs[0].data()), + bufs[0].size()).starts_with("HTTP/1.1 204")); + }()); + } + { + // A response whose framing runs to the end of the stream hides + // its leftovers from has_buffered_data. This is the shape of a + // 200 answer to CONNECT: the caller knows there is no body even + // though the response alone cannot say so, and buffered_data is + // the only way to recover the octets which follow. + capy::test::read_stream server; + capy::any_read_stream stream(&server); + test_parser pr({}, &stream); + + capy::test::run_blocking()([&]() -> capy::task<> + { + server.provide( + "HTTP/1.1 200 Connection Established\r\n" + "\r\n" + "\x16\x03\x01"); + + pr.start(); + auto [ec] = co_await pr.read_header(); + BOOST_TEST(!ec); + BOOST_TEST(pr.get().payload() == + http::payload::to_eof); + + BOOST_TEST(!pr.has_buffered_data()); + auto const bufs = pr.buffered_data(); + BOOST_TEST_EQ(capy::buffer_size(bufs), 3u); + BOOST_TEST(std::string_view( + static_cast(bufs[0].data()), + bufs[0].size()) == "\x16\x03\x01"); + }()); + } + } + + void + testEofWithOctets() + { + // A read which delivers the last octets together with eof must not + // lose them: they are committed before the contingency is acted on, + // and reported before eof is. + { + eager_eof_stream server( + "HTTP/1.1 200 OK\r\n" + "\r\n" + "hello"); + test_parser pr({}, nullptr); + capy::any_read_stream stream(&server); + pr.reset(&stream); + + capy::test::run_blocking()([&]() -> capy::task<> + { + pr.start(); + auto [ec] = co_await pr.read_header(); + BOOST_TEST(!ec); + + std::string got; + char buf[16]; + for(;;) + { + auto [rec, n] = co_await pr.read_some( + capy::make_buffer(buf)); + got.append(buf, n); + if(rec) + { + // any contingency ends the read; only eof is + // expected here + BOOST_TEST(rec == capy::cond::eof); + break; + } + } + BOOST_TEST(got == "hello"); + BOOST_TEST(pr.got_body()); + }()); + } + { + // same, through a decoder, which forces every octet to travel + // via the parser's buffer rather than the pass-through path + eager_eof_stream server( + "HTTP/1.1 200 OK\r\n" + "Content-Length: 5\r\n" + "\r\n" + "hello"); + test_parser pr({}, nullptr); + capy::any_read_stream stream(&server); + pr.reset(&stream); + test_decoder dec; + + capy::test::run_blocking()([&]() -> capy::task<> + { + pr.start(); + auto [ec] = co_await pr.read_header(); + BOOST_TEST(!ec); + pr.set_decoder(&dec); + + auto [bec, body] = co_await pr.read_body(); + BOOST_TEST(!bec); + BOOST_TEST(body == decoded("hello")); + }()); + } + } + void run() { @@ -2865,9 +3787,10 @@ class parser_test testHeaderEagerComplete(); testHeaderSyntaxError(); testHeaderPayloadError(); + testRequestBadTransferEncoding(); testEndOfStream(); testIncompleteHeader(); - //testHeaderLargerThanBuffer(); + testHeaderLargerThanBuffer(); testSizedReadSome(); testSizedPullConsume(); @@ -2878,6 +3801,7 @@ class parser_test testSizedReadBodyIncomplete(); testSizedBodyLimit(); testSizedBodyLimitViaReadBody(); + testSizedPullBodyLimitZero(); testSizedMixedStreamThenView(); testSizedReadBodyOverflow(); @@ -2888,12 +3812,19 @@ class parser_test testChunkedReadBodySplitMidChunk(); testChunkedReadBodyPipelined(); testChunkedTrailersAndExtensions(); + testChunkedPullSmallDest(); testChunkedBadFraming(); testChunkedBadFramingWithData(); + testChunkedBadChunkExtension(); + testChunkedSizeOverflow(); + testChunkedBadChunkTerminator(); + testChunkedBadTrailer(); testChunkedPullDrainThenError(); testChunkedIncomplete(); testChunkedFramingLargerThanBuffer(); + testChunkedReadBodyChunkLargerThanBuffer(); testChunkedBodyLimit(); + testChunkedBodyLimitViaReadBody(); testChunkedBodyLimitViaConsume(); testChunkedByteByByte(); testChunkedReadBodyOverflowThenStream(); @@ -2903,6 +3834,8 @@ class parser_test testToEofPull(); testToEofEmptyBody(); testToEofBodyLimit(); + testToEofBodyLimitViaReadBody(); + testToEofPullBodyLimitZero(); testSetBodyLimitEnforced(); testSetBodyLimitRaiseUnblocks(); @@ -2919,8 +3852,14 @@ class parser_test testDecoderEarlyEof(); testDecoderHardError(); testDecoderPullServesDataBeforeError(); + testDecoderNoPayload(); + testDecoderReadSomeEmptyBuffer(); + testDecoderBodyAfterStreaming(); + testDecoderReadBodyHardError(); + testDecoderPullTwiceWithoutConsume(); testDecoderChunked(); testDecoderChunkedEarlyEof(); + testDecoderChunkedIncomplete(); testDecoderToEof(); testDecoderToEofTrailerSeparateRead(); testDecoderReadBody(); @@ -2938,11 +3877,14 @@ class parser_test testStartCompactsIncompleteMessage(); testStartRetriesOverflowAtParkedBase(); testStartSkipsUndeliveredRemainder(); + + testDirectCapacity(); + testDirectReadBypassesBuffer(); + testBufferedData(); + testEofWithOctets(); } }; -TEST_SUITE(parser_test, "boost.burl.detail.parser"); - -} // namespace detail +TEST_SUITE(parser_test, "boost.burl.parser"); } // namespace burl } // namespace boost diff --git a/test/unit/detail/request_parser.cpp b/test/unit/request_parser.cpp similarity index 76% rename from test/unit/detail/request_parser.cpp rename to test/unit/request_parser.cpp index ca7ec55..bcbfee0 100644 --- a/test/unit/detail/request_parser.cpp +++ b/test/unit/request_parser.cpp @@ -8,10 +8,12 @@ // // Test that header file is self-contained. -#include +#include #include +#include +#include #include #include "test_suite.hpp" @@ -22,9 +24,6 @@ namespace boost { namespace burl { -namespace detail -{ - class request_parser_test { public: @@ -33,7 +32,7 @@ class request_parser_test { capy::test::read_stream server; capy::any_read_stream stream(&server); - request_parser pr({}, &stream); + request_parser pr(request_parser::config{}); capy::test::run_blocking()([&]() -> capy::task<> { @@ -45,7 +44,7 @@ class request_parser_test "hello"); pr.start(); - auto [ec] = co_await pr.read_header(); + auto [ec] = co_await message_reader{ &stream, &pr }.read_header(); BOOST_TEST(!ec); BOOST_TEST(pr.got_header()); @@ -53,7 +52,8 @@ class request_parser_test BOOST_TEST(pr.get().target() == "/index.html"); char buf[8]; - auto [bec, n] = co_await pr.read(capy::make_buffer(buf)); + auto [bec, n] = co_await message_reader{ &stream, &pr } + .read(capy::make_buffer(buf)); BOOST_TEST(bec == capy::cond::eof); BOOST_TEST_EQ(n, 5); BOOST_TEST(std::string_view(buf, n) == "hello"); @@ -67,8 +67,6 @@ class request_parser_test } }; -TEST_SUITE(request_parser_test, "boost.burl.detail.request_parser"); - -} // namespace detail +TEST_SUITE(request_parser_test, "boost.burl.request_parser"); } // namespace burl } // namespace boost diff --git a/test/unit/detail/response_parser.cpp b/test/unit/response_parser.cpp similarity index 79% rename from test/unit/detail/response_parser.cpp rename to test/unit/response_parser.cpp index 72f4d95..ddc6d7f 100644 --- a/test/unit/detail/response_parser.cpp +++ b/test/unit/response_parser.cpp @@ -8,10 +8,12 @@ // // Test that header file is self-contained. -#include +#include #include +#include +#include #include #include "test_suite.hpp" @@ -22,9 +24,6 @@ namespace boost { namespace burl { -namespace detail -{ - class response_parser_test { public: @@ -33,7 +32,7 @@ class response_parser_test { capy::test::read_stream server; capy::any_read_stream stream(&server); - response_parser pr({}, &stream); + response_parser pr(response_parser::config{}); capy::test::run_blocking()([&]() -> capy::task<> { @@ -44,7 +43,7 @@ class response_parser_test "hello"); pr.start(); - auto [ec] = co_await pr.read_header(); + auto [ec] = co_await message_reader{ &stream, &pr }.read_header(); BOOST_TEST(!ec); BOOST_TEST(pr.got_header()); // the whole body arrived with the header, so the message @@ -56,7 +55,8 @@ class response_parser_test BOOST_TEST(pr.get().reason() == "OK"); char buf[8]; - auto [bec, n] = co_await pr.read(capy::make_buffer(buf)); + auto [bec, n] = co_await message_reader{ &stream, &pr } + .read(capy::make_buffer(buf)); BOOST_TEST(bec == capy::cond::eof); BOOST_TEST_EQ(n, 5); BOOST_TEST(std::string_view(buf, n) == "hello"); @@ -68,7 +68,7 @@ class response_parser_test { capy::test::read_stream server; capy::any_read_stream stream(&server); - response_parser pr({}, &stream); + response_parser pr(response_parser::config{}); capy::test::run_blocking()([&]() -> capy::task<> { @@ -80,7 +80,7 @@ class response_parser_test "\r\n"); pr.start(true); - auto [ec] = co_await pr.read_header(); + auto [ec] = co_await message_reader{ &stream, &pr }.read_header(); BOOST_TEST(!ec); BOOST_TEST(pr.got_header()); BOOST_TEST(pr.got_body()); @@ -96,8 +96,6 @@ class response_parser_test } }; -TEST_SUITE(response_parser_test, "boost.burl.detail.response_parser"); - -} // namespace detail +TEST_SUITE(response_parser_test, "boost.burl.response_parser"); } // namespace burl } // namespace boost