Contention-aware per-event send budget for QUICStream

include/iocore/net/quic/QUICStream.h

@@ -59,6 +59,13 @@ class QUICStream
 public:
   using ErrorCode = uint64_t; //!<  recv/send stream application error codes.
 
+  // Guaranteed per-stream send budget for one write event when many streams are
+  // contending for the connection's write path this round.
+  static constexpr size_t MIN_STREAM_SEND_BYTES_PER_EVENT = 16 * 1024;
+  // Total send budget for one connection's write event, divided across its writable
+  // streams by compute_fair_send_budget(). A lone writable stream gets it all.
+  static constexpr size_t MAX_CONNECTION_SEND_BYTES_PER_EVENT = 256 * 1024;
+
   QUICStream() {}
   QUICStream(QUICConnectionInfoProvider *cinfo, QUICStreamId sid);
   virtual ~QUICStream();
@@ -76,7 +83,13 @@ class QUICStream
   void reset(QUICStreamErrorUPtr error);
 
   void    receive_data(QUICStreamIO &stream_io);
-  int64_t send_data(QUICStreamIO &stream_io);
+  int64_t send_data(QUICStreamIO &stream_io, size_t max_bytes_this_event);
+
+  // Computes the per-stream send budget for one write event given how many streams
+  // are writable this event on this connection. Scales down toward
+  // MIN_STREAM_SEND_BYTES_PER_EVENT under contention, up toward
+  // MAX_CONNECTION_SEND_BYTES_PER_EVENT when a stream has the write path to itself.
+  static size_t compute_fair_send_budget(size_t num_writable_streams);
 
   /*
    * QUICApplication need to call one of these functions when it process VC_EVENT_*

include/proxy/http3/Http3Frame.h

@@ -63,19 +63,6 @@ class Http3Frame
   bool _is_ready = false;
 };
 
-class Http3UnknownFrame : public Http3Frame
-{
-public:
-  Http3UnknownFrame() : Http3Frame() {}
-  Http3UnknownFrame(IOBufferReader &reader);
-
-  Ptr<IOBufferBlock> to_io_buffer_block() const override;
-
-protected:
-  const uint8_t *_buf     = nullptr;
-  size_t         _buf_len = 0;
-};
-
 //
 // DATA Frame
 //
@@ -112,7 +99,11 @@ class Http3HeadersFrame : public Http3Frame
 public:
   Http3HeadersFrame() : Http3Frame() {}
   Http3HeadersFrame(IOBufferReader &reader);
-  Http3HeadersFrame(ats_unique_buf header_block, size_t header_block_len);
+  // Shares the caller's buffer via a cloned reader instead of copying header_block_len bytes.
+  // Safe as long as the source MIOBuffer outlives this frame, which holds for the qmux/quic
+  // write path: the frame is created, serialized via to_io_buffer_block(), and destroyed, all
+  // synchronously, well within the lifetime of the Http3HeaderFramer that owns the source buffer.
+  Http3HeadersFrame(IOBufferReader &header_block_reader, size_t header_block_len);
   ~Http3HeadersFrame();
 
   Ptr<IOBufferBlock> to_io_buffer_block() const override;
@@ -125,9 +116,9 @@ class Http3HeadersFrame : public Http3Frame
   bool _parse() override;
 
 private:
-  uint8_t       *_header_block      = nullptr;
-  ats_unique_buf _header_block_uptr = {nullptr};
-  size_t         _header_block_len  = 0;
+  uint8_t        *_header_block        = nullptr;
+  size_t          _header_block_len    = 0;
+  IOBufferReader *_header_block_reader = nullptr;
 };
 
 //
@@ -247,7 +238,6 @@ class Http3FrameFactory
   /*
    * Creates a HEADERS frame.
    */
-  static Http3HeadersFrameUPtr create_headers_frame(const uint8_t *header_block, size_t header_block_len);
   static Http3HeadersFrameUPtr create_headers_frame(IOBufferReader *header_block_reader, size_t header_block_len);
 
   /*

include/proxy/http3/Http3FrameCollector.h

@@ -33,6 +33,10 @@ class QUICStreamVCAdapter;
 class Http3FrameCollector
 {
 public:
+  // Http3Transaction always adds exactly 2 generators (header framer, data framer) per
+  // transaction; reserving avoids the growth-triggered reallocation on the second add_generator().
+  Http3FrameCollector() { _generators.reserve(2); }
+
   Http3ErrorUPtr on_write_ready(QUICStreamId stream_id, MIOBuffer &writer, size_t &nread, bool &all_done);
 
   void add_generator(Http3FrameGenerator *generator);

include/proxy/http3/Http3FrameCounter.h

@@ -32,7 +32,7 @@ class Http3FrameCounter : public Http3FrameHandler
   Http3FrameCounter(){};
 
   // Http3FrameHandler
-  std::vector<Http3FrameType> interests() override;
+  std::vector<Http3FrameType> const &interests() override;
   Http3ErrorUPtr handle_frame(std::shared_ptr<const Http3Frame> frame, Http3StreamType s_type = Http3StreamType::UNKNOWN) override;
 
   uint64_t get_count(uint64_t type) const;

include/proxy/http3/Http3FrameDispatcher.h

@@ -26,7 +26,8 @@
 #include "iocore/net/quic/QUICApplication.h"
 #include "proxy/http3/Http3Frame.h"
 #include "proxy/http3/Http3FrameHandler.h"
-#include <vector>
+#include <array>
+#include <cstdint>
 
 class QUICStreamVCAdapter;
 
@@ -38,17 +39,25 @@ class Http3FrameDispatcher
   void add_handler(Http3FrameHandler *handler);
 
 private:
+  // At most a handful of handlers ever register interest in the same frame type (currently
+  // up to 3: the frame counter, the protocol enforcer, and one of the header/data handlers).
+  // Inline storage avoids a heap allocation per handler registration, which otherwise runs
+  // once per HTTP/3 request since this dispatcher is a per-transaction object.
+  static constexpr size_t MAX_HANDLERS_PER_TYPE = 4;
+
   enum READING_STATE {
     READING_TYPE_LEN,
     READING_LENGTH_LEN,
     READING_PAYLOAD_LEN,
     READING_PAYLOAD,
   } _reading_state = READING_TYPE_LEN;
-  int64_t                          _reading_frame_type_len;
-  int64_t                          _reading_frame_length_len;
-  uint64_t                         _reading_frame_payload_len;
-  uint64_t                         _bytes_to_skip;
-  Http3FrameFactory                _frame_factory;
-  std::shared_ptr<Http3Frame>      _current_frame = nullptr;
-  std::vector<Http3FrameHandler *> _handlers[256];
+  int64_t                     _reading_frame_type_len;
+  int64_t                     _reading_frame_length_len;
+  uint64_t                    _reading_frame_payload_len;
+  uint64_t                    _bytes_to_skip;
+  Http3FrameFactory           _frame_factory;
+  std::shared_ptr<Http3Frame> _current_frame = nullptr;
+
+  std::array<Http3FrameHandler *, MAX_HANDLERS_PER_TYPE> _handlers[256]      = {};
+  uint8_t                                                _handler_count[256] = {};
 };

include/proxy/http3/Http3FrameHandler.h

@@ -31,7 +31,7 @@ class Http3FrameHandler
 {
 public:
   virtual ~Http3FrameHandler(){};
-  virtual std::vector<Http3FrameType> interests()                                                     = 0;
-  virtual Http3ErrorUPtr              handle_frame(std::shared_ptr<const Http3Frame> frame,
-                                                   Http3StreamType                   s_type = Http3StreamType::UNKNOWN) = 0;
+  virtual std::vector<Http3FrameType> const &interests()                                                     = 0;
+  virtual Http3ErrorUPtr                     handle_frame(std::shared_ptr<const Http3Frame> frame,
+                                                          Http3StreamType                   s_type = Http3StreamType::UNKNOWN) = 0;
 };

include/proxy/http3/Http3HeaderVIOAdaptor.h

@@ -35,7 +35,7 @@ class Http3HeaderVIOAdaptor : public Continuation, public Http3FrameHandler
   ~Http3HeaderVIOAdaptor();
 
   // Http3FrameHandler
-  std::vector<Http3FrameType> interests() override;
+  std::vector<Http3FrameType> const &interests() override;
   Http3ErrorUPtr handle_frame(std::shared_ptr<const Http3Frame> frame, Http3StreamType s_type = Http3StreamType::UNKNOWN) override;
 
   bool is_complete();

include/proxy/http3/Http3ProtocolEnforcer.h

@@ -32,7 +32,7 @@ class Http3ProtocolEnforcer : public Http3FrameHandler
   Http3ProtocolEnforcer(){};
 
   // Http3FrameHandler
-  std::vector<Http3FrameType> interests() override;
+  std::vector<Http3FrameType> const &interests() override;
   Http3ErrorUPtr handle_frame(std::shared_ptr<const Http3Frame> frame, Http3StreamType s_type = Http3StreamType::UNKNOWN) override;
 
 private:

include/proxy/http3/Http3SettingsHandler.h

@@ -32,7 +32,7 @@ class Http3SettingsHandler : public Http3FrameHandler
   Http3SettingsHandler(Http3Session *session) : _session(session){};
 
   // Http3FrameHandler
-  std::vector<Http3FrameType> interests() override;
+  std::vector<Http3FrameType> const &interests() override;
   Http3ErrorUPtr handle_frame(std::shared_ptr<const Http3Frame> frame, Http3StreamType s_type = Http3StreamType::UNKNOWN) override;
 
 private:

include/proxy/http3/Http3StreamDataVIOAdaptor.h

@@ -34,12 +34,12 @@ class Http3StreamDataVIOAdaptor : public Http3FrameHandler
   virtual ~Http3StreamDataVIOAdaptor();
 
   // Http3FrameHandler
-  std::vector<Http3FrameType> interests() override;
+  std::vector<Http3FrameType> const &interests() override;
   Http3ErrorUPtr handle_frame(std::shared_ptr<const Http3Frame> frame, Http3StreamType s_type = Http3StreamType::UNKNOWN) override;
 
   // Http3StreamDataVIOAdaptor
   void finalize();
-  bool has_data();
+  bool has_data() const;
 
 private:
   VIO            *_sink_vio          = nullptr;

include/proxy/http3/Http3Transaction.h

@@ -28,18 +28,18 @@
 #include "iocore/net/quic/QUICStreamVCAdapter.h"
 #include "proxy/http3/Http3FrameDispatcher.h"
 #include "proxy/http3/Http3FrameCollector.h"
+#include "proxy/http3/Http3HeaderFramer.h"
+#include "proxy/http3/Http3DataFramer.h"
+#include "proxy/http3/Http3ProtocolEnforcer.h"
+#include "proxy/http3/Http3HeaderVIOAdaptor.h"
+#include "proxy/http3/Http3StreamDataVIOAdaptor.h"
 
 #include <functional>
 
 class QUICStreamIO;
 class HQSession;
 class Http09Session;
 class Http3Session;
-class Http3HeaderFramer;
-class Http3DataFramer;
-class Http3HeaderVIOAdaptor;
-class Http3ProtocolEnforcer;
-class Http3StreamDataVIOAdaptor;
 
 class HQTransaction : public ProxyTransaction
 {
@@ -151,13 +151,13 @@ class Http3Transaction : public HQTransaction
   void    _handle_error(const Http3Error &error);
 
   // These are for HTTP/3
-  Http3FrameDispatcher       _frame_dispatcher;
-  Http3FrameCollector        _frame_collector;
-  Http3ProtocolEnforcer     *_protocol_enforcer = nullptr;
-  Http3HeaderFramer         *_header_framer     = nullptr;
-  Http3DataFramer           *_data_framer       = nullptr;
-  Http3HeaderVIOAdaptor     *_header_handler    = nullptr;
-  Http3StreamDataVIOAdaptor *_data_handler      = nullptr;
+  Http3FrameDispatcher      _frame_dispatcher;
+  Http3FrameCollector       _frame_collector;
+  Http3ProtocolEnforcer     _protocol_enforcer;
+  Http3HeaderFramer         _header_framer;
+  Http3DataFramer           _data_framer;
+  Http3HeaderVIOAdaptor     _header_handler;
+  Http3StreamDataVIOAdaptor _data_handler;
 };
 
 /**

src/iocore/net/CMakeLists.txt

@@ -159,6 +159,9 @@ if(BUILD_TESTING)
   if(TS_USE_QUIC)
     target_sources(test_net PRIVATE unit_tests/test_QUICTokenKeyConfig.cc)
   endif()
+  if(TS_USE_QUIC OR TS_USE_QMUX)
+    target_sources(test_net PRIVATE unit_tests/test_QUICStream.cc)
+  endif()
   # Use link groups to solve circular dependency
   set(LINK_GROUP_LIBS
       ts::logging

src/iocore/net/OpenSSLQUICNetVConnection.cc

@@ -749,9 +749,9 @@ QUICNetVConnection::_process_openssl_streams()
     }
     if ((stream_type & SSL_STREAM_TYPE_WRITE) != 0 || stream->has_data_to_send()) {
       if (stream->has_data_to_send()) {
-        while (stream->has_data_to_send() && stream->send_data(*this) > 0) {}
+        while (stream->has_data_to_send() && stream->send_data(*this, QUICStream::MIN_STREAM_SEND_BYTES_PER_EVENT) > 0) {}
       } else {
-        stream->send_data(*this);
+        stream->send_data(*this, QUICStream::MIN_STREAM_SEND_BYTES_PER_EVENT);
       }
     }
   }

src/iocore/net/QUICNetProcessor.cc

@@ -84,7 +84,11 @@ QUICNetProcessor::start(int, size_t /* stacksize ATS_UNUSED */)
   QUICCertConfig::startup();
   QUICConfig::scoped_config params;
 
-  if (dbg_ctl_vv_quiche.tag_on()) {
+  // tag_on() only checks the tag pattern, not whether debug output is globally
+  // enabled -- use on() so a tag that happens to match "vv_quiche" as a substring
+  // (e.g. "vv_quic") doesn't permanently install quiche's trace-level Rust logger
+  // regardless of proxy.config.diags.debug.enabled.
+  if (dbg_ctl_vv_quiche.on()) {
     quiche_enable_debug_logging(debug_log, NULL);
   }
   this->_quiche_config = quiche_config_new(QUICHE_PROTOCOL_VERSION);

src/iocore/net/QUICNetVConnection.cc

@@ -688,6 +688,23 @@ void
 QUICNetVConnection::_handle_write_ready()
 {
   if (quiche_conn_is_established(this->_quiche_con)) {
+    // Count real contention for THIS event before deciding its budget, rather than
+    // sizing it from a previous event's count -- a stale count can be wrong in either
+    // direction whenever contention swings between events, not just on the first
+    // event. writable() is a pure, side-effect-free snapshot (verified against
+    // quiche's source), so draining it twice costs one extra O(n) collect and n extra
+    // FFI calls, n bounded by this connection's stream limit -- cheap next to the
+    // per-stream work that follows.
+    quiche_stream_iter *probe          = quiche_conn_writable(this->_quiche_con);
+    uint64_t            probe_id       = 0;
+    size_t              writable_count = 0;
+    while (quiche_stream_iter_next(probe, &probe_id)) {
+      ++writable_count;
+    }
+    quiche_stream_iter_free(probe);
+
+    const size_t budget = QUICStream::compute_fair_send_budget(writable_count);
+
     quiche_stream_iter *writable = quiche_conn_writable(this->_quiche_con);
     uint64_t            s        = 0;
     while (quiche_stream_iter_next(writable, &s)) {
@@ -696,7 +713,7 @@ QUICNetVConnection::_handle_write_ready()
         [[maybe_unused]] QUICConnectionError err;
         stream = this->_stream_manager->create_stream(s, err);
       }
-      stream->send_data(*this);
+      stream->send_data(*this, budget);
     }
     quiche_stream_iter_free(writable);
   }

src/iocore/net/qmux/QMuxConnection.cc

@@ -355,13 +355,30 @@ QMuxConnection::_handle_write_streams()
     return;
   }
 
+  // Count real contention for THIS event before deciding its budget, rather than
+  // sizing it from a previous event's count -- a stale count can be wrong in either
+  // direction whenever contention swings between events, not just on the first
+  // event. writable() is a pure, side-effect-free snapshot (verified against quiche's
+  // source), so draining it twice costs one extra O(n) collect and n extra FFI calls,
+  // n bounded by this connection's stream limit -- cheap next to the per-stream work
+  // that follows.
+  quiche_stream_iter *probe = quiche_conn_writable(_quiche_con);
+  uint64_t            probe_id;
+  size_t              writable_count = 0;
+  while (quiche_stream_iter_next(probe, &probe_id)) {
+    ++writable_count;
+  }
+  quiche_stream_iter_free(probe);
+
+  const size_t budget = QUICStream::compute_fair_send_budget(writable_count);
+
   quiche_stream_iter *writable = quiche_conn_writable(_quiche_con);
   uint64_t            stream_id;
 
   while (quiche_stream_iter_next(writable, &stream_id)) {
     QUICStream *stream = _stream_manager->find_stream(stream_id);
     if (stream != nullptr) {
-      stream->send_data(*this);
+      stream->send_data(*this, budget);
     }
   }
   quiche_stream_iter_free(writable);