From 697e671f57072cbaf83b611b7bf205d5577750ca Mon Sep 17 00:00:00 2001 From: randomkang <550941794@qq.com> Date: Tue, 4 Aug 2026 00:47:49 +0800 Subject: [PATCH 1/2] MTU negotiation --- src/brpc/rdma/rdma_endpoint.cpp | 19 +++- src/brpc/rdma/rdma_endpoint.h | 7 ++ src/brpc/rdma/rdma_handshake.cpp | 19 ++++ src/brpc/rdma/rdma_handshake.h | 8 ++ src/brpc/rdma/rdma_handshake.proto | 10 ++ src/brpc/rdma/rdma_helper.cpp | 8 ++ src/brpc/rdma/rdma_helper.h | 3 + test/brpc_rdma_unittest.cpp | 176 +++++++++++++++++++++++++++++ 8 files changed, 249 insertions(+), 1 deletion(-) diff --git a/src/brpc/rdma/rdma_endpoint.cpp b/src/brpc/rdma/rdma_endpoint.cpp index 6c20ea3994..506dc9d7f4 100644 --- a/src/brpc/rdma/rdma_endpoint.cpp +++ b/src/brpc/rdma/rdma_endpoint.cpp @@ -1188,7 +1188,24 @@ int RdmaEndpoint::BringUpQp(const ParsedHello& remote, bool is_server) { } attr.qp_state = IBV_QPS_RTR; - attr.path_mtu = IBV_MTU_1024; // TODO: support more mtu in future + // MTU negotiation: use the peer-advertised MTU if available, otherwise + // fall back to the legacy default (IBV_MTU_1024). + // Server side: remote.path_mtu is the client's active MTU; + // we compute min(local, client) here. + // Client side: remote.path_mtu is already the server's negotiated + // min(local, client) and we use it as-is. + uint32_t negotiated_mtu = IBV_MTU_1024; + if (remote.path_mtu.has_value()) { + if (is_server) { + uint32_t local_mtu = GetRdmaActiveMtu(); + negotiated_mtu = std::min(local_mtu, *remote.path_mtu); + // Store the negotiated MTU for the server hello reply. + _outgoing_mtu = negotiated_mtu; + } else { + negotiated_mtu = *remote.path_mtu; + } + } + attr.path_mtu = static_cast(negotiated_mtu); attr.ah_attr.grh.dgid = remote.gid; attr.ah_attr.grh.flow_label = 0; attr.ah_attr.grh.sgid_index = GetRdmaGidIndex(); diff --git a/src/brpc/rdma/rdma_endpoint.h b/src/brpc/rdma/rdma_endpoint.h index 36e22ad28d..03c2f15ec5 100644 --- a/src/brpc/rdma/rdma_endpoint.h +++ b/src/brpc/rdma/rdma_endpoint.h @@ -281,6 +281,13 @@ friend int v3_wire::WriteV3Hello(RdmaEndpoint*, const RdmaHello&); // QP reached RTS (filled in BringUpQp). butil::optional _outgoing_ece; + // MTU payload to advertise in the next local hello: + // Client: the locally queried active MTU (filled + // before C_HELLO_SEND); + // Server: the negotiated MTU = min(local_active_mtu, client_mtu) + // (filled in BringUpQp). + butil::optional _outgoing_mtu; + // rdma resource RdmaResource* _resource; diff --git a/src/brpc/rdma/rdma_handshake.cpp b/src/brpc/rdma/rdma_handshake.cpp index 180c2b3f0b..2e5a566b71 100644 --- a/src/brpc/rdma/rdma_handshake.cpp +++ b/src/brpc/rdma/rdma_handshake.cpp @@ -318,6 +318,15 @@ void FillLocalRdmaHello(const RdmaEndpoint* ep, RdmaHello* msg) { ece->set_options(ep->_outgoing_ece->options); ece->set_comp_mask(ep->_outgoing_ece->comp_mask); } + + // Advertise MTU if the endpoint has a value to advertise. + // Client side: queried local active MTU (filled before C_HELLO_SEND). + // Server side: negotiated MTU = min(local_mtu, client_mtu) + // (filled in BringUpQp). + // nullopt -> omit the field (peer falls back to IBV_MTU_1024). + if (ep->_outgoing_mtu.has_value()) { + msg->set_mtu(*ep->_outgoing_mtu); + } } int ReadAndParseV3Hello(RdmaEndpoint* ep, RdmaHello* out) { @@ -380,6 +389,9 @@ void TranslateHello(const RdmaHello& msg, ParsedHello* out) { ece.comp_mask = msg.ece().comp_mask(); out->ece = ece; } + if (msg.has_mtu()) { + out->path_mtu = msg.mtu(); + } } } // namespace v3_wire @@ -400,6 +412,13 @@ int RdmaHandshakeClientV3::SendLocalHello() { } } + // Query local active MTU so it can be advertised in the client hello. + // Best-effort: any failure just means we won't advertise MTU + // (the peer falls back to IBV_MTU_1024). + if (!g_skip_rdma_init) { + _ep->_outgoing_mtu = GetRdmaActiveMtu(); + } + RdmaHello local_msg{}; v3_wire::FillLocalRdmaHello(_ep, &local_msg); return v3_wire::WriteV3Hello(_ep, local_msg); diff --git a/src/brpc/rdma/rdma_handshake.h b/src/brpc/rdma/rdma_handshake.h index 6238d424f0..d91dccf04f 100644 --- a/src/brpc/rdma/rdma_handshake.h +++ b/src/brpc/rdma/rdma_handshake.h @@ -53,6 +53,14 @@ struct ParsedHello { // - on the server side: the client's queried ECE capabilities; // - on the client side: the server's reduced/negotiated ECE. butil::optional ece; + + // MTU negotiation, v3 handshake only. + // nullopt means the peer did not advertise an MTU (v2 peer or older v3 + // peer that predates MTU negotiation). When engaged: + // - on the server side: the client's active MTU; + // - on the client side: the server's negotiated MTU + // (= min(local_active_mtu, client_mtu)). + butil::optional path_mtu; }; // Result of reading/parsing a peer's hello (see ReceiveAndParseRemoteHello). diff --git a/src/brpc/rdma/rdma_handshake.proto b/src/brpc/rdma/rdma_handshake.proto index b5627b2e9d..08491657ba 100644 --- a/src/brpc/rdma/rdma_handshake.proto +++ b/src/brpc/rdma/rdma_handshake.proto @@ -55,6 +55,16 @@ message RdmaHello { // Server hello: carries the REDUCED/negotiated ECE // queried after the QP reached RTS. optional RdmaEce ece = 7; + + // MTU negotiation (v3 only). + // Optional: carries the sender's active MTU (IBV_MTU_256 .. IBV_MTU_4096). + // Absent on v2 peers and on older v3 peers that predate MTU negotiation; + // the receiver then falls back to IBV_MTU_1024. + // + // Semantics differ by sender role: + // Client hello: the locally queried active MTU of the RDMA port. + // Server hello: the negotiated MTU = min(local_active_mtu, client_mtu). + optional uint32 mtu = 8; } // Mirrors struct ibv_ece { uint32 vendor_id; uint32 options; uint32 comp_mask; }. diff --git a/src/brpc/rdma/rdma_helper.cpp b/src/brpc/rdma/rdma_helper.cpp index b0e13ad72c..c598f3cad1 100644 --- a/src/brpc/rdma/rdma_helper.cpp +++ b/src/brpc/rdma/rdma_helper.cpp @@ -83,6 +83,7 @@ static int g_gid_tbl_len = 0; static uint8_t g_gid_index = 0; static ibv_gid g_gid; static uint16_t g_lid; +static uint32_t g_active_mtu = IBV_MTU_1024; static int g_max_sge = 0; static uint8_t g_port_num = 1; @@ -458,6 +459,7 @@ static ibv_context* OpenDevice(int num_total, int* num_available_devices) { ret_context = context.release(); g_gid_tbl_len = attr.gid_tbl_len; g_lid = attr.lid; + g_active_mtu = attr.active_mtu; } else { LOG(INFO) << "Device name not match: " << context->device->name << " vs " << FLAGS_rdma_device; @@ -467,6 +469,7 @@ static ibv_context* OpenDevice(int num_total, int* num_available_devices) { ret_context = context.release(); g_gid_tbl_len = attr.gid_tbl_len; g_lid = attr.lid; + g_active_mtu = attr.active_mtu; } } return ret_context; @@ -516,6 +519,7 @@ static void GlobalRdmaInitializeOrDieImpl() { LOG(INFO) << "RDMA device: " << g_context->device->name; } LOG(INFO) << "RDMA LID: " << g_lid; + LOG(INFO) << "RDMA Active MTU: " << g_active_mtu; if (!FindRdmaGid(g_context)) { LOG(ERROR) << "Fail to find available RDMA GID"; ExitWithError(); @@ -696,6 +700,10 @@ uint16_t GetRdmaLid() { return g_lid; } +uint32_t GetRdmaActiveMtu() { + return g_active_mtu; +} + uint8_t GetRdmaGidIndex() { return g_gid_index; } diff --git a/src/brpc/rdma/rdma_helper.h b/src/brpc/rdma/rdma_helper.h index 052763325b..fc69a31f9e 100644 --- a/src/brpc/rdma/rdma_helper.h +++ b/src/brpc/rdma/rdma_helper.h @@ -68,6 +68,9 @@ ibv_gid GetRdmaGid(); // Return Global LID uint16_t GetRdmaLid(); +// Return active MTU of the RDMA port (IBV_MTU_256 .. IBV_MTU_4096). +uint32_t GetRdmaActiveMtu(); + // Return suggested comp vector for CQ int GetRdmaCompVector(); diff --git a/test/brpc_rdma_unittest.cpp b/test/brpc_rdma_unittest.cpp index e30ae09f35..87852f7197 100644 --- a/test/brpc_rdma_unittest.cpp +++ b/test/brpc_rdma_unittest.cpp @@ -1919,6 +1919,182 @@ TEST_F(RdmaTest, v3_server_reply_has_no_ece_without_hw_negotiation) { StopServer(); } +// Build a valid v3 hello that also carries an MTU value. +rdma::RdmaHello MakeValidV3HelloWithMtu(uint32_t mtu) { + rdma::RdmaHello msg = MakeValidV3Hello(); + msg.set_mtu(mtu); + return msg; +} + +// A client hello carrying MTU must not break the server handshake: +// the server still parses the hello and advances to S_ACK_WAIT. +TEST_F(RdmaTest, v3_server_accepts_client_hello_with_mtu) { + StartServer(); + + sockaddr_in addr; + bzero((char*)&addr, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(PORT); + butil::fd_guard sockfd(socket(AF_INET, SOCK_STREAM, 0)); + ASSERT_TRUE(sockfd >= 0); + ASSERT_EQ(0, connect(sockfd, (sockaddr*)&addr, sizeof(sockaddr))); + usleep(100000); + Socket* s = GetSocketFromServer(0); + ASSERT_TRUE(s != NULL); + + rdma::RdmaHello msg = MakeValidV3HelloWithMtu(IBV_MTU_4096); + std::string packet = MakeV3Packet(msg); + ASSERT_EQ((ssize_t)packet.size(), + write(sockfd, packet.data(), packet.size())); + usleep(100000); + + ASSERT_EQ(rdma::RdmaEndpoint::S_ACK_WAIT, + static_cast(s->_transport.get())->_rdma_ep->_state); + + rdma::RdmaHello reply; + ReadServerV3Reply(sockfd, &reply); + + // ACK flags=0 -> clean FALLBACK_TCP so the test ends without hardware. + uint32_t flags = butil::HostToNet32(0); + ASSERT_EQ((ssize_t)sizeof(flags), write(sockfd, &flags, sizeof(flags))); + usleep(100000); + ASSERT_EQ(rdma::RdmaEndpoint::FALLBACK_TCP, + static_cast(s->_transport.get())->_rdma_ep->_state); + + sockfd.reset(-1); + usleep(100000); + ASSERT_EQ(NULL, GetSocketFromServer(0)); + StopServer(); +} + +// The server reply must carry a negotiated MTU when the client advertised one. +// Since UT skips real QP bring-up, the server computes min(local, client) and +// stores it in _outgoing_mtu; FillLocalRdmaHello then includes it in the reply. +TEST_F(RdmaTest, v3_server_reply_has_negotiated_mtu) { + StartServer(); + + sockaddr_in addr; + bzero((char*)&addr, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(PORT); + butil::fd_guard sockfd(socket(AF_INET, SOCK_STREAM, 0)); + ASSERT_TRUE(sockfd >= 0); + ASSERT_EQ(0, connect(sockfd, (sockaddr*)&addr, sizeof(sockaddr))); + usleep(100000); + Socket* s = GetSocketFromServer(0); + ASSERT_TRUE(s != NULL); + + // Advertise a non-default MTU so we can verify negotiation. + rdma::RdmaHello msg = MakeValidV3HelloWithMtu(IBV_MTU_4096); + std::string packet = MakeV3Packet(msg); + ASSERT_EQ((ssize_t)packet.size(), + write(sockfd, packet.data(), packet.size())); + usleep(100000); + + rdma::RdmaHello reply; + ReadServerV3Reply(sockfd, &reply); + + // The server should advertise a negotiated MTU in its reply. + EXPECT_TRUE(reply.has_mtu()); + // Negotiated MTU = min(local_active_mtu, client_mtu). Since + // g_skip_rdma_init is true in UT, local_mtu defaults to IBV_MTU_1024. + // So negotiated = min(1024, 4096) = 1024. + EXPECT_EQ(IBV_MTU_1024, reply.mtu()); + + uint32_t flags = butil::HostToNet32(0); + ASSERT_EQ((ssize_t)sizeof(flags), write(sockfd, &flags, sizeof(flags))); + usleep(100000); + + sockfd.reset(-1); + usleep(100000); + StopServer(); +} + +// When the client does NOT advertise an MTU (e.g., v2 peer or older v3 peer), +// the server must NOT include an MTU in its reply (backward-compatible +// degradation to legacy IBV_MTU_1024 default). +TEST_F(RdmaTest, v3_server_reply_has_no_mtu_without_client_mtu) { + StartServer(); + + sockaddr_in addr; + bzero((char*)&addr, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(PORT); + butil::fd_guard sockfd(socket(AF_INET, SOCK_STREAM, 0)); + ASSERT_TRUE(sockfd >= 0); + ASSERT_EQ(0, connect(sockfd, (sockaddr*)&addr, sizeof(sockaddr))); + usleep(100000); + Socket* s = GetSocketFromServer(0); + ASSERT_TRUE(s != NULL); + + // Client hello without MTU field. + rdma::RdmaHello msg = MakeValidV3Hello(); + std::string packet = MakeV3Packet(msg); + ASSERT_EQ((ssize_t)packet.size(), + write(sockfd, packet.data(), packet.size())); + usleep(100000); + + rdma::RdmaHello reply; + ReadServerV3Reply(sockfd, &reply); + + // Server must not advertise MTU when client didn't. + EXPECT_FALSE(reply.has_mtu()); + + uint32_t flags = butil::HostToNet32(0); + ASSERT_EQ((ssize_t)sizeof(flags), write(sockfd, &flags, sizeof(flags))); + usleep(100000); + + sockfd.reset(-1); + usleep(100000); + StopServer(); +} + +// Verify the client includes its local MTU in the v3 hello. +TEST_F(RdmaTest, v3_client_hello_includes_mtu) { + HandshakeVersionFlag _hsv(3); + + butil::fd_guard sockfd(butil::tcp_listen(g_ep)); + EXPECT_TRUE(sockfd >= 0); + + Channel channel; + ChannelOptions chan_options; + chan_options.socket_mode = SOCKET_MODE_RDMA; + chan_options.connect_timeout_ms = 500; + chan_options.timeout_ms = 500; + chan_options.max_retry = 0; + ASSERT_EQ(0, channel.Init(g_ep, &chan_options)); + + Controller cntl; + test::EchoRequest req; + test::EchoResponse res; + req.set_message(__FUNCTION__); + google::protobuf::Closure* done = DoNothing(); + ::test::EchoService::Stub(&channel).Echo(&cntl, &req, &res, done); + + butil::fd_guard acc_fd(accept(sockfd, NULL, NULL)); + ASSERT_TRUE(acc_fd >= 0); + + // Read 4B magic + 4B pb_size + body from the client hello. + uint8_t hdr[8]; + ASSERT_EQ(8, read(acc_fd, hdr, 8)); + ASSERT_EQ(0, memcmp(hdr, "RDM3", 4)); + uint32_t pb_size = butil::NetToHost32(*reinterpret_cast(hdr + 4)); + ASSERT_GT(pb_size, 0u); + ASSERT_LE(pb_size, 4096u); + std::string body(pb_size, '\0'); + ASSERT_EQ((ssize_t)pb_size, read(acc_fd, &body[0], pb_size)); + + rdma::RdmaHello hello; + ASSERT_TRUE(hello.ParseFromString(body)); + + // In UT mode (g_skip_rdma_init=true), GetRdmaActiveMtu() returns the + // default IBV_MTU_1024, so the client hello should include mtu=1024. + EXPECT_TRUE(hello.has_mtu()); + EXPECT_EQ(IBV_MTU_1024, hello.mtu()); + + bthread_id_join(cntl.call_id()); +} + TEST_F(RdmaTest, try_global_disable_rdma) { StartServer(); rdma::g_rdma_available.store(false, butil::memory_order_relaxed); From edf36c2d6cb4d60f51314c5df3920eca18f0673f Mon Sep 17 00:00:00 2001 From: randomkang <550941794@qq.com> Date: Wed, 5 Aug 2026 00:03:25 +0800 Subject: [PATCH 2/2] fix --- src/brpc/rdma/rdma_endpoint.cpp | 43 ++++++++++++++++++------------ src/brpc/rdma/rdma_endpoint.h | 3 ++- src/brpc/rdma/rdma_handshake.cpp | 37 ++++++++++++++++++++----- src/brpc/rdma/rdma_handshake.proto | 4 +-- 4 files changed, 60 insertions(+), 27 deletions(-) diff --git a/src/brpc/rdma/rdma_endpoint.cpp b/src/brpc/rdma/rdma_endpoint.cpp index 506dc9d7f4..2d9175c6de 100644 --- a/src/brpc/rdma/rdma_endpoint.cpp +++ b/src/brpc/rdma/rdma_endpoint.cpp @@ -35,6 +35,7 @@ #include "brpc/rdma/rdma_handshake_constants.h" DECLARE_int32(task_group_ntags); +DECLARE_bool(rdma_mtu_negotiation); namespace brpc { namespace rdma { @@ -156,6 +157,7 @@ void RdmaEndpoint::Reset() { _state.store(UNINIT, butil::memory_order_relaxed); _handshake_version = 0; _outgoing_ece.reset(); + _outgoing_mtu.reset(); _resource = NULL; _send_cq_events = 0; _recv_cq_events = 0; @@ -1141,6 +1143,27 @@ int RdmaEndpoint::AllocateResources() { } int RdmaEndpoint::BringUpQp(const ParsedHello& remote, bool is_server) { + // MTU negotiation. This is pure integer arithmetic and does NOT depend on + // real hardware, so it runs even in UT mode (where the actual QP bring-up + // below is skipped via g_skip_rdma_init). Keeping it here ensures the + // server-side _outgoing_mtu is populated so the server hello advertises + // the negotiated MTU, which unit tests rely on. + // + // Server: remote.path_mtu is the client's active MTU; we compute + // min(local_active_mtu, client_mtu) and advertise it back. + // Client: remote.path_mtu is already the server's negotiated MTU; we + // just apply it to the QP. + uint32_t negotiated_mtu = IBV_MTU_1024; + if (FLAGS_rdma_mtu_negotiation && remote.path_mtu.has_value()) { + if (is_server) { + negotiated_mtu = std::min(GetRdmaActiveMtu(), *remote.path_mtu); + // Store the negotiated MTU for the server hello reply. + _outgoing_mtu = negotiated_mtu; + } else { + negotiated_mtu = *remote.path_mtu; + } + } + if (BAIDU_UNLIKELY(g_skip_rdma_init)) { // For UT return 0; @@ -1188,23 +1211,9 @@ int RdmaEndpoint::BringUpQp(const ParsedHello& remote, bool is_server) { } attr.qp_state = IBV_QPS_RTR; - // MTU negotiation: use the peer-advertised MTU if available, otherwise - // fall back to the legacy default (IBV_MTU_1024). - // Server side: remote.path_mtu is the client's active MTU; - // we compute min(local, client) here. - // Client side: remote.path_mtu is already the server's negotiated - // min(local, client) and we use it as-is. - uint32_t negotiated_mtu = IBV_MTU_1024; - if (remote.path_mtu.has_value()) { - if (is_server) { - uint32_t local_mtu = GetRdmaActiveMtu(); - negotiated_mtu = std::min(local_mtu, *remote.path_mtu); - // Store the negotiated MTU for the server hello reply. - _outgoing_mtu = negotiated_mtu; - } else { - negotiated_mtu = *remote.path_mtu; - } - } + // negotiated_mtu was computed at the top of this function: the server's + // min(local_active_mtu, client_mtu), or the client's echo of the server's + // advertised value. Falls back to IBV_MTU_1024 when no MTU was advertised. attr.path_mtu = static_cast(negotiated_mtu); attr.ah_attr.grh.dgid = remote.gid; attr.ah_attr.grh.flow_label = 0; diff --git a/src/brpc/rdma/rdma_endpoint.h b/src/brpc/rdma/rdma_endpoint.h index 03c2f15ec5..2c8608002f 100644 --- a/src/brpc/rdma/rdma_endpoint.h +++ b/src/brpc/rdma/rdma_endpoint.h @@ -281,7 +281,8 @@ friend int v3_wire::WriteV3Hello(RdmaEndpoint*, const RdmaHello&); // QP reached RTS (filled in BringUpQp). butil::optional _outgoing_ece; - // MTU payload to advertise in the next local hello: + // MTU payload to advertise in the next local hello. Populated only when + // FLAGS_rdma_mtu_negotiation is on: // Client: the locally queried active MTU (filled // before C_HELLO_SEND); // Server: the negotiated MTU = min(local_active_mtu, client_mtu) diff --git a/src/brpc/rdma/rdma_handshake.cpp b/src/brpc/rdma/rdma_handshake.cpp index 2e5a566b71..19e1ddeb59 100644 --- a/src/brpc/rdma/rdma_handshake.cpp +++ b/src/brpc/rdma/rdma_handshake.cpp @@ -57,6 +57,13 @@ DEFINE_bool(rdma_ece, false, "Enable end-to-end ECE (Enhanced Connection Establi "to no-ECE when the peer, the local libibverbs, or set_ece " "does not support it. Acts as a kill switch (default off)."); +DEFINE_bool(rdma_mtu_negotiation, true, + "Enable RDMA path-MTU negotiation in the v3 handshake. When on, " + "the two ends pick min(local_active_mtu, peer_active_mtu) instead " + "of the legacy fixed IBV_MTU_1024. Backward compatible: peers that " + "do not advertise an MTU (v2, or older v3) fall back to " + "IBV_MTU_1024. Acts as a kill switch (default on)."); + DECLARE_bool(rdma_trace_verbose); namespace v2_wire { @@ -324,7 +331,7 @@ void FillLocalRdmaHello(const RdmaEndpoint* ep, RdmaHello* msg) { // Server side: negotiated MTU = min(local_mtu, client_mtu) // (filled in BringUpQp). // nullopt -> omit the field (peer falls back to IBV_MTU_1024). - if (ep->_outgoing_mtu.has_value()) { + if (FLAGS_rdma_mtu_negotiation && ep->_outgoing_mtu.has_value()) { msg->set_mtu(*ep->_outgoing_mtu); } } @@ -389,8 +396,21 @@ void TranslateHello(const RdmaHello& msg, ParsedHello* out) { ece.comp_mask = msg.ece().comp_mask(); out->ece = ece; } - if (msg.has_mtu()) { - out->path_mtu = msg.mtu(); + if (FLAGS_rdma_mtu_negotiation && msg.has_mtu()) { + const uint32_t advertised = msg.mtu(); + // ibv_mtu only defines 256/512/1024/2048/4096. Reject any other value + // so a buggy/malicious peer cannot push an out-of-range enum through + // static_cast, which would make ibv_modify_qp fail (EINVAL) + // and force a silent TCP fallback. + if (advertised == IBV_MTU_256 || advertised == IBV_MTU_512 || + advertised == IBV_MTU_1024 || advertised == IBV_MTU_2048 || + advertised == IBV_MTU_4096) { + out->path_mtu = advertised; + } else { + LOG(WARNING) << "Peer advertised an invalid RDMA MTU (" + << advertised << "), ignoring it; falling back to " + << "IBV_MTU_1024"; + } } } @@ -412,10 +432,13 @@ int RdmaHandshakeClientV3::SendLocalHello() { } } - // Query local active MTU so it can be advertised in the client hello. - // Best-effort: any failure just means we won't advertise MTU - // (the peer falls back to IBV_MTU_1024). - if (!g_skip_rdma_init) { + // Advertise local active MTU in the client hello. GetRdmaActiveMtu() is a + // pure getter (always returns a valid enum value, defaulting to + // IBV_MTU_1024), so it is safe to call even in UT mode (no real RDMA + // hardware) and must NOT be gated on g_skip_rdma_init. + // Governed by FLAGS_rdma_mtu_negotiation: when off, the field is omitted + // and the peer falls back to IBV_MTU_1024. + if (FLAGS_rdma_mtu_negotiation) { _ep->_outgoing_mtu = GetRdmaActiveMtu(); } diff --git a/src/brpc/rdma/rdma_handshake.proto b/src/brpc/rdma/rdma_handshake.proto index 08491657ba..b72cec2186 100644 --- a/src/brpc/rdma/rdma_handshake.proto +++ b/src/brpc/rdma/rdma_handshake.proto @@ -56,9 +56,9 @@ message RdmaHello { // queried after the QP reached RTS. optional RdmaEce ece = 7; - // MTU negotiation (v3 only). + // MTU negotiation (v3 only). Toggled by FLAGS_rdma_mtu_negotiation. // Optional: carries the sender's active MTU (IBV_MTU_256 .. IBV_MTU_4096). - // Absent on v2 peers and on older v3 peers that predate MTU negotiation; + // Absent on v2 peers, older v3 peers, or when negotiation is disabled; // the receiver then falls back to IBV_MTU_1024. // // Semantics differ by sender role: