From 6a6aa5cd8af21f4465003512a0b46628319639a2 Mon Sep 17 00:00:00 2001 From: xulei25 Date: Mon, 21 Sep 2026 09:09:16 +0800 Subject: [PATCH 1/6] Add FlatBuffers RPC protocol support Add the optional fb_rpc transport with length-delimited metadata and stable method IDs, and integrate it with Channel, Controller and Server without changing the existing protobuf Protocol callback signatures. Keep request buffers immutable across retries and concurrent calls, and manage service ownership, asynchronous completion and concurrency limits. Reject invalid frames and unsupported authentication, compression, checksums and streams. The feature stays disabled by default in CMake, Make and Bazel. Add 33 protocol tests covering the wire format, service lifecycle, malformed input and real RPC round trips. --- BUILD.bazel | 5 +- CMakeLists.txt | 2 +- Makefile | 3 + docs/en/flatbuffers.md | 101 +- src/brpc/CMakeLists.txt | 2 + src/brpc/channel.cpp | 60 +- src/brpc/channel.h | 26 +- src/brpc/controller.cpp | 3 + src/brpc/controller.h | 15 + .../details/controller_private_accessor.h | 5 + src/brpc/details/server_private_accessor.h | 9 + src/brpc/global.cpp | 14 + src/brpc/options.proto | 1 + src/brpc/policy/flatbuffers_protocol.cpp | 463 +++++ src/brpc/policy/flatbuffers_protocol.h | 47 + src/brpc/server.cpp | 313 ++- src/brpc/server.h | 42 + test/BUILD.bazel | 21 +- test/CMakeLists.txt | 18 +- test/Makefile | 6 +- test/brpc_channel_unittest.cpp | 4 +- test/brpc_flatbuffers_protocol_unittest.cpp | 1825 +++++++++++++++++ tools/flatbuffers/README.md | 11 +- 23 files changed, 2953 insertions(+), 43 deletions(-) create mode 100644 src/brpc/policy/flatbuffers_protocol.cpp create mode 100644 src/brpc/policy/flatbuffers_protocol.h create mode 100644 test/brpc_flatbuffers_protocol_unittest.cpp diff --git a/BUILD.bazel b/BUILD.bazel index 5882a034da..67b2937b37 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -559,7 +559,10 @@ brpc_proto_library( visibility = ["//visibility:public"], ) -FLATBUFFERS_SRC_PATTERNS = ["src/brpc/flatbuffers/*.cpp"] +FLATBUFFERS_SRC_PATTERNS = [ + "src/brpc/flatbuffers/*.cpp", + "src/brpc/policy/flatbuffers_protocol.cpp", +] URMA_SRC_PATTERNS = [ "src/brpc/urma/*.cpp", diff --git a/CMakeLists.txt b/CMakeLists.txt index 0966595d13..48cce36cf0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,7 +23,7 @@ option(WITH_MESALINK "With MesaLink" OFF) option(WITH_BORINGSSL "With BoringSSL" OFF) option(WITH_DEBUG_SYMBOLS "With debug symbols" ON) option(WITH_THRIFT "With thrift framed protocol supported" OFF) -option(WITH_FLATBUFFERS "With FlatBuffers message support (headers only)" OFF) +option(WITH_FLATBUFFERS "With FlatBuffers message and RPC support (headers only)" OFF) option(WITH_BTHREAD_TRACER "With bthread tracer supported" OFF) option(WITH_SNAPPY "With snappy" OFF) option(WITH_RDMA "With RDMA" OFF) diff --git a/Makefile b/Makefile index 08df1f04e9..936299f363 100644 --- a/Makefile +++ b/Makefile @@ -216,6 +216,9 @@ BRPC_SOURCES_ALL = $(foreach d,$(BRPC_DIRS),$(wildcard $(addprefix $(d)/*,$(SRCE ifeq ($(URMA_USE_MOCK),0) BRPC_SOURCES_ALL := $(filter-out src/brpc/urma/mock_urma.cpp,$(BRPC_SOURCES_ALL)) endif +ifneq ($(WITH_FLATBUFFERS),1) +BRPC_SOURCES_ALL := $(filter-out src/brpc/policy/flatbuffers_protocol.cpp,$(BRPC_SOURCES_ALL)) +endif BRPC_SOURCES = $(filter-out $(THRIFT_SOURCES) $(EXCLUDE_SOURCES), $(BRPC_SOURCES_ALL)) BRPC_PROTOS = $(filter %.proto,$(BRPC_SOURCES)) BRPC_CFAMILIES = $(filter-out %.proto %.pb.cc,$(BRPC_SOURCES)) diff --git a/docs/en/flatbuffers.md b/docs/en/flatbuffers.md index 50748900ad..1b8a0f48bd 100644 --- a/docs/en/flatbuffers.md +++ b/docs/en/flatbuffers.md @@ -1,14 +1,12 @@ -# FlatBuffers messages +# FlatBuffers messages and RPC [中文版](../cn/flatbuffers.md) -bRPC supports IOBuf-backed FlatBuffers messages, builders, and service -descriptors, disabled by default. Message construction is based on -[apache/brpc#3196](https://github.com/apache/brpc/pull/3196). - -This component does not include `fb_rpc` transport or change `brpc::Channel` -or `brpc::Server`. Service generation and in-process dispatch tests do not -exercise network RPC or measure performance. +bRPC provides optional IOBuf-backed FlatBuffers messages, builders, service +descriptors, and the `fb_rpc` transport. The implementation builds on +[apache/brpc#3196](https://github.com/apache/brpc/pull/3196) and +[apache/brpc#3197](https://github.com/apache/brpc/pull/3197), while preserving +Protocol's existing protobuf callback signatures. ## Build @@ -22,8 +20,8 @@ With GoogleTest sources at `/usr/src/googletest`: cmake -S . -B build -DWITH_FLATBUFFERS=ON -DBUILD_UNIT_TESTS=ON \ -DBUILD_BRPC_TOOLS=OFF -DDOWNLOAD_GTEST=OFF \ -DBRPC_SYSTEM_GTEST_SOURCE_DIR=/usr/src/googletest -cmake --build build --target brpc_flatbuffers_unittest -j6 -ctest --test-dir build -R '^brpc_flatbuffers_unittest$' --output-on-failure +cmake --build build --target brpc_flatbuffers_unittest brpc_flatbuffers_protocol_unittest -j6 +ctest --test-dir build -R '^brpc_flatbuffers(_protocol)?_unittest$' --output-on-failure ``` For nonstandard installations, set `FLATBUFFERS_INCLUDE_DIR`, @@ -127,11 +125,78 @@ rpc_service BenchmarkService { success. Generated accessors use thread-safe function-local static initialization. -The generator in `tools/flatbuffers/` uses the upstream parser and links -`libflatbuffers` separately. See its [README](../../tools/flatbuffers/README.md) -for commands and limitations. - -Generated dispatch validates requests and rejects unknown, foreign, or -unimplemented methods. On failure it runs a non-null completion callback. After -successful dispatch, the application owns completion and must run it exactly -once. +The companion generator in `tools/flatbuffers/` uses the upstream parser to +produce service bindings. It is independently built; only that optional tool +needs `libflatbuffers`. See its [README](../../tools/flatbuffers/README.md) for +commands and limitations. Generated dispatch verifies requests, rejects +unknown/foreign methods, and runs non-null completion callbacks on failure, +including unimplemented methods. Successful implementations own completion and +must run their callback exactly once. + +## Network RPC + +Build both the library and its users with the same `BRPC_WITH_FLATBUFFERS` +setting; enabling the feature changes the Channel, Controller and Server ABI. +Register generated services with `server.AddFlatBuffersService(&service, +SERVER_DOESNT_OWN_SERVICE)`. Use `ChannelOptions::protocol = "fb_rpc"` and pass +that `brpc::Channel` to a generated stub. The stub calls `Channel::FBCallMethod`; +`Controller::flatbuffers_method()` returns the typed descriptor, while the +protobuf `Controller::method()` remains null. Manually supplied descriptors +must outlive their calls, including retries and backup requests. + +Service registration is separate from protobuf's AddService and ListServices +APIs. Add/remove/clear require a stopped (READY) server, not a server still +STOPPING; management operations must be externally serialized. Ownership +transfers only after successful registration. `RemoveFlatBuffersService` +deletes an owned service, while Stop/Join keep registrations for restart. +`GetFlatBuffersServiceCount()` reports this separate registry. Duplicate service +IDs (including hash collisions) and conflicting full protobuf/FlatBuffers method +names are rejected. String-based `MaxConcurrencyOf` supports FlatBuffers +methods, as do server-wide and default method concurrency limits. + +The transport supports synchronous/asynchronous calls, retries, backup requests, +single/pooled/short connections and request/response attachments. It allocates a +small independent header for each send and shares payload storage; it never +rewrites a const request's metadata prefix. Thus retries and concurrent calls +can safely share an immutable request. The application must still keep each +response and Controller alive until completion. + +Framing validates lengths, not application schemas. Generated services verify +requests before dispatch. Handwritten services must do the same. Callers must +verify received response schemas before using root accessors; an RPC succeeding +does not replace `response.Verify()`. + +Authentication, compression, checksums and streaming are not supported and are +rejected rather than silently ignored. FlatBuffers services cannot be accessed +through the internal, builtin-only port. SelectiveChannel/ParallelChannel, +HTTP/JSON mapping and RPC-dump replay are not provided by this transport. +Log IDs, user fields and distributed-tracing metadata are not transmitted. + +### FRPC framing and compatibility + +A frame is `[12-byte header][metadata][message][attachment]`. The header contains +`FRPC`, a big-endian uint32 body size, and a big-endian uint32 metadata size; body +size excludes the 12-byte header but includes all three following sections. +The metadata prefix is explicitly little-endian, matching the original FRPC +experiment on little-endian machines without relying on packed structs: + +* Request: uint32 service ID, int32 method ID, int32 message size, int32 attachment + size, uint64 correlation ID (24 bytes). +* Response: int32 error code, int32 message size, int32 attachment size, uint64 + correlation ID (20 bytes). + +Readers require the whole known prefix, then skip unknown trailing metadata +using the advertised metadata size. Future optional fields must be appended; +do not reorder, resize or repurpose existing fields. A shorter prefix, negative +size, inconsistent payload length or excessive body is rejected. Error replies +have a nonzero error code and no payload. Method IDs are sparse wire IDs, never +array positions: adding, deleting or reordering declarations preserves surviving +IDs only when explicit IDs are retained. This does not make legacy ordinal-ID +schemas compatible, and removed IDs must never be reused. + +The magic is `FRPC`, not `BRPC`. This format does not promise compatibility with +older unpublished variants that used different magic, service hashes or native +big-endian metadata. The library's global Protocol hook signatures remain +unchanged, so existing protocol callback implementations need no adaptation. +Private numeric protocol IDs must not overlap newly assigned builtin IDs; +`PROTOCOL_FLATBUFFERS_RPC` uses ID 30. diff --git a/src/brpc/CMakeLists.txt b/src/brpc/CMakeLists.txt index 03e45da465..9378d0c8e2 100644 --- a/src/brpc/CMakeLists.txt +++ b/src/brpc/CMakeLists.txt @@ -251,6 +251,8 @@ file(GLOB_RECURSE BRPC_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/*. file(GLOB_RECURSE THRIFT_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/thrift*.cpp") if(NOT WITH_FLATBUFFERS) list(FILTER BRPC_SOURCES EXCLUDE REGEX "/brpc/flatbuffers/.*\\.cpp$") + list(REMOVE_ITEM BRPC_SOURCES + "${CMAKE_CURRENT_SOURCE_DIR}/policy/flatbuffers_protocol.cpp") endif() file(GLOB_RECURSE EXCLUDE_SOURCES CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/event_dispatcher_*.cpp") diff --git a/src/brpc/channel.cpp b/src/brpc/channel.cpp index de0a1bde49..ee5aa64175 100644 --- a/src/brpc/channel.cpp +++ b/src/brpc/channel.cpp @@ -40,6 +40,9 @@ #include "brpc/transport_factory.h" #include "brpc/details/controller_private_accessor.h" #include "brpc/details/ssl_helper.h" +#if BRPC_WITH_FLATBUFFERS +#include "brpc/flatbuffers/message.h" +#endif namespace brpc { @@ -484,6 +487,29 @@ void Channel::CallMethod(const google::protobuf::MethodDescriptor* method, const google::protobuf::Message* request, google::protobuf::Message* response, google::protobuf::Closure* done) { +#if BRPC_WITH_FLATBUFFERS + static_cast(controller_base)->_flatbuffers_method = nullptr; +#endif + CallMethodInternal(method, controller_base, request, response, done, false); +} + +#if BRPC_WITH_FLATBUFFERS +void Channel::FBCallMethod(const flatbuffers::MethodDescriptor* method, + google::protobuf::RpcController* controller_base, + const flatbuffers::Message* request, + flatbuffers::Message* response, + google::protobuf::Closure* done) { + static_cast(controller_base)->_flatbuffers_method = method; + CallMethodInternal(nullptr, controller_base, request, response, done, true); +} +#endif + +void Channel::CallMethodInternal( + const google::protobuf::MethodDescriptor* method, + google::protobuf::RpcController* controller_base, + const google::protobuf::Message* request, + google::protobuf::Message* response, + google::protobuf::Closure* done, bool is_fb) { const int64_t start_send_real_us = butil::gettimeofday_us(); Controller* cntl = static_cast(controller_base); cntl->OnRPCBegin(start_send_real_us); @@ -538,12 +564,17 @@ void Channel::CallMethod(const google::protobuf::MethodDescriptor* method, return; } cntl->set_used_by_rpc(); + const bool is_fb_protocol = (_options.protocol == PROTOCOL_FLATBUFFERS_RPC); if (cntl->_sender == nullptr && IsTraceable(Span::tls_parent().get())) { const int64_t start_send_us = butil::cpuwide_time_us(); std::string method_name; - if (_get_method_name) { + if (_get_method_name && is_fb == is_fb_protocol) { method_name = butil::EnsureString(_get_method_name(method, cntl)); +#if BRPC_WITH_FLATBUFFERS + } else if (is_fb && cntl->_flatbuffers_method) { + method_name = cntl->_flatbuffers_method->full_name(); +#endif } else if (method) { method_name = butil::EnsureString(method->full_name()); } else { @@ -591,6 +622,33 @@ void Channel::CallMethod(const google::protobuf::MethodDescriptor* method, // Share the lb with controller. cntl->_lb = _lb; + const char* entry_error = nullptr; + if (is_fb != is_fb_protocol) { + entry_error = is_fb ? "FlatBuffers calls require the fb_rpc protocol" : + "The fb_rpc protocol requires FBCallMethod"; + } +#if BRPC_WITH_FLATBUFFERS + else if (is_fb) { + if (!cntl->_flatbuffers_method || !request || !response) { + entry_error = "FlatBuffers method, request and response " + "must not be null"; + } else if (_options.auth != nullptr) { + // An authenticated shared socket may skip the packer's auth check. + entry_error = "The fb_rpc protocol does not support authentication"; + } else if (!cntl->_request_streams.empty() || + !cntl->_response_streams.empty()) { + entry_error = "The fb_rpc protocol does not support streams"; + } + } +#endif + if (entry_error) { + // A custom retry policy must not pack an unserialized request. Keep + // the normal completion path; callbacks may delete the controller. + cntl->set_max_retry(0); + cntl->SetFailed(EINVAL, "%s", entry_error); + return cntl->HandleSendFailed(); + } + // Ensure that serialize_request is done before pack_request in all // possible executions, including: // HandleSendFailed => OnVersionedRPCReturned => IssueRPC(pack_request) diff --git a/src/brpc/channel.h b/src/brpc/channel.h index a778a1bd2a..ce490e203f 100644 --- a/src/brpc/channel.h +++ b/src/brpc/channel.h @@ -23,6 +23,7 @@ // on internal structures, use opaque pointers instead. #include // std::ostream +#include "butil/config.h" #include "bthread/errno.h" // Redefine errno #include "butil/intrusive_ptr.hpp" // butil::intrusive_ptr #include "butil/ptr_container.h" @@ -38,6 +39,9 @@ #include "brpc/naming_service_filter.h" #include "brpc/health_check_option.h" #include "brpc/socket_mode.h" +#if BRPC_WITH_FLATBUFFERS +#include "brpc/flatbuffers/service.h" +#endif namespace brpc { @@ -176,7 +180,11 @@ struct ChannelOptions { // channel.Init("bns://rdev.matrix.all", "rr", nullptr/*default options*/); // MyService_Stub stub(&channel); // stub.MyMethod(&controller, &request, &response, nullptr); -class Channel : public ChannelBase { +class Channel : public ChannelBase +#if BRPC_WITH_FLATBUFFERS + , public flatbuffers::RpcChannel +#endif +{ friend class Controller; friend class SelectiveChannel; public: @@ -228,6 +236,15 @@ friend class SelectiveChannel; google::protobuf::Message* response, google::protobuf::Closure* done); +#if BRPC_WITH_FLATBUFFERS + // The descriptor is borrowed and must remain valid until the RPC finishes. + void FBCallMethod(const flatbuffers::MethodDescriptor* method, + google::protobuf::RpcController* controller, + const flatbuffers::Message* request, + flatbuffers::Message* response, + google::protobuf::Closure* done) override; +#endif + // Get current options. const ChannelOptions& options() const { return _options; } @@ -238,6 +255,13 @@ friend class SelectiveChannel; int CheckHealth(); +private: + void CallMethodInternal(const google::protobuf::MethodDescriptor* method, + google::protobuf::RpcController* controller, + const google::protobuf::Message* request, + google::protobuf::Message* response, + google::protobuf::Closure* done, bool is_fb); + protected: bool SingleServer() const { return _lb.get() == nullptr; } diff --git a/src/brpc/controller.cpp b/src/brpc/controller.cpp index 4fff9fd2f4..ce5c8e933d 100644 --- a/src/brpc/controller.cpp +++ b/src/brpc/controller.cpp @@ -510,6 +510,9 @@ void Controller::ResetPods() { _accessed = nullptr; _pack_request = nullptr; _method = nullptr; +#if BRPC_WITH_FLATBUFFERS + _flatbuffers_method = nullptr; +#endif _auth = nullptr; _idl_names = idl_single_req_single_res; _idl_result = IDL_VOID_RESULT; diff --git a/src/brpc/controller.h b/src/brpc/controller.h index c05dbb75a7..25ebcb0999 100644 --- a/src/brpc/controller.h +++ b/src/brpc/controller.h @@ -26,6 +26,7 @@ #include // Users often need gflags #include #include +#include "butil/config.h" #include "butil/intrusive_ptr.hpp" // butil::intrusive_ptr #include "bthread/errno.h" // Redefine errno #include "butil/endpoint.h" // butil::EndPoint @@ -64,6 +65,11 @@ struct x509_st; } namespace brpc { +#if BRPC_WITH_FLATBUFFERS +namespace flatbuffers { +class MethodDescriptor; +} +#endif class Span; class Server; class SharedLoadBalancer; @@ -336,6 +342,12 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); ConnectionType connection_type() const { return _connection_type; } // Get the called method. May-be nullptr for non-pb services. const google::protobuf::MethodDescriptor* method() const { return _method; } +#if BRPC_WITH_FLATBUFFERS + // Borrowed descriptor. The caller must keep it alive until the RPC ends. + const flatbuffers::MethodDescriptor* flatbuffers_method() const { + return _flatbuffers_method; + } +#endif // Get the controllers for accessing sub channels in combo channels. // Ordinary channel: @@ -961,6 +973,9 @@ friend void policy::ProcessThriftRequest(InputMessageBase*); // Fields will be used when making requests Protocol::PackRequest _pack_request; const google::protobuf::MethodDescriptor* _method; +#if BRPC_WITH_FLATBUFFERS + const flatbuffers::MethodDescriptor* _flatbuffers_method; +#endif const Authenticator* _auth; butil::IOBuf _request_buf; IdlNames _idl_names; diff --git a/src/brpc/details/controller_private_accessor.h b/src/brpc/details/controller_private_accessor.h index d752d7fe5e..12200e3bc7 100644 --- a/src/brpc/details/controller_private_accessor.h +++ b/src/brpc/details/controller_private_accessor.h @@ -149,6 +149,11 @@ class ControllerPrivateAccessor { void set_method(const google::protobuf::MethodDescriptor* method) { _cntl->_method = method; } +#if BRPC_WITH_FLATBUFFERS + void set_flatbuffers_method(const flatbuffers::MethodDescriptor* method) + { _cntl->_flatbuffers_method = method; } +#endif + void set_readable_progressive_attachment(ReadableProgressiveAttachment* s) { _cntl->_rpa.reset(s); } diff --git a/src/brpc/details/server_private_accessor.h b/src/brpc/details/server_private_accessor.h index 0e6e4fbba8..90448a2db5 100644 --- a/src/brpc/details/server_private_accessor.h +++ b/src/brpc/details/server_private_accessor.h @@ -76,6 +76,15 @@ class ServerPrivateAccessor { return _server->FindMethodPropertyByNameAndIndex(service_name, method_index); } +#if BRPC_WITH_FLATBUFFERS + const Server::FlatBuffersMethodProperty* + FindFlatBuffersMethodPropertyByIndex(uint32_t service_index, + int32_t method_index) const { + return _server->FindFlatBuffersMethodPropertyByIndex( + service_index, method_index); + } +#endif + const Server::ServiceProperty* FindServicePropertyByFullName(const butil::StringPiece& fullname) const { return _server->FindServicePropertyByFullName(fullname); diff --git a/src/brpc/global.cpp b/src/brpc/global.cpp index 4cce93ed0c..0ab7a33405 100644 --- a/src/brpc/global.cpp +++ b/src/brpc/global.cpp @@ -71,6 +71,9 @@ #include "brpc/protocol.h" #include "brpc/policy/rdma_handshake_protocol.h" #include "brpc/policy/baidu_rpc_protocol.h" +#if BRPC_WITH_FLATBUFFERS +#include "brpc/policy/flatbuffers_protocol.h" +#endif #include "brpc/policy/http_rpc_protocol.h" #include "brpc/policy/http2_rpc_protocol.h" #include "brpc/policy/hulu_pbrpc_protocol.h" @@ -456,6 +459,17 @@ static void GlobalInitializeOrDieImpl() { exit(1); } +#if BRPC_WITH_FLATBUFFERS + Protocol flatbuffers_protocol = { + ParseFlatBuffersMessage, SerializeFlatBuffersRequest, PackFlatBuffersRequest, + ProcessFlatBuffersRequest, ProcessFlatBuffersResponse, + VerifyFlatBuffersRequest, nullptr, GetFlatBuffersMethodName, + CONNECTION_TYPE_ALL, "fb_rpc" }; + if (RegisterProtocol(PROTOCOL_FLATBUFFERS_RPC, flatbuffers_protocol) != 0) { + exit(1); + } +#endif + Protocol streaming_protocol = { ParseStreamingMessage, nullptr, nullptr, ProcessStreamingMessage, ProcessStreamingMessage, diff --git a/src/brpc/options.proto b/src/brpc/options.proto index 13b8b682e6..23f28c0a9e 100644 --- a/src/brpc/options.proto +++ b/src/brpc/options.proto @@ -67,6 +67,7 @@ enum ProtocolType { PROTOCOL_H2 = 27; PROTOCOL_COUCHBASE = 28; PROTOCOL_MYSQL = 29; // Client side only + PROTOCOL_FLATBUFFERS_RPC = 30; } enum CompressType { diff --git a/src/brpc/policy/flatbuffers_protocol.cpp b/src/brpc/policy/flatbuffers_protocol.cpp new file mode 100644 index 0000000000..d63f67a4bd --- /dev/null +++ b/src/brpc/policy/flatbuffers_protocol.cpp @@ -0,0 +1,463 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "brpc/policy/flatbuffers_protocol.h" + +#if BRPC_WITH_FLATBUFFERS +#include +#include +#include +#include +#include "butil/logging.h" +#include "butil/macros.h" +#include "brpc/controller.h" +#include "brpc/server.h" +#include "brpc/socket.h" +#include "brpc/flatbuffers/message.h" +#include "brpc/flatbuffers/service.h" +#include "brpc/policy/most_common_message.h" +#include "brpc/details/controller_private_accessor.h" +#include "brpc/details/server_private_accessor.h" +#include "brpc/details/usercode_backup_pool.h" + +extern "C" void bthread_assign_data(void* data); + +namespace brpc { +namespace policy { +namespace { + +const size_t kHeaderSize = 12; +const size_t kRequestMetaSize = 24; +const size_t kResponseMetaSize = 20; + +// The header is big-endian; the fixed metadata prefix is little-endian, as in +// the experimental FRPC format on x86/ARM. meta_size permits appended fields. +// Bytewise access also handles fragmented or unaligned input without packed +// structs, aliasing violations or host-endian dependencies. +uint32_t Load32(const unsigned char* p, bool big_endian = false) { + uint32_t result = 0; + for (size_t i = 0; i < 4; ++i) { + result |= static_cast(p[i]) << (8 * (big_endian ? 3 - i : i)); + } + return result; +} + +uint64_t Load64(const unsigned char* p) { + return Load32(p) | (static_cast(Load32(p + 4)) << 32); +} + +void Store32(unsigned char* p, uint32_t value, bool big_endian = false) { + for (size_t i = 0; i < 4; ++i) { + p[i] = static_cast(value >> (8 * (big_endian ? 3 - i : i))); + } +} + +void Store64(unsigned char* p, uint64_t value) { + Store32(p, static_cast(value)); + Store32(p + 4, static_cast(value >> 32)); +} + +bool ValidSizes(size_t message_size, size_t attachment_size, size_t meta_size) { + const size_t limit = static_cast(std::numeric_limits::max()); + return message_size <= limit && attachment_size <= limit && + meta_size <= limit && message_size <= limit - meta_size && + attachment_size <= limit - meta_size - message_size && + meta_size + message_size + attachment_size <= + static_cast(FLAGS_max_body_size); +} + +void PackHeader(unsigned char* header, size_t meta_size, size_t payload_size) { + memcpy(header, "FRPC", 4); + Store32(header + 4, static_cast(meta_size + payload_size), true); + Store32(header + 8, static_cast(meta_size), true); +} + +bool SerializePayload(const flatbuffers::Message& message, butil::IOBuf* out) { + butil::IOBuf body; + if (!message.append_msg_to_iobuf(body)) { + return false; + } + body.pop_front(message.get_meta_size()); + if (body.size() != message.size()) { + return false; + } + out->append(body); + return true; +} + +bool ValidPayloadSizes(uint32_t message_size, uint32_t attachment_size, + size_t payload_size) { + const uint32_t limit = static_cast(std::numeric_limits::max()); + return message_size <= limit && attachment_size <= limit && + attachment_size <= payload_size && + message_size == payload_size - attachment_size; +} + +// One closure owns the request, response, controller and concurrency accounting +// until the application completes, including asynchronous service methods. +class FlatBuffersCall : public google::protobuf::Closure { +public: + FlatBuffersCall(uint64_t id, int64_t received_us) + : correlation_id(id), received_us(received_us), cntl(new Controller), + service(nullptr), method(nullptr), status(nullptr) {} + + ~FlatBuffersCall() override { + { + ConcurrencyRemover remover(status, cntl.get(), received_us); + } + cntl->CallAfterRpcResp(&request, &response); + } + + void Run() override { + std::unique_ptr self(this); + ControllerPrivateAccessor accessor(cntl.get()); + Socket* socket = accessor.get_sending_socket(); + if (cntl->IsCloseConnection()) { + socket->SetFailed(); + return; + } + + butil::IOBuf payload; + size_t attachment_size = 0; + if (!cntl->Failed()) { + if (cntl->response_compress_type() != COMPRESS_TYPE_NONE || + cntl->response_checksum_type() != CHECKSUM_TYPE_NONE) { + cntl->SetFailed(ERESPONSE, "FRPC does not support compression or checksums"); + } else if (!SerializePayload(response, &payload)) { + cntl->SetFailed(ERESPONSE, "Empty or invalid FlatBuffers response"); + } else { + attachment_size = cntl->response_attachment().size(); + if (!ValidSizes(payload.size(), attachment_size, kResponseMetaSize)) { + cntl->SetFailed(ERESPONSE, "FlatBuffers response is too large"); + } + } + } + if (cntl->Failed()) { + payload.clear(); + attachment_size = 0; + } + + unsigned char header[kHeaderSize + kResponseMetaSize] = {}; + PackHeader(header, kResponseMetaSize, payload.size() + attachment_size); + Store32(header + 12, static_cast(cntl->ErrorCode())); + Store32(header + 16, static_cast(payload.size())); + Store32(header + 20, static_cast(attachment_size)); + Store64(header + 24, correlation_id); + butil::IOBuf wire; + if (wire.append(header, sizeof(header)) != 0) { + socket->SetFailed(ENOMEM, "Fail to allocate FlatBuffers response header"); + return; + } + wire.append(payload); + if (attachment_size) { + wire.append(cntl->response_attachment()); + } + Socket::WriteOptions options; + options.ignore_eovercrowded = true; + if (socket->Write(&wire, &options) != 0) { + cntl->SetFailed(errno, "Fail to write FlatBuffers response"); + } + } + + static void Invoke(void* arg) { + FlatBuffersCall* call = static_cast(arg); + call->service->FBCallMethod(call->method, call->cntl.get(), + &call->request, &call->response, call); + } + + uint64_t correlation_id; + int64_t received_us; + std::unique_ptr cntl; + flatbuffers::Message request; + flatbuffers::Message response; + flatbuffers::Service* service; + const flatbuffers::MethodDescriptor* method; + MethodStatus* status; + +private: + DISALLOW_COPY_AND_ASSIGN(FlatBuffersCall); +}; + +} // namespace + +ParseResult ParseFlatBuffersMessage(butil::IOBuf* source, Socket*, bool, + const void*) { + unsigned char header[kHeaderSize] = {}; + const size_t n = source->copy_to(header, sizeof(header)); + if (memcmp(header, "FRPC", std::min(n, size_t(4))) != 0) { + return MakeParseError(PARSE_ERROR_TRY_OTHERS); + } + if (n < sizeof(header)) { + return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); + } + const uint32_t body_size = Load32(header + 4, true); + const uint32_t meta_size = Load32(header + 8, true); + if (body_size > static_cast(FLAGS_max_body_size)) { + return MakeParseError(PARSE_ERROR_TOO_BIG_DATA); + } + if (meta_size > body_size) { + return MakeParseError(PARSE_ERROR_ABSOLUTELY_WRONG, + "FRPC metadata exceeds frame body"); + } + if (source->size() - kHeaderSize < body_size) { + return MakeParseError(PARSE_ERROR_NOT_ENOUGH_DATA); + } + MostCommonMessage* message = MostCommonMessage::Get(); + if (!message) { + return MakeParseError(PARSE_ERROR_NO_RESOURCE); + } + source->pop_front(kHeaderSize); + source->cutn(&message->meta, meta_size); + source->cutn(&message->payload, body_size - meta_size); + return MakeMessage(message); +} + +bool VerifyFlatBuffersRequest(const InputMessageBase* message) { + const Server* server = static_cast(message->arg()); + // No credential format has been defined for FRPC. Never silently bypass + // authentication configured for the server's other protocols. + return server->options().auth == nullptr; +} + +void ProcessFlatBuffersRequest(InputMessageBase* message_base) { + DestroyingPtr message( + static_cast(message_base)); + SocketUniquePtr socket_guard(message->ReleaseSocket()); + Socket* socket = socket_guard.get(); + const Server* server = static_cast(message->arg()); + ScopedNonServiceError non_service_error(server); + unsigned char meta[kRequestMetaSize]; + if (message->meta.copy_to(meta, sizeof(meta)) != sizeof(meta)) { + socket->SetFailed(EREQUEST, "Truncated FlatBuffers request metadata"); + return; + } + const uint32_t service_id = Load32(meta); + const uint32_t method_id = Load32(meta + 4); + const uint32_t message_size = Load32(meta + 8); + const uint32_t attachment_size = Load32(meta + 12); + auto call = std::make_unique(Load64(meta + 16), + message->received_us()); + Controller* cntl = call->cntl.get(); + ControllerPrivateAccessor accessor(cntl); + ServerPrivateAccessor server_accessor(server); + const bool security_mode = server->options().security_mode() && + socket->user() == server_accessor.acceptor(); + cntl->set_rpc_received_us(message->received_us()); + accessor.set_server(server) + .set_security_mode(security_mode) + .set_peer_id(socket->id()) + .set_remote_side(socket->remote_side()) + .set_local_side(socket->local_side()) + .set_auth_context(socket->auth_context()) + .set_request_protocol(PROTOCOL_FLATBUFFERS_RPC) + .set_begin_time_us(message->received_us()) + .move_in_server_receiving_sock(socket_guard); + if (server->thread_local_options().thread_local_data_factory) { + bthread_assign_data((void*)&server->thread_local_options()); + } + + do { + if (server->options().auth) { + cntl->SetFailed(ERPCAUTH, "FRPC does not support authentication"); + break; + } + if (RejectNonBuiltinAccessFromInternalPort(cntl, *server)) { + break; + } + if (!server->IsRunning()) { + cntl->SetFailed(ELOGOFF, "Server is stopping"); + break; + } + if (!server_accessor.AddConcurrency(cntl)) { + cntl->SetFailed(ELIMIT, "Reached server's max_concurrency"); + break; + } + if (FLAGS_usercode_in_pthread && TooManyUserCode()) { + cntl->SetFailed(ELIMIT, "Too many user code tasks"); + break; + } + if (!message_size || !ValidPayloadSizes(message_size, attachment_size, + message->payload.size())) { + cntl->SetFailed(EREQUEST, "Invalid FlatBuffers request sizes"); + break; + } + if (method_id > static_cast(std::numeric_limits::max())) { + cntl->SetFailed(ENOMETHOD, "Invalid FlatBuffers method ID"); + break; + } + const Server::FlatBuffersMethodProperty* property = + server_accessor.FindFlatBuffersMethodPropertyByIndex(service_id, method_id); + if (!property) { + cntl->SetFailed(ENOMETHOD, "Unknown FlatBuffers service=%u method=%u", + service_id, method_id); + break; + } + call->service = property->service; + call->method = property->method; + accessor.set_flatbuffers_method(property->method); + if (socket->is_overcrowded() && !server->options().ignore_eovercrowded && + !property->ignore_eovercrowded) { + cntl->SetFailed(EOVERCROWDED, "FlatBuffers connection is overcrowded"); + break; + } + non_service_error.release(); + if (property->status) { + call->status = property->status; + if (!call->status->OnRequested(nullptr, cntl)) { + cntl->SetFailed(ELIMIT, "Reached method's max_concurrency"); + break; + } + } + if (!server->AcceptRequest(cntl)) { + break; + } + butil::IOBuf payload; + message->payload.cutn(&payload, message_size); + if (!call->request.parse_msg_from_iobuf(payload, message_size, 0)) { + cntl->SetFailed(EREQUEST, "Fail to parse FlatBuffers request"); + break; + } + cntl->request_attachment().swap(message->payload); + message.reset(); + if (FLAGS_usercode_in_pthread) { + RunUserCode(&FlatBuffersCall::Invoke, call.release()); + } else { + FlatBuffersCall::Invoke(call.release()); + } + return; + } while (false); + call.release()->Run(); +} + +void ProcessFlatBuffersResponse(InputMessageBase* message_base) { + DestroyingPtr message( + static_cast(message_base)); + unsigned char meta[kResponseMetaSize]; + if (message->meta.copy_to(meta, sizeof(meta)) != sizeof(meta)) { + if (message->socket()) { + message->socket()->SetFailed(ERESPONSE, "Truncated FlatBuffers response metadata"); + } + return; + } + const uint32_t error_code = Load32(meta); + const uint32_t message_size = Load32(meta + 4); + const uint32_t attachment_size = Load32(meta + 8); + const bthread_id_t correlation_id = {Load64(meta + 12)}; + Controller* cntl = nullptr; + const int rc = bthread_id_lock(correlation_id, reinterpret_cast(&cntl)); + if (rc != 0) { + return; + } + ControllerPrivateAccessor accessor(cntl); + const int saved_error = cntl->ErrorCode(); + cntl->set_rpc_received_us(message->received_us()); + do { + if (!ValidPayloadSizes(message_size, attachment_size, message->payload.size())) { + cntl->SetFailed(ERESPONSE, "Invalid FlatBuffers response sizes"); + break; + } + if (error_code) { + if (message_size || attachment_size) { + cntl->SetFailed(ERESPONSE, "FlatBuffers error response has a payload"); + break; + } + cntl->SetFailed(static_cast(error_code), "FlatBuffers server error"); + break; + } + if (!message_size) { + cntl->SetFailed(ERESPONSE, "Empty FlatBuffers response"); + break; + } + butil::IOBuf payload; + message->payload.cutn(&payload, message_size); + if (cntl->response()) { + if (cntl->response()->GetDescriptor() != flatbuffers::Message::descriptor()) { + cntl->SetFailed(ERESPONSE, "Response is not a FlatBuffers Message"); + break; + } + flatbuffers::Message response; + if (!response.parse_msg_from_iobuf(payload, message_size, 0)) { + cntl->SetFailed(ERESPONSE, "Fail to parse FlatBuffers response"); + break; + } + *static_cast(cntl->response()) = std::move(response); + } + cntl->response_attachment().swap(message->payload); + } while (false); + message.reset(); + accessor.OnResponse(correlation_id, saved_error); +} + +void SerializeFlatBuffersRequest(butil::IOBuf* buf, Controller* cntl, + const google::protobuf::Message* request) { + if (!request || request->GetDescriptor() != flatbuffers::Message::descriptor()) { + cntl->SetFailed(EREQUEST, "Request is not a FlatBuffers Message"); + return; + } + if (cntl->request_compress_type() != COMPRESS_TYPE_NONE || + cntl->request_checksum_type() != CHECKSUM_TYPE_NONE) { + cntl->SetFailed(EREQUEST, "FRPC does not support compression or checksums"); + return; + } + if (!SerializePayload(*static_cast(request), buf)) { + cntl->SetFailed(EREQUEST, "Empty or invalid FlatBuffers request"); + } +} + +void PackFlatBuffersRequest(butil::IOBuf* buf, SocketMessage**, + uint64_t correlation_id, + const google::protobuf::MethodDescriptor* method, + Controller* cntl, const butil::IOBuf& request, + const Authenticator* auth) { + const flatbuffers::MethodDescriptor* fb_method = cntl->flatbuffers_method(); + if (method || !fb_method || fb_method->index() < 0) { + cntl->SetFailed(ENOMETHOD, "Use Channel::FBCallMethod with a FlatBuffers method"); + return; + } + if (auth) { + cntl->SetFailed(EREQUEST, "FRPC does not support authentication"); + return; + } + const size_t attachment_size = cntl->request_attachment().size(); + if (!request.size() || !ValidSizes(request.size(), attachment_size, kRequestMetaSize)) { + cntl->SetFailed(EREQUEST, "Invalid FlatBuffers request size"); + return; + } + unsigned char header[kHeaderSize + kRequestMetaSize] = {}; + PackHeader(header, kRequestMetaSize, request.size() + attachment_size); + Store32(header + 12, fb_method->service()->index()); + Store32(header + 16, static_cast(fb_method->index())); + Store32(header + 20, static_cast(request.size())); + Store32(header + 24, static_cast(attachment_size)); + Store64(header + 28, correlation_id); + if (buf->append(header, sizeof(header)) != 0) { + cntl->SetFailed(ENOMEM, "Fail to allocate FlatBuffers request header"); + return; + } + buf->append(request); + buf->append(cntl->request_attachment()); +} + +const std::string& GetFlatBuffersMethodName( + const google::protobuf::MethodDescriptor*, const Controller* cntl) { + static const std::string empty; + return cntl->flatbuffers_method() ? cntl->flatbuffers_method()->full_name() : empty; +} + +} // namespace policy +} // namespace brpc +#endif // BRPC_WITH_FLATBUFFERS diff --git a/src/brpc/policy/flatbuffers_protocol.h b/src/brpc/policy/flatbuffers_protocol.h new file mode 100644 index 0000000000..e3e0065b6d --- /dev/null +++ b/src/brpc/policy/flatbuffers_protocol.h @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef BRPC_POLICY_FLATBUFFERS_PROTOCOL_H +#define BRPC_POLICY_FLATBUFFERS_PROTOCOL_H + +#include "butil/config.h" + +#if BRPC_WITH_FLATBUFFERS +#include "brpc/protocol.h" + +namespace brpc { +namespace policy { + +ParseResult ParseFlatBuffersMessage(butil::IOBuf* source, Socket* socket, + bool read_eof, const void* arg); +void ProcessFlatBuffersRequest(InputMessageBase* msg); +void ProcessFlatBuffersResponse(InputMessageBase* msg); +bool VerifyFlatBuffersRequest(const InputMessageBase* msg); +void SerializeFlatBuffersRequest(butil::IOBuf* buf, Controller* cntl, + const google::protobuf::Message* request); +void PackFlatBuffersRequest(butil::IOBuf* buf, SocketMessage** user_message, + uint64_t correlation_id, + const google::protobuf::MethodDescriptor* method, + Controller* cntl, const butil::IOBuf& request, + const Authenticator* auth); +const std::string& GetFlatBuffersMethodName( + const google::protobuf::MethodDescriptor* method, const Controller* cntl); + +} // namespace policy +} // namespace brpc +#endif // BRPC_WITH_FLATBUFFERS +#endif // BRPC_POLICY_FLATBUFFERS_PROTOCOL_H diff --git a/src/brpc/server.cpp b/src/brpc/server.cpp index 19d58ac422..ec6d02d167 100644 --- a/src/brpc/server.cpp +++ b/src/brpc/server.cpp @@ -83,6 +83,10 @@ #include "brpc/rdma/rdma_helper.h" #include "brpc/baidu_master_service.h" #include "brpc/transport_factory.h" +#if BRPC_WITH_FLATBUFFERS +#include +#include "brpc/flatbuffers/service.h" +#endif inline std::ostream& operator<<(std::ostream& os, const timeval& tm) { const char old_fill = os.fill(); @@ -114,6 +118,58 @@ const char* status_str(Server::Status s) { return "UNKNOWN_STATUS"; } +#if BRPC_WITH_FLATBUFFERS +struct Server::FlatBuffersServiceMap { + struct Method { + Method(flatbuffers::Service* service, + const flatbuffers::MethodDescriptor* descriptor) + : status(std::make_unique()) + , property{service, descriptor, status.get(), false} {} + + std::unique_ptr status; + FlatBuffersMethodProperty property; + AdaptiveMaxConcurrency max_concurrency; + + private: + DISALLOW_COPY_AND_ASSIGN(Method); + }; + struct Service { + Service(flatbuffers::Service* svc, + const flatbuffers::ServiceDescriptor* desc) + : service(svc) + , descriptor(desc) + , ownership(SERVER_DOESNT_OWN_SERVICE) {} + ~Service() { + methods.clear(); + if (ownership == SERVER_OWNS_SERVICE) { + delete service; + } + } + + flatbuffers::Service* service; + const flatbuffers::ServiceDescriptor* descriptor; + ServiceOwnership ownership; + std::map > methods; + + private: + DISALLOW_COPY_AND_ASSIGN(Service); + }; + typedef std::map > ServiceMap; + ServiceMap services; + + Method* FindMethodByFullName(const butil::StringPiece& fullname) const { + for (const auto& service : services) { + for (const auto& method : service.second->methods) { + if (method.second->property.method->full_name() == fullname) { + return method.second.get(); + } + } + } + return nullptr; + } +}; +#endif + butil::static_atomic g_running_server_count = BUTIL_STATIC_ATOMIC_INIT(0); // Following services may have security issues and are disabled by default. @@ -361,6 +417,19 @@ void* Server::UpdateDerivedVars(void* arg) { it->second.status->Expose(mprefix); } } +#if BRPC_WITH_FLATBUFFERS + if (server->_flatbuffers_services) { + for (const auto& service : server->_flatbuffers_services->services) { + for (const auto& method : service.second->methods) { + mprefix.resize(prefix.size()); + mprefix.push_back('_'); + bvar::to_underscored_name( + &mprefix, method.second->property.method->full_name()); + method.second->status->Expose(mprefix); + } + } + } +#endif if (server->options().baidu_master_service) { server->options().baidu_master_service->Expose(prefix); } @@ -1105,6 +1174,27 @@ int Server::StartInternal(const butil::EndPoint& endpoint, it->second.max_concurrency.SetConcurrencyLimiter(cl); } } +#if BRPC_WITH_FLATBUFFERS + if (_flatbuffers_services) { + for (const auto& service : _flatbuffers_services->services) { + for (const auto& entry : service.second->methods) { + auto& method = *entry.second; + const AdaptiveMaxConcurrency* amc = &method.max_concurrency; + if (amc->type() == AdaptiveMaxConcurrency::UNLIMITED()) { + amc = &_options.method_max_concurrency; + } + ConcurrencyLimiter* cl = nullptr; + if (!CreateConcurrencyLimiter(*amc, &cl)) { + LOG(ERROR) << "Fail to create ConcurrencyLimiter for " + << method.property.method->full_name(); + return -1; + } + method.status->SetConcurrencyLimiter(cl); + method.max_concurrency.SetConcurrencyLimiter(cl); + } + } + } +#endif if (0 != SetServiceMaxConcurrency(_options.nshead_service)) { return -1; } @@ -1426,6 +1516,22 @@ int Server::AddServiceInternal(google::protobuf::Service* service, // defined `option (idl_support) = true' or not. const bool is_idl_support = sd->file()->options().GetExtension(idl_support); +#if BRPC_WITH_FLATBUFFERS + if (_flatbuffers_services) { + for (int i = 0; i < sd->method_count(); ++i) { + const auto* md = sd->method(i); + if (_flatbuffers_services->FindMethodByFullName(md->full_name()) || + (is_idl_support && sd->name() != sd->full_name() && + _flatbuffers_services->FindMethodByFullName( + butil::EnsureString(sd->name()) + "." + + butil::EnsureString(md->name())))) { + LOG(ERROR) << "Protobuf method conflicts with FlatBuffers method: " + << md->full_name(); + return -1; + } + } + } +#endif Tabbed* tabbed = dynamic_cast(service); for (int i = 0; i < sd->method_count(); ++i) { @@ -1656,6 +1762,125 @@ int Server::AddService(google::protobuf::Service* service, return AddServiceInternal(service, false, options); } +#if BRPC_WITH_FLATBUFFERS +int Server::AddFlatBuffersService(flatbuffers::Service* service, + ServiceOwnership ownership) { + if (service == nullptr || + (ownership != SERVER_OWNS_SERVICE && + ownership != SERVER_DOESNT_OWN_SERVICE)) { + LOG(ERROR) << "Invalid FlatBuffers service or ownership"; + return -1; + } + if (InitializeOnce() != 0 || status() != READY) { + LOG(ERROR) << "Can't add FlatBuffers service to Server[" << version() + << "] which is " << status_str(status()); + return -1; + } + const flatbuffers::ServiceDescriptor* sd = service->GetDescriptor(); + if (sd == nullptr || sd->name().empty() || sd->full_name().empty() || + sd->method_count() <= 0) { + LOG(ERROR) << "Invalid FlatBuffers service descriptor"; + return -1; + } + if (_flatbuffers_services) { + for (const auto& entry : _flatbuffers_services->services) { + if (entry.second->service == service || entry.first == sd->index() || + entry.second->descriptor->full_name() == sd->full_name()) { + LOG(ERROR) << "Duplicate FlatBuffers service or hash collision: " + << sd->full_name(); + return -1; + } + } + } + + // Keep ownership with the caller until the entire record is published. + auto record = std::make_unique(service, sd); + std::unordered_set method_names; + for (int i = 0; i < sd->method_count(); ++i) { + const flatbuffers::MethodDescriptor* md = sd->method(i); + if (md == nullptr || md->service() != sd || md->index() < 0 || + md->name().empty() || + md->full_name() != sd->full_name() + "." + md->name() || + sd->FindMethodByIndex(md->index()) != md || + !method_names.insert(md->name()).second || + record->methods.count(md->index()) != 0) { + LOG(ERROR) << "Invalid FlatBuffers method in " << sd->full_name(); + return -1; + } + if (_method_map.seek(md->full_name()) != nullptr) { + LOG(ERROR) << "FlatBuffers method conflicts with protobuf method: " + << md->full_name(); + return -1; + } + record->methods.emplace( + md->index(), + std::make_unique(service, md)); + } + if (!_flatbuffers_services) { + _flatbuffers_services = std::make_unique(); + } + auto inserted = _flatbuffers_services->services.emplace( + sd->index(), std::move(record)); + if (!inserted.second) { + return -1; + } + inserted.first->second->ownership = ownership; + return 0; +} + +int Server::RemoveFlatBuffersService(flatbuffers::Service* service) { + if (service == nullptr) { + LOG(ERROR) << "Parameter[service] is NULL"; + return -1; + } + if (InitializeOnce() != 0 || status() != READY) { + LOG(ERROR) << "Can't remove FlatBuffers service from Server[" << version() + << "] which is " << status_str(status()); + return -1; + } + if (_flatbuffers_services) { + for (auto it = _flatbuffers_services->services.begin(); + it != _flatbuffers_services->services.end(); ++it) { + if (it->second->service == service) { + std::unique_ptr removed( + std::move(it->second)); + _flatbuffers_services->services.erase(it); + return 0; + } + } + } + return -1; +} + +size_t Server::GetFlatBuffersServiceCount() const { + return _flatbuffers_services ? _flatbuffers_services->services.size() : 0; +} + +const Server::FlatBuffersMethodProperty* +Server::FindFlatBuffersMethodPropertyByIndex(uint32_t service_index, + int32_t method_index) const { + if (!_flatbuffers_services || method_index < 0) { + return nullptr; + } + auto service = _flatbuffers_services->services.find(service_index); + if (service == _flatbuffers_services->services.end()) { + return nullptr; + } + // Wire IDs are sparse, not declaration-order positions. + const flatbuffers::MethodDescriptor* md = + service->second->descriptor->FindMethodByIndex(method_index); + if (md == nullptr) { + return nullptr; + } + auto method = service->second->methods.find(method_index); + if (method == service->second->methods.end() || + method->second->property.method != md) { + return nullptr; + } + return &method->second->property; +} +#endif + int Server::AddBuiltinService(google::protobuf::Service* service) { ServiceOptions options; options.ownership = SERVER_OWNS_SERVICE; @@ -1775,6 +2000,11 @@ void Server::ClearServices() { << "] which is " << status_str(status()); return; } +#if BRPC_WITH_FLATBUFFERS + // Detach every FlatBuffers method before destroying owned services. + std::unique_ptr removed_flatbuffers_services; + removed_flatbuffers_services.swap(_flatbuffers_services); +#endif for (ServiceMap::const_iterator it = _fullname_service_map.begin(); it != _fullname_service_map.end(); ++it) { if (it->second.ownership == SERVER_OWNS_SERVICE) { @@ -1821,6 +2051,9 @@ void Server::GetStat(ServerStatistics* stat) const { stat->connection_count += _internal_am->ConnectionCount(); } stat->user_service_count = service_count(); +#if BRPC_WITH_FLATBUFFERS + stat->user_service_count += GetFlatBuffersServiceCount(); +#endif stat->builtin_service_count = builtin_service_count(); } @@ -1851,8 +2084,11 @@ void Server::GenerateVersionIfNeeded() { if (!_version.empty()) { return; } - int extra_count = !!_options.nshead_service + !!_options.rtmp_service + + size_t extra_count = !!_options.nshead_service + !!_options.rtmp_service + !!_options.thrift_service + !!_options.redis_service; +#if BRPC_WITH_FLATBUFFERS + extra_count += GetFlatBuffersServiceCount(); +#endif _version.reserve((extra_count + service_count()) * 20); for (ServiceMap::const_iterator it = _fullname_service_map.begin(); it != _fullname_service_map.end(); ++it) { @@ -1863,6 +2099,16 @@ void Server::GenerateVersionIfNeeded() { _version.append(butil::class_name_str(*it->second.service)); } } +#if BRPC_WITH_FLATBUFFERS + if (_flatbuffers_services) { + for (const auto& service : _flatbuffers_services->services) { + if (!_version.empty()) { + _version.push_back('+'); + } + _version.append(service.second->descriptor->full_name()); + } + } +#endif if (_options.nshead_service) { if (!_version.empty()) { _version.push_back('+'); @@ -2308,6 +2554,15 @@ AdaptiveMaxConcurrency& Server::MaxConcurrencyOf(const butil::StringPiece& full_ MethodProperty* mp = _method_map.seek(full_method_name); if (mp == nullptr) { +#if BRPC_WITH_FLATBUFFERS + if (_flatbuffers_services) { + auto* method = _flatbuffers_services->FindMethodByFullName( + full_method_name); + if (method) { + return method->max_concurrency; + } + } +#endif break; } return MaxConcurrencyOf(mp); @@ -2320,7 +2575,16 @@ AdaptiveMaxConcurrency& Server::MaxConcurrencyOf(const butil::StringPiece& full_ } int Server::MaxConcurrencyOf(const butil::StringPiece& full_method_name) const { - return MaxConcurrencyOf(_method_map.seek(full_method_name)); + const MethodProperty* mp = _method_map.seek(full_method_name); +#if BRPC_WITH_FLATBUFFERS + if (mp == nullptr && _flatbuffers_services) { + auto* method = _flatbuffers_services->FindMethodByFullName(full_method_name); + if (method) { + return method->max_concurrency; + } + } +#endif + return MaxConcurrencyOf(mp); } AdaptiveMaxConcurrency& Server::MaxConcurrencyOf(const butil::StringPiece& full_service_name, @@ -2328,6 +2592,15 @@ AdaptiveMaxConcurrency& Server::MaxConcurrencyOf(const butil::StringPiece& full_ MethodProperty* mp = const_cast( FindMethodPropertyByFullName(full_service_name, method_name)); if (mp == nullptr) { +#if BRPC_WITH_FLATBUFFERS + if (_flatbuffers_services) { + auto* method = _flatbuffers_services->FindMethodByFullName( + full_service_name.as_string() + "." + method_name.as_string()); + if (method) { + return method->max_concurrency; + } + } +#endif LOG(ERROR) << "Fail to find method=" << full_service_name << '/' << method_name; _failed_to_set_max_concurrency_of_method = true; @@ -2338,8 +2611,18 @@ AdaptiveMaxConcurrency& Server::MaxConcurrencyOf(const butil::StringPiece& full_ int Server::MaxConcurrencyOf(const butil::StringPiece& full_service_name, const butil::StringPiece& method_name) const { - return MaxConcurrencyOf(FindMethodPropertyByFullName( - full_service_name, method_name)); + const MethodProperty* mp = FindMethodPropertyByFullName( + full_service_name, method_name); +#if BRPC_WITH_FLATBUFFERS + if (mp == nullptr && _flatbuffers_services) { + auto* method = _flatbuffers_services->FindMethodByFullName( + full_service_name.as_string() + "." + method_name.as_string()); + if (method) { + return method->max_concurrency; + } + } +#endif + return MaxConcurrencyOf(mp); } AdaptiveMaxConcurrency& Server::MaxConcurrencyOf(google::protobuf::Service* service, @@ -2355,6 +2638,19 @@ int Server::MaxConcurrencyOf(google::protobuf::Service* service, bool& Server::IgnoreEovercrowdedOf(const butil::StringPiece& full_method_name) { MethodProperty* mp = _method_map.seek(full_method_name); if (mp == nullptr) { +#if BRPC_WITH_FLATBUFFERS + if (_flatbuffers_services) { + auto* method = _flatbuffers_services->FindMethodByFullName( + full_method_name); + if (method) { + if (status() != READY) { + LOG(WARNING) << "IgnoreEovercrowdedOf requires a stopped Server"; + return g_default_ignore_eovercrowded; + } + return method->property.ignore_eovercrowded; + } + } +#endif LOG(ERROR) << "Fail to find method=" << full_method_name; _failed_to_set_ignore_eovercrowded = true; return g_default_ignore_eovercrowded; @@ -2379,6 +2675,15 @@ bool Server::IgnoreEovercrowdedOf(const butil::StringPiece& full_method_name) co return g_default_ignore_eovercrowded; } if (mp == nullptr || mp->status == nullptr) { +#if BRPC_WITH_FLATBUFFERS + if (mp == nullptr && _flatbuffers_services) { + auto* method = _flatbuffers_services->FindMethodByFullName( + full_method_name); + if (method) { + return method->property.ignore_eovercrowded; + } + } +#endif return false; } return mp->ignore_eovercrowded; diff --git a/src/brpc/server.h b/src/brpc/server.h index 31dc69af2f..830f0391ca 100644 --- a/src/brpc/server.h +++ b/src/brpc/server.h @@ -26,6 +26,7 @@ #include "bthread/bthread.h" // Server may need some bthread functions, // e.g. bthread_usleep #include // google::protobuf::Service +#include "butil/config.h" #include "butil/macros.h" // DISALLOW_COPY_AND_ASSIGN #include "butil/containers/doubly_buffered_data.h" // DoublyBufferedData #include "bvar/bvar.h" @@ -49,6 +50,13 @@ namespace brpc { +#if BRPC_WITH_FLATBUFFERS +namespace flatbuffers { +class Service; +class MethodDescriptor; +} // namespace flatbuffers +#endif + class Acceptor; class MethodStatus; class NsheadService; @@ -450,6 +458,15 @@ class Server { }; typedef butil::FlatMap MethodMap; +#if BRPC_WITH_FLATBUFFERS + struct FlatBuffersMethodProperty { + flatbuffers::Service* service; + const flatbuffers::MethodDescriptor* method; + MethodStatus* status; + bool ignore_eovercrowded; + }; +#endif + struct ThreadLocalOptions { bthread_key_t tls_key; const DataFactory* thread_local_data_factory; @@ -521,6 +538,20 @@ class Server { // Returns 0 on success, -1 otherwise. int RemoveService(google::protobuf::Service* service); +#if BRPC_WITH_FLATBUFFERS + // FlatBuffers services are separate from the protobuf service maps. + // The service and its descriptor must remain valid until removal. + // Add/Remove require a stopped server (READY). Ownership transfers only + // after a successful Add; Remove deletes a server-owned service. + // Like AddService/RemoveService, these methods are not thread-safe. + // Full method names must not conflict with registered protobuf methods. + // Returns 0 on success, -1 otherwise. + int AddFlatBuffersService(flatbuffers::Service* service, + ServiceOwnership ownership); + int RemoveFlatBuffersService(flatbuffers::Service* service); + size_t GetFlatBuffersServiceCount() const; +#endif + // Remove all services from this server. // NOTE: clearing services when server is running is forbidden. void ClearServices(); @@ -612,6 +643,7 @@ class Server { // server.MaxConcurrencyOf("example.EchoService.Echo") = 10; // or server.MaxConcurrencyOf("example.EchoService", "Echo") = 10; // or server.MaxConcurrencyOf(&service, "Echo") = 10; + // The string-based forms also support registered FlatBuffers methods. // Note: These interfaces can ONLY be called before the server is started. // And you should NOT set the max_concurrency when you are going to choose // an auto concurrency limiter, eg `options.max_concurrency = "auto"`.If you @@ -712,6 +744,11 @@ friend struct RevertServerStatus; FindMethodPropertyByNameAndIndex(const butil::StringPiece& service_name, int method_index) const; +#if BRPC_WITH_FLATBUFFERS + const FlatBuffersMethodProperty* FindFlatBuffersMethodPropertyByIndex( + uint32_t service_index, int32_t method_index) const; +#endif + const ServiceProperty* FindServicePropertyByFullName(const butil::StringPiece& fullname) const; @@ -782,6 +819,11 @@ friend struct RevertServerStatus; Acceptor* _am; Acceptor* _internal_am; +#if BRPC_WITH_FLATBUFFERS + struct FlatBuffersServiceMap; + std::unique_ptr _flatbuffers_services; +#endif + // Use method->full_name() as key MethodMap _method_map; diff --git a/test/BUILD.bazel b/test/BUILD.bazel index a2c291eb7e..5eec8e014c 100644 --- a/test/BUILD.bazel +++ b/test/BUILD.bazel @@ -264,11 +264,30 @@ cc_test( ], ) +cc_test( + name = "brpc_flatbuffers_protocol_unittest", + srcs = [ + "brpc_flatbuffers_protocol_unittest.cpp", + ":flatbuffers_test_messages", + ], + copts = COPTS, + includes = ["."], + tags = ["exclusive"], + target_compatible_with = FLATBUFFERS_COMPATIBILITY, + deps = [ + "//:brpc", + "@com_google_googletest//:gtest_main", + ], +) + generate_unittests( name = "brpc_unittests", srcs = glob( ["brpc_*_unittest.cpp"], - exclude = ["brpc_flatbuffers_unittest.cpp"], + exclude = [ + "brpc_flatbuffers_unittest.cpp", + "brpc_flatbuffers_protocol_unittest.cpp", + ], ), deps = [ ":gperftools_helper", diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index e2e9a84757..925bf3b1e9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -306,19 +306,21 @@ if(WITH_FLATBUFFERS) "${FLATBUFFERS_FLATC_EXECUTABLE}" COMMENT "Generating FlatBuffers test messages" VERBATIM) - add_executable(brpc_flatbuffers_unittest - brpc_flatbuffers_unittest.cpp "${FLATBUFFERS_TEST_HEADER}") - target_include_directories(brpc_flatbuffers_unittest BEFORE PRIVATE - "${BRPC_GTEST_INCLUDE_DIR}") - target_link_libraries(brpc_flatbuffers_unittest PRIVATE - brpc-static brpc_test_config gtest_main ${DYNAMIC_LIB}) - add_test(NAME brpc_flatbuffers_unittest COMMAND brpc_flatbuffers_unittest) + foreach(FB_TEST brpc_flatbuffers_unittest brpc_flatbuffers_protocol_unittest) + add_executable(${FB_TEST} ${FB_TEST}.cpp "${FLATBUFFERS_TEST_HEADER}") + target_include_directories(${FB_TEST} BEFORE PRIVATE + "${BRPC_GTEST_INCLUDE_DIR}") + target_link_libraries(${FB_TEST} PRIVATE + brpc-static brpc_test_config gtest_main ${DYNAMIC_LIB}) + add_test(NAME ${FB_TEST} COMMAND ${FB_TEST}) + endforeach() endif() # brpc tests file(GLOB BRPC_UNITTESTS "brpc_*_unittest.cpp") list(REMOVE_ITEM BRPC_UNITTESTS - "${CMAKE_CURRENT_SOURCE_DIR}/brpc_flatbuffers_unittest.cpp") + "${CMAKE_CURRENT_SOURCE_DIR}/brpc_flatbuffers_unittest.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/brpc_flatbuffers_protocol_unittest.cpp") foreach(BRPC_UT ${BRPC_UNITTESTS}) get_filename_component(BRPC_UT_WE ${BRPC_UT} NAME_WE) add_executable(${BRPC_UT_WE} ${BRPC_UT} $) diff --git a/test/Makefile b/test/Makefile index f4d5462e63..3f9103e9fb 100644 --- a/test/Makefile +++ b/test/Makefile @@ -182,7 +182,7 @@ TEST_BTHREAD_OBJS = $(addsuffix .o, $(basename $(TEST_BTHREAD_SOURCES))) TEST_BRPC_SOURCES = $(wildcard brpc_*unittest.cpp) ifneq ($(WITH_FLATBUFFERS),1) -TEST_BRPC_SOURCES := $(filter-out brpc_flatbuffers_unittest.cpp,$(TEST_BRPC_SOURCES)) +TEST_BRPC_SOURCES := $(filter-out brpc_flatbuffers_unittest.cpp brpc_flatbuffers_protocol_unittest.cpp,$(TEST_BRPC_SOURCES)) endif TEST_BRPC_OBJS = $(addsuffix .o, $(basename $(TEST_BRPC_SOURCES))) @@ -248,11 +248,11 @@ $(FLATBUFFERS_TEST_HEADER):flatbuffers_message.fbs @mkdir -p $(FLATBUFFERS_TEST_GEN_DIR) $(FLATC) --cpp -o $(FLATBUFFERS_TEST_GEN_DIR) $< -brpc_flatbuffers_unittest.o:brpc_flatbuffers_unittest.cpp $(FLATBUFFERS_TEST_HEADER) | libbrpc.dbg.$(SOEXT) +brpc_flatbuffers_unittest.o brpc_flatbuffers_protocol_unittest.o:%.o:%.cpp $(FLATBUFFERS_TEST_HEADER) | libbrpc.dbg.$(SOEXT) @echo "> Compiling $@" $(CXX) -c $(FLATBUFFERS_TEST_HDRPATHS) $(CXXFLAGS) $< -o $@ -brpc_flatbuffers_unittest:brpc_flatbuffers_unittest.o | libbrpc.dbg.$(SOEXT) +brpc_flatbuffers_unittest brpc_flatbuffers_protocol_unittest: %: %.o | libbrpc.dbg.$(SOEXT) @echo "> Linking $@" ifeq ($(SYSTEM),Linux) $(CXX) -o $@ $(LIBPATHS) $(SOPATHS) -Xlinker "-(" $^ -Xlinker "-)" $(STATIC_LINKINGS) $(UT_DYNAMIC_LINKINGS) diff --git a/test/brpc_channel_unittest.cpp b/test/brpc_channel_unittest.cpp index 3663efa6e3..20bd168b1d 100644 --- a/test/brpc_channel_unittest.cpp +++ b/test/brpc_channel_unittest.cpp @@ -334,7 +334,9 @@ class ChannelTest : public ::testing::Test{ nullptr, ProcessRpcRequest, VerifyMyRequest, nullptr, nullptr, brpc::CONNECTION_TYPE_ALL, "baidu_std" }; - ASSERT_EQ(0, RegisterProtocol((brpc::ProtocolType)30, dummy_protocol)); + // Keep the test protocol outside the IDs assigned to builtins. + const auto dummy_type = static_cast(brpc::ProtocolType_MAX + 1); + ASSERT_EQ(0, RegisterProtocol(dummy_type, dummy_protocol)); } static void ProcessRpcRequest(brpc::InputMessageBase* msg_base) { diff --git a/test/brpc_flatbuffers_protocol_unittest.cpp b/test/brpc_flatbuffers_protocol_unittest.cpp new file mode 100644 index 0000000000..21e6749641 --- /dev/null +++ b/test/brpc_flatbuffers_protocol_unittest.cpp @@ -0,0 +1,1825 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "butil/config.h" +#if BRPC_WITH_FLATBUFFERS +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "butil/endpoint.h" +#include "butil/fd_guard.h" +#include "brpc/authenticator.h" +#include "brpc/channel.h" +#include "brpc/closure_guard.h" +#include "brpc/controller.h" +#include "brpc/details/controller_private_accessor.h" +#include "brpc/details/method_status.h" +#include "brpc/details/server_private_accessor.h" +#include "brpc/errno.pb.h" +#include "brpc/flatbuffers/message.h" +#include "brpc/flatbuffers/service.h" +#include "brpc/interceptor.h" +#include "brpc/retry_policy.h" +#include "brpc/policy/flatbuffers_protocol.h" +#include "brpc/policy/most_common_message.h" +#include "brpc/server.h" +#include "brpc/stream.h" +#include "flatbuffers_message_generated.h" + +namespace { + +using brpc::flatbuffers::BrpcDescriptorTable; +using brpc::flatbuffers::Message; +using brpc::flatbuffers::MessageBuilder; +using brpc::flatbuffers::MethodDescriptor; +using brpc::flatbuffers::ServiceDescriptor; +using brpc_fbtest::Payload; + +const int kRpcTimeoutMs = 1500; +const int kWaitTimeoutMs = 5000; +const size_t kHeaderSize = 12; +const size_t kRequestMetaSize = 24; +const size_t kResponseMetaSize = 20; + +// These independent byte-level helpers intentionally do not reuse the codec. +void Put32(std::string* bytes, size_t offset, uint32_t value, + bool big_endian = false) { + for (size_t i = 0; i < 4; ++i) { + (*bytes)[offset + i] = static_cast( + value >> (8 * (big_endian ? 3 - i : i))); + } +} + +uint32_t Get32(const std::string& bytes, size_t offset, + bool big_endian = false) { + uint32_t value = 0; + for (size_t i = 0; i < 4; ++i) { + value |= static_cast( + static_cast(bytes[offset + i])) + << (8 * (big_endian ? 3 - i : i)); + } + return value; +} + +void Put64(std::string* bytes, size_t offset, uint64_t value) { + Put32(bytes, offset, static_cast(value)); + Put32(bytes, offset + 4, static_cast(value >> 32)); +} + +uint64_t Get64(const std::string& bytes, size_t offset) { + return Get32(bytes, offset) | + (static_cast(Get32(bytes, offset + 4)) << 32); +} + +std::string Header(uint32_t body_size, uint32_t meta_size) { + std::string bytes(kHeaderSize, '\0'); + bytes.replace(0, 4, "FRPC"); + Put32(&bytes, 4, body_size, true); + Put32(&bytes, 8, meta_size, true); + return bytes; +} + +std::string Frame(const std::string& meta, const std::string& payload) { + return Header(meta.size() + payload.size(), meta.size()) + meta + payload; +} + +std::string Bytes(const Message& message) { + return std::string(reinterpret_cast(message.data()), + message.size()); +} + +Message MakePayload(size_t text_size = 16, int64_t value = 123, + size_t vector_size = 4) { + MessageBuilder builder(8); + auto text = builder.CreateString(std::string(text_size, 'x')); + std::vector numbers(vector_size); + for (size_t i = 0; i < numbers.size(); ++i) { + numbers[i] = static_cast(i) * -3; + } + auto values = builder.CreateVector(numbers); + builder.Finish(brpc_fbtest::CreatePayload(builder, value, text, values)); + return builder.ReleaseMessage(); +} + +void ExpectReply(const Message& response, const Message& request, + const MethodDescriptor* method) { + ASSERT_NE(nullptr, method); + ASSERT_TRUE(request.Verify()); + // Framing validation is not schema validation; the caller must do this. + ASSERT_TRUE(response.Verify()); + const Payload* in = request.GetRoot(); + const Payload* out = response.GetRoot(); + EXPECT_EQ(in->value() + method->index(), out->value()); + ASSERT_NE(nullptr, out->message()); + EXPECT_EQ(method->name() + ":" + in->message()->str(), + out->message()->str()); + ASSERT_NE(nullptr, out->values()); + ASSERT_EQ(in->values()->size(), out->values()->size()); + for (size_t i = 0; i < in->values()->size(); ++i) { + EXPECT_EQ(in->values()->Get(i), out->values()->Get(i)); + } +} + +std::string RequestMeta(const MethodDescriptor* method, uint32_t message_size, + uint32_t attachment_size, uint64_t correlation_id) { + std::string meta(kRequestMetaSize, '\0'); + Put32(&meta, 0, method->service()->index()); + Put32(&meta, 4, method->index()); + Put32(&meta, 8, message_size); + Put32(&meta, 12, attachment_size); + Put64(&meta, 16, correlation_id); + return meta; +} + +std::string ResponseMeta(uint32_t message_size, uint32_t attachment_size, + uint64_t correlation_id) { + std::string meta(kResponseMetaSize, '\0'); + Put32(&meta, 4, message_size); + Put32(&meta, 8, attachment_size); + Put64(&meta, 12, correlation_id); + return meta; +} + +class Completion : public google::protobuf::Closure { +public: + void Run() override { + std::lock_guard lock(_mutex); + ++_calls; + _condition.notify_all(); + } + bool Wait() { + std::unique_lock lock(_mutex); + return _condition.wait_for(lock, std::chrono::milliseconds(kWaitTimeoutMs), + [this] { return _calls != 0; }); + } + int calls() { + std::lock_guard lock(_mutex); + return _calls; + } +private: + std::mutex _mutex; + std::condition_variable _condition; + int _calls = 0; +}; + +class PayloadService : public brpc::flatbuffers::Service { +public: + explicit PayloadService( + BrpcDescriptorTable table = {"brpc_fbtest", "ProtocolService", + "Echo Alternate Removed", {7, 41, 99}}, + std::atomic* destroyed = nullptr) + : _destroyed(destroyed) { + EXPECT_EQ(0, _descriptor.init(table)); + } + ~PayloadService() override { + ReleaseHeld(); + if (_destroyed) { + ++*_destroyed; + } + } + const ServiceDescriptor* GetDescriptor() override { return &_descriptor; } + void FBCallMethod(const MethodDescriptor* method, + google::protobuf::RpcController* controller, + const Message* request, Message* response, + google::protobuf::Closure* done) override { + brpc::ClosureGuard done_guard(done); + brpc::Controller* cntl = static_cast(controller); + ++entered; + if (cntl->is_security_mode()) { + ++security_mode_calls; + } + if (!method || _descriptor.FindMethodByIndex(method->index()) != method || + cntl->flatbuffers_method() != method) { + cntl->SetFailed(brpc::ENOMETHOD, "Wrong FlatBuffers method context"); + return; + } + if (!request || !request->Verify()) { + ++rejected; + cntl->SetFailed(brpc::EREQUEST, "Invalid Payload schema"); + return; + } + // No application field is read before Verify succeeds. + const Payload* in = request->GetRoot(); + if (!in->message() || !in->values()) { + cntl->SetFailed(brpc::EREQUEST, "Missing application fields"); + return; + } + const int failure = fail_next_code.exchange(0); + if (failure != 0) { + cntl->SetFailed(failure, "Requested one-shot application failure"); + return; + } + MessageBuilder builder(8); + auto text = builder.CreateString(method->name() + ":" + + in->message()->str()); + const std::vector numbers(in->values()->begin(), + in->values()->end()); + auto values = builder.CreateVector(numbers); + builder.Finish(brpc_fbtest::CreatePayload( + builder, in->value() + method->index(), text, values)); + *response = builder.ReleaseMessage(); + cntl->response_attachment().append(cntl->request_attachment()); + if (response_compressed.load()) { + cntl->set_response_compress_type(brpc::COMPRESS_TYPE_GZIP); + } + if (response_checksummed.load()) { + cntl->set_response_checksum_type(brpc::CHECKSUM_TYPE_CRC32C); + } + std::lock_guard lock(_mutex); + if (_hold_next) { + _hold_next = false; + _held = done_guard.release(); + _condition.notify_all(); + } + } + void HoldNext() { + std::lock_guard lock(_mutex); + _hold_next = true; + } + bool WaitHeld() { + std::unique_lock lock(_mutex); + return _condition.wait_for(lock, std::chrono::milliseconds(kWaitTimeoutMs), + [this] { return _held != nullptr; }); + } + void ReleaseHeld() { + google::protobuf::Closure* done = nullptr; + { + std::lock_guard lock(_mutex); + _hold_next = false; + std::swap(done, _held); + } + if (done) { + done->Run(); + } + } + std::atomic entered{0}; + std::atomic rejected{0}; + std::atomic security_mode_calls{0}; + std::atomic fail_next_code{0}; + std::atomic response_compressed{false}; + std::atomic response_checksummed{false}; +private: + ServiceDescriptor _descriptor; + std::atomic* _destroyed; + std::mutex _mutex; + std::condition_variable _condition; + bool _hold_next = false; + google::protobuf::Closure* _held = nullptr; +}; + +int InitChannel(brpc::Channel* channel, const butil::EndPoint& endpoint, + const char* protocol = "fb_rpc", + const char* connection_type = "single") { + brpc::ChannelOptions options; + options.protocol = protocol; + options.connection_type = connection_type; + options.timeout_ms = kRpcTimeoutMs; + options.connect_timeout_ms = kRpcTimeoutMs; + options.max_retry = 0; + return channel->Init(endpoint, &options); +} + +class FlatBuffersProtocolTest : public ::testing::Test { +protected: + void SetUp() override { + ASSERT_EQ(0, server.AddFlatBuffersService( + &service, brpc::SERVER_DOESNT_OWN_SERVICE)); + ASSERT_EQ(0, server.Start("127.0.0.1:0", nullptr)); + ASSERT_EQ(0, InitChannel(&channel, server.listen_address())); + } + void TearDown() override { + service.ReleaseHeld(); + server.Stop(0); + server.Join(); + } + const MethodDescriptor* method(int id = 7) { + return service.GetDescriptor()->FindMethodByIndex(id); + } + PayloadService service; + brpc::Server server; + brpc::Channel channel; +}; + +TEST(FlatBuffersFramingTest, PartialFramesAndInvalidHeadersDoNotConsumeInput) { + const std::string wire = Frame(std::string(24, 'm'), "payload"); + for (size_t size = 0; size < wire.size(); ++size) { + SCOPED_TRACE(size); + butil::IOBuf input; + input.append(wire.data(), size); + const std::string before = input.to_string(); + brpc::ParseResult result = brpc::policy::ParseFlatBuffersMessage( + &input, nullptr, false, nullptr); + EXPECT_EQ(brpc::PARSE_ERROR_NOT_ENOUGH_DATA, result.error()); + EXPECT_EQ(before, input.to_string()); + } + for (const std::string& bad : {std::string("X"), std::string("FX"), + std::string("FRX"), std::string("FRPX")}) { + butil::IOBuf input; + input.append(bad); + EXPECT_EQ(brpc::PARSE_ERROR_TRY_OTHERS, + brpc::policy::ParseFlatBuffersMessage( + &input, nullptr, false, nullptr).error()); + EXPECT_EQ(bad, input.to_string()); + } + const std::vector > malformed = { + {Header(std::numeric_limits::max(), 24), + brpc::PARSE_ERROR_TOO_BIG_DATA}, + {Header(0, 1), brpc::PARSE_ERROR_ABSOLUTELY_WRONG}, + {Header(23, 24), brpc::PARSE_ERROR_ABSOLUTELY_WRONG} + }; + for (const auto& entry : malformed) { + butil::IOBuf input; + input.append(entry.first); + EXPECT_EQ(entry.second, brpc::policy::ParseFlatBuffersMessage( + &input, nullptr, false, nullptr).error()); + EXPECT_EQ(entry.first, input.to_string()); + } +} + +TEST(FlatBuffersFramingTest, FragmentedUnalignedAndExtendedMetadata) { + const Message payload = MakePayload(8193); + ServiceDescriptor descriptor; + ASSERT_EQ(0, descriptor.init({"brpc_fbtest", "Framing", "Echo", {41}})); + std::string meta = RequestMeta(descriptor.method(0), payload.size(), 3, + 0x1122334455667788ULL); + meta.append("\x01\x02\x03\x04\x05", 5); + const std::string wire = Frame(meta, Bytes(payload) + "att"); + for (size_t fragment_size : {1u, 7u, 13u, 4093u}) { + SCOPED_TRACE(fragment_size); + butil::IOBuf input; + for (size_t pos = 0; pos < wire.size(); pos += fragment_size) { + const size_t size = std::min(fragment_size, wire.size() - pos); + char* allocation = static_cast(malloc(size + 1)); + ASSERT_NE(nullptr, allocation); + memcpy(allocation + 1, wire.data() + pos, size); + ASSERT_EQ(0, input.append_user_data(allocation + 1, size, + [allocation](void*) { free(allocation); })); + } + ASSERT_GT(input.backing_block_num(), 1u); + input.append("next"); + brpc::ParseResult result = brpc::policy::ParseFlatBuffersMessage( + &input, nullptr, false, nullptr); + ASSERT_EQ(brpc::PARSE_OK, result.error()); + brpc::DestroyingPtr parsed( + static_cast(result.message())); + EXPECT_EQ(meta, parsed->meta.to_string()); + EXPECT_EQ(Bytes(payload) + "att", parsed->payload.to_string()); + EXPECT_EQ("next", input.to_string()); + Message decoded; + butil::IOBuf body; + parsed->payload.cutn(&body, payload.size()); + ASSERT_TRUE(decoded.parse_msg_from_iobuf(body, payload.size(), 0)); + ASSERT_TRUE(decoded.Verify()); + EXPECT_EQ(Bytes(payload), Bytes(decoded)); + } +} + +TEST(FlatBuffersFramingTest, EmptyMetadataIsSafelySeparatedForProcessorValidation) { + for (size_t meta_size : {0u, 1u, 19u, 23u, 24u}) { + butil::IOBuf input; + input.append(Frame(std::string(meta_size, '\xff'), "x")); + brpc::ParseResult result = brpc::policy::ParseFlatBuffersMessage( + &input, nullptr, false, nullptr); + ASSERT_EQ(brpc::PARSE_OK, result.error()); + brpc::DestroyingPtr parsed( + static_cast(result.message())); + EXPECT_EQ(meta_size, parsed->meta.size()); + EXPECT_EQ("x", parsed->payload.to_string()); + EXPECT_TRUE(input.empty()); + } +} + +TEST(FlatBuffersSerializationTest, RejectsNullProtobufEmptyCompressionAndChecksum) { + google::protobuf::DescriptorProto protobuf; + Message empty; + Message valid = MakePayload(); + const google::protobuf::Message* invalid[] = {nullptr, &protobuf, &empty}; + for (const auto* request : invalid) { + brpc::Controller cntl; + butil::IOBuf serialized; + brpc::policy::SerializeFlatBuffersRequest(&serialized, &cntl, request); + EXPECT_TRUE(cntl.Failed()); + EXPECT_EQ(brpc::EREQUEST, cntl.ErrorCode()); + EXPECT_TRUE(serialized.empty()); + } + for (int option = 0; option < 2; ++option) { + brpc::Controller cntl; + if (option == 0) { + cntl.set_request_compress_type(brpc::COMPRESS_TYPE_GZIP); + } else { + cntl.set_request_checksum_type(brpc::CHECKSUM_TYPE_CRC32C); + } + butil::IOBuf serialized; + brpc::policy::SerializeFlatBuffersRequest(&serialized, &cntl, &valid); + EXPECT_EQ(brpc::EREQUEST, cntl.ErrorCode()); + EXPECT_TRUE(serialized.empty()); + } +} + +TEST(FlatBuffersSerializationTest, PayloadOnlyAndIndependentRetryHeadersKeepConstInput) { + Message mutable_request = MakePayload(8192); + memset(mutable_request.mutable_buf_begin(), 0xa5, + mutable_request.get_meta_size()); + const Message request = std::move(mutable_request); + const std::string original = Bytes(request); + const size_t prefix_size = request.get_meta_size(); + const std::string prefix(reinterpret_cast(request.data()) - + prefix_size, prefix_size); + const uint8_t* data = request.data(); + ServiceDescriptor descriptor; + ASSERT_EQ(0, descriptor.init({"brpc_fbtest", "Pack", "Echo", {41}})); + brpc::Controller cntl; + brpc::ControllerPrivateAccessor(&cntl).set_flatbuffers_method( + descriptor.method(0)); + cntl.request_attachment().append("\0attachment", 11); + butil::IOBuf serialized; + brpc::policy::SerializeFlatBuffersRequest(&serialized, &cntl, &request); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ(original, serialized.to_string()); + EXPECT_EQ(request.size(), serialized.size()); + butil::IOBuf first; + brpc::policy::PackFlatBuffersRequest(&first, nullptr, + 0x0102030405060708ULL, nullptr, &cntl, serialized, nullptr); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + const std::string first_snapshot = first.to_string(); + cntl.request_attachment().clear(); + cntl.request_attachment().append("second"); + butil::IOBuf second; + brpc::policy::PackFlatBuffersRequest(&second, nullptr, + 0x8877665544332211ULL, nullptr, &cntl, serialized, nullptr); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + const std::string second_snapshot = second.to_string(); + ASSERT_GE(first_snapshot.size(), kHeaderSize + kRequestMetaSize); + ASSERT_GE(second_snapshot.size(), kHeaderSize + kRequestMetaSize); + EXPECT_EQ("FRPC", first_snapshot.substr(0, 4)); + EXPECT_EQ(first_snapshot.size() - 12, Get32(first_snapshot, 4, true)); + EXPECT_EQ(24u, Get32(first_snapshot, 8, true)); + EXPECT_EQ(descriptor.index(), Get32(first_snapshot, 12)); + EXPECT_EQ(41u, Get32(first_snapshot, 16)); + EXPECT_EQ(request.size(), Get32(first_snapshot, 20)); + EXPECT_EQ(11u, Get32(first_snapshot, 24)); + EXPECT_EQ(0x0102030405060708ULL, Get64(first_snapshot, 28)); + EXPECT_EQ(0x8877665544332211ULL, Get64(second_snapshot, 28)); + EXPECT_EQ(6u, Get32(second_snapshot, 24)); + EXPECT_EQ(first_snapshot, first.to_string()); + EXPECT_EQ(original, serialized.to_string()); + EXPECT_EQ(data, request.data()); + EXPECT_EQ(prefix_size, request.get_meta_size()); + EXPECT_EQ(prefix, std::string(reinterpret_cast(request.data()) - + prefix_size, prefix_size)); + EXPECT_EQ(original, Bytes(request)); + EXPECT_EQ(original, first_snapshot.substr(36, request.size())); + EXPECT_EQ(original, second_snapshot.substr(36, request.size())); +} + +class RejectingAuthenticator : public brpc::Authenticator { +public: + int GenerateCredential(std::string* credential) const override { + ++generated; + *credential = "test-credential"; + return 0; + } + int VerifyCredential(const std::string&, const butil::EndPoint&, + brpc::AuthContext*) const override { + return -1; + } + mutable std::atomic generated{0}; +}; + +TEST(FlatBuffersSerializationTest, PackRejectsMissingMethodEmptyPayloadAndAuth) { + ServiceDescriptor descriptor; + ASSERT_EQ(0, descriptor.init({"brpc_fbtest", "PackErrors", "Echo", {7}})); + const Message request = MakePayload(); + RejectingAuthenticator auth; + for (int variant = 0; variant < 3; ++variant) { + brpc::Controller cntl; + if (variant != 0) { + brpc::ControllerPrivateAccessor(&cntl).set_flatbuffers_method( + descriptor.method(0)); + } + butil::IOBuf serialized; + if (variant != 1) { + serialized.append(Bytes(request)); + } + butil::IOBuf packed; + brpc::policy::PackFlatBuffersRequest(&packed, nullptr, 1, nullptr, + &cntl, serialized, variant == 2 ? &auth : nullptr); + EXPECT_TRUE(cntl.Failed()); + EXPECT_TRUE(packed.empty()); + } +} + +TEST_F(FlatBuffersProtocolTest, SynchronousAndAsynchronousPayloadAttachmentVariants) { + EXPECT_EQ(30, static_cast(brpc::PROTOCOL_FLATBUFFERS_RPC)); + for (size_t size : {0u, 1u, 63u, 1024u, 8193u, 65537u}) { + for (bool asynchronous : {false, true}) { + SCOPED_TRACE(size); + SCOPED_TRACE(asynchronous); + const Message request = MakePayload(size, size + 31, size % 19); + Message response; + brpc::Controller cntl; + std::string attachment(size / 3, '\0'); + for (size_t i = 0; i < attachment.size(); ++i) { + attachment[i] = static_cast(i % 251); + } + cntl.request_attachment().append(attachment); + Completion done; + channel.FBCallMethod(method(41), &cntl, &request, &response, + asynchronous ? &done : nullptr); + if (asynchronous) { + ASSERT_TRUE(done.Wait()); + EXPECT_EQ(1, done.calls()); + } + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ(method(41), cntl.flatbuffers_method()); + EXPECT_EQ(brpc::PROTOCOL_FLATBUFFERS_RPC, cntl.request_protocol()); + ExpectReply(response, request, method(41)); + EXPECT_EQ(attachment, cntl.response_attachment().to_string()); + } + } +} + +TEST_F(FlatBuffersProtocolTest, ConcurrentCallsShareOneConstMessageWithoutMutation) { + Message mutable_request = MakePayload(16385, 9981, 256); + memset(mutable_request.mutable_buf_begin(), 0x3d, + mutable_request.get_meta_size()); + const Message request = std::move(mutable_request); + const std::string original = Bytes(request); + const size_t prefix_size = request.get_meta_size(); + const std::string prefix(reinterpret_cast(request.data()) - + prefix_size, prefix_size); + const MethodDescriptor* echo = method(); + const MethodDescriptor* alternate = method(41); + std::vector clients; + for (int client = 0; client < 6; ++client) { + clients.emplace_back([&, client] { + for (int call = 0; call < 8; ++call) { + brpc::Controller cntl; + Message response; + const std::string attachment = std::to_string(client) + ":" + + std::to_string(call); + cntl.request_attachment().append(attachment); + const MethodDescriptor* selected = call % 2 ? echo : alternate; + channel.FBCallMethod(selected, &cntl, &request, &response, nullptr); + EXPECT_FALSE(cntl.Failed()) << cntl.ErrorText(); + if (!cntl.Failed()) { + ExpectReply(response, request, selected); + EXPECT_EQ(attachment, cntl.response_attachment().to_string()); + } + } + }); + } + for (auto& client : clients) { + client.join(); + } + EXPECT_EQ(48, service.entered.load()); + EXPECT_EQ(original, Bytes(request)); + EXPECT_EQ(prefix_size, request.get_meta_size()); + EXPECT_EQ(prefix, std::string(reinterpret_cast(request.data()) - + prefix_size, prefix_size)); +} + +TEST_F(FlatBuffersProtocolTest, UnknownServiceAndMethodDoNotDispatch) { + ServiceDescriptor unknown_service; + ServiceDescriptor unknown_method; + ASSERT_EQ(0, unknown_service.init( + {"brpc_fbtest", "MissingService", "Echo", {7}})); + ASSERT_EQ(0, unknown_method.init( + {"brpc_fbtest", "ProtocolService", "Missing", {12345}})); + const Message request = MakePayload(); + for (const MethodDescriptor* missing : {unknown_service.method(0), + unknown_method.method(0)}) { + Message response; + brpc::Controller cntl; + channel.FBCallMethod(missing, &cntl, &request, &response, nullptr); + EXPECT_EQ(brpc::ENOMETHOD, cntl.ErrorCode()) << cntl.ErrorText(); + EXPECT_EQ(0u, response.size()); + } + EXPECT_EQ(0, service.entered.load()); + brpc::Controller cntl; + Message response; + channel.FBCallMethod(method(), &cntl, &request, &response, nullptr); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ExpectReply(response, request, method()); +} + +TEST_F(FlatBuffersProtocolTest, ServiceVerifiesInvalidPayloadAndRemainsUsable) { + for (size_t size : {1u, 3u, 4u, 16u, 65u}) { + butil::IOBuf bytes; + bytes.append(std::string(size, '\xff')); + Message invalid; + ASSERT_TRUE(invalid.parse_msg_from_iobuf(bytes, size, 0)); + ASSERT_FALSE(invalid.Verify()); + brpc::Controller cntl; + Message response; + channel.FBCallMethod(method(), &cntl, &invalid, &response, nullptr); + EXPECT_EQ(brpc::EREQUEST, cntl.ErrorCode()) << cntl.ErrorText(); + EXPECT_EQ(0u, response.size()); + } + EXPECT_EQ(5, service.rejected.load()); + const Message request = MakePayload(); + Message response; + brpc::Controller cntl; + channel.FBCallMethod(method(), &cntl, &request, &response, nullptr); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ExpectReply(response, request, method()); + EXPECT_EQ(6, service.entered.load()); +} + +TEST_F(FlatBuffersProtocolTest, AsynchronousValidationErrorsRunCallbackOnce) { + const Message request = MakePayload(); + brpc::Channel protobuf_channel; + ASSERT_EQ(0, InitChannel(&protobuf_channel, server.listen_address(), + "baidu_std")); + for (int variant = 0; variant < 4; ++variant) { + SCOPED_TRACE(variant); + Message response; + brpc::Controller cntl; + Completion done; + brpc::Channel* selected = variant == 0 ? &protobuf_channel : &channel; + selected->FBCallMethod(variant == 1 ? nullptr : method(), &cntl, + variant == 2 ? nullptr : &request, + variant == 3 ? nullptr : &response, &done); + ASSERT_TRUE(done.Wait()); + EXPECT_EQ(1, done.calls()); + EXPECT_TRUE(cntl.Failed()); + } + EXPECT_EQ(0, service.entered.load()); +} + +TEST_F(FlatBuffersProtocolTest, RejectsProtobufOnLegacyCallbackWithoutDispatch) { + google::protobuf::DescriptorProto request; + google::protobuf::DescriptorProto response; + request.set_name("not-a-flatbuffer"); + brpc::Controller cntl; + Completion done; + channel.CallMethod(nullptr, &cntl, &request, &response, &done); + ASSERT_TRUE(done.Wait()); + EXPECT_EQ(1, done.calls()); + EXPECT_TRUE(cntl.Failed()); + EXPECT_EQ(0, service.entered.load()); +} + +TEST_F(FlatBuffersProtocolTest, UnsupportedCompressionChecksumAndStreamsFailRpc) { + const Message request = MakePayload(); + for (int variant = 0; variant < 3; ++variant) { + brpc::Controller cntl; + Message response; + brpc::StreamId stream = brpc::INVALID_STREAM_ID; + if (variant == 0) { + cntl.set_request_compress_type(brpc::COMPRESS_TYPE_GZIP); + } else if (variant == 1) { + cntl.set_request_checksum_type(brpc::CHECKSUM_TYPE_CRC32C); + } else { + ASSERT_EQ(0, brpc::StreamCreate(&stream, cntl, nullptr)); + } + Completion done; + channel.FBCallMethod(method(), &cntl, &request, &response, &done); + ASSERT_TRUE(done.Wait()); + EXPECT_EQ(1, done.calls()); + EXPECT_TRUE(cntl.Failed()); + if (stream != brpc::INVALID_STREAM_ID) { + brpc::StreamClose(stream); + } + } + EXPECT_EQ(0, service.entered.load()); + for (int variant = 0; variant < 2; ++variant) { + service.response_compressed = variant == 0; + service.response_checksummed = variant == 1; + brpc::Controller cntl; + Message response; + channel.FBCallMethod(method(), &cntl, &request, &response, nullptr); + EXPECT_EQ(brpc::ERESPONSE, cntl.ErrorCode()) << cntl.ErrorText(); + EXPECT_EQ(0u, response.size()); + } +} + +TEST_F(FlatBuffersProtocolTest, ClientAuthenticationIsRejected) { + RejectingAuthenticator auth; + brpc::ChannelOptions options; + options.protocol = "fb_rpc"; + options.auth = &auth; + options.timeout_ms = kRpcTimeoutMs; + options.max_retry = 0; + brpc::Channel authenticated; + ASSERT_EQ(0, authenticated.Init(server.listen_address(), &options)); + const Message request = MakePayload(); + brpc::Controller cntl; + Message response; + authenticated.FBCallMethod(method(), &cntl, &request, &response, nullptr); + EXPECT_TRUE(cntl.Failed()); + EXPECT_EQ(0, service.entered.load()); +} + +TEST(FlatBuffersLifecycleTest, AuthenticationOnServerCannotBeBypassed) { + PayloadService service; + RejectingAuthenticator auth; + brpc::Server server; + ASSERT_EQ(0, server.AddFlatBuffersService( + &service, brpc::SERVER_DOESNT_OWN_SERVICE)); + brpc::ServerOptions options; + options.auth = &auth; + ASSERT_EQ(0, server.Start("127.0.0.1:0", &options)); + brpc::Channel channel; + ASSERT_EQ(0, InitChannel(&channel, server.listen_address())); + const Message request = MakePayload(); + Message response; + brpc::Controller cntl; + channel.FBCallMethod(service.GetDescriptor()->method(0), &cntl, &request, + &response, nullptr); + EXPECT_TRUE(cntl.Failed()); + EXPECT_EQ(0, service.entered.load()); + EXPECT_EQ(0, server.Stop(0)); + EXPECT_EQ(0, server.Join()); +} + +TEST(FlatBuffersLifecycleTest, StableIdsSurviveReorderRemovalAndRestart) { + PayloadService original; + PayloadService reordered({"brpc_fbtest", "ProtocolService", + "Alternate Echo", {41, 7}}); + brpc::Server server; + ASSERT_EQ(0, server.AddFlatBuffersService( + &original, brpc::SERVER_DOESNT_OWN_SERVICE)); + ASSERT_EQ(0, server.Start("127.0.0.1:0", nullptr)); + const ServiceDescriptor* client_descriptor = original.GetDescriptor(); + const Message request = MakePayload(); + for (int version = 0; version < 2; ++version) { + brpc::Channel channel; + ASSERT_EQ(0, InitChannel(&channel, server.listen_address())); + for (int id : {7, 41, 99}) { + SCOPED_TRACE(version); + SCOPED_TRACE(id); + const MethodDescriptor* method = client_descriptor->FindMethodByIndex(id); + ASSERT_NE(nullptr, method); + brpc::Controller cntl; + Message response; + channel.FBCallMethod(method, &cntl, &request, &response, nullptr); + if (version == 1 && id == 99) { + EXPECT_EQ(brpc::ENOMETHOD, cntl.ErrorCode()); + } else { + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ExpectReply(response, request, method); + } + } + ASSERT_EQ(0, server.Stop(0)); + ASSERT_EQ(0, server.Join()); + if (version == 0) { + ASSERT_EQ(0, server.RemoveFlatBuffersService(&original)); + EXPECT_EQ(0, server.GetFlatBuffersServiceCount()); + ASSERT_EQ(0, server.AddFlatBuffersService( + &reordered, brpc::SERVER_DOESNT_OWN_SERVICE)); + EXPECT_EQ(1, server.GetFlatBuffersServiceCount()); + ASSERT_EQ(0, server.Start("127.0.0.1:0", nullptr)); + } + } + EXPECT_EQ(3, original.entered.load()); + EXPECT_EQ(2, reordered.entered.load()); + server.ClearServices(); + EXPECT_EQ(0, server.GetFlatBuffersServiceCount()); +} + +class NullDescriptorService : public brpc::flatbuffers::Service { +public: + explicit NullDescriptorService(std::atomic* destroyed) + : _destroyed(destroyed) {} + ~NullDescriptorService() override { ++*_destroyed; } + const ServiceDescriptor* GetDescriptor() override { return nullptr; } + void FBCallMethod(const MethodDescriptor*, + google::protobuf::RpcController* controller, + const Message*, Message*, + google::protobuf::Closure* done) override { + brpc::ClosureGuard done_guard(done); + controller->SetFailed("Invalid service was dispatched"); + ADD_FAILURE() << "Invalid service was dispatched"; + } +private: + std::atomic* _destroyed; +}; + +TEST(FlatBuffersLifecycleTest, OwnershipFailureWrongInstanceRemoveAndClear) { + std::atomic owned_destroyed{0}; + std::atomic duplicate_destroyed{0}; + std::atomic invalid_destroyed{0}; + std::atomic borrowed_destroyed{0}; + { + PayloadService borrowed({"brpc_fbtest", "Borrowed", "Echo", {7}}, + &borrowed_destroyed); + brpc::Server server; + ASSERT_EQ(0, server.AddFlatBuffersService( + &borrowed, brpc::SERVER_DOESNT_OWN_SERVICE)); + std::unique_ptr owned(new PayloadService( + {"brpc_fbtest", "Owned", "Echo", {41}}, &owned_destroyed)); + ASSERT_EQ(0, server.AddFlatBuffersService( + owned.get(), brpc::SERVER_OWNS_SERVICE)); + owned.release(); + EXPECT_EQ(2, server.GetFlatBuffersServiceCount()); + for (brpc::ServiceOwnership ownership : {brpc::SERVER_OWNS_SERVICE, + brpc::SERVER_DOESNT_OWN_SERVICE}) { + std::unique_ptr duplicate(new PayloadService( + {"brpc_fbtest", "Owned", "Echo", {41}}, &duplicate_destroyed)); + const int before = duplicate_destroyed.load(); + EXPECT_NE(0, server.AddFlatBuffersService(duplicate.get(), ownership)); + EXPECT_EQ(before, duplicate_destroyed.load()); + EXPECT_NE(0, server.RemoveFlatBuffersService(duplicate.get())); + EXPECT_EQ(2, server.GetFlatBuffersServiceCount()); + std::unique_ptr invalid( + new NullDescriptorService(&invalid_destroyed)); + const int invalid_before = invalid_destroyed.load(); + EXPECT_NE(0, server.AddFlatBuffersService(invalid.get(), ownership)); + EXPECT_EQ(invalid_before, invalid_destroyed.load()); + } + EXPECT_EQ(2, duplicate_destroyed.load()); + EXPECT_EQ(2, invalid_destroyed.load()); + EXPECT_NE(0, server.AddFlatBuffersService(nullptr, brpc::SERVER_OWNS_SERVICE)); + EXPECT_NE(0, server.RemoveFlatBuffersService(nullptr)); + ASSERT_EQ(0, server.RemoveFlatBuffersService(&borrowed)); + EXPECT_EQ(0, borrowed_destroyed.load()); + EXPECT_EQ(1, server.GetFlatBuffersServiceCount()); + ASSERT_EQ(0, server.AddFlatBuffersService( + &borrowed, brpc::SERVER_DOESNT_OWN_SERVICE)); + server.ClearServices(); + EXPECT_EQ(0, server.GetFlatBuffersServiceCount()); + EXPECT_EQ(1, owned_destroyed.load()); + EXPECT_EQ(0, borrowed_destroyed.load()); + server.ClearServices(); + EXPECT_EQ(1, owned_destroyed.load()); + } + EXPECT_EQ(1, borrowed_destroyed.load()); +} + +bool WaitForServerConcurrency(const brpc::Server& server, int expected) { + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::milliseconds(kWaitTimeoutMs); + do { + if (server.Concurrency() == expected) { + return true; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } while (std::chrono::steady_clock::now() < deadline); + return server.Concurrency() == expected; +} + +class RejectAllInterceptor : public brpc::Interceptor { +public: + bool Accept(const brpc::Controller* cntl, int& error_code, + std::string& error_text) const override { + ++calls; + EXPECT_EQ(brpc::PROTOCOL_FLATBUFFERS_RPC, cntl->request_protocol()); + EXPECT_NE(nullptr, cntl->flatbuffers_method()); + error_code = EACCES; + error_text = "Rejected by test interceptor"; + return false; + } + mutable std::atomic calls{0}; +}; + +TEST(FlatBuffersLifecycleTest, InterceptorRejectsBeforeServiceAndReleasesConcurrency) { + PayloadService service; + RejectAllInterceptor interceptor; + brpc::Server server; + ASSERT_EQ(0, server.AddFlatBuffersService( + &service, brpc::SERVER_DOESNT_OWN_SERVICE)); + brpc::ServerOptions options; + options.interceptor = &interceptor; + options.max_concurrency = 1; + options.method_max_concurrency = 1; + ASSERT_EQ(0, server.Start("127.0.0.1:0", &options)); + brpc::Channel channel; + ASSERT_EQ(0, InitChannel(&channel, server.listen_address())); + const Message request = MakePayload(); + for (int call = 0; call < 3; ++call) { + brpc::Controller cntl; + Message response; + channel.FBCallMethod(service.GetDescriptor()->method(0), &cntl, + &request, &response, nullptr); + EXPECT_EQ(EACCES, cntl.ErrorCode()) << cntl.ErrorText(); + EXPECT_EQ(call + 1, interceptor.calls.load()); + EXPECT_EQ(0, service.entered.load()); + EXPECT_EQ(0u, response.size()); + EXPECT_TRUE(WaitForServerConcurrency(server, 0)); + } +} + +TEST(FlatBuffersLifecycleTest, PublicControllerSecurityModeMatchesServerOptions) { + for (bool security_mode : {false, true}) { + SCOPED_TRACE(security_mode); + PayloadService service; + brpc::Server server; + ASSERT_EQ(0, server.AddFlatBuffersService( + &service, brpc::SERVER_DOESNT_OWN_SERVICE)); + brpc::ServerOptions options; + options.has_builtin_services = !security_mode; + ASSERT_EQ(security_mode, options.security_mode()); + ASSERT_EQ(0, server.Start("127.0.0.1:0", &options)); + brpc::Channel channel; + ASSERT_EQ(0, InitChannel(&channel, server.listen_address())); + const Message request = MakePayload(); + brpc::Controller cntl; + Message response; + const MethodDescriptor* method = service.GetDescriptor()->method(0); + channel.FBCallMethod(method, &cntl, &request, &response, nullptr); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ExpectReply(response, request, method); + EXPECT_EQ(1, service.entered.load()); + EXPECT_EQ(security_mode ? 1 : 0, service.security_mode_calls.load()); + } +} + +int StartWithEphemeralInternalPort(brpc::Server* server, + brpc::ServerOptions* options) { + for (int attempt = 0; attempt < 10; ++attempt) { + butil::EndPoint endpoint; + if (butil::str2endpoint("127.0.0.1:0", &endpoint) != 0) { + return -1; + } + butil::fd_guard reserved(butil::tcp_listen(endpoint)); + if (reserved < 0 || butil::get_local_side(reserved, &endpoint) != 0) { + continue; + } + options->internal_port = endpoint.port; + // internal_port does not support 0. Retry if its ephemeral reservation + // is taken between releasing it and starting the two server listeners. + reserved.reset(-1); + if (server->Start("127.0.0.1:0", options) == 0) { + return 0; + } + } + return -1; +} + +TEST(FlatBuffersLifecycleTest, InternalPortRejectsOrdinaryFlatBuffersServices) { + PayloadService service; + brpc::Server server; + ASSERT_EQ(0, server.AddFlatBuffersService( + &service, brpc::SERVER_DOESNT_OWN_SERVICE)); + brpc::ServerOptions options; + ASSERT_EQ(0, StartWithEphemeralInternalPort(&server, &options)); + ASSERT_TRUE(server.options().security_mode()); + const butil::EndPoint internal_endpoint(server.listen_address().ip, + options.internal_port); + brpc::Channel internal_channel; + brpc::Channel public_channel; + ASSERT_EQ(0, InitChannel(&internal_channel, internal_endpoint)); + ASSERT_EQ(0, InitChannel(&public_channel, server.listen_address())); + const Message request = MakePayload(); + const MethodDescriptor* method = service.GetDescriptor()->method(0); + brpc::Controller rejected; + Message rejected_response; + internal_channel.FBCallMethod(method, &rejected, &request, + &rejected_response, nullptr); + EXPECT_EQ(EPERM, rejected.ErrorCode()) << rejected.ErrorText(); + EXPECT_EQ(0, service.entered.load()); + EXPECT_EQ(0u, rejected_response.size()); + + brpc::Controller accepted; + Message response; + public_channel.FBCallMethod(method, &accepted, &request, &response, nullptr); + ASSERT_FALSE(accepted.Failed()) << accepted.ErrorText(); + ExpectReply(response, request, method); + EXPECT_EQ(1, service.entered.load()); + EXPECT_EQ(1, service.security_mode_calls.load()); +} + +void ExerciseMethodConcurrency(bool specific_limit) { + PayloadService service; + brpc::Server server; + ASSERT_EQ(0, server.AddFlatBuffersService( + &service, brpc::SERVER_DOESNT_OWN_SERVICE)); + const MethodDescriptor* echo = service.GetDescriptor()->FindMethodByIndex(7); + if (specific_limit) { + server.MaxConcurrencyOf(echo->full_name()) = 1; + } + brpc::ServerOptions options; + options.max_concurrency = 8; + options.method_max_concurrency = specific_limit ? 2 : 1; + ASSERT_EQ(0, server.Start("127.0.0.1:0", &options)); + brpc::Channel channel; + ASSERT_EQ(0, InitChannel(&channel, server.listen_address())); + const Message request = MakePayload(); + for (int id : {7, 41}) { + SCOPED_TRACE(id); + const MethodDescriptor* method = service.GetDescriptor()->FindMethodByIndex(id); + const int limit = specific_limit && id == 41 ? 2 : 1; + const brpc::Server& const_server = server; + EXPECT_EQ(specific_limit && id == 7 ? 1 : 0, + const_server.MaxConcurrencyOf(method->full_name())); + const auto* property = brpc::ServerPrivateAccessor(&server). + FindFlatBuffersMethodPropertyByIndex(method->service()->index(), id); + ASSERT_NE(nullptr, property); + ASSERT_NE(nullptr, property->status); + EXPECT_EQ(limit, property->status->MaxConcurrency()); + const int entered_before = service.entered.load(); + service.HoldNext(); + brpc::Controller held_cntl; + held_cntl.set_timeout_ms(kWaitTimeoutMs); + Message held_response; + Completion held_done; + channel.FBCallMethod(method, &held_cntl, &request, &held_response, &held_done); + const bool held = service.WaitHeld(); + EXPECT_TRUE(held); + // No fatal assertions while a service completion is held: all exits + // below must release it before the server's destructor calls Join(). + if (held) { + EXPECT_TRUE(WaitForServerConcurrency(server, 1)); + for (int attempt = 0; attempt < 2; ++attempt) { + brpc::Controller contender; + Message contender_response; + channel.FBCallMethod(method, &contender, &request, + &contender_response, nullptr); + if (limit == 1) { + EXPECT_EQ(brpc::ELIMIT, contender.ErrorCode()) + << contender.ErrorText(); + EXPECT_EQ(entered_before + 1, service.entered.load()); + EXPECT_EQ(0u, contender_response.size()); + } else { + EXPECT_FALSE(contender.Failed()) << contender.ErrorText(); + if (!contender.Failed()) { + ExpectReply(contender_response, request, method); + } + EXPECT_EQ(entered_before + attempt + 2, service.entered.load()); + } + EXPECT_TRUE(WaitForServerConcurrency(server, 1)); + } + } + service.ReleaseHeld(); + const bool completed = held_done.Wait(); + EXPECT_TRUE(completed); + EXPECT_EQ(1, held_done.calls()); + EXPECT_FALSE(held_cntl.Failed()) << held_cntl.ErrorText(); + if (completed && !held_cntl.Failed()) { + ExpectReply(held_response, request, method); + } + EXPECT_TRUE(WaitForServerConcurrency(server, 0)); + brpc::Controller recovered; + Message recovered_response; + channel.FBCallMethod(method, &recovered, &request, &recovered_response, nullptr); + EXPECT_FALSE(recovered.Failed()) << recovered.ErrorText(); + if (!recovered.Failed()) { + ExpectReply(recovered_response, request, method); + } + EXPECT_TRUE(WaitForServerConcurrency(server, 0)); + } +} + +TEST(FlatBuffersLifecycleTest, DefaultMethodConcurrencyRecoversAfterRejection) { + ExerciseMethodConcurrency(false); +} + +TEST(FlatBuffersLifecycleTest, SpecificMethodConcurrencyOverridesDefaultAndRecovers) { + ExerciseMethodConcurrency(true); +} + +class RetryOnlyAccessDenied : public brpc::RetryPolicy { +public: + bool DoRetry(const brpc::Controller* cntl) const override { + if (cntl->ErrorCode() == EACCES) { + ++retries; + return true; + } + return false; + } + mutable std::atomic retries{0}; +}; + +TEST_F(FlatBuffersProtocolTest, RealNetworkRetryPreservesConstRequestAndAttachment) { + RetryOnlyAccessDenied policy; + brpc::ChannelOptions options; + options.protocol = "fb_rpc"; + options.connection_type = "single"; + options.timeout_ms = kRpcTimeoutMs; + options.connect_timeout_ms = kRpcTimeoutMs; + options.max_retry = 2; + options.retry_policy = &policy; + brpc::Channel retry_channel; + ASSERT_EQ(0, retry_channel.Init(server.listen_address(), &options)); + Message mutable_request = MakePayload(8193, 7654, 19); + memset(mutable_request.mutable_buf_begin(), 0x69, + mutable_request.get_meta_size()); + const Message request = std::move(mutable_request); + const std::string original = Bytes(request); + const size_t prefix_size = request.get_meta_size(); + const std::string prefix(reinterpret_cast(request.data()) - + prefix_size, prefix_size); + const std::string attachment("\0retry\xff", 7); + service.fail_next_code = EACCES; + brpc::Controller cntl; + cntl.request_attachment().append(attachment); + Message response; + Completion done; + retry_channel.FBCallMethod(method(41), &cntl, &request, &response, &done); + ASSERT_TRUE(done.Wait()); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ(1, done.calls()); + EXPECT_EQ(1, cntl.retried_count()); + EXPECT_EQ(1, policy.retries.load()); + EXPECT_EQ(2, service.entered.load()); + ExpectReply(response, request, method(41)); + EXPECT_EQ(attachment, cntl.response_attachment().to_string()); + EXPECT_EQ(original, Bytes(request)); + EXPECT_EQ(prefix_size, request.get_meta_size()); + EXPECT_EQ(prefix, std::string(reinterpret_cast(request.data()) - + prefix_size, prefix_size)); + + // The same policy must not retry a different application error. + service.fail_next_code = brpc::EREQUEST; + brpc::Controller no_retry; + Message no_retry_response; + retry_channel.FBCallMethod(method(), &no_retry, &request, + &no_retry_response, nullptr); + EXPECT_EQ(brpc::EREQUEST, no_retry.ErrorCode()); + EXPECT_EQ(0, no_retry.retried_count()); + EXPECT_EQ(1, policy.retries.load()); + EXPECT_EQ(3, service.entered.load()); +} + +TEST_F(FlatBuffersProtocolTest, PooledAndShortConnectionsSupportSyncAndAsyncCalls) { + for (const char* connection_type : {"pooled", "short"}) { + SCOPED_TRACE(connection_type); + brpc::Channel selected; + ASSERT_EQ(0, InitChannel(&selected, server.listen_address(), + "fb_rpc", connection_type)); + for (size_t size : {0u, 8193u}) { + for (bool asynchronous : {false, true}) { + const Message request = MakePayload(size, size + 23, 7); + brpc::Controller cntl; + cntl.request_attachment().append("connection-attachment"); + Message response; + Completion done; + selected.FBCallMethod(method(), &cntl, &request, &response, + asynchronous ? &done : nullptr); + if (asynchronous) { + ASSERT_TRUE(done.Wait()); + EXPECT_EQ(1, done.calls()); + } + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + const brpc::ConnectionType expected_type = + std::string(connection_type) == "pooled" ? + brpc::CONNECTION_TYPE_POOLED : brpc::CONNECTION_TYPE_SHORT; + EXPECT_EQ(expected_type, cntl.connection_type()); + ExpectReply(response, request, method()); + EXPECT_EQ("connection-attachment", + cntl.response_attachment().to_string()); + } + } + } + EXPECT_EQ(8, service.entered.load()); +} + +struct DeferredJoinState { + ~DeferredJoinState() { service.ReleaseHeld(); } + PayloadService service; + Message request = MakePayload(); + Message response; + brpc::Controller cntl; + Completion done; + // Destroy the server before the request/controller/service on early exits. + brpc::Server server; + std::mutex mutex; + std::condition_variable condition; + bool join_started = false; + bool joined = false; + int join_result = -1; +}; + +TEST(FlatBuffersLifecycleTest, StopRejectsRemovalUntilDeferredDoneAndJoin) { + auto state = std::make_shared(); + ASSERT_EQ(0, state->server.AddFlatBuffersService( + &state->service, brpc::SERVER_DOESNT_OWN_SERVICE)); + ASSERT_EQ(0, state->server.Start("127.0.0.1:0", nullptr)); + brpc::Channel channel; + ASSERT_EQ(0, InitChannel(&channel, state->server.listen_address())); + const MethodDescriptor* method = state->service.GetDescriptor()->method(0); + state->service.HoldNext(); + state->cntl.set_timeout_ms(kWaitTimeoutMs); + channel.FBCallMethod(method, &state->cntl, &state->request, + &state->response, &state->done); + EXPECT_TRUE(state->service.WaitHeld()); + EXPECT_EQ(0, state->done.calls()); + EXPECT_NE(0, state->server.RemoveFlatBuffersService(&state->service)); + EXPECT_EQ(0, state->server.Stop(0)); + EXPECT_NE(0, state->server.RemoveFlatBuffersService(&state->service)); + EXPECT_EQ(1, state->server.GetFlatBuffersServiceCount()); + + std::thread joiner([state] { + { + std::lock_guard lock(state->mutex); + state->join_started = true; + state->condition.notify_all(); + } + const int result = state->server.Join(); + { + std::lock_guard lock(state->mutex); + state->join_result = result; + state->joined = true; + state->condition.notify_all(); + } + }); + { + std::unique_lock lock(state->mutex); + EXPECT_TRUE(state->condition.wait_for( + lock, std::chrono::milliseconds(kWaitTimeoutMs), + [&state] { return state->join_started; })); + EXPECT_FALSE(state->condition.wait_for( + lock, std::chrono::milliseconds(100), + [&state] { return state->joined; })); + } + EXPECT_EQ(0, state->done.calls()); + state->service.ReleaseHeld(); + const bool completed = state->done.Wait(); + EXPECT_TRUE(completed); + EXPECT_EQ(1, state->done.calls()); + EXPECT_FALSE(state->cntl.Failed()) << state->cntl.ErrorText(); + if (completed && !state->cntl.Failed()) { + ExpectReply(state->response, state->request, method); + } + bool joined; + { + std::unique_lock lock(state->mutex); + joined = state->condition.wait_for( + lock, std::chrono::milliseconds(kWaitTimeoutMs), + [&state] { return state->joined; }); + } + if (!joined) { + // A broken Join must fail rather than hang the suite. The thread owns + // every object it may still access, so detaching cannot use dead locals. + joiner.detach(); + FAIL() << "Server::Join did not return after deferred done completed"; + } + joiner.join(); + EXPECT_EQ(0, state->join_result); + ASSERT_EQ(0, state->server.RemoveFlatBuffersService(&state->service)); + EXPECT_EQ(0, state->server.GetFlatBuffersServiceCount()); +} + +// A normal protobuf service built from descriptors avoids another generated +// test proto while exercising the actual baidu_std serializer and dispatcher. +class DynamicEchoService : public google::protobuf::Service { +public: + DynamicEchoService() { + google::protobuf::FileDescriptorProto file; + file.set_name("flatbuffers_protocol_pb_compat.proto"); + file.set_package("brpc_fbtest_pb"); + auto* message = file.add_message_type(); + message->set_name("Payload"); + auto* field = message->add_field(); + field->set_name("text"); + field->set_number(1); + field->set_type(google::protobuf::FieldDescriptorProto::TYPE_STRING); + field->set_label(google::protobuf::FieldDescriptorProto::LABEL_OPTIONAL); + auto* service = file.add_service(); + service->set_name("EchoService"); + auto* method = service->add_method(); + method->set_name("Echo"); + method->set_input_type(".brpc_fbtest_pb.Payload"); + method->set_output_type(".brpc_fbtest_pb.Payload"); + const auto* built = _pool.BuildFile(file); + if (built) { + _descriptor = built->service(0); + _prototype = _factory.GetPrototype(built->message_type(0)); + } + } + const google::protobuf::ServiceDescriptor* GetDescriptor() override { + return _descriptor; + } + const google::protobuf::Message& GetRequestPrototype( + const google::protobuf::MethodDescriptor*) const override { + return *_prototype; + } + const google::protobuf::Message& GetResponsePrototype( + const google::protobuf::MethodDescriptor*) const override { + return *_prototype; + } + void CallMethod(const google::protobuf::MethodDescriptor*, + google::protobuf::RpcController* controller, + const google::protobuf::Message* request, + google::protobuf::Message* response, + google::protobuf::Closure* done) override { + brpc::ClosureGuard done_guard(done); + auto* cntl = static_cast(controller); + EXPECT_EQ(nullptr, cntl->flatbuffers_method()); + response->CopyFrom(*request); + } +private: + google::protobuf::DescriptorPool _pool; + google::protobuf::DynamicMessageFactory _factory; + const google::protobuf::ServiceDescriptor* _descriptor = nullptr; + const google::protobuf::Message* _prototype = nullptr; +}; + +TEST(FlatBuffersLifecycleTest, ControllerResetClearsContextBeforeOrdinaryProtobufRpc) { + PayloadService flatbuffers_service; + DynamicEchoService protobuf_service; + ASSERT_NE(nullptr, protobuf_service.GetDescriptor()); + brpc::Server server; + ASSERT_EQ(0, server.AddFlatBuffersService( + &flatbuffers_service, brpc::SERVER_DOESNT_OWN_SERVICE)); + ASSERT_EQ(0, server.AddService(&protobuf_service, brpc::SERVER_DOESNT_OWN_SERVICE)); + ASSERT_EQ(0, server.Start("127.0.0.1:0", nullptr)); + brpc::Channel fb_channel; + brpc::Channel pb_channel; + ASSERT_EQ(0, InitChannel(&fb_channel, server.listen_address())); + ASSERT_EQ(0, InitChannel(&pb_channel, server.listen_address(), "baidu_std")); + const MethodDescriptor* fb_method = flatbuffers_service.GetDescriptor()->method(0); + const Message fb_request = MakePayload(); + Message fb_response; + brpc::Controller cntl; + fb_channel.FBCallMethod(fb_method, &cntl, &fb_request, &fb_response, nullptr); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ(fb_method, cntl.flatbuffers_method()); + cntl.Reset(); + EXPECT_EQ(nullptr, cntl.flatbuffers_method()); + EXPECT_EQ(nullptr, cntl.method()); + const auto* method = protobuf_service.GetDescriptor()->method(0); + std::unique_ptr request( + protobuf_service.GetRequestPrototype(method).New()); + std::unique_ptr response( + protobuf_service.GetResponsePrototype(method).New()); + const auto* field = request->GetDescriptor()->FindFieldByName("text"); + ASSERT_NE(nullptr, field); + request->GetReflection()->SetString(request.get(), field, "ordinary protobuf"); + pb_channel.CallMethod(method, &cntl, request.get(), response.get(), nullptr); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ(nullptr, cntl.flatbuffers_method()); + EXPECT_EQ(method, cntl.method()); + EXPECT_EQ("ordinary protobuf", + response->GetReflection()->GetString(*response, field)); + EXPECT_EQ(0, server.Stop(0)); + EXPECT_EQ(0, server.Join()); +} + +TEST(FlatBuffersLifecycleTest, AuthenticatedProtobufSocketCannotBypassFlatBuffersAuthCheck) { + PayloadService flatbuffers_service; + DynamicEchoService protobuf_service; + RejectingAuthenticator auth; + brpc::Server server; + ASSERT_NE(nullptr, protobuf_service.GetDescriptor()); + ASSERT_EQ(0, server.AddFlatBuffersService( + &flatbuffers_service, brpc::SERVER_DOESNT_OWN_SERVICE)); + ASSERT_EQ(0, server.AddService(&protobuf_service, brpc::SERVER_DOESNT_OWN_SERVICE)); + ASSERT_EQ(0, server.Start("127.0.0.1:0", nullptr)); + brpc::ChannelOptions options; + options.protocol = "baidu_std"; + options.connection_type = "single"; + options.auth = &auth; + options.timeout_ms = kRpcTimeoutMs; + options.connect_timeout_ms = kRpcTimeoutMs; + options.max_retry = 0; + brpc::Channel pb_channel; + brpc::Channel fb_channel; + ASSERT_EQ(0, pb_channel.Init(server.listen_address(), &options)); + // The socket pool key includes the auth pointer, but not the protocol. + options.protocol = "fb_rpc"; + ASSERT_EQ(0, fb_channel.Init(server.listen_address(), &options)); + const auto* pb_method = protobuf_service.GetDescriptor()->method(0); + std::unique_ptr pb_request( + protobuf_service.GetRequestPrototype(pb_method).New()); + std::unique_ptr pb_response( + protobuf_service.GetResponsePrototype(pb_method).New()); + const auto* field = pb_request->GetDescriptor()->FindFieldByName("text"); + ASSERT_NE(nullptr, field); + pb_request->GetReflection()->SetString(pb_request.get(), field, "prime auth socket"); + brpc::Controller first_pb; + pb_channel.CallMethod(pb_method, &first_pb, pb_request.get(), + pb_response.get(), nullptr); + ASSERT_FALSE(first_pb.Failed()) << first_pb.ErrorText(); + ASSERT_EQ(1, auth.generated.load()); + ASSERT_GT(first_pb.local_side().port, 0); + const butil::EndPoint first_connection = first_pb.local_side(); + brpc::ServerStatistics statistics; + server.GetStat(&statistics); + ASSERT_EQ(1u, statistics.connection_count); + + const Message request = MakePayload(); + Message response; + brpc::Controller rejected; + Completion done; + fb_channel.FBCallMethod(flatbuffers_service.GetDescriptor()->method(0), + &rejected, &request, &response, &done); + ASSERT_TRUE(done.Wait()); + EXPECT_EQ(1, done.calls()); + EXPECT_TRUE(rejected.Failed()) + << "Previously authenticated socket bypassed FRPC auth check"; + EXPECT_EQ(0u, response.size()); + EXPECT_EQ(0, flatbuffers_service.entered.load()); + EXPECT_EQ(1, auth.generated.load()); + + // Verify the primed connection stayed usable and did not authenticate again. + brpc::Controller second_pb; + pb_response->Clear(); + pb_channel.CallMethod(pb_method, &second_pb, pb_request.get(), + pb_response.get(), nullptr); + ASSERT_FALSE(second_pb.Failed()) << second_pb.ErrorText(); + EXPECT_EQ(first_connection, second_pb.local_side()); + EXPECT_EQ("prime auth socket", + pb_response->GetReflection()->GetString(*pb_response, field)); + EXPECT_EQ(1, auth.generated.load()); + server.GetStat(&statistics); + EXPECT_EQ(1u, statistics.connection_count); +} + +// Raw peers use both an absolute deadline and socket timeouts; a regression +// must fail a test instead of leaving accept/read/write blocked indefinitely. +bool ConfigureSocket(int fd) { + struct timeval timeout = {kRpcTimeoutMs / 1000, + (kRpcTimeoutMs % 1000) * 1000}; + if (setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)) != 0 || + setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(timeout)) != 0) { + return false; + } +#ifdef SO_NOSIGPIPE + const int enabled = 1; + if (setsockopt(fd, SOL_SOCKET, SO_NOSIGPIPE, &enabled, sizeof(enabled)) != 0) { + return false; + } +#endif + return true; +} + +bool WaitFd(int fd, short events, + const std::chrono::steady_clock::time_point& deadline) { + while (true) { + const auto remaining = std::chrono::duration_cast( + deadline - std::chrono::steady_clock::now()).count(); + if (remaining <= 0) { + return false; + } + struct pollfd descriptor = {fd, events, 0}; + const int rc = poll(&descriptor, 1, static_cast(remaining)); + if (rc > 0) { + return (descriptor.revents & (events | POLLHUP | POLLERR)) != 0; + } + if (rc == 0 || errno != EINTR) { + return false; + } + } +} + +bool ReadExactly(int fd, char* bytes, size_t size) { + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::milliseconds(kWaitTimeoutMs); + size_t offset = 0; + while (offset < size && WaitFd(fd, POLLIN, deadline)) { + const ssize_t count = recv(fd, bytes + offset, size - offset, 0); + if (count > 0) { + offset += count; + } else if (count == 0 || errno != EINTR) { + return false; + } + } + return offset == size; +} + +bool WriteExactly(int fd, const std::string& bytes) { + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::milliseconds(kWaitTimeoutMs); + size_t offset = 0; + while (offset < bytes.size() && WaitFd(fd, POLLOUT, deadline)) { +#ifdef MSG_NOSIGNAL + const int flags = MSG_NOSIGNAL; +#else + const int flags = 0; +#endif + const ssize_t count = send(fd, bytes.data() + offset, + bytes.size() - offset, flags); + if (count > 0) { + offset += count; + } else if (count == 0 || errno != EINTR) { + return false; + } + } + return offset == bytes.size(); +} + +bool ReadFrame(int fd, std::string* meta, std::string* payload) { + std::string header(kHeaderSize, '\0'); + if (!ReadExactly(fd, &header[0], header.size()) || + header.compare(0, 4, "FRPC") != 0) { + return false; + } + const uint32_t body_size = Get32(header, 4, true); + const uint32_t meta_size = Get32(header, 8, true); + if (meta_size > body_size || body_size > 1024 * 1024) { + return false; + } + std::string body(body_size, '\0'); + if (body_size && !ReadExactly(fd, &body[0], body.size())) { + return false; + } + *meta = body.substr(0, meta_size); + *payload = body.substr(meta_size); + return true; +} + +TEST_F(FlatBuffersProtocolTest, ExtendedRequestMetadataTraversesRealSocket) { + const Message request = MakePayload(8193, 7123, 17); + const std::string attachment("\0binary\xff", 8); + std::string meta = RequestMeta(method(41), request.size(), attachment.size(), + 0x1122334455667788ULL); + meta.append("unknown future fields", 21); + butil::fd_guard fd(butil::tcp_connect(server.listen_address(), nullptr, + kRpcTimeoutMs)); + ASSERT_GE(static_cast(fd), 0); + ASSERT_TRUE(ConfigureSocket(fd)); + const std::string wire = Frame(meta, Bytes(request) + attachment); + ASSERT_TRUE(WriteExactly(fd, wire.substr(0, 5))); + ASSERT_TRUE(WriteExactly(fd, wire.substr(5, 13))); + ASSERT_TRUE(WriteExactly(fd, wire.substr(18))); + std::string reply_meta; + std::string reply_payload; + ASSERT_TRUE(ReadFrame(fd, &reply_meta, &reply_payload)); + ASSERT_GE(reply_meta.size(), kResponseMetaSize); + EXPECT_EQ(0u, Get32(reply_meta, 0)); + EXPECT_EQ(0x1122334455667788ULL, Get64(reply_meta, 12)); + const uint32_t message_size = Get32(reply_meta, 4); + ASSERT_EQ(reply_payload.size(), message_size + attachment.size()); + EXPECT_EQ(attachment.size(), Get32(reply_meta, 8)); + EXPECT_EQ(attachment, reply_payload.substr(message_size)); + butil::IOBuf body; + body.append(reply_payload.data(), message_size); + Message response; + ASSERT_TRUE(response.parse_msg_from_iobuf(body, message_size, 0)); + ExpectReply(response, request, method(41)); +} + +TEST_F(FlatBuffersProtocolTest, MalformedRequestLengthsFailBeforeServiceDispatch) { + const Message request = MakePayload(); + const std::string payload = Bytes(request) + "att"; + for (int variant = 0; variant < 6; ++variant) { + SCOPED_TRACE(variant); + std::string meta = RequestMeta(method(), request.size(), 3, 991 + variant); + switch (variant) { + case 0: Put32(&meta, 8, 0); break; + case 1: Put32(&meta, 8, 0xffffffffu); break; + case 2: Put32(&meta, 12, 0xffffffffu); break; + case 3: Put32(&meta, 8, request.size() + 1); break; + case 4: Put32(&meta, 12, 2); break; + case 5: Put32(&meta, 4, 0xffffffffu); break; + } + butil::fd_guard fd(butil::tcp_connect(server.listen_address(), nullptr, + kRpcTimeoutMs)); + ASSERT_GE(static_cast(fd), 0); + ASSERT_TRUE(ConfigureSocket(fd)); + ASSERT_TRUE(WriteExactly(fd, Frame(meta, payload))); + std::string reply_meta; + std::string reply_payload; + ASSERT_TRUE(ReadFrame(fd, &reply_meta, &reply_payload)); + ASSERT_GE(reply_meta.size(), kResponseMetaSize); + EXPECT_EQ(static_cast(variant == 5 ? brpc::ENOMETHOD : + brpc::EREQUEST), + Get32(reply_meta, 0)); + EXPECT_EQ(static_cast(991 + variant), Get64(reply_meta, 12)); + EXPECT_EQ(0u, Get32(reply_meta, 4)); + EXPECT_EQ(0u, Get32(reply_meta, 8)); + EXPECT_TRUE(reply_payload.empty()); + } + EXPECT_EQ(0, service.entered.load()); + brpc::Controller cntl; + Message response; + channel.FBCallMethod(method(), &cntl, &request, &response, nullptr); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ExpectReply(response, request, method()); +} + +TEST_F(FlatBuffersProtocolTest, TruncatedMetadataAndImpossibleHeaderCloseConnection) { + std::vector frames; + for (size_t size : {0u, 1u, 8u, 19u, 23u}) { + frames.push_back(Frame(std::string(size, '\xff'), "invalid")); + } + frames.push_back(Header(0, 24)); + const Message warmup_request = MakePayload(); + int warmups = 0; + for (const auto& wire : frames) { + SCOPED_TRACE(wire.size()); + butil::fd_guard fd(butil::tcp_connect(server.listen_address(), nullptr, + kRpcTimeoutMs)); + ASSERT_GE(static_cast(fd), 0); + ASSERT_TRUE(ConfigureSocket(fd)); + // NSHEAD precedes FRPC during initial multi-protocol discovery and + // needs 28 bytes. First establish FRPC on this exact connection. + const uint64_t correlation_id = 9001 + warmups; + ASSERT_TRUE(WriteExactly(fd, Frame( + RequestMeta(method(), warmup_request.size(), 0, correlation_id), + Bytes(warmup_request)))); + std::string reply_meta; + std::string reply_payload; + ASSERT_TRUE(ReadFrame(fd, &reply_meta, &reply_payload)); + ASSERT_GE(reply_meta.size(), kResponseMetaSize); + ASSERT_EQ(0u, Get32(reply_meta, 0)); + EXPECT_EQ(correlation_id, Get64(reply_meta, 12)); + ASSERT_EQ(reply_payload.size(), Get32(reply_meta, 4)); + EXPECT_EQ(0u, Get32(reply_meta, 8)); + butil::IOBuf reply_bytes; + reply_bytes.append(reply_payload); + Message warmup_response; + ASSERT_TRUE(warmup_response.parse_msg_from_iobuf( + reply_bytes, reply_payload.size(), 0)); + ExpectReply(warmup_response, warmup_request, method()); + EXPECT_EQ(++warmups, service.entered.load()); + + ASSERT_TRUE(WriteExactly(fd, wire)); + char byte; + ssize_t count; + do { + count = recv(fd, &byte, 1, 0); + } while (count < 0 && errno == EINTR); + // A receive timeout is not evidence that the server rejected the frame. + EXPECT_TRUE(count == 0 || (count < 0 && errno == ECONNRESET)) + << "recv=" << count << " errno=" << errno; + EXPECT_EQ(warmups, service.entered.load()); + } + EXPECT_EQ(static_cast(frames.size()), service.entered.load()); +} + +class ScriptedPeer { +public: + using Reply = std::function; + bool Start(Reply reply) { + butil::EndPoint endpoint; + if (butil::str2endpoint("127.0.0.1:0", &endpoint) != 0) { + return false; + } + _listener.reset(butil::tcp_listen(endpoint)); + if (_listener < 0 || butil::get_local_side(_listener, &_endpoint) != 0) { + return false; + } + _thread = std::thread([this, reply] { + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::milliseconds(kWaitTimeoutMs); + if (!WaitFd(_listener, POLLIN, deadline)) { + return; + } + butil::fd_guard connection(accept(_listener, nullptr, nullptr)); + if (connection < 0 || !ConfigureSocket(connection)) { + return; + } + std::string request_meta; + std::string request_payload; + if (!ReadFrame(connection, &request_meta, &request_payload) || + request_meta.size() < kRequestMetaSize) { + return; + } + _served = WriteExactly(connection, reply(request_meta, request_payload)); + // Do not let EOF accidentally turn an accepted malformed response + // into an RPC failure. Keep the connection open until the assertion. + std::unique_lock lock(_mutex); + _condition.wait_for(lock, std::chrono::milliseconds(kWaitTimeoutMs), + [this] { return _release; }); + }); + return true; + } + ~ScriptedPeer() { Join(); } + void Join() { + { + std::lock_guard lock(_mutex); + _release = true; + _condition.notify_all(); + } + if (_thread.joinable()) { + _thread.join(); + } + } + butil::EndPoint endpoint() const { return _endpoint; } + bool served() const { return _served.load(); } +private: + butil::fd_guard _listener; + butil::EndPoint _endpoint; + std::thread _thread; + std::atomic _served{false}; + std::mutex _mutex; + std::condition_variable _condition; + bool _release = false; +}; + +TEST(FlatBuffersResponseTest, ExtendedResponseMetadataUsesDeclaredPayloadOffset) { + ScriptedPeer peer; + ASSERT_TRUE(peer.Start([](const std::string& request_meta, + const std::string& request_payload) { + const uint32_t message_size = Get32(request_meta, 8); + const uint32_t attachment_size = Get32(request_meta, 12); + std::string meta = ResponseMeta(message_size, attachment_size, + Get64(request_meta, 16)); + meta.append("\xff\x01\0\x7f\x03", 5); + return Frame(meta, request_payload); + })); + brpc::Channel channel; + ASSERT_EQ(0, InitChannel(&channel, peer.endpoint())); + ServiceDescriptor descriptor; + ASSERT_EQ(0, descriptor.init({"brpc_fbtest", "Peer", "Echo", {7}})); + const Message request = MakePayload(8193); + Message response; + brpc::Controller cntl; + cntl.request_attachment().append("\0tail", 5); + channel.FBCallMethod(descriptor.method(0), &cntl, &request, &response, nullptr); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ASSERT_TRUE(response.Verify()); + EXPECT_EQ(Bytes(request), Bytes(response)); + EXPECT_EQ(std::string("\0tail", 5), cntl.response_attachment().to_string()); + peer.Join(); + EXPECT_TRUE(peer.served()); +} + +TEST(FlatBuffersResponseTest, MalformedResponseFramesFailRpcNotMerelySchemaVerification) { + ServiceDescriptor descriptor; + ASSERT_EQ(0, descriptor.init({"brpc_fbtest", "Peer", "Echo", {41}})); + const Message request = MakePayload(); + for (int variant = 0; variant < 12; ++variant) { + SCOPED_TRACE(variant); + ScriptedPeer peer; + ASSERT_TRUE(peer.Start([variant](const std::string& request_meta, + const std::string& request_payload) { + std::string meta = ResponseMeta(Get32(request_meta, 8), 0, + Get64(request_meta, 16)); + std::string payload = request_payload; + switch (variant) { + case 0: meta.clear(); break; + case 1: meta.resize(1); break; + case 2: meta.resize(19); break; + case 3: Put32(&meta, 4, 0xffffffffu); break; + case 4: Put32(&meta, 8, 0xffffffffu); break; + case 5: Put32(&meta, 4, payload.size() + 1); break; + case 6: Put32(&meta, 8, 1); break; + case 7: payload.clear(); Put32(&meta, 4, 0); break; + case 8: return Header(0, 20); + case 9: return Header(0xffffffffu, 20); + case 10: Put64(&meta, 12, Get64(request_meta, 16) + (1ULL << 32)); break; + case 11: { + std::string wire = Frame(meta, payload); + wire.resize(wire.size() - 1); + return wire; + } + } + return Frame(meta, payload); + })); + brpc::Channel channel; + ASSERT_EQ(0, InitChannel(&channel, peer.endpoint())); + Message response; + brpc::Controller cntl; + Completion done; + channel.FBCallMethod(descriptor.method(0), &cntl, &request, &response, &done); + ASSERT_TRUE(done.Wait()); + EXPECT_EQ(1, done.calls()); + EXPECT_TRUE(cntl.Failed()) << "Malformed frame reported RPC success"; + if (variant >= 3 && variant <= 7) { + EXPECT_EQ(brpc::ERESPONSE, cntl.ErrorCode()) << cntl.ErrorText(); + } + EXPECT_EQ(0u, response.size()); + peer.Join(); + EXPECT_TRUE(peer.served()); + } +} + +TEST(FlatBuffersResponseTest, SchemaVerificationIsExplicitlyTheCallersResponsibility) { + ScriptedPeer peer; + ASSERT_TRUE(peer.Start([](const std::string& request_meta, const std::string&) { + const std::string invalid(16, '\xff'); + return Frame(ResponseMeta(invalid.size(), 0, Get64(request_meta, 16)), + invalid); + })); + brpc::Channel channel; + ASSERT_EQ(0, InitChannel(&channel, peer.endpoint())); + ServiceDescriptor descriptor; + ASSERT_EQ(0, descriptor.init({"brpc_fbtest", "Peer", "Echo", {7}})); + const Message request = MakePayload(); + Message response; + brpc::Controller cntl; + channel.FBCallMethod(descriptor.method(0), &cntl, &request, &response, nullptr); + // The wire sizes are valid; no typed schema is known by the FRPC transport. + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_FALSE(response.Verify()); + peer.Join(); + EXPECT_TRUE(peer.served()); +} + +} // namespace +#endif // BRPC_WITH_FLATBUFFERS diff --git a/tools/flatbuffers/README.md b/tools/flatbuffers/README.md index 93bc4f988d..3ed128eb23 100644 --- a/tools/flatbuffers/README.md +++ b/tools/flatbuffers/README.md @@ -1,7 +1,7 @@ # Standalone FlatBuffers service generator -`brpc_flatc` generates the bRPC **service abstraction**, not a network protocol or -an adapter for `brpc::Channel`. It uses the installed, official +`brpc_flatc` generates bRPC service bindings. The separate `fb_rpc` transport +implements their channel interface in `brpc::Channel`. The generator uses the installed, official `flatbuffers/idl.h` Parser and `libflatbuffers`; it does not download or modify a FlatBuffers fork. The official `flatc --cpp` remains responsible for table types. @@ -69,8 +69,11 @@ The generated names are `example::api::Echo`, They are **not** the old `EchoStub` spelling. Methods take protobuf `RpcController`/`Closure` and `brpc::flatbuffers::Message` parameters. Derive from `Echo` and override the schema methods; supply an implementation of -`brpc::flatbuffers::RpcChannel` to a stub. Channel ownership defaults to borrowed; -`Service::STUB_OWNS_CHANNEL` transfers ownership to the stub. +`brpc::flatbuffers::RpcChannel` to a stub, such as a `brpc::Channel` initialized +with `ChannelOptions::protocol = "fb_rpc"`. Register the implementation with +`Server::AddFlatBuffersService`. See [FlatBuffers RPC](../../docs/en/flatbuffers.md) +for ownership, protocol limitations, and response verification. Channel ownership +defaults to borrowed; `Service::STUB_OWNS_CHANNEL` transfers ownership to the stub. This intentionally bounded generator supports unary table RPCs, multiple services/methods, namespaces, included request/response tables, and absent From 7bddeade2232bf83d315477bc1c2cfa7888499f2 Mon Sep 17 00:00:00 2001 From: xulei25 Date: Mon, 21 Sep 2026 15:04:17 +0800 Subject: [PATCH 2/6] Add a generated FlatBuffers RPC example Add a client/server example built from a generated service, with a bounded loopback smoke test: 13 verified replies, 2 expected schema rejections and a clean shutdown. It is a functional check, not a benchmark. The example takes an installed FlatBuffers-enabled bRPC and writes the code generated by flatc and brpc_flatc into its own build directory. --- example/benchmark_fb/CMakeLists.txt | 168 +++++++++++++++++++++ example/benchmark_fb/README.md | 111 ++++++++++++++ example/benchmark_fb/client.cpp | 219 ++++++++++++++++++++++++++++ example/benchmark_fb/echo.fbs | 35 +++++ example/benchmark_fb/server.cpp | 120 +++++++++++++++ example/benchmark_fb/smoke.py | 164 +++++++++++++++++++++ 6 files changed, 817 insertions(+) create mode 100644 example/benchmark_fb/CMakeLists.txt create mode 100644 example/benchmark_fb/README.md create mode 100644 example/benchmark_fb/client.cpp create mode 100644 example/benchmark_fb/echo.fbs create mode 100644 example/benchmark_fb/server.cpp create mode 100644 example/benchmark_fb/smoke.py diff --git a/example/benchmark_fb/CMakeLists.txt b/example/benchmark_fb/CMakeLists.txt new file mode 100644 index 0000000000..8eb080c878 --- /dev/null +++ b/example/benchmark_fb/CMakeLists.txt @@ -0,0 +1,168 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +cmake_minimum_required(VERSION 3.16...3.28) +project(benchmark_fb LANGUAGES CXX) + +set(BRPC_ROOT "" CACHE PATH "FlatBuffers-enabled brpc output or install prefix") +set(BRPC_EXTRA_LIBRARIES "" CACHE STRING "Extra dependencies of this libbrpc, as a CMake list") +option(LINK_SO "Link the shared brpc library instead of libbrpc.a" OFF) +if(NOT BRPC_ROOT) + message(FATAL_ERROR "Set BRPC_ROOT to an existing WITH_FLATBUFFERS=ON output/install prefix") +endif() +get_filename_component(BRPC_ROOT "${BRPC_ROOT}" ABSOLUTE) +set(BRPC_INCLUDE_DIR "${BRPC_ROOT}/include") +if(NOT EXISTS "${BRPC_INCLUDE_DIR}/brpc/channel.h" OR + NOT EXISTS "${BRPC_INCLUDE_DIR}/butil/config.h") + message(FATAL_ERROR "BRPC_ROOT must contain include/brpc/channel.h and include/butil/config.h") +endif() +file(STRINGS "${BRPC_INCLUDE_DIR}/butil/config.h" _fb_enabled + REGEX "^#[ \t]*define[ \t]+BRPC_WITH_FLATBUFFERS[ \t]+1([ \t]|$)") +if(NOT _fb_enabled) + message(FATAL_ERROR + "BRPC_ROOT is not FlatBuffers-enabled. Rebuild brpc with WITH_FLATBUFFERS=ON; do not force BRPC_WITH_FLATBUFFERS with -D.") +endif() +# Re-resolve when BRPC_ROOT or LINK_SO changes; never select another installed brpc. +unset(_BRPC_LIBRARY CACHE) +if(LINK_SO) + set(_brpc_names "${CMAKE_SHARED_LIBRARY_PREFIX}brpc${CMAKE_SHARED_LIBRARY_SUFFIX}") +else() + set(_brpc_names libbrpc.a) +endif() +find_library(_BRPC_LIBRARY NAMES ${_brpc_names} + PATHS "${BRPC_ROOT}/lib" "${BRPC_ROOT}/lib64" NO_DEFAULT_PATH) +if(NOT _BRPC_LIBRARY) + message(FATAL_ERROR "The requested brpc library was not found under BRPC_ROOT/lib or lib64") +endif() + +find_path(FLATBUFFERS_INCLUDE_DIR NAMES flatbuffers/flatbuffers.h) +find_program(FLATC_EXECUTABLE NAMES flatc) +find_program(BRPC_FLATC_EXECUTABLE NAMES brpc_flatc HINTS "${BRPC_ROOT}/bin") +if(NOT FLATBUFFERS_INCLUDE_DIR OR NOT FLATC_EXECUTABLE OR NOT BRPC_FLATC_EXECUTABLE) + message(FATAL_ERROR "Set FLATBUFFERS_INCLUDE_DIR, FLATC_EXECUTABLE and BRPC_FLATC_EXECUTABLE") +endif() +if(NOT EXISTS "${FLATBUFFERS_INCLUDE_DIR}/flatbuffers/base.h") + message(FATAL_ERROR "FLATBUFFERS_INCLUDE_DIR must contain flatbuffers/base.h") +endif() +foreach(_part MAJOR MINOR REVISION) + file(STRINGS "${FLATBUFFERS_INCLUDE_DIR}/flatbuffers/base.h" _version_line + REGEX "^#define[ \t]+FLATBUFFERS_VERSION_${_part}[ \t]+[0-9]+") + if(NOT _version_line MATCHES "FLATBUFFERS_VERSION_${_part}[ \t]+([0-9]+)") + message(FATAL_ERROR "Cannot read FlatBuffers ${_part} version from base.h") + endif() + set(_fb_${_part} "${CMAKE_MATCH_1}") +endforeach() +set(_headers_version "${_fb_MAJOR}.${_fb_MINOR}.${_fb_REVISION}") +execute_process(COMMAND "${FLATC_EXECUTABLE}" --version + RESULT_VARIABLE _flatc_result OUTPUT_VARIABLE _flatc_version + ERROR_VARIABLE _flatc_error OUTPUT_STRIP_TRAILING_WHITESPACE TIMEOUT 5) +if(NOT _flatc_result STREQUAL "0" OR + NOT _flatc_version MATCHES "flatc version ([0-9]+\\.[0-9]+\\.[0-9]+)") + message(FATAL_ERROR "Cannot execute official flatc --version: ${_flatc_error}") +endif() +set(_compiler_version "${CMAKE_MATCH_1}") +if(NOT _compiler_version VERSION_EQUAL _headers_version) + message(FATAL_ERROR + "FlatBuffers headers ${_headers_version} do not match flatc ${_compiler_version}") +endif() +execute_process(COMMAND "${BRPC_FLATC_EXECUTABLE}" --help + RESULT_VARIABLE _brpc_flatc_result OUTPUT_QUIET ERROR_VARIABLE _brpc_flatc_error TIMEOUT 5) +if(NOT _brpc_flatc_result STREQUAL "0") + message(FATAL_ERROR "Cannot execute BRPC_FLATC_EXECUTABLE: ${_brpc_flatc_error}") +endif() + +find_package(Threads REQUIRED) +find_package(Protobuf CONFIG QUIET) +set(_protobuf_config ${Protobuf_FOUND}) +if(NOT Protobuf_FOUND) + find_package(Protobuf REQUIRED) +endif() +find_package(OpenSSL REQUIRED) +find_package(ZLIB REQUIRED) +find_path(GFLAGS_INCLUDE_DIR NAMES gflags/gflags.h) +find_library(GFLAGS_LIBRARY NAMES gflags) +find_library(LEVELDB_LIBRARY NAMES leveldb) +if(NOT GFLAGS_INCLUDE_DIR OR NOT GFLAGS_LIBRARY OR NOT LEVELDB_LIBRARY) + message(FATAL_ERROR "Install gflags and leveldb, or set CMAKE_PREFIX_PATH to the runtime dependency prefix") +endif() +set(_cxx_standard 14) +if(Protobuf_VERSION VERSION_GREATER 4.21) + set(_cxx_standard 17) + if(NOT _protobuf_config) + message(FATAL_ERROR "Protobuf 5+ needs its CMake config package to export transitive Abseil dependencies") + endif() +endif() + +add_library(benchmark_fb_dependencies INTERFACE) +# Keep the actual runtime configuration ahead of dependency include directories. +target_include_directories(benchmark_fb_dependencies INTERFACE + "${BRPC_INCLUDE_DIR}" "${FLATBUFFERS_INCLUDE_DIR}" "${GFLAGS_INCLUDE_DIR}") +target_compile_features(benchmark_fb_dependencies INTERFACE cxx_std_${_cxx_standard}) +target_link_libraries(benchmark_fb_dependencies INTERFACE + "${_BRPC_LIBRARY}" protobuf::libprotobuf "${GFLAGS_LIBRARY}" "${LEVELDB_LIBRARY}" + OpenSSL::SSL OpenSSL::Crypto ZLIB::ZLIB Threads::Threads ${CMAKE_DL_LIBS} + ${BRPC_EXTRA_LIBRARIES}) +file(STRINGS "${BRPC_INCLUDE_DIR}/butil/config.h" _glog_enabled + REGEX "^#[ \t]*define[ \t]+BRPC_WITH_GLOG[ \t]+1([ \t]|$)") +if(_glog_enabled) + find_package(glog CONFIG REQUIRED) + target_link_libraries(benchmark_fb_dependencies INTERFACE glog::glog) +endif() +if(APPLE) + target_link_libraries(benchmark_fb_dependencies INTERFACE + "-framework CoreFoundation" "-framework CoreGraphics" "-framework CoreData" + "-framework CoreText" "-framework Security" "-framework Foundation") + target_link_options(benchmark_fb_dependencies INTERFACE + "LINKER:-U,_MallocExtension_ReleaseFreeMemory" "LINKER:-U,_ProfilerStart" + "LINKER:-U,_ProfilerStop" "LINKER:-U,__Z13GetStackTracePPvii" + "LINKER:-U,_mallctl" "LINKER:-U,_malloc_stats_print") +endif() + +set(_generated "${CMAKE_CURRENT_BINARY_DIR}/generated") +set(_schema "${CMAKE_CURRENT_SOURCE_DIR}/echo.fbs") +set(_generated_files "${_generated}/echo_generated.h" + "${_generated}/echo.brpc.fb.h" "${_generated}/echo.brpc.fb.cpp") +add_custom_command(OUTPUT ${_generated_files} + COMMAND "${CMAKE_COMMAND}" -E make_directory "${_generated}" + COMMAND "${FLATC_EXECUTABLE}" --cpp -o "${_generated}" "${_schema}" + COMMAND "${BRPC_FLATC_EXECUTABLE}" -o "${_generated}" "${_schema}" + DEPENDS "${_schema}" "${FLATC_EXECUTABLE}" "${BRPC_FLATC_EXECUTABLE}" + VERBATIM) +add_custom_target(benchmark_fb_codegen DEPENDS ${_generated_files}) +add_library(benchmark_fb_schema STATIC "${_generated}/echo.brpc.fb.cpp") +add_dependencies(benchmark_fb_schema benchmark_fb_codegen) +target_include_directories(benchmark_fb_schema PUBLIC "${_generated}") +target_link_libraries(benchmark_fb_schema PUBLIC benchmark_fb_dependencies) +foreach(_program client server) + add_executable(benchmark_fb_${_program} "${_program}.cpp") + add_dependencies(benchmark_fb_${_program} benchmark_fb_codegen) + target_link_libraries(benchmark_fb_${_program} PRIVATE benchmark_fb_schema) + if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + target_compile_options(benchmark_fb_${_program} PRIVATE -Wall -Wextra) + endif() +endforeach() + +include(CTest) +if(BUILD_TESTING) + find_package(Python3 COMPONENTS Interpreter REQUIRED) + add_test(NAME benchmark_fb_smoke + COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/smoke.py" + --server "$" + --client "$") + set_tests_properties(benchmark_fb_smoke PROPERTIES TIMEOUT 50 LABELS "flatbuffers;smoke") +endif() +message(STATUS "benchmark_fb: brpc=${_BRPC_LIBRARY}, FlatBuffers=${_headers_version}") diff --git a/example/benchmark_fb/README.md b/example/benchmark_fb/README.md new file mode 100644 index 0000000000..3f63182c84 --- /dev/null +++ b/example/benchmark_fb/README.md @@ -0,0 +1,111 @@ +# FlatBuffers RPC example + +[English guide](../../docs/en/flatbuffers.md) | +[中文指南](../../docs/cn/flatbuffers.md) + +Independent client and server programs using the `fb_rpc` protocol, a generated +FlatBuffers service, and `brpc::flatbuffers::Message`. This is a **bounded +functional example**, not a performance comparison; the directory name is kept +for continuity. + +The schema declares `BenchmarkService.Echo` with stable wire method ID **7**. +The generated stub calls `Channel::FBCallMethod`, and the server registers the +generated service with `Server::AddFlatBuffersService`. + +## Build + +Dependencies, the FlatBuffers-enabled runtime install, the generator and the +example build are described in the guide's +[verification section](../../docs/en/flatbuffers.md#verify-with-the-clientserver-example). +`BRPC_ROOT` must be an install prefix (or a Make `output/` directory) of a bRPC +built with FlatBuffers ON; the example reads its `include/butil/config.h` and +rejects an OFF build. Both `flatc` and `brpc_flatc` write generated code only to +the build directory; do not commit `*_generated.h` or `*.brpc.fb.*`. + +To link the shared library, build and install the runtime with +`-DBUILD_SHARED_LIBS=ON` and the `brpc-shared` target, then configure the +example with `-DLINK_SO=ON`. Extra runtime dependencies can be passed with +`-DBRPC_EXTRA_LIBRARIES='library1;library2'`. + +## Run separately + +The server and client must run on the **same host**; both accept only IPv4 +loopback endpoints. In the build terminal: + +```sh +"$WORK/example/benchmark_fb_server" --listen_addr=127.0.0.1:0 --duration_s=300 +``` + +The server prints `BRPC_FB_READY 127.0.0.1:` after `Server::Start` +succeeds. Use that port in another terminal (shell variables are not shared, so +set `WORK` again): + +```sh +WORK=/absolute/path/printed/by/the/build +SERVER=127.0.0.1:PORT_FROM_READY +"$WORK/example/benchmark_fb_client" --server="$SERVER" \ + --request_count=16 --thread_num=2 --request_size=8193 --attachment_size=257 +"$WORK/example/benchmark_fb_client" --server="$SERVER" \ + --request_count=2 --thread_num=1 --request_size=0 --omit_message=true +"$WORK/example/benchmark_fb_client" --server="$SERVER" \ + --request_count=2 --thread_num=1 --corrupt_request=true +"$WORK/example/benchmark_fb_client" --server="$SERVER" \ + --request_count=2 --thread_num=1 +``` + +All four commands exit 0. The first prints +`{"completed":16,"successes":16,"expected_rejections":0,"failures":0}`. The +corrupt-request run prints `{"completed":2,"successes":0,"expected_rejections":2,"failures":0}`, +and the last one checks recovery on the same server (`completed=2`, +`successes=2`). + +The client verifies the response schema before reading it, then compares the +request ID, every string byte, optional-string presence, and the attachment. +`--corrupt_request=true` damages the FlatBuffers root offset but keeps the FRPC +framing valid. It succeeds only if every request gets the generated service's +schema rejection (error code -1); timeouts and connection errors are failures. +`failures` is a 0/1 flag, not a count, and no throughput is reported. + +### Bounded controls + +| Client option | Default | Allowed | +| --- | --- | --- | +| `server` | required | `127.0.0.1:` | +| `request_count` | 16 | 1..1000 total, not per thread | +| `thread_num` | 2 | 1..16 synchronous callers | +| `request_size` | 64 | 0..1048576 string bytes | +| `attachment_size` | 0 | 0..1048576 bytes | +| `timeout_ms` | 1500 | 1..5000 per RPC | +| `deadline_ms` | 30000 | 1..120000 for the client run | +| `connection_type` | `single` | `single`, `pooled`, `short` | +| `omit_message` | false | Preserve absent rather than empty string | +| `corrupt_request` | false | Require schema rejection | + +The server defaults to `127.0.0.1:0`, `duration_s=60` (1..3600) and +`max_concurrency=32` (1..64), and stops earlier on SIGINT/SIGTERM. Builtin +services are disabled. FRPC has no authentication, compression, checksums or +streaming, so this is not a production or public-network configuration. + +## Smoke test + +CTest runs the standard-library-only `smoke.py`, which starts its own server on +an ephemeral loopback port and checks: + +1. finite RPCs with binary attachments, empty and absent strings, and + single/pooled/short connections (13 verified replies); +2. two malformed-schema requests are rejected; +3. normal requests still work on the same server afterwards; +4. the server it started is stopped and reaped. + +The orchestration deadline is 35 seconds and CTest's timeout is 50 seconds. +Needing the SIGKILL fallback fails the test. The smoke test can also run +without CTest: + +```sh +python3 "$REPO/example/benchmark_fb/smoke.py" \ + --server "$WORK/example/benchmark_fb_server" \ + --client "$WORK/example/benchmark_fb_client" +``` + +For common failures see the guide's +[troubleshooting table](../../docs/en/flatbuffers.md#troubleshooting). diff --git a/example/benchmark_fb/client.cpp b/example/benchmark_fb/client.cpp new file mode 100644 index 0000000000..4900d366c9 --- /dev/null +++ b/example/benchmark_fb/client.cpp @@ -0,0 +1,219 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "butil/config.h" +#include "butil/endpoint.h" +#include "brpc/channel.h" +#include "brpc/controller.h" +#include "echo.brpc.fb.h" + +#if !BRPC_WITH_FLATBUFFERS +#error "benchmark_fb requires a FlatBuffers-enabled brpc library" +#endif + +DEFINE_string(server, "", "Required IPv4 loopback endpoint printed by benchmark_fb_server"); +DEFINE_string(connection_type, "single", "single, pooled, or short"); +DEFINE_int32(request_count, 16, "Total requests across all threads, 1..1000"); +DEFINE_int32(thread_num, 2, "Number of concurrent synchronous callers, 1..16"); +DEFINE_int32(request_size, 64, "Bytes in the optional request string, 0..1048576"); +DEFINE_int32(attachment_size, 0, "Bytes in the echoed attachment, 0..1048576"); +DEFINE_int32(timeout_ms, 1500, "Per-RPC timeout, 1..5000 milliseconds"); +DEFINE_int32(deadline_ms, 30000, "Overall client deadline, 1..120000 milliseconds"); +DEFINE_bool(omit_message, false, "Omit the optional string rather than sending an empty string"); +DEFINE_bool(corrupt_request, false, "Send an invalid root offset and require schema rejection"); + +namespace { +using brpc::flatbuffers::Message; +using brpc::flatbuffers::MessageBuilder; + +std::string MakeBytes(size_t size, unsigned multiplier) { + std::string bytes(size, '\0'); + for (size_t i = 0; i < size; ++i) { + bytes[i] = static_cast((i * multiplier) % 251); + } + return bytes; +} + +Message MakeRequest(uint64_t request_id, const std::string& text) { + MessageBuilder builder; + const auto message = FLAGS_omit_message ? + ::flatbuffers::Offset<::flatbuffers::String>() : builder.CreateString(text); + builder.Finish(benchmark_fb::CreateRequest(builder, request_id, message)); + Message request = builder.ReleaseMessage(); + if (FLAGS_corrupt_request) { + // Damage only the schema, not FRPC framing, so the server's generated + // dispatcher must reject the request before invoking the service. + memset(request.mutable_data(), 0xff, sizeof(::flatbuffers::uoffset_t)); + } + return request; +} + +bool CheckResponse(const Message& response, uint64_t request_id, + const std::string& text, const std::string& attachment, + const brpc::Controller& cntl) { + if (!response.Verify()) { + return false; + } + const auto* output = response.GetRoot(); + if (output->request_id() != request_id || + output->attachment_size() != attachment.size() || + cntl.response_attachment().to_string() != attachment) { + return false; + } + if (FLAGS_omit_message) { + return output->message() == nullptr; + } + return output->message() != nullptr && output->message()->str() == text; +} + +bool ValidFlags(butil::EndPoint* endpoint) { + return FLAGS_server.compare(0, 10, "127.0.0.1:") == 0 && + butil::str2endpoint(FLAGS_server.c_str(), endpoint) == 0 && endpoint->port > 0 && + (FLAGS_connection_type == "single" || FLAGS_connection_type == "pooled" || + FLAGS_connection_type == "short") && + FLAGS_request_count >= 1 && FLAGS_request_count <= 1000 && + FLAGS_thread_num >= 1 && FLAGS_thread_num <= 16 && + FLAGS_request_size >= 0 && FLAGS_request_size <= 1024 * 1024 && + FLAGS_attachment_size >= 0 && FLAGS_attachment_size <= 1024 * 1024 && + FLAGS_timeout_ms >= 1 && FLAGS_timeout_ms <= 5000 && + FLAGS_deadline_ms >= 1 && FLAGS_deadline_ms <= 120000; +} + +int RunClient(const butil::EndPoint& endpoint) { + brpc::Channel channel; + brpc::ChannelOptions options; + options.protocol = "fb_rpc"; + options.connection_type = FLAGS_connection_type; + options.timeout_ms = FLAGS_timeout_ms; + options.connect_timeout_ms = std::min(FLAGS_timeout_ms, 1000); + options.max_retry = 0; + if (channel.Init(endpoint, &options) != 0) { + std::cerr << "Cannot initialize FlatBuffers channel\n"; + return 1; + } + // Generated stub methods route through Channel::FBCallMethod. + benchmark_fb::BenchmarkService::Stub stub(&channel); + const std::string text = MakeBytes(FLAGS_omit_message ? 0 : FLAGS_request_size, 31); + const std::string attachment = MakeBytes(FLAGS_attachment_size, 17); + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::milliseconds(FLAGS_deadline_ms); + std::atomic next{0}; + std::atomic successes{0}; + std::atomic expected_rejections{0}; + std::atomic failed{false}; + std::mutex error_mutex; + auto fail = [&](const std::string& message) { + if (!failed.exchange(true)) { + std::lock_guard lock(error_mutex); + std::cerr << message << '\n'; + } + }; + auto worker = [&] { + try { + while (!failed.load()) { + const int index = next.fetch_add(1); + if (index >= FLAGS_request_count) { + break; + } + const int64_t remaining = std::chrono::duration_cast< + std::chrono::milliseconds>(deadline - + std::chrono::steady_clock::now()).count(); + if (remaining <= 0) { + fail("Overall client deadline exceeded"); + break; + } + const uint64_t request_id = static_cast(index) + 1; + const Message request = MakeRequest(request_id, text); + Message response; + brpc::Controller cntl; + cntl.set_timeout_ms(std::min(FLAGS_timeout_ms, remaining)); + cntl.request_attachment().append(attachment); + stub.Echo(&cntl, &request, &response, nullptr); + if (FLAGS_corrupt_request) { + // The generated generic RpcController::SetFailed(string) + // maps to -1. Timeouts and connection errors do not count. + if (!cntl.Failed() || cntl.ErrorCode() != -1 || + response.size() != 0 || !cntl.response_attachment().empty()) { + fail("Expected server schema rejection; error=" + cntl.ErrorText()); + break; + } + ++expected_rejections; + } else if (cntl.Failed()) { + fail("RPC failed: " + cntl.ErrorText()); + break; + } else if (!CheckResponse(response, request_id, text, attachment, cntl)) { + fail("Response schema, fields, optional string or attachment mismatch"); + break; + } else { + ++successes; + } + } + } catch (const std::exception& error) { + fail(std::string("Client worker failed: ") + error.what()); + } + }; + std::vector workers; + workers.reserve(FLAGS_thread_num); + try { + for (int i = 0; i < FLAGS_thread_num; ++i) { + workers.emplace_back(worker); + } + } catch (const std::exception& error) { + fail(std::string("Cannot start client worker: ") + error.what()); + } + for (auto& thread : workers) { + thread.join(); + } + const int completed = successes.load() + expected_rejections.load(); + if (completed != FLAGS_request_count) { + fail("Not all requested RPCs completed"); + } + std::cout << "{\"completed\":" << completed + << ",\"successes\":" << successes.load() + << ",\"expected_rejections\":" << expected_rejections.load() + << ",\"failures\":" << (failed.load() ? 1 : 0) << "}\n"; + return failed.load() ? 1 : 0; +} +} // namespace + +int main(int argc, char* argv[]) { + GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); + butil::EndPoint endpoint; + if (!ValidFlags(&endpoint)) { + std::cerr << "Invalid flags; --server=127.0.0.1:PORT is required. " + "See --help and README.md for bounded parameter ranges.\n"; + return 2; + } + try { + return RunClient(endpoint); + } catch (const std::exception& error) { + std::cerr << "Client failed: " << error.what() << '\n'; + return 1; + } +} diff --git a/example/benchmark_fb/echo.fbs b/example/benchmark_fb/echo.fbs new file mode 100644 index 0000000000..f7a40eae2b --- /dev/null +++ b/example/benchmark_fb/echo.fbs @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace benchmark_fb; + +table Request { + request_id:ulong; + message:string; +} + +table Response { + request_id:ulong; + message:string; + attachment_size:ulong; +} + +rpc_service BenchmarkService { + Echo(Request):Response (id: 7); +} + +root_type Request; diff --git a/example/benchmark_fb/server.cpp b/example/benchmark_fb/server.cpp new file mode 100644 index 0000000000..f8fa9244ef --- /dev/null +++ b/example/benchmark_fb/server.cpp @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include "butil/config.h" +#include "butil/endpoint.h" +#include "brpc/closure_guard.h" +#include "brpc/controller.h" +#include "brpc/errno.pb.h" +#include "brpc/server.h" +#include "echo.brpc.fb.h" + +#if !BRPC_WITH_FLATBUFFERS +#error "benchmark_fb requires a FlatBuffers-enabled brpc library" +#endif + +DEFINE_string(listen_addr, "127.0.0.1:0", "Loopback IPv4 address; port 0 selects a free port"); +DEFINE_int32(duration_s, 60, "Exit after 1..3600 seconds, or earlier on SIGINT/SIGTERM"); +DEFINE_int32(max_concurrency, 32, "Maximum concurrent requests, 1..64"); + +namespace { +const size_t kMaxBytes = 1024 * 1024; + +class BenchmarkServiceImpl : public benchmark_fb::BenchmarkService { +public: + void Echo(google::protobuf::RpcController* controller, + const brpc::flatbuffers::Message* request, + brpc::flatbuffers::Message* response, + google::protobuf::Closure* done) override { + brpc::ClosureGuard done_guard(done); + if (!controller) { + return; + } + auto* cntl = static_cast(controller); + // The generated dispatcher also verifies the schema. Keep this guard + // for callers invoking the typed service method directly. + if (!request || !response || !request->Verify()) { + cntl->SetFailed(brpc::EREQUEST, "Invalid FlatBuffers Request"); + return; + } + const auto* input = request->GetRoot(); + if ((input->message() && input->message()->size() > kMaxBytes) || + cntl->request_attachment().size() > kMaxBytes) { + cntl->SetFailed(brpc::EREQUEST, "Example payload limit is 1 MiB per field"); + return; + } + brpc::flatbuffers::MessageBuilder builder; + // Preserve absent versus empty strings, including embedded zero bytes. + const auto text = input->message() ? + builder.CreateString(input->message()->str()) : + ::flatbuffers::Offset<::flatbuffers::String>(); + builder.Finish(benchmark_fb::CreateResponse( + builder, input->request_id(), text, cntl->request_attachment().size())); + *response = builder.ReleaseMessage(); + cntl->response_attachment().append(cntl->request_attachment()); + } +}; +} // namespace + +int main(int argc, char* argv[]) { + GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); + butil::EndPoint endpoint; + if (FLAGS_listen_addr.compare(0, 10, "127.0.0.1:") != 0 || + butil::str2endpoint(FLAGS_listen_addr.c_str(), &endpoint) != 0 || + FLAGS_duration_s < 1 || FLAGS_duration_s > 3600 || + FLAGS_max_concurrency < 1 || FLAGS_max_concurrency > 64) { + std::cerr << "Use --listen_addr=127.0.0.1:PORT, --duration_s=1..3600 " + "and --max_concurrency=1..64\n"; + return 2; + } + // brpc defaults SIGTERM to immediate termination. This example always + // enables graceful shutdown, including the smoke's owned-process cleanup. + if (GFLAGS_NAMESPACE::SetCommandLineOption( + "graceful_quit_on_sigterm", "true").empty()) { + std::cerr << "Cannot enable graceful SIGTERM handling\n"; + return 1; + } + BenchmarkServiceImpl service; + brpc::Server server; + if (server.AddFlatBuffersService(&service, brpc::SERVER_DOESNT_OWN_SERVICE) != 0) { + std::cerr << "Cannot register FlatBuffers service\n"; + return 1; + } + brpc::ServerOptions options; + options.has_builtin_services = false; + options.enabled_protocols = "fb_rpc"; + options.max_concurrency = FLAGS_max_concurrency; + if (server.Start(endpoint, &options) != 0) { + std::cerr << "Cannot start FlatBuffers server\n"; + return 1; + } + // Install brpc's termination handlers before publishing readiness. + brpc::IsAskedToQuit(); + std::cout << "BRPC_FB_READY " << server.listen_address() << std::endl; + const auto deadline = std::chrono::steady_clock::now() + + std::chrono::seconds(FLAGS_duration_s); + while (!brpc::IsAskedToQuit() && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + const int stop_result = server.Stop(0); + const int join_result = server.Join(); + return stop_result == 0 && join_result == 0 ? 0 : 1; +} diff --git a/example/benchmark_fb/smoke.py b/example/benchmark_fb/smoke.py new file mode 100644 index 0000000000..5b9c621cfc --- /dev/null +++ b/example/benchmark_fb/smoke.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Bounded functional smoke: no network dependency and no throughput measurement.""" + +import argparse +import json +import os +from pathlib import Path +import re +import selectors +import signal +import subprocess +import sys +import tempfile +import time + + +def remaining(deadline): + seconds = deadline - time.monotonic() + if seconds <= 0: + raise RuntimeError("smoke deadline exceeded") + return seconds + + +def wait_ready(process, deadline): + buffer = b"" + os.set_blocking(process.stdout.fileno(), False) + with selectors.DefaultSelector() as selector: + selector.register(process.stdout, selectors.EVENT_READ) + while True: + if process.poll() is not None: + raise RuntimeError("server exited before readiness") + events = selector.select(min(0.2, remaining(deadline))) + if not events: + continue + chunk = os.read(process.stdout.fileno(), 4096) + if not chunk: + raise RuntimeError("server closed stdout before readiness") + buffer += chunk + if len(buffer) > 65536: + raise RuntimeError("unexpectedly large server readiness output") + while b"\n" in buffer: + line, buffer = buffer.split(b"\n", 1) + match = re.fullmatch(rb"BRPC_FB_READY (127\.0\.0\.1):([0-9]{1,5})", line) + if match: + port = int(match.group(2)) + if not 1 <= port <= 65535: + raise RuntimeError("server published an invalid port") + return "127.0.0.1:" + str(port) + + +def run_client(binary, endpoint, deadline, *, count, size, attachment, + threads=1, omit=False, corrupt=False, connection="single"): + command = [ + str(binary), "--server=" + endpoint, + "--request_count=" + str(count), "--thread_num=" + str(threads), + "--request_size=" + str(size), "--attachment_size=" + str(attachment), + "--connection_type=" + connection, + "--omit_message=" + str(omit).lower(), + "--corrupt_request=" + str(corrupt).lower(), + "--timeout_ms=1000", "--deadline_ms=5000", + ] + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, timeout=min(8.0, remaining(deadline)), check=False) + if result.returncode != 0: + raise RuntimeError("client failed ({}):\n{}\n{}".format( + result.returncode, result.stdout, result.stderr)) + try: + report = json.loads(result.stdout) + except (ValueError, TypeError) as error: + raise RuntimeError("invalid client summary: " + result.stdout) from error + expected = {"completed": count, "successes": 0 if corrupt else count, + "expected_rejections": count if corrupt else 0, "failures": 0} + if report != expected: + raise RuntimeError("unexpected client result: " + repr(report)) + + +def stop_owned_server(process): + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=3) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=3) + raise RuntimeError("server required SIGKILL instead of a clean Stop/Join") + if process.returncode != 0: + raise RuntimeError("server exited with status " + str(process.returncode)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--server", required=True, help="benchmark_fb_server executable") + parser.add_argument("--client", required=True, help="benchmark_fb_client executable") + args = parser.parse_args() + process = None + failure = None + deadline = time.monotonic() + 35 + with tempfile.TemporaryFile(mode="w+b") as server_log: + try: + server = Path(args.server).resolve(strict=True) + client = Path(args.client).resolve(strict=True) + process = subprocess.Popen( + [str(server), "--listen_addr=127.0.0.1:0", "--duration_s=45", + "--max_concurrency=8"], + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=server_log, + start_new_session=True, + ) + endpoint = wait_ready(process, min(deadline, time.monotonic() + 10)) + run_client(client, endpoint, deadline, + count=6, size=8193, attachment=257, threads=2) + run_client(client, endpoint, deadline, + count=2, size=0, attachment=31, connection="pooled") + run_client(client, endpoint, deadline, + count=2, size=0, attachment=0, omit=True, connection="short") + run_client(client, endpoint, deadline, + count=2, size=17, attachment=5, corrupt=True) + # Keep the same server alive after the malformed requests. + run_client(client, endpoint, deadline, count=3, size=33, attachment=9) + if process.poll() is not None: + raise RuntimeError("server exited during the smoke") + except (Exception, KeyboardInterrupt) as error: + failure = str(error) or type(error).__name__ + finally: + if process is not None: + try: + stop_owned_server(process) + except Exception as error: + failure = failure or str(error) + if process.stdout is not None: + process.stdout.close() + if failure: + server_log.seek(0) + log = server_log.read(65536).decode("utf-8", errors="replace") + sys.stderr.write("benchmark_fb smoke failed: " + failure + "\n" + log) + if failure: + return 1 + print("benchmark_fb smoke passed: 13 verified replies, 2 schema rejections, clean shutdown") + return 0 + + +def handle_termination(signum, frame): + raise KeyboardInterrupt() + + +if __name__ == "__main__": + signal.signal(signal.SIGTERM, handle_termination) + sys.exit(main()) From 93547c38de7679dc5e503eca4fa481522cb97b2f Mon Sep 17 00:00:00 2001 From: xulei25 Date: Mon, 21 Sep 2026 15:05:51 +0800 Subject: [PATCH 3/6] Add explicit FlatBuffers ON validation Add a runner that builds and tests the FlatBuffers-enabled configuration with CMake, Make or Bazel and rejects empty, filtered or skipped test runs. Add its self-tests and a GitHub Actions workflow that runs it on Linux and macOS when FlatBuffers-related paths change. --- .github/scripts/flatbuffers-on.py | 455 +++++++++++++++++++++++++ .github/scripts/test_flatbuffers_on.py | 351 +++++++++++++++++++ .github/workflows/flatbuffers-on.yml | 130 +++++++ .gitignore | 4 +- 4 files changed, 939 insertions(+), 1 deletion(-) create mode 100644 .github/scripts/flatbuffers-on.py create mode 100644 .github/scripts/test_flatbuffers_on.py create mode 100644 .github/workflows/flatbuffers-on.yml diff --git a/.github/scripts/flatbuffers-on.py b/.github/scripts/flatbuffers-on.py new file mode 100644 index 0000000000..8d6c02cb55 --- /dev/null +++ b/.github/scripts/flatbuffers-on.py @@ -0,0 +1,455 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Run the focused FlatBuffers ON gates; never accept empty or skipped tests.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import shlex +import shutil +import signal +import subprocess +import sys +import tarfile +import time +import xml.etree.ElementTree as ET + +# Keep the FlatBuffers pin synchronized with MODULE.bazel and WORKSPACE. +FLATBUFFERS_VERSION = "25.2.10" +FLATBUFFERS_SHA256 = "b9c2df49707c57a48fc0923d52b8c73beb72d675f9d44b2211e4569be40a7421" +GTEST_VERSION = "1.14.0" +GTEST_SHA256 = "8ad598c73ad796e0d8280b082cebd82a630d73e73cd3c70057938a6501bba5d7" +TESTS = {"brpc_flatbuffers_unittest": 22, "brpc_flatbuffers_protocol_unittest": 33} + + +def list_gtests(text): + names = set() + suite = None + for line in text.splitlines(): + value = line.split("#", 1)[0].strip() + if not value: + continue + if not line[0].isspace() and value.endswith("."): + suite = value + elif line[0].isspace() and suite and re.fullmatch(r"[A-Za-z0-9_/]+", value): + name = suite + value + if any(part.startswith("DISABLED_") for part in re.split(r"[./]", name)): + raise ValueError("Disabled test in ON gate: " + name) + if name in names: + raise ValueError("Duplicate discovered test: " + name) + names.add(name) + if not names: + raise ValueError("No GoogleTest cases discovered") + return names + + +def xml_cases(path): + path = Path(path) + if not path.is_file() or path.stat().st_size > 16 * 1024 * 1024: + raise ValueError("Missing or oversized test XML: " + str(path)) + root = ET.parse(path).getroot() + for suite in root.iter(): + if suite.tag in ("testsuites", "testsuite"): + for attribute in ("failures", "errors", "disabled", "skipped"): + if int(suite.get(attribute, "0")) != 0: + raise ValueError("Nonzero " + attribute + " in " + str(path)) + cases = list(root.iter("testcase")) + if not cases: + raise ValueError("Test XML contains no cases: " + str(path)) + for case in cases: + if (case.find("failure") is not None or case.find("error") is not None or + case.find("skipped") is not None or case.get("status") == "notrun" or + case.get("result") in ("skipped", "suppressed")): + raise ValueError("Failed or unexecuted case: " + str(case.attrib)) + return root, cases + + +def verify_gtests(xml, listing, expected_count): + expected = list_gtests(Path(listing).read_text()) + root, cases = xml_cases(xml) + actual = [case.get("classname", "") + "." + case.get("name", "") for case in cases] + if len(expected) != expected_count or len(actual) != len(set(actual)) or set(actual) != expected: + raise ValueError("Executed cases differ from discovery or the expected count: " + str(xml)) + if int(root.get("tests", "-1")) != len(actual): + raise ValueError("Incorrect XML test count: " + str(xml)) + return {"executed": len(actual), "failed": 0, "skipped": 0} + + +def verify_ctest(xml, required): + _, cases = xml_cases(xml) + names = [case.get("name", "") for case in cases] + if len(names) != len(set(names)) or not set(required) <= set(names): + raise ValueError("Required CTest cases did not execute: " + str(required)) + return {"executed": len(names), "names": names, "failed": 0, "skipped": 0} + + +def extract_archive(archive, digest, destination, prefix): + archive = Path(archive) + if hashlib.sha256(archive.read_bytes()).hexdigest() != digest: + raise ValueError("Dependency checksum mismatch: " + str(archive)) + destination = Path(destination) + if destination.exists(): + raise ValueError("Refusing to overwrite dependency directory") + with tarfile.open(archive, "r:gz") as source: + members = source.getmembers() + if len(members) > 30000 or sum(item.size for item in members) > 512 * 1024 * 1024: + raise ValueError("Dependency archive exceeds extraction limits") + paths = set() + links = {} + for member in members: + path = PurePosixPath(member.name) + if (not path.parts or path.is_absolute() or ".." in path.parts or + path.parts[0] != prefix or path in paths or + not (member.isdir() or member.isfile() or member.issym())): + raise ValueError("Unsafe dependency archive member: " + member.name) + paths.add(path) + if member.issym(): + if len(path.parts) < 2: + raise ValueError("Archive root must not be a symlink") + links[path] = member + for path in paths: + if any(parent in links for parent in path.parents): + raise ValueError("Archive member traverses a symlink: " + str(path)) + for path, member in links.items(): + target = PurePosixPath(member.linkname) + if not member.linkname or target.is_absolute(): + raise ValueError("Unsafe archive symlink: " + member.name) + parts = list(path.parent.parts) + for part in target.parts: + if part == "..": + if len(parts) <= 1: + raise ValueError("Escaping archive symlink: " + member.name) + parts.pop() + else: + parts.append(part) + if not parts or parts[0] != prefix or PurePosixPath(*parts) in links: + raise ValueError("Escaping or chained archive symlink: " + member.name) + destination.mkdir(parents=True) + # Create links only after all regular data, so extraction never writes + # through a symlink. The pinned archives need no hardlinks/link chains. + source.extractall(destination, members=[member for member in members if not member.issym()]) + for path, member in links.items(): + link = destination / str(path) + link.parent.mkdir(parents=True, exist_ok=True) + link.symlink_to(member.linkname) + return destination / prefix + + +def check_enabled(header): + if not re.search(r"^\s*#\s*define\s+BRPC_WITH_FLATBUFFERS\s+1\s*$", + Path(header).read_text(), re.MULTILINE): + raise ValueError("FlatBuffers is not enabled in " + str(header)) + + +def signal_group(process, signum): + try: + os.killpg(process.pid, signum) + return True + except ProcessLookupError: + return False + + +def stop_process(process, grace=5): + # The group may outlive its leader; waiting for the leader alone is not + # sufficient when a compiler/test child ignores SIGTERM. + if signal_group(process, signal.SIGTERM): + try: + process.wait(timeout=grace) + except subprocess.TimeoutExpired: + pass + signal_group(process, signal.SIGKILL) + process.wait() + + +class Runner: + def __init__(self, args): + self.args = args + self.source = args.source.resolve(strict=True) + self.work = args.work.resolve() + if self.work == self.source or self.source in self.work.parents: + raise ValueError("--work must be outside the source checkout") + self.work.mkdir(parents=True, exist_ok=False) + self.evidence = self.work / "evidence" + self.evidence.mkdir() + self.env = dict(os.environ) + for name in list(self.env): + if name.startswith("GTEST_"): + self.env.pop(name) + self.env.update(GTEST_FILTER="*", GTEST_REPEAT="1") + self.steps = [] + self.results = {} + self.compiler = self.env.get("CC", "cc") + self.cxx = self.env.get("CXX", "c++") + self.prefixes = [path.resolve(strict=True) for path in args.dependency_prefix] + + def save(self, status): + (self.evidence / "summary.json").write_text(json.dumps({ + "status": status, "system": self.args.build_system, + "source": str(self.source), "steps": self.steps, "tests": self.results, + }, indent=2) + "\n") + + def step(self, name, command, cwd=None, timeout=900, env=None): + command = [str(part) for part in command] + log = self.evidence / (name + ".log") + directory = Path(cwd or self.source) + (self.evidence / (name + ".command")).write_text( + "cwd=" + str(directory) + "\n" + shlex.join(command) + "\n") + print("START", name, flush=True) + start = time.monotonic() + code = 1 + with log.open("w") as output: + process = subprocess.Popen(command, cwd=directory, env=env or self.env, + stdout=output, stderr=subprocess.STDOUT, + start_new_session=True) + try: + code = process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + stop_process(process) + code = 124 + print("Step exceeded timeout", file=output) + except BaseException: + stop_process(process) + raise + finally: + self.steps.append({"name": name, "exitcode": code, + "seconds": round(time.monotonic() - start, 3)}) + (self.evidence / (name + ".exitcode")).write_text(str(code) + "\n") + self.save("running") + print("END", name, "exit=" + str(code), flush=True) + if code != 0: + raise RuntimeError(name + " failed; see " + str(log)) + return log + + def download(self, name, version, digest): + archive = self.work / (name + ".tar.gz") + self.step("download-" + name, ["curl", "--fail", "--location", "--retry", "2", + "--connect-timeout", "20", "--max-time", "180", + "https://github.com/google/" + name + "/archive/refs/tags/v" + version + ".tar.gz", + "--output", archive], timeout=600) + return extract_archive(archive, digest, self.work / (name + "-source"), name + "-" + version) + + def prepare_dependencies(self): + if self.args.flatbuffers_prefix: + self.fb = self.args.flatbuffers_prefix.resolve(strict=True) + else: + source = self.download("flatbuffers", FLATBUFFERS_VERSION, FLATBUFFERS_SHA256) + self.fb = self.work / "dependencies" + build = self.work / "flatbuffers-build" + self.step("flatbuffers-configure", ["cmake", "-S", source, "-B", build, + "-DCMAKE_BUILD_TYPE=Release", "-DCMAKE_POSITION_INDEPENDENT_CODE=ON", + "-DCMAKE_INSTALL_LIBDIR=lib", "-DCMAKE_INSTALL_PREFIX=" + str(self.fb), + "-DFLATBUFFERS_BUILD_TESTS=OFF", "-DFLATBUFFERS_BUILD_FLATC=ON", + "-DFLATBUFFERS_BUILD_FLATLIB=ON", "-DFLATBUFFERS_BUILD_SHAREDLIB=OFF", + "-DFLATBUFFERS_INSTALL=ON", "-DFLATBUFFERS_LIBCXX_WITH_CLANG=OFF"]) + self.step("flatbuffers-build", ["cmake", "--build", build, "--parallel", self.args.jobs]) + self.step("flatbuffers-install", ["cmake", "--install", build]) + self.flatc = self.fb / "bin/flatc" + version = self.step("flatc-version", [self.flatc, "--version"], timeout=10).read_text().strip() + base = (self.fb / "include/flatbuffers/base.h").read_text() + numbers = [re.search(r"#define\s+FLATBUFFERS_VERSION_" + part + r"\s+(\d+)", base) + for part in ("MAJOR", "MINOR", "REVISION")] + if (version != "flatc version " + FLATBUFFERS_VERSION or + not all(numbers) or ".".join(match.group(1) for match in numbers) != FLATBUFFERS_VERSION): + raise ValueError("Use matching pinned FlatBuffers headers and compiler") + self.fb_library = self.fb / "lib/libflatbuffers.a" + if not self.fb_library.is_file(): + raise ValueError("The generator needs " + str(self.fb_library)) + self.gtest = (self.args.gtest_source.resolve(strict=True) if self.args.gtest_source else + self.download("googletest", GTEST_VERSION, GTEST_SHA256)) + self.prefixes.insert(0, self.fb) + if self.args.build_system == "make": + build = self.work / "gtest-build" + gtest_prefix = self.work / "gtest-prefix" + self.step("gtest-configure", ["cmake", "-S", self.gtest, "-B", build, + "-DCMAKE_BUILD_TYPE=Release", "-DCMAKE_INSTALL_LIBDIR=lib", + "-DBUILD_SHARED_LIBS=OFF", "-DCMAKE_INSTALL_PREFIX=" + str(gtest_prefix)]) + self.step("gtest-build", ["cmake", "--build", build, "--parallel", self.args.jobs]) + self.step("gtest-install", ["cmake", "--install", build]) + self.prefixes.insert(0, gtest_prefix) + self.env["PATH"] = os.pathsep.join(str(prefix / "bin") for prefix in self.prefixes) + os.pathsep + self.env.get("PATH", "") + self.step("protoc-version", [self.args.protoc or shutil.which("protoc", path=self.env["PATH"]), "--version"], timeout=10) + + def common_cmake(self): + options = ["-DCMAKE_BUILD_TYPE=RelWithDebInfo", "-DCMAKE_POLICY_VERSION_MINIMUM=3.5", + "-DCMAKE_PREFIX_PATH=" + ";".join(str(prefix) for prefix in self.prefixes), + "-DCMAKE_C_COMPILER=" + self.compiler, "-DCMAKE_CXX_COMPILER=" + self.cxx] + for prefix in self.prefixes: + if (prefix / "include/openssl/ssl.h").is_file(): + options.append("-DOPENSSL_ROOT_DIR=" + str(prefix)) + break + if self.args.protoc: + options.append("-DProtobuf_PROTOC_EXECUTABLE=" + str(self.args.protoc.resolve(strict=True))) + return options + + def check_gtest(self, name, binary, ctest_build=None): + listing = self.step(name + "-list", [binary, "--gtest_list_tests"], timeout=30) + xml = self.evidence / (name + ".xml") + env = dict(self.env, GTEST_OUTPUT="xml:" + str(xml)) + command = (["ctest", "--test-dir", ctest_build, "--no-tests=error", "--output-on-failure", + "--timeout", "300", "-R", "^" + name + "$"] if ctest_build else [binary]) + self.step(name, command, timeout=330, env=env) + self.results[name] = verify_gtests(xml, listing, TESTS[name]) + self.save("running") + + def ctest(self, name, build, required): + xml = self.evidence / (name + ".xml") + self.step(name, ["ctest", "--test-dir", build, "--no-tests=error", "--output-on-failure", + "--timeout", "300", "--output-junit", xml], timeout=600) + self.results[name] = verify_ctest(xml, required) + self.save("running") + + def run_cmake(self): + build = self.work / "build" + runtime = self.work / "runtime" + self.step("configure", ["cmake", "-S", self.source, "-B", build] + self.common_cmake() + [ + "-DCMAKE_INSTALL_PREFIX=" + str(runtime), "-DINSTALL_GTEST=OFF", "-DWITH_FLATBUFFERS=ON", "-DBUILD_UNIT_TESTS=ON", "-DBUILD_BRPC_TOOLS=OFF", + "-DBUILD_SHARED_LIBS=ON", + "-DDOWNLOAD_GTEST=OFF", "-DBRPC_SYSTEM_GTEST_SOURCE_DIR=" + str(self.gtest), + "-DFLATBUFFERS_INCLUDE_DIR=" + str(self.fb / "include"), + "-DFLATBUFFERS_FLATC_EXECUTABLE=" + str(self.flatc)]) + self.step("build", ["cmake", "--build", build, "--target", *TESTS, "brpc-shared", + "--parallel", self.args.jobs]) + self.step("install", ["cmake", "--install", build]) + check_enabled(runtime / "include/butil/config.h") + for name in TESTS: + self.check_gtest(name, build / "test" / name, build) + codegen = self.work / "codegen" + shared_library = "libbrpc.dylib" if sys.platform == "darwin" else "libbrpc.so" + self.step("codegen-configure", ["cmake", "-S", self.source / "tools/flatbuffers", "-B", codegen] + self.common_cmake() + [ + "-DBUILD_TESTING=ON", "-DBRPC_CODEGEN_BRPC_LIBRARY=" + str(build / "output/lib" / shared_library), + "-DFLATBUFFERS_INCLUDE_DIR=" + str(self.fb / "include"), + "-DFLATBUFFERS_LIBRARY=" + str(self.fb_library), "-DFLATC_EXECUTABLE=" + str(self.flatc)]) + self.step("codegen-build", ["cmake", "--build", codegen, "--parallel", self.args.jobs]) + self.ctest("codegen", codegen, {"flatbuffers_codegen_acceptance", "flatbuffers_codegen_runtime"}) + example = self.work / "example" + self.step("example-configure", ["cmake", "-S", self.source / "example/benchmark_fb", "-B", example] + self.common_cmake() + [ + "-DBUILD_TESTING=ON", "-DBRPC_ROOT=" + str(runtime), + "-DFLATBUFFERS_INCLUDE_DIR=" + str(self.fb / "include"), + "-DFLATC_EXECUTABLE=" + str(self.flatc), + "-DBRPC_FLATC_EXECUTABLE=" + str(codegen / "brpc_flatc")]) + self.step("example-build", ["cmake", "--build", example, "--parallel", self.args.jobs]) + self.ctest("example", example, {"benchmark_fb_smoke"}) + + def copy_source(self): + listing = self.step("source-files", ["git", "ls-files", "--cached", "--others", "--exclude-standard", "-z"]) + destination = self.work / "source" + destination.mkdir() + count = 0 + for name in set(listing.read_bytes().split(b"\0")): + if not name: + continue + relative = Path(os.fsdecode(name)) + path = self.source / relative + if (relative.is_absolute() or ".." in relative.parts or path.is_symlink() or + self.source not in path.resolve().parents): + raise ValueError("Unsafe checkout path: " + str(relative)) + if not path.exists(): + continue + if not path.is_file(): + raise ValueError("Only regular source files are supported: " + str(relative)) + target = destination / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(path, target) + count += 1 + if not count: + raise ValueError("The source must be a nonempty Git checkout") + self.env["CCACHE_BASEDIR"] = str(destination) + return destination + + def run_make(self): + source = self.copy_source() + headers = [str(prefix / "include") for prefix in self.prefixes] + ["/usr/include"] + libs = [str(prefix / "lib") for prefix in self.prefixes] + ["/usr/lib", "/usr/lib64"] + self.step("configure", ["sh", "config_brpc.sh", "--with-flatbuffers", + "--headers=" + " ".join(headers), "--libs=" + " ".join(libs), + "--cc=" + self.compiler, "--cxx=" + self.cxx], cwd=source) + self.step("build", ["make", "-j" + str(self.args.jobs)], cwd=source) + check_enabled(source / "output/include/butil/config.h") + self.step("test-build", ["make", "-C", "test", "-j" + str(self.args.jobs), + "FLATC=" + str(self.flatc), *TESTS], cwd=source) + variable = "DYLD_LIBRARY_PATH" if sys.platform == "darwin" else "LD_LIBRARY_PATH" + self.env[variable] = os.pathsep.join([str(source / "test")] + + [str(prefix / "lib") for prefix in self.prefixes]) + for name in TESTS: + self.check_gtest(name, source / "test" / name) + + def run_bazel(self): + source = self.copy_source() + command = ["bazel", "--batch", "--output_user_root=" + str(self.work / "bazel-state"), + "test", "--define=BRPC_WITH_FLATBUFFERS=true", "--jobs=" + str(self.args.jobs), + "--local_test_jobs=1", "--test_timeout=300", "--cache_test_results=no", + "--runs_per_test=1", "--flaky_test_attempts=1", "--test_output=errors", + "--test_env=GTEST_FILTER=*", "--test_env=GTEST_REPEAT=1"] + try: + self.step("bazel-tests", command + ["//test:" + name for name in TESTS], cwd=source, timeout=1800) + finally: + for name in TESTS: + for filename in ("test.xml", "test.log"): + path = source / "bazel-testlogs/test" / name / filename + if path.is_file(): + shutil.copy2(path, self.evidence / (name + "." + filename)) + for name in TESTS: + listing = self.step(name + "-list", [source / "bazel-bin/test" / name, "--gtest_list_tests"], cwd=source, timeout=30) + self.results[name] = verify_gtests(self.evidence / (name + ".test.xml"), listing, TESTS[name]) + self.save("running") + + def run(self): + self.step("compiler-version", [self.cxx, "--version"], timeout=10) + self.step("cmake-version", ["cmake", "--version"], timeout=10) + if self.args.build_system == "bazel": + self.step("bazel-version", ["bazel", "--version"], timeout=30) + self.run_bazel() + else: + self.prepare_dependencies() + getattr(self, "run_" + self.args.build_system)() + self.save("passed") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--build-system", choices=("cmake", "make", "bazel"), required=True) + parser.add_argument("--source", type=Path, default=Path(__file__).resolve().parents[2]) + parser.add_argument("--work", type=Path, required=True) + parser.add_argument("--jobs", type=int, default=2) + parser.add_argument("--flatbuffers-prefix", type=Path) + parser.add_argument("--gtest-source", type=Path) + parser.add_argument("--dependency-prefix", type=Path, action="append", default=[]) + parser.add_argument("--protoc", type=Path) + args = parser.parse_args() + if not 1 <= args.jobs <= 16: + parser.error("--jobs must be in 1..16") + runner = None + try: + runner = Runner(args) + runner.run() + except (Exception, KeyboardInterrupt) as error: + if runner: + runner.save("failed") + (runner.evidence / "failure.txt").write_text(str(error) + "\n") + print("FlatBuffers ON gate failed:", error, file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + signal.signal(signal.SIGTERM, lambda *_: (_ for _ in ()).throw(KeyboardInterrupt())) + sys.exit(main()) diff --git a/.github/scripts/test_flatbuffers_on.py b/.github/scripts/test_flatbuffers_on.py new file mode 100644 index 0000000000..4071f97645 --- /dev/null +++ b/.github/scripts/test_flatbuffers_on.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import argparse +import base64 +import hashlib +import importlib.util +import io +import json +import os +from pathlib import Path +import signal +import select +import subprocess +import sys +import time +import tarfile +import tempfile +import unittest +from unittest import mock +import xml.etree.ElementTree as ET + +SCRIPT = Path(__file__).with_name("flatbuffers-on.py") +SPEC = importlib.util.spec_from_file_location("flatbuffers_on", SCRIPT) +gate = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(gate) + + +class GateTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.listing = self.root / "list.txt" + self.listing.write_text("Suite.\n First\n Second\n") + self.xml = self.root / "tests.xml" + + def report(self, names=("First", "Second"), **attributes): + root = ET.Element("testsuites", tests=str(len(names)), failures="0", errors="0", disabled="0") + root.attrib.update(attributes) + suite = ET.SubElement(root, "testsuite", name="Suite") + for name in names: + ET.SubElement(suite, "testcase", classname="Suite", name=name, + status="run", result="completed") + ET.ElementTree(root).write(self.xml) + return root + + def save(self, root): + ET.ElementTree(root).write(self.xml) + + def test_successful_report_matches_discovery(self): + self.report() + self.assertEqual(2, gate.verify_gtests(self.xml, self.listing, 2)["executed"]) + + def test_parameterized_listing(self): + self.assertEqual({"Typed/0.Works/1"}, gate.list_gtests( + "Running main from gtest\nTyped/0. # TypeParam = int\n Works/1 # GetParam = 1\n")) + + def test_empty_or_disabled_listing_rejected(self): + for text in ("", "no tests", "DISABLED_Suite.\n Test\n", "Suite.\n DISABLED_Test\n"): + with self.subTest(text=text), self.assertRaises(ValueError): + gate.list_gtests(text) + + def test_duplicate_listing_rejected(self): + with self.assertRaises(ValueError): + gate.list_gtests("Suite.\n First\n First\n") + + def test_empty_or_missing_xml_rejected(self): + with self.assertRaises(ValueError): + gate.verify_gtests(self.xml, self.listing, 1) + self.report(()) + with self.assertRaises(ValueError): + gate.verify_gtests(self.xml, self.listing, 1) + + def test_filtered_and_unexpected_count_rejected(self): + self.report(("First",)) + with self.assertRaises(ValueError): + gate.verify_gtests(self.xml, self.listing, 1) + self.report() + # The expected count is exact: both fewer and more cases are rejected. + for count in (1, 3): + with self.subTest(count=count), self.assertRaises(ValueError): + gate.verify_gtests(self.xml, self.listing, count) + + def test_failure_error_skip_disabled_rejected(self): + for tag in ("failure", "error", "skipped"): + root = self.report() + ET.SubElement(next(root.iter("testcase")), tag) + self.save(root) + with self.subTest(tag=tag), self.assertRaises(ValueError): + gate.verify_gtests(self.xml, self.listing, 2) + for attribute in ("failures", "errors", "disabled", "skipped"): + self.report(**{attribute: "1"}) + with self.subTest(attribute=attribute), self.assertRaises(ValueError): + gate.verify_gtests(self.xml, self.listing, 2) + + def test_unexecuted_and_duplicate_cases_rejected(self): + root = self.report() + next(root.iter("testcase")).set("status", "notrun") + self.save(root) + with self.assertRaises(ValueError): + gate.verify_gtests(self.xml, self.listing, 2) + self.report(("First", "First")) + with self.assertRaises(ValueError): + gate.verify_gtests(self.xml, self.listing, 2) + + def test_incorrect_root_count_rejected(self): + self.report(tests="999") + with self.assertRaises(ValueError): + gate.verify_gtests(self.xml, self.listing, 2) + + def test_ctest_requires_runtime_and_acceptance(self): + self.report(("acceptance", "runtime")) + self.assertEqual(2, gate.verify_ctest(self.xml, {"acceptance", "runtime"})["executed"]) + self.report(("acceptance",)) + with self.assertRaises(ValueError): + gate.verify_ctest(self.xml, {"acceptance", "runtime"}) + + def archive(self, name="pkg/file", link=False): + path = self.root / "archive.tar.gz" + with tarfile.open(path, "w:gz") as archive: + info = tarfile.TarInfo(name) + if link: + info.type = tarfile.SYMTYPE + info.linkname = "/outside" + archive.addfile(info) + else: + info.size = 4 + archive.addfile(info, io.BytesIO(b"data")) + return path, hashlib.sha256(path.read_bytes()).hexdigest() + + def test_pinned_archive_extracts(self): + path, digest = self.archive() + extracted = gate.extract_archive(path, digest, self.root / "extract", "pkg") + self.assertEqual(b"data", (extracted / "file").read_bytes()) + + def test_archive_checksum_paths_and_links_rejected(self): + path, _ = self.archive() + with self.assertRaises(ValueError): + gate.extract_archive(path, "0" * 64, self.root / "extract", "pkg") + for name, link in (("../outside", False), ("pkg/../outside", False), + ("/absolute", False), ("wrong/file", False), ("pkg/link", True)): + path, digest = self.archive(name, link) + with self.subTest(name=name), self.assertRaises(ValueError): + gate.extract_archive(path, digest, self.root / "extract", "pkg") + self.assertFalse((self.root / "extract").exists()) + + def linked_archive(self, entries): + path = self.root / "linked.tar.gz" + with tarfile.open(path, "w:gz") as archive: + for name, target in entries: + info = tarfile.TarInfo(name) + if target is not None: + info.type = tarfile.SYMTYPE + info.linkname = target + archive.addfile(info) + else: + info.size = 4 + archive.addfile(info, io.BytesIO(b"data")) + return path, hashlib.sha256(path.read_bytes()).hexdigest() + + def test_internal_archive_symlinks_extract_after_data(self): + path, digest = self.linked_archive([ + ("pkg/java/src/test/java/Example", "../../../../tests/Example"), + ("pkg/ts/package.json", "../package.json"), + ("pkg/tests/Example/schema.fbs", None), ("pkg/package.json", None)]) + extracted = gate.extract_archive(path, digest, self.root / "extract", "pkg") + self.assertTrue((extracted / "java/src/test/java/Example").is_symlink()) + self.assertEqual(b"data", (extracted / "java/src/test/java/Example/schema.fbs").read_bytes()) + self.assertEqual(b"data", (extracted / "ts/package.json").read_bytes()) + + def test_escaping_chained_and_ancestor_links_rejected(self): + cases = [ + [("pkg/link", "../../outside")], [("pkg", ".")], + [("pkg/link", "next"), ("pkg/next", "file"), ("pkg/file", None)], + [("pkg/dir", "target"), ("pkg/dir/file", None)], + [("pkg/a", "."), ("pkg/link", "nested/../a/../outside")], + ] + for entries in cases: + path, digest = self.linked_archive(entries) + with self.subTest(entries=entries), self.assertRaises(ValueError): + gate.extract_archive(path, digest, self.root / "extract", "pkg") + self.assertFalse((self.root / "extract").exists()) + + def test_duplicate_archive_members_rejected(self): + path, digest = self.linked_archive([("pkg/file", None), ("pkg/file", None)]) + with self.assertRaises(ValueError): + gate.extract_archive(path, digest, self.root / "extract", "pkg") + self.assertFalse((self.root / "extract").exists()) + + def runner(self): + source = self.root / "source" + source.mkdir() + return gate.Runner(argparse.Namespace(source=source, work=self.root / "work", + build_system="cmake", dependency_prefix=[])) + + def test_work_directory_is_exclusive(self): + runner = self.runner() + with self.assertRaises(FileExistsError): + gate.Runner(runner.args) + + def test_filter_environment_is_reset(self): + with mock.patch.dict(os.environ, {"GTEST_FILTER": "Wrong.*", "GTEST_SHARD_INDEX": "2"}): + runner = self.runner() + self.assertEqual("*", runner.env["GTEST_FILTER"]) + self.assertNotIn("GTEST_SHARD_INDEX", runner.env) + + def test_work_inside_checkout_rejected(self): + source = self.root / "checkout" + source.mkdir() + args = argparse.Namespace(source=source, work=source / "build", build_system="cmake", dependency_prefix=[]) + with self.assertRaises(ValueError): + gate.Runner(args) + + def test_source_directory_link_cannot_escape_checkout(self): + runner = self.runner() + outside = self.root / "outside" + outside.mkdir() + (outside / "file").write_text("must not be copied") + (runner.source / "link").symlink_to(outside, target_is_directory=True) + self.listing.write_bytes(b"link/file\0") + with mock.patch.object(runner, "step", return_value=self.listing): + with self.assertRaises(ValueError): + runner.copy_source() + self.assertFalse((runner.work / "source/link/file").exists()) + + def test_cmake_uses_explicit_openssl_prefix(self): + runner = self.runner() + runner.args.protoc = None + self.assertFalse(any(option.startswith("-DOPENSSL_ROOT_DIR=") + for option in runner.common_cmake())) + prefix = self.root / "openssl" + (prefix / "include/openssl").mkdir(parents=True) + (prefix / "include/openssl/ssl.h").write_text("header fixture") + runner.prefixes = [self.root / "unrelated", prefix] + self.assertIn("-DOPENSSL_ROOT_DIR=" + str(prefix), runner.common_cmake()) + + def test_make_builds_supplied_gtest_in_private_prefix(self): + runner = self.runner() + prefix = self.root / "provided-fb" + (prefix / "include/flatbuffers").mkdir(parents=True) + (prefix / "include/flatbuffers/base.h").write_text( + "#define FLATBUFFERS_VERSION_MAJOR 25\n#define FLATBUFFERS_VERSION_MINOR 2\n#define FLATBUFFERS_VERSION_REVISION 10\n") + (prefix / "lib").mkdir() + (prefix / "lib/libflatbuffers.a").touch() + source = self.root / "provided-gtest" + source.mkdir() + runner.args.build_system = "make" + runner.args.flatbuffers_prefix = prefix + runner.args.gtest_source = source + runner.args.jobs = 2 + runner.args.protoc = Path(sys.executable) + version = self.root / "version.log" + version.write_text("flatc version 25.2.10\n") + with mock.patch.object(runner, "step", return_value=version) as step: + with mock.patch.object(runner, "download") as download: + runner.prepare_dependencies() + download.assert_not_called() + configure = next(call.args[1] for call in step.call_args_list + if call.args[0] == "gtest-configure") + self.assertIn(source.resolve(), configure) + self.assertIn("-DCMAKE_INSTALL_PREFIX=" + str(runner.work / "gtest-prefix"), configure) + self.assertEqual(runner.work / "gtest-prefix", runner.prefixes[0]) + self.assertFalse((prefix / "include/gtest").exists()) + + def exercise_process_group(self, leader_exits): + ready = self.root / "child.ready" + child_code = ("import signal,time,pathlib; signal.signal(signal.SIGTERM, signal.SIG_IGN); " + "pathlib.Path(" + repr(str(ready)) + ").touch(); time.sleep(60)") + leader_code = ("import subprocess,sys,time,pathlib; " + "child=subprocess.Popen([sys.executable,'-c'," + repr(child_code) + "], " + "stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL); " + "print(child.pid,flush=True); " + + ("sys.exit(0)" if leader_exits else "time.sleep(60)")) + process = subprocess.Popen([sys.executable, "-c", leader_code], + stdout=subprocess.PIPE, text=True, start_new_session=True) + try: + readable, _, _ = select.select([process.stdout], [], [], 3) + self.assertTrue(readable, "group leader did not publish child PID") + child = int(process.stdout.readline()) + deadline = time.monotonic() + 3 + while not ready.exists() and time.monotonic() < deadline: + time.sleep(0.02) + self.assertTrue(ready.exists()) + if leader_exits: + process.wait(timeout=3) + gate.stop_process(process, grace=0.1) + deadline = time.monotonic() + 3 + while True: + result = subprocess.run(["ps", "-o", "stat=", "-p", str(child)], + stdout=subprocess.PIPE, text=True, check=False) + state = result.stdout.strip() + if not state or state.startswith("Z"): + break + self.assertLess(time.monotonic(), deadline, "owned child survived cleanup") + time.sleep(0.02) + finally: + gate.signal_group(process, signal.SIGKILL) + process.wait(timeout=3) + process.stdout.close() + + def test_cleanup_after_group_leader_exits(self): + self.exercise_process_group(True) + + def test_cleanup_kills_child_ignoring_term(self): + self.exercise_process_group(False) + + def test_timeout_is_a_recorded_failure(self): + runner = self.runner() + with self.assertRaises(RuntimeError): + runner.step("timeout", [sys.executable, "-c", "import time; time.sleep(60)"], timeout=0.05) + self.assertEqual(124, json.loads((runner.evidence / "summary.json").read_text())["steps"][0]["exitcode"]) + + def test_flatbuffers_pin_matches_both_bazel_definitions(self): + root = SCRIPT.resolve().parents[2] + for filename in ("MODULE.bazel", "WORKSPACE"): + text = (root / filename).read_text() + integrity = "sha256-" + base64.b64encode(bytes.fromhex(gate.FLATBUFFERS_SHA256)).decode() + self.assertTrue(gate.FLATBUFFERS_SHA256 in text or integrity in text, + filename + ": FlatBuffers checksum differs") + self.assertTrue("v" + gate.FLATBUFFERS_VERSION + ".tar.gz" in text, + filename + ": FlatBuffers version differs") + + def test_workflow_script_paths_and_characters(self): + root = SCRIPT.resolve().parents[2] + workflow = (root / ".github/workflows/flatbuffers-on.yml").read_text() + self.assertFalse(any(ord(char) < 32 and char not in "\n\r\t" for char in workflow)) + for filename in ("flatbuffers-on.py", "test_flatbuffers_on.py"): + self.assertIn(".github/scripts/" + filename, workflow) + self.assertTrue((SCRIPT.parent / filename).is_file()) + self.assertNotIn("pull_request_target", workflow) + self.assertNotIn("continue-on-error", workflow) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/flatbuffers-on.yml b/.github/workflows/flatbuffers-on.yml new file mode 100644 index 0000000000..3e754dd6b0 --- /dev/null +++ b/.github/workflows/flatbuffers-on.yml @@ -0,0 +1,130 @@ +name: FlatBuffers ON + +on: + push: + branches: [master] + paths: + - 'src/brpc/flatbuffers/**' + - 'src/brpc/policy/flatbuffers_protocol.*' + - 'src/brpc/channel.*' + - 'src/brpc/controller.*' + - 'src/brpc/server.*' + - 'src/brpc/global.cpp' + - 'src/brpc/options.proto' + - 'src/brpc/details/*_private_accessor.h' + - 'test/*flatbuffers*' + - 'test/CMakeLists.txt' + - 'test/Makefile' + - 'test/BUILD.bazel' + - 'tools/flatbuffers/**' + - 'example/benchmark_fb/**' + - 'CMakeLists.txt' + - 'src/**/CMakeLists.txt' + - 'Makefile' + - 'config_brpc.sh' + - 'BUILD.bazel' + - 'MODULE.bazel' + - 'WORKSPACE' + - '.github/scripts/*flatbuffers*' + - '.github/workflows/flatbuffers-on.yml' + pull_request: + paths: + - 'src/brpc/flatbuffers/**' + - 'src/brpc/policy/flatbuffers_protocol.*' + - 'src/brpc/channel.*' + - 'src/brpc/controller.*' + - 'src/brpc/server.*' + - 'src/brpc/global.cpp' + - 'src/brpc/options.proto' + - 'src/brpc/details/*_private_accessor.h' + - 'test/*flatbuffers*' + - 'test/CMakeLists.txt' + - 'test/Makefile' + - 'test/BUILD.bazel' + - 'tools/flatbuffers/**' + - 'example/benchmark_fb/**' + - 'CMakeLists.txt' + - 'src/**/CMakeLists.txt' + - 'Makefile' + - 'config_brpc.sh' + - 'BUILD.bazel' + - 'MODULE.bazel' + - 'WORKSPACE' + - '.github/scripts/*flatbuffers*' + - '.github/workflows/flatbuffers-on.yml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + build-test: + name: FB ON (${{ matrix.os }}, ${{ matrix.system }}, ${{ matrix.compiler }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + max-parallel: 2 + matrix: + include: + - {os: ubuntu-22.04, system: cmake, compiler: gcc, cc: gcc, cxx: g++, cache: ccache} + - {os: ubuntu-22.04, system: cmake, compiler: clang, cc: clang, cxx: clang++, cache: ccache} + - {os: ubuntu-22.04, system: make, compiler: gcc, cc: gcc, cxx: g++, cache: ccache} + - {os: ubuntu-22.04, system: make, compiler: clang, cc: clang, cxx: clang++, cache: ccache} + - {os: ubuntu-22.04, system: bazel, compiler: gcc, cc: gcc, cxx: g++, cache: bazel} + - {os: macos-latest, system: cmake, compiler: apple-clang, cc: clang, cxx: clang++, cache: ccache} + env: + CC: ${{ matrix.cc }} + CXX: ${{ matrix.cxx }} + FB_WORK: ${{ runner.temp }}/brpc-fb-${{ matrix.system }}-${{ matrix.compiler }} + BUILD_SYSTEM: ${{ matrix.system }} + steps: + - uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + + - name: Validate the gate's report checks + run: python3 .github/scripts/test_flatbuffers_on.py + + - name: Install focused Linux dependencies + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + build-essential cmake ninja-build clang ccache pkg-config python3 curl ca-certificates git \ + libssl-dev libgflags-dev libprotobuf-dev libprotoc-dev protobuf-compiler \ + libleveldb-dev libgoogle-perftools-dev zlib1g-dev + + - name: Install focused macOS dependencies + if: runner.os == 'macOS' + run: brew install cmake ninja pkg-config openssl@3 gflags leveldb protobuf@29 abseil + + - uses: ./.github/actions/setup-build-cache + with: + kind: ${{ matrix.cache }} + cache-key: fb-on-${{ matrix.system }}-${{ matrix.compiler }} + + - name: Build and run the FB ON gates + shell: bash + run: | + args=(--build-system "$BUILD_SYSTEM" --work "$FB_WORK" --jobs 2) + if [[ "$RUNNER_OS" == macOS ]]; then + for package in protobuf@29 abseil openssl@3 gflags leveldb; do + args+=(--dependency-prefix "$(brew --prefix "$package")") + done + args+=(--protoc "$(brew --prefix protobuf@29)/bin/protoc") + fi + python3 .github/scripts/flatbuffers-on.py "${args[@]}" + + - name: Upload commands, logs and test reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: flatbuffers-on-${{ matrix.os }}-${{ matrix.system }}-${{ matrix.compiler }} + path: ${{ env.FB_WORK }}/evidence/ + if-no-files-found: warn + retention-days: 14 diff --git a/.gitignore b/.gitignore index 7a9ade11d1..7c646771b1 100644 --- a/.gitignore +++ b/.gitignore @@ -17,8 +17,10 @@ /test/output build/ -# Ignore hidden files +# Ignore hidden files, but keep GitHub workflows and helpers discoverable. .* +!/.github/ +/.github/scripts/__pycache__/ *.swp # Ignore auto-generated files From 8cbb17b8efce6dd192814bb73fa18887ee324b76 Mon Sep 17 00:00:00 2001 From: xulei25 Date: Mon, 21 Sep 2026 15:07:16 +0800 Subject: [PATCH 4/6] Document FlatBuffers setup and example checks Add Chinese and English guides covering how to enable FlatBuffers, the dependency and ABI requirements, building the runtime, running the example and its smoke test, the network RPC API and the FRPC wire format. Update the generator README for the RPC example. --- README.md | 1 + README_cn.md | 1 + docs/cn/flatbuffers.md | 296 ++++++++++++++++++++++++++++++++---- docs/en/flatbuffers.md | 281 ++++++++++++++++++++++++++++++---- tools/flatbuffers/README.md | 47 +++++- 5 files changed, 557 insertions(+), 69 deletions(-) diff --git a/README.md b/README.md index d65366fafb..3fa3ea7f08 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ You can use it to: * Read [overview](docs/en/overview.md) to know where bRPC can be used and its advantages. * Read [getting started](docs/en/getting_started.md) for building steps and play with [examples](https://github.com/apache/brpc/tree/master/example/). * Docs: + * [Enable FlatBuffers RPC and verify the example](docs/en/flatbuffers.md) * [Performance benchmark](docs/en/benchmark.md) * [bvar](docs/en/bvar.md) * [bvar_c++](docs/en/bvar_c++.md) diff --git a/README_cn.md b/README_cn.md index 2cc686bd85..b3a5946c0d 100644 --- a/README_cn.md +++ b/README_cn.md @@ -32,6 +32,7 @@ * 通过[概述](docs/cn/overview.md)了解哪里可以用bRPC及其优势。 * 阅读[编译步骤](docs/cn/getting_started.md)了解如何开始使用, 之后可以运行一下[示例程序](https://github.com/apache/brpc/tree/master/example/). * 文档: + * [启用 FlatBuffers RPC 并通过 example 验证](docs/cn/flatbuffers.md) * [性能测试](docs/cn/benchmark.md) * [bvar](docs/cn/bvar.md) * [bvar_c++](docs/cn/bvar_c++.md) diff --git a/docs/cn/flatbuffers.md b/docs/cn/flatbuffers.md index 5e58801594..5cba140608 100644 --- a/docs/cn/flatbuffers.md +++ b/docs/cn/flatbuffers.md @@ -1,52 +1,261 @@ -# FlatBuffers 消息 +# 启用 FlatBuffers 消息与 RPC [English version](../en/flatbuffers.md) -bRPC 支持基于 IOBuf 的 FlatBuffers 消息、构造器和服务描述符，默认不启用。 -消息构造方案源自 [apache/brpc#3196](https://github.com/apache/brpc/pull/3196)。 +bRPC 提供基于 IOBuf 的 FlatBuffers 消息、构造器、服务描述及 `fb_rpc` 协议。 +支持默认关闭，需先编译启用该功能的 bRPC，再让客户端和服务端使用同一套头文件、 +生成配置和运行库。完整示例位于 [example/benchmark_fb](../../example/benchmark_fb/README.md)。 -本组件不包含 `fb_rpc` 传输协议，也不改动 `brpc::Channel` 或 `brpc::Server`。 -服务代码生成和进程内分派测试不涉及网络 RPC，也不是性能测试。 +## 1. 启用选项与依赖 -## 构建 +| 构建方式 | 启用选项 | 可供独立 example 使用的运行库前缀 | +| --- | --- | --- | +| CMake | `-DWITH_FLATBUFFERS=ON` | `/output` | +| Make | `config_brpc.sh --with-flatbuffers` | `/output` | +| Bazel | build/test 命令均传 `--define=BRPC_WITH_FLATBUFFERS=true` | 原始 Bazel 输出不是 example 所需的安装前缀 | -运行库只需要 FlatBuffers 头文件，不链接其库；测试还需要版本匹配的 `flatc`。 -版本不一致时应重新生成头文件，不要删除或放宽上游生成代码中的版本断言。 +**不能只给应用增加 `-DBRPC_WITH_FLATBUFFERS=1`。** 该功能会改变 Channel、Controller、 +Server 的 ABI；ON 头文件与 OFF 运行库混用不受支持。`butil/config.h` 中的宏始终为 0 或 1， +应用应使用 `#if BRPC_WITH_FLATBUFFERS`，而不是 `#ifdef`。 -假设 GoogleTest 源码位于 `/usr/src/googletest`： +先按[入门指南](getting_started.md)准备 C++ 工具链、Protobuf 编译器及开发库、gflags、 +LevelDB、OpenSSL、zlib。FlatBuffers RPC **并不消除 Protobuf 依赖**。 -```sh -cmake -S . -B build -DWITH_FLATBUFFERS=ON -DBUILD_UNIT_TESTS=ON \ - -DBUILD_BRPC_TOOLS=OFF -DDOWNLOAD_GTEST=OFF \ - -DBRPC_SYSTEM_GTEST_SOURCE_DIR=/usr/src/googletest -cmake --build build --target brpc_flatbuffers_unittest -j6 -ctest --test-dir build -R '^brpc_flatbuffers_unittest$' --output-on-failure -``` +| 组件 | FlatBuffers 或测试依赖 | +| --- | --- | +| bRPC 消息及 RPC 运行库 | FlatBuffers 头文件，不链接 `libflatbuffers` | +| 官方 schema 代码生成 | 与头文件版本一致的 `flatc` | +| bRPC 服务代码生成 | `brpc_flatc`，其构建另需匹配的官方头文件和 `libflatbuffers` | +| example 烟测 | Python 3，不需要 GoogleTest | +| 库单元测试 | GoogleTest 及项目原有测试依赖 | -非标准安装可设置 `FLATBUFFERS_INCLUDE_DIR`、`FLATBUFFERS_FLATC_EXECUTABLE` -和 `BRPC_SYSTEM_GTEST_SOURCE_DIR`。其他测试依赖与 bRPC 原有配置相同。 +本仓库的 Bazel 和 ON gate 固定使用 **FlatBuffers 25.2.10**。复现该流程时，应使用同版本的 +头文件、官方 `flatc` 和用于构建 `brpc_flatc` 的库。不要删除生成头文件中的版本断言。 -Make 使用 `config_brpc.sh --with-flatbuffers`，测试时可通过 -`FLATC=/path/to/flatc` 指定生成器。Bazel 使用 -`--define=BRPC_WITH_FLATBUFFERS=true`。 +三个容易混淆的参数： -运行库使用了 FlatBuffers builder 的内部结构，目前只支持 25.2.10，其他版本会在 -编译期报错。Bzlmod 和 WORKSPACE 使用同一个源码归档，并用校验和锁定内容。 -未采用 `bazel_dep`，是为了避免引入 gRPC、多语言工具以及与 bRPC 冲突的 BoringSSL -依赖。 +- 根项目 CMake 的库测试：`FLATBUFFERS_FLATC_EXECUTABLE` 指向官方 `flatc`。 +- 生成器验收及 example：`FLATC_EXECUTABLE` 指向官方 `flatc`。 +- example：`BRPC_FLATC_EXECUTABLE` 指向 bRPC 的 `brpc_flatc`，不能填成官方 `flatc`。 -公共头文件位于 `brpc/flatbuffers/`，命名空间为 `brpc::flatbuffers`。 -构造消息时包含 `message.h`；使用服务描述符和接口时包含 `service.h`。`butil/config.h` 中的 -`BRPC_WITH_FLATBUFFERS` 始终是 0 或 1，因此应使用 `#if` 判断。 +各阶段还必须使用兼容的同一套 Protobuf。当 CMake 检测到 `Protobuf_VERSION > 4.21` 时， +需要 C++17 及匹配的 Abseil 依赖；生成器验收和 example 需要 Protobuf 的 CMake config 包 +导出传递依赖。运行库与 example 最低要求 CMake 3.16；下文 CTest 命令使用 3.17+ 的 `--no-tests=error` +防止空测试误报成功。只有 3.16 时，可直接执行 `smoke.py`。以下按 Linux/macOS 的 +Unix Makefiles 或 Ninja 单配置生成器编写。 -旧版 flatc 2.0.x 可能生成未全限定名称。业务 schema 不要使用 `brpc` 命名空间，也不要依赖 include 顺序。 +运行库使用了 FlatBuffers builder 的内部结构，目前只支持 25.2.10，其他版本会在编译期报错。 +版本不一致时应重新生成头文件，不要删除或放宽上游生成代码中的版本断言。Bzlmod 和 WORKSPACE +使用同一个源码归档，并用校验和锁定内容；未采用 `bazel_dep`，是为了避免引入 gRPC、多语言工具 +以及与 bRPC 冲突的 BoringSSL 依赖。 如需使用其他 FlatBuffers 版本，需要修改运行库、生成器和 codegen 测试中的版本检查， 再用该版本完整运行 `tools/flatbuffers/verify.sh`。步骤见生成器 [README](../../tools/flatbuffers/README.md#using-another-flatbuffers-version)， 其中以 23.5.26 为例，仅适用于 CMake 构建。 -## 消息构造和所有权 +## 2. 用 CMake 构建启用后的运行库 + +从 bRPC checkout 开始。按需提前设置有效的 `CC`、`CXX`。 +CMake 构建把生成文件放在 build 目录，example 的绑定代码与构建产物放在 `WORK` 中。 +独立 example 需要公共头文件和 `config.h`，build 目录里没有这些文件，所以要把运行库安装到一个前缀。 + +将下列变量替换为已有安装前缀，即包含 `include/`、`lib/` 或 `lib64/` 的目录；多个变量 +可以指向同一前缀。 + +```sh +REPO="$PWD" +DEPS=/absolute/path/to/dependency-prefix +FB=/absolute/path/to/flatbuffers-prefix +OPENSSL=/absolute/path/to/openssl-prefix +WORK="$(mktemp -d "${TMPDIR:-/tmp}/brpc-benchmark-fb.XXXXXX")" +JOBS=2 +printf 'WORK=%s\n' "$WORK" +``` + +若尚未安装 FlatBuffers，可先从匹配版本的官方源码构建。已有完整安装时跳过此段。 +`FB` 必须是可写的安装前缀，不是源码目录： + +```sh +FB_SOURCE=/absolute/path/to/flatbuffers-25.2.10-source +cmake -S "$FB_SOURCE" -B "$WORK/flatbuffers" \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DCMAKE_INSTALL_PREFIX="$FB" -DCMAKE_INSTALL_LIBDIR=lib \ + -DFLATBUFFERS_BUILD_TESTS=OFF -DFLATBUFFERS_BUILD_FLATC=ON \ + -DFLATBUFFERS_BUILD_FLATLIB=ON -DFLATBUFFERS_BUILD_SHAREDLIB=OFF \ + -DFLATBUFFERS_INSTALL=ON -DFLATBUFFERS_LIBCXX_WITH_CLANG=OFF +cmake --build "$WORK/flatbuffers" --parallel "$JOBS" +cmake --install "$WORK/flatbuffers" +"$FB/bin/flatc" --version +``` + +构建静态运行库，不启用库单元测试，因此这一阶段不需要 GoogleTest： + +```sh +cmake -S "$REPO" -B "$WORK/runtime" \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_PREFIX_PATH="$DEPS;$FB;$OPENSSL" \ + -DOPENSSL_ROOT_DIR="$OPENSSL" \ + -DCMAKE_INSTALL_PREFIX="$WORK/prefix" \ + -DWITH_FLATBUFFERS=ON -DBUILD_SHARED_LIBS=OFF \ + -DFLATBUFFERS_INCLUDE_DIR="$FB/include" \ + -DBUILD_UNIT_TESTS=OFF -DDOWNLOAD_GTEST=OFF -DBUILD_BRPC_TOOLS=OFF +cmake --build "$WORK/runtime" --parallel "$JOBS" +cmake --install "$WORK/runtime" +grep -E '^#define BRPC_WITH_FLATBUFFERS +1$' "$WORK/prefix/include/butil/config.h" +``` + +最后一条检查必须输出 `#define BRPC_WITH_FLATBUFFERS 1`。公共接口位于 +`brpc/flatbuffers/message.h` 和 `brpc/flatbuffers/service.h`。 +上面的 policy 参数是兼容 CMake 4 下某些旧依赖策略的可选设置，不是启用 FlatBuffers 的 +开关，也不能替代编译器和依赖版本要求。 + +## 3. 通过 example 完成端到端验证 + +该示例是**有界功能验证，不是性能对比**。[echo.fbs](../../example/benchmark_fb/echo.fbs) +声明 `BenchmarkService.Echo`，显式 wire method ID 为 7。 +[server.cpp](../../example/benchmark_fb/server.cpp) 使用 `AddFlatBuffersService` 注册服务； +[client.cpp](../../example/benchmark_fb/client.cpp) 使用生成的 `BenchmarkService::Stub` +及 `fb_rpc` channel。 + +### 构建生成器和示例 + +在前一节同一个 shell 中继续，沿用 `REPO`、`DEPS`、`FB`、`OPENSSL`、`WORK`、`JOBS`： + +```sh +cmake -S "$REPO/tools/flatbuffers" -B "$WORK/codegen" \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DBUILD_TESTING=OFF \ + -DCMAKE_PREFIX_PATH="$FB;$DEPS" \ + -DFLATBUFFERS_INCLUDE_DIR="$FB/include" +cmake --build "$WORK/codegen" --parallel "$JOBS" + +cmake -S "$REPO/example/benchmark_fb" -B "$WORK/example" \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DBUILD_TESTING=ON \ + -DCMAKE_PREFIX_PATH="$DEPS;$FB;$OPENSSL" \ + -DOPENSSL_ROOT_DIR="$OPENSSL" \ + -DBRPC_ROOT="$WORK/prefix" \ + -DFLATBUFFERS_INCLUDE_DIR="$FB/include" \ + -DFLATC_EXECUTABLE="$FB/bin/flatc" \ + -DBRPC_FLATC_EXECUTABLE="$WORK/codegen/brpc_flatc" +cmake --build "$WORK/example" --parallel "$JOBS" +(cd "$WORK/example" && ctest -V --no-tests=error --output-on-failure -R '^benchmark_fb_smoke$') +``` + +构建时会执行两个生成器，文件位于 `WORK/example/generated/`： + +- 官方 `flatc`：`echo_generated.h`，定义 table 类型。 +- `brpc_flatc`：`echo.brpc.fb.h`、`echo.brpc.fb.cpp`，定义服务及 stub。 + +不要提交这些生成文件，也不要在更换版本后沿用旧头文件。生成器若选错库，可在其 CMake +配置中显式传入 `FLATBUFFERS_LIBRARY`。导入 schema、稳定方法 ID 等规则见 +[生成器说明](../../tools/flatbuffers/README.md)。 + +### 烟测的成功标准 + +必须看到名为 `benchmark_fb_smoke` 的 **1 项 CTest 通过**、命令退出码为 0，并输出： + +```text +benchmark_fb smoke passed: 13 verified replies, 2 schema rejections, clean shutdown +``` + +这是 15 次 RPC（13 次正常响应、2 次预期 schema 拒绝），不是 15 项 CTest。`No tests were found` +不算通过。烟测验证的内容、期限，以及不经 CTest 直接运行的方法见 +[example README](../../example/benchmark_fb/README.md#smoke-test)。 + +### 手工分别运行 server/client + +手工运行客户端的命令、预期结果和有界参数见 +[example README](../../example/benchmark_fb/README.md#run-separately)。 + +### 复用已有运行库或改为动态链接 + +已有兼容的 FB ON 运行库时，无需重编：将 example 配置中的 `BRPC_ROOT` 替换为其安装前缀 +（CMake）或 `output/`（Make），`BRPC_FLATC_EXECUTABLE` 指向匹配的生成器，其他步骤不变。 +example 会读取该前缀真实的 `include/butil/config.h`。 + +默认链接 `libbrpc.a`。使用 `LINK_SO=ON` 前，先用 `BUILD_SHARED_LIBS=ON` 构建 `brpc-shared` +并再次执行 `cmake --install`，再用 `-DLINK_SO=ON` 重新配置并构建 example；`LINK_SO` 本身 +不会构建运行库。运行库若增加可选依赖，可通过 `BRPC_EXTRA_LIBRARIES` 补充链接库。 + +## 4. Make、Bazel 和完整回归 + +### Make + +在独立可写 checkout 中配置，沿用前述依赖变量： + +```sh +sh config_brpc.sh --with-flatbuffers \ + --headers="$FB/include $DEPS/include $OPENSSL/include" \ + --libs="$DEPS/lib $OPENSSL/lib" --cc="${CC:-cc}" --cxx="${CXX:-c++}" +make -j"$JOBS" +``` + +按平台替换为实际 `lib64` 或库目录。产出的 `output/` 可以替换 example 命令中的 `$WORK/prefix`，作为 `BRPC_ROOT`。 +Make 单测通过 `FLATC="$FB/bin/flatc"` 选择官方生成器，但还需要已安装的 GoogleTest 库和 +gperftools，不是只提供 GoogleTest 源码即可；执行时也需能够加载 `test/libbrpc.dbg.*`。 +下方 ON runner 会构建 GoogleTest、设置测试库路径并校验报告；gperftools 等系统测试依赖 +仍需预先安装。 + +### Bazel + +```sh +bazel build --define=BRPC_WITH_FLATBUFFERS=true //:brpc +bazel test --define=BRPC_WITH_FLATBUFFERS=true --cache_test_results=no \ + //test:brpc_flatbuffers_unittest //test:brpc_flatbuffers_protocol_unittest +``` + +这些目标验证库，不会自动执行独立 example。不要将原始 `bazel-bin` 当成 `BRPC_ROOT`； +example 需要 CMake 安装前缀或 Make `output/` 的 include/lib 布局。 + +### 库单测与 ON gate + +单独配置库单测时，设置 `BUILD_UNIT_TESTS=ON`、匹配的 `FLATBUFFERS_FLATC_EXECUTABLE`； +若 `DOWNLOAD_GTEST=OFF`，还需指定 `BRPC_SYSTEM_GTEST_SOURCE_DIR`。 +构建并运行 `brpc_flatbuffers_unittest`、`brpc_flatbuffers_protocol_unittest`。 +仅打开库的 FlatBuffers 选项，不会自动执行测试。 + +要一次验证两组库单测、两项 codegen 验收及 example，可使用 +[ON runner](../../.github/scripts/flatbuffers-on.py)。完整 CMake 门禁需要 **CMake/CTest 3.21+** +以生成 JUnit XML 报告（`--output-junit`）。**`--work` 目录必须尚不存在**： + +```sh +python3 "$REPO/.github/scripts/flatbuffers-on.py" \ + --build-system cmake --source "$REPO" --work "$WORK/on-gate" --jobs "$JOBS" \ + --flatbuffers-prefix "$FB" \ + --dependency-prefix "$DEPS" --dependency-prefix "$OPENSSL" +``` + +该门禁要求 FlatBuffers 25.2.10。不传 `--flatbuffers-prefix` 时会下载、校验并构建该版本。 +GoogleTest 默认下载并校验固定的 1.14.0，也可用 `--gtest-source` 复用源码。 +平台常规依赖仍需预先安装。Make/Bazel 模式验证两组库测试，只有 CMake 模式额外构建 +生成器和 example。 + +检查 `WORK/on-gate/evidence/summary.json`：`status` 应为 `passed`，应包含所有预期测试， +执行数量非零，失败和跳过均为零。各步骤命令、日志及 XML 即使失败也会保留； +空测试、过滤后少跑或跳过不会被当作成功。 + +[CI 工作流](../../.github/workflows/flatbuffers-on.yml)覆盖 Linux CMake/Make 的 GCC、Clang， +Linux Bazel GCC，以及 macOS CMake。本地 gate 通过，不代表 GitHub 托管矩阵或另一套依赖 +组合已经通过。 + +## 5. 常见问题 + +| 现象 | 检查与处理 | +| --- | --- | +| `BRPC_ROOT is not FlatBuffers-enabled` 或缺少 FB 符号 | 用正确选项重编运行库和使用方，检查安装前缀里的 `include/butil/config.h`；不要强制宏或混用 ON 头文件与 OFF 库。 | +| 找不到 `flatbuffers/idl.h`、`libflatbuffers` | 生成器需要完整官方开发安装，单有运行时头文件不够；明确指定 include 和 library 路径。 | +| 头文件与 `flatc` 版本不匹配 | 检查 `flatbuffers/base.h` 和 `flatc --version`，统一版本，在新目录重新生成代码，不删除版本断言。 | +| Protobuf/Abseil 缺头文件或链接符号 | 统一 Protobuf，提供其 config 包和 Abseil 前缀，避免系统与私有安装混用。 | +| macOS 找不到 OpenSSL | 设置实际 `OPENSSL_ROOT_DIR` 和 `CMAKE_PREFIX_PATH`，保持编译器、SDK、架构一致。 | +| 无法写 `src/butil/config.h.tmp` | 只有 `config_brpc.sh`（Make）会往源码目录写这个文件，所以源码必须可写；用独立副本隔离。CMake 把 `config.h` 放在 build 目录。 | +| CMake 4 报旧依赖策略不兼容 | 对相关依赖使用适当的 policy compatibility 设置或升级依赖；不能借此降低 C++ 和依赖版本要求。 | +| `No tests were found` | 检查 example 的 build 目录、`BUILD_TESTING=ON` 和已构建的程序，或直接运行 `smoke.py`。 | +| `LINK_SO=ON` 找不到库或动态加载失败 | 先用 `BUILD_SHARED_LIBS=ON` 构建 `brpc-shared`，再检查共享库及传递依赖的运行时搜索路径。 | +| 连接拒绝或超时 | 等待 readiness，用当前端口，确保同一主机/容器且服务端未超过生命周期；不能计作 schema 拒绝成功。 | + +## 6. 消息构造和所有权 先用上游 `flatc --cpp` 生成 `*_generated.h`，再将 `brpc::flatbuffers::MessageBuilder` 传给生成的 `Create...` 函数。 @@ -86,7 +295,7 @@ schema 校验前复制到新存储，因此大小限制必须放在解析之前 `GetMutableRoot()`。合法消息中的可选 string/vector 仍可能为 null。 Framing 检查失败不会修改原 Message。 -## Service ID 和代码生成 +## 7. Service ID 和代码生成 `BrpcDescriptorTable` 保存 namespace、service 名称、按空白分隔的方法名和显式 方法 ID。ID 必须是唯一的非负 int32。手写 descriptor 可以传空 ID 列表，此时按 @@ -114,3 +323,28 @@ rpc_service BenchmarkService { 生成的 dispatch 会验证请求，拒绝未知、不属于当前 service 或尚未实现的方法。 失败时，它会执行非空 completion callback；成功分派后，completion 由业务实现负责， 并且必须恰好执行一次。 + +## 8. 网络 RPC + +库和使用方必须用相同的 `BRPC_WITH_FLATBUFFERS` 设置构建；启用该功能会改变 Channel、Controller、Server 的 ABI。用 `server.AddFlatBuffersService(&service, SERVER_DOESNT_OWN_SERVICE)` 注册生成的服务；将 `ChannelOptions::protocol` 设为 `"fb_rpc"`，并把这个 `brpc::Channel` 传给生成的 stub。stub 调用 `Channel::FBCallMethod`；`Controller::flatbuffers_method()` 返回带类型的 descriptor，protobuf 的 `Controller::method()` 仍为空。手工提供的 descriptor 必须活过整个调用，包括重试和 backup request。 + +服务注册独立于 protobuf 的 AddService 和 ListServices。Add/Remove/Clear 要求服务端处于已停止（READY）状态，不能是正在 STOPPING；管理操作必须由外部串行化。只有注册成功后所有权才转移。`RemoveFlatBuffersService` 会删除自有服务，Stop/Join 则保留注册以便重启。`GetFlatBuffersServiceCount()` 返回这个独立注册表的数量。重复的 service ID（包括 hash 冲突）以及与 protobuf 方法全名冲突的 FlatBuffers 方法会被拒绝。按字符串调用的 `MaxConcurrencyOf` 支持 FlatBuffers 方法，服务端整体并发和默认方法并发限制同样适用。 + +传输层支持同步/异步调用、重试、backup request、single/pooled/short 连接以及请求/响应附件。每次发送都会分配一个独立的小 header 并共享 payload 存储，不会改写 const 请求的 metadata 前缀，因此重试和并发调用可以安全共享同一个不可变请求。应用仍须保证每个 response 和 Controller 活到调用完成。 + +Framing 只检查长度，不检查应用 schema。生成的服务会在分派前验证请求，手写服务也必须这样做。调用方使用 root accessor 前必须验证收到的响应 schema，RPC 成功不能替代 `response.Verify()`。 + +不支持鉴权、压缩、校验和与 streaming，遇到时会明确拒绝而不是悄悄忽略。FlatBuffers 服务不能通过仅限内置服务的内部端口访问。本传输层不提供 SelectiveChannel/ParallelChannel、HTTP/JSON 映射和 RPC-dump 回放，也不传输 log ID、用户字段和分布式追踪元数据。 + +完整的客户端/服务端示例见[第 3 节](#3-通过-example-完成端到端验证)；示例演示这些 API，不做性能结论。 + +### FRPC 帧格式与兼容性 + +一帧由 `[12 字节 header][metadata][message][attachment]` 组成。header 包含 `FRPC`、大端 uint32 的 body 大小和大端 uint32 的 metadata 大小；body 大小不含 12 字节 header，但包含后面三段。metadata 前缀明确使用小端，与原先在小端机器上的 FRPC 实验一致，且不依赖 packed 结构体： + +* 请求：uint32 service ID、int32 method ID、int32 message 大小、int32 attachment 大小、uint64 correlation ID（共 24 字节）。 +* 响应：int32 错误码、int32 message 大小、int32 attachment 大小、uint64 correlation ID（共 20 字节）。 + +读取方要求完整的已知前缀，然后根据 header 中的 metadata 大小跳过未知的尾部。以后新增的可选字段只能追加，不要重排、改变大小或改变已有字段的含义。前缀过短、大小为负、payload 长度不一致或 body 过大都会被拒绝。错误响应带非零错误码且没有 payload。Method ID 是稀疏的 wire ID，不是数组下标：只有保留显式 ID 时，增加、删除或重排声明才不会改变已有方法的 ID。这并不意味着旧的按序号编号的 schema 能兼容，已删除的 ID 也绝不能复用。 + +magic 是 `FRPC`，不是 `BRPC`。该格式不保证与更早未发布、使用不同 magic、service hash 或本机大端 metadata 的版本兼容。库的全局 Protocol hook 签名保持不变，现有协议回调实现无需修改。自定义的数字协议 ID 不得与新分配的内置 ID 重叠，`PROTOCOL_FLATBUFFERS_RPC` 使用 ID 30。 diff --git a/docs/en/flatbuffers.md b/docs/en/flatbuffers.md index 1b8a0f48bd..2cd5ac88a3 100644 --- a/docs/en/flatbuffers.md +++ b/docs/en/flatbuffers.md @@ -8,47 +8,264 @@ descriptors, and the `fb_rpc` transport. The implementation builds on [apache/brpc#3197](https://github.com/apache/brpc/pull/3197), while preserving Protocol's existing protobuf callback signatures. -## Build +## Enable FlatBuffers + +FlatBuffers is **OFF by default**. Enable it when building bRPC, then build the +client/server against that same library and generated configuration header. +Adding `-DBRPC_WITH_FLATBUFFERS=1` to an application is not a substitute: the +feature changes the Channel, Controller and Server ABI. + +| Build system | Enable option | Exported runtime prefix | +| --- | --- | --- | +| CMake | `-DWITH_FLATBUFFERS=ON` | `cmake --install` prefix | +| Make | `config_brpc.sh --with-flatbuffers` | `/output` | +| Bazel | `--define=BRPC_WITH_FLATBUFFERS=true` on build/test commands | Bazel outputs, not an install prefix for the standalone example | + +### Dependencies and versions + +Prepare the normal bRPC dependencies described in [Getting started](getting_started.md): +a C++ toolchain, Protobuf compiler/development libraries, gflags, LevelDB, +OpenSSL and zlib. FlatBuffers RPC does **not** remove the Protobuf dependency. + +| Component | Additional requirement | +| --- | --- | +| bRPC message/RPC runtime | FlatBuffers headers; no `libflatbuffers` linkage | +| Official schema generation | `flatc` matching the runtime headers | +| bRPC service generation | `brpc_flatc`, built with matching official headers and `libflatbuffers` | +| Example smoke | Python 3; no GoogleTest requirement | +| Library unit tests | GoogleTest and the project's test dependencies | + +The repository's Bazel/ON gate pins FlatBuffers **25.2.10**. Use a complete, +matching installation to reproduce it; never remove the generated header's +version assertions. CMake runtime tests use `FLATBUFFERS_FLATC_EXECUTABLE`, +whereas generator acceptance and the example use `FLATC_EXECUTABLE`. +`BRPC_FLATC_EXECUTABLE` always means the bRPC generator, not official `flatc`. + +Use the same Protobuf installation across all builds. With `Protobuf_VERSION > +4.21`, C++17 and Abseil are required, and the generator and example builds need +Protobuf's CMake config package. CMake 3.16+ is enough to build; the CTest +commands below use 3.17+ for `--no-tests=error` (with 3.16, run `smoke.py` +directly). Use a single-configuration generator such as Unix Makefiles or Ninja. + +The runtime depends on FlatBuffers builder internals and supports only +25.2.10; other versions fail at compile time. Regenerate incompatible headers +rather than removing or weakening upstream version assertions. Bzlmod and +WORKSPACE use the same checksum-pinned archive; this avoids the gRPC, other +language tools and conflicting BoringSSL dependencies that `bazel_dep` would add. -The runtime needs only FlatBuffers headers; tests also need a matching `flatc`. -Regenerate incompatible headers rather than removing or weakening upstream -version assertions. +To use another FlatBuffers version, update the version checks in the runtime, +the generator and the codegen tests, then run `tools/flatbuffers/verify.sh` +against that version. See the generator +[README](../../tools/flatbuffers/README.md#using-another-flatbuffers-version) +for the steps, using 23.5.26 as an example. They apply to CMake builds only. + +### CMake runtime build + +Run from a checkout, with a working compiler selected through `CC`/`CXX` if +necessary. The CMake build keeps generated files in its build directory, and the +example bindings and binaries stay in `WORK`. Install the runtime into a prefix: +the build tree alone does not contain the public headers or `config.h` that the +standalone example needs. -With GoogleTest sources at `/usr/src/googletest`: +Replace the prefixes below with your installations (`include/`, `lib/` or +`lib64/`); one prefix may serve several dependencies. ```sh -cmake -S . -B build -DWITH_FLATBUFFERS=ON -DBUILD_UNIT_TESTS=ON \ - -DBUILD_BRPC_TOOLS=OFF -DDOWNLOAD_GTEST=OFF \ - -DBRPC_SYSTEM_GTEST_SOURCE_DIR=/usr/src/googletest -cmake --build build --target brpc_flatbuffers_unittest brpc_flatbuffers_protocol_unittest -j6 -ctest --test-dir build -R '^brpc_flatbuffers(_protocol)?_unittest$' --output-on-failure +REPO="$PWD" +DEPS=/absolute/path/to/dependency-prefix +FB=/absolute/path/to/flatbuffers-prefix +OPENSSL=/absolute/path/to/openssl-prefix +WORK="$(mktemp -d "${TMPDIR:-/tmp}/brpc-benchmark-fb.XXXXXX")" +JOBS=2 +printf 'WORK=%s\n' "$WORK" ``` -For nonstandard installations, set `FLATBUFFERS_INCLUDE_DIR`, -`FLATBUFFERS_FLATC_EXECUTABLE`, and `BRPC_SYSTEM_GTEST_SOURCE_DIR` as needed. -The usual bRPC test dependencies still apply. +If FlatBuffers is not installed, build a matching official source checkout first +(skip this block when `FB` already contains the required headers, compiler and +library). `FB` must be a writable installation prefix, not the source directory: -For Make, use `config_brpc.sh --with-flatbuffers` and set `FLATC=/path/to/flatc` -when testing. For Bazel, use `--define=BRPC_WITH_FLATBUFFERS=true`. +```sh +FB_SOURCE=/absolute/path/to/flatbuffers-25.2.10-source +cmake -S "$FB_SOURCE" -B "$WORK/flatbuffers" \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DCMAKE_INSTALL_PREFIX="$FB" -DCMAKE_INSTALL_LIBDIR=lib \ + -DFLATBUFFERS_BUILD_TESTS=OFF -DFLATBUFFERS_BUILD_FLATC=ON \ + -DFLATBUFFERS_BUILD_FLATLIB=ON -DFLATBUFFERS_BUILD_SHAREDLIB=OFF \ + -DFLATBUFFERS_INSTALL=ON -DFLATBUFFERS_LIBCXX_WITH_CLANG=OFF +cmake --build "$WORK/flatbuffers" --parallel "$JOBS" +cmake --install "$WORK/flatbuffers" +"$FB/bin/flatc" --version +``` -The runtime depends on FlatBuffers builder internals and supports only 25.2.10; -other versions fail at compile time. Bzlmod and WORKSPACE use the same -checksum-pinned archive. Using this instead of `bazel_dep` avoids gRPC, -other language tools, and conflicting BoringSSL dependencies. +Build the runtime without the unit-test dependencies: -Public headers are in `brpc/flatbuffers/`, with namespace `brpc::flatbuffers`. -Include `message.h` for message construction and `service.h` for service -descriptors and interfaces. `BRPC_WITH_FLATBUFFERS` in `butil/config.h` is -0 or 1; test it with `#if`. +```sh +cmake -S "$REPO" -B "$WORK/runtime" \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_PREFIX_PATH="$DEPS;$FB;$OPENSSL" \ + -DOPENSSL_ROOT_DIR="$OPENSSL" \ + -DCMAKE_INSTALL_PREFIX="$WORK/prefix" \ + -DWITH_FLATBUFFERS=ON -DBUILD_SHARED_LIBS=OFF \ + -DFLATBUFFERS_INCLUDE_DIR="$FB/include" \ + -DBUILD_UNIT_TESTS=OFF -DDOWNLOAD_GTEST=OFF -DBUILD_BRPC_TOOLS=OFF +cmake --build "$WORK/runtime" --parallel "$JOBS" +cmake --install "$WORK/runtime" +grep -E '^#define BRPC_WITH_FLATBUFFERS +1$' "$WORK/prefix/include/butil/config.h" +``` -Older flatc 2.0.x may emit unqualified names. Keep business schemas outside -the `brpc` namespace and do not rely on include order. +The final check must print `#define BRPC_WITH_FLATBUFFERS 1`. Public headers live +in `brpc/flatbuffers/`: `message.h` for messages/builders and `service.h` for +services/descriptors. The configuration macro is always 0 or 1; use `#if`, not +`#ifdef`. The policy argument above is an optional compatibility setting for +older dependencies under CMake 4, not a FlatBuffers switch or a replacement for +required dependency versions. -To use another FlatBuffers version, update the version checks in the runtime, -the generator and the codegen tests, then run `tools/flatbuffers/verify.sh` -against that version. See the generator -[README](../../tools/flatbuffers/README.md#using-another-flatbuffers-version) -for the steps, using 23.5.26 as an example. They apply to CMake builds only. +### Make and Bazel alternatives + +In a separate writable checkout, using the same dependency prefixes: + +```sh +sh config_brpc.sh --with-flatbuffers \ + --headers="$FB/include $DEPS/include $OPENSSL/include" \ + --libs="$DEPS/lib $OPENSSL/lib" --cc="${CC:-cc}" --cxx="${CXX:-c++}" +make -j"$JOBS" +``` + +Use `lib64` or the platform's library directory where appropriate. The resulting +`output/` can replace `WORK/prefix` in the example command below. Make +unit tests take `FLATC="$FB/bin/flatc"` and require installed GoogleTest libraries +and gperftools, not just GoogleTest sources; their `test/libbrpc.dbg.*` must also +be loadable. The ON runner below builds GoogleTest, sets the test library path, +and validates reports; system test dependencies such as gperftools must already +be installed. + +Bazel supplies its pinned dependencies; pass the feature flag to both commands: + +```sh +bazel build --define=BRPC_WITH_FLATBUFFERS=true //:brpc +bazel test --define=BRPC_WITH_FLATBUFFERS=true --cache_test_results=no \ + //test:brpc_flatbuffers_unittest //test:brpc_flatbuffers_protocol_unittest +``` + +These Bazel targets verify the library, not the standalone example. Do not use a +raw `bazel-bin` directory as `BRPC_ROOT`; the example requires the include/lib +layout of a CMake install prefix or a Make `output/` directory. + +## Verify with the client/server example + +[example/benchmark_fb](../../example/benchmark_fb/README.md) is a **bounded +functional example**, not a performance benchmark. Its [schema](../../example/benchmark_fb/echo.fbs) +uses `BenchmarkService.Echo` with explicit wire ID 7. The +[server](../../example/benchmark_fb/server.cpp) calls `AddFlatBuffersService`; +the [client](../../example/benchmark_fb/client.cpp) uses the generated +`BenchmarkService::Stub` and `fb_rpc` channel. + +### Generate, build and smoke-test + +Continue in the same shell with the variables and runtime from above. An +existing compatible FB ON runtime may be used instead by changing `BRPC_ROOT`. +Build the bRPC generator, then the example: + +```sh +cmake -S "$REPO/tools/flatbuffers" -B "$WORK/codegen" \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DBUILD_TESTING=OFF \ + -DCMAKE_PREFIX_PATH="$FB;$DEPS" \ + -DFLATBUFFERS_INCLUDE_DIR="$FB/include" +cmake --build "$WORK/codegen" --parallel "$JOBS" + +cmake -S "$REPO/example/benchmark_fb" -B "$WORK/example" \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DBUILD_TESTING=ON \ + -DCMAKE_PREFIX_PATH="$DEPS;$FB;$OPENSSL" \ + -DOPENSSL_ROOT_DIR="$OPENSSL" \ + -DBRPC_ROOT="$WORK/prefix" \ + -DFLATBUFFERS_INCLUDE_DIR="$FB/include" \ + -DFLATC_EXECUTABLE="$FB/bin/flatc" \ + -DBRPC_FLATC_EXECUTABLE="$WORK/codegen/brpc_flatc" +cmake --build "$WORK/example" --parallel "$JOBS" +(cd "$WORK/example" && ctest -V --no-tests=error --output-on-failure -R '^benchmark_fb_smoke$') +``` + +The example build runs **both** generators. `flatc` produces `echo_generated.h`; +`brpc_flatc` produces `echo.brpc.fb.h/.cpp`, all under `WORK/example/generated/`. +Do not commit them or reuse stale generated headers after changing versions. +If library discovery is ambiguous, pass `FLATBUFFERS_LIBRARY` explicitly when +configuring `brpc_flatc`. See the [generator guide](../../tools/flatbuffers/README.md) +for included schemas, explicit IDs and generator acceptance tests. + +Success means **one named CTest passed**, exit status 0, and this output: + +```text +benchmark_fb smoke passed: 13 verified replies, 2 schema rejections, clean shutdown +``` + +That is 15 RPCs, not 15 CTests. `No tests were found` is not a pass. What the +smoke test checks, its deadlines and how to run it without CTest are in the +[example README](../../example/benchmark_fb/README.md#smoke-test). + +The example links `libbrpc.a` by default. For `LINK_SO=ON`, build and install the +runtime with `BUILD_SHARED_LIBS=ON` and the `brpc-shared` target first; `LINK_SO` +alone cannot create a shared runtime. + +### Run the programs separately + +The [example README](../../example/benchmark_fb/README.md#run-separately) lists the +manual client commands, their expected results and the bounded client/server +options. + +## Additional tests and the ON gate + +For library unit tests, configure with `BUILD_UNIT_TESTS=ON`, +`FLATBUFFERS_FLATC_EXECUTABLE` pointing to the matching official compiler, and +`BRPC_SYSTEM_GTEST_SOURCE_DIR` pointing to GoogleTest sources when +`DOWNLOAD_GTEST=OFF`. Build and run `brpc_flatbuffers_unittest` and +`brpc_flatbuffers_protocol_unittest`; enabling the library alone does not run them. + +For the complete CMake path (both library suites, two codegen tests, and the +example smoke), the shared [ON runner](../../.github/scripts/flatbuffers-on.py) +can use the prefixes above. This full CMake gate requires **CMake/CTest 3.21+** +for JUnit XML output (`--output-junit`). **Its work directory must not already +exist**: + +```sh +python3 "$REPO/.github/scripts/flatbuffers-on.py" \ + --build-system cmake --source "$REPO" --work "$WORK/on-gate" --jobs "$JOBS" \ + --flatbuffers-prefix "$FB" \ + --dependency-prefix "$DEPS" --dependency-prefix "$OPENSSL" +``` + +This gate requires FlatBuffers 25.2.10. Omit `--flatbuffers-prefix` to let it +download/checksum/build that version; GoogleTest defaults to a checksum-pinned +1.14.0 download, or can be supplied with `--gtest-source`. Normal platform +dependencies must already be installed. Make/Bazel modes validate the two +library suites; only CMake mode also builds the generator and example. Inspect +`WORK/on-gate/evidence/summary.json` for `status: passed`, all required test names, +nonzero executed counts and zero failed/skipped cases. Commands, logs and XML +reports are retained even on failure. Empty/filtered/skipped runs are rejected. + +The [workflow](../../.github/workflows/flatbuffers-on.yml) runs Linux CMake/Make +with GCC and Clang, Linux Bazel with GCC, and macOS CMake. A local gate pass does +not imply the hosted GitHub matrix or a different dependency combination passed. + +## Troubleshooting + +| Symptom | Check / action | +| --- | --- | +| `BRPC_ROOT is not FlatBuffers-enabled` or missing FB symbols | Rebuild bRPC with the enable option, then rebuild all consumers. Check the installed prefix's `include/butil/config.h`; do not force the macro or mix an ON header with an OFF library. | +| Missing `flatbuffers/idl.h` or `libflatbuffers` | The service generator needs the full official development installation, not only the runtime headers. Set `FLATBUFFERS_INCLUDE_DIR` and, if needed, `FLATBUFFERS_LIBRARY`. | +| Header/compiler version mismatch | Check `flatc --version`, select matching headers/compiler/library, and regenerate bindings in a fresh build directory. Do not delete upstream version checks. | +| Missing Protobuf/Abseil headers or link symbols | Use one compatible Protobuf installation throughout and expose its CMake config and Abseil prefixes. Do not mix system and private headers/libraries. | +| OpenSSL not found on macOS | Set `OPENSSL_ROOT_DIR` and include the actual installed prefix in `CMAKE_PREFIX_PATH`; check compiler/SDK/architecture consistency. | +| Cannot write `src/butil/config.h.tmp` | Only `config_brpc.sh` (Make) writes this file into the checkout, so it needs a writable source tree. Use a private copy; CMake keeps `config.h` in its build directory. | +| Old dependency policy error with CMake 4 | Try the appropriate `CMAKE_POLICY_VERSION_MINIMUM` compatibility setting for that dependency, or update it; this does not change the required C++ or dependency versions. | +| `No tests were found` | Use the example build directory, configure `BUILD_TESTING=ON`, build the executables, then run the named test or `smoke.py` directly. | +| `LINK_SO=ON` cannot find a library or the loader fails | Build `brpc-shared` with runtime `BUILD_SHARED_LIBS=ON` first. Check the shared library's dependencies and runtime search paths. | +| Connection refused/timeout | Wait for the readiness line, use its current port on the same host/container, and check the server's finite lifetime. These are not successful schema rejections. | + +Flatc 2.0.x emits unqualified `flatbuffers::` names. Keep business schemas outside +the `brpc` namespace (for example `myapp.rpc`); do not depend on include order to +avoid shadowing. ## Message construction and ownership @@ -172,6 +389,10 @@ through the internal, builtin-only port. SelectiveChannel/ParallelChannel, HTTP/JSON mapping and RPC-dump replay are not provided by this transport. Log IDs, user fields and distributed-tracing metadata are not transmitted. +For a complete generated client/server, follow +[Verify with the client/server example](#verify-with-the-clientserver-example) +above. The example demonstrates these APIs without making a performance claim. + ### FRPC framing and compatibility A frame is `[12-byte header][metadata][message][attachment]`. The header contains diff --git a/tools/flatbuffers/README.md b/tools/flatbuffers/README.md index 3ed128eb23..23cda6bb58 100644 --- a/tools/flatbuffers/README.md +++ b/tools/flatbuffers/README.md @@ -1,5 +1,9 @@ # Standalone FlatBuffers service generator +For runtime enablement and a complete client/server check, see the +[English guide](../../docs/en/flatbuffers.md) or +[中文指南](../../docs/cn/flatbuffers.md). + `brpc_flatc` generates bRPC service bindings. The separate `fb_rpc` transport implements their channel interface in `brpc::Channel`. The generator uses the installed, official `flatbuffers/idl.h` Parser and `libflatbuffers`; it does not download or modify a @@ -20,6 +24,12 @@ flatc --cpp -o /tmp/brpc-generated test/flatbuffers_codegen/echo.fbs /tmp/brpc-codegen-build/brpc_flatc -o /tmp/brpc-generated test/flatbuffers_codegen/echo.fbs ``` +For a non-default installation, add `-DCMAKE_PREFIX_PATH=/path/to/flatbuffers` +and/or explicit `FLATBUFFERS_INCLUDE_DIR` and `FLATBUFFERS_LIBRARY` values. +Run official `flatc` from that matching installation, not an unrelated binary +found earlier on `PATH`. The runtime alone does not link `libflatbuffers`, but +this generator needs it. + The output directory must already exist. The command line is: ```text @@ -32,7 +42,8 @@ For `echo.fbs`, the two tools produce: - `brpc_flatc`: `echo.brpc.fb.h` and `echo.brpc.fb.cpp`, both Apache-licensed. Compile the generated `.cpp` into the application with C++14 or newer and bRPC -configured with FlatBuffers support. Keep the generated header, official table header, and bRPC +configured with FlatBuffers support. When the detected `Protobuf_VERSION` is +greater than 4.21, use C++17 and the matching Protobuf/Abseil dependency set. Keep the generated header, official table header, and bRPC headers on the include path. The generated service files do not pin a FlatBuffers version; use an official `flatc` matching your installed table headers, as required by official generated code. @@ -96,6 +107,10 @@ when possible; if rollback also fails, the generator attempts to remove both final outputs and reports that manual cleanup is needed when the filesystem refuses. The standalone generator currently requires POSIX file-locking semantics, including on Linux and macOS. +Service header guards preserve the exact filename and first exported service's +fully-qualified identity. They distinguish punctuation/case and same basenames +in different namespaces without depending on checkout or output paths. Schemas +included together must still define distinct fully-qualified C++ services. ## Descriptor and completion contracts @@ -205,12 +220,13 @@ Ordinary FlatBuffers-OFF jobs are not codegen evidence. No root build file needs modification. The default standalone build enables CTest and additionally needs matching official `flatc`, protobuf, and gflags -development headers: +development headers. The commands below use CMake/CTest 3.17+ for +`--no-tests=error` (the standalone generator itself has a CMake 3.10 minimum): ```sh cmake -S tools/flatbuffers -B /tmp/brpc-codegen-build -DBUILD_TESTING=ON cmake --build /tmp/brpc-codegen-build -j2 -ctest --test-dir /tmp/brpc-codegen-build --output-on-failure +(cd /tmp/brpc-codegen-build && ctest --no-tests=error --output-on-failure) ``` The build compiles the generated service against the real bRPC headers. CTest @@ -226,19 +242,34 @@ source tree. The name-shadowing cases are compile-only by default; pass `RUNTIME_LIBRARIES` when invoking `acceptance.cmake` directly to exercise their runtime checks as well. +Expect **one** CTest named `flatbuffers_codegen_acceptance` in this mode. Its +inner checks also compile colliding-basename headers alone and together in both +orders, and compare regenerated bytes after relocating the schema. An empty +CTest selection is not acceptance. + To enable additional behavior tests against an **already built** -FlatBuffers-enabled bRPC library: +FlatBuffers-enabled bRPC library, use matching runtime headers/dependencies and +an existing library. A shared library avoids having to reconstruct all static +platform link dependencies. Build it with root `BUILD_SHARED_LIBS=ON` and target +`brpc-shared` first; use `.dylib` on macOS instead of `.so`: ```sh +BRPC_LIBRARY=/absolute/path/to/FB-on/output/lib/libbrpc.so cmake -S tools/flatbuffers -B /tmp/brpc-codegen-build \ -DBUILD_TESTING=ON \ - -DBRPC_CODEGEN_BRPC_LIBRARY=/path/to/libbrpc.a + -DBRPC_CODEGEN_BRPC_LIBRARY="$BRPC_LIBRARY" cmake --build /tmp/brpc-codegen-build -j2 -ctest --test-dir /tmp/brpc-codegen-build --output-on-failure +(cd /tmp/brpc-codegen-build && ctest --no-tests=error --output-on-failure) ``` -Use `BRPC_CODEGEN_EXTRA_LIBRARIES` for additional dependencies of a custom bRPC -build. The runtime test covers concurrent first descriptor access, sparse +Expect **two** CTests in this mode: `flatbuffers_codegen_acceptance` and +`flatbuffers_codegen_runtime`. For non-default dependency installations, pass +their prefixes through `CMAKE_PREFIX_PATH`; modern Protobuf needs its CMake +config package and Abseil. A static `libbrpc.a` is also accepted, but the caller +must supply its complete transitive/platform link requirements (including macOS +frameworks where needed) via `BRPC_CODEGEN_EXTRA_LIBRARIES`. + +The runtime test covers concurrent first descriptor access, sparse stub/dispatch mapping, absent and present strings, invalid/foreign/forged methods, failed request verification, null arguments, unimplemented methods, exactly-once error callbacks (including self-deleting closures), deferred From 358a58a0e190a12e40eb1ca4fba9e6c6f1358fa6 Mon Sep 17 00:00:00 2001 From: xulei25 Date: Wed, 28 Oct 2026 11:58:32 +0800 Subject: [PATCH 5/6] Add a timed fb_rpc vs baidu_std comparison to the example The example client and server could only run a bounded loopback smoke test. Add an opt-in timed mode so the same binaries can compare fb_rpc with an equivalent protobuf service over baidu_std: - echo_pb.proto mirrors echo.fbs, and the server serves both protocols on one port. - --duration_s and --warmup_s run closed-loop callers and print QPS, latency percentiles and client CPU as one JSON line. - --protocol selects fb_rpc or baidu_std, --use_bthread runs callers as bthreads, and --allow_remote accepts non-loopback endpoints. - Request and reply builders reserve the payload size, so large messages are not grown by repeated reallocate-and-copy. Defaults are unchanged and the smoke test still passes. The guides now describe the example as a functional check with an optional timed mode. --- docs/cn/flatbuffers.md | 2 +- docs/en/flatbuffers.md | 5 +- example/benchmark_fb/CMakeLists.txt | 20 +- example/benchmark_fb/README.md | 54 +++- example/benchmark_fb/client.cpp | 397 ++++++++++++++++++++++------ example/benchmark_fb/echo_pb.proto | 39 +++ example/benchmark_fb/server.cpp | 60 ++++- 7 files changed, 469 insertions(+), 108 deletions(-) create mode 100644 example/benchmark_fb/echo_pb.proto diff --git a/docs/cn/flatbuffers.md b/docs/cn/flatbuffers.md index 5cba140608..fb2597b71b 100644 --- a/docs/cn/flatbuffers.md +++ b/docs/cn/flatbuffers.md @@ -112,7 +112,7 @@ grep -E '^#define BRPC_WITH_FLATBUFFERS +1$' "$WORK/prefix/include/butil/config. ## 3. 通过 example 完成端到端验证 -该示例是**有界功能验证，不是性能对比**。[echo.fbs](../../example/benchmark_fb/echo.fbs) +该示例默认做**有界功能验证**，另有可选的计时模式，用于对比 `fb_rpc` 和等价的 `baidu_std` 服务。[echo.fbs](../../example/benchmark_fb/echo.fbs) 声明 `BenchmarkService.Echo`，显式 wire method ID 为 7。 [server.cpp](../../example/benchmark_fb/server.cpp) 使用 `AddFlatBuffersService` 注册服务； [client.cpp](../../example/benchmark_fb/client.cpp) 使用生成的 `BenchmarkService::Stub` diff --git a/docs/en/flatbuffers.md b/docs/en/flatbuffers.md index 2cd5ac88a3..cc767fa721 100644 --- a/docs/en/flatbuffers.md +++ b/docs/en/flatbuffers.md @@ -153,8 +153,9 @@ layout of a CMake install prefix or a Make `output/` directory. ## Verify with the client/server example -[example/benchmark_fb](../../example/benchmark_fb/README.md) is a **bounded -functional example**, not a performance benchmark. Its [schema](../../example/benchmark_fb/echo.fbs) +[example/benchmark_fb](../../example/benchmark_fb/README.md) runs a **bounded +functional check** by default and has an opt-in timed mode that compares `fb_rpc` +with an equivalent `baidu_std` service. Its [schema](../../example/benchmark_fb/echo.fbs) uses `BenchmarkService.Echo` with explicit wire ID 7. The [server](../../example/benchmark_fb/server.cpp) calls `AddFlatBuffersService`; the [client](../../example/benchmark_fb/client.cpp) uses the generated diff --git a/example/benchmark_fb/CMakeLists.txt b/example/benchmark_fb/CMakeLists.txt index 8eb080c878..780b108583 100644 --- a/example/benchmark_fb/CMakeLists.txt +++ b/example/benchmark_fb/CMakeLists.txt @@ -143,7 +143,25 @@ add_custom_command(OUTPUT ${_generated_files} DEPENDS "${_schema}" "${FLATC_EXECUTABLE}" "${BRPC_FLATC_EXECUTABLE}" VERBATIM) add_custom_target(benchmark_fb_codegen DEPENDS ${_generated_files}) -add_library(benchmark_fb_schema STATIC "${_generated}/echo.brpc.fb.cpp") +# Protobuf control schema used for the baidu_std comparison. +if(TARGET protobuf::protoc) + set(_protoc "$") +elseif(Protobuf_PROTOC_EXECUTABLE) + set(_protoc "${Protobuf_PROTOC_EXECUTABLE}") +else() + message(FATAL_ERROR "Set Protobuf_PROTOC_EXECUTABLE to the protoc matching libprotobuf") +endif() +add_custom_command(OUTPUT "${_generated}/echo_pb.pb.h" "${_generated}/echo_pb.pb.cc" + COMMAND "${CMAKE_COMMAND}" -E make_directory "${_generated}" + COMMAND "${_protoc}" "--cpp_out=${_generated}" -I "${CMAKE_CURRENT_SOURCE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}/echo_pb.proto" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/echo_pb.proto" + VERBATIM) +add_custom_target(benchmark_fb_pb_codegen DEPENDS + "${_generated}/echo_pb.pb.h" "${_generated}/echo_pb.pb.cc") +add_dependencies(benchmark_fb_codegen benchmark_fb_pb_codegen) +add_library(benchmark_fb_schema STATIC "${_generated}/echo.brpc.fb.cpp" + "${_generated}/echo_pb.pb.cc") add_dependencies(benchmark_fb_schema benchmark_fb_codegen) target_include_directories(benchmark_fb_schema PUBLIC "${_generated}") target_link_libraries(benchmark_fb_schema PUBLIC benchmark_fb_dependencies) diff --git a/example/benchmark_fb/README.md b/example/benchmark_fb/README.md index 3f63182c84..a3713ecbc3 100644 --- a/example/benchmark_fb/README.md +++ b/example/benchmark_fb/README.md @@ -4,13 +4,16 @@ [中文指南](../../docs/cn/flatbuffers.md) Independent client and server programs using the `fb_rpc` protocol, a generated -FlatBuffers service, and `brpc::flatbuffers::Message`. This is a **bounded -functional example**, not a performance comparison; the directory name is kept -for continuity. +FlatBuffers service, and `brpc::flatbuffers::Message`. By default the client +runs a **bounded functional check**. A timed mode compares `fb_rpc` with an +equivalent protobuf service over `baidu_std` (see +[Timed comparison](#timed-comparison)). The schema declares `BenchmarkService.Echo` with stable wire method ID **7**. The generated stub calls `Channel::FBCallMethod`, and the server registers the -generated service with `Server::AddFlatBuffersService`. +generated service with `Server::AddFlatBuffersService`. The same server also +registers `benchmark_fb_pb.BenchmarkService` from `echo_pb.proto`, a protobuf +service with the same fields, and serves both `fb_rpc` and `baidu_std`. ## Build @@ -29,8 +32,8 @@ example with `-DLINK_SO=ON`. Extra runtime dependencies can be passed with ## Run separately -The server and client must run on the **same host**; both accept only IPv4 -loopback endpoints. In the build terminal: +By default the server and client accept only IPv4 loopback endpoints. In the +build terminal: ```sh "$WORK/example/benchmark_fb_server" --listen_addr=127.0.0.1:0 --duration_s=300 @@ -70,22 +73,49 @@ schema rejection (error code -1); timeouts and connection errors are failures. | Client option | Default | Allowed | | --- | --- | --- | -| `server` | required | `127.0.0.1:` | +| `server` | required | `127.0.0.1:`, any IPv4 with `allow_remote` | +| `protocol` | `fb_rpc` | `fb_rpc`, `baidu_std` (protobuf control service) | | `request_count` | 16 | 1..1000 total, not per thread | -| `thread_num` | 2 | 1..16 synchronous callers | +| `thread_num` | 2 | 1..16 synchronous callers (1..1024 when timed) | | `request_size` | 64 | 0..1048576 string bytes | | `attachment_size` | 0 | 0..1048576 bytes | | `timeout_ms` | 1500 | 1..5000 per RPC | | `deadline_ms` | 30000 | 1..120000 for the client run | | `connection_type` | `single` | `single`, `pooled`, `short` | | `omit_message` | false | Preserve absent rather than empty string | -| `corrupt_request` | false | Require schema rejection | - -The server defaults to `127.0.0.1:0`, `duration_s=60` (1..3600) and -`max_concurrency=32` (1..64), and stops earlier on SIGINT/SIGTERM. Builtin +| `corrupt_request` | false | Require schema rejection (`fb_rpc`, count mode only) | +| `duration_s` | 0 | 0 for count mode, 1..3600 for timed mode | +| `warmup_s` | 0 | 0..600 unmeasured seconds before a timed run | +| `allow_remote` | false | Accept a non-loopback `server` | +| `use_bthread` | false | Run callers as bthreads, like `multi_threaded_echo_c++ --use_bthread` | + +The server defaults to `127.0.0.1:0`, `duration_s=60` (1..86400) and +`max_concurrency=32` (1..4096), and stops earlier on SIGINT/SIGTERM. Use +`--allow_remote` with a non-loopback `listen_addr`. Builtin services are disabled. FRPC has no authentication, compression, checksums or streaming, so this is not a production or public-network configuration. +## Timed comparison + +With `--duration_s` the client keeps one synchronous RPC in flight per caller +thread, verifies every reply, and prints one JSON line with QPS, payload +throughput, latency percentiles and the client's CPU cores. Run each protocol +with identical sizes and concurrency against the same server: + +```sh +"$WORK/example/benchmark_fb_server" --listen_addr=SERVER_IP:8901 --allow_remote \ + --duration_s=3600 --max_concurrency=4096 +for p in fb_rpc baidu_std; do + "$WORK/example/benchmark_fb_client" --server=SERVER_IP:8901 --allow_remote \ + --protocol=$p --duration_s=30 --warmup_s=3 --thread_num=64 --request_size=8192 +done +``` + +Both services echo the request string and attachment; the FlatBuffers path +builds and verifies a FlatBuffers response, while the protobuf path parses and +serializes the equivalent message. The results are only comparable when +binaries, hosts, concurrency, sizes and duration are the same. + ## Smoke test CTest runs the standard-library-only `smoke.py`, which starts its own server on diff --git a/example/benchmark_fb/client.cpp b/example/benchmark_fb/client.cpp index 4900d366c9..d28d2268b3 100644 --- a/example/benchmark_fb/client.cpp +++ b/example/benchmark_fb/client.cpp @@ -16,13 +16,17 @@ // under the License. #include +#include #include #include #include #include #include #include +#include +#include #include +#include #include #include #include @@ -31,26 +35,40 @@ #include "butil/endpoint.h" #include "brpc/channel.h" #include "brpc/controller.h" +#include "bthread/bthread.h" #include "echo.brpc.fb.h" +#include "echo_pb.pb.h" #if !BRPC_WITH_FLATBUFFERS #error "benchmark_fb requires a FlatBuffers-enabled brpc library" #endif -DEFINE_string(server, "", "Required IPv4 loopback endpoint printed by benchmark_fb_server"); +DEFINE_string(server, "", "IPv4 endpoint printed by benchmark_fb_server; " + "a non-loopback address also needs --allow_remote"); +DEFINE_bool(allow_remote, false, "Allow a non-loopback --server for cross-host benchmarks"); +DEFINE_string(protocol, "fb_rpc", "fb_rpc, or baidu_std for the protobuf control service"); DEFINE_string(connection_type, "single", "single, pooled, or short"); -DEFINE_int32(request_count, 16, "Total requests across all threads, 1..1000"); -DEFINE_int32(thread_num, 2, "Number of concurrent synchronous callers, 1..16"); +DEFINE_int32(request_count, 16, "Count mode: total requests across all threads, 1..1000"); +DEFINE_int32(duration_s, 0, "0 selects count mode; 1..3600 runs a timed benchmark"); +DEFINE_int32(warmup_s, 0, "Timed mode: unmeasured warm-up seconds, 0..600"); +DEFINE_int32(thread_num, 2, "Concurrent synchronous callers: 1..16, or 1..1024 when timed"); DEFINE_int32(request_size, 64, "Bytes in the optional request string, 0..1048576"); DEFINE_int32(attachment_size, 0, "Bytes in the echoed attachment, 0..1048576"); DEFINE_int32(timeout_ms, 1500, "Per-RPC timeout, 1..5000 milliseconds"); -DEFINE_int32(deadline_ms, 30000, "Overall client deadline, 1..120000 milliseconds"); +DEFINE_int32(deadline_ms, 30000, "Count mode: overall client deadline, 1..120000 milliseconds"); DEFINE_bool(omit_message, false, "Omit the optional string rather than sending an empty string"); -DEFINE_bool(corrupt_request, false, "Send an invalid root offset and require schema rejection"); +DEFINE_bool(use_bthread, false, "Run callers as bthreads instead of pthreads"); +DEFINE_bool(corrupt_request, false, "fb_rpc only: send an invalid root offset and " + "require schema rejection"); namespace { using brpc::flatbuffers::Message; using brpc::flatbuffers::MessageBuilder; +using Clock = std::chrono::steady_clock; + +// Result of one RPC: kSuccess, kRejected (expected schema rejection) or kFailed. +enum CallResult { kSuccess, kRejected, kFailed }; +using CallFn = std::function; std::string MakeBytes(size_t size, unsigned multiplier) { std::string bytes(size, '\0'); @@ -61,9 +79,10 @@ std::string MakeBytes(size_t size, unsigned multiplier) { } Message MakeRequest(uint64_t request_id, const std::string& text) { - MessageBuilder builder; + MessageBuilder builder(text.size() + 256); const auto message = FLAGS_omit_message ? - ::flatbuffers::Offset<::flatbuffers::String>() : builder.CreateString(text); + ::flatbuffers::Offset<::flatbuffers::String>() : + builder.CreateString(text.data(), text.size()); builder.Finish(benchmark_fb::CreateRequest(builder, request_id, message)); Message request = builder.ReleaseMessage(); if (FLAGS_corrupt_request) { @@ -89,116 +108,331 @@ bool CheckResponse(const Message& response, uint64_t request_id, if (FLAGS_omit_message) { return output->message() == nullptr; } - return output->message() != nullptr && output->message()->str() == text; + return output->message() != nullptr && output->message()->size() == text.size() && + memcmp(output->message()->data(), text.data(), text.size()) == 0; +} + +bool CheckPbResponse(const benchmark_fb_pb::Response& response, uint64_t request_id, + const std::string& text, const std::string& attachment, + const brpc::Controller& cntl) { + if (response.request_id() != request_id || + response.attachment_size() != attachment.size() || + cntl.response_attachment().to_string() != attachment) { + return false; + } + if (FLAGS_omit_message) { + return !response.has_message(); + } + return response.has_message() && response.message() == text; +} + +bool IsLoopback(const std::string& server) { + return server.compare(0, 10, "127.0.0.1:") == 0; } bool ValidFlags(butil::EndPoint* endpoint) { - return FLAGS_server.compare(0, 10, "127.0.0.1:") == 0 && + const bool timed = FLAGS_duration_s > 0; + return (IsLoopback(FLAGS_server) || FLAGS_allow_remote) && butil::str2endpoint(FLAGS_server.c_str(), endpoint) == 0 && endpoint->port > 0 && + (FLAGS_protocol == "fb_rpc" || FLAGS_protocol == "baidu_std") && + !(FLAGS_corrupt_request && (FLAGS_protocol != "fb_rpc" || timed)) && (FLAGS_connection_type == "single" || FLAGS_connection_type == "pooled" || FLAGS_connection_type == "short") && FLAGS_request_count >= 1 && FLAGS_request_count <= 1000 && - FLAGS_thread_num >= 1 && FLAGS_thread_num <= 16 && + FLAGS_duration_s >= 0 && FLAGS_duration_s <= 3600 && + FLAGS_warmup_s >= 0 && FLAGS_warmup_s <= 600 && (timed || FLAGS_warmup_s == 0) && + FLAGS_thread_num >= 1 && FLAGS_thread_num <= (timed ? 1024 : 16) && FLAGS_request_size >= 0 && FLAGS_request_size <= 1024 * 1024 && FLAGS_attachment_size >= 0 && FLAGS_attachment_size <= 1024 * 1024 && FLAGS_timeout_ms >= 1 && FLAGS_timeout_ms <= 5000 && FLAGS_deadline_ms >= 1 && FLAGS_deadline_ms <= 120000; } -int RunClient(const butil::EndPoint& endpoint) { - brpc::Channel channel; - brpc::ChannelOptions options; - options.protocol = "fb_rpc"; - options.connection_type = FLAGS_connection_type; - options.timeout_ms = FLAGS_timeout_ms; - options.connect_timeout_ms = std::min(FLAGS_timeout_ms, 1000); - options.max_retry = 0; - if (channel.Init(endpoint, &options) != 0) { - std::cerr << "Cannot initialize FlatBuffers channel\n"; - return 1; - } - // Generated stub methods route through Channel::FBCallMethod. - benchmark_fb::BenchmarkService::Stub stub(&channel); - const std::string text = MakeBytes(FLAGS_omit_message ? 0 : FLAGS_request_size, 31); - const std::string attachment = MakeBytes(FLAGS_attachment_size, 17); - const auto deadline = std::chrono::steady_clock::now() + - std::chrono::milliseconds(FLAGS_deadline_ms); - std::atomic next{0}; - std::atomic successes{0}; - std::atomic expected_rejections{0}; +struct Shared { std::atomic failed{false}; std::mutex error_mutex; - auto fail = [&](const std::string& message) { + void Fail(const std::string& message) { if (!failed.exchange(true)) { std::lock_guard lock(error_mutex); std::cerr << message << '\n'; } - }; - auto worker = [&] { - try { - while (!failed.load()) { - const int index = next.fetch_add(1); - if (index >= FLAGS_request_count) { - break; - } - const int64_t remaining = std::chrono::duration_cast< - std::chrono::milliseconds>(deadline - - std::chrono::steady_clock::now()).count(); - if (remaining <= 0) { - fail("Overall client deadline exceeded"); - break; - } - const uint64_t request_id = static_cast(index) + 1; - const Message request = MakeRequest(request_id, text); - Message response; - brpc::Controller cntl; - cntl.set_timeout_ms(std::min(FLAGS_timeout_ms, remaining)); - cntl.request_attachment().append(attachment); - stub.Echo(&cntl, &request, &response, nullptr); - if (FLAGS_corrupt_request) { - // The generated generic RpcController::SetFailed(string) - // maps to -1. Timeouts and connection errors do not count. - if (!cntl.Failed() || cntl.ErrorCode() != -1 || - response.size() != 0 || !cntl.response_attachment().empty()) { - fail("Expected server schema rejection; error=" + cntl.ErrorText()); - break; - } - ++expected_rejections; - } else if (cntl.Failed()) { - fail("RPC failed: " + cntl.ErrorText()); - break; - } else if (!CheckResponse(response, request_id, text, attachment, cntl)) { - fail("Response schema, fields, optional string or attachment mismatch"); - break; - } else { - ++successes; - } + } +}; + +// Builds one synchronous RPC for the selected protocol. Both protocols carry +// the same request string and attachment and verify every echoed byte. +CallFn MakeCall(brpc::Channel* channel, const std::string& text, + const std::string& attachment) { + if (FLAGS_protocol == "baidu_std") { + auto stub = std::make_shared(channel); + return [stub, &text, &attachment](uint64_t request_id, int64_t timeout_ms, + std::string* error) { + benchmark_fb_pb::Request request; + request.set_request_id(request_id); + if (!FLAGS_omit_message) { + request.set_message(text); + } + benchmark_fb_pb::Response response; + brpc::Controller cntl; + cntl.set_timeout_ms(timeout_ms); + cntl.request_attachment().append(attachment); + stub->Echo(&cntl, &request, &response, nullptr); + if (cntl.Failed()) { + *error = "RPC failed: " + cntl.ErrorText(); + return kFailed; + } + if (!CheckPbResponse(response, request_id, text, attachment, cntl)) { + *error = "Response fields, optional string or attachment mismatch"; + return kFailed; } - } catch (const std::exception& error) { - fail(std::string("Client worker failed: ") + error.what()); + return kSuccess; + }; + } + // Generated stub methods route through Channel::FBCallMethod. + auto stub = std::make_shared(channel); + return [stub, &text, &attachment](uint64_t request_id, int64_t timeout_ms, + std::string* error) { + const Message request = MakeRequest(request_id, text); + Message response; + brpc::Controller cntl; + cntl.set_timeout_ms(timeout_ms); + cntl.request_attachment().append(attachment); + stub->Echo(&cntl, &request, &response, nullptr); + if (FLAGS_corrupt_request) { + // The generated generic RpcController::SetFailed(string) + // maps to -1. Timeouts and connection errors do not count. + if (!cntl.Failed() || cntl.ErrorCode() != -1 || + response.size() != 0 || !cntl.response_attachment().empty()) { + *error = "Expected server schema rejection; error=" + cntl.ErrorText(); + return kFailed; + } + return kRejected; } + if (cntl.Failed()) { + *error = "RPC failed: " + cntl.ErrorText(); + return kFailed; + } + if (!CheckResponse(response, request_id, text, attachment, cntl)) { + *error = "Response schema, fields, optional string or attachment mismatch"; + return kFailed; + } + return kSuccess; }; +} + +struct BthreadArg { + Shared* shared; + const std::function* body; + int index; +}; + +void* BthreadEntry(void* raw) { + std::unique_ptr arg(static_cast(raw)); + try { + (*arg->body)(arg->index); + } catch (const std::exception& error) { + arg->shared->Fail(std::string("Client worker failed: ") + error.what()); + } + return nullptr; +} + +void RunWorkers(Shared* shared, const std::function& body) { + if (FLAGS_use_bthread) { + std::vector ids(FLAGS_thread_num, INVALID_BTHREAD); + for (int i = 0; i < FLAGS_thread_num; ++i) { + auto* arg = new BthreadArg{shared, &body, i}; + if (bthread_start_background(&ids[i], nullptr, BthreadEntry, arg) != 0) { + delete arg; + ids[i] = INVALID_BTHREAD; + shared->Fail("Cannot start bthread worker"); + } + } + for (bthread_t id : ids) { + if (id != INVALID_BTHREAD) { + bthread_join(id, nullptr); + } + } + return; + } std::vector workers; workers.reserve(FLAGS_thread_num); try { for (int i = 0; i < FLAGS_thread_num; ++i) { - workers.emplace_back(worker); + workers.emplace_back([shared, &body, i] { + try { + body(i); + } catch (const std::exception& error) { + shared->Fail(std::string("Client worker failed: ") + error.what()); + } + }); } } catch (const std::exception& error) { - fail(std::string("Cannot start client worker: ") + error.what()); + shared->Fail(std::string("Cannot start client worker: ") + error.what()); } for (auto& thread : workers) { thread.join(); } +} + +// Bounded functional mode used by smoke.py. +int RunCount(const CallFn& call) { + const auto deadline = Clock::now() + std::chrono::milliseconds(FLAGS_deadline_ms); + std::atomic next{0}; + std::atomic successes{0}; + std::atomic expected_rejections{0}; + Shared shared; + RunWorkers(&shared, [&](int) { + while (!shared.failed.load()) { + const int index = next.fetch_add(1); + if (index >= FLAGS_request_count) { + break; + } + const int64_t remaining = std::chrono::duration_cast< + std::chrono::milliseconds>(deadline - Clock::now()).count(); + if (remaining <= 0) { + shared.Fail("Overall client deadline exceeded"); + break; + } + std::string error; + const CallResult result = call(static_cast(index) + 1, + std::min(FLAGS_timeout_ms, remaining), &error); + if (result == kFailed) { + shared.Fail(error); + break; + } + ++(result == kRejected ? expected_rejections : successes); + } + }); const int completed = successes.load() + expected_rejections.load(); if (completed != FLAGS_request_count) { - fail("Not all requested RPCs completed"); + shared.Fail("Not all requested RPCs completed"); } std::cout << "{\"completed\":" << completed << ",\"successes\":" << successes.load() << ",\"expected_rejections\":" << expected_rejections.load() - << ",\"failures\":" << (failed.load() ? 1 : 0) << "}\n"; - return failed.load() ? 1 : 0; + << ",\"failures\":" << (shared.failed.load() ? 1 : 0) << "}\n"; + return shared.failed.load() ? 1 : 0; +} + +double CpuSeconds() { + rusage usage; + getrusage(RUSAGE_SELF, &usage); + return usage.ru_utime.tv_sec + usage.ru_stime.tv_sec + + (usage.ru_utime.tv_usec + usage.ru_stime.tv_usec) / 1e6; +} + +// Closed-loop benchmark: every caller keeps one RPC in flight for +// warmup_s + duration_s and records per-call latency in the measured window. +int RunTimed(const CallFn& call) { + Shared shared; + std::vector> latencies(FLAGS_thread_num); + std::atomic next_id{1}; + std::atomic measuring{false}; + std::atomic stop{false}; + std::atomic failures{0}; + std::thread timer([&] { + std::this_thread::sleep_for(std::chrono::seconds(FLAGS_warmup_s)); + measuring = true; + std::this_thread::sleep_for(std::chrono::seconds(FLAGS_duration_s)); + stop = true; + }); + // Wait for the warm-up boundary on this thread to sample CPU and time. + auto measure_begin = Clock::now(); + double cpu_begin = 0; + std::thread sampler([&] { + while (!measuring.load() && !stop.load()) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + measure_begin = Clock::now(); + cpu_begin = CpuSeconds(); + }); + RunWorkers(&shared, [&](int index) { + std::vector& samples = latencies[index]; + samples.reserve(1 << 16); + while (!stop.load(std::memory_order_relaxed)) { + const bool measured = measuring.load(std::memory_order_relaxed); + const auto begin = Clock::now(); + std::string error; + const CallResult result = call(next_id.fetch_add(1, std::memory_order_relaxed), + FLAGS_timeout_ms, &error); + const auto end = Clock::now(); + if (result != kSuccess) { + if (failures.fetch_add(1) == 0) { + shared.Fail(error); + } + continue; + } + if (measured && !stop.load(std::memory_order_relaxed)) { + samples.push_back(static_cast(std::min( + std::chrono::duration_cast( + end - begin).count(), UINT32_MAX))); + } + } + }); + const auto measure_end = Clock::now(); + const double cpu_end = CpuSeconds(); + timer.join(); + sampler.join(); + std::vector all; + for (const auto& samples : latencies) { + all.insert(all.end(), samples.begin(), samples.end()); + } + std::sort(all.begin(), all.end()); + const double seconds = std::chrono::duration(measure_end - measure_begin).count(); + auto percentile = [&all](double p) -> uint32_t { + if (all.empty()) { + return 0; + } + const size_t rank = static_cast(p * (all.size() - 1) + 0.5); + return all[std::min(rank, all.size() - 1)]; + }; + double sum = 0; + for (uint32_t value : all) { + sum += value; + } + const double qps = seconds > 0 ? all.size() / seconds : 0; + const double payload = static_cast(FLAGS_omit_message ? 0 : FLAGS_request_size) + + FLAGS_attachment_size; + std::cout << std::fixed << std::setprecision(1) + << "{\"mode\":\"timed\",\"protocol\":\"" << FLAGS_protocol + << "\",\"connection_type\":\"" << FLAGS_connection_type + << "\",\"use_bthread\":" << (FLAGS_use_bthread ? "true" : "false") + << ",\"thread_num\":" << FLAGS_thread_num + << ",\"request_size\":" << FLAGS_request_size + << ",\"attachment_size\":" << FLAGS_attachment_size + << ",\"seconds\":" << seconds + << ",\"requests\":" << all.size() + << ",\"failures\":" << failures.load() + << ",\"qps\":" << qps + << ",\"payload_mib_per_s\":" << qps * payload / (1024.0 * 1024.0) + << ",\"avg_us\":" << (all.empty() ? 0 : sum / all.size()) + << ",\"p50_us\":" << percentile(0.50) + << ",\"p90_us\":" << percentile(0.90) + << ",\"p99_us\":" << percentile(0.99) + << ",\"p999_us\":" << percentile(0.999) + << ",\"max_us\":" << (all.empty() ? 0 : all.back()) + << ",\"client_cpu_cores\":" << std::setprecision(3) + << (seconds > 0 ? (cpu_end - cpu_begin) / seconds : 0) + << "}\n"; + return failures.load() == 0 && !all.empty() ? 0 : 1; +} + +int RunClient(const butil::EndPoint& endpoint) { + brpc::Channel channel; + brpc::ChannelOptions options; + options.protocol = FLAGS_protocol; + options.connection_type = FLAGS_connection_type; + options.timeout_ms = FLAGS_timeout_ms; + options.connect_timeout_ms = std::min(FLAGS_timeout_ms, 1000); + options.max_retry = 0; + if (channel.Init(endpoint, &options) != 0) { + std::cerr << "Cannot initialize " << FLAGS_protocol << " channel\n"; + return 1; + } + const std::string text = MakeBytes(FLAGS_omit_message ? 0 : FLAGS_request_size, 31); + const std::string attachment = MakeBytes(FLAGS_attachment_size, 17); + const CallFn call = MakeCall(&channel, text, attachment); + return FLAGS_duration_s > 0 ? RunTimed(call) : RunCount(call); } } // namespace @@ -206,8 +440,9 @@ int main(int argc, char* argv[]) { GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); butil::EndPoint endpoint; if (!ValidFlags(&endpoint)) { - std::cerr << "Invalid flags; --server=127.0.0.1:PORT is required. " - "See --help and README.md for bounded parameter ranges.\n"; + std::cerr << "Invalid flags; --server=127.0.0.1:PORT is required unless " + "--allow_remote is set. See --help and README.md for bounded " + "parameter ranges.\n"; return 2; } try { diff --git a/example/benchmark_fb/echo_pb.proto b/example/benchmark_fb/echo_pb.proto new file mode 100644 index 0000000000..bded8516a2 --- /dev/null +++ b/example/benchmark_fb/echo_pb.proto @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Protobuf control service with the same fields as echo.fbs, served over +// baidu_std so fb_rpc can be compared against an equivalent payload. +syntax = "proto2"; + +package benchmark_fb_pb; + +option cc_generic_services = true; + +message Request { + optional uint64 request_id = 1; + optional bytes message = 2; +} + +message Response { + optional uint64 request_id = 1; + optional bytes message = 2; + optional uint64 attachment_size = 3; +} + +service BenchmarkService { + rpc Echo(Request) returns (Response); +} diff --git a/example/benchmark_fb/server.cpp b/example/benchmark_fb/server.cpp index f8fa9244ef..7989573272 100644 --- a/example/benchmark_fb/server.cpp +++ b/example/benchmark_fb/server.cpp @@ -26,18 +26,28 @@ #include "brpc/errno.pb.h" #include "brpc/server.h" #include "echo.brpc.fb.h" +#include "echo_pb.pb.h" #if !BRPC_WITH_FLATBUFFERS #error "benchmark_fb requires a FlatBuffers-enabled brpc library" #endif -DEFINE_string(listen_addr, "127.0.0.1:0", "Loopback IPv4 address; port 0 selects a free port"); -DEFINE_int32(duration_s, 60, "Exit after 1..3600 seconds, or earlier on SIGINT/SIGTERM"); -DEFINE_int32(max_concurrency, 32, "Maximum concurrent requests, 1..64"); +DEFINE_string(listen_addr, "127.0.0.1:0", "IPv4 address; port 0 selects a free port. " + "A non-loopback address also needs --allow_remote"); +DEFINE_bool(allow_remote, false, "Allow a non-loopback --listen_addr for cross-host benchmarks"); +DEFINE_int32(duration_s, 60, "Exit after 1..86400 seconds, or earlier on SIGINT/SIGTERM"); +DEFINE_int32(max_concurrency, 32, "Maximum concurrent requests, 1..4096"); namespace { const size_t kMaxBytes = 1024 * 1024; +// Reserve the echoed string plus table overhead up front so a large reply +// does not grow the builder by repeated reallocate-and-copy. +size_t BuilderSize(const benchmark_fb::Request* input) { + const size_t text = input->message() ? input->message()->size() : 0; + return text + 256; +} + class BenchmarkServiceImpl : public benchmark_fb::BenchmarkService { public: void Echo(google::protobuf::RpcController* controller, @@ -61,10 +71,10 @@ class BenchmarkServiceImpl : public benchmark_fb::BenchmarkService { cntl->SetFailed(brpc::EREQUEST, "Example payload limit is 1 MiB per field"); return; } - brpc::flatbuffers::MessageBuilder builder; + brpc::flatbuffers::MessageBuilder builder(BuilderSize(input)); // Preserve absent versus empty strings, including embedded zero bytes. const auto text = input->message() ? - builder.CreateString(input->message()->str()) : + builder.CreateString(input->message()->data(), input->message()->size()) : ::flatbuffers::Offset<::flatbuffers::String>(); builder.Finish(benchmark_fb::CreateResponse( builder, input->request_id(), text, cntl->request_attachment().size())); @@ -72,17 +82,40 @@ class BenchmarkServiceImpl : public benchmark_fb::BenchmarkService { cntl->response_attachment().append(cntl->request_attachment()); } }; + +// Protobuf control service over baidu_std with the same echo semantics. +class PbBenchmarkServiceImpl : public benchmark_fb_pb::BenchmarkService { +public: + void Echo(google::protobuf::RpcController* controller, + const benchmark_fb_pb::Request* request, + benchmark_fb_pb::Response* response, + google::protobuf::Closure* done) override { + brpc::ClosureGuard done_guard(done); + auto* cntl = static_cast(controller); + if (request->message().size() > kMaxBytes || + cntl->request_attachment().size() > kMaxBytes) { + cntl->SetFailed(brpc::EREQUEST, "Example payload limit is 1 MiB per field"); + return; + } + response->set_request_id(request->request_id()); + if (request->has_message()) { + response->set_message(request->message()); + } + response->set_attachment_size(cntl->request_attachment().size()); + cntl->response_attachment().append(cntl->request_attachment()); + } +}; } // namespace int main(int argc, char* argv[]) { GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); butil::EndPoint endpoint; - if (FLAGS_listen_addr.compare(0, 10, "127.0.0.1:") != 0 || + if ((FLAGS_listen_addr.compare(0, 10, "127.0.0.1:") != 0 && !FLAGS_allow_remote) || butil::str2endpoint(FLAGS_listen_addr.c_str(), &endpoint) != 0 || - FLAGS_duration_s < 1 || FLAGS_duration_s > 3600 || - FLAGS_max_concurrency < 1 || FLAGS_max_concurrency > 64) { - std::cerr << "Use --listen_addr=127.0.0.1:PORT, --duration_s=1..3600 " - "and --max_concurrency=1..64\n"; + FLAGS_duration_s < 1 || FLAGS_duration_s > 86400 || + FLAGS_max_concurrency < 1 || FLAGS_max_concurrency > 4096) { + std::cerr << "Use --listen_addr=127.0.0.1:PORT (or --allow_remote), " + "--duration_s=1..86400 and --max_concurrency=1..4096\n"; return 2; } // brpc defaults SIGTERM to immediate termination. This example always @@ -98,9 +131,14 @@ int main(int argc, char* argv[]) { std::cerr << "Cannot register FlatBuffers service\n"; return 1; } + PbBenchmarkServiceImpl pb_service; + if (server.AddService(&pb_service, brpc::SERVER_DOESNT_OWN_SERVICE) != 0) { + std::cerr << "Cannot register protobuf control service\n"; + return 1; + } brpc::ServerOptions options; options.has_builtin_services = false; - options.enabled_protocols = "fb_rpc"; + options.enabled_protocols = "fb_rpc baidu_std"; options.max_concurrency = FLAGS_max_concurrency; if (server.Start(endpoint, &options) != 0) { std::cerr << "Cannot start FlatBuffers server\n"; From a6709e717f260d29f6095adfe1c0b8cf3a3ec84c Mon Sep 17 00:00:00 2001 From: xulei25 Date: Wed, 28 Oct 2026 17:13:19 +0800 Subject: [PATCH 6/6] Drop fb_rpc responses that arrive on another socket ProcessFlatBuffersResponse only checked that the correlation id was still live. A response on another connection that carried the id of an in-flight RPC could therefore modify and complete that RPC. Like the protobuf response handlers, check that the Controller exists and that the response arrived on the socket used by the attempt it answers, before touching the Controller or the response. Otherwise log a warning, unlock the id and drop the message. Add regression tests for a live id on a foreign socket, a response from a previous retry, and a preserved backup attempt that may only complete on its own socket. The ON runner now expects 36 protocol tests. --- .github/scripts/flatbuffers-on.py | 2 +- src/brpc/policy/flatbuffers_protocol.cpp | 13 +++ test/brpc_flatbuffers_protocol_unittest.cpp | 97 +++++++++++++++++++++ 3 files changed, 111 insertions(+), 1 deletion(-) diff --git a/.github/scripts/flatbuffers-on.py b/.github/scripts/flatbuffers-on.py index 8d6c02cb55..e860b5aa67 100644 --- a/.github/scripts/flatbuffers-on.py +++ b/.github/scripts/flatbuffers-on.py @@ -38,7 +38,7 @@ FLATBUFFERS_SHA256 = "b9c2df49707c57a48fc0923d52b8c73beb72d675f9d44b2211e4569be40a7421" GTEST_VERSION = "1.14.0" GTEST_SHA256 = "8ad598c73ad796e0d8280b082cebd82a630d73e73cd3c70057938a6501bba5d7" -TESTS = {"brpc_flatbuffers_unittest": 22, "brpc_flatbuffers_protocol_unittest": 33} +TESTS = {"brpc_flatbuffers_unittest": 22, "brpc_flatbuffers_protocol_unittest": 36} def list_gtests(text): diff --git a/src/brpc/policy/flatbuffers_protocol.cpp b/src/brpc/policy/flatbuffers_protocol.cpp index d63f67a4bd..91f7e7cd46 100644 --- a/src/brpc/policy/flatbuffers_protocol.cpp +++ b/src/brpc/policy/flatbuffers_protocol.cpp @@ -362,6 +362,19 @@ void ProcessFlatBuffersResponse(InputMessageBase* message_base) { if (rc != 0) { return; } + // A live correlation id alone does not prove that this response belongs + // to the attempt: it must also arrive on the socket the attempt used. + const SocketId response_socket_id = + message->socket() ? message->socket()->id() : INVALID_SOCKET_ID; + if (cntl == nullptr || !ControllerPrivateAccessor(cntl) + .does_response_match_request_socket(correlation_id, response_socket_id)) { + LOG(WARNING) << "correlation_id=" << correlation_id.value + << " of the FlatBuffers response from socket " + << response_socket_id + << " does not match a request sent over it, drop it"; + CHECK_EQ(0, bthread_id_unlock(correlation_id)); + return; + } ControllerPrivateAccessor accessor(cntl); const int saved_error = cntl->ErrorCode(); cntl->set_rpc_received_us(message->received_us()); diff --git a/test/brpc_flatbuffers_protocol_unittest.cpp b/test/brpc_flatbuffers_protocol_unittest.cpp index 21e6749641..e49f5c63f2 100644 --- a/test/brpc_flatbuffers_protocol_unittest.cpp +++ b/test/brpc_flatbuffers_protocol_unittest.cpp @@ -54,6 +54,7 @@ #include "brpc/policy/flatbuffers_protocol.h" #include "brpc/policy/most_common_message.h" #include "brpc/server.h" +#include "brpc/socket.h" #include "brpc/stream.h" #include "flatbuffers_message_generated.h" @@ -1821,5 +1822,101 @@ TEST(FlatBuffersResponseTest, SchemaVerificationIsExplicitlyTheCallersResponsibi EXPECT_TRUE(peer.served()); } +// Builds a response frame for `cid' as if it had arrived on `socket'. +brpc::policy::MostCommonMessage* MakeFlatBuffersResponse( + const Message& payload, brpc::CallId cid, brpc::Socket* socket) { + brpc::policy::MostCommonMessage* msg = brpc::policy::MostCommonMessage::Get(); + msg->meta.append(ResponseMeta(payload.size(), 0, cid.value)); + msg->payload.append(Bytes(payload)); + socket->ReAddress(&msg->_socket); + return msg; +} + +brpc::SocketUniquePtr CreateTestSocket() { + brpc::SocketId id; + brpc::SocketUniquePtr ptr; + EXPECT_EQ(0, brpc::Socket::Create(brpc::SocketOptions(), &id)); + EXPECT_EQ(0, brpc::Socket::Address(id, &ptr)); + return ptr; +} + +TEST(FlatBuffersResponseTest, ResponseFromForeignSocketIsDropped) { + const Message expected = MakePayload(64, 7); + Message response; + brpc::Controller cntl; + cntl._response = &response; + ASSERT_EQ(0, bthread_id_lock_and_reset_range(cntl.call_id(), nullptr, 2)); + ASSERT_EQ(0, bthread_id_unlock(cntl.current_id())); + brpc::SocketUniquePtr sending = CreateTestSocket(); + brpc::SocketUniquePtr foreign = CreateTestSocket(); + ASSERT_TRUE(sending && foreign); + sending->ReAddress(&cntl._current_call.sending_sock); + + // A live correlation id on another connection must not complete the RPC. + brpc::policy::ProcessFlatBuffersResponse( + MakeFlatBuffersResponse(expected, cntl.current_id(), foreign.get())); + EXPECT_EQ(0u, response.size()); + EXPECT_EQ(0, cntl.ErrorCode()); + + // The call stays available for the response on its own socket. + brpc::policy::ProcessFlatBuffersResponse( + MakeFlatBuffersResponse(expected, cntl.current_id(), sending.get())); + EXPECT_EQ(0, cntl.ErrorCode()) << cntl.ErrorText(); + EXPECT_EQ(Bytes(expected), Bytes(response)); +} + +TEST(FlatBuffersResponseTest, ResponseFromPreviousRetryIsDropped) { + const Message expected = MakePayload(64, 8); + Message response; + brpc::Controller cntl; + cntl._response = &response; + ASSERT_EQ(0, bthread_id_lock_and_reset_range(cntl.call_id(), nullptr, 3)); + const brpc::CallId previous_id = cntl.current_id(); + cntl._current_call.nretry = 1; + ASSERT_EQ(0, bthread_id_unlock(cntl.current_id())); + brpc::SocketUniquePtr sending = CreateTestSocket(); + ASSERT_TRUE(sending); + sending->ReAddress(&cntl._current_call.sending_sock); + + // Even on the right socket, a finished attempt cannot supply the result. + brpc::policy::ProcessFlatBuffersResponse( + MakeFlatBuffersResponse(expected, previous_id, sending.get())); + EXPECT_EQ(0u, response.size()); + EXPECT_EQ(0, cntl.ErrorCode()); + + brpc::policy::ProcessFlatBuffersResponse( + MakeFlatBuffersResponse(expected, cntl.current_id(), sending.get())); + EXPECT_EQ(0, cntl.ErrorCode()) << cntl.ErrorText(); + EXPECT_EQ(Bytes(expected), Bytes(response)); +} + +TEST(FlatBuffersResponseTest, UnfinishedBackupAttemptMatchesOnlyItsOwnSocket) { + const Message expected = MakePayload(64, 9); + Message response; + brpc::Controller cntl; + cntl._response = &response; + ASSERT_EQ(0, bthread_id_lock_and_reset_range(cntl.call_id(), nullptr, 3)); + ASSERT_EQ(0, bthread_id_unlock(cntl.current_id())); + const brpc::CallId first_id = cntl.current_id(); + brpc::SocketUniquePtr first = CreateTestSocket(); + brpc::SocketUniquePtr backup = CreateTestSocket(); + ASSERT_TRUE(first && backup); + first->ReAddress(&cntl._current_call.sending_sock); + // Mirror starting a backup request: keep the first attempt and its socket. + cntl._unfinished_call = new brpc::Controller::Call(&cntl._current_call); + ++cntl._current_call.nretry; + backup->ReAddress(&cntl._current_call.sending_sock); + + brpc::policy::ProcessFlatBuffersResponse( + MakeFlatBuffersResponse(expected, first_id, backup.get())); + EXPECT_EQ(0u, response.size()); + EXPECT_EQ(0, cntl.ErrorCode()); + + brpc::policy::ProcessFlatBuffersResponse( + MakeFlatBuffersResponse(expected, first_id, first.get())); + EXPECT_EQ(0, cntl.ErrorCode()) << cntl.ErrorText(); + EXPECT_EQ(Bytes(expected), Bytes(response)); +} + } // namespace #endif // BRPC_WITH_FLATBUFFERS