From 452a7448478b987f7d1fe0b9008d576f5ef8d17c Mon Sep 17 00:00:00 2001 From: Dave Allison Date: Sun, 2 Aug 2026 17:39:53 -0700 Subject: [PATCH 1/7] Add ROS-style struct frontend Add an optional value-like ROS API while preserving Phaser native layout and protobuf wire compatibility, including fixed arrays, oneof variants, and ROS1 intrinsic types. --- MODULE.bazel | 1 + MODULE.bazel.lock | 3 +- README.md | 32 + phaser/BUILD.bazel | 48 + phaser/compiler/BUILD.bazel | 1 + phaser/compiler/gen.cc | 102 +- phaser/compiler/gen.h | 7 +- phaser/compiler/message_gen.cc | 657 ++++++- phaser/compiler/message_gen.h | 44 +- phaser/docs/phaser_user_guide.md | 163 ++ phaser/options.proto | 12 + phaser/phaser_library.bzl | 34 +- phaser/ros_compile_test.cc | 666 +++++++ phaser/ros_intrinsics_test.cc | 109 ++ .../ros_native_frontend_compatibility_test.cc | 98 ++ phaser/runtime/BUILD.bazel | 2 + phaser/runtime/arrays.h | 1547 +++++++++++++++++ phaser/runtime/fields.h | 198 ++- phaser/runtime/message.h | 17 +- phaser/runtime/ros.h | 220 +++ phaser/runtime/runtime.h | 1 + phaser/runtime/union.h | 119 ++ phaser/runtime/vectors.h | 305 +++- phaser/testdata/BUILD | 135 ++ phaser/testdata/InvalidArraySize.proto | 9 + phaser/testdata/InvalidRosIntrinsic.proto | 9 + phaser/testdata/RosCompile.proto | 40 + phaser/testdata/RosHeader.proto | 11 + phaser/testdata/RosIntrinsics.proto | 32 + .../RosIntrinsicsProtobufFrontend.proto | 32 + phaser/testdata/invalid_array_size_test.sh | 24 + phaser/testdata/invalid_ros_intrinsic_test.sh | 23 + .../ros_intrinsics_phaser_wire_tool.cc | 112 ++ .../ros_intrinsics_protobuf_wire_tool.cc | 113 ++ .../ros_intrinsics_wire_compatibility_test.sh | 16 + phaser/testdata/ros_shim/ros/time.h | 33 + phaser/testdata/ros_shim/std_msgs/Header.h | 21 + 37 files changed, 4903 insertions(+), 93 deletions(-) create mode 100644 phaser/options.proto create mode 100644 phaser/ros_compile_test.cc create mode 100644 phaser/ros_intrinsics_test.cc create mode 100644 phaser/ros_native_frontend_compatibility_test.cc create mode 100644 phaser/runtime/arrays.h create mode 100644 phaser/runtime/ros.h create mode 100644 phaser/testdata/InvalidArraySize.proto create mode 100644 phaser/testdata/InvalidRosIntrinsic.proto create mode 100644 phaser/testdata/RosCompile.proto create mode 100644 phaser/testdata/RosHeader.proto create mode 100644 phaser/testdata/RosIntrinsics.proto create mode 100644 phaser/testdata/RosIntrinsicsProtobufFrontend.proto create mode 100755 phaser/testdata/invalid_array_size_test.sh create mode 100755 phaser/testdata/invalid_ros_intrinsic_test.sh create mode 100644 phaser/testdata/ros_intrinsics_phaser_wire_tool.cc create mode 100644 phaser/testdata/ros_intrinsics_protobuf_wire_tool.cc create mode 100755 phaser/testdata/ros_intrinsics_wire_compatibility_test.sh create mode 100644 phaser/testdata/ros_shim/ros/time.h create mode 100644 phaser/testdata/ros_shim/std_msgs/Header.h diff --git a/MODULE.bazel b/MODULE.bazel index 8562487..a0d12ea 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -13,6 +13,7 @@ bazel_dep(name = "googletest", version = "1.17.0.bcr.2", repo_name = "com_google bazel_dep(name = "protobuf", version = "34.1", repo_name = "com_google_protobuf") bazel_dep(name = "rules_cc", version = "0.2.17") +bazel_dep(name = "rules_shell", version = "0.8.0") bazel_dep(name = "rules_pkg", version = "1.0.1") bazel_dep(name = "zlib", version = "1.3.1.bcr.5") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 343a1ac..4618626 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -245,7 +245,8 @@ "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", "https://bcr.bazel.build/modules/rules_shell/0.6.1/MODULE.bazel": "72e76b0eea4e81611ef5452aa82b3da34caca0c8b7b5c0c9584338aa93bae26b", - "https://bcr.bazel.build/modules/rules_shell/0.6.1/source.json": "20ec05cd5e592055e214b2da8ccb283c7f2a421ea0dc2acbf1aa792e11c03d0c", + "https://bcr.bazel.build/modules/rules_shell/0.8.0/MODULE.bazel": "f6a89f1d6a669a26f28fe814503857055d76306b79cfc11d12399af08d0b80ae", + "https://bcr.bazel.build/modules/rules_shell/0.8.0/source.json": "eb53cc815bc503c6683c5fe12d943f98883f81fc22f51403ec8a95610cba4195", "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", "https://bcr.bazel.build/modules/rules_swift/2.4.0/MODULE.bazel": "1639617eb1ede28d774d967a738b4a68b0accb40650beadb57c21846beab5efd", diff --git a/README.md b/README.md index 4c0e136..978a844 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,7 @@ proto_library( phaser_library( name = "foo_phaser", add_namespace = "phaser", # optional: avoids clashing with protobuf classes + frontend = "protobuf", # default; use "ros" for public field proxies deps = [":foo_proto"], ) ``` @@ -126,6 +127,37 @@ any protobuf header: #include "foo/bar/Foo.phaser.h" ``` +Phaser can generate either the default protobuf-style accessors or a ROS-style +struct interface. The ROS frontend preserves the same native payload layout and +protobuf wire transcoding: + +```python +phaser_library( + name = "foo_ros_phaser", + frontend = "ros", + deps = [":foo_proto"], +) +``` + +```c++ +Foo msg; +msg.count = 3; +msg.name = "sensor"; +msg.samples.push_back(1.5); +``` + +Fixed-size ROS fields are repeated protobuf fields annotated with +`[(phaser.array_size) = N]`; import `phaser/options.proto` in the schema. ROS +`oneof` fields expose a variant-like proxy with generated alternative tags. +See the user guide for the complete array and oneof APIs. + +The ROS frontend also maps singular `google.protobuf.Timestamp`, +`google.protobuf.Duration`, and `std_msgs.Header` message fields to +`ros::Time`, `ros::Duration`, and `std_msgs::Header`. This allows existing ROS1 +functions taking values, const references, or mutable references to accept the +generated fields unchanged. Add the corresponding ROS C++ targets through the +`cc_deps` attribute. + ### 2. Create and use a message Creating a message looks just like protobuf — the binary data is backed by a dynamic diff --git a/phaser/BUILD.bazel b/phaser/BUILD.bazel index c059c2f..943bc7f 100644 --- a/phaser/BUILD.bazel +++ b/phaser/BUILD.bazel @@ -1,8 +1,21 @@ +load("@com_google_protobuf//bazel:cc_proto_library.bzl", "cc_proto_library") +load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") load("//phaser:copts.bzl", "PHASER_COPTS") package(default_visibility = ["//visibility:public"]) +proto_library( + name = "options_proto", + srcs = ["options.proto"], + deps = ["@com_google_protobuf//:descriptor_proto"], +) + +cc_proto_library( + name = "options_cc_proto", + deps = [":options_proto"], +) + exports_files(["valgrind.supp"]) # True when building with clang, which gates the clang-only -Weverything set in @@ -50,6 +63,41 @@ cc_test( ], ) +cc_test( + name = "ros_compile_test", + srcs = ["ros_compile_test.cc"], + copts = PHASER_COPTS, + deps = [ + "//phaser/runtime:phaser_runtime", + "//phaser/testdata:ros_compile_cc_proto", + "//phaser/testdata:ros_compile_phaser", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "ros_intrinsics_test", + srcs = ["ros_intrinsics_test.cc"], + copts = PHASER_COPTS, + deps = [ + "//phaser/runtime:phaser_runtime", + "//phaser/testdata:ros_intrinsics_phaser", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "ros_native_frontend_compatibility_test", + srcs = ["ros_native_frontend_compatibility_test.cc"], + copts = PHASER_COPTS, + deps = [ + "//phaser/runtime:phaser_runtime", + "//phaser/testdata:ros_intrinsics_phaser", + "//phaser/testdata:ros_intrinsics_protobuf_phaser", + "@com_google_googletest//:gtest_main", + ], +) + cc_test( name = "all_types_test", srcs = ["all_types_test.cc"], diff --git a/phaser/compiler/BUILD.bazel b/phaser/compiler/BUILD.bazel index 90b36a6..7410082 100644 --- a/phaser/compiler/BUILD.bazel +++ b/phaser/compiler/BUILD.bazel @@ -17,6 +17,7 @@ cc_library( "message_gen.h", ], deps = [ + "//phaser:options_cc_proto", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/container:flat_hash_set", "@com_google_absl//absl/status", diff --git a/phaser/compiler/gen.cc b/phaser/compiler/gen.cc index 4a1f38b..716a5ed 100644 --- a/phaser/compiler/gen.cc +++ b/phaser/compiler/gen.cc @@ -12,6 +12,46 @@ namespace phaser { +static FrontendStyle EffectiveFrontendStyle( + const google::protobuf::FileDescriptor* file, FrontendStyle requested) { + if (requested == FrontendStyle::kRos && + file->package().rfind("google.protobuf", 0) == 0) { + // Well-known protobuf imports keep the protobuf-style layout so dependency + // graphs (notably descriptor.proto) remain compilable in ROS targets. + return FrontendStyle::kProtobuf; + } + return requested; +} + +static bool UsesRos1Intrinsic(const google::protobuf::Descriptor* message) { + for (int i = 0; i < message->field_count(); ++i) { + const auto* field = message->field(i); + if (field->type() != google::protobuf::FieldDescriptor::TYPE_MESSAGE) { + continue; + } + const auto name = field->message_type()->full_name(); + if (name == "google.protobuf.Timestamp" || + name == "google.protobuf.Duration" || name == "std_msgs.Header") { + return true; + } + } + for (int i = 0; i < message->nested_type_count(); ++i) { + if (UsesRos1Intrinsic(message->nested_type(i))) { + return true; + } + } + return false; +} + +static bool UsesRos1Intrinsic(const google::protobuf::FileDescriptor* file) { + for (int i = 0; i < file->message_type_count(); ++i) { + if (UsesRos1Intrinsic(file->message_type(i))) { + return true; + } + } + return false; +} + static void WriteToZeroCopyStream( const std::string& data, google::protobuf::io::ZeroCopyOutputStream* stream) { @@ -67,11 +107,35 @@ bool CodeGenerator::Generate( generate_active_message_ = option.second.empty() || option.second == "true" || option.second == "1"; + } else if (option.first == "frontend") { + if (option.second == "protobuf" || option.second.empty()) { + frontend_style_ = FrontendStyle::kProtobuf; + } else if (option.second == "ros") { + frontend_style_ = FrontendStyle::kRos; + } else { + *error = absl::StrFormat( + "Unknown frontend value: %s (expected protobuf or ros)", + option.second); + return false; + } } } + const FrontendStyle effective_frontend = + EffectiveFrontendStyle(file, frontend_style_); + + // Custom option schemas and other message-free protos need no C++ output. + // descriptor.proto is imported for extensions but must not be emitted as a + // Phaser message graph (it is huge and not a runtime payload type here). + if (file->message_type_count() == 0 && file->enum_type_count() == 0) { + return true; + } + if (file->name() == std::string("google/protobuf/descriptor.proto")) { + return true; + } + Generator gen(file, added_namespace_, package_name_, target_name_, - generate_active_message_); + generate_active_message_, effective_frontend); std::string filename = GeneratedFilename(package_name_, target_name_, std::string(file->name())); @@ -95,7 +159,12 @@ bool CodeGenerator::Generate( return false; } std::stringstream header_stream; - gen.GenerateHeaders(header_stream); + std::string validation_error; + gen.GenerateHeaders(header_stream, &validation_error); + if (!validation_error.empty()) { + *error = validation_error; + return false; + } std::filesystem::path cp(filename); cp.replace_extension(".phaser.cc"); @@ -137,16 +206,18 @@ void Generator::CloseNamespace(std::ostream& os) { Generator::Generator(const google::protobuf::FileDescriptor* file, const std::string& ns, const std::string& pn, - const std::string& tn, bool generate_active_message) + const std::string& tn, bool generate_active_message, + FrontendStyle frontend_style) : file_(file), added_namespace_(ns), package_name_(pn), target_name_(tn), - generate_active_message_(generate_active_message) { + generate_active_message_(generate_active_message), + frontend_style_(frontend_style) { for (int i = 0; i < file->message_type_count(); i++) { message_gens_.push_back(std::make_unique( file->message_type(i), added_namespace_, std::string(file->package()), - generate_active_message_)); + generate_active_message_, frontend_style_)); } // Enums for (int i = 0; i < file->enum_type_count(); i++) { @@ -154,15 +225,25 @@ Generator::Generator(const google::protobuf::FileDescriptor* file, } } -void Generator::GenerateHeaders(std::ostream& os) { +void Generator::GenerateHeaders(std::ostream& os, std::string* error) { os << "#pragma once\n"; os << "#include \"phaser/runtime/runtime.h\"\n"; + if (frontend_style_ == FrontendStyle::kRos && UsesRos1Intrinsic(file_)) { + os << "#include \"phaser/runtime/ros.h\"\n"; + } if (generate_active_message_) { os << "#include \n"; } for (int i = 0; i < file_->dependency_count(); i++) { + const google::protobuf::FileDescriptor* dep = file_->dependency(i); + if (dep->message_type_count() == 0 && dep->enum_type_count() == 0) { + continue; + } + if (dep->name() == std::string("google/protobuf/descriptor.proto")) { + continue; + } std::string base = GeneratedFilename( - package_name_, target_name_, std::string(file_->dependency(i)->name())); + package_name_, target_name_, std::string(dep->name())); std::filesystem::path p(base); p.replace_extension(".phaser.h"); os << "#include \"" << p.string() << "\"\n"; @@ -180,7 +261,12 @@ void Generator::GenerateHeaders(std::ostream& os) { } for (auto& msg_gen : message_gens_) { - msg_gen->GenerateHeader(os); + if (absl::Status status = msg_gen->GenerateHeader(os); !status.ok()) { + if (error != nullptr) { + *error = std::string(status.message()); + } + return; + } } CloseNamespace(os); diff --git a/phaser/compiler/gen.h b/phaser/compiler/gen.h index b885940..762a1c7 100644 --- a/phaser/compiler/gen.h +++ b/phaser/compiler/gen.h @@ -36,15 +36,17 @@ class CodeGenerator : public google::protobuf::compiler::CodeGenerator { // field. Enabled via the `active_message=true` plugin command-line option // (set by phaser_library(enable_active_message = True)). mutable bool generate_active_message_ = false; + mutable FrontendStyle frontend_style_ = FrontendStyle::kProtobuf; }; class Generator { public: Generator(const google::protobuf::FileDescriptor* file, const std::string& ns, const std::string& pn, const std::string& tn, - bool generate_active_message = false); + bool generate_active_message = false, + FrontendStyle frontend_style = FrontendStyle::kProtobuf); - void GenerateHeaders(std::ostream& os); + void GenerateHeaders(std::ostream& os, std::string* error); void GenerateSources(std::ostream& os); private: @@ -58,6 +60,7 @@ class Generator { const std::string& package_name_; const std::string& target_name_; bool generate_active_message_; + FrontendStyle frontend_style_; }; } // namespace phaser diff --git a/phaser/compiler/message_gen.cc b/phaser/compiler/message_gen.cc index f0a72f0..a686387 100644 --- a/phaser/compiler/message_gen.cc +++ b/phaser/compiler/message_gen.cc @@ -12,6 +12,7 @@ #include "absl/strings/str_format.h" #include "absl/strings/str_replace.h" +#include "phaser/options.pb.h" namespace phaser { @@ -118,6 +119,136 @@ static bool IsCppReservedWord(const std::string& s) { return reserved_words.contains(s); } +std::string MessageGenerator::SanitizedIdentifier( + const std::string& name) const { + if (IsCppReservedWord(name)) { + return name + "_"; + } + return name; +} + +std::string MessageGenerator::MemberVariableName( + const std::string& proto_name) const { + if (IsRosFrontend()) { + return SanitizedIdentifier(proto_name); + } + return proto_name + "_"; +} + +std::string MessageGenerator::OneofVariantTypeName( + const google::protobuf::OneofDescriptor* oneof) const { + std::string name; + bool capitalize = true; + for (char c : oneof->name()) { + if (c == '_') { + capitalize = true; + continue; + } + name.push_back(capitalize ? static_cast(std::toupper(c)) : c); + capitalize = false; + } + return SanitizedIdentifier(name + "Variant"); +} + +std::string MessageGenerator::OneofAlternativeTypeName( + const google::protobuf::FieldDescriptor* field) const { + std::string name(field->camelcase_name()); + if (!name.empty()) { + name[0] = static_cast(std::toupper(name[0])); + } + return SanitizedIdentifier(name + "Alternative"); +} + +int MessageGenerator::GetArraySize( + const google::protobuf::FieldDescriptor* field) const { + if (!field->options().HasExtension(phaser::array_size)) { + return 0; + } + return static_cast(field->options().GetExtension(phaser::array_size)); +} + +bool MessageGenerator::UsesArrayFacade( + const google::protobuf::FieldDescriptor* field) const { + return IsRosFrontend() && GetArraySize(field) > 0; +} + +absl::Status MessageGenerator::ValidateArraySizeOption( + const google::protobuf::FieldDescriptor* field) const { + if (!field->options().HasExtension(phaser::array_size)) { + return absl::OkStatus(); + } + const int array_size = GetArraySize(field); + const std::string context = absl::StrFormat("%s.%s", message_->full_name(), + field->name()); + if (array_size <= 0) { + return absl::InvalidArgumentError( + absl::StrFormat("phaser.array_size must be positive on field %s", + context)); + } + if (!field->is_repeated()) { + return absl::InvalidArgumentError(absl::StrFormat( + "phaser.array_size is only valid on repeated fields: %s", context)); + } + if (field->is_map()) { + return absl::InvalidArgumentError(absl::StrFormat( + "phaser.array_size is not valid on map fields: %s", context)); + } + return absl::OkStatus(); +} + +absl::Status MessageGenerator::ValidateFieldOptions() const { + if (IsRosFrontend() && IsRosHeader(message_) && added_namespace_.empty()) { + return absl::InvalidArgumentError( + "ROS frontend generation for std_msgs.Header requires add_namespace " + "to avoid colliding with the ROS std_msgs::Header type"); + } + if (IsRosFrontend()) { + if (absl::Status status = ValidateRosHeaderDescriptor(); !status.ok()) { + return status; + } + } + for (int i = 0; i < message_->field_count(); i++) { + const auto* field = message_->field(i); + if (absl::Status status = ValidateArraySizeOption(field); + !status.ok()) { + return status; + } + if (IsRosFrontend() && IsRosIntrinsic(field) && + (field->is_repeated() || field->containing_oneof() != nullptr)) { + return absl::InvalidArgumentError(absl::StrFormat( + "ROS intrinsic field %s.%s must be singular and cannot be in a " + "oneof", + message_->full_name(), field->name())); + } + } + for (const auto& nested : nested_message_gens_) { + if (absl::Status status = nested->ValidateFieldOptions(); !status.ok()) { + return status; + } + } + return absl::OkStatus(); +} + +absl::Status MessageGenerator::ValidateRosHeaderDescriptor() const { + if (!IsRosHeader(message_)) { + return absl::OkStatus(); + } + const auto* seq = message_->FindFieldByName("seq"); + const auto* stamp = message_->FindFieldByName("stamp"); + const auto* frame_id = message_->FindFieldByName("frame_id"); + if (seq == nullptr || + seq->type() != google::protobuf::FieldDescriptor::TYPE_UINT32 || + stamp == nullptr || + stamp->type() != google::protobuf::FieldDescriptor::TYPE_MESSAGE || + !IsRosTime(stamp->message_type()) || frame_id == nullptr || + frame_id->type() != google::protobuf::FieldDescriptor::TYPE_STRING) { + return absl::InvalidArgumentError( + "std_msgs.Header must declare uint32 seq, " + "google.protobuf.Timestamp stamp, and string frame_id"); + } + return absl::OkStatus(); +} + std::string MessageGenerator::EnumName( const google::protobuf::EnumDescriptor* desc) { std::string name(desc->name()); @@ -192,6 +323,9 @@ std::string MessageGenerator::FieldCFieldType( if (IsAny(field)) { return "AnyField"; } + if (IsRosFrontend() && IsRosIntrinsic(field)) { + return RosIntrinsicFieldType(field); + } return "IndirectMessageField<" + MessageName(field->message_type(), true) + ">"; @@ -278,6 +412,9 @@ std::string MessageGenerator::FieldCType( case google::protobuf::FieldDescriptor::TYPE_BYTES: return "std::string_view"; case google::protobuf::FieldDescriptor::TYPE_MESSAGE: + if (IsRosFrontend() && IsRosIntrinsic(field)) { + return RosIntrinsicCType(field); + } return MessageName(field->message_type(), true); case google::protobuf::FieldDescriptor::TYPE_GROUP: std::cerr << "Groups are not supported\n"; @@ -289,6 +426,15 @@ std::string MessageGenerator::FieldCType( std::string MessageGenerator::FieldRepeatedCType( const google::protobuf::FieldDescriptor* field) { + const int array_size = GetArraySize(field); + if (IsRosFrontend() && array_size > 0) { + return FieldRepeatedArrayCType(field, array_size); + } + return FieldRepeatedVectorCType(field); +} + +std::string MessageGenerator::FieldRepeatedVectorCType( + const google::protobuf::FieldDescriptor* field) { std::string packed = field->is_packed() ? ", true>" : ", false>"; switch (field->type()) { case google::protobuf::FieldDescriptor::TYPE_INT32: @@ -335,6 +481,65 @@ std::string MessageGenerator::FieldRepeatedCType( abort(); } +std::string MessageGenerator::FieldRepeatedArrayCType( + const google::protobuf::FieldDescriptor* field, int array_size) { + const std::string extent = std::to_string(array_size); + const std::string packed = field->is_packed() ? ", true>" : ", false>"; + switch (field->type()) { + case google::protobuf::FieldDescriptor::TYPE_INT32: + return "PrimitiveArrayFieldenum_type()) + ", " + extent + + ", " + EnumName(field->enum_type()) + "Stringizer, " + + EnumName(field->enum_type()) + "Parser" + packed; + case google::protobuf::FieldDescriptor::TYPE_STRING: + case google::protobuf::FieldDescriptor::TYPE_BYTES: + return "StringArrayField<" + extent + ">"; + case google::protobuf::FieldDescriptor::TYPE_MESSAGE: + return "MessageArrayField<" + MessageName(field->message_type(), true) + + ", " + extent + ">"; + case google::protobuf::FieldDescriptor::TYPE_GROUP: + std::cerr << "Groups are not supported\n"; + exit(1); + } + abort(); +} + std::string MessageGenerator::FieldUnionCType( const google::protobuf::FieldDescriptor* field) { switch (field->type()) { @@ -424,6 +629,58 @@ bool MessageGenerator::IsAny(const google::protobuf::Descriptor* desc) { return desc->full_name() == "google.protobuf.Any"; } +bool MessageGenerator::IsRosTime( + const google::protobuf::Descriptor* desc) const { + return desc != nullptr && + desc->full_name() == "google.protobuf.Timestamp"; +} + +bool MessageGenerator::IsRosDuration( + const google::protobuf::Descriptor* desc) const { + return desc != nullptr && + desc->full_name() == "google.protobuf.Duration"; +} + +bool MessageGenerator::IsRosHeader( + const google::protobuf::Descriptor* desc) const { + return desc != nullptr && desc->full_name() == "std_msgs.Header"; +} + +bool MessageGenerator::IsRosIntrinsic( + const google::protobuf::FieldDescriptor* field) const { + if (field == nullptr || + field->type() != google::protobuf::FieldDescriptor::TYPE_MESSAGE) { + return false; + } + const auto* desc = field->message_type(); + return IsRosTime(desc) || IsRosDuration(desc) || IsRosHeader(desc); +} + +std::string MessageGenerator::RosIntrinsicFieldType( + const google::protobuf::FieldDescriptor* field) { + const std::string backend = MessageName(field->message_type(), true); + if (IsRosTime(field->message_type())) { + return "RosTimeField<" + backend + ">"; + } + if (IsRosDuration(field->message_type())) { + return "RosDurationField<" + backend + ">"; + } + assert(IsRosHeader(field->message_type())); + return "RosHeaderField<" + backend + ">"; +} + +std::string MessageGenerator::RosIntrinsicCType( + const google::protobuf::FieldDescriptor* field) { + if (IsRosTime(field->message_type())) { + return "::ros::Time"; + } + if (IsRosDuration(field->message_type())) { + return "::ros::Duration"; + } + assert(IsRosHeader(field->message_type())); + return "::std_msgs::Header"; +} + bool MessageGenerator::IsAny(const google::protobuf::FieldDescriptor* field) { return field->type() == google::protobuf::FieldDescriptor::TYPE_MESSAGE && field->message_type()->full_name() == "google.protobuf.Any"; @@ -453,7 +710,8 @@ void MessageGenerator::CompileUnions() { union_info->member_type += "::phaser::" + field_type; uint32_t field_size = FieldBinarySize(field); union_info->members.push_back(std::make_shared( - field, 0, union_info->id, std::string(field->name()) + "_", field_type, + field, 0, union_info->id, + MemberVariableName(std::string(field->name())), field_type, FieldCType(field), field_size)); union_info->binary_size = std::max(union_info->binary_size, 4 + field_size); union_info->id++; @@ -483,7 +741,7 @@ void MessageGenerator::CompileFields() { auto it = unions_.find(oneof); if (it == unions_.end()) { auto union_info = std::make_shared( - oneof, 4, std::string(oneof->name()) + "_", "UnionField"); + oneof, 4, MemberVariableName(std::string(oneof->name())), "UnionField"); unions_[oneof] = union_info; fields_in_order_.push_back(union_info); } @@ -505,8 +763,8 @@ void MessageGenerator::CompileFields() { } offset = (offset + (field_size - 1)) & ~(field_size - 1); fields_.push_back(std::make_shared( - field, offset, id, std::string(field->name()) + "_", field_type, - FieldCType(field), field_size)); + field, offset, id, MemberVariableName(std::string(field->name())), + field_type, FieldCType(field), field_size)); fields_in_order_.push_back(fields_.back()); offset += field_size; id = next_id; @@ -550,16 +808,25 @@ void MessageGenerator::FinalizeOffsetsAndSizes() { binary_size_ = size; } -void MessageGenerator::GenerateHeader(std::ostream& os) { +absl::Status MessageGenerator::GenerateHeader(std::ostream& os) { + if (absl::Status status = ValidateFieldOptions(); !status.ok()) { + return status; + } for (const auto& nested : nested_message_gens_) { - nested->GenerateHeader(os); + if (absl::Status status = nested->GenerateHeader(os); !status.ok()) { + return status; + } } CompileFields(); CompileUnions(); FinalizeOffsetsAndSizes(); - os << "class " << MessageName(message_) << " : public ::phaser::Message {\n"; + os << (IsRosFrontend() ? "struct " : "class ") << MessageName(message_) + << " : public ::phaser::Message {\n"; os << " public:\n"; + if (IsRosFrontend()) { + GenerateRosOneofTypes(os); + } if (generate_active_message_) { os << " // Optional user-attached payload, not part of the wire format.\n"; os << " std::any active_message;\n\n"; @@ -572,6 +839,9 @@ void MessageGenerator::GenerateHeader(std::ostream& os) { GenerateCreators(os, true); // Generate clear function. GenerateClear(os, true); + if (IsRosFrontend()) { + GenerateRosSyncToPayload(os); + } // Generate field metadata. GenerateFieldMetadata(os); @@ -602,8 +872,15 @@ void MessageGenerator::GenerateHeader(std::ostream& os) { GenerateCopy(os, true); GenerateDebugString(os); + if (IsRosFrontend()) { + GenerateRosOwnerCopyMove(os, true); + GeneratePublicFieldDeclarations(os); + } + // Generate protobuf accessors. - GenerateProtobufAccessors(os); + if (!IsRosFrontend()) { + GenerateProtobufAccessors(os); + } GenerateProtobufSerialization(os); @@ -614,13 +891,104 @@ void MessageGenerator::GenerateHeader(std::ostream& os) { // Generate deserializer. GenerateDeserializer(os, true); - os << " private:\n"; - GenerateFieldDeclarations(os); + if (!IsRosFrontend()) { + os << " private:\n"; + GenerateFieldDeclarations(os); + } os << "};\n\n"; // Steamer outside the class. GenerateStreamer(os); GenerateCopy(os, false); + return absl::OkStatus(); +} + +void MessageGenerator::GenerateRosSyncToPayload(std::ostream& os) { + os << " void SyncToPayload() const override {\n"; + for (const auto& field : fields_) { + if (field->field->type() != + google::protobuf::FieldDescriptor::TYPE_MESSAGE || + (!field->field->is_repeated() && IsAny(field->field))) { + continue; + } + os << " " << field->member_name << ".SyncToPayload();\n"; + } + for (const auto& [oneof, union_info] : unions_) { + os << " switch (" << union_info->member_name + << ".Discriminator()) {\n"; + for (size_t i = 0; i < union_info->members.size(); ++i) { + const auto& member = union_info->members[i]; + if (member->field->type() != + google::protobuf::FieldDescriptor::TYPE_MESSAGE) { + continue; + } + os << " case " << member->field->number() << ":\n"; + os << " " << union_info->member_name << ".template GetReference<" + << i << ", " << member->c_type << ">().SyncToPayload();\n"; + os << " break;\n"; + } + os << " default:\n"; + os << " break;\n"; + os << " }\n"; + } + os << " }\n\n"; +} + +void MessageGenerator::GenerateRosOwnerCopyMove(std::ostream& os, bool decl) { + if (!IsRosFrontend()) { + return; + } + const std::string name = MessageName(message_); + if (decl) { + os << " " << name << "(const " << name << "& other);\n"; + os << " " << name << "& operator=(const " << name << "& other);\n"; + os << " " << name << "(" << name << "&& other) noexcept;\n"; + os << " " << name << "& operator=(" << name << "&& other) noexcept;\n\n"; + return; + } + + os << name << "::" << name << "(const " << name << "& other)\n"; + GenerateFieldInitializers(os); + os << R"XXX({ + size_t initial_size = other.BinarySize() * 2; + if (initial_size < 8192) { + initial_size = 8192; + } + InitDynamicMutable(initial_size, ::phaser::Tuning::kPerformance); + (void)CloneFrom(other); +} + +)XXX"; + + os << name << "& " << name << "::operator=(const " << name << "& other) {\n"; + os << " if (this != &other) {\n"; + os << " (void)CloneFrom(other);\n"; + os << " }\n"; + os << " return *this;\n"; + os << "}\n\n"; + + os << name << "::" << name << "(" << name << "&& other) noexcept\n"; + os << " : Message(std::move(other))\n"; + const char* sep = ", "; + for (auto& field : fields_) { + os << sep << field->member_name << "(std::move(other." << field->member_name + << "))\n"; + sep = ", "; + } + for (auto& [oneof, u] : unions_) { + os << sep << u->member_name << "(std::move(other." << u->member_name + << "))\n"; + } + os << "{}\n\n"; + + os << name << "& " << name << "::operator=(" << name << "&& other) noexcept " + << "{\n"; + os << " if (this != &other) {\n"; + os << " (void)CloneFrom(other);\n"; + os << " other.Clear();\n"; + os << " }\n"; + os << " return *this;\n"; + os << "}\n\n"; } void MessageGenerator::GenerateSource(std::ostream& os) { @@ -629,6 +997,9 @@ void MessageGenerator::GenerateSource(std::ostream& os) { } GenerateConstructors(os, false); + if (IsRosFrontend()) { + GenerateRosOwnerCopyMove(os, false); + } // Generate creators. GenerateCreators(os, false); @@ -654,10 +1025,58 @@ void MessageGenerator::GenerateFieldDeclarations(std::ostream& os) { << ";\n"; } for (auto& [oneof, u] : unions_) { - os << " ::phaser::" << u->member_type << " " << u->member_name << ";\n"; + if (IsRosFrontend()) { + os << " " << OneofVariantTypeName(oneof) << " " << u->member_name + << ";\n"; + } else { + os << " ::phaser::" << u->member_type << " " << u->member_name << ";\n"; + } + } +} + +void MessageGenerator::GenerateRosOneofTypes(std::ostream& os) { + for (auto& [oneof, u] : unions_) { + const std::string variant_name = OneofVariantTypeName(oneof); + os << " struct " << variant_name << " : public ::phaser::" + << u->member_type << " {\n"; + os << " using Base = ::phaser::" << u->member_type << ";\n"; + os << " using Base::Base;\n"; + for (size_t i = 0; i < u->members.size(); ++i) { + const auto& field = u->members[i]; + const std::string alternative_name = + OneofAlternativeTypeName(field->field); + os << " struct " << alternative_name << " {\n"; + os << " using value_type = " << field->c_type << ";\n"; + os << " static constexpr size_t kIndex = " << i << ";\n"; + os << " static constexpr int kFieldNumber = " + << field->field->number() << ";\n"; + os << " static constexpr bool kIsMessage = " + << (field->field->type() == + google::protobuf::FieldDescriptor::TYPE_MESSAGE + ? "true" + : "false") + << ";\n"; + os << " };\n"; + } + os << " };\n"; + for (const auto& field : u->members) { + const std::string alternative_name = + OneofAlternativeTypeName(field->field); + os << " using " << alternative_name << " = " << variant_name << "::" + << alternative_name << ";\n"; + } + os << "\n"; } } +void MessageGenerator::GeneratePublicFieldDeclarations(std::ostream& os) { + if (fields_.empty() && unions_.empty()) { + return; + } + os << "\n"; + GenerateFieldDeclarations(os); +} + void MessageGenerator::GenerateEnums(std::ostream& os) { // Nested enums. for (auto& msg : nested_message_gens_) { @@ -938,6 +1357,9 @@ void MessageGenerator::GenerateClear(std::ostream& os, bool decl) { } void MessageGenerator::GenerateProtobufAccessors(std::ostream& os) { + if (IsRosFrontend()) { + return; + } // Generate field accessors. GenerateFieldProtobufAccessors(os); // Union accessors. @@ -1377,6 +1799,21 @@ void MessageGenerator::GenerateDeserializer(std::ostream& os, bool decl) { } os << "absl::Status " << MessageName(message_) << "::Deserialize(::phaser::ProtoBuffer &buffer) {"; + bool has_array_fields = false; + for (auto& field : fields_) { + if (UsesArrayFacade(field->field)) { + has_array_fields = true; + break; + } + } + if (has_array_fields) { + os << "\n"; + for (auto& field : fields_) { + if (UsesArrayFacade(field->field)) { + os << " " << field->member_name << ".BeginDeserialize();\n"; + } + } + } os << R"XXX( while (!buffer.Eof()) { absl::StatusOr tag = @@ -1411,6 +1848,15 @@ void MessageGenerator::GenerateDeserializer(std::ostream& os, bool decl) { } } )XXX"; + if (has_array_fields) { + os << "\n"; + for (auto& field : fields_) { + if (UsesArrayFacade(field->field)) { + os << " if (absl::Status status = " << field->member_name + << ".FinalizeDeserialize(); !status.ok()) return status;\n"; + } + } + } os << " return absl::OkStatus();\n"; os << "}\n\n"; } @@ -1546,45 +1992,153 @@ void MessageGenerator::GenerateCopy(std::ostream& os, bool decl) { os << "template \n"; os << "inline absl::Status " << MessageName(message_) << "::CloneFrom([[maybe_unused]] const T & other) {\n"; - for (auto& field : fields_) { - if (field->field->is_repeated()) { - os << " for (auto& v : other." << field->field->name() << "()) {\n"; - if (field->field->type() == - google::protobuf::FieldDescriptor::TYPE_MESSAGE) { - os << " auto* m = add_" << field->field->name() << "();\n"; - os << " if (absl::Status s = m->CloneFrom(v.Msg()); !s.ok()) return " - "s;\n"; + if (IsRosFrontend()) { + for (auto& field : fields_) { + if (field->field->is_repeated()) { + os << " " << field->member_name << ".Clear();\n"; + if (UsesArrayFacade(field->field)) { + const int array_size = GetArraySize(field->field); + if (field->field->type() == + google::protobuf::FieldDescriptor::TYPE_MESSAGE) { + os << " for (size_t i = 0; i < static_cast(" << array_size + << "); i++) {\n"; + os << " if (!other." << field->member_name << "[i].empty()) {\n"; + os << " if (absl::Status s = " << field->member_name + << "[i].Mutable()->CloneFrom(other." << field->member_name + << ".Get(i)); !s.ok()) return s;\n"; + os << " }\n"; + os << " }\n"; + } else if (field->field->type() == + google::protobuf::FieldDescriptor::TYPE_STRING || + field->field->type() == + google::protobuf::FieldDescriptor::TYPE_BYTES) { + os << " for (size_t i = 0; i < static_cast(" << array_size + << "); i++) {\n"; + os << " " << field->member_name << ".Set(i, other." + << field->member_name << ".Get(i));\n"; + os << " }\n"; + } else { + os << " for (size_t i = 0; i < static_cast(" << array_size + << "); i++) {\n"; + os << " " << field->member_name << ".Set(i, other." + << field->member_name << ".Get(i));\n"; + os << " }\n"; + } + } else if (field->field->type() == + google::protobuf::FieldDescriptor::TYPE_MESSAGE) { + os << " for (auto& v : other." << field->member_name << ") {\n"; + os << " auto* m = " << field->member_name << ".Add();\n"; + os << " if (absl::Status s = m->CloneFrom(v.Msg()); !s.ok()) return " + "s;\n"; + os << " }\n"; + } else { + os << " for (auto& v : other." << field->member_name << ") {\n"; + os << " " << field->member_name << ".Add(v);\n"; + os << " }\n"; + } + } else if (field->field->type() == + google::protobuf::FieldDescriptor::TYPE_MESSAGE) { + os << " if (other." << field->member_name << ".IsPresent()) {\n"; + if (IsRosIntrinsic(field->field)) { + os << " " << field->member_name << ".Set(other." + << field->member_name << ".Get());\n"; + } else { + os << " if (absl::Status s = " << field->member_name + << ".Mutable()->CloneFrom(other." << field->member_name + << ".Get()); !s.ok()) return s;\n"; + } + os << " } else {\n"; + os << " " << field->member_name << ".Clear();\n"; + os << " }\n"; + } else if (field->field->type() == + google::protobuf::FieldDescriptor::TYPE_STRING || + field->field->type() == + google::protobuf::FieldDescriptor::TYPE_BYTES) { + os << " if (other." << field->member_name << ".IsPresent()) {\n"; + os << " " << field->member_name << ".Set(other." + << field->member_name << ".Get());\n"; + os << " } else {\n"; + os << " " << field->member_name << ".Clear();\n"; + os << " }\n"; } else { - os << " add_" << field->field->name() << "(v);\n"; + os << " if (other." << field->member_name << ".IsPresent()) {\n"; + os << " " << field->member_name << ".Set(other." + << field->member_name << ".Get());\n"; + os << " } else {\n"; + os << " " << field->member_name << ".Clear();\n"; + os << " }\n"; } - os << " }\n"; + } + if (!unions_.empty()) { + for (auto& [oneof, u] : unions_) { + os << " switch (other." << u->member_name << ".Discriminator()) {\n"; + for (size_t i = 0; i < u->members.size(); i++) { + auto& field = u->members[i]; + os << " case " << field->field->number() << ":\n"; + if (field->field->type() == + google::protobuf::FieldDescriptor::TYPE_MESSAGE) { + os << " if (absl::Status s = " << u->member_name + << ".template CloneFrom<" << i << ">(other." << u->member_name + << ".template GetReference<" << i << ", " + << MessageName(field->field->message_type()) << ">()); !s.ok()) " + "return s;\n"; + } else { + os << " if (absl::Status s = " << u->member_name + << ".template CloneFrom<" << i << ">(other." << u->member_name + << ".template GetValue<" << i << ", " << field->c_type + << ">()); !s.ok()) return s;\n"; + } + os << " break;\n"; + } + os << " default:\n"; + for (size_t i = 0; i < u->members.size(); i++) { + os << " " << u->member_name << ".Clear<" << i << ">();\n"; + } + os << " break;\n"; + os << " }\n"; + } + } + } else { + for (auto& field : fields_) { + if (field->field->is_repeated()) { + os << " for (auto& v : other." << field->field->name() << "()) {\n"; + if (field->field->type() == + google::protobuf::FieldDescriptor::TYPE_MESSAGE) { + os << " auto* m = add_" << field->field->name() << "();\n"; + os << " if (absl::Status s = m->CloneFrom(v.Msg()); !s.ok()) return " + "s;\n"; + } else { + os << " add_" << field->field->name() << "(v);\n"; + } + os << " }\n"; - } else { - os << " if (other." << field->member_name << ".IsPresent()) {\n"; - if (field->field->type() == - google::protobuf::FieldDescriptor::TYPE_MESSAGE) { - os << " auto* m = mutable_" << field->field->name() << "();\n"; - os << " if (absl::Status s = m->CloneFrom(other." - << field->field->name() << "()); !s.ok()) return s;\n"; } else { - os << " set_" << field->field->name() << "(other." - << field->field->name() << "());\n"; + os << " if (other." << field->member_name << ".IsPresent()) {\n"; + if (field->field->type() == + google::protobuf::FieldDescriptor::TYPE_MESSAGE) { + os << " auto* m = mutable_" << field->field->name() << "();\n"; + os << " if (absl::Status s = m->CloneFrom(other." + << field->field->name() << "()); !s.ok()) return s;\n"; + } else { + os << " set_" << field->field->name() << "(other." + << field->field->name() << "());\n"; + } + os << " }\n"; } - os << " }\n"; } - } - if (!unions_.empty()) { - for (auto& [oneof, u] : unions_) { - os << " switch (other." << u->member_name << ".Discriminator()) {\n"; - for (size_t i = 0; i < u->members.size(); i++) { - auto& field = u->members[i]; - os << " case " << field->field->number() << ":\n"; - os << " if (absl::Status s = " << u->member_name - << ".template CloneFrom<" << i << ">(other." << field->field->name() - << "()); !s.ok()) return s;\n"; - os << " break;\n"; + if (!unions_.empty()) { + for (auto& [oneof, u] : unions_) { + os << " switch (other." << u->member_name << ".Discriminator()) {\n"; + for (size_t i = 0; i < u->members.size(); i++) { + auto& field = u->members[i]; + os << " case " << field->field->number() << ":\n"; + os << " if (absl::Status s = " << u->member_name + << ".template CloneFrom<" << i << ">(other." << field->field->name() + << "()); !s.ok()) return s;\n"; + os << " break;\n"; + } + os << " }\n"; } - os << " }\n"; } } os << " return absl::OkStatus();\n"; @@ -1689,7 +2243,13 @@ void MessageGenerator::GeneratePhaserBank(std::ostream& os) { os << " switch (number) {\n"; for (auto& field : fields_) { os << " case " << field->field->number() << ":\n"; - if (field->field->is_repeated()) { + if (IsRosFrontend()) { + if (field->field->is_repeated()) { + os << " return m->" << field->member_name << ".Size() > 0;\n"; + } else { + os << " return m->" << field->member_name << ".IsPresent();\n"; + } + } else if (field->field->is_repeated()) { os << " return m->" << field->field->name() << "_size() > 0;\n"; } else { os << " return m->has_" << field->field->name() << "();\n"; @@ -1699,8 +2259,13 @@ void MessageGenerator::GeneratePhaserBank(std::ostream& os) { for (size_t i = 0; i < u->members.size(); i++) { auto& field = u->members[i]; os << " case " << field->field->number() << ":\n"; - os << " return m->" << oneof->name() - << "_case() == " << field->field->number() << ";\n"; + if (IsRosFrontend()) { + os << " return m->" << u->member_name << ".Discriminator() == " + << field->field->number() << ";\n"; + } else { + os << " return m->" << oneof->name() + << "_case() == " << field->field->number() << ";\n"; + } } } os << " }\n"; diff --git a/phaser/compiler/message_gen.h b/phaser/compiler/message_gen.h index 97f8dc3..c03f19a 100644 --- a/phaser/compiler/message_gen.h +++ b/phaser/compiler/message_gen.h @@ -16,6 +16,8 @@ namespace phaser { +enum class FrontendStyle { kProtobuf, kRos }; + struct FieldInfo { // Constructor. FieldInfo(const google::protobuf::FieldDescriptor* f, uint32_t o, uint32_t i, @@ -54,15 +56,17 @@ class MessageGenerator { MessageGenerator(const google::protobuf::Descriptor* message, const std::string& added_namespace, const std::string& package_name, - bool generate_active_message = false) + bool generate_active_message = false, + FrontendStyle frontend_style = FrontendStyle::kProtobuf) : message_(message), added_namespace_(added_namespace), package_name_(package_name), - generate_active_message_(generate_active_message) { + generate_active_message_(generate_active_message), + frontend_style_(frontend_style) { for (int i = 0; i < message_->nested_type_count(); i++) { nested_message_gens_.push_back(std::make_unique( message_->nested_type(i), added_namespace, package_name, - generate_active_message)); + generate_active_message, frontend_style)); } // Enums for (int i = 0; i < message_->enum_type_count(); i++) { @@ -71,7 +75,7 @@ class MessageGenerator { } } - void GenerateHeader(std::ostream& os); + absl::Status GenerateHeader(std::ostream& os); void GenerateSource(std::ostream& os); void GenerateFieldDeclarations(std::ostream& os); @@ -93,6 +97,10 @@ class MessageGenerator { void GenerateCreators(std::ostream& os, bool decl); void GenerateClear(std::ostream& os, bool decl); + void GeneratePublicFieldDeclarations(std::ostream& os); + void GenerateRosOneofTypes(std::ostream& os); + void GenerateRosOwnerCopyMove(std::ostream& os, bool decl); + void GenerateRosSyncToPayload(std::ostream& os); void GenerateProtobufAccessors(std::ostream& os); void GenerateFieldProtobufAccessors(std::ostream& os); void GenerateFieldProtobufAccessors(std::shared_ptr field, @@ -128,9 +136,36 @@ class MessageGenerator { std::string FieldCType(const google::protobuf::FieldDescriptor* field); std::string FieldRepeatedCType( const google::protobuf::FieldDescriptor* field); + std::string FieldRepeatedVectorCType( + const google::protobuf::FieldDescriptor* field); + std::string FieldRepeatedArrayCType( + const google::protobuf::FieldDescriptor* field, int array_size); std::string FieldUnionCType(const google::protobuf::FieldDescriptor* field); uint32_t FieldBinarySize(const google::protobuf::FieldDescriptor* field); std::string FieldInfoType(const google::protobuf::FieldDescriptor* field); + std::string SanitizedIdentifier(const std::string& name) const; + std::string MemberVariableName(const std::string& proto_name) const; + std::string OneofVariantTypeName( + const google::protobuf::OneofDescriptor* oneof) const; + std::string OneofAlternativeTypeName( + const google::protobuf::FieldDescriptor* field) const; + int GetArraySize(const google::protobuf::FieldDescriptor* field) const; + bool UsesArrayFacade(const google::protobuf::FieldDescriptor* field) const; + bool IsRosTime(const google::protobuf::Descriptor* desc) const; + bool IsRosDuration(const google::protobuf::Descriptor* desc) const; + bool IsRosHeader(const google::protobuf::Descriptor* desc) const; + bool IsRosIntrinsic(const google::protobuf::FieldDescriptor* field) const; + std::string RosIntrinsicFieldType( + const google::protobuf::FieldDescriptor* field); + std::string RosIntrinsicCType( + const google::protobuf::FieldDescriptor* field); + absl::Status ValidateFieldOptions() const; + absl::Status ValidateArraySizeOption( + const google::protobuf::FieldDescriptor* field) const; + absl::Status ValidateRosHeaderDescriptor() const; + bool IsRosFrontend() const { + return frontend_style_ == FrontendStyle::kRos; + } const google::protobuf::Descriptor* message_; std::vector> nested_message_gens_; @@ -144,6 +179,7 @@ class MessageGenerator { std::string added_namespace_; std::string package_name_; bool generate_active_message_ = false; + FrontendStyle frontend_style_ = FrontendStyle::kProtobuf; }; } // namespace phaser diff --git a/phaser/docs/phaser_user_guide.md b/phaser/docs/phaser_user_guide.md index 1abbe31..8683ec6 100644 --- a/phaser/docs/phaser_user_guide.md +++ b/phaser/docs/phaser_user_guide.md @@ -221,6 +221,169 @@ The actual command used by Bazel to build the Phaser files is: I haven't done this with `cmake` or any other build system but I guess it will be reasonably easy. +## Protobuf and ROS frontends +`phaser_library` supports two generated C++ interfaces over the same Phaser +payload layout: + +- `frontend = "protobuf"` (the default) generates protobuf-style getters and + setters. +- `frontend = "ros"` generates a struct with public field proxies. + +```python +phaser_library( + name = "foo_ros_phaser", + frontend = "ros", + deps = [":foo_proto"], +) +``` + +When invoking the plugin directly, pass `frontend=ros` in `--phaser_out`. +Unknown frontend values are rejected. + +### ROS field syntax +Scalar, enum, string, message, and repeated fields provide value-like syntax, +but remain handles into the message's `PayloadBuffer`: + +```c++ +Foo msg; +msg.count = 10; +int count = msg.count; + +msg.name = "camera"; +std::string_view name = msg.name; + +msg.child->id = 7; // materializes child when needed +msg.values.push_back(1); +msg.values[0] = 2; +for (int value : msg.values) { /* ... */ } +``` + +Assigning one field proxy to another copies the field value; it does not rebind +the destination proxy to the source message. Copying a generated ROS message +deep-copies its payload. Iterators, references, pointers, and string views into +a dynamic message can be invalidated by an operation that grows or compacts its +payload buffer, so reacquire them after mutation. + +`CreateReadonly` messages support const field access and serialization. A +non-const proxy operation requires a mutable message. + +### ROS1 intrinsic types +The ROS frontend recognizes three singular protobuf message declarations and +presents their fields as the corresponding ROS1 C++ types: + +- `google.protobuf.Timestamp` as `ros::Time` +- `google.protobuf.Duration` as `ros::Duration` +- `std_msgs.Header` as `std_msgs::Header` + +`std_msgs.Header` is expected to contain `uint32 seq`, a +`google.protobuf.Timestamp stamp`, and `string frame_id`. Generate it with a +non-empty `add_namespace` so the Phaser backend type does not collide with the +real ROS class. + +The proxies support values and both const and mutable references, so existing +functions can be called without adapters: + +```c++ +void Advance(ros::Time& stamp); +void UpdateHeader(std_msgs::Header& header); +void Observe(const std_msgs::Header& header); + +Advance(msg.stamp); +UpdateHeader(msg.header); +Observe(msg.header); +``` + +`ros::Time`, `ros::Duration`, and especially `std_msgs::Header` are cached +source objects rather than in-buffer overlays. Mutable-reference changes are +automatically copied into the payload before native `Data()`/size access, +protobuf serialization, copying, and recursive parent synchronization. Avoid +accessing `runtime->pb` directly while such a mutable borrow may be dirty. + +Pass the ROS libraries needed by generated headers through `cc_deps`: + +```python +phaser_library( + name = "messages_phaser", + add_namespace = "phaser", + frontend = "ros", + deps = [":messages_proto"], + cc_deps = [ + "@ros//:roscpp", + "@ros//std_msgs", + ], +) +``` + +Intrinsic fields are currently required to be singular and outside a `oneof`; +the generator rejects unsupported repeated or union forms rather than silently +emitting a non-ROS-compatible type. + +### Fixed arrays +Fixed arrays use a repeated protobuf field plus the `phaser.array_size` option: + +```proto +import "phaser/options.proto"; + +message Scan { + repeated float ranges = 1 [(phaser.array_size) = 360]; + repeated string frame_names = 2 [(phaser.array_size) = 2]; +} +``` + +The option is used as a fixed-array facade only by the ROS frontend. The field +has `size() == N`, indexing, front/back, and iteration, while its native payload +storage remains the same vector-backed representation as a repeated field. +During protobuf wire parsing, fewer than `N` elements are default-filled and +more than `N` elements are rejected. `array_size` must be positive and may only +annotate a non-map repeated field. + +The proto target must depend on Phaser's options proto: + +```python +proto_library( + name = "scan_proto", + srcs = ["Scan.proto"], + deps = ["@phaser//phaser:options_proto"], +) +``` + +### Variant-like oneofs +In the ROS frontend, each `oneof` is a public variant-like proxy. Phaser +generates a named tag for every arm: + +```proto +oneof command { + int32 sequence = 3; + string frame = 4; + Child child = 5; +} +``` + +```c++ +using Sequence = Foo::SequenceAlternative; +using Frame = Foo::FrameAlternative; +using ChildArm = Foo::ChildAlternative; + +msg.command.emplace(12); +if (msg.command.holds_alternative()) { + int sequence = msg.command.get(); +} + +msg.command.emplace("map"); // clears the sequence arm +msg.command.emplace().id = 9; // returns the mutable child +msg.command.reset(); +``` + +`index()` returns the zero-based arm index or `std::variant_npos`; +`case_number()` returns the active protobuf field number or zero. `get()` +throws `std::bad_variant_access` for an inactive arm. Switching arms clears and +releases any string or message storage owned by the previous arm. + +Both frontends use the same native binary metadata and protobuf wire +serialization. A protobuf-style target and a ROS-style target generated from +the same schema can therefore exchange native Phaser buffers and protobuf wire +bytes. + ## Creating a message In protobuf, you generally create messages on the local stack frame or from the heap. Submessages (fields whose type is a message) are allocated from the heap. diff --git a/phaser/options.proto b/phaser/options.proto new file mode 100644 index 0000000..63938ec --- /dev/null +++ b/phaser/options.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package phaser; + +import "google/protobuf/descriptor.proto"; + +extend google.protobuf.FieldOptions { + // Fixed logical extent for a repeated field. Must be a positive integer. + // Only valid on non-map repeated fields. ROS frontend generation exposes a + // fixed-extent array facade; protobuf frontend keeps repeated accessors. + uint32 array_size = 50001; +} diff --git a/phaser/phaser_library.bzl b/phaser/phaser_library.bzl index 4b16d44..64e8d29 100644 --- a/phaser/phaser_library.bzl +++ b/phaser/phaser_library.bzl @@ -21,7 +21,9 @@ def _phaser_action( package_name, outputs, add_namespace, - target_name): + target_name, + frontend, + enable_active_message): # The protobuf compiler allow plugins to get arguments specified in the --plugin_out # argument. The args are passed as a comma separated list of key=value pairs followed # by a colon and the output directory. @@ -30,7 +32,8 @@ def _phaser_action( options.append("add_namespace={}".format(add_namespace)) options.append("package_name={}".format(package_name)) options.append("target_name={}".format(target_name)) - if ctx.attr.enable_active_message: + options.append("frontend={}".format(frontend)) + if enable_active_message: options.append("active_message=true") options_and_out_dir = "--phaser_out={}:{}".format(",".join(options), out_dir) @@ -89,6 +92,14 @@ def _proto_output_base(source_file): file_path = v[1].split("/", 1)[1] return file_path +def _skip_phaser_generation(source_file): + base = _proto_output_base(source_file) + if base == "google/protobuf/descriptor.proto": + return True + if base == "phaser/options.proto": + return True + return False + def _phaser_aspect_impl(target, _ctx): direct_sources = [] transitive_sources = depset() @@ -106,6 +117,8 @@ def _phaser_aspect_impl(target, _ctx): transitive_sources = target[ProtoInfo].transitive_sources direct_paths = {s.path: True for s in _to_list(target[ProtoInfo].direct_sources)} for s in _to_list(transitive_sources): + if _skip_phaser_generation(s): + continue direct_sources.append(s) add_output(_proto_output_base(s), s.path in direct_paths) @@ -125,6 +138,10 @@ phaser_aspect = aspect( # The phaser rule runs the Phaser plugin from the protoc compiler. # The deps for the rule are proto_libraries that contain the protobuf files. def _phaser_impl(ctx): + frontend = ctx.attr.frontend + if frontend not in ("protobuf", "ros"): + fail("phaser_library frontend must be 'protobuf' or 'ros', got: {}".format(frontend)) + outputs = [] direct_sources = [] @@ -172,6 +189,8 @@ def _phaser_impl(ctx): cpp_outputs, ctx.attr.add_namespace, ctx.attr.target_name, + frontend, + ctx.attr.enable_active_message, ) return [DefaultInfo(files = depset(outputs))] @@ -194,6 +213,7 @@ _phaser_gen = rule( "add_namespace": attr.string(), "package_name": attr.string(), "target_name": attr.string(), + "frontend": attr.string(default = "protobuf"), "enable_active_message": attr.bool(default = False), }, implementation = _phaser_impl, @@ -215,7 +235,7 @@ _split_files = rule( implementation = _split_files_impl, ) -def phaser_library(name, deps = [], runtime = "@phaser//phaser/runtime:phaser_runtime", add_namespace = "", enable_active_message = False): +def phaser_library(name, deps = [], runtime = "@phaser//phaser/runtime:phaser_runtime", add_namespace = "", enable_active_message = False, frontend = "protobuf", cc_deps = []): """ Generate a cc_libary for protobuf files specified in deps. @@ -228,7 +248,13 @@ def phaser_library(name, deps = [], runtime = "@phaser//phaser/runtime:phaser_ru enable_active_message: if True, generated message types get a public `std::any active_message` field (also enableable via the `active_message=true` plugin command-line option). + frontend: generated C++ API style, either "protobuf" (default) or "ros". + cc_deps: additional C++ dependencies required by generated headers, + such as ROS1 message/runtime libraries for intrinsic ROS fields. """ + if frontend not in ("protobuf", "ros"): + fail("phaser_library frontend must be 'protobuf' or 'ros', got: {}".format(frontend)) + phaser = name + "_phaser" _phaser_gen( @@ -238,6 +264,7 @@ def phaser_library(name, deps = [], runtime = "@phaser//phaser/runtime:phaser_ru package_name = native.package_name(), target_name = name, enable_active_message = enable_active_message, + frontend = frontend, ) srcs = name + "_srcs" @@ -261,6 +288,7 @@ def phaser_library(name, deps = [], runtime = "@phaser//phaser/runtime:phaser_ru if runtime != "": libdeps = libdeps + [runtime] + libdeps = libdeps + cc_deps cc_library( name = name, diff --git a/phaser/ros_compile_test.cc b/phaser/ros_compile_test.cc new file mode 100644 index 0000000..de39c98 --- /dev/null +++ b/phaser/ros_compile_test.cc @@ -0,0 +1,666 @@ +// Compile and runtime fixture for frontend=ros generated messages. +#include "phaser/testdata/RosCompile.phaser.h" +#include "phaser/testdata/RosCompile.pb.h" + +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" + +namespace foo::bar::phaser { +namespace { + +TEST(RosCompileTest, ScalarConversionAndAssignment) { + RosCompileMessage msg; + msg.x = 42; + msg.flag = true; + msg.value = 3.5; + msg.color = RosColor::ROS_COLOR_RED; + + EXPECT_TRUE(msg.x.IsPresent()); + EXPECT_TRUE(msg.flag.IsPresent()); + EXPECT_TRUE(msg.value.IsPresent()); + EXPECT_TRUE(msg.color.IsPresent()); + + int32_t x = msg.x; + bool flag = msg.flag; + double value = msg.value; + RosColor color = msg.color; + EXPECT_EQ(x, 42); + EXPECT_TRUE(flag); + EXPECT_DOUBLE_EQ(value, 3.5); + EXPECT_EQ(color, RosColor::ROS_COLOR_RED); + + msg.x = 7; + msg.flag = false; + msg.color = RosColor::ROS_COLOR_BLUE; + EXPECT_EQ(msg.x.Get(), 7); + EXPECT_FALSE(static_cast(msg.flag)); + EXPECT_EQ(msg.color.Get(), RosColor::ROS_COLOR_BLUE); +} + +TEST(RosCompileTest, StringConversionAndAssignment) { + RosCompileMessage msg; + msg.name = "hello"; + EXPECT_TRUE(msg.name.IsPresent()); + EXPECT_EQ(std::string_view(msg.name), "hello"); + + msg.name = std::string("world"); + EXPECT_EQ(msg.name.Get(), "world"); + + const char* cstr = "from_cstr"; + msg.name = cstr; + EXPECT_EQ(msg.name.Get(), "from_cstr"); + + RosCompileMessage other; + other.name = "proxy source"; + msg.name = other.name; + EXPECT_EQ(msg.name.Get(), "proxy source"); + EXPECT_TRUE(other.name.IsPresent()); +} + +TEST(RosCompileTest, IndirectMessageAccess) { + RosCompileMessage msg; + EXPECT_FALSE(msg.inner.IsPresent()); + + const RosCompileMessage& cmsg = msg; + EXPECT_FALSE(cmsg.inner.IsPresent()); + EXPECT_EQ(cmsg.inner->id.Get(), 0); + + msg.inner->id = 99; + EXPECT_TRUE(msg.inner.IsPresent()); + EXPECT_EQ(msg.inner->id.Get(), 99); + EXPECT_EQ((*msg.inner).id.Get(), 99); + + RosInner& inner_ref = msg.inner; + inner_ref.id = 100; + EXPECT_EQ(msg.inner->id.Get(), 100); + + const RosInner& inner_cref = cmsg.inner; + EXPECT_EQ(inner_cref.id.Get(), 100); +} + +TEST(RosCompileTest, PrimitiveVectorSyntax) { + RosCompileMessage msg; + msg.xs.push_back(1); + msg.xs.push_back(2); + msg.xs.reserve(8); + EXPECT_EQ(msg.xs.size(), 2u); + EXPECT_EQ(msg.xs[0], 1); + EXPECT_EQ(msg.xs[1], 2); + + msg.xs[0] = 10; + EXPECT_EQ(msg.xs.front(), 10); + EXPECT_EQ(msg.xs.back(), 2); + + msg.xs.resize(4); + EXPECT_EQ(msg.xs.size(), 4u); + msg.xs[3] = 40; + EXPECT_EQ(msg.xs[3], 40); + + std::vector seen; + for (int32_t v : msg.xs) { + seen.push_back(v); + } + EXPECT_EQ(seen.size(), 4u); + EXPECT_EQ(seen[0], 10); + EXPECT_EQ(seen[3], 40); + + msg.xs.clear(); + EXPECT_TRUE(msg.xs.empty()); +} + +TEST(RosCompileTest, EnumVectorSyntax) { + RosCompileMessage msg; + msg.colors.push_back(RosColor::ROS_COLOR_RED); + msg.colors.push_back(RosColor::ROS_COLOR_BLUE); + EXPECT_EQ(msg.colors.size(), 2u); + EXPECT_EQ(msg.colors[0], RosColor::ROS_COLOR_RED); + msg.colors[1] = RosColor::ROS_COLOR_UNSPECIFIED; + EXPECT_EQ(msg.colors[1], RosColor::ROS_COLOR_UNSPECIFIED); +} + +TEST(RosCompileTest, StringVectorSyntax) { + RosCompileMessage msg; + msg.names.push_back("a"); + msg.names.push_back("b"); + EXPECT_EQ(msg.names.size(), 2u); + EXPECT_EQ(std::string_view(msg.names[0]), "a"); + + msg.names[1] = "beta"; + EXPECT_EQ(msg.names[1].Get(), "beta"); + + msg.names[0] = msg.names[1]; + EXPECT_EQ(msg.names[0].Get(), "beta"); + + size_t count = 0; + for (const auto& s : msg.names) { + EXPECT_FALSE(s.Get().empty()); + ++count; + } + EXPECT_EQ(count, 2u); +} + +TEST(RosCompileTest, MessageVectorSyntax) { + RosCompileMessage msg; + RosInner* a = msg.inners.Add(); + a->id = 1; + RosInner* b = msg.inners.Add(); + b->id = 2; + EXPECT_EQ(msg.inners.size(), 2u); + EXPECT_EQ(msg.inners[0]->id.Get(), 1); + EXPECT_EQ(msg.inners[1]->id.Get(), 2); + + msg.inners[0]->id = 11; + EXPECT_EQ(msg.inners.front()->id.Get(), 11); + + size_t count = 0; + for (auto& elem : msg.inners) { + EXPECT_TRUE(elem->id.IsPresent()); + ++count; + } + EXPECT_EQ(count, 2u); +} + +TEST(RosCompileTest, ProxyAssignmentDoesNotRebind) { + RosCompileMessage dst; + RosCompileMessage src; + src.x = 5; + src.name = "src"; + src.inner->id = 77; + + dst.x = src.x; + dst.name = src.name; + dst.inner = src.inner; + + EXPECT_EQ(dst.x.Get(), 5); + EXPECT_EQ(dst.name.Get(), "src"); + EXPECT_EQ(dst.inner->id.Get(), 77); + + src.x = 999; + src.name = "mutated"; + src.inner->id = 0; + EXPECT_EQ(dst.x.Get(), 5); + EXPECT_EQ(dst.name.Get(), "src"); + EXPECT_EQ(dst.inner->id.Get(), 77); +} + +TEST(RosCompileTest, ProxyMoveAssignIsValueSemantics) { + RosCompileMessage dst; + RosCompileMessage src; + src.x = 5; + src.name = "move-me"; + src.inner->id = 12; + + dst.x = std::move(src.x); + dst.name = std::move(src.name); + dst.inner = std::move(src.inner); + + EXPECT_EQ(dst.x.Get(), 5); + EXPECT_EQ(dst.name.Get(), "move-me"); + EXPECT_EQ(dst.inner->id.Get(), 12); + + // Source proxies keep their owner binding and values (no structural rebind). + EXPECT_EQ(src.x.Get(), 5); + EXPECT_EQ(src.name.Get(), "move-me"); + EXPECT_EQ(src.inner->id.Get(), 12); + + src.x = 999; + EXPECT_EQ(dst.x.Get(), 5); +} + + +TEST(RosCompileTest, MessageCopyAssignUsesCloneFrom) { + RosCompileMessage src; + src.x = 11; + src.name = "copy"; + src.xs.push_back(3); + + RosCompileMessage dst; + dst = src; + + EXPECT_EQ(dst.x.Get(), 11); + EXPECT_EQ(dst.name.Get(), "copy"); + ASSERT_EQ(dst.xs.size(), 1u); + EXPECT_EQ(dst.xs[0], 3); + + EXPECT_NE(dst.runtime.get(), src.runtime.get()); + src.x = 0; + EXPECT_EQ(dst.x.Get(), 11); +} + +TEST(RosCompileTest, MessageMoveAssignUsesCloneFrom) { + RosCompileMessage src; + src.x = 21; + src.name = "moved"; + src.xs.push_back(9); + + RosCompileMessage dst; + dst = std::move(src); + + EXPECT_EQ(dst.x.Get(), 21); + EXPECT_EQ(dst.name.Get(), "moved"); + ASSERT_EQ(dst.xs.size(), 1u); + EXPECT_EQ(dst.xs[0], 9); + EXPECT_FALSE(src.x.IsPresent()); + EXPECT_FALSE(src.name.IsPresent()); + EXPECT_TRUE(src.xs.empty()); +} + +TEST(RosCompileTest, MessageCopyCtorDeepCopies) { + RosCompileMessage src; + src.x = 31; + src.names.push_back("deep"); + + RosCompileMessage copy(src); + EXPECT_EQ(copy.x.Get(), 31); + ASSERT_EQ(copy.names.size(), 1u); + EXPECT_EQ(copy.names[0].Get(), "deep"); + EXPECT_NE(copy.runtime.get(), src.runtime.get()); + + src.x = 0; + EXPECT_EQ(copy.x.Get(), 31); +} + +TEST(RosCompileTest, MessageMoveCtorRebindsStringCaches) { + auto source = std::make_unique(); + source->names.push_back("vector"); + source->fixed_names[0] = "array"; + // Populate both source-side caches before moving the owner. + EXPECT_EQ(source->names[0].Get(), "vector"); + EXPECT_EQ(source->fixed_names[0].Get(), "array"); + + RosCompileMessage moved(std::move(*source)); + source.reset(); + + EXPECT_EQ(moved.names[0].Get(), "vector"); + EXPECT_EQ(moved.fixed_names[0].Get(), "array"); + moved.names[0] = "updated"; + moved.fixed_names[0] = "updated-array"; + EXPECT_EQ(moved.names[0].Get(), "updated"); + EXPECT_EQ(moved.fixed_names[0].Get(), "updated-array"); +} + +TEST(RosCompileTest, EnumVectorDataAccessor) { + RosCompileMessage msg; + msg.colors.push_back(RosColor::ROS_COLOR_RED); + ASSERT_NE(msg.colors.data(), nullptr); + EXPECT_EQ(msg.colors.data()[0], RosColor::ROS_COLOR_RED); + + const RosCompileMessage& cmsg = msg; + ASSERT_NE(cmsg.colors.data(), nullptr); + EXPECT_EQ(cmsg.colors.data()[0], RosColor::ROS_COLOR_RED); +} + +TEST(RosCompileTest, CloneFromCopyAndBufferGrowth) { + RosCompileMessage src; + src.x = 1; + src.name = "grow"; + src.xs.reserve(64); + for (int i = 0; i < 32; i++) { + src.xs.push_back(i); + src.names.push_back(std::to_string(i)); + } + src.inner->id = 42; + EXPECT_TRUE(src.fixed_names.Get(0).empty()); + + RosCompileMessage copy; + ASSERT_TRUE(copy.CloneFrom(src).ok()); + EXPECT_EQ(copy.x.Get(), 1); + EXPECT_EQ(copy.name.Get(), "grow"); + EXPECT_EQ(copy.xs.size(), 32u); + EXPECT_EQ(copy.names.size(), 32u); + EXPECT_EQ(copy.inner->id.Get(), 42); + + RosCompileMessage moved_from; + moved_from.x = 99; + moved_from.name = "move"; + moved_from.xs = src.xs; + moved_from.names = src.names; + moved_from.inner = src.inner; + + RosCompileMessage move_dst; + ASSERT_TRUE(move_dst.CloneFrom(moved_from).ok()); + EXPECT_EQ(move_dst.x.Get(), 99); + EXPECT_EQ(move_dst.names.size(), 32u); + EXPECT_EQ(move_dst.inner->id.Get(), 42); + + copy.xs.reserve(128); + for (int i = 0; i < 64; i++) { + copy.xs.push_back(1000 + i); + } + EXPECT_EQ(copy.xs.size(), 96u); + EXPECT_EQ(copy.xs[95], 1000 + 63); +} + +TEST(RosCompileTest, FixedPrimitiveArrayExtentAndMutation) { + RosCompileMessage msg; + EXPECT_EQ(msg.fixed_ints.size(), 4u); + EXPECT_EQ(msg.fixed_ints.max_size(), 4u); + + msg.fixed_ints[0] = 10; + msg.fixed_ints[1] = 20; + msg.fixed_ints[3] = 40; + EXPECT_EQ(msg.fixed_ints.front(), 10); + EXPECT_EQ(msg.fixed_ints.back(), 40); + EXPECT_EQ(msg.fixed_ints[2], 0); + + std::vector seen; + for (int32_t v : msg.fixed_ints) { + seen.push_back(v); + } + ASSERT_EQ(seen.size(), 4u); + EXPECT_EQ(seen[1], 20); +} + +TEST(RosCompileTest, FixedEnumArrayExtent) { + RosCompileMessage msg; + EXPECT_EQ(msg.fixed_colors.size(), 3u); + msg.fixed_colors[0] = RosColor::ROS_COLOR_RED; + msg.fixed_colors[2] = RosColor::ROS_COLOR_BLUE; + EXPECT_EQ(msg.fixed_colors[0], RosColor::ROS_COLOR_RED); + EXPECT_EQ(msg.fixed_colors[1], RosColor::ROS_COLOR_UNSPECIFIED); + EXPECT_EQ(msg.fixed_colors[2], RosColor::ROS_COLOR_BLUE); +} + +TEST(RosCompileTest, FixedStringArrayExtentAndAssignment) { + RosCompileMessage msg; + EXPECT_EQ(msg.fixed_names.size(), 2u); + msg.fixed_names[0] = "alpha"; + msg.fixed_names[1] = "beta"; + EXPECT_EQ(msg.fixed_names[0].Get(), "alpha"); + EXPECT_EQ(std::string_view(msg.fixed_names[1]), "beta"); + + msg.fixed_names[0] = msg.fixed_names[1]; + EXPECT_EQ(msg.fixed_names[0].Get(), "beta"); +} + +TEST(RosCompileTest, UntouchedFixedStringArrayReadsDefault) { + const RosCompileMessage msg; + EXPECT_TRUE(msg.fixed_names.Get(0).empty()); + EXPECT_TRUE(msg.fixed_names.Get(1).empty()); +} + +TEST(RosCompileTest, FixedMessageArrayExtent) { + RosCompileMessage msg; + EXPECT_EQ(msg.fixed_inners.size(), 2u); + msg.fixed_inners[0]->id = 7; + msg.fixed_inners[1]->id = 8; + EXPECT_EQ(msg.fixed_inners[0]->id.Get(), 7); + EXPECT_EQ(msg.fixed_inners[1]->id.Get(), 8); + + size_t count = 0; + for (auto& inner : msg.fixed_inners) { + EXPECT_TRUE(inner->id.IsPresent()); + ++count; + } + EXPECT_EQ(count, 2u); +} + +TEST(RosCompileTest, FixedArrayClearRestoresLogicalDefaults) { + RosCompileMessage msg; + msg.fixed_ints[0] = 99; + msg.fixed_ints[3] = 42; + msg.fixed_names[0] = "keep"; + msg.fixed_inners[1]->id = 5; + + msg.fixed_ints.clear(); + msg.fixed_names.clear(); + msg.fixed_inners.clear(); + + EXPECT_EQ(msg.fixed_ints.size(), 4u); + EXPECT_EQ(msg.fixed_ints[0], 0); + EXPECT_EQ(msg.fixed_ints[3], 0); + EXPECT_EQ(msg.fixed_names.size(), 2u); + EXPECT_TRUE(msg.fixed_names[0].Get().empty()); + EXPECT_EQ(msg.fixed_inners.size(), 2u); + EXPECT_FALSE(msg.fixed_inners[1]->id.IsPresent()); +} + +TEST(RosCompileTest, FixedArrayWireRoundtrip) { + RosCompileMessage msg; + msg.fixed_ints[0] = 1; + msg.fixed_ints[2] = 3; + msg.fixed_colors[1] = RosColor::ROS_COLOR_RED; + msg.fixed_names[0] = "wire"; + msg.fixed_inners[0]->id = 55; + + std::string wire = msg.SerializeAsString(); + ASSERT_FALSE(wire.empty()); + + RosCompileMessage parsed; + ASSERT_TRUE(parsed.ParseFromString(wire)); + EXPECT_EQ(parsed.fixed_ints.size(), 4u); + EXPECT_EQ(parsed.fixed_ints[0], 1); + EXPECT_EQ(parsed.fixed_ints[2], 3); + EXPECT_EQ(parsed.fixed_colors[1], RosColor::ROS_COLOR_RED); + EXPECT_EQ(parsed.fixed_names[0].Get(), "wire"); + EXPECT_EQ(parsed.fixed_inners[0]->id.Get(), 55); +} + +TEST(RosCompileTest, FixedArrayPartialWireDefaultFillsExtent) { + RosCompileMessage msg; + msg.fixed_ints[0] = 100; + std::string wire = msg.SerializeAsString(); + + RosCompileMessage parsed; + ASSERT_TRUE(parsed.ParseFromString(wire)); + EXPECT_EQ(parsed.fixed_ints.size(), 4u); + EXPECT_EQ(parsed.fixed_ints[0], 100); + EXPECT_EQ(parsed.fixed_ints[1], 0); + EXPECT_EQ(parsed.fixed_ints[3], 0); +} + +TEST(RosCompileTest, FixedArrayOverflowRejected) { + RosCompileMessage msg; + msg.fixed_ints[0] = 1; + msg.fixed_ints[1] = 2; + msg.fixed_ints[2] = 3; + msg.fixed_ints[3] = 4; + std::string good = msg.SerializeAsString(); + + // Append another packed fixed_ints field (tag 11, wire type 2) with one int. + std::string overflow = good; + overflow.push_back(static_cast(0x5A)); // (11 << 3) | 2 + overflow.push_back(static_cast(0x04)); // length 4 + overflow.push_back(static_cast(0x05)); + overflow.push_back(static_cast(0x00)); + overflow.push_back(static_cast(0x00)); + overflow.push_back(static_cast(0x00)); + + RosCompileMessage parsed; + EXPECT_FALSE(parsed.ParseFromString(overflow)); +} + +TEST(RosCompileTest, FixedArrayConstViewAfterWireParse) { + RosCompileMessage msg; + msg.fixed_ints[1] = 22; + msg.fixed_names[0] = "readonly"; + std::string wire = msg.SerializeAsString(); + + RosCompileMessage parsed; + ASSERT_TRUE(parsed.ParseFromString(wire)); + + const RosCompileMessage& view = parsed; + EXPECT_EQ(view.fixed_ints.size(), 4u); + EXPECT_EQ(view.fixed_ints[1], 22); + EXPECT_EQ(view.fixed_names[0].Get(), "readonly"); +} + +TEST(RosCompileTest, FixedArrayCreateReadonlyConstAccess) { + RosCompileMessage msg; + msg.fixed_ints[0] = 10; + msg.fixed_ints[2] = 30; + msg.fixed_names[1] = "ro"; + msg.fixed_inners[0]->id = 4; + + std::vector buffer(msg.ByteSizeLong()); + std::memcpy(buffer.data(), msg.Data(), buffer.size()); + + RosCompileMessage ro = + RosCompileMessage::CreateReadonly(buffer.data(), buffer.size()); + const RosCompileMessage& view = ro; + + EXPECT_EQ(view.fixed_ints.size(), 4u); + EXPECT_EQ(view.fixed_ints[0], 10); + EXPECT_EQ(view.fixed_ints[1], 0); + EXPECT_EQ(view.fixed_ints[2], 30); + EXPECT_EQ(view.fixed_names[0].Get(), ""); + EXPECT_EQ(view.fixed_names[1].Get(), "ro"); + EXPECT_EQ(view.fixed_inners[0]->id.Get(), 4); + EXPECT_FALSE(view.fixed_inners[1]->id.IsPresent()); + + std::string wire = view.SerializeAsString(); + EXPECT_FALSE(wire.empty()); + + RosCompileMessage reparsed; + ASSERT_TRUE(reparsed.ParseFromString(wire)); + EXPECT_EQ(reparsed.fixed_ints[0], 10); + EXPECT_EQ(reparsed.fixed_ints[2], 30); + EXPECT_EQ(reparsed.fixed_names[1].Get(), "ro"); + EXPECT_EQ(reparsed.fixed_inners[0]->id.Get(), 4); +} + +TEST(RosCompileTest, FixedArrayReadonlyShortBufferDefaults) { + RosCompileMessage msg; + msg.fixed_ints[0] = 7; + std::string partial = msg.SerializeAsString(); + + std::vector buffer(msg.ByteSizeLong()); + std::memcpy(buffer.data(), msg.Data(), buffer.size()); + + RosCompileMessage ro = + RosCompileMessage::CreateReadonly(buffer.data(), buffer.size()); + const RosCompileMessage& view = ro; + + EXPECT_EQ(view.fixed_ints[0], 7); + EXPECT_EQ(view.fixed_ints[1], 0); + EXPECT_EQ(view.fixed_ints[3], 0); + EXPECT_TRUE(view.fixed_names[0].Get().empty()); + (void)partial; +} + +TEST(RosCompileTest, FixedArrayCloneFromCopiesExtent) { + RosCompileMessage src; + src.fixed_ints[0] = 9; + src.fixed_names[1] = "clone"; + src.fixed_inners[0]->id = 3; + + RosCompileMessage dst; + ASSERT_TRUE(dst.CloneFrom(src).ok()); + EXPECT_EQ(dst.fixed_ints.size(), 4u); + EXPECT_EQ(dst.fixed_ints[0], 9); + EXPECT_EQ(dst.fixed_names[1].Get(), "clone"); + EXPECT_EQ(dst.fixed_inners[0]->id.Get(), 3); +} + +TEST(RosCompileTest, VariantOneofNamedAlternatives) { + using Count = RosCompileMessage::ChoiceCountAlternative; + using Code = RosCompileMessage::ChoiceCodeAlternative; + using Name = RosCompileMessage::ChoiceNameAlternative; + using Inner = RosCompileMessage::ChoiceInnerAlternative; + + RosCompileMessage msg; + EXPECT_EQ(msg.choice.index(), std::variant_npos); + EXPECT_TRUE(msg.choice.valueless_by_exception()); + + EXPECT_EQ(msg.choice.emplace(42), 42); + EXPECT_EQ(msg.choice.index(), 0u); + EXPECT_EQ(msg.choice.case_number(), 15); + EXPECT_TRUE(msg.choice.holds_alternative()); + EXPECT_FALSE(msg.choice.holds_alternative()); + EXPECT_EQ(msg.choice.get(), 42); + EXPECT_THROW((void)msg.choice.get(), std::bad_variant_access); + + EXPECT_EQ(msg.choice.emplace(7), 7); + EXPECT_FALSE(msg.choice.holds_alternative()); + EXPECT_TRUE(msg.choice.holds_alternative()); + + EXPECT_EQ(msg.choice.emplace("laser"), "laser"); + EXPECT_TRUE(msg.choice.holds_alternative()); + EXPECT_EQ(msg.choice.get(), "laser"); + + RosInner& inner = msg.choice.emplace(); + inner.id = 99; + EXPECT_TRUE(msg.choice.holds_alternative()); + EXPECT_EQ(msg.choice.get().id.Get(), 99); + + msg.choice.reset(); + EXPECT_EQ(msg.choice.index(), std::variant_npos); + EXPECT_EQ(msg.choice.case_number(), 0); +} + +TEST(RosCompileTest, VariantOneofSwitchingCleansVariableArms) { + using Count = RosCompileMessage::ChoiceCountAlternative; + using Name = RosCompileMessage::ChoiceNameAlternative; + using Inner = RosCompileMessage::ChoiceInnerAlternative; + + RosCompileMessage msg(256, ::phaser::Tuning::kSize); + for (int i = 0; i < 100; ++i) { + msg.choice.emplace(std::string(128, static_cast('a' + i % 26))); + EXPECT_EQ(msg.choice.get().size(), 128u); + msg.choice.emplace().id = i; + EXPECT_EQ(msg.choice.get().id.Get(), i); + msg.choice.emplace(i); + EXPECT_EQ(msg.choice.get(), i); + } + msg.choice.reset(); + EXPECT_EQ(msg.choice.case_number(), 0); +} + +TEST(RosCompileTest, VariantOneofWireRoundtrip) { + using Name = RosCompileMessage::ChoiceNameAlternative; + using Inner = RosCompileMessage::ChoiceInnerAlternative; + + RosCompileMessage source; + source.choice.emplace("wire"); + std::string wire = source.SerializeAsString(); + + RosCompileMessage parsed; + ASSERT_TRUE(parsed.ParseFromString(wire)); + ASSERT_TRUE(parsed.choice.holds_alternative()); + EXPECT_EQ(parsed.choice.get(), "wire"); + + source.choice.emplace().id = 123; + wire = source.SerializeAsString(); + ASSERT_TRUE(parsed.ParseFromString(wire)); + ASSERT_TRUE(parsed.choice.holds_alternative()); + EXPECT_EQ(parsed.choice.get().id.Get(), 123); +} + +TEST(RosCompileTest, StandardProtobufWireCompatibility) { + using Name = RosCompileMessage::ChoiceNameAlternative; + + RosCompileMessage ros; + ros.x = 41; + ros.name = "phaser"; + ros.fixed_ints[0] = 5; + ros.fixed_ints[3] = 8; + ros.choice.emplace("map"); + + ::foo::bar::RosCompileMessage protobuf; + ASSERT_TRUE(protobuf.ParseFromString(ros.SerializeAsString())); + EXPECT_EQ(protobuf.x(), 41); + EXPECT_EQ(protobuf.name(), "phaser"); + ASSERT_EQ(protobuf.fixed_ints_size(), 4); + EXPECT_EQ(protobuf.fixed_ints(0), 5); + EXPECT_EQ(protobuf.fixed_ints(3), 8); + EXPECT_EQ(protobuf.choice_name(), "map"); + + protobuf.set_x(73); + protobuf.set_name("protobuf"); + protobuf.set_choice_name("odom"); + + RosCompileMessage parsed; + ASSERT_TRUE(parsed.ParseFromString(protobuf.SerializeAsString())); + EXPECT_EQ(parsed.x.Get(), 73); + EXPECT_EQ(parsed.name.Get(), "protobuf"); + ASSERT_TRUE(parsed.choice.holds_alternative()); + EXPECT_EQ(parsed.choice.get(), "odom"); +} + +} // namespace +} // namespace foo::bar::phaser diff --git a/phaser/ros_intrinsics_test.cc b/phaser/ros_intrinsics_test.cc new file mode 100644 index 0000000..5441ad3 --- /dev/null +++ b/phaser/ros_intrinsics_test.cc @@ -0,0 +1,109 @@ +#include "phaser/testdata/RosIntrinsics.phaser.h" + +#include +#include +#include + +#include "gtest/gtest.h" + +namespace foo::bar::phaser { +namespace { + +void MutateTime(::ros::Time& value) { + value.sec = 12; + value.nsec = 345; +} + +void MutateDuration(::ros::Duration& value) { + value.sec = -4; + value.nsec = 500; +} + +void MutateHeader(::std_msgs::Header& value) { + value.seq = 9; + value.stamp = ::ros::Time(21, 654); + value.frame_id = "map"; +} + +uint32_t ReadSeconds(const ::ros::Time& value) { return value.sec; } +uint32_t ReadSecondsByValue(::ros::Time value) { return value.sec; } + +std::string ReadFrame(const ::std_msgs::Header& value) { + return value.frame_id; +} +std::string ReadFrameByValue(::std_msgs::Header value) { + return value.frame_id; +} + +TEST(RosIntrinsicsTest, ExistingMutableReferenceFunctionsWorkUnchanged) { + RosIntrinsicMessage message; + + MutateTime(message.stamp); + MutateDuration(message.timeout); + MutateHeader(message.header); + + EXPECT_EQ(ReadSeconds(message.stamp), 12u); + EXPECT_EQ(ReadSecondsByValue(message.stamp), 12u); + EXPECT_EQ(message.stamp->nsec, 345u); + EXPECT_EQ(message.timeout->sec, -4); + EXPECT_EQ(ReadFrame(message.header), "map"); + EXPECT_EQ(ReadFrameByValue(message.header), "map"); + EXPECT_EQ(message.header->stamp.sec, 21u); +} + +TEST(RosIntrinsicsTest, NativePayloadAccessFlushesMutableBorrows) { + RosIntrinsicMessage message; + MutateTime(message.stamp); + MutateDuration(message.timeout); + MutateHeader(message.header); + + const size_t size = message.ByteSizeLong(); + const void* data = message.Data(); + std::vector buffer(size); + std::memcpy(buffer.data(), data, size); + + RosIntrinsicMessage readonly = + RosIntrinsicMessage::CreateReadonly(buffer.data(), buffer.size()); + const RosIntrinsicMessage& view = readonly; + EXPECT_EQ(ReadSeconds(view.stamp), 12u); + EXPECT_EQ(view.stamp->nsec, 345u); + EXPECT_EQ(view.timeout->sec, -4); + EXPECT_EQ(view.timeout->nsec, 500); + EXPECT_EQ(view.header->seq, 9u); + EXPECT_EQ(view.header->stamp.sec, 21u); + EXPECT_EQ(ReadFrame(view.header), "map"); +} + +TEST(RosIntrinsicsTest, ProtobufWireRoundtripFlushesMutableBorrows) { + RosIntrinsicMessage phaser_message; + MutateTime(phaser_message.stamp); + MutateDuration(phaser_message.timeout); + MutateHeader(phaser_message.header); + + RosIntrinsicMessage parsed; + ASSERT_TRUE(parsed.ParseFromString(phaser_message.SerializeAsString())); + EXPECT_EQ(ReadSeconds(parsed.stamp), 12u); + EXPECT_EQ(parsed.stamp->nsec, 345u); + EXPECT_EQ(parsed.timeout->sec, -4); + EXPECT_EQ(parsed.timeout->nsec, 500); + EXPECT_EQ(parsed.header->seq, 9u); + EXPECT_EQ(parsed.header->stamp.sec, 21u); + EXPECT_EQ(ReadFrame(parsed.header), "map"); +} + +TEST(RosIntrinsicsTest, CopyAndMovePreserveDeferredValues) { + RosIntrinsicMessage source; + MutateTime(source.stamp); + MutateHeader(source.header); + + RosIntrinsicMessage copy(source); + EXPECT_EQ(ReadSeconds(copy.stamp), 12u); + EXPECT_EQ(ReadFrame(copy.header), "map"); + + RosIntrinsicMessage moved(std::move(source)); + EXPECT_EQ(ReadSeconds(moved.stamp), 12u); + EXPECT_EQ(ReadFrame(moved.header), "map"); +} + +} // namespace +} // namespace foo::bar::phaser diff --git a/phaser/ros_native_frontend_compatibility_test.cc b/phaser/ros_native_frontend_compatibility_test.cc new file mode 100644 index 0000000..b2836f3 --- /dev/null +++ b/phaser/ros_native_frontend_compatibility_test.cc @@ -0,0 +1,98 @@ +#include "phaser/testdata/RosIntrinsics.phaser.h" +#include "phaser/testdata/RosIntrinsicsProtobufFrontend.phaser.h" + +#include +#include + +#include "gtest/gtest.h" + +namespace { + +using RosMessage = ::foo::bar::phaser::RosIntrinsicMessage; +using ProtobufMessage = + ::foo::bar::pb::protobuf_phaser::RosIntrinsicMessage; + +TEST(RosNativeFrontendCompatibilityTest, + ProtobufFrontendReadsRosFrontendNativePayload) { + RosMessage ros_message; + ros_message.stamp = ::ros::Time(12, 345); + ros_message.timeout = ::ros::Duration(-4, -500); + + ::std_msgs::Header header; + header.seq = 9; + header.stamp = ::ros::Time(21, 654); + header.frame_id = "map"; + ros_message.header = header; + + ros_message.count = 42; + ros_message.name = "native"; + ros_message.samples.push_back(3); + ros_message.samples.push_back(5); + ros_message.tags.push_back("front"); + ros_message.tags.push_back("rear"); + + auto* first_child = ros_message.children.Add(); + first_child->id = 101; + first_child->label = "left"; + auto* second_child = ros_message.children.Add(); + second_child->id = 202; + second_child->label = "right"; + + ros_message.fixed_names[0] = "fixed-a"; + ros_message.fixed_names[1] = "fixed-b"; + ros_message.fixed_children[0]->id = 301; + ros_message.fixed_children[0]->label = "fixed-left"; + ros_message.fixed_children[1]->id = 302; + ros_message.fixed_children[1]->label = "fixed-right"; + + using ChoiceChild = RosMessage::ChoiceChildAlternative; + auto& selected = ros_message.choice.emplace(); + selected.id = 404; + selected.label = "selected"; + + const size_t native_size = ros_message.ByteSizeLong(); + std::vector native_payload(native_size); + std::memcpy(native_payload.data(), ros_message.Data(), native_size); + + const ProtobufMessage protobuf_view = ProtobufMessage::CreateReadonly( + native_payload.data(), native_payload.size()); + + EXPECT_EQ(protobuf_view.stamp().seconds(), 12); + EXPECT_EQ(protobuf_view.stamp().nanos(), 345); + EXPECT_EQ(protobuf_view.timeout().seconds(), -4); + EXPECT_EQ(protobuf_view.timeout().nanos(), -500); + EXPECT_EQ(protobuf_view.header().seq(), 9u); + EXPECT_EQ(protobuf_view.header().stamp().seconds(), 21); + EXPECT_EQ(protobuf_view.header().stamp().nanos(), 654); + EXPECT_EQ(protobuf_view.header().frame_id(), "map"); + + EXPECT_EQ(protobuf_view.count(), 42); + EXPECT_EQ(protobuf_view.name(), "native"); + ASSERT_EQ(protobuf_view.samples_size(), 2u); + EXPECT_EQ(protobuf_view.samples(0), 3); + EXPECT_EQ(protobuf_view.samples(1), 5); + ASSERT_EQ(protobuf_view.tags_size(), 2u); + EXPECT_EQ(protobuf_view.tags(0), "front"); + EXPECT_EQ(protobuf_view.tags(1), "rear"); + + ASSERT_EQ(protobuf_view.children_size(), 2u); + EXPECT_EQ(protobuf_view.children(0).id(), 101); + EXPECT_EQ(protobuf_view.children(0).label(), "left"); + EXPECT_EQ(protobuf_view.children(1).id(), 202); + EXPECT_EQ(protobuf_view.children(1).label(), "right"); + + ASSERT_EQ(protobuf_view.fixed_names_size(), 2u); + EXPECT_EQ(protobuf_view.fixed_names(0), "fixed-a"); + EXPECT_EQ(protobuf_view.fixed_names(1), "fixed-b"); + ASSERT_EQ(protobuf_view.fixed_children_size(), 2u); + EXPECT_EQ(protobuf_view.fixed_children(0).id(), 301); + EXPECT_EQ(protobuf_view.fixed_children(0).label(), "fixed-left"); + EXPECT_EQ(protobuf_view.fixed_children(1).id(), 302); + EXPECT_EQ(protobuf_view.fixed_children(1).label(), "fixed-right"); + + ASSERT_TRUE(protobuf_view.has_choice_child()); + EXPECT_EQ(protobuf_view.choice_child().id(), 404); + EXPECT_EQ(protobuf_view.choice_child().label(), "selected"); +} + +} // namespace diff --git a/phaser/runtime/BUILD.bazel b/phaser/runtime/BUILD.bazel index 51b8f12..b3b310d 100644 --- a/phaser/runtime/BUILD.bazel +++ b/phaser/runtime/BUILD.bazel @@ -12,6 +12,7 @@ cc_library( copts = PHASER_COPTS, hdrs = [ "any.h", + "arrays.h", "fields.h", "iterators.h", "message.h", @@ -20,6 +21,7 @@ cc_library( "vectors.h", "wireformat.h", "phaser_bank.h", + "ros.h", ], deps = [ "@com_google_absl//absl/container:flat_hash_map", diff --git a/phaser/runtime/arrays.h b/phaser/runtime/arrays.h new file mode 100644 index 0000000..018073c --- /dev/null +++ b/phaser/runtime/arrays.h @@ -0,0 +1,1547 @@ +// Copyright 2024-2026 David Allison +// All Rights Reserved +// See LICENSE file for licensing information. + +#pragma once + +// Fixed-extent array facades backed by the same VectorHeader layout as repeated +// vector fields. Wire encoding remains standard protobuf repeated-field format. + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "phaser/runtime/fields.h" +#include "phaser/runtime/iterators.h" +#include "phaser/runtime/message.h" +#include "phaser/runtime/vectors.h" +#include "phaser/runtime/wireformat.h" +#include "toolbelt/payload_buffer.h" + +namespace phaser { + +inline bool HasMutablePayload( + const std::shared_ptr& runtime) { + return runtime != nullptr && + dynamic_cast(runtime.get()) != nullptr; +} + +template +struct FixedArrayConstIterator { + FixedArrayConstIterator(const Field* f, size_t idx, bool reverse = false) + : field(f), index(idx), reverse(reverse) {} + + FixedArrayConstIterator& operator++() { + if (reverse) { + if (index == static_cast(-1)) { + return *this; + } + --index; + } else { + ++index; + } + return *this; + } + FixedArrayConstIterator& operator--() { + if (reverse) { + ++index; + } else { + if (index == 0) { + index = static_cast(-1); + } else { + --index; + } + } + return *this; + } + FixedArrayConstIterator operator+(size_t i) const { + if (reverse) { + return FixedArrayConstIterator(field, index - i, true); + } + return FixedArrayConstIterator(field, index + i); + } + FixedArrayConstIterator operator-(size_t i) const { + if (reverse) { + return FixedArrayConstIterator(field, index + i, true); + } + return FixedArrayConstIterator(field, index - i); + } + + const Value& operator*() const { return field->ConstRefAt(index); } + + bool operator==(const FixedArrayConstIterator& it) const { + return field == it.field && index == it.index && reverse == it.reverse; + } + bool operator!=(const FixedArrayConstIterator& it) const { + return !operator==(it); + } + + const Field* field; + size_t index; + bool reverse; +}; + +#define DECLARE_FIXED_ARRAY_BITS(classname, vtype, itype, ctype, utype, extent) \ + using value_type = vtype; \ + using reference = value_type&; \ + using const_reference = value_type&; \ + using pointer = value_type*; \ + using const_pointer = const value_type*; \ + using size_type = size_t; \ + using difference_type = ptrdiff_t; \ + \ + using iterator = itype; \ + using const_iterator = ctype; \ + using reverse_iterator = itype; \ + using const_reverse_iterator = ctype; \ + \ + iterator begin() { \ + EnsureExtent(); \ + return iterator(this, BaseOffset()); \ + } \ + iterator end() { \ + EnsureExtent(); \ + return iterator( \ + this, BaseOffset() + static_cast<::toolbelt::BufferOffset>(extent * \ + sizeof( \ + utype))); \ + } \ + const_iterator begin() const { \ + return const_iterator(this, 0); \ + } \ + const_iterator end() const { \ + return const_iterator(this, extent); \ + } \ + const_iterator cbegin() const { return begin(); } \ + const_iterator cend() const { return end(); } \ + reverse_iterator rbegin() { \ + return reverse_iterator(this, BaseOffset(), true); \ + } \ + reverse_iterator rend() { \ + return reverse_iterator( \ + this, BaseOffset() + static_cast<::toolbelt::BufferOffset>(extent * \ + sizeof( \ + utype)), \ + true); \ + } \ + const_reverse_iterator rbegin() const { \ + return const_reverse_iterator( \ + this, extent == 0 ? static_cast(-1) : extent - 1, true); \ + } \ + const_reverse_iterator rend() const { \ + return const_reverse_iterator(this, static_cast(-1), true); \ + } \ + const_reverse_iterator crbegin() const { return rbegin(); } \ + const_reverse_iterator crend() const { return rend(); } + +template +class PrimitiveArrayField : public Field { + public: + static constexpr size_t kExtent = N; + + PrimitiveArrayField() = default; + explicit PrimitiveArrayField(uint32_t source_offset, + uint32_t relative_binary_offset, int id, + int number) + : Field(id, number), + source_offset_(source_offset), + relative_binary_offset_(relative_binary_offset) {} + PrimitiveArrayField(const PrimitiveArrayField&) = default; + PrimitiveArrayField(PrimitiveArrayField&&) = default; + + T& operator[](size_t index) { + EnsureExtent(); + return data()[index]; + } + + T operator[](size_t index) const { return GetAt(index); } + + T& front() { return (*this)[0]; } + const T front() const { return GetAt(0); } + T& back() { return (*this)[N - 1]; } + const T back() const { return GetAt(N - 1); } + + T Get(size_t index) const { return GetAt(index); } + + const T& ConstRefAt(size_t index) const { + if (index >= N) { + static const T kDefault{}; + return kDefault; + } + const size_t count = NumElements(); + if (index >= count) { + static const T kDefault{}; + return kDefault; + } + const T* base = StoragePointer(); + if (base == nullptr) { + static const T kDefault{}; + return kDefault; + } + return base[index]; + } + + void Set(size_t index, T v) { (*this)[index] = v; } + + std::array Get() const { + std::array v; + for (size_t i = 0; i < N; i++) { + v[i] = GetAt(i); + } + return v; + } + +#define ITYPE FieldIterator +#define CTYPE FixedArrayConstIterator + DECLARE_FIXED_ARRAY_BITS(PrimitiveArrayField, T, ITYPE, CTYPE, T, N) +#undef ITYPE +#undef CTYPE + + void BeginDeserialize() { + ::toolbelt::PayloadBuffer::VectorClear(GetBufferAddr(), + Header(relative_binary_offset_)); + parsed_count_ = 0; + } + + absl::Status FinalizeDeserialize() { + if (parsed_count_ > N) { + return absl::InvalidArgumentError("array_size overflow"); + } + EnsureExtent(); + parsed_count_ = 0; + return absl::OkStatus(); + } + + void Clear() { + ::toolbelt::PayloadBuffer::VectorClear(GetBufferAddr(), + Header(relative_binary_offset_)); + parsed_count_ = 0; + EnsureStorage(N); + T* base = GetRuntime()->template ToAddress(BaseOffset()); + for (size_t i = 0; i < N; i++) { + base[i] = T{}; + } + SetActiveCount(N); + this->ResetFieldCache(); + } + void clear() { Clear(); } + + PrimitiveArrayField& operator=(const PrimitiveArrayField& other) { + if (this == &other) { + return *this; + } + Clear(); + for (size_t i = 0; i < N; i++) { + (*this)[i] = other.GetAt(i); + } + this->ResetFieldCache(); + return *this; + } + PrimitiveArrayField& operator=(PrimitiveArrayField&& other) noexcept { + return operator=(static_cast(other)); + } + + size_t size() const { return N; } + size_t Size() const { return N; } + size_t max_size() const { return N; } + bool empty() const { return N == 0; } + + T* data() { + EnsureExtent(); + return GetRuntime()->template ToAddress(BaseOffset()); + } + const T* data() const { return StoragePointer(); } + + ::toolbelt::BufferOffset BinaryEndOffset() const { + return relative_binary_offset_ + sizeof(toolbelt::VectorHeader); + } + ::toolbelt::BufferOffset BinaryOffset() const { + return relative_binary_offset_; + } + + bool operator==(const PrimitiveArrayField& other) const { + for (size_t i = 0; i < N; i++) { + if (GetAt(i) != other.GetAt(i)) { + return false; + } + } + return true; + } + bool operator!=(const PrimitiveArrayField& other) const { + return !(*this == other); + } + + size_t SerializedSize() const { + size_t sz = ActiveElementCount(); + if (sz == 0) { + return 0; + } + size_t length = 0; + if constexpr (Packed) { + if constexpr (FixedSize) { + return ProtoBuffer::LengthDelimitedSize(Number(), sz * sizeof(T)); + } else { + for (size_t i = 0; i < sz; i++) { + length += ProtoBuffer::VarintSize(GetAt(i)); + } + return ProtoBuffer::LengthDelimitedSize(Number(), length); + } + } + if constexpr (FixedSize) { + length += sz * (ProtoBuffer::TagSize(Number(), + ProtoBuffer::FixedWireType()) + + sizeof(T)); + } else { + for (size_t i = 0; i < sz; i++) { + length += ProtoBuffer::TagSize(Number(), WireType::kVarint) + + ProtoBuffer::VarintSize(GetAt(i)); + } + } + return length; + } + + absl::Status Serialize(ProtoBuffer& buffer) const { + size_t sz = ActiveElementCount(); + if (sz == 0) { + return absl::OkStatus(); + } + if constexpr (Packed) { + if constexpr (FixedSize) { + const T* base = StoragePointer(); + if (base == nullptr) { + return absl::OkStatus(); + } + return buffer.SerializeLengthDelimited( + Number(), reinterpret_cast(base), sz * sizeof(T)); + } else { + size_t length = 0; + for (size_t i = 0; i < sz; i++) { + length += ProtoBuffer::VarintSize(GetAt(i)); + } + if (absl::Status status = + buffer.SerializeLengthDelimitedHeader(Number(), length); + !status.ok()) { + return status; + } + for (size_t i = 0; i < sz; i++) { + if (absl::Status status = + buffer.SerializeRawVarint(GetAt(i)); + !status.ok()) { + return status; + } + } + return absl::OkStatus(); + } + } + if constexpr (FixedSize) { + for (size_t i = 0; i < sz; i++) { + if (absl::Status status = buffer.SerializeFixed(Number(), GetAt(i)); + !status.ok()) { + return status; + } + } + } else { + for (size_t i = 0; i < sz; i++) { + if (absl::Status status = + buffer.SerializeVarint(Number(), GetAt(i)); + !status.ok()) { + return status; + } + } + } + return absl::OkStatus(); + } + + absl::Status Deserialize(ProtoBuffer& buffer) { + if constexpr (Packed) { + absl::StatusOr> payload = + buffer.DeserializeLengthDelimited(); + if (!payload.ok()) { + return payload.status(); + } + size_t count = payload->size() / sizeof(T); + if (parsed_count_ + count > N) { + return absl::InvalidArgumentError("array_size overflow"); + } + if constexpr (FixedSize) { + EnsureStorage(N); + T* base = GetRuntime()->template ToAddress(BaseOffset()); + memcpy(base + parsed_count_, payload->data(), payload->size()); + parsed_count_ += count; + SetActiveCount(parsed_count_); + } else { + ProtoBuffer sub_buffer(*payload); + while (!sub_buffer.Eof()) { + if (parsed_count_ >= N) { + return absl::InvalidArgumentError("array_size overflow"); + } + absl::StatusOr v = sub_buffer.DeserializeVarint(); + if (!v.ok()) { + return v.status(); + } + EnsureStorage(parsed_count_ + 1); + GetRuntime()->template ToAddress(BaseOffset())[parsed_count_] = + *v; + parsed_count_++; + SetActiveCount(parsed_count_); + } + } + } else { + if constexpr (FixedSize) { + if (parsed_count_ >= N) { + return absl::InvalidArgumentError("array_size overflow"); + } + absl::StatusOr v = buffer.DeserializeFixed(); + if (!v.ok()) { + return v.status(); + } + EnsureStorage(parsed_count_ + 1); + GetRuntime()->template ToAddress(BaseOffset())[parsed_count_] = *v; + parsed_count_++; + SetActiveCount(parsed_count_); + } else { + if (parsed_count_ >= N) { + return absl::InvalidArgumentError("array_size overflow"); + } + absl::StatusOr v = buffer.DeserializeVarint(); + if (!v.ok()) { + return v.status(); + } + EnsureStorage(parsed_count_ + 1); + GetRuntime()->template ToAddress(BaseOffset())[parsed_count_] = *v; + parsed_count_++; + SetActiveCount(parsed_count_); + } + } + return absl::OkStatus(); + } + + private: + friend FieldIterator; + friend FieldIterator; + + ::toolbelt::BufferOffset BaseOffset() const { + toolbelt::VectorHeader* hdr = Header(relative_binary_offset_); + if (hdr == nullptr) { + return 0; + } + return hdr->data; + } + + toolbelt::VectorHeader* Header(uint32_t offset) const { + return GetRuntime()->template ToAddress( + Message::GetMessageBinaryStart(this, source_offset_) + offset); + } + + size_t NumElements() const { + toolbelt::VectorHeader* hdr = Header(relative_binary_offset_); + if (hdr == nullptr) { + return 0; + } + return hdr->num_elements; + } + + void SetActiveCount(size_t count) { + toolbelt::VectorHeader* hdr = Header(relative_binary_offset_); + hdr->num_elements = static_cast(count); + } + + void EnsureStorage(size_t count) { + ::toolbelt::PayloadBuffer::VectorReserve(GetBufferAddr(), + Header(relative_binary_offset_), + count); + ::toolbelt::PayloadBuffer::VectorResize(GetBufferAddr(), + Header(relative_binary_offset_), + count); + } + + void EnsureExtent() { + if (!IsMutable()) { + return; + } + const size_t current = NumElements(); + if (current >= N) { + return; + } + EnsureStorage(N); + T* base = GetRuntime()->template ToAddress(BaseOffset()); + if (base == nullptr) { + return; + } + for (size_t i = current; i < N; i++) { + base[i] = T{}; + } + SetActiveCount(N); + } + + T GetAt(size_t index) const { + if (index >= N) { + return T{}; + } + const size_t count = NumElements(); + if (index >= count) { + return T{}; + } + const T* base = StoragePointer(); + if (base == nullptr) { + return T{}; + } + return base[index]; + } + + const T* StoragePointer() const { + if (BaseOffset() == 0) { + return nullptr; + } + return GetRuntime()->template ToAddress(BaseOffset()); + } + + size_t ActiveElementCount() const { + size_t count = NumElements(); + if (count > N) { + return N; + } + return count; + } + + bool IsMutable() const { return HasMutablePayload(GetRuntime()); } + + ::toolbelt::PayloadBuffer* GetBuffer() const { + return Message::GetBuffer(this, source_offset_); + } + + ::toolbelt::PayloadBuffer** GetBufferAddr() const { + return Message::GetBufferAddr(this, source_offset_); + } + + const std::shared_ptr& GetRuntime() const { + return Message::GetRuntime(this, source_offset_); + } + + uint32_t source_offset_; + ::toolbelt::BufferOffset relative_binary_offset_; + size_t parsed_count_ = 0; +}; + +template +class EnumArrayField : public Field { + public: + static constexpr size_t kExtent = N; + + EnumArrayField() = default; + explicit EnumArrayField(uint32_t source_offset, + uint32_t relative_binary_offset, int id, int number) + : Field(id, number), + source_offset_(source_offset), + relative_binary_offset_(relative_binary_offset) {} + EnumArrayField(const EnumArrayField&) = default; + EnumArrayField(EnumArrayField&&) = default; + + using T = typename std::underlying_type::type; + + Enum& operator[](size_t index) { + EnsureExtent(); + return *reinterpret_cast(&data()[index]); + } + + const Enum operator[](size_t index) const { return GetAt(index); } + + Enum& front() { return (*this)[0]; } + const Enum front() const { return GetAt(0); } + Enum& back() { return (*this)[N - 1]; } + const Enum back() const { return GetAt(N - 1); } + + Enum Get(size_t index) const { return GetAt(index); } + + const Enum& ConstRefAt(size_t index) const { + if (index >= N) { + static const Enum kDefault = static_cast(T{}); + return kDefault; + } + const size_t count = NumElements(); + if (index >= count) { + static const Enum kDefault = static_cast(T{}); + return kDefault; + } + const T* base = StoragePointer(); + if (base == nullptr) { + static const Enum kDefault = static_cast(T{}); + return kDefault; + } + return *reinterpret_cast(&base[index]); + } + + void Set(size_t index, Enum v) { + EnsureExtent(); + GetRuntime()->template ToAddress(BaseOffset())[index] = static_cast(v); + } + + std::array Get() const { + std::array r; + for (size_t i = 0; i < N; i++) { + r[i] = GetAt(i); + } + return r; + } + +#define ITYPE EnumFieldIterator +#define CTYPE FixedArrayConstIterator + DECLARE_FIXED_ARRAY_BITS(EnumArrayField, Enum, ITYPE, CTYPE, T, N) +#undef ITYPE +#undef CTYPE + + void BeginDeserialize() { + ::toolbelt::PayloadBuffer::VectorClear(GetBufferAddr(), + Header(relative_binary_offset_)); + parsed_count_ = 0; + } + + absl::Status FinalizeDeserialize() { + if (parsed_count_ > N) { + return absl::InvalidArgumentError("array_size overflow"); + } + EnsureExtent(); + parsed_count_ = 0; + return absl::OkStatus(); + } + + void Clear() { + ::toolbelt::PayloadBuffer::VectorClear(GetBufferAddr(), + Header(relative_binary_offset_)); + parsed_count_ = 0; + EnsureStorage(N); + T* base = GetRuntime()->template ToAddress(BaseOffset()); + for (size_t i = 0; i < N; i++) { + base[i] = T{}; + } + SetActiveCount(N); + this->ResetFieldCache(); + } + void clear() { Clear(); } + + EnumArrayField& operator=(const EnumArrayField& other) { + if (this == &other) { + return *this; + } + Clear(); + for (size_t i = 0; i < N; i++) { + (*this)[i] = other.GetAt(i); + } + this->ResetFieldCache(); + return *this; + } + EnumArrayField& operator=(EnumArrayField&& other) noexcept { + return operator=(static_cast(other)); + } + + size_t size() const { return N; } + size_t Size() const { return N; } + size_t max_size() const { return N; } + bool empty() const { return N == 0; } + + Enum* data() { + EnsureExtent(); + return reinterpret_cast(GetRuntime()->template ToAddress( + BaseOffset())); + } + const Enum* data() const { + return reinterpret_cast(StoragePointer()); + } + + ::toolbelt::BufferOffset BinaryEndOffset() const { + return relative_binary_offset_ + sizeof(toolbelt::VectorHeader); + } + ::toolbelt::BufferOffset BinaryOffset() const { + return relative_binary_offset_; + } + + bool operator==(const EnumArrayField& other) const { + for (size_t i = 0; i < N; i++) { + if (GetAt(i) != other.GetAt(i)) { + return false; + } + } + return true; + } + bool operator!=(const EnumArrayField& other) const { + return !(*this == other); + } + + size_t SerializedSize() const { + size_t sz = ActiveElementCount(); + if (sz == 0) { + return 0; + } + size_t length = 0; + if constexpr (Packed) { + for (size_t i = 0; i < sz; i++) { + length += ProtoBuffer::VarintSize( + static_cast(GetAt(i))); + } + return ProtoBuffer::LengthDelimitedSize(Number(), length); + } + for (size_t i = 0; i < sz; i++) { + const T raw = static_cast(GetAt(i)); + length += ProtoBuffer::TagSize(Number(), WireType::kVarint) + + ProtoBuffer::VarintSize(raw); + } + return length; + } + + absl::Status Serialize(ProtoBuffer& buffer) const { + size_t sz = ActiveElementCount(); + if (sz == 0) { + return absl::OkStatus(); + } + if constexpr (Packed) { + size_t length = 0; + for (size_t i = 0; i < sz; i++) { + length += ProtoBuffer::VarintSize(static_cast(GetAt(i))); + } + if (absl::Status status = + buffer.SerializeLengthDelimitedHeader(Number(), length); + !status.ok()) { + return status; + } + for (size_t i = 0; i < sz; i++) { + const T raw = static_cast(GetAt(i)); + if (absl::Status status = buffer.SerializeRawVarint(raw); + !status.ok()) { + return status; + } + } + return absl::OkStatus(); + } + for (size_t i = 0; i < sz; i++) { + const T raw = static_cast(GetAt(i)); + if (absl::Status status = + buffer.SerializeVarint(Number(), raw); + !status.ok()) { + return status; + } + } + return absl::OkStatus(); + } + + absl::Status Deserialize(ProtoBuffer& buffer) { + if constexpr (Packed) { + absl::StatusOr> data = + buffer.DeserializeLengthDelimited(); + if (!data.ok()) { + return data.status(); + } + ProtoBuffer sub_buffer(*data); + while (!sub_buffer.Eof()) { + if (parsed_count_ >= N) { + return absl::InvalidArgumentError("array_size overflow"); + } + absl::StatusOr v = sub_buffer.DeserializeVarint(); + if (!v.ok()) { + return v.status(); + } + EnsureStorage(parsed_count_ + 1); + GetRuntime()->template ToAddress(BaseOffset())[parsed_count_] = *v; + parsed_count_++; + SetActiveCount(parsed_count_); + } + } else { + if (parsed_count_ >= N) { + return absl::InvalidArgumentError("array_size overflow"); + } + absl::StatusOr v = buffer.DeserializeVarint(); + if (!v.ok()) { + return v.status(); + } + EnsureStorage(parsed_count_ + 1); + GetRuntime()->template ToAddress(BaseOffset())[parsed_count_] = *v; + parsed_count_++; + SetActiveCount(parsed_count_); + } + return absl::OkStatus(); + } + + private: + friend EnumFieldIterator; + friend EnumFieldIterator; + friend FieldIterator; + friend FieldIterator; + + ::toolbelt::BufferOffset BaseOffset() const { + toolbelt::VectorHeader* hdr = Header(relative_binary_offset_); + if (hdr == nullptr) { + return 0; + } + return hdr->data; + } + + toolbelt::VectorHeader* Header(uint32_t offset) const { + return GetRuntime()->template ToAddress( + Message::GetMessageBinaryStart(this, source_offset_) + offset); + } + + size_t NumElements() const { + toolbelt::VectorHeader* hdr = Header(relative_binary_offset_); + if (hdr == nullptr) { + return 0; + } + return hdr->num_elements; + } + + void SetActiveCount(size_t count) { + toolbelt::VectorHeader* hdr = Header(relative_binary_offset_); + hdr->num_elements = static_cast(count); + } + + void EnsureStorage(size_t count) { + ::toolbelt::PayloadBuffer::VectorReserve(GetBufferAddr(), + Header(relative_binary_offset_), + count); + ::toolbelt::PayloadBuffer::VectorResize(GetBufferAddr(), + Header(relative_binary_offset_), + count); + } + + void EnsureExtent() { + if (!IsMutable()) { + return; + } + const size_t current = NumElements(); + if (current >= N) { + return; + } + EnsureStorage(N); + T* base = GetRuntime()->template ToAddress(BaseOffset()); + if (base == nullptr) { + return; + } + for (size_t i = current; i < N; i++) { + base[i] = T{}; + } + SetActiveCount(N); + } + + Enum GetAt(size_t index) const { + if (index >= N) { + return static_cast(T{}); + } + const size_t count = NumElements(); + if (index >= count) { + return static_cast(T{}); + } + const T* base = StoragePointer(); + if (base == nullptr) { + return static_cast(T{}); + } + return static_cast(base[index]); + } + + const T* StoragePointer() const { + if (BaseOffset() == 0) { + return nullptr; + } + return GetRuntime()->template ToAddress(BaseOffset()); + } + + size_t ActiveElementCount() const { + size_t count = NumElements(); + if (count > N) { + return N; + } + return count; + } + + bool IsMutable() const { return HasMutablePayload(GetRuntime()); } + + ::toolbelt::PayloadBuffer* GetBuffer() const { + return Message::GetBuffer(this, source_offset_); + } + + ::toolbelt::PayloadBuffer** GetBufferAddr() const { + return Message::GetBufferAddr(this, source_offset_); + } + + const std::shared_ptr& GetRuntime() const { + return Message::GetRuntime(this, source_offset_); + } + + uint32_t source_offset_; + ::toolbelt::BufferOffset relative_binary_offset_; + size_t parsed_count_ = 0; +}; + +template +class MessageArrayField : public MessageVectorField { + public: + static constexpr size_t kExtent = N; + + MessageArrayField() = default; + explicit MessageArrayField(uint32_t source_offset, + uint32_t relative_binary_offset, int id, + int number) + : MessageVectorField(source_offset, relative_binary_offset, id, + number) {} + MessageArrayField(const MessageArrayField&) = default; + MessageArrayField(MessageArrayField&&) = default; + + const MessageObject& operator[](size_t index) const { + return ConstObject(index); + } + + MessageObject& operator[](size_t index) { return MutableObject(index); } + + MessageObject& front() { return (*this)[0]; } + const MessageObject& front() const { return (*this)[0]; } + MessageObject& back() { return (*this)[N - 1]; } + const MessageObject& back() const { return (*this)[N - 1]; } + + using typename MessageVectorField::iterator; + using typename MessageVectorField::const_iterator; + using typename MessageVectorField::reverse_iterator; + using typename MessageVectorField::const_reverse_iterator; + + iterator begin() { + EnsureExtent(); + return MessageVectorField::begin(); + } + iterator end() { + EnsureExtent(); + return MessageVectorField::end(); + } + const_iterator begin() const { + return MessageVectorField::begin(); + } + const_iterator end() const { + return MessageVectorField::end(); + } + const_iterator cbegin() const { return begin(); } + const_iterator cend() const { return end(); } + reverse_iterator rbegin() { + EnsureExtent(); + return MessageVectorField::rbegin(); + } + reverse_iterator rend() { + EnsureExtent(); + return MessageVectorField::rend(); + } + const_reverse_iterator rbegin() const { + return MessageVectorField::rbegin(); + } + const_reverse_iterator rend() const { + return MessageVectorField::rend(); + } + const_reverse_iterator crbegin() const { return rbegin(); } + const_reverse_iterator crend() const { return rend(); } + + void BeginDeserialize() { + MessageVectorField::Clear(); + parsed_count_ = 0; + } + + absl::Status FinalizeDeserialize() { + if (parsed_count_ > N) { + return absl::InvalidArgumentError("array_size overflow"); + } + while (MessageVectorField::Get().size() < N) { + MessageVectorField::Add(); + } + parsed_count_ = 0; + return absl::OkStatus(); + } + + void Clear() { + if (!HasMutablePayload(MessageVectorField::GetRuntime())) { + return; + } + MessageVectorField::Clear(); + parsed_count_ = 0; + for (size_t i = 0; i < N; i++) { + MessageVectorField::Add(); + } + this->ResetFieldCache(); + } + void clear() { Clear(); } + + MessageArrayField& operator=(const MessageArrayField& other) { + if (this == &other) { + return *this; + } + Clear(); + for (size_t i = 0; i < N; i++) { + if (absl::Status s = MutableObject(i).Mutable()->CloneFrom(other[i].Get()); + !s.ok()) { + return *this; + } + } + this->ResetFieldCache(); + return *this; + } + MessageArrayField& operator=(MessageArrayField&& other) noexcept { + return operator=(static_cast(other)); + } + + size_t size() const { return N; } + size_t Size() const { return N; } + size_t max_size() const { return N; } + bool empty() const { return N == 0; } + + const T& Get(size_t index) const { + const MessageObject& obj = ConstObject(index); + if (obj.empty()) { + return EmptyObject().Get(); + } + return obj.Get(); + } + + T* Mutable(size_t index) { return MutableObject(index).Mutable(); } + + ::toolbelt::BufferOffset BinaryEndOffset() const { + return MessageVectorField::BinaryEndOffset(); + } + ::toolbelt::BufferOffset BinaryOffset() const { + return MessageVectorField::BinaryOffset(); + } + + bool operator==(const MessageArrayField& other) const { + for (size_t i = 0; i < N; i++) { + if ((*this)[i].Get() != other[i].Get()) { + return false; + } + } + return true; + } + bool operator!=(const MessageArrayField& other) const { + return !(*this == other); + } + + size_t SerializedSize() const { + size_t length = 0; + for (size_t i = 0; i < parsed_count_for_serialize(); i++) { + length += phaser::ProtoBuffer::LengthDelimitedSize( + Field::Number(), (*this)[i].SerializedSize()); + } + return length; + } + + absl::Status Serialize(ProtoBuffer& buffer) const { + size_t count = parsed_count_for_serialize(); + for (size_t i = 0; i < count; i++) { + if (absl::Status status = buffer.SerializeLengthDelimitedHeader( + Field::Number(), (*this)[i].SerializedSize()); + !status.ok()) { + return status; + } + if (absl::Status status = (*this)[i].Serialize(buffer); !status.ok()) { + return status; + } + } + return absl::OkStatus(); + } + + absl::Status Deserialize(ProtoBuffer& buffer) { + if (parsed_count_ >= N) { + return absl::InvalidArgumentError("array_size overflow"); + } + absl::StatusOr> v = buffer.DeserializeLengthDelimited(); + if (!v.ok()) { + return v.status(); + } + T* msg = nullptr; + if (parsed_count_ < MessageVectorField::Get().size()) { + msg = MessageVectorField::Mutable(parsed_count_); + } else { + msg = MessageVectorField::Add(); + } + ProtoBuffer msg_buffer(*v); + if (absl::Status status = msg->Deserialize(msg_buffer); !status.ok()) { + return status; + } + parsed_count_++; + return absl::OkStatus(); + } + + private: + static const MessageObject& EmptyObject() { + static const MessageObject empty; + return empty; + } + + const MessageObject& ConstObject(size_t index) const { + if (index >= N) { + return EmptyObject(); + } + return MessageVectorField::operator[](static_cast(index)); + } + + MessageObject& MutableObject(size_t index) { + if (!HasMutablePayload(MessageVectorField::GetRuntime())) { + return const_cast&>(ConstObject(index)); + } + while (MessageVectorField::Get().size() <= index) { + MessageVectorField::Add(); + } + while (MessageVectorField::Get().size() < N) { + MessageVectorField::Add(); + } + return MessageVectorField::operator[](static_cast(index)); + } + + void EnsureExtent() { + if (!HasMutablePayload(MessageVectorField::GetRuntime())) { + return; + } + while (MessageVectorField::Get().size() < N) { + MessageVectorField::Add(); + } + } + + size_t parsed_count_for_serialize() const { + size_t count = MessageVectorField::size(); + if (count == 0) { + return 0; + } + if (count > N) { + return N; + } + return count; + } + + size_t parsed_count_ = 0; +}; + +template +class StringArrayField : public Field { + public: + static constexpr size_t kExtent = N; + + StringArrayField() = default; + explicit StringArrayField(uint32_t source_offset, + uint32_t relative_binary_offset, int id, + int number) + : Field(id, number), + source_offset_(source_offset), + relative_binary_offset_(relative_binary_offset) {} + StringArrayField(const StringArrayField&) = default; + StringArrayField(StringArrayField&& other) noexcept + : Field(std::move(other)), + source_offset_(other.source_offset_), + relative_binary_offset_(other.relative_binary_offset_), + parsed_count_(other.parsed_count_) {} + + const NonEmbeddedStringField& operator[](size_t index) const { + return ConstSlot(index); + } + + NonEmbeddedStringField& operator[](size_t index) { + EnsureMutableExtent(); + return strings_[index]; + } + + const NonEmbeddedStringField& front() const { return ConstSlot(0); } + NonEmbeddedStringField& front() { return (*this)[0]; } + const NonEmbeddedStringField& back() const { return ConstSlot(N - 1); } + NonEmbeddedStringField& back() { return (*this)[N - 1]; } + + using value_type = NonEmbeddedStringField; + using reference = value_type&; + using const_reference = value_type&; + using size_type = size_t; + using difference_type = ptrdiff_t; + using iterator = typename std::array::iterator; + struct ConstIterator { + const StringArrayField* field = nullptr; + size_t index = 0; + + ConstIterator() = default; + ConstIterator(const StringArrayField* f, size_t i) : field(f), index(i) {} + + ConstIterator& operator++() { + ++index; + return *this; + } + ConstIterator& operator--() { + --index; + return *this; + } + const NonEmbeddedStringField& operator*() const { + return field->ConstSlot(index); + } + bool operator==(const ConstIterator& it) const { + return field == it.field && index == it.index; + } + bool operator!=(const ConstIterator& it) const { return !operator==(it); } + }; + struct ConstReverseIterator { + const StringArrayField* field = nullptr; + size_t index = 0; + + ConstReverseIterator() = default; + ConstReverseIterator(const StringArrayField* f, size_t i) + : field(f), index(i) {} + + ConstReverseIterator& operator++() { + if (index == static_cast(-1)) { + return *this; + } + --index; + return *this; + } + ConstReverseIterator& operator--() { + ++index; + return *this; + } + const NonEmbeddedStringField& operator*() const { + return field->ConstSlot(index); + } + bool operator==(const ConstReverseIterator& it) const { + return field == it.field && index == it.index; + } + bool operator!=(const ConstReverseIterator& it) const { + return !operator==(it); + } + }; + using const_iterator = ConstIterator; + using reverse_iterator = + typename std::array::reverse_iterator; + using const_reverse_iterator = ConstReverseIterator; + + iterator begin() { + EnsureMutableExtent(); + return strings_.begin(); + } + iterator end() { + EnsureMutableExtent(); + return strings_.end(); + } + const_iterator begin() const { return const_iterator(this, 0); } + const_iterator end() const { return const_iterator(this, N); } + const_iterator cbegin() const { return begin(); } + const_iterator cend() const { return end(); } + reverse_iterator rbegin() { + EnsureMutableExtent(); + return strings_.rbegin(); + } + reverse_iterator rend() { + EnsureMutableExtent(); + return strings_.rend(); + } + const_reverse_iterator rbegin() const { + return const_reverse_iterator( + this, N == 0 ? static_cast(-1) : N - 1); + } + const_reverse_iterator rend() const { + return const_reverse_iterator(this, static_cast(-1)); + } + const_reverse_iterator crbegin() const { return rbegin(); } + const_reverse_iterator crend() const { return rend(); } + + void BeginDeserialize() { + if (NumElements() > 0) { + ClearParsedElements(); + } else { + ResetSlots(); + } + parsed_count_ = 0; + } + + absl::Status FinalizeDeserialize() { + if (parsed_count_ > N) { + return absl::InvalidArgumentError("array_size overflow"); + } + EnsureMutableExtent(); + parsed_count_ = 0; + return absl::OkStatus(); + } + + void Clear() { + if (!IsMutable()) { + return; + } + parsed_count_ = 0; + if (NumElements() > 0) { + ClearParsedElements(); + } else { + ResetSlots(); + } + EnsureMutableExtent(); + this->ResetFieldCache(); + } + void clear() { Clear(); } + + StringArrayField& operator=(const StringArrayField& other) { + if (this == &other) { + return *this; + } + Clear(); + for (size_t i = 0; i < N; i++) { + (*this)[i] = other.ConstSlot(i); + } + this->ResetFieldCache(); + return *this; + } + StringArrayField& operator=(StringArrayField&& other) noexcept { + return operator=(static_cast(other)); + } + + size_t size() const { return N; } + size_t Size() const { return N; } + size_t max_size() const { return N; } + bool empty() const { return N == 0; } + + NonEmbeddedStringField* data() { + EnsureMutableExtent(); + return strings_.data(); + } + const NonEmbeddedStringField* data() const { return strings_.data(); } + + std::string_view Get(size_t index) const { + const NonEmbeddedStringField& slot = ConstSlot(index); + if (slot.IsPlaceholder()) { + return {}; + } + return slot.Get(); + } + + template + void Set(size_t index, Str s) { + (*this)[index].Set(s); + } + + ::toolbelt::BufferOffset BinaryEndOffset() const { + return relative_binary_offset_ + sizeof(toolbelt::VectorHeader); + } + ::toolbelt::BufferOffset BinaryOffset() const { + return relative_binary_offset_; + } + + bool operator==(const StringArrayField& other) const { + for (size_t i = 0; i < N; i++) { + if (ConstSlot(i).Get() != other.ConstSlot(i).Get()) { + return false; + } + } + return true; + } + bool operator!=(const StringArrayField& other) const { + return !(*this == other); + } + + size_t SerializedSize() const { + size_t length = 0; + size_t count = parsed_count_for_serialize(); + for (size_t i = 0; i < count; i++) { + length += phaser::ProtoBuffer::LengthDelimitedSize( + Number(), ConstSlot(i).SerializedSize()); + } + return length; + } + + absl::Status Serialize(ProtoBuffer& buffer) const { + size_t count = parsed_count_for_serialize(); + for (size_t i = 0; i < count; i++) { + const NonEmbeddedStringField& slot = ConstSlot(i); + if (absl::Status status = buffer.SerializeLengthDelimited( + Number(), slot.data(), slot.size()); + !status.ok()) { + return status; + } + } + return absl::OkStatus(); + } + + absl::Status Deserialize(ProtoBuffer& buffer) { + if (parsed_count_ >= N) { + return absl::InvalidArgumentError("array_size overflow"); + } + EnsureStorage(parsed_count_ + 1); + absl::StatusOr v = buffer.DeserializeString(); + if (!v.ok()) { + return v.status(); + } + void* str_hdr = ::toolbelt::PayloadBuffer::Allocate( + GetBufferAddr(), sizeof(toolbelt::StringHeader)); + ::toolbelt::BufferOffset hdr_offset = GetRuntime()->ToOffset(str_hdr); + ::toolbelt::PayloadBuffer::SetString(GetBufferAddr(), *v, hdr_offset); + toolbelt::VectorHeader* hdr = Header(); + ::toolbelt::BufferOffset* data = + GetRuntime()->template ToAddress<::toolbelt::BufferOffset>(hdr->data); + data[parsed_count_] = hdr_offset; + hdr = Header(); + data = GetRuntime()->template ToAddress<::toolbelt::BufferOffset>(hdr->data); + strings_[parsed_count_] = NonEmbeddedStringField( + Message::GetMessage(this, source_offset_), data[parsed_count_]); + parsed_count_++; + SetActiveCount(parsed_count_); + return absl::OkStatus(); + } + + private: + toolbelt::VectorHeader* Header() const { + return GetRuntime()->template ToAddress( + Message::GetMessageBinaryStart(this, source_offset_) + + relative_binary_offset_); + } + + size_t NumElements() const { + toolbelt::VectorHeader* hdr = Header(); + if (hdr == nullptr) { + return 0; + } + return hdr->num_elements; + } + + void SetActiveCount(size_t count) { + Header()->num_elements = static_cast(count); + } + + void EnsureStorage(size_t count) { + ::toolbelt::PayloadBuffer::VectorReserve<::toolbelt::BufferOffset>( + GetBufferAddr(), Header(), count); + ::toolbelt::PayloadBuffer::VectorResize<::toolbelt::BufferOffset>( + GetBufferAddr(), Header(), count); + } + + size_t parsed_count_for_serialize() const { + size_t count = NumElements(); + if (count == 0) { + return 0; + } + if (count > N) { + return N; + } + return count; + } + + void ClearParsedElements() { + for (auto& s : strings_) { + if (!s.IsPlaceholder()) { + s.Clear(); + } + } + ::toolbelt::PayloadBuffer::VectorClear<::toolbelt::BufferOffset>( + GetBufferAddr(), Header()); + ResetSlots(); + } + + void ResetSlots() { + for (auto& string : strings_) { + string = NonEmbeddedStringField(); + } + } + + void EnsureMutableExtent() { + if (!IsMutable()) { + return; + } + const size_t current = NumElements(); + if (current < N) { + EnsureStorage(N); + for (size_t i = current; i < N; i++) { + AllocateStringSlot(i); + } + SetActiveCount(N); + } + BindExistingSlots(0, N); + } + + const NonEmbeddedStringField& ConstSlot(size_t index) const { + if (index >= N) { + return empty_; + } + if (FindFieldOffset(source_offset_) < 0) { + return empty_; + } + toolbelt::VectorHeader* hdr = Header(); + if (hdr == nullptr || hdr->data == 0) { + return empty_; + } + const size_t count = hdr->num_elements; + if (index >= count) { + return empty_; + } + if (!strings_[index].IsPlaceholder()) { + return strings_[index]; + } + ::toolbelt::BufferOffset* data = + GetRuntime()->template ToAddress<::toolbelt::BufferOffset>(hdr->data); + if (data[index] == 0) { + return empty_; + } + auto* self = const_cast(this); + self->strings_[index] = NonEmbeddedStringField( + Message::GetMessage(this, source_offset_), data[index]); + return strings_[index]; + } + + void AllocateStringSlot(size_t index) { + toolbelt::VectorHeader* hdr = Header(); + ::toolbelt::BufferOffset* data = + GetRuntime()->template ToAddress<::toolbelt::BufferOffset>(hdr->data); + if (data[index] == 0) { + void* str_hdr = ::toolbelt::PayloadBuffer::Allocate( + GetBufferAddr(), sizeof(toolbelt::StringHeader)); + hdr = Header(); + data = GetRuntime()->template ToAddress<::toolbelt::BufferOffset>( + hdr->data); + data[index] = GetRuntime()->ToOffset(str_hdr); + } + strings_[index] = NonEmbeddedStringField( + Message::GetMessage(this, source_offset_), data[index]); + } + + void BindExistingSlots(size_t start, size_t end) const { + if (start >= end || end > N) { + return; + } + toolbelt::VectorHeader* hdr = Header(); + if (hdr == nullptr || hdr->data == 0) { + return; + } + ::toolbelt::BufferOffset* data = + GetRuntime()->template ToAddress<::toolbelt::BufferOffset>(hdr->data); + auto* self = const_cast(this); + for (size_t i = start; i < end; i++) { + if (data[i] == 0) { + continue; + } + if (self->strings_[i].IsPlaceholder()) { + self->strings_[i] = NonEmbeddedStringField( + Message::GetMessage(this, source_offset_), data[i]); + } + } + } + + bool IsMutable() const { return HasMutablePayload(GetRuntime()); } + + ::toolbelt::PayloadBuffer* GetBuffer() const { + return Message::GetBuffer(this, source_offset_); + } + + ::toolbelt::PayloadBuffer** GetBufferAddr() const { + return Message::GetBufferAddr(this, source_offset_); + } + + const std::shared_ptr& GetRuntime() const { + return Message::GetRuntime(this, source_offset_); + } + + uint32_t source_offset_; + ::toolbelt::BufferOffset relative_binary_offset_; + mutable std::array strings_; + size_t parsed_count_ = 0; + NonEmbeddedStringField empty_; +}; + +#undef DECLARE_FIXED_ARRAY_BITS + +} // namespace phaser diff --git a/phaser/runtime/fields.h b/phaser/runtime/fields.h index af250c1..0f7377a 100644 --- a/phaser/runtime/fields.h +++ b/phaser/runtime/fields.h @@ -9,8 +9,11 @@ #include #include +#include +#include #include #include +#include #include #include "absl/container/flat_hash_map.h" @@ -85,6 +88,12 @@ class Field { int GetIndent() const { return indent_; } + protected: + void ResetFieldCache() { + cached_offset_ = 0xffffffff; + cached_field_id_ = -1; + } + protected: int id_ = 0; int number_ = 0; @@ -102,6 +111,28 @@ class Field { : Field(id, number), \ source_offset_(boff), \ relative_binary_offset_(offset) {} \ + cname##Field(const cname##Field&) = default; \ + cname##Field(cname##Field&&) = default; \ + cname##Field& operator=(const cname##Field& other) { \ + if (this == &other) { \ + return *this; \ + } \ + if (other.IsPresent()) { \ + Set(other.Get()); \ + } else { \ + Clear(); \ + } \ + ResetFieldCache(); \ + return *this; \ + } \ + cname##Field& operator=(cname##Field&& other) noexcept { \ + return operator=(static_cast(other)); \ + } \ + operator type() const { return Get(); } \ + cname##Field& operator=(type v) { \ + Set(v); \ + return *this; \ + } \ type Get() const { \ int32_t offset = FindFieldOffset(source_offset_); \ if (offset < 0) { \ @@ -203,6 +234,33 @@ class EnumField : public Field { : Field(id, number), source_offset_(boff), relative_binary_offset_(offset) {} + EnumField(const EnumField&) = default; + EnumField(EnumField&&) = default; + EnumField& operator=(const EnumField& other) { + if (this == &other) { + return *this; + } + if (other.IsPresent()) { + Set(other.Get()); + } else { + Clear(); + } + ResetFieldCache(); + return *this; + } + EnumField& operator=(EnumField&& other) noexcept { + return operator=(static_cast(other)); + } + operator Enum() const { return Get(); } + operator T() const { return GetUnderlying(); } + EnumField& operator=(Enum e) { + Set(e); + return *this; + } + EnumField& operator=(T e) { + Set(e); + return *this; + } Enum Get() const { int32_t offset = FindFieldOffset(source_offset_); @@ -298,6 +356,38 @@ class StringField : public Field { : Field(id, number), source_offset_(source_offset), relative_binary_offset_(relative_binary_offset) {} + StringField(const StringField&) = default; + StringField(StringField&&) = default; + StringField& operator=(const StringField& other) { + if (this == &other) { + return *this; + } + if (other.IsPresent()) { + Set(other.Get()); + } else { + Clear(); + } + ResetFieldCache(); + return *this; + } + StringField& operator=(StringField&& other) noexcept { + return operator=(static_cast(other)); + } + operator std::string_view() const { return Get(); } + StringField& operator=(const std::string& s) { + Set(s); + return *this; + } + StringField& operator=(std::string_view s) { + Set(s); + return *this; + } + StringField& operator=(const char* s) { + ::toolbelt::PayloadBuffer::SetString( + GetBufferAddr(), std::string_view(s, std::strlen(s)), + GetMessageBinaryStart() + relative_binary_offset_); + return *this; + } std::string_view Get() const { int32_t offset = FindFieldOffset(source_offset_); @@ -399,7 +489,7 @@ class StringField : public Field { } private: - template + template friend class StringArrayField; const std::shared_ptr& GetRuntime() const { @@ -430,18 +520,56 @@ class NonEmbeddedStringField { explicit NonEmbeddedStringField(const Message* msg, uint32_t absolute_binary_offset) : msg_(msg), absolute_binary_offset_(absolute_binary_offset) {} + NonEmbeddedStringField(const NonEmbeddedStringField&) = default; + NonEmbeddedStringField(NonEmbeddedStringField&&) = default; + NonEmbeddedStringField& operator=(const NonEmbeddedStringField& other) { + if (this == &other) { + return *this; + } + if (other.IsPlaceholder()) { + return *this; + } + Set(other.Get()); + return *this; + } + NonEmbeddedStringField& operator=(NonEmbeddedStringField&& other) noexcept = + default; + operator std::string_view() const { return Get(); } + NonEmbeddedStringField& operator=(const std::string& s) { + Set(s); + return *this; + } + NonEmbeddedStringField& operator=(std::string_view s) { + Set(s); + return *this; + } + NonEmbeddedStringField& operator=(const char* s) { + ::toolbelt::PayloadBuffer::SetString( + GetBufferAddr(), std::string_view(s, std::strlen(s)), + absolute_binary_offset_); + return *this; + } std::string_view Get() const { + if (IsPlaceholder()) { + return {}; + } return GetBuffer()->GetStringView(absolute_binary_offset_); } template void Set(Str s) { + if (IsPlaceholder()) { + return; + } ::toolbelt::PayloadBuffer::SetString(GetBufferAddr(), s, absolute_binary_offset_); } void Clear() { + if (IsPlaceholder()) { + return; + } ::toolbelt::PayloadBuffer::ClearString(GetBufferAddr(), absolute_binary_offset_); } @@ -454,10 +582,16 @@ class NonEmbeddedStringField { } size_t size() const { + if (IsPlaceholder()) { + return 0; + } return GetBuffer()->StringSize(absolute_binary_offset_); } const char* data() const { + if (IsPlaceholder()) { + return ""; + } return GetBuffer()->StringData(absolute_binary_offset_); } bool empty() const { return size() == 0; } @@ -477,9 +611,9 @@ class NonEmbeddedStringField { return &msg_->runtime->pb; } - const Message* msg_; + const Message* msg_ = nullptr; ::toolbelt::BufferOffset - absolute_binary_offset_; // Offset into + absolute_binary_offset_ = 0; // Offset into // ::toolbelt::PayloadBuffer of // toolbelt::StringHeader }; @@ -508,6 +642,31 @@ class IndirectMessageField : public Field { source_offset_(source_offset), relative_binary_offset_(relative_binary_offset), msg_(InternalDefault{}) {} + IndirectMessageField(const IndirectMessageField&) = default; + IndirectMessageField(IndirectMessageField&&) = default; + IndirectMessageField& operator=(const IndirectMessageField& other) { + if (this == &other) { + return *this; + } + if (other.IsPresent()) { + if (absl::Status s = Mutable()->CloneFrom(other.Get()); !s.ok()) { + return *this; + } + } else { + Clear(); + } + ResetFieldCache(); + return *this; + } + IndirectMessageField& operator=(IndirectMessageField&& other) noexcept { + return operator=(static_cast(other)); + } + operator const MessageType&() const { return Get(); } + const MessageType& operator*() const { return Get(); } + const MessageType* operator->() const { return &Get(); } + operator MessageType&() { return *Mutable(); } + MessageType& operator*() { return *Mutable(); } + MessageType* operator->() { return Mutable(); } const MessageType& Msg() const { return msg_; } MessageType& MutableMsg() { return msg_; } @@ -588,7 +747,7 @@ class IndirectMessageField : public Field { } bool operator==(const IndirectMessageField& other) const { - return msg_ != other.msg_; + return Get() == other.Get(); } bool operator!=(const IndirectMessageField& other) const { return !(*this == other); @@ -657,6 +816,12 @@ class IndirectMessageField : public Field { return msg_.Deserialize(sub_buffer); } + void SyncToPayload() const { + if (IsPresent()) { + Get().SyncToPayload(); + } + } + void Indent(int indent) const { Field::Indent(indent); msg_.Indent(indent); @@ -696,17 +861,40 @@ class MessageObject { explicit MessageObject(std::shared_ptr runtime, uint32_t absolute_binary_offset) : msg_(runtime, absolute_binary_offset) {} + MessageObject(const MessageObject& other) + : msg_(other.msg_.runtime, other.msg_.absolute_binary_offset), + indent_(other.indent_) {} + MessageObject(MessageObject&& other) noexcept + : msg_(other.msg_.runtime, other.msg_.absolute_binary_offset), + indent_(other.indent_) {} + MessageObject& operator=(const MessageObject& other) { + if (this == &other) { + return *this; + } + this->~MessageObject(); + new (this) MessageObject(other); + return *this; + } + MessageObject& operator=(MessageObject&& other) noexcept { + if (this == &other) { + return *this; + } + this->~MessageObject(); + new (this) MessageObject(std::move(other)); + return *this; + } const MessageType& Get() const { return msg_; } const MessageType& operator*() const { return msg_; } MessageType& operator*() { return msg_; } + const MessageType* operator->() const { return &msg_; } MessageType* operator->() { return &msg_; } MessageType* Mutable() { return &msg_; } bool operator==(const MessageObject& other) const { - return msg_ != other.msg_; + return msg_ == other.msg_; } bool operator!=(const MessageObject& other) const { return !(*this == other); diff --git a/phaser/runtime/message.h b/phaser/runtime/message.h index 5f72ddc..b38b8b1 100644 --- a/phaser/runtime/message.h +++ b/phaser/runtime/message.h @@ -243,6 +243,7 @@ struct Message { virtual std::string GetFullName() const { return "phaser.Message"; } virtual void Clear() {} virtual void CopyFrom(const Message& /*src*/) {} + virtual void SyncToPayload() const {} std::shared_ptr runtime; ::toolbelt::BufferOffset absolute_binary_offset; @@ -363,13 +364,23 @@ struct Message { int32_t FindFieldId(uint32_t field_number) const; void* BinaryData() const { + SyncToPayload(); return runtime->pb->ToAddress(absolute_binary_offset); } - void* Data() const { return reinterpret_cast(runtime->pb); } + void* Data() const { + SyncToPayload(); + return reinterpret_cast(runtime->pb); + } - size_t Size() const { return runtime->pb->Size(); } - size_t ZeroCopySize() const { return runtime->pb->Size(); } + size_t Size() const { + SyncToPayload(); + return runtime->pb->Size(); + } + size_t ZeroCopySize() const { + SyncToPayload(); + return runtime->pb->Size(); + } }; ::toolbelt::PayloadBuffer* NewDynamicBuffer( diff --git a/phaser/runtime/ros.h b/phaser/runtime/ros.h new file mode 100644 index 0000000..8b439e2 --- /dev/null +++ b/phaser/runtime/ros.h @@ -0,0 +1,220 @@ +// Copyright 2024-2026 David Allison +// All Rights Reserved. +// See LICENSE file for licensing information. + +#pragma once + +// ROS1 compatibility proxies for protobuf message fields. This header is only +// included by generated files that use a ROS intrinsic, so non-ROS Phaser +// users do not need ROS headers or libraries. + +#include +#include + +#include +#include +#include +#include + +#include "phaser/runtime/fields.h" + +namespace phaser { +namespace internal { + +struct RosTimeTraits { + using RosType = ::ros::Time; + + template + static void Load(const Backend& backend, RosType& value) { + value.sec = static_cast(backend.seconds()); + value.nsec = static_cast(backend.nanos()); + } + + template + static void Store(const RosType& value, Backend& backend) { + backend.set_seconds(static_cast(value.sec)); + backend.set_nanos(static_cast(value.nsec)); + } + + static void Print(std::ostream& os, const RosType& value) { + os << "sec: " << value.sec << " nsec: " << value.nsec; + } +}; + +struct RosDurationTraits { + using RosType = ::ros::Duration; + + template + static void Load(const Backend& backend, RosType& value) { + value.sec = static_cast(backend.seconds()); + value.nsec = static_cast(backend.nanos()); + } + + template + static void Store(const RosType& value, Backend& backend) { + backend.set_seconds(static_cast(value.sec)); + backend.set_nanos(static_cast(value.nsec)); + } + + static void Print(std::ostream& os, const RosType& value) { + os << "sec: " << value.sec << " nsec: " << value.nsec; + } +}; + +struct RosHeaderTraits { + using RosType = ::std_msgs::Header; + + template + static void Load(const Backend& backend, RosType& value) { + value.seq = static_cast(backend.seq.Get()); + value.stamp = backend.stamp.Get(); + value.frame_id = std::string(backend.frame_id.Get()); + } + + template + static void Store(const RosType& value, Backend& backend) { + backend.seq = value.seq; + backend.stamp = value.stamp; + backend.frame_id = value.frame_id; + backend.SyncToPayload(); + } + + static void Print(std::ostream& os, const RosType& value) { + os << "seq: " << value.seq << " stamp {"; + RosTimeTraits::Print(os, value.stamp); + os << "} frame_id: \"" << value.frame_id << "\""; + } +}; + +} // namespace internal + +template +class RosMessageField : public IndirectMessageField { + public: + using Base = IndirectMessageField; + using RosType = typename Traits::RosType; + using Base::Base; + + RosMessageField() = default; + RosMessageField(const RosMessageField&) = default; + RosMessageField(RosMessageField&&) = default; + + RosMessageField& operator=(const RosMessageField& other) { + if (this != &other) { + Set(other.Get()); + } + return *this; + } + + RosMessageField& operator=(RosMessageField&& other) { + if (this != &other) { + Set(other.Get()); + } + return *this; + } + + RosMessageField& operator=(const RosType& value) { + Set(value); + return *this; + } + + operator const RosType&() const { return Get(); } + operator RosType&() { return MutableRos(); } + + const RosType& operator*() const { return Get(); } + RosType& operator*() { return MutableRos(); } + const RosType* operator->() const { return &Get(); } + RosType* operator->() { return &MutableRos(); } + + const RosType& Get() const { + LoadCache(); + return cache_; + } + + RosType& MutableRos() { + LoadCache(); + dirty_ = true; + return cache_; + } + + void Set(const RosType& value) { + cache_ = value; + cache_loaded_ = true; + dirty_ = true; + } + + bool IsPresent() const { return dirty_ || Base::IsPresent(); } + + void Clear() { + Base::Clear(); + cache_ = RosType(); + cache_loaded_ = false; + dirty_ = false; + } + + void SyncToPayload() const { + if (!dirty_) { + if (Base::IsPresent()) { + Base::Get().SyncToPayload(); + } + return; + } + Backend* backend = const_cast(this)->Base::Mutable(); + Traits::Store(cache_, *backend); + dirty_ = false; + cache_loaded_ = true; + } + + size_t SerializedSize() const { + SyncToPayload(); + return Base::SerializedSize(); + } + + absl::Status Serialize(ProtoBuffer& buffer) const { + SyncToPayload(); + return Base::Serialize(buffer); + } + + absl::Status Deserialize(ProtoBuffer& buffer) { + absl::Status status = Base::Deserialize(buffer); + if (status.ok()) { + cache_loaded_ = false; + dirty_ = false; + } + return status; + } + + friend std::ostream& operator<<(std::ostream& os, + const RosMessageField& field) { + Traits::Print(os, field.Get()); + return os; + } + + private: + void LoadCache() const { + if (cache_loaded_ || dirty_) { + return; + } + cache_ = RosType(); + if (Base::IsPresent()) { + Traits::Load(Base::Get(), cache_); + } + cache_loaded_ = true; + } + + mutable RosType cache_; + mutable bool cache_loaded_ = false; + mutable bool dirty_ = false; +}; + +template +using RosTimeField = RosMessageField; + +template +using RosDurationField = + RosMessageField; + +template +using RosHeaderField = RosMessageField; + +} // namespace phaser diff --git a/phaser/runtime/runtime.h b/phaser/runtime/runtime.h index c930038..a5580e4 100644 --- a/phaser/runtime/runtime.h +++ b/phaser/runtime/runtime.h @@ -6,6 +6,7 @@ #include #include "phaser/runtime/any.h" +#include "phaser/runtime/arrays.h" #include "phaser/runtime/fields.h" #include "phaser/runtime/iterators.h" #include "phaser/runtime/message.h" diff --git a/phaser/runtime/union.h b/phaser/runtime/union.h index 1be4a93..3189c46 100644 --- a/phaser/runtime/union.h +++ b/phaser/runtime/union.h @@ -11,6 +11,9 @@ #include #include #include +#include +#include +#include #include #include "absl/container/flat_hash_map.h" @@ -503,6 +506,74 @@ class UnionField : public Field { relative_binary_offset_(relative_binary_offset), field_numbers_(field_numbers) {} + size_t index() const { + const int32_t discriminator = Discriminator(); + for (size_t i = 0; i < field_numbers_.size(); ++i) { + if (discriminator == static_cast(field_numbers_[i])) { + return i; + } + } + return std::variant_npos; + } + + int32_t case_number() const { return Discriminator(); } + bool valueless_by_exception() const { + return index() == std::variant_npos; + } + + void reset() { ClearActive(); } + + template + bool holds_alternative() const { + static_assert(Alternative::kIndex < sizeof...(T), + "oneof alternative index is out of range"); + return IsPresent(); + } + + template + decltype(auto) get() const { + if (!holds_alternative()) { + throw std::bad_variant_access(); + } + if constexpr (Alternative::kIsMessage) { + return GetReference(); + } else { + return GetValue(); + } + } + + template + decltype(auto) get() { + if (!holds_alternative()) { + throw std::bad_variant_access(); + } + if constexpr (Alternative::kIsMessage) { + return *Mutable(); + } else { + return GetValue(); + } + } + + template + decltype(auto) emplace(Args&&... args) { + static_assert(Alternative::kIndex < sizeof...(T), + "oneof alternative index is out of range"); + if constexpr (Alternative::kIsMessage) { + static_assert(sizeof...(Args) == 0, + "message oneof alternatives are emplaced empty and then " + "mutated through the returned reference"); + return *Mutable(); + } else { + using Value = typename Alternative::value_type; + Value value(std::forward(args)...); + Set(value); + return GetValue(); + } + } + template const F& GetReference() const { int32_t relative_offset = Message::GetMessage(this, source_offset_) @@ -550,6 +621,7 @@ class UnionField : public Field { template void Set(const U& v) { + PrepareArm(); // Write the field number into the discriminator. int32_t* discrim = GetRuntime()->template ToAddress( GetMessageBinaryStart() + relative_binary_offset_); @@ -563,6 +635,7 @@ class UnionField : public Field { template U* Mutable() { + PrepareArm(); // Write the field number into the discriminator. int32_t* discrim = GetRuntime()->template ToAddress( GetMessageBinaryStart() + relative_binary_offset_); @@ -578,6 +651,7 @@ class UnionField : public Field { // Only valid for strings and bytes. template absl::Span Allocate(size_t size) { + PrepareArm(); // Write the field number into the discriminator. int32_t* discrim = GetRuntime()->template ToAddress( GetMessageBinaryStart() + relative_binary_offset_); @@ -672,6 +746,7 @@ class UnionField : public Field { if (relative_offset < 0) { // Field not present. return absl::OkStatus(); } + PrepareArm(); if (absl::Status status = std::get(value_).Deserialize( buffer, GetRuntime(), GetMessageBinaryStart() + @@ -693,6 +768,7 @@ class UnionField : public Field { if (relative_offset < 0) { // Field not present. return absl::OkStatus(); } + PrepareArm(); int32_t* discrim = GetRuntime()->template ToAddress( GetMessageBinaryStart() + relative_binary_offset_); *discrim = static_cast(field_numbers_[Id]); @@ -719,6 +795,7 @@ class UnionField : public Field { template void SetOffset(toolbelt::BufferOffset offset) { + PrepareArm(); int32_t* discrim = GetRuntime()->template ToAddress( GetMessageBinaryStart() + relative_binary_offset_); *discrim = static_cast(field_numbers_[Id]); @@ -732,6 +809,48 @@ class UnionField : public Field { } private: + template + void ClearByIndex(size_t active_index) { + if constexpr (Id < sizeof...(T)) { + if (active_index == Id) { + Clear(Id)>(); + return; + } + ClearByIndex(active_index); + } + } + + void ClearActive() { + const size_t active_index = index(); + if (active_index != std::variant_npos) { + ClearByIndex(active_index); + return; + } + int32_t* discriminator = GetRuntime()->template ToAddress( + GetMessageBinaryStart() + relative_binary_offset_); + if (discriminator != nullptr) { + *discriminator = 0; + } + } + + template + void PrepareArm() { + static_assert(Id >= 0 && Id < static_cast(sizeof...(T)), + "oneof arm index is out of range"); + const int32_t desired = static_cast(field_numbers_[Id]); + const int32_t current = Discriminator(); + if (current == desired) { + return; + } + ClearActive(); + ::toolbelt::BufferOffset* slot = + GetRuntime()->template ToAddress<::toolbelt::BufferOffset>( + GetMessageBinaryStart() + relative_binary_offset_ + 4); + if (slot != nullptr) { + *slot = 0; + } + } + ::toolbelt::PayloadBuffer* GetBuffer() const { return Message::GetBuffer(this, source_offset_); } diff --git a/phaser/runtime/vectors.h b/phaser/runtime/vectors.h index c593c17..1f2f6ca 100644 --- a/phaser/runtime/vectors.h +++ b/phaser/runtime/vectors.h @@ -11,6 +11,7 @@ #include #include +#include #include #include "absl/container/flat_hash_map.h" @@ -133,8 +134,10 @@ class PrimitiveVectorField : public Field { : Field(id, number), source_offset_(source_offset), relative_binary_offset_(relative_binary_offset) {} + PrimitiveVectorField(const PrimitiveVectorField&) = default; + PrimitiveVectorField(PrimitiveVectorField&&) = default; - const T& operator[](int index) { + T& operator[](int index) { static T empty; T* base = GetRuntime()->template ToAddress(BaseOffset()); if (base == nullptr) { @@ -205,8 +208,27 @@ class PrimitiveVectorField : public Field { } void clear() { Clear(); } // STL compatibility. + PrimitiveVectorField& operator=(const PrimitiveVectorField& other) { + if (this == &other) { + return *this; + } + Clear(); + reserve(other.size()); + for (size_t i = 0; i < other.size(); i++) { + push_back(other[i]); + } + ResetFieldCache(); + return *this; + } + PrimitiveVectorField& operator=(PrimitiveVectorField&& other) noexcept { + return operator=(static_cast(other)); + } + size_t size() const { return NumElements(); } - T* data() const { return GetRuntime()->template ToAddress(BaseOffset()); } + T* data() { return GetRuntime()->template ToAddress(BaseOffset()); } + const T* data() const { + return GetRuntime()->template ToAddress(BaseOffset()); + } size_t Size() const { return NumElements(); } absl::Span AsMutableSpan() { @@ -236,7 +258,12 @@ class PrimitiveVectorField : public Field { bool empty() const { return size() == 0; } size_t capacity() const { - ::toolbelt::BufferOffset* addr = BaseOffset(); + ::toolbelt::BufferOffset offset = BaseOffset(); + if (offset == 0) { + return 0; + } + ::toolbelt::BufferOffset* addr = + GetRuntime()->template ToAddress<::toolbelt::BufferOffset>(offset); if (addr == nullptr) { return 0; } @@ -462,13 +489,16 @@ class EnumVectorField : public Field { : Field(id, number), source_offset_(source_offset), relative_binary_offset_(relative_binary_offset) {} + EnumVectorField(const EnumVectorField&) = default; + EnumVectorField(EnumVectorField&&) = default; using T = typename std::underlying_type::type; - Enum operator[](int index) { + Enum& operator[](int index) { T* base = GetRuntime()->template ToAddress(BaseOffset()); if (base == nullptr) { - return static_cast(0); + static Enum empty = static_cast(0); + return *reinterpret_cast(&empty); } return *reinterpret_cast(&base[index]); } @@ -557,8 +587,27 @@ class EnumVectorField : public Field { } void clear() { Clear(); } // STL compatibility. + EnumVectorField& operator=(const EnumVectorField& other) { + if (this == &other) { + return *this; + } + Clear(); + reserve(other.size()); + for (size_t i = 0; i < other.size(); i++) { + push_back(other[i]); + } + ResetFieldCache(); + return *this; + } + EnumVectorField& operator=(EnumVectorField&& other) noexcept { + return operator=(static_cast(other)); + } + size_t size() const { return NumElements(); } - Enum* data() const { GetRuntime()->template ToAddress(BaseOffset()); } + Enum* data() { return GetRuntime()->template ToAddress(BaseOffset()); } + const Enum* data() const { + return GetRuntime()->template ToAddress(BaseOffset()); + } bool empty() const { return size() == 0; } size_t Size() const { return NumElements(); } @@ -754,6 +803,8 @@ class MessageVectorField : public Field { : Field(id, number), source_offset_(source_offset), relative_binary_offset_(relative_binary_offset) {} + MessageVectorField(const MessageVectorField&) = default; + MessageVectorField(MessageVectorField&&) = default; const MessageObject& operator[](int index) const { int32_t offset = FindFieldOffset(source_offset_); @@ -779,14 +830,90 @@ class MessageVectorField : public Field { return msgs_[static_cast(index)]; } - MessageObject& front() { return msgs_.front(); } - const MessageObject& front() const { return msgs_.front(); } - MessageObject& back() { return msgs_.back(); } - const MessageObject& back() const { return msgs_.back(); } + MessageObject& operator[](int index) { + return const_cast&>( + static_cast(this)->operator[](index)); + } -#define RTYPE std::vector> - DECLARE_RELAY_VECTOR_BITS(MessageObject, RTYPE, msgs_) -#undef RTYPE + MessageObject& front() { + Populate(); + return msgs_.front(); + } + const MessageObject& front() const { + Populate(); + return msgs_.front(); + } + MessageObject& back() { + Populate(); + return msgs_.back(); + } + const MessageObject& back() const { + Populate(); + return msgs_.back(); + } + + using value_type = MessageObject; + using reference = value_type&; + using const_reference = value_type&; + using pointer = value_type*; + using const_pointer = const value_type*; + using size_type = size_t; + using difference_type = ptrdiff_t; + using iterator = typename std::vector>::iterator; + using const_iterator = typename std::vector>::const_iterator; + using reverse_iterator = + typename std::vector>::reverse_iterator; + using const_reverse_iterator = + typename std::vector>::const_reverse_iterator; + + iterator begin() { + Populate(); + return msgs_.begin(); + } + iterator end() { + Populate(); + return msgs_.end(); + } + reverse_iterator rbegin() { + Populate(); + return msgs_.rbegin(); + } + reverse_iterator rend() { + Populate(); + return msgs_.rend(); + } + const_iterator begin() const { + Populate(); + return msgs_.begin(); + } + const_iterator end() const { + Populate(); + return msgs_.end(); + } + const_iterator cbegin() const { + Populate(); + return msgs_.cbegin(); + } + const_iterator cend() const { + Populate(); + return msgs_.cend(); + } + const_reverse_iterator rbegin() const { + Populate(); + return msgs_.rbegin(); + } + const_reverse_iterator rend() const { + Populate(); + return msgs_.rend(); + } + const_reverse_iterator crbegin() const { + Populate(); + return msgs_.crbegin(); + } + const_reverse_iterator crend() const { + Populate(); + return msgs_.crend(); + } void push_back(const T& v) { ::toolbelt::BufferOffset offset = v.absolute_binary_offset; @@ -940,6 +1067,26 @@ class MessageVectorField : public Field { bool empty() const { return size() == 0; } size_t Size() const { return NumElements(); } + MessageVectorField& operator=(const MessageVectorField& other) { + if (this == &other) { + return *this; + } + Clear(); + other.Populate(); + reserve(other.size()); + for (size_t i = 0; i < other.size(); i++) { + T* m = Add(); + if (absl::Status s = m->CloneFrom(other[i].Get()); !s.ok()) { + return *this; + } + } + ResetFieldCache(); + return *this; + } + MessageVectorField& operator=(MessageVectorField&& other) noexcept { + return operator=(static_cast(other)); + } + ::toolbelt::BufferOffset BinaryEndOffset() const { return relative_binary_offset_ + sizeof(toolbelt::VectorHeader); } @@ -1024,6 +1171,15 @@ class MessageVectorField : public Field { return absl::OkStatus(); } + void SyncToPayload() const { + Populate(); + for (const auto& message : msgs_) { + if (!message.empty()) { + message.Get().SyncToPayload(); + } + } + } + private: friend FieldIterator; friend FieldIterator; @@ -1063,6 +1219,7 @@ class MessageVectorField : public Field { return Message::GetMessageBinaryStart(this, source_offset_); } + protected: const std::shared_ptr& GetRuntime() const { return Message::GetRuntime(this, source_offset_); } @@ -1097,6 +1254,11 @@ class StringVectorField : public Field { : Field(id, number), source_offset_(source_offset), relative_binary_offset_(relative_binary_offset) {} + StringVectorField(const StringVectorField&) = default; + StringVectorField(StringVectorField&& other) noexcept + : Field(std::move(other)), + source_offset_(other.source_offset_), + relative_binary_offset_(other.relative_binary_offset_) {} const NonEmbeddedStringField& operator[](int index) const { int32_t offset = FindFieldOffset(source_offset_); @@ -1122,22 +1284,121 @@ class StringVectorField : public Field { return strings_[static_cast(index)]; } -#define RTYPE std::vector - DECLARE_RELAY_VECTOR_BITS(NonEmbeddedStringField, RTYPE, strings_) -#undef RTYPE + NonEmbeddedStringField& operator[](int index) { + return const_cast( + static_cast(this)->operator[](index)); + } + + using value_type = NonEmbeddedStringField; + using reference = value_type&; + using const_reference = value_type&; + using pointer = value_type*; + using const_pointer = const value_type*; + using size_type = size_t; + using difference_type = ptrdiff_t; + using iterator = typename std::vector::iterator; + using const_iterator = + typename std::vector::const_iterator; + using reverse_iterator = + typename std::vector::reverse_iterator; + using const_reverse_iterator = + typename std::vector::const_reverse_iterator; + + iterator begin() { + Populate(); + return strings_.begin(); + } + iterator end() { + Populate(); + return strings_.end(); + } + reverse_iterator rbegin() { + Populate(); + return strings_.rbegin(); + } + reverse_iterator rend() { + Populate(); + return strings_.rend(); + } + const_iterator begin() const { + Populate(); + return strings_.begin(); + } + const_iterator end() const { + Populate(); + return strings_.end(); + } + const_iterator cbegin() const { + Populate(); + return strings_.cbegin(); + } + const_iterator cend() const { + Populate(); + return strings_.cend(); + } + const_reverse_iterator rbegin() const { + Populate(); + return strings_.rbegin(); + } + const_reverse_iterator rend() const { + Populate(); + return strings_.rend(); + } + const_reverse_iterator crbegin() const { + Populate(); + return strings_.crbegin(); + } + const_reverse_iterator crend() const { + Populate(); + return strings_.crend(); + } size_t size() const { return NumElements(); } NonEmbeddedStringField* data() { Populate(); return strings_.data(); } + const NonEmbeddedStringField* data() const { + Populate(); + return strings_.data(); + } bool empty() const { return size() == 0; } size_t Size() const { return NumElements(); } - NonEmbeddedStringField& front() { return strings_.front(); } - const NonEmbeddedStringField& front() const { return strings_.front(); } - NonEmbeddedStringField& back() { return strings_.back(); } - const NonEmbeddedStringField& back() const { return strings_.back(); } + NonEmbeddedStringField& front() { + Populate(); + return strings_.front(); + } + const NonEmbeddedStringField& front() const { + Populate(); + return strings_.front(); + } + NonEmbeddedStringField& back() { + Populate(); + return strings_.back(); + } + const NonEmbeddedStringField& back() const { + Populate(); + return strings_.back(); + } + + StringVectorField& operator=(const StringVectorField& other) { + if (this == &other) { + return *this; + } + Clear(); + other.Populate(); + reserve(other.size()); + for (size_t i = 0; i < other.size(); i++) { + push_back(other[i].Get()); + } + ResetFieldCache(); + return *this; + } + + StringVectorField& operator=(StringVectorField&& other) noexcept { + return operator=(static_cast(other)); + } template void push_back(Str s) { @@ -1221,7 +1482,9 @@ class StringVectorField : public Field { void Clear() { for (auto& s : strings_) { - s.Clear(); + if (!s.IsPlaceholder()) { + s.Clear(); + } } strings_.clear(); ::toolbelt::PayloadBuffer::VectorClear<::toolbelt::BufferOffset>( diff --git a/phaser/testdata/BUILD b/phaser/testdata/BUILD index 64d1594..207f4f0 100644 --- a/phaser/testdata/BUILD +++ b/phaser/testdata/BUILD @@ -1,5 +1,7 @@ load("@com_google_protobuf//bazel:cc_proto_library.bzl", "cc_proto_library") load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") +load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library") +load("@rules_shell//shell:sh_test.bzl", "sh_test") load("//phaser:phaser_library.bzl", "phaser_library") package(default_visibility = ["//visibility:public"]) @@ -66,6 +68,139 @@ phaser_library( deps = [":foo_proto"], ) +proto_library( + name = "ros_compile_proto", + srcs = ["RosCompile.proto"], + deps = ["//phaser:options_proto"], +) + +cc_proto_library( + name = "ros_compile_cc_proto", + deps = [":ros_compile_proto"], +) + +phaser_library( + name = "ros_compile_phaser", + add_namespace = "phaser", + frontend = "ros", + runtime = "//phaser/runtime:phaser_runtime", + deps = [":ros_compile_proto"], +) + +cc_library( + name = "ros1_shim", + hdrs = glob(["ros_shim/**/*.h"]), + strip_include_prefix = "ros_shim", +) + +proto_library( + name = "ros_header_proto", + srcs = ["RosHeader.proto"], + deps = ["@com_google_protobuf//:timestamp_proto"], +) + +proto_library( + name = "ros_intrinsics_proto", + srcs = ["RosIntrinsics.proto"], + deps = [ + ":ros_header_proto", + "//phaser:options_proto", + "@com_google_protobuf//:duration_proto", + "@com_google_protobuf//:timestamp_proto", + ], +) + +cc_proto_library( + name = "ros_intrinsics_cc_proto", + deps = [":ros_intrinsics_proto"], +) + +phaser_library( + name = "ros_intrinsics_phaser", + add_namespace = "phaser", + cc_deps = [":ros1_shim"], + frontend = "ros", + runtime = "//phaser/runtime:phaser_runtime", + deps = [":ros_intrinsics_proto"], +) + +proto_library( + name = "ros_intrinsics_protobuf_frontend_proto", + srcs = ["RosIntrinsicsProtobufFrontend.proto"], + deps = [ + ":ros_header_proto", + "//phaser:options_proto", + "@com_google_protobuf//:duration_proto", + "@com_google_protobuf//:timestamp_proto", + ], +) + +phaser_library( + name = "ros_intrinsics_protobuf_phaser", + add_namespace = "protobuf_phaser", + frontend = "protobuf", + runtime = "//phaser/runtime:phaser_runtime", + deps = [":ros_intrinsics_protobuf_frontend_proto"], +) + +cc_binary( + name = "ros_intrinsics_phaser_wire_tool", + srcs = ["ros_intrinsics_phaser_wire_tool.cc"], + deps = [":ros_intrinsics_phaser"], +) + +cc_binary( + name = "ros_intrinsics_protobuf_wire_tool", + srcs = ["ros_intrinsics_protobuf_wire_tool.cc"], + deps = [":ros_intrinsics_cc_proto"], +) + +sh_test( + name = "ros_intrinsics_wire_compatibility_test", + srcs = ["ros_intrinsics_wire_compatibility_test.sh"], + args = [ + "$(rootpath :ros_intrinsics_phaser_wire_tool)", + "$(rootpath :ros_intrinsics_protobuf_wire_tool)", + ], + data = [ + ":ros_intrinsics_phaser_wire_tool", + ":ros_intrinsics_protobuf_wire_tool", + ], +) + +sh_test( + name = "invalid_array_size_test", + srcs = ["invalid_array_size_test.sh"], + args = [ + "$(rootpath @com_google_protobuf//:protoc)", + "$(rootpath //phaser/compiler:phaser)", + "$(rootpath InvalidArraySize.proto)", + ], + data = [ + "InvalidArraySize.proto", + "//phaser:options_proto", + "//phaser/compiler:phaser", + "@com_google_protobuf//:protoc", + "@com_google_protobuf//:well_known_protos", + ], +) + +sh_test( + name = "invalid_ros_intrinsic_test", + srcs = ["invalid_ros_intrinsic_test.sh"], + args = [ + "$(rootpath @com_google_protobuf//:protoc)", + "$(rootpath //phaser/compiler:phaser)", + "$(rootpath InvalidRosIntrinsic.proto)", + ], + data = [ + "InvalidRosIntrinsic.proto", + "//phaser/compiler:phaser", + "@com_google_protobuf//:protoc", + "@com_google_protobuf//:well_known_protos", + ], +) + proto_library( name = "coverage_proto", srcs = ["coverage.proto"], diff --git a/phaser/testdata/InvalidArraySize.proto b/phaser/testdata/InvalidArraySize.proto new file mode 100644 index 0000000..99add3c --- /dev/null +++ b/phaser/testdata/InvalidArraySize.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +package foo.bar; + +import "phaser/options.proto"; + +message InvalidArraySizeMessage { + int32 bad = 1 [(phaser.array_size) = 4]; +} diff --git a/phaser/testdata/InvalidRosIntrinsic.proto b/phaser/testdata/InvalidRosIntrinsic.proto new file mode 100644 index 0000000..a44740c --- /dev/null +++ b/phaser/testdata/InvalidRosIntrinsic.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +package invalid; + +import "google/protobuf/timestamp.proto"; + +message InvalidRosIntrinsic { + repeated google.protobuf.Timestamp stamps = 1; +} diff --git a/phaser/testdata/RosCompile.proto b/phaser/testdata/RosCompile.proto new file mode 100644 index 0000000..58ff714 --- /dev/null +++ b/phaser/testdata/RosCompile.proto @@ -0,0 +1,40 @@ +syntax = "proto3"; + +package foo.bar; + +import "phaser/options.proto"; + +enum RosColor { + ROS_COLOR_UNSPECIFIED = 0; + ROS_COLOR_RED = 1; + ROS_COLOR_BLUE = 2; +} + +message RosInner { + int32 id = 1; +} + +message RosCompileMessage { + int32 x = 1; + string name = 2; + bool flag = 3; + double value = 4; + RosColor color = 5; + RosInner inner = 6; + repeated int32 xs = 7; + repeated string names = 8; + repeated RosColor colors = 9; + repeated RosInner inners = 10; + + repeated int32 fixed_ints = 11 [(phaser.array_size) = 4]; + repeated RosColor fixed_colors = 12 [(phaser.array_size) = 3]; + repeated string fixed_names = 13 [(phaser.array_size) = 2]; + repeated RosInner fixed_inners = 14 [(phaser.array_size) = 2]; + + oneof choice { + int32 choice_count = 15; + int32 choice_code = 16; + string choice_name = 17; + RosInner choice_inner = 18; + } +} diff --git a/phaser/testdata/RosHeader.proto b/phaser/testdata/RosHeader.proto new file mode 100644 index 0000000..a94d523 --- /dev/null +++ b/phaser/testdata/RosHeader.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; + +package std_msgs; + +import "google/protobuf/timestamp.proto"; + +message Header { + uint32 seq = 1; + google.protobuf.Timestamp stamp = 2; + string frame_id = 3; +} diff --git a/phaser/testdata/RosIntrinsics.proto b/phaser/testdata/RosIntrinsics.proto new file mode 100644 index 0000000..5ef6d44 --- /dev/null +++ b/phaser/testdata/RosIntrinsics.proto @@ -0,0 +1,32 @@ +syntax = "proto3"; + +package foo.bar; + +import "google/protobuf/duration.proto"; +import "google/protobuf/timestamp.proto"; +import "phaser/options.proto"; +import "phaser/testdata/RosHeader.proto"; + +message WireChild { + int32 id = 1; + string label = 2; +} + +message RosIntrinsicMessage { + google.protobuf.Timestamp stamp = 1; + google.protobuf.Duration timeout = 2; + std_msgs.Header header = 3; + int32 count = 4; + string name = 5; + repeated int32 samples = 6; + repeated string tags = 7; + repeated WireChild children = 8; + repeated string fixed_names = 9 [(phaser.array_size) = 2]; + repeated WireChild fixed_children = 10 [(phaser.array_size) = 2]; + + oneof choice { + int32 choice_number = 11; + string choice_text = 12; + WireChild choice_child = 13; + } +} diff --git a/phaser/testdata/RosIntrinsicsProtobufFrontend.proto b/phaser/testdata/RosIntrinsicsProtobufFrontend.proto new file mode 100644 index 0000000..a2dd101 --- /dev/null +++ b/phaser/testdata/RosIntrinsicsProtobufFrontend.proto @@ -0,0 +1,32 @@ +syntax = "proto3"; + +package foo.bar.pb; + +import "google/protobuf/duration.proto"; +import "google/protobuf/timestamp.proto"; +import "phaser/options.proto"; +import "phaser/testdata/RosHeader.proto"; + +message WireChild { + int32 id = 1; + string label = 2; +} + +message RosIntrinsicMessage { + google.protobuf.Timestamp stamp = 1; + google.protobuf.Duration timeout = 2; + std_msgs.Header header = 3; + int32 count = 4; + string name = 5; + repeated int32 samples = 6; + repeated string tags = 7; + repeated WireChild children = 8; + repeated string fixed_names = 9 [(phaser.array_size) = 2]; + repeated WireChild fixed_children = 10 [(phaser.array_size) = 2]; + + oneof choice { + int32 choice_number = 11; + string choice_text = 12; + WireChild choice_child = 13; + } +} diff --git a/phaser/testdata/invalid_array_size_test.sh b/phaser/testdata/invalid_array_size_test.sh new file mode 100755 index 0000000..2242d55 --- /dev/null +++ b/phaser/testdata/invalid_array_size_test.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +protoc="$1" +plugin="$2" +invalid_proto="$3" +out_dir="$(mktemp -d)" +trap 'rm -rf "${out_dir}"' EXIT + +root="${TEST_SRCDIR}/${TEST_WORKSPACE}" +proto_include="${TEST_SRCDIR}/protobuf+/src" +if ! "${protoc}" \ + --plugin="protoc-gen-phaser=${plugin}" \ + -I"${root}" \ + -I"${proto_include}" \ + --phaser_out="frontend=ros:${out_dir}" \ + "${invalid_proto}" 2>"${out_dir}/err.txt"; then + grep -q "array_size is only valid on repeated fields" "${out_dir}/err.txt" + exit 0 +fi + +echo "expected phaser plugin to reject invalid array_size annotation" >&2 +cat "${out_dir}/err.txt" >&2 || true +exit 1 diff --git a/phaser/testdata/invalid_ros_intrinsic_test.sh b/phaser/testdata/invalid_ros_intrinsic_test.sh new file mode 100755 index 0000000..bf2a8e1 --- /dev/null +++ b/phaser/testdata/invalid_ros_intrinsic_test.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +protoc="$1" +plugin="$2" +invalid_proto="$3" +out_dir="$(mktemp -d)" +trap 'rm -rf "${out_dir}"' EXIT + +root="${TEST_SRCDIR}/${TEST_WORKSPACE}" +proto_include="${TEST_SRCDIR}/protobuf+/src" +if ! "${protoc}" \ + --plugin="protoc-gen-phaser=${plugin}" \ + -I"${root}" \ + -I"${proto_include}" \ + --phaser_out="frontend=ros,add_namespace=phaser:${out_dir}" \ + "${invalid_proto}" 2>"${out_dir}/err.txt"; then + grep -q "must be singular and cannot be in a oneof" "${out_dir}/err.txt" + exit 0 +fi + +echo "expected phaser plugin to reject repeated ROS intrinsic" >&2 +exit 1 diff --git a/phaser/testdata/ros_intrinsics_phaser_wire_tool.cc b/phaser/testdata/ros_intrinsics_phaser_wire_tool.cc new file mode 100644 index 0000000..4fde34a --- /dev/null +++ b/phaser/testdata/ros_intrinsics_phaser_wire_tool.cc @@ -0,0 +1,112 @@ +#include "phaser/testdata/RosIntrinsics.phaser.h" + +#include +#include +#include + +namespace { + +using Message = ::foo::bar::phaser::RosIntrinsicMessage; + +bool Write(const char* path) { + Message message; + message.stamp = ::ros::Time(12, 345); + message.timeout = ::ros::Duration(-4, -500); + + ::std_msgs::Header header; + header.seq = 9; + header.stamp = ::ros::Time(21, 654); + header.frame_id = "map"; + message.header = header; + + message.count = 42; + message.name = "wire"; + message.samples.push_back(3); + message.samples.push_back(5); + message.tags.push_back("front"); + message.tags.push_back("rear"); + + auto* first_child = message.children.Add(); + first_child->id = 101; + first_child->label = "left"; + auto* second_child = message.children.Add(); + second_child->id = 202; + second_child->label = "right"; + + message.fixed_names[0] = "fixed-a"; + message.fixed_names[1] = "fixed-b"; + message.fixed_children[0]->id = 301; + message.fixed_children[0]->label = "fixed-left"; + message.fixed_children[1]->id = 302; + message.fixed_children[1]->label = "fixed-right"; + + using ChoiceChild = Message::ChoiceChildAlternative; + auto& choice = message.choice.emplace(); + choice.id = 404; + choice.label = "selected"; + + std::string wire; + if (!message.SerializeToString(&wire)) { + return false; + } + std::ofstream output(path, std::ios::binary); + output.write(wire.data(), static_cast(wire.size())); + return output.good(); +} + +bool Verify(const char* path) { + std::ifstream input(path, std::ios::binary); + std::string wire((std::istreambuf_iterator(input)), + std::istreambuf_iterator()); + Message message; + if (!input.good() && !input.eof()) { + return false; + } + if (!message.ParseFromString(wire)) { + return false; + } + + if (message.samples.size() != 2 || message.tags.size() != 2 || + message.children.size() != 2 || + !message.choice + .holds_alternative()) { + return false; + } + const auto& choice = + message.choice.get(); + return message.stamp->sec == 12 && message.stamp->nsec == 345 && + message.timeout->sec == -4 && message.timeout->nsec == -500 && + message.header->seq == 9 && message.header->stamp.sec == 21 && + message.header->stamp.nsec == 654 && + message.header->frame_id == "map" && message.count.Get() == 42 && + message.name.Get() == "wire" && message.samples[0] == 3 && + message.samples[1] == 5 && message.tags[0].Get() == "front" && + message.tags[1].Get() == "rear" && + message.children[0]->id.Get() == 101 && + message.children[0]->label.Get() == "left" && + message.children[1]->id.Get() == 202 && + message.children[1]->label.Get() == "right" && + message.fixed_names[0].Get() == "fixed-a" && + message.fixed_names[1].Get() == "fixed-b" && + message.fixed_children[0]->id.Get() == 301 && + message.fixed_children[0]->label.Get() == "fixed-left" && + message.fixed_children[1]->id.Get() == 302 && + message.fixed_children[1]->label.Get() == "fixed-right" && + choice.id.Get() == 404 && choice.label.Get() == "selected"; +} + +} // namespace + +int main(int argc, char** argv) { + if (argc != 3) { + return 2; + } + const std::string command(argv[1]); + if (command == "write") { + return Write(argv[2]) ? 0 : 1; + } + if (command == "verify") { + return Verify(argv[2]) ? 0 : 1; + } + return 2; +} diff --git a/phaser/testdata/ros_intrinsics_protobuf_wire_tool.cc b/phaser/testdata/ros_intrinsics_protobuf_wire_tool.cc new file mode 100644 index 0000000..f80648a --- /dev/null +++ b/phaser/testdata/ros_intrinsics_protobuf_wire_tool.cc @@ -0,0 +1,113 @@ +#include "phaser/testdata/RosIntrinsics.pb.h" + +#include +#include +#include + +namespace { + +using Message = ::foo::bar::RosIntrinsicMessage; + +bool Write(const char* path) { + Message message; + message.mutable_stamp()->set_seconds(12); + message.mutable_stamp()->set_nanos(345); + message.mutable_timeout()->set_seconds(-4); + message.mutable_timeout()->set_nanos(-500); + message.mutable_header()->set_seq(9); + message.mutable_header()->mutable_stamp()->set_seconds(21); + message.mutable_header()->mutable_stamp()->set_nanos(654); + message.mutable_header()->set_frame_id("map"); + + message.set_count(42); + message.set_name("wire"); + message.add_samples(3); + message.add_samples(5); + message.add_tags("front"); + message.add_tags("rear"); + + auto* first_child = message.add_children(); + first_child->set_id(101); + first_child->set_label("left"); + auto* second_child = message.add_children(); + second_child->set_id(202); + second_child->set_label("right"); + + message.add_fixed_names("fixed-a"); + message.add_fixed_names("fixed-b"); + auto* first_fixed_child = message.add_fixed_children(); + first_fixed_child->set_id(301); + first_fixed_child->set_label("fixed-left"); + auto* second_fixed_child = message.add_fixed_children(); + second_fixed_child->set_id(302); + second_fixed_child->set_label("fixed-right"); + + auto* choice = message.mutable_choice_child(); + choice->set_id(404); + choice->set_label("selected"); + + std::string wire; + if (!message.SerializeToString(&wire)) { + return false; + } + std::ofstream output(path, std::ios::binary); + output.write(wire.data(), static_cast(wire.size())); + return output.good(); +} + +bool Verify(const char* path) { + std::ifstream input(path, std::ios::binary); + std::string wire((std::istreambuf_iterator(input)), + std::istreambuf_iterator()); + Message message; + if (!input.good() && !input.eof()) { + return false; + } + if (!message.ParseFromString(wire)) { + return false; + } + + if (message.samples_size() != 2 || message.tags_size() != 2 || + message.children_size() != 2 || message.fixed_names_size() != 2 || + message.fixed_children_size() != 2 || !message.has_choice_child()) { + return false; + } + return message.stamp().seconds() == 12 && + message.stamp().nanos() == 345 && + message.timeout().seconds() == -4 && + message.timeout().nanos() == -500 && + message.header().seq() == 9 && + message.header().stamp().seconds() == 21 && + message.header().stamp().nanos() == 654 && + message.header().frame_id() == "map" && message.count() == 42 && + message.name() == "wire" && message.samples(0) == 3 && + message.samples(1) == 5 && message.tags(0) == "front" && + message.tags(1) == "rear" && message.children(0).id() == 101 && + message.children(0).label() == "left" && + message.children(1).id() == 202 && + message.children(1).label() == "right" && + message.fixed_names(0) == "fixed-a" && + message.fixed_names(1) == "fixed-b" && + message.fixed_children(0).id() == 301 && + message.fixed_children(0).label() == "fixed-left" && + message.fixed_children(1).id() == 302 && + message.fixed_children(1).label() == "fixed-right" && + message.choice_child().id() == 404 && + message.choice_child().label() == "selected"; +} + +} // namespace + +int main(int argc, char** argv) { + if (argc != 3) { + return 2; + } + const std::string command(argv[1]); + if (command == "write") { + return Write(argv[2]) ? 0 : 1; + } + if (command == "verify") { + return Verify(argv[2]) ? 0 : 1; + } + return 2; +} diff --git a/phaser/testdata/ros_intrinsics_wire_compatibility_test.sh b/phaser/testdata/ros_intrinsics_wire_compatibility_test.sh new file mode 100755 index 0000000..5ed97ea --- /dev/null +++ b/phaser/testdata/ros_intrinsics_wire_compatibility_test.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +phaser_tool="$1" +protobuf_tool="$2" +tmp_dir="$(mktemp -d)" +trap 'rm -rf "${tmp_dir}"' EXIT + +phaser_wire="${tmp_dir}/phaser.wire" +protobuf_wire="${tmp_dir}/protobuf.wire" + +"${phaser_tool}" write "${phaser_wire}" +"${protobuf_tool}" verify "${phaser_wire}" + +"${protobuf_tool}" write "${protobuf_wire}" +"${phaser_tool}" verify "${protobuf_wire}" diff --git a/phaser/testdata/ros_shim/ros/time.h b/phaser/testdata/ros_shim/ros/time.h new file mode 100644 index 0000000..8a5ecf6 --- /dev/null +++ b/phaser/testdata/ros_shim/ros/time.h @@ -0,0 +1,33 @@ +#pragma once + +#include + +namespace ros { + +struct Time { + Time() = default; + Time(uint32_t seconds, uint32_t nanoseconds) + : sec(seconds), nsec(nanoseconds) {} + + uint32_t sec = 0; + uint32_t nsec = 0; + + bool operator==(const Time& other) const { + return sec == other.sec && nsec == other.nsec; + } +}; + +struct Duration { + Duration() = default; + Duration(int32_t seconds, int32_t nanoseconds) + : sec(seconds), nsec(nanoseconds) {} + + int32_t sec = 0; + int32_t nsec = 0; + + bool operator==(const Duration& other) const { + return sec == other.sec && nsec == other.nsec; + } +}; + +} // namespace ros diff --git a/phaser/testdata/ros_shim/std_msgs/Header.h b/phaser/testdata/ros_shim/std_msgs/Header.h new file mode 100644 index 0000000..8d08cf5 --- /dev/null +++ b/phaser/testdata/ros_shim/std_msgs/Header.h @@ -0,0 +1,21 @@ +#pragma once + +#include +#include + +#include + +namespace std_msgs { + +struct Header { + uint32_t seq = 0; + ::ros::Time stamp; + std::string frame_id; + + bool operator==(const Header& other) const { + return seq == other.seq && stamp == other.stamp && + frame_id == other.frame_id; + } +}; + +} // namespace std_msgs From 04a934f41d1ba886b09e011d79371801a0fd8186 Mon Sep 17 00:00:00 2001 From: Dave Allison Date: Sun, 2 Aug 2026 19:39:55 -0700 Subject: [PATCH 2/7] Add ROS1 wire transcoding Generate one-way conversion from protobuf or native Phaser payloads, with deterministic format inference and byte-level compatibility coverage. --- README.md | 23 ++ phaser/BUILD.bazel | 15 ++ phaser/compiler/message_gen.cc | 336 ++++++++++++++++++++++++++ phaser/compiler/message_gen.h | 11 + phaser/docs/phaser_user_guide.md | 58 ++++- phaser/ros_wire_conversion_test.cc | 303 +++++++++++++++++++++++ phaser/runtime/BUILD.bazel | 11 + phaser/runtime/ros_wireformat.h | 224 +++++++++++++++++ phaser/runtime/ros_wireformat_test.cc | 65 +++++ phaser/runtime/runtime.h | 1 + phaser/runtime/wireformat.h | 196 +++++++++++++++ phaser/runtime/wireformat_test.cc | 26 ++ 12 files changed, 1266 insertions(+), 3 deletions(-) create mode 100644 phaser/ros_wire_conversion_test.cc create mode 100644 phaser/runtime/ros_wireformat.h create mode 100644 phaser/runtime/ros_wireformat_test.cc diff --git a/README.md b/README.md index 978a844..91d62ff 100644 --- a/README.md +++ b/README.md @@ -158,6 +158,29 @@ functions taking values, const references, or mutable references to accept the generated fields unchanged. Add the corresponding ROS C++ targets through the `cc_deps` attribute. +Every generated message, in either frontend style, can also produce ROS1 wire +bytes: + +```c++ +::phaser::ROSBuffer ros_output; +absl::Status status = msg.SerializeToROS(ros_output); + +// Convert serialized protobuf or a native Phaser payload without first +// constructing the user-facing message. +status = Foo::ProtobufToROS(protobuf_bytes, ros_output); +status = Foo::PhaserToROS(phaser_bytes, ros_output); +status = Foo::ConvertToROS(input_bytes, ros_output); // infers input format +``` + +`ROSBuffer` can own a growing allocation or wrap caller-provided output memory. +ROS output uses little-endian ROS1 layout, is one-way only, and writes an active +`oneof` arm without a discriminator (matching Sato's convention). That `oneof` +encoding is custom and requires the receiver to know which arm is active. +`InferMessageWireFormat` validates both the protobuf structure and Phaser +payload header; magic alone is not enough because the same four-byte prefix can +begin a valid protobuf tag. Malformed or genuinely ambiguous input is reported +explicitly. + ### 2. Create and use a message Creating a message looks just like protobuf — the binary data is backed by a dynamic diff --git a/phaser/BUILD.bazel b/phaser/BUILD.bazel index 943bc7f..4c11a2f 100644 --- a/phaser/BUILD.bazel +++ b/phaser/BUILD.bazel @@ -98,6 +98,21 @@ cc_test( ], ) +cc_test( + name = "ros_wire_conversion_test", + srcs = ["ros_wire_conversion_test.cc"], + copts = PHASER_COPTS, + deps = [ + "//phaser/runtime:phaser_runtime", + "//phaser/testdata:ros_compile_cc_proto", + "//phaser/testdata:ros_compile_phaser", + "//phaser/testdata:ros_intrinsics_phaser", + "//phaser/testdata:ros_intrinsics_protobuf_phaser", + "@com_google_absl//absl/types:span", + "@com_google_googletest//:gtest_main", + ], +) + cc_test( name = "all_types_test", srcs = ["all_types_test.cc"], diff --git a/phaser/compiler/message_gen.cc b/phaser/compiler/message_gen.cc index a686387..6870a32 100644 --- a/phaser/compiler/message_gen.cc +++ b/phaser/compiler/message_gen.cc @@ -883,6 +883,7 @@ absl::Status MessageGenerator::GenerateHeader(std::ostream& os) { } GenerateProtobufSerialization(os); + GenerateROSSerialization(os, true); // Generate serialized size. GenerateSerializedSize(os, true); @@ -1014,6 +1015,7 @@ void MessageGenerator::GenerateSource(std::ostream& os) { GenerateSerializer(os, false); // Generate deserializer. GenerateDeserializer(os, false); + GenerateROSSerialization(os, false); // Phaser bank GeneratePhaserBank(os); @@ -1727,6 +1729,340 @@ void MessageGenerator::GenerateFieldNumbers(std::ostream& os) { } } +std::string MessageGenerator::ROSFieldValueExpression( + const std::shared_ptr& field, + const std::shared_ptr& union_field, int union_index) const { + if (union_field == nullptr) { + return field->member_name + ".Get()"; + } + if (field->field->type() == + google::protobuf::FieldDescriptor::TYPE_MESSAGE) { + return union_field->member_name + ".template GetReference<" + + std::to_string(union_index) + ", " + field->c_type + ">()"; + } + return union_field->member_name + ".template GetValue<" + + std::to_string(union_index) + ", " + field->c_type + ">()"; +} + +void MessageGenerator::GenerateROSFieldSize( + std::ostream& os, const google::protobuf::FieldDescriptor* field, + const std::string& value_expression, const std::string& indent) { + switch (field->type()) { + case google::protobuf::FieldDescriptor::TYPE_INT32: + case google::protobuf::FieldDescriptor::TYPE_SINT32: + case google::protobuf::FieldDescriptor::TYPE_SFIXED32: + case google::protobuf::FieldDescriptor::TYPE_UINT32: + case google::protobuf::FieldDescriptor::TYPE_FIXED32: + case google::protobuf::FieldDescriptor::TYPE_FLOAT: + case google::protobuf::FieldDescriptor::TYPE_ENUM: + os << indent << "size += 4;\n"; + return; + case google::protobuf::FieldDescriptor::TYPE_INT64: + case google::protobuf::FieldDescriptor::TYPE_SINT64: + case google::protobuf::FieldDescriptor::TYPE_SFIXED64: + case google::protobuf::FieldDescriptor::TYPE_UINT64: + case google::protobuf::FieldDescriptor::TYPE_FIXED64: + case google::protobuf::FieldDescriptor::TYPE_DOUBLE: + os << indent << "size += 8;\n"; + return; + case google::protobuf::FieldDescriptor::TYPE_BOOL: + os << indent << "size += 1;\n"; + return; + case google::protobuf::FieldDescriptor::TYPE_STRING: + case google::protobuf::FieldDescriptor::TYPE_BYTES: + os << indent << "size += 4 + (" << value_expression << ").size();\n"; + return; + case google::protobuf::FieldDescriptor::TYPE_MESSAGE: + if (IsAny(field)) { + // Any has no static ROS1 type. An absent Any is represented by its two + // empty declared fields; a populated Any is rejected by the writer. + os << indent << "size += 8;\n"; + } else if (IsRosFrontend() && IsRosTime(field->message_type())) { + os << indent << "size += 8;\n"; + } else if (IsRosFrontend() && IsRosDuration(field->message_type())) { + os << indent << "size += 8;\n"; + } else if (IsRosFrontend() && IsRosHeader(field->message_type())) { + os << indent << "size += 16 + (" << value_expression + << ").frame_id.size();\n"; + } else { + os << indent << "size += (" << value_expression + << ").ROSSerializedSize();\n"; + } + return; + case google::protobuf::FieldDescriptor::TYPE_GROUP: + abort(); + } + abort(); +} + +void MessageGenerator::GenerateROSFieldWrite( + std::ostream& os, const google::protobuf::FieldDescriptor* field, + const std::string& value_expression, const std::string& indent) { + auto write = [&](const std::string& expression) { + os << indent << "if (absl::Status status = buffer.Write(" << expression + << "); !status.ok()) return status;\n"; + }; + switch (field->type()) { + case google::protobuf::FieldDescriptor::TYPE_INT32: + case google::protobuf::FieldDescriptor::TYPE_SINT32: + case google::protobuf::FieldDescriptor::TYPE_SFIXED32: + write("static_cast(" + value_expression + ")"); + return; + case google::protobuf::FieldDescriptor::TYPE_INT64: + case google::protobuf::FieldDescriptor::TYPE_SINT64: + case google::protobuf::FieldDescriptor::TYPE_SFIXED64: + write("static_cast(" + value_expression + ")"); + return; + case google::protobuf::FieldDescriptor::TYPE_UINT32: + case google::protobuf::FieldDescriptor::TYPE_FIXED32: + write("static_cast(" + value_expression + ")"); + return; + case google::protobuf::FieldDescriptor::TYPE_UINT64: + case google::protobuf::FieldDescriptor::TYPE_FIXED64: + write("static_cast(" + value_expression + ")"); + return; + case google::protobuf::FieldDescriptor::TYPE_DOUBLE: + write("static_cast(" + value_expression + ")"); + return; + case google::protobuf::FieldDescriptor::TYPE_FLOAT: + write("static_cast(" + value_expression + ")"); + return; + case google::protobuf::FieldDescriptor::TYPE_BOOL: + write("static_cast(" + value_expression + ")"); + return; + case google::protobuf::FieldDescriptor::TYPE_ENUM: + write("static_cast(" + value_expression + ")"); + return; + case google::protobuf::FieldDescriptor::TYPE_STRING: + case google::protobuf::FieldDescriptor::TYPE_BYTES: + os << indent + << "if (absl::Status status = buffer.WriteString(" + << value_expression << "); !status.ok()) return status;\n"; + return; + case google::protobuf::FieldDescriptor::TYPE_MESSAGE: + if (IsAny(field)) { + os << indent << "if ((" << value_expression << ").has_type_url() || (" + << value_expression << ").has_value()) {\n"; + os << indent + << " return absl::UnimplementedError(\"ROS1 serialization of a " + "populated google.protobuf.Any is unsupported\");\n"; + os << indent << "}\n"; + os << indent + << "if (absl::Status status = buffer.WriteString({}); " + "!status.ok()) return status;\n"; + os << indent + << "if (absl::Status status = buffer.WriteString({}); " + "!status.ok()) return status;\n"; + } else if (IsRosFrontend() && IsRosTime(field->message_type())) { + write("static_cast((" + value_expression + ").sec)"); + write("static_cast((" + value_expression + ").nsec)"); + } else if (IsRosFrontend() && + IsRosDuration(field->message_type())) { + write("static_cast((" + value_expression + ").sec)"); + write("static_cast((" + value_expression + ").nsec)"); + } else if (IsRosFrontend() && IsRosHeader(field->message_type())) { + write("static_cast((" + value_expression + ").seq)"); + write("static_cast((" + value_expression + ").stamp.sec)"); + write("static_cast((" + value_expression + ").stamp.nsec)"); + os << indent + << "if (absl::Status status = buffer.WriteString((" + << value_expression + << ").frame_id); !status.ok()) return status;\n"; + } else { + os << indent << "if (absl::Status status = (" << value_expression + << ").SerializeToROS(buffer); !status.ok()) return status;\n"; + } + return; + case google::protobuf::FieldDescriptor::TYPE_GROUP: + abort(); + } + abort(); +} + +void MessageGenerator::GenerateROSSerialization(std::ostream& os, bool decl) { + const std::string name = MessageName(message_); + if (decl) { + os << " size_t ROSSerializedSize() const;\n"; + os << " absl::Status SerializeToROS(::phaser::ROSBuffer& buffer) const;\n"; + os << R"XXX( absl::Status SerializeToROSArray(void* data, size_t size) const { + ::phaser::ROSBuffer buffer(data, size); + return SerializeToROS(buffer); + } + absl::Status SerializeToROSString(std::string* output) const { + if (output == nullptr) { + return absl::InvalidArgumentError("ROS output string is null"); + } + output->resize(ROSSerializedSize()); + ::phaser::ROSBuffer buffer(output->data(), output->size()); + absl::Status status = SerializeToROS(buffer); + if (!status.ok()) { + output->clear(); + } + return status; + } +)XXX"; + os << " static absl::Status ProtobufToROS(" + "std::string_view protobuf, ::phaser::ROSBuffer& output);\n"; + os << " static absl::Status PhaserToROS(" + "absl::Span phaser, ::phaser::ROSBuffer& output);\n\n"; + os << " static absl::Status ConvertToROS(" + "absl::Span input, ::phaser::ROSBuffer& output);\n\n"; + return; + } + + os << "size_t " << name << "::ROSSerializedSize() const {\n"; + os << " SyncToPayload();\n"; + if (IsRosTime(message_) || IsRosDuration(message_)) { + os << " return 8;\n"; + } else { + os << " size_t size = 0;\n"; + for (const auto& item : fields_in_order_) { + if (item->IsUnion()) { + auto union_info = std::static_pointer_cast(item); + os << " switch (" << union_info->member_name + << ".Discriminator()) {\n"; + for (size_t i = 0; i < union_info->members.size(); ++i) { + const auto& field = union_info->members[i]; + os << " case " << field->field->number() << ":\n"; + GenerateROSFieldSize( + os, field->field, + ROSFieldValueExpression(field, union_info, static_cast(i)), + " "); + os << " break;\n"; + } + os << " default:\n"; + os << " break;\n"; + os << " }\n"; + continue; + } + + const auto* descriptor = item->field; + if (!descriptor->is_repeated()) { + GenerateROSFieldSize(os, descriptor, ROSFieldValueExpression(item), + " "); + continue; + } + + const int fixed_extent = GetArraySize(descriptor); + if (fixed_extent <= 0) { + os << " size += 4;\n"; + } + const std::string count = + fixed_extent > 0 ? std::to_string(fixed_extent) + : item->member_name + ".size()"; + os << " for (size_t ros_index = 0; ros_index < " << count + << "; ++ros_index) {\n"; + GenerateROSFieldSize(os, descriptor, + item->member_name + ".Get(ros_index)", " "); + os << " }\n"; + } + os << " return size;\n"; + } + os << "}\n\n"; + + os << "absl::Status " << name + << "::SerializeToROS(::phaser::ROSBuffer& buffer) const {\n"; + os << " SyncToPayload();\n"; + if (IsRosTime(message_)) { + os << " if (absl::Status status = buffer.Write(" + "static_cast(seconds())); !status.ok()) return status;\n"; + os << " if (absl::Status status = buffer.Write(" + "static_cast(nanos())); !status.ok()) return status;\n"; + } else if (IsRosDuration(message_)) { + os << " if (absl::Status status = buffer.Write(" + "static_cast(seconds())); !status.ok()) return status;\n"; + os << " if (absl::Status status = buffer.Write(" + "static_cast(nanos())); !status.ok()) return status;\n"; + } else { + for (const auto& item : fields_in_order_) { + if (item->IsUnion()) { + auto union_info = std::static_pointer_cast(item); + os << " switch (" << union_info->member_name + << ".Discriminator()) {\n"; + for (size_t i = 0; i < union_info->members.size(); ++i) { + const auto& field = union_info->members[i]; + os << " case " << field->field->number() << ":\n"; + GenerateROSFieldWrite( + os, field->field, + ROSFieldValueExpression(field, union_info, static_cast(i)), + " "); + os << " break;\n"; + } + os << " default:\n"; + os << " break;\n"; + os << " }\n"; + continue; + } + + const auto* descriptor = item->field; + if (!descriptor->is_repeated()) { + GenerateROSFieldWrite(os, descriptor, ROSFieldValueExpression(item), + " "); + continue; + } + + const int fixed_extent = GetArraySize(descriptor); + if (fixed_extent <= 0) { + os << " if (absl::Status status = buffer.WriteSequenceLength(" + << item->member_name + << ".size()); !status.ok()) return status;\n"; + } + const std::string count = + fixed_extent > 0 ? std::to_string(fixed_extent) + : item->member_name + ".size()"; + os << " for (size_t ros_index = 0; ros_index < " << count + << "; ++ros_index) {\n"; + GenerateROSFieldWrite(os, descriptor, + item->member_name + ".Get(ros_index)", " "); + os << " }\n"; + } + } + os << " return absl::OkStatus();\n"; + os << "}\n\n"; + + os << "absl::Status " << name + << "::ProtobufToROS(std::string_view protobuf, " + "::phaser::ROSBuffer& output) {\n"; + os << " " << name << " message;\n"; + os << " ::phaser::ProtoBuffer input(protobuf);\n"; + os << " if (absl::Status status = message.Deserialize(input); " + "!status.ok()) return status;\n"; + os << " return message.SerializeToROS(output);\n"; + os << "}\n\n"; + + os << "absl::Status " << name + << "::PhaserToROS(absl::Span phaser, " + "::phaser::ROSBuffer& output) {\n"; + os << " if (phaser.empty()) {\n"; + os << " return absl::InvalidArgumentError(" + "\"Native Phaser payload is empty\");\n"; + os << " }\n"; + os << " " << name + << " message = CreateReadonly(phaser.data(), phaser.size());\n"; + os << " return message.SerializeToROS(output);\n"; + os << "}\n\n"; + + os << "absl::Status " << name + << "::ConvertToROS(absl::Span input, " + "::phaser::ROSBuffer& output) {\n"; + os << " switch (::phaser::InferMessageWireFormat(input)) {\n"; + os << " case ::phaser::MessageWireFormat::kProtobuf:\n"; + os << " return ProtobufToROS(" + "std::string_view(input.data(), input.size()), output);\n"; + os << " case ::phaser::MessageWireFormat::kPhaser:\n"; + os << " return PhaserToROS(input, output);\n"; + os << " case ::phaser::MessageWireFormat::kAmbiguous:\n"; + os << " return absl::InvalidArgumentError(" + "\"Input is structurally valid as both Phaser and protobuf\");\n"; + os << " case ::phaser::MessageWireFormat::kUnknown:\n"; + os << " return absl::InvalidArgumentError(" + "\"Input is neither a valid Phaser payload nor protobuf wire " + "message\");\n"; + os << " }\n"; + os << " return absl::InvalidArgumentError(\"Unknown input format\");\n"; + os << "}\n\n"; +} + void MessageGenerator::GenerateSerializedSize(std::ostream& os, bool decl) { if (decl) { os << " size_t SerializedSize() const;\n"; diff --git a/phaser/compiler/message_gen.h b/phaser/compiler/message_gen.h index c03f19a..c7f9801 100644 --- a/phaser/compiler/message_gen.h +++ b/phaser/compiler/message_gen.h @@ -112,6 +112,17 @@ class MessageGenerator { void GenerateSerializedSize(std::ostream& os, bool decl); void GenerateSerializer(std::ostream& os, bool decl); void GenerateDeserializer(std::ostream& os, bool decl); + void GenerateROSSerialization(std::ostream& os, bool decl); + void GenerateROSFieldSize( + std::ostream& os, const google::protobuf::FieldDescriptor* field, + const std::string& value_expression, const std::string& indent); + void GenerateROSFieldWrite( + std::ostream& os, const google::protobuf::FieldDescriptor* field, + const std::string& value_expression, const std::string& indent); + std::string ROSFieldValueExpression( + const std::shared_ptr& field, + const std::shared_ptr& union_field = nullptr, + int union_index = -1) const; void GenerateProtobufSerialization(std::ostream& os); void GenerateIndent(std::ostream& os); diff --git a/phaser/docs/phaser_user_guide.md b/phaser/docs/phaser_user_guide.md index 8683ec6..cb87961 100644 --- a/phaser/docs/phaser_user_guide.md +++ b/phaser/docs/phaser_user_guide.md @@ -379,6 +379,59 @@ msg.command.reset(); throws `std::bad_variant_access` for an inactive arm. Switching arms clears and releases any string or message storage owned by the previous arm. +### ROS1 wire conversion +Every generated message supports one-way conversion to ROS1 serialization, +regardless of whether its C++ frontend is protobuf-style or ROS-style: + +```c++ +size_t ROSSerializedSize() const; +absl::Status SerializeToROS(::phaser::ROSBuffer& output) const; +absl::Status SerializeToROSArray(void* output, size_t capacity) const; +absl::Status SerializeToROSString(std::string* output) const; + +static absl::Status ProtobufToROS( + std::string_view protobuf, ::phaser::ROSBuffer& output); +static absl::Status PhaserToROS( + absl::Span phaser, ::phaser::ROSBuffer& output); +static absl::Status ConvertToROS( + absl::Span input, ::phaser::ROSBuffer& output); +``` + +`SerializeToROS` reads a live message. `ProtobufToROS` parses protobuf wire +bytes with the generated Phaser parser and then writes ROS bytes. +`PhaserToROS` attaches a read-only message to a valid native Phaser payload for +the duration of the conversion. `ConvertToROS` calls +`InferMessageWireFormat` and selects either input path. Inference validates the +complete generic protobuf field structure and the Phaser payload header rather +than checking only the four-byte magic: those magic bytes can also begin a +valid protobuf tag. The result enum can report `kProtobuf`, `kPhaser`, +`kUnknown`, or `kAmbiguous`; automatic conversion rejects the latter two. +There is intentionally no ROS-to-protobuf or ROS-to-Phaser conversion. + +`ROSBuffer` is declared in `phaser/runtime/ros_wireformat.h`. Its default +constructor owns a dynamically growing allocation; constructing it with a +pointer and size wraps fixed caller-owned output memory. Writes return an +`absl::Status`, including insufficient-capacity and malformed-protobuf errors. + +The generated layout follows ROS1 serialization rules: + +- signed/unsigned integer, floating-point, boolean, and enum values are written + little-endian (`bool` is one byte and enums are `int32`); +- strings and bytes have a `uint32` byte-length prefix; +- variable repeated fields have a `uint32` element count; +- fields annotated with `phaser.array_size` are fixed arrays and omit the + element count; +- nested messages are written inline in declaration order; +- `Timestamp`, `Duration`, and `Header` use the ROS1 time, duration, and header + layouts described above. + +ROS1 has no standard union encoding. Phaser follows Sato's convention for a +protobuf `oneof`: only the active arm is written, with no discriminator, and an +unset `oneof` writes no bytes. The result is not self-describing; the receiver +must know which arm is active through an external contract. A populated +`google.protobuf.Any` is rejected because its dynamic type has no static ROS1 +layout. + Both frontends use the same native binary metadata and protobuf wire serialization. A protobuf-style target and a ROS-style target generated from the same schema can therefore exchange native Phaser buffers and protobuf wire @@ -796,9 +849,8 @@ The `MutableAny` function creates a mutable message of type `T` in the `value` f the `type_url`. You can then create the message as you would do normally. ## Serialization and deserialization -Phaser doesn't do serialization, but protobuf does. In order to give you a way to convert -from Phaser wire-format to protobuf, some transcoding serialization functions are provided. -These are: +Phaser's native format does not require serialization. To interoperate with +protobuf, generated messages provide these protobuf transcoding functions: ```c++ size_t SerializedSize() const; diff --git a/phaser/ros_wire_conversion_test.cc b/phaser/ros_wire_conversion_test.cc new file mode 100644 index 0000000..3b660ec --- /dev/null +++ b/phaser/ros_wire_conversion_test.cc @@ -0,0 +1,303 @@ +// Copyright 2024-2026 David Allison +// All Rights Reserved. +// See LICENSE file for licensing information. + +#include + +#include +#include +#include +#include +#include + +#include "absl/types/span.h" +#include "phaser/runtime/ros_wireformat.h" +#include "phaser/testdata/RosCompile.pb.h" +#include "phaser/testdata/RosCompile.phaser.h" +#include "phaser/testdata/RosIntrinsics.phaser.h" +#include "phaser/testdata/RosIntrinsicsProtobufFrontend.phaser.h" + +namespace { + +using RosCompileMessage = ::foo::bar::phaser::RosCompileMessage; +using RosInner = ::foo::bar::phaser::RosInner; +using RosIntrinsicMessage = ::foo::bar::phaser::RosIntrinsicMessage; +using ProtobufFrontendIntrinsicMessage = + ::foo::bar::pb::protobuf_phaser::RosIntrinsicMessage; +using RosColor = ::foo::bar::phaser::RosColor; + +template +void AppendIntegral(std::string& bytes, T value) { + static_assert(std::is_integral_v); + using U = std::make_unsigned_t; + U bits = static_cast(value); + for (size_t i = 0; i < sizeof(U); ++i) { + bytes.push_back(static_cast(bits & static_cast(0xff))); + if constexpr (sizeof(U) > 1) { + bits >>= 8; + } + } +} + +void AppendDouble(std::string& bytes, double value) { + uint64_t bits = 0; + static_assert(sizeof(bits) == sizeof(value)); + std::memcpy(&bits, &value, sizeof(bits)); + AppendIntegral(bytes, bits); +} + +void AppendString(std::string& bytes, std::string_view value) { + AppendIntegral(bytes, static_cast(value.size())); + bytes.append(value); +} + +void PopulatePhaserMessage(RosCompileMessage& message) { + message.x = -7; + message.name = "robot"; + message.flag = true; + message.value = 1.5; + message.color = RosColor::ROS_COLOR_RED; + message.inner->id = 42; + + message.xs.push_back(10); + message.xs.push_back(-20); + message.names.push_back("a"); + message.names.push_back("beta"); + message.colors.push_back(RosColor::ROS_COLOR_RED); + message.colors.push_back(RosColor::ROS_COLOR_BLUE); + message.inners.Add()->id = 100; + message.inners.Add()->id = 200; + + message.fixed_ints[0] = 1; + message.fixed_ints[1] = 2; + message.fixed_ints[2] = 3; + message.fixed_ints[3] = 4; + message.fixed_colors[0] = RosColor::ROS_COLOR_RED; + message.fixed_colors[1] = RosColor::ROS_COLOR_BLUE; + message.fixed_colors[2] = RosColor::ROS_COLOR_UNSPECIFIED; + message.fixed_names[0] = "left"; + message.fixed_names[1] = "right"; + message.fixed_inners[0]->id = 300; + message.fixed_inners[1]->id = 400; + + using ChoiceName = RosCompileMessage::ChoiceNameAlternative; + message.choice.emplace("selected"); +} + +void PopulateProtobufMessage(::foo::bar::RosCompileMessage& message) { + message.set_x(-7); + message.set_name("robot"); + message.set_flag(true); + message.set_value(1.5); + message.set_color(::foo::bar::ROS_COLOR_RED); + message.mutable_inner()->set_id(42); + + message.add_xs(10); + message.add_xs(-20); + message.add_names("a"); + message.add_names("beta"); + message.add_colors(::foo::bar::ROS_COLOR_RED); + message.add_colors(::foo::bar::ROS_COLOR_BLUE); + message.add_inners()->set_id(100); + message.add_inners()->set_id(200); + + for (int32_t value : {1, 2, 3, 4}) { + message.add_fixed_ints(value); + } + message.add_fixed_colors(::foo::bar::ROS_COLOR_RED); + message.add_fixed_colors(::foo::bar::ROS_COLOR_BLUE); + message.add_fixed_colors(::foo::bar::ROS_COLOR_UNSPECIFIED); + message.add_fixed_names("left"); + message.add_fixed_names("right"); + message.add_fixed_inners()->set_id(300); + message.add_fixed_inners()->set_id(400); + message.set_choice_name("selected"); +} + +std::string ExpectedRosCompileBytes(bool include_oneof = true) { + std::string bytes; + AppendIntegral(bytes, static_cast(-7)); + AppendString(bytes, "robot"); + AppendIntegral(bytes, static_cast(1)); + AppendDouble(bytes, 1.5); + AppendIntegral(bytes, static_cast(RosColor::ROS_COLOR_RED)); + AppendIntegral(bytes, static_cast(42)); + + AppendIntegral(bytes, static_cast(2)); + AppendIntegral(bytes, static_cast(10)); + AppendIntegral(bytes, static_cast(-20)); + AppendIntegral(bytes, static_cast(2)); + AppendString(bytes, "a"); + AppendString(bytes, "beta"); + AppendIntegral(bytes, static_cast(2)); + AppendIntegral(bytes, static_cast(RosColor::ROS_COLOR_RED)); + AppendIntegral(bytes, static_cast(RosColor::ROS_COLOR_BLUE)); + AppendIntegral(bytes, static_cast(2)); + AppendIntegral(bytes, static_cast(100)); + AppendIntegral(bytes, static_cast(200)); + + for (int32_t value : {1, 2, 3, 4}) { + AppendIntegral(bytes, value); + } + AppendIntegral(bytes, static_cast(RosColor::ROS_COLOR_RED)); + AppendIntegral(bytes, static_cast(RosColor::ROS_COLOR_BLUE)); + AppendIntegral(bytes, + static_cast(RosColor::ROS_COLOR_UNSPECIFIED)); + AppendString(bytes, "left"); + AppendString(bytes, "right"); + AppendIntegral(bytes, static_cast(300)); + AppendIntegral(bytes, static_cast(400)); + + if (include_oneof) { + AppendString(bytes, "selected"); + } + return bytes; +} + +std::string ExpectedIntrinsicBytes() { + std::string bytes; + AppendIntegral(bytes, static_cast(12)); + AppendIntegral(bytes, static_cast(345)); + AppendIntegral(bytes, static_cast(-4)); + AppendIntegral(bytes, static_cast(500)); + AppendIntegral(bytes, static_cast(9)); + AppendIntegral(bytes, static_cast(21)); + AppendIntegral(bytes, static_cast(654)); + AppendString(bytes, "map"); + + AppendIntegral(bytes, static_cast(0)); // count + AppendString(bytes, ""); // name + AppendIntegral(bytes, static_cast(0)); // samples + AppendIntegral(bytes, static_cast(0)); // tags + AppendIntegral(bytes, static_cast(0)); // children + AppendString(bytes, ""); // fixed_names[0] + AppendString(bytes, ""); // fixed_names[1] + for (int i = 0; i < 2; ++i) { + AppendIntegral(bytes, static_cast(0)); // child id + AppendString(bytes, ""); // child label + } + return bytes; +} + +TEST(ROSWireConversionTest, LiveProtobufAndNativePathsMatchKnownBytes) { + RosCompileMessage phaser_message; + PopulatePhaserMessage(phaser_message); + const std::string expected = ExpectedRosCompileBytes(); + + ::phaser::ROSBuffer live_output(16); + ASSERT_TRUE(phaser_message.SerializeToROS(live_output).ok()); + EXPECT_EQ(live_output.AsString(), expected); + EXPECT_EQ(phaser_message.ROSSerializedSize(), expected.size()); + + std::string string_output; + ASSERT_TRUE(phaser_message.SerializeToROSString(&string_output).ok()); + EXPECT_EQ(string_output, expected); + + ::foo::bar::RosCompileMessage protobuf_message; + PopulateProtobufMessage(protobuf_message); + const std::string protobuf_wire = protobuf_message.SerializeAsString(); + EXPECT_EQ(::phaser::InferMessageWireFormat(protobuf_wire), + ::phaser::MessageWireFormat::kProtobuf); + ::phaser::ROSBuffer protobuf_output; + ASSERT_TRUE(RosCompileMessage::ProtobufToROS(protobuf_wire, protobuf_output) + .ok()); + EXPECT_EQ(protobuf_output.AsString(), expected); + + ::phaser::ROSBuffer native_output; + const auto* native_data = + reinterpret_cast(phaser_message.Data()); + const absl::Span native_bytes(native_data, + phaser_message.Size()); + EXPECT_EQ(::phaser::InferMessageWireFormat(native_bytes), + ::phaser::MessageWireFormat::kPhaser); + ASSERT_TRUE(RosCompileMessage::PhaserToROS( + native_bytes, native_output) + .ok()); + EXPECT_EQ(native_output.AsString(), expected); + + ::phaser::ROSBuffer inferred_protobuf_output; + ASSERT_TRUE(RosCompileMessage::ConvertToROS( + absl::Span(protobuf_wire.data(), + protobuf_wire.size()), + inferred_protobuf_output) + .ok()); + EXPECT_EQ(inferred_protobuf_output.AsString(), expected); + + ::phaser::ROSBuffer inferred_native_output; + ASSERT_TRUE( + RosCompileMessage::ConvertToROS(native_bytes, inferred_native_output) + .ok()); + EXPECT_EQ(inferred_native_output.AsString(), expected); +} + +TEST(ROSWireConversionTest, FixedOutputAndErrorsAreReported) { + RosCompileMessage message; + PopulatePhaserMessage(message); + const std::string expected = ExpectedRosCompileBytes(); + + std::vector exact(expected.size()); + ASSERT_TRUE(message.SerializeToROSArray(exact.data(), exact.size()).ok()); + EXPECT_EQ(std::string(exact.data(), exact.size()), expected); + + std::vector too_small(expected.size() - 1); + EXPECT_FALSE( + message.SerializeToROSArray(too_small.data(), too_small.size()).ok()); + EXPECT_FALSE(message.SerializeToROSString(nullptr).ok()); + + ::phaser::ROSBuffer output; + EXPECT_FALSE( + RosCompileMessage::ProtobufToROS(std::string(1, '\x80'), output).ok()); + EXPECT_TRUE(output.empty()); + EXPECT_FALSE(RosCompileMessage::PhaserToROS({}, output).ok()); + EXPECT_FALSE(RosCompileMessage::ConvertToROS( + absl::Span("\0", 1), output) + .ok()); +} + +TEST(ROSWireConversionTest, UnsetOneofWritesNoBytesForTheUnion) { + RosCompileMessage message; + PopulatePhaserMessage(message); + message.choice.reset(); + + ::phaser::ROSBuffer output; + ASSERT_TRUE(message.SerializeToROS(output).ok()); + EXPECT_EQ(output.AsString(), ExpectedRosCompileBytes(false)); +} + +TEST(ROSWireConversionTest, ROS1IntrinsicsUseNativeLayoutsAndFlushCaches) { + RosIntrinsicMessage message; + message.stamp = ::ros::Time(12, 345); + message.timeout = ::ros::Duration(-4, 500); + message.header->seq = 9; + message.header->stamp = ::ros::Time(21, 654); + message.header->frame_id = "map"; + const std::string expected = ExpectedIntrinsicBytes(); + + ::phaser::ROSBuffer live_output; + ASSERT_TRUE(message.SerializeToROS(live_output).ok()); + EXPECT_EQ(live_output.AsString(), expected); + EXPECT_EQ(message.ROSSerializedSize(), expected.size()); + + ::phaser::ROSBuffer protobuf_output; + ASSERT_TRUE(RosIntrinsicMessage::ProtobufToROS(message.SerializeAsString(), + protobuf_output) + .ok()); + EXPECT_EQ(protobuf_output.AsString(), expected); + + ::phaser::ROSBuffer native_output; + const auto* native_data = reinterpret_cast(message.Data()); + ASSERT_TRUE(RosIntrinsicMessage::PhaserToROS( + absl::Span(native_data, message.Size()), + native_output) + .ok()); + EXPECT_EQ(native_output.AsString(), expected); + + ::phaser::ROSBuffer protobuf_frontend_native_output; + ASSERT_TRUE(ProtobufFrontendIntrinsicMessage::PhaserToROS( + absl::Span(native_data, message.Size()), + protobuf_frontend_native_output) + .ok()); + EXPECT_EQ(protobuf_frontend_native_output.AsString(), expected); +} + +} // namespace diff --git a/phaser/runtime/BUILD.bazel b/phaser/runtime/BUILD.bazel index b3b310d..3c39194 100644 --- a/phaser/runtime/BUILD.bazel +++ b/phaser/runtime/BUILD.bazel @@ -17,6 +17,7 @@ cc_library( "iterators.h", "message.h", "runtime.h", + "ros_wireformat.h", "union.h", "vectors.h", "wireformat.h", @@ -64,3 +65,13 @@ cc_test( "@cpp_toolbelt//toolbelt", ], ) + +cc_test( + name = "ros_wireformat_test", + srcs = ["ros_wireformat_test.cc"], + copts = PHASER_COPTS, + deps = [ + ":phaser_runtime", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/phaser/runtime/ros_wireformat.h b/phaser/runtime/ros_wireformat.h new file mode 100644 index 0000000..0fadbf5 --- /dev/null +++ b/phaser/runtime/ros_wireformat.h @@ -0,0 +1,224 @@ +// Copyright 2024-2026 David Allison +// All Rights Reserved. +// See LICENSE file for licensing information. + +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/strings/str_format.h" +#include "absl/types/span.h" + +namespace phaser { + +// A sequential ROS1 serialization buffer. ROS1 primitives are always encoded +// little-endian, strings and variable-length sequences use uint32 length +// prefixes, and messages do not carry tags or alignment padding. +class ROSBuffer { + public: + explicit ROSBuffer(size_t initial_size = 16) + : owned_(true), capacity_(std::max(initial_size, 16)) { + data_ = static_cast(malloc(capacity_)); + allocation_failed_ = data_ == nullptr; + } + + ROSBuffer(void* data, size_t size) + : data_(static_cast(data)), capacity_(size) {} + + ROSBuffer(const ROSBuffer&) = delete; + ROSBuffer& operator=(const ROSBuffer&) = delete; + + ROSBuffer(ROSBuffer&& other) noexcept { MoveFrom(std::move(other)); } + + ROSBuffer& operator=(ROSBuffer&& other) noexcept { + if (this != &other) { + Release(); + MoveFrom(std::move(other)); + } + return *this; + } + + ~ROSBuffer() { Release(); } + + size_t Size() const { return size_; } + size_t size() const { return Size(); } + size_t Capacity() const { return capacity_; } + bool empty() const { return size_ == 0; } + + char* Data() { return data_; } + const char* Data() const { return data_; } + char* data() { return Data(); } + const char* data() const { return Data(); } + + std::string AsString() const { + return data_ == nullptr ? std::string() : std::string(data_, size_); + } + + absl::Span AsSpan() const { + return absl::Span(data_, size_); + } + + void Clear() { size_ = 0; } + + absl::Status WriteRaw(const void* source, size_t length) { + if (length == 0) { + return absl::OkStatus(); + } + if (source == nullptr) { + return absl::InvalidArgumentError( + "Cannot write non-empty data from a null pointer"); + } + if (absl::Status status = EnsureSpace(length); !status.ok()) { + return status; + } + memcpy(data_ + size_, source, length); + size_ += length; + return absl::OkStatus(); + } + + absl::Status WriteString(std::string_view value) { + if (value.size() > std::numeric_limits::max()) { + return absl::InvalidArgumentError("ROS1 string exceeds uint32 length"); + } + const size_t total = sizeof(uint32_t) + value.size(); + if (absl::Status status = EnsureSpace(total); !status.ok()) { + return status; + } + WriteLittleEndianUnchecked(static_cast(value.size())); + if (!value.empty()) { + memcpy(data_ + size_, value.data(), value.size()); + size_ += value.size(); + } + return absl::OkStatus(); + } + + absl::Status WriteSequenceLength(size_t length) { + if (length > std::numeric_limits::max()) { + return absl::InvalidArgumentError( + "ROS1 sequence exceeds uint32 length"); + } + return Write(static_cast(length)); + } + + absl::Status Write(bool value) { + return Write(static_cast(value ? 1 : 0)); + } + + template && + !std::is_same_v, bool>, + int> = 0> + absl::Status Write(T value) { + using U = std::make_unsigned_t; + if (absl::Status status = EnsureSpace(sizeof(T)); !status.ok()) { + return status; + } + WriteLittleEndianUnchecked(static_cast(value)); + return absl::OkStatus(); + } + + absl::Status Write(float value) { + uint32_t bits = 0; + static_assert(sizeof(bits) == sizeof(value)); + memcpy(&bits, &value, sizeof(bits)); + return Write(bits); + } + + absl::Status Write(double value) { + uint64_t bits = 0; + static_assert(sizeof(bits) == sizeof(value)); + memcpy(&bits, &value, sizeof(bits)); + return Write(bits); + } + + private: + absl::Status EnsureSpace(size_t length) { + if (allocation_failed_) { + return absl::ResourceExhaustedError( + "Unable to allocate ROS serialization buffer"); + } + if (data_ == nullptr && capacity_ != 0) { + return absl::InvalidArgumentError("ROS output buffer is null"); + } + if (length > std::numeric_limits::max() - size_) { + return absl::ResourceExhaustedError("ROS output size overflow"); + } + const size_t required = size_ + length; + if (required <= capacity_) { + return absl::OkStatus(); + } + if (!owned_) { + return absl::ResourceExhaustedError(absl::StrFormat( + "No space in ROS output buffer: capacity %d, need %d", capacity_, + required)); + } + + size_t new_capacity = capacity_; + while (new_capacity < required) { + if (new_capacity > std::numeric_limits::max() / 2) { + new_capacity = required; + break; + } + new_capacity *= 2; + } + void* replacement = realloc(data_, new_capacity); + if (replacement == nullptr) { + allocation_failed_ = true; + return absl::ResourceExhaustedError( + "Unable to grow ROS serialization buffer"); + } + data_ = static_cast(replacement); + capacity_ = new_capacity; + return absl::OkStatus(); + } + + template + void WriteLittleEndianUnchecked(U value) { + static_assert(std::is_unsigned_v); + for (size_t i = 0; i < sizeof(U); ++i) { + data_[size_++] = static_cast(value & static_cast(0xff)); + if constexpr (sizeof(U) > 1) { + value >>= 8; + } + } + } + + void Release() { + if (owned_) { + free(data_); + } + } + + void MoveFrom(ROSBuffer&& other) { + owned_ = other.owned_; + allocation_failed_ = other.allocation_failed_; + data_ = other.data_; + capacity_ = other.capacity_; + size_ = other.size_; + + other.owned_ = false; + other.allocation_failed_ = false; + other.data_ = nullptr; + other.capacity_ = 0; + other.size_ = 0; + } + + bool owned_ = false; + bool allocation_failed_ = false; + char* data_ = nullptr; + size_t capacity_ = 0; + size_t size_ = 0; +}; + +} // namespace phaser diff --git a/phaser/runtime/ros_wireformat_test.cc b/phaser/runtime/ros_wireformat_test.cc new file mode 100644 index 0000000..03d8a66 --- /dev/null +++ b/phaser/runtime/ros_wireformat_test.cc @@ -0,0 +1,65 @@ +// Copyright 2024-2026 David Allison +// All Rights Reserved. +// See LICENSE file for licensing information. + +#include "phaser/runtime/ros_wireformat.h" + +#include + +#include +#include +#include + +namespace phaser { +namespace { + +TEST(ROSWireformatTest, WritesCanonicalLittleEndianBytes) { + ROSBuffer buffer; + ASSERT_TRUE(buffer.Write(static_cast(-2)).ok()); + ASSERT_TRUE(buffer.Write(static_cast(0x12345678)).ok()); + ASSERT_TRUE(buffer.Write(true).ok()); + ASSERT_TRUE(buffer.WriteString("hi").ok()); + + const std::array expected = { + 0xfe, 0xff, 0x78, 0x56, 0x34, 0x12, 0x01, + 0x02, 0x00, 0x00, 0x00, 'h', 'i', + }; + ASSERT_EQ(buffer.Size(), expected.size()); + for (size_t i = 0; i < expected.size(); ++i) { + EXPECT_EQ(static_cast(buffer.data()[i]), expected[i]) + << "byte " << i; + } +} + +TEST(ROSWireformatTest, DynamicBufferGrowsAndCanBeReused) { + ROSBuffer buffer(16); + const std::string value(4096, 'x'); + ASSERT_TRUE(buffer.WriteString(value).ok()); + EXPECT_EQ(buffer.Size(), value.size() + sizeof(uint32_t)); + EXPECT_GE(buffer.Capacity(), buffer.Size()); + + buffer.Clear(); + EXPECT_TRUE(buffer.empty()); + ASSERT_TRUE(buffer.Write(static_cast(7)).ok()); + EXPECT_EQ(buffer.Size(), sizeof(uint64_t)); +} + +TEST(ROSWireformatTest, FixedBufferFailureDoesNotAdvanceCursor) { + std::array storage = {}; + ROSBuffer buffer(storage.data(), storage.size()); + EXPECT_FALSE(buffer.Write(static_cast(1)).ok()); + EXPECT_EQ(buffer.Size(), 0u); + + EXPECT_FALSE(buffer.WriteString("x").ok()); + EXPECT_EQ(buffer.Size(), 0u); +} + +TEST(ROSWireformatTest, RejectsInvalidRawWrite) { + ROSBuffer buffer; + EXPECT_FALSE(buffer.WriteRaw(nullptr, 1).ok()); + EXPECT_EQ(buffer.Size(), 0u); + EXPECT_TRUE(buffer.WriteRaw(nullptr, 0).ok()); +} + +} // namespace +} // namespace phaser diff --git a/phaser/runtime/runtime.h b/phaser/runtime/runtime.h index a5580e4..a7ca165 100644 --- a/phaser/runtime/runtime.h +++ b/phaser/runtime/runtime.h @@ -11,6 +11,7 @@ #include "phaser/runtime/iterators.h" #include "phaser/runtime/message.h" #include "phaser/runtime/phaser_bank.h" +#include "phaser/runtime/ros_wireformat.h" #include "phaser/runtime/union.h" #include "phaser/runtime/vectors.h" #include "toolbelt/hexdump.h" diff --git a/phaser/runtime/wireformat.h b/phaser/runtime/wireformat.h index 1255cc4..da03ed1 100644 --- a/phaser/runtime/wireformat.h +++ b/phaser/runtime/wireformat.h @@ -15,9 +15,205 @@ #include "absl/status/statusor.h" #include "absl/strings/str_format.h" #include "absl/types/span.h" +#include "toolbelt/payload_buffer.h" namespace phaser { +enum class MessageWireFormat { + kProtobuf, + kPhaser, + kUnknown, + kAmbiguous, +}; + +namespace internal { + +inline uint32_t LoadWireUint32(absl::Span data, size_t offset) { + uint32_t value = 0; + for (size_t i = 0; i < sizeof(value); ++i) { + value |= static_cast( + static_cast(data[offset + i])) + << static_cast(i * 8); + } + return value; +} + +inline bool ConsumeWireVarint(absl::Span data, size_t* offset, + size_t max_bytes, uint8_t max_last_byte, + uint64_t* value) { + uint64_t result = 0; + for (size_t i = 0; i < max_bytes; ++i) { + if (*offset >= data.size()) { + return false; + } + const uint8_t byte = + static_cast(static_cast(data[(*offset)++])); + if (i + 1 == max_bytes && + ((byte & 0x80U) != 0 || (byte & 0x7fU) > max_last_byte)) { + return false; + } + result |= static_cast(byte & 0x7fU) + << static_cast(i * 7); + if ((byte & 0x80U) == 0) { + *value = result; + return true; + } + } + return false; +} + +inline bool ConsumeProtobufFields(absl::Span data, size_t* offset, + uint32_t expected_end_group = 0, + size_t depth = 0) { + constexpr uint32_t kMaxFieldNumber = (uint32_t{1} << 29) - 1; + constexpr size_t kMaxGroupDepth = 100; + if (depth > kMaxGroupDepth) { + return false; + } + while (*offset < data.size()) { + uint64_t tag = 0; + if (!ConsumeWireVarint(data, offset, 5, 0x0f, &tag)) { + return false; + } + const uint32_t field_number = static_cast(tag >> 3); + const uint32_t wire_type = static_cast(tag & 7); + if (field_number == 0 || field_number > kMaxFieldNumber) { + return false; + } + if (wire_type == 4) { + return expected_end_group != 0 && field_number == expected_end_group; + } + + uint64_t value = 0; + switch (wire_type) { + case 0: + if (!ConsumeWireVarint(data, offset, 10, 1, &value)) { + return false; + } + break; + case 1: + if (data.size() - *offset < sizeof(uint64_t)) { + return false; + } + *offset += sizeof(uint64_t); + break; + case 2: + if (!ConsumeWireVarint(data, offset, 10, 1, &value) || + value > data.size() - *offset) { + return false; + } + *offset += static_cast(value); + break; + case 3: + if (!ConsumeProtobufFields(data, offset, field_number, depth + 1)) { + return false; + } + break; + case 5: + if (data.size() - *offset < sizeof(uint32_t)) { + return false; + } + *offset += sizeof(uint32_t); + break; + default: + return false; + } + } + return expected_end_group == 0; +} + +inline bool IsStructurallyValidProtobuf(absl::Span data) { + size_t offset = 0; + return ConsumeProtobufFields(data, &offset) && offset == data.size(); +} + +inline bool IsStructurallyValidPhaser(absl::Span data) { + constexpr size_t kMagicOffset = + offsetof(::toolbelt::PayloadBuffer, magic); + constexpr size_t kMessageOffset = + offsetof(::toolbelt::PayloadBuffer, message); + constexpr size_t kHwmOffset = offsetof(::toolbelt::PayloadBuffer, hwm); + constexpr size_t kFullSizeOffset = + offsetof(::toolbelt::PayloadBuffer, full_size); + constexpr size_t kFreeListOffset = + offsetof(::toolbelt::PayloadBuffer, free_list); + constexpr size_t kMetadataOffset = + offsetof(::toolbelt::PayloadBuffer, metadata); + constexpr size_t kBitmapsOffset = + offsetof(::toolbelt::PayloadBuffer, bitmaps); + + if (data.size() < sizeof(::toolbelt::PayloadBuffer)) { + return false; + } + const uint32_t magic = LoadWireUint32(data, kMagicOffset); + const uint32_t base_magic = magic & ::toolbelt::kBitMapMask; + const bool movable = base_magic == ::toolbelt::kMovableBufferMagic; + if (!movable && base_magic != ::toolbelt::kFixedBufferMagic) { + return false; + } + + const size_t minimum_header = + sizeof(::toolbelt::PayloadBuffer) + + (movable ? sizeof(::toolbelt::Resizer*) : 0); + const uint32_t message = LoadWireUint32(data, kMessageOffset); + const uint32_t hwm = LoadWireUint32(data, kHwmOffset); + const uint32_t full_size = LoadWireUint32(data, kFullSizeOffset); + const uint32_t free_list = LoadWireUint32(data, kFreeListOffset); + const uint32_t metadata = LoadWireUint32(data, kMetadataOffset); + + if (full_size < minimum_header || hwm < minimum_header || + hwm > full_size || hwm > data.size() || message < minimum_header || + (message & 7U) != 0 || message > hwm - sizeof(uint32_t)) { + return false; + } + if (free_list != 0 && + (free_list < minimum_header || + free_list > full_size - sizeof(::toolbelt::FreeBlockHeader))) { + return false; + } + if (metadata != 0 && + (metadata < minimum_header || metadata >= hwm)) { + return false; + } + for (size_t i = 0; i < ::toolbelt::kNumBitmapRuns; ++i) { + const uint32_t bitmap = + LoadWireUint32(data, kBitmapsOffset + i * sizeof(uint32_t)); + if (bitmap != 0 && (bitmap < minimum_header || bitmap >= hwm)) { + return false; + } + } + return true; +} + +} // namespace internal + +// Infers a format from its complete byte representation. Both formats are +// validated structurally; the magic prefix alone is deliberately insufficient +// because it can also begin a valid protobuf field tag. +inline MessageWireFormat InferMessageWireFormat( + absl::Span data) { + const bool phaser = internal::IsStructurallyValidPhaser(data); + const bool protobuf = internal::IsStructurallyValidProtobuf(data); + if (phaser && protobuf) { + return MessageWireFormat::kAmbiguous; + } + if (phaser) { + return MessageWireFormat::kPhaser; + } + if (protobuf) { + return MessageWireFormat::kProtobuf; + } + return MessageWireFormat::kUnknown; +} + +inline MessageWireFormat InferMessageWireFormat(std::string_view data) { + return InferMessageWireFormat(absl::Span(data.data(), data.size())); +} + +inline MessageWireFormat InferMessageWireFormat(const std::string& data) { + return InferMessageWireFormat(std::string_view(data)); +} + enum class WireType { kVarint = 0, kFixed64 = 1, diff --git a/phaser/runtime/wireformat_test.cc b/phaser/runtime/wireformat_test.cc index 5574c06..a3c0aa3 100644 --- a/phaser/runtime/wireformat_test.cc +++ b/phaser/runtime/wireformat_test.cc @@ -15,6 +15,7 @@ using ProtoBuffer = phaser::ProtoBuffer; using WireType = phaser::WireType; +using MessageWireFormat = phaser::MessageWireFormat; TEST(Wireformat, Sizes) { ASSERT_EQ(1, (ProtoBuffer::VarintSize(1))); @@ -33,6 +34,31 @@ TEST(Wireformat, Sizes) { ASSERT_EQ(7, ProtoBuffer::StringSize(1, "hello")); } +TEST(Wireformat, InfersStructurallyValidProtobuf) { + EXPECT_EQ(MessageWireFormat::kProtobuf, + phaser::InferMessageWireFormat(std::string_view())); + + const std::string ordinary = "\x08\x96\x01"; + EXPECT_EQ(MessageWireFormat::kProtobuf, + phaser::InferMessageWireFormat(ordinary)); +} + +TEST(Wireformat, MagicPrefixAloneDoesNotImplyPhaser) { + // The first four bytes are kFixedBufferMagic with the bitmap flag. Together + // with the fifth byte they form a valid fixed32 protobuf tag, followed by its + // four-byte value. + const std::string protobuf_with_magic_prefix( + "\xc5\xf1\xf6\xe5\x01\x12\x34\x56\x78", 9); + EXPECT_EQ(MessageWireFormat::kProtobuf, + phaser::InferMessageWireFormat(protobuf_with_magic_prefix)); +} + +TEST(Wireformat, RejectsStructurallyInvalidInput) { + const std::string invalid(1, '\0'); + EXPECT_EQ(MessageWireFormat::kUnknown, + phaser::InferMessageWireFormat(invalid)); +} + TEST(Wireformat, ZigZagKnownValues) { // Canonical protobuf zigzag mappings. ZigZag() returns the encoded value // reinterpreted as the signed type, so compare via the unsigned bit pattern. From c9a3d9a7f08079699082c2e9d89fcda40d8ce21c Mon Sep 17 00:00:00 2001 From: Dave Allison Date: Sun, 2 Aug 2026 19:54:22 -0700 Subject: [PATCH 3/7] Fix Valgrind test runfiles Include the shared suppression file in newly added ROS test runfiles so CI can launch them under Valgrind. --- phaser/BUILD.bazel | 4 ++++ phaser/runtime/BUILD.bazel | 1 + phaser/testdata/BUILD | 3 +++ 3 files changed, 8 insertions(+) diff --git a/phaser/BUILD.bazel b/phaser/BUILD.bazel index 4c11a2f..05fbd77 100644 --- a/phaser/BUILD.bazel +++ b/phaser/BUILD.bazel @@ -67,6 +67,7 @@ cc_test( name = "ros_compile_test", srcs = ["ros_compile_test.cc"], copts = PHASER_COPTS, + data = ["valgrind.supp"], deps = [ "//phaser/runtime:phaser_runtime", "//phaser/testdata:ros_compile_cc_proto", @@ -79,6 +80,7 @@ cc_test( name = "ros_intrinsics_test", srcs = ["ros_intrinsics_test.cc"], copts = PHASER_COPTS, + data = ["valgrind.supp"], deps = [ "//phaser/runtime:phaser_runtime", "//phaser/testdata:ros_intrinsics_phaser", @@ -90,6 +92,7 @@ cc_test( name = "ros_native_frontend_compatibility_test", srcs = ["ros_native_frontend_compatibility_test.cc"], copts = PHASER_COPTS, + data = ["valgrind.supp"], deps = [ "//phaser/runtime:phaser_runtime", "//phaser/testdata:ros_intrinsics_phaser", @@ -102,6 +105,7 @@ cc_test( name = "ros_wire_conversion_test", srcs = ["ros_wire_conversion_test.cc"], copts = PHASER_COPTS, + data = ["valgrind.supp"], deps = [ "//phaser/runtime:phaser_runtime", "//phaser/testdata:ros_compile_cc_proto", diff --git a/phaser/runtime/BUILD.bazel b/phaser/runtime/BUILD.bazel index 3c39194..3ae8eba 100644 --- a/phaser/runtime/BUILD.bazel +++ b/phaser/runtime/BUILD.bazel @@ -70,6 +70,7 @@ cc_test( name = "ros_wireformat_test", srcs = ["ros_wireformat_test.cc"], copts = PHASER_COPTS, + data = ["//phaser:valgrind.supp"], deps = [ ":phaser_runtime", "@com_google_googletest//:gtest_main", diff --git a/phaser/testdata/BUILD b/phaser/testdata/BUILD index 207f4f0..30cef86 100644 --- a/phaser/testdata/BUILD +++ b/phaser/testdata/BUILD @@ -165,6 +165,7 @@ sh_test( data = [ ":ros_intrinsics_phaser_wire_tool", ":ros_intrinsics_protobuf_wire_tool", + "//phaser:valgrind.supp", ], ) @@ -178,6 +179,7 @@ sh_test( ], data = [ "InvalidArraySize.proto", + "//phaser:valgrind.supp", "//phaser:options_proto", "//phaser/compiler:phaser", "@com_google_protobuf//:protoc", @@ -195,6 +197,7 @@ sh_test( ], data = [ "InvalidRosIntrinsic.proto", + "//phaser:valgrind.supp", "//phaser/compiler:phaser", "@com_google_protobuf//:protoc", "@com_google_protobuf//:well_known_protos", From 301e18869df31de93ec1a0e38da56e931f114d7a Mon Sep 17 00:00:00 2001 From: Dave Allison Date: Tue, 11 Aug 2026 11:41:57 -0700 Subject: [PATCH 4/7] Add ROS1 wire deserialization Decode ROS1 data into native payloads with deterministic oneof handling and bulk primitive I/O. Reject malformed packed fixed-width protobuf fields. --- README.md | 10 +- phaser/all_types_test.cc | 12 + phaser/compiler/message_gen.cc | 387 +++++++++++++++++++++++++- phaser/compiler/message_gen.h | 5 + phaser/docs/phaser_user_guide.md | 31 ++- phaser/ros_wire_conversion_test.cc | 185 +++++++++++- phaser/runtime/arrays.h | 6 + phaser/runtime/ros_wireformat.h | 220 +++++++++++++++ phaser/runtime/ros_wireformat_test.cc | 118 ++++++++ phaser/runtime/vectors.h | 7 + 10 files changed, 958 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 91d62ff..4dadcf2 100644 --- a/README.md +++ b/README.md @@ -170,12 +170,16 @@ absl::Status status = msg.SerializeToROS(ros_output); status = Foo::ProtobufToROS(protobuf_bytes, ros_output); status = Foo::PhaserToROS(phaser_bytes, ros_output); status = Foo::ConvertToROS(input_bytes, ros_output); // infers input format + +// Decode received ROS1 bytes directly into msg's native PayloadBuffer. +status = msg.ParseFromROS(ros_bytes); ``` `ROSBuffer` can own a growing allocation or wrap caller-provided output memory. -ROS output uses little-endian ROS1 layout, is one-way only, and writes an active -`oneof` arm without a discriminator (matching Sato's convention). That `oneof` -encoding is custom and requires the receiver to know which arm is active. +`ROSReader` provides bounds-checked input decoding. ROS wire data uses +little-endian ROS1 layout. Phaser's custom `oneof` layout writes a `uint32` +protobuf field-number discriminator before the selected arm, with zero meaning +unset, so it can be decoded without an external arm selection. `InferMessageWireFormat` validates both the protobuf structure and Phaser payload header; magic alone is not enough because the same four-byte prefix can begin a valid protobuf tag. Malformed or genuinely ambiguous input is reported diff --git a/phaser/all_types_test.cc b/phaser/all_types_test.cc index 87b69cd..db13372 100644 --- a/phaser/all_types_test.cc +++ b/phaser/all_types_test.cc @@ -779,6 +779,18 @@ TEST(AllTypesTest, WireFormatRepeatedPackedBidirectional) { }); } +TEST(AllTypesTest, RejectsPartialPackedFixedWidthElement) { + // Field 3 is repeated fixed64. A nine-byte packed body contains one complete + // value plus a partial value and must not be copied into an eight-byte slot. + std::string malformed; + malformed.push_back(static_cast(0x1a)); // (3 << 3) | length-delimited + malformed.push_back(static_cast(0x09)); + malformed.append(9, '\0'); + + RepeatedPrimitivesPacked message; + EXPECT_FALSE(message.ParseFromString(malformed)); +} + // Proto has [packed=false], but phaser currently emits a length-delimited // packed payload for scalar repeated fields. Protobuf accepts that wire; verify // both directions through protobuf bytes. diff --git a/phaser/compiler/message_gen.cc b/phaser/compiler/message_gen.cc index 6870a32..f28c5c9 100644 --- a/phaser/compiler/message_gen.cc +++ b/phaser/compiler/message_gen.cc @@ -1744,6 +1744,39 @@ std::string MessageGenerator::ROSFieldValueExpression( std::to_string(union_index) + ", " + field->c_type + ">()"; } +static std::string ROSBulkPrimitiveType( + const google::protobuf::FieldDescriptor* field) { + switch (field->type()) { + case google::protobuf::FieldDescriptor::TYPE_INT32: + case google::protobuf::FieldDescriptor::TYPE_SINT32: + case google::protobuf::FieldDescriptor::TYPE_SFIXED32: + return "int32_t"; + case google::protobuf::FieldDescriptor::TYPE_INT64: + case google::protobuf::FieldDescriptor::TYPE_SINT64: + case google::protobuf::FieldDescriptor::TYPE_SFIXED64: + return "int64_t"; + case google::protobuf::FieldDescriptor::TYPE_UINT32: + case google::protobuf::FieldDescriptor::TYPE_FIXED32: + return "uint32_t"; + case google::protobuf::FieldDescriptor::TYPE_UINT64: + case google::protobuf::FieldDescriptor::TYPE_FIXED64: + return "uint64_t"; + case google::protobuf::FieldDescriptor::TYPE_DOUBLE: + return "double"; + case google::protobuf::FieldDescriptor::TYPE_FLOAT: + return "float"; + case google::protobuf::FieldDescriptor::TYPE_BOOL: + return "bool"; + case google::protobuf::FieldDescriptor::TYPE_ENUM: + case google::protobuf::FieldDescriptor::TYPE_STRING: + case google::protobuf::FieldDescriptor::TYPE_BYTES: + case google::protobuf::FieldDescriptor::TYPE_MESSAGE: + case google::protobuf::FieldDescriptor::TYPE_GROUP: + return ""; + } + return ""; +} + void MessageGenerator::GenerateROSFieldSize( std::ostream& os, const google::protobuf::FieldDescriptor* field, const std::string& value_expression, const std::string& indent) { @@ -1879,11 +1912,196 @@ void MessageGenerator::GenerateROSFieldWrite( abort(); } +void MessageGenerator::GenerateROSFieldRead( + std::ostream& os, const google::protobuf::FieldDescriptor* field, + const std::string& target_expression, const std::string& indent, + bool append, const std::string& index_expression, int union_index) { + auto set_value = [&](const std::string& value) { + if (union_index >= 0) { + os << indent << target_expression << ".template Set<" << union_index + << ">(" << value << ");\n"; + } else if (!index_expression.empty()) { + os << indent << target_expression << ".Set(" << index_expression << ", " + << value << ");\n"; + } else if (append) { + os << indent << target_expression << ".Add(" << value << ");\n"; + } else { + os << indent << target_expression << ".Set(" << value << ");\n"; + } + }; + auto mutable_message = [&]() { + if (union_index >= 0) { + return target_expression + ".template Mutable<" + + std::to_string(union_index) + ", " + + MessageName(field->message_type(), true) + ">()"; + } + if (!index_expression.empty()) { + return target_expression + ".Mutable(" + index_expression + ")"; + } + if (append) { + return target_expression + ".Add()"; + } + return target_expression + ".Mutable()"; + }; + auto read_value = [&](const std::string& type, + const std::string& conversion = "") { + os << indent << "{\n"; + os << indent << " absl::StatusOr<" << type + << "> ros_value = buffer.Read<" << type << ">();\n"; + os << indent + << " if (!ros_value.ok()) return ros_value.status();\n"; + const std::string value = + conversion.empty() ? "*ros_value" : conversion + "(*ros_value)"; + set_value(value); + os << indent << "}\n"; + }; + + switch (field->type()) { + case google::protobuf::FieldDescriptor::TYPE_INT32: + case google::protobuf::FieldDescriptor::TYPE_SINT32: + case google::protobuf::FieldDescriptor::TYPE_SFIXED32: + read_value("int32_t"); + return; + case google::protobuf::FieldDescriptor::TYPE_INT64: + case google::protobuf::FieldDescriptor::TYPE_SINT64: + case google::protobuf::FieldDescriptor::TYPE_SFIXED64: + read_value("int64_t"); + return; + case google::protobuf::FieldDescriptor::TYPE_UINT32: + case google::protobuf::FieldDescriptor::TYPE_FIXED32: + read_value("uint32_t"); + return; + case google::protobuf::FieldDescriptor::TYPE_UINT64: + case google::protobuf::FieldDescriptor::TYPE_FIXED64: + read_value("uint64_t"); + return; + case google::protobuf::FieldDescriptor::TYPE_DOUBLE: + read_value("double"); + return; + case google::protobuf::FieldDescriptor::TYPE_FLOAT: + read_value("float"); + return; + case google::protobuf::FieldDescriptor::TYPE_BOOL: + read_value("bool"); + return; + case google::protobuf::FieldDescriptor::TYPE_ENUM: + read_value("int32_t", "static_cast<" + EnumName(field->enum_type()) + ">"); + return; + case google::protobuf::FieldDescriptor::TYPE_STRING: + case google::protobuf::FieldDescriptor::TYPE_BYTES: + os << indent << "{\n"; + os << indent + << " absl::StatusOr ros_value = " + "buffer.ReadString();\n"; + os << indent + << " if (!ros_value.ok()) return ros_value.status();\n"; + set_value("*ros_value"); + os << indent << "}\n"; + return; + case google::protobuf::FieldDescriptor::TYPE_MESSAGE: + if (IsAny(field)) { + os << indent << "{\n"; + os << indent + << " absl::StatusOr ros_type_url = " + "buffer.ReadString();\n"; + os << indent + << " if (!ros_type_url.ok()) return ros_type_url.status();\n"; + os << indent + << " absl::StatusOr ros_any_value = " + "buffer.ReadString();\n"; + os << indent + << " if (!ros_any_value.ok()) return ros_any_value.status();\n"; + os << indent + << " if (!ros_type_url->empty() || !ros_any_value->empty()) {\n"; + os << indent + << " return absl::UnimplementedError(\"ROS1 deserialization of " + "a populated google.protobuf.Any is unsupported\");\n"; + os << indent << " }\n"; + os << indent << " " << mutable_message() << "->Clear();\n"; + os << indent << "}\n"; + } else if (IsRosFrontend() && IsRosTime(field->message_type())) { + os << indent << "{\n"; + os << indent + << " absl::StatusOr ros_sec = " + "buffer.Read();\n"; + os << indent << " if (!ros_sec.ok()) return ros_sec.status();\n"; + os << indent + << " absl::StatusOr ros_nsec = " + "buffer.Read();\n"; + os << indent << " if (!ros_nsec.ok()) return ros_nsec.status();\n"; + os << indent << " ::ros::Time ros_value;\n"; + os << indent << " ros_value.sec = *ros_sec;\n"; + os << indent << " ros_value.nsec = *ros_nsec;\n"; + set_value("ros_value"); + os << indent << "}\n"; + } else if (IsRosFrontend() && + IsRosDuration(field->message_type())) { + os << indent << "{\n"; + os << indent + << " absl::StatusOr ros_sec = " + "buffer.Read();\n"; + os << indent << " if (!ros_sec.ok()) return ros_sec.status();\n"; + os << indent + << " absl::StatusOr ros_nsec = " + "buffer.Read();\n"; + os << indent << " if (!ros_nsec.ok()) return ros_nsec.status();\n"; + os << indent << " ::ros::Duration ros_value;\n"; + os << indent << " ros_value.sec = *ros_sec;\n"; + os << indent << " ros_value.nsec = *ros_nsec;\n"; + set_value("ros_value"); + os << indent << "}\n"; + } else if (IsRosFrontend() && IsRosHeader(field->message_type())) { + os << indent << "{\n"; + os << indent + << " absl::StatusOr ros_seq = " + "buffer.Read();\n"; + os << indent << " if (!ros_seq.ok()) return ros_seq.status();\n"; + os << indent + << " absl::StatusOr ros_sec = " + "buffer.Read();\n"; + os << indent << " if (!ros_sec.ok()) return ros_sec.status();\n"; + os << indent + << " absl::StatusOr ros_nsec = " + "buffer.Read();\n"; + os << indent << " if (!ros_nsec.ok()) return ros_nsec.status();\n"; + os << indent + << " absl::StatusOr ros_frame_id = " + "buffer.ReadString();\n"; + os << indent + << " if (!ros_frame_id.ok()) return ros_frame_id.status();\n"; + os << indent << " ::std_msgs::Header ros_value;\n"; + os << indent << " ros_value.seq = *ros_seq;\n"; + os << indent << " ros_value.stamp.sec = *ros_sec;\n"; + os << indent << " ros_value.stamp.nsec = *ros_nsec;\n"; + os << indent + << " ros_value.frame_id.assign(ros_frame_id->data(), " + "ros_frame_id->size());\n"; + set_value("ros_value"); + os << indent << "}\n"; + } else { + os << indent << "{\n"; + os << indent << " auto* ros_message = " << mutable_message() << ";\n"; + os << indent + << " if (absl::Status status = " + "ros_message->DeserializeFromROS(buffer); !status.ok()) " + "return status;\n"; + os << indent << "}\n"; + } + return; + case google::protobuf::FieldDescriptor::TYPE_GROUP: + abort(); + } + abort(); +} + void MessageGenerator::GenerateROSSerialization(std::ostream& os, bool decl) { const std::string name = MessageName(message_); if (decl) { os << " size_t ROSSerializedSize() const;\n"; os << " absl::Status SerializeToROS(::phaser::ROSBuffer& buffer) const;\n"; + os << " absl::Status DeserializeFromROS(" + "::phaser::ROSReader& buffer);\n"; + os << " absl::Status ParseFromROS(absl::Span input);\n"; os << R"XXX( absl::Status SerializeToROSArray(void* data, size_t size) const { ::phaser::ROSBuffer buffer(data, size); return SerializeToROS(buffer); @@ -1919,6 +2137,7 @@ void MessageGenerator::GenerateROSSerialization(std::ostream& os, bool decl) { for (const auto& item : fields_in_order_) { if (item->IsUnion()) { auto union_info = std::static_pointer_cast(item); + os << " size += 4;\n"; os << " switch (" << union_info->member_name << ".Discriminator()) {\n"; for (size_t i = 0; i < union_info->members.size(); ++i) { @@ -1950,11 +2169,20 @@ void MessageGenerator::GenerateROSSerialization(std::ostream& os, bool decl) { const std::string count = fixed_extent > 0 ? std::to_string(fixed_extent) : item->member_name + ".size()"; - os << " for (size_t ros_index = 0; ros_index < " << count - << "; ++ros_index) {\n"; - GenerateROSFieldSize(os, descriptor, - item->member_name + ".Get(ros_index)", " "); - os << " }\n"; + std::string bulk_type = ROSBulkPrimitiveType(descriptor); + if (descriptor->type() == + google::protobuf::FieldDescriptor::TYPE_ENUM) { + bulk_type = EnumName(descriptor->enum_type()); + } + if (!bulk_type.empty()) { + os << " size += " << count << " * sizeof(" << bulk_type << ");\n"; + } else { + os << " for (size_t ros_index = 0; ros_index < " << count + << "; ++ros_index) {\n"; + GenerateROSFieldSize(os, descriptor, + item->member_name + ".Get(ros_index)", " "); + os << " }\n"; + } } os << " return size;\n"; } @@ -1977,6 +2205,9 @@ void MessageGenerator::GenerateROSSerialization(std::ostream& os, bool decl) { for (const auto& item : fields_in_order_) { if (item->IsUnion()) { auto union_info = std::static_pointer_cast(item); + os << " if (absl::Status status = buffer.Write(static_cast(" + << union_info->member_name + << ".Discriminator())); !status.ok()) return status;\n"; os << " switch (" << union_info->member_name << ".Discriminator()) {\n"; for (size_t i = 0; i < union_info->members.size(); ++i) { @@ -2010,16 +2241,152 @@ void MessageGenerator::GenerateROSSerialization(std::ostream& os, bool decl) { const std::string count = fixed_extent > 0 ? std::to_string(fixed_extent) : item->member_name + ".size()"; - os << " for (size_t ros_index = 0; ros_index < " << count - << "; ++ros_index) {\n"; - GenerateROSFieldWrite(os, descriptor, - item->member_name + ".Get(ros_index)", " "); - os << " }\n"; + std::string bulk_type = ROSBulkPrimitiveType(descriptor); + if (descriptor->type() == + google::protobuf::FieldDescriptor::TYPE_ENUM) { + bulk_type = EnumName(descriptor->enum_type()); + } + if (!bulk_type.empty()) { + if (fixed_extent > 0) { + os << " if (" << item->member_name << ".data() == nullptr) {\n"; + os << " if (absl::Status status = buffer.WriteZeros(" << count + << " * sizeof(" << bulk_type + << ")); !status.ok()) return status;\n"; + os << " } else {\n"; + os << " if (absl::Status status = buffer.WriteArray<" << bulk_type + << ">(absl::Span(" + << item->member_name << ".data(), " << count + << ")); !status.ok()) return status;\n"; + os << " }\n"; + } else { + os << " if (absl::Status status = buffer.WriteArray<" << bulk_type + << ">(absl::Span(" + << item->member_name << ".data(), " << count + << ")); !status.ok()) return status;\n"; + } + } else { + os << " for (size_t ros_index = 0; ros_index < " << count + << "; ++ros_index) {\n"; + GenerateROSFieldWrite(os, descriptor, + item->member_name + ".Get(ros_index)", " "); + os << " }\n"; + } } } os << " return absl::OkStatus();\n"; os << "}\n\n"; + os << "absl::Status " << name + << "::DeserializeFromROS(::phaser::ROSReader& buffer) {\n"; + os << " Clear();\n"; + if (IsRosTime(message_)) { + os << " absl::StatusOr ros_sec = buffer.Read();\n"; + os << " if (!ros_sec.ok()) return ros_sec.status();\n"; + os << " absl::StatusOr ros_nsec = buffer.Read();\n"; + os << " if (!ros_nsec.ok()) return ros_nsec.status();\n"; + os << " set_seconds(static_cast(*ros_sec));\n"; + os << " set_nanos(static_cast(*ros_nsec));\n"; + } else if (IsRosDuration(message_)) { + os << " absl::StatusOr ros_sec = buffer.Read();\n"; + os << " if (!ros_sec.ok()) return ros_sec.status();\n"; + os << " absl::StatusOr ros_nsec = buffer.Read();\n"; + os << " if (!ros_nsec.ok()) return ros_nsec.status();\n"; + os << " set_seconds(static_cast(*ros_sec));\n"; + os << " set_nanos(*ros_nsec);\n"; + } else { + for (const auto& item : fields_in_order_) { + if (item->IsUnion()) { + auto union_info = std::static_pointer_cast(item); + os << " {\n"; + os << " absl::StatusOr ros_discriminator = " + "buffer.Read();\n"; + os << " if (!ros_discriminator.ok()) return " + "ros_discriminator.status();\n"; + os << " switch (*ros_discriminator) {\n"; + os << " case 0:\n"; + os << " " << union_info->member_name << ".reset();\n"; + os << " break;\n"; + for (size_t i = 0; i < union_info->members.size(); ++i) { + const auto& field = union_info->members[i]; + os << " case " << field->field->number() << ":\n"; + GenerateROSFieldRead(os, field->field, union_info->member_name, + " ", false, "", + static_cast(i)); + os << " break;\n"; + } + os << " default:\n"; + os << " return absl::InvalidArgumentError(" + "\"Unknown ROS oneof discriminator\");\n"; + os << " }\n"; + os << " }\n"; + continue; + } + + const auto* descriptor = item->field; + if (!descriptor->is_repeated()) { + GenerateROSFieldRead(os, descriptor, item->member_name, " "); + continue; + } + + const int fixed_extent = GetArraySize(descriptor); + std::string bulk_type = ROSBulkPrimitiveType(descriptor); + if (descriptor->type() == + google::protobuf::FieldDescriptor::TYPE_ENUM) { + bulk_type = EnumName(descriptor->enum_type()); + } + if (fixed_extent <= 0) { + os << " {\n"; + os << " absl::StatusOr ros_count = " + "buffer.ReadSequenceLength();\n"; + os << " if (!ros_count.ok()) return ros_count.status();\n"; + if (!bulk_type.empty()) { + os << " if (*ros_count > buffer.Remaining() / sizeof(" + << bulk_type << ")) {\n"; + os << " return absl::InvalidArgumentError(" + "\"ROS sequence length exceeds remaining input\");\n"; + os << " }\n"; + os << " " << item->member_name << ".resize(*ros_count);\n"; + os << " if (absl::Status status = buffer.ReadArray<" << bulk_type + << ">(absl::Span<" << bulk_type << ">(" << item->member_name + << ".data(), static_cast(*ros_count))); " + "!status.ok()) return status;\n"; + } else { + os << " for (uint32_t ros_index = 0; ros_index < *ros_count; " + "++ros_index) {\n"; + GenerateROSFieldRead(os, descriptor, item->member_name, " ", + true); + os << " }\n"; + } + os << " }\n"; + } else if (!bulk_type.empty()) { + os << " if (absl::Status status = buffer.ReadArray<" << bulk_type + << ">(absl::Span<" << bulk_type << ">(" << item->member_name + << ".data(), " << fixed_extent + << ")); !status.ok()) return status;\n"; + } else { + os << " for (size_t ros_index = 0; ros_index < " << fixed_extent + << "; ++ros_index) {\n"; + GenerateROSFieldRead(os, descriptor, item->member_name, " ", false, + "ros_index"); + os << " }\n"; + } + } + } + os << " return absl::OkStatus();\n"; + os << "}\n\n"; + + os << "absl::Status " << name + << "::ParseFromROS(absl::Span input) {\n"; + os << " ::phaser::ROSReader buffer(input);\n"; + os << " if (absl::Status status = DeserializeFromROS(buffer); " + "!status.ok()) return status;\n"; + os << " if (!buffer.Eof()) {\n"; + os << " return absl::InvalidArgumentError(" + "\"Trailing bytes after ROS message\");\n"; + os << " }\n"; + os << " return absl::OkStatus();\n"; + os << "}\n\n"; + os << "absl::Status " << name << "::ProtobufToROS(std::string_view protobuf, " "::phaser::ROSBuffer& output) {\n"; diff --git a/phaser/compiler/message_gen.h b/phaser/compiler/message_gen.h index c7f9801..8b97297 100644 --- a/phaser/compiler/message_gen.h +++ b/phaser/compiler/message_gen.h @@ -119,6 +119,11 @@ class MessageGenerator { void GenerateROSFieldWrite( std::ostream& os, const google::protobuf::FieldDescriptor* field, const std::string& value_expression, const std::string& indent); + void GenerateROSFieldRead( + std::ostream& os, const google::protobuf::FieldDescriptor* field, + const std::string& target_expression, const std::string& indent, + bool append = false, const std::string& index_expression = "", + int union_index = -1); std::string ROSFieldValueExpression( const std::shared_ptr& field, const std::shared_ptr& union_field = nullptr, diff --git a/phaser/docs/phaser_user_guide.md b/phaser/docs/phaser_user_guide.md index cb87961..5b57442 100644 --- a/phaser/docs/phaser_user_guide.md +++ b/phaser/docs/phaser_user_guide.md @@ -380,14 +380,17 @@ throws `std::bad_variant_access` for an inactive arm. Switching arms clears and releases any string or message storage owned by the previous arm. ### ROS1 wire conversion -Every generated message supports one-way conversion to ROS1 serialization, -regardless of whether its C++ frontend is protobuf-style or ROS-style: +Every generated message supports conversion to ROS1 serialization and decoding +from ROS1 bytes into its native Phaser payload, regardless of whether its C++ +frontend is protobuf-style or ROS-style: ```c++ size_t ROSSerializedSize() const; absl::Status SerializeToROS(::phaser::ROSBuffer& output) const; absl::Status SerializeToROSArray(void* output, size_t capacity) const; absl::Status SerializeToROSString(std::string* output) const; +absl::Status DeserializeFromROS(::phaser::ROSReader& input); +absl::Status ParseFromROS(absl::Span input); static absl::Status ProtobufToROS( std::string_view protobuf, ::phaser::ROSBuffer& output); @@ -406,12 +409,22 @@ complete generic protobuf field structure and the Phaser payload header rather than checking only the four-byte magic: those magic bytes can also begin a valid protobuf tag. The result enum can report `kProtobuf`, `kPhaser`, `kUnknown`, or `kAmbiguous`; automatic conversion rejects the latter two. -There is intentionally no ROS-to-protobuf or ROS-to-Phaser conversion. + +`ParseFromROS` clears the target message, decodes fields in schema order, and +requires the complete input span to be consumed. A default-constructed target +stores the result in its dynamically allocated native `PayloadBuffer`; a target +created with `CreateMutable` stores it in caller-provided payload memory. +`DeserializeFromROS` accepts an existing `ROSReader` and is used recursively for +inline nested messages. Malformed input returns an `absl::Status` and may leave +the target partially populated, so callers should discard or clear it after an +error. `ROSBuffer` is declared in `phaser/runtime/ros_wireformat.h`. Its default constructor owns a dynamically growing allocation; constructing it with a pointer and size wraps fixed caller-owned output memory. Writes return an `absl::Status`, including insufficient-capacity and malformed-protobuf errors. +`ROSReader` is a non-owning view over received bytes and checks every primitive, +string length, and sequence-length read against the remaining input. The generated layout follows ROS1 serialization rules: @@ -425,12 +438,12 @@ The generated layout follows ROS1 serialization rules: - `Timestamp`, `Duration`, and `Header` use the ROS1 time, duration, and header layouts described above. -ROS1 has no standard union encoding. Phaser follows Sato's convention for a -protobuf `oneof`: only the active arm is written, with no discriminator, and an -unset `oneof` writes no bytes. The result is not self-describing; the receiver -must know which arm is active through an external contract. A populated -`google.protobuf.Any` is rejected because its dynamic type has no static ROS1 -layout. +ROS1 has no standard union encoding. Phaser uses a custom protobuf `oneof` +layout: a little-endian `uint32` containing the active protobuf field number is +written before the arm value, and zero represents an unset `oneof`. Decoding +rejects unknown discriminator values. This replaces the earlier +discriminator-free convention. A populated `google.protobuf.Any` is rejected +in either direction because its dynamic type has no static ROS1 layout. Both frontends use the same native binary metadata and protobuf wire serialization. A protobuf-style target and a ROS-style target generated from diff --git a/phaser/ros_wire_conversion_test.cc b/phaser/ros_wire_conversion_test.cc index 3b660ec..f49fe24 100644 --- a/phaser/ros_wire_conversion_test.cc +++ b/phaser/ros_wire_conversion_test.cc @@ -8,6 +8,7 @@ #include #include #include +#include #include #include "absl/types/span.h" @@ -149,7 +150,10 @@ std::string ExpectedRosCompileBytes(bool include_oneof = true) { AppendIntegral(bytes, static_cast(400)); if (include_oneof) { + AppendIntegral(bytes, static_cast(17)); AppendString(bytes, "selected"); + } else { + AppendIntegral(bytes, static_cast(0)); } return bytes; } @@ -176,6 +180,7 @@ std::string ExpectedIntrinsicBytes() { AppendIntegral(bytes, static_cast(0)); // child id AppendString(bytes, ""); // child label } + AppendIntegral(bytes, static_cast(0)); // choice unset return bytes; } @@ -254,7 +259,7 @@ TEST(ROSWireConversionTest, FixedOutputAndErrorsAreReported) { .ok()); } -TEST(ROSWireConversionTest, UnsetOneofWritesNoBytesForTheUnion) { +TEST(ROSWireConversionTest, OneofWritesFieldNumberDiscriminator) { RosCompileMessage message; PopulatePhaserMessage(message); message.choice.reset(); @@ -300,4 +305,182 @@ TEST(ROSWireConversionTest, ROS1IntrinsicsUseNativeLayoutsAndFlushCaches) { EXPECT_EQ(protobuf_frontend_native_output.AsString(), expected); } +TEST(ROSWireConversionTest, ParsesKnownROSBytesIntoNativePayload) { + RosCompileMessage message; + const std::string input = ExpectedRosCompileBytes(); + ASSERT_TRUE(message.ParseFromROS( + absl::Span(input.data(), input.size())) + .ok()); + + EXPECT_EQ(message.x.Get(), -7); + EXPECT_EQ(message.name.Get(), "robot"); + EXPECT_TRUE(message.flag.Get()); + EXPECT_DOUBLE_EQ(message.value.Get(), 1.5); + EXPECT_EQ(message.color.Get(), RosColor::ROS_COLOR_RED); + EXPECT_EQ(message.inner->id.Get(), 42); + ASSERT_EQ(message.xs.size(), 2u); + EXPECT_EQ(message.xs[0], 10); + EXPECT_EQ(message.xs[1], -20); + ASSERT_EQ(message.names.size(), 2u); + EXPECT_EQ(message.names[0].Get(), "a"); + EXPECT_EQ(message.names[1].Get(), "beta"); + ASSERT_EQ(message.inners.size(), 2u); + EXPECT_EQ(message.inners[0]->id.Get(), 100); + EXPECT_EQ(message.inners[1]->id.Get(), 200); + EXPECT_EQ(message.fixed_ints[0], 1); + EXPECT_EQ(message.fixed_ints[3], 4); + EXPECT_EQ(message.fixed_names[0].Get(), "left"); + EXPECT_EQ(message.fixed_inners[1]->id.Get(), 400); + + using ChoiceName = RosCompileMessage::ChoiceNameAlternative; + ASSERT_TRUE(message.choice.holds_alternative()); + EXPECT_EQ(message.choice.get(), "selected"); + + ::foo::bar::RosCompileMessage protobuf; + ASSERT_TRUE(protobuf.ParseFromString(message.SerializeAsString())); + EXPECT_EQ(protobuf.x(), -7); + EXPECT_EQ(protobuf.name(), "robot"); + ASSERT_EQ(protobuf.xs_size(), 2); + EXPECT_EQ(protobuf.xs(1), -20); + EXPECT_EQ(protobuf.fixed_inners(1).id(), 400); + EXPECT_EQ(protobuf.choice_name(), "selected"); +} + +TEST(ROSWireConversionTest, ParsedROSPayloadUsesEitherFrontend) { + RosIntrinsicMessage ros_message; + const std::string input = ExpectedIntrinsicBytes(); + ASSERT_TRUE(ros_message + .ParseFromROS( + absl::Span(input.data(), input.size())) + .ok()); + + EXPECT_EQ(ros_message.stamp->sec, 12u); + EXPECT_EQ(ros_message.stamp->nsec, 345u); + EXPECT_EQ(ros_message.timeout->sec, -4); + EXPECT_EQ(ros_message.timeout->nsec, 500); + EXPECT_EQ(ros_message.header->seq, 9u); + EXPECT_EQ(ros_message.header->stamp.sec, 21u); + EXPECT_EQ(ros_message.header->stamp.nsec, 654u); + EXPECT_EQ(ros_message.header->frame_id, "map"); + EXPECT_EQ(ros_message.choice.index(), std::variant_npos); + + const size_t native_size = ros_message.Size(); + std::vector native_payload(native_size); + std::memcpy(native_payload.data(), ros_message.Data(), native_size); + const ProtobufFrontendIntrinsicMessage protobuf_view = + ProtobufFrontendIntrinsicMessage::CreateReadonly(native_payload.data(), + native_payload.size()); + EXPECT_EQ(protobuf_view.stamp().seconds(), 12); + EXPECT_EQ(protobuf_view.stamp().nanos(), 345); + EXPECT_EQ(protobuf_view.timeout().seconds(), -4); + EXPECT_EQ(protobuf_view.timeout().nanos(), 500); + EXPECT_EQ(protobuf_view.header().seq(), 9u); + EXPECT_EQ(protobuf_view.header().frame_id(), "map"); + EXPECT_FALSE(protobuf_view.has_choice_number()); + EXPECT_FALSE(protobuf_view.has_choice_text()); + EXPECT_FALSE(protobuf_view.has_choice_child()); + + ProtobufFrontendIntrinsicMessage parsed_protobuf_frontend; + ASSERT_TRUE(parsed_protobuf_frontend + .ParseFromROS( + absl::Span(input.data(), input.size())) + .ok()); + EXPECT_EQ(parsed_protobuf_frontend.stamp().seconds(), 12); + EXPECT_EQ(parsed_protobuf_frontend.timeout().nanos(), 500); + EXPECT_EQ(parsed_protobuf_frontend.header().stamp().nanos(), 654); + EXPECT_EQ(parsed_protobuf_frontend.header().frame_id(), "map"); +} + +TEST(ROSWireConversionTest, ParsesScalarAndMessageOneofArms) { + std::string scalar_input = ExpectedRosCompileBytes(false); + scalar_input.resize(scalar_input.size() - sizeof(uint32_t)); + AppendIntegral(scalar_input, static_cast(15)); + AppendIntegral(scalar_input, static_cast(123)); + + RosCompileMessage scalar_message; + ASSERT_TRUE( + scalar_message + .ParseFromROS( + absl::Span(scalar_input.data(), scalar_input.size())) + .ok()); + using ChoiceCount = RosCompileMessage::ChoiceCountAlternative; + ASSERT_TRUE(scalar_message.choice.holds_alternative()); + EXPECT_EQ(scalar_message.choice.get(), 123); + + std::string message_input = ExpectedRosCompileBytes(false); + message_input.resize(message_input.size() - sizeof(uint32_t)); + AppendIntegral(message_input, static_cast(18)); + AppendIntegral(message_input, static_cast(456)); + + RosCompileMessage message; + ASSERT_TRUE( + message + .ParseFromROS( + absl::Span(message_input.data(), message_input.size())) + .ok()); + using ChoiceInner = RosCompileMessage::ChoiceInnerAlternative; + ASSERT_TRUE(message.choice.holds_alternative()); + EXPECT_EQ(message.choice.get().id.Get(), 456); +} + +TEST(ROSWireConversionTest, RejectsMalformedROSInput) { + const std::string valid = ExpectedRosCompileBytes(); + + std::string truncated = valid; + truncated.pop_back(); + RosCompileMessage truncated_message; + EXPECT_FALSE(truncated_message + .ParseFromROS(absl::Span(truncated.data(), + truncated.size())) + .ok()); + + std::string trailing = valid; + trailing.push_back('\0'); + RosCompileMessage trailing_message; + EXPECT_FALSE( + trailing_message + .ParseFromROS( + absl::Span(trailing.data(), trailing.size())) + .ok()); + + std::string invalid_length = valid; + invalid_length[4] = static_cast(0xff); + invalid_length[5] = static_cast(0xff); + invalid_length[6] = static_cast(0xff); + invalid_length[7] = static_cast(0x7f); + RosCompileMessage invalid_length_message; + EXPECT_FALSE( + invalid_length_message + .ParseFromROS(absl::Span(invalid_length.data(), + invalid_length.size())) + .ok()); + + std::string invalid_discriminator = valid; + const size_t discriminator_offset = + invalid_discriminator.size() - sizeof(uint32_t) - sizeof(uint32_t) - 8; + invalid_discriminator[discriminator_offset] = 99; + RosCompileMessage invalid_discriminator_message; + EXPECT_FALSE( + invalid_discriminator_message + .ParseFromROS(absl::Span(invalid_discriminator.data(), + invalid_discriminator.size())) + .ok()); + + std::string oversized_sequence; + AppendIntegral(oversized_sequence, static_cast(0)); + AppendString(oversized_sequence, ""); + AppendIntegral(oversized_sequence, static_cast(0)); + AppendDouble(oversized_sequence, 0); + AppendIntegral(oversized_sequence, static_cast(0)); + AppendIntegral(oversized_sequence, static_cast(0)); + AppendIntegral(oversized_sequence, static_cast(100)); + RosCompileMessage oversized_sequence_message; + EXPECT_FALSE( + oversized_sequence_message + .ParseFromROS(absl::Span(oversized_sequence.data(), + oversized_sequence.size())) + .ok()); + EXPECT_TRUE(oversized_sequence_message.xs.empty()); +} + } // namespace diff --git a/phaser/runtime/arrays.h b/phaser/runtime/arrays.h index 018073c..0d93ac0 100644 --- a/phaser/runtime/arrays.h +++ b/phaser/runtime/arrays.h @@ -368,6 +368,12 @@ class PrimitiveArrayField : public Field { if (!payload.ok()) { return payload.status(); } + if constexpr (FixedSize) { + if (payload->size() % sizeof(T) != 0) { + return absl::InvalidArgumentError( + "Packed fixed-width array field has a partial element"); + } + } size_t count = payload->size() / sizeof(T); if (parsed_count_ + count > N) { return absl::InvalidArgumentError("array_size overflow"); diff --git a/phaser/runtime/ros_wireformat.h b/phaser/runtime/ros_wireformat.h index 0fadbf5..2f377e5 100644 --- a/phaser/runtime/ros_wireformat.h +++ b/phaser/runtime/ros_wireformat.h @@ -17,11 +17,167 @@ #include #include "absl/status/status.h" +#include "absl/status/statusor.h" #include "absl/strings/str_format.h" #include "absl/types/span.h" namespace phaser { +// A non-owning sequential reader for ROS1 wire data. +class ROSReader { + public: + explicit ROSReader(absl::Span data) : data_(data) {} + explicit ROSReader(std::string_view data) + : data_(data.data(), data.size()) {} + explicit ROSReader(const std::string& data) + : ROSReader(std::string_view(data)) {} + ROSReader(std::string&&) = delete; + + size_t Position() const { return position_; } + size_t Remaining() const { return data_.size() - position_; } + bool Eof() const { return position_ == data_.size(); } + + template + absl::StatusOr Read() { + static_assert(std::is_arithmetic_v, + "ROSReader::Read only supports arithmetic types"); + if (Remaining() < sizeof(T)) { + return absl::InvalidArgumentError(absl::StrFormat( + "Truncated ROS input at byte %d: need %d bytes, have %d", position_, + sizeof(T), Remaining())); + } + + if constexpr (std::is_same_v, bool>) { + const uint8_t value = static_cast( + static_cast(data_[position_++])); + if (value > 1) { + return absl::InvalidArgumentError(absl::StrFormat( + "Invalid ROS bool value %d at byte %d", value, position_ - 1)); + } + return value != 0; + } else if constexpr (std::is_integral_v) { + using U = std::make_unsigned_t; + U value = 0; + for (size_t i = 0; i < sizeof(U); ++i) { + const U byte = static_cast( + static_cast(data_[position_++])); + value |= static_cast(static_cast(byte) + << static_cast(i * 8)); + } + T result = 0; + memcpy(&result, &value, sizeof(result)); + return result; + } else { + using U = std::conditional_t; + static_assert(sizeof(U) == sizeof(T)); + U bits = 0; + for (size_t i = 0; i < sizeof(U); ++i) { + const U byte = static_cast( + static_cast(data_[position_++])); + bits |= static_cast(static_cast(byte) + << static_cast(i * 8)); + } + T value = 0; + memcpy(&value, &bits, sizeof(value)); + return value; + } + } + + template + absl::Status ReadArray(absl::Span values) { + static_assert(std::is_arithmetic_v || std::is_enum_v, + "ROSReader::ReadArray only supports primitive types"); + constexpr size_t kWireElementSize = + std::is_enum_v ? sizeof(int32_t) : sizeof(T); + if constexpr (std::is_enum_v) { + static_assert(sizeof(T) == sizeof(int32_t), + "ROS enum storage must be 32 bits"); + } + if (values.size() > + std::numeric_limits::max() / kWireElementSize) { + return absl::InvalidArgumentError("ROS array byte size overflow"); + } + const size_t byte_size = values.size() * kWireElementSize; + if (Remaining() < byte_size) { + return absl::InvalidArgumentError(absl::StrFormat( + "Truncated ROS array at byte %d: need %d bytes, have %d", position_, + byte_size, Remaining())); + } + if (values.empty()) { + return absl::OkStatus(); + } + + if constexpr (std::is_same_v, bool>) { + static_assert(sizeof(bool) == sizeof(uint8_t)); + for (size_t i = 0; i < values.size(); ++i) { + const uint8_t value = static_cast( + static_cast(data_[position_ + i])); + if (value > 1) { + return absl::InvalidArgumentError(absl::StrFormat( + "Invalid ROS bool value %d at byte %d", value, position_ + i)); + } + } + for (size_t i = 0; i < values.size(); ++i) { + values[i] = data_[position_ + i] != 0; + } + position_ += byte_size; + return absl::OkStatus(); + } + + const uint16_t endian_marker = 1; + const bool little_endian = + *reinterpret_cast(&endian_marker) == 1; + if (little_endian) { + memcpy(values.data(), data_.data() + position_, byte_size); + position_ += byte_size; + return absl::OkStatus(); + } + + if constexpr (std::is_enum_v) { + for (T& value : values) { + absl::StatusOr decoded = Read(); + if (!decoded.ok()) { + return decoded.status(); + } + value = static_cast(*decoded); + } + } else { + for (T& value : values) { + absl::StatusOr decoded = Read(); + if (!decoded.ok()) { + return decoded.status(); + } + value = *decoded; + } + } + return absl::OkStatus(); + } + + absl::StatusOr ReadString() { + absl::StatusOr length = Read(); + if (!length.ok()) { + return length.status(); + } + if (*length > Remaining()) { + return absl::InvalidArgumentError(absl::StrFormat( + "Truncated ROS string at byte %d: length %d exceeds remaining %d", + position_, *length, Remaining())); + } + const std::string_view value(data_.data() + position_, *length); + position_ += *length; + return value; + } + + absl::StatusOr ReadSequenceLength() { + return Read(); + } + + private: + absl::Span data_; + size_t position_ = 0; +}; + // A sequential ROS1 serialization buffer. ROS1 primitives are always encoded // little-endian, strings and variable-length sequences use uint32 length // prefixes, and messages do not carry tags or alignment padding. @@ -87,6 +243,70 @@ class ROSBuffer { return absl::OkStatus(); } + absl::Status WriteZeros(size_t length) { + if (absl::Status status = EnsureSpace(length); !status.ok()) { + return status; + } + if (length != 0) { + memset(data_ + size_, 0, length); + size_ += length; + } + return absl::OkStatus(); + } + + template + absl::Status WriteArray(absl::Span values) { + static_assert(std::is_arithmetic_v || std::is_enum_v, + "ROSBuffer::WriteArray only supports primitive types"); + constexpr size_t kWireElementSize = + std::is_enum_v ? sizeof(int32_t) : sizeof(T); + if constexpr (std::is_enum_v) { + static_assert(sizeof(T) == sizeof(int32_t), + "ROS enum storage must be 32 bits"); + } + if (values.size() > + std::numeric_limits::max() / kWireElementSize) { + return absl::ResourceExhaustedError("ROS array byte size overflow"); + } + const size_t byte_size = values.size() * kWireElementSize; + if (absl::Status status = EnsureSpace(byte_size); !status.ok()) { + return status; + } + if (values.empty()) { + return absl::OkStatus(); + } + + if constexpr (std::is_same_v, bool>) { + static_assert(sizeof(bool) == sizeof(uint8_t)); + for (bool value : values) { + data_[size_++] = static_cast(value ? 1 : 0); + } + return absl::OkStatus(); + } + + const uint16_t endian_marker = 1; + const bool little_endian = + *reinterpret_cast(&endian_marker) == 1; + if (little_endian) { + memcpy(data_ + size_, values.data(), byte_size); + size_ += byte_size; + return absl::OkStatus(); + } + + for (const T& value : values) { + absl::Status status; + if constexpr (std::is_enum_v) { + status = Write(static_cast(value)); + } else { + status = Write(value); + } + if (!status.ok()) { + return status; + } + } + return absl::OkStatus(); + } + absl::Status WriteString(std::string_view value) { if (value.size() > std::numeric_limits::max()) { return absl::InvalidArgumentError("ROS1 string exceeds uint32 length"); diff --git a/phaser/runtime/ros_wireformat_test.cc b/phaser/runtime/ros_wireformat_test.cc index 03d8a66..166c9d3 100644 --- a/phaser/runtime/ros_wireformat_test.cc +++ b/phaser/runtime/ros_wireformat_test.cc @@ -13,6 +13,11 @@ namespace phaser { namespace { +enum TestEnum : int { + kFirst = 1, + kSecond = -2, +}; + TEST(ROSWireformatTest, WritesCanonicalLittleEndianBytes) { ROSBuffer buffer; ASSERT_TRUE(buffer.Write(static_cast(-2)).ok()); @@ -61,5 +66,118 @@ TEST(ROSWireformatTest, RejectsInvalidRawWrite) { EXPECT_TRUE(buffer.WriteRaw(nullptr, 0).ok()); } +TEST(ROSWireformatTest, ReadsCanonicalLittleEndianBytes) { + ROSBuffer buffer; + ASSERT_TRUE(buffer.Write(static_cast(-2)).ok()); + ASSERT_TRUE(buffer.Write(static_cast(0xf2345678)).ok()); + ASSERT_TRUE(buffer.Write(true).ok()); + ASSERT_TRUE(buffer.Write(1.5F).ok()); + ASSERT_TRUE(buffer.Write(-2.25).ok()); + ASSERT_TRUE(buffer.WriteString("hello").ok()); + + ROSReader reader(buffer.AsSpan()); + absl::StatusOr int16_value = reader.Read(); + ASSERT_TRUE(int16_value.ok()); + EXPECT_EQ(*int16_value, -2); + absl::StatusOr uint32_value = reader.Read(); + ASSERT_TRUE(uint32_value.ok()); + EXPECT_EQ(*uint32_value, 0xf2345678u); + absl::StatusOr bool_value = reader.Read(); + ASSERT_TRUE(bool_value.ok()); + EXPECT_TRUE(*bool_value); + absl::StatusOr float_value = reader.Read(); + ASSERT_TRUE(float_value.ok()); + EXPECT_FLOAT_EQ(*float_value, 1.5F); + absl::StatusOr double_value = reader.Read(); + ASSERT_TRUE(double_value.ok()); + EXPECT_DOUBLE_EQ(*double_value, -2.25); + absl::StatusOr string_value = reader.ReadString(); + ASSERT_TRUE(string_value.ok()); + EXPECT_EQ(*string_value, "hello"); + EXPECT_TRUE(reader.Eof()); + EXPECT_EQ(reader.Remaining(), 0u); +} + +TEST(ROSWireformatTest, ReaderRejectsTruncatedAndInvalidValues) { + const std::array truncated_integer = {1, 2, 3}; + ROSReader integer_reader(absl::MakeConstSpan(truncated_integer)); + EXPECT_FALSE(integer_reader.Read().ok()); + EXPECT_EQ(integer_reader.Position(), 0u); + + const std::array invalid_bool = {2}; + ROSReader bool_reader(absl::MakeConstSpan(invalid_bool)); + EXPECT_FALSE(bool_reader.Read().ok()); + + const std::array truncated_string = {5, 0, 0, 0, 'a', 'b'}; + ROSReader string_reader(absl::MakeConstSpan(truncated_string)); + EXPECT_FALSE(string_reader.ReadString().ok()); + EXPECT_EQ(string_reader.Position(), sizeof(uint32_t)); +} + +TEST(ROSWireformatTest, ReadsPrimitiveArraysInBulk) { + ROSBuffer buffer; + for (int32_t value : {1, -2, 3, -4}) { + ASSERT_TRUE(buffer.Write(value).ok()); + } + + std::array values = {}; + ROSReader reader(buffer.AsSpan()); + ASSERT_TRUE(reader.ReadArray(absl::MakeSpan(values)).ok()); + EXPECT_EQ(values, (std::array{1, -2, 3, -4})); + EXPECT_TRUE(reader.Eof()); +} + +TEST(ROSWireformatTest, WritesPrimitiveArraysInBulk) { + const std::array integers = {1, -2, 3, -4}; + const std::array bools = {true, false, true}; + const std::array enums = {kFirst, kSecond}; + + ROSBuffer buffer; + ASSERT_TRUE(buffer.WriteArray(absl::MakeConstSpan(integers)).ok()); + ASSERT_TRUE(buffer.WriteArray(absl::MakeConstSpan(bools)).ok()); + ASSERT_TRUE(buffer.WriteArray(absl::MakeConstSpan(enums)).ok()); + ASSERT_TRUE(buffer.WriteZeros(3).ok()); + + ROSReader reader(buffer.AsSpan()); + std::array decoded_integers = {}; + std::array decoded_bools = {}; + std::array decoded_enums = {}; + ASSERT_TRUE( + reader.ReadArray(absl::MakeSpan(decoded_integers)).ok()); + ASSERT_TRUE(reader.ReadArray(absl::MakeSpan(decoded_bools)).ok()); + ASSERT_TRUE(reader.ReadArray(absl::MakeSpan(decoded_enums)).ok()); + EXPECT_EQ(decoded_integers, integers); + EXPECT_EQ(decoded_bools, bools); + EXPECT_EQ(decoded_enums, enums); + EXPECT_EQ(reader.Remaining(), 3u); + for (size_t i = 0; i < 3; ++i) { + absl::StatusOr zero = reader.Read(); + ASSERT_TRUE(zero.ok()); + EXPECT_EQ(*zero, 0); + } +} + +TEST(ROSWireformatTest, BulkWriteFailureDoesNotAdvanceCursor) { + std::array storage = {}; + const std::array values = {1, 2}; + ROSBuffer buffer(storage.data(), storage.size()); + EXPECT_FALSE(buffer.WriteArray(absl::MakeConstSpan(values)).ok()); + EXPECT_EQ(buffer.Size(), 0u); +} + +TEST(ROSWireformatTest, BulkReadValidatesBeforeAdvancing) { + const std::array truncated = {}; + std::array integers = {}; + ROSReader integer_reader(absl::MakeConstSpan(truncated)); + EXPECT_FALSE(integer_reader.ReadArray(absl::MakeSpan(integers)).ok()); + EXPECT_EQ(integer_reader.Position(), 0u); + + const std::array invalid_bool = {0, 2, 1}; + std::array bools = {}; + ROSReader bool_reader(absl::MakeConstSpan(invalid_bool)); + EXPECT_FALSE(bool_reader.ReadArray(absl::MakeSpan(bools)).ok()); + EXPECT_EQ(bool_reader.Position(), 0u); +} + } // namespace } // namespace phaser diff --git a/phaser/runtime/vectors.h b/phaser/runtime/vectors.h index 1f2f6ca..b100151 100644 --- a/phaser/runtime/vectors.h +++ b/phaser/runtime/vectors.h @@ -404,7 +404,14 @@ class PrimitiveVectorField : public Field { return data.status(); } if constexpr (FixedSize) { + if (data->size() % sizeof(T) != 0) { + return absl::InvalidArgumentError( + "Packed fixed-width field has a partial element"); + } resize(data->size() / sizeof(T)); + if (data->empty()) { + return absl::OkStatus(); + } T* base = GetRuntime()->template ToAddress(BaseOffset()); memcpy(base, data->data(), data->size()); return absl::OkStatus(); From 9778222894b6ed13cbc2695f13f43e3b06a53e1b Mon Sep 17 00:00:00 2001 From: Dave Allison Date: Wed, 12 Aug 2026 14:27:22 -0700 Subject: [PATCH 5/7] Make receive paths allocation-free Move runtime metadata into payload storage, return lightweight field views, and transcode ROS wire data directly to avoid heap work on receive paths. --- README.md | 45 +- phaser/BUILD.bazel | 15 + phaser/all_types_test.cc | 20 +- phaser/compiler/message_gen.cc | 1143 +++++++++++++++-- phaser/compiler/message_gen.h | 29 + phaser/docs/phaser_user_guide.md | 103 +- phaser/perf_test.cc | 280 +++- phaser/phaser_test.cc | 58 + phaser/receive_allocation_test.cc | 492 +++++++ phaser/ros_compile_test.cc | 20 +- phaser/ros_intrinsics_test.cc | 13 +- .../ros_native_frontend_compatibility_test.cc | 4 +- phaser/ros_wire_conversion_test.cc | 107 ++ phaser/runtime/any.h | 104 +- phaser/runtime/arrays.h | 130 +- phaser/runtime/fields.h | 89 +- phaser/runtime/message.cc | 67 - phaser/runtime/message.h | 372 +++++- phaser/runtime/message_test.cc | 115 +- phaser/runtime/phaser_bank.cc | 44 +- phaser/runtime/phaser_bank.h | 29 +- phaser/runtime/ros.h | 209 ++- phaser/runtime/ros_wireformat.h | 36 +- phaser/runtime/ros_wireformat_test.cc | 14 +- phaser/runtime/runtime.h | 2 +- phaser/runtime/union.h | 7 +- phaser/runtime/vectors.h | 564 ++++---- phaser/runtime/wireformat.h | 99 ++ phaser/stress_test.cc | 2 +- phaser/testdata/RosCompile.proto | 10 + phaser/testdata/TestMessage.proto | 170 ++- .../ros_intrinsics_phaser_wire_tool.cc | 4 +- 32 files changed, 3520 insertions(+), 876 deletions(-) create mode 100644 phaser/receive_allocation_test.cc diff --git a/README.md b/README.md index 4dadcf2..4caa5ff 100644 --- a/README.md +++ b/README.md @@ -85,9 +85,14 @@ The key idea is the split between two representations: - When you call `set_x(...)`, the value is written **directly into the binary buffer**. - When you call `x()`, the value is read back **from the binary buffer**, located via a - small per-message **field-metadata** array. That indirection is what enables protobuf's - version compatibility: a reader built with a different schema version can still find the - fields present in the data. + compact per-message **field-metadata** index. Dense field-number ranges use direct lookup; + outliers use binary search. That indirection is what enables protobuf's version + compatibility: a reader built with a different schema version can still find the fields + present in the data. + +The hybrid metadata layout is a new native Phaser payload format. New runtimes can +read legacy payload metadata, but older runtimes cannot read newly generated native +payloads. Protobuf and ROS wire formats are unchanged. The `PayloadBuffer` (from the [cpp_toolbelt](https://github.com/dallison/cpp_toolbelt) library) is a relocatable heap — a malloc/free/realloc allocator that uses only offsets @@ -153,10 +158,11 @@ See the user guide for the complete array and oneof APIs. The ROS frontend also maps singular `google.protobuf.Timestamp`, `google.protobuf.Duration`, and `std_msgs.Header` message fields to -`ros::Time`, `ros::Duration`, and `std_msgs::Header`. This allows existing ROS1 -functions taking values, const references, or mutable references to accept the -generated fields unchanged. Add the corresponding ROS C++ targets through the -`cc_deps` attribute. +`ros::Time`, `ros::Duration`, and `std_msgs::Header`. Mutable Header access +remains compatible with existing ROS1 reference-taking functions. Read-only +Header access returns `phaser::RosHeaderView`, whose `frame_id` is a +`std::string_view`; call `ToOwned()` when an owning `std_msgs::Header` is +required. Add the corresponding ROS C++ targets through the `cc_deps` attribute. Every generated message, in either frontend style, can also produce ROS1 wire bytes: @@ -168,6 +174,8 @@ absl::Status status = msg.SerializeToROS(ros_output); // Convert serialized protobuf or a native Phaser payload without first // constructing the user-facing message. status = Foo::ProtobufToROS(protobuf_bytes, ros_output); +::phaser::ProtoBuffer protobuf_output(output_data, output_capacity); +status = Foo::ROSToProtobuf(ros_bytes, protobuf_output); status = Foo::PhaserToROS(phaser_bytes, ros_output); status = Foo::ConvertToROS(input_bytes, ros_output); // infers input format @@ -213,6 +221,29 @@ auto msg = foo::bar::phaser::TestMessage::CreateReadonly(buffer, size); int x = msg.x(); ``` +Caller-buffer `CreateMutable`, typed mutation, `CreateReadonly`, typed field +traversal, typed `Any`, caller-buffer protobuf serialization, and fixed-buffer +ROS serialization do not use the system heap. Runtime bookkeeping and generated +type metadata live inside the `PayloadBuffer`; fixed-buffer creation therefore +needs enough room for both message data and this small control data. Repeated +strings iterate as `std::string_view`; repeated-message +indexing and iteration return lightweight message handles by value. These views +remain valid while the caller-owned receive buffer remains alive and unchanged. +Protobuf/ROS deserialization into a fixed mutable message and +`ProtobufToROS` with a sufficiently large fixed `ROSBuffer` are also +system-heap-allocation-free, including registered `Any` payloads. +`ProtobufToROS` scans protobuf wire fields directly and emits ROS bytes without +constructing an intermediate protobuf or Phaser message. +`ROSToProtobuf` performs the reverse direct conversion through `ROSReader` and +`ProtoWriter`; nested and packed protobuf lengths are computed with allocation- +free counting passes. ROS Header decoding writes `frame_id` directly from its +wire view into payload storage without an owning `std::string` intermediate. +Dynamic messages, reflection/debug output, unknown `Any` error handling, and +explicit owning conversions such as `RosHeaderView::ToOwned()` are outside this +guarantee. +Owning conversion/copy helpers, mutation, reflection, debug printing, and +string-returning serialization helpers are outside this guarantee. + ### 3. Zero-copy field access Beyond the standard protobuf accessors, Phaser adds helpers that hand you the final storage diff --git a/phaser/BUILD.bazel b/phaser/BUILD.bazel index 05fbd77..d983157 100644 --- a/phaser/BUILD.bazel +++ b/phaser/BUILD.bazel @@ -101,6 +101,20 @@ cc_test( ], ) +cc_test( + name = "receive_allocation_test", + srcs = ["receive_allocation_test.cc"], + copts = PHASER_COPTS, + data = ["valgrind.supp"], + deps = [ + "//phaser/runtime:phaser_runtime", + "//phaser/testdata:ros_compile_phaser", + "//phaser/testdata:ros_intrinsics_phaser", + "//phaser/testdata:test_message_phaser", + "@com_google_googletest//:gtest_main", + ], +) + cc_test( name = "ros_wire_conversion_test", srcs = ["ros_wire_conversion_test.cc"], @@ -161,6 +175,7 @@ cc_test( copts = PHASER_COPTS, deps = [ "//phaser/runtime:phaser_runtime", + "//phaser/testdata:test_message_phaser", "//phaser/testdata:vision_cc_proto", "//phaser/testdata:vision_phaser", "@com_google_absl//absl/strings:str_format", diff --git a/phaser/all_types_test.cc b/phaser/all_types_test.cc index db13372..473cf8a 100644 --- a/phaser/all_types_test.cc +++ b/phaser/all_types_test.cc @@ -166,10 +166,10 @@ void ExpectBidirectionalWireRoundTrip(FillPhaser fill_phaser, FillPb fill_pb, } void FillMapHolder(MapHolder& msg) { - auto* a = msg.add_values(); + auto a = msg.add_values(); a->set_key("alpha"); a->set_value(-7); - auto* b = msg.add_values(); + auto b = msg.add_values(); b->set_key("beta"); b->set_value(42); } @@ -303,9 +303,9 @@ void ExpectRepeatedBytesMatchPb(const PbRepeatedBytes& pb, } void FillRepeatedMessages(RepeatedMessages& msg) { - auto* m0 = msg.add_items(); - FillAllScalars(*m0); - auto* m1 = msg.add_items(); + auto m0 = msg.add_items(); + FillAllScalars(m0); + auto m1 = msg.add_items(); m1->set_f_int32(99); m1->set_f_string("nested"); } @@ -572,10 +572,10 @@ TEST(AllTypesTest, LargeSintWireCompat) { TEST(AllTypesTest, MapInsertOverwriteAndClear) { MapHolder msg; - auto* e1 = msg.add_values(); + auto e1 = msg.add_values(); e1->set_key("alpha"); e1->set_value(1); - auto* e2 = msg.add_values(); + auto e2 = msg.add_values(); e2->set_key("beta"); e2->set_value(2); @@ -591,7 +591,7 @@ TEST(AllTypesTest, MapInsertOverwriteAndClear) { msg.clear_values(); EXPECT_EQ(0u, msg.values_size()); - auto* e3 = msg.add_values(); + auto e3 = msg.add_values(); e3->set_key("gamma"); e3->set_value(3); EXPECT_EQ(1u, msg.values_size()); @@ -664,9 +664,9 @@ TEST(AllTypesTest, RepeatedBytesWithNulls) { TEST(AllTypesTest, RepeatedMessagesSparseMutable) { RepeatedMessages msg; - auto* m5 = msg.mutable_items(5); + auto m5 = msg.mutable_items(5); m5->set_f_string("slot-five"); - auto* m0 = msg.mutable_items(0); + auto m0 = msg.mutable_items(0); m0->set_f_int32(7); EXPECT_EQ(7, msg.items(0).f_int32()); EXPECT_EQ("slot-five", msg.items(5).f_string()); diff --git a/phaser/compiler/message_gen.cc b/phaser/compiler/message_gen.cc index f28c5c9..5667c1b 100644 --- a/phaser/compiler/message_gen.cc +++ b/phaser/compiler/message_gen.cc @@ -7,8 +7,11 @@ #include #include +#include #include #include +#include +#include #include "absl/strings/str_format.h" #include "absl/strings/str_replace.h" @@ -119,6 +122,22 @@ static bool IsCppReservedWord(const std::string& s) { return reserved_words.contains(s); } +static bool IsFixedWireType( + const google::protobuf::FieldDescriptor* field) { + using Field = google::protobuf::FieldDescriptor; + switch (field->type()) { + case Field::TYPE_FIXED32: + case Field::TYPE_SFIXED32: + case Field::TYPE_FLOAT: + case Field::TYPE_FIXED64: + case Field::TYPE_SFIXED64: + case Field::TYPE_DOUBLE: + return true; + default: + return false; + } +} + std::string MessageGenerator::SanitizedIdentifier( const std::string& name) const { if (IsCppReservedWord(name)) { @@ -178,12 +197,11 @@ absl::Status MessageGenerator::ValidateArraySizeOption( return absl::OkStatus(); } const int array_size = GetArraySize(field); - const std::string context = absl::StrFormat("%s.%s", message_->full_name(), - field->name()); + const std::string context = + absl::StrFormat("%s.%s", message_->full_name(), field->name()); if (array_size <= 0) { - return absl::InvalidArgumentError( - absl::StrFormat("phaser.array_size must be positive on field %s", - context)); + return absl::InvalidArgumentError(absl::StrFormat( + "phaser.array_size must be positive on field %s", context)); } if (!field->is_repeated()) { return absl::InvalidArgumentError(absl::StrFormat( @@ -209,8 +227,7 @@ absl::Status MessageGenerator::ValidateFieldOptions() const { } for (int i = 0; i < message_->field_count(); i++) { const auto* field = message_->field(i); - if (absl::Status status = ValidateArraySizeOption(field); - !status.ok()) { + if (absl::Status status = ValidateArraySizeOption(field); !status.ok()) { return status; } if (IsRosFrontend() && IsRosIntrinsic(field) && @@ -517,8 +534,7 @@ std::string MessageGenerator::FieldRepeatedArrayCType( return "PrimitiveArrayFieldfull_name() == "google.protobuf.Timestamp"; + return desc != nullptr && desc->full_name() == "google.protobuf.Timestamp"; } bool MessageGenerator::IsRosDuration( const google::protobuf::Descriptor* desc) const { - return desc != nullptr && - desc->full_name() == "google.protobuf.Duration"; + return desc != nullptr && desc->full_name() == "google.protobuf.Duration"; } bool MessageGenerator::IsRosHeader( @@ -741,7 +755,8 @@ void MessageGenerator::CompileFields() { auto it = unions_.find(oneof); if (it == unions_.end()) { auto union_info = std::make_shared( - oneof, 4, MemberVariableName(std::string(oneof->name())), "UnionField"); + oneof, 4, MemberVariableName(std::string(oneof->name())), + "UnionField"); unions_[oneof] = union_info; fields_in_order_.push_back(union_info); } @@ -831,8 +846,24 @@ absl::Status MessageGenerator::GenerateHeader(std::ostream& os) { os << " // Optional user-attached payload, not part of the wire format.\n"; os << " std::any active_message;\n\n"; } + for (const auto& [oneof, u] : unions_) { + os << " inline static constexpr uint32_t " << u->member_name + << "_field_numbers[] = {"; + const char* separator = ""; + for (const auto& field : u->members) { + os << separator << field->field->number(); + separator = ", "; + } + os << "};\n"; + } + if (!unions_.empty()) { + os << "\n"; + } // Generate constructors. GenerateConstructors(os, true); + os << " " << MessageName(message_) << "* operator->() { return this; }\n"; + os << " const " << MessageName(message_) + << "* operator->() const { return this; }\n"; // Generate size functions. GenerateSizeFunctions(os); // Generate creators. @@ -844,14 +875,18 @@ absl::Status MessageGenerator::GenerateHeader(std::ostream& os) { } // Generate field metadata. GenerateFieldMetadata(os); + os << " static constexpr size_t MetadataTypeCount() { return " + << ReachableMessageTypeCount() << "; }\n"; - os << " static std::string FullName() { return \"" << message_->full_name() - << "\"; }\n"; - os << " static std::string Name() { return \"" << message_->name() - << "\"; }\n\n"; + os << " static constexpr std::string_view FullName() { return \"" + << message_->full_name() << "\"; }\n"; + os << " static constexpr std::string_view Name() { return \"" + << message_->name() << "\"; }\n\n"; - os << " std::string GetName() const override { return Name(); }\n"; - os << " std::string GetFullName() const override { return FullName(); }\n"; + os << " std::string GetName() const override { return std::string(Name()); " + "}\n"; + os << " std::string GetFullName() const override { return " + "std::string(FullName()); }\n"; os << " friend std::ostream &operator<<(std::ostream &os, const " << MessageName(message_) << " &msg);\n\n"; @@ -915,8 +950,7 @@ void MessageGenerator::GenerateRosSyncToPayload(std::ostream& os) { os << " " << field->member_name << ".SyncToPayload();\n"; } for (const auto& [oneof, union_info] : unions_) { - os << " switch (" << union_info->member_name - << ".Discriminator()) {\n"; + os << " switch (" << union_info->member_name << ".Discriminator()) {\n"; for (size_t i = 0; i < union_info->members.size(); ++i) { const auto& member = union_info->members[i]; if (member->field->type() != @@ -948,9 +982,13 @@ void MessageGenerator::GenerateRosOwnerCopyMove(std::ostream& os, bool decl) { return; } - os << name << "::" << name << "(const " << name << "& other)\n"; - GenerateFieldInitializers(os); + os << name << "::" << name << "(const " << name + << "& other) : Message(other)\n"; + GenerateFieldInitializers(os, ", "); os << R"XXX({ + if (other.runtime != nullptr && other.runtime.use_count() == 0) { + return; + } size_t initial_size = other.BinarySize() * 2; if (initial_size < 8192) { initial_size = 8192; @@ -1039,8 +1077,8 @@ void MessageGenerator::GenerateFieldDeclarations(std::ostream& os) { void MessageGenerator::GenerateRosOneofTypes(std::ostream& os) { for (auto& [oneof, u] : unions_) { const std::string variant_name = OneofVariantTypeName(oneof); - os << " struct " << variant_name << " : public ::phaser::" - << u->member_type << " {\n"; + os << " struct " << variant_name + << " : public ::phaser::" << u->member_type << " {\n"; os << " using Base = ::phaser::" << u->member_type << ";\n"; os << " using Base::Base;\n"; for (size_t i = 0; i < u->members.size(); ++i) { @@ -1064,8 +1102,8 @@ void MessageGenerator::GenerateRosOneofTypes(std::ostream& os) { for (const auto& field : u->members) { const std::string alternative_name = OneofAlternativeTypeName(field->field); - os << " using " << alternative_name << " = " << variant_name << "::" - << alternative_name << ";\n"; + os << " using " << alternative_name << " = " << variant_name + << "::" << alternative_name << ";\n"; } os << "\n"; } @@ -1163,13 +1201,8 @@ void MessageGenerator::GenerateFieldInitializers(std::ostream& os, } for (auto& [oneof, u] : unions_) { os << sep << u->member_name << "(offsetof(" << MessageName(message_) << ", " - << u->member_name << "), " << u->offset << ", 0, 0, {"; - const char* num_sep = ""; - for (auto& field : u->members) { - os << num_sep << field->field->number(); - num_sep = ","; - } - os << "})\n"; + << u->member_name << "), " << u->offset << ", 0, 0, " + << "absl::MakeConstSpan(" << u->member_name << "_field_numbers))\n"; sep = ", "; } os << "#pragma clang diagnostic pop\n\n"; @@ -1205,11 +1238,13 @@ void MessageGenerator::GenerateCreators(std::ostream& os, bool decl) { " ::toolbelt::PayloadBuffer::AllocateMainMessage(&pb, " << MessageName(message_) << "::BinarySize());\n" - " auto runtime = " - "std::make_shared<::phaser::MutableMessageRuntime>(pb);\n" + " ::phaser::InitializeRuntimeControl(&pb, " + << MessageName(message_) + << "::MetadataTypeCount());\n" + " ::phaser::MessageRuntime runtime(pb, true);\n" " auto msg = " << MessageName(message_) - << "(runtime, pb->message);\n" + << "(BorrowRuntime(runtime), pb->message);\n" " msg.InstallMetadata<" << MessageName(message_) << ">();\n" @@ -1224,11 +1259,10 @@ void MessageGenerator::GenerateCreators(std::ostream& os, bool decl) { " ::toolbelt::PayloadBuffer *pb =" "reinterpret_cast<::toolbelt::PayloadBuffer " "*>(const_cast(addr));\n" - " auto runtime = std::make_shared<::phaser::MessageRuntime>(pb, " - "size);\n" + " ::phaser::MessageRuntime runtime(pb, size);\n" " return " << MessageName(message_) - << "(runtime, pb->message);\n" + << "(BorrowRuntime(runtime), pb->message);\n" "}\n\n"; os << "// Create a message in a dynamically resized buffer allocated from " "the heap.\n"; @@ -1247,6 +1281,9 @@ void MessageGenerator::GenerateCreators(std::ostream& os, bool decl) { " ::toolbelt::PayloadBuffer::AllocateMainMessage(&pb, " << MessageName(message_) << "::BinarySize());\n" + " ::phaser::InitializeRuntimeControl(&pb, " + << MessageName(message_) + << "::MetadataTypeCount());\n" " auto runtime = " "std::make_shared<::phaser::DynamicMutableMessageRuntime>(pb, " "std::move(free));\n" @@ -1276,6 +1313,9 @@ void MessageGenerator::GenerateCreators(std::ostream& os, bool decl) { " ::toolbelt::PayloadBuffer::AllocateMainMessage(&pb, " << MessageName(message_) << "::BinarySize());\n" + " ::phaser::InitializeRuntimeControl(&pb, " + << MessageName(message_) + << "::MetadataTypeCount());\n" " auto runtime = " "std::make_shared<::phaser::DynamicMutableMessageRuntime>(pb, " "::free);\n" @@ -1296,6 +1336,25 @@ void MessageGenerator::GenerateSizeFunctions(std::ostream& os) { "PresenceMaskSize(); }\n"; } +size_t MessageGenerator::ReachableMessageTypeCount() const { + std::set reachable; + std::function visit = + [&](const google::protobuf::Descriptor* descriptor) { + if (descriptor == nullptr || !reachable.insert(descriptor).second) { + return; + } + for (int i = 0; i < descriptor->field_count(); ++i) { + const auto* field = descriptor->field(i); + if (field->type() == + google::protobuf::FieldDescriptor::TYPE_MESSAGE) { + visit(field->message_type()); + } + } + }; + visit(message_); + return reachable.size(); +} + void MessageGenerator::GenerateFieldMetadata(std::ostream& os) { // Build a vector of fields from the fields an unions, sorted by field number. std::vector> all_fields; @@ -1313,31 +1372,116 @@ void MessageGenerator::GenerateFieldMetadata(std::ostream& os) { return a->field->number() < b->field->number(); }); + // Find the interval with the greatest metadata-size saving. For [i, j], + // sparse entries cost 8 bytes each while the dense representation costs + // 4 bytes per field-number slot. Missing slots use a zero field offset. + // Separating the start- and end-dependent terms finds the best interval in + // O(number_of_fields). + size_t dense_begin = 0; + size_t dense_end = 0; + int64_t best_saving = 0; + int64_t best_start_score = std::numeric_limits::min(); + size_t best_start_index = 0; + for (size_t end = 0; end < all_fields.size(); ++end) { + const int64_t number = all_fields[end]->field->number(); + const int64_t start_score = -8 * static_cast(end) + 4 * number; + if (start_score > best_start_score) { + best_start_score = start_score; + best_start_index = end; + } + + const int64_t end_score = + 8 * static_cast(end + 1) - 4 * (number + 1); + const int64_t saving = end_score + best_start_score; + if (saving > best_saving) { + best_saving = saving; + dense_begin = best_start_index; + dense_end = end; + } + } + + const bool has_dense_range = best_saving > 0; + const uint32_t dense_base = + has_dense_range + ? static_cast(all_fields[dense_begin]->field->number()) + : 0; + const uint32_t dense_span = + has_dense_range + ? static_cast(all_fields[dense_end]->field->number() - + all_fields[dense_begin]->field->number() + 1) + : 0; + const size_t dense_count = has_dense_range ? dense_end - dense_begin + 1 : 0; + const size_t sparse_count = all_fields.size() - dense_count; + os << " struct " << MessageName(message_) << "FieldData {"; - os << R"( - uint32_t num; - struct Field { - uint32_t number; - uint32_t offset : 24; - uint32_t id : 8; -)"; - // A message with no fields would otherwise emit `fields[0]`, a zero-length - // array (a non-standard extension). Reserve one (zero-initialized, never read - // because num == 0) element instead so the generated header stays clean. - os << " } fields[" << (all_fields.empty() ? size_t{1} : all_fields.size()) - << "];\n"; + os << "\n ::phaser::HybridFieldData header;\n"; + if (dense_span != 0) { + os << " ::phaser::FieldValue dense_fields[" << dense_span << "];\n"; + } + if (sparse_count != 0) { + os << " ::phaser::SparseFieldData sparse_fields[" << sparse_count + << "];\n"; + } os << " };\n"; + const size_t metadata_size = + 4 * sizeof(uint32_t) + + static_cast(dense_span) * sizeof(uint32_t) + + sparse_count * 2 * sizeof(uint32_t); + os << " static_assert(sizeof(" << MessageName(message_) + << "FieldData) == " << metadata_size + << "u, \"Unexpected hybrid field metadata padding\");\n"; + + uint32_t max_offset = 0; + uint32_t max_id = 0; + for (const auto& field : all_fields) { + max_offset = std::max(max_offset, field->offset); + max_id = std::max(max_id, field->id); + } + os << " static_assert(" << max_offset + << "u <= 0x00ffffffu, \"Field offset exceeds 24 bits\");\n"; + os << " static_assert(" << max_id + << "u <= 0xffu, \"Field presence id exceeds 8 bits\");\n"; - // Generate the field data. os << " static constexpr " << MessageName(message_) << "FieldData field_data = {\n"; - os << " .num = " << all_fields.size() << ",\n"; - os << " .fields = {\n"; - for (auto& field : all_fields) { - os << " { .number = " << field->field->number() - << ", .offset = " << field->offset << ", .id = " << field->id << " },\n"; + os << " .header = {\n"; + os << " .magic = ::phaser::kHybridFieldDataMagic,\n"; + os << " .dense_base = " << dense_base << ",\n"; + os << " .dense_span = " << dense_span << ",\n"; + os << " .sparse_count = " << sparse_count << ",\n"; + os << " },\n"; + + if (dense_span != 0) { + os << " .dense_fields = {\n"; + size_t field_index = dense_begin; + for (uint32_t dense_index = 0; dense_index < dense_span; ++dense_index) { + const uint32_t field_number = dense_base + dense_index; + if (field_index <= dense_end && + static_cast(all_fields[field_index]->field->number()) == + field_number) { + const auto& field = all_fields[field_index++]; + os << " { .offset = " << field->offset << ", .id = " << field->id + << " },\n"; + } else { + os << " {},\n"; + } + } + os << " },\n"; + } + + if (sparse_count != 0) { + os << " .sparse_fields = {\n"; + for (size_t i = 0; i < all_fields.size(); ++i) { + if (has_dense_range && i >= dense_begin && i <= dense_end) { + continue; + } + const auto& field = all_fields[i]; + os << " { .number = " << field->field->number() + << ", .offset = " << field->offset << ", .id = " << field->id + << " },\n"; + } + os << " },\n"; } - os << " }\n"; os << " };\n"; } @@ -1466,7 +1610,6 @@ void MessageGenerator::GenerateFieldProtobufAccessors( os << " }\n"; os << " const ::phaser::StringVectorField& " << sanitized_field_name << "() const {\n"; - os << " " << member_name << ".Populate();\n"; os << " return " << member_name << ";\n"; os << " }\n"; } else { @@ -1511,20 +1654,19 @@ void MessageGenerator::GenerateFieldProtobufAccessors( os << " void clear_" << field_name << "() {\n"; os << " " << member_name << ".Clear();\n"; os << " }\n"; - os << " const " << field->c_type << "& " << sanitized_field_name + os << " " << field->c_type << " " << sanitized_field_name << "(size_t index) const {\n"; os << " return " << member_name << ".Get(index);\n"; os << " }\n"; - os << " " << field->c_type << "* mutable_" << field_name + os << " " << field->c_type << " mutable_" << field_name << "(size_t index) {\n"; os << " return " << member_name << ".Mutable(index);\n"; os << " }\n"; - os << " " << field->c_type << "* add_" << field_name << "() {\n"; + os << " " << field->c_type << " add_" << field_name << "() {\n"; os << " return " << member_name << ".Add();\n"; os << " }\n"; os << " const ::phaser::MessageVectorField<" << field->c_type << ">& " << sanitized_field_name << "() const {\n"; - os << " " << member_name << ".Populate();\n"; os << " return " << member_name << ";\n"; os << " }\n"; os << " void reserve_" << field_name << "(size_t num) {\n"; @@ -1533,7 +1675,7 @@ void MessageGenerator::GenerateFieldProtobufAccessors( os << " void resize_" << field_name << "(size_t num) {\n"; os << " " << member_name << ".resize(num);\n"; os << " }\n"; - os << " std::vector<" << field->c_type << "*> allocate_" << field_name + os << " std::vector<" << field->c_type << "> allocate_" << field_name << "(size_t n) {\n"; os << " return " << member_name << ".Allocate(n);\n"; os << " }\n"; @@ -1735,8 +1877,7 @@ std::string MessageGenerator::ROSFieldValueExpression( if (union_field == nullptr) { return field->member_name + ".Get()"; } - if (field->field->type() == - google::protobuf::FieldDescriptor::TYPE_MESSAGE) { + if (field->field->type() == google::protobuf::FieldDescriptor::TYPE_MESSAGE) { return union_field->member_name + ".template GetReference<" + std::to_string(union_index) + ", " + field->c_type + ">()"; } @@ -1868,8 +2009,7 @@ void MessageGenerator::GenerateROSFieldWrite( return; case google::protobuf::FieldDescriptor::TYPE_STRING: case google::protobuf::FieldDescriptor::TYPE_BYTES: - os << indent - << "if (absl::Status status = buffer.WriteString(" + os << indent << "if (absl::Status status = buffer.WriteString(" << value_expression << "); !status.ok()) return status;\n"; return; case google::protobuf::FieldDescriptor::TYPE_MESSAGE: @@ -1889,18 +2029,15 @@ void MessageGenerator::GenerateROSFieldWrite( } else if (IsRosFrontend() && IsRosTime(field->message_type())) { write("static_cast((" + value_expression + ").sec)"); write("static_cast((" + value_expression + ").nsec)"); - } else if (IsRosFrontend() && - IsRosDuration(field->message_type())) { + } else if (IsRosFrontend() && IsRosDuration(field->message_type())) { write("static_cast((" + value_expression + ").sec)"); write("static_cast((" + value_expression + ").nsec)"); } else if (IsRosFrontend() && IsRosHeader(field->message_type())) { write("static_cast((" + value_expression + ").seq)"); write("static_cast((" + value_expression + ").stamp.sec)"); write("static_cast((" + value_expression + ").stamp.nsec)"); - os << indent - << "if (absl::Status status = buffer.WriteString((" - << value_expression - << ").frame_id); !status.ok()) return status;\n"; + os << indent << "if (absl::Status status = buffer.WriteString((" + << value_expression << ").frame_id); !status.ok()) return status;\n"; } else { os << indent << "if (absl::Status status = (" << value_expression << ").SerializeToROS(buffer); !status.ok()) return status;\n"; @@ -1946,10 +2083,9 @@ void MessageGenerator::GenerateROSFieldRead( auto read_value = [&](const std::string& type, const std::string& conversion = "") { os << indent << "{\n"; - os << indent << " absl::StatusOr<" << type - << "> ros_value = buffer.Read<" << type << ">();\n"; - os << indent - << " if (!ros_value.ok()) return ros_value.status();\n"; + os << indent << " absl::StatusOr<" << type << "> ros_value = buffer.Read<" + << type << ">();\n"; + os << indent << " if (!ros_value.ok()) return ros_value.status();\n"; const std::string value = conversion.empty() ? "*ros_value" : conversion + "(*ros_value)"; set_value(value); @@ -1985,7 +2121,8 @@ void MessageGenerator::GenerateROSFieldRead( read_value("bool"); return; case google::protobuf::FieldDescriptor::TYPE_ENUM: - read_value("int32_t", "static_cast<" + EnumName(field->enum_type()) + ">"); + read_value("int32_t", + "static_cast<" + EnumName(field->enum_type()) + ">"); return; case google::protobuf::FieldDescriptor::TYPE_STRING: case google::protobuf::FieldDescriptor::TYPE_BYTES: @@ -1993,8 +2130,7 @@ void MessageGenerator::GenerateROSFieldRead( os << indent << " absl::StatusOr ros_value = " "buffer.ReadString();\n"; - os << indent - << " if (!ros_value.ok()) return ros_value.status();\n"; + os << indent << " if (!ros_value.ok()) return ros_value.status();\n"; set_value("*ros_value"); os << indent << "}\n"; return; @@ -2034,8 +2170,7 @@ void MessageGenerator::GenerateROSFieldRead( os << indent << " ros_value.nsec = *ros_nsec;\n"; set_value("ros_value"); os << indent << "}\n"; - } else if (IsRosFrontend() && - IsRosDuration(field->message_type())) { + } else if (IsRosFrontend() && IsRosDuration(field->message_type())) { os << indent << "{\n"; os << indent << " absl::StatusOr ros_sec = " @@ -2069,18 +2204,25 @@ void MessageGenerator::GenerateROSFieldRead( "buffer.ReadString();\n"; os << indent << " if (!ros_frame_id.ok()) return ros_frame_id.status();\n"; - os << indent << " ::std_msgs::Header ros_value;\n"; - os << indent << " ros_value.seq = *ros_seq;\n"; - os << indent << " ros_value.stamp.sec = *ros_sec;\n"; - os << indent << " ros_value.stamp.nsec = *ros_nsec;\n"; - os << indent - << " ros_value.frame_id.assign(ros_frame_id->data(), " - "ros_frame_id->size());\n"; - set_value("ros_value"); + if (union_index < 0 && !append && index_expression.empty()) { + os << indent << " auto ros_value = " << target_expression + << ".Mutable();\n"; + os << indent << " ros_value.seq = *ros_seq;\n"; + os << indent << " ros_value.stamp.sec = *ros_sec;\n"; + os << indent << " ros_value.stamp.nsec = *ros_nsec;\n"; + os << indent << " ros_value.frame_id = *ros_frame_id;\n"; + } else { + os << indent << " auto ros_value = " << mutable_message() << ";\n"; + os << indent << " ros_value->seq = *ros_seq;\n"; + os << indent << " ros_value->stamp = " + << "::ros::Time(*ros_sec, *ros_nsec);\n"; + os << indent << " ros_value->frame_id = *ros_frame_id;\n"; + os << indent << " ros_value->SyncToPayload();\n"; + } os << indent << "}\n"; } else { os << indent << "{\n"; - os << indent << " auto* ros_message = " << mutable_message() << ";\n"; + os << indent << " auto ros_message = " << mutable_message() << ";\n"; os << indent << " if (absl::Status status = " "ros_message->DeserializeFromROS(buffer); !status.ok()) " @@ -2094,6 +2236,678 @@ void MessageGenerator::GenerateROSFieldRead( abort(); } +std::string MessageGenerator::DirectProtobufValueType( + const google::protobuf::FieldDescriptor* field) const { + using Field = google::protobuf::FieldDescriptor; + switch (field->type()) { + case Field::TYPE_INT32: + case Field::TYPE_SINT32: + case Field::TYPE_SFIXED32: + case Field::TYPE_ENUM: + return "int32_t"; + case Field::TYPE_INT64: + case Field::TYPE_SINT64: + case Field::TYPE_SFIXED64: + return "int64_t"; + case Field::TYPE_UINT32: + case Field::TYPE_FIXED32: + return "uint32_t"; + case Field::TYPE_UINT64: + case Field::TYPE_FIXED64: + return "uint64_t"; + case Field::TYPE_FLOAT: + return "float"; + case Field::TYPE_DOUBLE: + return "double"; + case Field::TYPE_BOOL: + return "bool"; + case Field::TYPE_STRING: + case Field::TYPE_BYTES: + case Field::TYPE_MESSAGE: + return "std::string_view"; + case Field::TYPE_GROUP: + break; + } + abort(); +} + +void MessageGenerator::GenerateDirectProtobufReadValue( + std::ostream& os, const google::protobuf::FieldDescriptor* field, + const std::string& buffer, const std::string& value, + const std::string& indent) { + using Field = google::protobuf::FieldDescriptor; + const std::string type = DirectProtobufValueType(field); + if (field->type() == Field::TYPE_STRING || + field->type() == Field::TYPE_BYTES) { + os << indent << "absl::StatusOr ros_parsed = " << buffer + << ".DeserializeString();\n"; + os << indent << "if (!ros_parsed.ok()) return ros_parsed.status();\n"; + os << indent << value << " = *ros_parsed;\n"; + return; + } + if (field->type() == Field::TYPE_MESSAGE) { + os << indent << "absl::StatusOr> ros_parsed = " << buffer + << ".DeserializeLengthDelimited();\n"; + os << indent << "if (!ros_parsed.ok()) return ros_parsed.status();\n"; + os << indent << value + << " = std::string_view(ros_parsed->data(), ros_parsed->size());\n"; + return; + } + + const bool fixed = field->type() == Field::TYPE_FIXED32 || + field->type() == Field::TYPE_SFIXED32 || + field->type() == Field::TYPE_FLOAT || + field->type() == Field::TYPE_FIXED64 || + field->type() == Field::TYPE_SFIXED64 || + field->type() == Field::TYPE_DOUBLE; + const bool is_signed = field->type() == Field::TYPE_SINT32 || + field->type() == Field::TYPE_SINT64; + os << indent << "absl::StatusOr<" << type << "> ros_parsed = " << buffer + << (fixed ? ".DeserializeFixed<" : ".DeserializeVarint<") << type; + if (fixed) { + os << ">();\n"; + } else { + os << ", " << (is_signed ? "true" : "false") << ">();\n"; + } + os << indent << "if (!ros_parsed.ok()) return ros_parsed.status();\n"; + os << indent << value << " = *ros_parsed;\n"; +} + +void MessageGenerator::GenerateDirectROSWriteValue( + std::ostream& os, const google::protobuf::FieldDescriptor* field, + const std::string& value, const std::string& indent) { + using Field = google::protobuf::FieldDescriptor; + switch (field->type()) { + case Field::TYPE_STRING: + case Field::TYPE_BYTES: + os << indent << "if (absl::Status status = output.WriteString(" << value + << "); !status.ok()) return status;\n"; + return; + case Field::TYPE_MESSAGE: + if (IsAny(field)) { + os << indent << "{\n"; + os << indent << " ::phaser::ProtoBuffer any_scan(" << value << ");\n"; + os << indent << " while (!any_scan.Eof()) {\n"; + os << indent + << " absl::StatusOr any_tag = " + "any_scan.DeserializeVarint();\n"; + os << indent << " if (!any_tag.ok()) return any_tag.status();\n"; + os << indent + << " const uint32_t any_number = " + "*any_tag >> ::phaser::ProtoBuffer::kFieldIdShift;\n"; + os << indent << " if (any_number == 1 || any_number == 2) {\n"; + os << indent + << " return absl::UnimplementedError(\"ROS1 serialization of " + "a populated google.protobuf.Any is unsupported\");\n"; + os << indent << " }\n"; + os << indent + << " if (absl::Status status = any_scan.SkipTag(*any_tag); " + "!status.ok()) return status;\n"; + os << indent << " }\n"; + os << indent + << " if (absl::Status status = output.WriteString({}); " + "!status.ok()) return status;\n"; + os << indent + << " if (absl::Status status = output.WriteString({}); " + "!status.ok()) return status;\n"; + os << indent << "}\n"; + } else { + os << indent << "if (absl::Status status = " + << MessageName(field->message_type(), true) << "::ProtobufWireToROS(" + << value << ", output); !status.ok()) return status;\n"; + } + return; + case Field::TYPE_ENUM: + os << indent + << "if (absl::Status status = output.Write(static_cast(" + << value << ")); !status.ok()) return status;\n"; + return; + default: + os << indent << "if (absl::Status status = output.Write(" << value + << "); !status.ok()) return status;\n"; + return; + } +} + +void MessageGenerator::GenerateDirectProtobufSingularField( + std::ostream& os, const google::protobuf::FieldDescriptor* field, + const std::string& indent) { + const std::string type = DirectProtobufValueType(field); + os << indent << "{\n"; + os << indent << " " << type << " ros_value{};\n"; + os << indent << " ::phaser::ProtoBuffer ros_scan(protobuf);\n"; + os << indent << " while (!ros_scan.Eof()) {\n"; + os << indent + << " absl::StatusOr ros_tag = " + "ros_scan.DeserializeVarint();\n"; + os << indent << " if (!ros_tag.ok()) return ros_tag.status();\n"; + os << indent + << " const uint32_t ros_number = " + "*ros_tag >> ::phaser::ProtoBuffer::kFieldIdShift;\n"; + os << indent << " if (ros_number == " << field->number() << ") {\n"; + os << indent << " {\n"; + GenerateDirectProtobufReadValue(os, field, "ros_scan", "ros_value", + indent + " "); + os << indent << " }\n"; + os << indent << " } else {\n"; + os << indent + << " if (absl::Status status = ros_scan.SkipTag(*ros_tag); " + "!status.ok()) return status;\n"; + os << indent << " }\n"; + os << indent << " }\n"; + GenerateDirectROSWriteValue(os, field, "ros_value", indent + " "); + os << indent << "}\n"; +} + +void MessageGenerator::GenerateDirectProtobufField( + std::ostream& os, const google::protobuf::FieldDescriptor* field, + const std::string& indent) { + if (!field->is_repeated()) { + GenerateDirectProtobufSingularField(os, field, indent); + return; + } + + const std::string type = DirectProtobufValueType(field); + const int fixed_extent = GetArraySize(field); + const bool fixed_wire_type = IsFixedWireType(field); + os << indent << "{\n"; + if (fixed_extent <= 0) { + os << indent << " size_t ros_count = 0;\n"; + os << indent << " ::phaser::ProtoBuffer ros_count_scan(protobuf);\n"; + os << indent << " while (!ros_count_scan.Eof()) {\n"; + os << indent + << " absl::StatusOr ros_tag = " + "ros_count_scan.DeserializeVarint();\n"; + os << indent << " if (!ros_tag.ok()) return ros_tag.status();\n"; + os << indent + << " const uint32_t ros_number = " + "*ros_tag >> ::phaser::ProtoBuffer::kFieldIdShift;\n"; + os << indent << " if (ros_number == " << field->number() << ") {\n"; + if (field->is_packable()) { + os << indent + << " if ((*ros_tag & 7u) == " + "static_cast(::phaser::WireType::kLengthDelimited)) {\n"; + os << indent + << " absl::StatusOr> ros_packed = " + "ros_count_scan.DeserializeLengthDelimited();\n"; + os << indent + << " if (!ros_packed.ok()) return ros_packed.status();\n"; + if (fixed_wire_type) { + os << indent << " if (ros_packed->size() % sizeof(" << type + << ") != 0) {\n"; + os << indent + << " return absl::InvalidArgumentError(" + "\"packed fixed-width field has a partial element\");\n"; + os << indent << " }\n"; + os << indent << " ros_count += ros_packed->size() / sizeof(" + << type << ");\n"; + } else { + os << indent + << " ::phaser::ProtoBuffer ros_values(*ros_packed);\n"; + os << indent << " while (!ros_values.Eof()) {\n"; + os << indent << " " << type << " ros_ignored{};\n"; + os << indent << " {\n"; + GenerateDirectProtobufReadValue(os, field, "ros_values", "ros_ignored", + indent + " "); + os << indent << " }\n"; + os << indent << " ++ros_count;\n"; + os << indent << " }\n"; + } + os << indent << " } else {\n"; + os << indent << " " << type << " ros_ignored{};\n"; + os << indent << " {\n"; + GenerateDirectProtobufReadValue(os, field, "ros_count_scan", + "ros_ignored", indent + " "); + os << indent << " }\n"; + os << indent << " ++ros_count;\n"; + os << indent << " }\n"; + } else { + os << indent << " " << type << " ros_ignored{};\n"; + os << indent << " {\n"; + GenerateDirectProtobufReadValue(os, field, "ros_count_scan", + "ros_ignored", indent + " "); + os << indent << " }\n"; + os << indent << " ++ros_count;\n"; + } + os << indent << " } else {\n"; + os << indent + << " if (absl::Status status = " + "ros_count_scan.SkipTag(*ros_tag); !status.ok()) return status;\n"; + os << indent << " }\n"; + os << indent << " }\n"; + os << indent + << " if (absl::Status status = output.WriteSequenceLength(ros_count); " + "!status.ok()) return status;\n"; + } + + os << indent << " size_t ros_emitted = 0;\n"; + os << indent << " ::phaser::ProtoBuffer ros_scan(protobuf);\n"; + os << indent << " while (!ros_scan.Eof()) {\n"; + os << indent + << " absl::StatusOr ros_tag = " + "ros_scan.DeserializeVarint();\n"; + os << indent << " if (!ros_tag.ok()) return ros_tag.status();\n"; + os << indent + << " const uint32_t ros_number = " + "*ros_tag >> ::phaser::ProtoBuffer::kFieldIdShift;\n"; + os << indent << " if (ros_number == " << field->number() << ") {\n"; + auto emit_value = [&](const std::string& buffer, + const std::string& emit_indent) { + os << emit_indent << type << " ros_value{};\n"; + os << emit_indent << "{\n"; + GenerateDirectProtobufReadValue(os, field, buffer, "ros_value", + emit_indent + " "); + os << emit_indent << "}\n"; + if (fixed_extent > 0) { + os << emit_indent << "if (ros_emitted >= " << fixed_extent << ") {\n"; + os << emit_indent + << " return absl::InvalidArgumentError(" + "\"protobuf input exceeds fixed ROS array extent\");\n"; + os << emit_indent << "}\n"; + } + GenerateDirectROSWriteValue(os, field, "ros_value", emit_indent); + os << emit_indent << "++ros_emitted;\n"; + }; + if (field->is_packable()) { + os << indent + << " if ((*ros_tag & 7u) == " + "static_cast(::phaser::WireType::kLengthDelimited)) {\n"; + os << indent + << " absl::StatusOr> ros_packed = " + "ros_scan.DeserializeLengthDelimited();\n"; + os << indent + << " if (!ros_packed.ok()) return ros_packed.status();\n"; + if (fixed_wire_type) { + os << indent << " if (ros_packed->size() % sizeof(" << type + << ") != 0) {\n"; + os << indent + << " return absl::InvalidArgumentError(" + "\"packed fixed-width field has a partial element\");\n"; + os << indent << " }\n"; + os << indent << " const size_t ros_packed_count = " + << "ros_packed->size() / sizeof(" << type << ");\n"; + if (fixed_extent > 0) { + os << indent << " if (ros_packed_count > " << fixed_extent + << " - ros_emitted) {\n"; + os << indent + << " return absl::InvalidArgumentError(" + "\"protobuf input exceeds fixed ROS array extent\");\n"; + os << indent << " }\n"; + } + os << indent + << " if (absl::Status status = output.WriteRaw(" + "ros_packed->data(), ros_packed->size()); !status.ok()) " + "return status;\n"; + os << indent << " ros_emitted += ros_packed_count;\n"; + } else { + os << indent << " ::phaser::ProtoBuffer ros_values(*ros_packed);\n"; + os << indent << " while (!ros_values.Eof()) {\n"; + emit_value("ros_values", indent + " "); + os << indent << " }\n"; + } + os << indent << " } else {\n"; + emit_value("ros_scan", indent + " "); + os << indent << " }\n"; + } else { + emit_value("ros_scan", indent + " "); + } + os << indent << " } else {\n"; + os << indent + << " if (absl::Status status = ros_scan.SkipTag(*ros_tag); " + "!status.ok()) return status;\n"; + os << indent << " }\n"; + os << indent << " }\n"; + if (fixed_extent > 0) { + os << indent << " while (ros_emitted < " << fixed_extent << ") {\n"; + os << indent << " " << type << " ros_value{};\n"; + GenerateDirectROSWriteValue(os, field, "ros_value", indent + " "); + os << indent << " ++ros_emitted;\n"; + os << indent << " }\n"; + } + os << indent << "}\n"; +} + +void MessageGenerator::GenerateDirectProtobufToROS(std::ostream& os) { + const std::string name = MessageName(message_); + os << "absl::Status " << name + << "::ProtobufWireToROS(std::string_view protobuf, " + "::phaser::ROSBuffer& output) {\n"; + if (IsRosTime(message_) || IsRosDuration(message_)) { + os << " int64_t ros_seconds = 0;\n"; + os << " int32_t ros_nanos = 0;\n"; + os << " ::phaser::ProtoBuffer ros_scan(protobuf);\n"; + os << " while (!ros_scan.Eof()) {\n"; + os << " absl::StatusOr ros_tag = " + "ros_scan.DeserializeVarint();\n"; + os << " if (!ros_tag.ok()) return ros_tag.status();\n"; + os << " switch (*ros_tag >> ::phaser::ProtoBuffer::kFieldIdShift) {\n"; + os << " case 1: {\n"; + os << " absl::StatusOr value = " + "ros_scan.DeserializeVarint();\n"; + os << " if (!value.ok()) return value.status();\n"; + os << " ros_seconds = *value;\n"; + os << " break;\n"; + os << " }\n"; + os << " case 2: {\n"; + os << " absl::StatusOr value = " + "ros_scan.DeserializeVarint();\n"; + os << " if (!value.ok()) return value.status();\n"; + os << " ros_nanos = *value;\n"; + os << " break;\n"; + os << " }\n"; + os << " default:\n"; + os << " if (absl::Status status = ros_scan.SkipTag(*ros_tag); " + "!status.ok()) return status;\n"; + os << " }\n"; + os << " }\n"; + const std::string wire_type = IsRosTime(message_) ? "uint32_t" : "int32_t"; + os << " if (absl::Status status = output.Write(static_cast<" << wire_type + << ">(ros_seconds)); !status.ok()) return status;\n"; + os << " if (absl::Status status = output.Write(static_cast<" << wire_type + << ">(ros_nanos)); !status.ok()) return status;\n"; + } else { + for (const auto& item : fields_in_order_) { + if (!item->IsUnion()) { + GenerateDirectProtobufField(os, item->field, " "); + continue; + } + auto union_info = std::static_pointer_cast(item); + os << " {\n"; + os << " uint32_t ros_discriminator = 0;\n"; + os << " ::phaser::ProtoBuffer ros_scan(protobuf);\n"; + os << " while (!ros_scan.Eof()) {\n"; + os << " absl::StatusOr ros_tag = " + "ros_scan.DeserializeVarint();\n"; + os << " if (!ros_tag.ok()) return ros_tag.status();\n"; + os << " const uint32_t ros_number = " + "*ros_tag >> ::phaser::ProtoBuffer::kFieldIdShift;\n"; + os << " switch (ros_number) {\n"; + for (const auto& member : union_info->members) { + os << " case " << member->field->number() << ":\n"; + os << " ros_discriminator = ros_number;\n"; + os << " break;\n"; + } + os << " default:\n"; + os << " break;\n"; + os << " }\n"; + os << " if (absl::Status status = ros_scan.SkipTag(*ros_tag); " + "!status.ok()) return status;\n"; + os << " }\n"; + os << " if (absl::Status status = output.Write(ros_discriminator); " + "!status.ok()) return status;\n"; + os << " switch (ros_discriminator) {\n"; + for (const auto& member : union_info->members) { + os << " case " << member->field->number() << ":\n"; + GenerateDirectProtobufSingularField(os, member->field, " "); + os << " break;\n"; + } + os << " default:\n"; + os << " break;\n"; + os << " }\n"; + os << " }\n"; + } + } + os << " return absl::OkStatus();\n"; + os << "}\n\n"; +} + +void MessageGenerator::GenerateDirectROSReadValue( + std::ostream& os, const google::protobuf::FieldDescriptor* field, + const std::string& reader, const std::string& value, + const std::string& indent) { + using Field = google::protobuf::FieldDescriptor; + if (field->type() == Field::TYPE_STRING || + field->type() == Field::TYPE_BYTES) { + os << indent << "absl::StatusOr ros_value = " << reader + << ".ReadString();\n"; + os << indent << "if (!ros_value.ok()) return ros_value.status();\n"; + os << indent << value << " = *ros_value;\n"; + return; + } + if (field->type() == Field::TYPE_MESSAGE) { + abort(); + } + const std::string type = DirectProtobufValueType(field); + os << indent << "absl::StatusOr<" << type << "> ros_value = " << reader + << ".Read<" << type << ">();\n"; + os << indent << "if (!ros_value.ok()) return ros_value.status();\n"; + os << indent << value << " = *ros_value;\n"; +} + +void MessageGenerator::GenerateDirectProtoWriteValue( + std::ostream& os, const google::protobuf::FieldDescriptor* field, + const std::string& value, const std::string& field_number, + const std::string& indent, bool raw) { + using Field = google::protobuf::FieldDescriptor; + const std::string type = DirectProtobufValueType(field); + if (field->type() == Field::TYPE_STRING || + field->type() == Field::TYPE_BYTES) { + os << indent << "if (absl::Status status = output.SerializeLengthDelimited(" + << field_number << ", " << value << ".data(), " << value + << ".size()); !status.ok()) return status;\n"; + return; + } + if (field->type() == Field::TYPE_MESSAGE) { + abort(); + } + const bool fixed = field->type() == Field::TYPE_FIXED32 || + field->type() == Field::TYPE_SFIXED32 || + field->type() == Field::TYPE_FLOAT || + field->type() == Field::TYPE_FIXED64 || + field->type() == Field::TYPE_SFIXED64 || + field->type() == Field::TYPE_DOUBLE; + const bool is_signed = field->type() == Field::TYPE_SINT32 || + field->type() == Field::TYPE_SINT64; + if (raw && fixed) { + os << indent << "if (absl::Status status = output.SerializeRaw(&" << value + << ", sizeof(" << value << ")); !status.ok()) return status;\n"; + return; + } + os << indent << "if (absl::Status status = output."; + if (fixed) { + os << "SerializeFixed(" << field_number << ", " << value << ")"; + } else if (raw) { + os << "SerializeRawVarint<" << type << ", " + << (is_signed ? "true" : "false") << ">(" << value << ")"; + } else { + os << "SerializeVarint<" << type << ", " << (is_signed ? "true" : "false") + << ">(" << field_number << ", " << value << ")"; + } + os << "; !status.ok()) return status;\n"; +} + +void MessageGenerator::GenerateDirectROSFieldToProtobuf( + std::ostream& os, const google::protobuf::FieldDescriptor* field, + const std::string& indent) { + const std::string type = DirectProtobufValueType(field); + const int fixed_extent = GetArraySize(field); + const bool fixed_wire_type = IsFixedWireType(field); + const std::string count = + fixed_extent > 0 ? std::to_string(fixed_extent) : "ros_count"; + + os << indent << "{\n"; + if (field->is_repeated() && fixed_extent <= 0) { + os << indent + << " absl::StatusOr ros_count_value = " + "ros.ReadSequenceLength();\n"; + os << indent + << " if (!ros_count_value.ok()) return ros_count_value.status();\n"; + os << indent << " const size_t ros_count = *ros_count_value;\n"; + } + + auto emit_one = [&](const std::string& reader, const std::string& writer, + const std::string& emit_indent, bool raw) { + if (field->type() == google::protobuf::FieldDescriptor::TYPE_MESSAGE) { + if (IsAny(field)) { + os << emit_indent + << "absl::StatusOr ros_type_url = " << reader + << ".ReadString();\n"; + os << emit_indent + << "if (!ros_type_url.ok()) return ros_type_url.status();\n"; + os << emit_indent + << "absl::StatusOr ros_any_value = " << reader + << ".ReadString();\n"; + os << emit_indent + << "if (!ros_any_value.ok()) return ros_any_value.status();\n"; + os << emit_indent + << "if (!ros_type_url->empty() || !ros_any_value->empty()) {\n"; + os << emit_indent + << " return absl::UnimplementedError(\"ROS1 conversion of a " + "populated google.protobuf.Any is unsupported\");\n"; + os << emit_indent << "}\n"; + os << emit_indent << "if (absl::Status status = " << writer + << ".SerializeLengthDelimitedHeader(" << field->number() + << ", 0); !status.ok()) return status;\n"; + } else { + const std::string nested = MessageName(field->message_type(), true); + os << emit_indent << "::phaser::ROSReader ros_size_reader = " << reader + << ";\n"; + os << emit_indent << "::phaser::ProtoWriter ros_size_output;\n"; + os << emit_indent << "if (absl::Status status = " << nested + << "::ROSReaderToProtobuf(ros_size_reader, ros_size_output); " + "!status.ok()) return status;\n"; + os << emit_indent << "if (absl::Status status = " << writer + << ".SerializeLengthDelimitedHeader(" << field->number() + << ", ros_size_output.Size()); !status.ok()) return status;\n"; + os << emit_indent << "if (absl::Status status = " << nested + << "::ROSReaderToProtobuf(" << reader << ", " << writer + << "); !status.ok()) return status;\n"; + } + return; + } + + os << emit_indent << type << " ros_field_value{};\n"; + os << emit_indent << "{\n"; + GenerateDirectROSReadValue(os, field, reader, "ros_field_value", + emit_indent + " "); + os << emit_indent << "}\n"; + const std::string old_output = "output"; + if (writer != old_output) { + os << emit_indent << "{\n"; + os << emit_indent << " auto& output = " << writer << ";\n"; + GenerateDirectProtoWriteValue(os, field, "ros_field_value", + std::to_string(field->number()), + emit_indent + " ", raw); + os << emit_indent << "}\n"; + } else { + GenerateDirectProtoWriteValue(os, field, "ros_field_value", + std::to_string(field->number()), + emit_indent, raw); + } + }; + + if (!field->is_repeated()) { + emit_one("ros", "output", indent + " ", false); + os << indent << "}\n"; + return; + } + + if (field->is_packable() && field->is_packed() && fixed_wire_type) { + os << indent << " const uint64_t ros_packed_byte_size = " + << "static_cast(" << count << ") * sizeof(" << type << ");\n"; + os << indent + << " if (ros_packed_byte_size > ros.Remaining()) {\n"; + os << indent + << " return absl::InvalidArgumentError(" + "\"truncated ROS packed fixed-width field\");\n"; + os << indent << " }\n"; + os << indent + << " absl::StatusOr> ros_packed = " + "ros.ReadRaw(static_cast(ros_packed_byte_size));\n"; + os << indent << " if (!ros_packed.ok()) return ros_packed.status();\n"; + os << indent + << " if (absl::Status status = output.SerializeLengthDelimited(" + << field->number() + << ", ros_packed->data(), ros_packed->size()); !status.ok()) " + "return status;\n"; + } else if (field->is_packable() && field->is_packed()) { + os << indent << " ::phaser::ROSReader ros_packed_reader = ros;\n"; + os << indent << " ::phaser::ProtoWriter ros_packed_size;\n"; + os << indent << " for (size_t ros_index = 0; ros_index < " << count + << "; ++ros_index) {\n"; + emit_one("ros_packed_reader", "ros_packed_size", indent + " ", true); + os << indent << " }\n"; + os << indent + << " if (absl::Status status = " + "output.SerializeLengthDelimitedHeader(" + << field->number() + << ", ros_packed_size.Size()); !status.ok()) return status;\n"; + os << indent << " for (size_t ros_index = 0; ros_index < " << count + << "; ++ros_index) {\n"; + emit_one("ros", "output", indent + " ", true); + os << indent << " }\n"; + } else { + os << indent << " for (size_t ros_index = 0; ros_index < " << count + << "; ++ros_index) {\n"; + emit_one("ros", "output", indent + " ", false); + os << indent << " }\n"; + } + os << indent << "}\n"; +} + +void MessageGenerator::GenerateDirectROSToProtobuf(std::ostream& os) { + const std::string name = MessageName(message_); + os << "absl::Status " << name + << "::ROSReaderToProtobuf(::phaser::ROSReader& ros, " + "::phaser::ProtoWriter& output) {\n"; + if (IsRosTime(message_) || IsRosDuration(message_)) { + const std::string seconds_type = + IsRosTime(message_) ? "uint32_t" : "int32_t"; + os << " absl::StatusOr<" << seconds_type << "> ros_seconds = ros.Read<" + << seconds_type << ">();\n"; + os << " if (!ros_seconds.ok()) return ros_seconds.status();\n"; + os << " absl::StatusOr<" << seconds_type << "> ros_nanos = ros.Read<" + << seconds_type << ">();\n"; + os << " if (!ros_nanos.ok()) return ros_nanos.status();\n"; + os << " if (absl::Status status = " + "output.SerializeVarint(1, " + "static_cast(*ros_seconds)); !status.ok()) return status;\n"; + os << " if (absl::Status status = " + "output.SerializeVarint(2, " + "static_cast(*ros_nanos)); !status.ok()) return status;\n"; + } else { + for (const auto& item : fields_in_order_) { + if (!item->IsUnion()) { + GenerateDirectROSFieldToProtobuf(os, item->field, " "); + continue; + } + auto union_info = std::static_pointer_cast(item); + os << " {\n"; + os << " absl::StatusOr ros_discriminator = " + "ros.Read();\n"; + os << " if (!ros_discriminator.ok()) return " + "ros_discriminator.status();\n"; + os << " switch (*ros_discriminator) {\n"; + os << " case 0:\n"; + os << " break;\n"; + for (const auto& member : union_info->members) { + os << " case " << member->field->number() << ":\n"; + GenerateDirectROSFieldToProtobuf(os, member->field, " "); + os << " break;\n"; + } + os << " default:\n"; + os << " return absl::InvalidArgumentError(" + "\"Unknown ROS oneof discriminator\");\n"; + os << " }\n"; + os << " }\n"; + } + } + os << " return absl::OkStatus();\n"; + os << "}\n\n"; + + os << "absl::Status " << name + << "::ROSToProtobuf(absl::Span ros_bytes, " + "::phaser::ProtoBuffer& output) {\n"; + os << " ::phaser::ROSReader ros(ros_bytes);\n"; + os << " ::phaser::ProtoWriter writer(output);\n"; + os << " if (absl::Status status = ROSReaderToProtobuf(ros, writer); " + "!status.ok()) return status;\n"; + os << " if (!ros.Eof()) return absl::InvalidArgumentError(" + "\"Trailing bytes after ROS message\");\n"; + os << " return absl::OkStatus();\n"; + os << "}\n\n"; +} + void MessageGenerator::GenerateROSSerialization(std::ostream& os, bool decl) { const std::string name = MessageName(message_); if (decl) { @@ -2121,6 +2935,18 @@ void MessageGenerator::GenerateROSSerialization(std::ostream& os, bool decl) { )XXX"; os << " static absl::Status ProtobufToROS(" "std::string_view protobuf, ::phaser::ROSBuffer& output);\n"; + os << " static absl::Status ProtobufWireToROS(" + "std::string_view protobuf, ::phaser::ROSBuffer& output);\n"; + os << " static absl::Status ROSReaderToProtobuf(" + "::phaser::ROSReader& ros, ::phaser::ProtoWriter& output);\n"; + os << " static absl::Status ROSToProtobuf(" + "absl::Span ros, ::phaser::ProtoBuffer& output);\n"; + os << R"XXX( static bool ROSToProtobufArray( + absl::Span ros, void* output, size_t output_size) { + ::phaser::ProtoBuffer buffer(static_cast(output), output_size); + return ROSToProtobuf(ros, buffer).ok(); + } +)XXX"; os << " static absl::Status PhaserToROS(" "absl::Span phaser, ::phaser::ROSBuffer& output);\n\n"; os << " static absl::Status ConvertToROS(" @@ -2166,12 +2992,11 @@ void MessageGenerator::GenerateROSSerialization(std::ostream& os, bool decl) { if (fixed_extent <= 0) { os << " size += 4;\n"; } - const std::string count = - fixed_extent > 0 ? std::to_string(fixed_extent) - : item->member_name + ".size()"; + const std::string count = fixed_extent > 0 + ? std::to_string(fixed_extent) + : item->member_name + ".size()"; std::string bulk_type = ROSBulkPrimitiveType(descriptor); - if (descriptor->type() == - google::protobuf::FieldDescriptor::TYPE_ENUM) { + if (descriptor->type() == google::protobuf::FieldDescriptor::TYPE_ENUM) { bulk_type = EnumName(descriptor->enum_type()); } if (!bulk_type.empty()) { @@ -2235,15 +3060,13 @@ void MessageGenerator::GenerateROSSerialization(std::ostream& os, bool decl) { const int fixed_extent = GetArraySize(descriptor); if (fixed_extent <= 0) { os << " if (absl::Status status = buffer.WriteSequenceLength(" - << item->member_name - << ".size()); !status.ok()) return status;\n"; + << item->member_name << ".size()); !status.ok()) return status;\n"; } - const std::string count = - fixed_extent > 0 ? std::to_string(fixed_extent) - : item->member_name + ".size()"; + const std::string count = fixed_extent > 0 + ? std::to_string(fixed_extent) + : item->member_name + ".size()"; std::string bulk_type = ROSBulkPrimitiveType(descriptor); - if (descriptor->type() == - google::protobuf::FieldDescriptor::TYPE_ENUM) { + if (descriptor->type() == google::protobuf::FieldDescriptor::TYPE_ENUM) { bulk_type = EnumName(descriptor->enum_type()); } if (!bulk_type.empty()) { @@ -2254,15 +3077,13 @@ void MessageGenerator::GenerateROSSerialization(std::ostream& os, bool decl) { << ")); !status.ok()) return status;\n"; os << " } else {\n"; os << " if (absl::Status status = buffer.WriteArray<" << bulk_type - << ">(absl::Span(" - << item->member_name << ".data(), " << count - << ")); !status.ok()) return status;\n"; + << ">(absl::Span(" << item->member_name + << ".data(), " << count << ")); !status.ok()) return status;\n"; os << " }\n"; } else { os << " if (absl::Status status = buffer.WriteArray<" << bulk_type - << ">(absl::Span(" - << item->member_name << ".data(), " << count - << ")); !status.ok()) return status;\n"; + << ">(absl::Span(" << item->member_name + << ".data(), " << count << ")); !status.ok()) return status;\n"; } } else { os << " for (size_t ros_index = 0; ros_index < " << count @@ -2310,8 +3131,7 @@ void MessageGenerator::GenerateROSSerialization(std::ostream& os, bool decl) { const auto& field = union_info->members[i]; os << " case " << field->field->number() << ":\n"; GenerateROSFieldRead(os, field->field, union_info->member_name, - " ", false, "", - static_cast(i)); + " ", false, "", static_cast(i)); os << " break;\n"; } os << " default:\n"; @@ -2330,8 +3150,7 @@ void MessageGenerator::GenerateROSSerialization(std::ostream& os, bool decl) { const int fixed_extent = GetArraySize(descriptor); std::string bulk_type = ROSBulkPrimitiveType(descriptor); - if (descriptor->type() == - google::protobuf::FieldDescriptor::TYPE_ENUM) { + if (descriptor->type() == google::protobuf::FieldDescriptor::TYPE_ENUM) { bulk_type = EnumName(descriptor->enum_type()); } if (fixed_extent <= 0) { @@ -2340,8 +3159,8 @@ void MessageGenerator::GenerateROSSerialization(std::ostream& os, bool decl) { "buffer.ReadSequenceLength();\n"; os << " if (!ros_count.ok()) return ros_count.status();\n"; if (!bulk_type.empty()) { - os << " if (*ros_count > buffer.Remaining() / sizeof(" - << bulk_type << ")) {\n"; + os << " if (*ros_count > buffer.Remaining() / sizeof(" << bulk_type + << ")) {\n"; os << " return absl::InvalidArgumentError(" "\"ROS sequence length exceeds remaining input\");\n"; os << " }\n"; @@ -2387,14 +3206,16 @@ void MessageGenerator::GenerateROSSerialization(std::ostream& os, bool decl) { os << " return absl::OkStatus();\n"; os << "}\n\n"; + GenerateDirectProtobufToROS(os); + GenerateDirectROSToProtobuf(os); + os << "absl::Status " << name << "::ProtobufToROS(std::string_view protobuf, " "::phaser::ROSBuffer& output) {\n"; - os << " " << name << " message;\n"; - os << " ::phaser::ProtoBuffer input(protobuf);\n"; - os << " if (absl::Status status = message.Deserialize(input); " - "!status.ok()) return status;\n"; - os << " return message.SerializeToROS(output);\n"; + os << " output.Clear();\n"; + os << " absl::Status status = ProtobufWireToROS(protobuf, output);\n"; + os << " if (!status.ok()) output.Clear();\n"; + os << " return status;\n"; os << "}\n\n"; os << "absl::Status " << name @@ -2647,13 +3468,19 @@ void MessageGenerator::GenerateStreamer(std::ostream& os) { } if (field->field->is_repeated()) { - os << " for (auto& v : msg." << field->member_name << ") {\n"; + os << " for (auto v : msg." << field->member_name << ") {\n"; os << " msg." << field->member_name << ".PrintIndent(os);\n"; if (field->field->type() == google::protobuf::FieldDescriptor::TYPE_ENUM) { os << " os << \"" << field->field->name() << ": \" << " << EnumName(field->field->enum_type()) << "Stringizer()(v) << std::endl;\n"; + } else if (field->field->type() == + google::protobuf::FieldDescriptor::TYPE_STRING || + field->field->type() == + google::protobuf::FieldDescriptor::TYPE_BYTES) { + os << " os << \"" << field->field->name() + << ": \\\"\" << v << \"\\\"\" << std::endl;\n"; } else { os << " os << \"" << field->field->name() << ": \" << v << std::endl;\n"; @@ -2705,10 +3532,13 @@ void MessageGenerator::GenerateCopy(std::ostream& os, bool decl) { google::protobuf::FieldDescriptor::TYPE_MESSAGE) { os << " for (size_t i = 0; i < static_cast(" << array_size << "); i++) {\n"; - os << " if (!other." << field->member_name << "[i].empty()) {\n"; - os << " if (absl::Status s = " << field->member_name - << "[i].Mutable()->CloneFrom(other." << field->member_name - << ".Get(i)); !s.ok()) return s;\n"; + os << " auto source = other." << field->member_name + << ".Get(i);\n"; + os << " if (source.IsBound()) {\n"; + os << " auto destination = " << field->member_name + << ".Mutable(i);\n"; + os << " if (absl::Status s = destination.CloneFrom(source); " + "!s.ok()) return s;\n"; os << " }\n"; os << " }\n"; } else if (field->field->type() == @@ -2728,14 +3558,14 @@ void MessageGenerator::GenerateCopy(std::ostream& os, bool decl) { os << " }\n"; } } else if (field->field->type() == - google::protobuf::FieldDescriptor::TYPE_MESSAGE) { - os << " for (auto& v : other." << field->member_name << ") {\n"; - os << " auto* m = " << field->member_name << ".Add();\n"; - os << " if (absl::Status s = m->CloneFrom(v.Msg()); !s.ok()) return " + google::protobuf::FieldDescriptor::TYPE_MESSAGE) { + os << " for (auto v : other." << field->member_name << ") {\n"; + os << " auto m = " << field->member_name << ".Add();\n"; + os << " if (absl::Status s = m.CloneFrom(v); !s.ok()) return " "s;\n"; os << " }\n"; } else { - os << " for (auto& v : other." << field->member_name << ") {\n"; + os << " for (auto v : other." << field->member_name << ") {\n"; os << " " << field->member_name << ".Add(v);\n"; os << " }\n"; } @@ -2783,7 +3613,8 @@ void MessageGenerator::GenerateCopy(std::ostream& os, bool decl) { os << " if (absl::Status s = " << u->member_name << ".template CloneFrom<" << i << ">(other." << u->member_name << ".template GetReference<" << i << ", " - << MessageName(field->field->message_type()) << ">()); !s.ok()) " + << MessageName(field->field->message_type()) + << ">()); !s.ok()) " "return s;\n"; } else { os << " if (absl::Status s = " << u->member_name @@ -2804,11 +3635,11 @@ void MessageGenerator::GenerateCopy(std::ostream& os, bool decl) { } else { for (auto& field : fields_) { if (field->field->is_repeated()) { - os << " for (auto& v : other." << field->field->name() << "()) {\n"; + os << " for (auto v : other." << field->field->name() << "()) {\n"; if (field->field->type() == google::protobuf::FieldDescriptor::TYPE_MESSAGE) { - os << " auto* m = add_" << field->field->name() << "();\n"; - os << " if (absl::Status s = m->CloneFrom(v.Msg()); !s.ok()) return " + os << " auto m = add_" << field->field->name() << "();\n"; + os << " if (absl::Status s = m.CloneFrom(v); !s.ok()) return " "s;\n"; } else { os << " add_" << field->field->name() << "(v);\n"; @@ -2836,8 +3667,8 @@ void MessageGenerator::GenerateCopy(std::ostream& os, bool decl) { auto& field = u->members[i]; os << " case " << field->field->number() << ":\n"; os << " if (absl::Status s = " << u->member_name - << ".template CloneFrom<" << i << ">(other." << field->field->name() - << "()); !s.ok()) return s;\n"; + << ".template CloneFrom<" << i << ">(other." + << field->field->name() << "()); !s.ok()) return s;\n"; os << " break;\n"; } os << " }\n"; @@ -2894,6 +3725,30 @@ void MessageGenerator::GeneratePhaserBank(std::ostream& os) { os << " return m->SerializedSize();\n"; os << "}\n\n"; + os << "static absl::Status " << MessageName(message_) + << "SerializeAtOffset(std::shared_ptr<::phaser::MessageRuntime> runtime, " + "::toolbelt::BufferOffset offset, ::phaser::ProtoBuffer& buffer) {\n"; + os << " const " << MessageName(message_) << " message(runtime, offset);\n"; + os << " return message.Serialize(buffer);\n"; + os << "}\n\n"; + + os << "static absl::Status " << MessageName(message_) + << "DeserializeAtOffset(" + "std::shared_ptr<::phaser::MessageRuntime> runtime, " + "::toolbelt::BufferOffset offset, ::phaser::ProtoBuffer& buffer) {\n"; + os << " " << MessageName(message_) << " message(runtime, offset);\n"; + os << " message.InstallMetadata<" << MessageName(message_) << ">();\n"; + os << " return message.Deserialize(buffer);\n"; + os << "}\n\n"; + + os << "static size_t " << MessageName(message_) + << "SerializedSizeAtOffset(" + "std::shared_ptr<::phaser::MessageRuntime> runtime, " + "::toolbelt::BufferOffset offset) {\n"; + os << " const " << MessageName(message_) << " message(runtime, offset);\n"; + os << " return message.SerializedSize();\n"; + os << "}\n\n"; + os << "static ::phaser::Message* " << MessageName(message_) << "AllocateAtOffset(std::shared_ptr<::phaser::MessageRuntime> runtime, " "::toolbelt::BufferOffset offset) {\n"; @@ -2963,8 +3818,8 @@ void MessageGenerator::GeneratePhaserBank(std::ostream& os) { auto& field = u->members[i]; os << " case " << field->field->number() << ":\n"; if (IsRosFrontend()) { - os << " return m->" << u->member_name << ".Discriminator() == " - << field->field->number() << ";\n"; + os << " return m->" << u->member_name + << ".Discriminator() == " << field->field->number() << ";\n"; } else { os << " return m->" << oneof->name() << "_case() == " << field->field->number() << ";\n"; @@ -3021,6 +3876,12 @@ void MessageGenerator::GeneratePhaserBank(std::ostream& os) { os << " .deserialize_from_buffer = " << MessageName(message_) << "DeserializeFromBuffer,\n"; os << " .serialized_size = " << MessageName(message_) << "SerializedSize,\n"; + os << " .serialize_at_offset = " << MessageName(message_) + << "SerializeAtOffset,\n"; + os << " .deserialize_at_offset = " << MessageName(message_) + << "DeserializeAtOffset,\n"; + os << " .serialized_size_at_offset = " << MessageName(message_) + << "SerializedSizeAtOffset,\n"; os << " .allocate_at_offset = " << MessageName(message_) << "AllocateAtOffset,\n"; os << " .allocate = " << MessageName(message_) << "Allocate,\n"; diff --git a/phaser/compiler/message_gen.h b/phaser/compiler/message_gen.h index 8b97297..2742810 100644 --- a/phaser/compiler/message_gen.h +++ b/phaser/compiler/message_gen.h @@ -93,6 +93,7 @@ class MessageGenerator { void GenerateConstructors(std::ostream& os, bool decl); void GenerateFieldInitializers(std::ostream& os, const char* sep = ": "); void GenerateSizeFunctions(std::ostream& os); + size_t ReachableMessageTypeCount() const; void GenerateFieldMetadata(std::ostream& os); void GenerateCreators(std::ostream& os, bool decl); void GenerateClear(std::ostream& os, bool decl); @@ -113,6 +114,34 @@ class MessageGenerator { void GenerateSerializer(std::ostream& os, bool decl); void GenerateDeserializer(std::ostream& os, bool decl); void GenerateROSSerialization(std::ostream& os, bool decl); + void GenerateDirectProtobufToROS(std::ostream& os); + void GenerateDirectProtobufField( + std::ostream& os, const google::protobuf::FieldDescriptor* field, + const std::string& indent); + void GenerateDirectProtobufSingularField( + std::ostream& os, const google::protobuf::FieldDescriptor* field, + const std::string& indent); + void GenerateDirectProtobufReadValue( + std::ostream& os, const google::protobuf::FieldDescriptor* field, + const std::string& buffer, const std::string& value, + const std::string& indent); + void GenerateDirectROSWriteValue( + std::ostream& os, const google::protobuf::FieldDescriptor* field, + const std::string& value, const std::string& indent); + void GenerateDirectROSToProtobuf(std::ostream& os); + void GenerateDirectROSFieldToProtobuf( + std::ostream& os, const google::protobuf::FieldDescriptor* field, + const std::string& indent); + void GenerateDirectROSReadValue( + std::ostream& os, const google::protobuf::FieldDescriptor* field, + const std::string& reader, const std::string& value, + const std::string& indent); + void GenerateDirectProtoWriteValue( + std::ostream& os, const google::protobuf::FieldDescriptor* field, + const std::string& value, const std::string& field_number, + const std::string& indent, bool raw = false); + std::string DirectProtobufValueType( + const google::protobuf::FieldDescriptor* field) const; void GenerateROSFieldSize( std::ostream& os, const google::protobuf::FieldDescriptor* field, const std::string& value_expression, const std::string& indent); diff --git a/phaser/docs/phaser_user_guide.md b/phaser/docs/phaser_user_guide.md index 5b57442..b9ad2ad 100644 --- a/phaser/docs/phaser_user_guide.md +++ b/phaser/docs/phaser_user_guide.md @@ -49,11 +49,12 @@ So, when you set a field using the `set_` function, the data is placed in the `b buffer using information held in the `source` class. This makes the source classes very lightweight, containing just metadata about the binary message. -When you read a field from a source class, the data is read from the binary message. However, +When you read a field from a source class, the data is read from the binary message. However, this is not done directly since the binary message may be a different version from that -doing the reading (fields may be been added or changed). Therefore, reading a field results -in the use of `field metadata` that is held in a small array inside the binary message. This -allows the software to use newer or older version of messages, one of the mainstays of +doing the reading (fields may have been added or changed). Reading therefore uses `field +metadata` stored inside the binary message. Dense field-number ranges use a direct-indexed +table, while fields outside that range use a sorted sparse table and binary search. This +allows the software to use newer or older versions of messages, one of the mainstays of protobuf's popularity. ## What's wrong with serialization? @@ -259,45 +260,56 @@ for (int value : msg.values) { /* ... */ } ``` Assigning one field proxy to another copies the field value; it does not rebind -the destination proxy to the source message. Copying a generated ROS message -deep-copies its payload. Iterators, references, pointers, and string views into -a dynamic message can be invalidated by an operation that grows or compacts its -payload buffer, so reacquire them after mutation. +the destination proxy to the source message. Copying an owning generated ROS +message deep-copies its payload, while copying a read-only message handle keeps +a borrowed view of the caller-owned receive buffer. Iterators and string views +into a dynamic message can be invalidated by an operation that grows or compacts +its payload buffer, so reacquire them after mutation. `CreateReadonly` messages support const field access and serialization. A non-const proxy operation requires a mutable message. +Repeated strings index and iterate as `std::string_view`. Repeated and +fixed-array messages index and iterate as generated message handles returned by +value; `add_*` and `mutable_*` message accessors also return handles by value. +The collection `Get()` helpers materialize owning vectors and may allocate. + ### ROS1 intrinsic types The ROS frontend recognizes three singular protobuf message declarations and presents their fields as the corresponding ROS1 C++ types: - `google.protobuf.Timestamp` as `ros::Time` - `google.protobuf.Duration` as `ros::Duration` -- `std_msgs.Header` as `std_msgs::Header` +- mutable `std_msgs.Header` access as `std_msgs::Header` +- read-only `std_msgs.Header` access as `phaser::RosHeaderView` `std_msgs.Header` is expected to contain `uint32 seq`, a `google.protobuf.Timestamp stamp`, and `string frame_id`. Generate it with a non-empty `add_namespace` so the Phaser backend type does not collide with the real ROS class. -The proxies support values and both const and mutable references, so existing +Mutable proxies support values and mutable references, so existing mutation functions can be called without adapters: ```c++ void Advance(ros::Time& stamp); void UpdateHeader(std_msgs::Header& header); -void Observe(const std_msgs::Header& header); Advance(msg.stamp); UpdateHeader(msg.header); -Observe(msg.header); + +const auto& received = msg; +phaser::RosHeaderView header = received.header.Get(); +std::string_view frame = header.frame_id; +std_msgs::Header owned = header.ToOwned(); // May allocate. ``` -`ros::Time`, `ros::Duration`, and especially `std_msgs::Header` are cached +`ros::Time`, `ros::Duration`, and mutable `std_msgs::Header` values are cached source objects rather than in-buffer overlays. Mutable-reference changes are automatically copied into the payload before native `Data()`/size access, protobuf serialization, copying, and recursive parent synchronization. Avoid accessing `runtime->pb` directly while such a mutable borrow may be dirty. +`RosHeaderView` instead reads the payload directly and does not allocate. Pass the ROS libraries needed by generated headers through `cc_deps`: @@ -394,14 +406,21 @@ absl::Status ParseFromROS(absl::Span input); static absl::Status ProtobufToROS( std::string_view protobuf, ::phaser::ROSBuffer& output); +static absl::Status ROSToProtobuf( + absl::Span ros, ::phaser::ProtoBuffer& output); +static bool ROSToProtobufArray( + absl::Span ros, void* output, size_t output_size); static absl::Status PhaserToROS( absl::Span phaser, ::phaser::ROSBuffer& output); static absl::Status ConvertToROS( absl::Span input, ::phaser::ROSBuffer& output); ``` -`SerializeToROS` reads a live message. `ProtobufToROS` parses protobuf wire -bytes with the generated Phaser parser and then writes ROS bytes. +`SerializeToROS` reads a live message. `ProtobufToROS` scans protobuf wire +fields directly and emits ROS bytes in schema order. `ROSToProtobuf` scans ROS +wire fields in schema order and writes protobuf tags directly; it uses counting +passes to determine nested-message and packed-field lengths without staging +encoded bytes or constructing a native message. `PhaserToROS` attaches a read-only message to a valid native Phaser payload for the duration of the conversion. `ConvertToROS` calls `InferMessageWireFormat` and selects either input path. Inference validates the @@ -672,6 +691,28 @@ void Receive(const char* buffer, size_t buffer_size) { The runtime access to fields in the message validates that nothing can go outside of the buffer you pass to `CreateReadonly`. +Caller-buffer `CreateMutable`, typed scalar/string/nested/repeated/oneof +mutation, typed `Any::MutableAny()`, native `CreateReadonly`, typed traversal, +protobuf `SerializeToArray`, and fixed-buffer ROS serialization do not use the +system heap. Runtime control data and exact generated type names are stored in +the `PayloadBuffer`; generated roots reserve capacity for all statically +reachable message types, and typed `Any` growth uses that same allocator. +Successful protobuf and ROS deserialization into a fixed mutable message also +use only the payload allocator. This includes protobuf `Any` values whose type +is registered in the Phaser bank. `ProtobufToROS` is allocation-free when passed +a sufficiently large fixed `ROSBuffer`; generated field scanners read protobuf +wire values directly and emit fields in ROS schema order without constructing an +intermediate protobuf or Phaser message. `ROSToProtobuf` provides the symmetric +guarantee with a caller-provided `ProtoBuffer`. ROS Header deserialization also +writes `frame_id` directly into payload storage from the wire view, avoiding an +owning `std_msgs::Header` string allocation. +Returned handles and `std::string_view` values borrow the receive buffer; the +caller must keep that buffer alive and unchanged. Dynamic message ownership, +reflection, debug output, unknown-type error construction, and APIs returning +`std::string` or other owning containers are not covered by this contract. +Mutable ROS Headers use the payload-backed `header.Mutable()` view; convert with +`ToOwned()` only when an owning `std_msgs::Header` is required. + ## Setting and getting fields Creating a message usually involves setting the values in its fields, and using one @@ -861,6 +902,11 @@ directly. It does not check that the message is of the given type, so make sure The `MutableAny` function creates a mutable message of type `T` in the `value` field and sets the `type_url`. You can then create the message as you would do normally. +For a native read-only message, `Is()`, `As()`, serialized-size +calculation, and caller-buffer protobuf serialization construct the concrete +embedded handle on the stack and do not allocate. `UnpackTo`, mutation, +reflection, and debug helpers remain copying or owning operations. + ## Serialization and deserialization Phaser's native format does not require serialization. To interoperate with protobuf, generated messages provide these protobuf transcoding functions: @@ -1290,15 +1336,24 @@ The binary version of a message can be seen in the following diagram: -The first 4 bytes of a message contains the offset to the message's `metadata`. This is -a fixed size array of field information specifying, for each field, where that field -is located (offset from the start of the PayloadBuffer) and its id. The field id -is used as the a bit number for the `presence mask` which is located immediately -after the metadata in the message. The presence mask has a bit position for every -primitive field and may not be present if there are no primitive fields. The purpose -of the metadata is to allow a message reader to locate the field in received -data. The message might have been created by a different version of the software -and field locations might have changed or fields added or removed. +The first 4 bytes of a message contain the offset to the message's field metadata. +The generator selects the field-number interval for which a direct table saves +space over individual entries. That interval stores an occupancy bitmap and one +packed offset/id value per field-number slot. Fields outside the interval are +stored as sorted `(number, offset, id)` entries and found by binary search. A +message with dense fields `1..20` and an outlier at `50000`, for example, uses +direct lookup for the first group and one sparse entry for the outlier. + +The field id is used as a bit number for the `presence mask`, which is located +immediately after the metadata offset in the message. The presence mask has a +bit position for every primitive field and may not be present if there are no +primitive fields. Field proxies cache both the resolved offset and id after +their first access, including a missing-field result. + +Hybrid metadata changes the native Phaser payload format. This runtime can +still read legacy metadata using its previous all-fields binary-search layout, +but older runtimes cannot read payloads generated with hybrid metadata. This +does not affect protobuf or ROS wire-format compatibility. The rest of the message consists of the fixed size portion of the message. This has space for every field in the message. Primitive fields (like int32, double, etc.) diff --git a/phaser/perf_test.cc b/phaser/perf_test.cc index 6fdb6fc..d8d31d4 100644 --- a/phaser/perf_test.cc +++ b/phaser/perf_test.cc @@ -11,6 +11,7 @@ #include "absl/strings/str_format.h" #include "phaser/runtime/runtime.h" +#include "phaser/testdata/TestMessage.phaser.h" #include "phaser/testdata/vision.pb.h" #include "phaser/testdata/vision.phaser.h" #include "toolbelt/clock.h" @@ -312,7 +313,7 @@ TEST(PerfTest, PhaserAllLidarsPush) { constexpr int kNumBeams = 100000; lidars.reserve_scans(kNumLidars); for (int j = 0; j < kNumLidars; ++j) { - robot::phaser::LidarScan* scan = lidars.add_scans(); + auto scan = lidars.add_scans(); scan->mutable_header()->set_timestamp(1234567890); scan->reserve_beams(kNumBeams); @@ -357,9 +358,9 @@ TEST(PerfTest, PhaserAllLidarsZeroCopy) { constexpr int kNumBeams = 100000; // Allocate all the scans at once. - std::vector lidar_scans = + std::vector lidar_scans = lidars.allocate_scans(kNumLidars); - for (auto scan : lidar_scans) { + for (auto& scan : lidar_scans) { scan->mutable_header()->set_timestamp(1234567890); scan->resize_beams(kNumBeams); @@ -375,7 +376,7 @@ TEST(PerfTest, PhaserAllLidarsZeroCopy) { ASSERT_EQ(lidars2.scans_size(), kNumLidars); auto& scans = lidars2.scans(); for (int j = 0; j < kNumLidars; ++j) { - const robot::phaser::LidarScan& scan = *(scans[j]); + const robot::phaser::LidarScan scan = scans[j]; ASSERT_EQ(scan.header().timestamp(), 1234567890); ASSERT_EQ(scan.beams_size(), kNumBeams); absl::Span beams = scan.beams_as_span(); @@ -389,6 +390,277 @@ TEST(PerfTest, PhaserAllLidarsZeroCopy) { std::cout << absl::StrFormat("Phaser zero-copy: %d ns\n", end - start); } +TEST(PerfTest, HybridFieldLookup) { + foo::bar::phaser::HybridLookupMessage hybrid; + foo::bar::phaser::SparseLookupMessage sparse; + constexpr int kIterations = 10000000; + volatile int64_t result = 0; + + uint64_t start = toolbelt::Now(); + for (int i = 0; i < kIterations; ++i) { + result += hybrid.FindField(10).offset; + } + uint64_t dense_end = toolbelt::Now(); + + for (int i = 0; i < kIterations; ++i) { + result += sparse.FindField(10000).offset; + } + uint64_t sparse_end = toolbelt::Now(); + + std::cout << absl::StrFormat("Dense direct lookup: %d ns\n", + dense_end - start); + std::cout << absl::StrFormat("Sparse binary lookup: %d ns\n", + sparse_end - dense_end); + EXPECT_NE(result, 0); +} + +TEST(PerfTest, DenseVsSparsePhaserMessageReadsAndWrites) { + using Dense = foo::bar::phaser::DenseLookupBenchmarkMessage; + using Sparse = foo::bar::phaser::SparseLookupBenchmarkMessage; + + std::vector dense_storage(64 * 1024); + std::vector sparse_storage(64 * 1024); + Dense dense_seed = + Dense::CreateMutable(dense_storage.data(), dense_storage.size()); + Sparse sparse_seed = + Sparse::CreateMutable(sparse_storage.data(), sparse_storage.size()); + + const auto write_fields = [](auto& message, int32_t base) { + message.set_value_01(base + 1); + message.set_value_02(base + 2); + message.set_value_03(base + 3); + message.set_value_04(base + 4); + message.set_value_05(base + 5); + message.set_value_06(base + 6); + message.set_value_07(base + 7); + message.set_value_08(base + 8); + message.set_value_09(base + 9); + message.set_value_10(base + 10); + message.set_value_11(base + 11); + message.set_value_12(base + 12); + message.set_value_13(base + 13); + message.set_value_14(base + 14); + message.set_value_15(base + 15); + message.set_value_16(base + 16); + message.set_value_17(base + 17); + message.set_value_18(base + 18); + message.set_value_19(base + 19); + message.set_value_20(base + 20); + message.set_value_21(base + 21); + message.set_value_22(base + 22); + message.set_value_23(base + 23); + message.set_value_24(base + 24); + message.set_value_25(base + 25); + message.set_value_26(base + 26); + message.set_value_27(base + 27); + message.set_value_28(base + 28); + message.set_value_29(base + 29); + message.set_value_30(base + 30); + message.set_value_31(base + 31); + message.set_value_32(base + 32); + message.set_value_33(base + 33); + message.set_value_34(base + 34); + message.set_value_35(base + 35); + message.set_value_36(base + 36); + message.set_value_37(base + 37); + message.set_value_38(base + 38); + message.set_value_39(base + 39); + message.set_value_40(base + 40); + message.set_value_41(base + 41); + message.set_value_42(base + 42); + message.set_value_43(base + 43); + message.set_value_44(base + 44); + message.set_value_45(base + 45); + message.set_value_46(base + 46); + message.set_value_47(base + 47); + message.set_value_48(base + 48); + message.set_value_49(base + 49); + message.set_value_50(base + 50); + message.set_value_51(base + 51); + message.set_value_52(base + 52); + message.set_value_53(base + 53); + message.set_value_54(base + 54); + message.set_value_55(base + 55); + message.set_value_56(base + 56); + message.set_value_57(base + 57); + message.set_value_58(base + 58); + message.set_value_59(base + 59); + message.set_value_60(base + 60); + message.set_value_61(base + 61); + message.set_value_62(base + 62); + message.set_value_63(base + 63); + message.set_value_64(base + 64); + }; + const auto read_fields = [](const auto& message) -> int64_t { + return static_cast(message.value_01()) + message.value_02() + + message.value_03() + message.value_04() + message.value_05() + + message.value_06() + message.value_07() + message.value_08() + + message.value_09() + message.value_10() + message.value_11() + + message.value_12() + message.value_13() + message.value_14() + + message.value_15() + message.value_16() + message.value_17() + + message.value_18() + message.value_19() + message.value_20() + + message.value_21() + message.value_22() + message.value_23() + + message.value_24() + message.value_25() + message.value_26() + + message.value_27() + message.value_28() + message.value_29() + + message.value_30() + message.value_31() + message.value_32() + + message.value_33() + message.value_34() + message.value_35() + + message.value_36() + message.value_37() + message.value_38() + + message.value_39() + message.value_40() + message.value_41() + + message.value_42() + message.value_43() + message.value_44() + + message.value_45() + message.value_46() + message.value_47() + + message.value_48() + message.value_49() + message.value_50() + + message.value_51() + message.value_52() + message.value_53() + + message.value_54() + message.value_55() + message.value_56() + + message.value_57() + message.value_58() + message.value_59() + + message.value_60() + message.value_61() + message.value_62() + + message.value_63() + message.value_64(); + }; + + write_fields(dense_seed, 0); + write_fields(sparse_seed, 0); + const size_t dense_size = dense_seed.Size(); + const size_t sparse_size = sparse_seed.Size(); + + constexpr int kFirstTouchIterations = 1000000; + constexpr int kIterations = 5000000; + constexpr int kFieldsPerIteration = 64; + volatile int64_t checksum = 0; + + uint64_t first_touch_start = toolbelt::Now(); + for (int i = 0; i < kFirstTouchIterations; ++i) { + const Dense message = + Dense::CreateReadonly(dense_storage.data(), dense_size); + asm volatile("" ::: "memory"); + checksum += read_fields(message); + } + const uint64_t dense_first_touch_end = toolbelt::Now(); + + for (int i = 0; i < kFirstTouchIterations; ++i) { + const Sparse message = + Sparse::CreateReadonly(sparse_storage.data(), sparse_size); + asm volatile("" ::: "memory"); + checksum += read_fields(message); + } + const uint64_t sparse_first_touch_end = toolbelt::Now(); + + const Dense dense_message = + Dense::CreateReadonly(dense_storage.data(), dense_size); + const Sparse sparse_message = + Sparse::CreateReadonly(sparse_storage.data(), sparse_size); + + // Resolve each proxy once before timing. The benchmark below measures field + // reads and writes on already-constructed Phaser handles, not construction or + // first-access metadata lookup. + checksum += read_fields(dense_message); + checksum += read_fields(sparse_message); + + const uint64_t cached_start = toolbelt::Now(); + for (int i = 0; i < kIterations; ++i) { + asm volatile("" ::: "memory"); + checksum += read_fields(dense_message); + } + const uint64_t dense_read_end = toolbelt::Now(); + + for (int i = 0; i < kIterations; ++i) { + asm volatile("" ::: "memory"); + checksum += read_fields(sparse_message); + } + const uint64_t sparse_read_end = toolbelt::Now(); + + for (int i = 0; i < kIterations; ++i) { + write_fields(dense_seed, i); + asm volatile("" ::: "memory"); + } + const uint64_t dense_write_end = toolbelt::Now(); + + for (int i = 0; i < kIterations; ++i) { + write_fields(sparse_seed, i); + asm volatile("" ::: "memory"); + } + const uint64_t sparse_write_end = toolbelt::Now(); + + const auto report = [&](const char* label, uint64_t elapsed, int iterations) { + const double ns_per_field = + static_cast(elapsed) / + static_cast(iterations * kFieldsPerIteration); + std::cout << absl::StrFormat("%s: %d ns total, %.3f ns/field\n", label, + elapsed, ns_per_field); + }; + report("Dense first-touch Phaser reads", + dense_first_touch_end - first_touch_start, kFirstTouchIterations); + report("Sparse first-touch Phaser reads", + sparse_first_touch_end - dense_first_touch_end, kFirstTouchIterations); + report("Dense cached Phaser reads", dense_read_end - cached_start, + kIterations); + report("Sparse cached Phaser reads", sparse_read_end - dense_read_end, + kIterations); + report("Dense Phaser writes", dense_write_end - sparse_read_end, kIterations); + report("Sparse Phaser writes", sparse_write_end - dense_write_end, + kIterations); + + checksum += dense_seed.value_64(); + checksum += sparse_seed.value_64(); + EXPECT_GT(checksum, 0); +} + +TEST(PerfTest, AllocationFreeReceiveTree) { + std::vector buffer(1024 * 1024); + auto source = + robot::phaser::AllLidars::CreateMutable(buffer.data(), buffer.size()); + constexpr int kNumLidars = 64; + constexpr int kNumBeams = 128; + for (int i = 0; i < kNumLidars; ++i) { + auto scan = source.add_scans(); + scan->resize_beams(kNumBeams); + absl::Span beams = scan->beams_as_mutable_span(); + for (int j = 0; j < kNumBeams; ++j) { + beams[j] = i + j; + } + } + const size_t payload_size = source.Size(); + + constexpr int kIterations = 10000; + volatile double checksum = 0; + const uint64_t start = toolbelt::Now(); + for (int iteration = 0; iteration < kIterations; ++iteration) { + const auto received = + robot::phaser::AllLidars::CreateReadonly(buffer.data(), payload_size); + for (auto scan : received.scans()) { + const absl::Span beams = scan.beams_as_span(); + checksum += beams.front(); + checksum += beams.back(); + } + } + const uint64_t end = toolbelt::Now(); + std::cout << absl::StrFormat("Allocation-free receive tree: %d ns\n", + end - start); + EXPECT_GT(checksum, 0); +} + +TEST(PerfTest, AllocationFreeFixedBufferConstruction) { + std::vector buffer(1024 * 1024); + constexpr int kIterations = 10000; + volatile size_t checksum = 0; + const uint64_t start = toolbelt::Now(); + for (int iteration = 0; iteration < kIterations; ++iteration) { + auto message = foo::bar::phaser::TestMessage::CreateMutable(buffer.data(), + buffer.size()); + message.set_x(iteration); + message.set_s("fixed-buffer"); + message.add_vi32(iteration); + message.add_vstr("value"); + message.add_vm()->set_str("nested"); + auto embedded = + message.mutable_any()->MutableAny(); + embedded.set_str("any"); + checksum += message.Size(); + } + const uint64_t end = toolbelt::Now(); + std::cout << absl::StrFormat("Allocation-free fixed output: %d ns\n", + end - start); + EXPECT_GT(checksum, 0u); +} + int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); diff --git a/phaser/phaser_test.cc b/phaser/phaser_test.cc index f78c1ef..49ace76 100644 --- a/phaser/phaser_test.cc +++ b/phaser/phaser_test.cc @@ -186,6 +186,64 @@ TEST(PhaserTest, DeletedFieldsBasic) { ASSERT_EQ(msg2.s(), msg.s()); } +TEST(PhaserTest, HybridFieldMetadataLayout) { + using Hybrid = foo::bar::phaser::HybridLookupMessage; + EXPECT_EQ(Hybrid::field_data.header.magic, ::phaser::kHybridFieldDataMagic); + EXPECT_EQ(Hybrid::field_data.header.dense_base, 10u); + EXPECT_EQ(Hybrid::field_data.header.dense_span, 6u); + EXPECT_EQ(Hybrid::field_data.header.sparse_count, 1u); + EXPECT_EQ(Hybrid::field_data.dense_fields[2].offset, 0u); + EXPECT_EQ(Hybrid::field_data.sparse_fields[0].number, 1000u); + + Hybrid message; + message.set_dense_10(10); + message.set_dense_11(11); + message.set_dense_13(13); + message.set_dense_14(14); + message.set_dense_15(15); + message.set_sparse_1000(1000); + EXPECT_EQ(message.dense_10(), 10); + EXPECT_EQ(message.dense_15(), 15); + EXPECT_EQ(message.sparse_1000(), 1000); + EXPECT_EQ(message.FindField(12).offset, -1); + EXPECT_EQ(message.FindField(1000).offset, + static_cast(Hybrid::field_data.sparse_fields[0].offset)); +} + +TEST(PhaserTest, SparseFieldMetadataLayout) { + using Sparse = foo::bar::phaser::SparseLookupMessage; + EXPECT_EQ(Sparse::field_data.header.magic, ::phaser::kHybridFieldDataMagic); + EXPECT_EQ(Sparse::field_data.header.dense_base, 1u); + EXPECT_EQ(Sparse::field_data.header.dense_span, 1u); + EXPECT_EQ(Sparse::field_data.header.sparse_count, 2u); + EXPECT_NE(Sparse::field_data.dense_fields[0].offset, 0u); + EXPECT_EQ(Sparse::field_data.sparse_fields[0].number, 100u); + EXPECT_EQ(Sparse::field_data.sparse_fields[1].number, 10000u); + + Sparse message; + message.set_field_1(1); + message.set_field_100(100); + message.set_field_10000(10000); + EXPECT_EQ(message.field_1(), 1); + EXPECT_EQ(message.field_100(), 100); + EXPECT_EQ(message.field_10000(), 10000); + EXPECT_EQ(message.FindField(99).offset, -1); +} + +TEST(PhaserTest, DenseMetadataHandlesOneofAndOutlierFields) { + using Message = foo::bar::phaser::TestMessage; + ASSERT_LE(Message::field_data.header.dense_base, 107u); + ASSERT_GT(Message::field_data.header.dense_base + + Message::field_data.header.dense_span, + 108u); + const size_t first_index = 107u - Message::field_data.header.dense_base; + const size_t second_index = 108u - Message::field_data.header.dense_base; + EXPECT_EQ(Message::field_data.dense_fields[first_index].offset, + Message::field_data.dense_fields[second_index].offset); + ASSERT_EQ(Message::field_data.header.sparse_count, 1u); + EXPECT_EQ(Message::field_data.sparse_fields[0].number, 200u); +} + TEST(PhaserTest, CopySimple) { foo::bar::phaser::TestMessage msg; msg.set_x(1234); diff --git a/phaser/receive_allocation_test.cc b/phaser/receive_allocation_test.cc new file mode 100644 index 0000000..c2a93a4 --- /dev/null +++ b/phaser/receive_allocation_test.cc @@ -0,0 +1,492 @@ +#include "phaser/testdata/RosCompile.phaser.h" +#include "phaser/testdata/RosIntrinsics.phaser.h" +#include "phaser/testdata/TestMessage.phaser.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" + +namespace { + +std::atomic g_count_allocations = false; +std::atomic g_allocation_count = 0; + +void CountAllocation() { + if (g_count_allocations.load(std::memory_order_relaxed)) { + g_allocation_count.fetch_add(1, std::memory_order_relaxed); + } +} + +void* Allocate(size_t size) { + CountAllocation(); + if (void* result = std::malloc(size == 0 ? 1 : size); result != nullptr) { + return result; + } + throw std::bad_alloc(); +} + +void* AllocateAligned(size_t size, size_t alignment) { + CountAllocation(); + void* result = nullptr; + if (posix_memalign(&result, alignment, size == 0 ? 1 : size) == 0) { + return result; + } + throw std::bad_alloc(); +} + +struct ExerciseResult { + bool values_match = true; + bool protobuf_serialized = true; + bool ros_serialized = true; +}; + +template +std::vector CopyPayload(const Message& message) { + const size_t size = message.ByteSizeLong(); + std::vector result(size); + std::memcpy(result.data(), message.Data(), size); + return result; +} + +ExerciseResult Exercise( + const std::vector& ros_compile_payload, + const std::vector& ros_intrinsic_payload, + const std::vector& any_payload, + const std::vector& dense_payload, + const std::vector& sparse_payload) { + ExerciseResult result; + + auto ros_message = + foo::bar::phaser::RosCompileMessage::CreateReadonly( + ros_compile_payload.data(), ros_compile_payload.size()); + const auto& ros_view = ros_message; + result.values_match &= ros_view.x.Get() == 42; + result.values_match &= ros_view.name.Get() == "root"; + result.values_match &= + ros_view.choice + .get() == + 17; + result.values_match &= ros_view.names.size() == 2; + size_t string_count = 0; + for (std::string_view value : ros_view.names) { + result.values_match &= !value.empty(); + ++string_count; + } + result.values_match &= string_count == 2; + size_t message_count = 0; + for (auto value : ros_view.inners) { + result.values_match &= value.id.Get() > 0; + ++message_count; + } + result.values_match &= message_count == 2; + const auto first_inner = ros_view.inners[0]; + auto copied_inner = first_inner; + const auto moved_inner = std::move(copied_inner); + result.values_match &= moved_inner.id.Get() == 1; + result.values_match &= ros_view.fixed_names[0] == "fixed"; + result.values_match &= ros_view.fixed_inners[0].id.Get() == 9; + + char protobuf_output[4096]; + result.protobuf_serialized &= + ros_view.SerializeToArray(protobuf_output, sizeof(protobuf_output)); + char ros_output[4096]; + result.ros_serialized &= + ros_view.SerializeToROSArray(ros_output, sizeof(ros_output)).ok(); + + auto intrinsic_message = + foo::bar::phaser::RosIntrinsicMessage::CreateReadonly( + ros_intrinsic_payload.data(), ros_intrinsic_payload.size()); + const auto& intrinsic_view = intrinsic_message; + const phaser::RosHeaderView header = intrinsic_view.header.Get(); + result.values_match &= header.seq == 7; + result.values_match &= header.stamp.sec == 11; + result.values_match &= header.frame_id == "allocation-free-frame"; + + auto any_message = foo::bar::phaser::TestMessage::CreateReadonly( + any_payload.data(), any_payload.size()); + const auto& any_view = any_message; + result.values_match &= + any_view.any().Is(); + const auto embedded = + any_view.any().As(); + result.values_match &= embedded.str() == "embedded"; + result.values_match &= any_view.has_u3b(); + result.values_match &= any_view.u3b().str() == "oneof-message"; + result.protobuf_serialized &= + any_view.SerializeToArray(protobuf_output, sizeof(protobuf_output)); + + const auto dense_view = + foo::bar::phaser::HybridLookupMessage::CreateReadonly( + dense_payload.data(), dense_payload.size()); + const auto sparse_view = + foo::bar::phaser::SparseLookupMessage::CreateReadonly( + sparse_payload.data(), sparse_payload.size()); + result.values_match &= dense_view.dense_10() == 10; + result.values_match &= dense_view.sparse_1000() == 1000; + result.values_match &= sparse_view.field_1() == 1; + result.values_match &= sparse_view.field_10000() == 10000; + + return result; +} + +ExerciseResult ExerciseFixedOutput() { + ExerciseResult result; + alignas(std::max_align_t) std::array ros_storage{}; + alignas(std::max_align_t) std::array header_storage{}; + alignas(std::max_align_t) std::array any_storage{}; + std::array protobuf_output{}; + std::array ros_output{}; + + auto ros = foo::bar::phaser::RosCompileMessage::CreateMutable( + ros_storage.data(), ros_storage.size()); + ros.x = 42; + ros.name = "fixed-output"; + ros.flag = true; + ros.choice + .emplace< + foo::bar::phaser::RosCompileMessage::ChoiceCountAlternative>(17); + ros.names.push_back("first"); + ros.names.push_back("second"); + ros.inners.Add()->id = 7; + ros.fixed_names[0] = "array"; + ros.fixed_inners[0]->id = 8; + result.values_match &= ros.x.Get() == 42; + result.values_match &= ros.name.Get() == "fixed-output"; + result.values_match &= ros.Data() == ros_storage.data(); + result.values_match &= ros.Size() > 0; + result.protobuf_serialized &= + ros.SerializeToArray(protobuf_output.data(), protobuf_output.size()); + result.ros_serialized &= + ros.SerializeToROSArray(ros_output.data(), ros_output.size()).ok(); + + auto intrinsic = foo::bar::phaser::RosIntrinsicMessage::CreateMutable( + header_storage.data(), header_storage.size()); + { + auto header = intrinsic.header.Mutable(); + header.seq = 9; + header.stamp = ::ros::Time(21, 654); + header.frame_id = "map"; + } + result.values_match &= intrinsic.header.Get().frame_id == "map"; + result.protobuf_serialized &= intrinsic.SerializeToArray( + protobuf_output.data(), protobuf_output.size()); + result.ros_serialized &= + intrinsic.SerializeToROSArray(ros_output.data(), ros_output.size()).ok(); + + auto any = foo::bar::phaser::TestMessage::CreateMutable( + any_storage.data(), any_storage.size()); + any.set_x(5); + any.set_s("outer"); + any.add_vstr("one"); + any.add_vstr("two"); + any.add_vm()->set_str("nested"); + any.set_u2b("union"); + auto embedded = + any.mutable_any()->MutableAny(); + embedded.set_str("typed-any"); + result.values_match &= + any.any().Is() && + any.any().As().str() == "typed-any"; + result.protobuf_serialized &= + any.SerializeToArray(protobuf_output.data(), protobuf_output.size()); + + return result; +} + +ExerciseResult ExerciseWireDeserialization(std::string_view any_wire, + std::string_view protobuf_ros_wire, + std::string_view ros_wire) { + ExerciseResult result; + alignas(std::max_align_t) std::array message_storage{}; + alignas(std::max_align_t) std::array header_storage{}; + alignas(std::max_align_t) std::array ros_storage{}; + std::array protobuf_storage{}; + + auto parsed = foo::bar::phaser::TestMessage::CreateMutable( + message_storage.data(), message_storage.size()); + result.protobuf_serialized &= + parsed.ParseFromArray(any_wire.data(), any_wire.size()); + result.values_match &= + parsed.any().Is() && + parsed.any().As().str() == "wire-any"; + + ::phaser::ROSBuffer ros_output(ros_storage.data(), ros_storage.size()); + result.ros_serialized &= + foo::bar::phaser::RosCompileMessage::ProtobufToROS(protobuf_ros_wire, + ros_output) + .ok(); + result.values_match &= !ros_output.empty(); + + auto intrinsic = foo::bar::phaser::RosIntrinsicMessage::CreateMutable( + header_storage.data(), header_storage.size()); + result.ros_serialized &= + intrinsic + .ParseFromROS( + absl::Span(ros_wire.data(), ros_wire.size())) + .ok(); + result.values_match &= intrinsic.header.Get().frame_id == "wire-map"; + + ::phaser::ProtoBuffer protobuf_output(protobuf_storage.data(), + protobuf_storage.size()); + result.protobuf_serialized &= + foo::bar::phaser::RosIntrinsicMessage::ROSToProtobuf( + absl::Span(ros_wire.data(), ros_wire.size()), + protobuf_output) + .ok(); + result.values_match &= protobuf_output.Size() > 0; + return result; +} + +} // namespace + +void* operator new(size_t size) { return Allocate(size); } +void* operator new[](size_t size) { return Allocate(size); } +void* operator new(size_t size, const std::nothrow_t&) noexcept { + try { + return Allocate(size); + } catch (...) { + return nullptr; + } +} +void* operator new[](size_t size, const std::nothrow_t&) noexcept { + try { + return Allocate(size); + } catch (...) { + return nullptr; + } +} +void* operator new(size_t size, std::align_val_t alignment) { + return AllocateAligned(size, static_cast(alignment)); +} +void* operator new[](size_t size, std::align_val_t alignment) { + return AllocateAligned(size, static_cast(alignment)); +} +void operator delete(void* ptr) noexcept { std::free(ptr); } +void operator delete[](void* ptr) noexcept { std::free(ptr); } +void operator delete(void* ptr, size_t) noexcept { std::free(ptr); } +void operator delete[](void* ptr, size_t) noexcept { std::free(ptr); } +void operator delete(void* ptr, const std::nothrow_t&) noexcept { + std::free(ptr); +} +void operator delete[](void* ptr, const std::nothrow_t&) noexcept { + std::free(ptr); +} +void operator delete(void* ptr, std::align_val_t) noexcept { std::free(ptr); } +void operator delete[](void* ptr, std::align_val_t) noexcept { + std::free(ptr); +} +void operator delete(void* ptr, size_t, std::align_val_t) noexcept { + std::free(ptr); +} +void operator delete[](void* ptr, size_t, std::align_val_t) noexcept { + std::free(ptr); +} + +TEST(ReceiveAllocationTest, TypedNativeReceiveAndSerializationAllocateNothing) { + foo::bar::phaser::RosCompileMessage ros_message; + ros_message.x = 42; + ros_message.name = "root"; + ros_message.choice.emplace< + foo::bar::phaser::RosCompileMessage::ChoiceCountAlternative>(17); + ros_message.names.push_back("first"); + ros_message.names.push_back("second"); + ros_message.inners.Add()->id = 1; + ros_message.inners.Add()->id = 2; + ros_message.fixed_names[0] = "fixed"; + ros_message.fixed_inners[0]->id = 9; + const std::vector ros_payload = CopyPayload(ros_message); + + foo::bar::phaser::RosIntrinsicMessage intrinsic_message; + intrinsic_message.header->seq = 7; + intrinsic_message.header->stamp = ::ros::Time(11, 13); + intrinsic_message.header->frame_id = "allocation-free-frame"; + const std::vector intrinsic_payload = CopyPayload(intrinsic_message); + + foo::bar::phaser::TestMessage any_message; + auto embedded = + any_message.mutable_any()->MutableAny(); + embedded.set_str("embedded"); + any_message.mutable_u3b()->set_str("oneof-message"); + const std::vector any_payload = CopyPayload(any_message); + + foo::bar::phaser::HybridLookupMessage dense_message; + dense_message.set_dense_10(10); + dense_message.set_sparse_1000(1000); + const std::vector dense_payload = CopyPayload(dense_message); + foo::bar::phaser::SparseLookupMessage sparse_message; + sparse_message.set_field_1(1); + sparse_message.set_field_10000(10000); + const std::vector sparse_payload = CopyPayload(sparse_message); + + // Prime process-wide bank and status internals before measuring the receive + // path itself. + ExerciseResult warmup = + Exercise(ros_payload, intrinsic_payload, any_payload, dense_payload, + sparse_payload); + ASSERT_TRUE(warmup.values_match); + ASSERT_TRUE(warmup.protobuf_serialized); + ASSERT_TRUE(warmup.ros_serialized); + + g_allocation_count.store(0, std::memory_order_relaxed); + g_count_allocations.store(true, std::memory_order_relaxed); + ExerciseResult measured = + Exercise(ros_payload, intrinsic_payload, any_payload, dense_payload, + sparse_payload); + g_count_allocations.store(false, std::memory_order_relaxed); + + EXPECT_TRUE(measured.values_match); + EXPECT_TRUE(measured.protobuf_serialized); + EXPECT_TRUE(measured.ros_serialized); + EXPECT_EQ(g_allocation_count.load(std::memory_order_relaxed), 0u); +} + +TEST(OutputAllocationTest, FixedBufferTypedMutationAndSerializationAllocateNothing) { + ExerciseResult warmup = ExerciseFixedOutput(); + ASSERT_TRUE(warmup.values_match); + ASSERT_TRUE(warmup.protobuf_serialized); + ASSERT_TRUE(warmup.ros_serialized); + + g_allocation_count.store(0, std::memory_order_relaxed); + g_count_allocations.store(true, std::memory_order_relaxed); + ExerciseResult measured = ExerciseFixedOutput(); + g_count_allocations.store(false, std::memory_order_relaxed); + + EXPECT_TRUE(measured.values_match); + EXPECT_TRUE(measured.protobuf_serialized); + EXPECT_TRUE(measured.ros_serialized); + EXPECT_EQ(g_allocation_count.load(std::memory_order_relaxed), 0u); +} + +TEST(OutputAllocationTest, + AnyDeserializationAndProtobufToROSAllocateNothingWithFixedBuffers) { + foo::bar::phaser::TestMessage any_source; + any_source.mutable_any() + ->MutableAny() + .set_str("wire-any"); + const std::string any_wire = any_source.SerializeAsString(); + + foo::bar::phaser::RosCompileMessage ros_source; + ros_source.x = 42; + ros_source.name = "protobuf-to-ros"; + ros_source.names.push_back("one"); + ros_source.inners.Add()->id = 7; + const std::string protobuf_ros_wire = ros_source.SerializeAsString(); + + foo::bar::phaser::RosIntrinsicMessage intrinsic_source; + intrinsic_source.stamp = ::ros::Time(12, 34); + { + auto header = intrinsic_source.header.Mutable(); + header.seq = 9; + header.stamp = ::ros::Time(21, 654); + header.frame_id = "wire-map"; + } + std::string ros_wire; + ASSERT_TRUE(intrinsic_source.SerializeToROSString(&ros_wire).ok()); + + ExerciseResult warmup = + ExerciseWireDeserialization(any_wire, protobuf_ros_wire, ros_wire); + ASSERT_TRUE(warmup.values_match); + ASSERT_TRUE(warmup.protobuf_serialized); + ASSERT_TRUE(warmup.ros_serialized); + + g_allocation_count.store(0, std::memory_order_relaxed); + g_count_allocations.store(true, std::memory_order_relaxed); + ExerciseResult measured = + ExerciseWireDeserialization(any_wire, protobuf_ros_wire, ros_wire); + g_count_allocations.store(false, std::memory_order_relaxed); + + EXPECT_TRUE(measured.values_match); + EXPECT_TRUE(measured.protobuf_serialized); + EXPECT_TRUE(measured.ros_serialized); + EXPECT_EQ(g_allocation_count.load(std::memory_order_relaxed), 0u); +} + +TEST(OutputAllocationTest, RuntimeControlPreservesUserMetadataAndHandleLifetime) { + alignas(std::max_align_t) std::array storage{}; + auto message = foo::bar::phaser::TestMessage::CreateMutable( + storage.data(), storage.size()); + void* user_data = message.Allocate(sizeof(uint32_t)); + *static_cast(user_data) = 0x12345678; + ASSERT_TRUE(message.SetUserMetadata(message.ToOffset(user_data)).ok()); + EXPECT_EQ(*static_cast(message.GetUserMetadata()), 0x12345678u); + + auto copy = message; + auto moved = std::move(copy); + moved.set_x(42); + EXPECT_EQ(message.x(), 42); + EXPECT_TRUE(moved.runtime->GetRuntimeControl() != nullptr); +} + +TEST(OutputAllocationTest, MetadataEntriesAreReusedAndTypedAnyCanGrowTable) { + alignas(std::max_align_t) std::array storage{}; + auto fixed = foo::bar::phaser::TestMessage::CreateMutable( + storage.data(), storage.size()); + fixed.add_vm()->set_str("first"); + fixed.add_vm()->set_str("second"); + ASSERT_NE(fixed.runtime->GetRuntimeControl(), nullptr); + EXPECT_EQ(fixed.runtime->GetRuntimeControl()->count, 2u); + + foo::bar::phaser::TestMessage dynamic; + const uint32_t initial_capacity = + dynamic.runtime->GetRuntimeControl()->capacity; + dynamic.mutable_any() + ->MutableAny() + .name = "first-any-type"; + dynamic.mutable_any() + ->MutableAny() + .name = "second-any-type"; + dynamic.mutable_any() + ->MutableAny() + .set_dense_10(1); + EXPECT_GT(dynamic.runtime->GetRuntimeControl()->capacity, initial_capacity); +} + +TEST(OutputAllocationTest, LegacyMetadataSlotRemainsReadable) { + alignas(std::max_align_t) std::array storage{}; + auto message = foo::bar::phaser::TestMessage::CreateMutable( + storage.data(), storage.size()); + void* user_data = message.Allocate(sizeof(uint32_t)); + *static_cast(user_data) = 99; + message.runtime->pb->metadata = message.ToOffset(user_data); + EXPECT_EQ(*static_cast(message.GetUserMetadata()), 99u); +} + +TEST(OutputAllocationTest, ReadonlyMutationIsRejected) { + alignas(std::max_align_t) std::array storage{}; + auto mutable_message = foo::bar::phaser::TestMessage::CreateMutable( + storage.data(), storage.size()); + mutable_message.set_x(7); + auto readonly = foo::bar::phaser::TestMessage::CreateReadonly( + mutable_message.Data(), mutable_message.Size()); + EXPECT_THROW(readonly.set_x(8), std::logic_error); + EXPECT_EQ(readonly.x(), 7); +} + +TEST(ReceiveAllocationTest, ReturnedViewsDependOnlyOnReceiveBuffer) { + foo::bar::phaser::RosCompileMessage source; + source.names.push_back("borrowed-name"); + source.inners.Add()->id = 23; + std::vector payload = CopyPayload(source); + + std::string_view borrowed_name; + auto borrowed_inner = [&]() { + auto root = foo::bar::phaser::RosCompileMessage::CreateReadonly( + payload.data(), payload.size()); + borrowed_name = root.names[0]; + return root.inners[0]; + }(); + + auto copied_inner = borrowed_inner; + auto moved_inner = std::move(copied_inner); + EXPECT_EQ(borrowed_name, "borrowed-name"); + EXPECT_EQ(borrowed_inner.id.Get(), 23); + EXPECT_EQ(moved_inner.id.Get(), 23); +} diff --git a/phaser/ros_compile_test.cc b/phaser/ros_compile_test.cc index de39c98..16e2730 100644 --- a/phaser/ros_compile_test.cc +++ b/phaser/ros_compile_test.cc @@ -137,8 +137,8 @@ TEST(RosCompileTest, StringVectorSyntax) { EXPECT_EQ(msg.names[0].Get(), "beta"); size_t count = 0; - for (const auto& s : msg.names) { - EXPECT_FALSE(s.Get().empty()); + for (std::string_view s : msg.names) { + EXPECT_FALSE(s.empty()); ++count; } EXPECT_EQ(count, 2u); @@ -146,9 +146,9 @@ TEST(RosCompileTest, StringVectorSyntax) { TEST(RosCompileTest, MessageVectorSyntax) { RosCompileMessage msg; - RosInner* a = msg.inners.Add(); + auto a = msg.inners.Add(); a->id = 1; - RosInner* b = msg.inners.Add(); + auto b = msg.inners.Add(); b->id = 2; EXPECT_EQ(msg.inners.size(), 2u); EXPECT_EQ(msg.inners[0]->id.Get(), 1); @@ -158,7 +158,7 @@ TEST(RosCompileTest, MessageVectorSyntax) { EXPECT_EQ(msg.inners.front()->id.Get(), 11); size_t count = 0; - for (auto& elem : msg.inners) { + for (auto elem : msg.inners) { EXPECT_TRUE(elem->id.IsPresent()); ++count; } @@ -393,7 +393,7 @@ TEST(RosCompileTest, FixedMessageArrayExtent) { EXPECT_EQ(msg.fixed_inners[1]->id.Get(), 8); size_t count = 0; - for (auto& inner : msg.fixed_inners) { + for (auto inner : msg.fixed_inners) { EXPECT_TRUE(inner->id.IsPresent()); ++count; } @@ -487,7 +487,7 @@ TEST(RosCompileTest, FixedArrayConstViewAfterWireParse) { const RosCompileMessage& view = parsed; EXPECT_EQ(view.fixed_ints.size(), 4u); EXPECT_EQ(view.fixed_ints[1], 22); - EXPECT_EQ(view.fixed_names[0].Get(), "readonly"); + EXPECT_EQ(view.fixed_names[0], "readonly"); } TEST(RosCompileTest, FixedArrayCreateReadonlyConstAccess) { @@ -508,8 +508,8 @@ TEST(RosCompileTest, FixedArrayCreateReadonlyConstAccess) { EXPECT_EQ(view.fixed_ints[0], 10); EXPECT_EQ(view.fixed_ints[1], 0); EXPECT_EQ(view.fixed_ints[2], 30); - EXPECT_EQ(view.fixed_names[0].Get(), ""); - EXPECT_EQ(view.fixed_names[1].Get(), "ro"); + EXPECT_EQ(view.fixed_names[0], ""); + EXPECT_EQ(view.fixed_names[1], "ro"); EXPECT_EQ(view.fixed_inners[0]->id.Get(), 4); EXPECT_FALSE(view.fixed_inners[1]->id.IsPresent()); @@ -539,7 +539,7 @@ TEST(RosCompileTest, FixedArrayReadonlyShortBufferDefaults) { EXPECT_EQ(view.fixed_ints[0], 7); EXPECT_EQ(view.fixed_ints[1], 0); EXPECT_EQ(view.fixed_ints[3], 0); - EXPECT_TRUE(view.fixed_names[0].Get().empty()); + EXPECT_TRUE(view.fixed_names[0].empty()); (void)partial; } diff --git a/phaser/ros_intrinsics_test.cc b/phaser/ros_intrinsics_test.cc index 5441ad3..ab7c75f 100644 --- a/phaser/ros_intrinsics_test.cc +++ b/phaser/ros_intrinsics_test.cc @@ -19,7 +19,9 @@ void MutateDuration(::ros::Duration& value) { value.nsec = 500; } -void MutateHeader(::std_msgs::Header& value) { +template +void MutateHeader(HeaderField& field) { + auto value = field.Mutable(); value.seq = 9; value.stamp = ::ros::Time(21, 654); value.frame_id = "map"; @@ -28,8 +30,9 @@ void MutateHeader(::std_msgs::Header& value) { uint32_t ReadSeconds(const ::ros::Time& value) { return value.sec; } uint32_t ReadSecondsByValue(::ros::Time value) { return value.sec; } -std::string ReadFrame(const ::std_msgs::Header& value) { - return value.frame_id; +template +std::string ReadFrame(const HeaderField& value) { + return std::string(value.Get().frame_id); } std::string ReadFrameByValue(::std_msgs::Header value) { return value.frame_id; @@ -47,7 +50,7 @@ TEST(RosIntrinsicsTest, ExistingMutableReferenceFunctionsWorkUnchanged) { EXPECT_EQ(message.stamp->nsec, 345u); EXPECT_EQ(message.timeout->sec, -4); EXPECT_EQ(ReadFrame(message.header), "map"); - EXPECT_EQ(ReadFrameByValue(message.header), "map"); + EXPECT_EQ(ReadFrameByValue(message.header.ToOwned()), "map"); EXPECT_EQ(message.header->stamp.sec, 21u); } @@ -71,7 +74,7 @@ TEST(RosIntrinsicsTest, NativePayloadAccessFlushesMutableBorrows) { EXPECT_EQ(view.timeout->nsec, 500); EXPECT_EQ(view.header->seq, 9u); EXPECT_EQ(view.header->stamp.sec, 21u); - EXPECT_EQ(ReadFrame(view.header), "map"); + EXPECT_EQ(view.header.Get().frame_id, "map"); } TEST(RosIntrinsicsTest, ProtobufWireRoundtripFlushesMutableBorrows) { diff --git a/phaser/ros_native_frontend_compatibility_test.cc b/phaser/ros_native_frontend_compatibility_test.cc index b2836f3..e1eb4de 100644 --- a/phaser/ros_native_frontend_compatibility_test.cc +++ b/phaser/ros_native_frontend_compatibility_test.cc @@ -31,10 +31,10 @@ TEST(RosNativeFrontendCompatibilityTest, ros_message.tags.push_back("front"); ros_message.tags.push_back("rear"); - auto* first_child = ros_message.children.Add(); + auto first_child = ros_message.children.Add(); first_child->id = 101; first_child->label = "left"; - auto* second_child = ros_message.children.Add(); + auto second_child = ros_message.children.Add(); second_child->id = 202; second_child->label = "right"; diff --git a/phaser/ros_wire_conversion_test.cc b/phaser/ros_wire_conversion_test.cc index f49fe24..7ff9df0 100644 --- a/phaser/ros_wire_conversion_test.cc +++ b/phaser/ros_wire_conversion_test.cc @@ -23,6 +23,7 @@ namespace { using RosCompileMessage = ::foo::bar::phaser::RosCompileMessage; using RosInner = ::foo::bar::phaser::RosInner; using RosIntrinsicMessage = ::foo::bar::phaser::RosIntrinsicMessage; +using RosPackedFixedMessage = ::foo::bar::phaser::RosPackedFixedMessage; using ProtobufFrontendIntrinsicMessage = ::foo::bar::pb::protobuf_phaser::RosIntrinsicMessage; using RosColor = ::foo::bar::phaser::RosColor; @@ -47,6 +48,13 @@ void AppendDouble(std::string& bytes, double value) { AppendIntegral(bytes, bits); } +void AppendFloat(std::string& bytes, float value) { + uint32_t bits = 0; + static_assert(sizeof(bits) == sizeof(value)); + std::memcpy(&bits, &value, sizeof(bits)); + AppendIntegral(bytes, bits); +} + void AppendString(std::string& bytes, std::string_view value) { AppendIntegral(bytes, static_cast(value.size())); bytes.append(value); @@ -207,6 +215,14 @@ TEST(ROSWireConversionTest, LiveProtobufAndNativePathsMatchKnownBytes) { ASSERT_TRUE(RosCompileMessage::ProtobufToROS(protobuf_wire, protobuf_output) .ok()); EXPECT_EQ(protobuf_output.AsString(), expected); + std::vector exact_output(expected.size()); + ::phaser::ROSBuffer fixed_protobuf_output(exact_output.data(), + exact_output.size()); + ASSERT_TRUE( + RosCompileMessage::ProtobufToROS(protobuf_wire, fixed_protobuf_output) + .ok()); + EXPECT_EQ(fixed_protobuf_output.AsSpan(), + absl::Span(expected.data(), expected.size())); ::phaser::ROSBuffer native_output; const auto* native_data = @@ -235,6 +251,73 @@ TEST(ROSWireConversionTest, LiveProtobufAndNativePathsMatchKnownBytes) { EXPECT_EQ(inferred_native_output.AsString(), expected); } +TEST(ROSWireConversionTest, DirectPackedFixedFieldsUseCompatibleRawLayout) { + ::foo::bar::RosPackedFixedMessage protobuf; + protobuf.add_fixed32_values(0x01020304u); + protobuf.add_fixed32_values(0xfedcba98u); + protobuf.add_sfixed32_values(-1); + protobuf.add_sfixed32_values(-1234567); + protobuf.add_float_values(1.5f); + protobuf.add_float_values(-0.0f); + protobuf.add_fixed64_values(0x0102030405060708ULL); + protobuf.add_sfixed64_values(-1234567890123LL); + protobuf.add_double_values(1.25); + protobuf.add_double_values(-2.5); + protobuf.add_fixed_array(0xfedcba9876543210ULL); + protobuf.add_fixed_array(0); + protobuf.add_fixed_array(0x1122334455667788ULL); + + std::string expected; + AppendIntegral(expected, uint32_t{2}); + AppendIntegral(expected, uint32_t{0x01020304}); + AppendIntegral(expected, uint32_t{0xfedcba98}); + AppendIntegral(expected, uint32_t{2}); + AppendIntegral(expected, int32_t{-1}); + AppendIntegral(expected, int32_t{-1234567}); + AppendIntegral(expected, uint32_t{2}); + AppendFloat(expected, 1.5f); + AppendFloat(expected, -0.0f); + AppendIntegral(expected, uint32_t{1}); + AppendIntegral(expected, uint64_t{0x0102030405060708}); + AppendIntegral(expected, uint32_t{1}); + AppendIntegral(expected, int64_t{-1234567890123LL}); + AppendIntegral(expected, uint32_t{2}); + AppendDouble(expected, 1.25); + AppendDouble(expected, -2.5); + AppendIntegral(expected, uint64_t{0xfedcba9876543210}); + AppendIntegral(expected, uint64_t{0}); + AppendIntegral(expected, uint64_t{0x1122334455667788}); + + ::phaser::ROSBuffer ros_output; + ASSERT_TRUE(RosPackedFixedMessage::ProtobufToROS( + protobuf.SerializeAsString(), ros_output) + .ok()); + EXPECT_EQ(ros_output.AsString(), expected); + + std::vector protobuf_storage(1024); + ::phaser::ProtoBuffer protobuf_output(protobuf_storage.data(), + protobuf_storage.size()); + ASSERT_TRUE(RosPackedFixedMessage::ROSToProtobuf( + absl::Span(expected.data(), expected.size()), + protobuf_output) + .ok()); + ::foo::bar::RosPackedFixedMessage reparsed; + ASSERT_TRUE(reparsed.ParseFromArray(protobuf_storage.data(), + protobuf_output.Size())); + EXPECT_EQ(reparsed.SerializeAsString(), protobuf.SerializeAsString()); +} + +TEST(ROSWireConversionTest, DirectPackedFixedFieldRejectsPartialElement) { + std::string malformed; + malformed.push_back(static_cast(0x0a)); // field 1, packed + malformed.push_back(static_cast(0x03)); + malformed.append("\x01\x02\x03", 3); + + ::phaser::ROSBuffer output; + EXPECT_FALSE( + RosPackedFixedMessage::ProtobufToROS(malformed, output).ok()); +} + TEST(ROSWireConversionTest, FixedOutputAndErrorsAreReported) { RosCompileMessage message; PopulatePhaserMessage(message); @@ -344,6 +427,17 @@ TEST(ROSWireConversionTest, ParsesKnownROSBytesIntoNativePayload) { EXPECT_EQ(protobuf.xs(1), -20); EXPECT_EQ(protobuf.fixed_inners(1).id(), 400); EXPECT_EQ(protobuf.choice_name(), "selected"); + + std::vector direct_wire(4096); + ::phaser::ProtoBuffer direct_output(direct_wire.data(), direct_wire.size()); + ASSERT_TRUE(RosCompileMessage::ROSToProtobuf( + absl::Span(input.data(), input.size()), + direct_output) + .ok()); + ::foo::bar::RosCompileMessage direct_protobuf; + ASSERT_TRUE(direct_protobuf.ParseFromArray(direct_wire.data(), + direct_output.Size())); + EXPECT_EQ(direct_protobuf.SerializeAsString(), protobuf.SerializeAsString()); } TEST(ROSWireConversionTest, ParsedROSPayloadUsesEitherFrontend) { @@ -389,6 +483,19 @@ TEST(ROSWireConversionTest, ParsedROSPayloadUsesEitherFrontend) { EXPECT_EQ(parsed_protobuf_frontend.timeout().nanos(), 500); EXPECT_EQ(parsed_protobuf_frontend.header().stamp().nanos(), 654); EXPECT_EQ(parsed_protobuf_frontend.header().frame_id(), "map"); + + std::vector direct_wire(4096); + ::phaser::ProtoBuffer direct_output(direct_wire.data(), direct_wire.size()); + ASSERT_TRUE(RosIntrinsicMessage::ROSToProtobuf( + absl::Span(input.data(), input.size()), + direct_output) + .ok()); + ProtobufFrontendIntrinsicMessage direct_protobuf; + ASSERT_TRUE(direct_protobuf.ParseFromArray(direct_wire.data(), + direct_output.Size())); + EXPECT_EQ(direct_protobuf.stamp().seconds(), 12); + EXPECT_EQ(direct_protobuf.timeout().nanos(), 500); + EXPECT_EQ(direct_protobuf.header().frame_id(), "map"); } TEST(ROSWireConversionTest, ParsesScalarAndMessageOneofArms) { diff --git a/phaser/runtime/any.h b/phaser/runtime/any.h index 81e0873..5ec0f45 100644 --- a/phaser/runtime/any.h +++ b/phaser/runtime/any.h @@ -15,6 +15,7 @@ // #include +#include #include #include #include @@ -58,10 +59,14 @@ class AnyMessage : public Message { {.number = 1, .offset = 4, .id = 0}, {.number = 2, .offset = 8, .id = 0}, }}; - static std::string Name() { return "Any"; } - static std::string FullName() { return "google.protobuf.Any"; } - std::string GetName() const override { return Name(); } - std::string GetFullName() const override { return FullName(); } + static constexpr std::string_view Name() { return "Any"; } + static constexpr std::string_view FullName() { + return "google.protobuf.Any"; + } + std::string GetName() const override { return std::string(Name()); } + std::string GetFullName() const override { + return std::string(FullName()); + } friend std::ostream& operator<<(std::ostream& os, const AnyMessage& msg); @@ -109,13 +114,9 @@ class AnyMessage : public Message { size += type_url_.SerializedSize(); } if (value_.IsPresent()) { - absl::StatusOr embedded = EmbeddedMessageForWire(); - if (!embedded.ok()) { - return 0; - } - std::unique_ptr embedded_owner(*embedded); - absl::StatusOr inner_size = - PhaserBankSerializedSize(MessageTypeName(), **embedded); + absl::StatusOr inner_size = PhaserBankSerializedSizeAtOffset( + MessageTypeName(), runtime, + runtime->ToOffset(const_cast(value().data()))); if (!inner_size.ok()) { return 0; } @@ -131,14 +132,11 @@ class AnyMessage : public Message { } } if (value_.IsPresent()) { - absl::StatusOr embedded = EmbeddedMessageForWire(); - if (!embedded.ok()) { - return embedded.status(); - } - std::unique_ptr embedded_owner(*embedded); - const std::string type = MessageTypeName(); + const std::string_view type = MessageTypeName(); + const toolbelt::BufferOffset offset = + runtime->ToOffset(const_cast(value().data())); absl::StatusOr inner_size = - PhaserBankSerializedSize(type, **embedded); + PhaserBankSerializedSizeAtOffset(type, runtime, offset); if (!inner_size.ok()) { return inner_size.status(); } @@ -148,7 +146,7 @@ class AnyMessage : public Message { return status; } if (absl::Status status = - PhaserBankSerializeToBuffer(type, **embedded, buffer); + PhaserBankSerializeAtOffset(type, runtime, offset, buffer); !status.ok()) { return status; } @@ -160,8 +158,8 @@ class AnyMessage : public Message { clear_type_url(); clear_value(); - std::optional pending_type_url; - std::optional pending_wire_value; + std::optional pending_type_url; + std::optional> pending_wire_value; while (!buffer.Eof()) { absl::StatusOr tag = @@ -176,7 +174,7 @@ class AnyMessage : public Message { if (!url.ok()) { return url.status(); } - pending_type_url = std::string(*url); + pending_type_url = *url; break; } case 2: { @@ -185,8 +183,7 @@ class AnyMessage : public Message { if (!wire.ok()) { return wire.status(); } - pending_wire_value = - std::string(wire->data(), wire->data() + wire->size()); + pending_wire_value = *wire; break; } default: @@ -200,16 +197,18 @@ class AnyMessage : public Message { type_url_.Set(*pending_type_url); } if (pending_wire_value) { - return MaterializeValueFromProtobufWire(*pending_wire_value); + return MaterializeValueFromProtobufWire( + std::string_view(pending_wire_value->data(), + pending_wire_value->size())); } return absl::OkStatus(); } - std::string MessageTypeName() const { - std::string type = std::string(type_url()); + std::string_view MessageTypeName() const { + std::string_view type = type_url(); size_t pos = type.find('/'); if (pos != std::string::npos) { - return type.substr(pos + 1); + type.remove_prefix(pos + 1); } return type; } @@ -223,7 +222,7 @@ class AnyMessage : public Message { return absl::FailedPreconditionError( "Any value without type_url cannot be cloned"); } - const std::string type = msg.MessageTypeName(); + const std::string type(msg.MessageTypeName()); absl::StatusOr dest = AllocateEmbeddedMessage(type); if (!dest.ok()) { return dest.status(); @@ -250,7 +249,12 @@ class AnyMessage : public Message { // the value. template T MutableAny() { - set_type_url("type.googleapis.com/" + T::FullName()); + constexpr std::string_view prefix = "type.googleapis.com/"; + constexpr std::string_view full_name = T::FullName(); + absl::Span url = + type_url_.Allocate(prefix.size() + full_name.size(), false); + memcpy(url.data(), prefix.data(), prefix.size()); + memcpy(url.data() + prefix.size(), full_name.data(), full_name.size()); size_t size = T::BinarySize(); absl::Span memory = value_.Allocate(size, true); auto msg = T(runtime, runtime->ToOffset(memory.data())); @@ -275,24 +279,22 @@ class AnyMessage : public Message { template bool UnpackTo(T* msg) const { const char* addr = value().data(); - std::unique_ptr embedded_msg = std::make_unique( - runtime, runtime->ToOffset(const_cast(addr))); - return msg->CloneFrom(*embedded_msg).ok(); + const T embedded_msg(runtime, + runtime->ToOffset(const_cast(addr))); + return msg->CloneFrom(embedded_msg).ok(); } template absl::Status UnpackToOrStatus(T* msg) const { const char* addr = value().data(); - std::unique_ptr embedded_msg = std::make_unique( - runtime, runtime->ToOffset(const_cast(addr))); - return msg->CloneFrom(*embedded_msg); + const T embedded_msg(runtime, + runtime->ToOffset(const_cast(addr))); + return msg->CloneFrom(embedded_msg); } template bool Is() const { - const std::string& msg_type = T::FullName(); - std::string t = MessageTypeName(); - return t == msg_type; + return MessageTypeName() == T::FullName(); } // Gets the value of the any field as a message of type T. This does not @@ -343,15 +345,6 @@ class AnyMessage : public Message { private: static constexpr int kValueFieldNumber = 2; - absl::StatusOr EmbeddedMessageForWire() const { - if (!has_type_url() || !has_value()) { - return absl::FailedPreconditionError( - "Any is missing type_url or embedded value"); - } - const std::string type = MessageTypeName(); - return PhaserBankMakeExisting(type, runtime, value().data()); - } - absl::StatusOr AllocateEmbeddedMessage(const std::string& type) { absl::StatusOr binary_size = PhaserBankBinarySize(type); if (!binary_size.ok()) { @@ -362,18 +355,19 @@ class AnyMessage : public Message { runtime->ToOffset(memory.data())); } - absl::Status MaterializeValueFromProtobufWire(const std::string& wire) { + absl::Status MaterializeValueFromProtobufWire(std::string_view wire) { if (!has_type_url()) { return absl::InvalidArgumentError("Any value on wire requires type_url"); } - const std::string type = MessageTypeName(); - absl::StatusOr embedded = AllocateEmbeddedMessage(type); - if (!embedded.ok()) { - return embedded.status(); + const std::string_view type = MessageTypeName(); + absl::StatusOr binary_size = PhaserBankBinarySize(type); + if (!binary_size.ok()) { + return binary_size.status(); } - std::unique_ptr embedded_owner(*embedded); + absl::Span memory = value_.Allocate(*binary_size, true); ProtoBuffer sub(wire); - return PhaserBankDeserializeFromBuffer(type, **embedded, sub); + return PhaserBankDeserializeAtOffset( + type, runtime, runtime->ToOffset(memory.data()), sub); } phaser::StringField type_url_; diff --git a/phaser/runtime/arrays.h b/phaser/runtime/arrays.h index 0d93ac0..9f19db0 100644 --- a/phaser/runtime/arrays.h +++ b/phaser/runtime/arrays.h @@ -30,8 +30,7 @@ namespace phaser { inline bool HasMutablePayload( const std::shared_ptr& runtime) { - return runtime != nullptr && - dynamic_cast(runtime.get()) != nullptr; + return runtime != nullptr && runtime->IsMutable(); } template @@ -900,16 +899,16 @@ class MessageArrayField : public MessageVectorField { MessageArrayField(const MessageArrayField&) = default; MessageArrayField(MessageArrayField&&) = default; - const MessageObject& operator[](size_t index) const { - return ConstObject(index); + T operator[](size_t index) const { + return MessageVectorField::operator[](static_cast(index)); } - MessageObject& operator[](size_t index) { return MutableObject(index); } + T operator[](size_t index) { return MutableObject(index); } - MessageObject& front() { return (*this)[0]; } - const MessageObject& front() const { return (*this)[0]; } - MessageObject& back() { return (*this)[N - 1]; } - const MessageObject& back() const { return (*this)[N - 1]; } + T front() { return (*this)[0]; } + T front() const { return (*this)[0]; } + T back() { return (*this)[N - 1]; } + T back() const { return (*this)[N - 1]; } using typename MessageVectorField::iterator; using typename MessageVectorField::const_iterator; @@ -958,7 +957,7 @@ class MessageArrayField : public MessageVectorField { if (parsed_count_ > N) { return absl::InvalidArgumentError("array_size overflow"); } - while (MessageVectorField::Get().size() < N) { + while (MessageVectorField::size() < N) { MessageVectorField::Add(); } parsed_count_ = 0; @@ -984,7 +983,7 @@ class MessageArrayField : public MessageVectorField { } Clear(); for (size_t i = 0; i < N; i++) { - if (absl::Status s = MutableObject(i).Mutable()->CloneFrom(other[i].Get()); + if (absl::Status s = MutableObject(i).CloneFrom(other.Get(i)); !s.ok()) { return *this; } @@ -1001,15 +1000,9 @@ class MessageArrayField : public MessageVectorField { size_t max_size() const { return N; } bool empty() const { return N == 0; } - const T& Get(size_t index) const { - const MessageObject& obj = ConstObject(index); - if (obj.empty()) { - return EmptyObject().Get(); - } - return obj.Get(); - } + T Get(size_t index) const { return (*this)[index]; } - T* Mutable(size_t index) { return MutableObject(index).Mutable(); } + T Mutable(size_t index) { return MutableObject(index); } ::toolbelt::BufferOffset BinaryEndOffset() const { return MessageVectorField::BinaryEndOffset(); @@ -1020,7 +1013,7 @@ class MessageArrayField : public MessageVectorField { bool operator==(const MessageArrayField& other) const { for (size_t i = 0; i < N; i++) { - if ((*this)[i].Get() != other[i].Get()) { + if (Get(i) != other.Get(i)) { return false; } } @@ -1062,14 +1055,11 @@ class MessageArrayField : public MessageVectorField { if (!v.ok()) { return v.status(); } - T* msg = nullptr; - if (parsed_count_ < MessageVectorField::Get().size()) { - msg = MessageVectorField::Mutable(parsed_count_); - } else { - msg = MessageVectorField::Add(); - } + T msg = parsed_count_ < MessageVectorField::size() + ? MessageVectorField::Mutable(parsed_count_) + : MessageVectorField::Add(); ProtoBuffer msg_buffer(*v); - if (absl::Status status = msg->Deserialize(msg_buffer); !status.ok()) { + if (absl::Status status = msg.Deserialize(msg_buffer); !status.ok()) { return status; } parsed_count_++; @@ -1077,36 +1067,24 @@ class MessageArrayField : public MessageVectorField { } private: - static const MessageObject& EmptyObject() { - static const MessageObject empty; - return empty; - } - - const MessageObject& ConstObject(size_t index) const { - if (index >= N) { - return EmptyObject(); - } - return MessageVectorField::operator[](static_cast(index)); - } - - MessageObject& MutableObject(size_t index) { + T MutableObject(size_t index) { if (!HasMutablePayload(MessageVectorField::GetRuntime())) { - return const_cast&>(ConstObject(index)); + return MessageVectorField::operator[](static_cast(index)); } - while (MessageVectorField::Get().size() <= index) { + while (MessageVectorField::size() <= index) { MessageVectorField::Add(); } - while (MessageVectorField::Get().size() < N) { + while (MessageVectorField::size() < N) { MessageVectorField::Add(); } - return MessageVectorField::operator[](static_cast(index)); + return MessageVectorField::Mutable(index); } void EnsureExtent() { if (!HasMutablePayload(MessageVectorField::GetRuntime())) { return; } - while (MessageVectorField::Get().size() < N) { + while (MessageVectorField::size() < N) { MessageVectorField::Add(); } } @@ -1144,27 +1122,30 @@ class StringArrayField : public Field { relative_binary_offset_(other.relative_binary_offset_), parsed_count_(other.parsed_count_) {} - const NonEmbeddedStringField& operator[](size_t index) const { - return ConstSlot(index); - } + std::string_view operator[](size_t index) const { return Get(index); } - NonEmbeddedStringField& operator[](size_t index) { + NonEmbeddedStringField operator[](size_t index) { EnsureMutableExtent(); - return strings_[index]; + return ConstSlot(index); } - const NonEmbeddedStringField& front() const { return ConstSlot(0); } - NonEmbeddedStringField& front() { return (*this)[0]; } - const NonEmbeddedStringField& back() const { return ConstSlot(N - 1); } - NonEmbeddedStringField& back() { return (*this)[N - 1]; } + std::string_view front() const { return Get(0); } + NonEmbeddedStringField front() { return (*this)[0]; } + std::string_view back() const { return Get(N - 1); } + NonEmbeddedStringField back() { return (*this)[N - 1]; } using value_type = NonEmbeddedStringField; - using reference = value_type&; - using const_reference = value_type&; + using reference = NonEmbeddedStringField; + using const_reference = std::string_view; using size_type = size_t; using difference_type = ptrdiff_t; - using iterator = typename std::array::iterator; struct ConstIterator { + using iterator_category = std::bidirectional_iterator_tag; + using value_type = std::string_view; + using difference_type = ptrdiff_t; + using pointer = void; + using reference = std::string_view; + const StringArrayField* field = nullptr; size_t index = 0; @@ -1179,9 +1160,7 @@ class StringArrayField : public Field { --index; return *this; } - const NonEmbeddedStringField& operator*() const { - return field->ConstSlot(index); - } + std::string_view operator*() const { return field->Get(index); } bool operator==(const ConstIterator& it) const { return field == it.field && index == it.index; } @@ -1206,9 +1185,7 @@ class StringArrayField : public Field { ++index; return *this; } - const NonEmbeddedStringField& operator*() const { - return field->ConstSlot(index); - } + std::string_view operator*() const { return field->Get(index); } bool operator==(const ConstReverseIterator& it) const { return field == it.field && index == it.index; } @@ -1216,30 +1193,22 @@ class StringArrayField : public Field { return !operator==(it); } }; + using iterator = ConstIterator; using const_iterator = ConstIterator; - using reverse_iterator = - typename std::array::reverse_iterator; + using reverse_iterator = ConstReverseIterator; using const_reverse_iterator = ConstReverseIterator; - iterator begin() { - EnsureMutableExtent(); - return strings_.begin(); - } - iterator end() { - EnsureMutableExtent(); - return strings_.end(); - } + iterator begin() { return iterator(this, 0); } + iterator end() { return iterator(this, N); } const_iterator begin() const { return const_iterator(this, 0); } const_iterator end() const { return const_iterator(this, N); } const_iterator cbegin() const { return begin(); } const_iterator cend() const { return end(); } reverse_iterator rbegin() { - EnsureMutableExtent(); - return strings_.rbegin(); + return reverse_iterator(this, N == 0 ? static_cast(-1) : N - 1); } reverse_iterator rend() { - EnsureMutableExtent(); - return strings_.rend(); + return reverse_iterator(this, static_cast(-1)); } const_reverse_iterator rbegin() const { return const_reverse_iterator( @@ -1304,11 +1273,8 @@ class StringArrayField : public Field { size_t max_size() const { return N; } bool empty() const { return N == 0; } - NonEmbeddedStringField* data() { - EnsureMutableExtent(); - return strings_.data(); - } - const NonEmbeddedStringField* data() const { return strings_.data(); } + NonEmbeddedStringField* data() = delete; + const NonEmbeddedStringField* data() const = delete; std::string_view Get(size_t index) const { const NonEmbeddedStringField& slot = ConstSlot(index); diff --git a/phaser/runtime/fields.h b/phaser/runtime/fields.h index 0f7377a..0aef80a 100644 --- a/phaser/runtime/fields.h +++ b/phaser/runtime/fields.h @@ -62,19 +62,12 @@ class Field { int Number() const { return number_; } int32_t FindFieldOffset(uint32_t source_offset) const { - if (cached_offset_ == 0xffffffff) { - cached_offset_ = static_cast<::toolbelt::BufferOffset>( - Message::GetMessage(this, source_offset) - ->FindFieldOffset(static_cast(number_))); - } - return static_cast(cached_offset_); + ResolveField(source_offset); + return cached_offset_; } int32_t FindFieldId(uint32_t source_offset) const { - if (cached_field_id_ == -1) { - cached_field_id_ = Message::GetMessage(this, source_offset) - ->FindFieldId(static_cast(number_)); - } + ResolveField(source_offset); return cached_field_id_; } @@ -89,16 +82,39 @@ class Field { int GetIndent() const { return indent_; } protected: + void RequireMutable(uint32_t source_offset) const { + if (!Message::GetRuntime(this, source_offset)->IsMutable()) { + throw std::logic_error("cannot mutate a readonly Phaser message"); + } + } + + void ResolveField(uint32_t source_offset) const { + if (field_cache_resolved_) { + return; + } + const Message* message = Message::GetMessage(this, source_offset); + if (message->runtime == nullptr) { + return; + } + const FieldLocation location = + message->FindField(static_cast(number_)); + cached_offset_ = location.offset; + cached_field_id_ = location.id; + field_cache_resolved_ = true; + } + void ResetFieldCache() { - cached_offset_ = 0xffffffff; + cached_offset_ = -1; cached_field_id_ = -1; + field_cache_resolved_ = false; } protected: int id_ = 0; int number_ = 0; - mutable ::toolbelt::BufferOffset cached_offset_ = 0xffffffff; + mutable int32_t cached_offset_ = -1; mutable int32_t cached_field_id_ = -1; + mutable bool field_cache_resolved_ = false; mutable int indent_ = 0; }; @@ -150,10 +166,14 @@ class Field { } \ \ void Set(type v) { \ + RequireMutable(source_offset_); \ GetBuffer()->Set(GetMessageBinaryStart() + relative_binary_offset_, v); \ SetPresence(GetBuffer(), GetPresenceMaskStart()); \ } \ - void Clear() { ClearPresence(GetBuffer(), GetPresenceMaskStart()); } \ + void Clear() { \ + RequireMutable(source_offset_); \ + ClearPresence(GetBuffer(), GetPresenceMaskStart()); \ + } \ bool operator==(const cname##Field& other) const { \ return Get() == other.Get(); \ } \ @@ -522,18 +542,50 @@ class NonEmbeddedStringField { : msg_(msg), absolute_binary_offset_(absolute_binary_offset) {} NonEmbeddedStringField(const NonEmbeddedStringField&) = default; NonEmbeddedStringField(NonEmbeddedStringField&&) = default; - NonEmbeddedStringField& operator=(const NonEmbeddedStringField& other) { + NonEmbeddedStringField& operator=(const NonEmbeddedStringField& other) & { + return AssignValue(other); + } + NonEmbeddedStringField& operator=(const NonEmbeddedStringField& other) && { + return AssignValue(other); + } + NonEmbeddedStringField& operator=(NonEmbeddedStringField&& other) & noexcept { + msg_ = other.msg_; + absolute_binary_offset_ = other.absolute_binary_offset_; + return *this; + } + NonEmbeddedStringField& operator=(NonEmbeddedStringField&& other) && { + return AssignValue(other); + } + + private: + NonEmbeddedStringField& AssignValue(const NonEmbeddedStringField& other) { if (this == &other) { return *this; } if (other.IsPlaceholder()) { return *this; } - Set(other.Get()); + std::string_view value = other.Get(); + if (GetBuffer() != other.GetBuffer()) { + Set(value); + return *this; + } + if (value.empty()) { + Set(value); + return *this; + } + // Preserve an offset rather than an address: allocating the destination + // may relocate a dynamic payload, but offsets remain valid. + const ::toolbelt::BufferOffset source = + GetBuffer()->ToOffset(const_cast(value.data())); + absl::Span destination = ::toolbelt::PayloadBuffer::AllocateString( + GetBufferAddr(), value.size(), absolute_binary_offset_, false); + memmove(destination.data(), GetBuffer()->ToAddress(source), + value.size()); return *this; } - NonEmbeddedStringField& operator=(NonEmbeddedStringField&& other) noexcept = - default; + + public: operator std::string_view() const { return Get(); } NonEmbeddedStringField& operator=(const std::string& s) { Set(s); @@ -608,6 +660,9 @@ class NonEmbeddedStringField { ::toolbelt::PayloadBuffer* GetBuffer() const { return msg_->runtime->pb; } ::toolbelt::PayloadBuffer** GetBufferAddr() const { + if (!msg_->runtime->IsMutable()) { + throw std::logic_error("cannot mutate a readonly Phaser string"); + } return &msg_->runtime->pb; } diff --git a/phaser/runtime/message.cc b/phaser/runtime/message.cc index a11ddec..0f24e6f 100644 --- a/phaser/runtime/message.cc +++ b/phaser/runtime/message.cc @@ -10,73 +10,6 @@ namespace phaser { -int32_t Message::FindFieldOffset(uint32_t field_number) const { - if (runtime == nullptr) { - return -1; - } - // First 4 bytes of message are the the offset to the field data. - ::toolbelt::BufferOffset* field_data = - runtime->ToAddress<::toolbelt::BufferOffset>(absolute_binary_offset); - if (field_data == nullptr) { - return -1; - } - // Dereference offset to get a pointer to the field data (in the payload - // buffer)l - FieldData* fd = runtime->ToAddress(*field_data); - if (fd == nullptr) { - return -1; - } - // Search for number using binary search. This must be sorted by field - // number. - uint32_t left = 0; - uint32_t right = fd->num; - while (left < right) { - uint32_t mid = left + (right - left) / 2; - if (fd->fields[mid].number == field_number) { - return int32_t(fd->fields[mid].offset); - } else if (fd->fields[mid].number < field_number) { - left = mid + 1; - } else { - right = mid; - } - } - return -1; -} - -int32_t Message::FindFieldId(uint32_t field_number) const { - if (runtime == nullptr) { - return -1; - } - // First 4 bytes of message are the the offset to the field data. - ::toolbelt::BufferOffset* field_data = - runtime->ToAddress<::toolbelt::BufferOffset>(absolute_binary_offset); - - if (field_data == nullptr) { - return -1; - } - // Dereference offset to get a pointer to the field data (in the payload - // buffer)l - FieldData* fd = runtime->ToAddress(*field_data); - if (fd == nullptr) { - return -1; - } - // Search for number using binary search. This must be sorted by field - // number. - uint32_t left = 0; - uint32_t right = fd->num; - while (left < right) { - uint32_t mid = left + (right - left) / 2; - if (fd->fields[mid].number == field_number) { - return int32_t(fd->fields[mid].id); - } else if (fd->fields[mid].number < field_number) { - left = mid + 1; - } else { - right = mid; - } - } - return -1; -} - ::toolbelt::PayloadBuffer* NewDynamicBuffer(size_t initial_size, Tuning tuning) { absl::StatusOr<::toolbelt::PayloadBuffer*> r = NewDynamicBuffer( diff --git a/phaser/runtime/message.h b/phaser/runtime/message.h index b38b8b1..8b20d72 100644 --- a/phaser/runtime/message.h +++ b/phaser/runtime/message.h @@ -6,10 +6,15 @@ #include +#include +#include +#include #include #include #include +#include #include +#include #include "absl/container/flat_hash_map.h" #include "absl/status/status.h" @@ -40,6 +45,105 @@ struct FieldData { }; #pragma clang diagnostic pop +// Hybrid field metadata stores a compact direct-indexed range followed by +// sorted sparse entries. The arrays immediately following HybridFieldData are: +// FieldValue dense_fields[dense_span] +// SparseFieldData sparse_fields[sparse_count] +// A zero dense-field offset marks a field number absent from the schema. Real +// field offsets are nonzero because every message begins with its header. +inline constexpr uint32_t kHybridFieldDataMagic = 0x50484431; + +struct FieldValue { + uint32_t offset : 24; + uint32_t id : 8; +}; +static_assert(sizeof(FieldValue) == sizeof(uint32_t)); + +struct SparseFieldData { + uint32_t number; + uint32_t offset : 24; + uint32_t id : 8; +}; +static_assert(sizeof(SparseFieldData) == 2 * sizeof(uint32_t)); + +struct HybridFieldData { + uint32_t magic; + uint32_t dense_base; + uint32_t dense_span; + uint32_t sparse_count; +}; +static_assert(sizeof(HybridFieldData) == 4 * sizeof(uint32_t)); + +struct FieldLocation { + int32_t offset = -1; + int32_t id = -1; +}; + +namespace internal { + +inline FieldLocation FindLegacyField(const FieldData* field_data, + uint32_t field_number) { + uint32_t left = 0; + uint32_t right = field_data->num; + while (left < right) { + const uint32_t mid = left + (right - left) / 2; + if (field_data->fields[mid].number == field_number) { + return { + .offset = static_cast(field_data->fields[mid].offset), + .id = static_cast(field_data->fields[mid].id), + }; + } + if (field_data->fields[mid].number < field_number) { + left = mid + 1; + } else { + right = mid; + } + } + return {}; +} + +inline FieldLocation FindHybridField(const HybridFieldData* field_data, + uint32_t field_number) { + const auto* dense_fields = + reinterpret_cast(field_data + 1); + const auto* sparse_fields = reinterpret_cast( + dense_fields + field_data->dense_span); + + if (field_number >= field_data->dense_base) { + const uint32_t dense_index = field_number - field_data->dense_base; + if (dense_index < field_data->dense_span) { + const FieldValue& field = dense_fields[dense_index]; + if (field.offset == 0) { + return {}; + } + return { + .offset = static_cast(field.offset), + .id = static_cast(field.id), + }; + } + } + + uint32_t left = 0; + uint32_t right = field_data->sparse_count; + while (left < right) { + const uint32_t mid = left + (right - left) / 2; + if (sparse_fields[mid].number == field_number) { + return { + .offset = static_cast(sparse_fields[mid].offset), + .id = static_cast(sparse_fields[mid].id), + }; + } + if (sparse_fields[mid].number < field_number) { + left = mid + 1; + } else { + right = mid; + } + } + return {}; +} + +} // namespace internal + enum class FieldType { kFieldInt32, kFieldInt64, @@ -106,14 +210,64 @@ struct MessageInfo { std::vector> fields_in_order; }; -// Each message contains a std::shared_ptr to one of these, allocated from -// the heap. This is used when creating a message in the payload buffer -// so that we know where the metadata for each message is stored. The -// metadata offset is held in the message header. +inline constexpr uint32_t kRuntimeControlMagic = 0x52544850; // "PHTR" +inline constexpr uint32_t kRuntimeControlVersion = 1; + +struct RuntimeMetadataEntry { + ::toolbelt::BufferOffset name = 0; + ::toolbelt::BufferOffset field_data = 0; + uint32_t name_size = 0; +}; + +struct RuntimeControlBlock { + uint32_t magic = kRuntimeControlMagic; + uint32_t version = kRuntimeControlVersion; + ::toolbelt::BufferOffset user_metadata = 0; + uint32_t count = 0; + uint32_t capacity = 0; + RuntimeMetadataEntry entries[1]; +}; + +inline size_t RuntimeControlSize(size_t capacity) { + return offsetof(RuntimeControlBlock, entries) + + capacity * sizeof(RuntimeMetadataEntry); +} + +inline RuntimeControlBlock* InitializeRuntimeControl( + ::toolbelt::PayloadBuffer** pb, size_t capacity) { + capacity = std::max(capacity, 1); + void* memory = ::toolbelt::PayloadBuffer::Allocate( + pb, static_cast(RuntimeControlSize(capacity)), true); + auto* control = new (memory) RuntimeControlBlock; + control->capacity = static_cast(capacity); + (*pb)->metadata = (*pb)->ToOffset(control); + return control; +} + +// Mutable messages share one of these through an owning shared_ptr. Read-only +// messages copy this small runtime into the generated handle and use a +// non-owning shared_ptr alias without allocating a control block. +enum class RuntimeHandleMode { + kBorrowedReadonly, + kFixedMutable, + kOwnedDynamic, +}; + struct MessageRuntime { - MessageRuntime(::toolbelt::PayloadBuffer* p) : pb(p) {} - MessageRuntime(::toolbelt::PayloadBuffer* p, size_t size) - : pb(p), buffer_size(size) {} + MessageRuntime(::toolbelt::PayloadBuffer* p, bool is_mutable = false) + : pb(p), + is_mutable_(is_mutable), + mode_(is_mutable ? RuntimeHandleMode::kFixedMutable + : RuntimeHandleMode::kBorrowedReadonly) {} + MessageRuntime(::toolbelt::PayloadBuffer* p, size_t size, + bool is_mutable = false) + : pb(p), + buffer_size(size), + is_mutable_(is_mutable), + mode_(is_mutable ? RuntimeHandleMode::kFixedMutable + : RuntimeHandleMode::kBorrowedReadonly) {} + MessageRuntime(const MessageRuntime&) = default; + MessageRuntime& operator=(const MessageRuntime&) = default; virtual ~MessageRuntime() = default; ::toolbelt::PayloadBuffer* pb; @@ -123,13 +277,70 @@ struct MessageRuntime { // are looking at received data (someone could set it to anything and we // have no way to check it's valid). size_t buffer_size = 0; + bool is_mutable_ = false; + RuntimeHandleMode mode_ = RuntimeHandleMode::kBorrowedReadonly; - virtual void AddMetadata(const std::string& /*name*/, - ::toolbelt::BufferOffset /*offset*/) {} - virtual ::toolbelt::BufferOffset GetMetadata(const std::string& /*name*/) { + virtual void AddMetadata(std::string_view name, + ::toolbelt::BufferOffset offset) { + RuntimeControlBlock* control = GetRuntimeControl(); + if (control == nullptr) { + return; + } + if (control->count == control->capacity) { + const ::toolbelt::BufferOffset old_offset = pb->metadata; + const uint32_t new_capacity = control->capacity * 2; + void* memory = ::toolbelt::PayloadBuffer::Realloc( + &pb, pb->ToAddress(old_offset), + static_cast(RuntimeControlSize(new_capacity)), true); + control = static_cast(memory); + control->capacity = new_capacity; + pb->metadata = pb->ToOffset(control); + } + void* name_memory = ::toolbelt::PayloadBuffer::Allocate( + &pb, static_cast(name.size()), false); + if (!name.empty()) { + memcpy(name_memory, name.data(), name.size()); + } + control = GetRuntimeControl(); + RuntimeMetadataEntry& entry = control->entries[control->count++]; + entry.name = pb->ToOffset(name_memory); + entry.name_size = static_cast(name.size()); + entry.field_data = offset; + } + virtual ::toolbelt::BufferOffset GetMetadata(std::string_view name) { + RuntimeControlBlock* control = GetRuntimeControl(); + if (control == nullptr) { + return 0; + } + for (uint32_t i = 0; i < control->count; ++i) { + const RuntimeMetadataEntry& entry = control->entries[i]; + if (entry.name_size != name.size()) { + continue; + } + const char* stored = pb->ToAddress(entry.name); + if (name.empty() || memcmp(stored, name.data(), name.size()) == 0) { + return entry.field_data; + } + } return 0; } + bool IsMutable() const { return is_mutable_; } + RuntimeHandleMode Mode() const { return mode_; } + + RuntimeControlBlock* GetRuntimeControl() const { + if (pb == nullptr || pb->metadata == 0) { + return nullptr; + } + auto* control = + pb->ToAddress(pb->metadata, buffer_size); + if (control == nullptr || control->magic != kRuntimeControlMagic || + control->version != kRuntimeControlVersion) { + return nullptr; + } + return control; + } + template T* ToAddress(toolbelt::BufferOffset offset) { return pb->ToAddress(offset, buffer_size); @@ -154,21 +365,8 @@ struct MessageRuntime { // This is a message runtime for a message that is mutable. It holds a mapping // for each message name to the offset of the metadata in the payload buffer. struct MutableMessageRuntime : public MessageRuntime { - MutableMessageRuntime(::toolbelt::PayloadBuffer* p) : MessageRuntime(p) {} - - absl::flat_hash_map metadata_offsets; - void AddMetadata(const std::string& name, - ::toolbelt::BufferOffset offset) override { - metadata_offsets[name] = offset; - } - - ::toolbelt::BufferOffset GetMetadata(const std::string& name) override { - auto it = metadata_offsets.find(name); - if (it == metadata_offsets.end()) { - return 0; - } - return it->second; - } + MutableMessageRuntime(::toolbelt::PayloadBuffer* p) + : MessageRuntime(p, true) {} }; // Dynamically allocated payload buffer. Must be allocated in memory @@ -176,7 +374,9 @@ struct MutableMessageRuntime : public MessageRuntime { struct DynamicMutableMessageRuntime : public MutableMessageRuntime { DynamicMutableMessageRuntime(::toolbelt::PayloadBuffer* p, std::function free) - : MutableMessageRuntime(p), free_(std::move(free)) {} + : MutableMessageRuntime(p), free_(std::move(free)) { + mode_ = RuntimeHandleMode::kOwnedDynamic; + } ~DynamicMutableMessageRuntime() override { if (free_ != nullptr) { pb->~PayloadBuffer(); @@ -186,6 +386,8 @@ struct DynamicMutableMessageRuntime : public MutableMessageRuntime { std::function free_; }; +using RuntimeHandle = std::shared_ptr; + struct InternalDefault {}; // Tuning parameters for messages. The kPerformance tuning uses a bitmap @@ -231,11 +433,40 @@ enum class Tuning { // +---------------+ +-------------+ struct Message { - Message() = default; - Message(std::shared_ptr rt, ::toolbelt::BufferOffset start) - : runtime(rt), absolute_binary_offset(start) {} - Message(const Message&) = default; - Message& operator=(const Message&) = default; + private: + MessageRuntime readonly_runtime_; + + public: + Message() : readonly_runtime_(nullptr, size_t{0}, false) {} + Message(RuntimeHandle rt, ::toolbelt::BufferOffset start) + : readonly_runtime_(nullptr, size_t{0}, false), + absolute_binary_offset(start) { + BindRuntime(std::move(rt)); + } + Message(const Message& other) + : readonly_runtime_(nullptr, size_t{0}, false), + absolute_binary_offset(other.absolute_binary_offset) { + BindRuntime(other.runtime); + } + Message(Message&& other) noexcept + : readonly_runtime_(nullptr, size_t{0}, false), + absolute_binary_offset(other.absolute_binary_offset) { + BindRuntime(std::move(other.runtime)); + } + Message& operator=(const Message& other) { + if (this != &other) { + absolute_binary_offset = other.absolute_binary_offset; + BindRuntime(other.runtime); + } + return *this; + } + Message& operator=(Message&& other) noexcept { + if (this != &other) { + absolute_binary_offset = other.absolute_binary_offset; + BindRuntime(std::move(other.runtime)); + } + return *this; + } virtual ~Message() = default; virtual const MessageInfo* GetMessageInfo() const { return nullptr; } @@ -245,8 +476,12 @@ struct Message { virtual void CopyFrom(const Message& /*src*/) {} virtual void SyncToPayload() const {} - std::shared_ptr runtime; - ::toolbelt::BufferOffset absolute_binary_offset; + Message* operator->() { return this; } + const Message* operator->() const { return this; } + bool IsBound() const { return runtime != nullptr; } + + RuntimeHandle runtime; + ::toolbelt::BufferOffset absolute_binary_offset = 0; // 'field' is the offset from the start of the message to the field (positive) // Subtract the field offset from the field to get the address of the @@ -262,18 +497,19 @@ struct Message { uint32_t offset) { const Message* msg = reinterpret_cast( reinterpret_cast(field) - offset); + if (!msg->runtime->IsMutable()) { + throw std::logic_error("cannot mutate a readonly Phaser message"); + } return &msg->runtime->pb; } - static std::shared_ptr& GetRuntime(void* field, - uint32_t offset) { + static RuntimeHandle& GetRuntime(void* field, uint32_t offset) { Message* msg = reinterpret_cast(reinterpret_cast(field) - offset); return msg->runtime; } - static const std::shared_ptr& GetRuntime(const void* field, - uint32_t offset) { + static const RuntimeHandle& GetRuntime(const void* field, uint32_t offset) { const Message* msg = reinterpret_cast( reinterpret_cast(field) - offset); return msg->runtime; @@ -302,12 +538,22 @@ struct Message { if (offset >= runtime->pb->hwm) { return absl::InternalError("Invalid metadata offset"); } - runtime->pb->metadata = offset; + if (RuntimeControlBlock* control = runtime->GetRuntimeControl(); + control != nullptr) { + control->user_metadata = offset; + } else { + runtime->pb->metadata = offset; + } return absl::OkStatus(); } void* GetUserMetadata() { - return runtime->pb->ToAddress(runtime->pb->metadata); + toolbelt::BufferOffset offset = runtime->pb->metadata; + if (RuntimeControlBlock* control = runtime->GetRuntimeControl(); + control != nullptr) { + offset = control->user_metadata; + } + return runtime->pb->ToAddress(offset); } void* Allocate(size_t size, size_t alignment = 4, bool clear = true) { @@ -358,10 +604,39 @@ struct Message { // Looks for the field number in the field data. Returns the offset of the // field if found, -1 otherwise. - int32_t FindFieldOffset(uint32_t field_number) const; + int32_t FindFieldOffset(uint32_t field_number) const { + return FindField(field_number).offset; + } // Similar for field id for presence bit mask. - int32_t FindFieldId(uint32_t field_number) const; + int32_t FindFieldId(uint32_t field_number) const { + return FindField(field_number).id; + } + + // Resolves both values in one metadata lookup. + FieldLocation FindField(uint32_t field_number) const { + if (runtime == nullptr) { + return {}; + } + // First 4 bytes of the message are the offset to the field data. + const ::toolbelt::BufferOffset* field_data = + runtime->ToAddress<::toolbelt::BufferOffset>(absolute_binary_offset); + if (field_data == nullptr) { + return {}; + } + const void* metadata = runtime->ToAddress(*field_data); + if (metadata == nullptr) { + return {}; + } + + const uint32_t first_word = *static_cast(metadata); + if (first_word == kHybridFieldDataMagic) { + return internal::FindHybridField( + static_cast(metadata), field_number); + } + return internal::FindLegacyField(static_cast(metadata), + field_number); + } void* BinaryData() const { SyncToPayload(); @@ -381,6 +656,21 @@ struct Message { SyncToPayload(); return runtime->pb->Size(); } + + protected: + static RuntimeHandle BorrowRuntime(MessageRuntime& runtime) { + return RuntimeHandle(RuntimeHandle(), &runtime); + } + + private: + void BindRuntime(RuntimeHandle other) { + if (other != nullptr && other.use_count() == 0) { + readonly_runtime_ = *other; + runtime = BorrowRuntime(readonly_runtime_); + } else { + runtime = std::move(other); + } + } }; ::toolbelt::PayloadBuffer* NewDynamicBuffer( diff --git a/phaser/runtime/message_test.cc b/phaser/runtime/message_test.cc index eefa7de..6815c9c 100644 --- a/phaser/runtime/message_test.cc +++ b/phaser/runtime/message_test.cc @@ -526,6 +526,10 @@ static struct InnerMessageBankRegister { } inner_message_bank_register; struct TestMessage : public Message { + inline static constexpr uint32_t kU1FieldNumbers[] = {107, 108}; + inline static constexpr uint32_t kU2FieldNumbers[] = {109, 110}; + inline static constexpr uint32_t kU3FieldNumbers[] = {111, 112}; + // Default constructor makes a dynamic payload buffer. TestMessage(size_t initial_size = 1024) : x_(offsetof(TestMessage, x_), HeaderSize() + 0, 0, 100), @@ -535,9 +539,12 @@ struct TestMessage : public Message { vi32_(offsetof(TestMessage, vi32_), HeaderSize() + 24, 0, 104), vstr_(offsetof(TestMessage, vstr_), HeaderSize() + 32, 0, 105), vm_(offsetof(TestMessage, vm_), HeaderSize() + 40, 0, 106), - u1_(offsetof(TestMessage, u1_), HeaderSize() + 48, 0, 0, {107, 108}), - u2_(offsetof(TestMessage, u2_), HeaderSize() + 56, 0, 0, {109, 110}), - u3_(offsetof(TestMessage, u3_), HeaderSize() + 64, 0, 0, {111, 112}) { + u1_(offsetof(TestMessage, u1_), HeaderSize() + 48, 0, 0, + absl::MakeConstSpan(kU1FieldNumbers)), + u2_(offsetof(TestMessage, u2_), HeaderSize() + 56, 0, 0, + absl::MakeConstSpan(kU2FieldNumbers)), + u3_(offsetof(TestMessage, u3_), HeaderSize() + 64, 0, 0, + absl::MakeConstSpan(kU3FieldNumbers)) { InitDynamicMutable(initial_size); } @@ -551,9 +558,12 @@ struct TestMessage : public Message { vi32_(offsetof(TestMessage, vi32_), HeaderSize() + 24, 0, 104), vstr_(offsetof(TestMessage, vstr_), HeaderSize() + 32, 0, 105), vm_(offsetof(TestMessage, vm_), HeaderSize() + 40, 0, 106), - u1_(offsetof(TestMessage, u1_), HeaderSize() + 48, 0, 0, {107, 108}), - u2_(offsetof(TestMessage, u2_), HeaderSize() + 56, 0, 0, {109, 110}), - u3_(offsetof(TestMessage, u3_), HeaderSize() + 64, 0, 0, {111, 112}) {} + u1_(offsetof(TestMessage, u1_), HeaderSize() + 48, 0, 0, + absl::MakeConstSpan(kU1FieldNumbers)), + u2_(offsetof(TestMessage, u2_), HeaderSize() + 56, 0, 0, + absl::MakeConstSpan(kU2FieldNumbers)), + u3_(offsetof(TestMessage, u3_), HeaderSize() + 64, 0, 0, + absl::MakeConstSpan(kU3FieldNumbers)) {} static TestMessage CreateMutable(void* addr, size_t size) { ::toolbelt::PayloadBuffer* pb = @@ -926,8 +936,8 @@ struct TestMessage : public Message { add_vstr(src.vstr(i)); } for (size_t i = 0; i < src.vm_size(); ++i) { - auto* m = add_vm(); - if (absl::Status s = m->CloneFrom(src.vm(i)); !s.ok()) { + auto m = add_vm(); + if (absl::Status s = m.CloneFrom(src.vm(i)); !s.ok()) { return s; } } @@ -1022,9 +1032,9 @@ struct TestMessage : public Message { } size_t vm_size() const { return vm_.size(); } - const InnerMessage& vm(size_t i) const { return vm_.Get(i); } - InnerMessage* mutable_vm(size_t i) { return vm_.Mutable(i); } - InnerMessage* add_vm() { return vm_.Add(); } + InnerMessage vm(size_t i) const { return vm_.Get(i); } + InnerMessage mutable_vm(size_t i) { return vm_.Mutable(i); } + InnerMessage add_vm() { return vm_.Add(); } void clear_vm() { vm_.Clear(); } phaser::MessageVectorField& vm() { vm_.Populate(); @@ -1135,12 +1145,12 @@ inline std::ostream& operator<<(std::ostream& os, const TestMessage& msg) { os << "vi32: " << v << std::endl; } - for (auto& v : msg.vstr_) { + for (auto v : msg.vstr_) { msg.vstr_.PrintIndent(os); os << "vstr: " << v << std::endl; } - for (auto& v : msg.vm_) { + for (auto v : msg.vm_) { msg.vm_.PrintIndent(os); os << "vm: " << v << std::endl; } @@ -1336,6 +1346,21 @@ TEST(MessageTest, Basic) { free(buffer); } +TEST(MessageTest, LegacyFieldMetadataFallback) { + char* buffer = static_cast(calloc(4096, 1)); + TestMessage msg = TestMessage::CreateMutable(buffer, 4096); + + const phaser::FieldLocation x = msg.FindField(100); + EXPECT_EQ(x.offset, 8); + EXPECT_EQ(x.id, 0); + const phaser::FieldLocation union_a = msg.FindField(111); + const phaser::FieldLocation union_b = msg.FindField(112); + EXPECT_EQ(union_a.offset, union_b.offset); + EXPECT_EQ(msg.FindField(999).offset, -1); + + free(buffer); +} + TEST(MessageTest, RepeatedPrimitive) { char* buffer = static_cast(calloc(4096, 1)); TestMessage msg = TestMessage::CreateMutable(buffer, 4096); @@ -1386,8 +1411,8 @@ TEST(MessageTest, RepeatedString) { const char* strings[] = {"one", "two", "three", "four"}; int i = 0; - for (auto& v : msg.vstr_) { - ASSERT_EQ(strings[i], v.Get()); + for (auto v : msg.vstr_) { + ASSERT_EQ(strings[i], v); i++; } @@ -1405,8 +1430,8 @@ TEST(MessageTest, RepeatedString) { TestMessage ro_msg = TestMessage::CreateReadonly(buffer2); int ro_i = 0; - for (auto& v : ro_msg.vstr_) { - ASSERT_EQ(strings[ro_i], v.Get()); + for (auto v : ro_msg.vstr_) { + ASSERT_EQ(strings[ro_i], v); ro_i++; } ASSERT_EQ("one", ro_msg.vstr_.Get(0)); @@ -1422,22 +1447,22 @@ TEST(MessageTest, RepeatedMessage) { char* buffer = static_cast(calloc(4096, 1)); TestMessage msg = TestMessage::CreateMutable(buffer, 4096); - InnerMessage* inner1 = msg.vm_.Add(); - inner1->str_.Set("one"); - inner1->f_.Set(0xdeadbeef); + auto inner1 = msg.vm_.Add(); + inner1.str_.Set("one"); + inner1.f_.Set(0xdeadbeef); - InnerMessage* inner2 = msg.vm_.Mutable(2); - inner2->str_.Set("two"); - inner2->f_.Set(0x1234); + auto inner2 = msg.vm_.Mutable(2); + inner2.str_.Set("two"); + inner2.f_.Set(0x1234); msg.DebugDump(); { - auto& ro_inner1 = msg.vm_.Get(0); + auto ro_inner1 = msg.vm_.Get(0); ASSERT_EQ("one", ro_inner1.str_.Get()); ASSERT_EQ(0xdeadbeef, ro_inner1.f_.Get()); - auto& ro_inner2 = msg.vm_.Get(2); + auto ro_inner2 = msg.vm_.Get(2); ASSERT_EQ("two", ro_inner2.str_.Get()); ASSERT_EQ(0x1234, ro_inner2.f_.Get()); } @@ -1448,11 +1473,11 @@ TEST(MessageTest, RepeatedMessage) { memcpy(buffer2, buffer, 4096); TestMessage ro_msg = TestMessage::CreateReadonly(buffer2); - auto& ro_inner1 = ro_msg.vm_.Get(0); + auto ro_inner1 = ro_msg.vm_.Get(0); ASSERT_EQ("one", ro_inner1.str_.Get()); ASSERT_EQ(0xdeadbeef, ro_inner1.f_.Get()); - auto& ro_inner2 = ro_msg.vm_.Get(2); + auto ro_inner2 = ro_msg.vm_.Get(2); ASSERT_EQ("two", ro_inner2.str_.Get()); ASSERT_EQ(0x1234, ro_inner2.f_.Get()); free(buffer2); @@ -1578,13 +1603,13 @@ TEST(MessageTest, ClearRepeated) { msg.vstr_.Add("one"); msg.vstr_.Add("two"); - auto* inner1 = msg.vm_.Add(); - inner1->str_.Set("one"); - inner1->f_.Set(0xdeadbeef); + auto inner1 = msg.vm_.Add(); + inner1.str_.Set("one"); + inner1.f_.Set(0xdeadbeef); - auto* inner2 = msg.vm_.Add(); - inner2->str_.Set("two"); - inner2->f_.Set(0x1234); + auto inner2 = msg.vm_.Add(); + inner2.str_.Set("two"); + inner2.f_.Set(0x1234); msg.DebugDump(); @@ -1818,13 +1843,13 @@ TEST(MessageTest, ProtobufSerializationRepeated) { msg.add_vstr("two"); msg.add_vstr("three"); - auto* inner1 = msg.vm_.Add(); - inner1->set_str("one"); - inner1->set_f(0xdeadbeef); + auto inner1 = msg.vm_.Add(); + inner1.set_str("one"); + inner1.set_f(0xdeadbeef); - auto* inner2 = msg.vm_.Add(); - inner2->set_str("two"); - inner2->set_f(0x1234); + auto inner2 = msg.vm_.Add(); + inner2.set_str("two"); + inner2.set_f(0x1234); ASSERT_TRUE(msg.Serialize(pb).ok()); foo::bar::TestMessage pb_msg; @@ -2013,13 +2038,13 @@ TEST(MessageTest, Print) { msg.vstr_.Add("one"); msg.vstr_.Add("two"); - auto* inner1 = msg.vm_.Add(); - inner1->str_.Set("one"); - inner1->f_.Set(0xdeadbeef); + auto inner1 = msg.vm_.Add(); + inner1.str_.Set("one"); + inner1.f_.Set(0xdeadbeef); - auto* inner2 = msg.vm_.Add(); - inner2->str_.Set("two"); - inner2->f_.Set(0x1234); + auto inner2 = msg.vm_.Add(); + inner2.str_.Set("two"); + inner2.f_.Set(0x1234); // Unions. msg.u1_.Set<0>(1234u); diff --git a/phaser/runtime/phaser_bank.cc b/phaser/runtime/phaser_bank.cc index e2df153..158e1ee 100644 --- a/phaser/runtime/phaser_bank.cc +++ b/phaser/runtime/phaser_bank.cc @@ -12,7 +12,10 @@ namespace phaser { std::unique_ptr> phaser_banks_; -absl::StatusOr GetPhaserBankInfo(std::string message_type) { +absl::StatusOr GetPhaserBankInfo(std::string_view message_type) { + if (!phaser_banks_) { + return absl::InternalError("Phaser message bank is not initialized"); + } auto it = phaser_banks_->find(message_type); if (it == phaser_banks_->end()) { return absl::InternalError( @@ -21,13 +24,13 @@ absl::StatusOr GetPhaserBankInfo(std::string message_type) { return &it->second; } -void PhaserBankRegisterMessage(const std::string& name, const BankInfo& info) { +void PhaserBankRegisterMessage(std::string_view name, const BankInfo& info) { if (!phaser_banks_) { // Lazy init because we can't guarantee the order of static initialization. phaser_banks_ = std::make_unique>(); } - (*phaser_banks_)[name] = info; + (*phaser_banks_)[std::string(name)] = info; } absl::Status PhaserStreamTo(const std::string& message_type, const Message& msg, @@ -80,6 +83,39 @@ absl::StatusOr PhaserBankSerializedSize(const std::string& message_type, return (*bank_info)->serialized_size(msg); } +absl::Status PhaserBankSerializeAtOffset( + std::string_view message_type, + std::shared_ptr<::phaser::MessageRuntime> runtime, + toolbelt::BufferOffset offset, ProtoBuffer& buffer) { + absl::StatusOr bank_info = GetPhaserBankInfo(message_type); + if (!bank_info.ok()) { + return bank_info.status(); + } + return (*bank_info)->serialize_at_offset(std::move(runtime), offset, buffer); +} + +absl::Status PhaserBankDeserializeAtOffset( + std::string_view message_type, + std::shared_ptr<::phaser::MessageRuntime> runtime, + toolbelt::BufferOffset offset, ProtoBuffer& buffer) { + absl::StatusOr bank_info = GetPhaserBankInfo(message_type); + if (!bank_info.ok()) { + return bank_info.status(); + } + return (*bank_info)->deserialize_at_offset(std::move(runtime), offset, buffer); +} + +absl::StatusOr PhaserBankSerializedSizeAtOffset( + std::string_view message_type, + std::shared_ptr<::phaser::MessageRuntime> runtime, + toolbelt::BufferOffset offset) { + absl::StatusOr bank_info = GetPhaserBankInfo(message_type); + if (!bank_info.ok()) { + return bank_info.status(); + } + return (*bank_info)->serialized_size_at_offset(std::move(runtime), offset); +} + absl::StatusOr PhaserBankAllocateAtOffset( const std::string& message_type, std::shared_ptr<::phaser::MessageRuntime> runtime, @@ -119,7 +155,7 @@ absl::StatusOr PhaserBankMakeExisting( return (*bank_info)->make_existing(runtime, data); } -absl::StatusOr PhaserBankBinarySize(const std::string& message_type) { +absl::StatusOr PhaserBankBinarySize(std::string_view message_type) { absl::StatusOr bank_info = GetPhaserBankInfo(message_type); if (!bank_info.ok()) { return bank_info.status(); diff --git a/phaser/runtime/phaser_bank.h b/phaser/runtime/phaser_bank.h index 0995dde..1d90930 100644 --- a/phaser/runtime/phaser_bank.h +++ b/phaser/runtime/phaser_bank.h @@ -3,6 +3,8 @@ // See LICENSE file for licensing information. #pragma once +#include + #include "absl/container/flat_hash_map.h" #include "absl/status/status.h" #include "phaser/runtime/message.h" @@ -15,6 +17,15 @@ struct BankInfo { absl::Status (*serialize_to_buffer)(const Message& msg, ProtoBuffer& buffer); absl::Status (*deserialize_from_buffer)(Message& msg, ProtoBuffer& buffer); size_t (*serialized_size)(const Message& msg); + absl::Status (*serialize_at_offset)( + std::shared_ptr<::phaser::MessageRuntime> runtime, + toolbelt::BufferOffset offset, ProtoBuffer& buffer); + absl::Status (*deserialize_at_offset)( + std::shared_ptr<::phaser::MessageRuntime> runtime, + toolbelt::BufferOffset offset, ProtoBuffer& buffer); + size_t (*serialized_size_at_offset)( + std::shared_ptr<::phaser::MessageRuntime> runtime, + toolbelt::BufferOffset offset); Message* (*allocate_at_offset)( std::shared_ptr<::phaser::MessageRuntime> runtime, toolbelt::BufferOffset offset); @@ -34,9 +45,9 @@ struct BankInfo { extern std::unique_ptr> phaser_banks_; -absl::StatusOr GetPhaserBankInfo(std::string message_type); +absl::StatusOr GetPhaserBankInfo(std::string_view message_type); -void PhaserBankRegisterMessage(const std::string& name, const BankInfo& info); +void PhaserBankRegisterMessage(std::string_view name, const BankInfo& info); absl::Status PhaserStreamTo(const std::string& message_type, const Message& msg, std::ostream& os, int indent); @@ -50,6 +61,18 @@ absl::Status PhaserBankDeserializeFromBuffer(const std::string& message_type, Message& msg, ProtoBuffer& buffer); absl::StatusOr PhaserBankSerializedSize(const std::string& message_type, const Message& msg); +absl::Status PhaserBankSerializeAtOffset( + std::string_view message_type, + std::shared_ptr<::phaser::MessageRuntime> runtime, + toolbelt::BufferOffset offset, ProtoBuffer& buffer); +absl::Status PhaserBankDeserializeAtOffset( + std::string_view message_type, + std::shared_ptr<::phaser::MessageRuntime> runtime, + toolbelt::BufferOffset offset, ProtoBuffer& buffer); +absl::StatusOr PhaserBankSerializedSizeAtOffset( + std::string_view message_type, + std::shared_ptr<::phaser::MessageRuntime> runtime, + toolbelt::BufferOffset offset); // This allocates a message from the heap (using new) with its storage in the // payload buffer. The ownership of the heap memory is passed back to the @@ -92,7 +115,7 @@ absl::StatusOr PhaserBankMakeExisting( const std::string& message_type, std::shared_ptr<::phaser::MessageRuntime> runtime, const void* data); -absl::StatusOr PhaserBankBinarySize(const std::string& message_type); +absl::StatusOr PhaserBankBinarySize(std::string_view message_type); absl::StatusOr PhaserBankMessageInfo( const std::string& message_type); diff --git a/phaser/runtime/ros.h b/phaser/runtime/ros.h index 8b439e2..506d3cc 100644 --- a/phaser/runtime/ros.h +++ b/phaser/runtime/ros.h @@ -14,11 +14,29 @@ #include #include #include +#include #include #include "phaser/runtime/fields.h" namespace phaser { + +struct RosHeaderView { + uint32_t seq = 0; + ::ros::Time stamp; + std::string_view frame_id; + + ::std_msgs::Header ToOwned() const { + ::std_msgs::Header result; + result.seq = seq; + result.stamp = stamp; + if (!frame_id.empty()) { + result.frame_id.assign(frame_id.data(), frame_id.size()); + } + return result; + } +}; + namespace internal { struct RosTimeTraits { @@ -84,6 +102,21 @@ struct RosHeaderTraits { RosTimeTraits::Print(os, value.stamp); os << "} frame_id: \"" << value.frame_id << "\""; } + + template + static RosHeaderView LoadView(const Backend& backend) { + return { + .seq = static_cast(backend.seq.Get()), + .stamp = backend.stamp.Get(), + .frame_id = backend.frame_id.Get(), + }; + } + + static void Print(std::ostream& os, RosHeaderView value) { + os << "seq: " << value.seq << " stamp {"; + RosTimeTraits::Print(os, value.stamp); + os << "} frame_id: \"" << value.frame_id << "\""; + } }; } // namespace internal @@ -214,7 +247,181 @@ template using RosDurationField = RosMessageField; +template +class RosHeaderMutableView { + public: + class FrameIdProxy { + public: + explicit FrameIdProxy(Owner* owner) : owner_(owner) {} + operator std::string_view() const { return owner_->Get().frame_id; } + std::string_view Get() const { return owner_->Get().frame_id; } + friend bool operator==(const FrameIdProxy& lhs, std::string_view rhs) { + return lhs.Get() == rhs; + } + friend bool operator==(std::string_view lhs, const FrameIdProxy& rhs) { + return lhs == rhs.Get(); + } + friend bool operator!=(const FrameIdProxy& lhs, std::string_view rhs) { + return !(lhs == rhs); + } + friend bool operator!=(std::string_view lhs, const FrameIdProxy& rhs) { + return !(lhs == rhs); + } + template + FrameIdProxy& operator=(String value) { + owner_->SetFrameId(value); + return *this; + } + + private: + Owner* owner_; + }; + + explicit RosHeaderMutableView(Owner* owner) + : owner_(owner), + seq(owner->Get().seq), + stamp(owner->Get().stamp), + frame_id(owner) {} + RosHeaderMutableView(const RosHeaderMutableView&) = delete; + RosHeaderMutableView& operator=(const RosHeaderMutableView&) = delete; + RosHeaderMutableView(RosHeaderMutableView&& other) noexcept + : owner_(other.owner_), + seq(other.seq), + stamp(other.stamp), + frame_id(owner_) { + other.active_ = false; + } + ~RosHeaderMutableView() { + if (active_) { + owner_->CommitMutable(seq, stamp); + } + } + + RosHeaderView Get() const { + return {.seq = seq, .stamp = stamp, .frame_id = frame_id.Get()}; + } + ::std_msgs::Header ToOwned() const { return Get().ToOwned(); } + + private: + Owner* owner_; + bool active_ = true; + + public: + uint32_t seq; + ::ros::Time stamp; + FrameIdProxy frame_id; +}; + template -using RosHeaderField = RosMessageField; +class RosHeaderField : public IndirectMessageField { + public: + using Base = IndirectMessageField; + using MutableView = RosHeaderMutableView>; + using Base::Base; + + struct ConstArrow { + RosHeaderView view; + const RosHeaderView* operator->() const { return &view; } + }; + struct MutableArrow { + MutableView view; + MutableView* operator->() { return &view; } + }; + + RosHeaderField() = default; + RosHeaderField(const RosHeaderField&) = default; + RosHeaderField(RosHeaderField&&) = default; + + RosHeaderField& operator=(const RosHeaderField& other) { + if (this != &other) { + Set(other.Get()); + } + return *this; + } + RosHeaderField& operator=(RosHeaderField&& other) { + if (this != &other) { + Set(other.Get()); + } + return *this; + } + RosHeaderField& operator=(const ::std_msgs::Header& value) { + Set(value); + return *this; + } + + operator RosHeaderView() const { return Get(); } + RosHeaderView operator*() const { return Get(); } + MutableView operator*() { return Mutable(); } + ConstArrow operator->() const { return ConstArrow{Get()}; } + MutableArrow operator->() { return MutableArrow{Mutable()}; } + + RosHeaderView Get() const { + if (!Base::IsPresent()) { + return {}; + } + return internal::RosHeaderTraits::LoadView(Base::Get()); + } + + ::std_msgs::Header ToOwned() const { return Get().ToOwned(); } + + MutableView Mutable() { + Base::Mutable(); + return MutableView(this); + } + + template + void SetFrameId(String value) { + Backend* backend = Base::Mutable(); + backend->frame_id = value; + } + + void CommitMutable(uint32_t seq, const ::ros::Time& stamp) { + Backend* backend = Base::Mutable(); + backend->seq = seq; + backend->stamp = stamp; + backend->SyncToPayload(); + } + + void Set(const ::std_msgs::Header& value) { + auto backend = Mutable(); + backend.seq = value.seq; + backend.stamp = value.stamp; + backend.frame_id = value.frame_id; + } + void Set(RosHeaderView value) { + auto backend = Mutable(); + backend.seq = value.seq; + backend.stamp = value.stamp; + backend.frame_id = value.frame_id; + } + + bool IsPresent() const { return Base::IsPresent(); } + + void Clear() { Base::Clear(); } + + void SyncToPayload() const { + if (Base::IsPresent()) { + Base::Get().SyncToPayload(); + } + } + + size_t SerializedSize() const { + SyncToPayload(); + return Base::SerializedSize(); + } + absl::Status Serialize(ProtoBuffer& buffer) const { + SyncToPayload(); + return Base::Serialize(buffer); + } + absl::Status Deserialize(ProtoBuffer& buffer) { + return Base::Deserialize(buffer); + } + + friend std::ostream& operator<<(std::ostream& os, + const RosHeaderField& field) { + internal::RosHeaderTraits::Print(os, field.Get()); + return os; + } +}; } // namespace phaser diff --git a/phaser/runtime/ros_wireformat.h b/phaser/runtime/ros_wireformat.h index 2f377e5..ee229bd 100644 --- a/phaser/runtime/ros_wireformat.h +++ b/phaser/runtime/ros_wireformat.h @@ -48,13 +48,7 @@ class ROSReader { } if constexpr (std::is_same_v, bool>) { - const uint8_t value = static_cast( - static_cast(data_[position_++])); - if (value > 1) { - return absl::InvalidArgumentError(absl::StrFormat( - "Invalid ROS bool value %d at byte %d", value, position_ - 1)); - } - return value != 0; + return data_[position_++] != 0; } else if constexpr (std::is_integral_v) { using U = std::make_unsigned_t; U value = 0; @@ -110,17 +104,7 @@ class ROSReader { if constexpr (std::is_same_v, bool>) { static_assert(sizeof(bool) == sizeof(uint8_t)); - for (size_t i = 0; i < values.size(); ++i) { - const uint8_t value = static_cast( - static_cast(data_[position_ + i])); - if (value > 1) { - return absl::InvalidArgumentError(absl::StrFormat( - "Invalid ROS bool value %d at byte %d", value, position_ + i)); - } - } - for (size_t i = 0; i < values.size(); ++i) { - values[i] = data_[position_ + i] != 0; - } + memcpy(values.data(), data_.data() + position_, byte_size); position_ += byte_size; return absl::OkStatus(); } @@ -173,6 +157,17 @@ class ROSReader { return Read(); } + absl::StatusOr> ReadRaw(size_t length) { + if (length > Remaining()) { + return absl::InvalidArgumentError(absl::StrFormat( + "Truncated ROS input at byte %d: need %d bytes, have %d", position_, + length, Remaining())); + } + absl::Span value(data_.data() + position_, length); + position_ += length; + return value; + } + private: absl::Span data_; size_t position_ = 0; @@ -278,9 +273,8 @@ class ROSBuffer { if constexpr (std::is_same_v, bool>) { static_assert(sizeof(bool) == sizeof(uint8_t)); - for (bool value : values) { - data_[size_++] = static_cast(value ? 1 : 0); - } + memcpy(data_ + size_, values.data(), byte_size); + size_ += byte_size; return absl::OkStatus(); } diff --git a/phaser/runtime/ros_wireformat_test.cc b/phaser/runtime/ros_wireformat_test.cc index 166c9d3..16ba61d 100644 --- a/phaser/runtime/ros_wireformat_test.cc +++ b/phaser/runtime/ros_wireformat_test.cc @@ -98,16 +98,12 @@ TEST(ROSWireformatTest, ReadsCanonicalLittleEndianBytes) { EXPECT_EQ(reader.Remaining(), 0u); } -TEST(ROSWireformatTest, ReaderRejectsTruncatedAndInvalidValues) { +TEST(ROSWireformatTest, ReaderRejectsTruncatedValues) { const std::array truncated_integer = {1, 2, 3}; ROSReader integer_reader(absl::MakeConstSpan(truncated_integer)); EXPECT_FALSE(integer_reader.Read().ok()); EXPECT_EQ(integer_reader.Position(), 0u); - const std::array invalid_bool = {2}; - ROSReader bool_reader(absl::MakeConstSpan(invalid_bool)); - EXPECT_FALSE(bool_reader.Read().ok()); - const std::array truncated_string = {5, 0, 0, 0, 'a', 'b'}; ROSReader string_reader(absl::MakeConstSpan(truncated_string)); EXPECT_FALSE(string_reader.ReadString().ok()); @@ -165,18 +161,12 @@ TEST(ROSWireformatTest, BulkWriteFailureDoesNotAdvanceCursor) { EXPECT_EQ(buffer.Size(), 0u); } -TEST(ROSWireformatTest, BulkReadValidatesBeforeAdvancing) { +TEST(ROSWireformatTest, BulkReadValidatesSizeBeforeAdvancing) { const std::array truncated = {}; std::array integers = {}; ROSReader integer_reader(absl::MakeConstSpan(truncated)); EXPECT_FALSE(integer_reader.ReadArray(absl::MakeSpan(integers)).ok()); EXPECT_EQ(integer_reader.Position(), 0u); - - const std::array invalid_bool = {0, 2, 1}; - std::array bools = {}; - ROSReader bool_reader(absl::MakeConstSpan(invalid_bool)); - EXPECT_FALSE(bool_reader.ReadArray(absl::MakeSpan(bools)).ok()); - EXPECT_EQ(bool_reader.Position(), 0u); } } // namespace diff --git a/phaser/runtime/runtime.h b/phaser/runtime/runtime.h index a7ca165..171e670 100644 --- a/phaser/runtime/runtime.h +++ b/phaser/runtime/runtime.h @@ -135,7 +135,7 @@ inline std::ostream& operator<<(std::ostream& os, const AnyMessage& msg) { os << "{\n"; msg.value_.Indent(2); msg.value_.PrintIndent(os); - std::string type = msg.MessageTypeName(); + std::string type(msg.MessageTypeName()); os << "[" << msg.type_url() << "] {\n"; msg.value_.Indent(2); absl::StatusOr s = diff --git a/phaser/runtime/union.h b/phaser/runtime/union.h index 3189c46..c4dc8e6 100644 --- a/phaser/runtime/union.h +++ b/phaser/runtime/union.h @@ -14,11 +14,11 @@ #include #include #include -#include #include "absl/container/flat_hash_map.h" #include "absl/status/status.h" #include "absl/status/statusor.h" +#include "absl/types/span.h" #include "phaser/runtime/iterators.h" #include "phaser/runtime/message.h" #include "toolbelt/payload_buffer.h" @@ -500,7 +500,7 @@ class UnionField : public Field { public: UnionField() = default; UnionField(uint32_t source_offset, uint32_t relative_binary_offset, int id, - int number, std::vector field_numbers) + int number, absl::Span field_numbers) : Field(id, number), source_offset_(source_offset), relative_binary_offset_(relative_binary_offset), @@ -868,7 +868,8 @@ class UnionField : public Field { uint32_t source_offset_; ::toolbelt::BufferOffset relative_binary_offset_; - std::vector field_numbers_; // field number for each tuple type + absl::Span + field_numbers_; // Static field number for each tuple type. mutable std::tuple value_; }; } // namespace phaser diff --git a/phaser/runtime/vectors.h b/phaser/runtime/vectors.h index b100151..2b3ddad 100644 --- a/phaser/runtime/vectors.h +++ b/phaser/runtime/vectors.h @@ -813,134 +813,126 @@ class MessageVectorField : public Field { MessageVectorField(const MessageVectorField&) = default; MessageVectorField(MessageVectorField&&) = default; - const MessageObject& operator[](int index) const { + T operator[](int index) const { int32_t offset = FindFieldOffset(source_offset_); if (offset == -1) { - return empty_; + return T(InternalDefault{}); } auto hdr = Header(static_cast(offset)); if (static_cast(index) >= hdr->num_elements) { - return empty_; + return T(InternalDefault{}); } ::toolbelt::BufferOffset* data = GetRuntime()->template ToAddress<::toolbelt::BufferOffset>(hdr->data); if (data[index] == 0) { - return empty_; - } - if (static_cast(index) >= msgs_.size()) { - msgs_.resize(static_cast(index) + 1); + return T(InternalDefault{}); } - if (msgs_[static_cast(index)].empty()) { - msgs_[static_cast(index)] = - MessageObject(GetRuntime(), data[index]); - } - return msgs_[static_cast(index)]; - } - - MessageObject& operator[](int index) { - return const_cast&>( - static_cast(this)->operator[](index)); + return T(GetRuntime(), data[index]); } - MessageObject& front() { - Populate(); - return msgs_.front(); - } - const MessageObject& front() const { - Populate(); - return msgs_.front(); - } - MessageObject& back() { - Populate(); - return msgs_.back(); - } - const MessageObject& back() const { - Populate(); - return msgs_.back(); + T operator[](int index) { + return static_cast(this)->operator[](index); } - using value_type = MessageObject; - using reference = value_type&; - using const_reference = value_type&; - using pointer = value_type*; - using const_pointer = const value_type*; + T front() { return (*this)[0]; } + T front() const { return (*this)[0]; } + T back() { return (*this)[static_cast(size() - 1)]; } + T back() const { return (*this)[static_cast(size() - 1)]; } + + using value_type = T; + using reference = T; + using const_reference = T; + using pointer = void; + using const_pointer = void; using size_type = size_t; using difference_type = ptrdiff_t; - using iterator = typename std::vector>::iterator; - using const_iterator = typename std::vector>::const_iterator; - using reverse_iterator = - typename std::vector>::reverse_iterator; - using const_reverse_iterator = - typename std::vector>::const_reverse_iterator; + class const_iterator { + public: + using iterator_category = std::bidirectional_iterator_tag; + using value_type = T; + using difference_type = ptrdiff_t; + using pointer = void; + using reference = T; + + const_iterator() = default; + const_iterator(const MessageVectorField* field, size_t index) + : field_(field), index_(index) {} + T operator*() const { return field_->Get(index_); } + const_iterator& operator++() { + ++index_; + return *this; + } + const_iterator operator++(int) { + const_iterator result = *this; + ++*this; + return result; + } + const_iterator& operator--() { + --index_; + return *this; + } + const_iterator operator--(int) { + const_iterator result = *this; + --*this; + return result; + } + bool operator==(const const_iterator& other) const { + return field_ == other.field_ && index_ == other.index_; + } + bool operator!=(const const_iterator& other) const { + return !(*this == other); + } - iterator begin() { - Populate(); - return msgs_.begin(); - } - iterator end() { - Populate(); - return msgs_.end(); - } - reverse_iterator rbegin() { - Populate(); - return msgs_.rbegin(); - } - reverse_iterator rend() { - Populate(); - return msgs_.rend(); - } + private: + const MessageVectorField* field_ = nullptr; + size_t index_ = 0; + }; + using iterator = const_iterator; + using reverse_iterator = std::reverse_iterator; + using const_reverse_iterator = std::reverse_iterator; + + iterator begin() { return iterator(this, 0); } + iterator end() { return iterator(this, size()); } + reverse_iterator rbegin() { return reverse_iterator(end()); } + reverse_iterator rend() { return reverse_iterator(begin()); } const_iterator begin() const { - Populate(); - return msgs_.begin(); + return const_iterator(this, 0); } const_iterator end() const { - Populate(); - return msgs_.end(); + return const_iterator(this, size()); } const_iterator cbegin() const { - Populate(); - return msgs_.cbegin(); + return begin(); } const_iterator cend() const { - Populate(); - return msgs_.cend(); + return end(); } const_reverse_iterator rbegin() const { - Populate(); - return msgs_.rbegin(); + return const_reverse_iterator(end()); } const_reverse_iterator rend() const { - Populate(); - return msgs_.rend(); + return const_reverse_iterator(begin()); } const_reverse_iterator crbegin() const { - Populate(); - return msgs_.crbegin(); + return rbegin(); } const_reverse_iterator crend() const { - Populate(); - return msgs_.crend(); + return rend(); } void push_back(const T& v) { ::toolbelt::BufferOffset offset = v.absolute_binary_offset; ::toolbelt::PayloadBuffer::VectorPush<::toolbelt::BufferOffset>( GetBufferAddr(), Header(), offset); - MessageObject obj(GetRuntime(), offset); - obj.msg_ = v; - msgs_.push_back(std::move(obj)); } void push_back(T&& v) { ::toolbelt::BufferOffset offset = v.absolute_binary_offset; ::toolbelt::PayloadBuffer::VectorPush<::toolbelt::BufferOffset>( GetBufferAddr(), Header(), offset); - MessageObject obj(GetRuntime(), offset); - obj.msg_ = v; - msgs_.push_back(std::move(obj)); } - T* Add() { + T Add() { // Allocate a new message. void* binary = ::toolbelt::PayloadBuffer::Allocate(GetBufferAddr(), T::BinarySize()); @@ -948,61 +940,59 @@ class MessageVectorField : public Field { GetRuntime()->ToOffset(binary); ::toolbelt::PayloadBuffer::VectorPush<::toolbelt::BufferOffset>( GetBufferAddr(), Header(), absolute_binary_offset); - auto obj = MessageObject(GetRuntime(), absolute_binary_offset); - obj.InstallMetadata(); - msgs_.push_back(std::move(obj)); - return msgs_.back().Mutable(); + T result(GetRuntime(), absolute_binary_offset); + result.template InstallMetadata(); + return result; } - const T& Get(size_t index) const { - return (*this)[static_cast(index)].Get(); - } + T Get(size_t index) const { return (*this)[static_cast(index)]; } - T* Mutable(size_t index) { - if (static_cast(index) >= msgs_.size()) { + T Mutable(size_t index) { + if (index >= size()) { ::toolbelt::PayloadBuffer::VectorResize<::toolbelt::BufferOffset>( - GetBufferAddr(), Header(), static_cast(index) + 1); - msgs_.resize(static_cast(index) + 1); + GetBufferAddr(), Header(), index + 1); } - if (msgs_[static_cast(index)].IsPlaceholder()) { + auto hdr = Header(); + ::toolbelt::BufferOffset* data = + GetRuntime()->template ToAddress<::toolbelt::BufferOffset>(hdr->data); + if (data[index] == 0) { void* binary = ::toolbelt::PayloadBuffer::Allocate(GetBufferAddr(), T::BinarySize()); ::toolbelt::BufferOffset absolute_binary_offset = GetRuntime()->ToOffset(binary); - auto hdr = Header(); - ::toolbelt::BufferOffset* data = + hdr = Header(); + data = GetRuntime()->template ToAddress<::toolbelt::BufferOffset>(hdr->data); data[index] = absolute_binary_offset; - - auto obj = MessageObject(GetRuntime(), absolute_binary_offset); - obj.InstallMetadata(); - msgs_[static_cast(index)] = std::move(obj); + T result(GetRuntime(), absolute_binary_offset); + result.template InstallMetadata(); + return result; } - return msgs_[static_cast(index)].Mutable(); + return T(GetRuntime(), data[index]); } void SetOffset(int index, toolbelt::BufferOffset offset) { - if (static_cast(index) >= msgs_.size()) { - msgs_.resize(static_cast(index) + 1); + if (static_cast(index) >= size()) { + ::toolbelt::PayloadBuffer::VectorResize<::toolbelt::BufferOffset>( + GetBufferAddr(), Header(), static_cast(index) + 1); } auto hdr = Header(); ::toolbelt::BufferOffset* data = GetRuntime()->template ToAddress<::toolbelt::BufferOffset>(hdr->data); - if (!msgs_[static_cast(index)].IsPlaceholder() && - data[index] != 0) { + if (data[index] != 0) { // Already set, free the current value. - msgs_[static_cast(index)].Clear(); + T(GetRuntime(), data[index]).Clear(); + GetBuffer()->Free(GetRuntime()->ToAddress(data[index])); } data[index] = offset; - msgs_[static_cast(index)] = MessageObject(GetRuntime(), offset); } // Allocate a bunch of empty messages. - std::vector Allocate(size_t n) { - std::vector result; - result.resize(n); + std::vector Allocate(size_t n) { + std::vector result; + result.reserve(n); this->resize(n); // Allocate memory for n messages in the payload buffer. std::vector addrs = ::toolbelt::PayloadBuffer::AllocateMany( @@ -1012,15 +1002,11 @@ class MessageVectorField : public Field { ::toolbelt::BufferOffset* data = GetRuntime()->template ToAddress<::toolbelt::BufferOffset>(hdr->data); - // Fill in the msgs_ vector with MessageObject objects referring to the - // allocated memory. for (size_t i = 0; i < n; i++) { - auto& msg = msgs_[i].MutableMsg(); - msg.runtime = GetRuntime(); toolbelt::BufferOffset offset = GetRuntime()->ToOffset(addrs[i]); - msg.absolute_binary_offset = offset; - msgs_[i].InstallMetadata(); - result[i] = &msg; + T msg(GetRuntime(), offset); + msg.template InstallMetadata(); + result.push_back(std::move(msg)); data[i] = offset; } return result; @@ -1041,14 +1027,12 @@ class MessageVectorField : public Field { void reserve(size_t n) { ::toolbelt::PayloadBuffer::VectorReserve<::toolbelt::BufferOffset>( GetBufferAddr(), Header(), n); - msgs_.reserve(n); } void resize(size_t n) { // Resize the vector data in the binary. This contains BufferOffets. ::toolbelt::PayloadBuffer::VectorResize<::toolbelt::BufferOffset>( GetBufferAddr(), Header(), n); - msgs_.resize(n); } void Clear() { @@ -1056,18 +1040,14 @@ class MessageVectorField : public Field { auto data = GetRuntime()->template ToAddress<::toolbelt::BufferOffset>(hdr->data); for (uint32_t i = 0; i < hdr->num_elements; i++) { - if (msgs_[i].empty()) { - continue; - } - msgs_[i].Clear(); if (data[i] == 0) { continue; } + T(GetRuntime(), data[i]).Clear(); GetBuffer()->Free(GetRuntime()->ToAddress(data[i])); } ::toolbelt::PayloadBuffer::VectorClear<::toolbelt::BufferOffset>( GetBufferAddr(), Header()); - msgs_.clear(); } size_t size() const { return NumElements(); } @@ -1079,11 +1059,10 @@ class MessageVectorField : public Field { return *this; } Clear(); - other.Populate(); reserve(other.size()); for (size_t i = 0; i < other.size(); i++) { - T* m = Add(); - if (absl::Status s = m->CloneFrom(other[i].Get()); !s.ok()) { + T m = Add(); + if (absl::Status s = m.CloneFrom(other.Get(i)); !s.ok()) { return *this; } } @@ -1102,57 +1081,49 @@ class MessageVectorField : public Field { } bool operator==(const MessageVectorField& other) const { - return msgs_ != other.msgs_; + if (size() != other.size()) { + return false; + } + for (size_t i = 0; i < size(); ++i) { + if (Get(i) != other.Get(i)) { + return false; + } + } + return true; } bool operator!=(const MessageVectorField& other) const { - return !(*this == other); + return !operator==(other); } - std::vector>& Get() { return msgs_; } - - const std::vector>& Get() const { return msgs_; } - - void Populate() const { - if (!msgs_.empty()) { - return; - } - // Populate the msgs vector with MessageObject objects referring to the - // binary messages. - int32_t offset = FindFieldOffset(source_offset_); - if (offset == -1) { - return; - } - auto hdr = Header(static_cast(offset)); - msgs_.resize(hdr->num_elements); - ::toolbelt::BufferOffset* data = - GetRuntime()->template ToAddress<::toolbelt::BufferOffset>(hdr->data); - for (uint32_t i = 0; i < hdr->num_elements; i++) { - if (data[i] == 0) { - continue; - } - MessageObject obj(GetRuntime(), data[i]); - msgs_[i] = std::move(obj); + std::vector Get() const { + std::vector result; + result.reserve(size()); + for (size_t i = 0; i < size(); ++i) { + result.push_back(Get(i)); } + return result; } + void Populate() const {} + size_t SerializedSize() const { - Populate(); size_t length = 0; for (size_t i = 0; i < size(); i++) { + const T message = Get(i); length += phaser::ProtoBuffer::LengthDelimitedSize( - Number(), msgs_[i].SerializedSize()); + Number(), message.SerializedSize()); } return length; } absl::Status Serialize(ProtoBuffer& buffer) const { - Populate(); size_t sz = size(); if (sz == 0) { return absl::OkStatus(); } - for (const auto& msg : msgs_) { + for (size_t i = 0; i < sz; ++i) { + const T msg = Get(i); if (absl::Status status = buffer.SerializeLengthDelimitedHeader( Number(), msg.SerializedSize()); !status.ok()) { @@ -1171,19 +1142,16 @@ class MessageVectorField : public Field { return v.status(); } ProtoBuffer msg_buffer(*v); - T* msg = Add(); - if (absl::Status status = msg->Deserialize(msg_buffer); !status.ok()) { + T msg = Add(); + if (absl::Status status = msg.Deserialize(msg_buffer); !status.ok()) { return status; } return absl::OkStatus(); } void SyncToPayload() const { - Populate(); - for (const auto& message : msgs_) { - if (!message.empty()) { - message.Get().SyncToPayload(); - } + for (size_t i = 0; i < size(); ++i) { + Get(i).SyncToPayload(); } } @@ -1233,8 +1201,6 @@ class MessageVectorField : public Field { uint32_t source_offset_; ::toolbelt::BufferOffset relative_binary_offset_; - mutable std::vector> msgs_; - MessageObject empty_; }; // This is a little more complex. The binary vector contains a set of @@ -1267,137 +1233,138 @@ class StringVectorField : public Field { source_offset_(other.source_offset_), relative_binary_offset_(other.relative_binary_offset_) {} - const NonEmbeddedStringField& operator[](int index) const { + std::string_view operator[](int index) const { int32_t offset = FindFieldOffset(source_offset_); if (offset == -1) { - return empty_; + return {}; } auto hdr = Header(static_cast(offset)); if (static_cast(index) >= hdr->num_elements) { - return empty_; + return {}; } ::toolbelt::BufferOffset* data = GetRuntime()->template ToAddress<::toolbelt::BufferOffset>(hdr->data); if (data[index] == 0) { - return empty_; - } - if (static_cast(index) >= strings_.size()) { - strings_.resize(static_cast(index) + 1); - } - if (strings_[static_cast(index)].IsPlaceholder()) { - strings_[static_cast(index)] = NonEmbeddedStringField( - Message::GetMessage(this, source_offset_), data[index]); + return {}; } - return strings_[static_cast(index)]; + return GetBuffer()->GetStringView(data[index]); } - NonEmbeddedStringField& operator[](int index) { - return const_cast( - static_cast(this)->operator[](index)); + NonEmbeddedStringField operator[](int index) { + int32_t offset = FindFieldOffset(source_offset_); + if (offset == -1) { + return {}; + } + auto hdr = Header(static_cast(offset)); + if (static_cast(index) >= hdr->num_elements) { + return {}; + } + ::toolbelt::BufferOffset* data = + GetRuntime()->template ToAddress<::toolbelt::BufferOffset>(hdr->data); + return NonEmbeddedStringField(Message::GetMessage(this, source_offset_), + data[index]); } - using value_type = NonEmbeddedStringField; - using reference = value_type&; - using const_reference = value_type&; - using pointer = value_type*; - using const_pointer = const value_type*; + using value_type = std::string_view; + using reference = std::string_view; + using const_reference = std::string_view; + using pointer = void; + using const_pointer = void; using size_type = size_t; using difference_type = ptrdiff_t; - using iterator = typename std::vector::iterator; - using const_iterator = - typename std::vector::const_iterator; - using reverse_iterator = - typename std::vector::reverse_iterator; - using const_reverse_iterator = - typename std::vector::const_reverse_iterator; - - iterator begin() { - Populate(); - return strings_.begin(); - } - iterator end() { - Populate(); - return strings_.end(); - } - reverse_iterator rbegin() { - Populate(); - return strings_.rbegin(); - } - reverse_iterator rend() { - Populate(); - return strings_.rend(); - } + class const_iterator { + public: + using iterator_category = std::bidirectional_iterator_tag; + using value_type = std::string_view; + using difference_type = ptrdiff_t; + using pointer = void; + using reference = std::string_view; + + const_iterator() = default; + const_iterator(const StringVectorField* field, size_t index) + : field_(field), index_(index) {} + std::string_view operator*() const { return field_->Get(index_); } + const_iterator& operator++() { + ++index_; + return *this; + } + const_iterator operator++(int) { + const_iterator result = *this; + ++*this; + return result; + } + const_iterator& operator--() { + --index_; + return *this; + } + const_iterator operator--(int) { + const_iterator result = *this; + --*this; + return result; + } + bool operator==(const const_iterator& other) const { + return field_ == other.field_ && index_ == other.index_; + } + bool operator!=(const const_iterator& other) const { + return !(*this == other); + } + + private: + const StringVectorField* field_ = nullptr; + size_t index_ = 0; + }; + using iterator = const_iterator; + using reverse_iterator = std::reverse_iterator; + using const_reverse_iterator = std::reverse_iterator; + + iterator begin() { return iterator(this, 0); } + iterator end() { return iterator(this, size()); } + reverse_iterator rbegin() { return reverse_iterator(end()); } + reverse_iterator rend() { return reverse_iterator(begin()); } const_iterator begin() const { - Populate(); - return strings_.begin(); + return const_iterator(this, 0); } const_iterator end() const { - Populate(); - return strings_.end(); + return const_iterator(this, size()); } const_iterator cbegin() const { - Populate(); - return strings_.cbegin(); + return begin(); } const_iterator cend() const { - Populate(); - return strings_.cend(); + return end(); } const_reverse_iterator rbegin() const { - Populate(); - return strings_.rbegin(); + return const_reverse_iterator(end()); } const_reverse_iterator rend() const { - Populate(); - return strings_.rend(); + return const_reverse_iterator(begin()); } const_reverse_iterator crbegin() const { - Populate(); - return strings_.crbegin(); + return rbegin(); } const_reverse_iterator crend() const { - Populate(); - return strings_.crend(); + return rend(); } size_t size() const { return NumElements(); } - NonEmbeddedStringField* data() { - Populate(); - return strings_.data(); - } - const NonEmbeddedStringField* data() const { - Populate(); - return strings_.data(); - } + NonEmbeddedStringField* data() = delete; + const NonEmbeddedStringField* data() const = delete; bool empty() const { return size() == 0; } size_t Size() const { return NumElements(); } - NonEmbeddedStringField& front() { - Populate(); - return strings_.front(); - } - const NonEmbeddedStringField& front() const { - Populate(); - return strings_.front(); - } - NonEmbeddedStringField& back() { - Populate(); - return strings_.back(); - } - const NonEmbeddedStringField& back() const { - Populate(); - return strings_.back(); - } + NonEmbeddedStringField front() { return (*this)[0]; } + std::string_view front() const { return Get(0); } + NonEmbeddedStringField back() { return (*this)[static_cast(size() - 1)]; } + std::string_view back() const { return Get(size() - 1); } StringVectorField& operator=(const StringVectorField& other) { if (this == &other) { return *this; } Clear(); - other.Populate(); reserve(other.size()); for (size_t i = 0; i < other.size(); i++) { - push_back(other[i].Get()); + push_back(other.Get(i)); } ResetFieldCache(); return *this; @@ -1419,47 +1386,44 @@ class StringVectorField : public Field { ::toolbelt::PayloadBuffer::VectorPush<::toolbelt::BufferOffset>( GetBufferAddr(), Header(), hdr_offset); - // Add a source string field. - NonEmbeddedStringField field(Message::GetMessage(this, source_offset_), - hdr_offset); - strings_.push_back(std::move(field)); } - void Add(const char* s, size_t len) { push_back(std::string(s, len)); } + void Add(const char* s, size_t len) { + push_back(std::string_view(s, len)); + } template void Add(Str s) { push_back(s); } std::string_view Get(size_t index) const { - return (*this)[static_cast(index)].Get(); + return (*this)[static_cast(index)]; } template void Set(size_t index, Str s) { - if (static_cast(index) >= strings_.size()) { + if (index >= size()) { ::toolbelt::PayloadBuffer::VectorResize<::toolbelt::BufferOffset>( - GetBufferAddr(), Header(), static_cast(index) + 1); - strings_.resize(static_cast(index) + 1); + GetBufferAddr(), Header(), index + 1); } - if (strings_[static_cast(index)].IsPlaceholder()) { + auto hdr = Header(); + ::toolbelt::BufferOffset* data = + GetRuntime()->template ToAddress<::toolbelt::BufferOffset>(hdr->data); + if (data[index] == 0) { // Allocate string header in buffer. void* str_hdr = ::toolbelt::PayloadBuffer::Allocate( GetBufferAddr(), sizeof(toolbelt::StringHeader)); ::toolbelt::BufferOffset hdr_offset = GetRuntime()->ToOffset(str_hdr); - auto hdr = Header(); - ::toolbelt::BufferOffset* data = + hdr = Header(); + data = GetRuntime()->template ToAddress<::toolbelt::BufferOffset>(hdr->data); data[index] = hdr_offset; - - // Add a source string field. - NonEmbeddedStringField field(Message::GetMessage(this, source_offset_), - hdr_offset); - strings_[static_cast(index)] = std::move(field); } - strings_[static_cast(index)].Set(s); + NonEmbeddedStringField(Message::GetMessage(this, source_offset_), + data[index]) + .Set(s); } size_t capacity() const { @@ -1477,23 +1441,20 @@ class StringVectorField : public Field { void reserve(size_t n) { ::toolbelt::PayloadBuffer::VectorReserve<::toolbelt::BufferOffset>( GetBufferAddr(), Header(), n); - strings_.reserve(n); } void resize(size_t n) { // Resize the vector data in the binary. This contains BufferOffets. ::toolbelt::PayloadBuffer::VectorResize<::toolbelt::BufferOffset>( GetBufferAddr(), Header(), n); - strings_.resize(n); } void Clear() { - for (auto& s : strings_) { - if (!s.IsPlaceholder()) { - s.Clear(); - } + const size_t count = size(); + for (size_t i = 0; i < count; ++i) { + NonEmbeddedStringField field = (*this)[static_cast(i)]; + field.Clear(); } - strings_.clear(); ::toolbelt::PayloadBuffer::VectorClear<::toolbelt::BufferOffset>( GetBufferAddr(), Header()); } @@ -1508,61 +1469,48 @@ class StringVectorField : public Field { } bool operator==(const StringVectorField& other) const { - return strings_ == other.strings_; + if (size() != other.size()) { + return false; + } + for (size_t i = 0; i < size(); ++i) { + if (Get(i) != other.Get(i)) { + return false; + } + } + return true; } bool operator!=(const StringVectorField& other) const { return !(*this == other); } - // Populate the vector with the strings from the binary message. This must be - // called before you access the vector via iterators. - void Populate() const { - if (!strings_.empty()) { - return; - } - int32_t offset = FindFieldOffset(source_offset_); - if (offset == -1) { - return; - } - auto hdr = Header(static_cast(offset)); - strings_.resize(hdr->num_elements); - ::toolbelt::BufferOffset* data = - GetRuntime()->template ToAddress<::toolbelt::BufferOffset>(hdr->data); - for (uint32_t i = 0; i < hdr->num_elements; i++) { - if (data[i] == 0) { - continue; - } - strings_[i] = NonEmbeddedStringField( - Message::GetMessage(this, source_offset_), data[i]); - } - } + void Populate() const {} std::vector Get() const { std::vector r; - for (const auto& s : strings_) { - r.push_back(s.Get()); + r.reserve(size()); + for (size_t i = 0; i < size(); ++i) { + r.push_back(Get(i)); } return r; } size_t SerializedSize() const { - Populate(); size_t length = 0; for (size_t i = 0; i < size(); i++) { - length += phaser::ProtoBuffer::LengthDelimitedSize( - Number(), strings_[i].SerializedSize()); + length += + phaser::ProtoBuffer::LengthDelimitedSize(Number(), Get(i).size()); } return length; } absl::Status Serialize(ProtoBuffer& buffer) const { - Populate(); size_t sz = size(); if (sz == 0) { return absl::OkStatus(); } - for (const auto& s : strings_) { + for (size_t i = 0; i < sz; ++i) { + const std::string_view s = Get(i); if (absl::Status status = buffer.SerializeLengthDelimited(Number(), s.data(), s.size()); !status.ok()) { @@ -1623,8 +1571,6 @@ class StringVectorField : public Field { } uint32_t source_offset_; ::toolbelt::BufferOffset relative_binary_offset_; - mutable std::vector strings_; - NonEmbeddedStringField empty_; }; #undef DECLARE_ZERO_COPY_VECTOR_BITS diff --git a/phaser/runtime/wireformat.h b/phaser/runtime/wireformat.h index da03ed1..39ee98c 100644 --- a/phaser/runtime/wireformat.h +++ b/phaser/runtime/wireformat.h @@ -646,4 +646,103 @@ class ProtoBuffer { char* end_ = nullptr; }; +// A protobuf sink that either writes to a ProtoBuffer or only counts bytes. +// The counting mode is used to size nested and packed fields without staging +// their encoded bytes in a temporary allocation. +class ProtoWriter { + public: + ProtoWriter() = default; + explicit ProtoWriter(ProtoBuffer& buffer) : buffer_(&buffer) {} + + size_t Size() const { return size_; } + + template + absl::Status SerializeVarint(int field_number, T value) { + const size_t bytes = + ProtoBuffer::TagSize(field_number, WireType::kVarint) + + ProtoBuffer::VarintSize(value); + if (buffer_ != nullptr) { + if (absl::Status status = + buffer_->SerializeVarint(field_number, value); + !status.ok()) { + return status; + } + } + size_ += bytes; + return absl::OkStatus(); + } + + template + absl::Status SerializeRawVarint(T value) { + const size_t bytes = ProtoBuffer::VarintSize(value); + if (buffer_ != nullptr) { + if (absl::Status status = buffer_->SerializeRawVarint(value); + !status.ok()) { + return status; + } + } + size_ += bytes; + return absl::OkStatus(); + } + + template + absl::Status SerializeFixed(int field_number, T value) { + const size_t bytes = + ProtoBuffer::TagSize(field_number, ProtoBuffer::FixedWireType()) + + sizeof(T); + if (buffer_ != nullptr) { + if (absl::Status status = buffer_->SerializeFixed(field_number, value); + !status.ok()) { + return status; + } + } + size_ += bytes; + return absl::OkStatus(); + } + + absl::Status SerializeLengthDelimited(int field_number, const void* data, + size_t length) { + const size_t bytes = ProtoBuffer::LengthDelimitedSize(field_number, length); + if (buffer_ != nullptr) { + if (absl::Status status = + buffer_->SerializeLengthDelimited(field_number, data, length); + !status.ok()) { + return status; + } + } + size_ += bytes; + return absl::OkStatus(); + } + + absl::Status SerializeLengthDelimitedHeader(int field_number, + size_t length) { + const size_t bytes = ProtoBuffer::LengthDelimitedSize(field_number, length) - + length; + if (buffer_ != nullptr) { + if (absl::Status status = + buffer_->SerializeLengthDelimitedHeader(field_number, length); + !status.ok()) { + return status; + } + } + size_ += bytes; + return absl::OkStatus(); + } + + absl::Status SerializeRaw(const void* data, size_t length) { + if (buffer_ != nullptr) { + if (absl::Status status = buffer_->SerializeRaw(data, length); + !status.ok()) { + return status; + } + } + size_ += length; + return absl::OkStatus(); + } + + private: + ProtoBuffer* buffer_ = nullptr; + size_t size_ = 0; +}; + } // namespace phaser diff --git a/phaser/stress_test.cc b/phaser/stress_test.cc index 0671939..70f326f 100644 --- a/phaser/stress_test.cc +++ b/phaser/stress_test.cc @@ -115,7 +115,7 @@ TEST(StressTest, MapManyEntries) { TestMessage msg(4096); ASSERT_FALSE(msg.allocate_buffer(128 * 1024).empty()); for (int i = 0; i < 500; i++) { - auto* e = msg.add_values(); + auto e = msg.add_values(); e->set_key(::phaser::test::MakePatternString( 8, static_cast('k' + (i % 10)))); e->set_value(i); diff --git a/phaser/testdata/RosCompile.proto b/phaser/testdata/RosCompile.proto index 58ff714..e3a5a16 100644 --- a/phaser/testdata/RosCompile.proto +++ b/phaser/testdata/RosCompile.proto @@ -38,3 +38,13 @@ message RosCompileMessage { RosInner choice_inner = 18; } } + +message RosPackedFixedMessage { + repeated fixed32 fixed32_values = 1; + repeated sfixed32 sfixed32_values = 2; + repeated float float_values = 3; + repeated fixed64 fixed64_values = 4; + repeated sfixed64 sfixed64_values = 5; + repeated double double_values = 6; + repeated fixed64 fixed_array = 7 [(phaser.array_size) = 3]; +} diff --git a/phaser/testdata/TestMessage.proto b/phaser/testdata/TestMessage.proto index a535c51..22cf961 100644 --- a/phaser/testdata/TestMessage.proto +++ b/phaser/testdata/TestMessage.proto @@ -1,18 +1,19 @@ syntax = "proto3"; -// There are tests in message_test.cc that use both the messages hand-coded in there -// and InnnerMessage and TestMessage defined here. The messages defined here should -// be serialization-compatible with the hand-coded messages in message_test.cc. -// So this means that the field numbers and types should match up. Also, don't -// delete fields from these messages, as the tests in message_test.cc rely on them. -// You can add fields to these messages without also adding them to the hand-coded -// messages because this is compatible for serialization. +// There are tests in message_test.cc that use both the messages hand-coded in +// there and InnnerMessage and TestMessage defined here. The messages defined +// here should be serialization-compatible with the hand-coded messages in +// message_test.cc. So this means that the field numbers and types should match +// up. Also, don't delete fields from these messages, as the tests in +// message_test.cc rely on them. You can add fields to these messages without +// also adding them to the hand-coded messages because this is compatible for +// serialization. package foo.bar; import "google/protobuf/any.proto"; enum EnumTest { - UNSET = 0; - FOO = 0xda; + UNSET = 0; + FOO = 0xda; BAR = 0xad; } @@ -113,4 +114,155 @@ message TestMessageDeletedFields { int64 u3a = 111; InnerMessage u3b = 112; } +} + +message HybridLookupMessage { + int32 dense_10 = 10; + int32 dense_11 = 11; + int32 dense_13 = 13; + int32 dense_14 = 14; + int32 dense_15 = 15; + int32 sparse_1000 = 1000; +} + +message SparseLookupMessage { + int32 field_1 = 1; + int32 field_100 = 100; + int32 field_10000 = 10000; +} + +// Same shape and field types, but deliberately different field-number +// distributions for end-to-end Phaser field-access benchmarks. +message DenseLookupBenchmarkMessage { + int32 value_01 = 1; + int32 value_02 = 2; + int32 value_03 = 3; + int32 value_04 = 4; + int32 value_05 = 5; + int32 value_06 = 6; + int32 value_07 = 7; + int32 value_08 = 8; + int32 value_09 = 9; + int32 value_10 = 10; + int32 value_11 = 11; + int32 value_12 = 12; + int32 value_13 = 13; + int32 value_14 = 14; + int32 value_15 = 15; + int32 value_16 = 16; + int32 value_17 = 17; + int32 value_18 = 18; + int32 value_19 = 19; + int32 value_20 = 20; + int32 value_21 = 21; + int32 value_22 = 22; + int32 value_23 = 23; + int32 value_24 = 24; + int32 value_25 = 25; + int32 value_26 = 26; + int32 value_27 = 27; + int32 value_28 = 28; + int32 value_29 = 29; + int32 value_30 = 30; + int32 value_31 = 31; + int32 value_32 = 32; + int32 value_33 = 33; + int32 value_34 = 34; + int32 value_35 = 35; + int32 value_36 = 36; + int32 value_37 = 37; + int32 value_38 = 38; + int32 value_39 = 39; + int32 value_40 = 40; + int32 value_41 = 41; + int32 value_42 = 42; + int32 value_43 = 43; + int32 value_44 = 44; + int32 value_45 = 45; + int32 value_46 = 46; + int32 value_47 = 47; + int32 value_48 = 48; + int32 value_49 = 49; + int32 value_50 = 50; + int32 value_51 = 51; + int32 value_52 = 52; + int32 value_53 = 53; + int32 value_54 = 54; + int32 value_55 = 55; + int32 value_56 = 56; + int32 value_57 = 57; + int32 value_58 = 58; + int32 value_59 = 59; + int32 value_60 = 60; + int32 value_61 = 61; + int32 value_62 = 62; + int32 value_63 = 63; + int32 value_64 = 64; +} + +message SparseLookupBenchmarkMessage { + int32 value_01 = 1; + int32 value_02 = 101; + int32 value_03 = 201; + int32 value_04 = 301; + int32 value_05 = 401; + int32 value_06 = 501; + int32 value_07 = 601; + int32 value_08 = 701; + int32 value_09 = 801; + int32 value_10 = 901; + int32 value_11 = 1001; + int32 value_12 = 1101; + int32 value_13 = 1201; + int32 value_14 = 1301; + int32 value_15 = 1401; + int32 value_16 = 1501; + int32 value_17 = 1601; + int32 value_18 = 1701; + int32 value_19 = 1801; + int32 value_20 = 1901; + int32 value_21 = 2001; + int32 value_22 = 2101; + int32 value_23 = 2201; + int32 value_24 = 2301; + int32 value_25 = 2401; + int32 value_26 = 2501; + int32 value_27 = 2601; + int32 value_28 = 2701; + int32 value_29 = 2801; + int32 value_30 = 2901; + int32 value_31 = 3001; + int32 value_32 = 3101; + int32 value_33 = 3201; + int32 value_34 = 3301; + int32 value_35 = 3401; + int32 value_36 = 3501; + int32 value_37 = 3601; + int32 value_38 = 3701; + int32 value_39 = 3801; + int32 value_40 = 3901; + int32 value_41 = 4001; + int32 value_42 = 4101; + int32 value_43 = 4201; + int32 value_44 = 4301; + int32 value_45 = 4401; + int32 value_46 = 4501; + int32 value_47 = 4601; + int32 value_48 = 4701; + int32 value_49 = 4801; + int32 value_50 = 4901; + int32 value_51 = 5001; + int32 value_52 = 5101; + int32 value_53 = 5201; + int32 value_54 = 5301; + int32 value_55 = 5401; + int32 value_56 = 5501; + int32 value_57 = 5601; + int32 value_58 = 5701; + int32 value_59 = 5801; + int32 value_60 = 5901; + int32 value_61 = 6001; + int32 value_62 = 6101; + int32 value_63 = 6201; + int32 value_64 = 6301; } \ No newline at end of file diff --git a/phaser/testdata/ros_intrinsics_phaser_wire_tool.cc b/phaser/testdata/ros_intrinsics_phaser_wire_tool.cc index 4fde34a..79d0433 100644 --- a/phaser/testdata/ros_intrinsics_phaser_wire_tool.cc +++ b/phaser/testdata/ros_intrinsics_phaser_wire_tool.cc @@ -26,10 +26,10 @@ bool Write(const char* path) { message.tags.push_back("front"); message.tags.push_back("rear"); - auto* first_child = message.children.Add(); + auto first_child = message.children.Add(); first_child->id = 101; first_child->label = "left"; - auto* second_child = message.children.Add(); + auto second_child = message.children.Add(); second_child->id = 202; second_child->label = "right"; From 294bac7ef4e918829e1a221a19c7dba90b041cfd Mon Sep 17 00:00:00 2001 From: Dave Allison Date: Wed, 12 Aug 2026 16:04:12 -0700 Subject: [PATCH 6/7] Document frontends and wire conversions Explain both generated APIs and show direct protobuf, ROS, and native wire conversion workflows. --- README.md | 131 ++++++++++++++++----- phaser/docs/phaser_user_guide.md | 191 +++++++++++++++++++++++++------ 2 files changed, 264 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index 4caa5ff..6727a96 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,9 @@ wire-format, instead of in a tree of heap-allocated objects. Once a message is b it can be written to disk, placed in shared memory, or sent over an IPC system **without a serialization step** — the bytes in the buffer *are* the message. -The generated API is intentionally almost identical to the standard protobuf C++ API, so -if you know protobuf, you already know Phaser. +Choose a familiar protobuf-style C++ API or a ROS-style public-field API. Both +frontends use the same native payload and expose the same protobuf and ROS wire +conversion backends. > 📖 For the full reference, see the **[Phaser User Guide](phaser/docs/phaser_user_guide.md)**. @@ -47,20 +48,26 @@ costs nothing until you actually touch a field. interoperate via per-message field metadata. - **Protobuf wire-format transcoding** is available when you *do* need it (e.g. storing in systems like BigQuery that expect protobuf bytes). -- **Familiar API** — same accessor names as protobuf, with extra zero-copy helpers. +- **Two C++ frontends** — protobuf-style accessors or ROS-style public field proxies. +- **Direct wire-to-wire conversion** between protobuf and ROS1 without constructing + an intermediate user-facing message. +- **Allocation-free receive paths** when native payloads and wire conversions use + caller-provided buffers. ## Features 1. proto3 (primary) and proto2 IDL support -2. Message printing to `std::ostream` -3. Fixed- and variable-sized buffers -4. User-supplied per-buffer metadata -5. Full `google.protobuf.Any` support (zero-copy) -6. Enum printing and parsing -7. Message reflection -8. Field presence masks -9. Bazel build integration -10. Modern C++17 with [Abseil](https://abseil.io) +2. Protobuf and ROS-style generated C++ frontends +3. Native Phaser, protobuf, and ROS1 wire input/output +4. Direct protobuf-to-ROS and ROS-to-protobuf backend conversion +5. Fixed, caller-owned, and dynamically growing buffers +6. Allocation-free typed receive, traversal, and fixed-buffer transcoding +7. Full zero-copy `google.protobuf.Any` support +8. Repeated vectors, ROS fixed-array facades, and variant-like ROS `oneof` fields +9. ROS1 `Time`, `Duration`, and `Header` intrinsic mappings +10. Hybrid dense/sparse field metadata and protobuf version compatibility +11. Reflection, enum conversion, field presence, and user metadata +12. Bazel integration and modern C++17 with [Abseil](https://abseil.io) ## How it works @@ -120,6 +127,7 @@ phaser_library( name = "foo_phaser", add_namespace = "phaser", # optional: avoids clashing with protobuf classes frontend = "protobuf", # default; use "ros" for public field proxies + enable_active_message = False, # optionally add std::any active_message deps = [":foo_proto"], ) ``` @@ -133,24 +141,43 @@ any protobuf header: ``` Phaser can generate either the default protobuf-style accessors or a ROS-style -struct interface. The ROS frontend preserves the same native payload layout and -protobuf wire transcoding: +struct interface: ```python phaser_library( name = "foo_ros_phaser", + add_namespace = "ros_api", frontend = "ros", deps = [":foo_proto"], ) ``` +The frontend changes only the C++ access syntax: + ```c++ -Foo msg; -msg.count = 3; -msg.name = "sensor"; -msg.samples.push_back(1.5); +// frontend = "protobuf" +foo::bar::phaser::Foo protobuf_api; +protobuf_api.set_count(3); +protobuf_api.set_name("sensor"); +protobuf_api.add_samples(1.5); + +// frontend = "ros" +foo::bar::ros_api::Foo ros_api; +ros_api.count = 3; +ros_api.name = "sensor"; +ros_api.samples.push_back(1.5); ``` +Both frontends preserve the same native payload layout. Code generated from the +same schema can therefore attach to the same received bytes with +`CreateReadonly`, regardless of which frontend produced them. Both also expose +protobuf serialization, ROS1 serialization, and the direct wire conversion +APIs described below. + +Set `enable_active_message = True` if each generated source object should also +carry an application-owned `std::any active_message`. This transient member is +not part of the native payload and is not serialized. + Fixed-size ROS fields are repeated protobuf fields annotated with `[(phaser.array_size) = N]`; import `phaser/options.proto` in the schema. ROS `oneof` fields expose a variant-like proxy with generated alternative tags. @@ -164,8 +191,8 @@ Header access returns `phaser::RosHeaderView`, whose `frame_id` is a `std::string_view`; call `ToOwned()` when an owning `std_msgs::Header` is required. Add the corresponding ROS C++ targets through the `cc_deps` attribute. -Every generated message, in either frontend style, can also produce ROS1 wire -bytes: +Every generated message, in either frontend style, can also produce and consume +ROS1 wire bytes: ```c++ ::phaser::ROSBuffer ros_output; @@ -241,8 +268,6 @@ wire view into payload storage without an owning `std::string` intermediate. Dynamic messages, reflection/debug output, unknown `Any` error handling, and explicit owning conversions such as `RosHeaderView::ToOwned()` are outside this guarantee. -Owning conversion/copy helpers, mutation, reflection, debug printing, and -string-returning serialization helpers are outside this guarantee. ### 3. Zero-copy field access @@ -257,8 +282,8 @@ absl::Span dst = msg.allocate_s(len); msg.resize_vi32(n); absl::Span data = msg.vi32_as_mutable_span(); -// Repeated messages: allocate many at once (one allocation). -std::vector items = msg.allocate_vm(n); +// Repeated messages: allocate many at once (one payload allocation). +std::vector items = msg.allocate_vm(n); ``` ## Protobuf interoperability @@ -279,6 +304,59 @@ bool ParseFromString(const std::string& str); binary message you can access directly (via `Is()` / `As()` / `MutableAny()`), with `PackFrom` / `UnpackTo` provided for protobuf-compatible copying. +## Wire-to-wire backend conversions + +Wire conversion APIs are generated for every message in both frontend styles. +They scan the source wire format and write the destination format directly; +they do not construct an intermediate protobuf object or Phaser source message. + +Convert protobuf bytes directly to ROS1: + +```c++ +std::string protobuf_wire = GetProtobufBytes(); +::phaser::ROSBuffer ros_output; // owns a growing output buffer +absl::Status status = + foo::bar::phaser::Foo::ProtobufToROS(protobuf_wire, ros_output); +if (status.ok()) { + SendROS(ros_output.data(), ros_output.size()); +} +``` + +Convert ROS1 bytes directly to protobuf using caller-owned output memory: + +```c++ +absl::Span ros_wire = ReceiveROS(); +std::array storage; +::phaser::ProtoBuffer protobuf_output(storage.data(), storage.size()); + +absl::Status status = + foo::bar::phaser::Foo::ROSToProtobuf(ros_wire, protobuf_output); +if (status.ok()) { + SendProtobuf(storage.data(), protobuf_output.Size()); +} +``` + +Convert a native Phaser payload to ROS1, or let Phaser distinguish native and +protobuf input: + +```c++ +const auto* data = static_cast(msg.Data()); +absl::Span native(data, msg.Size()); + +::phaser::ROSBuffer ros_output; +absl::Status status = + foo::bar::phaser::Foo::PhaserToROS(native, ros_output); + +// Accept either protobuf wire bytes or a native Phaser payload. +status = foo::bar::phaser::Foo::ConvertToROS(input, ros_output); +``` + +`ProtobufToROS` and `ROSToProtobuf` preserve schema order, recursively convert +nested messages, transform protobuf varints when required, and bulk-copy +wire-compatible packed fixed-width arrays. With fixed `ROSBuffer` and +`ProtoBuffer` instances, successful conversion performs no system-heap +allocation. `ParseFromROS` is the corresponding ROS1-to-native operation. + ## The Phaser Bank (type-erased operations & reflection) The **Phaser Bank** lets you operate on messages given only their type *name* — stream, @@ -325,13 +403,14 @@ and dependencies are available: ```bash protoc --plugin=protoc-gen-phaser=DIR/bin/phaser/compiler/phaser \ - --phaser_out=add_namespace=NS,package_name=PACKAGE,target_name=TARGET:OUTPUT_DIR \ + --phaser_out=frontend=ros,add_namespace=NS,package_name=PACKAGE,target_name=TARGET:OUTPUT_DIR \ -I IPATH \ FILE... ``` Output is written to `OUTPUT_DIR/PACKAGE/TARGET`. See the user guide for the full argument -reference. +reference. Use `frontend=protobuf` for the default accessor API and optionally +add `active_message=true`. ## Project layout diff --git a/phaser/docs/phaser_user_guide.md b/phaser/docs/phaser_user_guide.md index b9ad2ad..8117190 100644 --- a/phaser/docs/phaser_user_guide.md +++ b/phaser/docs/phaser_user_guide.md @@ -11,33 +11,33 @@ versions of messages interchangeably. This allows you to store the binary messa and later retrieve them with newer software, or to communicate with computers running different software versions. -The wire-format for Phaser messages is not the same as serialized protobuf format, -but for compatibility with protobuf, Phaser supports full protobuf serialization. This -is useful when storing messages in a format that is recognized by services such as -`BigQuery` where protobuf wire-format is required. +The native Phaser wire format is not the same as serialized protobuf or ROS1, +but Phaser supports both wire formats when interoperability is needed. Generated +backends can convert protobuf wire bytes directly to ROS1 and ROS1 directly to +protobuf without constructing an intermediate user-facing message. Other features include: 1. Target IDL is proto3, but proto2 is also supported -2. Message printing to std::ostream -3. Fixed and variable-sized buffers can be used -4. User-supplied metadata can be added to the message. -5. Full support for `google.protobuf.Any` -6. Enum printing and parsing -7. Message reflection -8. Presence masks -9. Bazel build -10. Modern C++17 with Google Abseil +2. Protobuf-style and ROS-style generated C++ frontends +3. Native Phaser, protobuf, and ROS1 wire input/output +4. Direct protobuf-to-ROS and ROS-to-protobuf conversion +5. Fixed, caller-owned, and dynamically growing buffers +6. Allocation-free typed receive and fixed-buffer transcoding +7. Full zero-copy support for `google.protobuf.Any` +8. ROS fixed arrays, variant-like oneofs, and ROS1 intrinsic types +9. Hybrid field metadata, reflection, enums, and presence masks +10. Bazel integration and modern C++17 with Google Abseil ## How Phaser works Phaser works as a plugin for the Protocol Buffers compiler (protoc). This means that protoc does the parsing of the `.proto` files and hands off to Phaser to generate code. Only C++ output is supported. -To the programmer, the messages generated by Phaser look just like those generated by -the regular C++ protobuf backend, with a few enhancements and minor incompatibilities. They -are generated as C++ classes with accessor functions for each field - with the same names -as those generated by protobuf. +The default generated frontend looks like the regular C++ protobuf backend, +with a few enhancements and minor incompatibilities. The optional ROS frontend +instead exposes public field proxies while retaining the same native payload +and wire-conversion APIs. However, while protobuf creates a tree of allocated objects holding the message data, Phaser's message classes do not store the actual field values directly, but rather the values are @@ -139,6 +139,8 @@ proto_library( phaser_library( name = "foo_phaser", add_namespace = "phaser", + frontend = "protobuf", # Default; use "ros" for public field proxies. + enable_active_message = False, deps = [":foo_proto"], ) ``` @@ -167,7 +169,7 @@ Phaser runs as a protoc plugin. To run it use the following command format: ``` $ protoc --plugin=protoc-gen-phaser=DIR/bin/phaser/compiler/phaser \ - --phaser_out=add_namespace=NS,package_name=PACKAGE,target_name=TARGET:OUTPUT_DIR \ + --phaser_out=frontend=protobuf,add_namespace=NS,package_name=PACKAGE,target_name=TARGET:OUTPUT_DIR \ -I IPATH... FILE ... ``` @@ -231,8 +233,16 @@ payload layout: - `frontend = "ros"` generates a struct with public field proxies. ```python +phaser_library( + name = "foo_protobuf_phaser", + add_namespace = "protobuf_api", + frontend = "protobuf", + deps = [":foo_proto"], +) + phaser_library( name = "foo_ros_phaser", + add_namespace = "ros_api", frontend = "ros", deps = [":foo_proto"], ) @@ -241,6 +251,43 @@ phaser_library( When invoking the plugin directly, pass `frontend=ros` in `--phaser_out`. Unknown frontend values are rejected. +The frontend controls source-level syntax, not storage or wire compatibility: + +```c++ +// frontend = "protobuf" +foo::bar::protobuf_api::Foo protobuf_message; +protobuf_message.set_count(10); +protobuf_message.set_name("camera"); +protobuf_message.add_values(1.5); + +// frontend = "ros" +foo::bar::ros_api::Foo ros_message; +ros_message.count = 10; +ros_message.name = "camera"; +ros_message.values.push_back(1.5); +``` + +Both types use the same native Phaser payload layout. A receiver can attach the +other frontend to the same bytes without conversion: + +```c++ +foo::bar::protobuf_api::Foo source; +source.set_count(10); + +const auto* bytes = static_cast(source.Data()); +auto ros_view = foo::bar::ros_api::Foo::CreateReadonly(bytes, source.Size()); +int count = ros_view.count; +``` + +Both frontends also generate the complete protobuf and ROS1 serialization and +wire-to-wire conversion APIs. You do not need the ROS frontend to produce or +consume ROS1 wire bytes. + +Set `enable_active_message = True` on `phaser_library`, or pass +`active_message=true` to the plugin, to add a public `std::any active_message` +member to every generated message. This is application-side transient state; +it is not stored in the native payload or serialized to protobuf or ROS. + ### ROS field syntax Scalar, enum, string, message, and repeated fields provide value-like syntax, but remain handles into the message's `PayloadBuffer`: @@ -406,6 +453,10 @@ absl::Status ParseFromROS(absl::Span input); static absl::Status ProtobufToROS( std::string_view protobuf, ::phaser::ROSBuffer& output); +static absl::Status ProtobufWireToROS( + std::string_view protobuf, ::phaser::ROSBuffer& output); +static absl::Status ROSReaderToProtobuf( + ::phaser::ROSReader& ros, ::phaser::ProtoWriter& output); static absl::Status ROSToProtobuf( absl::Span ros, ::phaser::ProtoBuffer& output); static bool ROSToProtobufArray( @@ -421,6 +472,9 @@ fields directly and emits ROS bytes in schema order. `ROSToProtobuf` scans ROS wire fields in schema order and writes protobuf tags directly; it uses counting passes to determine nested-message and packed-field lengths without staging encoded bytes or constructing a native message. +`ProtobufWireToROS` and `ROSReaderToProtobuf` are the lower-level scanners used +recursively by those public wrappers; most callers should use +`ProtobufToROS` and `ROSToProtobuf`. `PhaserToROS` attaches a read-only message to a valid native Phaser payload for the duration of the conversion. `ConvertToROS` calls `InferMessageWireFormat` and selects either input path. Inference validates the @@ -429,6 +483,72 @@ than checking only the four-byte magic: those magic bytes can also begin a valid protobuf tag. The result enum can report `kProtobuf`, `kPhaser`, `kUnknown`, or `kAmbiguous`; automatic conversion rejects the latter two. +#### Direct protobuf wire to ROS1 + +The output buffer can own a growing allocation: + +```c++ +std::string protobuf_wire = ReceiveProtobuf(); +::phaser::ROSBuffer ros_output; + +absl::Status status = + foo::bar::phaser::Foo::ProtobufToROS(protobuf_wire, ros_output); +if (status.ok()) { + PublishROS(ros_output.data(), ros_output.size()); +} +``` + +No protobuf or Phaser source message is constructed. On failure, +`ProtobufToROS` clears the output buffer. + +#### Direct ROS1 wire to protobuf + +Wrap caller-provided memory in a `ProtoBuffer` for an allocation-free output +path: + +```c++ +absl::Span ros_wire = ReceiveROS(); +std::array storage; +::phaser::ProtoBuffer protobuf_output(storage.data(), storage.size()); + +absl::Status status = + foo::bar::phaser::Foo::ROSToProtobuf(ros_wire, protobuf_output); +if (status.ok()) { + StoreProtobuf(storage.data(), protobuf_output.Size()); +} +``` + +`ROSToProtobufArray` is a thin boolean wrapper for callers that manage the +output extent separately. It does not report the encoded byte count; use +`ROSToProtobuf` when the size or detailed error status is required. + +#### Native Phaser or inferred input to ROS1 + +```c++ +const auto* native_data = static_cast(message.Data()); +absl::Span native(native_data, message.Size()); + +::phaser::ROSBuffer ros_output; +absl::Status status = + foo::bar::phaser::Foo::PhaserToROS(native, ros_output); + +// input may instead be either protobuf wire bytes or a native Phaser payload. +status = foo::bar::phaser::Foo::ConvertToROS(input, ros_output); +``` + +#### ROS1 wire to a native Phaser payload + +```c++ +alignas(std::max_align_t) std::array payload; +auto message = + foo::bar::phaser::Foo::CreateMutable(payload.data(), payload.size()); + +absl::Status status = message.ParseFromROS(ros_wire); +if (status.ok()) { + Consume(message); +} +``` + `ParseFromROS` clears the target message, decodes fields in schema order, and requires the complete input span to be consumed. A default-constructed target stores the result in its dynamically allocated native `PayloadBuffer`; a target @@ -442,8 +562,10 @@ error. constructor owns a dynamically growing allocation; constructing it with a pointer and size wraps fixed caller-owned output memory. Writes return an `absl::Status`, including insufficient-capacity and malformed-protobuf errors. -`ROSReader` is a non-owning view over received bytes and checks every primitive, -string length, and sequence-length read against the remaining input. +`ROSReader` is a non-owning view over received bytes and bounds-checks every +primitive, string, array, and raw read against the remaining input. It validates +structure and truncation, but intentionally does not reject unknown enum values +or nonzero boolean byte representations. The generated layout follows ROS1 serialization rules: @@ -469,6 +591,11 @@ serialization. A protobuf-style target and a ROS-style target generated from the same schema can therefore exchange native Phaser buffers and protobuf wire bytes. +The direct conversion backends preserve schema order, recursively convert +nested messages, and transform protobuf varints where ROS uses fixed-width +values. Packed fixed-width protobuf arrays whose byte layout already matches +ROS1 are copied in bulk rather than decoded element by element. + ## Creating a message In protobuf, you generally create messages on the local stack frame or from the heap. Submessages (fields whose type is a message) are allocated from the heap. @@ -629,10 +756,8 @@ void Foo() { } ``` -For completeness, the default constructor for a message (for one on the stack) can also -be achieved using the function `CreateDynamicMutable`, which also allows you to -specify the initial buffer size (with a 1K default) to be allocated for the binary message from the -heap: +The default constructor starts with an 8K buffer. The equivalent +`CreateDynamicMutable` factory takes an explicit initial buffer size: ```c++ void Foo() { @@ -682,7 +807,8 @@ call the function `CreateReadonly`: ```c++ void Receive(const char* buffer, size_t buffer_size) { - auto msg = foo::bar::phaser::TestMessage::CreateReadonly(buffer, size); + auto msg = + foo::bar::phaser::TestMessage::CreateReadonly(buffer, buffer_size); int x = msg.x(); // ... } @@ -802,19 +928,20 @@ field: ```c++ size_t vm_size() const; void clear_vm(); - const InnerMessage& vm(size_t index) const; - InnerMessage* mutable_vm(size_t index); - InnerMessage* add_vm(); + InnerMessage vm(size_t index) const; + InnerMessage mutable_vm(size_t index); + InnerMessage add_vm(); const ::phaser::MessageVectorField& vm() const; void reserve_vm(size_t num); void resize_vm(size_t num); - std::vector allocate_vm(size_t n); + std::vector allocate_vm(size_t n); ``` The `resize` and `reserve` functions operate similarly to `std::vector`. The -`allocate` function allocates `n` messages in a block and provides pointers -to the source messages allocated. This is a performance enhancement to allow -faster creation of a common pattern in protobuf message. +`allocate` function allocates `n` messages in a block and returns lightweight +message handles by value. The handles retain the shared mutable runtime and +remain valid if the payload buffer relocates. This is a performance enhancement +for a common protobuf message-building pattern. ### Oneof fields A `oneof` field is a discriminated union that holds only one of a number of From b4db0940ce055f4634bc0a8170e071ed06ad5e99 Mon Sep 17 00:00:00 2001 From: Dave Allison Date: Wed, 12 Aug 2026 18:46:22 -0700 Subject: [PATCH 7/7] Fix nested message view lifetimes Rebind cached nested views before use and keep union field-number spans backed by static storage so sanitizer builds do not access stale payload or stack memory. --- phaser/runtime/fields.h | 28 +++++++++++++++++++-------- phaser/runtime/message_test.cc | 8 ++++++-- phaser/runtime/union.h | 35 +++++++++++++++++++++++++++------- 3 files changed, 54 insertions(+), 17 deletions(-) diff --git a/phaser/runtime/fields.h b/phaser/runtime/fields.h index 0aef80a..d122b73 100644 --- a/phaser/runtime/fields.h +++ b/phaser/runtime/fields.h @@ -723,21 +723,22 @@ class IndirectMessageField : public Field { MessageType& operator*() { return *Mutable(); } MessageType* operator->() { return Mutable(); } - const MessageType& Msg() const { return msg_; } - MessageType& MutableMsg() { return msg_; } + const MessageType& Msg() const { return Get(); } + MessageType& MutableMsg() { return *Mutable(); } const MessageType& Get() const { int32_t offset = FindFieldOffset(source_offset_); if (offset < 0) { - return msg_; + return DefaultMessage(); } ::toolbelt::BufferOffset* addr = GetIndirectAddress(static_cast(offset)); - if (*addr != 0) { - // Load up the message if it's already been allocated. - msg_.runtime = GetRuntime(); - msg_.absolute_binary_offset = *addr; + if (*addr == 0) { + return DefaultMessage(); } + // Load up the message if it's already been allocated. + msg_.runtime = GetRuntime(); + msg_.absolute_binary_offset = *addr; return msg_; } @@ -756,6 +757,8 @@ class IndirectMessageField : public Field { GetIndirectAddress(relative_binary_offset_); if (*addr != 0) { // Already allocated. + msg_.runtime = GetRuntime(); + msg_.absolute_binary_offset = *addr; return &msg_; } // Allocate a new message. @@ -793,11 +796,15 @@ class IndirectMessageField : public Field { if (*addr == 0) { return; } + const ::toolbelt::BufferOffset old_offset = *addr; // Clear the message. + msg_.runtime = GetRuntime(); + msg_.absolute_binary_offset = old_offset; msg_.Clear(); // Delete the memory in the payload buffer. - GetBuffer()->Free(GetRuntime()->ToAddress(*addr)); + GetBuffer()->Free(GetRuntime()->ToAddress(old_offset)); // Zero out the offset to the message. + addr = GetIndirectAddress(relative_binary_offset_); *addr = 0; } @@ -883,6 +890,11 @@ class IndirectMessageField : public Field { } protected: + static const MessageType& DefaultMessage() { + static const MessageType message(InternalDefault{}); + return message; + } + ::toolbelt::PayloadBuffer* GetBuffer() const { return Message::GetBuffer(this, source_offset_); } diff --git a/phaser/runtime/message_test.cc b/phaser/runtime/message_test.cc index 6815c9c..5a03024 100644 --- a/phaser/runtime/message_test.cc +++ b/phaser/runtime/message_test.cc @@ -50,12 +50,15 @@ struct EnumTestParser { }; struct InnerMessage : public Message { + inline static constexpr uint32_t kUvFieldNumbers[] = {50, 60}; + InnerMessage(phaser::InternalDefault /*d*/) : str_(offsetof(InnerMessage, str_), HeaderSize() + 0, 0, 10), f_(offsetof(InnerMessage, f_), HeaderSize() + 8, 1, 20), e_(offsetof(InnerMessage, e_), HeaderSize() + 16, 2, 30), ev_(offsetof(InnerMessage, ev_), HeaderSize() + 20, 0, 40), - uv_(offsetof(InnerMessage, uv_), HeaderSize() + 28, 0, 0, {50, 60}) {} + uv_(offsetof(InnerMessage, uv_), HeaderSize() + 28, 0, 0, + absl::MakeConstSpan(kUvFieldNumbers)) {} InnerMessage(std::shared_ptr rt, ::toolbelt::BufferOffset offset) @@ -64,7 +67,8 @@ struct InnerMessage : public Message { f_(offsetof(InnerMessage, f_), HeaderSize() + 8, 1, 20), e_(offsetof(InnerMessage, e_), HeaderSize() + 16, 2, 30), ev_(offsetof(InnerMessage, ev_), HeaderSize() + 20, 0, 40), - uv_(offsetof(InnerMessage, uv_), HeaderSize() + 28, 0, 0, {50, 60}) {} + uv_(offsetof(InnerMessage, uv_), HeaderSize() + 28, 0, 0, + absl::MakeConstSpan(kUvFieldNumbers)) {} // uv_ is a union of: // uint32_t diff --git a/phaser/runtime/union.h b/phaser/runtime/union.h index c4dc8e6..c788e3b 100644 --- a/phaser/runtime/union.h +++ b/phaser/runtime/union.h @@ -311,11 +311,10 @@ class UnionMessageField : public UnionMemberField { uint32_t abs_offset) const { ::toolbelt::BufferOffset* addr = GetIndirectAddress(runtime, abs_offset); if (addr == nullptr || *addr == 0) { - return msg_; + return DefaultMessage(); } // Populate msg_ with the information from the message. - msg_.runtime = runtime; - msg_.absolute_binary_offset = *addr; + Bind(runtime, *addr); return msg_; } @@ -336,7 +335,7 @@ class UnionMessageField : public UnionMemberField { bool IsPresent(const std::shared_ptr& runtime, uint32_t abs_offset) const { ::toolbelt::BufferOffset* addr = GetIndirectAddress(runtime, abs_offset); - return addr == nullptr || *addr != 0; + return addr != nullptr && *addr != 0; } MessageType* Mutable(const std::shared_ptr& runtime, @@ -344,6 +343,9 @@ class UnionMessageField : public UnionMemberField { ::toolbelt::BufferOffset* addr = GetIndirectAddress(runtime, abs_offset); if (addr == nullptr || *addr != 0) { // Already allocated. + if (addr != nullptr) { + Bind(runtime, *addr); + } return &msg_; } // Allocate a new message. @@ -391,8 +393,10 @@ class UnionMessageField : public UnionMemberField { return; } if (*addr != 0) { + const ::toolbelt::BufferOffset old_offset = *addr; + Bind(runtime, old_offset); msg_.Clear(); - GetBuffer(runtime)->Free(runtime->ToAddress(*addr)); + GetBuffer(runtime)->Free(runtime->ToAddress(old_offset)); } // Allocate a new message. void* msg_addr = ::toolbelt::PayloadBuffer::Allocate( @@ -422,9 +426,15 @@ class UnionMessageField : public UnionMemberField { return; } if (*addr != 0) { + const ::toolbelt::BufferOffset old_offset = *addr; + Bind(runtime, old_offset); msg_.Clear(); - GetBuffer(runtime)->Free(runtime->ToAddress(*addr)); - *addr = 0; + GetBuffer(runtime)->Free(runtime->ToAddress(old_offset)); + // Clearing the nested message may move the payload buffer. + addr = GetIndirectAddress(runtime, abs_offset); + if (addr != nullptr) { + *addr = 0; + } } } @@ -475,6 +485,17 @@ class UnionMessageField : public UnionMemberField { } private: + static const MessageType& DefaultMessage() { + static const MessageType message(InternalDefault{}); + return message; + } + + void Bind(const std::shared_ptr& runtime, + ::toolbelt::BufferOffset offset) const { + msg_.runtime = runtime; + msg_.absolute_binary_offset = offset; + } + ::toolbelt::BufferOffset* GetIndirectAddress( const std::shared_ptr& runtime, uint32_t abs_offset) const {