From a3f4999a6853a197238de437b8f0f62a28b423ce Mon Sep 17 00:00:00 2001 From: PhilipDeegan Date: Sun, 7 Jun 2026 19:27:39 +0200 Subject: [PATCH] binary serializer --- include/cppconfig/io/bin.hpp | 474 +++++++++++++++++++++++++++++++++++ meson.build | 2 +- tests/binary/test.cpp | 122 +++++++++ 3 files changed, 597 insertions(+), 1 deletion(-) create mode 100644 include/cppconfig/io/bin.hpp create mode 100644 tests/binary/test.cpp diff --git a/include/cppconfig/io/bin.hpp b/include/cppconfig/io/bin.hpp new file mode 100644 index 0000000..f0daf5e --- /dev/null +++ b/include/cppconfig/io/bin.hpp @@ -0,0 +1,474 @@ +/*------------------------------------------------------------------------------ +-- This file is a part of the C++ Config library +-- Copyright (C) 2022, Plasma Physics Laboratory - CNRS +-- +-- This program is free software; you can redistribute it and/or modify +-- it under the terms of the GNU General Public License as published by +-- the Free Software Foundation; either version 2 of the License, or +-- (at your option) any later version. +-- +-- This program is distributed in the hope that it will be useful, +-- but WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-- GNU General Public License for more details. +-- +-- You should have received a copy of the GNU General Public License +-- along with this program; if not, write to the Free Software +-- Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +-------------------------------------------------------------------------------*/ +/*-- Author : Alexis Jeandet +-- Mail : alexis.jeandet@lpp.polytechnique.fr +----------------------------------------------------------------------------*/ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace cppconfig::config_binary +{ +template +class DictSerializer; + +template +class DictDeSerializer; + +template +void save_config(const std::string& file, const Config& config) +{ + DictSerializer { file }(config); +} + +template +Config load_config(const std::string& file) +{ + return DictDeSerializer { file }(); +} + +template +auto constexpr dict_types_as_tuple(cppdict::Dict const&) +{ + return std::tuple {}; +} + + +template +void _read_data(T data, std::size_t const size, std::ifstream& in) +{ + in.read(reinterpret_cast(data), size); +} + +template +void _write_data(T const data, std::size_t const size, std::ofstream& out) +{ + out.write(reinterpret_cast(data), size); +} + +template +using tryToInstanciate = void; + +template +struct is_spannable : std::false_type +{ +}; + +template +struct is_spannable().data())>, + tryToInstanciate().size())>> : std::true_type +{ +}; + +template +bool constexpr static is_spannable_v = is_spannable::value; + + +template +struct is_custom_serializable : std::false_type +{ +}; + +template +auto constexpr static has_custom_serialize(Dict& dict, T const& data) + -> decltype(serialize(dict, data), bool()) +{ + return true; +} +template +auto constexpr static has_custom_serialize(Args&&...) +{ + return false; +} +template +auto constexpr static has_custom_deserialize(Dict const& dict, T& data) + -> decltype(deserialize(dict, data), bool()) +{ + return true; +} +template +auto constexpr static has_custom_deserialize(Args&&...) +{ + return false; +} + + +template +struct is_custom_serializable(), std::declval()))>, + tryToInstanciate(), std::declval()))>> + : std::true_type +{ +}; + + +template +bool constexpr static is_custom_serializable_v = is_custom_serializable::value; + + +template +auto constexpr static _custom_serialize(Dict& dict, T const& data) + -> decltype(serialize(dict, data), bool()) +{ + serialize(dict, data); + return true; +} + +template +auto constexpr static _custom_serialize(Args&&...) +{ + return false; +} + + +template +auto constexpr static _custom_deserialize(Dict const& dict, T& data) + -> decltype(deserialize(dict, data), bool()) +{ + deserialize(dict, data); + return true; +} + +template +auto constexpr static _custom_deserialize(Args&&...) +{ + return false; +} + + +template +struct varient_visitor_overloads : Ts... +{ + using Ts::operator()...; +}; + +template +varient_visitor_overloads(Ts&&...) -> varient_visitor_overloads...>; + + +template +class DictSerializer +{ + using Dict = Config; + using data_t = typename Dict::data_t; + using node_t = typename Dict::node_t; + using Tuple = std::decay_t()))>; + std::size_t constexpr static n_types = std::tuple_size_v; + std::size_t constexpr static base = 2; // 0 = map, 1 = map_key_string + using NodeVisitor = std::function; + +public: + DictSerializer(std::string const& filename_) : filename { filename_ } { } + + + void operator()(Dict const& dict) { _serialize(dict); } + + +private: + void _serialize(Config const& dict, std::string const& in = "") + { + auto const value_visitor = [&](const std::string& key, const auto& v) + { + using El = std::decay_t; + static_assert(!std::is_same_v); + + if constexpr (std::is_same_v) + { + std::size_t const type = 1; + _write_data(&type, sizeof(type), out); + std::size_t const size = key.size(); + _write_data(&size, sizeof(size), out); + _write_data(key.data(), size, out); + + _serialize(v); + } + else if constexpr (Dict::template is_value::value) + { + std::size_t const type = 1; + _write_data(&type, sizeof(type), out); + std::size_t const size = key.size(); + _write_data(&size, sizeof(size), out); + _write_data(key.data(), size, out); + + write_type_id(v, std::make_integer_sequence {}); + _serialize_data(v); + } + else + for (const auto& [k, node] : v) + _serialize(node, k); + }; + + if (dict.isNode()) + dict.visit( + cppdict::visit_all_nodes, node_visitor, + [&](const std::string&, const typename Dict::empty_leaf_t&) { }, value_visitor); + else if (dict.isValue()) + { + if (in.empty()) + throw std::runtime_error("_serialize: value node has no key"); + value_visitor(in, dict.data); + } + else + throw std::runtime_error("_serialize: dict is neither node nor value"); + } + + + template + void _serialize(std::variant const& val) + { + auto const overloads + = varient_visitor_overloads { [&](const typename Config::empty_leaf_t&) {}, + [&](auto&& value) + { + using El = std::decay_t; + static_assert(!std::is_same_v); + if constexpr (Config::template is_value::value) + { + write_type_id(value, std::make_integer_sequence {}); + _serialize_data(value); + } + else + throw std::runtime_error("_serialize: variant holds unexpected map type"); + } }; + + std::visit(overloads, val); + } + + + NodeVisitor const node_visitor = [&](std::string const& key, node_t const& v) + { + { + std::size_t const type = 0; + _write_data(&type, sizeof(type), out); + std::size_t const size = key.size(); + _write_data(&size, sizeof(size), out); + std::size_t const keys = v.size(); + _write_data(&keys, sizeof(keys), out); + _write_data(key.data(), size, out); + } + + for (const auto& [k, node] : v) + { + + if (node->isNode()) + node_visitor(k, std::get(node->data)); + else if (node->isValue()) + { + + _serialize(*node, k); + } + else + throw std::runtime_error("node_visitor: node is neither node nor value"); + } + }; + + + + template + void _serialize_data(T const& data) + { + if constexpr (is_spannable_v) + { + std::size_t const size = data.size(); + _write_data(&size, sizeof(size), out); + _write_data(data.data(), size * sizeof(typename T::value_type), out); + } + else if constexpr (std::is_fundamental::value) + { + std::size_t const size = sizeof(T); + _write_data(&data, size, out); + } + else if constexpr (is_custom_serializable_v) + { + Dict temp; + _custom_serialize(temp, data); + // Write entry count so the deserializer knows how many chunks to consume + auto const& root_map = std::get(temp.data); + std::size_t const entry_count = root_map.size(); + _write_data(&entry_count, sizeof(entry_count), out); + (*this)(temp); + } + else + _serialize(data); + } + + + template + void write_type_id(T const&, bool& b) + { + if (b) + return; + if constexpr (std::is_same_v>) + { + std::size_t const type = I + base; + _write_data(&type, sizeof(type), out); + b = 1; + } + } + + template + void write_type_id(T const& t, std::integer_sequence) + { + bool b = 0; + (write_type_id(t, b), ...); + if (b == 0) + throw std::runtime_error("write_type_id: type not found in tuple"); + } + + std::string const filename; + std::ofstream out { filename, std::ios::binary }; +}; + + +template +class DictDeSerializer +{ + using Dict = Config; + using data_t = typename Dict::data_t; + using Tuple = std::decay_t()))>; + std::size_t constexpr static n_types = std::tuple_size_v; + std::size_t constexpr static base = 2; // 0 = map, 1 = map_key_string + +public: + DictDeSerializer(std::string const& filename_) : filename { filename_ } { } + + template + auto operator()() + { + Dict dict; + while (!in.eof()) + { + _read_chunk(dict); + in.peek(); + } + return dict; + } + +private: + template + void _deserialize_data(Dict& node) + { + if (!node.isEmpty()) + throw std::runtime_error("_deserialize_data: target node is not empty"); + + if constexpr (is_spannable_v) + { + std::size_t size = 0; + + in.read(reinterpret_cast(&size), sizeof(std::size_t)); + + T s; + s.resize(size); + _read_data(s.data(), size * sizeof(typename T::value_type), in); + node = s; + } + else if constexpr (std::is_fundamental::value) + { + T d {}; + in.read(reinterpret_cast(&d), sizeof(T)); + node = d; + } + else if constexpr (is_custom_serializable_v) + { + std::size_t entry_count = 0; + in.read(reinterpret_cast(&entry_count), sizeof(entry_count)); + Dict loader; + for (std::size_t i = 0; i < entry_count; ++i) + _read_chunk(loader); + T t; + deserialize(loader, t); + node = t; + } + else + throw std::runtime_error("_deserialize_data: unsupported type"); + } + + void _read_chunk(Dict& node) + { + std::size_t type = 0; + in.read(reinterpret_cast(&type), sizeof(std::size_t)); + if (type > n_types + 2) + throw std::runtime_error("_read_chunk: unknown type id " + std::to_string(type)); + + if (type == 0) + { + std::size_t size = 0; + in.read(reinterpret_cast(&size), sizeof(std::size_t)); + std::string s; + s.resize(size); + std::size_t keys = 0; + in.read(reinterpret_cast(&keys), sizeof(std::size_t)); + _read_data(s.data(), size, in); + auto& child = node[s]; + for (std::size_t i = 0; i < keys; ++i) + _read_chunk(child); + } + else if (type == 1) + { + std::size_t size = 0; + in.read(reinterpret_cast(&size), sizeof(std::size_t)); + std::string s; + s.resize(size); + _read_data(s.data(), size, in); + _read_chunk(node[s]); + } + else + { + read(node, type, std::make_integer_sequence {}); + } + } + + template + void _read(Dict& node, int& found, std::size_t const& type) + { + using El = std::tuple_element_t; + + if constexpr (Dict::template is_value::value) + if (I + base == type) + { + _deserialize_data(node); + found += 1; + } + } + + template + void read(Dict& node, std::size_t const& type, std::integer_sequence) + { + int found = 0; + (_read(node, found, type), ...); + if (found != 1) + throw std::runtime_error("_read: expected exactly one matching type, found " + std::to_string(found)); + } + + + std::string const filename; + std::ifstream in { filename, std::ios::binary }; +}; + + +} // namespace cppdict + + diff --git a/meson.build b/meson.build index cde1f84..7355c4e 100644 --- a/meson.build +++ b/meson.build @@ -18,7 +18,7 @@ cppconfig_dep = declare_dependency( library('cppconfig', extra_files:['include/cppconfig/cppconfig.hpp', 'include/cppconfig/io/yaml.hpp', 'include/cppconfig/io/json.hpp']) -foreach test:['yaml','json'] +foreach test:['binary', 'yaml', 'json'] exe = executable(test,'tests/'+test+'/test.cpp', dependencies:[cppconfig_dep, catch_dep], cpp_args : '-DCATCH_CONFIG_NO_POSIX_SIGNALS', diff --git a/tests/binary/test.cpp b/tests/binary/test.cpp new file mode 100644 index 0000000..201523e --- /dev/null +++ b/tests/binary/test.cpp @@ -0,0 +1,122 @@ + +// #include +// #include +// #include +#include + +#include "cppconfig/io/bin.hpp" + + +TEST_CASE("can read/write", "[cppconfig]") +{ + using MyDict = cppdict::Dict>; + + std::string const file("outfile.dat"); + + { + MyDict md; + md["PI"] = 3.14; + md["test"]["super"] = 2; + md["Key"] = std::string { "Value" }; + md["key2"] = 2; + md["vec"] = std::vector { 1, 2, 3 }; + + cppconfig::config_binary::save_config(file, md); + } + + auto const md = cppconfig::config_binary::load_config(file); + + SECTION("A dict") + { + REQUIRE(md["Key"].to() == "Value"); + REQUIRE(md["key2"].to() == 2); + REQUIRE(md["PI"].to() == 3.14); + REQUIRE(md["test"]["super"].to() == 2); + REQUIRE(md["vec"].to>() == std::vector { 1, 2, 3 }); + } +} + +struct CustomSerializable +{ + int a = 111; + double b = 333; + + bool operator==(CustomSerializable const& cs) { return a == cs.a and b == cs.b; } + bool operator!=(CustomSerializable const& cs) { return !(*this == cs); } +}; + + +template +void serialize(cppdict::Dict& dict, CustomSerializable const& cm) +{ + dict["cm.a"] = cm.a; + dict["cm.b"] = cm.b; +} + +template +void deserialize(cppdict::Dict const& dict, CustomSerializable& cm) +{ + cm.a = dict["cm.a"].template to(); + cm.b = dict["cm.b"].template to(); +} + + +TEST_CASE("can read/write custom serializable", "[cppconfig]") +{ + using MyDict = cppdict::Dict; + static_assert(cppconfig::config_binary::is_custom_serializable_v); + + CustomSerializable const cs {}; + + { + std::string const file("outfile.0.dat"); + + { + MyDict md; + md["key"] = std::string { "Value" }; + md["custom"]["data"]["cm.a"] = cs.a; + md["custom"]["data"]["cm.b"] = cs.b; + md["custom"]["other"] = 3.3; + md["key2"] = 2; + static_assert(cppconfig::config_binary::has_custom_serialize(md, cs)); + + cppconfig::config_binary::save_config(file, md); + } + + auto const md = cppconfig::config_binary::load_config(file); + + SECTION("A dict") + { + REQUIRE(md["key"].to() == "Value"); + REQUIRE(md["custom"]["data"]["cm.a"].to() == cs.a); + REQUIRE(md["custom"]["data"]["cm.b"].to() == cs.b); + REQUIRE(md["custom"]["other"].to() == 3.3); + REQUIRE(md["key2"].to() == 2); + } + } + + { + std::string const file("outfile.1.dat"); + + { + MyDict md; + md["key"] = std::string { "Value" }; + md["custom"]["data"] = cs; + md["custom"]["other"] = 3.3; + md["key2"] = 2; + static_assert(cppconfig::config_binary::has_custom_serialize(md, cs)); + + cppconfig::config_binary::save_config(file, md); + } + + auto const md = cppconfig::config_binary::load_config(file); + + SECTION("B dict - direct custom type") + { + REQUIRE(md["key"].to() == "Value"); + REQUIRE(md["custom"]["other"].to() == 3.3); + REQUIRE(md["custom"]["data"].to() == cs); + REQUIRE(md["key2"].to() == 2); + } + } +}