diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0c3049a64a..72ce416e4f 100755 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -251,6 +251,8 @@ jobs: target: esp32s3 - path: 'components/nvs/example' target: esp32s3 + - path: 'components/persistent_data/example' + target: esp32s3 - path: 'components/odrive_ascii/example' target: esp32 - path: 'components/odrive_native/example' diff --git a/.github/workflows/persistent_data_tests.yml b/.github/workflows/persistent_data_tests.yml new file mode 100644 index 0000000000..dcbb02860e --- /dev/null +++ b/.github/workflows/persistent_data_tests.yml @@ -0,0 +1,47 @@ +name: Persistent data host test + +# Builds and runs components/persistent_data/test/persistent_data_host_test.cpp +# (file format, version list, migration chain, atomic file helpers) on the +# host with g++ -std=c++23. Runs whenever the component or the serialization +# component it encodes with changes. + +# Minimal token scope: build + run tests, never writes. +permissions: + contents: read + +on: + pull_request: + paths: + - "components/persistent_data/**" + - "components/serialization/**" + - ".github/workflows/persistent_data_tests.yml" + push: + branches: + - main + paths: + - "components/persistent_data/**" + - "components/serialization/**" + - ".github/workflows/persistent_data_tests.yml" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + host-test: + name: C++ host test (codec, migration chain, file helpers) + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v7 + with: + submodules: recursive + - name: Build and run persistent_data_host_test + run: | + g++ --version | head -1 + g++ -std=c++23 -Wall -Wextra -Werror \ + -I components/persistent_data/include -I components/serialization/include \ + -isystem components/serialization/detail/alpaca/include \ + components/persistent_data/test/persistent_data_host_test.cpp -o persistent_data_host_test + ./persistent_data_host_test diff --git a/.github/workflows/upload_components.yml b/.github/workflows/upload_components.yml index d076dfa814..75ac4dd2ab 100755 --- a/.github/workflows/upload_components.yml +++ b/.github/workflows/upload_components.yml @@ -149,6 +149,7 @@ jobs: components/odrive_native components/ota components/pcf85063 + components/persistent_data components/pi4ioe5v components/pid components/ping diff --git a/components/persistent_data/CMakeLists.txt b/components/persistent_data/CMakeLists.txt new file mode 100644 index 0000000000..5f2da9aaf8 --- /dev/null +++ b/components/persistent_data/CMakeLists.txt @@ -0,0 +1,4 @@ +idf_component_register( + INCLUDE_DIRS "include" + REQUIRES base_component serialization + ) diff --git a/components/persistent_data/README.md b/components/persistent_data/README.md new file mode 100644 index 0000000000..54de4c3511 --- /dev/null +++ b/components/persistent_data/README.md @@ -0,0 +1,223 @@ +# Persistent Data Component + +[![Badge](https://components.espressif.com/components/espp/persistent_data/badge.svg)](https://components.espressif.com/components/espp/persistent_data) + +The `persistent_data` component stores one data structure (a settings struct, +calibration data, a paired-device list) in a file on any filesystem reachable +through the VFS (LittleFS, SPIFFS, SD card), tracks the schema version of what +is on disk, and migrates older files forward when the structure changes. + + +**Table of Contents** + +- [Persistent Data Component](#persistent-data-component) + - [Overview](#overview) + - [Defining versions](#defining-versions) + - [Using it](#using-it) + - [File format](#file-format) + - [Errors](#errors) + - [Other backends](#other-backends) + - [Thread safety](#thread-safety) + - [Host test](#host-test) + - [Example](#example) + + + +## Overview + +`espp::PersistentData` takes the schema versions of a structure as +a type list, newest first; the first type is the current one. Every file carries +a 20-byte header (magic, schema version, layout fingerprint, payload length, +CRC32) in front of the payload, which is serialized with the +[serialization](../serialization) component (alpaca). A load therefore +distinguishes, without guessing: + +| Situation | Result | +|---------------------------------------|--------------------------------------------| +| no file yet | `file_not_found` | +| file of the current version | loaded | +| file of an older listed version | loaded, migrated, rewritten (configurable) | +| file of a version not in the list | `unknown_version` (newer firmware wrote it, or support was dropped) | +| truncated file, wrong magic, trailing bytes | `bad_header` | +| bit error | `bad_checksum` | +| same version number, different layout | `layout_mismatch` (header fingerprint) | + +Saves are atomic: the data is written to `.tmp`, flushed and synced, +then renamed over the file, so a reset in the middle of a save keeps the +previous contents. + +## Defining versions + +Each version is an aggregate (no user-declared constructors; that is what alpaca +serializes) with a `static constexpr uint32_t VERSION`, strictly decreasing +along the list (newest first). Every version except the last provides `static T +migrate(const Older &)` from the version after it in the list; an older file is +walked toward the front of the list one step at a time, so each version only +knows how to come from the one before it. The structs themselves are declared +oldest first (C++ needs the older type complete before `migrate` can mention +it); only the type list is newest first. + +```cpp +struct SettingsV1 { + static constexpr uint32_t VERSION = 1; + uint8_t brightness{50}; + std::string device_name{"espp"}; +}; + +struct SettingsV2 { + static constexpr uint32_t VERSION = 2; + uint8_t brightness{50}; + std::string device_name{"espp"}; + uint32_t color_rgb{0x00FF00}; + + static SettingsV2 migrate(const SettingsV1 &v1) { + return {.brightness = v1.brightness, .device_name = v1.device_name}; + } +}; + +struct Settings { // current + static constexpr uint32_t VERSION = 3; + std::string device_name{"espp"}; + float brightness{0.5f}; + uint32_t color_rgb{0x00FF00}; + uint32_t boot_count{0}; + + static Settings migrate(const SettingsV2 &v2) { + return {.device_name = v2.device_name, + .brightness = v2.brightness / 100.0f, + .color_rgb = v2.color_rgb}; + } +}; + +using PersistentSettings = espp::PersistentData; +``` + +A new version is prepended to the list. Dropping support for the oldest +version is removing the last entry; such files then load with +`unknown_version` and the application decides what to do. +A missing `VERSION`, a non-decreasing version, or a missing `migrate` is a +compile error with a message saying which requirement failed. + +## Using it + +```cpp +std::error_code ec; +PersistentSettings settings({.file_path = fs.get_root_path() / "settings.bin", + .auto_load = false}); + +// load, or create the file from defaults if there is none; a corrupt file or +// one written by newer firmware is reported, not silently replaced +if (!settings.load_or_create(Settings{}, ec)) { + logger.error("settings: {}", ec.message()); +} + +// read-modify-write under one lock, saved atomically +settings.update([](Settings &s) { s.boot_count++; }, ec); + +// or explicitly +Settings s = settings.get(); // a copy +s.brightness = 1.0f; +settings.save(s, ec); + +// get() / set() are also available as conversion / assignment operators; +// saving stays explicit because it can fail +Settings t = settings; // == settings.get() +t.boot_count = 0; +settings = t; // == settings.set(t) +settings.save(ec); +``` + +With `auto_load` (the default) the constructor loads the file and logs any +problem; `is_loaded()`, `loaded_version()` and `was_migrated()` say what +happened. `erase()` removes the file (succeeding when there is none). + +## File format + +All fields little-endian: + +| offset | size | field | +|--------|------|------------------------------------------------------------| +| 0 | 4 | magic `"ESPD"` | +| 4 | 4 | schema version (`VERSION` of the type that wrote it) | +| 8 | 4 | layout fingerprint of the type that wrote it (below) | +| 12 | 4 | payload length | +| 16 | 4 | CRC32 over the four fields above and the payload | +| 20 | n | payload: alpaca, `fixed_length_encoding \| strict` | + +The format is the same on every target and on the host: alpaca encodes +container lengths as 32-bit values, and nothing in the header depends on the +platform, so a PC tool can read and write a device's file. + +### Layout fingerprint + +The fingerprint is a CRC32 over a byte sequence describing the type exactly as +alpaca serializes it, built from alpaca's `field_type` codes +(`components/serialization/detail/alpaca/include/alpaca/detail/field_type.h`), +recursively, one entry per type; multi-byte counts are little-endian: + +| type | bytes | +|-----------------------------|-----------------------------------------| +| scalar (`bool` ... `double`) | code | +| enum | `enum_class`, underlying integer | +| `std::string` | `string` | +| `std::vector` | `vector`, T | +| `std::array` | `array`, u32 N, T | +| `std::list` / `deque` | `list` / `deque`, T | +| `std::map` | `map`, K, V (`unordered_map` likewise) | +| `std::set` | `set`, T (`unordered_set` likewise) | +| `std::optional` | `optional`, T | +| `std::pair` | `pair`, A, B | +| `std::tuple` | `tuple`, u16 count, Ts... | +| `std::variant` | `variant`, u16 count, Ts... | +| aggregate | `struct_`, u16 field count, fields... | + +A version number reused for a different layout therefore loads with +`layout_mismatch` instead of producing garbage, and a reader without the C++ +type (a Python tool) can compute the same fingerprint from a field spec. + +alpaca's own `with_version` option is deliberately not used: its hash folds +`sizeof(struct)` in, which differs between a 32-bit ESP32 and a 64-bit host +(a struct of a string, a vector and an integer is 40 bytes on the ESP32, 56 on +macOS and 64 on Linux), so a file written on one would not load on the other. + +## Errors + +Every operation takes a `std::error_code &` and returns `bool`. The codes are +`espp::PersistentDataErrc` (`file_not_found`, `io_error`, `file_too_large`, +`bad_header`, `bad_checksum`, `unknown_version`, `layout_mismatch`, +`deserialize_failed`, `serialize_failed`) and compare directly: `ec == espp::PersistentDataErrc::file_not_found`. + +## Other backends + +`PersistentData::encode()` and `decode()` produce and consume the same bytes +as the file (header included, migrations applied on decode) without touching +the filesystem, for storing the structure as an NVS blob or sending it over a +transport. + +## Thread safety + +All methods lock an internal mutex; `get()` returns a copy, and `update()` +runs the caller's function and the save under the same lock. A save blocks +other callers for the duration of the flash write. + +## Host (cross-platform library) + +The component is part of the cross-platform C++ library in `lib/` (included +by `espp.hpp`), so the same class and file format work on a PC against the +installed `espp::espp` target, for desktop tools that read or write a device's +settings file, or for host-side tests of migrations. It is not in the Python +bindings: `PersistentData` is a class template over the application's own +types, which the generated bindings cannot instantiate generically. + +## Host test + +The file format, version list, migration chain and file helpers live in +`include/detail/persistent_data_codec.hpp` (standard library + alpaca only) and +are covered by `test/persistent_data_host_test.cpp`, which builds and runs on +the host; the build line is in the file's header comment. + +## Example + +The [example](./example) stores a settings struct on a LittleFS partition, +migrates a version-1 file written by "old firmware" and validates every +operation with a pass / fail summary. diff --git a/components/persistent_data/example/CMakeLists.txt b/components/persistent_data/example/CMakeLists.txt new file mode 100644 index 0000000000..3551373ca4 --- /dev/null +++ b/components/persistent_data/example/CMakeLists.txt @@ -0,0 +1,22 @@ +# The following lines of boilerplate have to be in your project's CMakeLists +# in this exact order for cmake to work correctly +cmake_minimum_required(VERSION 3.20) + +set(ENV{IDF_COMPONENT_MANAGER} "0") +include($ENV{IDF_PATH}/tools/cmake/project.cmake) + +# add the component directories that we want to use +set(EXTRA_COMPONENT_DIRS + "../../../components/" +) + +set( + COMPONENTS + "main esptool_py logger file_system persistent_data" + CACHE STRING + "List of components to include" + ) + +project(persistent_data_example) + +set(CMAKE_CXX_STANDARD 23) diff --git a/components/persistent_data/example/README.md b/components/persistent_data/example/README.md new file mode 100644 index 0000000000..6625cc2a24 --- /dev/null +++ b/components/persistent_data/example/README.md @@ -0,0 +1,86 @@ +# Persistent Data Example + +This example shows how the `persistent_data` component stores a settings +structure in a file on a [littlefs](https://github.com/littlefs-project/littlefs) +partition, how the structure evolves over firmware versions (a version-1 file +written by old firmware is migrated v1 -> v2 -> v3 and rewritten), and how +corrupt, truncated and newer-firmware files are reported instead of guessed at. + +The example ends with a validation section that exercises every operation +(create, auto-load, `update()`, migration, newer-version refusal, corruption +detection, `encode()` / `decode()`) and prints a pass / fail summary. + +## How to use example + +### Hardware Required + +Any ESP32 with a `littlefs` partition (see `partitions.csv`); the partition +label is configured through `menuconfig` (`CONFIG_ESPP_FILE_SYSTEM_PARTITION_LABEL`). + +### Build and Flash + +Build the project and flash it to the board, then run monitor tool to view serial output: + +``` +idf.py -p PORT flash monitor +``` + +(Replace PORT with the name of the serial port to use.) + +(To exit the serial monitor, type ``Ctrl-]``.) + +See the Getting Started Guide for full steps to configure and use ESP-IDF to build projects. + +## Expected Output + +``` +I (…) persistent data example: Starting persistent data example! +I (…) persistent data example: Day-to-day use: load or create, then update +I (…) PersistentData:settings.bin: No file at '/littlefs/settings.bin' +I (…) PersistentData:settings.bin: No file at '/littlefs/settings.bin', creating it with defaults +I (…) persistent data example: Loaded (file version 0): Settings{name: 'espp', brightness: 0.50, color: 0x00ff00, favorites: [], boot_count: 0} +I (…) persistent data example: After update: Settings{name: 'espp', brightness: 1.00, color: 0x00ff00, favorites: ["red", "green"], boot_count: 1} +I (…) persistent data example: Validating persistent data operations... + [PASS] erase succeeds + [PASS] file does not exist after erase + [PASS] load of a missing file reports file_not_found + [PASS] is_loaded() is false before any load or save + [PASS] load_or_create creates the file + [PASS] file exists after create + [PASS] is_loaded() is true after create + [PASS] auto_load loads the created file + [PASS] loaded data matches + [PASS] loaded_version() is the current version + [PASS] was_migrated() is false for a current-version file + [PASS] update() saves + [PASS] update() result persisted + [PASS] update() returning false does not save + [PASS] conversion operator returns the loaded data + [PASS] assignment operator sets the data in memory + [PASS] assignment does not write the file + [PASS] save() after assignment persists it + [PASS] v1 file written + [PASS] v1 file loads + [PASS] v1 file was migrated + [PASS] migration kept the name + [PASS] migration converted brightness 80 -> 0.80 + [PASS] migration filled v2 defaults + [PASS] migration filled v3 defaults + [PASS] file was rewritten as the current version + [PASS] newer file reports unknown_version + [PASS] memory untouched after a failed load + [PASS] load_or_create does not replace a newer file + [PASS] read the raw file + [PASS] write the corrupted file + [PASS] corrupt file reports bad_checksum + [PASS] truncated file reports bad_header + [PASS] encode + [PASS] decode + [PASS] decode round trip + [PASS] final save +Persistent data validation: 37/37 checks passed - PASS +I (…) persistent data example: Persistent data example complete! +``` + +On the next boot the file exists, so the first section loads it (boot count 1) +and the update bumps it to 2. diff --git a/components/persistent_data/example/main/CMakeLists.txt b/components/persistent_data/example/main/CMakeLists.txt new file mode 100644 index 0000000000..a941e22ba7 --- /dev/null +++ b/components/persistent_data/example/main/CMakeLists.txt @@ -0,0 +1,2 @@ +idf_component_register(SRC_DIRS "." + INCLUDE_DIRS ".") diff --git a/components/persistent_data/example/main/persistent_data_example.cpp b/components/persistent_data/example/main/persistent_data_example.cpp new file mode 100644 index 0000000000..cdb6cc27ed --- /dev/null +++ b/components/persistent_data/example/main/persistent_data_example.cpp @@ -0,0 +1,287 @@ +#include +#include +#include +#include + +#include "file_system.hpp" +#include "format.hpp" +#include "logger.hpp" +#include "persistent_data.hpp" + +using namespace std::chrono_literals; + +// ---------------------------------------------------------------------------- +// The data we persist, as it evolved over three firmware releases. Every +// version is a plain aggregate (the serialization component needs that) with +// a VERSION constant, and each version knows how to migrate from the one +// before it. (They are declared oldest first because C++ needs the previous +// type complete; the PersistentData type list below is newest first.) +// ---------------------------------------------------------------------------- + +// Version 1: what the first firmware stored. +struct SettingsV1 { + static constexpr uint32_t VERSION = 1; + uint8_t brightness{50}; + std::string device_name{"espp"}; +}; + +// Version 2 added the colour and a list of favourites. +struct SettingsV2 { + static constexpr uint32_t VERSION = 2; + uint8_t brightness{50}; + std::string device_name{"espp"}; + uint32_t color_rgb{0x00FF00}; + std::vector favorites{}; + + static SettingsV2 migrate(const SettingsV1 &v1) { + return {.brightness = v1.brightness, .device_name = v1.device_name}; + } +}; + +// Version 3 (current) replaced the 8-bit brightness with a float, nested the +// display settings, and added a boot counter. +struct Settings { + static constexpr uint32_t VERSION = 3; + struct Display { + float brightness{0.5f}; // 0..1 + uint32_t color_rgb{0x00FF00}; + }; + std::string device_name{"espp"}; + Display display{}; + std::vector favorites{}; + uint32_t boot_count{0}; + + static Settings migrate(const SettingsV2 &v2) { + return {.device_name = v2.device_name, + .display = {.brightness = v2.brightness / 100.0f, .color_rgb = v2.color_rgb}, + .favorites = v2.favorites}; + } +}; + +// A hypothetical version 4 that this firmware does not know about, used by the +// validation section to write a "newer firmware" file. +struct SettingsV4 { + static constexpr uint32_t VERSION = 4; + std::string device_name{"future"}; +}; + +template <> struct fmt::formatter { + constexpr auto parse(format_parse_context &ctx) { return ctx.begin(); } + template auto format(const Settings &s, FormatContext &ctx) const { + return fmt::format_to(ctx.out(), + "Settings{{name: '{}', brightness: {:.2f}, color: {:#08x}, favorites: " + "{}, boot_count: {}}}", + s.device_name, s.display.brightness, s.display.color_rgb, s.favorites, + s.boot_count); + } +}; + +extern "C" void app_main(void) { + static espp::Logger logger( + {.tag = "persistent data example", .level = espp::Logger::Verbosity::INFO}); + logger.info("Starting persistent data example!"); + + // the file system must be mounted first (partition label from menuconfig) + auto &fs = espp::FileSystem::get(); + std::filesystem::path settings_path = fs.get_root_path() / "settings.bin"; + + int passed = 0; + int failed = 0; + auto check = [&](const char *name, bool ok) { + fmt::print(" [{}] {}\n", ok ? "PASS" : "FAIL", name); + if (ok) { + passed++; + } else { + failed++; + } + }; + + { + logger.info("Day-to-day use: load or create, then update"); + //! [persistent data example] + // The type list is newest first; the first type is the current one. A file + // written by any listed version loads and is migrated to Settings. + using PersistentSettings = espp::PersistentData; + std::error_code ec; + PersistentSettings settings({ + .file_path = settings_path, + .auto_load = false, // we want to handle "no file yet" ourselves below + .log_level = espp::Logger::Verbosity::INFO, + }); + + // load the file, or create it from defaults if there is none; any other + // error (corrupt file, newer firmware's file) is reported, not papered over + if (!settings.load_or_create(Settings{}, ec)) { + logger.error("Could not load or create settings: {}", ec.message()); + } + logger.info("Loaded (file version {}): {}", settings.loaded_version(), settings.get()); + + // read-modify-write under one lock, saved atomically + settings.update([](Settings &s) { s.boot_count++; }, ec); + if (ec) { + logger.error("Could not save settings: {}", ec.message()); + } + + // get() returns a copy; set() only changes memory, save() writes it + auto copy = settings.get(); + copy.favorites = {"red", "green"}; + settings.set(copy); + settings.save(ec); + + // the same through the conversion / assignment operators + Settings s = settings; // == settings.get() + s.display.brightness = 1.0f; + settings = s; // == settings.set(s) + settings.save(ec); + //! [persistent data example] + logger.info("After update: {}", settings.get()); + } + + { + logger.info("Validating persistent data operations..."); + std::error_code ec; + using PersistentSettings = espp::PersistentData; + + // start clean + { + PersistentSettings settings({.file_path = settings_path, .auto_load = false}); + check("erase succeeds", settings.erase(ec) && !ec); + check("file does not exist after erase", !settings.exists()); + check("load of a missing file reports file_not_found", + !settings.load(ec) && ec == espp::PersistentDataErrc::file_not_found); + check("is_loaded() is false before any load or save", !settings.is_loaded()); + } + + // create + reload + { + PersistentSettings settings({.file_path = settings_path, .auto_load = false}); + Settings defaults{.device_name = "unit", .favorites = {"a"}, .boot_count = 1}; + check("load_or_create creates the file", settings.load_or_create(defaults, ec) && !ec); + check("file exists after create", settings.exists()); + check("is_loaded() is true after create", settings.is_loaded()); + PersistentSettings again( + {.file_path = settings_path, .log_level = espp::Logger::Verbosity::WARN}); + check("auto_load loads the created file", again.is_loaded()); + check("loaded data matches", again.get().device_name == "unit" && + again.get().favorites == std::vector{"a"} && + again.get().boot_count == 1); + check("loaded_version() is the current version", + again.loaded_version() == PersistentSettings::CURRENT_VERSION); + check("was_migrated() is false for a current-version file", !again.was_migrated()); + } + + // update() round trip + { + PersistentSettings settings({.file_path = settings_path}); + bool saved = settings.update( + [](Settings &s) { + s.boot_count += 10; + s.display.brightness = 0.75f; + }, + ec); + check("update() saves", saved && !ec); + PersistentSettings again({.file_path = settings_path}); + check("update() result persisted", + again.get().boot_count == 11 && again.get().display.brightness == 0.75f); + bool declined = settings.update([](Settings &) { return false; }, ec); + check("update() returning false does not save", !declined && !ec); + } + + // assignment / conversion operators + { + PersistentSettings settings({.file_path = settings_path}); + Settings s = settings; // conversion == get() + check("conversion operator returns the loaded data", s.boot_count == 11); + s.boot_count = 42; + settings = s; // assignment == set() + check("assignment operator sets the data in memory", settings.get().boot_count == 42); + PersistentSettings again({.file_path = settings_path}); + check("assignment does not write the file", again.get().boot_count == 11); + settings.save(ec); + PersistentSettings after_save({.file_path = settings_path}); + check("save() after assignment persists it", !ec && after_save.get().boot_count == 42); + } + + //! [persistent data migration example] + // Write a version-1 file the way the first firmware would have, then load + // it with the full version list: it comes back migrated v1 -> v2 -> v3 and + // (save_after_migration, the default) the file is rewritten as version 3. + { + espp::PersistentData old_firmware( + {.file_path = settings_path, .auto_load = false}); + old_firmware.save(SettingsV1{.brightness = 80, .device_name = "legacy"}, ec); + check("v1 file written", !ec); + + PersistentSettings settings({.file_path = settings_path}); + check("v1 file loads", settings.is_loaded() && settings.loaded_version() == 1); + check("v1 file was migrated", settings.was_migrated()); + auto s = settings.get(); + check("migration kept the name", s.device_name == "legacy"); + check("migration converted brightness 80 -> 0.80", s.display.brightness == 0.80f); + check("migration filled v2 defaults", s.display.color_rgb == 0x00FF00 && s.favorites.empty()); + check("migration filled v3 defaults", s.boot_count == 0); + + PersistentSettings again({.file_path = settings_path}); + check("file was rewritten as the current version", + again.loaded_version() == PersistentSettings::CURRENT_VERSION && !again.was_migrated()); + } + //! [persistent data migration example] + + // a file from newer firmware is refused, memory untouched + { + espp::PersistentData future({.file_path = settings_path, .auto_load = false}); + future.save(SettingsV4{}, ec); + PersistentSettings settings({.file_path = settings_path, .auto_load = false}); + settings.set(Settings{.device_name = "memory"}); + check("newer file reports unknown_version", + !settings.load(ec) && ec == espp::PersistentDataErrc::unknown_version); + check("memory untouched after a failed load", settings.get().device_name == "memory"); + check("load_or_create does not replace a newer file", + !settings.load_or_create(Settings{}, ec) && + ec == espp::PersistentDataErrc::unknown_version); + } + + // a corrupt file is detected + { + std::vector bytes; + check("read the raw file", espp::persistent_data::read_file(settings_path, 4096, bytes, ec)); + bytes[PersistentSettings::HEADER_SIZE] ^= 0xFF; + check("write the corrupted file", + espp::persistent_data::write_file_atomic(settings_path, bytes, ec)); + PersistentSettings settings({.file_path = settings_path, .auto_load = false}); + check("corrupt file reports bad_checksum", + !settings.load(ec) && ec == espp::PersistentDataErrc::bad_checksum); + bytes.resize(PersistentSettings::HEADER_SIZE + 3); + espp::persistent_data::write_file_atomic(settings_path, bytes, ec); + check("truncated file reports bad_header", + !settings.load(ec) && ec == espp::PersistentDataErrc::bad_header); + } + + // encode / decode without the filesystem (for NVS blobs, transports) + { + std::vector bytes; + Settings s{.device_name = "bytes", .boot_count = 5}; + check("encode", PersistentSettings::encode(s, bytes, ec) && !ec); + Settings out{}; + espp::persistent_data::LoadInfo info; + check("decode", PersistentSettings::decode(bytes, out, info, ec) && !ec); + check("decode round trip", out.device_name == "bytes" && out.boot_count == 5 && + info.file_version == 3 && !info.migrated); + } + + // leave the file in a sane state for the next boot + { + PersistentSettings settings({.file_path = settings_path, .auto_load = false}); + settings.save(Settings{.device_name = "espp", .boot_count = 1}, ec); + check("final save", !ec); + } + + fmt::print("Persistent data validation: {}/{} checks passed - {}\n", passed, passed + failed, + failed == 0 ? "PASS" : "FAIL"); + } + + logger.info("Persistent data example complete!"); + while (true) { + std::this_thread::sleep_for(1s); + } +} diff --git a/components/persistent_data/example/partitions.csv b/components/persistent_data/example/partitions.csv new file mode 100644 index 0000000000..4196003405 --- /dev/null +++ b/components/persistent_data/example/partitions.csv @@ -0,0 +1,5 @@ +# Name, Type, SubType, Offset, Size +nvs, data, nvs, 0x9000, 0x6000 +phy_init, data, phy, 0xf000, 0x1000 +factory, app, factory, 0x10000, 2M +littlefs, data, littlefs, , 1M diff --git a/components/persistent_data/example/sdkconfig.defaults b/components/persistent_data/example/sdkconfig.defaults new file mode 100644 index 0000000000..169de32462 --- /dev/null +++ b/components/persistent_data/example/sdkconfig.defaults @@ -0,0 +1,12 @@ +# Common ESP-related +# +CONFIG_ESP_SYSTEM_EVENT_TASK_STACK_SIZE=4096 +CONFIG_ESP_MAIN_TASK_STACK_SIZE=8192 + +CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y + +# +# Partition Table +# +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" diff --git a/components/persistent_data/idf_component.yml b/components/persistent_data/idf_component.yml new file mode 100644 index 0000000000..3c631fa78a --- /dev/null +++ b/components/persistent_data/idf_component.yml @@ -0,0 +1,21 @@ +## IDF Component Manager Manifest File +license: "MIT" +description: "File-backed storage of a versioned data structure with forward migration for ESP-IDF" +url: "https://github.com/esp-cpp/espp/tree/main/components/persistent_data" +repository: "git://github.com/esp-cpp/espp.git" +maintainers: + - William Emfinger +documentation: "https://esp-cpp.github.io/espp/storage/persistent_data.html" +examples: + - path: example +tags: + - cpp + - Component + - Storage + - Configuration + - Serialization +dependencies: + idf: + version: '>=5.0' + espp/base_component: '>=1.0' + espp/serialization: '>=1.0' diff --git a/components/persistent_data/include/detail/persistent_data_codec.hpp b/components/persistent_data/include/detail/persistent_data_codec.hpp new file mode 100644 index 0000000000..b2214f4f10 --- /dev/null +++ b/components/persistent_data/include/detail/persistent_data_codec.hpp @@ -0,0 +1,576 @@ +#pragma once + +// File format, version list and file I/O helpers for espp::PersistentData. +// Standard library + alpaca only so it can be built and tested on the host +// (see test/persistent_data_host_test.cpp); espp::PersistentData adds the +// logger, the mutex and the Config on top of these functions. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if __has_include() +#include +#endif + +#include "serialization.hpp" + +#include "persistent_data_errc.hpp" + +namespace espp::persistent_data { + +/// @brief Serialization options used for the payload. +/// @details `strict` rejects payloads that are shorter than the type; +/// `fixed_length_encoding` keeps the encoding independent of the +/// values stored. alpaca's own `with_version` is deliberately not used: +/// its hash folds `sizeof(struct)` in, which differs between a 32-bit +/// ESP32 and a 64-bit host, so a file written on one would not load on +/// the other. The header's layout fingerprint (see layout::fingerprint) +/// plays that role portably. +inline constexpr alpaca::options SERIALIZATION_OPTIONS = + alpaca::options::fixed_length_encoding | alpaca::options::strict; + +/// @brief Magic number at the start of every file ("ESPD" stored little-endian). +inline constexpr uint32_t MAGIC = 0x44505345; + +/// @brief Size of the file header in bytes. +/// @details Layout (all fields little-endian): +/// - u32 magic (MAGIC) +/// - u32 version (schema version of the payload, the type's `VERSION`) +/// - u32 layout (fingerprint of the payload type's layout, layout::fingerprint) +/// - u32 length (payload length in bytes) +/// - u32 crc32 (over the four fields above and the payload) +/// - payload (alpaca, SERIALIZATION_OPTIONS) +inline constexpr size_t HEADER_SIZE = 20; + +inline void put_u32(uint8_t *p, uint32_t v) { + p[0] = static_cast(v); + p[1] = static_cast(v >> 8); + p[2] = static_cast(v >> 16); + p[3] = static_cast(v >> 24); +} + +inline uint32_t get_u32(const uint8_t *p) { + return static_cast(p[0]) | (static_cast(p[1]) << 8) | + (static_cast(p[2]) << 16) | (static_cast(p[3]) << 24); +} + +/** + * @brief Portable description of a type's serialized layout. + * + * @details The fingerprint is a CRC32 over a byte sequence that describes the + * type exactly as alpaca serializes it, using alpaca's own + * `field_type` codes, and nothing platform-dependent (no sizeof, no + * padding). It is the same on every target and host, and a tool + * without the C++ type (a Python reader) can derive it from a field + * spec. Grammar (one entry per type, recursive): + * - scalar code + * - enum enum_class, + * - std::string string + * - std::vector vector, + * - std::array array, u32 N, + * - std::list / deque list / deque, + * - std::map map, , (unordered_map likewise) + * - std::set set, (unordered_set likewise) + * - std::optional optional, + * - std::pair pair, , + * - std::tuple tuple, u16 count, ... + * - std::variant variant, u16 count, ... + * - aggregate struct_, u16 field count, ... + * Multi-byte counts are little-endian. + */ +namespace layout { + +using alpaca::detail::field_type; + +template struct always_false : std::false_type {}; + +inline void code(std::vector &out, field_type t) { + out.push_back(static_cast(t)); +} +inline void put_u16(std::vector &out, uint16_t v) { + out.push_back(static_cast(v)); + out.push_back(static_cast(v >> 8)); +} +inline void put_u32(std::vector &out, uint32_t v) { + out.push_back(static_cast(v)); + out.push_back(static_cast(v >> 8)); + out.push_back(static_cast(v >> 16)); + out.push_back(static_cast(v >> 24)); +} + +template void describe(std::vector &out); + +template void describe_fields(std::vector &out) { + if constexpr (I < N) { + T ref{}; + using Field = std::decay_t(ref))>; + describe(out); + describe_fields(out); + } +} + +template void describe_tuple(std::vector &out) { + if constexpr (I < N) { + describe>(out); + describe_tuple(out); + } +} + +template void describe_variant(std::vector &out) { + if constexpr (I < N) { + describe>(out); + describe_variant(out); + } +} + +template void describe(std::vector &out) { + using U = std::remove_cv_t; + using alpaca::detail::is_specialization; + if constexpr (std::is_same_v) { + code(out, field_type::bool_); + } else if constexpr (std::is_same_v) { + code(out, field_type::char_); + } else if constexpr (std::is_same_v) { + code(out, field_type::uint8); + } else if constexpr (std::is_same_v) { + code(out, field_type::uint16); + } else if constexpr (std::is_same_v) { + code(out, field_type::uint32); + } else if constexpr (std::is_same_v) { + code(out, field_type::uint64); + } else if constexpr (std::is_same_v) { + code(out, field_type::int8); + } else if constexpr (std::is_same_v) { + code(out, field_type::int16); + } else if constexpr (std::is_same_v) { + code(out, field_type::int32); + } else if constexpr (std::is_same_v) { + code(out, field_type::int64); + } else if constexpr (std::is_same_v) { + code(out, field_type::float32); + } else if constexpr (std::is_same_v) { + code(out, field_type::float64); + } else if constexpr (std::is_enum_v) { + code(out, field_type::enum_class); + describe>(out); + } else if constexpr (is_specialization::value) { + code(out, field_type::string); + } else if constexpr (is_specialization::value) { + code(out, field_type::vector); + describe(out); + } else if constexpr (alpaca::detail::is_array_type::value) { + code(out, field_type::array); + put_u32(out, static_cast(std::tuple_size_v)); + describe(out); + } else if constexpr (is_specialization::value) { + code(out, field_type::list); + describe(out); + } else if constexpr (is_specialization::value) { + code(out, field_type::deque); + describe(out); + } else if constexpr (is_specialization::value) { + code(out, field_type::map); + describe(out); + describe(out); + } else if constexpr (is_specialization::value) { + code(out, field_type::unordered_map); + describe(out); + describe(out); + } else if constexpr (is_specialization::value) { + code(out, field_type::set); + describe(out); + } else if constexpr (is_specialization::value) { + code(out, field_type::unordered_set); + describe(out); + } else if constexpr (is_specialization::value) { + code(out, field_type::optional); + describe(out); + } else if constexpr (is_specialization::value) { + code(out, field_type::pair); + describe(out); + describe(out); + } else if constexpr (is_specialization::value) { + code(out, field_type::tuple); + put_u16(out, static_cast(std::tuple_size_v)); + describe_tuple, 0>(out); + } else if constexpr (is_specialization::value) { + code(out, field_type::variant); + put_u16(out, static_cast(std::variant_size_v)); + describe_variant, 0>(out); + } else if constexpr (std::is_aggregate_v) { + constexpr size_t N = alpaca::detail::aggregate_arity::size(); + code(out, field_type::struct_); + put_u16(out, static_cast(N)); + describe_fields(out); + } else { + static_assert(always_false::value, + "persistent_data: this field type has no layout description (add it to " + "layout::describe, or use a type alpaca serializes)"); + } +} + +/// @brief The layout description bytes of T (see the grammar above). +template std::vector describe() { + std::vector out; + describe(out); + return out; +} + +/// @brief CRC32 of describe(); the value stored in the file header. +template uint32_t fingerprint() { + static const uint32_t value = [] { + auto bytes = describe(); + return crc32_fast(bytes.data(), bytes.size()); + }(); + return value; +} + +} // namespace layout + +/// @brief A type usable as a schema version: an aggregate with a `static constexpr uint32_t +/// VERSION` member. +template +concept VersionedType = std::is_aggregate_v && requires { + { T::VERSION } -> std::convertible_to; +}; + +/// @brief `Next` can be produced from `Prev` by `Next::migrate(const Prev &)`. +template +concept MigratableFrom = requires(const Prev &prev) { + { Next::migrate(prev) } -> std::convertible_to; +}; + +/// @brief What the header of a well-formed file says. +struct Header { + uint32_t version{0}; ///< Schema version of the payload + uint32_t layout{0}; ///< Layout fingerprint of the type that wrote the payload +}; + +/// @brief Encode `value` (schema version `version`) into the file format. +/// @return true on success; on failure ec is set and `out` is unspecified. +template +bool encode(const T &value, uint32_t version, std::vector &out, std::error_code &ec) { + out.clear(); + out.resize(HEADER_SIZE); + // alpaca appends to the vector, so the payload lands right after the header + size_t payload_len = espp::serialize(value, out); + if (payload_len == 0 || out.size() != HEADER_SIZE + payload_len) { + ec = PersistentDataErrc::serialize_failed; + return false; + } + put_u32(out.data(), MAGIC); + put_u32(out.data() + 4, version); + put_u32(out.data() + 8, layout::fingerprint()); + put_u32(out.data() + 12, static_cast(payload_len)); + // crc over the header fields (the crc field itself is skipped) plus the payload + uint32_t crc = crc32_fast(out.data(), 16); + crc = crc32_fast(out.data() + HEADER_SIZE, payload_len, crc); + put_u32(out.data() + 16, crc); + ec.clear(); + return true; +} + +/// @brief Validate the header + checksum of an encoded file. +/// @param[in] bytes The whole file +/// @param[out] header The schema version and layout fingerprint recorded in the header +/// @param[out] payload The payload bytes (a view into `bytes`) +/// @return true if the file is well-formed; ec says why not otherwise. +inline bool parse(std::span bytes, Header &header, std::span &payload, + std::error_code &ec) { + if (bytes.size() < HEADER_SIZE || get_u32(bytes.data()) != MAGIC) { + ec = PersistentDataErrc::bad_header; + return false; + } + uint32_t length = get_u32(bytes.data() + 12); + if (length != bytes.size() - HEADER_SIZE) { + // truncated write, or trailing bytes that we never wrote + ec = PersistentDataErrc::bad_header; + return false; + } + uint32_t crc = crc32_fast(bytes.data(), 16); + crc = crc32_fast(bytes.data() + HEADER_SIZE, length, crc); + if (crc != get_u32(bytes.data() + 16)) { + ec = PersistentDataErrc::bad_checksum; + return false; + } + header.version = get_u32(bytes.data() + 4); + header.layout = get_u32(bytes.data() + 8); + payload = bytes.subspan(HEADER_SIZE); + ec.clear(); + return true; +} + +/// @brief Decode a payload as type T after checking its layout fingerprint. +template +bool decode_payload(uint32_t layout, std::span payload, T &out, + std::error_code &ec) { + if (layout != layout::fingerprint()) { + ec = PersistentDataErrc::layout_mismatch; + return false; + } + std::error_code alpaca_ec; + out = alpaca::deserialize(payload, payload.size(), alpaca_ec); + if (alpaca_ec) { + ec = PersistentDataErrc::deserialize_failed; + return false; + } + ec.clear(); + return true; +} + +/** + * @brief An ordered list of schema versions of one data structure, newest first. + * @tparam Versions Aggregate types, each with `static constexpr uint32_t VERSION` + * (strictly decreasing along the list); the first is the current type. + * Every type except the last must provide `static T migrate(const Older &)` + * from the type after it, so an old file is migrated one step at a + * time toward the front of the list. + */ +template struct VersionList { + static_assert(sizeof...(Versions) > 0, "at least one version type is required"); + + static constexpr size_t count = sizeof...(Versions); + template using At = std::tuple_element_t>; + using Current = At<0>; + static constexpr std::array versions{ + static_cast(Versions::VERSION)...}; + static constexpr uint32_t current_version = versions[0]; + +private: + static constexpr bool strictly_decreasing() { + for (size_t i = 1; i < count; i++) { + if (versions[i] >= versions[i - 1]) + return false; + } + return true; + } + static_assert(strictly_decreasing(), + "VERSION values must strictly decrease along the list (newest first)"); + + template static constexpr bool chain_migratable(std::index_sequence) { + return (MigratableFrom, At> && ...); + } + static_assert(chain_migratable(std::make_index_sequence{}), + "every version except the last needs 'static T migrate(const Older &)' from " + "the version after it in the list"); + +public: + /// @brief Whether a file of schema version `version` can be loaded. + static constexpr bool supports(uint32_t version) { + return std::any_of(versions.begin(), versions.end(), + [version](uint32_t v) { return v == version; }); + } + + /// @brief Run the migration chain from the version at index I up to Current (index 0). + template static Current migrate_from(At &&value) { + if constexpr (I == 0) { + return std::move(value); + } else { + return migrate_from(At::migrate(value)); + } + } + + /// @brief Decode a payload described by `header` and migrate it to Current. + /// @return true on success; ec is unknown_version, layout_mismatch or + /// deserialize_failed otherwise. + static bool decode(const Header &header, std::span payload, Current &out, + std::error_code &ec) { + return decode_impl(header, payload, out, ec, std::make_index_sequence{}); + } + +private: + template + static bool decode_at(uint32_t layout, std::span payload, Current &out, + std::error_code &ec) { + At value{}; + if (!decode_payload(layout, payload, value, ec)) + return false; + out = migrate_from(std::move(value)); + return true; + } + + template + static bool decode_impl(const Header &header, std::span payload, Current &out, + std::error_code &ec, std::index_sequence) { + bool ok = false; + bool matched = ((versions[Is] == header.version + ? (ok = decode_at(header.layout, payload, out, ec), true) + : false) || + ...); + if (!matched) { + ec = PersistentDataErrc::unknown_version; + return false; + } + return ok; + } +}; + +// ------------------------------- file helpers -------------------------------- + +/// @brief Whether a regular file exists at `path`. +inline bool file_exists(const std::filesystem::path &path) { + std::error_code ec; + return std::filesystem::is_regular_file(path, ec); +} + +/// @brief Read a whole file. +/// @param max_size Files larger than this are rejected (file_too_large) before +/// any allocation, so a corrupt directory entry cannot exhaust memory. +inline bool read_file(const std::filesystem::path &path, size_t max_size, std::vector &out, + std::error_code &ec) { + // path.c_str() is wchar_t on Windows; string() gives the narrow form everywhere + const std::string path_str = path.string(); + FILE *f = std::fopen(path_str.c_str(), "rb"); + if (!f) { + ec = errno == ENOENT ? PersistentDataErrc::file_not_found : PersistentDataErrc::io_error; + return false; + } + bool ok = false; + do { + if (std::fseek(f, 0, SEEK_END) != 0) { + ec = PersistentDataErrc::io_error; + break; + } + long size = std::ftell(f); + if (size < 0 || std::fseek(f, 0, SEEK_SET) != 0) { + ec = PersistentDataErrc::io_error; + break; + } + if (static_cast(size) > max_size) { + ec = PersistentDataErrc::file_too_large; + break; + } + out.resize(static_cast(size)); + if (size > 0 && std::fread(out.data(), 1, out.size(), f) != out.size()) { + ec = PersistentDataErrc::io_error; + break; + } + ok = true; + } while (false); + std::fclose(f); + if (ok) + ec.clear(); + return ok; +} + +/// @brief Write `bytes` to `path` atomically. +/// @details Writes `.tmp`, flushes and syncs it, then renames it over +/// `path`, so a reset in the middle of a save leaves the previous file +/// intact. If the filesystem cannot rename over an existing file +/// (SPIFFS), the old file is removed first and the rename retried; +/// that short window is the only non-atomic step. +inline bool write_file_atomic(const std::filesystem::path &path, std::span bytes, + std::error_code &ec) { + std::filesystem::path tmp = path; + tmp += ".tmp"; + const std::string tmp_str = tmp.string(); + FILE *f = std::fopen(tmp_str.c_str(), "wb"); + if (!f) { + ec = PersistentDataErrc::io_error; + return false; + } + bool ok = std::fwrite(bytes.data(), 1, bytes.size(), f) == bytes.size(); + ok = (std::fflush(f) == 0) && ok; +#if __has_include() + ok = (fsync(fileno(f)) == 0) && ok; +#endif + ok = (std::fclose(f) == 0) && ok; + std::error_code fs_ec; + if (!ok) { + std::filesystem::remove(tmp, fs_ec); + ec = PersistentDataErrc::io_error; + return false; + } + // std::filesystem::rename replaces an existing file atomically on POSIX + // (rename(2)) and on Windows (MoveFileEx REPLACE_EXISTING) + std::filesystem::rename(tmp, path, fs_ec); + if (fs_ec) { + // some filesystems (SPIFFS) refuse to replace an existing file: remove it + // first and retry; that short window is the only non-atomic step + std::filesystem::remove(path, fs_ec); + std::filesystem::rename(tmp, path, fs_ec); + if (fs_ec) { + std::filesystem::remove(tmp, fs_ec); + ec = PersistentDataErrc::io_error; + return false; + } + } + ec.clear(); + return true; +} + +/// @brief Remove the file at `path` (and a leftover `.tmp`). +/// @return true if the file is gone afterwards, including when there was none. +inline bool remove_file(const std::filesystem::path &path, std::error_code &ec) { + std::filesystem::path tmp = path; + tmp += ".tmp"; + std::error_code fs_ec; + std::filesystem::remove(tmp, fs_ec); + // remove() returns false without an error when there is nothing to remove + std::filesystem::remove(path, fs_ec); + if (fs_ec) { + ec = PersistentDataErrc::io_error; + return false; + } + ec.clear(); + return true; +} + +// --------------------------- load / save a VersionList ------------------------ + +/// @brief Result of a successful load. +struct LoadInfo { + uint32_t file_version{0}; ///< Schema version found in the file + bool migrated{false}; ///< The file was older than Current and was migrated +}; + +/// @brief Read, validate, decode and migrate the file at `path` into `out`. +template +bool load_file(const std::filesystem::path &path, size_t max_size, typename List::Current &out, + LoadInfo &info, std::error_code &ec) { + std::vector bytes; + if (!read_file(path, max_size, bytes, ec)) + return false; + Header header; + std::span payload; + if (!parse(bytes, header, payload, ec)) + return false; + if (!List::decode(header, payload, out, ec)) + return false; + info.file_version = header.version; + info.migrated = header.version != List::current_version; + return true; +} + +/// @brief Encode `value` as the current version and write it atomically to `path`. +template +bool save_file(const std::filesystem::path &path, const typename List::Current &value, + std::error_code &ec) { + std::vector bytes; + if (!encode(value, List::current_version, bytes, ec)) + return false; + return write_file_atomic(path, bytes, ec); +} + +} // namespace espp::persistent_data diff --git a/components/persistent_data/include/persistent_data.hpp b/components/persistent_data/include/persistent_data.hpp new file mode 100644 index 0000000000..84aefa9c30 --- /dev/null +++ b/components/persistent_data/include/persistent_data.hpp @@ -0,0 +1,334 @@ +#pragma once + +#include +#include +#include +#include + +#include "base_component.hpp" + +#include "detail/persistent_data_codec.hpp" +#include "persistent_data_errc.hpp" + +namespace espp { + +/** + * @brief File-backed storage for one data structure with schema versioning and + * forward migration. + * + * @details Stores a serializable aggregate (see the serialization component) + * in a single file on any filesystem reachable through the VFS + * (LittleFS, SPIFFS, SD card). Every file carries a small header with + * a magic number, the schema version of the data it holds, a portable + * fingerprint of the type's layout, the payload length and a CRC32, + * so a load distinguishes "no file yet", "corrupt or truncated", + * "written by newer firmware", "version number reused for a different + * layout" and "old version, migrated" instead of guessing by trying + * to parse. The format is identical on every target and on the host, + * so a PC tool can read and write a device's file. + * + * Versions are given as a type list, newest first; the first type is + * the current one. Each type declares `static constexpr uint32_t + * VERSION` (strictly decreasing along the list) and every type except + * the last provides `static T migrate(const Older &)` from the type + * after it. A file of an older version is decoded as that version and + * walked toward the front of the list one step at a time, so each + * version only knows how to come from the one before it. A new + * version is prepended; dropping support for the oldest version is + * removing the last entry: such files then load with + * PersistentDataErrc::unknown_version and the application decides + * what to do (typically load_or_create()). + * + * Saves are atomic: the data is written to `.tmp`, synced, then + * renamed over the file, so a reset mid-save keeps the previous + * contents. + * + * All methods are thread-safe; get() returns a copy and update() + * runs the caller's function and the save under the same lock. A save + * blocks other callers for the duration of the flash write. For + * convenience the object converts to a copy of the data (get()) and + * can be assigned from it (set()); saving stays an explicit call + * because it can fail. + * + * @tparam Versions The schema versions, newest first; the first is the current + * type. Each must be an aggregate the serialization component can + * (de)serialize. + * + * \section persistent_data_ex1 Persistent Data Example + * \snippet persistent_data_example.cpp persistent data example + * \section persistent_data_ex2 Migration Example + * \snippet persistent_data_example.cpp persistent data migration example + */ +template class PersistentData : public BaseComponent { + using List = persistent_data::VersionList; + +public: + /// The current (newest) version type: what get(), set() and save() work with. + using Current = typename List::Current; + + /// Schema version written by save(). + static constexpr uint32_t CURRENT_VERSION = List::current_version; + + /// Bytes of header in front of the payload in every file. + static constexpr size_t HEADER_SIZE = persistent_data::HEADER_SIZE; + + /** + * @brief Configuration for the PersistentData. + */ + struct Config { + std::filesystem::path file_path; ///< File to load from / save to + bool auto_load{true}; ///< Call load() in the constructor (its errors are logged, a + ///< missing file at info level) + bool save_after_migration{true}; ///< After loading an older version, save the migrated data + ///< so the next load is a plain read + size_t max_file_size{64 * 1024}; ///< Files larger than this are rejected before being read + Logger::Verbosity log_level{Logger::Verbosity::WARN}; ///< Verbosity of the logger + }; + + /** + * @brief Construct the PersistentData and, if configured, load the file. + * @param config The configuration. + */ + explicit PersistentData(const Config &config) + : BaseComponent("PersistentData", config.log_level) + , file_path_(config.file_path) + , save_after_migration_(config.save_after_migration) + , max_file_size_(config.max_file_size) { + set_log_tag("PersistentData:" + file_path_.filename().string()); + if (config.auto_load) { + std::error_code ec; + load(ec); + } + } + + /// @brief The file this object loads from and saves to. + const std::filesystem::path &file_path() const { return file_path_; } + + /// @brief Whether a file exists at the configured path. + bool exists() const { return persistent_data::file_exists(file_path_); } + + /// @brief Whether the data in memory came from, or was saved to, the file. + /// @details False until a load or save succeeds; get() then returns a + /// default-constructed Current. + bool is_loaded() const { + std::lock_guard lock(mutex_); + return loaded_; + } + + /// @brief Schema version found in the file by the last successful load. + /// @return The version, or 0 if nothing was loaded (a save does not change it). + uint32_t loaded_version() const { + std::lock_guard lock(mutex_); + return load_info_.file_version; + } + + /// @brief Whether the last successful load migrated an older file. + bool was_migrated() const { + std::lock_guard lock(mutex_); + return load_info_.migrated; + } + + /// @brief Get a copy of the data in memory. + /// @note Does not read the file; call load() for that. + Current get() const { + std::lock_guard lock(mutex_); + return data_; + } + + /// @brief Replace the data in memory. + /// @note Does not write the file; call save() for that, or use save(data) / update(). + void set(const Current &data) { + std::lock_guard lock(mutex_); + data_ = data; + } + + /// @brief Assign new data, the same as set(): `settings = Settings{...};`. + /// @note In memory only; save() still writes the file (and reports errors). + PersistentData &operator=(const Current &data) { + set(data); + return *this; + } + + /// @brief Convert to a copy of the data in memory, the same as get(): + /// `Settings s = settings;`. + operator Current() const { return get(); } + + /** + * @brief Load the file, migrating it to the current version if needed. + * @details On success the data in memory is replaced. On failure it is left + * as it was and ec says why: file_not_found, io_error, + * file_too_large, bad_header, bad_checksum, unknown_version or + * deserialize_failed. + * @param[out] ec The error, if any. + * @return True if the data was loaded. + */ + bool load(std::error_code &ec) { + std::lock_guard lock(mutex_); + return load_locked(ec); + } + + /** + * @brief Load the file, or create it from `defaults` if there is none. + * @details Calls load(); if that fails with file_not_found, `defaults` is + * saved and becomes the data in memory. Any other load error is + * returned untouched (a corrupt file is not silently replaced; call + * save(defaults) or erase() for that). + * @param defaults The data to save when no file exists. + * @param[out] ec The error, if any. + * @return True if the data in memory is now valid (loaded or created). + */ + bool load_or_create(const Current &defaults, std::error_code &ec) { + std::lock_guard lock(mutex_); + if (load_locked(ec)) + return true; + if (ec != PersistentDataErrc::file_not_found) + return false; + logger_.info("No file at '{}', creating it with defaults", file_path_.string()); + data_ = defaults; + return save_locked(ec); + } + + /** + * @brief Save the data in memory to the file (atomically). + * @param[out] ec The error, if any (serialize_failed or io_error). + * @return True if the file was written. + */ + bool save(std::error_code &ec) { + std::lock_guard lock(mutex_); + return save_locked(ec); + } + + /** + * @brief Replace the data in memory and save it. + * @param data The new data. + * @param[out] ec The error, if any. + * @return True if the file was written. + */ + bool save(const Current &data, std::error_code &ec) { + std::lock_guard lock(mutex_); + data_ = data; + return save_locked(ec); + } + + /** + * @brief Modify the data in memory and save it, under one lock. + * @param fn Callable taking `Current &`; it may return void, or a bool where + * false means "do not save" (the in-memory change is kept). + * @param[out] ec The error, if any. + * @return True if the file was written (false also when fn declined). + */ + template bool update(Fn &&fn, std::error_code &ec) { + std::lock_guard lock(mutex_); + if constexpr (std::is_same_v, bool>) { + if (!fn(data_)) { + ec.clear(); + return false; + } + } else { + fn(data_); + } + return save_locked(ec); + } + + /** + * @brief Remove the file (and a leftover temporary file). + * @details The data in memory is kept; is_loaded() becomes false. + * @param[out] ec The error, if any. + * @return True if no file exists afterwards (also when there was none). + */ + bool erase(std::error_code &ec) { + std::lock_guard lock(mutex_); + if (!persistent_data::remove_file(file_path_, ec)) { + logger_.error("Could not remove '{}': {}", file_path_.string(), ec.message()); + return false; + } + loaded_ = false; + load_info_ = {}; + return true; + } + + /** + * @brief Encode data in the file format (header + payload) without writing it. + * @details For storing the same bytes elsewhere (NVS blob, a transport); + * decode() reads them back, migrations included. + */ + static bool encode(const Current &data, std::vector &out, std::error_code &ec) { + return persistent_data::encode(data, CURRENT_VERSION, out, ec); + } + + /** + * @brief Decode bytes produced by encode() (any supported version) into `out`. + * @param[in] bytes The encoded bytes. + * @param[out] out The decoded, migrated data. + * @param[out] info Version found and whether it was migrated. + * @param[out] ec The error, if any. + */ + static bool decode(std::span bytes, Current &out, persistent_data::LoadInfo &info, + std::error_code &ec) { + persistent_data::Header header; + std::span payload; + if (!persistent_data::parse(bytes, header, payload, ec)) + return false; + if (!List::decode(header, payload, out, ec)) + return false; + info.file_version = header.version; + info.migrated = header.version != CURRENT_VERSION; + return true; + } + + /// @brief Portable fingerprint of the current type's serialized layout + /// (what save() writes into the header; see layout::fingerprint). + static uint32_t layout_fingerprint() { return persistent_data::layout::fingerprint(); } + +protected: + bool load_locked(std::error_code &ec) { + Current loaded{}; + persistent_data::LoadInfo info; + if (!persistent_data::load_file(file_path_, max_file_size_, loaded, info, ec)) { + if (ec == PersistentDataErrc::file_not_found) { + logger_.info("No file at '{}'", file_path_.string()); + } else { + logger_.error("Could not load '{}': {}", file_path_.string(), ec.message()); + } + return false; + } + data_ = std::move(loaded); + loaded_ = true; + load_info_ = info; + if (info.migrated) { + logger_.info("Migrated '{}' from version {} to {}", file_path_.string(), info.file_version, + CURRENT_VERSION); + if (save_after_migration_) { + std::error_code save_ec; + if (!save_locked(save_ec)) { + // the data is loaded fine; only the rewrite failed, so report success + logger_.warn("Could not save migrated data: {}", save_ec.message()); + } + } + } else { + logger_.debug("Loaded '{}' (version {})", file_path_.string(), info.file_version); + } + ec.clear(); + return true; + } + + bool save_locked(std::error_code &ec) { + if (!persistent_data::save_file(file_path_, data_, ec)) { + logger_.error("Could not save '{}': {}", file_path_.string(), ec.message()); + return false; + } + loaded_ = true; + logger_.debug("Saved '{}' (version {})", file_path_.string(), CURRENT_VERSION); + return true; + } + + std::filesystem::path file_path_; + bool save_after_migration_{true}; + size_t max_file_size_{64 * 1024}; + Current data_{}; + bool loaded_{false}; + persistent_data::LoadInfo load_info_{}; + mutable std::mutex mutex_; +}; + +} // namespace espp diff --git a/components/persistent_data/include/persistent_data_errc.hpp b/components/persistent_data/include/persistent_data_errc.hpp new file mode 100644 index 0000000000..84decdda18 --- /dev/null +++ b/components/persistent_data/include/persistent_data_errc.hpp @@ -0,0 +1,70 @@ +#pragma once + +#include +#include + +namespace espp { + +/** + * @brief Error codes reported by espp::PersistentData. + * @details Convertible to std::error_code; compare with `ec == PersistentDataErrc::file_not_found`. + */ +enum class PersistentDataErrc { + // no 0: 0 is "no error" + file_not_found = 1, ///< No file at the configured path (nothing has been saved yet) + io_error, ///< The filesystem refused to open / read / write / rename the file + file_too_large, ///< The file is larger than Config::max_file_size + bad_header, ///< Wrong magic, wrong length or truncated file (not written by us or + ///< cut short) + bad_checksum, ///< The header + payload CRC32 does not match (corrupt file) + unknown_version, ///< The file's schema version is not in the version list (written by + ///< newer firmware, or support for that version was dropped) + deserialize_failed, ///< The payload does not decode as the type registered for its version + serialize_failed, ///< The data could not be serialized + layout_mismatch, ///< The payload was written by a type with a different layout than the + ///< one registered for its version (a version number was reused) +}; + +/** + * @brief The std::error_category for PersistentDataErrc. + */ +inline const std::error_category &persistent_data_category() { + struct Category : std::error_category { + const char *name() const noexcept override { return "persistent_data"; } + std::string message(int ev) const override { + switch (static_cast(ev)) { + case PersistentDataErrc::file_not_found: + return "File not found"; + case PersistentDataErrc::io_error: + return "Filesystem I/O error"; + case PersistentDataErrc::file_too_large: + return "File larger than the configured maximum"; + case PersistentDataErrc::bad_header: + return "Bad file header (wrong magic, wrong length or truncated)"; + case PersistentDataErrc::bad_checksum: + return "Checksum mismatch (corrupt file)"; + case PersistentDataErrc::unknown_version: + return "Unsupported schema version"; + case PersistentDataErrc::deserialize_failed: + return "Payload does not decode as the type registered for its version"; + case PersistentDataErrc::serialize_failed: + return "Could not serialize the data"; + case PersistentDataErrc::layout_mismatch: + return "Payload layout does not match the type registered for its version"; + default: + return "Unknown persistent_data error"; + } + } + }; + static const Category category; + return category; +} + +/// @brief Build a std::error_code from a PersistentDataErrc. +inline std::error_code make_error_code(PersistentDataErrc e) { + return {static_cast(e), persistent_data_category()}; +} + +} // namespace espp + +template <> struct std::is_error_code_enum : std::true_type {}; diff --git a/components/persistent_data/test/persistent_data_host_test.cpp b/components/persistent_data/test/persistent_data_host_test.cpp new file mode 100644 index 0000000000..50b7f15e84 --- /dev/null +++ b/components/persistent_data/test/persistent_data_host_test.cpp @@ -0,0 +1,414 @@ +// Host-buildable test for the PersistentData file format, version list, +// migration chain and file helpers (include/detail/persistent_data_codec.hpp -- +// standard library + alpaca only). Build & run from the repo root, as one +// command (alpaca is third-party, hence -isystem so its own warnings do not +// trip -Werror): +// c++ -std=c++23 -Wall -Wextra -Werror +// -I components/persistent_data/include -I components/serialization/include +// -isystem components/serialization/detail/alpaca/include +// components/persistent_data/test/persistent_data_host_test.cpp -o test && ./test + +#include +#include +#include +#include +#include +#include + +#include "detail/persistent_data_codec.hpp" + +namespace pd = espp::persistent_data; + +static int g_failures = 0; +static int g_checks = 0; +#define CHECK(cond) \ + do { \ + ++g_checks; \ + if (!(cond)) { \ + std::printf(" FAIL: %s (line %d)\n", #cond, __LINE__); \ + ++g_failures; \ + } \ + } while (0) + +// ------------------------------ test types ----------------------------------- + +struct ConfigV1 { + static constexpr uint32_t VERSION = 1; + uint32_t counter{0}; + std::string name{}; +}; + +struct ConfigV2 { + static constexpr uint32_t VERSION = 2; + uint32_t counter{0}; + std::string name{}; + std::vector gains{}; + + static ConfigV2 migrate(const ConfigV1 &v1) { + return {.counter = v1.counter, .name = v1.name, .gains = {1.0f, 1.0f}}; + } +}; + +struct ConfigV3 { + static constexpr uint32_t VERSION = 3; + // the counter was dropped, a nested struct and a flag were added + struct Limits { + float min{0}; + float max{1}; + }; + std::string name{}; + std::vector gains{}; + Limits limits{}; + bool enabled{true}; + + static ConfigV3 migrate(const ConfigV2 &v2) { + return {.name = v2.name + " (migrated)", + .gains = v2.gains, + .limits = {.min = -1.0f, .max = 1.0f}, + .enabled = v2.counter > 0}; + } +}; + +// a different type that claims VERSION 3 as well, with another layout +struct OtherV3 { + static constexpr uint32_t VERSION = 3; + uint64_t a{0}; + uint64_t b{0}; +}; + +// a hypothetical future version this firmware does not know about +struct ConfigV4 { + static constexpr uint32_t VERSION = 4; + std::string name{}; + uint8_t flags{0}; +}; + +using Full = pd::VersionList; +using Recent = pd::VersionList; +using Single = pd::VersionList; + +static_assert(Full::count == 3); +static_assert(Full::current_version == 3); +static_assert(Full::supports(1) && Full::supports(2) && Full::supports(3) && !Full::supports(4)); +static_assert(!Recent::supports(1)); +static_assert(std::is_same_v); +static_assert(pd::VersionedType && !pd::VersionedType); +static_assert(pd::MigratableFrom && !pd::MigratableFrom); + +// ------------------------------ helpers -------------------------------------- + +template static std::vector encoded(const T &value) { + std::vector bytes; + std::error_code ec; + bool ok = pd::encode(value, T::VERSION, bytes, ec); + CHECK(ok && !ec); + return bytes; +} + +static bool decode_full(const std::vector &bytes, ConfigV3 &out, uint32_t &version, + std::error_code &ec) { + std::span payload; + pd::Header header; + if (!pd::parse(bytes, header, payload, ec)) + return false; + version = header.version; + return Full::decode(header, payload, out, ec); +} + +// ------------------------------ tests ---------------------------------------- + +static void test_header_layout() { + std::printf("header layout\n"); + ConfigV3 v3{.name = "abc", .gains = {0.5f}, .limits = {.min = 0, .max = 2}, .enabled = false}; + auto bytes = encoded(v3); + CHECK(bytes.size() > pd::HEADER_SIZE); + CHECK(pd::get_u32(bytes.data()) == pd::MAGIC); + CHECK(bytes[0] == 'E' && bytes[1] == 'S' && bytes[2] == 'P' && bytes[3] == 'D'); + CHECK(pd::get_u32(bytes.data() + 4) == 3); + CHECK(pd::get_u32(bytes.data() + 8) == pd::layout::fingerprint()); + CHECK(pd::get_u32(bytes.data() + 12) == bytes.size() - pd::HEADER_SIZE); + // encoding is deterministic + CHECK(encoded(v3) == bytes); +} + +static void test_round_trip_current() { + std::printf("round trip of the current version\n"); + ConfigV3 v3{ + .name = "abc", .gains = {0.5f, 0.25f}, .limits = {.min = -3, .max = 3}, .enabled = false}; + auto bytes = encoded(v3); + ConfigV3 out{}; + uint32_t version = 0; + std::error_code ec; + CHECK(decode_full(bytes, out, version, ec)); + CHECK(!ec); + CHECK(version == 3); + CHECK(out.name == "abc"); + CHECK(out.gains == std::vector({0.5f, 0.25f})); + CHECK(out.limits.min == -3 && out.limits.max == 3); + CHECK(out.enabled == false); +} + +static void test_migration_chain() { + std::printf("migration chain v1 -> v2 -> v3\n"); + { + auto bytes = encoded(ConfigV1{.counter = 7, .name = "one"}); + ConfigV3 out{}; + uint32_t version = 0; + std::error_code ec; + CHECK(decode_full(bytes, out, version, ec)); + CHECK(version == 1); + CHECK(out.name == "one (migrated)"); + CHECK(out.gains == std::vector({1.0f, 1.0f})); // from v2's migrate + CHECK(out.limits.min == -1.0f && out.limits.max == 1.0f); + CHECK(out.enabled == true); // counter 7 > 0 + } + { + auto bytes = encoded(ConfigV2{.counter = 0, .name = "two", .gains = {2.0f}}); + ConfigV3 out{}; + uint32_t version = 0; + std::error_code ec; + CHECK(decode_full(bytes, out, version, ec)); + CHECK(version == 2); + CHECK(out.name == "two (migrated)"); + CHECK(out.gains == std::vector({2.0f})); + CHECK(out.enabled == false); // counter 0 + } + { + // a list that dropped v1 support reports unknown_version for a v1 file + auto bytes = encoded(ConfigV1{.counter = 7, .name = "one"}); + std::span payload; + pd::Header header; + std::error_code ec; + CHECK(pd::parse(bytes, header, payload, ec)); + ConfigV3 out{}; + CHECK(!Recent::decode(header, payload, out, ec)); + CHECK(ec == espp::PersistentDataErrc::unknown_version); + // and the single-version list behaves the same for a v2 file + bytes = encoded(ConfigV2{.counter = 1, .name = "two"}); + CHECK(pd::parse(bytes, header, payload, ec)); + CHECK(!Single::decode(header, payload, out, ec)); + CHECK(ec == espp::PersistentDataErrc::unknown_version); + } +} + +static void test_newer_version_rejected() { + std::printf("file from newer firmware\n"); + auto bytes = encoded(ConfigV4{.name = "future", .flags = 3}); + ConfigV3 out{.name = "untouched"}; + uint32_t version = 0; + std::error_code ec; + CHECK(!decode_full(bytes, out, version, ec)); + CHECK(ec == espp::PersistentDataErrc::unknown_version); + CHECK(version == 4); // parse() reports what the header says + CHECK(out.name == "untouched"); +} + +static void test_same_version_different_layout() { + std::printf("same version number, different layout\n"); + // the header's layout fingerprint catches a type that reuses a version number + auto bytes = encoded(OtherV3{.a = 1, .b = 2}); + ConfigV3 out{}; + uint32_t version = 0; + std::error_code ec; + CHECK(!decode_full(bytes, out, version, ec)); + CHECK(ec == espp::PersistentDataErrc::layout_mismatch); + // a forged fingerprint gets past the header check and fails in alpaca instead + pd::put_u32(bytes.data() + 8, pd::layout::fingerprint()); + uint32_t crc = crc32_fast(bytes.data(), 16); + crc = crc32_fast(bytes.data() + pd::HEADER_SIZE, bytes.size() - pd::HEADER_SIZE, crc); + pd::put_u32(bytes.data() + 16, crc); + CHECK(!decode_full(bytes, out, version, ec)); + CHECK(ec == espp::PersistentDataErrc::deserialize_failed); +} + +static void test_layout_fingerprint() { + std::printf("layout fingerprint grammar\n"); + using ft = alpaca::detail::field_type; + auto c = [](ft f) { return static_cast(f); }; + // ConfigV3 { string name; vector gains; Limits{float, float} limits; bool enabled; } + std::vector expected{ + c(ft::struct_), 4, 0, c(ft::string), c(ft::vector), c(ft::float32), + c(ft::struct_), 2, 0, c(ft::float32), c(ft::float32), c(ft::bool_)}; + CHECK(pd::layout::describe() == expected); + CHECK(pd::layout::fingerprint() == crc32_fast(expected.data(), expected.size())); + // nothing platform-dependent went in: the description is exactly the grammar + CHECK(pd::layout::describe().size() == expected.size()); + // different layouts, different fingerprints; same layout, same fingerprint + CHECK(pd::layout::fingerprint() != pd::layout::fingerprint()); + CHECK(pd::layout::fingerprint() != pd::layout::fingerprint()); + struct SameAsV1 { + uint32_t counter{0}; + std::string name{}; + }; + CHECK(pd::layout::fingerprint() == pd::layout::fingerprint()); + // enums carry their underlying type, arrays their length + enum class Small : uint8_t { a }; + enum class Wide : uint32_t { a }; + struct E1 { + Small s{}; + }; + struct E2 { + Wide w{}; + }; + CHECK(pd::layout::fingerprint() != pd::layout::fingerprint()); + CHECK(pd::layout::describe() == + std::vector({c(ft::struct_), 1, 0, c(ft::enum_class), c(ft::uint8)})); + struct A3 { + std::array v{}; + }; + struct A4 { + std::array v{}; + }; + CHECK(pd::layout::fingerprint() != pd::layout::fingerprint()); + CHECK(pd::layout::describe() == + std::vector({c(ft::struct_), 1, 0, c(ft::array), 3, 0, 0, 0, c(ft::uint8)})); +} + +static void test_corruption() { + std::printf("corrupt / truncated / foreign files\n"); + ConfigV3 v3{.name = "abc", .gains = {0.5f}, .limits = {}, .enabled = true}; + auto good = encoded(v3); + ConfigV3 out{}; + uint32_t version = 0; + std::error_code ec; + + // empty file + CHECK(!decode_full({}, out, version, ec)); + CHECK(ec == espp::PersistentDataErrc::bad_header); + // shorter than a header + CHECK(!decode_full(std::vector(good.begin(), good.begin() + 10), out, version, ec)); + CHECK(ec == espp::PersistentDataErrc::bad_header); + // wrong magic + auto bad = good; + bad[0] = 'X'; + CHECK(!decode_full(bad, out, version, ec)); + CHECK(ec == espp::PersistentDataErrc::bad_header); + // truncated payload (a reset mid-write without the atomic rename) + CHECK(!decode_full(std::vector(good.begin(), good.end() - 1), out, version, ec)); + CHECK(ec == espp::PersistentDataErrc::bad_header); + // trailing junk + bad = good; + bad.push_back(0); + CHECK(!decode_full(bad, out, version, ec)); + CHECK(ec == espp::PersistentDataErrc::bad_header); + // one flipped payload bit + bad = good; + bad[pd::HEADER_SIZE + 2] ^= 0x01; + CHECK(!decode_full(bad, out, version, ec)); + CHECK(ec == espp::PersistentDataErrc::bad_checksum); + // a flipped version byte is caught by the crc too, so the version field can be trusted + bad = good; + bad[4] ^= 0x01; + CHECK(!decode_full(bad, out, version, ec)); + CHECK(ec == espp::PersistentDataErrc::bad_checksum); + // a length that claims more than the file holds + bad = good; + bad[12] += 1; + CHECK(!decode_full(bad, out, version, ec)); + CHECK(ec == espp::PersistentDataErrc::bad_header); +} + +static void test_files() { + std::printf("file helpers (atomic save, load, remove)\n"); + namespace fs = std::filesystem; + fs::path dir = fs::temp_directory_path() / "espp_persistent_data_test"; + fs::remove_all(dir); + fs::create_directories(dir); + fs::path path = dir / "config.bin"; + fs::path tmp = dir / "config.bin.tmp"; + std::error_code ec; + + // nothing there yet + CHECK(!pd::file_exists(path)); + ConfigV3 out{}; + pd::LoadInfo info; + CHECK(!pd::load_file(path, 1024, out, info, ec)); + CHECK(ec == espp::PersistentDataErrc::file_not_found); + // removing a missing file is fine + CHECK(pd::remove_file(path, ec)); + CHECK(!ec); + + // save + load the current version + ConfigV3 v3{.name = "disk", .gains = {1, 2, 3}, .limits = {.min = 0, .max = 9}, .enabled = true}; + CHECK(pd::save_file(path, v3, ec)); + CHECK(!ec); + CHECK(pd::file_exists(path)); + CHECK(!fs::exists(tmp)); // the temporary was renamed away + CHECK(pd::load_file(path, 1024, out, info, ec)); + CHECK(!ec); + CHECK(out.name == "disk" && out.gains.size() == 3 && out.limits.max == 9); + CHECK(info.file_version == 3 && !info.migrated); + + // a save replaces the existing file (rename over it) and keeps no temporary + v3.name = "disk2"; + CHECK(pd::save_file(path, v3, ec)); + CHECK(!fs::exists(tmp)); + CHECK(pd::load_file(path, 1024, out, info, ec)); + CHECK(out.name == "disk2"); + + // an old-version file on disk is migrated on load + CHECK(pd::save_file>(path, ConfigV1{.counter = 1, .name = "old"}, ec)); + CHECK(pd::load_file(path, 1024, out, info, ec)); + CHECK(!ec); + CHECK(info.file_version == 1 && info.migrated); + CHECK(out.name == "old (migrated)" && out.enabled); + + // a leftover temporary from an interrupted save does not affect the load... + { + FILE *f = std::fopen(tmp.c_str(), "wb"); + CHECK(f != nullptr); + if (f) { + std::fputs("partial", f); + std::fclose(f); + } + } + CHECK(pd::load_file(path, 1024, out, info, ec)); + CHECK(out.name == "old (migrated)"); + // ...and the next save overwrites it + CHECK(pd::save_file(path, v3, ec)); + CHECK(!fs::exists(tmp)); + + // the size cap rejects before reading + CHECK(!pd::load_file(path, 8, out, info, ec)); + CHECK(ec == espp::PersistentDataErrc::file_too_large); + + // a corrupt file on disk + { + std::vector bytes; + CHECK(pd::read_file(path, 1024, bytes, ec)); + bytes[pd::HEADER_SIZE] ^= 0xFF; + CHECK(pd::write_file_atomic(path, bytes, ec)); + } + CHECK(!pd::load_file(path, 1024, out, info, ec)); + CHECK(ec == espp::PersistentDataErrc::bad_checksum); + + // remove (also clears a temporary) + { + FILE *f = std::fopen(tmp.c_str(), "wb"); + CHECK(f != nullptr); + if (f) + std::fclose(f); + } + CHECK(pd::remove_file(path, ec)); + CHECK(!pd::file_exists(path)); + CHECK(!fs::exists(tmp)); + + // a save into a directory that does not exist reports io_error + CHECK(!pd::save_file(dir / "missing" / "config.bin", v3, ec)); + CHECK(ec == espp::PersistentDataErrc::io_error); + + fs::remove_all(dir); +} + +int main() { + test_header_layout(); + test_round_trip_current(); + test_migration_chain(); + test_newer_version_rejected(); + test_same_version_different_layout(); + test_layout_fingerprint(); + test_corruption(); + test_files(); + std::printf("%d checks, %d failures\n", g_checks, g_failures); + return g_failures == 0 ? 0 : 1; +} diff --git a/doc/Doxyfile b/doc/Doxyfile index 32bf8529f7..386b4e65a6 100755 --- a/doc/Doxyfile +++ b/doc/Doxyfile @@ -161,6 +161,7 @@ EXAMPLE_PATH = \ $(PROJECT_PATH)/components/mt6701/example/main/mt6701_example.cpp \ $(PROJECT_PATH)/components/neopixel/example/main/neopixel_example.cpp \ $(PROJECT_PATH)/components/nvs/example/main/nvs_example.cpp \ + $(PROJECT_PATH)/components/persistent_data/example/main/persistent_data_example.cpp \ $(PROJECT_PATH)/components/odrive_ascii/example/main/odrive_ascii_example.cpp \ $(PROJECT_PATH)/components/odrive_native/example/main/odrive_native_example.cpp \ $(PROJECT_PATH)/components/odrive_native/usb_example/main/odrive_usb_example.cpp \ @@ -407,6 +408,8 @@ INPUT = \ $(PROJECT_PATH)/components/neopixel/include/neopixel.hpp \ $(PROJECT_PATH)/components/nvs/include/nvs.hpp \ $(PROJECT_PATH)/components/nvs/include/nvs_handle_espp.hpp \ + $(PROJECT_PATH)/components/persistent_data/include/persistent_data.hpp \ + $(PROJECT_PATH)/components/persistent_data/include/persistent_data_errc.hpp \ $(PROJECT_PATH)/components/mcp23x17/include/mcp23x17.hpp \ $(PROJECT_PATH)/components/meshtastic/include/meshtastic.hpp \ $(PROJECT_PATH)/components/meshtastic/include/meshtastic_crypto.hpp \ diff --git a/doc/en/storage/index.rst b/doc/en/storage/index.rst index 63829759d0..c5a7fbb6c0 100644 --- a/doc/en/storage/index.rst +++ b/doc/en/storage/index.rst @@ -9,4 +9,5 @@ storage. file_system nvs + persistent_data sdcard diff --git a/doc/en/storage/persistent_data.rst b/doc/en/storage/persistent_data.rst new file mode 100644 index 0000000000..817e25109c --- /dev/null +++ b/doc/en/storage/persistent_data.rst @@ -0,0 +1,31 @@ +Persistent Data APIs +******************** + +Persistent Data +--------------- + +The `PersistentData` class stores one data structure (a settings struct, +calibration data, a paired-device list) in a file on any filesystem reachable +through the VFS, tracks the schema version of what is on disk in a small +header (magic, version, a portable layout fingerprint, length, CRC32) and migrates older files forward +through a chain of `migrate()` steps when the structure changes. Saves are +atomic (temporary file plus rename), every operation reports a +`std::error_code` (`PersistentDataErrc`), and `encode()` / `decode()` expose +the same bytes for other storage backends. + +Code examples for the Persistent Data API are provided in the `persistent_data` +example folder. + +.. ------------------------------- Example ------------------------------------- + +.. toctree:: + + persistent_data_example + +.. ---------------------------- API Reference ---------------------------------- + +API Reference +------------- + +.. include-build-file:: inc/persistent_data.inc +.. include-build-file:: inc/persistent_data_errc.inc diff --git a/doc/en/storage/persistent_data_example.md b/doc/en/storage/persistent_data_example.md new file mode 100644 index 0000000000..e1c408273b --- /dev/null +++ b/doc/en/storage/persistent_data_example.md @@ -0,0 +1,2 @@ +```{include} ../../../components/persistent_data/example/README.md +``` diff --git a/lib/autogenerate_bindings.py b/lib/autogenerate_bindings.py index e73f9e47df..4bc36db682 100644 --- a/lib/autogenerate_bindings.py +++ b/lib/autogenerate_bindings.py @@ -616,6 +616,9 @@ def autogenerate() -> None: # serialization (template header): # include_dir + "serialization/include/serialization.hpp", + + # persistent_data (class template over the app's own version types): + # include_dir + "persistent_data/include/persistent_data.hpp", ] pydef_file = output_dir + "/pybind_espp.cpp" diff --git a/lib/espp.cmake b/lib/espp.cmake index c7dd60e575..6d235e3a97 100644 --- a/lib/espp.cmake +++ b/lib/espp.cmake @@ -259,6 +259,7 @@ set(ESPP_INCLUDES ${ESPP_COMPONENTS}/math/include ${ESPP_COMPONENTS}/ndef/include ${ESPP_COMPONENTS}/odrive_native/include + ${ESPP_COMPONENTS}/persistent_data/include ${ESPP_COMPONENTS}/pid/include ${ESPP_COMPONENTS}/rtps/include ${ESPP_COMPONENTS}/rtsp/include diff --git a/lib/include/espp.hpp b/lib/include/espp.hpp index 9e9cdd8251..81e53b3efc 100644 --- a/lib/include/espp.hpp +++ b/lib/include/espp.hpp @@ -43,6 +43,7 @@ extern "C" { #include "mjpeg_packetizer.hpp" #include "ndef.hpp" #include "odrive_native.hpp" +#include "persistent_data.hpp" #include "pid.hpp" #include "range_mapper.hpp" #include "rtp_depacketizer.hpp"