From 8299fcc8b8fcf113091ae85bbcf11f867f8c5908 Mon Sep 17 00:00:00 2001 From: Auke van Slooten Date: Wed, 23 Sep 2026 16:52:08 +0200 Subject: [PATCH 01/15] Record authorization for file-backed SimplyStore cycle --- .spiral/cycles/CYC-20260923-TTZ7C-44.md | 20 +++++++++++++++++++ .spiral/cycles/CYC-20260923-TTZ7C-44.ttl | 9 +++++++++ .spiral/sources/SRC-20260923-TTZ7C-45.md | 24 +++++++++++++++++++++++ .spiral/sources/SRC-20260923-TTZ7C-45.ttl | 11 +++++++++++ 4 files changed, 64 insertions(+) create mode 100644 .spiral/cycles/CYC-20260923-TTZ7C-44.md create mode 100644 .spiral/cycles/CYC-20260923-TTZ7C-44.ttl create mode 100644 .spiral/sources/SRC-20260923-TTZ7C-45.md create mode 100644 .spiral/sources/SRC-20260923-TTZ7C-45.ttl diff --git a/.spiral/cycles/CYC-20260923-TTZ7C-44.md b/.spiral/cycles/CYC-20260923-TTZ7C-44.md new file mode 100644 index 0000000..361e103 --- /dev/null +++ b/.spiral/cycles/CYC-20260923-TTZ7C-44.md @@ -0,0 +1,20 @@ +--- +id: CYC-20260923-TTZ7C-44 +--- + +# File-backed dataset runtime + +Goal: replace shared-memory dataset transport and retention with file-backed +reads throughout live operation and administrative replay, preserving remote +behavior, disk formats, durable command ordering and recovery restrictions. + +Branch: spiral/CYC-20260923-TTZ7C-44-file-backed, from master dbb0703. +Status: Active. Human authorization is retained in SRC-20260923-TTZ7C-45. +Continue the file-only roadmap direction; prioritize it ahead of rebuild under +the maintainer's explicit direction. Library readiness was established first. + +Evaluate with existing durability/recovery tests, file-only worker messages, +ID lookup of unread records, sparse overlays, no-op commands, worker replacement +and shutdown, index/framing/integrity failures and a bounded-memory large-file +probe. Source changes must remain confined to disposable fixtures during tests. +Client version metadata/audit history and sandbox replacement remain later work. diff --git a/.spiral/cycles/CYC-20260923-TTZ7C-44.ttl b/.spiral/cycles/CYC-20260923-TTZ7C-44.ttl new file mode 100644 index 0000000..2dc39ae --- /dev/null +++ b/.spiral/cycles/CYC-20260923-TTZ7C-44.ttl @@ -0,0 +1,9 @@ +@prefix sd: . +@prefix dcterms: . +@prefix project: . + +project:CYC-20260923-TTZ7C-44 + a sd:Cycle ; + dcterms:identifier "CYC-20260923-TTZ7C-44" ; + sd:repositoryPath ".spiral/cycles/CYC-20260923-TTZ7C-44.md" ; + sd:status sd:Active . diff --git a/.spiral/sources/SRC-20260923-TTZ7C-45.md b/.spiral/sources/SRC-20260923-TTZ7C-45.md new file mode 100644 index 0000000..d73e7dd --- /dev/null +++ b/.spiral/sources/SRC-20260923-TTZ7C-45.md @@ -0,0 +1,24 @@ +--- +id: SRC-20260923-TTZ7C-45 +--- + +# File-backed migration authorization + +The maintainer first requested file-backed data before client-visible version +metadata and audit trails, and agreed to harden od-jsontag first. After reviewing +the dependency changes and the explanation of required SimplyStore integration, +the maintainer stated on 2026-09-23: + +> Read through it, and the code is readable and well supported by tests. Lets continue with simplystore and switch to filebased data instead of shared memory. + +The preceding explanation covered file sources and offset indexes, descriptor +lifetime, ID lookup independent of the proxy cache, index loaders, integrity, +command isolation, and streamed large output. This is explicit authorization to +implement that integration. It continues the roadmap file-only direction, with +priority ahead of rebuild as previously requested. It does not authorize a merge, +release, production-data mutation, new sandbox, or audit feature. + +Reviewed dependency: od-jsontag commit +24f47eb616c3a24391487197496804ed1c56837b, available on its upstream indexedParsing +branch. The library keeps format/API compatibility and has bounded clean-record +caching; indexes and mutable sessions still grow with their contents. diff --git a/.spiral/sources/SRC-20260923-TTZ7C-45.ttl b/.spiral/sources/SRC-20260923-TTZ7C-45.ttl new file mode 100644 index 0000000..0f37b63 --- /dev/null +++ b/.spiral/sources/SRC-20260923-TTZ7C-45.ttl @@ -0,0 +1,11 @@ +@prefix sd: . +@prefix dcterms: . +@prefix project: . + +project:SRC-20260923-TTZ7C-45 + a sd:Source ; + dcterms:identifier "SRC-20260923-TTZ7C-45" ; + sd:repositoryPath ".spiral/sources/SRC-20260923-TTZ7C-45.md" ; + sd:status sd:Active ; + sd:sourceAvailability sd:Retained ; + sd:provenanceConfidence sd:Explicit . From c276b42cbe89a3c7204dd26f1c66522e59553b90 Mon Sep 17 00:00:00 2001 From: Auke van Slooten Date: Wed, 23 Sep 2026 16:52:23 +0200 Subject: [PATCH 02/15] Define confirmed file-backed runtime understanding --- .../understandings/UND-20260923-TTZ7C-48.md | 22 +++++++++++++++++++ .../understandings/UND-20260923-TTZ7C-48.ttl | 10 +++++++++ 2 files changed, 32 insertions(+) create mode 100644 .spiral/understandings/UND-20260923-TTZ7C-48.md create mode 100644 .spiral/understandings/UND-20260923-TTZ7C-48.ttl diff --git a/.spiral/understandings/UND-20260923-TTZ7C-48.md b/.spiral/understandings/UND-20260923-TTZ7C-48.md new file mode 100644 index 0000000..e563ef3 --- /dev/null +++ b/.spiral/understandings/UND-20260923-TTZ7C-48.md @@ -0,0 +1,22 @@ +--- +id: UND-20260923-TTZ7C-48 +--- + +# File-backed runtime boundary + +Confirmed scope: make files the sole dataset source for live queries, commands, +startup assessment and administrative replay. SharedArrayBuffer is currently +used for the loaded base and all committed changesets. Worker ID lookup assumes +all records have cache entries. Inspection also retains full file buffers. +Those observed paths prevent the requested outcome despite file support in the +new dependency. The current 118-test suite passed with the hardened dependency. + +Keep the accepted format, remote API and durability semantics. Immutable base +and changeset files remain available while a store owns its dataset; each worker +owns its descriptors. Index metadata may scale with record count; command edits +and response/output buffers may scale with a command or query. Bounded clean +query caching is the dependency contract, not constant total memory. Full +startup integrity scanning remains necessary, but can use bounded byte buffers. + +The human's instruction follows the concrete integration explanation and is the +commitment boundary; no second permission ceremony is needed. diff --git a/.spiral/understandings/UND-20260923-TTZ7C-48.ttl b/.spiral/understandings/UND-20260923-TTZ7C-48.ttl new file mode 100644 index 0000000..d17b831 --- /dev/null +++ b/.spiral/understandings/UND-20260923-TTZ7C-48.ttl @@ -0,0 +1,10 @@ +@prefix sd: . +@prefix dcterms: . +@prefix project: . + +project:UND-20260923-TTZ7C-48 + a sd:Understanding ; + dcterms:identifier "UND-20260923-TTZ7C-48" ; + sd:repositoryPath ".spiral/understandings/UND-20260923-TTZ7C-48.md" ; + sd:status sd:Accepted ; + sd:interprets [ a sd:ArtifactReference ; sd:artifact project:SRC-20260923-TTZ7C-45 ; sd:gitCommit "8299fcc8b8fcf113091ae85bbcf11f867f8c5908" ] . From a75db831ee6779bbf297e94b9d1baf694692cb84 Mon Sep 17 00:00:00 2001 From: Auke van Slooten Date: Wed, 23 Sep 2026 16:52:38 +0200 Subject: [PATCH 03/15] Specify file-backed migration acceptance contract --- .spiral/requests/REQ-20260923-TTZ7C-46.md | 22 ++++++++++++++++++++++ .spiral/requests/REQ-20260923-TTZ7C-46.ttl | 10 ++++++++++ 2 files changed, 32 insertions(+) create mode 100644 .spiral/requests/REQ-20260923-TTZ7C-46.md create mode 100644 .spiral/requests/REQ-20260923-TTZ7C-46.ttl diff --git a/.spiral/requests/REQ-20260923-TTZ7C-46.md b/.spiral/requests/REQ-20260923-TTZ7C-46.md new file mode 100644 index 0000000..c60c9f8 --- /dev/null +++ b/.spiral/requests/REQ-20260923-TTZ7C-46.md @@ -0,0 +1,22 @@ +--- +id: REQ-20260923-TTZ7C-46 +--- + +# File-backed operation with durable compatibility + +Replace retained/transferred dataset byte buffers with ordered immutable file +sources. Queries lazily read indexed records, including ID lookups before any +traversal. Commands use the exact committed prefix and publish new sources only +after durable completion. Reinitialization, failure, timeout and shutdown release +owned handles. Administrative replay uses the same source contract. + +Preserve remote query/command behavior, disk framing and log-authoritative +ordering, integrity validation, custom index finalization and recovery rules. +Existing stores without usable sidecars remain readable via derived indexes; +never trust stale offsets over canonical data. Validation must not load whole +data files merely to hash them. No production data rewrites or format migration. + +Evidence: existing regression suite plus direct file lifecycle, partial-ID-read, +sparse update, worker publication/replacement, malformed framing/integrity and +large-file memory tests. Internal worker messages may change. New dependency +revision must be reproducible. No sandbox/audit/rebuild feature in this cycle. diff --git a/.spiral/requests/REQ-20260923-TTZ7C-46.ttl b/.spiral/requests/REQ-20260923-TTZ7C-46.ttl new file mode 100644 index 0000000..39c021d --- /dev/null +++ b/.spiral/requests/REQ-20260923-TTZ7C-46.ttl @@ -0,0 +1,10 @@ +@prefix sd: . +@prefix dcterms: . +@prefix project: . + +project:REQ-20260923-TTZ7C-46 + a sd:Request ; + dcterms:identifier "REQ-20260923-TTZ7C-46" ; + sd:repositoryPath ".spiral/requests/REQ-20260923-TTZ7C-46.md" ; + sd:status sd:Active ; + sd:derivedFrom [ a sd:ArtifactReference ; sd:artifact project:UND-20260923-TTZ7C-48 ; sd:gitCommit "c276b42cbe89a3c7204dd26f1c66522e59553b90" ] . From 9b8acd229291fa75423c45a9019245edc0c19ea9 Mon Sep 17 00:00:00 2001 From: Auke van Slooten Date: Wed, 23 Sep 2026 16:52:53 +0200 Subject: [PATCH 04/15] Design owned file sessions and ordered worker publication --- .spiral/designs/DES-20260923-TTZ7C-47.md | 37 +++++++++++++++++++++++ .spiral/designs/DES-20260923-TTZ7C-47.ttl | 10 ++++++ 2 files changed, 47 insertions(+) create mode 100644 .spiral/designs/DES-20260923-TTZ7C-47.md create mode 100644 .spiral/designs/DES-20260923-TTZ7C-47.ttl diff --git a/.spiral/designs/DES-20260923-TTZ7C-47.md b/.spiral/designs/DES-20260923-TTZ7C-47.md new file mode 100644 index 0000000..2096fbf --- /dev/null +++ b/.spiral/designs/DES-20260923-TTZ7C-47.md @@ -0,0 +1,37 @@ +--- +id: DES-20260923-TTZ7C-47 +--- + +# Ordered file sources and owned reader sessions + +Transport ordered source descriptors (path, byte offsets, size and digest), not +open descriptors or dataset bytes. A small file-reader session owns handles and +an od-jsontag parser; queries reuse it, commands open the committed prefix for +one execution, and all exits close handles. Resolve IDs through getLineProxy. + +Scan canonical OD-JSONTag framing incrementally while hashing to derive exact +sparse offsets; these derived indexes avoid requiring trusted/up-to-date +sidecars. Existing persisted sidecars remain produced and their loaders are +corrected. Reconstruct IDs from final live records with bounded clean caching, +without invoking user index hooks during inspection. Source maps and IDs remain +O(record count). Verify sources against integrity manifests before publication. + +Startup inspection returns plain file sources instead of retained data/buffers. +Administrative recovery uses the same loader/session. Temporary command output +may remain buffered for the existing custom finalize(serialized, meta, uuid) +hook; it is written durably and returned as a file descriptor object. This +preserves the hook and avoids holding the whole dataset in memory. Streaming +full-dataset serialization is unnecessary once startup stops compacting it. + +Worker updates must serialize with queries and replacement workers must receive +the current committed source list. Await termination before releasing store +ownership. OS/Node worker teardown closes descriptors on timeout/failure. + +Alternative considered: keep a buffered fallback or merely replace query reads. +That would leave startup/recovery retaining entire datasets and duplicate modes; +a shared file-session boundary gives one implementation and test surface. + +Pin the reviewed od-jsontag Git commit until a separately authorized release. +No dependency publish, remote deployment or user data mutation is part of this +implementation. Validate behavior with independent byte-offset oracles and the +existing durability suite. diff --git a/.spiral/designs/DES-20260923-TTZ7C-47.ttl b/.spiral/designs/DES-20260923-TTZ7C-47.ttl new file mode 100644 index 0000000..6a21ba6 --- /dev/null +++ b/.spiral/designs/DES-20260923-TTZ7C-47.ttl @@ -0,0 +1,10 @@ +@prefix sd: . +@prefix dcterms: . +@prefix project: . + +project:DES-20260923-TTZ7C-47 + a sd:Design ; + dcterms:identifier "DES-20260923-TTZ7C-47" ; + sd:repositoryPath ".spiral/designs/DES-20260923-TTZ7C-47.md" ; + sd:status sd:Active ; + sd:satisfies [ a sd:ArtifactReference ; sd:artifact project:REQ-20260923-TTZ7C-46 ; sd:gitCommit "a75db831ee6779bbf297e94b9d1baf694692cb84" ] . From bfeed2c5c45cd3ac58dd42aac868585f31c59d19 Mon Sep 17 00:00:00 2001 From: Auke van Slooten Date: Wed, 23 Sep 2026 17:10:00 +0200 Subject: [PATCH 05/15] Read SimplyStore datasets from owned file sources --- .../implementations/IMP-20260919-TTZ7C-26.md | 15 + .../implementations/IMP-20260919-TTZ7C-26.ttl | 7 +- .../implementations/IMP-20260919-TTZ7C-33.md | 26 ++ .../implementations/IMP-20260919-TTZ7C-33.ttl | 23 +- .../implementations/IMP-20260919-TTZ7C-34.md | 9 + .../implementations/IMP-20260919-TTZ7C-34.ttl | 10 +- .spiral/project-context.md | 16 +- .spiral/project.ttl | 7 +- DURABILITY.md | 10 + README.md | 13 +- docs/file-data.md | 110 +++++++ package.json | 5 +- scripts/benchmark-file-data.mjs | 80 +++++ src/admin-recovery.mjs | 49 ++- src/command-worker-module.mjs | 92 +++--- src/command-worker.mjs | 5 +- src/file-data.mjs | 262 +++++++++++++++ src/index.id.mjs | 6 +- src/index.mjs | 4 +- src/index.offset.mjs | 6 +- src/integrity.mjs | 29 +- src/load-worker.mjs | 69 +--- src/query-worker-module.mjs | 73 +++-- src/store-inspection.mjs | 61 ++-- src/store-runtime.mjs | 21 +- src/workerPool.mjs | 242 ++++++++------ test/admin-recovery.mjs | 19 +- test/command-offsets.mjs | 25 +- test/durability-startup.mjs | 14 +- test/file-data.mjs | 297 ++++++++++++++++++ test/power-loss-baseline.mjs | 6 +- test/store-runtime.mjs | 6 +- 32 files changed, 1257 insertions(+), 360 deletions(-) create mode 100644 docs/file-data.md create mode 100644 scripts/benchmark-file-data.mjs create mode 100644 src/file-data.mjs create mode 100644 test/file-data.mjs diff --git a/.spiral/implementations/IMP-20260919-TTZ7C-26.md b/.spiral/implementations/IMP-20260919-TTZ7C-26.md index 025b556..09cfc1b 100644 --- a/.spiral/implementations/IMP-20260919-TTZ7C-26.md +++ b/.spiral/implementations/IMP-20260919-TTZ7C-26.md @@ -119,3 +119,18 @@ retain ownership for assessment; successful initialization releases it. Regression coverage converts the actual bundled dataset, verifies all final record offsets and reads typed URL values, including a linked homeworld. No canonical format, storage barrier or existing-store overwrite policy changes. + +## Cycle 44: File-backed durability integration + +Command workers now open immutable file sources in a fresh mutable session, +serialize only their changes into an ordinary Buffer for existing index hooks, +and return a scanned source descriptor. Persist/write/finalize/integrity ordering +is preserved. A changed canonical file after finalization is refused. Default +index loaders now use the supplied metadata and correct UUID filename. + +Inspection hashes and scans files using bounded byte buffers and returns plain +source descriptors. Replay opens those sources instead of compacting buffers; +retained expected digests are checked against the resulting file bytes before +replacement integrity and completion records. Existing administrative copy +operations still buffer an individual file temporarily; this is outside the +retrieval/replay memory claim. diff --git a/.spiral/implementations/IMP-20260919-TTZ7C-26.ttl b/.spiral/implementations/IMP-20260919-TTZ7C-26.ttl index 718ba4f..9a275cf 100644 --- a/.spiral/implementations/IMP-20260919-TTZ7C-26.ttl +++ b/.spiral/implementations/IMP-20260919-TTZ7C-26.ttl @@ -33,6 +33,7 @@ project:IMP-20260919-TTZ7C-26 a sd:Implementation ; sd:implements [ a sd:ArtifactReference ; sd:artifact project:DES-003 ; sd:gitCommit "59179b6c93b7e21c3b2c69db46c53fbe706927c3" ] ; sd:implements [ a sd:ArtifactReference ; sd:artifact project:DES-20260917-TTZ7C-14 ; sd:gitCommit "59179b6c93b7e21c3b2c69db46c53fbe706927c3" ] ; sd:implements [ a sd:ArtifactReference ; sd:artifact project:DES-20260819-09ZEF-7 ; sd:gitCommit "59179b6c93b7e21c3b2c69db46c53fbe706927c3" ] ; - sd:changeCausedBy [ a sd:ArtifactReference ; sd:artifact project:SRC-20260919-TTZ7C-18 ; sd:gitCommit "9b4a340facd57f22edbead07a195700b0237c9ad" ] ; - sd:transforms [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-26 ; sd:gitCommit "9b4a340facd57f22edbead07a195700b0237c9ad" ] ; - sd:implementationChangeKind sd:SemanticChange . + sd:implements [ a sd:ArtifactReference ; sd:artifact project:DES-20260923-TTZ7C-47 ; sd:gitCommit "9b8acd229291fa75423c45a9019245edc0c19ea9" ] ; + sd:changeCausedBy [ a sd:ArtifactReference ; sd:artifact project:DES-20260923-TTZ7C-47 ; sd:gitCommit "9b8acd229291fa75423c45a9019245edc0c19ea9" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-26 ; sd:gitCommit "d4136eaf7b0014726a1c90a0d0b8488ece2d6b27" ] ; + sd:implementationChangeKind sd:MixedChange . diff --git a/.spiral/implementations/IMP-20260919-TTZ7C-33.md b/.spiral/implementations/IMP-20260919-TTZ7C-33.md index 704e8e2..388b626 100644 --- a/.spiral/implementations/IMP-20260919-TTZ7C-33.md +++ b/.spiral/implementations/IMP-20260919-TTZ7C-33.md @@ -101,3 +101,29 @@ and shares cleanup failure; both persistence-boundary failures reject acceptance known ID-only retries return status while incomplete new commands fail; and a real process receiving both shutdown signals commits, unlocks and restarts. EVD-20260919-TTZ7C-41 preserves the baseline and records final suite results. + +## Cycle 44: File-backed runtime and reader lifetime + +DES-20260923-TTZ7C-47 changes the internal runtime representation to ordered file +sources, with file paths, sparse offsets, immutable file identity and SHA-256. +FileDataset owns parser and descriptors; scanDataFile validates framing and +hashes with a 64 KiB buffer. Startup builds IDs from final records without +requiring trustworthy sidecars. Runtime/query/command messages contain sources, +not dataset bytes. Query and command ID lookups use the complete record catalog. + +The worker pool queues updates ahead of new queries, independently for each +worker, and initializes replacements with the current source list. Failed source +updates reject further queries and stop command acceptance. Shutdown awaits +termination (including workers already being replaced) before ownership release. + +A QueryParser specialization gives raw query arrays a non-configurable undefined +constructor before proxy wrapping. Local vm2 3.12.2 recognizes this as its +already-neutralized array-species case. Tests cover map, JAQT, and refused array +mutation/definition. This preserves the query API without weakening immutable +traps or changing the sandbox implementation. It is not a sandbox safety claim. + +Existing tests were adapted only where internal messages/report fields changed; +remote results, recovery eligibility, ordering and failure expectations remain. +New tests cover unread IDs, sparse overlays, corrupt framing, short reads/EOF, +file changes and closure, byte-free messages, index loaders, real runtime restart, +failed publication, worker replacement, and asynchronous update ordering. diff --git a/.spiral/implementations/IMP-20260919-TTZ7C-33.ttl b/.spiral/implementations/IMP-20260919-TTZ7C-33.ttl index bee07a8..2919a54 100644 --- a/.spiral/implementations/IMP-20260919-TTZ7C-33.ttl +++ b/.spiral/implementations/IMP-20260919-TTZ7C-33.ttl @@ -24,18 +24,13 @@ project:IMP-20260919-TTZ7C-33 sd:artifact project:DES-20260919-TTZ7C-40 ; sd:gitCommit "665a97d1c366b4cce2b9004d6e649a389d18a2c4" ] ; - sd:changeCausedBy [ - a sd:ArtifactReference ; - sd:artifact project:EVD-20260919-TTZ7C-41 ; - sd:gitCommit "959e2cd92181be07f3fe38d38328f10ba41923b3" - ], [ - a sd:ArtifactReference ; - sd:artifact project:DES-20260919-TTZ7C-40 ; - sd:gitCommit "665a97d1c366b4cce2b9004d6e649a389d18a2c4" - ] ; - sd:transforms [ - a sd:ArtifactReference ; - sd:artifact project:IMP-20260919-TTZ7C-33 ; - sd:gitCommit "959e2cd92181be07f3fe38d38328f10ba41923b3" - ] ; + sd:implementationLocation "src/file-data.mjs" ; + sd:implementationLocation "src/load-worker.mjs" ; + sd:implementationLocation "src/query-worker-module.mjs" ; + sd:implementationLocation "src/workerPool.mjs" ; + sd:implementationLocation "test/file-data.mjs" ; + sd:implementationLocation "scripts/benchmark-file-data.mjs" ; + sd:implements [ a sd:ArtifactReference ; sd:artifact project:DES-20260923-TTZ7C-47 ; sd:gitCommit "9b8acd229291fa75423c45a9019245edc0c19ea9" ] ; + sd:changeCausedBy [ a sd:ArtifactReference ; sd:artifact project:DES-20260923-TTZ7C-47 ; sd:gitCommit "9b8acd229291fa75423c45a9019245edc0c19ea9" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-33 ; sd:gitCommit "5d9e81218d5a51d8d3171f7132e73c2e4bfc298a" ] ; sd:implementationChangeKind sd:MixedChange . diff --git a/.spiral/implementations/IMP-20260919-TTZ7C-34.md b/.spiral/implementations/IMP-20260919-TTZ7C-34.md index a93abf0..7fb571c 100644 --- a/.spiral/implementations/IMP-20260919-TTZ7C-34.md +++ b/.spiral/implementations/IMP-20260919-TTZ7C-34.md @@ -32,3 +32,12 @@ A clean temporary repository copy installed both packages, built the example, started it, served a JSON query, accepted an example UUID command and reported that command `done`. Existing generated files in the working example were not modified. + +## Cycle 44: Reviewed file parser dependency + +Pin od-jsontag to reviewed upstream Git commit +24f47eb616c3a24391487197496804ed1c56837b. The published 0.4.6 version lacks the +file-backed API; using a range on that unchanged version would not reproduce +this implementation. No other dependency is upgraded, no npm version is bumped, +and no package is published. The default test command includes file-data tests; +packaged documentation includes the internal contract and measured boundaries. diff --git a/.spiral/implementations/IMP-20260919-TTZ7C-34.ttl b/.spiral/implementations/IMP-20260919-TTZ7C-34.ttl index 7d096b5..edd4627 100644 --- a/.spiral/implementations/IMP-20260919-TTZ7C-34.ttl +++ b/.spiral/implementations/IMP-20260919-TTZ7C-34.ttl @@ -17,9 +17,7 @@ project:IMP-20260919-TTZ7C-34 sd:artifact project:DES-20260919-TTZ7C-32 ; sd:gitCommit "dd0382ef49a50cada7a3699cb021df0929d02315" ] ; - sd:transforms [ - a sd:ArtifactReference ; - sd:artifact project:IMP-20260919-TTZ7C-34 ; - sd:gitCommit "fe516f08163642939e3c7478d5eb7b851de273ab" - ] ; - sd:implementationChangeKind sd:BehaviorPreservingChange . + sd:implements [ a sd:ArtifactReference ; sd:artifact project:DES-20260923-TTZ7C-47 ; sd:gitCommit "9b8acd229291fa75423c45a9019245edc0c19ea9" ] ; + sd:changeCausedBy [ a sd:ArtifactReference ; sd:artifact project:DES-20260923-TTZ7C-47 ; sd:gitCommit "9b8acd229291fa75423c45a9019245edc0c19ea9" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-34 ; sd:gitCommit "c4571b4389835302b47b4096d6f0c3cb77b62f1f" ] ; + sd:implementationChangeKind sd:MixedChange . diff --git a/.spiral/project-context.md b/.spiral/project-context.md index 1532c75..07dab14 100644 --- a/.spiral/project-context.md +++ b/.spiral/project-context.md @@ -18,12 +18,18 @@ Project causal-graph namespace: `https://github.com/simplyedit/simplystore/spira Spiral core source: `.spiral-core/`, git submodule for `https://github.com/muze-labs/spiral-developer.git`, currently checked out at `8d4b2c738a413abd4cccca740ce958f486e5f7af`. +Active cycle: CYC-20260923-TTZ7C-44, File-backed dataset runtime, on +spiral/CYC-20260923-TTZ7C-44-file-backed. The maintainer authorized replacing +shared-memory data after reviewing od-jsontag hardening. This continues the +file-only roadmap direction, prioritized ahead of rebuild. SRC-45, UND-48, +REQ-46 and DES-47 (20260923-TTZ7C namespace) retain the commitment. Implementation +uses ordered file sources, bounded clean-record caches, worker-owned handles, +streamed framing/hash scans and file-backed administrative replay. Evaluation +and human acceptance remain pending. Client-visible version metadata and audit +trails follow this migration; no such feature is implemented by this cycle. + Latest accepted cycle: CYC-20260919-TTZ7C-42, Updated Roadmap And Planning -Context, on spiral/CYC-20260919-TTZ7C-42-roadmap. The maintainer accepted the -evaluated branch tip 10a39df3e946944a168614fd59514f2aeb4aec05 on 2026-09-23. -The documentation cycle is closed; integration into master remains a separate -decision. No next product task is selected; Spiral risk analysis selects it -using ROADMAP.md and current evidence. +Context. Accepted on 2026-09-23 and integrated through PR #24 at dbb0703. Previously accepted and integrated cycle: CYC-20260919-TTZ7C-36, Reliable Runtime Contracts And Readable Workflows, on spiral/CYC-20260919-TTZ7C-36-runtime-contracts. The maintainer diff --git a/.spiral/project.ttl b/.spiral/project.ttl index 25585ee..24c3ab5 100644 --- a/.spiral/project.ttl +++ b/.spiral/project.ttl @@ -15,11 +15,8 @@ project:CTX-001 sd:artifact project:SRC-20260919-TTZ7C-43 ; sd:gitCommit "79fcf670d4558c75b9933743e6f2ea5616221eb0" ] ; - sd:transforms [ - a sd:ArtifactReference ; - sd:artifact project:CTX-001 ; - sd:gitCommit "b482053bdf0d3b3fc8902cce3169408d31ed9f6e" - ] ; + sd:derivedFrom [ a sd:ArtifactReference ; sd:artifact project:SRC-20260923-TTZ7C-45 ; sd:gitCommit "8299fcc8b8fcf113091ae85bbcf11f867f8c5908" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:CTX-001 ; sd:gitCommit "e446a315e4c86709f9d1eae98453a058491a9309" ] ; sd:developerEvaluationReadinessBar "Developers should be able to evaluate bounded durability claims from invariants, executable tests, evidence artifacts, explicit failure behavior, and documented known gaps." ; sd:hasAudience project:AUD-DEVELOPERS, project:AUD-END-USERS, project:AUD-OPERATORS ; sd:hasDownstreamProject project:DOWNSTREAM-CURRICULUM-STORE ; diff --git a/DURABILITY.md b/DURABILITY.md index cd8366e..223cd04 100644 --- a/DURABILITY.md +++ b/DURABILITY.md @@ -45,3 +45,13 @@ Custom hooks must await their work; declare additional required artifacts with their own conformance checks. Unsupported directory-sync operations fail rather than silently weakening the guarantee. Broader filesystems, random/soak testing, and deployment-specific power-loss testing remain future validation work. + +## File-backed readers + +The runtime retains ordered file sources instead of shared dataset bytes. Each +worker owns read handles; committed sources remain immutable while the store is +open. Source publication still follows the durable `done` record. Inspection +hashes/scans data with a fixed-size byte buffer, and shutdown waits for readers +to terminate before releasing ownership. See [file-backed data](docs/file-data.md) +for the source contract and memory boundaries. The existing filesystem durability +envelope and recovery restrictions remain in force. diff --git a/README.md b/README.md index 1210d77..59dd6b8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # SimplyStore -SimplyStore is a radically simpler backend storage server. It does not have a database, certainly no SQL or GraphQL, it is not REST. In return it has a well defined API that is automatically derived from your dataset. It supports JSONTag to allow for semantically meaningful data, without having to do the full switch to Linked Data and triple stores. The query format is javascript, you can post javascript queries that will run on the server. All data is read into memory and is available to these javascript queries without needing (or allowing) disk access or indexes. +SimplyStore is a radically simpler backend storage server. It does not have a database, certainly no SQL or GraphQL, it is not REST. In return it has a well defined API that is automatically derived from your dataset. It supports JSONTag to allow for semantically meaningful data, without having to do the full switch to Linked Data and triple stores. The query format is javascript, you can post javascript queries that will run on the server. Dataset records are read lazily from indexed files. Javascript queries use ordinary objects and arrays; SimplyStore manages file access and indexes. [JSONTag](https://github.com/muze-nl/jsontag) is an enhancement over JSON that allows you to tag JSON data with metadata using HTML-like tags. Javascript queries are run in a [VM2](https://www.npmjs.com/package/vm2) sandbox. @@ -13,6 +13,7 @@ Note: _There are known security issues in VM2, so the project will switch to V8- - [Background](#background) - [Install](#install) - [Usage](#usage) +- [File-backed Data](docs/file-data.md) - [Custom Index Modules](#custom-index-modules) - [Example Query](#examples) - [Goals](#goals) @@ -67,6 +68,12 @@ You should be able to go http://localhost:3000/query/ and see something like thi See the [durability contract](DURABILITY.md) and [administrator recovery guide](docs/recovery.md). Command-log order governs execution. Acceptance and completion await file and directory barriers. Uncertain or pending work found at startup requires administrator assessment; missing data does not prove that external effects did not happen. Existing store formats are retained without migration. +## File-backed data + +See the [file-backed data guide](docs/file-data.md) for worker/file lifetimes, +index rebuilding, memory limits, and the internal worker-message changes. +Existing stores retain their current formats and need no conversion. + ## Custom Index Modules Use `indexFile` to configure a module whose default export provides the existing @@ -177,7 +184,7 @@ SimplyStore is a more defined and usable REST like service, out of the box. One The SimplyStore design is predicated on the following realisations: - 1. Most data today will fit comfortably in memory in a commodity server. + 1. Files provide lazy record access; indexes and active edits still use memory. 2. REST today is usually JSON-over-HTTP, but JSON crucially misses a type. 3. JSON is never just JSON. You need additional things like JSON-LD or JSON-Schema, to make sense of it. 4. There is no clear onramp from JSON to Linked Data. @@ -185,7 +192,7 @@ The SimplyStore design is predicated on the following realisations: So the scope for SimplyStore is: -- datasets that will fit comfortably in memory, for now I've set a test goal of about 1GB of data. +- datasets whose indexes and active working set fit in memory, with file-backed record storage. - usecases that are mostly-read, with sparse updates. - scale-in-depth, so scale up is limited to the limits of a single computer system - linked data (RDF et al) is not an immediate concern, but there must be a plausible onramp / conversion to and from linked data. diff --git a/docs/file-data.md b/docs/file-data.md new file mode 100644 index 0000000..68b9695 --- /dev/null +++ b/docs/file-data.md @@ -0,0 +1,110 @@ +# File-backed datasets + +SimplyStore reads its base and committed changeset files through od-jsontag. +The runtime and its workers exchange file paths and index metadata; they no +longer retain or transfer the dataset as SharedArrayBuffers. Remote query and +command APIs and the existing OD-JSONTag/log formats are unchanged. + +## Opening and reading a store + +Startup inspects command-log order and the committed prefix, as before. A +64 KiB byte buffer scans each canonical data file's framing and SHA-256 digest. +That scan derives exact record offsets, including sparse changesets and UTF-8 +payloads. Stored offset/ID sidecars remain produced by the index hooks, but +startup derives its authoritative indexes from data files, so absent or stale +sidecars cannot redirect reads. Inspection does not import custom index modules. +The loader reconstructs the ID map from the final live records. + +Each query worker opens its own read-only descriptors and registers the ordered +base/changeset sources with its parser. ID lookups use the complete record +catalog, including objects not yet read. The parser retains at most 256 clean +record bodies by default; evicted bodies are read again when needed. + +Canonical source files must remain immutable while the store is open. Opening a +source checks its device, inode, size and modification/change timestamps against +the scanned identity. Store ownership and immutable publication are still the +operational boundary; this is not protection against another process editing an +already-open file in place. Use the existing stopped-store recovery workflow for +administrative changes. + +## Commands and publication + +Each command worker opens the exact committed source list in a fresh mutable +session. Touched objects and uncommitted edits remain available until that +session ends. Custom `update` and `finalize` hooks retain their existing calls. +The changeset is temporarily serialized to an ordinary Buffer for finalization, +then published using the existing complete-write, file-sync, rename and +directory-sync sequence. Finalization must leave canonical bytes unchanged. + +The worker returns the new file's descriptor and metadata. The runtime publishes +that source to query workers only after the required files and `done` status +have been synchronized. Empty/no-op changesets are valid sources. Query pools +finish pending updates before dispatching another query. Replacement workers +initialize at the current committed head. + +Workers own their file handles. Session reinitialization and failure close them; +worker termination also closes descriptors opened through Node's tracked file +APIs. Runtime shutdown waits for both pools to terminate before releasing store +ownership. A failed source update makes the pool unavailable and stops further +command acceptance rather than allowing the worker to skip an update. + +Administrative replay opens file sources too. `inspectStore()` now returns a +plain `sources` array instead of live `data` proxies and retained `buffers`. +Callers that need to read an inspected snapshot can use a `FileDataset` inside a +`try`/`finally` and call `close()` when finished. Proxies are valid only while +their owning session remains open. + +## Internal integration changes + +Custom worker implementations need to adopt these messages: + +| Boundary | Input/output | +| --- | --- | +| Loader result | `{ sources, meta }` | +| Query initialization | `req.sources`, `req.meta`, `req.access` | +| Query update | `req.source`, `req.meta` | +| Command initialization | `sources`, `meta`, existing command/configuration fields | +| Successful command result | `source`, `meta`, existing result/status fields | + +A source contains `file`, `offsets`, `identity`, `size` and `digest`; it contains +neither bytes nor an open descriptor. It is an internal trusted worker contract. +Use `scanDataFile()` to construct one and `FileDataset` to own the parser and +handles. Existing custom index modules need no changes. Default index loaders +now correctly accept `load(meta, uuid)`. + +The query parser prepares read-only arrays with an immutable undefined +`constructor` before wrapping them. This matches vm2's existing safe array-species +case, allowing `map`, `filter` and JAQT operations without permitting temporary +writes through the dataset proxy. This compatibility adapter does not replace +or establish a new security assessment of the query sandbox. + +## Memory and startup costs + +File-backed operation does not imply constant total memory. Record offsets, +source catalogs and ID maps scale with the dataset. Each worker keeps one open +handle per source file. Mutable sessions retain touched records; large individual +records, query results, command output and custom hooks can require substantial +memory. Command/status logs and schemas are still read into memory. Backup/copy +utilities still temporarily buffer individual files when copying them; this +cycle changes dataset retrieval and replay, not their copy algorithm. + +Startup still reads all canonical bytes for validation and decodes live records +to reconstruct IDs. It trades startup work for lazy subsequent reads. A generated +328,877,813-byte / 20,000-record store opened and answered an ID lookup with a +48 MB JavaScript heap limit. One local run measured approximately 17.5 seconds +open and 93 ms query (including worker readiness), with 8.9 MB parent heap and +24.4 MB query-worker heap. Retained byte buffers were approximately 25 KiB and +58 KiB respectively. These are warm local measurements, not a cold-storage, +physical-RAM or total-process-memory bound. + +Reproduce the disposable-fixture probe with: + +```sh +node --expose-gc --max-old-space-size=48 scripts/benchmark-file-data.mjs +``` + +The dependency is pinned to reviewed od-jsontag Git commit +`24f47eb616c3a24391487197496804ed1c56837b`, because its file-backed API has not +received a new npm version. Installation therefore requires Git access to its +public repository. Replacing that pin with a published release is separate +release work. diff --git a/package.json b/package.json index 3bf1641..205d2cc 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "type": "module", "scripts": { "start": "node src/run.mjs", - "test": "node --test test/store-runtime.mjs test/durability-startup.mjs test/conversion-offsets.mjs test/command-offsets.mjs", + "test": "node --test test/file-data.mjs test/store-runtime.mjs test/durability-startup.mjs test/conversion-offsets.mjs test/command-offsets.mjs", "test:acid": "node --test test/acid-baseline.mjs", "test:durability": "node --test test/durability-crash.mjs", "test:handlers": "node --test test/after-change-handlers.mjs", @@ -25,7 +25,7 @@ "dependencies": { "@muze-nl/jaqt": "^0.10.6", "@muze-nl/jsontag": "^0.10.4", - "@muze-nl/od-jsontag": "^0.4.6", + "@muze-nl/od-jsontag": "git+https://github.com/muze-nl/od-jsontag.git#24f47eb616c3a24391487197496804ed1c56837b", "express": "^5.2.1", "vm2": "^3.12.2" }, @@ -38,6 +38,7 @@ "README.md", "DURABILITY.md", "docs/recovery.md", + "docs/file-data.md", "docs/power-loss-filesystem.md", "LICENSE", "src/", diff --git a/scripts/benchmark-file-data.mjs b/scripts/benchmark-file-data.mjs new file mode 100644 index 0000000..b5d7468 --- /dev/null +++ b/scripts/benchmark-file-data.mjs @@ -0,0 +1,80 @@ +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { performance } from 'node:perf_hooks' +import { fileURLToPath } from 'node:url' +import StoreRuntime from '../src/store-runtime.mjs' + +const args = Object.fromEntries(process.argv.slice(2).map(argument => { + return argument.replace(/^--/, '').split('=') +})) +const records = Number(args.records || 20000) +const payload = Number(args.payload || 16384) +if (!Number.isSafeInteger(records) || records < 1 || + !Number.isSafeInteger(payload) || payload < 1) { + throw new Error('records and payload must be positive integers') +} +const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'simplystore-file-bench-')) +const datafile = path.join(dir, 'data.jsontag') +let runtime +try { + const fd = fs.openSync(datafile, 'w') + try { + const writeRecord = text => { + const bytes = Buffer.from(`(${Buffer.byteLength(text)})${text}\n`) + let offset = 0 + while (offset < bytes.length) { + offset += fs.writeSync(fd, bytes, offset) + } + } + writeRecord(`{"items":[~1-${records}]}`) + const padding = 'x'.repeat(payload) + for (let number = 1; number <= records; number++) { + writeRecord(`${JSON.stringify({ + number, padding + })}`) + } + } + finally { + fs.closeSync(fd) + } + const commandLog = path.join(dir, 'command-log.jsontag') + const commandStatus = path.join(dir, 'command-status.jsontag') + fs.writeFileSync(commandLog, '') + fs.writeFileSync(commandStatus, '') + const started = performance.now() + runtime = await StoreRuntime.open({ + datafile, commandLog, commandStatus, maxWorkers: 1, + loadTimeout: 0, slowTimeout: 120000, + commandsFile: fileURLToPath(new URL('../src/commands.mjs', import.meta.url)) + }) + const openMs = performance.now() - started + const queried = performance.now() + const response = await runtime.runQuery({ + path: '/', jsontag: false, + body: `meta.index.id.get('item-${records}').number` + }, { slow: true }) + if (response.code || JSON.parse(response.body) !== records) { + throw new Error(`Query failed: ${JSON.stringify(response)}`) + } + const queryMs = performance.now() - queried + global.gc?.() + const memory = process.memoryUsage() + const workerMemory = await runtime.slowQueryWorkerPool.memoryUsage() + console.log(JSON.stringify({ + bytes: fs.statSync(datafile).size, records, openMs, queryMs, + sources: runtime.sources.length, + idCount: runtime.meta.index.id.size, + parent: { + heapUsed: memory.heapUsed, arrayBuffers: memory.arrayBuffers + }, + worker: { + heapUsed: workerMemory.heapUsed, + arrayBuffers: workerMemory.arrayBuffers + } + }, null, 2)) +} +finally { + await runtime?.close() + fs.rmSync(dir, { recursive: true, force: true }) +} diff --git a/src/admin-recovery.mjs b/src/admin-recovery.mjs index c889a41..c04dffd 100644 --- a/src/admin-recovery.mjs +++ b/src/admin-recovery.mjs @@ -4,8 +4,7 @@ import path from 'node:path' import { fileURLToPath } from 'node:url' import JSONTag from '@muze-nl/jsontag' import { from, _, anyOf, not } from '@muze-nl/jaqt' -import Parser from '@muze-nl/od-jsontag/src/parse.mjs' -import serialize from '@muze-nl/od-jsontag/src/serialize.mjs' +import { loadFileData, scanDataFile } from './file-data.mjs' import { inspectStore, inventory, @@ -23,7 +22,7 @@ import { } from './storage.mjs' import { executeWorker } from './execute-worker.mjs' import { nextActiveCommandStatus } from './recovery.mjs' -import { loadIntegrityManifest, verifyIntegrity } from './integrity.mjs' +import { loadIntegrityManifest, verifyDigest, appendIntegrityDigest } from './integrity.mjs' const defaultWorker = fileURLToPath( new URL('./command-worker.mjs', import.meta.url) @@ -238,7 +237,7 @@ class RecoveryApplication { this.journal = null this.planHash = null this.meta = null - this.buffers = null + this.sources = null this.expectedManifest = null } @@ -363,23 +362,13 @@ class RecoveryApplication { if (inspection.errors.length) { throw new Error(inspection.errors.join('; ')) } - const parser = new Parser() - let data - for (const bytes of inspection.buffers) { - data = parser.parse(bytes) - } - // Rebuild parser indexes without invoking custom hooks. - this.meta = { - index: { id: new Map() }, - data: path.dirname(this.config.datafile), - parts: inspection.committed.length - } - this.buffers = [serialize(data, { meta: this.meta })] - if (this.config.schemaFile) { - this.meta.schema = JSONTag.parse( - await fs.readFile(this.config.schemaFile, 'utf8') - ) - } + const loaded = await loadFileData({ + dataFile: this.config.datafile, + commands: inspection.committed, + schemaFile: this.config.schemaFile + }) + this.meta = loaded.meta + this.sources = loaded.sources const integrityFile = this.plan.config.integrityFile if (this.plan.files[integrityFile] != null) { this.expectedManifest = await loadIntegrityManifest(integrityFile) @@ -431,7 +420,7 @@ class RecoveryApplication { id, command: command.line, meta: this.meta, - data: this.buffers, + sources: this.sources, commandsFile: this.plan.code.commandsFile, indexFile: this.plan.code.indexFile, datafile: this.config.datafile, @@ -475,23 +464,24 @@ class RecoveryApplication { for (const file of this.config.requiredFiles) { await syncFile(file) } + const source = scanDataFile(result.source.file) if (this.expectedManifest && command.status === 'done') { - verifyIntegrity( + verifyDigest( this.expectedManifest, this.plan.config.integrityFile, command.file, - result.data, + source.digest, { required: true } ) } if (this.config.integrity || this.expectedManifest) { - await this.appendResultIntegrity(command, result.data) + await this.appendResultIntegrity(command, source.digest) } await appendRecord( this.config.commandStatus, JSONTag.stringify({ command: id, code: 200, status: 'done' }) ) - this.buffers.push(result.data) + this.sources.push(source) Object.assign(this.meta, result.meta) await appendRecord( this.journal, @@ -499,16 +489,15 @@ class RecoveryApplication { ) } - async appendResultIntegrity(command, data) { - const { appendIntegrityRecord } = await import('./integrity.mjs') + async appendResultIntegrity(command, digest) { const targetFile = path.join( path.dirname(this.config.datafile), path.basename(command.file) ) - await appendIntegrityRecord( + await appendIntegrityDigest( this.config.integrityFile, targetFile, - data + digest ) } diff --git a/src/command-worker-module.mjs b/src/command-worker-module.mjs index a602493..be1f806 100644 --- a/src/command-worker-module.mjs +++ b/src/command-worker-module.mjs @@ -1,27 +1,25 @@ import JSONTag from '@muze-nl/jsontag' import { getIndex, - resultSet, isChanged } from '@muze-nl/od-jsontag/src/symbols.mjs' -import Parser from '@muze-nl/od-jsontag/src/parse.mjs' -import serialize from '@muze-nl/od-jsontag/src/serialize.mjs' +import { FileDataset, scanDataFile } from './file-data.mjs' +import { serializeChunks } from '@muze-nl/od-jsontag/src/serialize.mjs' import { publishFile as writeFileAtomic, storageError } from './storage.mjs' import { faultPoint } from './faults.mjs' -import { appendIntegrityRecord } from './integrity.mjs' +import { appendIntegrityRecord, digestBuffer } from './integrity.mjs' import { finalizeIndex } from './index.mjs' let commands = {} let index = {} -let resultArr = [] +let dataset +let parser let dataspace let datafile, basefile, extension, integrityFile let meta = {} let metaProxy = { index: {} } -const parser = new Parser() -parser.immutable = false export const metaIdProxy = { forEach: callback => { @@ -29,7 +27,7 @@ export const metaIdProxy = { callback( { deref: () => { - return resultArr[ref] + return parser.getLineProxy(ref) } }, id @@ -47,7 +45,7 @@ export const metaIdProxy = { } else { let line = parser.meta.index.id.get(id) - resultArr[line] = ref + parser.meta.resultArray[line] = ref } }, get: id => { @@ -55,7 +53,7 @@ export const metaIdProxy = { if (index || index === 0) { return { deref: () => { - return resultArr[index] + return parser.getLineProxy(index) } } } @@ -73,29 +71,34 @@ const metaReadProxy = { } export async function initialize(task) { - if (task.meta) { - parser.meta = task.meta - } - for (let jsontag of task.data) { - dataspace = parser.parse(jsontag) + close() + try { + dataset = new FileDataset(task.meta, false) + parser = dataset.parser + dataspace = dataset.open(task.sources) + meta = parser.meta + metaProxy.index.id = metaIdProxy + if (meta.schema) { + metaProxy.schema = meta.schema + } + datafile = task.datafile + integrityFile = task.integrityFile + extension = datafile.split('.').pop() + // Include the dot before the extension. + basefile = datafile.substring( + 0, datafile.length - (extension.length + 1) + ) + commands = await import(task.commandsFile).then(mod => { + return mod.default + }) + index = await import(task.indexFile).then(mod => { + return mod.default + }) } - resultArr = dataspace[resultSet] - meta = task.meta - metaProxy.index.id = metaIdProxy - if (meta.schema) { - metaProxy.schema = meta.schema + catch (error) { + close() + throw error } - datafile = task.datafile - integrityFile = task.integrityFile - extension = datafile.split('.').pop() - // Include the dot before the extension. - basefile = datafile.substring(0, datafile.length - (extension.length + 1)) - commands = await import(task.commandsFile).then(mod => { - return mod.default - }) - index = await import(task.indexFile).then(mod => { - return mod.default - }) } export default async function runCommand(commandStr) { @@ -115,7 +118,7 @@ export default async function runCommand(commandStr) { let time = Date.now() await commands[task.name](dataspace, task, undefined, metaProxy) // TODO: if command/task makes no changes, skip updating - // data.jsontag and writing it; skip response.data. + // data.jsontag and writing it. const changes = meta.resultArray.filter(e => e[isChanged]) //FIXME: new entities should also report isChanged = true @@ -124,8 +127,9 @@ export default async function runCommand(commandStr) { await index.update(dataspace, meta, changes) } // Serialize only changes. - const uint8sab = serialize(dataspace, { meta, changes: true }) - response.data = uint8sab + const serialized = Buffer.concat([ + ...serializeChunks(dataspace, { meta, changes: true }) + ]) response.meta = { index: { id: meta.index.id @@ -137,17 +141,21 @@ export default async function runCommand(commandStr) { let newfilename = basefile + '.' + task.id + '.' + extension publishing = true await faultPoint('before-command-changeset-write') - await writeFileAtomic(newfilename, uint8sab) + await writeFileAtomic(newfilename, serialized) // Final bytes include new records and mutations made by the custom // index hook. - await finalizeIndex(index, uint8sab, meta, task.id) + await finalizeIndex(index, serialized, meta, task.id) if (integrityFile) { await appendIntegrityRecord( integrityFile, newfilename, - uint8sab + serialized ) } + response.source = scanDataFile(newfilename) + if (response.source.digest !== digestBuffer(serialized)) { + throw new Error('Changeset changed during finalization') + } await faultPoint('after-command-changeset-write') meta.parts++ response.meta.parts = meta.parts @@ -168,3 +176,13 @@ export default async function runCommand(commandStr) { } return response } + +export function close() { + if (dataset) { + dataset.close() + dataset = undefined + parser = undefined + dataspace = undefined + meta = {} + } +} diff --git a/src/command-worker.mjs b/src/command-worker.mjs index aa98509..fe43399 100644 --- a/src/command-worker.mjs +++ b/src/command-worker.mjs @@ -1,5 +1,5 @@ import { parentPort } from 'node:worker_threads' -import runCommand, { initialize } from '../src/command-worker-module.mjs' +import runCommand, { initialize, close } from '../src/command-worker-module.mjs' parentPort.on('message', async data => { let result @@ -15,5 +15,8 @@ parentPort.on('message', async data => { storageFailure: Boolean(error.storageFailure) } } + finally { + close() + } parentPort.postMessage(result) }) diff --git a/src/file-data.mjs b/src/file-data.mjs new file mode 100644 index 0000000..1591fa3 --- /dev/null +++ b/src/file-data.mjs @@ -0,0 +1,262 @@ +import fs from 'node:fs' +import path from 'node:path' +import { createHash } from 'node:crypto' +import JSONTag from '@muze-nl/jsontag' +import Parser from '@muze-nl/od-jsontag/src/parse.mjs' +import { RecoveryIntegrityError, assertChangesetExists } from './recovery.mjs' +import { loadIntegrityManifest, verifyDigest } from './integrity.mjs' + +function identity(stat) { + return { + device: stat.dev, inode: stat.ino, size: stat.size, + modified: stat.mtimeMs, changed: stat.ctimeMs + } +} + +function assertIdentity(fd, source) { + if (JSON.stringify(identity(fs.fstatSync(fd))) !== + JSON.stringify(source.identity)) { + throw new Error(`Dataset source changed: ${source.file}`) + } +} + +// A fixed-size buffer covers both hashing and framing; payloads are skipped +// without decoding or allocating a buffer the size of a record/file. +class FileScan { + constructor(fd, file) { + this.fd = fd + this.file = file + this.buffer = Buffer.allocUnsafe(64 * 1024) + this.cursor = 0 + this.available = 0 + this.offset = 0 + this.digest = createHash('sha256') + } + + peek() { + if (this.cursor === this.available) { + this.available = fs.readSync( + this.fd, this.buffer, 0, this.buffer.length, this.offset + ) + this.cursor = 0 + if (!this.available) { + return undefined + } + this.digest.update(this.buffer.subarray(0, this.available)) + } + return this.buffer[this.cursor] + } + + advance(length = 1) { + while (length > 0) { + if (this.peek() === undefined) { + this.fail('truncated record payload') + } + const consumed = Math.min(length, this.available - this.cursor) + this.cursor += consumed + this.offset += consumed + length -= consumed + } + } + + number(kind) { + let digits = 0 + let value = 0 + while (this.peek() >= 48 && this.peek() <= 57) { + value = value * 10 + this.peek() - 48 + this.advance() + digits++ + if (!Number.isSafeInteger(value)) { + this.fail(`invalid ${kind}`) + } + } + if (!digits) { + this.fail(`malformed ${kind}`) + } + return value + } + + fail(message) { + throw new RecoveryIntegrityError(`Invalid OD-JSONTag data: ${message}`, { + file: this.file, + recordKind: 'OD-JSONTag data' + }) + } +} + +export function hashFile(file) { + const fd = fs.openSync(file, 'r') + try { + const source = { file, identity: identity(fs.fstatSync(fd)) } + const scan = new FileScan(fd, file) + while (scan.peek() !== undefined) { + scan.advance(scan.available - scan.cursor) + } + assertIdentity(fd, source) + if (scan.offset !== source.identity.size) { + throw new Error(`Unexpected EOF while hashing: ${file}`) + } + return scan.digest.digest('hex') + } + finally { + fs.closeSync(fd) + } +} + +export function scanDataFile(filename) { + const file = path.resolve(filename) + const fd = fs.openSync(file, 'r') + try { + const source = { + file, identity: identity(fs.fstatSync(fd)), offsets: {} + } + const scan = new FileScan(fd, file) + let record = 0 + while (scan.peek() !== undefined) { + const byte = scan.peek() + if (byte === 10 || byte === 13) { + scan.advance() + continue + } + if (byte === 43) { + scan.advance() + record += scan.number('skip record') + if (scan.peek() !== undefined && + scan.peek() !== 10 && scan.peek() !== 13) { + scan.fail('malformed skip record') + } + } + else { + if (byte !== 40) { + scan.fail('expected record length') + } + scan.advance() + const length = scan.number('record length') + if (scan.peek() !== 41 || length === 0) { + scan.fail('malformed record length') + } + scan.advance() + const start = scan.offset + scan.advance(length) + source.offsets[record++] = [start, scan.offset] + } + if (!Number.isSafeInteger(record) || record >= 0xffffffff) { + scan.fail('invalid record number') + } + } + assertIdentity(fd, source) + if (scan.offset !== source.identity.size) { + scan.fail('unexpected EOF') + } + source.size = scan.offset + source.digest = scan.digest.digest('hex') + return source + } + finally { + fs.closeSync(fd) + } +} + +export class FileDataset { + constructor(meta = {}, immutable = true, ParserType = Parser) { + this.parser = new ParserType(undefined, immutable) + this.parser.meta = { ...meta, resultArray: [] } + this.handles = [] + this.sources = [] + this.root = undefined + } + + append(source) { + const fd = fs.openSync(source.file, 'r') + try { + assertIdentity(fd, source) + this.root = this.parser.parse(fd, source.offsets) + this.handles.push(fd) + this.sources.push(source) + return this.root + } + catch (error) { + fs.closeSync(fd) + throw error + } + } + + open(sources) { + try { + for (const source of sources) { + this.append(source) + } + return this.root + } + catch (error) { + this.close() + throw error + } + } + + rebuildIds() { + const records = new Set() + for (const source of this.sources) { + for (const number of Object.keys(source.offsets)) { + records.add(Number(number)) + } + } + const ids = new Map() + for (const number of [...records].sort((a, b) => a - b)) { + const value = this.parser.getLineProxy(number) + const id = JSONTag.getAttribute(value, 'id') + if (id) { + ids.set(id, number) + } + } + this.parser.meta.index = { ...this.parser.meta.index, id: ids } + return ids + } + + close() { + for (const fd of this.handles.splice(0)) { + fs.closeSync(fd) + } + this.sources = [] + this.root = undefined + } +} + +export async function loadFileData(files) { + const manifest = files.integrityFile + ? await loadIntegrityManifest(files.integrityFile) + : null + const paths = [files.dataFile, ...files.commands.map(id => { + return assertChangesetExists(files.dataFile, id) + })] + const sources = paths.map(file => { + const source = scanDataFile(file) + if (manifest) { + verifyDigest(manifest, files.integrityFile, file, source.digest, { + required: files.integrityRequired + }) + } + return source + }) + if (!sources[0].size) { + throw new Error('Empty base dataset') + } + const dataset = new FileDataset() + try { + dataset.open(sources) + const meta = { + data: path.dirname(path.resolve(files.dataFile)), + parts: files.commands.length, + index: { id: dataset.rebuildIds() } + } + if (files.schemaFile) { + meta.schema = JSONTag.parse( + fs.readFileSync(files.schemaFile, 'utf8') + ) + } + return { sources, meta } + } + finally { + dataset.close() + } +} diff --git a/src/index.id.mjs b/src/index.id.mjs index 6383758..7b4f1ac 100644 --- a/src/index.id.mjs +++ b/src/index.id.mjs @@ -38,14 +38,14 @@ export default { JSON.stringify(index) ) }, - load(uuid = null) { + load(meta, uuid = null) { let filename if (!uuid) { filename = 'index.id.json' } else { - filename = 'index.id.' + filename + '.json' + filename = 'index.id.' + uuid + '.json' } - return JSON.parse(fs.readFileSync(meta.data + filename)) + return JSON.parse(fs.readFileSync(meta.data + '/' + filename)) } } diff --git a/src/index.mjs b/src/index.mjs index ed8b33d..02c3f64 100644 --- a/src/index.mjs +++ b/src/index.mjs @@ -15,8 +15,8 @@ const defaultIndex = { }, load(meta, uuid=null) { return { - id: idIndex.load(uuid), - offset: offsetIndex.load(uuid) + id: idIndex.load(meta, uuid), + offset: offsetIndex.load(meta, uuid) } } } diff --git a/src/index.offset.mjs b/src/index.offset.mjs index 99b6c7d..f90b803 100644 --- a/src/index.offset.mjs +++ b/src/index.offset.mjs @@ -63,14 +63,14 @@ export default { JSON.stringify(index) ) }, - load(uuid = null) { + load(meta, uuid = null) { let filename if (!uuid) { filename = 'index.offset.json' } else { - filename = 'index.offset.' + filename + '.json' + filename = 'index.offset.' + uuid + '.json' } - return JSON.parse(fs.readFileSync(meta.data + filename)) + return JSON.parse(fs.readFileSync(meta.data + '/' + filename)) } } diff --git a/src/integrity.mjs b/src/integrity.mjs index acf7d8c..4b6cd17 100644 --- a/src/integrity.mjs +++ b/src/integrity.mjs @@ -9,7 +9,9 @@ const integrityAlgorithm = 'sha256' export function getDefaultIntegrityFile(dataFile) { const extension = dataFile.split('.').pop() - const basefile = dataFile.substring(0, dataFile.length - (extension.length + 1)) + const basefile = dataFile.substring( + 0, dataFile.length - (extension.length + 1) + ) return `${basefile}.integrity.${extension}` } @@ -26,7 +28,8 @@ export async function loadIntegrityManifest(integrityFile) { let text try { text = await fs.readFile(integrityFile, 'utf8') - } catch (error) { + } + catch (error) { if (error.code === 'ENOENT') { return entries } @@ -39,7 +42,8 @@ export async function loadIntegrityManifest(integrityFile) { let record try { record = JSONTag.parse(line) - } catch (cause) { + } + catch (cause) { throw new RecoveryIntegrityError('Invalid integrity manifest record', { file: integrityFile, lineNumber: index + 1, @@ -73,7 +77,17 @@ export async function loadIntegrityManifest(integrityFile) { return entries } -export function verifyIntegrity(manifest, integrityFile, file, buffer, options = {}) { +export function verifyIntegrity( + manifest, integrityFile, file, buffer, options = {} +) { + return verifyDigest( + manifest, integrityFile, file, digestBuffer(buffer), options + ) +} + +export function verifyDigest( + manifest, integrityFile, file, actual, options = {} +) { const key = integrityFileKey(integrityFile, file) const record = manifest.get(key) if (!record) { @@ -85,7 +99,6 @@ export function verifyIntegrity(manifest, integrityFile, file, buffer, options = } return false } - const actual = digestBuffer(buffer) if (record.digest !== actual) { throw new RecoveryIntegrityError(`Integrity mismatch for ${key}`, { file, @@ -96,10 +109,14 @@ export function verifyIntegrity(manifest, integrityFile, file, buffer, options = } export async function appendIntegrityRecord(integrityFile, file, buffer) { + return appendIntegrityDigest(integrityFile, file, digestBuffer(buffer)) +} + +export async function appendIntegrityDigest(integrityFile, file, digest) { const record = { file: integrityFileKey(integrityFile, file), algorithm: integrityAlgorithm, - digest: digestBuffer(buffer) + digest } await appendFile(integrityFile, JSONTag.stringify(record)) return record diff --git a/src/load-worker.mjs b/src/load-worker.mjs index c90bcf1..a02cae2 100644 --- a/src/load-worker.mjs +++ b/src/load-worker.mjs @@ -1,69 +1,6 @@ import { parentPort } from 'node:worker_threads' -import JSONTag from '@muze-nl/jsontag' -import Parser from '@muze-nl/od-jsontag/src/parse.mjs' -import fs from 'fs' -import path from 'path' -import serialize from '@muze-nl/od-jsontag/src/serialize.mjs' -import { assertChangesetExists, assertOdJsonTagFraming } from './recovery.mjs' -import { loadIntegrityManifest, verifyIntegrity } from './integrity.mjs' - -const parser = new Parser() - -parentPort.on('message', async (files) => { - let meta = { - index: { - id: new Map() - } - } - - await import(files.indexFile) - const extension = files.dataFile.split('.').pop() - const basefile = files.dataFile.substring(0, files.dataFile.length - (extension.length + 1)) //+1 for . character - meta.data = path.dirname(basefile) - const integrityManifest = files.integrityFile - ? await loadIntegrityManifest(files.integrityFile) - : null - let data - let jsontag - // TODO - // - only load index files - // - for each command id - // - load files as raw bytes - // - index.id.*.jsontag and index.offset.*.jsontag to create proxies that will get the correct offset on access - // - do the same for resultSet[0] - the dataspace root entity - // don't parse entire files with od-jsontag - // add version info in proxies with a symbol to get that information - if (fs.existsSync(files.dataFile)) { - jsontag = fs.readFileSync(files.dataFile) - if (integrityManifest) { - verifyIntegrity(integrityManifest, files.integrityFile, files.dataFile, jsontag, { - required: files.integrityRequired - }) - } - assertOdJsonTagFraming(jsontag, files.dataFile, 'base OD-JSONTag data') - data = parser.parse(jsontag) - } - for (let command of files.commands) { - const changesetFile = assertChangesetExists(files.dataFile, command) - jsontag = fs.readFileSync(changesetFile) - if (integrityManifest) { - verifyIntegrity(integrityManifest, files.integrityFile, changesetFile, jsontag, { - required: files.integrityRequired - }) - } - assertOdJsonTagFraming(jsontag, changesetFile, 'changeset OD-JSONTag data') - data = parser.parse(jsontag) - } - if (files.schemaFile) { - jsontag = fs.readFileSync(files.schemaFile, 'utf-8') - meta.schema = JSONTag.parse(jsontag) - } - - const sab = serialize(data, {meta}) - - parentPort.postMessage({ - data: sab, - meta - }) +import { loadFileData } from './file-data.mjs' +parentPort.on('message', async files => { + parentPort.postMessage(await loadFileData(files)) }) diff --git a/src/query-worker-module.mjs b/src/query-worker-module.mjs index 6477182..91f780b 100644 --- a/src/query-worker-module.mjs +++ b/src/query-worker-module.mjs @@ -2,23 +2,40 @@ import {VM} from 'vm2' import { memoryUsage } from 'node:process' import JSONTag from '@muze-nl/jsontag' import {source} from '@muze-nl/od-jsontag/src/symbols.mjs' +import { FileDataset } from './file-data.mjs' import Parser from '@muze-nl/od-jsontag/src/parse.mjs' import {_,from,not,anyOf,allOf,asc,desc,sum,count,avg,max,min,many,one,first,distinct} from '@muze-nl/jaqt' import process from 'node:process' +// vm2 protects array species before host calls. Give it the already-safe +// constructor descriptor on raw query arrays before the read-only proxy exists, +// so methods like map/filter need no temporary mutation through that proxy. +class QueryParser extends Parser { + getArrayProxy(array, parent) { + Object.defineProperty(array, 'constructor', { + value: undefined, + writable: false, + enumerable: false, + configurable: false + }) + return super.getArrayProxy(array, parent) + } +} + let dataspace let metaProxy = { index: { } } -const parser = new Parser() +let dataset +let parser const metaIdProxy = { get: (id) => { let index = parser.meta.index.id.get(id) if (index || index===0) { - return parser.meta.resultArray[index] + return parser.getLineProxy(index) } }, has: (id) => { @@ -28,6 +45,11 @@ const metaIdProxy = { const tasks = { init: async (task) => { + if (dataset) { + dataset.close() + } + dataset = new FileDataset(task.req.meta, true, QueryParser) + parser = dataset.parser if (task.req.access) { task.req.access = await import(task.req.access) task.req.access = task.req.access.default @@ -39,9 +61,7 @@ const tasks = { if (task.req.meta.schema) { parser.meta.schema = task.req.meta.schema } - for (let sab of task.req.body) { //body contains an array of sharedArrayBuffers with initial data and changes - dataspace = parser.parse(sab) - } + dataspace = dataset.open(task.req.sources) metaProxy.index.id = metaIdProxy metaProxy.schema = parser.meta.schema //@TODO: add meta.index.references? and baseURL @@ -51,7 +71,7 @@ const tasks = { if (task.req.meta.index) { parser.meta.index = task.req.meta.index } - dataspace = parser.parse(task.req.body) //update only has a single changeset + dataspace = dataset.append(task.req.source) return true }, query: async (task) => { @@ -67,8 +87,12 @@ const tasks = { export default tasks export function runQuery(pointer, request, query, timeout=1000) { - if (!pointer) { throw new Error('missing pointer parameter')} - if (!request) { throw new Error('missing request parameter')} + if (!pointer) { + throw new Error('missing pointer parameter') +} + if (!request) { + throw new Error('missing request parameter') +} let response = { jsontag: request.jsontag } @@ -81,7 +105,7 @@ export function runQuery(pointer, request, query, timeout=1000) { timeout: timeout, allowAsync: false, sandbox: { - root: dataspace, //@TODO: if we don't pass the root, we can later shard + root: dataspace, data: result, meta: metaProxy, _, @@ -110,33 +134,38 @@ export function runQuery(pointer, request, query, timeout=1000) { result = vm.run(query) let used = Math.round(process.memoryUsage().heapUsed / 1024 / 1024); console.log(`(${used} MB)`); - } catch(err) { + } + catch(err) { console.log(err) response.code = 422; if (request.jsontag) { response.body = '{"message":'+JSON.stringify(''+err)+',"code":422}' - } else { + } + else { response.body = JSON.stringify({message:err, code: 422}) } } - } else { + } + else { result = linkReplacer(result, path+'/') } if (!response.code) { if (response.jsontag) { try { // JSONTag.stringify doesn't handle tagName/attributes well - // with od-jsontag, since some entries in result haven't been parsed yet + // with od-jsontag until result entries have been parsed. // only after parsing will these be available - // so force parsing of all result - od-jsontag should have its own stringify + // Force parsing before formatting the query response. parseAllObjects(result) response.body = JSONTag.stringify(result) - } catch(err) { + } + catch(err) { console.log(err) response.code = 500 response.body = '{"message":'+JSON.stringify(''+err)+',"code":500}' } - } else { + } + else { //@FIXME: replace recursive links response.body = JSON.stringify(result) } @@ -160,7 +189,8 @@ function parseAllObjects(o, reset=true) { parseAllObjects(v, false) } } - } else if (o && typeof o == 'object') { + } + else if (o && typeof o == 'object') { for (let k of Object.keys(o)) { if (o[k] && typeof o[k]=='object') { parseAllObjects(o[k], false) @@ -193,9 +223,11 @@ export function linkReplacer(data, baseURL) { data = data.map((entry,index) => { return linkReplacer(data[index], baseURL+index+'/') }) - } else if (type === 'link') { + } + else if (type === 'link') { // do nothing - } else if (data && typeof data === 'object') { + } + else if (data && typeof data === 'object') { if (data[source]) { data = data[source] } @@ -203,7 +235,8 @@ export function linkReplacer(data, baseURL) { Object.keys(data).forEach(key => { if (Array.isArray(data[key])) { data[key] = new JSONTag.Link(baseURL+key+'/') - } else if (data[key] && typeof data[key] === 'object') { + } + else if (data[key] && typeof data[key] === 'object') { if (JSONTag.getType(data[key])!=='link') { let id=JSONTag.getAttribute(data[key], 'id') if (!id) { diff --git a/src/store-inspection.mjs b/src/store-inspection.mjs index 64673d2..5a4fb70 100644 --- a/src/store-inspection.mjs +++ b/src/store-inspection.mjs @@ -2,13 +2,13 @@ import fs from 'node:fs/promises' import path from 'node:path' import JSONTag from '@muze-nl/jsontag' import { from, _, anyOf, not } from '@muze-nl/jaqt' -import Parser from '@muze-nl/od-jsontag/src/parse.mjs' +import { FileDataset, hashFile, scanDataFile } from './file-data.mjs' import { createHash } from 'node:crypto' -import { assertOdJsonTagFraming, getChangesetPath } from './recovery.mjs' +import { getChangesetPath } from './recovery.mjs' import { getDefaultIntegrityFile, loadIntegrityManifest, - verifyIntegrity + verifyDigest } from './integrity.mjs' export const hash = bytes => createHash('sha256').update(bytes).digest('hex') @@ -101,7 +101,7 @@ export async function inventory(config) { throw new Error(`Symlink in store artifact: ${file}`) } if (entry.isFile()) { - files[file] = hash(await fs.readFile(file)) + files[file] = hashFile(file) } } } @@ -116,7 +116,7 @@ export async function inventory(config) { for (const file of required) { if (!(file in files)) { try { - files[file] = hash(await fs.readFile(file)) + files[file] = hashFile(file) } catch (error) { if (error.code !== 'ENOENT') { @@ -147,22 +147,26 @@ class StoreInspector { this.commands = [] this.commandsById = new Map() this.manifest = null - this.parser = new Parser() - this.buffers = [] + this.dataset = new FileDataset() + this.verified = new Map() this.committed = [] - this.data = undefined this.prefixValid = false } async inspect() { - await this.validateConfiguredPaths() - this.files = await inventory(this.config) - await this.loadCommandHistory() - await this.loadIntegrityManifest() - await this.reconstructCommittedState() - this.validateStoreArtifacts() - await this.verifyStableSnapshot() - return this.createReport() + try { + await this.validateConfiguredPaths() + this.files = await inventory(this.config) + await this.loadCommandHistory() + await this.loadIntegrityManifest() + await this.reconstructCommittedState() + this.validateStoreArtifacts() + await this.verifyStableSnapshot() + return this.createReport() + } + finally { + this.dataset.close() + } } async loadCommandHistory() { @@ -344,18 +348,18 @@ class StoreInspector { } async readVerifiedData(file) { - const bytes = await fs.readFile(file) - assertOdJsonTagFraming(bytes, file) + const source = scanDataFile(file) if (this.manifest) { - verifyIntegrity( + verifyDigest( this.manifest, this.config.integrityFile, file, - bytes, + source.digest, { required: true } ) } - return bytes + this.verified.set(file, source) + return source } async reconstructCommittedState() { @@ -369,12 +373,11 @@ class StoreInspector { async readBaseDataset() { try { - const bytes = await this.readVerifiedData(this.config.datafile) - if (!bytes.length) { + const source = await this.readVerifiedData(this.config.datafile) + if (!source.size) { throw new Error('Empty base dataset') } - this.data = this.parser.parse(bytes) - this.buffers.push(bytes) + this.dataset.append(source) } catch (error) { this.errors.push(error.message) @@ -427,9 +430,8 @@ class StoreInspector { return } try { - const bytes = await this.readVerifiedData(command.file) - this.data = this.parser.parse(bytes) - this.buffers.push(bytes) + const source = this.verified.get(command.file) + this.dataset.append(source) this.committed.push(command.id) } catch (error) { @@ -508,8 +510,7 @@ class StoreInspector { commands: this.commands, committed: this.committed, ready, - data: this.data, - buffers: this.buffers + sources: [...this.dataset.sources] } } } diff --git a/src/store-runtime.mjs b/src/store-runtime.mjs index 71ba36c..e636826 100644 --- a/src/store-runtime.mjs +++ b/src/store-runtime.mjs @@ -165,7 +165,7 @@ export class StoreRuntime { constructor(configuration, mechanisms) { this.configuration = configuration this.mechanisms = mechanisms - this.data = [] + this.sources = [] this.meta = {} this.status = new Map() this.commandQueue = [] @@ -239,7 +239,7 @@ export class StoreRuntime { } initializeLoadedState(inspection, loaded) { - this.data = [loaded.data] + this.sources = loaded.sources this.meta = loaded.meta this.status = new Map( inspection.commands.map(command => { @@ -261,13 +261,16 @@ export class StoreRuntime { config.queryWorker, this.queryWorkerInitialTask(config.slowTimeout) ) + for (const pool of [this.queryWorkerPool, this.slowQueryWorkerPool]) { + pool.on?.('error', error => this.failStorage(error)) + } } queryWorkerInitialTask(timeout) { return { name: 'init', req: { - body: this.data, + sources: this.sources, meta: this.meta, access: this.configuration.access }, @@ -486,7 +489,7 @@ export class StoreRuntime { { ...command, meta: this.meta, - data: this.data, + sources: this.sources, commandsFile: config.commandsFile, indexFile: config.indexFile, datafile: config.store.datafile, @@ -559,14 +562,14 @@ export class StoreRuntime { } publishCommandResult(result) { - if (!result.data) { + if (!result.source) { return } - this.data.push(result.data) + this.sources.push(result.source) Object.assign(this.meta, result.meta) const task = { name: 'update', - req: { body: result.data, meta: this.meta } + req: { source: result.source, meta: this.meta } } this.queryWorkerPool.update(task) this.slowQueryWorkerPool.update(task) @@ -593,8 +596,8 @@ export class StoreRuntime { async closeResources() { await this.serializeAcceptance() await this.runner - this.queryWorkerPool.close() - this.slowQueryWorkerPool.close() + await this.queryWorkerPool.close() + await this.slowQueryWorkerPool.close() if (!this.storageFailed && this.ownership) { await this.ownership.release() this.ownership = null diff --git a/src/workerPool.mjs b/src/workerPool.mjs index 710e30e..8ccb692 100644 --- a/src/workerPool.mjs +++ b/src/workerPool.mjs @@ -1,114 +1,160 @@ -import { AsyncResource } from 'async_hooks' -import { EventEmitter } from 'events' -import path from 'path' -import { Worker } from 'worker_threads' +import { AsyncResource } from 'node:async_hooks' +import { EventEmitter } from 'node:events' +import path from 'node:path' +import { Worker } from 'node:worker_threads' -const kTaskInfo = Symbol('kTaskInfo'); -const kWorkerFreedEvent = Symbol('kWorkerFreedEvent'); - -class WorkerPoolTaskInfo extends AsyncResource { - constructor(callback) { - super('WorkerPoolTaskInfo'); - this.callback = callback; - } +class QueryTask extends AsyncResource { + constructor(task, resolve, reject) { + super('WorkerPoolTaskInfo') + this.task = task + this.resolve = resolve + this.reject = reject + } - done(err, result) { - this.runInAsyncScope(this.callback, null, err, result); - this.emitDestroy(); // `TaskInfo`s are used only once. - } + done(error, result) { + if (error) { + this.runInAsyncScope(this.reject, null, error) + } + else { + this.runInAsyncScope(this.resolve, null, result) + } + this.emitDestroy() + } } -//@TODO: only create new workers when needed, not immediately - export default class WorkerPool extends EventEmitter { - constructor(numThreads, workerFile, initTask) { - super() - this.numThreads = numThreads - this.workerFile = workerFile - this.initTask = initTask - this.workers = [] - this.freeWorkers = [] - this.priority = new Map() - for (let i = 0; i < numThreads; i++) - this.addNewWorker(); - } + constructor(numThreads, workerFile, initTask) { + super() + this.workerFile = path.resolve(workerFile) + this.initTask = { + ...initTask, + req: { ...initTask.req, sources: [...initTask.req.sources] } + } + this.workers = new Map() + this.waiting = [] + this.terminating = new Set() + this.closed = false + this.failure = null + for (let i = 0; i < numThreads; i++) { + this.addNewWorker() + } + } - addNewWorker() { - const worker = new Worker(path.resolve(this.workerFile)); - this.priority[worker] = [] - worker.on('message', (result) => { - // In case of success: Call the callback that was passed to `runTask`, - // remove the `TaskInfo` associated with the Worker, and mark it as free - // again. - if (worker[kTaskInfo]) { - worker[kTaskInfo].done(null, result); - worker[kTaskInfo] = null; - } - if (this.priority[worker].length) { - let task = this.priority[worker].shift() + addNewWorker() { + const worker = new Worker(this.workerFile) + const state = { pending: [this.initTask], active: null, query: null } + this.workers.set(worker, state) + worker.on('message', result => { + const query = state.query + state.active = null + state.query = null + if (query) { + query.done(null, result) + } + this.dispatch(worker, state) + }) + worker.on('error', error => this.workerFailed(worker, error)) + worker.on('exit', code => { + if (!this.closed && this.workers.has(worker)) { + this.workerFailed(worker, new Error(`Query worker exited (${code})`)) + } + }) + this.dispatch(worker, state) + } + + dispatch(worker, state) { + if (this.closed || this.failure || state.active) { + return + } + let task = state.pending.shift() + if (!task) { + const query = this.waiting.shift() + if (!query) { + return + } + state.query = query + task = query.task + } + state.active = task worker.postMessage(task) - } else { - this.freeWorkers.push(worker); - this.emit(kWorkerFreedEvent); - } - }); - worker.on('error', (err) => { - // In case of an uncaught exception: Call the callback that was passed to - // `runTask` with the error. - if (worker[kTaskInfo]) - worker[kTaskInfo].done(err, null); - else - this.emit('error', err); - // Remove the worker from the list and start a new Worker to replace the - // current one. - this.workers.splice(this.workers.indexOf(worker), 1); - this.addNewWorker(); - }); - this.workers.push(worker); - worker.postMessage(this.initTask); - } + } - async run(name, req, options={}) { - return new Promise((resolve, reject) => { - this.runTask({name,req,...options}, (error, result) => { - if (error) { - return reject(error) + workerFailed(worker, error) { + const state = this.workers.get(worker) + if (!state) { + return + } + this.workers.delete(worker) + const termination = worker.terminate() + this.terminating.add(termination) + termination.then(() => this.terminating.delete(termination)) + if (state.query) { + state.query.done(error) + state.query = null + } + if (this.closed || this.failure) { + return } - return resolve(result) - }) - }) - } + // A failed source update cannot be skipped: stop publication/use of + // this pool. Query failures may replace the worker at the current head. + if (state.active?.name === 'init' || state.active?.name === 'update') { + this.failure = error + for (const query of this.waiting.splice(0)) { + query.done(error) + } + this.emit('error', error) + } + else if (!this.closed) { + this.addNewWorker() + } + } - runTask(task, callback) { - if (this.freeWorkers.length === 0) { - // No free threads, wait until a worker thread becomes free. - console.log('no free worker.. waiting') - this.once(kWorkerFreedEvent, () => this.runTask(task, callback)); - return; + run(name, req, options = {}) { + if (this.closed || this.failure) { + return Promise.reject(this.failure || new Error('Query pool closed')) + } + return new Promise((resolve, reject) => { + const task = { name, req, ...options } + this.waiting.push(new QueryTask(task, resolve, reject)) + for (const [worker, state] of this.workers) { + this.dispatch(worker, state) + } + }) } - const worker = this.freeWorkers.pop(); - worker[kTaskInfo] = new WorkerPoolTaskInfo(callback); - worker.postMessage(task); - } - update(task) { - for (let worker of this.workers) { - if (worker[kTaskInfo]) { // worker is busy - this.priority[worker].push(task) // push task on priority list for this specific worker - } else { // worker is free - worker.postMessage(task) // run task immediately - } + update(task) { + this.initTask = { + ...this.initTask, + req: { + ...this.initTask.req, + sources: [...this.initTask.req.sources, task.req.source], + meta: task.req.meta + } + } + for (const [worker, state] of this.workers) { + state.pending.push(task) + this.dispatch(worker, state) + } } - } - memoryUsage() { - for (let worker of this.freeWorkers) { - worker[kTaskInfo] = new WorkerPoolTaskInfo(() => {}) - worker.postMessage({name:'memoryUsage'}) + memoryUsage() { + return this.run('memoryUsage', {}) } - } - close() { - for (const worker of this.workers) worker.terminate(); - } -} \ No newline at end of file + async close() { + this.closed = true + const error = new Error('Query pool closed') + for (const query of this.waiting.splice(0)) { + query.done(error) + } + await Promise.all([...this.workers].map(async ([worker, state]) => { + await worker.terminate() + if (state.query) { + state.query.done(error) + state.query = null + } + })) + await Promise.all(this.terminating) + this.workers.clear() + } +} diff --git a/test/admin-recovery.mjs b/test/admin-recovery.mjs index fa9ec53..0ac7973 100644 --- a/test/admin-recovery.mjs +++ b/test/admin-recovery.mjs @@ -1,3 +1,4 @@ +import { FileDataset } from '../src/file-data.mjs' import { Buffer } from 'node:buffer' import test from 'node:test' import assert from 'node:assert/strict' @@ -77,7 +78,7 @@ test('administrator recovers trailing missing/waiting commands in log order on a 'Inspection does not expose JAQT proxies' ) assert.deepEqual( - actual.data.persons.map(p => p.name), + personNames(actual), ['A', 'B'] ) assert.deepEqual(await fs.readFile(store.commandStatus), before) @@ -164,7 +165,7 @@ test('backup and restore validate saved bytes without executing commands', async sourceQuiescent: true }) assert.deepEqual( - (await inspectStore(restored.config)).data.persons.map(p => p.name), + personNames(await inspectStore(restored.config)), ['A'] ) assert.deepEqual(restored.missingFromBackup, []) @@ -390,7 +391,7 @@ test('approved recovery resumes from completed candidate prefix without repeatin await fs.mkdir(retry) const result = await runRecovery(partial, retry, plan) assert.deepEqual( - (await inspectStore(result.config)).data.persons.map(p => p.name), + personNames(await inspectStore(result.config)), ['A', 'B'] ) assert.equal(await fs.readFile(witness, 'utf8'), 'A\nB\n') @@ -547,7 +548,7 @@ test('verified A prefix permits only missing B, waiting C and missing D in log o await fs.mkdir(next) const result = await runRecovery(source, next, plan) assert.deepEqual( - (await inspectStore(result.config)).data.persons.map(p => p.name), + personNames(await inspectStore(result.config)), ['A', 'B', 'C', 'D'] ) assert.deepEqual( @@ -708,3 +709,13 @@ test('backup coverage distinguishes changed history and data from missing source ) } }) + +function personNames(inspection) { + const dataset = new FileDataset() + try { + return dataset.open(inspection.sources).persons.map(p => p.name) + } + finally { + dataset.close() + } +} diff --git a/test/command-offsets.mjs b/test/command-offsets.mjs index e24b1f3..bb996df 100644 --- a/test/command-offsets.mjs +++ b/test/command-offsets.mjs @@ -7,7 +7,9 @@ import {fileURLToPath} from 'node:url' import JSONTag from '@muze-nl/jsontag' import Parser from '@muze-nl/od-jsontag' import serialize from '@muze-nl/od-jsontag/src/serialize.mjs' -import runCommand, {initialize} from '../src/command-worker-module.mjs' +import runCommand, {initialize, close} from '../src/command-worker-module.mjs' + +import { scanDataFile } from '../src/file-data.mjs' const baseIndex = new URL('../src/index.mjs', import.meta.url) @@ -26,8 +28,11 @@ async function fixture(t, indexSource) { await fs.writeFile(indexFile, indexSource) } const base = serialize(JSONTag.parse('{"items":[{"name":"first"},{"name":"untouched"}]}')) + const datafile = path.join(dir, 'data.jsontag') + await fs.writeFile(datafile, base) + t.after(close) const task = { - data: [base], datafile: path.join(dir, 'data.jsontag'), commandsFile, indexFile, + sources: [scanDataFile(datafile)], datafile: path.join(dir, 'data.jsontag'), commandsFile, indexFile, meta: {data: dir, parts: 0, index: {id: new Map()}, resultArray: []} } await initialize(task) @@ -107,7 +112,7 @@ test('successive command indexes are relative to their own changeset files', asy const {dir, base, task} = await fixture(t) await runCommand(JSONTag.stringify({id: 'first', name: 'edit', value: 'longer first value'})) const first = await verifyOffsets(dir, 'first') - await initialize({...task, data: [base, first.bytes], meta: {data: dir, parts: 1, index: {id: new Map()}, resultArray: []}}) + await initialize({...task, sources: [task.sources[0], scanDataFile(path.join(dir, 'data.first.jsontag'))], meta: {data: dir, parts: 1, index: {id: new Map()}, resultArray: []}}) await runCommand(JSONTag.stringify({id: 'second', name: 'edit', item: 1, value: 'x'})) const second = await verifyOffsets(dir, 'second') assert.deepEqual(Object.keys(second.offsets), ['2']) @@ -169,3 +174,17 @@ test('custom finalization rejection propagates from the command worker', async t await assert.rejects(runCommand('{"id":"rejected","name":"edit","value":"changed"}'), /custom finalization failed/) await assert.rejects(fs.access(path.join(dir, 'index.offset.rejected.json')), /ENOENT/) }) + +test('a finalizer cannot publish changed canonical bytes as the command result', async t => { + await fixture(t, `import fs from 'node:fs/promises' +export default { + update() {}, + async finalize(serialized, meta, uuid) { + await fs.appendFile(meta.data + '/data.' + uuid + '.jsontag', '\\n') + } +}`) + await assert.rejects( + runCommand('{"id":"tampered","name":"edit","value":"changed"}'), + /Changeset changed during finalization/ + ) +}) diff --git a/test/durability-startup.mjs b/test/durability-startup.mjs index b5ad9cc..5a3ce22 100644 --- a/test/durability-startup.mjs +++ b/test/durability-startup.mjs @@ -8,6 +8,7 @@ import { Worker } from 'node:worker_threads' import { fileURLToPath } from 'node:url' import JSONTag from '@muze-nl/jsontag' import Parser from '@muze-nl/od-jsontag/src/parse.mjs' +import { FileDataset } from '../src/file-data.mjs' import serialize from '@muze-nl/od-jsontag/src/serialize.mjs' import { appendIntegrityRecord, @@ -26,9 +27,10 @@ import { const rootDir = path.dirname(path.dirname(fileURLToPath(import.meta.url))) const loadWorker = path.join(rootDir, 'src/load-worker.mjs') -function parseOd(buffer) { - const parser = new Parser() - return parser.parse(buffer) +function parseOd(sources, t) { + const dataset = new FileDataset() + t.after(() => dataset.close()) + return dataset.open(sources) } async function makeFixture(t) { @@ -126,7 +128,7 @@ test('accepted command changeset is not treated as committed state on startup', ...fixture, commands: getCommittedCommandIds(status) }) - const data = parseOd(result.data) + const data = parseOd(result.sources, t) assert.equal( data.persons.length, @@ -235,7 +237,7 @@ test('malformed uncommitted changeset is ignored during committed startup recons ...fixture, commands: getCommittedCommandIds(status) }) - const data = parseOd(result.data) + const data = parseOd(result.sources, t) assert.equal(data.persons.length, 0) }) @@ -264,7 +266,7 @@ test('durable status file selects only done command changesets for startup', asy ...fixture, commands: getCommittedCommandIds(status) }) - const data = parseOd(result.data) + const data = parseOd(result.sources, t) assert.deepEqual( data.persons.map(person => person.name), diff --git a/test/file-data.mjs b/test/file-data.mjs new file mode 100644 index 0000000..45c5017 --- /dev/null +++ b/test/file-data.mjs @@ -0,0 +1,297 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import test from 'node:test' +import { createHash } from 'node:crypto' +import { fileURLToPath } from 'node:url' +import JSONTag from '@muze-nl/jsontag' +import serialize from '@muze-nl/od-jsontag/src/serialize.mjs' +import { FileDataset, scanDataFile, hashFile, loadFileData } from '../src/file-data.mjs' +import { appendIntegrityRecord } from '../src/integrity.mjs' +import index from '../src/index.mjs' +import WorkerPool from '../src/workerPool.mjs' +import StoreRuntime from '../src/store-runtime.mjs' +import { makeServerFixture } from './durability-helpers.mjs' +import runCommand, { initialize, close } from '../src/command-worker-module.mjs' + +function fixture(t, text = '{"items":[{"name":"first"},{"name":"last"}]}') { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'simplystore-files-')) + t.after(() => fs.rmSync(dir, { recursive: true, force: true })) + const file = path.join(dir, 'data.jsontag') + fs.writeFileSync(file, serialize(JSONTag.parse(text))) + const source = scanDataFile(file) + return { dir, file, source } +} + +function open(t, sources, meta, immutable = true) { + const dataset = new FileDataset(meta, immutable) + t.after(() => dataset.close()) + dataset.open(sources) + return dataset +} + +test('file loading resolves unread IDs and sends metadata without dataset bytes', async t => { + const { file, dir } = fixture(t) + // Legacy missing/stale derived indexes must not override canonical bytes. + fs.writeFileSync(path.join(dir, 'index.offset.json'), '{"0":[9,999999]}') + fs.writeFileSync(path.join(dir, 'index.id.json'), '{"last":999}') + const loaded = await loadFileData({ dataFile: file, commands: [] }) + assert.equal(loaded.data, undefined) + assert.equal(loaded.meta.index.id.get('last'), 2) + assert.deepEqual(structuredClone(loaded), loaded) + const dataset = open(t, loaded.sources, loaded.meta) + assert.equal(dataset.parser.meta.resultArray[2], undefined) + assert.equal(dataset.parser.getLineProxy(2).name, 'last') + assert.equal(dataset.root.items[1], dataset.parser.getLineProxy(2)) +}) + +test('sparse file overlays retain unread records and allocate beyond the full base', t => { + const { dir, source } = fixture(t) + const patch = path.join(dir, 'data.patch.jsontag') + const payload = '{"name":"changed"}' + fs.writeFileSync(patch, `+1\n(${Buffer.byteLength(payload)})${payload}`) + const dataset = open(t, [source, scanDataFile(patch)], {}, false) + dataset.root.items.push({ name: 'new' }) + assert.deepEqual(dataset.root.items.map(item => item.name), [ + 'changed', 'last', 'new' + ]) + const output = path.join(dir, 'output.jsontag') + fs.writeFileSync(output, serialize(dataset.root)) + const copy = open(t, [scanDataFile(output)]) + assert.deepEqual(copy.root.items.map(item => item.name), [ + 'changed', 'last', 'new' + ]) +}) + +test('framing scan handles short reads, UTF-8 and payloads beyond its byte buffer', t => { + const { file } = fixture(t, JSON.stringify({ text: '漢😀'.repeat(20000) })) + const read = fs.readSync + const mock = t.mock.method(fs, 'readSync', (fd, buffer, offset, length, position) => { + return read(fd, buffer, offset, Math.min(length, 997), position) + }) + const source = scanDataFile(file) + assert.ok(mock.mock.callCount() > 100) + assert.equal(source.digest, hashFile(file)) + const bytes = fs.readFileSync(file) + assert.equal(source.digest, createHash('sha256').update(bytes).digest('hex')) + const header = /^\((\d+)\)/.exec(bytes.toString('utf8', 0, 32)) + assert.deepEqual(source.offsets, { + 0: [header[0].length, header[0].length + Number(header[1])] + }) +}) + +for (const content of ['(9){}', '(x){}', '+x\n', '+1x\n', '(2){}x', + '+4294967295\n', '(9007199254740999){}']) { + test(`file framing rejects ${JSON.stringify(content)}`, t => { + const { file } = fixture(t) + fs.writeFileSync(file, content) + assert.throws(() => scanDataFile(file), /Invalid OD-JSONTag/) + }) +} + +test('sessions close all handles on failure and reject changed source identities', t => { + const { source, file } = fixture(t) + const dataset = new FileDataset() + const handles = [] + const openSync = fs.openSync + t.mock.method(fs, 'openSync', (...args) => { + const fd = openSync(...args) + handles.push(fd) + return fd + }) + assert.throws(() => dataset.open([ + source, { ...source, file: file + '.missing' } + ]), /ENOENT/) + for (const fd of handles) { + assert.throws(() => fs.fstatSync(fd), /EBADF/) + } + fs.appendFileSync(file, '\n') + assert.throws(() => new FileDataset().open([source]), /source changed/) +}) + +test('opening and partial reads never read the entire file through readFile', async t => { + const { file } = fixture(t) + t.mock.method(fs, 'readFileSync', () => { + throw new Error('whole-file read is forbidden') + }) + const loaded = await loadFileData({ dataFile: file, commands: [] }) + const dataset = open(t, loaded.sources, loaded.meta) + assert.equal(dataset.root.items[1].name, 'last') +}) + +test('file loading checks full-file integrity before returning sources', async t => { + const { dir, file } = fixture(t) + const integrityFile = path.join(dir, 'integrity.jsontag') + await appendIntegrityRecord(integrityFile, file, fs.readFileSync(file)) + const options = { + dataFile: file, commands: [], integrityFile, integrityRequired: true + } + await loadFileData(options) + const bytes = fs.readFileSync(file) + fs.writeFileSync(file, bytes.toString().replace('last', 'lost')) + await assert.rejects(loadFileData(options), /Integrity mismatch/) +}) + +test('persisted index loaders use the requested base and command names', t => { + const { dir } = fixture(t) + for (const suffix of ['', '.command']) { + fs.writeFileSync(path.join(dir, `index.id${suffix}.json`), '{"id":2}') + fs.writeFileSync(path.join(dir, `index.offset${suffix}.json`), '{"2":[1,4]}') + } + for (const command of [null, 'command']) { + assert.deepEqual(index.load({ data: dir }, command), { + id: { id: 2 }, offset: { 2: [1, 4] } + }) + } +}) + +test('command ID lookups resolve unread objects and output only the changed file', async t => { + const { dir, file } = fixture(t) + const loaded = await loadFileData({ dataFile: file, commands: [] }) + const commandsFile = path.join(dir, 'commands.mjs') + fs.writeFileSync(commandsFile, `export default { + edit(data, command, unused, meta) { + meta.index.id.get(command.target).deref().name = 'edited by ID' + } + }`) + await initialize({ + ...loaded, datafile: file, commandsFile, + indexFile: fileURLToPath(new URL('../src/index.mjs', import.meta.url)) + }) + t.after(close) + const result = await runCommand('{"id":"edit","name":"edit","target":"last"}') + assert.equal(result.data, undefined) + assert.deepEqual(Object.keys(result.source.offsets), ['2']) + const dataset = open(t, [...loaded.sources, result.source], result.meta) + assert.equal(dataset.root.items[1].name, 'edited by ID') + assert.equal(dataset.root.items[0].name, 'first') +}) + +async function poolFixture(t) { + const { dir } = fixture(t) + const workerFile = path.join(dir, 'worker.mjs') + fs.writeFileSync(workerFile, `import {parentPort} from 'node:worker_threads' + let version = 0 + parentPort.on('message', async task => { + if (task.req?.crash) { throw new Error('query crashed') } + if (task.name === 'init' || task.name === 'update') { + await new Promise(resolve => setTimeout(resolve, 25)) + version = task.req.meta.version + } + parentPort.postMessage(version) + })`) + const pool = new WorkerPool(2, workerFile, { + name: 'init', req: { sources: [], meta: { version: 0 } } + }) + t.after(() => pool.close()) + return pool +} + +test('updates finish before queries even during initialization and replacement', async t => { + const pool = await poolFixture(t) + pool.update({ name: 'update', req: { source: { file: 'a' }, meta: { version: 1 } } }) + pool.update({ name: 'update', req: { source: { file: 'b' }, meta: { version: 2 } } }) + const results = await Promise.all(Array.from({ length: 8 }, () => { + return pool.run('query', {}) + })) + assert.deepEqual(results, Array(8).fill(2)) + await assert.rejects(pool.run('query', { crash: true }), /query crashed/) + const next = await Promise.all(Array.from({ length: 8 }, () => { + return pool.run('query', {}) + })) + assert.deepEqual(next, Array(8).fill(2)) + assert.equal(pool.initTask.req.sources.length, 2) +}) + +test('closing a pool awaits termination and rejects outstanding queries', async t => { + const pool = await poolFixture(t) + const pending = pool.run('query', {}) + const rejected = assert.rejects(pending, /pool closed/) + await pool.close() + await rejected + assert.equal(pool.workers.size, 0) + await assert.rejects(pool.run('query', {}), /pool closed/) +}) + +test('live runtime retains only files through commit, query and reopen', async t => { + let runtime + let reopened + t.after(async () => { + await reopened?.close() + await runtime?.close() + }) + const fixture = await makeServerFixture(t) + const options = { ...fixture, maxWorkers: 2 } + runtime = await StoreRuntime.open(options) + assert.equal(runtime.data, undefined) + assert.equal(runtime.sources.length, 1) + const accepted = await runtime.acceptCommand( + '{"id":"A","name":"addPerson","value":{"name":"A"}}' + ) + assert.equal(accepted.code, 202) + await runtime.runQueuedCommands() + assert.equal(runtime.getCommandStatus('A').value.status, 'done') + assert.equal(runtime.sources.length, 2) + assert.ok(runtime.sources.every(source => typeof source.file === 'string')) + const response = await runtime.runQuery({ + path: '/', body: 'data.persons.map(p => p.name)', jsontag: false + }) + assert.deepEqual(JSON.parse(response.body), ['A']) + const selected = await runtime.runQuery({ + path: '/', + body: 'from(data.persons).where({name: "A"}).select({name: _})', + jsontag: false + }) + assert.deepEqual(JSON.parse(selected.body), [{name: 'A'}]) + const mutation = await runtime.runQuery({ + path: '/', body: 'data.persons.push({name: "forbidden"})', + jsontag: false + }) + assert.equal(mutation.code, 422) + const definition = await runtime.runQuery({ + path: '/', body: 'Object.defineProperty(data.persons, "0", {value: null})', + jsontag: false + }) + assert.equal(definition.code, 422) + await runtime.close() + reopened = await StoreRuntime.open(options) + assert.equal(reopened.sources.length, 2) + const after = await reopened.runQuery({ + path: '/', body: 'data.persons.map(p => p.name)', jsontag: false + }) + assert.deepEqual(JSON.parse(after.body), ['A']) +}) + +test('a failed worker update refuses further queries instead of serving stale data', async t => { + const pool = await poolFixture(t) + const failed = new Promise(resolve => pool.once('error', resolve)) + pool.update({ name: 'update', req: { + source: {file: 'bad'}, meta: {version: 1}, crash: true + } }) + const query = assert.rejects(pool.run('query', {}), /query crashed/) + await failed + await query + await assert.rejects(pool.run('query', {}), /query crashed/) +}) + +test('premature EOF cannot turn an existing data file into an empty source', t => { + const { file } = fixture(t) + t.mock.method(fs, 'readSync', () => 0) + assert.throws(() => scanDataFile(file), /unexpected EOF/) + assert.throws(() => hashFile(file), /Unexpected EOF/) +}) + +test('ID precedence follows record order when a patch fills an earlier hole', async t => { + const { file, dir } = fixture(t) + const record = text => `(${Buffer.byteLength(text)})${text}` + fs.writeFileSync(file, + record('{"items":[~2,~5]}') + '\n+4\n' + + record('{"name":"higher"}') + ) + fs.writeFileSync(path.join(dir, 'data.fill.jsontag'), + '+2\n' + record('{"name":"lower"}') + ) + const loaded = await loadFileData({dataFile: file, commands: ['fill']}) + assert.equal(loaded.meta.index.id.get('duplicate'), 5) +}) diff --git a/test/power-loss-baseline.mjs b/test/power-loss-baseline.mjs index e88b00d..153bfb0 100644 --- a/test/power-loss-baseline.mjs +++ b/test/power-loss-baseline.mjs @@ -1,3 +1,4 @@ +import { FileDataset } from '../src/file-data.mjs' import { Buffer } from 'node:buffer' // The original baseline is retained at EVD-20260919-TTZ7C-23's Git version. // These assertions implement the corrected command-only / authoritative-log @@ -101,8 +102,11 @@ test('PL10 an actual short changeset write is completed before acknowledgment', path.basename(e.file).startsWith('data.A.jsontag.') ).length >= 2 ) + const dataset = new FileDataset() + t.after(() => dataset.close()) + const data = dataset.open((await inspectStore(store)).sources) assert.deepEqual( - (await inspectStore(store)).data.persons.map(p => p.name), + data.persons.map(p => p.name), ['A'] ) }) diff --git a/test/store-runtime.mjs b/test/store-runtime.mjs index fa83d57..07743ba 100644 --- a/test/store-runtime.mjs +++ b/test/store-runtime.mjs @@ -69,7 +69,7 @@ async function openRuntime(t, overrides = {}) { async runWorker() { events.push('committed data loaded') return { - data: new Uint8Array([1]), + sources: [{file: fixture.datafile}], meta: { index: { id: new Map() }, parts: 1 @@ -102,7 +102,7 @@ async function openRuntime(t, overrides = {}) { test('runtime opens committed state and closes its owned resources', async t => { const opened = await openRuntime(t) - assert.equal(opened.runtime.data.length, 1) + assert.equal(opened.runtime.sources.length, 1) assert.equal(opened.runtime.meta.parts, 1) assert.equal(opened.pools.length, 2) assert.equal(opened.ownershipReleased(), false) @@ -177,7 +177,7 @@ test('runtime keeps concurrent acceptance and execution in command-log order', const parsed = JSONTag.parse(task.command) starts.push(parsed.id) return { - data: new Uint8Array([starts.length + 1]), + source: { file: `data.${parsed.id}.jsontag` }, meta: { parts: starts.length + 1 } } } From a64b81373b2ef3d8252b400d7a18b9242038adb2 Mon Sep 17 00:00:00 2001 From: Auke van Slooten Date: Wed, 23 Sep 2026 17:12:12 +0200 Subject: [PATCH 06/15] Evaluate file-backed runtime migration and document its bounds --- .spiral/cycles/CYC-20260923-TTZ7C-44.md | 8 +++ .spiral/cycles/CYC-20260923-TTZ7C-44.ttl | 4 +- .spiral/evidence/EVD-20260923-TTZ7C-49.md | 82 ++++++++++++++++++++++ .spiral/evidence/EVD-20260923-TTZ7C-49.ttl | 12 ++++ .spiral/project-context.md | 4 +- .spiral/project.ttl | 2 +- 6 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 .spiral/evidence/EVD-20260923-TTZ7C-49.md create mode 100644 .spiral/evidence/EVD-20260923-TTZ7C-49.ttl diff --git a/.spiral/cycles/CYC-20260923-TTZ7C-44.md b/.spiral/cycles/CYC-20260923-TTZ7C-44.md index 361e103..b07d66a 100644 --- a/.spiral/cycles/CYC-20260923-TTZ7C-44.md +++ b/.spiral/cycles/CYC-20260923-TTZ7C-44.md @@ -18,3 +18,11 @@ ID lookup of unread records, sparse overlays, no-op commands, worker replacement and shutdown, index/framing/integrity failures and a bounded-memory large-file probe. Source changes must remain confined to disposable fixtures during tests. Client version metadata/audit history and sandbox replacement remain later work. + +## Evaluation + +Implementation is complete at `bfeed2c5c45cd3ac58dd42aac868585f31c59d19`. EVD-20260923-TTZ7C-49 +records 140 passing tests, clean focused lint, fresh-package installation, a +329 MB/48 MB-heap runtime probe and read-only curriculum evidence. +See [the integration guide](../../docs/file-data.md) for the source contract and +limits. Human acceptance remains pending; the cycle stays Active. diff --git a/.spiral/cycles/CYC-20260923-TTZ7C-44.ttl b/.spiral/cycles/CYC-20260923-TTZ7C-44.ttl index 2dc39ae..fc4825b 100644 --- a/.spiral/cycles/CYC-20260923-TTZ7C-44.ttl +++ b/.spiral/cycles/CYC-20260923-TTZ7C-44.ttl @@ -6,4 +6,6 @@ project:CYC-20260923-TTZ7C-44 a sd:Cycle ; dcterms:identifier "CYC-20260923-TTZ7C-44" ; sd:repositoryPath ".spiral/cycles/CYC-20260923-TTZ7C-44.md" ; - sd:status sd:Active . + sd:status sd:Active ; + sd:derivedFrom [ a sd:ArtifactReference ; sd:artifact project:REQ-20260923-TTZ7C-46 ; sd:gitCommit "a75db831ee6779bbf297e94b9d1baf694692cb84" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:CYC-20260923-TTZ7C-44 ; sd:gitCommit "8299fcc8b8fcf113091ae85bbcf11f867f8c5908" ] . diff --git a/.spiral/evidence/EVD-20260923-TTZ7C-49.md b/.spiral/evidence/EVD-20260923-TTZ7C-49.md new file mode 100644 index 0000000..6f51abf --- /dev/null +++ b/.spiral/evidence/EVD-20260923-TTZ7C-49.md @@ -0,0 +1,82 @@ +--- +id: EVD-20260923-TTZ7C-49 +--- + +# File-backed runtime evaluation + +Implementation: `bfeed2c5c45cd3ac58dd42aac868585f31c59d19` on +`spiral/CYC-20260923-TTZ7C-44-file-backed`. + +## Evaluation + +The authorized migration is implemented and ready for human review. Runtime, +query workers, command workers, startup inspection and administrative replay use +ordered file sources. The remote query/command API and stored data/log formats +remain unchanged. No production data, deployment, merge or release was changed. +Existing untracked example indexes and package backup were left untouched. + +- Full regression command below: **140 tests pass**, zero failures/cancellations + (118 existing cases plus 22 new). Runtime/startup, conversion and offsets, + ACID, process crashes, handlers, power-loss baseline, storage and administrative + recovery remain covered. Adaptations to old tests change internal source/report + fixtures; existing remote behavior and durability assertions remain. +- New regressions cover sparse overlays and record allocation, untouched IDs, + byte-free worker messages, stale/missing sidecars, UTF-8 and short reads, + malformed framing/overflow/premature EOF, source identity changes and handle + cleanup, integrity, index loader paths, ID-based command edits, no-op changes, + update ordering while workers initialize, replacement at current head, + failed update refusal, awaited shutdown, real commit/query/reopen, and finalizer + tampering. Duplicate-ID precedence follows record number even when a patch + fills an earlier hole. +- Array `map`, JAQT projection and mutation/defineProperty refusal are exercised + through the real query sandbox. Those new checks exposed vm2's temporary array + constructor mutation, which the QueryParser compatibility adapter avoids + using vm2's already-safe constructor shape. No sandbox replacement or broader + security assessment is claimed. +- Focused ESLint for all changed runtime modules, new tests and benchmark: + **zero errors and warnings**. `git diff --check` passes. +- Local package creation and installation into a fresh temporary consumer pass; + the installed package reads an indexed file. New source, documentation and + benchmark are included. The dependency resolves the pinned upstream od-jsontag + revision. No package was published. +- End-to-end generated store: **328,877,813 bytes / 20,000 records**, with a + **48 MB JavaScript heap limit**. Startup and a direct last-record ID query + pass. One run measured 17,498 ms opening and 93 ms for the query including + worker readiness. Parent retained heap: 8,905,216 bytes; query-worker heap: + 24,427,720 bytes. Array buffers: 24,823 and 57,562 bytes respectively. This is + evidence against retained dataset buffers, not a total-process-memory or + cold-storage bound. The benchmark uses and removes only a disposable fixture. +- Read-only curriculum probe: **77,952,430 bytes / 181,724 records / 180,416 IDs**, + 1,003 sampled ID reads, clean cache capped at 256 records, about 7.9 seconds + including ID reconstruction. Source SHA-256 before and after matched. No + dataset or sidecar was edited. + +```sh +node --test --test-reporter=tap test/file-data.mjs test/store-runtime.mjs \ + test/durability-startup.mjs test/conversion-offsets.mjs \ + test/command-offsets.mjs test/acid-baseline.mjs test/durability-crash.mjs \ + test/after-change-handlers.mjs test/power-loss-baseline.mjs \ + test/storage.mjs test/admin-recovery.mjs +node --expose-gc --max-old-space-size=48 scripts/benchmark-file-data.mjs +``` + +## Boundaries and review disposition + +Startup hashes all canonical bytes and reconstructs IDs by decoding final +records. Index catalogs remain O(records), handles O(source files per worker), +and mutable sessions retain touched data. Command output/custom hooks/query +results may be large. Schemas and logs remain in memory; administrative copying +still temporarily buffers individual files. These are documented boundaries, +not a constant-memory claim. The large-file result does not establish a broader +filesystem or hardware durability envelope; the separate QEMU power-cut campaign +was not repeated. + +Internal custom worker implementations must use the new source contract; +`inspectStore` returns `sources` instead of live `data`/`buffers`. Existing custom +index finalizers retain their call contract. Source files must be immutable +while open under store ownership. A Git dependency pin is used until a separate +od-jsontag npm release. Client version metadata/audit history and query sandbox +selection remain later work. + +Human outcome acceptance is pending. This Active cycle is not eligible for +integration until accepted and its exact prospective merge is validated. diff --git a/.spiral/evidence/EVD-20260923-TTZ7C-49.ttl b/.spiral/evidence/EVD-20260923-TTZ7C-49.ttl new file mode 100644 index 0000000..1b7da56 --- /dev/null +++ b/.spiral/evidence/EVD-20260923-TTZ7C-49.ttl @@ -0,0 +1,12 @@ +@prefix sd: . +@prefix dcterms: . +@prefix project: . + +project:EVD-20260923-TTZ7C-49 + a sd:VerificationEvidence ; + dcterms:identifier "EVD-20260923-TTZ7C-49" ; + sd:repositoryPath ".spiral/evidence/EVD-20260923-TTZ7C-49.md" ; + sd:status sd:Active ; + sd:verifies [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-26 ; sd:gitCommit "bfeed2c5c45cd3ac58dd42aac868585f31c59d19" ] ; + sd:verifies [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-33 ; sd:gitCommit "bfeed2c5c45cd3ac58dd42aac868585f31c59d19" ] ; + sd:verifies [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-34 ; sd:gitCommit "bfeed2c5c45cd3ac58dd42aac868585f31c59d19" ] . diff --git a/.spiral/project-context.md b/.spiral/project-context.md index 07dab14..3616989 100644 --- a/.spiral/project-context.md +++ b/.spiral/project-context.md @@ -24,8 +24,8 @@ shared-memory data after reviewing od-jsontag hardening. This continues the file-only roadmap direction, prioritized ahead of rebuild. SRC-45, UND-48, REQ-46 and DES-47 (20260923-TTZ7C namespace) retain the commitment. Implementation uses ordered file sources, bounded clean-record caches, worker-owned handles, -streamed framing/hash scans and file-backed administrative replay. Evaluation -and human acceptance remain pending. Client-visible version metadata and audit +streamed framing/hash scans and file-backed administrative replay. EVD-20260923-TTZ7C-49 records 140 passing tests, clean lint, a fresh-package +installation and large-file/real-data probes. Human acceptance remains pending. Client-visible version metadata and audit trails follow this migration; no such feature is implemented by this cycle. Latest accepted cycle: CYC-20260919-TTZ7C-42, Updated Roadmap And Planning diff --git a/.spiral/project.ttl b/.spiral/project.ttl index 24c3ab5..09baea9 100644 --- a/.spiral/project.ttl +++ b/.spiral/project.ttl @@ -16,7 +16,7 @@ project:CTX-001 sd:gitCommit "79fcf670d4558c75b9933743e6f2ea5616221eb0" ] ; sd:derivedFrom [ a sd:ArtifactReference ; sd:artifact project:SRC-20260923-TTZ7C-45 ; sd:gitCommit "8299fcc8b8fcf113091ae85bbcf11f867f8c5908" ] ; - sd:transforms [ a sd:ArtifactReference ; sd:artifact project:CTX-001 ; sd:gitCommit "e446a315e4c86709f9d1eae98453a058491a9309" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:CTX-001 ; sd:gitCommit "bfeed2c5c45cd3ac58dd42aac868585f31c59d19" ] ; sd:developerEvaluationReadinessBar "Developers should be able to evaluate bounded durability claims from invariants, executable tests, evidence artifacts, explicit failure behavior, and documented known gaps." ; sd:hasAudience project:AUD-DEVELOPERS, project:AUD-END-USERS, project:AUD-OPERATORS ; sd:hasDownstreamProject project:DOWNSTREAM-CURRICULUM-STORE ; From 4bb10c2013fd2a7cb989b02586b992b936121362 Mon Sep 17 00:00:00 2001 From: Auke van Slooten Date: Wed, 23 Sep 2026 17:16:31 +0200 Subject: [PATCH 07/15] Reproduce late file-read failure classification gap --- .spiral/defects/DEF-20260923-TTZ7C-50.md | 28 ++++++++++++++ .spiral/defects/DEF-20260923-TTZ7C-50.ttl | 11 ++++++ test/file-data.mjs | 46 +++++++++++++++++++++++ 3 files changed, 85 insertions(+) create mode 100644 .spiral/defects/DEF-20260923-TTZ7C-50.md create mode 100644 .spiral/defects/DEF-20260923-TTZ7C-50.ttl diff --git a/.spiral/defects/DEF-20260923-TTZ7C-50.md b/.spiral/defects/DEF-20260923-TTZ7C-50.md new file mode 100644 index 0000000..11bec5e --- /dev/null +++ b/.spiral/defects/DEF-20260923-TTZ7C-50.md @@ -0,0 +1,28 @@ +--- +id: DEF-20260923-TTZ7C-50 +--- + +# Late file reads need storage-failure semantics + +Final review of the file-backed migration identified a changed failure boundary: +source reads can now fail inside a command or query after initialization. The +prior buffered runtime read source bytes before execution. DES-47 preserved +publication/storage failures but did not explicitly carry late read failures +through the ordinary command/query exception handlers. + +Two new regressions on implementation bfeed2c fail (21 other focused tests pass): +a command's unread record raises Unexpected end of file without storageFailure; +a query that catches its unread-record error can return success, leaving future +command acceptance enabled. The fixtures truncate only disposable data files +after both query pools have initialized. This models unavailable backing bytes; +normal operation still requires immutable source files. + +Required correction within the existing migration goal: the file parser records +read/decode/reference failures in host-owned session state. Commands must refuse +publication even if handler code catches that failure. Query workers consult +that host state, not a user-thrown storageFailure property; a spoofed flag must +remain an ordinary query error. The runtime marks storage failure and refuses +further commands. Preserve ownership/recovery evidence on failure. + +This corrects the migration design's failure classification, not the sandbox or +remote query language. The baseline tests are committed before the repair. diff --git a/.spiral/defects/DEF-20260923-TTZ7C-50.ttl b/.spiral/defects/DEF-20260923-TTZ7C-50.ttl new file mode 100644 index 0000000..3c4afa1 --- /dev/null +++ b/.spiral/defects/DEF-20260923-TTZ7C-50.ttl @@ -0,0 +1,11 @@ +@prefix sd: . +@prefix dcterms: . +@prefix project: . + +project:DEF-20260923-TTZ7C-50 + a sd:Defect ; + dcterms:identifier "DEF-20260923-TTZ7C-50" ; + sd:repositoryPath ".spiral/defects/DEF-20260923-TTZ7C-50.md" ; + sd:status sd:Active ; + sd:provenanceConfidence sd:Evidenced ; + sd:observes [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-33 ; sd:gitCommit "bfeed2c5c45cd3ac58dd42aac868585f31c59d19" ] . diff --git a/test/file-data.mjs b/test/file-data.mjs index 45c5017..77a8938 100644 --- a/test/file-data.mjs +++ b/test/file-data.mjs @@ -295,3 +295,49 @@ test('ID precedence follows record order when a patch fills an earlier hole', as const loaded = await loadFileData({dataFile: file, commands: ['fill']}) assert.equal(loaded.meta.index.id.get('duplicate'), 5) }) + +test('a lazy read failure during a command is a storage failure', async t => { + const { dir, file } = fixture(t) + const loaded = await loadFileData({dataFile: file, commands: []}) + const commandsFile = path.join(dir, 'commands.mjs') + fs.writeFileSync(commandsFile, `export default { + read(data) { data.items[1].name } + }`) + await initialize({ + ...loaded, datafile: file, commandsFile, + indexFile: fileURLToPath(new URL('../src/index.mjs', import.meta.url)) + }) + t.after(close) + fs.truncateSync(file, 0) + await assert.rejects(runCommand('{"id":"read","name":"read"}'), error => { + return error.storageFailure === true + }) +}) + +test('lazy query read failures stop mutation, while query-thrown flags do not', async t => { + let runtime + t.after(async () => runtime?.close()) + const fixture = await makeServerFixture(t, { + initialData: '{"persons":[{"name":"unread"}]}' + }) + runtime = await StoreRuntime.open({...fixture, maxWorkers: 1}) + const request = body => ({path: '/', body, jsontag: false}) + await runtime.runQuery(request('data.persons.length')) + // Finish both initializations before altering a disposable source file. + await runtime.runQuery(request('data.persons.length'), {slow: true}) + const spoofed = await runtime.runQuery(request( + 'throw {storageFailure: true}' + )) + assert.equal(spoofed.code, 422) + assert.equal(runtime.storageFailed, false) + fs.truncateSync(fixture.datafile, 0) + const result = await runtime.runQuery(request( + 'try { data.persons[0].name } catch (error) {} "caught"' + )) + assert.equal(result.code, 500) + assert.equal(runtime.storageFailed, true) + const accepted = await runtime.acceptCommand( + '{"id":"after-failure","name":"addPerson","value":{"name":"bad"}}' + ) + assert.equal(accepted.code, 503) +}) From faa3e896f40aa5af89b9e3216018a0bca659c48e Mon Sep 17 00:00:00 2001 From: Auke van Slooten Date: Wed, 23 Sep 2026 17:18:41 +0200 Subject: [PATCH 08/15] Stop mutation after lazy dataset read failures --- .../implementations/IMP-20260919-TTZ7C-26.md | 9 ++++++ .../implementations/IMP-20260919-TTZ7C-26.ttl | 6 ++-- .../implementations/IMP-20260919-TTZ7C-33.md | 9 ++++++ .../implementations/IMP-20260919-TTZ7C-33.ttl | 6 ++-- docs/file-data.md | 4 +++ src/command-worker-module.mjs | 9 ++++++ src/file-data.mjs | 26 +++++++++++++++- src/query-worker-module.mjs | 30 ++++++++++++++++--- src/store-runtime.mjs | 8 +++-- test/file-data.mjs | 6 +++- 10 files changed, 99 insertions(+), 14 deletions(-) diff --git a/.spiral/implementations/IMP-20260919-TTZ7C-26.md b/.spiral/implementations/IMP-20260919-TTZ7C-26.md index 09cfc1b..42c45a0 100644 --- a/.spiral/implementations/IMP-20260919-TTZ7C-26.md +++ b/.spiral/implementations/IMP-20260919-TTZ7C-26.md @@ -134,3 +134,12 @@ retained expected digests are checked against the resulting file bytes before replacement integrity and completion records. Existing administrative copy operations still buffer an individual file temporarily; this is outside the retrieval/replay memory claim. + +### Late read failures + +DEF-20260923-TTZ7C-50 corrects a failure-classification gap introduced by lazy +reads. FileParser retains read/decode/reference failures in host-owned state. +Commands refuse publication even if a handler catches the error; query workers +report storage failure even if query code catches it. User-thrown error flags +cannot forge parser state. Runtime marks storage failure and refuses further +mutations, preserving ownership and administrative recovery evidence. diff --git a/.spiral/implementations/IMP-20260919-TTZ7C-26.ttl b/.spiral/implementations/IMP-20260919-TTZ7C-26.ttl index 9a275cf..bee2dbc 100644 --- a/.spiral/implementations/IMP-20260919-TTZ7C-26.ttl +++ b/.spiral/implementations/IMP-20260919-TTZ7C-26.ttl @@ -34,6 +34,6 @@ project:IMP-20260919-TTZ7C-26 a sd:Implementation ; sd:implements [ a sd:ArtifactReference ; sd:artifact project:DES-20260917-TTZ7C-14 ; sd:gitCommit "59179b6c93b7e21c3b2c69db46c53fbe706927c3" ] ; sd:implements [ a sd:ArtifactReference ; sd:artifact project:DES-20260819-09ZEF-7 ; sd:gitCommit "59179b6c93b7e21c3b2c69db46c53fbe706927c3" ] ; sd:implements [ a sd:ArtifactReference ; sd:artifact project:DES-20260923-TTZ7C-47 ; sd:gitCommit "9b8acd229291fa75423c45a9019245edc0c19ea9" ] ; - sd:changeCausedBy [ a sd:ArtifactReference ; sd:artifact project:DES-20260923-TTZ7C-47 ; sd:gitCommit "9b8acd229291fa75423c45a9019245edc0c19ea9" ] ; - sd:transforms [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-26 ; sd:gitCommit "d4136eaf7b0014726a1c90a0d0b8488ece2d6b27" ] ; - sd:implementationChangeKind sd:MixedChange . + sd:changeCausedBy [ a sd:ArtifactReference ; sd:artifact project:DEF-20260923-TTZ7C-50 ; sd:gitCommit "4bb10c2013fd2a7cb989b02586b992b936121362" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-26 ; sd:gitCommit "bfeed2c5c45cd3ac58dd42aac868585f31c59d19" ] ; + sd:implementationChangeKind sd:SemanticChange . diff --git a/.spiral/implementations/IMP-20260919-TTZ7C-33.md b/.spiral/implementations/IMP-20260919-TTZ7C-33.md index 388b626..8680fcb 100644 --- a/.spiral/implementations/IMP-20260919-TTZ7C-33.md +++ b/.spiral/implementations/IMP-20260919-TTZ7C-33.md @@ -127,3 +127,12 @@ remote results, recovery eligibility, ordering and failure expectations remain. New tests cover unread IDs, sparse overlays, corrupt framing, short reads/EOF, file changes and closure, byte-free messages, index loaders, real runtime restart, failed publication, worker replacement, and asynchronous update ordering. + +### Late read failures + +DEF-20260923-TTZ7C-50 corrects a failure-classification gap introduced by lazy +reads. FileParser retains read/decode/reference failures in host-owned state. +Commands refuse publication even if a handler catches the error; query workers +report storage failure even if query code catches it. User-thrown error flags +cannot forge parser state. Runtime marks storage failure and refuses further +mutations, preserving ownership and administrative recovery evidence. diff --git a/.spiral/implementations/IMP-20260919-TTZ7C-33.ttl b/.spiral/implementations/IMP-20260919-TTZ7C-33.ttl index 2919a54..34bf8f5 100644 --- a/.spiral/implementations/IMP-20260919-TTZ7C-33.ttl +++ b/.spiral/implementations/IMP-20260919-TTZ7C-33.ttl @@ -31,6 +31,6 @@ project:IMP-20260919-TTZ7C-33 sd:implementationLocation "test/file-data.mjs" ; sd:implementationLocation "scripts/benchmark-file-data.mjs" ; sd:implements [ a sd:ArtifactReference ; sd:artifact project:DES-20260923-TTZ7C-47 ; sd:gitCommit "9b8acd229291fa75423c45a9019245edc0c19ea9" ] ; - sd:changeCausedBy [ a sd:ArtifactReference ; sd:artifact project:DES-20260923-TTZ7C-47 ; sd:gitCommit "9b8acd229291fa75423c45a9019245edc0c19ea9" ] ; - sd:transforms [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-33 ; sd:gitCommit "5d9e81218d5a51d8d3171f7132e73c2e4bfc298a" ] ; - sd:implementationChangeKind sd:MixedChange . + sd:changeCausedBy [ a sd:ArtifactReference ; sd:artifact project:DEF-20260923-TTZ7C-50 ; sd:gitCommit "4bb10c2013fd2a7cb989b02586b992b936121362" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-33 ; sd:gitCommit "bfeed2c5c45cd3ac58dd42aac868585f31c59d19" ] ; + sd:implementationChangeKind sd:SemanticChange . diff --git a/docs/file-data.md b/docs/file-data.md index 68b9695..ef5cc59 100644 --- a/docs/file-data.md +++ b/docs/file-data.md @@ -47,6 +47,10 @@ worker termination also closes descriptors opened through Node's tracked file APIs. Runtime shutdown waits for both pools to terminate before releasing store ownership. A failed source update makes the pool unavailable and stops further command acceptance rather than allowing the worker to skip an update. +Lazy read/decode/reference failures are recorded in host-owned parser state. +Even if a command or query catches the error, publication is refused and the +runtime stops accepting mutations. Query code cannot forge this state by throwing +an object with a `storageFailure` property. Administrative replay opens file sources too. `inspectStore()` now returns a plain `sources` array instead of live `data` proxies and retained `buffers`. diff --git a/src/command-worker-module.mjs b/src/command-worker-module.mjs index be1f806..3732dbe 100644 --- a/src/command-worker-module.mjs +++ b/src/command-worker-module.mjs @@ -117,6 +117,9 @@ export default async function runCommand(commandStr) { if (commands[task.name]) { let time = Date.now() await commands[task.name](dataspace, task, undefined, metaProxy) + if (parser.readFailure) { + throw parser.readFailure + } // TODO: if command/task makes no changes, skip updating // data.jsontag and writing it. @@ -126,6 +129,9 @@ export default async function runCommand(commandStr) { changes.uuid = task.id await index.update(dataspace, meta, changes) } + if (parser.readFailure) { + throw parser.readFailure + } // Serialize only changes. const serialized = Buffer.concat([ ...serializeChunks(dataspace, { meta, changes: true }) @@ -172,6 +178,9 @@ export default async function runCommand(commandStr) { } catch (err) { console.error('task error', err) + if (parser.readFailure) { + throw parser.readFailure + } throw publishing ? storageError(err) : err } return response diff --git a/src/file-data.mjs b/src/file-data.mjs index 1591fa3..5f2caa4 100644 --- a/src/file-data.mjs +++ b/src/file-data.mjs @@ -5,6 +5,7 @@ import JSONTag from '@muze-nl/jsontag' import Parser from '@muze-nl/od-jsontag/src/parse.mjs' import { RecoveryIntegrityError, assertChangesetExists } from './recovery.mjs' import { loadIntegrityManifest, verifyDigest } from './integrity.mjs' +import { storageError } from './storage.mjs' function identity(stat) { return { @@ -157,8 +158,31 @@ export function scanDataFile(filename) { } } +// Keep read failure in host-owned state even when a handler/query catches it. +export class FileParser extends Parser { + firstParse(target) { + try { + return super.firstParse(target) + } + catch (error) { + this.readFailure = storageError(error) + throw this.readFailure + } + } + + getLineProxy(index) { + try { + return super.getLineProxy(index) + } + catch (error) { + this.readFailure = storageError(error) + throw this.readFailure + } + } +} + export class FileDataset { - constructor(meta = {}, immutable = true, ParserType = Parser) { + constructor(meta = {}, immutable = true, ParserType = FileParser) { this.parser = new ParserType(undefined, immutable) this.parser.meta = { ...meta, resultArray: [] } this.handles = [] diff --git a/src/query-worker-module.mjs b/src/query-worker-module.mjs index 91f780b..fe41f7d 100644 --- a/src/query-worker-module.mjs +++ b/src/query-worker-module.mjs @@ -2,15 +2,14 @@ import {VM} from 'vm2' import { memoryUsage } from 'node:process' import JSONTag from '@muze-nl/jsontag' import {source} from '@muze-nl/od-jsontag/src/symbols.mjs' -import { FileDataset } from './file-data.mjs' -import Parser from '@muze-nl/od-jsontag/src/parse.mjs' +import { FileDataset, FileParser } from './file-data.mjs' import {_,from,not,anyOf,allOf,asc,desc,sum,count,avg,max,min,many,one,first,distinct} from '@muze-nl/jaqt' import process from 'node:process' // vm2 protects array species before host calls. Give it the already-safe // constructor descriptor on raw query arrays before the read-only proxy exists, // so methods like map/filter need no temporary mutation through that proxy. -class QueryParser extends Parser { +class QueryParser extends FileParser { getArrayProxy(array, parent) { Object.defineProperty(array, 'constructor', { value: undefined, @@ -75,7 +74,30 @@ const tasks = { return true }, query: async (task) => { - return runQuery(task.req.path, task.req, task.req.body, task.timeout) + let response + try { + response = runQuery( + task.req.path, task.req, task.req.body, task.timeout + ) + } + catch (error) { + if (!parser.readFailure) { + throw error + } + } + // Query code can catch errors or throw its own properties. Only the + // parser's host-owned failure state identifies a storage problem. + if (parser.readFailure) { + const error = {code: 500, message: 'Unable to read committed data'} + return { + code: 500, + storageFailure: true, + body: task.req.jsontag + ? JSONTag.stringify(error) + : JSON.stringify(error) + } + } + return response }, memoryUsage: async () => { let result = memoryUsage() diff --git a/src/store-runtime.mjs b/src/store-runtime.mjs index e636826..eb087f3 100644 --- a/src/store-runtime.mjs +++ b/src/store-runtime.mjs @@ -278,14 +278,18 @@ export class StoreRuntime { } } - runQuery(request, { slow = false } = {}) { + async runQuery(request, { slow = false } = {}) { let pool = this.queryWorkerPool let timeout = this.configuration.timeout if (slow) { pool = this.slowQueryWorkerPool timeout = this.configuration.slowTimeout } - return pool.run('query', request, { timeout }) + const result = await pool.run('query', request, { timeout }) + if (result.storageFailure) { + this.failStorage(new Error('Unable to read committed data')) + } + return result } getCommandStatus(commandId) { diff --git a/test/file-data.mjs b/test/file-data.mjs index 77a8938..1d8679e 100644 --- a/test/file-data.mjs +++ b/test/file-data.mjs @@ -301,7 +301,10 @@ test('a lazy read failure during a command is a storage failure', async t => { const loaded = await loadFileData({dataFile: file, commands: []}) const commandsFile = path.join(dir, 'commands.mjs') fs.writeFileSync(commandsFile, `export default { - read(data) { data.items[1].name } + read(data) { + try { data.items[1].name } catch (error) {} + data.changedAfterFailure = true + } }`) await initialize({ ...loaded, datafile: file, commandsFile, @@ -312,6 +315,7 @@ test('a lazy read failure during a command is a storage failure', async t => { await assert.rejects(runCommand('{"id":"read","name":"read"}'), error => { return error.storageFailure === true }) + assert.equal(fs.existsSync(path.join(dir, 'data.read.jsontag')), false) }) test('lazy query read failures stop mutation, while query-thrown flags do not', async t => { From cdc1564744989c955a187fdce4baed22ec5d0fd7 Mon Sep 17 00:00:00 2001 From: Auke van Slooten Date: Wed, 23 Sep 2026 17:19:48 +0200 Subject: [PATCH 09/15] Preserve negotiated format for dataset read errors --- .spiral/implementations/IMP-20260919-TTZ7C-33.md | 2 ++ .spiral/implementations/IMP-20260919-TTZ7C-33.ttl | 2 +- src/query-worker-module.mjs | 1 + test/file-data.mjs | 5 +++++ 4 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.spiral/implementations/IMP-20260919-TTZ7C-33.md b/.spiral/implementations/IMP-20260919-TTZ7C-33.md index 8680fcb..6bc98fb 100644 --- a/.spiral/implementations/IMP-20260919-TTZ7C-33.md +++ b/.spiral/implementations/IMP-20260919-TTZ7C-33.md @@ -136,3 +136,5 @@ Commands refuse publication even if a handler catches the error; query workers report storage failure even if query code catches it. User-thrown error flags cannot forge parser state. Runtime marks storage failure and refuses further mutations, preserving ownership and administrative recovery evidence. + +Storage read error responses retain the negotiated JSON/JSONTag format. diff --git a/.spiral/implementations/IMP-20260919-TTZ7C-33.ttl b/.spiral/implementations/IMP-20260919-TTZ7C-33.ttl index 34bf8f5..041a409 100644 --- a/.spiral/implementations/IMP-20260919-TTZ7C-33.ttl +++ b/.spiral/implementations/IMP-20260919-TTZ7C-33.ttl @@ -32,5 +32,5 @@ project:IMP-20260919-TTZ7C-33 sd:implementationLocation "scripts/benchmark-file-data.mjs" ; sd:implements [ a sd:ArtifactReference ; sd:artifact project:DES-20260923-TTZ7C-47 ; sd:gitCommit "9b8acd229291fa75423c45a9019245edc0c19ea9" ] ; sd:changeCausedBy [ a sd:ArtifactReference ; sd:artifact project:DEF-20260923-TTZ7C-50 ; sd:gitCommit "4bb10c2013fd2a7cb989b02586b992b936121362" ] ; - sd:transforms [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-33 ; sd:gitCommit "bfeed2c5c45cd3ac58dd42aac868585f31c59d19" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-33 ; sd:gitCommit "faa3e896f40aa5af89b9e3216018a0bca659c48e" ] ; sd:implementationChangeKind sd:SemanticChange . diff --git a/src/query-worker-module.mjs b/src/query-worker-module.mjs index fe41f7d..84ce8a9 100644 --- a/src/query-worker-module.mjs +++ b/src/query-worker-module.mjs @@ -92,6 +92,7 @@ const tasks = { return { code: 500, storageFailure: true, + jsontag: task.req.jsontag, body: task.req.jsontag ? JSONTag.stringify(error) : JSON.stringify(error) diff --git a/test/file-data.mjs b/test/file-data.mjs index 1d8679e..a74b98b 100644 --- a/test/file-data.mjs +++ b/test/file-data.mjs @@ -340,6 +340,11 @@ test('lazy query read failures stop mutation, while query-thrown flags do not', )) assert.equal(result.code, 500) assert.equal(runtime.storageFailed, true) + const tagged = await runtime.runQuery({ + ...request('42'), jsontag: true + }) + assert.equal(tagged.jsontag, true) + assert.equal(JSONTag.parse(tagged.body).code, 500) const accepted = await runtime.acceptCommand( '{"id":"after-failure","name":"addPerson","value":{"name":"bad"}}' ) From 829a8dd81da5447c8e113a73312952a50d1c037d Mon Sep 17 00:00:00 2001 From: Auke van Slooten Date: Wed, 23 Sep 2026 17:21:05 +0200 Subject: [PATCH 10/15] Record verified late-read correction in migration evaluation --- .spiral/cycles/CYC-20260923-TTZ7C-44.md | 9 +++++++-- .spiral/cycles/CYC-20260923-TTZ7C-44.ttl | 2 +- .spiral/defects/DEF-20260923-TTZ7C-50.md | 7 +++++++ .spiral/defects/DEF-20260923-TTZ7C-50.ttl | 4 +++- .spiral/evidence/EVD-20260923-TTZ7C-49.md | 22 +++++++++++++++++++--- .spiral/evidence/EVD-20260923-TTZ7C-49.ttl | 7 ++++--- .spiral/project-context.md | 2 +- .spiral/project.ttl | 2 +- 8 files changed, 43 insertions(+), 12 deletions(-) diff --git a/.spiral/cycles/CYC-20260923-TTZ7C-44.md b/.spiral/cycles/CYC-20260923-TTZ7C-44.md index b07d66a..81ea7fd 100644 --- a/.spiral/cycles/CYC-20260923-TTZ7C-44.md +++ b/.spiral/cycles/CYC-20260923-TTZ7C-44.md @@ -21,8 +21,13 @@ Client version metadata/audit history and sandbox replacement remain later work. ## Evaluation -Implementation is complete at `bfeed2c5c45cd3ac58dd42aac868585f31c59d19`. EVD-20260923-TTZ7C-49 -records 140 passing tests, clean focused lint, fresh-package installation, a +Implementation is complete at `cdc1564744989c955a187fdce4baed22ec5d0fd7`. EVD-20260923-TTZ7C-49 +records 142 passing tests, clean focused lint, fresh-package installation, a 329 MB/48 MB-heap runtime probe and read-only curriculum evidence. See [the integration guide](../../docs/file-data.md) for the source contract and limits. Human acceptance remains pending; the cycle stays Active. + +Final review added DEF-20260923-TTZ7C-50 and two reproducing tests. The correction +stops mutation after late lazy-read failures, including caught errors, and keeps +query-supplied flags from masquerading as parser failures. EVD-49 records the +full 142-test pass and the final 23-test response-format check. diff --git a/.spiral/cycles/CYC-20260923-TTZ7C-44.ttl b/.spiral/cycles/CYC-20260923-TTZ7C-44.ttl index fc4825b..7db27c2 100644 --- a/.spiral/cycles/CYC-20260923-TTZ7C-44.ttl +++ b/.spiral/cycles/CYC-20260923-TTZ7C-44.ttl @@ -8,4 +8,4 @@ project:CYC-20260923-TTZ7C-44 sd:repositoryPath ".spiral/cycles/CYC-20260923-TTZ7C-44.md" ; sd:status sd:Active ; sd:derivedFrom [ a sd:ArtifactReference ; sd:artifact project:REQ-20260923-TTZ7C-46 ; sd:gitCommit "a75db831ee6779bbf297e94b9d1baf694692cb84" ] ; - sd:transforms [ a sd:ArtifactReference ; sd:artifact project:CYC-20260923-TTZ7C-44 ; sd:gitCommit "8299fcc8b8fcf113091ae85bbcf11f867f8c5908" ] . + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:CYC-20260923-TTZ7C-44 ; sd:gitCommit "a64b81373b2ef3d8252b400d7a18b9242038adb2" ] . diff --git a/.spiral/defects/DEF-20260923-TTZ7C-50.md b/.spiral/defects/DEF-20260923-TTZ7C-50.md index 11bec5e..395dfe0 100644 --- a/.spiral/defects/DEF-20260923-TTZ7C-50.md +++ b/.spiral/defects/DEF-20260923-TTZ7C-50.md @@ -26,3 +26,10 @@ further commands. Preserve ownership/recovery evidence on failure. This corrects the migration design's failure classification, not the sandbox or remote query language. The baseline tests are committed before the repair. + +## Correction + +Implemented at faa3e89 and refined at cdc1564. EVD-20260923-TTZ7C-49 records +the full 142-test pass and focused response-format verification. The tests also +show that swallowing the read error cannot permit command publication, and +query-thrown error properties cannot trigger host storage-failure state. diff --git a/.spiral/defects/DEF-20260923-TTZ7C-50.ttl b/.spiral/defects/DEF-20260923-TTZ7C-50.ttl index 3c4afa1..e6bd4b3 100644 --- a/.spiral/defects/DEF-20260923-TTZ7C-50.ttl +++ b/.spiral/defects/DEF-20260923-TTZ7C-50.ttl @@ -8,4 +8,6 @@ project:DEF-20260923-TTZ7C-50 sd:repositoryPath ".spiral/defects/DEF-20260923-TTZ7C-50.md" ; sd:status sd:Active ; sd:provenanceConfidence sd:Evidenced ; - sd:observes [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-33 ; sd:gitCommit "bfeed2c5c45cd3ac58dd42aac868585f31c59d19" ] . + sd:observes [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-33 ; sd:gitCommit "bfeed2c5c45cd3ac58dd42aac868585f31c59d19" ] ; + sd:observes [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-33 ; sd:gitCommit "cdc1564744989c955a187fdce4baed22ec5d0fd7" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:DEF-20260923-TTZ7C-50 ; sd:gitCommit "4bb10c2013fd2a7cb989b02586b992b936121362" ] . diff --git a/.spiral/evidence/EVD-20260923-TTZ7C-49.md b/.spiral/evidence/EVD-20260923-TTZ7C-49.md index 6f51abf..c52ec35 100644 --- a/.spiral/evidence/EVD-20260923-TTZ7C-49.md +++ b/.spiral/evidence/EVD-20260923-TTZ7C-49.md @@ -4,7 +4,7 @@ id: EVD-20260923-TTZ7C-49 # File-backed runtime evaluation -Implementation: `bfeed2c5c45cd3ac58dd42aac868585f31c59d19` on +Implementation: `cdc1564744989c955a187fdce4baed22ec5d0fd7` on `spiral/CYC-20260923-TTZ7C-44-file-backed`. ## Evaluation @@ -15,8 +15,8 @@ ordered file sources. The remote query/command API and stored data/log formats remain unchanged. No production data, deployment, merge or release was changed. Existing untracked example indexes and package backup were left untouched. -- Full regression command below: **140 tests pass**, zero failures/cancellations - (118 existing cases plus 22 new). Runtime/startup, conversion and offsets, +- Full regression command below: **142 tests pass**, zero failures/cancellations + (118 existing cases plus 24 new). Runtime/startup, conversion and offsets, ACID, process crashes, handlers, power-loss baseline, storage and administrative recovery remain covered. Adaptations to old tests change internal source/report fixtures; existing remote behavior and durability assertions remain. @@ -60,6 +60,22 @@ node --test --test-reporter=tap test/file-data.mjs test/store-runtime.mjs \ node --expose-gc --max-old-space-size=48 scripts/benchmark-file-data.mjs ``` +## Final read-failure correction + +DEF-20260923-TTZ7C-50 records two failing regressions introduced during final +review. Late lazy reads could be treated as ordinary command errors or swallowed +inside query code. FileParser now records read/decode/reference failure in +host-owned session state. Commands refuse publication even when a handler catches +the failure; query workers signal it to the runtime even when query code catches +it. Forged user-thrown storageFailure properties do not change that state. +The runtime stops mutation and retains ownership/recovery evidence. + +The complete suite passes 142 tests after that correction (faa3e89). The final +response-format refinement (cdc1564) additionally passes all 23 focused file-data +tests, including JSONTag error negotiation. Focused lint remains clean. Earlier +memory, curriculum and package probes establish the unchanged file-source and +distribution contracts; they are not separate security or hardware claims. + ## Boundaries and review disposition Startup hashes all canonical bytes and reconstructs IDs by decoding final diff --git a/.spiral/evidence/EVD-20260923-TTZ7C-49.ttl b/.spiral/evidence/EVD-20260923-TTZ7C-49.ttl index 1b7da56..b4b8194 100644 --- a/.spiral/evidence/EVD-20260923-TTZ7C-49.ttl +++ b/.spiral/evidence/EVD-20260923-TTZ7C-49.ttl @@ -7,6 +7,7 @@ project:EVD-20260923-TTZ7C-49 dcterms:identifier "EVD-20260923-TTZ7C-49" ; sd:repositoryPath ".spiral/evidence/EVD-20260923-TTZ7C-49.md" ; sd:status sd:Active ; - sd:verifies [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-26 ; sd:gitCommit "bfeed2c5c45cd3ac58dd42aac868585f31c59d19" ] ; - sd:verifies [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-33 ; sd:gitCommit "bfeed2c5c45cd3ac58dd42aac868585f31c59d19" ] ; - sd:verifies [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-34 ; sd:gitCommit "bfeed2c5c45cd3ac58dd42aac868585f31c59d19" ] . + sd:verifies [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-26 ; sd:gitCommit "cdc1564744989c955a187fdce4baed22ec5d0fd7" ] ; + sd:verifies [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-33 ; sd:gitCommit "cdc1564744989c955a187fdce4baed22ec5d0fd7" ] ; + sd:verifies [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-34 ; sd:gitCommit "cdc1564744989c955a187fdce4baed22ec5d0fd7" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:EVD-20260923-TTZ7C-49 ; sd:gitCommit "a64b81373b2ef3d8252b400d7a18b9242038adb2" ] . diff --git a/.spiral/project-context.md b/.spiral/project-context.md index 3616989..bdd8db3 100644 --- a/.spiral/project-context.md +++ b/.spiral/project-context.md @@ -24,7 +24,7 @@ shared-memory data after reviewing od-jsontag hardening. This continues the file-only roadmap direction, prioritized ahead of rebuild. SRC-45, UND-48, REQ-46 and DES-47 (20260923-TTZ7C namespace) retain the commitment. Implementation uses ordered file sources, bounded clean-record caches, worker-owned handles, -streamed framing/hash scans and file-backed administrative replay. EVD-20260923-TTZ7C-49 records 140 passing tests, clean lint, a fresh-package +streamed framing/hash scans and file-backed administrative replay. EVD-20260923-TTZ7C-49 records 142 passing tests, clean lint, a fresh-package installation and large-file/real-data probes. Human acceptance remains pending. Client-visible version metadata and audit trails follow this migration; no such feature is implemented by this cycle. diff --git a/.spiral/project.ttl b/.spiral/project.ttl index 09baea9..bd3239d 100644 --- a/.spiral/project.ttl +++ b/.spiral/project.ttl @@ -16,7 +16,7 @@ project:CTX-001 sd:gitCommit "79fcf670d4558c75b9933743e6f2ea5616221eb0" ] ; sd:derivedFrom [ a sd:ArtifactReference ; sd:artifact project:SRC-20260923-TTZ7C-45 ; sd:gitCommit "8299fcc8b8fcf113091ae85bbcf11f867f8c5908" ] ; - sd:transforms [ a sd:ArtifactReference ; sd:artifact project:CTX-001 ; sd:gitCommit "bfeed2c5c45cd3ac58dd42aac868585f31c59d19" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:CTX-001 ; sd:gitCommit "a64b81373b2ef3d8252b400d7a18b9242038adb2" ] ; sd:developerEvaluationReadinessBar "Developers should be able to evaluate bounded durability claims from invariants, executable tests, evidence artifacts, explicit failure behavior, and documented known gaps." ; sd:hasAudience project:AUD-DEVELOPERS, project:AUD-END-USERS, project:AUD-OPERATORS ; sd:hasDownstreamProject project:DOWNSTREAM-CURRICULUM-STORE ; From 91be30fdf8351afe1003e8fa8dcb7b67302341c2 Mon Sep 17 00:00:00 2001 From: Auke van Slooten Date: Wed, 23 Sep 2026 17:41:48 +0200 Subject: [PATCH 11/15] Record persisted-index completion and maintainer example check --- .spiral/sources/SRC-20260923-TTZ7C-45.md | 12 ++++++++++++ .spiral/sources/SRC-20260923-TTZ7C-45.ttl | 3 ++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.spiral/sources/SRC-20260923-TTZ7C-45.md b/.spiral/sources/SRC-20260923-TTZ7C-45.md index d73e7dd..17895cf 100644 --- a/.spiral/sources/SRC-20260923-TTZ7C-45.md +++ b/.spiral/sources/SRC-20260923-TTZ7C-45.md @@ -22,3 +22,15 @@ Reviewed dependency: od-jsontag commit 24f47eb616c3a24391487197496804ed1c56837b, available on its upstream indexedParsing branch. The library keeps format/API compatibility and has bounded clean-record caching; indexes and mutable sessions still grow with their contents. + +## Persisted-index completion, 2026-09-23 + +After the assistant explained that startup still scanned canonical files and +rebuilt IDs, and that persisted index loading with validation/fallback remained +unimplemented, the maintainer replied: + +> I think that needs to be in this version. I verified that SimplyStore still works with the example folder. + +This authorizes finishing persisted offset/ID loading in the active file-backed +cycle. The example check is maintainer-reported evidence, not acceptance of the +still-incomplete cycle or an independently reproduced agent test. diff --git a/.spiral/sources/SRC-20260923-TTZ7C-45.ttl b/.spiral/sources/SRC-20260923-TTZ7C-45.ttl index 0f37b63..0b859b7 100644 --- a/.spiral/sources/SRC-20260923-TTZ7C-45.ttl +++ b/.spiral/sources/SRC-20260923-TTZ7C-45.ttl @@ -8,4 +8,5 @@ project:SRC-20260923-TTZ7C-45 sd:repositoryPath ".spiral/sources/SRC-20260923-TTZ7C-45.md" ; sd:status sd:Active ; sd:sourceAvailability sd:Retained ; - sd:provenanceConfidence sd:Explicit . + sd:provenanceConfidence sd:Explicit ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:SRC-20260923-TTZ7C-45 ; sd:gitCommit "829a8dd81da5447c8e113a73312952a50d1c037d" ] . From b279aefe78a262a041b59dff2e2c199233fbcf7d Mon Sep 17 00:00:00 2001 From: Auke van Slooten Date: Wed, 23 Sep 2026 17:42:37 +0200 Subject: [PATCH 12/15] Require validated persisted indexes in the file-backed cycle --- .spiral/requests/REQ-20260923-TTZ7C-46.md | 12 ++++++++++++ .spiral/requests/REQ-20260923-TTZ7C-46.ttl | 4 +++- .spiral/understandings/UND-20260923-TTZ7C-48.md | 8 ++++++++ .spiral/understandings/UND-20260923-TTZ7C-48.ttl | 4 +++- 4 files changed, 26 insertions(+), 2 deletions(-) diff --git a/.spiral/requests/REQ-20260923-TTZ7C-46.md b/.spiral/requests/REQ-20260923-TTZ7C-46.md index c60c9f8..b9f212a 100644 --- a/.spiral/requests/REQ-20260923-TTZ7C-46.md +++ b/.spiral/requests/REQ-20260923-TTZ7C-46.md @@ -20,3 +20,15 @@ Evidence: existing regression suite plus direct file lifecycle, partial-ID-read, sparse update, worker publication/replacement, malformed framing/integrity and large-file memory tests. Internal worker messages may change. New dependency revision must be reproducible. No sandbox/audit/rebuild feature in this cycle. + +## Required persisted-index completion + +Load index.offset.json and index.id.json and their committed-command variants +in startup. Validate against canonical bytes; derive replacements for absent, +malformed, incomplete or stale sidecars. Preserve sparse overlay ordering and +ID removal/rename/duplicate precedence. Avoid decoding every complete record +merely to check IDs. Read-only loading must not rewrite data or sidecars. + +This directly implements the maintainer's explicit same-version correction +retained in SRC-20260923-TTZ7C-45; the previous reconstruction-only behavior is +insufficient. diff --git a/.spiral/requests/REQ-20260923-TTZ7C-46.ttl b/.spiral/requests/REQ-20260923-TTZ7C-46.ttl index 39c021d..e53402e 100644 --- a/.spiral/requests/REQ-20260923-TTZ7C-46.ttl +++ b/.spiral/requests/REQ-20260923-TTZ7C-46.ttl @@ -7,4 +7,6 @@ project:REQ-20260923-TTZ7C-46 dcterms:identifier "REQ-20260923-TTZ7C-46" ; sd:repositoryPath ".spiral/requests/REQ-20260923-TTZ7C-46.md" ; sd:status sd:Active ; - sd:derivedFrom [ a sd:ArtifactReference ; sd:artifact project:UND-20260923-TTZ7C-48 ; sd:gitCommit "c276b42cbe89a3c7204dd26f1c66522e59553b90" ] . + sd:derivedFrom [ a sd:ArtifactReference ; sd:artifact project:UND-20260923-TTZ7C-48 ; sd:gitCommit "c276b42cbe89a3c7204dd26f1c66522e59553b90" ] ; + sd:changeCausedBy [ a sd:ArtifactReference ; sd:artifact project:SRC-20260923-TTZ7C-45 ; sd:gitCommit "91be30fdf8351afe1003e8fa8dcb7b67302341c2" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:REQ-20260923-TTZ7C-46 ; sd:gitCommit "91be30fdf8351afe1003e8fa8dcb7b67302341c2" ] . diff --git a/.spiral/understandings/UND-20260923-TTZ7C-48.md b/.spiral/understandings/UND-20260923-TTZ7C-48.md index e563ef3..bdaa63d 100644 --- a/.spiral/understandings/UND-20260923-TTZ7C-48.md +++ b/.spiral/understandings/UND-20260923-TTZ7C-48.md @@ -20,3 +20,11 @@ startup integrity scanning remains necessary, but can use bounded byte buffers. The human's instruction follows the concrete integration explanation and is the commitment boundary; no second permission ceremony is needed. + +## Confirmed completion within this version + +The maintainer's response to the explicit persisted-index gap authorizes using +base and committed changeset offset/ID sidecars in this same cycle, with +validation and fallback. loadFileData currently never calls their loaders. +The example-folder check is supporting human feedback; cycle acceptance remains +pending. Preserve canonical framing and remote APIs. diff --git a/.spiral/understandings/UND-20260923-TTZ7C-48.ttl b/.spiral/understandings/UND-20260923-TTZ7C-48.ttl index d17b831..8e9a168 100644 --- a/.spiral/understandings/UND-20260923-TTZ7C-48.ttl +++ b/.spiral/understandings/UND-20260923-TTZ7C-48.ttl @@ -7,4 +7,6 @@ project:UND-20260923-TTZ7C-48 dcterms:identifier "UND-20260923-TTZ7C-48" ; sd:repositoryPath ".spiral/understandings/UND-20260923-TTZ7C-48.md" ; sd:status sd:Accepted ; - sd:interprets [ a sd:ArtifactReference ; sd:artifact project:SRC-20260923-TTZ7C-45 ; sd:gitCommit "8299fcc8b8fcf113091ae85bbcf11f867f8c5908" ] . + sd:interprets [ a sd:ArtifactReference ; sd:artifact project:SRC-20260923-TTZ7C-45 ; sd:gitCommit "8299fcc8b8fcf113091ae85bbcf11f867f8c5908" ] ; + sd:changeCausedBy [ a sd:ArtifactReference ; sd:artifact project:SRC-20260923-TTZ7C-45 ; sd:gitCommit "91be30fdf8351afe1003e8fa8dcb7b67302341c2" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:UND-20260923-TTZ7C-48 ; sd:gitCommit "91be30fdf8351afe1003e8fa8dcb7b67302341c2" ] . From df973f7b6912f0a73b4b5980e1ede04a3b983d56 Mon Sep 17 00:00:00 2001 From: Auke van Slooten Date: Wed, 23 Sep 2026 17:43:30 +0200 Subject: [PATCH 13/15] Design persisted-index validation from framing and record headers --- .spiral/designs/DES-20260923-TTZ7C-47.md | 27 ++++++++++++++++++----- .spiral/designs/DES-20260923-TTZ7C-47.ttl | 3 ++- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/.spiral/designs/DES-20260923-TTZ7C-47.md b/.spiral/designs/DES-20260923-TTZ7C-47.md index 2096fbf..8f79698 100644 --- a/.spiral/designs/DES-20260923-TTZ7C-47.md +++ b/.spiral/designs/DES-20260923-TTZ7C-47.md @@ -9,12 +9,27 @@ open descriptors or dataset bytes. A small file-reader session owns handles and an od-jsontag parser; queries reuse it, commands open the committed prefix for one execution, and all exits close handles. Resolve IDs through getLineProxy. -Scan canonical OD-JSONTag framing incrementally while hashing to derive exact -sparse offsets; these derived indexes avoid requiring trusted/up-to-date -sidecars. Existing persisted sidecars remain produced and their loaders are -corrected. Reconstruct IDs from final live records with bounded clean caching, -without invoking user index hooks during inspection. Source maps and IDs remain -O(record count). Verify sources against integrity manifests before publication. +Load default offset/ID sidecars for the base and log-authoritative committed +commands. Keep the fixed-buffer framing/hash scan: legacy sidecars have no +cryptographic binding to canonical bytes. Validate every persisted offset range +and record key against framing, and pass the loaded offsets to od-jsontag only +when the complete map agrees. Missing, malformed or stale maps use derived +offsets. Do not import custom index code during inspection/loading. + +During the same scan, collect only each record's leading JSONTag metadata, +respecting quoted/escaped delimiters, and use JSONTag's own tag parser for IDs. +Skip record bodies. Merge per-record IDs in committed source order, then apply +ascending record-number precedence. Compare the composed persisted ID map with +this complete final map; use persisted IDs when equal and reconstructed IDs +otherwise. This handles removed/renamed IDs, duplicate IDs and sparse holes +without trusting an incomplete sidecar. Unusual reference-only records use the +existing parser reconstruction fallback. Lazy body errors remain storage errors +when read; startup validates framing, headers and configured full-file integrity, +not all record-body syntax. + +Read-only load never rewrites sidecars or requires a new format/cache certificate. +The scan remains O(file bytes), and catalogs/IDs O(record count); the improvement +is avoiding full live-record decoding, not skipping integrity reads. Startup inspection returns plain file sources instead of retained data/buffers. Administrative recovery uses the same loader/session. Temporary command output diff --git a/.spiral/designs/DES-20260923-TTZ7C-47.ttl b/.spiral/designs/DES-20260923-TTZ7C-47.ttl index 6a21ba6..234a1cb 100644 --- a/.spiral/designs/DES-20260923-TTZ7C-47.ttl +++ b/.spiral/designs/DES-20260923-TTZ7C-47.ttl @@ -7,4 +7,5 @@ project:DES-20260923-TTZ7C-47 dcterms:identifier "DES-20260923-TTZ7C-47" ; sd:repositoryPath ".spiral/designs/DES-20260923-TTZ7C-47.md" ; sd:status sd:Active ; - sd:satisfies [ a sd:ArtifactReference ; sd:artifact project:REQ-20260923-TTZ7C-46 ; sd:gitCommit "a75db831ee6779bbf297e94b9d1baf694692cb84" ] . + sd:satisfies [ a sd:ArtifactReference ; sd:artifact project:REQ-20260923-TTZ7C-46 ; sd:gitCommit "b279aefe78a262a041b59dff2e2c199233fbcf7d" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:DES-20260923-TTZ7C-47 ; sd:gitCommit "b279aefe78a262a041b59dff2e2c199233fbcf7d" ] . From d9d8ff860ed2b74585ea66454382c9baf4a09250 Mon Sep 17 00:00:00 2001 From: Auke van Slooten Date: Wed, 23 Sep 2026 17:48:28 +0200 Subject: [PATCH 14/15] Load validated persisted indexes without decoding every record --- .../implementations/IMP-20260919-TTZ7C-33.md | 14 ++ .../implementations/IMP-20260919-TTZ7C-33.ttl | 4 +- docs/file-data.md | 48 +++-- scripts/benchmark-file-data.mjs | 26 ++- src/file-data.mjs | 120 ++++++++++++- test/file-data.mjs | 170 +++++++++++++++++- 6 files changed, 351 insertions(+), 31 deletions(-) diff --git a/.spiral/implementations/IMP-20260919-TTZ7C-33.md b/.spiral/implementations/IMP-20260919-TTZ7C-33.md index 6bc98fb..41b3463 100644 --- a/.spiral/implementations/IMP-20260919-TTZ7C-33.md +++ b/.spiral/implementations/IMP-20260919-TTZ7C-33.md @@ -138,3 +138,17 @@ cannot forge parser state. Runtime marks storage failure and refuses further mutations, preserving ownership and administrative recovery evidence. Storage read error responses retain the negotiated JSON/JSONTag format. + +## Persisted index completion + +The loader reads base and committed-command offset/ID sidecars. Complete offset +maps must match canonical framing. Composed persisted IDs must match final +record-header IDs; otherwise loading reconstructs the map. The scan validates +metadata through JSONTag's tag parser and attribute API without decoding every +body. Sparse overlays remove/rename IDs and preserve higher-record duplicate +precedence. No sidecar or canonical file is rewritten. + +The benchmark now generates independent persisted indexes by default and also +supports --indexes=missing. Focused regressions cover actual loader calls and +use of persisted offsets, unread bodies, fallback, source order, ID overlays, +Unicode/buffer boundaries, malformed tags and lazy body failures. diff --git a/.spiral/implementations/IMP-20260919-TTZ7C-33.ttl b/.spiral/implementations/IMP-20260919-TTZ7C-33.ttl index 041a409..209da68 100644 --- a/.spiral/implementations/IMP-20260919-TTZ7C-33.ttl +++ b/.spiral/implementations/IMP-20260919-TTZ7C-33.ttl @@ -30,7 +30,7 @@ project:IMP-20260919-TTZ7C-33 sd:implementationLocation "src/workerPool.mjs" ; sd:implementationLocation "test/file-data.mjs" ; sd:implementationLocation "scripts/benchmark-file-data.mjs" ; - sd:implements [ a sd:ArtifactReference ; sd:artifact project:DES-20260923-TTZ7C-47 ; sd:gitCommit "9b8acd229291fa75423c45a9019245edc0c19ea9" ] ; + sd:implements [ a sd:ArtifactReference ; sd:artifact project:DES-20260923-TTZ7C-47 ; sd:gitCommit "df973f7b6912f0a73b4b5980e1ede04a3b983d56" ] ; sd:changeCausedBy [ a sd:ArtifactReference ; sd:artifact project:DEF-20260923-TTZ7C-50 ; sd:gitCommit "4bb10c2013fd2a7cb989b02586b992b936121362" ] ; - sd:transforms [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-33 ; sd:gitCommit "faa3e896f40aa5af89b9e3216018a0bca659c48e" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-33 ; sd:gitCommit "df973f7b6912f0a73b4b5980e1ede04a3b983d56" ] ; sd:implementationChangeKind sd:SemanticChange . diff --git a/docs/file-data.md b/docs/file-data.md index ef5cc59..e6f10a5 100644 --- a/docs/file-data.md +++ b/docs/file-data.md @@ -7,13 +7,23 @@ command APIs and the existing OD-JSONTag/log formats are unchanged. ## Opening and reading a store -Startup inspects command-log order and the committed prefix, as before. A -64 KiB byte buffer scans each canonical data file's framing and SHA-256 digest. -That scan derives exact record offsets, including sparse changesets and UTF-8 -payloads. Stored offset/ID sidecars remain produced by the index hooks, but -startup derives its authoritative indexes from data files, so absent or stale -sidecars cannot redirect reads. Inspection does not import custom index modules. -The loader reconstructs the ID map from the final live records. +Startup inspects command-log order and the committed prefix, as before. It loads +`index.offset.json` and `index.id.json` for the base, followed by +`index.offset..json` and `index.id..json` for committed +changesets only. Existing index hooks continue writing these files. + +A 64 KiB byte buffer scans canonical framing and SHA-256 digests. Persisted +offsets must match every framed record range. The same scan reads leading +JSONTag headers to validate the complete composed ID map, including ID removal, +renaming and duplicate-ID precedence. Valid persisted indexes are used; missing, +unreadable, malformed, incomplete or stale indexes are reconstructed in memory. +Loading never rewrites data or sidecars and does not import custom index modules. + +Record bodies remain lazy, except for opening the root and the unusual fallback +for reference-only records. Malformed framing or metadata fails loading; body +syntax errors surface as storage failures when read. Configured integrity +checks still cover all canonical bytes. This does not claim full syntax +validation of every unread body. Each query worker opens its own read-only descriptors and registers the ordered base/changeset sources with its parser. ID lookups use the complete record @@ -92,14 +102,20 @@ memory. Command/status logs and schemas are still read into memory. Backup/copy utilities still temporarily buffer individual files when copying them; this cycle changes dataset retrieval and replay, not their copy algorithm. -Startup still reads all canonical bytes for validation and decodes live records -to reconstruct IDs. It trades startup work for lazy subsequent reads. A generated -328,877,813-byte / 20,000-record store opened and answered an ID lookup with a -48 MB JavaScript heap limit. One local run measured approximately 17.5 seconds -open and 93 ms query (including worker readiness), with 8.9 MB parent heap and -24.4 MB query-worker heap. Retained byte buffers were approximately 25 KiB and -58 KiB respectively. These are warm local measurements, not a cold-storage, -physical-RAM or total-process-memory bound. +Startup still reads all canonical bytes for validation. It reads tag headers +instead of decoding every live record to check IDs; sidecars have no independent +binding to the data contents, so merely loading them cannot safely replace this +scan. The same header scan supplies fallback IDs when sidecars are unusable. + +A generated 328,877,813-byte / 20,000-record store with persisted indexes opened +and answered an ID lookup with a 48 MB JavaScript heap limit. One local run +measured approximately 1.08 seconds open and 91 ms query (including worker +readiness), with 9.0 MB parent heap and 24.6 MB query-worker heap. The earlier +full-record reconstruction took approximately 17.5 seconds open on the same +fixture. Retained byte buffers were approximately 25 KiB and 58 KiB respectively. +These are warm local measurements, not a cold-storage, physical-RAM or +total-process-memory bound. The improvement comes from header-only validation; +fallback also benefits, rather than requiring persisted indexes for lazy loading. Reproduce the disposable-fixture probe with: @@ -107,6 +123,8 @@ Reproduce the disposable-fixture probe with: node --expose-gc --max-old-space-size=48 scripts/benchmark-file-data.mjs ``` +Pass `--indexes=missing` to exercise reconstruction without sidecars. + The dependency is pinned to reviewed od-jsontag Git commit `24f47eb616c3a24391487197496804ed1c56837b`, because its file-backed API has not received a new npm version. Installation therefore requires Git access to its diff --git a/scripts/benchmark-file-data.mjs b/scripts/benchmark-file-data.mjs index b5d7468..d678ce1 100644 --- a/scripts/benchmark-file-data.mjs +++ b/scripts/benchmark-file-data.mjs @@ -20,8 +20,21 @@ let runtime try { const fd = fs.openSync(datafile, 'w') try { - const writeRecord = text => { - const bytes = Buffer.from(`(${Buffer.byteLength(text)})${text}\n`) + const offsets = {} + const ids = {} + let fileOffset = 0 + let record = 0 + const writeRecord = (text, id) => { + const prefix = `(${Buffer.byteLength(text)})` + const bytes = Buffer.from(`${prefix}${text}\n`) + offsets[record] = [ + fileOffset + prefix.length, fileOffset + bytes.length - 1 + ] + if (id) { + ids[id] = record + } + record++ + fileOffset += bytes.length let offset = 0 while (offset < bytes.length) { offset += fs.writeSync(fd, bytes, offset) @@ -32,7 +45,13 @@ try { for (let number = 1; number <= records; number++) { writeRecord(`${JSON.stringify({ number, padding - })}`) + })}`, `item-${number}`) + } + if (args.indexes !== 'missing') { + fs.writeFileSync(path.join(dir, 'index.offset.json'), + JSON.stringify(offsets)) + fs.writeFileSync(path.join(dir, 'index.id.json'), + JSON.stringify(ids)) } } finally { @@ -64,6 +83,7 @@ try { console.log(JSON.stringify({ bytes: fs.statSync(datafile).size, records, openMs, queryMs, sources: runtime.sources.length, + persistedIndexes: args.indexes !== 'missing', idCount: runtime.meta.index.id.size, parent: { heapUsed: memory.heapUsed, arrayBuffers: memory.arrayBuffers diff --git a/src/file-data.mjs b/src/file-data.mjs index 5f2caa4..49e3399 100644 --- a/src/file-data.mjs +++ b/src/file-data.mjs @@ -1,6 +1,9 @@ import fs from 'node:fs' import path from 'node:path' import { createHash } from 'node:crypto' +import { isDeepStrictEqual } from 'node:util' +import idIndex from './index.id.mjs' +import offsetIndex from './index.offset.mjs' import JSONTag from '@muze-nl/jsontag' import Parser from '@muze-nl/od-jsontag/src/parse.mjs' import { RecoveryIntegrityError, assertChangesetExists } from './recovery.mjs' @@ -60,6 +63,56 @@ class FileScan { } } + recordId(end) { + while (this.offset < end && [9, 10, 13, 32].includes(this.peek())) { + this.advance() + } + if (this.offset === end) { + return undefined + } + if (this.peek() === 126) { + // A reference-only record needs the full parser to resolve its ID. + return null + } + if (this.peek() !== 60) { + return undefined + } + const bytes = [] + let quoted = false + let escaped = false + while (this.offset < end) { + const byte = this.peek() + this.advance() + bytes.push(byte) + if (escaped) { + escaped = false + } + else if (quoted && byte === 92) { + escaped = true + } + else if (byte === 34) { + quoted = !quoted + } + else if (!quoted && byte === 62) { + // Share the actual tag grammar, including Unicode escapes. + const parser = this.tagParser ??= new JSONTag.Parser() + parser.input = Buffer.from(bytes).toString('utf8') + parser.at = 0 + parser.next() + try { + const attributes = parser.tag().attributes + const value = {} + JSONTag.setAttributes(value, attributes) + return JSONTag.getAttribute(value, 'id') + } + catch (error) { + this.fail(`invalid record tag: ${error.message}`) + } + } + } + this.fail('unterminated record tag') + } + number(kind) { let digits = 0 let value = 0 @@ -104,7 +157,7 @@ export function hashFile(file) { } } -export function scanDataFile(filename) { +export function scanDataFile(filename, recordIds = null) { const file = path.resolve(filename) const fd = fs.openSync(file, 'r') try { @@ -138,7 +191,11 @@ export function scanDataFile(filename) { } scan.advance() const start = scan.offset - scan.advance(length) + const end = start + length + if (recordIds) { + recordIds.set(record, scan.recordId(end)) + } + scan.advance(end - scan.offset) source.offsets[record++] = [start, scan.offset] } if (!Number.isSafeInteger(record) || record >= 0xffffffff) { @@ -246,6 +303,29 @@ export class FileDataset { } } +function optionalIndex(index, meta, command) { + try { + return index.load(meta, command) + } + catch (error) { + if (error instanceof SyntaxError || error.code) { + return undefined + } + throw error + } +} + +function idsFromRecords(records) { + const ids = new Map() + for (const number of [...records.keys()].sort((a, b) => a - b)) { + const id = records.get(number) + if (id) { + ids.set(id, number) + } + } + return ids +} + export async function loadFileData(files) { const manifest = files.integrityFile ? await loadIntegrityManifest(files.integrityFile) @@ -253,13 +333,34 @@ export async function loadFileData(files) { const paths = [files.dataFile, ...files.commands.map(id => { return assertChangesetExists(files.dataFile, id) })] - const sources = paths.map(file => { - const source = scanDataFile(file) + const meta = { + data: path.dirname(path.resolve(files.dataFile)), + parts: files.commands.length + } + const recordIds = new Map() + const persistedIds = new Map() + let completeIds = true + const sources = paths.map((file, part) => { + const command = part === 0 ? null : files.commands[part - 1] + const source = scanDataFile(file, recordIds) if (manifest) { verifyDigest(manifest, files.integrityFile, file, source.digest, { required: files.integrityRequired }) } + const offsets = optionalIndex(offsetIndex, meta, command) + if (isDeepStrictEqual(offsets, source.offsets)) { + source.offsets = offsets + } + const ids = optionalIndex(idIndex, meta, command) + if (ids && typeof ids === 'object' && !Array.isArray(ids)) { + for (const [id, number] of Object.entries(ids)) { + persistedIds.set(id, number) + } + } + else { + completeIds = false + } return source }) if (!sources[0].size) { @@ -268,11 +369,14 @@ export async function loadFileData(files) { const dataset = new FileDataset() try { dataset.open(sources) - const meta = { - data: path.dirname(path.resolve(files.dataFile)), - parts: files.commands.length, - index: { id: dataset.rebuildIds() } + let ids = idsFromRecords(recordIds) + if ([...recordIds.values()].includes(null)) { + ids = dataset.rebuildIds() + } + if (completeIds && isDeepStrictEqual(persistedIds, ids)) { + ids = persistedIds } + meta.index = { id: ids } if (files.schemaFile) { meta.schema = JSONTag.parse( fs.readFileSync(files.schemaFile, 'utf8') diff --git a/test/file-data.mjs b/test/file-data.mjs index a74b98b..c70579d 100644 --- a/test/file-data.mjs +++ b/test/file-data.mjs @@ -7,9 +7,12 @@ import { createHash } from 'node:crypto' import { fileURLToPath } from 'node:url' import JSONTag from '@muze-nl/jsontag' import serialize from '@muze-nl/od-jsontag/src/serialize.mjs' -import { FileDataset, scanDataFile, hashFile, loadFileData } from '../src/file-data.mjs' +import { FileDataset, FileParser, scanDataFile, hashFile, loadFileData } from '../src/file-data.mjs' import { appendIntegrityRecord } from '../src/integrity.mjs' import index from '../src/index.mjs' +import offsetIndex from '../src/index.offset.mjs' +import idIndex from '../src/index.id.mjs' +import { getIndex } from '@muze-nl/od-jsontag/src/symbols.mjs' import WorkerPool from '../src/workerPool.mjs' import StoreRuntime from '../src/store-runtime.mjs' import { makeServerFixture } from './durability-helpers.mjs' @@ -112,8 +115,10 @@ test('sessions close all handles on failure and reject changed source identities test('opening and partial reads never read the entire file through readFile', async t => { const { file } = fixture(t) - t.mock.method(fs, 'readFileSync', () => { - throw new Error('whole-file read is forbidden') + const read = fs.readFileSync + t.mock.method(fs, 'readFileSync', (name, ...args) => { + assert.notEqual(name, file, 'whole dataset read is forbidden') + return read(name, ...args) }) const loaded = await loadFileData({ dataFile: file, commands: [] }) const dataset = open(t, loaded.sources, loaded.meta) @@ -350,3 +355,162 @@ test('lazy query read failures stop mutation, while query-thrown flags do not', ) assert.equal(accepted.code, 503) }) + +function writeIndexes(dir, offsets, ids, command = '') { + const suffix = command ? `.${command}` : '' + fs.writeFileSync(path.join(dir, `index.offset${suffix}.json`), + JSON.stringify(offsets)) + fs.writeFileSync(path.join(dir, `index.id${suffix}.json`), + JSON.stringify(ids)) +} + +test('startup consumes valid sidecars without decoding unread record bodies', async t => { + const { file, dir, source } = fixture(t) + writeIndexes(dir, source.offsets, {first: 1, last: 2}) + const before = fs.readdirSync(dir).map(name => [ + name, fs.readFileSync(path.join(dir, name)) + ]) + const loadedOffsets = [] + const loadOffsets = offsetIndex.load + t.mock.method(offsetIndex, 'load', (...args) => { + const offsets = loadOffsets(...args) + loadedOffsets.push(offsets) + return offsets + }) + const loadIds = t.mock.method(idIndex, 'load') + const parsed = new Set() + const firstParse = FileParser.prototype.firstParse + t.mock.method(FileParser.prototype, 'firstParse', function (target) { + parsed.add(target[getIndex]) + return firstParse.call(this, target) + }) + t.mock.method(FileDataset.prototype, 'rebuildIds', () => { + throw new Error('full-record ID reconstruction is forbidden') + }) + const loaded = await loadFileData({dataFile: file, commands: []}) + assert.equal(loaded.sources[0].offsets, loadedOffsets[0]) + assert.equal(loadIds.mock.callCount(), 1) + assert.deepEqual([...loaded.meta.index.id], [['first', 1], ['last', 2]]) + assert.deepEqual([...parsed], [0]) + assert.deepEqual(fs.readdirSync(dir).map(name => [ + name, fs.readFileSync(path.join(dir, name)) + ]), before) +}) + +test('startup loads only committed command sidecars in authoritative order', async t => { + const { file, dir, source } = fixture(t) + writeIndexes(dir, source.offsets, {first: 1, last: 2}) + const payload = '{"name":"committed"}' + const patch = path.join(dir, 'data.edit.jsontag') + fs.writeFileSync(patch, `+2\n(${Buffer.byteLength(payload)})${payload}`) + writeIndexes(dir, scanDataFile(patch).offsets, {last: 2}, 'edit') + writeIndexes(dir, {'2': [1, 99]}, {uncommitted: 2}, 'uncommitted') + const loadIds = t.mock.method(idIndex, 'load') + const loadOffsets = t.mock.method(offsetIndex, 'load') + const loaded = await loadFileData({dataFile: file, commands: ['edit']}) + for (const mock of [loadIds, loadOffsets]) { + assert.deepEqual(mock.mock.calls.map(call => call.arguments[1]), + [null, 'edit']) + } + assert.deepEqual([...loaded.meta.index.id], [['first', 1], ['last', 2]]) + const dataset = open(t, loaded.sources, loaded.meta) + assert.equal(dataset.parser.getLineProxy(2).name, 'committed') +}) + +for (const invalid of ['{', 'null', '[]', '{}', + '{"last":1}', '{"first":1,"last":2,"extra":2}']) { + test(`invalid/incomplete ID sidecars fall back: ${invalid}`, async t => { + const { file, dir, source } = fixture(t) + writeIndexes(dir, source.offsets, {}) + fs.writeFileSync(path.join(dir, 'index.id.json'), invalid) + const loaded = await loadFileData({dataFile: file, commands: []}) + assert.deepEqual([...loaded.meta.index.id], [['first', 1], ['last', 2]]) + }) +} + +test('offset sidecars cannot omit, redirect or add records', async t => { + const { file, dir, source } = fixture(t) + const invalid = [null, [], {}, {'0': source.offsets[0]}, + {...source.offsets, '1': source.offsets[2]}, + {...source.offsets, '9': source.offsets[2]}] + for (const offsets of invalid) { + writeIndexes(dir, offsets, {first: 1, last: 2}) + const loaded = await loadFileData({dataFile: file, commands: []}) + assert.deepEqual(loaded.sources[0].offsets, source.offsets) + const dataset = new FileDataset(loaded.meta) + try { + dataset.open(loaded.sources) + assert.equal(dataset.root.items[0].name, 'first') + } + finally { + dataset.close() + } + } + fs.writeFileSync(path.join(dir, 'index.offset.json'), '{') + const loaded = await loadFileData({dataFile: file, commands: []}) + assert.deepEqual(loaded.sources[0].offsets, source.offsets) +}) + +test('ID overlays remove old IDs and restore lower duplicates after renames', async t => { + const { file, dir } = fixture(t) + const record = text => `(${Buffer.byteLength(text)})${text}\n` + fs.writeFileSync(file, record('{"items":[~1,~2,~3]}') + + record('{"name":"lower"}') + + record('{"name":"higher"}') + + record('{}')) + writeIndexes(dir, scanDataFile(file).offsets, {duplicate: 2, removed: 3}) + const patch = path.join(dir, 'data.rename.jsontag') + fs.writeFileSync(patch, '+2\n' + record('{}') + + record('{}')) + writeIndexes(dir, scanDataFile(patch).offsets, {renamed: 2}, 'rename') + const loaded = await loadFileData({dataFile: file, commands: ['rename']}) + assert.deepEqual([...loaded.meta.index.id], [['duplicate', 1], ['renamed', 2]]) + const dataset = open(t, loaded.sources, loaded.meta) + assert.equal(dataset.parser.getLineProxy(1).name, 'lower') +}) + +test('ID header validation handles escaped delimiters and buffer boundaries', async t => { + const { file, dir } = fixture(t) + const ids = ['漢😀>\\tail', '__proto__', 'constructor', 'x'.repeat(70000)] + const record = text => `(${Buffer.byteLength(text)})${text}\n` + fs.writeFileSync(file, record('{"items":[~1-4]}') + ids.map(id => { + return record(` {"text":"ignored"}`) + }).join('')) + const expected = Object.fromEntries(ids.map((id, i) => [id, i + 1])) + writeIndexes(dir, scanDataFile(file).offsets, expected) + const read = fs.readSync + t.mock.method(fs, 'readSync', (fd, buffer, offset, length, position) => { + return read(fd, buffer, offset, Math.min(31, length), position) + }) + const loaded = await loadFileData({dataFile: file, commands: []}) + assert.deepEqual(loaded.meta.index.id, new Map(Object.entries(expected))) + const dataset = open(t, loaded.sources, loaded.meta) + assert.deepEqual(dataset.rebuildIds(), loaded.meta.index.id) +}) + +test('unterminated or invalid tags cannot be hidden by valid-looking indexes', async t => { + const { file, dir } = fixture(t) + for (const payload of ['{}']) { + fs.writeFileSync(file, `(${Buffer.byteLength(payload)})${payload}`) + writeIndexes(dir, scanDataFile(file).offsets, {bad: 0}) + await assert.rejects(loadFileData({dataFile: file, commands: []}), + /Invalid OD-JSONTag data/) + } +}) + +test('body syntax stays lazy while malformed reads still mark storage failure', async t => { + const { file, dir } = fixture(t) + const record = text => `(${Buffer.byteLength(text)})${text}\n` + fs.writeFileSync(file, record('{"items":[~1]}') + + record('{"name":invalid}')) + writeIndexes(dir, scanDataFile(file).offsets, {unread: 1}) + const loaded = await loadFileData({dataFile: file, commands: []}) + assert.equal(loaded.meta.index.id.get('unread'), 1) + const dataset = open(t, loaded.sources, loaded.meta) + assert.equal(dataset.parser.readFailure, undefined) + assert.throws(() => dataset.parser.getLineProxy(1).name, error => { + return error.storageFailure === true + }) + assert.equal(dataset.parser.readFailure.storageFailure, true) +}) From 37968ecce058f738900821c38b53d203a91a9845 Mon Sep 17 00:00:00 2001 From: Auke van Slooten Date: Wed, 23 Sep 2026 17:49:49 +0200 Subject: [PATCH 15/15] Evaluate persisted indexes and record startup improvements --- .spiral/cycles/CYC-20260923-TTZ7C-44.md | 11 ++++- .spiral/cycles/CYC-20260923-TTZ7C-44.ttl | 4 +- .spiral/evidence/EVD-20260923-TTZ7C-49.md | 52 +++++++++++++++++++++- .spiral/evidence/EVD-20260923-TTZ7C-49.ttl | 8 ++-- .spiral/project-context.md | 5 ++- .spiral/project.ttl | 4 +- 6 files changed, 71 insertions(+), 13 deletions(-) diff --git a/.spiral/cycles/CYC-20260923-TTZ7C-44.md b/.spiral/cycles/CYC-20260923-TTZ7C-44.md index 81ea7fd..5cac032 100644 --- a/.spiral/cycles/CYC-20260923-TTZ7C-44.md +++ b/.spiral/cycles/CYC-20260923-TTZ7C-44.md @@ -21,8 +21,9 @@ Client version metadata/audit history and sandbox replacement remain later work. ## Evaluation -Implementation is complete at `cdc1564744989c955a187fdce4baed22ec5d0fd7`. EVD-20260923-TTZ7C-49 -records 142 passing tests, clean focused lint, fresh-package installation, a +Implementation, including requested persisted-index loading, is complete at +`d9d8ff860ed2b74585ea66454382c9baf4a09250`. EVD-20260923-TTZ7C-49 +records 155 passing tests, clean focused lint, fresh-package installation, a 329 MB/48 MB-heap runtime probe and read-only curriculum evidence. See [the integration guide](../../docs/file-data.md) for the source contract and limits. Human acceptance remains pending; the cycle stays Active. @@ -31,3 +32,9 @@ Final review added DEF-20260923-TTZ7C-50 and two reproducing tests. The correcti stops mutation after late lazy-read failures, including caught errors, and keeps query-supplied flags from masquerading as parser failures. EVD-49 records the full 142-test pass and the final 23-test response-format check. + +The maintainer's example-folder verification and requirement to finish persisted +index loading are retained in the revised SRC-45. REQ-46/DES-47 now require +validated sidecar use and fallback. The extension removes full-body ID decoding +from normal startup, while retaining canonical byte scans and lazy read-failure +handling. EVD-49 records exact full/focused tests and new measurements. diff --git a/.spiral/cycles/CYC-20260923-TTZ7C-44.ttl b/.spiral/cycles/CYC-20260923-TTZ7C-44.ttl index 7db27c2..e4524c5 100644 --- a/.spiral/cycles/CYC-20260923-TTZ7C-44.ttl +++ b/.spiral/cycles/CYC-20260923-TTZ7C-44.ttl @@ -7,5 +7,5 @@ project:CYC-20260923-TTZ7C-44 dcterms:identifier "CYC-20260923-TTZ7C-44" ; sd:repositoryPath ".spiral/cycles/CYC-20260923-TTZ7C-44.md" ; sd:status sd:Active ; - sd:derivedFrom [ a sd:ArtifactReference ; sd:artifact project:REQ-20260923-TTZ7C-46 ; sd:gitCommit "a75db831ee6779bbf297e94b9d1baf694692cb84" ] ; - sd:transforms [ a sd:ArtifactReference ; sd:artifact project:CYC-20260923-TTZ7C-44 ; sd:gitCommit "a64b81373b2ef3d8252b400d7a18b9242038adb2" ] . + sd:derivedFrom [ a sd:ArtifactReference ; sd:artifact project:REQ-20260923-TTZ7C-46 ; sd:gitCommit "b279aefe78a262a041b59dff2e2c199233fbcf7d" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:CYC-20260923-TTZ7C-44 ; sd:gitCommit "d9d8ff860ed2b74585ea66454382c9baf4a09250" ] . diff --git a/.spiral/evidence/EVD-20260923-TTZ7C-49.md b/.spiral/evidence/EVD-20260923-TTZ7C-49.md index c52ec35..ce009cd 100644 --- a/.spiral/evidence/EVD-20260923-TTZ7C-49.md +++ b/.spiral/evidence/EVD-20260923-TTZ7C-49.md @@ -76,10 +76,58 @@ tests, including JSONTag error negotiation. Focused lint remains clean. Earlier memory, curriculum and package probes establish the unchanged file-source and distribution contracts; they are not separate security or hardware claims. +## Persisted-index completion, 2026-09-23 + +The maintainer required persisted index loading in this version after review +identified its absence, and reported successful operation with the example +folder. SRC-45 retains that feedback. This is supporting evidence, not cycle +acceptance; the agent did not modify or rerun the live example folder. + +Implementation d9d8ff8 reads base and committed-command offset/ID sidecars. +Valid offsets are passed directly to the parser; a complete validated composed +ID map is used. Canonical framing and record headers supply validation and +fallback, without decoding every record body or writing sidecars. + +- Complete suite: **155 tests pass** (142 previous plus 13 index/lazy-body cases), + `/tmp/simplystore-index-full-final.log`. The final edit only removes a redundant + escape in a test string without changing its value; all **36 focused tests** + pass afterwards (`/tmp/simplystore-index-focused-final.log`). +- Focused ESLint on file-data, its tests and the benchmark: zero errors/warnings + (`/tmp/simplystore-index-lint-final.log`). Whitespace checks pass. +- Tests observe actual base/committed-sidecar loader calls and the returned + offset object's identity; root-only parsing is checked with a parser spy and + full-record ID reconstruction is forbidden in that case. Missing, malformed, + partial, redirected and extra index entries fall back correctly. Overlays + handle ID removal/renaming and restoration of a lower duplicate. +- Header tests cross buffer boundaries with short reads and 70 KB metadata, + Unicode, backslashes, quoted delimiters and prototype-named IDs. They compare + results with full parser reconstruction. Invalid tags fail; unread invalid + bodies raise host-owned storage failures when accessed. JSONTag's attribute + API is used as well as its tag parser, preserving attribute validation. +- Same 328,877,813-byte / 20,000-record fixture and 48 MB JS heap limit, with + independently generated persisted offsets/IDs: open **1,076.6 ms**, query + **91.2 ms**, parent heap **8,990,544 bytes**, worker heap **24,561,248 bytes**. + Retained buffers: **24,823 / 57,562 bytes**. Log: + `/tmp/simplystore-index-memory.log`. Earlier full-body ID reconstruction was + about 17.5 seconds. The main improvement is header-only validation. +- Same fixture with `--indexes=missing`: open **1,123.1 ms**, query **97.6 ms**; + passes the heap limit and ID lookup. Log: + `/tmp/simplystore-index-missing-memory.log`. +- Real curriculum file, read-only: **77,952,430 bytes / 181,724 records / 180,416 + IDs**. Loader took **1,047.2 ms**; independent full-parser ID reconstruction + took **7,436.6 ms**. The complete Maps agree and before/after source digests + match (`/tmp/simplystore-index-curriculum.log`). No dataset or sidecar changed. + +These local warm measurements do not establish a general latency or total-RAM +bound. Startup still scans every canonical byte; no new index format, trusted +certificate cache, dependency change or separate power-cut campaign was added. +The earlier fresh-package probe remains applicable to the distribution contract; +this extension changes existing packaged files only. + ## Boundaries and review disposition -Startup hashes all canonical bytes and reconstructs IDs by decoding final -records. Index catalogs remain O(records), handles O(source files per worker), +Startup hashes all canonical bytes and validates persisted IDs from record +headers, with reconstruction as fallback. Index catalogs remain O(records), handles O(source files per worker), and mutable sessions retain touched data. Command output/custom hooks/query results may be large. Schemas and logs remain in memory; administrative copying still temporarily buffers individual files. These are documented boundaries, diff --git a/.spiral/evidence/EVD-20260923-TTZ7C-49.ttl b/.spiral/evidence/EVD-20260923-TTZ7C-49.ttl index b4b8194..c6eaff2 100644 --- a/.spiral/evidence/EVD-20260923-TTZ7C-49.ttl +++ b/.spiral/evidence/EVD-20260923-TTZ7C-49.ttl @@ -7,7 +7,7 @@ project:EVD-20260923-TTZ7C-49 dcterms:identifier "EVD-20260923-TTZ7C-49" ; sd:repositoryPath ".spiral/evidence/EVD-20260923-TTZ7C-49.md" ; sd:status sd:Active ; - sd:verifies [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-26 ; sd:gitCommit "cdc1564744989c955a187fdce4baed22ec5d0fd7" ] ; - sd:verifies [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-33 ; sd:gitCommit "cdc1564744989c955a187fdce4baed22ec5d0fd7" ] ; - sd:verifies [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-34 ; sd:gitCommit "cdc1564744989c955a187fdce4baed22ec5d0fd7" ] ; - sd:transforms [ a sd:ArtifactReference ; sd:artifact project:EVD-20260923-TTZ7C-49 ; sd:gitCommit "a64b81373b2ef3d8252b400d7a18b9242038adb2" ] . + sd:verifies [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-26 ; sd:gitCommit "d9d8ff860ed2b74585ea66454382c9baf4a09250" ] ; + sd:verifies [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-33 ; sd:gitCommit "d9d8ff860ed2b74585ea66454382c9baf4a09250" ] ; + sd:verifies [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-34 ; sd:gitCommit "d9d8ff860ed2b74585ea66454382c9baf4a09250" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:EVD-20260923-TTZ7C-49 ; sd:gitCommit "d9d8ff860ed2b74585ea66454382c9baf4a09250" ] . diff --git a/.spiral/project-context.md b/.spiral/project-context.md index bdd8db3..143c76a 100644 --- a/.spiral/project-context.md +++ b/.spiral/project-context.md @@ -24,7 +24,10 @@ shared-memory data after reviewing od-jsontag hardening. This continues the file-only roadmap direction, prioritized ahead of rebuild. SRC-45, UND-48, REQ-46 and DES-47 (20260923-TTZ7C namespace) retain the commitment. Implementation uses ordered file sources, bounded clean-record caches, worker-owned handles, -streamed framing/hash scans and file-backed administrative replay. EVD-20260923-TTZ7C-49 records 142 passing tests, clean lint, a fresh-package +streamed framing/hash scans and file-backed administrative replay. Persisted +offset/ID sidecars are loaded and validated against canonical framing and +record headers, with derived fallback. The maintainer verified example-folder +operation and required this persisted-index completion in the same version. EVD-20260923-TTZ7C-49 records 155 passing tests, clean lint, a fresh-package installation and large-file/real-data probes. Human acceptance remains pending. Client-visible version metadata and audit trails follow this migration; no such feature is implemented by this cycle. diff --git a/.spiral/project.ttl b/.spiral/project.ttl index bd3239d..c3dcdaf 100644 --- a/.spiral/project.ttl +++ b/.spiral/project.ttl @@ -15,8 +15,8 @@ project:CTX-001 sd:artifact project:SRC-20260919-TTZ7C-43 ; sd:gitCommit "79fcf670d4558c75b9933743e6f2ea5616221eb0" ] ; - sd:derivedFrom [ a sd:ArtifactReference ; sd:artifact project:SRC-20260923-TTZ7C-45 ; sd:gitCommit "8299fcc8b8fcf113091ae85bbcf11f867f8c5908" ] ; - sd:transforms [ a sd:ArtifactReference ; sd:artifact project:CTX-001 ; sd:gitCommit "a64b81373b2ef3d8252b400d7a18b9242038adb2" ] ; + sd:derivedFrom [ a sd:ArtifactReference ; sd:artifact project:SRC-20260923-TTZ7C-45 ; sd:gitCommit "91be30fdf8351afe1003e8fa8dcb7b67302341c2" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:CTX-001 ; sd:gitCommit "d9d8ff860ed2b74585ea66454382c9baf4a09250" ] ; sd:developerEvaluationReadinessBar "Developers should be able to evaluate bounded durability claims from invariants, executable tests, evidence artifacts, explicit failure behavior, and documented known gaps." ; sd:hasAudience project:AUD-DEVELOPERS, project:AUD-END-USERS, project:AUD-OPERATORS ; sd:hasDownstreamProject project:DOWNSTREAM-CURRICULUM-STORE ;