diff --git a/.spiral/cycles/CYC-20260923-TTZ7C-44.md b/.spiral/cycles/CYC-20260923-TTZ7C-44.md new file mode 100644 index 0000000..fe6646d --- /dev/null +++ b/.spiral/cycles/CYC-20260923-TTZ7C-44.md @@ -0,0 +1,65 @@ +--- +id: CYC-20260923-TTZ7C-44 +--- + +# File-backed dataset runtime + +Goal: replace shared-memory dataset transport and retention with file-backed +reads throughout live operation and administrative replay, preserving remote +behavior, disk formats, durable command ordering and recovery restrictions. + +Branch: spiral/CYC-20260923-TTZ7C-44-file-backed, from master dbb0703. +Status: Accepted on 2026-09-23. Authorization is retained in SRC-45; +human acceptance and independent example checks are recorded below. +Continue the file-only roadmap direction; prioritize it ahead of rebuild under +the maintainer's explicit direction. Library readiness was established first. + +Evaluate with existing durability/recovery tests, file-only worker messages, +ID lookup of unread records, sparse overlays, no-op commands, worker replacement +and shutdown, index/framing/integrity failures and a bounded-memory large-file +probe. Source changes must remain confined to disposable fixtures during tests. +Client version metadata/audit history and sandbox replacement remain later work. + +## Evaluation + +Implementation, including requested persisted-index loading, is complete at +`46fb22fe5c4fecbb23c3b8d3b59e5ff6e58e11c2`. EVD-20260923-TTZ7C-49 +records 178 passing tests, clean focused lint, fresh-package installation, a +329 MB/48 MB-heap runtime probe and read-only curriculum evidence. +See [the integration guide](../../docs/file-data.md) for the source contract and +limits. Human acceptance is recorded below; integration remains pending. + +Final review added DEF-20260923-TTZ7C-50 and two reproducing tests. The correction +stops mutation after late lazy-read failures, including caught errors, and keeps +query-supplied flags from masquerading as parser failures. EVD-49 records the +full 142-test pass and the final 23-test response-format check. + +The maintainer's example-folder verification and subsequent corrections are +retained in SRC-45. REQ-46/DES-47 now require direct trusted ID loading, reliable +final ID publication, errors for duplicate IDs and no legacy index compatibility. +Missing indexes reconstruct; validation/reconstruction can be explicitly +requested. EVD-49 records the 170-test final pass and direct-index memory probe. +This evaluation preceded the human acceptance recorded below. + +The maintainer then confirmed offset trust and file hashes. Normal startup now +loads both indexes directly, including inspection; explicit validation or missing +offsets alone reconstruct framing. Existing integrity manifests cover final +standard indexes too. EVD-49 records 178 passing tests, clean lint, and the updated +large-file probe. Human review concluded with the acceptance below. + +## Human acceptance + +On 2026-09-23, after the completed implementation and evaluation at +`8013c0c74fb1f01bb882b0b221c4502ef4970b85`, the maintainer confirmed: + +> confirmed tests, example works, ran a command, change was applied, and restarting kept it. Happy. + +This accepts the file-backed runtime cycle, including trusted committed offset +and ID indexes and independent file integrity hashes. The maintainer reports +confirming the tests and independently observing the example apply a command +and retain the change after restart. These are human-reported checks, separate +from the 178 automated tests retained in EVD-20260923-TTZ7C-49. + +The cycle is closed and ready to propose for integration into master with a +normal merge commit. Integration remains a separate decision; no next cycle +is selected. The branch identity was rechecked when recording acceptance. diff --git a/.spiral/cycles/CYC-20260923-TTZ7C-44.ttl b/.spiral/cycles/CYC-20260923-TTZ7C-44.ttl new file mode 100644 index 0000000..7897251 --- /dev/null +++ b/.spiral/cycles/CYC-20260923-TTZ7C-44.ttl @@ -0,0 +1,11 @@ +@prefix sd: . +@prefix dcterms: . +@prefix project: . + +project:CYC-20260923-TTZ7C-44 + a sd:Cycle ; + dcterms:identifier "CYC-20260923-TTZ7C-44" ; + sd:repositoryPath ".spiral/cycles/CYC-20260923-TTZ7C-44.md" ; + sd:status sd:Accepted ; + sd:derivedFrom [ a sd:ArtifactReference ; sd:artifact project:REQ-20260923-TTZ7C-46 ; sd:gitCommit "46fb22fe5c4fecbb23c3b8d3b59e5ff6e58e11c2" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:CYC-20260923-TTZ7C-44 ; sd:gitCommit "8013c0c74fb1f01bb882b0b221c4502ef4970b85" ] . diff --git a/.spiral/cycles/CYC-20260923-TTZ7C-51.md b/.spiral/cycles/CYC-20260923-TTZ7C-51.md new file mode 100644 index 0000000..d27a84a --- /dev/null +++ b/.spiral/cycles/CYC-20260923-TTZ7C-51.md @@ -0,0 +1,46 @@ +# Required integrity and existing-store initialization + +Status: Accepted on 2026-09-23. Continue the roadmap durability/file-backed direction under +SRC-20260923-TTZ7C-52. The confirmed goal is mandatory artifact integrity, +explicit initialization for existing stores, and measured command latency. + +Branch: spiral/CYC-20260923-TTZ7C-51-required-integrity. +Base exception: start from accepted CYC-44 tip `4680c76c8faabde53ecd712e1b903f2e8b0eddb6` because it supplies the +required file/index behavior and awaits separate integration authorization. +Do not merge it into master as part of this task. Preserve accepted history. + +Scope: conversion, startup/loader, commands, recovery, stopped-store initializer, +example documentation, regression coverage and before/after latency evidence. +Keep trusted index loading; no automatic re-baselining, command replay during +initialization, data rewrite, version/audit feature, or new hash format. +Preserve the maintainer's example data and command. Test with disposable stores. + +## Evaluation + +Implementation is complete at `1e119dd101dd231a7cf24ba1b9d6c3e4999e8f60`. EVD-55 records 193 passing tests, +clean focused lint, successful initialization of a disposable copy of the actual +example with its command retained, and a 329 MB/48 MB-heap probe. Three alternating +30-command latency runs on local ext4 measured roughly 55 ms baseline versus +57 ms required-integrity median completion. There is one durable hash append +per artifact set. Human acceptance is recorded below; integration is pending. + +The example now documents npm run init-integrity for a stopped existing store. +Actual user data/index/log files were preserved. Integration of accepted CYC-44 +and this cycle is still a separate decision. + +## Human acceptance + +After testing the evaluated branch tip `9515993531c55176e1c3444f71dc3a23d5172df0`, the maintainer confirmed +on 2026-09-23: + +> works as expected, fails to start without integrity now, init works, commands work. + +This accepts the mandatory-integrity and existing-store initialization cycle. +The maintainer independently reports startup refusal without integrity, successful +initialization, and working commands. These are human-reported checks alongside +the 193 automated tests and latency evidence in EVD-20260923-TTZ7C-55. + +The cycle is closed and ready to propose for integration with a normal merge +commit. This branch includes accepted CYC-44 and CYC-51. Integration remains a +separate decision; no next cycle is selected. The actual branch identity was +rechecked when recording acceptance. diff --git a/.spiral/cycles/CYC-20260923-TTZ7C-51.ttl b/.spiral/cycles/CYC-20260923-TTZ7C-51.ttl new file mode 100644 index 0000000..ff80ee8 --- /dev/null +++ b/.spiral/cycles/CYC-20260923-TTZ7C-51.ttl @@ -0,0 +1,11 @@ +@prefix sd: . +@prefix dcterms: . +@prefix project: . + +project:CYC-20260923-TTZ7C-51 + a sd:Cycle ; + dcterms:identifier "CYC-20260923-TTZ7C-51" ; + sd:repositoryPath ".spiral/cycles/CYC-20260923-TTZ7C-51.md" ; + sd:derivedFrom [ a sd:ArtifactReference ; sd:artifact project:REQ-20260923-TTZ7C-53 ; sd:gitCommit "1e119dd101dd231a7cf24ba1b9d6c3e4999e8f60" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:CYC-20260923-TTZ7C-51 ; sd:gitCommit "9515993531c55176e1c3444f71dc3a23d5172df0" ] ; + sd:status sd:Accepted . diff --git a/.spiral/defects/DEF-20260923-TTZ7C-50.md b/.spiral/defects/DEF-20260923-TTZ7C-50.md new file mode 100644 index 0000000..395dfe0 --- /dev/null +++ b/.spiral/defects/DEF-20260923-TTZ7C-50.md @@ -0,0 +1,35 @@ +--- +id: DEF-20260923-TTZ7C-50 +--- + +# Late file reads need storage-failure semantics + +Final review of the file-backed migration identified a changed failure boundary: +source reads can now fail inside a command or query after initialization. The +prior buffered runtime read source bytes before execution. DES-47 preserved +publication/storage failures but did not explicitly carry late read failures +through the ordinary command/query exception handlers. + +Two new regressions on implementation bfeed2c fail (21 other focused tests pass): +a command's unread record raises Unexpected end of file without storageFailure; +a query that catches its unread-record error can return success, leaving future +command acceptance enabled. The fixtures truncate only disposable data files +after both query pools have initialized. This models unavailable backing bytes; +normal operation still requires immutable source files. + +Required correction within the existing migration goal: the file parser records +read/decode/reference failures in host-owned session state. Commands must refuse +publication even if handler code catches that failure. Query workers consult +that host state, not a user-thrown storageFailure property; a spoofed flag must +remain an ordinary query error. The runtime marks storage failure and refuses +further commands. Preserve ownership/recovery evidence on failure. + +This corrects the migration design's failure classification, not the sandbox or +remote query language. The baseline tests are committed before the repair. + +## Correction + +Implemented at faa3e89 and refined at cdc1564. EVD-20260923-TTZ7C-49 records +the full 142-test pass and focused response-format verification. The tests also +show that swallowing the read error cannot permit command publication, and +query-thrown error properties cannot trigger host storage-failure state. diff --git a/.spiral/defects/DEF-20260923-TTZ7C-50.ttl b/.spiral/defects/DEF-20260923-TTZ7C-50.ttl new file mode 100644 index 0000000..e6bd4b3 --- /dev/null +++ b/.spiral/defects/DEF-20260923-TTZ7C-50.ttl @@ -0,0 +1,13 @@ +@prefix sd: . +@prefix dcterms: . +@prefix project: . + +project:DEF-20260923-TTZ7C-50 + a sd:Defect ; + dcterms:identifier "DEF-20260923-TTZ7C-50" ; + sd:repositoryPath ".spiral/defects/DEF-20260923-TTZ7C-50.md" ; + sd:status sd:Active ; + sd:provenanceConfidence sd:Evidenced ; + sd:observes [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-33 ; sd:gitCommit "bfeed2c5c45cd3ac58dd42aac868585f31c59d19" ] ; + sd:observes [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-33 ; sd:gitCommit "cdc1564744989c955a187fdce4baed22ec5d0fd7" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:DEF-20260923-TTZ7C-50 ; sd:gitCommit "4bb10c2013fd2a7cb989b02586b992b936121362" ] . diff --git a/.spiral/designs/DES-20260923-TTZ7C-47.md b/.spiral/designs/DES-20260923-TTZ7C-47.md new file mode 100644 index 0000000..0049179 --- /dev/null +++ b/.spiral/designs/DES-20260923-TTZ7C-47.md @@ -0,0 +1,88 @@ +--- +id: DES-20260923-TTZ7C-47 +--- + +# Ordered file sources and owned reader sessions + +Transport ordered source descriptors (path, byte offsets, size and digest), not +open descriptors or dataset bytes. A small file-reader session owns handles and +an od-jsontag parser; queries reuse it, commands open the committed prefix for +one execution, and all exits close handles. Resolve IDs through getLineProxy. + +Load base and log-authoritative committed ID sidecars directly. Validate their +JSON shape, record numbers and uniqueness using catalog metadata; do not read +record tags to check a present index's contents/completeness in normal startup. +Missing ID sidecars are reconstructed from their source's headers. Explicit +validateIndexes checks persisted content; rebuildIndexes ignores persisted IDs +and reconstructs in memory. Malformed present ID indexes fail normally. Offset +framing/hash scans and configured integrity validation remain unchanged. + +Use the existing compact ID-to-record JSON map; no legacy format path or +duplicate-ID winner selection. Apply each changeset by removing mappings whose +records it replaces, then inserting its IDs. Two distinct current records with +the same ID are an error; swaps and same-record updates remain valid. + +JSONTag attribute setters mutate the tag container in place; the reviewed +od-jsontag revision can therefore miss ID-only changes. Before calling update +and again after it returns, compare only materialized records' IDs with the +committed ID map and mark differences dirty. Conversion performs the same +check around create. This belongs at the store's ID publication boundary and +requires no full-record traversal or dependency revision. Regression tests must +use real JSONTag setters, including attribute-only commands. + +Extract IDs from final serialized bytes after mutation hooks. Prepare the ID +map and check global uniqueness before publishing the canonical changeset. +The core finalizeIndex boundary awaits custom finalization, durably writes the +canonical ID sidecar, and updates returned metadata. Custom finalizers retain +their call contract and control their own offset behavior, but cannot bypass +core ID finalization. Conversion does the same prepublication preparation. +Capture the expected serialized digest before custom finalization and reject +changes to canonical bytes before command or conversion success. Empty commands publish {}. + +The loader and commit preparation share removal/uniqueness rules. Command +preparation uses a snapshot of committed IDs, not a mutable hook/parser map. +Tests must compare live results with restart and prove complete indexes avoid +header parsing. No persistent reconstruction or canonical rewrite occurs at +startup. The only byte scan remains framing/hash validation unless an ID index +is missing or maintenance validation/reconstruction is requested. + +Startup inspection returns plain file sources instead of retained data/buffers. +Administrative recovery uses the same loader/session. Temporary command output +may remain buffered for the existing custom finalize(serialized, meta, uuid) +hook; it is written durably and returned as a file descriptor object. This +preserves the hook and avoids holding the whole dataset in memory. Streaming +full-dataset serialization is unnecessary once startup stops compacting it. + +Worker updates must serialize with queries and replacement workers must receive +the current committed source list. Await termination before releasing store +ownership. OS/Node worker teardown closes descriptors on timeout/failure. + +Alternative considered: keep a buffered fallback or merely replace query reads. +That would leave startup/recovery retaining entire datasets and duplicate modes; +a shared file-session boundary gives one implementation and test surface. + +Pin the reviewed od-jsontag Git commit until a separately authorized release. +No dependency publish, remote deployment or user data mutation is part of this +implementation. Validate behavior with independent byte-offset oracles and the +existing durability suite. + +## Offset trust correction, 2026-09-23 + +Share source loading between inspection, loader and publication: load offsets, +validate structural ranges, hash canonical bytes with a fixed buffer and retain +the stable file identity. Only missing offsets or explicit maintenance parse +framing. When IDs alone are missing, read headers at the stored offsets. + +Hash the exact index bytes that are parsed, using the existing SHA-256 manifest. +With integrity enabled, every present standard sidecar requires a manifest entry; +missing sidecars can reconstruct in memory from verified canonical data. Explicit +rebuild ignores sidecars including their hashes but still verifies canonical data. +No automatic mismatch repair and no legacy compatibility path. + +Conversion and command/recovery publication append hashes for final standard +sidecars after hooks and durable index writes, before done/success. Custom +finalizers may omit offsets; hash only present files. Inspection checks committed +sidecars too without importing custom hooks. Existing inventory byte hashes and +immutable source identity checks remain independent of offset reconstruction. +No claim that hashing proves index semantics: publication establishes consistency; +explicit validation diagnoses externally generated or modified indexes. diff --git a/.spiral/designs/DES-20260923-TTZ7C-47.ttl b/.spiral/designs/DES-20260923-TTZ7C-47.ttl new file mode 100644 index 0000000..39c8976 --- /dev/null +++ b/.spiral/designs/DES-20260923-TTZ7C-47.ttl @@ -0,0 +1,12 @@ +@prefix sd: . +@prefix dcterms: . +@prefix project: . + +project:DES-20260923-TTZ7C-47 + a sd:Design ; + dcterms:identifier "DES-20260923-TTZ7C-47" ; + sd:repositoryPath ".spiral/designs/DES-20260923-TTZ7C-47.md" ; + sd:status sd:Active ; + sd:satisfies [ a sd:ArtifactReference ; sd:artifact project:REQ-20260923-TTZ7C-46 ; sd:gitCommit "15088d395ae6466231edaf932ec7e889171729de" ] ; + sd:changeCausedBy [ a sd:ArtifactReference ; sd:artifact project:SRC-20260923-TTZ7C-45 ; sd:gitCommit "7e1ea39dc4056beec311c8e6d832d97d2a550d58" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:DES-20260923-TTZ7C-47 ; sd:gitCommit "7e1ea39dc4056beec311c8e6d832d97d2a550d58" ] . diff --git a/.spiral/designs/DES-20260923-TTZ7C-54.md b/.spiral/designs/DES-20260923-TTZ7C-54.md new file mode 100644 index 0000000..a92fdad --- /dev/null +++ b/.spiral/designs/DES-20260923-TTZ7C-54.md @@ -0,0 +1,35 @@ +# Mandatory verification with explicit initialization + +Always resolve a manifest path in runtime, converter, loader and inspection. +Ignore the former integrity=false/required=false toggles; they cannot disable +verification. Keep --integrity as a harmless redundant converter flag. Missing +manifest diagnostics direct operators to the explicit initialization workflow. + +Factor artifact hash collection and manifest serialization: canonical digest +already verified after finalization plus SHA-256 of final index bytes. Append +all entries with a single durable appendRecord before done; conversion uses the +same set. Administrative replay defers its worker's manifest append to its +coordinator so retained expected digests are checked before replacement hashes +and done. That is an internal publication boundary, not a store opt-out. + +Add initializeIntegrity and recover.mjs init-integrity --store config.json. +Acquire normal ownership on all mutable directories, refuse any existing +manifest, inspect an unsealed store via a separate explicit internal entry point, +validate framing/index contents/IDs and decode records through FileDataset. +Require a ready, stable committed prefix. Compare inventory after validation, +then atomically and durably publish hashes from that snapshot for base and +committed changesets and present standard sidecars. Release ownership on read-only +validation failures; retain it if durable publication becomes uncertain. +Never invoke custom hooks or replay commands; never modify data/index/log files. +This establishes trust in current validated bytes, not proof of past correctness. + +Normal inspect/load functions remain mandatory; only the named unsealed +validation entry points used by initialization can operate without a manifest. +Existing stores require this operation explicitly. Tests create manifests when +constructing fixtures, not as a side effect of opening the product. + +Evaluate mandatory startup and false-toggle rejection, same-length tampering, +index verification, missing index fallback, lifecycle and crash barriers, +initialization/rejection/no rewrite/lock cases and real command/restart behavior. +Compare repeated local latency runs against accepted 4680c76 with the same +30-command fixture; report observed variance and filesystem limitations. diff --git a/.spiral/designs/DES-20260923-TTZ7C-54.ttl b/.spiral/designs/DES-20260923-TTZ7C-54.ttl new file mode 100644 index 0000000..e0c8fd1 --- /dev/null +++ b/.spiral/designs/DES-20260923-TTZ7C-54.ttl @@ -0,0 +1,8 @@ +@prefix sd: . +@prefix dcterms: . +@prefix project: . +project:DES-20260923-TTZ7C-54 a sd:Design ; + dcterms:identifier "DES-20260923-TTZ7C-54" ; + sd:repositoryPath ".spiral/designs/DES-20260923-TTZ7C-54.md" ; + sd:status sd:Active ; + sd:derivedFrom [ a sd:ArtifactReference ; sd:artifact project:SRC-20260923-TTZ7C-52 ; sd:gitCommit "44827fde518ed8eca5fbcb606f731aa5e95d5167" ] . diff --git a/.spiral/evidence/EVD-20260923-TTZ7C-49.md b/.spiral/evidence/EVD-20260923-TTZ7C-49.md new file mode 100644 index 0000000..c95d49b --- /dev/null +++ b/.spiral/evidence/EVD-20260923-TTZ7C-49.md @@ -0,0 +1,237 @@ +--- +id: EVD-20260923-TTZ7C-49 +--- + +# File-backed runtime evaluation + +## Latest evaluation: trusted offsets and artifact hashes + +Implementation: `46fb22fe5c4fecbb23c3b8d3b59e5ff6e58e11c2`. +The same active cycle now trusts both committed offset and ID indexes in startup +inspection and loading. Earlier evaluation sections below are historical; their +scan-and-compare and duplicate-precedence descriptions have been superseded. + +- Full runtime, file-data, ID, conversion, command-offset, startup, ACID, + process-crash, handler, power-loss-baseline, storage and recovery suite: + **178/178 tests pass**, zero failures, cancellations or skips (8.44 seconds). +- Focused ESLint on all changed JavaScript modules/tests: clean, no warnings. +- Tests deliberately replace an inter-record separator outside indexed payloads. + Explicit framing scans reject it, while indexed loading and inspection succeed + and ID lookup reads the correct record. This proves the normal paths do not + silently reconstruct framing, including when IDs alone are missing. +- Shape/bounds/overlap failures reject present indexes. Structurally valid partial + indexes are trusted normally; explicit validation detects the discrepancy and + rebuild reconstructs without rewriting disk. Runtime rebuild bypasses malformed + offsets during inspection as well as loading. +- SHA-256 checks reject valid-JSON modifications to either sidecar. Missing hashes + reject present indexes under required integrity; missing indexes reconstruct + from verified canonical data. Conversion, real commands including no-op, + reopen and administrative recovery all publish and verify final index hashes. +- Disposable 328,877,813-byte / 20,000-record runtime fixture with complete + indexes passes under a 48 MB JavaScript heap limit: open 1073.8 ms, query 98.8 ms, + parent heap 9,009,096 bytes, worker heap 24,502,488 bytes, retained byte buffers + 24,823/57,562 bytes. Whole-process RSS was 189,423,616 bytes. This confirms + bounded dataset buffering, not a total RAM bound or a startup speedup. Snapshot + hashing continues reading canonical files independently of offset reconstruction. + +Logs: `/tmp/simplystore-offset-reviewed-tests.log`, +`/tmp/simplystore-offset-reviewed-lint.log`, +`/tmp/simplystore-offset-memory.log`. Existing reproduction commands below apply. +No separate hardware/QEMU power-cut experiment or repeat package-install smoke +was run for this correction. No dependency, public API or canonical data format +change was made. Human example indexes and package backup remain untouched. +Cycle acceptance and integration remain pending. + +## Earlier migration evaluation (superseded where refined above) + +Implementation: `cdc1564744989c955a187fdce4baed22ec5d0fd7` on +`spiral/CYC-20260923-TTZ7C-44-file-backed`. + +## Evaluation + +The authorized migration is implemented and ready for human review. Runtime, +query workers, command workers, startup inspection and administrative replay use +ordered file sources. The remote query/command API and stored data/log formats +remain unchanged. No production data, deployment, merge or release was changed. +Existing untracked example indexes and package backup were left untouched. + +- Full regression command below: **142 tests pass**, zero failures/cancellations + (118 existing cases plus 24 new). Runtime/startup, conversion and offsets, + ACID, process crashes, handlers, power-loss baseline, storage and administrative + recovery remain covered. Adaptations to old tests change internal source/report + fixtures; existing remote behavior and durability assertions remain. +- New regressions cover sparse overlays and record allocation, untouched IDs, + byte-free worker messages, stale/missing sidecars, UTF-8 and short reads, + malformed framing/overflow/premature EOF, source identity changes and handle + cleanup, integrity, index loader paths, ID-based command edits, no-op changes, + update ordering while workers initialize, replacement at current head, + failed update refusal, awaited shutdown, real commit/query/reopen, and finalizer + tampering. Duplicate-ID precedence follows record number even when a patch + fills an earlier hole. +- Array `map`, JAQT projection and mutation/defineProperty refusal are exercised + through the real query sandbox. Those new checks exposed vm2's temporary array + constructor mutation, which the QueryParser compatibility adapter avoids + using vm2's already-safe constructor shape. No sandbox replacement or broader + security assessment is claimed. +- Focused ESLint for all changed runtime modules, new tests and benchmark: + **zero errors and warnings**. `git diff --check` passes. +- Local package creation and installation into a fresh temporary consumer pass; + the installed package reads an indexed file. New source, documentation and + benchmark are included. The dependency resolves the pinned upstream od-jsontag + revision. No package was published. +- End-to-end generated store: **328,877,813 bytes / 20,000 records**, with a + **48 MB JavaScript heap limit**. Startup and a direct last-record ID query + pass. One run measured 17,498 ms opening and 93 ms for the query including + worker readiness. Parent retained heap: 8,905,216 bytes; query-worker heap: + 24,427,720 bytes. Array buffers: 24,823 and 57,562 bytes respectively. This is + evidence against retained dataset buffers, not a total-process-memory or + cold-storage bound. The benchmark uses and removes only a disposable fixture. +- Read-only curriculum probe: **77,952,430 bytes / 181,724 records / 180,416 IDs**, + 1,003 sampled ID reads, clean cache capped at 256 records, about 7.9 seconds + including ID reconstruction. Source SHA-256 before and after matched. No + dataset or sidecar was edited. + +```sh +node --test --test-reporter=tap test/file-data.mjs test/store-runtime.mjs \ + test/durability-startup.mjs test/conversion-offsets.mjs \ + test/command-offsets.mjs test/acid-baseline.mjs test/durability-crash.mjs \ + test/after-change-handlers.mjs test/power-loss-baseline.mjs \ + test/storage.mjs test/admin-recovery.mjs +node --expose-gc --max-old-space-size=48 scripts/benchmark-file-data.mjs +``` + +## Final read-failure correction + +DEF-20260923-TTZ7C-50 records two failing regressions introduced during final +review. Late lazy reads could be treated as ordinary command errors or swallowed +inside query code. FileParser now records read/decode/reference failure in +host-owned session state. Commands refuse publication even when a handler catches +the failure; query workers signal it to the runtime even when query code catches +it. Forged user-thrown storageFailure properties do not change that state. +The runtime stops mutation and retains ownership/recovery evidence. + +The complete suite passes 142 tests after that correction (faa3e89). The final +response-format refinement (cdc1564) additionally passes all 23 focused file-data +tests, including JSONTag error negotiation. Focused lint remains clean. Earlier +memory, curriculum and package probes establish the unchanged file-source and +distribution contracts; they are not separate security or hardware claims. + +## Persisted-index completion, 2026-09-23 + +The maintainer required persisted index loading in this version after review +identified its absence, and reported successful operation with the example +folder. SRC-45 retains that feedback. This is supporting evidence, not cycle +acceptance; the agent did not modify or rerun the live example folder. + +Implementation d9d8ff8 reads base and committed-command offset/ID sidecars. +Valid offsets are passed directly to the parser; a complete validated composed +ID map is used. Canonical framing and record headers supply validation and +fallback, without decoding every record body or writing sidecars. + +- Complete suite: **155 tests pass** (142 previous plus 13 index/lazy-body cases), + `/tmp/simplystore-index-full-final.log`. The final edit only removes a redundant + escape in a test string without changing its value; all **36 focused tests** + pass afterwards (`/tmp/simplystore-index-focused-final.log`). +- Focused ESLint on file-data, its tests and the benchmark: zero errors/warnings + (`/tmp/simplystore-index-lint-final.log`). Whitespace checks pass. +- Tests observe actual base/committed-sidecar loader calls and the returned + offset object's identity; root-only parsing is checked with a parser spy and + full-record ID reconstruction is forbidden in that case. Missing, malformed, + partial, redirected and extra index entries fall back correctly. Overlays + handle ID removal/renaming and restoration of a lower duplicate. +- Header tests cross buffer boundaries with short reads and 70 KB metadata, + Unicode, backslashes, quoted delimiters and prototype-named IDs. They compare + results with full parser reconstruction. Invalid tags fail; unread invalid + bodies raise host-owned storage failures when accessed. JSONTag's attribute + API is used as well as its tag parser, preserving attribute validation. +- Same 328,877,813-byte / 20,000-record fixture and 48 MB JS heap limit, with + independently generated persisted offsets/IDs: open **1,076.6 ms**, query + **91.2 ms**, parent heap **8,990,544 bytes**, worker heap **24,561,248 bytes**. + Retained buffers: **24,823 / 57,562 bytes**. Log: + `/tmp/simplystore-index-memory.log`. Earlier full-body ID reconstruction was + about 17.5 seconds. The main improvement is header-only validation. +- Same fixture with `--indexes=missing`: open **1,123.1 ms**, query **97.6 ms**; + passes the heap limit and ID lookup. Log: + `/tmp/simplystore-index-missing-memory.log`. +- Real curriculum file, read-only: **77,952,430 bytes / 181,724 records / 180,416 + IDs**. Loader took **1,047.2 ms**; independent full-parser ID reconstruction + took **7,436.6 ms**. The complete Maps agree and before/after source digests + match (`/tmp/simplystore-index-curriculum.log`). No dataset or sidecar changed. + +These local warm measurements do not establish a general latency or total-RAM +bound. Startup still scans every canonical byte; no new index format, trusted +certificate cache, dependency change or separate power-cut campaign was added. +The earlier fresh-package probe remains applicable to the distribution contract; +this extension changes existing packaged files only. + +## Trusted unique ID indexes, 2026-09-23 + +The maintainer corrected the prior design: committed ID indexes should be +trusted, no legacy index compatibility is needed, and duplicate IDs are errors. +SRC-45 retains those statements and the agreement to implement them. REQ-46 +and DES-47 supersede routine tag comparison and duplicate winner selection. +The current implementation is 7689e26, following 439cfa0. + +- **170/170 tests pass**, including all previous runtime, durability, recovery + and power-loss baseline tests (`/tmp/simplystore-trust-reviewed-tests.log`, + approximately 7.8 seconds). The default npm test command includes index-ids. +- Focused ESLint on changed implementation modules, conversion and changed/new + tests reports zero errors/warnings (`/tmp/simplystore-trust-reviewed-lint.log`). + Whitespace checks pass. +- Normal startup with present indexes succeeds while tag parsing and full ID + reconstruction are forbidden by test spies. Structurally valid index contents + are trusted; explicit validation catches mismatch, and explicit reconstruction + recovers in memory without changing the files. Malformed present indexes fail. +- Commit/restart cases cover same-record IDs, renames, attribute removal, atomic + swaps, ID-only changes, post-update hook renames/additions, custom finalizer + writes, no-op indexes, failed ID publication, and direct live ID lookup. +- Duplicate IDs within changed records or against untouched records fail before + the canonical changeset is written. The runtime records failed status, keeps + accepting commands and reopens with the same ID map. Conversion also rejects + duplicate IDs before canonical publication. No duplicate precedence remains. +- Actual JSONTag attribute setters initially exposed a missed-change problem: + they mutate the existing attribute container, so the dependency may not mark + an ID-only edit dirty. The commit boundary now compares materialized IDs with + the committed map before/after hooks. Tests verify untouched records are not + parsed. This fixes the required ID behavior without a dependency revision or + a broader metadata/history change. +- Finalizers may not change canonical bytes or the supplied serialization buffer. + A final-review buffer-only mutation test failed at 439cfa0; the buffer-plus-file + case already passed (`/tmp/simplystore-trust-buffer-baseline.log`). The fix + checks both against the pre-finalization digest, and the complete 170-test + suite passes. Conversion has an equivalent final-file check. +- Direct-index 328,877,813-byte / 20,000-record runtime probe with the same 48 MB + JS heap limit passes: **1,085.2 ms** open, **97.1 ms** query; parent heap + **8,997,984 bytes**, worker heap **24,485,632 bytes**, retained buffers + **24,823 / 57,562 bytes** (`/tmp/simplystore-trust-memory.log`). Missing-index + reconstruction also passes: **1,076.2 ms** open and **94.2 ms** query + (`/tmp/simplystore-trust-missing-memory.log`). These warm local measurements + establish bounded byte retention, not a speed advantage over header fallback. + +The startup probes precede the final buffer-consistency guard; that correction +only changes command finalization. Earlier real-data/package evidence is retained +as earlier evidence, not a new test of external index contents under the revised +trust contract. No user dataset or sidecar was rewritten. Canonical framing and +hashing still scan bytes at startup. No legacy index compatibility path, new +ID-index format, dependency release or hardware power-cut campaign was added. + +## Boundaries and review disposition + +Startup hashes all canonical bytes and trusts structurally valid committed ID +indexes. Missing-index fallback and explicit maintenance read record headers. Index catalogs remain O(records), handles O(source files per worker), +and mutable sessions retain touched data. Command output/custom hooks/query +results may be large. Schemas and logs remain in memory; administrative copying +still temporarily buffers individual files. These are documented boundaries, +not a constant-memory claim. The large-file result does not establish a broader +filesystem or hardware durability envelope; the separate QEMU power-cut campaign +was not repeated. + +Internal custom worker implementations must use the new source contract; +`inspectStore` returns `sources` instead of live `data`/`buffers`. Existing custom +index finalizers retain their call contract. Source files must be immutable +while open under store ownership. A Git dependency pin is used until a separate +od-jsontag npm release. Client version metadata/audit history and query sandbox +selection remain later work. + +Human outcome acceptance is pending. This Active cycle is not eligible for +integration until accepted and its exact prospective merge is validated. diff --git a/.spiral/evidence/EVD-20260923-TTZ7C-49.ttl b/.spiral/evidence/EVD-20260923-TTZ7C-49.ttl new file mode 100644 index 0000000..9e48d36 --- /dev/null +++ b/.spiral/evidence/EVD-20260923-TTZ7C-49.ttl @@ -0,0 +1,13 @@ +@prefix sd: . +@prefix dcterms: . +@prefix project: . + +project:EVD-20260923-TTZ7C-49 + a sd:VerificationEvidence ; + dcterms:identifier "EVD-20260923-TTZ7C-49" ; + sd:repositoryPath ".spiral/evidence/EVD-20260923-TTZ7C-49.md" ; + sd:status sd:Active ; + sd:verifies [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-26 ; sd:gitCommit "46fb22fe5c4fecbb23c3b8d3b59e5ff6e58e11c2" ] ; + sd:verifies [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-33 ; sd:gitCommit "46fb22fe5c4fecbb23c3b8d3b59e5ff6e58e11c2" ] ; + sd:verifies [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-34 ; sd:gitCommit "46fb22fe5c4fecbb23c3b8d3b59e5ff6e58e11c2" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:EVD-20260923-TTZ7C-49 ; sd:gitCommit "46fb22fe5c4fecbb23c3b8d3b59e5ff6e58e11c2" ] . diff --git a/.spiral/evidence/EVD-20260923-TTZ7C-55.md b/.spiral/evidence/EVD-20260923-TTZ7C-55.md new file mode 100644 index 0000000..2685874 --- /dev/null +++ b/.spiral/evidence/EVD-20260923-TTZ7C-55.md @@ -0,0 +1,85 @@ +# Mandatory integrity evaluation + +Implementation: `1e119dd101dd231a7cf24ba1b9d6c3e4999e8f60` on +spiral/CYC-20260923-TTZ7C-51-required-integrity. The confirmed change is implemented +and ready for review; acceptance remains pending. It continues reliability work +from accepted CYC-44 without integrating either branch into master. + +## Behavior and correctness + +- All normal opens/loaders/inspections require a manifest and required hashes; + previous false toggles cannot disable verification. Conversion creates the + manifest without --integrity. Location remains configurable. +- Commands append final canonical/index digests as one durable batch; canonical + digest reuse avoids redundant hashing at the manifest boundary. Recovery defers + publication to its coordinator, checking retained expected hashes first. +- The explicit stopped-store initializer acquires ownership, checks committed + history, framing, available indexes, IDs and record contents, syncs the prefix, + rechecks the inventory and atomically publishes its hashes. It does not execute + handlers, replay commands, rewrite data/logs/indexes, or replace a manifest. +- Invalid/stale indexes, duplicate IDs, malformed unread bodies, pending history, + unexplained files, existing ownership, and changing files reject initialization. + Prepublication validation failure releases ownership; uncertain publication + retains it. Missing indexes remain reconstructable from verified data. +- The manifest synchronization failure test rejects command success. The batching + test counts one manifest append/open and one data synchronization for all three + artifact hashes, alongside the existing parent-directory synchronization. + +Full regression run: **193/193 passing**, zero failures/cancellations/skips, +11.54 seconds. This includes runtime/loader/indexes, conversion, commands, startup, +ACID, process crashes, handler behavior, power-loss baseline, storage and recovery, +plus 15 explicit required-integrity/initialization tests. Focused lint across all +changed JavaScript modules/tests is clean. Existing fixtures now create hashes +explicitly; missing committed recovery outputs retain their expected digests. + +Run the full suite with: + +```sh +node --test --test-reporter=tap test/required-integrity.mjs test/file-data.mjs test/index-ids.mjs test/store-runtime.mjs test/durability-startup.mjs test/conversion-offsets.mjs test/command-offsets.mjs test/acid-baseline.mjs test/durability-crash.mjs test/after-change-handlers.mjs test/power-loss-baseline.mjs test/storage.mjs test/admin-recovery.mjs +``` + +## Existing example and large-file checks + +A disposable byte-for-byte copy of the maintainer's current example initialized +successfully: six manifest entries, 269 IDs, committed command +62262f33-41b7-483c-ae12-0258f5d85dc6 retained. Normal inspection and loading then +succeeded. Every copied original file was compared with its source afterwards; +the actual example files were unchanged. The real example still requires the +explicit npm run init-integrity operation while stopped. No manifest was written +to the maintainer's store as part of verification. + +The 328,877,813-byte / 20,000-record runtime benchmark passed with required hashes +under --max-old-space-size=48: open 1115.4 ms, query 106.6 ms, parent heap +9,082,592 bytes, worker heap 24,848,704 bytes, retained buffers 24,823/57,562 bytes. +Whole-process RSS 189,829,120 bytes. This is a JavaScript-heap/buffering result, +not a 48 MB total RAM guarantee or a controlled large-file speed comparison. +Reproduce with node --expose-gc --max-old-space-size=48 scripts/benchmark-file-data.mjs. + +## Command latency comparison + +Baseline: accepted 4680c76, archived src/test/package.json under /tmp with the +same node_modules, integrity disabled. Candidate: required integrity with batched +appends. Three alternating baseline/candidate runs, 30 commands each, same +benchmark and local ext4 directory selected through TMPDIR; Node 24.7.0. +The benchmark includes server startup, HTTP acceptance, a fresh command worker, +command completion and 2 ms polling. It is not isolated checksum CPU timing. + +| Run | Baseline p50 completion | Required p50 completion | Baseline p95 | Required p95 | +| --- | ---: | ---: | ---: | ---: | +| 1 | 54.57 ms | 57.13 ms | 64.85 ms | 73.23 ms | +| 2 | 55.91 ms | 57.05 ms | 63.14 ms | 62.88 ms | +| 3 | 55.09 ms | 56.26 ms | 65.94 ms | 63.44 ms | + +Median of run medians: 55.09 vs 57.05 ms, approximately 1.95 ms (3.5%) more. +Startup ranged 152.6–153.9 vs 153.6–154.3 ms. Acceptance medians ranged 2.95–2.98 +vs 2.91–2.98 ms. Ninety commands per mode provide bounded local evidence; p95 +variance and different storage/command workloads preclude a universal overhead +claim. No per-query hashing was introduced. Fixture cleanup removed the ext4 +benchmark directory. + +Logs: /tmp/simplystore-required-reviewed-tests.log, +/tmp/simplystore-required-reviewed-lint.log, /tmp/simplystore-required-memory.log, +and /tmp/simplystore-latency-optional-N.log / required-N.log (N=1,2,3). +No separate QEMU/hardware power-cut campaign or package-install smoke was repeated. +Existing crash/storage tests remain the durability regression boundary. No new +dependency, canonical format, audit feature, deployment or merge was introduced. diff --git a/.spiral/evidence/EVD-20260923-TTZ7C-55.ttl b/.spiral/evidence/EVD-20260923-TTZ7C-55.ttl new file mode 100644 index 0000000..8501cab --- /dev/null +++ b/.spiral/evidence/EVD-20260923-TTZ7C-55.ttl @@ -0,0 +1,9 @@ +@prefix sd: . +@prefix dcterms: . +@prefix project: . +project:EVD-20260923-TTZ7C-55 a sd:VerificationEvidence ; + dcterms:identifier "EVD-20260923-TTZ7C-55" ; + sd:repositoryPath ".spiral/evidence/EVD-20260923-TTZ7C-55.md" ; + sd:status sd:Active ; + sd:verifies [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-26 ; sd:gitCommit "1e119dd101dd231a7cf24ba1b9d6c3e4999e8f60" ] ; + sd:verifies [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-33 ; sd:gitCommit "1e119dd101dd231a7cf24ba1b9d6c3e4999e8f60" ] . diff --git a/.spiral/implementations/IMP-20260919-TTZ7C-26.md b/.spiral/implementations/IMP-20260919-TTZ7C-26.md index 025b556..2873002 100644 --- a/.spiral/implementations/IMP-20260919-TTZ7C-26.md +++ b/.spiral/implementations/IMP-20260919-TTZ7C-26.md @@ -119,3 +119,95 @@ retain ownership for assessment; successful initialization releases it. Regression coverage converts the actual bundled dataset, verifies all final record offsets and reads typed URL values, including a linked homeworld. No canonical format, storage barrier or existing-store overwrite policy changes. + +## Cycle 44: File-backed durability integration + +Command workers now open immutable file sources in a fresh mutable session, +serialize only their changes into an ordinary Buffer for existing index hooks, +and return a scanned source descriptor. Persist/write/finalize/integrity ordering +is preserved. A changed canonical file after finalization is refused. Default +index loaders now use the supplied metadata and correct UUID filename. + +Inspection hashes and scans files using bounded byte buffers and returns plain +source descriptors. Replay opens those sources instead of compacting buffers; +retained expected digests are checked against the resulting file bytes before +replacement integrity and completion records. Existing administrative copy +operations still buffer an individual file temporarily; this is outside the +retrieval/replay memory claim. + +### Late read failures + +DEF-20260923-TTZ7C-50 corrects a failure-classification gap introduced by lazy +reads. FileParser retains read/decode/reference failures in host-owned state. +Commands refuse publication even if a handler catches the error; query workers +report storage failure even if query code catches it. User-thrown error flags +cannot forge parser state. Runtime marks storage failure and refuses further +mutations, preserving ownership and administrative recovery evidence. + +## Trusted unique ID indexes + +The confirmed correction replaces routine header reconstruction with direct +committed ID-index loading. Missing indexes reconstruct; explicit validation +and reconstruction are available as store/loader options. Malformed present +indexes fail. Overlay application clears mappings for replaced records, then +adds unique IDs. Duplicate IDs are errors, not precedence cases. + +The core prepares ID entries from final serialized bytes and checks uniqueness +before data publication. It awaits custom finalization, writes the ID sidecar +durably and returns matching live metadata. Conversion follows the same rules. +ID-only mutations are detected among materialized records before/after hooks. +Expected digests captured before finalizers prevent later canonical changes +from leaving successful publication with stale indexes. No legacy index path, +new index format or od-jsontag revision is needed. + +Tests cover normal loading with tag parsing forbidden, ID-only edits without +reading untouched records, renames, removals, swaps, duplicate rejection before +publication, custom-hook final IDs, index-write failure, runtime/restart +agreement, conversion, and explicit maintenance. + +Final review checks both the published file and the mutable serialization buffer +against the digest captured before custom finalization. Otherwise a finalizer +could change only the buffer and cause a later integrity entry to hash bytes +different from the published file. Two regression cases cover buffer-only and +buffer-plus-file mutation; the former reproduced the gap before correction. + +## Direct offset loading and artifact hashes + +The offset correction shares loadDataSource across startup inspection, loading, +command publication and recovery. Present offsets are checked for structural +ranges/bounds, then used directly. Only missing offsets or explicit maintenance +reconstruct framing. Missing IDs use stored ranges to read record headers. +Malformed present indexes fail; rebuild ignores them without changing disk. + +index-files.mjs hashes the exact index bytes parsed against the existing manifest. +When integrity is enabled, conversion, commands and recovery append final ID and +offset hashes before success/done. Present indexes need hashes; missing files +can reconstruct from verified canonical data. Canonical hashing and file identity +checks remain bounded and independent of index reconstruction. + +Regression coverage includes bypassing framing in both loader and inspection, +structurally valid content trust, malformed bounds, explicit rebuild, hash +mismatches/missing entries, conversion hashes, committed/no-op sidecars through +runtime reopen, and recovery of a fingerprinted prefix. + +## Mandatory integrity and initialization (CYC-51) + +DES-20260923-TTZ7C-54 supersedes optional verification in the earlier design. +Runtime, inspection and file loading always resolve and require a manifest; +former false settings cannot bypass checks. Conversion always publishes hashes. +Each command appends its already verified data digest and final index digests +as one durable batch before done. Recovery owns the deferred append and checks +retained expected digests before replacement entries. + +initializeIntegrity and recover.mjs init-integrity validate a stopped existing +store, including index contents and record decoding, under ownership. They +refuse existing manifests, stale indexes, uncertain history and changing files. +They sync the committed prefix, check the final inventory, and atomically publish +its baseline without rewriting data or replaying commands. Publication uncertainty +retains ownership; prepublication validation errors release it. Named unsealed +validation helpers are confined to this explicit administrative path. + +The example exposes npm run init-integrity with a declarative store.json. +Regression coverage and disposable-copy example verification preserve existing +commands. Performance comparison uses the accepted optional baseline and identical +30-command runs; canonical hashes are reused and index entries share one barrier. diff --git a/.spiral/implementations/IMP-20260919-TTZ7C-26.ttl b/.spiral/implementations/IMP-20260919-TTZ7C-26.ttl index 718ba4f..7827b87 100644 --- a/.spiral/implementations/IMP-20260919-TTZ7C-26.ttl +++ b/.spiral/implementations/IMP-20260919-TTZ7C-26.ttl @@ -14,7 +14,13 @@ project:IMP-20260919-TTZ7C-26 a sd:Implementation ; sd:implementationLocation "src/store-ownership.mjs" ; sd:implementationLocation "src/admin-recovery.mjs" ; sd:implementationLocation "src/index.offset.mjs" ; + sd:implementationLocation "src/index-files.mjs" ; + sd:implementationLocation "src/integrity.mjs" ; + sd:implementationLocation "src/initialize-integrity.mjs" ; + sd:implementationLocation "test/required-integrity.mjs" ; + sd:implementationLocation "example/store.json" ; sd:implementationLocation "src/index.id.mjs" ; + sd:implementationLocation "src/index.mjs" ; sd:implementationLocation "src/recovery.mjs" ; sd:implementationLocation "scripts/convert.mjs" ; sd:implementationLocation "example/package.json" ; @@ -33,6 +39,9 @@ project:IMP-20260919-TTZ7C-26 a sd:Implementation ; sd:implements [ a sd:ArtifactReference ; sd:artifact project:DES-003 ; sd:gitCommit "59179b6c93b7e21c3b2c69db46c53fbe706927c3" ] ; sd:implements [ a sd:ArtifactReference ; sd:artifact project:DES-20260917-TTZ7C-14 ; sd:gitCommit "59179b6c93b7e21c3b2c69db46c53fbe706927c3" ] ; sd:implements [ a sd:ArtifactReference ; sd:artifact project:DES-20260819-09ZEF-7 ; sd:gitCommit "59179b6c93b7e21c3b2c69db46c53fbe706927c3" ] ; - sd:changeCausedBy [ a sd:ArtifactReference ; sd:artifact project:SRC-20260919-TTZ7C-18 ; sd:gitCommit "9b4a340facd57f22edbead07a195700b0237c9ad" ] ; - sd:transforms [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-26 ; sd:gitCommit "9b4a340facd57f22edbead07a195700b0237c9ad" ] ; + sd:implements [ a sd:ArtifactReference ; sd:artifact project:DES-20260923-TTZ7C-47 ; sd:gitCommit "83edb03bc406b9e651ef93bed69b87ece17dcf47" ] ; + sd:changeCausedBy [ a sd:ArtifactReference ; sd:artifact project:DEF-20260923-TTZ7C-50 ; sd:gitCommit "4bb10c2013fd2a7cb989b02586b992b936121362" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-26 ; sd:gitCommit "c8a3fd9464abfd5e7b699a2752789d82af4da2da" ] ; + sd:implements [ a sd:ArtifactReference ; sd:artifact project:DES-20260923-TTZ7C-54 ; sd:gitCommit "c8a3fd9464abfd5e7b699a2752789d82af4da2da" ] ; + sd:changeCausedBy [ a sd:ArtifactReference ; sd:artifact project:REQ-20260923-TTZ7C-53 ; sd:gitCommit "c8a3fd9464abfd5e7b699a2752789d82af4da2da" ] ; sd:implementationChangeKind sd:SemanticChange . diff --git a/.spiral/implementations/IMP-20260919-TTZ7C-33.md b/.spiral/implementations/IMP-20260919-TTZ7C-33.md index 704e8e2..978ee00 100644 --- a/.spiral/implementations/IMP-20260919-TTZ7C-33.md +++ b/.spiral/implementations/IMP-20260919-TTZ7C-33.md @@ -101,3 +101,116 @@ and shares cleanup failure; both persistence-boundary failures reject acceptance known ID-only retries return status while incomplete new commands fail; and a real process receiving both shutdown signals commits, unlocks and restarts. EVD-20260919-TTZ7C-41 preserves the baseline and records final suite results. + +## Cycle 44: File-backed runtime and reader lifetime + +DES-20260923-TTZ7C-47 changes the internal runtime representation to ordered file +sources, with file paths, sparse offsets, immutable file identity and SHA-256. +FileDataset owns parser and descriptors; scanDataFile validates framing and +hashes with a 64 KiB buffer. Startup builds IDs from final records without +requiring trustworthy sidecars. Runtime/query/command messages contain sources, +not dataset bytes. Query and command ID lookups use the complete record catalog. + +The worker pool queues updates ahead of new queries, independently for each +worker, and initializes replacements with the current source list. Failed source +updates reject further queries and stop command acceptance. Shutdown awaits +termination (including workers already being replaced) before ownership release. + +A QueryParser specialization gives raw query arrays a non-configurable undefined +constructor before proxy wrapping. Local vm2 3.12.2 recognizes this as its +already-neutralized array-species case. Tests cover map, JAQT, and refused array +mutation/definition. This preserves the query API without weakening immutable +traps or changing the sandbox implementation. It is not a sandbox safety claim. + +Existing tests were adapted only where internal messages/report fields changed; +remote results, recovery eligibility, ordering and failure expectations remain. +New tests cover unread IDs, sparse overlays, corrupt framing, short reads/EOF, +file changes and closure, byte-free messages, index loaders, real runtime restart, +failed publication, worker replacement, and asynchronous update ordering. + +### Late read failures + +DEF-20260923-TTZ7C-50 corrects a failure-classification gap introduced by lazy +reads. FileParser retains read/decode/reference failures in host-owned state. +Commands refuse publication even if a handler catches the error; query workers +report storage failure even if query code catches it. User-thrown error flags +cannot forge parser state. Runtime marks storage failure and refuses further +mutations, preserving ownership and administrative recovery evidence. + +Storage read error responses retain the negotiated JSON/JSONTag format. + +## Persisted index completion + +The loader reads base and committed-command offset/ID sidecars. Complete offset +maps must match canonical framing. Composed persisted IDs must match final +record-header IDs; otherwise loading reconstructs the map. The scan validates +metadata through JSONTag's tag parser and attribute API without decoding every +body. Sparse overlays remove/rename IDs and preserve higher-record duplicate +precedence. No sidecar or canonical file is rewritten. + +The benchmark now generates independent persisted indexes by default and also +supports --indexes=missing. Focused regressions cover actual loader calls and +use of persisted offsets, unread bodies, fallback, source order, ID overlays, +Unicode/buffer boundaries, malformed tags and lazy body failures. + +## Trusted unique ID indexes + +The confirmed correction replaces routine header reconstruction with direct +committed ID-index loading. Missing indexes reconstruct; explicit validation +and reconstruction are available as store/loader options. Malformed present +indexes fail. Overlay application clears mappings for replaced records, then +adds unique IDs. Duplicate IDs are errors, not precedence cases. + +The core prepares ID entries from final serialized bytes and checks uniqueness +before data publication. It awaits custom finalization, writes the ID sidecar +durably and returns matching live metadata. Conversion follows the same rules. +ID-only mutations are detected among materialized records before/after hooks. +Expected digests captured before finalizers prevent later canonical changes +from leaving successful publication with stale indexes. No legacy index path, +new index format or od-jsontag revision is needed. + +Tests cover normal loading with tag parsing forbidden, ID-only edits without +reading untouched records, renames, removals, swaps, duplicate rejection before +publication, custom-hook final IDs, index-write failure, runtime/restart +agreement, conversion, and explicit maintenance. + +## Direct offset loading and artifact hashes + +The offset correction shares loadDataSource across startup inspection, loading, +command publication and recovery. Present offsets are checked for structural +ranges/bounds, then used directly. Only missing offsets or explicit maintenance +reconstruct framing. Missing IDs use stored ranges to read record headers. +Malformed present indexes fail; rebuild ignores them without changing disk. + +index-files.mjs hashes the exact index bytes parsed against the existing manifest. +When integrity is enabled, conversion, commands and recovery append final ID and +offset hashes before success/done. Present indexes need hashes; missing files +can reconstruct from verified canonical data. Canonical hashing and file identity +checks remain bounded and independent of index reconstruction. + +Regression coverage includes bypassing framing in both loader and inspection, +structurally valid content trust, malformed bounds, explicit rebuild, hash +mismatches/missing entries, conversion hashes, committed/no-op sidecars through +runtime reopen, and recovery of a fingerprinted prefix. + +## Mandatory integrity and initialization (CYC-51) + +DES-20260923-TTZ7C-54 supersedes optional verification in the earlier design. +Runtime, inspection and file loading always resolve and require a manifest; +former false settings cannot bypass checks. Conversion always publishes hashes. +Each command appends its already verified data digest and final index digests +as one durable batch before done. Recovery owns the deferred append and checks +retained expected digests before replacement entries. + +initializeIntegrity and recover.mjs init-integrity validate a stopped existing +store, including index contents and record decoding, under ownership. They +refuse existing manifests, stale indexes, uncertain history and changing files. +They sync the committed prefix, check the final inventory, and atomically publish +its baseline without rewriting data or replaying commands. Publication uncertainty +retains ownership; prepublication validation errors release it. Named unsealed +validation helpers are confined to this explicit administrative path. + +The example exposes npm run init-integrity with a declarative store.json. +Regression coverage and disposable-copy example verification preserve existing +commands. Performance comparison uses the accepted optional baseline and identical +30-command runs; canonical hashes are reused and index entries share one barrier. diff --git a/.spiral/implementations/IMP-20260919-TTZ7C-33.ttl b/.spiral/implementations/IMP-20260919-TTZ7C-33.ttl index bee07a8..3e92bce 100644 --- a/.spiral/implementations/IMP-20260919-TTZ7C-33.ttl +++ b/.spiral/implementations/IMP-20260919-TTZ7C-33.ttl @@ -24,18 +24,16 @@ project:IMP-20260919-TTZ7C-33 sd:artifact project:DES-20260919-TTZ7C-40 ; sd:gitCommit "665a97d1c366b4cce2b9004d6e649a389d18a2c4" ] ; - sd:changeCausedBy [ - a sd:ArtifactReference ; - sd:artifact project:EVD-20260919-TTZ7C-41 ; - sd:gitCommit "959e2cd92181be07f3fe38d38328f10ba41923b3" - ], [ - a sd:ArtifactReference ; - sd:artifact project:DES-20260919-TTZ7C-40 ; - sd:gitCommit "665a97d1c366b4cce2b9004d6e649a389d18a2c4" - ] ; - sd:transforms [ - a sd:ArtifactReference ; - sd:artifact project:IMP-20260919-TTZ7C-33 ; - sd:gitCommit "959e2cd92181be07f3fe38d38328f10ba41923b3" - ] ; - sd:implementationChangeKind sd:MixedChange . + sd:implementationLocation "src/file-data.mjs" ; + sd:implementationLocation "src/load-worker.mjs" ; + sd:implementationLocation "src/query-worker-module.mjs" ; + sd:implementationLocation "src/workerPool.mjs" ; + sd:implementationLocation "test/file-data.mjs" ; + sd:implementationLocation "test/index-ids.mjs" ; + sd:implementationLocation "scripts/benchmark-file-data.mjs" ; + sd:implements [ a sd:ArtifactReference ; sd:artifact project:DES-20260923-TTZ7C-47 ; sd:gitCommit "83edb03bc406b9e651ef93bed69b87ece17dcf47" ] ; + sd:changeCausedBy [ a sd:ArtifactReference ; sd:artifact project:DEF-20260923-TTZ7C-50 ; sd:gitCommit "4bb10c2013fd2a7cb989b02586b992b936121362" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-33 ; sd:gitCommit "c8a3fd9464abfd5e7b699a2752789d82af4da2da" ] ; + sd:implements [ a sd:ArtifactReference ; sd:artifact project:DES-20260923-TTZ7C-54 ; sd:gitCommit "c8a3fd9464abfd5e7b699a2752789d82af4da2da" ] ; + sd:changeCausedBy [ a sd:ArtifactReference ; sd:artifact project:REQ-20260923-TTZ7C-53 ; sd:gitCommit "c8a3fd9464abfd5e7b699a2752789d82af4da2da" ] ; + sd:implementationChangeKind sd:SemanticChange . diff --git a/.spiral/implementations/IMP-20260919-TTZ7C-34.md b/.spiral/implementations/IMP-20260919-TTZ7C-34.md index a93abf0..7fb571c 100644 --- a/.spiral/implementations/IMP-20260919-TTZ7C-34.md +++ b/.spiral/implementations/IMP-20260919-TTZ7C-34.md @@ -32,3 +32,12 @@ A clean temporary repository copy installed both packages, built the example, started it, served a JSON query, accepted an example UUID command and reported that command `done`. Existing generated files in the working example were not modified. + +## Cycle 44: Reviewed file parser dependency + +Pin od-jsontag to reviewed upstream Git commit +24f47eb616c3a24391487197496804ed1c56837b. The published 0.4.6 version lacks the +file-backed API; using a range on that unchanged version would not reproduce +this implementation. No other dependency is upgraded, no npm version is bumped, +and no package is published. The default test command includes file-data tests; +packaged documentation includes the internal contract and measured boundaries. diff --git a/.spiral/implementations/IMP-20260919-TTZ7C-34.ttl b/.spiral/implementations/IMP-20260919-TTZ7C-34.ttl index 7d096b5..edd4627 100644 --- a/.spiral/implementations/IMP-20260919-TTZ7C-34.ttl +++ b/.spiral/implementations/IMP-20260919-TTZ7C-34.ttl @@ -17,9 +17,7 @@ project:IMP-20260919-TTZ7C-34 sd:artifact project:DES-20260919-TTZ7C-32 ; sd:gitCommit "dd0382ef49a50cada7a3699cb021df0929d02315" ] ; - sd:transforms [ - a sd:ArtifactReference ; - sd:artifact project:IMP-20260919-TTZ7C-34 ; - sd:gitCommit "fe516f08163642939e3c7478d5eb7b851de273ab" - ] ; - sd:implementationChangeKind sd:BehaviorPreservingChange . + sd:implements [ a sd:ArtifactReference ; sd:artifact project:DES-20260923-TTZ7C-47 ; sd:gitCommit "9b8acd229291fa75423c45a9019245edc0c19ea9" ] ; + sd:changeCausedBy [ a sd:ArtifactReference ; sd:artifact project:DES-20260923-TTZ7C-47 ; sd:gitCommit "9b8acd229291fa75423c45a9019245edc0c19ea9" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:IMP-20260919-TTZ7C-34 ; sd:gitCommit "c4571b4389835302b47b4096d6f0c3cb77b62f1f" ] ; + sd:implementationChangeKind sd:MixedChange . diff --git a/.spiral/project-context.md b/.spiral/project-context.md index 1532c75..a6f5eee 100644 --- a/.spiral/project-context.md +++ b/.spiral/project-context.md @@ -18,12 +18,43 @@ Project causal-graph namespace: `https://github.com/simplyedit/simplystore/spira Spiral core source: `.spiral-core/`, git submodule for `https://github.com/muze-labs/spiral-developer.git`, currently checked out at `8d4b2c738a413abd4cccca740ce958f486e5f7af`. -Latest accepted cycle: CYC-20260919-TTZ7C-42, Updated Roadmap And Planning -Context, on spiral/CYC-20260919-TTZ7C-42-roadmap. The maintainer accepted the -evaluated branch tip 10a39df3e946944a168614fd59514f2aeb4aec05 on 2026-09-23. -The documentation cycle is closed; integration into master remains a separate -decision. No next product task is selected; Spiral risk analysis selects it -using ROADMAP.md and current evidence. +Active cycle: none. Latest accepted cycle: CYC-20260923-TTZ7C-51, required +integrity and existing-store initialization, accepted on 2026-09-23, awaiting +integration on spiral/CYC-20260923-TTZ7C-51-required-integrity. This continues +the reliability roadmap from the accepted file-backed tip while integration is +pending. The maintainer explicitly confirmed mandatory hashes, initialization +and latency measurement. Implementation and evaluation are complete: 193 tests +pass, lint is clean, the copied example retains its command after initialization, +and local ext4 median completion increased roughly 2 ms. EVD-55 retains evidence. +Normal startup now requires hashes; existing stores use explicit init-integrity. +The maintainer confirmed startup refusal without integrity, successful +initialization and working commands. Acceptance is retained in CYC-51. +Both CYC-44 and CYC-51 are accepted; integration remains a separate decision. +No next cycle is selected. + +Previously accepted cycle: CYC-20260923-TTZ7C-44, +File-backed dataset runtime, accepted on 2026-09-23, awaiting integration on +spiral/CYC-20260923-TTZ7C-44-file-backed. The maintainer authorized replacing +shared-memory data after reviewing od-jsontag hardening. This continues the +file-only roadmap direction, prioritized ahead of rebuild. SRC-45, UND-48, +REQ-46 and DES-47 (20260923-TTZ7C namespace) retain the commitment. Implementation +uses ordered file sources, bounded clean-record caches, worker-owned handles, +bounded hashing and file-backed administrative replay. Committed offset and ID +indexes are trusted and loaded directly; missing indexes reconstruct, and +validation/reconstruction are explicit maintenance options. ID indexes are +finalized from serialized bytes and durable before success. Integrity manifests +cover canonical data and present standard index files. Duplicate IDs are +errors; no legacy index compatibility is required. The maintainer verified the +example folder and explicitly confirmed these corrections. +EVD-20260923-TTZ7C-49 records 178 passing tests, clean lint and large-file probes. +The maintainer confirmed the tests, successful example command application and +persistence after restart. Acceptance is retained in the cycle record. +Integration remains pending; accepted CYC-51 adds mandatory integrity. +Client-visible version metadata and audit trails follow this migration; +neither is implemented here. + +Latest integrated cycle: CYC-20260919-TTZ7C-42, Updated Roadmap And Planning +Context. Accepted on 2026-09-23 and integrated through PR #24 at dbb0703. Previously accepted and integrated cycle: CYC-20260919-TTZ7C-36, Reliable Runtime Contracts And Readable Workflows, on spiral/CYC-20260919-TTZ7C-36-runtime-contracts. The maintainer diff --git a/.spiral/project.ttl b/.spiral/project.ttl index 25585ee..cba4391 100644 --- a/.spiral/project.ttl +++ b/.spiral/project.ttl @@ -15,11 +15,8 @@ project:CTX-001 sd:artifact project:SRC-20260919-TTZ7C-43 ; sd:gitCommit "79fcf670d4558c75b9933743e6f2ea5616221eb0" ] ; - sd:transforms [ - a sd:ArtifactReference ; - sd:artifact project:CTX-001 ; - sd:gitCommit "b482053bdf0d3b3fc8902cce3169408d31ed9f6e" - ] ; + sd:derivedFrom [ a sd:ArtifactReference ; sd:artifact project:SRC-20260923-TTZ7C-45 ; sd:gitCommit "46fb22fe5c4fecbb23c3b8d3b59e5ff6e58e11c2" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:CTX-001 ; sd:gitCommit "9515993531c55176e1c3444f71dc3a23d5172df0" ] ; sd:developerEvaluationReadinessBar "Developers should be able to evaluate bounded durability claims from invariants, executable tests, evidence artifacts, explicit failure behavior, and documented known gaps." ; sd:hasAudience project:AUD-DEVELOPERS, project:AUD-END-USERS, project:AUD-OPERATORS ; sd:hasDownstreamProject project:DOWNSTREAM-CURRICULUM-STORE ; diff --git a/.spiral/requests/REQ-20260923-TTZ7C-46.md b/.spiral/requests/REQ-20260923-TTZ7C-46.md new file mode 100644 index 0000000..27a55cd --- /dev/null +++ b/.spiral/requests/REQ-20260923-TTZ7C-46.md @@ -0,0 +1,51 @@ +--- +id: REQ-20260923-TTZ7C-46 +--- + +# File-backed operation with durable compatibility + +Replace retained/transferred dataset byte buffers with ordered immutable file +sources. Queries lazily read indexed records, including ID lookups before any +traversal. Commands use the exact committed prefix and publish new sources only +after durable completion. Reinitialization, failure, timeout and shutdown release +owned handles. Administrative replay uses the same source contract. + +Preserve remote query/command behavior, disk framing and log-authoritative +ordering, integrity validation, custom index finalization and recovery rules. +Missing indexes remain reconstructable; offset framing and configured data +integrity checks remain in force. Validation must not load whole +data files merely to hash them. No production data rewrites or format migration. + +Evidence: existing regression suite plus direct file lifecycle, partial-ID-read, +sparse update, worker publication/replacement, malformed framing/integrity and +large-file memory tests. Internal worker messages may change. New dependency +revision must be reproducible. No sandbox/audit/rebuild feature in this cycle. + +## Trusted committed ID indexes + +Normal startup reads index.id.json and committed-command ID sidecars directly, +without extracting record IDs again. Make index publication reliable instead of +reconstructing IDs to validate them on every startup. ID indexes describe final +serialized bytes after mutation hooks and must be durable before command success. +Index-write failure must prevent success. Runtime ID lookup and restart must agree. + +Replacing a record removes its previous ID mapping before applying the new one. +Renames, ID removal and atomic ID swaps must work. Duplicate IDs on distinct +current records are errors; replacement of the same record may retain its ID. +Reject duplicate-ID commands before publishing their canonical changeset. + +Reconstruct missing indexes. Provide explicit validation and reconstruction for +maintenance; present malformed indexes fail clearly. Structurally valid indexes +are trusted in normal operation, including completeness. No silent content +comparison or legacy index compatibility is required. The maintainer explicitly +confirmed this is not yet used. Preserve OD-JSONTag/log formats and ordinary +custom hook calls; canonical ID publication belongs to the core commit boundary. + +## Offset trust correction, 2026-09-23 + +Load present offset indexes directly in startup inspection and loading. Check +only shape, safe record numbers, ordered nonoverlapping ranges and file bounds. +Missing indexes may reconstruct; malformed present indexes fail. Explicit +validateIndexes compares contents and rebuildIndexes ignores persisted indexes. +Extend configured integrity manifests to canonical data and present standard +ID/offset files, verifying their hashes before use. Preserve duplicate-ID errors. diff --git a/.spiral/requests/REQ-20260923-TTZ7C-46.ttl b/.spiral/requests/REQ-20260923-TTZ7C-46.ttl new file mode 100644 index 0000000..762a6ea --- /dev/null +++ b/.spiral/requests/REQ-20260923-TTZ7C-46.ttl @@ -0,0 +1,12 @@ +@prefix sd: . +@prefix dcterms: . +@prefix project: . + +project:REQ-20260923-TTZ7C-46 + a sd:Request ; + dcterms:identifier "REQ-20260923-TTZ7C-46" ; + sd:repositoryPath ".spiral/requests/REQ-20260923-TTZ7C-46.md" ; + sd:status sd:Active ; + sd:derivedFrom [ a sd:ArtifactReference ; sd:artifact project:UND-20260923-TTZ7C-48 ; sd:gitCommit "c276b42cbe89a3c7204dd26f1c66522e59553b90" ] ; + sd:changeCausedBy [ a sd:ArtifactReference ; sd:artifact project:SRC-20260923-TTZ7C-45 ; sd:gitCommit "7e1ea39dc4056beec311c8e6d832d97d2a550d58" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:REQ-20260923-TTZ7C-46 ; sd:gitCommit "7e1ea39dc4056beec311c8e6d832d97d2a550d58" ] . diff --git a/.spiral/requests/REQ-20260923-TTZ7C-53.md b/.spiral/requests/REQ-20260923-TTZ7C-53.md new file mode 100644 index 0000000..f2c05d8 --- /dev/null +++ b/.spiral/requests/REQ-20260923-TTZ7C-53.md @@ -0,0 +1,16 @@ +# Mandatory integrity + +All normal conversion, store opening, file loading and committed publication +require hashes of canonical data and present standard ID/offset sidecars. +There is no opt-out; only the manifest location remains configurable. +Keep direct index trust, missing-index reconstruction and explicit maintenance. +Hash mismatch or missing required evidence fails before successful publication. + +Provide explicit initialization for stopped existing stores lacking a manifest. +Validate committed history, framing, IDs and available index consistency; do not +replay commands, rewrite data, replace an existing manifest or claim to establish +historical correctness. Refuse uncertain stores and concurrent ownership. + +Batch each committed artifact set into one durable manifest append; reuse the +already verified canonical digest. Measure added command latency against the +accepted optional implementation. Preserve durability, recovery and user files. diff --git a/.spiral/requests/REQ-20260923-TTZ7C-53.ttl b/.spiral/requests/REQ-20260923-TTZ7C-53.ttl new file mode 100644 index 0000000..f232e03 --- /dev/null +++ b/.spiral/requests/REQ-20260923-TTZ7C-53.ttl @@ -0,0 +1,8 @@ +@prefix sd: . +@prefix dcterms: . +@prefix project: . +project:REQ-20260923-TTZ7C-53 a sd:Request ; + dcterms:identifier "REQ-20260923-TTZ7C-53" ; + sd:repositoryPath ".spiral/requests/REQ-20260923-TTZ7C-53.md" ; + sd:status sd:Active ; + sd:derivedFrom [ a sd:ArtifactReference ; sd:artifact project:SRC-20260923-TTZ7C-52 ; sd:gitCommit "44827fde518ed8eca5fbcb606f731aa5e95d5167" ] . diff --git a/.spiral/sources/SRC-20260923-TTZ7C-45.md b/.spiral/sources/SRC-20260923-TTZ7C-45.md new file mode 100644 index 0000000..94670a1 --- /dev/null +++ b/.spiral/sources/SRC-20260923-TTZ7C-45.md @@ -0,0 +1,74 @@ +--- +id: SRC-20260923-TTZ7C-45 +--- + +# File-backed migration authorization + +The maintainer first requested file-backed data before client-visible version +metadata and audit trails, and agreed to harden od-jsontag first. After reviewing +the dependency changes and the explanation of required SimplyStore integration, +the maintainer stated on 2026-09-23: + +> Read through it, and the code is readable and well supported by tests. Lets continue with simplystore and switch to filebased data instead of shared memory. + +The preceding explanation covered file sources and offset indexes, descriptor +lifetime, ID lookup independent of the proxy cache, index loaders, integrity, +command isolation, and streamed large output. This is explicit authorization to +implement that integration. It continues the roadmap file-only direction, with +priority ahead of rebuild as previously requested. It does not authorize a merge, +release, production-data mutation, new sandbox, or audit feature. + +Reviewed dependency: od-jsontag commit +24f47eb616c3a24391487197496804ed1c56837b, available on its upstream indexedParsing +branch. The library keeps format/API compatibility and has bounded clean-record +caching; indexes and mutable sessions still grow with their contents. + +## Persisted-index completion, 2026-09-23 + +After the assistant explained that startup still scanned canonical files and +rebuilt IDs, and that persisted index loading with validation/fallback remained +unimplemented, the maintainer replied: + +> I think that needs to be in this version. I verified that SimplyStore still works with the example folder. + +This authorizes finishing persisted offset/ID loading in the active file-backed +cycle. The example check is maintainer-reported evidence, not acceptance of the +still-incomplete cycle or an independently reproduced agent test. + +## Trust committed indexes, 2026-09-23 + +The maintainer challenged reconstructing IDs on every startup: “If we do that, +we don't need the index.id.json, right?” and “Why would we not trust +index.id.json?” The assistant acknowledged redundant validation and identified +two actual consistency gaps: ID publication before final hook mutations and +stale mappings after ID removal/rename. The assistant proposed making committed +indexes reliable, loading them directly, and reserving reconstruction for +missing indexes or explicit validation/recovery. The maintainer confirmed: + +> Agreed, lets do that + +This revises the earlier same-cycle scan-and-compare design. It does not accept +the cycle or authorize integration. + +The maintainer further clarified: + +> There is no legacy index compatibility to worry about, this is not yet used. + +> Duplicate ID's are an error + +These explicit corrections remove legacy index support and duplicate-ID winner +selection from the intended contract. Replacing the same record may retain its +ID; two distinct current records may not share an ID. + +## Trust offset indexes and hash artifacts, 2026-09-23 + +After the assistant acknowledged that retaining offset reconstruction was an +incomplete application of the agreed trust model, the maintainer directed: + +> Lets trust the offset index. If there is a reason to check the integrity of these files, we should add a hash of the files somewhere, so that we can check that, not do the work they were intended to avoid anyway. + +This confirms direct offset loading in the same active cycle. Both startup +inspection and loading currently reconstruct framing; both must change. Missing +indexes and explicit maintenance can reconstruct. Integrity belongs in the +existing SHA-256 manifest, covering canonical data and present standard indexes. +There is no new acceptance, integration authorization or legacy requirement. diff --git a/.spiral/sources/SRC-20260923-TTZ7C-45.ttl b/.spiral/sources/SRC-20260923-TTZ7C-45.ttl new file mode 100644 index 0000000..f554fc7 --- /dev/null +++ b/.spiral/sources/SRC-20260923-TTZ7C-45.ttl @@ -0,0 +1,12 @@ +@prefix sd: . +@prefix dcterms: . +@prefix project: . + +project:SRC-20260923-TTZ7C-45 + a sd:Source ; + dcterms:identifier "SRC-20260923-TTZ7C-45" ; + sd:repositoryPath ".spiral/sources/SRC-20260923-TTZ7C-45.md" ; + sd:status sd:Active ; + sd:sourceAvailability sd:Retained ; + sd:provenanceConfidence sd:Explicit ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:SRC-20260923-TTZ7C-45 ; sd:gitCommit "d5100e4aa78cacfcfb3c103b70d8f39afed57d70" ] . diff --git a/.spiral/sources/SRC-20260923-TTZ7C-52.md b/.spiral/sources/SRC-20260923-TTZ7C-52.md new file mode 100644 index 0000000..4fc89b4 --- /dev/null +++ b/.spiral/sources/SRC-20260923-TTZ7C-52.md @@ -0,0 +1,19 @@ +# Mandatory file integrity authorization + +After accepting file-backed operation, the maintainer discovered that the +example had no manifest. The assistant explained the inherited optional +integrity setting, missing initialization for existing stores, and mandatory +creation/startup/publication rules. It explained that data bytes are already +hashed at startup, queries have no per-query hashing, and command overhead +includes durable manifest appends that can be batched and reuse known digests. + +The maintainer said: “It would make the data much more reliable I think”. +The assistant proposed making integrity mandatory, with one-time initialization +for existing stores and measurement of added command latency. The maintainer +confirmed: “Lets do that”. This is the execution commitment after the concrete +intent/current-behavior/gap discussion; no repeat confirmation is needed. + +Understanding: every normal store verifies persisted hashes, new stores and +commits always publish them, and existing stores have an explicit stopped-store +initialization path. Hashing detects changes after the baseline; it cannot prove +historical correctness or repair corruption. No merge or deployment is authorized. diff --git a/.spiral/sources/SRC-20260923-TTZ7C-52.ttl b/.spiral/sources/SRC-20260923-TTZ7C-52.ttl new file mode 100644 index 0000000..8153750 --- /dev/null +++ b/.spiral/sources/SRC-20260923-TTZ7C-52.ttl @@ -0,0 +1,11 @@ +@prefix sd: . +@prefix dcterms: . +@prefix project: . + +project:SRC-20260923-TTZ7C-52 + a sd:Source ; + dcterms:identifier "SRC-20260923-TTZ7C-52" ; + sd:repositoryPath ".spiral/sources/SRC-20260923-TTZ7C-52.md" ; + sd:sourceAvailability sd:Retained ; + sd:provenanceConfidence sd:Explicit ; + sd:status sd:Active . diff --git a/.spiral/understandings/UND-20260923-TTZ7C-48.md b/.spiral/understandings/UND-20260923-TTZ7C-48.md new file mode 100644 index 0000000..c5a836f --- /dev/null +++ b/.spiral/understandings/UND-20260923-TTZ7C-48.md @@ -0,0 +1,44 @@ +--- +id: UND-20260923-TTZ7C-48 +--- + +# File-backed runtime boundary + +Confirmed scope: make files the sole dataset source for live queries, commands, +startup assessment and administrative replay. SharedArrayBuffer is currently +used for the loaded base and all committed changesets. Worker ID lookup assumes +all records have cache entries. Inspection also retains full file buffers. +Those observed paths prevent the requested outcome despite file support in the +new dependency. The current 118-test suite passed with the hardened dependency. + +Keep the accepted format, remote API and durability semantics. Immutable base +and changeset files remain available while a store owns its dataset; each worker +owns its descriptors. Index metadata may scale with record count; command edits +and response/output buffers may scale with a command or query. Bounded clean +query caching is the dependency contract, not constant total memory. Full +startup integrity scanning remains necessary, but can use bounded byte buffers. + +The human's instruction follows the concrete integration explanation and is the +commitment boundary; no second permission ceremony is needed. + +## Confirmed completion within this version + +The maintainer's response to the explicit persisted-index gap authorizes using +base and committed changeset offset/ID sidecars in this same cycle, with +validation and fallback. loadFileData currently never calls their loaders. +The example-folder check is supporting human feedback; cycle acceptance remains +pending. Preserve canonical framing and remote APIs. + +## Revised trust and uniqueness contract + +SRC-45 now records agreement to trust committed indexes, no legacy index +compatibility requirement, and duplicate IDs as errors. Earlier header scanning +against every sidecar and highest-record duplicate precedence are superseded. +The evidenced gap is early ID-index writing plus incomplete merge/removal +semantics. Correct generation and direct loading satisfy the confirmed intent. + +## Offset trust correction, 2026-09-23 + +Normal opening must trust committed offset indexes as well as ID indexes. +Reconstructing their contents defeats their purpose; verify integrity with +file hashes when configured. This also applies to startup inspection. diff --git a/.spiral/understandings/UND-20260923-TTZ7C-48.ttl b/.spiral/understandings/UND-20260923-TTZ7C-48.ttl new file mode 100644 index 0000000..a8eaa71 --- /dev/null +++ b/.spiral/understandings/UND-20260923-TTZ7C-48.ttl @@ -0,0 +1,12 @@ +@prefix sd: . +@prefix dcterms: . +@prefix project: . + +project:UND-20260923-TTZ7C-48 + a sd:Understanding ; + dcterms:identifier "UND-20260923-TTZ7C-48" ; + sd:repositoryPath ".spiral/understandings/UND-20260923-TTZ7C-48.md" ; + sd:status sd:Accepted ; + sd:interprets [ a sd:ArtifactReference ; sd:artifact project:SRC-20260923-TTZ7C-45 ; sd:gitCommit "8299fcc8b8fcf113091ae85bbcf11f867f8c5908" ] ; + sd:changeCausedBy [ a sd:ArtifactReference ; sd:artifact project:SRC-20260923-TTZ7C-45 ; sd:gitCommit "7e1ea39dc4056beec311c8e6d832d97d2a550d58" ] ; + sd:transforms [ a sd:ArtifactReference ; sd:artifact project:UND-20260923-TTZ7C-48 ; sd:gitCommit "7e1ea39dc4056beec311c8e6d832d97d2a550d58" ] . diff --git a/DURABILITY.md b/DURABILITY.md index cd8366e..9550488 100644 --- a/DURABILITY.md +++ b/DURABILITY.md @@ -1,7 +1,9 @@ # Durability contract -SimplyStore retains its existing base, changeset, command/status and optional -integrity formats. There is no data migration. The command log is authoritative: +SimplyStore retains its existing base, changeset, command/status and +integrity formats. Integrity manifests are required; existing stores without +one use the explicit initialization workflow, without rewriting canonical data. +The command log is authoritative: acceptance serialization keeps execution in log order, including concurrent requests and duplicate IDs. A command supplies all inputs beyond its preceding dataset; handlers receive `undefined` as the third argument and metadata as the @@ -25,8 +27,10 @@ Startup validates existing canonical files before executing user modules or changing command status. Complete consistent stores reopen normally. Missing logs, malformed records, missing/corrupt committed data, or pending/uncertain commands stop startup for administrator assessment. Upgrading cannot prove that -older writers preserved the whole history. Explicit integrity configuration is -needed to detect a lost manifest; existence-based detection alone cannot do so. +older writers preserved the whole history. A missing manifest or required entry now stops startup even without explicit +integrity configuration. Creation, commands and recovery persist hashes of +canonical data and present standard indexes before success. Each artifact set +uses one durable manifest append; normal startup never creates a new baseline. The [administrator guide](docs/recovery.md) provides read-only inspection, source-bound preview, explicitly approved suffix recovery on a copy, interrupted @@ -45,3 +49,13 @@ Custom hooks must await their work; declare additional required artifacts with their own conformance checks. Unsupported directory-sync operations fail rather than silently weakening the guarantee. Broader filesystems, random/soak testing, and deployment-specific power-loss testing remain future validation work. + +## File-backed readers + +The runtime retains ordered file sources instead of shared dataset bytes. Each +worker owns read handles; committed sources remain immutable while the store is +open. Source publication still follows the durable `done` record. Inspection +hashes/scans data with a fixed-size byte buffer, and shutdown waits for readers +to terminate before releasing ownership. See [file-backed data](docs/file-data.md) +for the source contract and memory boundaries. The existing filesystem durability +envelope and recovery restrictions remain in force. diff --git a/README.md b/README.md index 1210d77..5289963 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # SimplyStore -SimplyStore is a radically simpler backend storage server. It does not have a database, certainly no SQL or GraphQL, it is not REST. In return it has a well defined API that is automatically derived from your dataset. It supports JSONTag to allow for semantically meaningful data, without having to do the full switch to Linked Data and triple stores. The query format is javascript, you can post javascript queries that will run on the server. All data is read into memory and is available to these javascript queries without needing (or allowing) disk access or indexes. +SimplyStore is a radically simpler backend storage server. It does not have a database, certainly no SQL or GraphQL, it is not REST. In return it has a well defined API that is automatically derived from your dataset. It supports JSONTag to allow for semantically meaningful data, without having to do the full switch to Linked Data and triple stores. The query format is javascript, you can post javascript queries that will run on the server. Dataset records are read lazily from indexed files. Javascript queries use ordinary objects and arrays; SimplyStore manages file access and indexes. [JSONTag](https://github.com/muze-nl/jsontag) is an enhancement over JSON that allows you to tag JSON data with metadata using HTML-like tags. Javascript queries are run in a [VM2](https://www.npmjs.com/package/vm2) sandbox. @@ -13,6 +13,7 @@ Note: _There are known security issues in VM2, so the project will switch to V8- - [Background](#background) - [Install](#install) - [Usage](#usage) +- [File-backed Data](docs/file-data.md) - [Custom Index Modules](#custom-index-modules) - [Example Query](#examples) - [Goals](#goals) @@ -65,7 +66,18 @@ You should be able to go http://localhost:3000/query/ and see something like thi ## Durability and recovery -See the [durability contract](DURABILITY.md) and [administrator recovery guide](docs/recovery.md). Command-log order governs execution. Acceptance and completion await file and directory barriers. Uncertain or pending work found at startup requires administrator assessment; missing data does not prove that external effects did not happen. Existing store formats are retained without migration. +See the [durability contract](DURABILITY.md) and [administrator recovery guide](docs/recovery.md). Command-log order governs execution. Acceptance and completion await file and directory barriers. Uncertain or pending work found at startup requires administrator assessment; missing data does not prove that external effects did not happen. Canonical data formats remain unchanged. + +## File-backed data + +See the [file-backed data guide](docs/file-data.md) for worker/file lifetimes, +index rebuilding, memory limits, and the internal worker-message changes. +Existing stores retain their current formats and need no conversion. + +Integrity hashes are mandatory for data and present standard index files. +New-store conversion creates the manifest automatically. Existing stores without +one need the stopped-store [initialization command](docs/recovery.md#initialize-integrity-for-an-existing-store) +before opening; startup never invents a replacement baseline. ## Custom Index Modules @@ -177,7 +189,7 @@ SimplyStore is a more defined and usable REST like service, out of the box. One The SimplyStore design is predicated on the following realisations: - 1. Most data today will fit comfortably in memory in a commodity server. + 1. Files provide lazy record access; indexes and active edits still use memory. 2. REST today is usually JSON-over-HTTP, but JSON crucially misses a type. 3. JSON is never just JSON. You need additional things like JSON-LD or JSON-Schema, to make sense of it. 4. There is no clear onramp from JSON to Linked Data. @@ -185,7 +197,7 @@ The SimplyStore design is predicated on the following realisations: So the scope for SimplyStore is: -- datasets that will fit comfortably in memory, for now I've set a test goal of about 1GB of data. +- datasets whose indexes and active working set fit in memory, with file-backed record storage. - usecases that are mostly-read, with sparse updates. - scale-in-depth, so scale up is limited to the limits of a single computer system - linked data (RDF et al) is not an immediate concern, but there must be a plausible onramp / conversion to and from linked data. diff --git a/docs/file-data.md b/docs/file-data.md new file mode 100644 index 0000000..94c7f0f --- /dev/null +++ b/docs/file-data.md @@ -0,0 +1,174 @@ +# File-backed datasets + +SimplyStore reads its base and committed changeset files through od-jsontag. +The runtime and its workers exchange file paths and index metadata; they no +longer retain or transfer the dataset as SharedArrayBuffers. Remote query and +command APIs and the existing OD-JSONTag/log formats are unchanged. + +## Opening and reading a store + +Startup inspects command-log order and the committed prefix, as before. It loads +`index.offset.json` and `index.id.json` for the base, followed by +`index.offset..json` and `index.id..json` for committed +changesets only. Existing index hooks continue writing these files. + +Normal loading trusts committed offset and ID indexes. It checks their JSON +structure, ordered nonoverlapping offset ranges, file bounds and +record numbers, then combines them in committed order without reconstructing +framing or scanning record tags. A changeset first removes mappings for every record it replaces, then +applies its new IDs. This handles renames, ID removal and ID swaps. Duplicate +IDs on distinct records are errors; updating the same record may retain its ID. + +A 64 KiB byte buffer hashes canonical bytes without interpreting their framing. +Missing offset indexes are reconstructed by scanning framing. If only an ID index +is missing, the loader reads leading JSONTag headers at the stored offsets. +Present malformed indexes fail loading rather than silently being replaced. + +Store options support deliberate maintenance: + +- `validateIndexes: true` compares offsets with canonical framing and IDs with + record headers, failing on mismatches or duplicate IDs. +- `rebuildIndexes: true` ignores both persisted indexes and reconstructs them in + memory. This takes precedence over validation. + +Both options are also accepted by `loadFileData()`. Loading never rewrites data +or sidecars. Normal loading assumes committed indexes are correct, including +completeness; use explicit validation after external changes. There is no legacy +index compatibility path. Record bodies remain lazy except for opening the root +and the unusual reconstruction fallback for reference-only records. Required +integrity checks cover canonical data and present standard index files using the +existing SHA-256 manifest. Index hashes cover the exact bytes parsed. +Every present standard index requires a manifest entry; +missing indexes can reconstruct from verified data. Explicit rebuild ignores +sidecar hashes as well as contents, while still checking canonical data hashes. +Hash mismatches fail opening; normal loading never repairs them automatically. +The manifest location remains configurable with `integrityFile`; integrity has +no opt-out. Existing stores without a manifest require the explicit +[initialization operation](recovery.md#initialize-integrity-for-an-existing-store). + +Each query worker opens its own read-only descriptors and registers the ordered +base/changeset sources with its parser. ID lookups use the complete record +catalog, including objects not yet read. The parser retains at most 256 clean +record bodies by default; evicted bodies are read again when needed. + +Canonical source files must remain immutable while the store is open. Opening a +source checks its device, inode, size and modification/change timestamps against +the hashed identity. Store ownership and immutable publication are still the +operational boundary; this is not protection against another process editing an +already-open file in place. Use the existing stopped-store recovery workflow for +administrative changes. + +## Commands and publication + +Each command worker opens the exact committed source list in a fresh mutable +session. Touched objects and uncommitted edits remain available until that +session ends. Custom `update` and `finalize` hooks retain their existing calls. +The changeset is temporarily serialized to an ordinary Buffer for finalization, +then published using the existing complete-write, file-sync, rename and +directory-sync sequence. Finalization must leave canonical bytes unchanged. + +ID indexes are prepared from final serialization after mutation hooks. Duplicate +IDs are rejected before canonical publication. JSONTag ID-only edits are detected +on materialized records before and after the update hook, without reading +untouched objects. The core finalization boundary awaits custom finalization, +then durably writes the ID sidecar and returns the updated ID map. Conversion +uses the same preparation/finalization rules. Empty commands write an empty ID +sidecar. A failed ID write prevents success. Conversion, +commands and administrative replay append hashes of final standard index files +after finalization and before success or durable `done`. The data digest is +reused and all entries share one durable append. Custom finalizers may +omit offsets; absent files are not fingerprinted. Hashing detects later file +changes; it does not independently prove a custom index was generated correctly. + +The worker returns the new file's descriptor and metadata. The runtime publishes +that source to query workers only after the required files and `done` status +have been synchronized. Empty/no-op changesets are valid sources. Query pools +finish pending updates before dispatching another query. Replacement workers +initialize at the current committed head. + +Workers own their file handles. Session reinitialization and failure close them; +worker termination also closes descriptors opened through Node's tracked file +APIs. Runtime shutdown waits for both pools to terminate before releasing store +ownership. A failed source update makes the pool unavailable and stops further +command acceptance rather than allowing the worker to skip an update. +Lazy read/decode/reference failures are recorded in host-owned parser state. +Even if a command or query catches the error, publication is refused and the +runtime stops accepting mutations. Query code cannot forge this state by throwing +an object with a `storageFailure` property. + +Administrative replay opens file sources too. `inspectStore()` now returns a +plain `sources` array instead of live `data` proxies and retained `buffers`. +Callers that need to read an inspected snapshot can use a `FileDataset` inside a +`try`/`finally` and call `close()` when finished. Proxies are valid only while +their owning session remains open. + +## Internal integration changes + +Custom worker implementations need to adopt these messages: + +| Boundary | Input/output | +| --- | --- | +| Loader result | `{ sources, meta }` | +| Query initialization | `req.sources`, `req.meta`, `req.access` | +| Query update | `req.source`, `req.meta` | +| Command initialization | `sources`, `meta`, existing command/configuration fields | +| Successful command result | `source`, `meta`, existing result/status fields | + +A source contains `file`, `offsets`, `identity`, `size` and `digest`; it contains +neither bytes nor an open descriptor. It is an internal trusted worker contract. +Use `loadDataSource()` to load persisted offsets (or `scanDataFile()` to +explicitly reconstruct them) and `FileDataset` to own the parser and handles. Custom hook signatures are unchanged. Custom finalizers can still +replace offset finalization; core ID publication always runs after them. +Default index loaders accept `load(meta, uuid)`. + +The query parser prepares read-only arrays with an immutable undefined +`constructor` before wrapping them. This matches vm2's existing safe array-species +case, allowing `map`, `filter` and JAQT operations without permitting temporary +writes through the dataset proxy. This compatibility adapter does not replace +or establish a new security assessment of the query sandbox. + +## Memory and startup costs + +File-backed operation does not imply constant total memory. Record offsets, +source catalogs and ID maps scale with the dataset. Each worker keeps one open +handle per source file. Mutable sessions retain touched records; large individual +records, query results, command output and custom hooks can require substantial +memory. Command/status logs and schemas are still read into memory. Backup/copy +utilities still temporarily buffer individual files when copying them; this +cycle changes dataset retrieval and replay, not their copy algorithm. + +Startup still reads canonical bytes for hashing and snapshot checks. With +complete indexes, neither inspection nor loading reconstructs offsets or IDs. +Missing-index fallback and explicit maintenance read headers instead of fully +decoding record bodies. Offset catalogs and ID maps still scale with the data. + +The accepted file-backed baseline used a generated 328,877,813-byte / 20,000-record store with persisted indexes opened +and answered an ID lookup with a 48 MB JavaScript heap limit. One local run +measured approximately 1.07 seconds open and 99 ms query (including worker +readiness), with 9.0 MB parent heap and 24.5 MB query-worker heap. Retained byte +buffers were approximately 25 KiB and 58 KiB respectively. These are warm local +measurements, not a cold-storage, physical-RAM or total-process-memory bound. +Direct index loading removes framing and ID reconstruction. Existing snapshot +hashing still reads the files; this run does not establish a startup speedup. + +Mandatory-integrity command latency was compared with the accepted optional +implementation (`4680c76`, integrity disabled), using three alternating runs of +30 commands per mode on local ext4, Node 24.7.0. Per-run median completion times +were 54.6–55.9 ms before and 56.3–57.1 ms with required integrity (roughly 2 ms +extra). Completion includes a fresh worker and HTTP polling; p95 varied from +63.1–65.9 ms before and 62.9–73.2 ms after. These are local observations, not a +storage-independent overhead guarantee. Queries perform no additional hash check. + +Reproduce the disposable-fixture probe with: + +```sh +node --expose-gc --max-old-space-size=48 scripts/benchmark-file-data.mjs +``` + +Pass `--indexes=missing` to exercise reconstruction without sidecars. + +The dependency is pinned to reviewed od-jsontag Git commit +`24f47eb616c3a24391487197496804ed1c56837b`, because its file-backed API has not +received a new npm version. Installation therefore requires Git access to its +public repository. Replacing that pin with a published release is separate +release work. diff --git a/docs/recovery.md b/docs/recovery.md index 2cc9d94..12af6ed 100644 --- a/docs/recovery.md +++ b/docs/recovery.md @@ -1,7 +1,8 @@ # Administrator recovery SimplyStore preserves the existing OD-JSONTag data, command-log, status-log and -optional integrity formats. The command log defines command order. Commands must +integrity manifest formats. Integrity verification is mandatory. The command log +defines command order. Commands must contain every invocation input beyond the dataset; HTTP request context is no longer supplied to command workers. The fourth handler argument still provides metadata/index access; the former third request argument is `undefined`. @@ -25,8 +26,10 @@ Tools accept a declarative JSON file, not executable server configuration: } ``` -Omit `integrityFile` if integrity is unused. Set it explicitly when integrity is -required: existence-based detection cannot detect a lost manifest. Relative paths +Omit `integrityFile` to use the default next to the data file, such as +`data.integrity.jsontag`. Setting it changes the location, not whether hashes +are required. The former `integrity: false` option cannot disable checks. +Relative paths resolve against the command's working directory, as server paths do. `requiredFiles` can list additional required files; these are checked at startup and synced before done. Keep custom artifact paths inside the declared directories or list them @@ -35,15 +38,37 @@ explicitly. Custom hooks must finish all required work before resolving. For a new store, create an empty destination directory and run: ```sh -node scripts/convert.mjs input.jsontag /srv/new-store/data.jsontag --integrity +node scripts/convert.mjs input.jsontag /srv/new-store/data.jsontag ``` The converter creates empty command/status logs alongside the base, finishes -indexes and optional integrity, and releases ownership after durable completion. +indexes and the required integrity manifest, and releases ownership after +durable completion. It refuses to overwrite an existing base/log/manifest. Opening an existing store never creates missing logs. A bare historical base is not proof of an empty history: preserve it and its surrounding evidence before deciding on initialization. -There is no migration or format-conversion step for an existing complete store. +An existing store with a complete manifest needs no migration. Stores without +a manifest need the explicit initialization below. + +## Initialize integrity for an existing store + +Stop the server and run: + +```sh +node scripts/recover.mjs init-integrity --store store.json +``` + +This takes the normal ownership locks, validates the committed history, framing, +record contents, IDs and available indexes, and creates the missing manifest +atomically. It preserves data, indexes and logs and does not run command handlers +or replay commands. Pending/uncertain history, malformed data or stale indexes +must be resolved first. Existing manifests are never replaced, including when +verification fails. Normal startup never initializes or repairs hashes. + +Initialization establishes a baseline for the current validated bytes; it cannot +prove that those bytes match their historical contents. Afterwards, changes are +checked against the recorded hashes. An uncertain publication failure retains +ownership for administrator inspection, as with other durable operations. ## Inspect before executing anything diff --git a/example/README.md b/example/README.md index 8a23b67..59422fa 100644 --- a/example/README.md +++ b/example/README.md @@ -12,10 +12,22 @@ Then convert the dataset `swdb.jsontag` to internal SimplyStore format: npm run build ``` -This creates `data.jsontag`, its indexes, and empty command and status logs +This creates `data.jsontag`, its indexes, `data.integrity.jsontag`, and empty +command and status logs for the first startup. Build requires a new store and refuses to overwrite existing data or logs. Run it once before starting the example. +If you already have an example store from before integrity became mandatory, +stop its server and initialize hashes once, preserving its existing commands: + +```shell +npm run init-integrity +``` + +This validates the current files and creates the missing manifest. It refuses to +replace an existing manifest or initialize an uncertain/inconsistent store. New +stores created with `npm run build` do not need this step. + Then start the server: ```shell npm start diff --git a/example/package.json b/example/package.json index dfdf2ec..b9de07e 100644 --- a/example/package.json +++ b/example/package.json @@ -3,7 +3,8 @@ "main": "server.mjs", "scripts": { "build": "node ../scripts/convert.mjs swdb.jsontag data.jsontag", - "start": "node server.mjs" + "start": "node server.mjs", + "init-integrity": "node ../scripts/recover.mjs init-integrity --store store.json" }, "dependencies": { "@muze-nl/jsontag": "^0.10.4", diff --git a/example/store.json b/example/store.json new file mode 100644 index 0000000..e6d1c3f --- /dev/null +++ b/example/store.json @@ -0,0 +1,5 @@ +{ + "datafile": "./data.jsontag", + "commandLog": "./command-log.jsontag", + "commandStatus": "./command-status.jsontag" +} diff --git a/package.json b/package.json index 3bf1641..40c18c5 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "type": "module", "scripts": { "start": "node src/run.mjs", - "test": "node --test test/store-runtime.mjs test/durability-startup.mjs test/conversion-offsets.mjs test/command-offsets.mjs", + "test": "node --test test/required-integrity.mjs test/file-data.mjs test/index-ids.mjs test/store-runtime.mjs test/durability-startup.mjs test/conversion-offsets.mjs test/command-offsets.mjs", "test:acid": "node --test test/acid-baseline.mjs", "test:durability": "node --test test/durability-crash.mjs", "test:handlers": "node --test test/after-change-handlers.mjs", @@ -25,7 +25,7 @@ "dependencies": { "@muze-nl/jaqt": "^0.10.6", "@muze-nl/jsontag": "^0.10.4", - "@muze-nl/od-jsontag": "^0.4.6", + "@muze-nl/od-jsontag": "git+https://github.com/muze-nl/od-jsontag.git#24f47eb616c3a24391487197496804ed1c56837b", "express": "^5.2.1", "vm2": "^3.12.2" }, @@ -38,6 +38,7 @@ "README.md", "DURABILITY.md", "docs/recovery.md", + "docs/file-data.md", "docs/power-loss-filesystem.md", "LICENSE", "src/", diff --git a/scripts/benchmark-file-data.mjs b/scripts/benchmark-file-data.mjs new file mode 100644 index 0000000..bf653f5 --- /dev/null +++ b/scripts/benchmark-file-data.mjs @@ -0,0 +1,105 @@ +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { performance } from 'node:perf_hooks' +import { fileURLToPath } from 'node:url' +import StoreRuntime from '../src/store-runtime.mjs' +import { hashFile } from '../src/file-data.mjs' +import { appendArtifactIntegrity } from '../src/index-files.mjs' +import { getDefaultIntegrityFile } from '../src/integrity.mjs' + +const args = Object.fromEntries(process.argv.slice(2).map(argument => { + return argument.replace(/^--/, '').split('=') +})) +const records = Number(args.records || 20000) +const payload = Number(args.payload || 16384) +if (!Number.isSafeInteger(records) || records < 1 || + !Number.isSafeInteger(payload) || payload < 1) { + throw new Error('records and payload must be positive integers') +} +const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'simplystore-file-bench-')) +const datafile = path.join(dir, 'data.jsontag') +let runtime +try { + const fd = fs.openSync(datafile, 'w') + try { + const offsets = {} + const ids = {} + let fileOffset = 0 + let record = 0 + const writeRecord = (text, id) => { + const prefix = `(${Buffer.byteLength(text)})` + const bytes = Buffer.from(`${prefix}${text}\n`) + offsets[record] = [ + fileOffset + prefix.length, fileOffset + bytes.length - 1 + ] + if (id) { + ids[id] = record + } + record++ + fileOffset += bytes.length + let offset = 0 + while (offset < bytes.length) { + offset += fs.writeSync(fd, bytes, offset) + } + } + writeRecord(`{"items":[~1-${records}]}`) + const padding = 'x'.repeat(payload) + for (let number = 1; number <= records; number++) { + writeRecord(`${JSON.stringify({ + number, padding + })}`, `item-${number}`) + } + if (args.indexes !== 'missing') { + fs.writeFileSync(path.join(dir, 'index.offset.json'), + JSON.stringify(offsets)) + fs.writeFileSync(path.join(dir, 'index.id.json'), + JSON.stringify(ids)) + } + } + finally { + fs.closeSync(fd) + } + const commandLog = path.join(dir, 'command-log.jsontag') + const commandStatus = path.join(dir, 'command-status.jsontag') + fs.writeFileSync(commandLog, '') + fs.writeFileSync(commandStatus, '') + await appendArtifactIntegrity(getDefaultIntegrityFile(datafile), + datafile, hashFile(datafile), {data: dir}) + const started = performance.now() + runtime = await StoreRuntime.open({ + datafile, commandLog, commandStatus, maxWorkers: 1, + loadTimeout: 0, slowTimeout: 120000, + commandsFile: fileURLToPath(new URL('../src/commands.mjs', import.meta.url)) + }) + const openMs = performance.now() - started + const queried = performance.now() + const response = await runtime.runQuery({ + path: '/', jsontag: false, + body: `meta.index.id.get('item-${records}').number` + }, { slow: true }) + if (response.code || JSON.parse(response.body) !== records) { + throw new Error(`Query failed: ${JSON.stringify(response)}`) + } + const queryMs = performance.now() - queried + global.gc?.() + const memory = process.memoryUsage() + const workerMemory = await runtime.slowQueryWorkerPool.memoryUsage() + console.log(JSON.stringify({ + bytes: fs.statSync(datafile).size, records, openMs, queryMs, + sources: runtime.sources.length, + persistedIndexes: args.indexes !== 'missing', + idCount: runtime.meta.index.id.size, + parent: { + heapUsed: memory.heapUsed, arrayBuffers: memory.arrayBuffers + }, + worker: { + heapUsed: workerMemory.heapUsed, + arrayBuffers: workerMemory.arrayBuffers + } + }, null, 2)) +} +finally { + await runtime?.close() + fs.rmSync(dir, { recursive: true, force: true }) +} diff --git a/scripts/convert.mjs b/scripts/convert.mjs index cf0f0d9..4486cd4 100644 --- a/scripts/convert.mjs +++ b/scripts/convert.mjs @@ -4,22 +4,24 @@ import JSONTag from '@muze-nl/jsontag' import serialize, { stringify } from '@muze-nl/od-jsontag/src/serialize.mjs' import Parser from '@muze-nl/od-jsontag' import { finalizeIndex } from '../src/index.mjs' +import { markIdChanges, prepareIdIndex } from '../src/index.id.mjs' import fs from 'node:fs' import path from 'node:path' import { publishFile, syncFile } from '../src/storage.mjs' import { acquireOwnership } from '../src/store-ownership.mjs' +import { hashFile } from '../src/file-data.mjs' +import { appendArtifactIntegrity } from '../src/index-files.mjs' import { - appendIntegrityRecord, + digestBuffer, getDefaultIntegrityFile } from '../src/integrity.mjs' const __dirname = import.meta.dirname -const integrity = process.argv.includes('--integrity') const args = process.argv.slice(2).filter(value => value !== '--integrity') if (args.length < 2) { console.log( - 'usage: node ./convert.mjs {inputfile} {outputfile} {indexlib?} {schema?} [--integrity]' + 'usage: node ./convert.mjs {inputfile} {outputfile} {indexlib?} {schema?}' ) process.exit() } @@ -97,22 +99,24 @@ async function main() { meta.index.id.set(JSONTag.getAttribute(ob, 'id'), ob) } + const originalIds = prepareIdIndex(strData).ids await index.create(odData, meta) + markIdChanges(meta, originalIds) console.log('Indexes created') strData = stringify(serialize(odData)) + const prepared = prepareIdIndex(strData) + const expectedDigest = digestBuffer(Buffer.from(strData)) await publishFile(outputFile, strData) // Custom indexes may change record sizes or add records after initial // parsing. - await finalizeIndex(index, strData, meta) - if (integrity) { - await appendIntegrityRecord( - getDefaultIntegrityFile(outputFile), - outputFile, - Buffer.from(strData) - ) + await finalizeIndex(index, strData, meta, null, prepared) + if (hashFile(outputFile) !== expectedDigest) { + throw new Error('Dataset changed during finalization') } + await appendArtifactIntegrity(getDefaultIntegrityFile(outputFile), + outputFile, expectedDigest, meta) for (const file of logs) { await publishFile(file, '') } diff --git a/scripts/recover.mjs b/scripts/recover.mjs index 26fdb12..4ad89e5 100644 --- a/scripts/recover.mjs +++ b/scripts/recover.mjs @@ -12,6 +12,7 @@ import { verifyCandidate } from '../src/admin-recovery.mjs' import { inspectStore } from '../src/store-inspection.mjs' +import { initializeIntegrity } from '../src/initialize-integrity.mjs' const [action, ...args] = process.argv.slice(2) const options = {} @@ -37,6 +38,8 @@ const required = key => { } async function main() { switch (action) { + case 'init-integrity': + return initializeIntegrity(await read(required('store'))) case 'inspect': { const plan = await planRecovery(await read(required('store')), { quiescent: options.quiescent === true @@ -121,7 +124,7 @@ async function main() { } default: throw new Error( - 'Usage: recover.mjs inspect|apply|backup|restore|unlock|finish|verify --store config.json ...; see docs/recovery.md' + 'Usage: recover.mjs init-integrity|inspect|apply|backup|restore|unlock|finish|verify --store config.json ...; see docs/recovery.md' ) } } diff --git a/src/admin-recovery.mjs b/src/admin-recovery.mjs index c889a41..b5146ae 100644 --- a/src/admin-recovery.mjs +++ b/src/admin-recovery.mjs @@ -4,8 +4,7 @@ import path from 'node:path' import { fileURLToPath } from 'node:url' import JSONTag from '@muze-nl/jsontag' import { from, _, anyOf, not } from '@muze-nl/jaqt' -import Parser from '@muze-nl/od-jsontag/src/parse.mjs' -import serialize from '@muze-nl/od-jsontag/src/serialize.mjs' +import { loadFileData, loadDataSource } from './file-data.mjs' import { inspectStore, inventory, @@ -23,7 +22,8 @@ import { } from './storage.mjs' import { executeWorker } from './execute-worker.mjs' import { nextActiveCommandStatus } from './recovery.mjs' -import { loadIntegrityManifest, verifyIntegrity } from './integrity.mjs' +import { appendArtifactIntegrity } from './index-files.mjs' +import { loadIntegrityManifest, verifyDigest } from './integrity.mjs' const defaultWorker = fileURLToPath( new URL('./command-worker.mjs', import.meta.url) @@ -238,7 +238,7 @@ class RecoveryApplication { this.journal = null this.planHash = null this.meta = null - this.buffers = null + this.sources = null this.expectedManifest = null } @@ -363,23 +363,14 @@ class RecoveryApplication { if (inspection.errors.length) { throw new Error(inspection.errors.join('; ')) } - const parser = new Parser() - let data - for (const bytes of inspection.buffers) { - data = parser.parse(bytes) - } - // Rebuild parser indexes without invoking custom hooks. - this.meta = { - index: { id: new Map() }, - data: path.dirname(this.config.datafile), - parts: inspection.committed.length - } - this.buffers = [serialize(data, { meta: this.meta })] - if (this.config.schemaFile) { - this.meta.schema = JSONTag.parse( - await fs.readFile(this.config.schemaFile, 'utf8') - ) - } + const loaded = await loadFileData({ + dataFile: this.config.datafile, + commands: inspection.committed, + schemaFile: this.config.schemaFile, + integrityFile: this.config.integrityFile + }) + this.meta = loaded.meta + this.sources = loaded.sources const integrityFile = this.plan.config.integrityFile if (this.plan.files[integrityFile] != null) { this.expectedManifest = await loadIntegrityManifest(integrityFile) @@ -431,14 +422,13 @@ class RecoveryApplication { id, command: command.line, meta: this.meta, - data: this.buffers, + sources: this.sources, commandsFile: this.plan.code.commandsFile, indexFile: this.plan.code.indexFile, datafile: this.config.datafile, // Retained digests are checked before replacement manifest and // done records are allowed. - integrityFile: null, - integrityRequired: false + deferIntegrityPublication: true }, 30000 ) @@ -475,23 +465,23 @@ class RecoveryApplication { for (const file of this.config.requiredFiles) { await syncFile(file) } + const source = loadDataSource(result.source.file, this.meta, id) if (this.expectedManifest && command.status === 'done') { - verifyIntegrity( + verifyDigest( this.expectedManifest, this.plan.config.integrityFile, command.file, - result.data, + source.digest, { required: true } ) } - if (this.config.integrity || this.expectedManifest) { - await this.appendResultIntegrity(command, result.data) - } + await appendArtifactIntegrity(this.config.integrityFile, + source.file, source.digest, this.meta, id) await appendRecord( this.config.commandStatus, JSONTag.stringify({ command: id, code: 200, status: 'done' }) ) - this.buffers.push(result.data) + this.sources.push(source) Object.assign(this.meta, result.meta) await appendRecord( this.journal, @@ -499,19 +489,6 @@ class RecoveryApplication { ) } - async appendResultIntegrity(command, data) { - const { appendIntegrityRecord } = await import('./integrity.mjs') - const targetFile = path.join( - path.dirname(this.config.datafile), - path.basename(command.file) - ) - await appendIntegrityRecord( - this.config.integrityFile, - targetFile, - data - ) - } - async inspectCompletedCandidate() { const complete = await inspectStore(this.config) if (complete.ready) { diff --git a/src/command-worker-module.mjs b/src/command-worker-module.mjs index a602493..d04dd35 100644 --- a/src/command-worker-module.mjs +++ b/src/command-worker-module.mjs @@ -1,27 +1,28 @@ import JSONTag from '@muze-nl/jsontag' import { getIndex, - resultSet, isChanged } from '@muze-nl/od-jsontag/src/symbols.mjs' -import Parser from '@muze-nl/od-jsontag/src/parse.mjs' -import serialize from '@muze-nl/od-jsontag/src/serialize.mjs' +import { FileDataset, loadDataSource } from './file-data.mjs' +import { serializeChunks } from '@muze-nl/od-jsontag/src/serialize.mjs' import { publishFile as writeFileAtomic, storageError } from './storage.mjs' import { faultPoint } from './faults.mjs' -import { appendIntegrityRecord } from './integrity.mjs' +import { appendArtifactIntegrity } from './index-files.mjs' +import { getDefaultIntegrityFile, digestBuffer } from './integrity.mjs' import { finalizeIndex } from './index.mjs' +import { markIdChanges, prepareIdIndex } from './index.id.mjs' let commands = {} let index = {} -let resultArr = [] +let dataset +let parser let dataspace let datafile, basefile, extension, integrityFile +let committedIds = new Map() let meta = {} let metaProxy = { index: {} } -const parser = new Parser() -parser.immutable = false export const metaIdProxy = { forEach: callback => { @@ -29,7 +30,7 @@ export const metaIdProxy = { callback( { deref: () => { - return resultArr[ref] + return parser.getLineProxy(ref) } }, id @@ -47,7 +48,7 @@ export const metaIdProxy = { } else { let line = parser.meta.index.id.get(id) - resultArr[line] = ref + parser.meta.resultArray[line] = ref } }, get: id => { @@ -55,7 +56,7 @@ export const metaIdProxy = { if (index || index === 0) { return { deref: () => { - return resultArr[index] + return parser.getLineProxy(index) } } } @@ -73,29 +74,36 @@ const metaReadProxy = { } export async function initialize(task) { - if (task.meta) { - parser.meta = task.meta - } - for (let jsontag of task.data) { - dataspace = parser.parse(jsontag) + close() + try { + committedIds = new Map(task.meta.index.id) + dataset = new FileDataset(task.meta, false) + parser = dataset.parser + dataspace = dataset.open(task.sources) + meta = parser.meta + metaProxy.index.id = metaIdProxy + if (meta.schema) { + metaProxy.schema = meta.schema + } + datafile = task.datafile + integrityFile = task.deferIntegrityPublication ? null : + task.integrityFile || getDefaultIntegrityFile(datafile) + extension = datafile.split('.').pop() + // Include the dot before the extension. + basefile = datafile.substring( + 0, datafile.length - (extension.length + 1) + ) + commands = await import(task.commandsFile).then(mod => { + return mod.default + }) + index = await import(task.indexFile).then(mod => { + return mod.default + }) } - resultArr = dataspace[resultSet] - meta = task.meta - metaProxy.index.id = metaIdProxy - if (meta.schema) { - metaProxy.schema = meta.schema + catch (error) { + close() + throw error } - datafile = task.datafile - integrityFile = task.integrityFile - extension = datafile.split('.').pop() - // Include the dot before the extension. - basefile = datafile.substring(0, datafile.length - (extension.length + 1)) - commands = await import(task.commandsFile).then(mod => { - return mod.default - }) - index = await import(task.indexFile).then(mod => { - return mod.default - }) } export default async function runCommand(commandStr) { @@ -114,39 +122,48 @@ export default async function runCommand(commandStr) { if (commands[task.name]) { let time = Date.now() await commands[task.name](dataspace, task, undefined, metaProxy) + if (parser.readFailure) { + throw parser.readFailure + } // TODO: if command/task makes no changes, skip updating - // data.jsontag and writing it; skip response.data. + // data.jsontag and writing it. + markIdChanges(meta, committedIds) const changes = meta.resultArray.filter(e => e[isChanged]) //FIXME: new entities should also report isChanged = true if (changes.length) { changes.uuid = task.id await index.update(dataspace, meta, changes) } - // Serialize only changes. - const uint8sab = serialize(dataspace, { meta, changes: true }) - response.data = uint8sab - response.meta = { - index: { - id: meta.index.id - } + if (parser.readFailure) { + throw parser.readFailure } + markIdChanges(meta, committedIds) + // Serialize only changes. + const serialized = Buffer.concat([ + ...serializeChunks(dataspace, { meta, changes: true }) + ]) + const prepared = prepareIdIndex(serialized, committedIds) + const expectedDigest = digestBuffer(serialized) // TODO: write data every x commands or x minutes, // in a separate thread? let newfilename = basefile + '.' + task.id + '.' + extension publishing = true await faultPoint('before-command-changeset-write') - await writeFileAtomic(newfilename, uint8sab) + await writeFileAtomic(newfilename, serialized) // Final bytes include new records and mutations made by the custom // index hook. - await finalizeIndex(index, uint8sab, meta, task.id) + await finalizeIndex(index, serialized, meta, task.id, prepared) + response.source = loadDataSource(newfilename, meta, task.id) + if (response.source.digest !== expectedDigest || + digestBuffer(serialized) !== expectedDigest) { + throw new Error('Changeset changed during finalization') + } + response.meta = { index: { id: prepared.ids } } if (integrityFile) { - await appendIntegrityRecord( - integrityFile, - newfilename, - uint8sab - ) + await appendArtifactIntegrity(integrityFile, newfilename, + expectedDigest, meta, task.id) } await faultPoint('after-command-changeset-write') meta.parts++ @@ -164,7 +181,21 @@ export default async function runCommand(commandStr) { } catch (err) { console.error('task error', err) + if (parser.readFailure) { + throw parser.readFailure + } throw publishing ? storageError(err) : err } return response } + +export function close() { + if (dataset) { + dataset.close() + dataset = undefined + parser = undefined + dataspace = undefined + meta = {} + committedIds = new Map() + } +} diff --git a/src/command-worker.mjs b/src/command-worker.mjs index aa98509..fe43399 100644 --- a/src/command-worker.mjs +++ b/src/command-worker.mjs @@ -1,5 +1,5 @@ import { parentPort } from 'node:worker_threads' -import runCommand, { initialize } from '../src/command-worker-module.mjs' +import runCommand, { initialize, close } from '../src/command-worker-module.mjs' parentPort.on('message', async data => { let result @@ -15,5 +15,8 @@ parentPort.on('message', async data => { storageFailure: Boolean(error.storageFailure) } } + finally { + close() + } parentPort.postMessage(result) }) diff --git a/src/file-data.mjs b/src/file-data.mjs new file mode 100644 index 0000000..58bd177 --- /dev/null +++ b/src/file-data.mjs @@ -0,0 +1,466 @@ +import fs from 'node:fs' +import path from 'node:path' +import { createHash } from 'node:crypto' +import { isDeepStrictEqual } from 'node:util' +import idIndex, { + addUniqueId, readIdEntries, mergeIdIndex +} from './index.id.mjs' +import offsetIndex from './index.offset.mjs' +import { loadStoredIndex } from './index-files.mjs' +import JSONTag from '@muze-nl/jsontag' +import Parser from '@muze-nl/od-jsontag/src/parse.mjs' +import { RecoveryIntegrityError, assertChangesetExists } from './recovery.mjs' +import { getDefaultIntegrityFile, loadIntegrityManifest, verifyDigest } + from './integrity.mjs' +import { storageError } from './storage.mjs' + +function identity(stat) { + return { + device: stat.dev, inode: stat.ino, size: stat.size, + modified: stat.mtimeMs, changed: stat.ctimeMs + } +} + +function assertIdentity(fd, source) { + if (JSON.stringify(identity(fs.fstatSync(fd))) !== + JSON.stringify(source.identity)) { + throw new Error(`Dataset source changed: ${source.file}`) + } +} + +// A fixed-size buffer covers both hashing and framing; payloads are skipped +// without decoding or allocating a buffer the size of a record/file. +class FileScan { + constructor(fd, file) { + this.fd = fd + this.file = file + this.buffer = Buffer.allocUnsafe(64 * 1024) + this.cursor = 0 + this.available = 0 + this.offset = 0 + this.digest = createHash('sha256') + } + + peek() { + if (this.cursor === this.available) { + this.available = fs.readSync( + this.fd, this.buffer, 0, this.buffer.length, this.offset + ) + this.cursor = 0 + if (!this.available) { + return undefined + } + this.digest.update(this.buffer.subarray(0, this.available)) + } + return this.buffer[this.cursor] + } + + advance(length = 1) { + while (length > 0) { + if (this.peek() === undefined) { + this.fail('truncated record payload') + } + const consumed = Math.min(length, this.available - this.cursor) + this.cursor += consumed + this.offset += consumed + length -= consumed + } + } + + recordId(end) { + while (this.offset < end && [9, 10, 13, 32].includes(this.peek())) { + this.advance() + } + if (this.offset === end) { + return undefined + } + if (this.peek() === 126) { + // A reference-only record needs the full parser to resolve its ID. + return null + } + if (this.peek() !== 60) { + return undefined + } + const bytes = [] + let quoted = false + let escaped = false + while (this.offset < end) { + const byte = this.peek() + this.advance() + bytes.push(byte) + if (escaped) { + escaped = false + } + else if (quoted && byte === 92) { + escaped = true + } + else if (byte === 34) { + quoted = !quoted + } + else if (!quoted && byte === 62) { + // Share the actual tag grammar, including Unicode escapes. + const parser = this.tagParser ??= new JSONTag.Parser() + parser.input = Buffer.from(bytes).toString('utf8') + parser.at = 0 + parser.next() + try { + const attributes = parser.tag().attributes + const value = {} + JSONTag.setAttributes(value, attributes) + return JSONTag.getAttribute(value, 'id') + } + catch (error) { + this.fail(`invalid record tag: ${error.message}`) + } + } + } + this.fail('unterminated record tag') + } + + number(kind) { + let digits = 0 + let value = 0 + while (this.peek() >= 48 && this.peek() <= 57) { + value = value * 10 + this.peek() - 48 + this.advance() + digits++ + if (!Number.isSafeInteger(value)) { + this.fail(`invalid ${kind}`) + } + } + if (!digits) { + this.fail(`malformed ${kind}`) + } + return value + } + + fail(message) { + throw new RecoveryIntegrityError(`Invalid OD-JSONTag data: ${message}`, { + file: this.file, + recordKind: 'OD-JSONTag data' + }) + } +} + +export function hashFile(file) { + return hashSource(file).digest +} + +function hashSource(file) { + const fd = fs.openSync(file, 'r') + try { + const source = { file, identity: identity(fs.fstatSync(fd)) } + const scan = new FileScan(fd, file) + while (scan.peek() !== undefined) { + scan.advance(scan.available - scan.cursor) + } + assertIdentity(fd, source) + if (scan.offset !== source.identity.size) { + throw new Error(`Unexpected EOF while hashing: ${file}`) + } + return { ...source, size: scan.offset, digest: scan.digest.digest('hex') } + } + finally { + fs.closeSync(fd) + } +} + +export function scanDataFile(filename, recordIds = null) { + const file = path.resolve(filename) + const fd = fs.openSync(file, 'r') + try { + const source = { + file, identity: identity(fs.fstatSync(fd)), offsets: {} + } + const scan = new FileScan(fd, file) + let record = 0 + while (scan.peek() !== undefined) { + const byte = scan.peek() + if (byte === 10 || byte === 13) { + scan.advance() + continue + } + if (byte === 43) { + scan.advance() + record += scan.number('skip record') + if (scan.peek() !== undefined && + scan.peek() !== 10 && scan.peek() !== 13) { + scan.fail('malformed skip record') + } + } + else { + if (byte !== 40) { + scan.fail('expected record length') + } + scan.advance() + const length = scan.number('record length') + if (scan.peek() !== 41 || length === 0) { + scan.fail('malformed record length') + } + scan.advance() + const start = scan.offset + const end = start + length + if (recordIds) { + recordIds.set(record, scan.recordId(end)) + } + scan.advance(end - scan.offset) + source.offsets[record++] = [start, scan.offset] + } + if (!Number.isSafeInteger(record) || record >= 0xffffffff) { + scan.fail('invalid record number') + } + } + assertIdentity(fd, source) + if (scan.offset !== source.identity.size) { + scan.fail('unexpected EOF') + } + source.size = scan.offset + source.digest = scan.digest.digest('hex') + return source + } + finally { + fs.closeSync(fd) + } +} + +// Keep read failure in host-owned state even when a handler/query catches it. +export class FileParser extends Parser { + firstParse(target) { + try { + return super.firstParse(target) + } + catch (error) { + this.readFailure = storageError(error) + throw this.readFailure + } + } + + getLineProxy(index) { + try { + return super.getLineProxy(index) + } + catch (error) { + this.readFailure = storageError(error) + throw this.readFailure + } + } +} + +export class FileDataset { + constructor(meta = {}, immutable = true, ParserType = FileParser) { + this.parser = new ParserType(undefined, immutable) + this.parser.meta = { ...meta, resultArray: [] } + this.handles = [] + this.sources = [] + this.root = undefined + } + + append(source) { + const fd = fs.openSync(source.file, 'r') + try { + assertIdentity(fd, source) + this.root = this.parser.parse(fd, source.offsets) + this.handles.push(fd) + this.sources.push(source) + return this.root + } + catch (error) { + fs.closeSync(fd) + throw error + } + } + + open(sources) { + try { + for (const source of sources) { + this.append(source) + } + return this.root + } + catch (error) { + this.close() + throw error + } + } + + rebuildIds() { + const records = new Set() + for (const source of this.sources) { + for (const number of Object.keys(source.offsets)) { + records.add(Number(number)) + } + } + const ids = new Map() + for (const number of [...records].sort((a, b) => a - b)) { + const value = this.parser.getLineProxy(number) + const id = JSONTag.getAttribute(value, 'id') + addUniqueId(ids, id, number) + } + this.parser.meta.index = { ...this.parser.meta.index, id: ids } + return ids + } + + close() { + for (const fd of this.handles.splice(0)) { + fs.closeSync(fd) + } + this.sources = [] + this.root = undefined + } +} + +function validateOffsets(offsets, source) { + const fail = () => { + throw new Error(`Invalid offset index: ${source.file}`) + } + if (!offsets || typeof offsets !== 'object' || Array.isArray(offsets)) { + fail() + } + let previousEnd = 0 + for (const [key, range] of Object.entries(offsets)) { + const number = Number(key) + if (!Number.isSafeInteger(number) || number < 0 || + number >= 0xfffffffe || String(number) !== key || + !Array.isArray(range) || range.length !== 2) { + fail() + } + const [start, end] = range + if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || + start < previousEnd || start < 3 || end <= start || + end > source.size) { + fail() + } + previousEnd = end + } +} + +function readRecordIds(source, recordIds) { + const fd = fs.openSync(source.file, 'r') + try { + assertIdentity(fd, source) + const scan = new FileScan(fd, source.file) + for (const [number, [start, end]] of Object.entries(source.offsets)) { + // Reuse buffered bytes when adjacent headers share a read. + const bufferStart = scan.offset - scan.cursor + if (start >= bufferStart && start < bufferStart + scan.available) { + scan.cursor = start - bufferStart + } + else { + scan.cursor = 0 + scan.available = 0 + } + scan.offset = start + recordIds.set(Number(number), scan.recordId(end)) + } + assertIdentity(fd, source) + } + finally { + fs.closeSync(fd) + } +} + +export function loadDataSource(filename, meta, command = null, options = {}, + recordIds = null) { + const file = path.resolve(filename) + const offsets = loadStoredIndex(offsetIndex, meta, command, options) + let source + if (offsets === undefined || options.validateIndexes) { + source = scanDataFile(file, recordIds) + if (offsets !== undefined && + !isDeepStrictEqual(offsets, source.offsets)) { + throw new Error(`Offset index does not match data: ${file}`) + } + } + else { + source = hashSource(file) + validateOffsets(offsets, source) + source.offsets = offsets + } + if (options.manifest) { + verifyDigest(options.manifest, options.integrityFile, file, + source.digest, { required: options.integrityRequired }) + } + if (recordIds && offsets !== undefined && !options.validateIndexes) { + readRecordIds(source, recordIds) + } + return source +} + +function idsFromRecords(records) { + const ids = new Map() + for (const number of [...records.keys()].sort((a, b) => a - b)) { + const id = records.get(number) + addUniqueId(ids, id, number) + } + return ids +} + +export async function loadFileData(files) { + const integrityFile = files.integrityFile || + getDefaultIntegrityFile(files.dataFile) + const manifest = await loadIntegrityManifest(integrityFile) + return readFileData({...files, integrityFile}, manifest) +} + +// Initialization validates current bytes before any trusted baseline exists. +export function validateUnsealedData(files) { + return readFileData({...files, + validateIndexes: true, rebuildIndexes: false}, null) +} + +async function readFileData(files, manifest) { + const paths = [files.dataFile, ...files.commands.map(id => { + return assertChangesetExists(files.dataFile, id) + })] + const meta = { + data: path.dirname(path.resolve(files.dataFile)), + parts: files.commands.length + } + let ids = new Map() + let unresolvedReferences = false + const sources = paths.map((file, part) => { + const command = part === 0 ? null : files.commands[part - 1] + const options = { ...files, manifest, integrityRequired: true } + const storedIds = loadStoredIndex(idIndex, meta, command, options) + let recordIds = null + if (storedIds === undefined || files.validateIndexes) { + recordIds = new Map() + } + const source = loadDataSource(file, meta, command, options, recordIds) + const records = new Set(Object.keys(source.offsets).map(Number)) + let entries + if (storedIds === undefined) { + entries = idsFromRecords(recordIds) + unresolvedReferences ||= [...recordIds.values()].includes(null) + } + else { + entries = readIdEntries(storedIds, records) + if (files.validateIndexes && + !isDeepStrictEqual(entries, idsFromRecords(recordIds))) { + throw new Error(`ID index does not match data: ${file}`) + } + } + mergeIdIndex(ids, records, entries) + return source + }) + if (!sources[0].size) { + throw new Error('Empty base dataset') + } + const dataset = new FileDataset() + try { + dataset.open(sources) + if (unresolvedReferences) { + ids = dataset.rebuildIds() + } + meta.index = { id: ids } + if (files.schemaFile) { + meta.schema = JSONTag.parse( + fs.readFileSync(files.schemaFile, 'utf8') + ) + } + return { sources, meta } + } + finally { + dataset.close() + } +} diff --git a/src/index-files.mjs b/src/index-files.mjs new file mode 100644 index 0000000..521ca7c --- /dev/null +++ b/src/index-files.mjs @@ -0,0 +1,68 @@ +import fs from 'node:fs' +import path from 'node:path' +import { appendIntegrityDigests, digestBuffer, verifyIntegrity } + from './integrity.mjs' + +export function indexPath(meta, kind, command = null) { + const suffix = command === null ? '' : `.${command}` + return path.join(meta.data, `index.${kind}${suffix}.json`) +} + +export function readIndexFile(file, options = {}) { + const bytes = fs.readFileSync(file) + if (options.manifest) { + verifyIntegrity(options.manifest, options.integrityFile, file, bytes, { + required: options.integrityRequired + }) + } + return JSON.parse(bytes) +} + +export function loadStoredIndex(index, meta, command, options = {}) { + if (options.rebuildIndexes) { + return undefined + } + try { + return index.load(meta, command, options) + } + catch (error) { + if (error.code !== 'ENOENT') { + throw error + } + return undefined + } +} + +// Finalizers have completed all writes before these bytes are fingerprinted. +export function indexDigests(meta, command = null) { + const digests = [] + for (const kind of ['offset', 'id']) { + const file = indexPath(meta, kind, command) + let bytes + try { + bytes = fs.readFileSync(file) + } + catch (error) { + if (error.code === 'ENOENT') { + continue + } + throw error + } + digests.push([file, digestBuffer(bytes)]) + } + return digests +} + +export async function appendIndexIntegrity( + integrityFile, meta, command = null +) { + await appendIntegrityDigests(integrityFile, indexDigests(meta, command)) +} + +export async function appendArtifactIntegrity( + integrityFile, file, digest, meta, command = null +) { + await appendIntegrityDigests(integrityFile, [ + [file, digest], ...indexDigests(meta, command) + ]) +} diff --git a/src/index.id.mjs b/src/index.id.mjs index 6383758..940f6db 100644 --- a/src/index.id.mjs +++ b/src/index.id.mjs @@ -1,51 +1,112 @@ -import { publishFileSync } from './storage.mjs' -import fs from 'fs' +import { readIndexFile } from './index-files.mjs' +import path from 'node:path' import JSONTag from '@muze-nl/jsontag' -import { getIndex } from '@muze-nl/od-jsontag/src/symbols.mjs' +import { getIndex, isChanged } from '@muze-nl/od-jsontag/src/symbols.mjs' +import { publishFileSync } from './storage.mjs' +import { scanOdJsonTagRecords } from './recovery.mjs' + +export function addUniqueId(ids, id, number) { + if (!id) { + return + } + if (typeof id !== 'string') { + throw new Error('Record IDs must be strings') + } + if (ids.has(id) && ids.get(id) !== number) { + throw new Error(`Duplicate ID: ${id}`) + } + ids.set(id, number) +} + +export function readIdEntries(value, records) { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error('Invalid ID index: expected an object') + } + const entries = new Map() + const assigned = new Set() + for (const [id, number] of Object.entries(value)) { + if (!id || !Number.isSafeInteger(number) || number < 0 || + !records.has(number) || assigned.has(number)) { + throw new Error(`Invalid ID index entry: ${id}`) + } + assigned.add(number) + entries.set(id, number) + } + return entries +} + +export function mergeIdIndex(ids, records, entries) { + // Remove all replaced records first, allowing ID swaps within one command. + for (const [id, number] of ids) { + if (records.has(number)) { + ids.delete(id) + } + } + for (const [id, number] of entries) { + addUniqueId(ids, id, number) + } + return ids +} + +export function markIdChanges(meta, committedIds) { + const original = new Map([...committedIds].map(([id, number]) => { + return [number, id] + })) + // JSONTag edits its attribute object in place. Check only materialized + // records so ID-only mutations participate in change serialization. + for (const value of Object.values(meta.resultArray)) { + if (value && JSONTag.getAttribute(value, 'id') !== + original.get(value[getIndex])) { + value[isChanged] = true + } + } +} + +export function prepareIdIndex(serialized, committedIds = new Map()) { + let bytes + if (typeof serialized === 'string') { + bytes = Buffer.from(serialized) + } + else { + bytes = Buffer.from(serialized.buffer, + serialized.byteOffset, serialized.byteLength) + } + const entries = new Map() + const records = new Set() + const parser = new JSONTag.Parser() + scanOdJsonTagRecords(bytes, 'serialized data', 'ID index source', + (number, start, end) => { + records.add(number) + // Command output is already buffered. Decode one record at a time, + // but parse only its leading tag; references need no resolution. + parser.input = bytes.toString('utf8', start, end) + parser.at = 0 + parser.ch = ' ' + parser.whitespace() + if (parser.ch === '<') { + const value = {} + JSONTag.setAttributes(value, parser.tag().attributes) + addUniqueId(entries, JSONTag.getAttribute(value, 'id'), number) + } + }) + const ids = mergeIdIndex(new Map(committedIds), records, entries) + return { entries, ids } +} + +function filename(meta, uuid) { + let name = 'index.id.json' + if (uuid !== null) { + name = `index.id.${uuid}.json` + } + return path.join(meta.data, name) +} export default { - create(data, meta) { - console.log('creating ' + meta.data + '/index.id.json') - // jsontag parse automatically fills meta.index.id, so no need to create - // anything - // just store meta.index.id in index.id.json - const index = {} - for (const key of meta.index.id.keys()) { - if (!key) { - continue - } - const entity = meta.index.id.get(key) - if (!entity) { - continue - } - index[key] = entity[getIndex] - } - publishFileSync(meta.data + '/index.id.json', JSON.stringify(index)) - }, - update(data, meta, changes) { - if (!changes.length) { - return - } - const index = {} - for (const entry of changes) { - const id = JSONTag.getAttribute(entry, 'id') - if (id) { - index[id] = entry[getIndex] - } - } - publishFileSync( - meta.data + '/index.id.' + changes.uuid + '.json', - JSON.stringify(index) - ) - }, - load(uuid = null) { - let filename - if (!uuid) { - filename = 'index.id.json' - } - else { - filename = 'index.id.' + filename + '.json' - } - return JSON.parse(fs.readFileSync(meta.data + filename)) - } + write(meta, entries, uuid = null) { + publishFileSync(filename(meta, uuid), + JSON.stringify(Object.fromEntries(entries))) + }, + load(meta, uuid = null, options = {}) { + return readIndexFile(filename(meta, uuid), options) + } } diff --git a/src/index.mjs b/src/index.mjs index ed8b33d..e5ed022 100644 --- a/src/index.mjs +++ b/src/index.mjs @@ -1,13 +1,11 @@ -import idIndex from './index.id.mjs' +import idIndex, { prepareIdIndex } from './index.id.mjs' import offsetIndex from './index.offset.mjs' const defaultIndex = { create(data, meta) { - idIndex.create(data, meta) offsetIndex.create(data, meta) }, update(data, meta, changes) { - idIndex.update(data, meta, changes) offsetIndex.update(data, meta, changes) }, finalize(serialized, meta, uuid = null) { @@ -15,15 +13,19 @@ const defaultIndex = { }, load(meta, uuid=null) { return { - id: idIndex.load(uuid), - offset: offsetIndex.load(uuid) + id: idIndex.load(meta, uuid), + offset: offsetIndex.load(meta, uuid) } } } -export function finalizeIndex(index, serialized, meta, uuid = null) { +export async function finalizeIndex( + index, serialized, meta, uuid = null, prepared = prepareIdIndex(serialized) +) { const finalize = index.finalize ?? defaultIndex.finalize - return finalize.call(index, serialized, meta, uuid) + await finalize.call(index, serialized, meta, uuid) + idIndex.write(meta, prepared.entries, uuid) + meta.index = { ...meta.index, id: prepared.ids } } export default defaultIndex diff --git a/src/index.offset.mjs b/src/index.offset.mjs index 99b6c7d..f9f0544 100644 --- a/src/index.offset.mjs +++ b/src/index.offset.mjs @@ -1,5 +1,5 @@ import { publishFileSync } from './storage.mjs' -import fs from 'fs' +import { indexPath, readIndexFile } from './index-files.mjs' import path from 'node:path' import { Buffer } from 'node:buffer' import { publishFile as writeFileAtomic } from './storage.mjs' @@ -63,14 +63,7 @@ export default { JSON.stringify(index) ) }, - load(uuid = null) { - let filename - if (!uuid) { - filename = 'index.offset.json' - } - else { - filename = 'index.offset.' + filename + '.json' - } - return JSON.parse(fs.readFileSync(meta.data + filename)) + load(meta, uuid = null, options = {}) { + return readIndexFile(indexPath(meta, 'offset', uuid), options) } } diff --git a/src/initialize-integrity.mjs b/src/initialize-integrity.mjs new file mode 100644 index 0000000..8a2bf31 --- /dev/null +++ b/src/initialize-integrity.mjs @@ -0,0 +1,91 @@ +import fs from 'node:fs/promises' +import path from 'node:path' +import { FileDataset, validateUnsealedData } from './file-data.mjs' +import { indexPath } from './index-files.mjs' +import { serializeIntegrityRecords } from './integrity.mjs' +import { publishFile, syncFile } from './storage.mjs' +import { acquireOwnership } from './store-ownership.mjs' +import { + storePaths, mutableDirectories, inspectUnsealedStore, inventory, hash +} from './store-inspection.mjs' + +export async function initializeIntegrity(options = {}) { + const config = storePaths(options) + const ownership = await acquireOwnership(mutableDirectories(config), { + purpose: 'initialize-integrity' + }) + let publishing = false + try { + try { + await fs.lstat(config.integrityFile) + throw new Error('Integrity manifest already exists; initialization ' + + 'never replaces an established baseline') + } + catch (error) { + if (error.code !== 'ENOENT') { + throw error + } + } + const report = await inspectUnsealedStore(config) + if (!report.ready) { + const problems = [...report.errors, + ...report.commands.map(command => command.problem) + .filter(Boolean)] + throw new Error('Store is not ready for integrity initialization: ' + + (problems.join('; ') || 'pending or uncertain command history')) + } + const loaded = await validateUnsealedData({ + dataFile: config.datafile, commands: report.committed, + schemaFile: config.schemaFile + }) + const dataset = new FileDataset(loaded.meta) + try { + dataset.open(loaded.sources) + // Decode every record during this explicit baseline operation. + dataset.rebuildIds() + } + finally { + dataset.close() + } + const meta = {data: path.dirname(config.datafile)} + const digests = [] + for (const [part, source] of loaded.sources.entries()) { + const command = part === 0 ? null : report.committed[part - 1] + if (source.digest !== report.files[source.file]) { + throw new Error('Store changed during integrity initialization') + } + digests.push([source.file, source.digest]) + for (const kind of ['offset', 'id']) { + const file = indexPath(meta, kind, command) + if (report.files[file] != null) { + digests.push([file, report.files[file]]) + } + } + } + // Preserve the committed prefix before publishing its baseline. + const files = new Set([...digests.map(([file]) => file), + config.commandLog, config.commandStatus, ...config.requiredFiles]) + for (const file of files) { + await syncFile(file) + } + const current = hash(JSON.stringify(await inventory(config))) + if (current !== report.fingerprint) { + throw new Error('Store changed during integrity initialization') + } + publishing = true + await publishFile(config.integrityFile, + serializeIntegrityRecords(config.integrityFile, digests) + '\n') + await ownership.release() + return { + integrityFile: config.integrityFile, + files: digests.length, + committed: report.committed + } + } + catch (error) { + if (!publishing) { + await ownership.release() + } + throw error + } +} diff --git a/src/integrity.mjs b/src/integrity.mjs index acf7d8c..ed56d61 100644 --- a/src/integrity.mjs +++ b/src/integrity.mjs @@ -9,7 +9,9 @@ const integrityAlgorithm = 'sha256' export function getDefaultIntegrityFile(dataFile) { const extension = dataFile.split('.').pop() - const basefile = dataFile.substring(0, dataFile.length - (extension.length + 1)) + const basefile = dataFile.substring( + 0, dataFile.length - (extension.length + 1) + ) return `${basefile}.integrity.${extension}` } @@ -26,9 +28,14 @@ export async function loadIntegrityManifest(integrityFile) { let text try { text = await fs.readFile(integrityFile, 'utf8') - } catch (error) { + } + catch (error) { if (error.code === 'ENOENT') { - return entries + throw new RecoveryIntegrityError( + `Missing integrity manifest ${integrityFile}; ` + + 'use recover.mjs init-integrity for a stopped existing store', + { file: integrityFile, recordKind: 'integrity manifest' } + ) } throw error } @@ -39,7 +46,8 @@ export async function loadIntegrityManifest(integrityFile) { let record try { record = JSONTag.parse(line) - } catch (cause) { + } + catch (cause) { throw new RecoveryIntegrityError('Invalid integrity manifest record', { file: integrityFile, lineNumber: index + 1, @@ -73,7 +81,17 @@ export async function loadIntegrityManifest(integrityFile) { return entries } -export function verifyIntegrity(manifest, integrityFile, file, buffer, options = {}) { +export function verifyIntegrity( + manifest, integrityFile, file, buffer, options = {} +) { + return verifyDigest( + manifest, integrityFile, file, digestBuffer(buffer), options + ) +} + +export function verifyDigest( + manifest, integrityFile, file, actual, options = {} +) { const key = integrityFileKey(integrityFile, file) const record = manifest.get(key) if (!record) { @@ -85,7 +103,6 @@ export function verifyIntegrity(manifest, integrityFile, file, buffer, options = } return false } - const actual = digestBuffer(buffer) if (record.digest !== actual) { throw new RecoveryIntegrityError(`Integrity mismatch for ${key}`, { file, @@ -96,11 +113,30 @@ export function verifyIntegrity(manifest, integrityFile, file, buffer, options = } export async function appendIntegrityRecord(integrityFile, file, buffer) { - const record = { - file: integrityFileKey(integrityFile, file), - algorithm: integrityAlgorithm, - digest: digestBuffer(buffer) - } - await appendFile(integrityFile, JSONTag.stringify(record)) - return record + return appendIntegrityDigest(integrityFile, file, digestBuffer(buffer)) +} + +export function integrityRecords(integrityFile, digests) { + return digests.map(([file, digest]) => ({ + file: integrityFileKey(integrityFile, file), + algorithm: integrityAlgorithm, + digest + })) +} + +export function serializeIntegrityRecords(integrityFile, digests) { + return integrityRecords(integrityFile, digests) + .map(record => JSONTag.stringify(record)).join('\n') +} + +export async function appendIntegrityDigests(integrityFile, digests) { + if (digests.length) { + await appendFile(integrityFile, + serializeIntegrityRecords(integrityFile, digests)) + } +} + +export async function appendIntegrityDigest(integrityFile, file, digest) { + await appendIntegrityDigests(integrityFile, [[file, digest]]) + return integrityRecords(integrityFile, [[file, digest]])[0] } diff --git a/src/load-worker.mjs b/src/load-worker.mjs index c90bcf1..a02cae2 100644 --- a/src/load-worker.mjs +++ b/src/load-worker.mjs @@ -1,69 +1,6 @@ import { parentPort } from 'node:worker_threads' -import JSONTag from '@muze-nl/jsontag' -import Parser from '@muze-nl/od-jsontag/src/parse.mjs' -import fs from 'fs' -import path from 'path' -import serialize from '@muze-nl/od-jsontag/src/serialize.mjs' -import { assertChangesetExists, assertOdJsonTagFraming } from './recovery.mjs' -import { loadIntegrityManifest, verifyIntegrity } from './integrity.mjs' - -const parser = new Parser() - -parentPort.on('message', async (files) => { - let meta = { - index: { - id: new Map() - } - } - - await import(files.indexFile) - const extension = files.dataFile.split('.').pop() - const basefile = files.dataFile.substring(0, files.dataFile.length - (extension.length + 1)) //+1 for . character - meta.data = path.dirname(basefile) - const integrityManifest = files.integrityFile - ? await loadIntegrityManifest(files.integrityFile) - : null - let data - let jsontag - // TODO - // - only load index files - // - for each command id - // - load files as raw bytes - // - index.id.*.jsontag and index.offset.*.jsontag to create proxies that will get the correct offset on access - // - do the same for resultSet[0] - the dataspace root entity - // don't parse entire files with od-jsontag - // add version info in proxies with a symbol to get that information - if (fs.existsSync(files.dataFile)) { - jsontag = fs.readFileSync(files.dataFile) - if (integrityManifest) { - verifyIntegrity(integrityManifest, files.integrityFile, files.dataFile, jsontag, { - required: files.integrityRequired - }) - } - assertOdJsonTagFraming(jsontag, files.dataFile, 'base OD-JSONTag data') - data = parser.parse(jsontag) - } - for (let command of files.commands) { - const changesetFile = assertChangesetExists(files.dataFile, command) - jsontag = fs.readFileSync(changesetFile) - if (integrityManifest) { - verifyIntegrity(integrityManifest, files.integrityFile, changesetFile, jsontag, { - required: files.integrityRequired - }) - } - assertOdJsonTagFraming(jsontag, changesetFile, 'changeset OD-JSONTag data') - data = parser.parse(jsontag) - } - if (files.schemaFile) { - jsontag = fs.readFileSync(files.schemaFile, 'utf-8') - meta.schema = JSONTag.parse(jsontag) - } - - const sab = serialize(data, {meta}) - - parentPort.postMessage({ - data: sab, - meta - }) +import { loadFileData } from './file-data.mjs' +parentPort.on('message', async files => { + parentPort.postMessage(await loadFileData(files)) }) diff --git a/src/query-worker-module.mjs b/src/query-worker-module.mjs index 6477182..84ce8a9 100644 --- a/src/query-worker-module.mjs +++ b/src/query-worker-module.mjs @@ -2,23 +2,39 @@ import {VM} from 'vm2' import { memoryUsage } from 'node:process' import JSONTag from '@muze-nl/jsontag' import {source} from '@muze-nl/od-jsontag/src/symbols.mjs' -import Parser from '@muze-nl/od-jsontag/src/parse.mjs' +import { FileDataset, FileParser } from './file-data.mjs' import {_,from,not,anyOf,allOf,asc,desc,sum,count,avg,max,min,many,one,first,distinct} from '@muze-nl/jaqt' import process from 'node:process' +// vm2 protects array species before host calls. Give it the already-safe +// constructor descriptor on raw query arrays before the read-only proxy exists, +// so methods like map/filter need no temporary mutation through that proxy. +class QueryParser extends FileParser { + getArrayProxy(array, parent) { + Object.defineProperty(array, 'constructor', { + value: undefined, + writable: false, + enumerable: false, + configurable: false + }) + return super.getArrayProxy(array, parent) + } +} + let dataspace let metaProxy = { index: { } } -const parser = new Parser() +let dataset +let parser const metaIdProxy = { get: (id) => { let index = parser.meta.index.id.get(id) if (index || index===0) { - return parser.meta.resultArray[index] + return parser.getLineProxy(index) } }, has: (id) => { @@ -28,6 +44,11 @@ const metaIdProxy = { const tasks = { init: async (task) => { + if (dataset) { + dataset.close() + } + dataset = new FileDataset(task.req.meta, true, QueryParser) + parser = dataset.parser if (task.req.access) { task.req.access = await import(task.req.access) task.req.access = task.req.access.default @@ -39,9 +60,7 @@ const tasks = { if (task.req.meta.schema) { parser.meta.schema = task.req.meta.schema } - for (let sab of task.req.body) { //body contains an array of sharedArrayBuffers with initial data and changes - dataspace = parser.parse(sab) - } + dataspace = dataset.open(task.req.sources) metaProxy.index.id = metaIdProxy metaProxy.schema = parser.meta.schema //@TODO: add meta.index.references? and baseURL @@ -51,11 +70,35 @@ const tasks = { if (task.req.meta.index) { parser.meta.index = task.req.meta.index } - dataspace = parser.parse(task.req.body) //update only has a single changeset + dataspace = dataset.append(task.req.source) return true }, query: async (task) => { - return runQuery(task.req.path, task.req, task.req.body, task.timeout) + let response + try { + response = runQuery( + task.req.path, task.req, task.req.body, task.timeout + ) + } + catch (error) { + if (!parser.readFailure) { + throw error + } + } + // Query code can catch errors or throw its own properties. Only the + // parser's host-owned failure state identifies a storage problem. + if (parser.readFailure) { + const error = {code: 500, message: 'Unable to read committed data'} + return { + code: 500, + storageFailure: true, + jsontag: task.req.jsontag, + body: task.req.jsontag + ? JSONTag.stringify(error) + : JSON.stringify(error) + } + } + return response }, memoryUsage: async () => { let result = memoryUsage() @@ -67,8 +110,12 @@ const tasks = { export default tasks export function runQuery(pointer, request, query, timeout=1000) { - if (!pointer) { throw new Error('missing pointer parameter')} - if (!request) { throw new Error('missing request parameter')} + if (!pointer) { + throw new Error('missing pointer parameter') +} + if (!request) { + throw new Error('missing request parameter') +} let response = { jsontag: request.jsontag } @@ -81,7 +128,7 @@ export function runQuery(pointer, request, query, timeout=1000) { timeout: timeout, allowAsync: false, sandbox: { - root: dataspace, //@TODO: if we don't pass the root, we can later shard + root: dataspace, data: result, meta: metaProxy, _, @@ -110,33 +157,38 @@ export function runQuery(pointer, request, query, timeout=1000) { result = vm.run(query) let used = Math.round(process.memoryUsage().heapUsed / 1024 / 1024); console.log(`(${used} MB)`); - } catch(err) { + } + catch(err) { console.log(err) response.code = 422; if (request.jsontag) { response.body = '{"message":'+JSON.stringify(''+err)+',"code":422}' - } else { + } + else { response.body = JSON.stringify({message:err, code: 422}) } } - } else { + } + else { result = linkReplacer(result, path+'/') } if (!response.code) { if (response.jsontag) { try { // JSONTag.stringify doesn't handle tagName/attributes well - // with od-jsontag, since some entries in result haven't been parsed yet + // with od-jsontag until result entries have been parsed. // only after parsing will these be available - // so force parsing of all result - od-jsontag should have its own stringify + // Force parsing before formatting the query response. parseAllObjects(result) response.body = JSONTag.stringify(result) - } catch(err) { + } + catch(err) { console.log(err) response.code = 500 response.body = '{"message":'+JSON.stringify(''+err)+',"code":500}' } - } else { + } + else { //@FIXME: replace recursive links response.body = JSON.stringify(result) } @@ -160,7 +212,8 @@ function parseAllObjects(o, reset=true) { parseAllObjects(v, false) } } - } else if (o && typeof o == 'object') { + } + else if (o && typeof o == 'object') { for (let k of Object.keys(o)) { if (o[k] && typeof o[k]=='object') { parseAllObjects(o[k], false) @@ -193,9 +246,11 @@ export function linkReplacer(data, baseURL) { data = data.map((entry,index) => { return linkReplacer(data[index], baseURL+index+'/') }) - } else if (type === 'link') { + } + else if (type === 'link') { // do nothing - } else if (data && typeof data === 'object') { + } + else if (data && typeof data === 'object') { if (data[source]) { data = data[source] } @@ -203,7 +258,8 @@ export function linkReplacer(data, baseURL) { Object.keys(data).forEach(key => { if (Array.isArray(data[key])) { data[key] = new JSONTag.Link(baseURL+key+'/') - } else if (data[key] && typeof data[key] === 'object') { + } + else if (data[key] && typeof data[key] === 'object') { if (JSONTag.getType(data[key])!=='link') { let id=JSONTag.getAttribute(data[key], 'id') if (!id) { diff --git a/src/store-inspection.mjs b/src/store-inspection.mjs index 64673d2..5ed9756 100644 --- a/src/store-inspection.mjs +++ b/src/store-inspection.mjs @@ -2,13 +2,14 @@ import fs from 'node:fs/promises' import path from 'node:path' import JSONTag from '@muze-nl/jsontag' import { from, _, anyOf, not } from '@muze-nl/jaqt' -import Parser from '@muze-nl/od-jsontag/src/parse.mjs' +import { FileDataset, hashFile, loadDataSource } from './file-data.mjs' +import idIndex from './index.id.mjs' +import { loadStoredIndex } from './index-files.mjs' import { createHash } from 'node:crypto' -import { assertOdJsonTagFraming, getChangesetPath } from './recovery.mjs' +import { getChangesetPath } from './recovery.mjs' import { getDefaultIntegrityFile, - loadIntegrityManifest, - verifyIntegrity + loadIntegrityManifest } from './integrity.mjs' export const hash = bytes => createHash('sha256').update(bytes).digest('hex') @@ -16,13 +17,6 @@ export const validCommandId = id => typeof id === 'string' && id.length > 0 && !/[/\\\0]/.test(id) export function storePaths(options = {}) { const datafile = path.resolve(options.datafile || './data.od-jsontag') - let integrity - if (options.integrity === undefined) { - integrity = Boolean(options.integrityFile) - } - else { - integrity = Boolean(options.integrity) - } const config = { datafile, commandLog: path.resolve(options.commandLog || './command-log.jsontag'), @@ -32,7 +26,9 @@ export function storePaths(options = {}) { integrityFile: path.resolve( options.integrityFile || getDefaultIntegrityFile(datafile) ), - integrity, + integrity: true, + validateIndexes: Boolean(options.validateIndexes), + rebuildIndexes: Boolean(options.rebuildIndexes), requiredFiles: (options.requiredFiles || []).map(file => path.resolve(file) ) @@ -101,7 +97,7 @@ export async function inventory(config) { throw new Error(`Symlink in store artifact: ${file}`) } if (entry.isFile()) { - files[file] = hash(await fs.readFile(file)) + files[file] = hashFile(file) } } } @@ -116,7 +112,7 @@ export async function inventory(config) { for (const file of required) { if (!(file in files)) { try { - files[file] = hash(await fs.readFile(file)) + files[file] = hashFile(file) } catch (error) { if (error.code !== 'ENOENT') { @@ -138,8 +134,16 @@ export async function inspectStore(options = {}) { return inspector.inspect() } +// Only the explicit initialization workflow may inspect an unsealed store. +export async function inspectUnsealedStore(options = {}) { + const config = storePaths({...options, + validateIndexes: true, rebuildIndexes: false}) + return new StoreInspector(config, true).inspect() +} + class StoreInspector { - constructor(config) { + constructor(config, unsealed = false) { + this.unsealed = unsealed this.config = config this.errors = [] this.warnings = [] @@ -147,22 +151,28 @@ class StoreInspector { this.commands = [] this.commandsById = new Map() this.manifest = null - this.parser = new Parser() - this.buffers = [] + this.dataset = new FileDataset() + this.verified = new Map() this.committed = [] - this.data = undefined this.prefixValid = false } async inspect() { - await this.validateConfiguredPaths() - this.files = await inventory(this.config) - await this.loadCommandHistory() - await this.loadIntegrityManifest() - await this.reconstructCommittedState() - this.validateStoreArtifacts() - await this.verifyStableSnapshot() - return this.createReport() + try { + await this.validateConfiguredPaths() + this.files = await inventory(this.config) + await this.loadCommandHistory() + if (!this.unsealed) { + await this.loadIntegrityManifest() + } + await this.reconstructCommittedState() + this.validateStoreArtifacts() + await this.verifyStableSnapshot() + return this.createReport() + } + finally { + this.dataset.close() + } } async loadCommandHistory() { @@ -327,13 +337,9 @@ class StoreInspector { async loadIntegrityManifest() { try { const integrityFile = this.config.integrityFile - const enabled = - this.config.integrity || this.files[integrityFile] != null - if (!enabled) { - return - } if (this.files[integrityFile] == null) { - throw new Error(`Missing integrity manifest ${integrityFile}`) + throw new Error(`Missing integrity manifest ${integrityFile}; ` + + 'use recover.mjs init-integrity for a stopped existing store') } await this.readRecords(integrityFile, 'integrity manifest') this.manifest = await loadIntegrityManifest(integrityFile) @@ -343,19 +349,16 @@ class StoreInspector { } } - async readVerifiedData(file) { - const bytes = await fs.readFile(file) - assertOdJsonTagFraming(bytes, file) - if (this.manifest) { - verifyIntegrity( - this.manifest, - this.config.integrityFile, - file, - bytes, - { required: true } - ) + async readVerifiedData(file, command = null) { + const meta = { data: path.dirname(this.config.datafile) } + const options = { + ...this.config, manifest: this.manifest, integrityRequired: true } - return bytes + // Verify present ID sidecars too; normal inspection never parses tags. + loadStoredIndex(idIndex, meta, command, options) + const source = loadDataSource(file, meta, command, options) + this.verified.set(file, source) + return source } async reconstructCommittedState() { @@ -369,12 +372,11 @@ class StoreInspector { async readBaseDataset() { try { - const bytes = await this.readVerifiedData(this.config.datafile) - if (!bytes.length) { + const source = await this.readVerifiedData(this.config.datafile) + if (!source.size) { throw new Error('Empty base dataset') } - this.data = this.parser.parse(bytes) - this.buffers.push(bytes) + this.dataset.append(source) } catch (error) { this.errors.push(error.message) @@ -395,7 +397,7 @@ class StoreInspector { } if (command.present) { try { - await this.readVerifiedData(command.file) + await this.readVerifiedData(command.file, command.id) } catch (error) { command.condition = 'corrupt' @@ -427,9 +429,8 @@ class StoreInspector { return } try { - const bytes = await this.readVerifiedData(command.file) - this.data = this.parser.parse(bytes) - this.buffers.push(bytes) + const source = this.verified.get(command.file) + this.dataset.append(source) this.committed.push(command.id) } catch (error) { @@ -508,8 +509,7 @@ class StoreInspector { commands: this.commands, committed: this.committed, ready, - data: this.data, - buffers: this.buffers + sources: [...this.dataset.sources] } } } diff --git a/src/store-runtime.mjs b/src/store-runtime.mjs index 71ba36c..74605b0 100644 --- a/src/store-runtime.mjs +++ b/src/store-runtime.mjs @@ -1,4 +1,3 @@ -import fs from 'node:fs' import path from 'node:path' import { fileURLToPath } from 'node:url' import JSONTag from '@muze-nl/jsontag' @@ -55,26 +54,20 @@ function createRuntimeConfiguration(options) { const datafile = options.datafile || './data.od-jsontag' const integrityFile = options.integrityFile || getDefaultIntegrityFile(datafile) - const integrity = Boolean( - options.integrity || - options.integrityFile || - fs.existsSync(integrityFile) - ) const storeOptions = { ...options, datafile, commandLog: options.commandLog || './command-log.jsontag', commandStatus: options.commandStatus || './command-status.jsontag', - integrity - } - if (integrity) { - storeOptions.integrityFile = integrityFile + integrityFile } const store = storePaths(storeOptions) return { store, schemaFile: options.schemaFile || null, + validateIndexes: Boolean(options.validateIndexes), + rebuildIndexes: Boolean(options.rebuildIndexes), maxWorkers: options.maxWorkers || 8, queryWorker: options.queryWorker || rootDirectory + '/src/query-worker.mjs', @@ -165,7 +158,7 @@ export class StoreRuntime { constructor(configuration, mechanisms) { this.configuration = configuration this.mechanisms = mechanisms - this.data = [] + this.sources = [] this.meta = {} this.status = new Map() this.commandQueue = [] @@ -225,11 +218,10 @@ export class StoreRuntime { dataFile: config.store.datafile, indexFile: config.indexFile, schemaFile: config.schemaFile, + validateIndexes: config.validateIndexes, + rebuildIndexes: config.rebuildIndexes, commands, - integrityFile: config.store.integrity - ? config.store.integrityFile - : null, - integrityRequired: config.store.integrity + integrityFile: config.store.integrityFile }, { timeout: config.loadTimeout, @@ -239,7 +231,7 @@ export class StoreRuntime { } initializeLoadedState(inspection, loaded) { - this.data = [loaded.data] + this.sources = loaded.sources this.meta = loaded.meta this.status = new Map( inspection.commands.map(command => { @@ -261,13 +253,16 @@ export class StoreRuntime { config.queryWorker, this.queryWorkerInitialTask(config.slowTimeout) ) + for (const pool of [this.queryWorkerPool, this.slowQueryWorkerPool]) { + pool.on?.('error', error => this.failStorage(error)) + } } queryWorkerInitialTask(timeout) { return { name: 'init', req: { - body: this.data, + sources: this.sources, meta: this.meta, access: this.configuration.access }, @@ -275,14 +270,18 @@ export class StoreRuntime { } } - runQuery(request, { slow = false } = {}) { + async runQuery(request, { slow = false } = {}) { let pool = this.queryWorkerPool let timeout = this.configuration.timeout if (slow) { pool = this.slowQueryWorkerPool timeout = this.configuration.slowTimeout } - return pool.run('query', request, { timeout }) + const result = await pool.run('query', request, { timeout }) + if (result.storageFailure) { + this.failStorage(new Error('Unable to read committed data')) + } + return result } getCommandStatus(commandId) { @@ -486,14 +485,11 @@ export class StoreRuntime { { ...command, meta: this.meta, - data: this.data, + sources: this.sources, commandsFile: config.commandsFile, indexFile: config.indexFile, datafile: config.store.datafile, - integrityFile: config.store.integrity - ? config.store.integrityFile - : null, - integrityRequired: config.store.integrity + integrityFile: config.store.integrityFile }, config.commandTimeout ) @@ -559,14 +555,14 @@ export class StoreRuntime { } publishCommandResult(result) { - if (!result.data) { + if (!result.source) { return } - this.data.push(result.data) + this.sources.push(result.source) Object.assign(this.meta, result.meta) const task = { name: 'update', - req: { body: result.data, meta: this.meta } + req: { source: result.source, meta: this.meta } } this.queryWorkerPool.update(task) this.slowQueryWorkerPool.update(task) @@ -593,8 +589,8 @@ export class StoreRuntime { async closeResources() { await this.serializeAcceptance() await this.runner - this.queryWorkerPool.close() - this.slowQueryWorkerPool.close() + await this.queryWorkerPool.close() + await this.slowQueryWorkerPool.close() if (!this.storageFailed && this.ownership) { await this.ownership.release() this.ownership = null diff --git a/src/workerPool.mjs b/src/workerPool.mjs index 710e30e..8ccb692 100644 --- a/src/workerPool.mjs +++ b/src/workerPool.mjs @@ -1,114 +1,160 @@ -import { AsyncResource } from 'async_hooks' -import { EventEmitter } from 'events' -import path from 'path' -import { Worker } from 'worker_threads' +import { AsyncResource } from 'node:async_hooks' +import { EventEmitter } from 'node:events' +import path from 'node:path' +import { Worker } from 'node:worker_threads' -const kTaskInfo = Symbol('kTaskInfo'); -const kWorkerFreedEvent = Symbol('kWorkerFreedEvent'); - -class WorkerPoolTaskInfo extends AsyncResource { - constructor(callback) { - super('WorkerPoolTaskInfo'); - this.callback = callback; - } +class QueryTask extends AsyncResource { + constructor(task, resolve, reject) { + super('WorkerPoolTaskInfo') + this.task = task + this.resolve = resolve + this.reject = reject + } - done(err, result) { - this.runInAsyncScope(this.callback, null, err, result); - this.emitDestroy(); // `TaskInfo`s are used only once. - } + done(error, result) { + if (error) { + this.runInAsyncScope(this.reject, null, error) + } + else { + this.runInAsyncScope(this.resolve, null, result) + } + this.emitDestroy() + } } -//@TODO: only create new workers when needed, not immediately - export default class WorkerPool extends EventEmitter { - constructor(numThreads, workerFile, initTask) { - super() - this.numThreads = numThreads - this.workerFile = workerFile - this.initTask = initTask - this.workers = [] - this.freeWorkers = [] - this.priority = new Map() - for (let i = 0; i < numThreads; i++) - this.addNewWorker(); - } + constructor(numThreads, workerFile, initTask) { + super() + this.workerFile = path.resolve(workerFile) + this.initTask = { + ...initTask, + req: { ...initTask.req, sources: [...initTask.req.sources] } + } + this.workers = new Map() + this.waiting = [] + this.terminating = new Set() + this.closed = false + this.failure = null + for (let i = 0; i < numThreads; i++) { + this.addNewWorker() + } + } - addNewWorker() { - const worker = new Worker(path.resolve(this.workerFile)); - this.priority[worker] = [] - worker.on('message', (result) => { - // In case of success: Call the callback that was passed to `runTask`, - // remove the `TaskInfo` associated with the Worker, and mark it as free - // again. - if (worker[kTaskInfo]) { - worker[kTaskInfo].done(null, result); - worker[kTaskInfo] = null; - } - if (this.priority[worker].length) { - let task = this.priority[worker].shift() + addNewWorker() { + const worker = new Worker(this.workerFile) + const state = { pending: [this.initTask], active: null, query: null } + this.workers.set(worker, state) + worker.on('message', result => { + const query = state.query + state.active = null + state.query = null + if (query) { + query.done(null, result) + } + this.dispatch(worker, state) + }) + worker.on('error', error => this.workerFailed(worker, error)) + worker.on('exit', code => { + if (!this.closed && this.workers.has(worker)) { + this.workerFailed(worker, new Error(`Query worker exited (${code})`)) + } + }) + this.dispatch(worker, state) + } + + dispatch(worker, state) { + if (this.closed || this.failure || state.active) { + return + } + let task = state.pending.shift() + if (!task) { + const query = this.waiting.shift() + if (!query) { + return + } + state.query = query + task = query.task + } + state.active = task worker.postMessage(task) - } else { - this.freeWorkers.push(worker); - this.emit(kWorkerFreedEvent); - } - }); - worker.on('error', (err) => { - // In case of an uncaught exception: Call the callback that was passed to - // `runTask` with the error. - if (worker[kTaskInfo]) - worker[kTaskInfo].done(err, null); - else - this.emit('error', err); - // Remove the worker from the list and start a new Worker to replace the - // current one. - this.workers.splice(this.workers.indexOf(worker), 1); - this.addNewWorker(); - }); - this.workers.push(worker); - worker.postMessage(this.initTask); - } + } - async run(name, req, options={}) { - return new Promise((resolve, reject) => { - this.runTask({name,req,...options}, (error, result) => { - if (error) { - return reject(error) + workerFailed(worker, error) { + const state = this.workers.get(worker) + if (!state) { + return + } + this.workers.delete(worker) + const termination = worker.terminate() + this.terminating.add(termination) + termination.then(() => this.terminating.delete(termination)) + if (state.query) { + state.query.done(error) + state.query = null + } + if (this.closed || this.failure) { + return } - return resolve(result) - }) - }) - } + // A failed source update cannot be skipped: stop publication/use of + // this pool. Query failures may replace the worker at the current head. + if (state.active?.name === 'init' || state.active?.name === 'update') { + this.failure = error + for (const query of this.waiting.splice(0)) { + query.done(error) + } + this.emit('error', error) + } + else if (!this.closed) { + this.addNewWorker() + } + } - runTask(task, callback) { - if (this.freeWorkers.length === 0) { - // No free threads, wait until a worker thread becomes free. - console.log('no free worker.. waiting') - this.once(kWorkerFreedEvent, () => this.runTask(task, callback)); - return; + run(name, req, options = {}) { + if (this.closed || this.failure) { + return Promise.reject(this.failure || new Error('Query pool closed')) + } + return new Promise((resolve, reject) => { + const task = { name, req, ...options } + this.waiting.push(new QueryTask(task, resolve, reject)) + for (const [worker, state] of this.workers) { + this.dispatch(worker, state) + } + }) } - const worker = this.freeWorkers.pop(); - worker[kTaskInfo] = new WorkerPoolTaskInfo(callback); - worker.postMessage(task); - } - update(task) { - for (let worker of this.workers) { - if (worker[kTaskInfo]) { // worker is busy - this.priority[worker].push(task) // push task on priority list for this specific worker - } else { // worker is free - worker.postMessage(task) // run task immediately - } + update(task) { + this.initTask = { + ...this.initTask, + req: { + ...this.initTask.req, + sources: [...this.initTask.req.sources, task.req.source], + meta: task.req.meta + } + } + for (const [worker, state] of this.workers) { + state.pending.push(task) + this.dispatch(worker, state) + } } - } - memoryUsage() { - for (let worker of this.freeWorkers) { - worker[kTaskInfo] = new WorkerPoolTaskInfo(() => {}) - worker.postMessage({name:'memoryUsage'}) + memoryUsage() { + return this.run('memoryUsage', {}) } - } - close() { - for (const worker of this.workers) worker.terminate(); - } -} \ No newline at end of file + async close() { + this.closed = true + const error = new Error('Query pool closed') + for (const query of this.waiting.splice(0)) { + query.done(error) + } + await Promise.all([...this.workers].map(async ([worker, state]) => { + await worker.terminate() + if (state.query) { + state.query.done(error) + state.query = null + } + })) + await Promise.all(this.terminating) + this.workers.clear() + } +} diff --git a/test/admin-recovery.mjs b/test/admin-recovery.mjs index fa9ec53..c94fa94 100644 --- a/test/admin-recovery.mjs +++ b/test/admin-recovery.mjs @@ -1,3 +1,4 @@ +import { FileDataset } from '../src/file-data.mjs' import { Buffer } from 'node:buffer' import test from 'node:test' import assert from 'node:assert/strict' @@ -5,6 +6,10 @@ import fs from 'node:fs/promises' import path from 'node:path' import os from 'node:os' import JSONTag from '@muze-nl/jsontag' +import Parser from '@muze-nl/od-jsontag/src/parse.mjs' +import serialize from '@muze-nl/od-jsontag/src/serialize.mjs' +import { appendIntegrityRecord, getDefaultIntegrityFile } + from '../src/integrity.mjs' import { makeServerFixture } from './durability-helpers.mjs' import { planRecovery, @@ -43,6 +48,22 @@ async function setup(t, entries) { } const statusLines = statuses.map(status => JSONTag.stringify(status)) await fs.writeFile(store.commandStatus, statusLines.join('\n') + '\n') + // A lost committed output still has its original digest in the manifest. + const previous = [await fs.readFile(store.datafile)] + for (const [id, status] of entries) { + if (status === 'done') { + const parser = new Parser(undefined, false) + let data + for (const bytes of previous) { + data = parser.parse(bytes) + } + data.persons.push({name: id}) + const bytes = serialize(data, {meta: parser.meta, changes: true}) + await appendIntegrityRecord(getDefaultIntegrityFile(store.datafile), + path.join(store.dir, `data.${id}.jsontag`), bytes) + previous.push(bytes) + } + } return { store, outputs } } async function runRecovery(store, outputs, plan) { @@ -77,7 +98,7 @@ test('administrator recovers trailing missing/waiting commands in log order on a 'Inspection does not expose JAQT proxies' ) assert.deepEqual( - actual.data.persons.map(p => p.name), + personNames(actual), ['A', 'B'] ) assert.deepEqual(await fs.readFile(store.commandStatus), before) @@ -164,7 +185,7 @@ test('backup and restore validate saved bytes without executing commands', async sourceQuiescent: true }) assert.deepEqual( - (await inspectStore(restored.config)).data.persons.map(p => p.name), + personNames(await inspectStore(restored.config)), ['A'] ) assert.deepEqual(restored.missingFromBackup, []) @@ -390,7 +411,7 @@ test('approved recovery resumes from completed candidate prefix without repeatin await fs.mkdir(retry) const result = await runRecovery(partial, retry, plan) assert.deepEqual( - (await inspectStore(result.config)).data.persons.map(p => p.name), + personNames(await inspectStore(result.config)), ['A', 'B'] ) assert.equal(await fs.readFile(witness, 'utf8'), 'A\nB\n') @@ -440,6 +461,7 @@ test('multi-directory integrity paths and file permissions survive backup and re store.commandStatus = path.join(logdir, 'status.jsontag') await fs.writeFile(store.commandLog, '') await fs.writeFile(store.commandStatus, '') + await fs.unlink(getDefaultIntegrityFile(store.datafile)) store.integrityFile = path.join(logdir, 'integrity.jsontag') const { appendIntegrityRecord } = await import('../src/integrity.mjs') await appendIntegrityRecord( @@ -547,7 +569,7 @@ test('verified A prefix permits only missing B, waiting C and missing D in log o await fs.mkdir(next) const result = await runRecovery(source, next, plan) assert.deepEqual( - (await inspectStore(result.config)).data.persons.map(p => p.name), + personNames(await inspectStore(result.config)), ['A', 'B', 'C', 'D'] ) assert.deepEqual( @@ -708,3 +730,44 @@ test('backup coverage distinguishes changed history and data from missing source ) } }) + +function personNames(inspection) { + const dataset = new FileDataset() + try { + return dataset.open(inspection.sources).persons.map(p => p.name) + } + finally { + dataset.close() + } +} + +test('recovery fingerprints final sidecars and verifies the copied prefix', async t => { + const { store, outputs } = await setup(t, [['A', 'accepted']]) + const { appendIntegrityRecord, loadIntegrityManifest, verifyIntegrity } = + await import('../src/integrity.mjs') + const { appendIndexIntegrity } = await import('../src/index-files.mjs') + const { default: offsetIndex } = await import('../src/index.offset.mjs') + const { default: idIndex, prepareIdIndex } = + await import('../src/index.id.mjs') + const bytes = await fs.readFile(store.datafile) + const meta = {data: store.dir} + await offsetIndex.writeSerialized(bytes, meta) + idIndex.write(meta, prepareIdIndex(bytes).entries) + store.integrityFile = path.join(store.dir, 'data.integrity.jsontag') + await appendIntegrityRecord(store.integrityFile, store.datafile, bytes) + await appendIndexIntegrity(store.integrityFile, meta) + const plan = await planRecovery(store, {quiescent: true}) + assert.equal(plan.actionable, true) + const result = await runRecovery(store, outputs, plan) + const report = await inspectStore(result.config) + assert.equal(report.ready, true) + const manifest = await loadIntegrityManifest(result.config.integrityFile) + for (const suffix of ['', '.A']) { + for (const kind of ['id', 'offset']) { + const file = path.join(path.dirname(result.config.datafile), + `index.${kind}${suffix}.json`) + assert.equal(verifyIntegrity(manifest, result.config.integrityFile, + file, await fs.readFile(file), {required: true}), true) + } + } +}) diff --git a/test/command-offsets.mjs b/test/command-offsets.mjs index e24b1f3..bb996df 100644 --- a/test/command-offsets.mjs +++ b/test/command-offsets.mjs @@ -7,7 +7,9 @@ import {fileURLToPath} from 'node:url' import JSONTag from '@muze-nl/jsontag' import Parser from '@muze-nl/od-jsontag' import serialize from '@muze-nl/od-jsontag/src/serialize.mjs' -import runCommand, {initialize} from '../src/command-worker-module.mjs' +import runCommand, {initialize, close} from '../src/command-worker-module.mjs' + +import { scanDataFile } from '../src/file-data.mjs' const baseIndex = new URL('../src/index.mjs', import.meta.url) @@ -26,8 +28,11 @@ async function fixture(t, indexSource) { await fs.writeFile(indexFile, indexSource) } const base = serialize(JSONTag.parse('{"items":[{"name":"first"},{"name":"untouched"}]}')) + const datafile = path.join(dir, 'data.jsontag') + await fs.writeFile(datafile, base) + t.after(close) const task = { - data: [base], datafile: path.join(dir, 'data.jsontag'), commandsFile, indexFile, + sources: [scanDataFile(datafile)], datafile: path.join(dir, 'data.jsontag'), commandsFile, indexFile, meta: {data: dir, parts: 0, index: {id: new Map()}, resultArray: []} } await initialize(task) @@ -107,7 +112,7 @@ test('successive command indexes are relative to their own changeset files', asy const {dir, base, task} = await fixture(t) await runCommand(JSONTag.stringify({id: 'first', name: 'edit', value: 'longer first value'})) const first = await verifyOffsets(dir, 'first') - await initialize({...task, data: [base, first.bytes], meta: {data: dir, parts: 1, index: {id: new Map()}, resultArray: []}}) + await initialize({...task, sources: [task.sources[0], scanDataFile(path.join(dir, 'data.first.jsontag'))], meta: {data: dir, parts: 1, index: {id: new Map()}, resultArray: []}}) await runCommand(JSONTag.stringify({id: 'second', name: 'edit', item: 1, value: 'x'})) const second = await verifyOffsets(dir, 'second') assert.deepEqual(Object.keys(second.offsets), ['2']) @@ -169,3 +174,17 @@ test('custom finalization rejection propagates from the command worker', async t await assert.rejects(runCommand('{"id":"rejected","name":"edit","value":"changed"}'), /custom finalization failed/) await assert.rejects(fs.access(path.join(dir, 'index.offset.rejected.json')), /ENOENT/) }) + +test('a finalizer cannot publish changed canonical bytes as the command result', async t => { + await fixture(t, `import fs from 'node:fs/promises' +export default { + update() {}, + async finalize(serialized, meta, uuid) { + await fs.appendFile(meta.data + '/data.' + uuid + '.jsontag', '\\n') + } +}`) + await assert.rejects( + runCommand('{"id":"tampered","name":"edit","value":"changed"}'), + /Changeset changed during finalization/ + ) +}) diff --git a/test/conversion-offsets.mjs b/test/conversion-offsets.mjs index 1b25ff9..47cb941 100644 --- a/test/conversion-offsets.mjs +++ b/test/conversion-offsets.mjs @@ -200,17 +200,22 @@ test('conversion propagates rejection from custom finalization', async t => { ) }) -test('conversion initializes existing-format logs and optional base integrity without overwriting a store', async t => { +test('conversion always initializes logs and integrity without overwriting a store', async t => { const files = await fixture(t) await run( process.execPath, - [converter, files.input, files.output, '--integrity'], + [converter, files.input, files.output], { timeout: 10000 } ) const { loadIntegrityManifest, verifyIntegrity } = await import( '../src/integrity.mjs' ) const integrity = path.join(files.dir, 'data.integrity.jsontag') + for (const name of ['index.offset.json', 'index.id.json']) { + const file = path.join(files.dir, name) + assert.equal(verifyIntegrity(await loadIntegrityManifest(integrity), + integrity, file, await fs.readFile(file), {required: true}), true) + } assert.equal( verifyIntegrity( await loadIntegrityManifest(integrity), @@ -236,3 +241,47 @@ test('conversion initializes existing-format logs and optional base integrity wi await assert.rejects(convert(files), /existing store is not overwritten/) assert.deepEqual(await fs.readFile(files.output), before) }) + + +test('conversion ID indexes include final hook renames and additions', async t => { + const files = await fixture(t) + const hook = path.join(files.dir, 'index.mjs') + await fs.writeFile(hook, `import base from ${JSON.stringify(baseIndex)} + import JSONTag from ${JSON.stringify(import.meta.resolve('@muze-nl/jsontag'))} + export default { + create(data, meta) { + base.create(data, meta) + JSONTag.setAttribute(data.items[0], 'id', 'renamed') + data.items.push(JSONTag.parse('{}')) + } + }`) + await convert(files, hook) + const ids = JSON.parse(await fs.readFile( + path.join(files.dir, 'index.id.json'), 'utf8')) + assert.deepEqual(ids, {renamed: 1, '/second': 2, added: 3}) +}) + +test('conversion rejects duplicate IDs before publishing canonical data', async t => { + const files = await fixture(t) + await fs.writeFile(files.input, + '{"items":[{},{}]}') + await assert.rejects(convert(files), /Duplicate ID: duplicate/) + await assert.rejects(fs.access(files.output), /ENOENT/) + await assert.rejects(fs.access(path.join(files.dir, 'index.id.json')), /ENOENT/) +}) + + +test('conversion rejects canonical mutation during custom finalization', async t => { + const files = await fixture(t) + const hook = path.join(files.dir, 'index.mjs') + await fs.writeFile(hook, `import fs from 'node:fs/promises' + export default { + create() {}, + async finalize(bytes, meta) { + await fs.appendFile(meta.data + '/data.jsontag', '\\n') + } + }`) + await assert.rejects(convert(files, hook), /Dataset changed during finalization/) + await assert.rejects(fs.access(path.join(files.dir, 'command-log.jsontag')), + /ENOENT/) +}) diff --git a/test/durability-helpers.mjs b/test/durability-helpers.mjs index 5957609..16c709f 100644 --- a/test/durability-helpers.mjs +++ b/test/durability-helpers.mjs @@ -8,6 +8,8 @@ import { fileURLToPath } from 'node:url' import JSONTag from '@muze-nl/jsontag' import Parser from '@muze-nl/od-jsontag/src/parse.mjs' import serialize from '@muze-nl/od-jsontag/src/serialize.mjs' +import { appendIntegrityRecord, getDefaultIntegrityFile } + from '../src/integrity.mjs' import { getChangesetPath } from '../src/recovery.mjs' export const rootDir = path.dirname( @@ -67,6 +69,8 @@ SimplyStore.run(options) ` ) + await appendIntegrityRecord(getDefaultIntegrityFile(datafile), + datafile, await fs.readFile(datafile)) return { dir, datafile, diff --git a/test/durability-startup.mjs b/test/durability-startup.mjs index b5ad9cc..06753b4 100644 --- a/test/durability-startup.mjs +++ b/test/durability-startup.mjs @@ -8,6 +8,7 @@ import { Worker } from 'node:worker_threads' import { fileURLToPath } from 'node:url' import JSONTag from '@muze-nl/jsontag' import Parser from '@muze-nl/od-jsontag/src/parse.mjs' +import { FileDataset } from '../src/file-data.mjs' import serialize from '@muze-nl/od-jsontag/src/serialize.mjs' import { appendIntegrityRecord, @@ -26,9 +27,10 @@ import { const rootDir = path.dirname(path.dirname(fileURLToPath(import.meta.url))) const loadWorker = path.join(rootDir, 'src/load-worker.mjs') -function parseOd(buffer) { - const parser = new Parser() - return parser.parse(buffer) +function parseOd(sources, t) { + const dataset = new FileDataset() + t.after(() => dataset.close()) + return dataset.open(sources) } async function makeFixture(t) { @@ -47,6 +49,7 @@ async function makeFixture(t) { 'export default { create() {}, update() {}, load() { return {} } }\n' ) + await writeIntegrityEntry(getDefaultIntegrityFile(dataFile), dataFile) return { dir, dataFile, indexFile } } @@ -76,6 +79,8 @@ async function writeChangeset(dataFile, commandId, change) { changes: true }) ) + await writeIntegrityEntry(getDefaultIntegrityFile(dataFile), + `${basefile}.${commandId}.${extension}`) } function loadDataset({ @@ -126,7 +131,7 @@ test('accepted command changeset is not treated as committed state on startup', ...fixture, commands: getCommittedCommandIds(status) }) - const data = parseOd(result.data) + const data = parseOd(result.sources, t) assert.equal( data.persons.length, @@ -235,7 +240,7 @@ test('malformed uncommitted changeset is ignored during committed startup recons ...fixture, commands: getCommittedCommandIds(status) }) - const data = parseOd(result.data) + const data = parseOd(result.sources, t) assert.equal(data.persons.length, 0) }) @@ -264,7 +269,7 @@ test('durable status file selects only done command changesets for startup', asy ...fixture, commands: getCommittedCommandIds(status) }) - const data = parseOd(result.data) + const data = parseOd(result.sources, t) assert.deepEqual( data.persons.map(person => person.name), @@ -504,8 +509,9 @@ test('integrity manifest detects same-length altered committed changeset payload ) }) -test('integrity enabled requires manifest entry for base data', async t => { +test('integrity requires manifest entry for base data', async t => { const fixture = await makeFixture(t) + await fs.writeFile(getDefaultIntegrityFile(fixture.dataFile), '') await assert.rejects( loadDataset({ diff --git a/test/file-data.mjs b/test/file-data.mjs new file mode 100644 index 0000000..02d2b56 --- /dev/null +++ b/test/file-data.mjs @@ -0,0 +1,640 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import test from 'node:test' +import { createHash } from 'node:crypto' +import { fileURLToPath } from 'node:url' +import JSONTag from '@muze-nl/jsontag' +import serialize from '@muze-nl/od-jsontag/src/serialize.mjs' +import { FileDataset, FileParser, scanDataFile, hashFile, loadFileData } from '../src/file-data.mjs' +import { appendIndexIntegrity } from '../src/index-files.mjs' +import { inspectStore } from '../src/store-inspection.mjs' +import { getDefaultIntegrityFile, serializeIntegrityRecords } + from '../src/integrity.mjs' +import { appendIntegrityRecord } from '../src/integrity.mjs' +import index from '../src/index.mjs' +import offsetIndex from '../src/index.offset.mjs' +import idIndex from '../src/index.id.mjs' +import { getIndex } from '@muze-nl/od-jsontag/src/symbols.mjs' +import WorkerPool from '../src/workerPool.mjs' +import StoreRuntime from '../src/store-runtime.mjs' +import { makeServerFixture } from './durability-helpers.mjs' +import runCommand, { initialize, close } from '../src/command-worker-module.mjs' + + +function sealFixture(file, indexes = []) { + const digests = [file, ...indexes].map(file => [file, hashFile(file)]) + const base = path.join(path.dirname(file), 'data.jsontag') + fs.appendFileSync(getDefaultIntegrityFile(base), + serializeIntegrityRecords(getDefaultIntegrityFile(base), digests) + '\n') +} + +function fixture(t, text = '{"items":[{"name":"first"},{"name":"last"}]}') { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'simplystore-files-')) + t.after(() => fs.rmSync(dir, { recursive: true, force: true })) + const file = path.join(dir, 'data.jsontag') + fs.writeFileSync(file, serialize(JSONTag.parse(text))) + const source = scanDataFile(file) + sealFixture(file) + return { dir, file, source } +} + +function open(t, sources, meta, immutable = true) { + const dataset = new FileDataset(meta, immutable) + t.after(() => dataset.close()) + dataset.open(sources) + return dataset +} + +test('file loading resolves unread IDs and sends metadata without dataset bytes', async t => { + const { file } = fixture(t) + // Missing indexes reconstruct without retaining dataset bytes. + const loaded = await loadFileData({ dataFile: file, commands: [] }) + assert.equal(loaded.data, undefined) + assert.equal(loaded.meta.index.id.get('last'), 2) + assert.deepEqual(structuredClone(loaded), loaded) + const dataset = open(t, loaded.sources, loaded.meta) + assert.equal(dataset.parser.meta.resultArray[2], undefined) + assert.equal(dataset.parser.getLineProxy(2).name, 'last') + assert.equal(dataset.root.items[1], dataset.parser.getLineProxy(2)) +}) + +test('sparse file overlays retain unread records and allocate beyond the full base', t => { + const { dir, source } = fixture(t) + const patch = path.join(dir, 'data.patch.jsontag') + const payload = '{"name":"changed"}' + fs.writeFileSync(patch, `+1\n(${Buffer.byteLength(payload)})${payload}`) + const dataset = open(t, [source, scanDataFile(patch)], {}, false) + dataset.root.items.push({ name: 'new' }) + assert.deepEqual(dataset.root.items.map(item => item.name), [ + 'changed', 'last', 'new' + ]) + const output = path.join(dir, 'output.jsontag') + fs.writeFileSync(output, serialize(dataset.root)) + const copy = open(t, [scanDataFile(output)]) + assert.deepEqual(copy.root.items.map(item => item.name), [ + 'changed', 'last', 'new' + ]) +}) + +test('framing scan handles short reads, UTF-8 and payloads beyond its byte buffer', t => { + const { file } = fixture(t, JSON.stringify({ text: '漢😀'.repeat(20000) })) + const read = fs.readSync + const mock = t.mock.method(fs, 'readSync', (fd, buffer, offset, length, position) => { + return read(fd, buffer, offset, Math.min(length, 997), position) + }) + const source = scanDataFile(file) + assert.ok(mock.mock.callCount() > 100) + assert.equal(source.digest, hashFile(file)) + const bytes = fs.readFileSync(file) + assert.equal(source.digest, createHash('sha256').update(bytes).digest('hex')) + const header = /^\((\d+)\)/.exec(bytes.toString('utf8', 0, 32)) + assert.deepEqual(source.offsets, { + 0: [header[0].length, header[0].length + Number(header[1])] + }) +}) + +for (const content of ['(9){}', '(x){}', '+x\n', '+1x\n', '(2){}x', + '+4294967295\n', '(9007199254740999){}']) { + test(`file framing rejects ${JSON.stringify(content)}`, t => { + const { file } = fixture(t) + fs.writeFileSync(file, content) + assert.throws(() => scanDataFile(file), /Invalid OD-JSONTag/) + }) +} + +test('sessions close all handles on failure and reject changed source identities', t => { + const { source, file } = fixture(t) + const dataset = new FileDataset() + const handles = [] + const openSync = fs.openSync + t.mock.method(fs, 'openSync', (...args) => { + const fd = openSync(...args) + handles.push(fd) + return fd + }) + assert.throws(() => dataset.open([ + source, { ...source, file: file + '.missing' } + ]), /ENOENT/) + for (const fd of handles) { + assert.throws(() => fs.fstatSync(fd), /EBADF/) + } + fs.appendFileSync(file, '\n') + assert.throws(() => new FileDataset().open([source]), /source changed/) +}) + +test('opening and partial reads never read the entire file through readFile', async t => { + const { file } = fixture(t) + const read = fs.readFileSync + t.mock.method(fs, 'readFileSync', (name, ...args) => { + assert.notEqual(name, file, 'whole dataset read is forbidden') + return read(name, ...args) + }) + const loaded = await loadFileData({ dataFile: file, commands: [] }) + const dataset = open(t, loaded.sources, loaded.meta) + assert.equal(dataset.root.items[1].name, 'last') +}) + +test('file loading checks full-file integrity before returning sources', async t => { + const { dir, file } = fixture(t) + const integrityFile = path.join(dir, 'integrity.jsontag') + await appendIntegrityRecord(integrityFile, file, fs.readFileSync(file)) + const options = { + dataFile: file, commands: [], integrityFile, integrityRequired: true + } + await loadFileData(options) + const bytes = fs.readFileSync(file) + fs.writeFileSync(file, bytes.toString().replace('last', 'lost')) + await assert.rejects(loadFileData(options), /Integrity mismatch/) +}) + +test('persisted index loaders use the requested base and command names', t => { + const { dir } = fixture(t) + for (const suffix of ['', '.command']) { + fs.writeFileSync(path.join(dir, `index.id${suffix}.json`), '{"id":2}') + fs.writeFileSync(path.join(dir, `index.offset${suffix}.json`), '{"2":[1,4]}') + } + for (const command of [null, 'command']) { + assert.deepEqual(index.load({ data: dir }, command), { + id: { id: 2 }, offset: { 2: [1, 4] } + }) + } +}) + +test('command ID lookups resolve unread objects and output only the changed file', async t => { + const { dir, file } = fixture(t) + const loaded = await loadFileData({ dataFile: file, commands: [] }) + const commandsFile = path.join(dir, 'commands.mjs') + fs.writeFileSync(commandsFile, `export default { + edit(data, command, unused, meta) { + meta.index.id.get(command.target).deref().name = 'edited by ID' + } + }`) + await initialize({ + ...loaded, datafile: file, commandsFile, + indexFile: fileURLToPath(new URL('../src/index.mjs', import.meta.url)) + }) + t.after(close) + const result = await runCommand('{"id":"edit","name":"edit","target":"last"}') + assert.equal(result.data, undefined) + assert.deepEqual(Object.keys(result.source.offsets), ['2']) + const dataset = open(t, [...loaded.sources, result.source], result.meta) + assert.equal(dataset.root.items[1].name, 'edited by ID') + assert.equal(dataset.root.items[0].name, 'first') +}) + +async function poolFixture(t) { + const { dir } = fixture(t) + const workerFile = path.join(dir, 'worker.mjs') + fs.writeFileSync(workerFile, `import {parentPort} from 'node:worker_threads' + let version = 0 + parentPort.on('message', async task => { + if (task.req?.crash) { throw new Error('query crashed') } + if (task.name === 'init' || task.name === 'update') { + await new Promise(resolve => setTimeout(resolve, 25)) + version = task.req.meta.version + } + parentPort.postMessage(version) + })`) + const pool = new WorkerPool(2, workerFile, { + name: 'init', req: { sources: [], meta: { version: 0 } } + }) + t.after(() => pool.close()) + return pool +} + +test('updates finish before queries even during initialization and replacement', async t => { + const pool = await poolFixture(t) + pool.update({ name: 'update', req: { source: { file: 'a' }, meta: { version: 1 } } }) + pool.update({ name: 'update', req: { source: { file: 'b' }, meta: { version: 2 } } }) + const results = await Promise.all(Array.from({ length: 8 }, () => { + return pool.run('query', {}) + })) + assert.deepEqual(results, Array(8).fill(2)) + await assert.rejects(pool.run('query', { crash: true }), /query crashed/) + const next = await Promise.all(Array.from({ length: 8 }, () => { + return pool.run('query', {}) + })) + assert.deepEqual(next, Array(8).fill(2)) + assert.equal(pool.initTask.req.sources.length, 2) +}) + +test('closing a pool awaits termination and rejects outstanding queries', async t => { + const pool = await poolFixture(t) + const pending = pool.run('query', {}) + const rejected = assert.rejects(pending, /pool closed/) + await pool.close() + await rejected + assert.equal(pool.workers.size, 0) + await assert.rejects(pool.run('query', {}), /pool closed/) +}) + +test('live runtime retains only files through commit, query and reopen', async t => { + let runtime + let reopened + t.after(async () => { + await reopened?.close() + await runtime?.close() + }) + const fixture = await makeServerFixture(t) + const options = { ...fixture, maxWorkers: 2 } + runtime = await StoreRuntime.open(options) + assert.equal(runtime.data, undefined) + assert.equal(runtime.sources.length, 1) + const accepted = await runtime.acceptCommand( + '{"id":"A","name":"addPerson","value":{"name":"A"}}' + ) + assert.equal(accepted.code, 202) + await runtime.runQueuedCommands() + assert.equal(runtime.getCommandStatus('A').value.status, 'done') + assert.equal(runtime.sources.length, 2) + assert.ok(runtime.sources.every(source => typeof source.file === 'string')) + const response = await runtime.runQuery({ + path: '/', body: 'data.persons.map(p => p.name)', jsontag: false + }) + assert.deepEqual(JSON.parse(response.body), ['A']) + const selected = await runtime.runQuery({ + path: '/', + body: 'from(data.persons).where({name: "A"}).select({name: _})', + jsontag: false + }) + assert.deepEqual(JSON.parse(selected.body), [{name: 'A'}]) + const mutation = await runtime.runQuery({ + path: '/', body: 'data.persons.push({name: "forbidden"})', + jsontag: false + }) + assert.equal(mutation.code, 422) + const definition = await runtime.runQuery({ + path: '/', body: 'Object.defineProperty(data.persons, "0", {value: null})', + jsontag: false + }) + assert.equal(definition.code, 422) + await runtime.close() + reopened = await StoreRuntime.open(options) + assert.equal(reopened.sources.length, 2) + const after = await reopened.runQuery({ + path: '/', body: 'data.persons.map(p => p.name)', jsontag: false + }) + assert.deepEqual(JSON.parse(after.body), ['A']) +}) + +test('a failed worker update refuses further queries instead of serving stale data', async t => { + const pool = await poolFixture(t) + const failed = new Promise(resolve => pool.once('error', resolve)) + pool.update({ name: 'update', req: { + source: {file: 'bad'}, meta: {version: 1}, crash: true + } }) + const query = assert.rejects(pool.run('query', {}), /query crashed/) + await failed + await query + await assert.rejects(pool.run('query', {}), /query crashed/) +}) + +test('premature EOF cannot turn an existing data file into an empty source', t => { + const { file } = fixture(t) + t.mock.method(fs, 'readSync', () => 0) + assert.throws(() => scanDataFile(file), /unexpected EOF/) + assert.throws(() => hashFile(file), /Unexpected EOF/) +}) + +test('duplicate IDs are rejected when a patch fills an earlier hole', async t => { + const { file, dir } = fixture(t) + const record = text => `(${Buffer.byteLength(text)})${text}` + fs.writeFileSync(file, + record('{"items":[~2,~5]}') + '\n+4\n' + + record('{"name":"higher"}') + ) + fs.writeFileSync(path.join(dir, 'data.fill.jsontag'), + '+2\n' + record('{"name":"lower"}') + ) + sealFixture(file) + sealFixture(path.join(dir, 'data.fill.jsontag')) + await assert.rejects(loadFileData({dataFile: file, commands: ['fill']}), + /Duplicate ID: duplicate/) +}) + +test('a lazy read failure during a command is a storage failure', async t => { + const { dir, file } = fixture(t) + const loaded = await loadFileData({dataFile: file, commands: []}) + const commandsFile = path.join(dir, 'commands.mjs') + fs.writeFileSync(commandsFile, `export default { + read(data) { + try { data.items[1].name } catch (error) {} + data.changedAfterFailure = true + } + }`) + await initialize({ + ...loaded, datafile: file, commandsFile, + indexFile: fileURLToPath(new URL('../src/index.mjs', import.meta.url)) + }) + t.after(close) + fs.truncateSync(file, 0) + await assert.rejects(runCommand('{"id":"read","name":"read"}'), error => { + return error.storageFailure === true + }) + assert.equal(fs.existsSync(path.join(dir, 'data.read.jsontag')), false) +}) + +test('lazy query read failures stop mutation, while query-thrown flags do not', async t => { + let runtime + t.after(async () => runtime?.close()) + const fixture = await makeServerFixture(t, { + initialData: '{"persons":[{"name":"unread"}]}' + }) + runtime = await StoreRuntime.open({...fixture, maxWorkers: 1}) + const request = body => ({path: '/', body, jsontag: false}) + await runtime.runQuery(request('data.persons.length')) + // Finish both initializations before altering a disposable source file. + await runtime.runQuery(request('data.persons.length'), {slow: true}) + const spoofed = await runtime.runQuery(request( + 'throw {storageFailure: true}' + )) + assert.equal(spoofed.code, 422) + assert.equal(runtime.storageFailed, false) + fs.truncateSync(fixture.datafile, 0) + const result = await runtime.runQuery(request( + 'try { data.persons[0].name } catch (error) {} "caught"' + )) + assert.equal(result.code, 500) + assert.equal(runtime.storageFailed, true) + const tagged = await runtime.runQuery({ + ...request('42'), jsontag: true + }) + assert.equal(tagged.jsontag, true) + assert.equal(JSONTag.parse(tagged.body).code, 500) + const accepted = await runtime.acceptCommand( + '{"id":"after-failure","name":"addPerson","value":{"name":"bad"}}' + ) + assert.equal(accepted.code, 503) +}) + +function writeIndexes(dir, offsets, ids, command = '') { + const suffix = command ? `.${command}` : '' + fs.writeFileSync(path.join(dir, `index.offset${suffix}.json`), + JSON.stringify(offsets)) + fs.writeFileSync(path.join(dir, `index.id${suffix}.json`), + JSON.stringify(ids)) + const file = path.join(dir, command ? `data.${command}.jsontag` : + 'data.jsontag') + if (fs.existsSync(file)) { + sealFixture(file, ['offset', 'id'].map(kind => + path.join(dir, `index.${kind}${suffix}.json`))) + } +} + +test('startup consumes valid sidecars without parsing record tags or bodies', async t => { + const { file, dir, source } = fixture(t) + writeIndexes(dir, source.offsets, {first: 1, last: 2}) + const before = fs.readdirSync(dir).map(name => [ + name, fs.readFileSync(path.join(dir, name)) + ]) + const loadedOffsets = [] + const loadOffsets = offsetIndex.load + t.mock.method(offsetIndex, 'load', (...args) => { + const offsets = loadOffsets(...args) + loadedOffsets.push(offsets) + return offsets + }) + const loadIds = t.mock.method(idIndex, 'load') + const parsed = new Set() + const firstParse = FileParser.prototype.firstParse + t.mock.method(FileParser.prototype, 'firstParse', function (target) { + parsed.add(target[getIndex]) + return firstParse.call(this, target) + }) + t.mock.method(FileDataset.prototype, 'rebuildIds', () => { + throw new Error('full-record ID reconstruction is forbidden') + }) + t.mock.method(JSONTag.Parser.prototype, 'tag', () => { + throw new Error('record tag parsing is forbidden') + }) + const loaded = await loadFileData({dataFile: file, commands: []}) + assert.equal(loaded.sources[0].offsets, loadedOffsets[0]) + assert.equal(loadIds.mock.callCount(), 1) + assert.deepEqual([...loaded.meta.index.id], [['first', 1], ['last', 2]]) + assert.deepEqual([...parsed], [0]) + assert.deepEqual(fs.readdirSync(dir).map(name => [ + name, fs.readFileSync(path.join(dir, name)) + ]), before) +}) + +test('startup loads only committed command sidecars in authoritative order', async t => { + const { file, dir, source } = fixture(t) + writeIndexes(dir, source.offsets, {first: 1, last: 2}) + const payload = '{"name":"committed"}' + const patch = path.join(dir, 'data.edit.jsontag') + fs.writeFileSync(patch, `+2\n(${Buffer.byteLength(payload)})${payload}`) + writeIndexes(dir, scanDataFile(patch).offsets, {last: 2}, 'edit') + writeIndexes(dir, {'2': [1, 99]}, {uncommitted: 2}, 'uncommitted') + const loadIds = t.mock.method(idIndex, 'load') + const loadOffsets = t.mock.method(offsetIndex, 'load') + const loaded = await loadFileData({dataFile: file, commands: ['edit']}) + for (const mock of [loadIds, loadOffsets]) { + assert.deepEqual(mock.mock.calls.map(call => call.arguments[1]), + [null, 'edit']) + } + assert.deepEqual([...loaded.meta.index.id], [['first', 1], ['last', 2]]) + const dataset = open(t, loaded.sources, loaded.meta) + assert.equal(dataset.parser.getLineProxy(2).name, 'committed') +}) + +for (const invalid of ['{', 'null', '[]', '{"last":999}', + '{"first":1,"last":2,"extra":2}']) { + test(`malformed ID indexes fail; explicit rebuild recovers: ${invalid}`, async t => { + const { file, dir, source } = fixture(t) + writeIndexes(dir, source.offsets, {}) + fs.writeFileSync(path.join(dir, 'index.id.json'), invalid) + await assert.rejects(loadFileData({dataFile: file, commands: []})) + const loaded = await loadFileData({ + dataFile: file, commands: [], rebuildIndexes: true + }) + assert.deepEqual([...loaded.meta.index.id], [['first', 1], ['last', 2]]) + assert.equal(fs.readFileSync(path.join(dir, 'index.id.json'), 'utf8'), + invalid) + }) +} + +test('normal loading trusts index contents; explicit validation detects mismatch', async t => { + const { file, dir, source } = fixture(t) + for (const stored of [{}, {last: 1}]) { + writeIndexes(dir, source.offsets, stored) + const options = {dataFile: file, commands: []} + const loaded = await loadFileData(options) + assert.deepEqual(loaded.meta.index.id, new Map(Object.entries(stored))) + await assert.rejects(loadFileData({...options, validateIndexes: true}), + /ID index does not match data/) + const rebuilt = await loadFileData({...options, rebuildIndexes: true}) + assert.deepEqual([...rebuilt.meta.index.id], [['first', 1], ['last', 2]]) + } +}) + +test('malformed offset indexes fail; explicit rebuild leaves disk unchanged', async t => { + const { file, dir, source } = fixture(t) + const invalid = ['{', 'null', '[]', '{"-1":[3,8]}', + '{"01":[3,8]}', '{"4294967295":[3,8]}', + '{"0":[3,999999]}', '{"0":[3,3]}', '{"0":[3.5,8]}', + '{"0":["3",8]}', '{"0":[3,8,9]}', + JSON.stringify({...source.offsets, '1': source.offsets[2]})] + for (const offsets of invalid) { + writeIndexes(dir, source.offsets, {first: 1, last: 2}) + fs.writeFileSync(path.join(dir, 'index.offset.json'), offsets) + await assert.rejects(loadFileData({dataFile: file, commands: []})) + const loaded = await loadFileData({ + dataFile: file, commands: [], rebuildIndexes: true + }) + assert.deepEqual(loaded.sources[0].offsets, source.offsets) + assert.equal(fs.readFileSync(path.join(dir, 'index.offset.json'), + 'utf8'), offsets) + } +}) + +test('offset contents are trusted unless validation is requested', async t => { + const { file, dir, source } = fixture(t) + // Opening only needs the root; completeness is a producer promise. + const offsets = {'0': source.offsets[0]} + writeIndexes(dir, offsets, {}) + const options = {dataFile: file, commands: []} + const loaded = await loadFileData(options) + assert.deepEqual(loaded.sources[0].offsets, offsets) + await assert.rejects(loadFileData({...options, validateIndexes: true}), + /Offset index does not match data/) + const rebuilt = await loadFileData({...options, rebuildIndexes: true}) + assert.deepEqual(rebuilt.sources[0].offsets, source.offsets) +}) + +for (const missingIds of [false, true]) { + test(`loader and inspection bypass framing reconstruction (missing IDs: ${missingIds})`, async t => { + const { file, dir, source } = fixture(t) + writeIndexes(dir, source.offsets, {first: 1, last: 2}) + if (missingIds) { + fs.unlinkSync(path.join(dir, 'index.id.json')) + } + // Corrupt only an inter-record separator, outside every payload/prefix. + // A framing reconstruction must fail; indexed record reads still work. + const bytes = fs.readFileSync(file) + assert.equal(bytes[source.offsets[0][1]], 10) + bytes[source.offsets[0][1]] = 120 + fs.writeFileSync(file, bytes) + sealFixture(file) + assert.throws(() => scanDataFile(file), /expected record length/) + const loaded = await loadFileData({dataFile: file, commands: []}) + assert.deepEqual(loaded.sources[0].offsets, source.offsets) + const dataset = open(t, loaded.sources, loaded.meta) + assert.equal(dataset.parser.getLineProxy(2).name, 'last') + const config = {datafile: file, + commandLog: path.join(dir, 'commands.jsontag'), + commandStatus: path.join(dir, 'status.jsontag')} + fs.writeFileSync(config.commandLog, '') + fs.writeFileSync(config.commandStatus, '') + assert.equal((await inspectStore(config)).ready, true) + assert.equal((await inspectStore({...config, + validateIndexes: true})).ready, false) + }) +} + +for (const kind of ['offset', 'id']) { + test(`hash checks detect valid-JSON ${kind} index changes before use`, async t => { + const { file, dir, source } = fixture(t) + writeIndexes(dir, source.offsets, {first: 1, last: 2}) + const integrityFile = path.join(dir, 'integrity.jsontag') + await appendIntegrityRecord(integrityFile, file, fs.readFileSync(file)) + await appendIndexIntegrity(integrityFile, {data: dir}) + const options = {dataFile: file, commands: [], integrityFile, + integrityRequired: true} + await loadFileData(options) + const indexFile = path.join(dir, `index.${kind}.json`) + fs.writeFileSync(indexFile, '{}') + await assert.rejects(loadFileData(options), /Integrity mismatch/) + const config = {datafile: file, integrityFile, + commandLog: path.join(dir, 'commands.jsontag'), + commandStatus: path.join(dir, 'status.jsontag')} + fs.writeFileSync(config.commandLog, '') + fs.writeFileSync(config.commandStatus, '') + const inspected = await inspectStore(config) + assert.equal(inspected.ready, false) + assert.ok(inspected.errors.some(error => /Integrity mismatch/.test(error))) + // Explicit rebuild ignores sidecars, but still verifies canonical data. + const rebuilt = await loadFileData({...options, rebuildIndexes: true}) + assert.deepEqual(rebuilt.sources[0].offsets, source.offsets) + fs.unlinkSync(indexFile) + await loadFileData(options) + }) +} + +test('integrity mode requires hashes for present indexes', async t => { + const { file, dir, source } = fixture(t) + writeIndexes(dir, source.offsets, {first: 1, last: 2}) + const integrityFile = path.join(dir, 'integrity.jsontag') + await appendIntegrityRecord(integrityFile, file, fs.readFileSync(file)) + await assert.rejects(loadFileData({dataFile: file, commands: [], + integrityFile, integrityRequired: true}), /Missing integrity manifest entry/) +}) + +test('ID overlays remove old IDs after renames and attribute removal', async t => { + const { file, dir } = fixture(t) + const record = text => `(${Buffer.byteLength(text)})${text}\n` + fs.writeFileSync(file, record('{"items":[~1,~2,~3]}') + + record('{"name":"lower"}') + + record('{"name":"higher"}') + + record('{}')) + writeIndexes(dir, scanDataFile(file).offsets, {kept: 1, old: 2, removed: 3}) + const patch = path.join(dir, 'data.rename.jsontag') + fs.writeFileSync(patch, '+2\n' + record('{}') + + record('{}')) + writeIndexes(dir, scanDataFile(patch).offsets, {renamed: 2}, 'rename') + const loaded = await loadFileData({dataFile: file, commands: ['rename']}) + assert.deepEqual([...loaded.meta.index.id], [['kept', 1], ['renamed', 2]]) + const dataset = open(t, loaded.sources, loaded.meta) + assert.equal(dataset.parser.getLineProxy(1).name, 'lower') +}) + +test('ID header validation handles escaped delimiters and buffer boundaries', async t => { + const { file, dir } = fixture(t) + const ids = ['漢😀>\\tail', '__proto__', 'constructor', 'x'.repeat(70000)] + const record = text => `(${Buffer.byteLength(text)})${text}\n` + fs.writeFileSync(file, record('{"items":[~1-4]}') + ids.map(id => { + return record(` {"text":"ignored"}`) + }).join('')) + const expected = Object.fromEntries(ids.map((id, i) => [id, i + 1])) + writeIndexes(dir, scanDataFile(file).offsets, expected) + const read = fs.readSync + t.mock.method(fs, 'readSync', (fd, buffer, offset, length, position) => { + return read(fd, buffer, offset, Math.min(31, length), position) + }) + const loaded = await loadFileData({ + dataFile: file, commands: [], validateIndexes: true + }) + assert.deepEqual(loaded.meta.index.id, new Map(Object.entries(expected))) + const dataset = open(t, loaded.sources, loaded.meta) + assert.deepEqual(dataset.rebuildIds(), loaded.meta.index.id) +}) + +test('explicit validation rejects invalid tags despite valid-looking indexes', async t => { + const { file, dir } = fixture(t) + for (const payload of ['{}']) { + fs.writeFileSync(file, `(${Buffer.byteLength(payload)})${payload}`) + writeIndexes(dir, scanDataFile(file).offsets, {bad: 0}) + await assert.rejects(loadFileData({ + dataFile: file, commands: [], validateIndexes: true + }), + /Invalid OD-JSONTag data/) + } +}) + +test('body syntax stays lazy while malformed reads still mark storage failure', async t => { + const { file, dir } = fixture(t) + const record = text => `(${Buffer.byteLength(text)})${text}\n` + fs.writeFileSync(file, record('{"items":[~1]}') + + record('{"name":invalid}')) + writeIndexes(dir, scanDataFile(file).offsets, {unread: 1}) + const loaded = await loadFileData({dataFile: file, commands: []}) + assert.equal(loaded.meta.index.id.get('unread'), 1) + const dataset = open(t, loaded.sources, loaded.meta) + assert.equal(dataset.parser.readFailure, undefined) + assert.throws(() => dataset.parser.getLineProxy(1).name, error => { + return error.storageFailure === true + }) + assert.equal(dataset.parser.readFailure.storageFailure, true) +}) diff --git a/test/index-ids.mjs b/test/index-ids.mjs new file mode 100644 index 0000000..bb0d79d --- /dev/null +++ b/test/index-ids.mjs @@ -0,0 +1,286 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import path from 'node:path' +import test from 'node:test' +import { fileURLToPath } from 'node:url' +import { FileParser, loadFileData } from '../src/file-data.mjs' +import { getIndex } from '@muze-nl/od-jsontag/src/symbols.mjs' +import idIndex, { prepareIdIndex } from '../src/index.id.mjs' +import offsetIndex from '../src/index.offset.mjs' +import { appendIndexIntegrity } from '../src/index-files.mjs' +import { getDefaultIntegrityFile } from '../src/integrity.mjs' +import runCommand, { initialize, close } from '../src/command-worker-module.mjs' +import StoreRuntime from '../src/store-runtime.mjs' +import { makeServerFixture } from './durability-helpers.mjs' + +const tagModule = import.meta.resolve('@muze-nl/jsontag') +const defaultIndex = new URL('../src/index.mjs', import.meta.url) +const commandsSource = `import JSONTag from ${JSON.stringify(tagModule)} +export default { + rename(data, command) { + JSONTag.setAttribute(data.items[command.item || 0], 'id', command.value) + }, + swap(data) { + JSONTag.setAttribute(data.items[0], 'id', 'last') + JSONTag.setAttribute(data.items[1], 'id', 'first') + }, + duplicateBoth(data) { + JSONTag.setAttribute(data.items[0], 'id', 'duplicate') + JSONTag.setAttribute(data.items[1], 'id', 'duplicate') + }, + edit(data) { data.items[0].name = 'edited' }, + noop() {} +}` + +async function fixture(t, options = {}) { + const files = await makeServerFixture(t, { + initialData: '{"items":[{"name":"first"},' + + '{"name":"last"}]}', + commandsSource, ...options + }) + if (!options.indexSource) { + files.indexFile = fileURLToPath(defaultIndex) + } + const bytes = fs.readFileSync(files.datafile) + idIndex.write({data: files.dir}, prepareIdIndex(bytes).entries) + await offsetIndex.writeSerialized(bytes, {data: files.dir}) + await appendIndexIntegrity(getDefaultIntegrityFile(files.datafile), + {data: files.dir}) + return files +} + +async function command(files, previous, task) { + const loaded = await loadFileData({ + dataFile: files.datafile, commands: previous + }) + await initialize({...loaded, ...files}) + try { + return await runCommand(JSON.stringify(task)) + } + finally { + close() + } +} + +function storedIds(files, id) { + return JSON.parse(fs.readFileSync( + path.join(files.dir, `index.id.${id}.json`), 'utf8')) +} + +test('same-record IDs, renames, removal and swaps agree after restart', async t => { + const files = await fixture(t) + const previous = [] + const steps = [ + [{id: 'edit', name: 'edit'}, {first: 1}, {first: 1, last: 2}], + [{id: 'swap', name: 'swap'}, {last: 1, first: 2}, {last: 1, first: 2}], + [{id: 'rename', name: 'rename', value: 'renamed'}, + {renamed: 1}, {first: 2, renamed: 1}], + [{id: 'remove', name: 'rename', value: ''}, {}, {first: 2}] + ] + for (const [task, part, expected] of steps) { + const result = await command(files, previous, task) + previous.push(task.id) + assert.deepEqual(storedIds(files, task.id), part) + const reopened = await loadFileData({ + dataFile: files.datafile, commands: previous, validateIndexes: true + }) + assert.deepEqual(result.meta.index.id, + new Map(Object.entries(expected))) + assert.deepEqual(reopened.meta.index.id, result.meta.index.id) + } +}) + +test('IDs reflect mutations after the default update hook', async t => { + const files = await fixture(t, { + indexSource: `import base from ${JSON.stringify(defaultIndex.href)} + import JSONTag from ${JSON.stringify(tagModule)} + export default { + update(data, meta, changes) { + base.update(data, meta, changes) + JSONTag.setAttribute(data.items[1], 'id', 'hook-renamed') + data.items.push(JSONTag.parse('{}')) + } + }` + }) + const result = await command(files, [], {id: 'hook', name: 'edit'}) + assert.deepEqual(storedIds(files, 'hook'), { + first: 1, 'hook-renamed': 2, added: 3 + }) + const reopened = await loadFileData({ + dataFile: files.datafile, commands: ['hook'], validateIndexes: true + }) + assert.deepEqual(result.meta.index.id, reopened.meta.index.id) +}) + +test('custom finalization cannot leave a stale canonical ID index', async t => { + const files = await fixture(t, { + indexSource: `import fs from 'node:fs/promises' + export default { + update() {}, + async finalize(bytes, meta, uuid) { + await fs.writeFile(meta.data + '/index.id.' + uuid + '.json', + '{"stale":999}') + } + }` + }) + await command(files, [], {id: 'custom', name: 'edit'}) + assert.deepEqual(storedIds(files, 'custom'), {first: 1}) +}) + +test('no-op commands finalize an empty ID index', async t => { + const files = await fixture(t) + fs.writeFileSync(path.join(files.dir, 'index.id.noop.json'), '{"stale":1}') + const result = await command(files, [], {id: 'noop', name: 'noop'}) + assert.deepEqual(storedIds(files, 'noop'), {}) + assert.deepEqual(result.meta.index.id, new Map([['first', 1], ['last', 2]])) +}) + +for (const task of [ + {id: 'duplicate', name: 'rename', item: 1, value: 'first'}, + {id: 'duplicate', name: 'duplicateBoth'} +]) { + test(`duplicate IDs reject ${task.name} before data publication`, async t => { + const files = await fixture(t) + await assert.rejects(command(files, [], task), error => { + assert.match(error.message, /Duplicate ID/) + assert.notEqual(error.storageFailure, true) + return true + }) + assert.equal(fs.existsSync(path.join(files.dir, + 'data.duplicate.jsontag')), false) + assert.equal(fs.existsSync(path.join(files.dir, + 'index.id.duplicate.json')), false) + }) +} + +test('ID-index publication failure prevents command success', async t => { + const files = await fixture(t) + fs.mkdirSync(path.join(files.dir, 'index.id.blocked.json')) + await assert.rejects(command(files, [], {id: 'blocked', name: 'edit'}), + error => error.storageFailure === true) +}) + +test('live ID lookup, duplicate rejection and restart preserve the same map', async t => { + let runtime + t.after(async () => runtime?.close()) + const files = await fixture(t) + runtime = await StoreRuntime.open({...files, maxWorkers: 1}) + await runtime.acceptCommand(JSON.stringify({ + id: 'rename', name: 'rename', value: 'renamed' + })) + await runtime.runQueuedCommands() + assert.equal(runtime.getCommandStatus('rename').value.status, 'done') + const query = { + path: '/', jsontag: false, + body: '[meta.index.id.has("first"), meta.index.id.get("renamed").name]' + } + const live = await runtime.runQuery(query) + assert.deepEqual(JSON.parse(live.body), [false, 'first']) + await runtime.acceptCommand(JSON.stringify({ + id: 'duplicate', name: 'rename', item: 1, value: 'renamed' + })) + await runtime.runQueuedCommands() + assert.equal(runtime.getCommandStatus('duplicate').value.status, 'failed') + assert.equal(runtime.storageFailed, false) + assert.equal(fs.existsSync(path.join(files.dir, + 'data.duplicate.jsontag')), false) + await runtime.close() + runtime = await StoreRuntime.open({...files, maxWorkers: 1, + validateIndexes: true}) + assert.deepEqual(JSON.parse((await runtime.runQuery(query)).body), + [false, 'first']) + await runtime.acceptCommand('{"id":"after","name":"edit"}') + await runtime.runQueuedCommands() + assert.equal(runtime.getCommandStatus('after').value.status, 'done') +}) + +test('runtime exposes explicit index validation and reconstruction', async t => { + const files = await fixture(t) + fs.writeFileSync(path.join(files.dir, 'index.id.json'), '{}') + await assert.rejects(StoreRuntime.open({...files, maxWorkers: 1, + validateIndexes: true}), /Integrity mismatch/) + fs.writeFileSync(path.join(files.dir, 'index.offset.json'), '{') + const runtime = await StoreRuntime.open({...files, maxWorkers: 1, + rebuildIndexes: true}) + try { + assert.deepEqual(runtime.meta.index.id, + new Map([['first', 1], ['last', 2]])) + } + finally { + await runtime.close() + } + assert.equal(fs.readFileSync(path.join(files.dir, 'index.id.json'), 'utf8'), + '{}') +}) + + +test('ID-only edits do not parse untouched records during command preparation', async t => { + const files = await fixture(t) + const parsed = new Set() + const firstParse = FileParser.prototype.firstParse + t.mock.method(FileParser.prototype, 'firstParse', function (target) { + parsed.add(target[getIndex]) + return firstParse.call(this, target) + }) + const result = await command(files, [], { + id: 'rename', name: 'rename', value: 'renamed' + }) + assert.equal(result.meta.index.id.get('renamed'), 1) + assert.equal(parsed.has(2), false) +}) + +for (const changeFile of [false, true]) { + test(`finalizers must preserve serialization (change file: ${changeFile})`, async t => { + const files = await fixture(t, { + indexSource: `import fs from 'node:fs/promises' + export default { + update() {}, + async finalize(bytes, meta, uuid) { + bytes[bytes.indexOf('edited')] = 120 + if (${changeFile}) { + await fs.writeFile(meta.data + '/data.' + uuid + + '.jsontag', bytes) + } + } + }` + }) + await assert.rejects(command(files, [], {id: 'tamper', name: 'edit'}), + /Changeset changed during finalization/) + }) +} + +test('integrity-enabled commands publish data and index hashes before reopening', async t => { + let runtime + t.after(async () => { + await runtime?.close() + }) + const files = await fixture(t) + const { appendIntegrityRecord, loadIntegrityManifest, verifyIntegrity } = + await import('../src/integrity.mjs') + const { appendIndexIntegrity } = await import('../src/index-files.mjs') + files.integrityFile = path.join(files.dir, 'data.integrity.jsontag') + await appendIntegrityRecord(files.integrityFile, files.datafile, + fs.readFileSync(files.datafile)) + await appendIndexIntegrity(files.integrityFile, {data: files.dir}) + runtime = await StoreRuntime.open({...files, maxWorkers: 1}) + for (const task of [{id: 'rename', name: 'rename', value: 'renamed'}, + {id: 'noop', name: 'noop'}]) { + await runtime.acceptCommand(JSON.stringify(task)) + await runtime.runQueuedCommands() + assert.equal(runtime.getCommandStatus(task.id).value.status, 'done') + const manifest = await loadIntegrityManifest(files.integrityFile) + for (const name of [`data.${task.id}.jsontag`, + `index.id.${task.id}.json`, `index.offset.${task.id}.json`]) { + const file = path.join(files.dir, name) + assert.equal(verifyIntegrity(manifest, files.integrityFile, file, + fs.readFileSync(file), {required: true}), true) + } + } + await runtime.close() + runtime = await StoreRuntime.open({...files, maxWorkers: 1}) + assert.equal(runtime.meta.index.id.get('renamed'), 1) + await runtime.close() + fs.writeFileSync(path.join(files.dir, 'index.offset.rename.json'), '{}') + await assert.rejects(StoreRuntime.open({...files, maxWorkers: 1}), + /Integrity mismatch/) +}) diff --git a/test/power-loss-baseline.mjs b/test/power-loss-baseline.mjs index e88b00d..153bfb0 100644 --- a/test/power-loss-baseline.mjs +++ b/test/power-loss-baseline.mjs @@ -1,3 +1,4 @@ +import { FileDataset } from '../src/file-data.mjs' import { Buffer } from 'node:buffer' // The original baseline is retained at EVD-20260919-TTZ7C-23's Git version. // These assertions implement the corrected command-only / authoritative-log @@ -101,8 +102,11 @@ test('PL10 an actual short changeset write is completed before acknowledgment', path.basename(e.file).startsWith('data.A.jsontag.') ).length >= 2 ) + const dataset = new FileDataset() + t.after(() => dataset.close()) + const data = dataset.open((await inspectStore(store)).sources) assert.deepEqual( - (await inspectStore(store)).data.persons.map(p => p.name), + data.persons.map(p => p.name), ['A'] ) }) diff --git a/test/required-integrity.mjs b/test/required-integrity.mjs new file mode 100644 index 0000000..a8a6e9d --- /dev/null +++ b/test/required-integrity.mjs @@ -0,0 +1,259 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs/promises' +import path from 'node:path' +import test from 'node:test' +import { execFile } from 'node:child_process' +import { promisify } from 'node:util' +import { fileURLToPath } from 'node:url' +import { initializeIntegrity } from '../src/initialize-integrity.mjs' +import { loadFileData, scanDataFile } from '../src/file-data.mjs' +import { inspectStore } from '../src/store-inspection.mjs' +import { acquireOwnership } from '../src/store-ownership.mjs' +import StoreRuntime from '../src/store-runtime.mjs' +import { getDefaultIntegrityFile, loadIntegrityManifest, verifyIntegrity } + from '../src/integrity.mjs' +import { appendArtifactIntegrity } from '../src/index-files.mjs' +import idIndex, { prepareIdIndex } from '../src/index.id.mjs' +import offsetIndex from '../src/index.offset.mjs' +import runCommand, { initialize, close } from '../src/command-worker-module.mjs' +import { makeServerFixture } from './durability-helpers.mjs' + +async function fixture(t) { + const files = await makeServerFixture(t, { + initialData: '{"persons":[{"name":"first"}]}' + }) + files.integrityFile = getDefaultIntegrityFile(files.datafile) + const bytes = await fs.readFile(files.datafile) + await offsetIndex.writeSerialized(bytes, {data: files.dir}) + idIndex.write({data: files.dir}, prepareIdIndex(bytes).entries) + await fs.unlink(files.integrityFile) + return files +} + +async function snapshot(files) { + const entries = [] + for (const name of await fs.readdir(files.dir)) { + if (name !== path.basename(files.integrityFile)) { + entries.push([name, await fs.readFile(path.join(files.dir, name))]) + } + } + return entries +} + +async function assertUnlocked(files) { + const owner = await acquireOwnership([files.dir]) + await owner.release() +} + +test('normal startup and loader require integrity even with old false options', async t => { + const files = await fixture(t) + const before = await snapshot(files) + await assert.rejects(StoreRuntime.open({...files, integrity: false}), + /Missing integrity manifest.*init-integrity/) + await assert.rejects(loadFileData({dataFile: files.datafile, commands: [], + integrityFile: null, integrityRequired: false}), + /Missing integrity manifest/) + const report = await inspectStore({...files, integrity: false}) + assert.equal(report.ready, false) + assert.deepEqual(await snapshot(files), before) + await assert.rejects(fs.access(files.integrityFile), /ENOENT/) + await assertUnlocked(files) +}) + +test('initialization preserves commands and original bytes', async t => { + let runtime + t.after(async () => { + await runtime?.close() + }) + const files = await fixture(t) + await initializeIntegrity(files) + runtime = await StoreRuntime.open({...files, maxWorkers: 1}) + await runtime.acceptCommand(JSON.stringify({id: 'added', name: 'addPerson', + value: {name: 'kept after restart'}})) + await runtime.runQueuedCommands() + assert.equal(runtime.getCommandStatus('added').value.status, 'done') + await runtime.close() + await fs.unlink(files.integrityFile) + const before = await snapshot(files) + const result = await initializeIntegrity(files) + assert.equal(result.files, 6) + assert.deepEqual(result.committed, ['added']) + assert.deepEqual(await snapshot(files), before) + const manifest = await loadIntegrityManifest(files.integrityFile) + assert.equal(manifest.size, 6) + for (const [name] of manifest) { + const file = path.join(files.dir, name) + assert.equal(verifyIntegrity(manifest, files.integrityFile, + file, await fs.readFile(file), {required: true}), true) + } + runtime = await StoreRuntime.open({...files, maxWorkers: 1}) + const response = await runtime.runQuery({path: '/', jsontag: false, + body: 'data.persons.map(person => person.name)'}) + assert.deepEqual(JSON.parse(response.body), ['first', 'kept after restart']) +}) + +test('initialization CLI supports a configured manifest path without importing hooks', async t => { + const files = await fixture(t) + const logdir = path.join(files.dir, 'logs') + await fs.mkdir(logdir) + files.integrityFile = path.join(logdir, 'checksums.jsontag') + await fs.writeFile(files.commandsFile, 'throw new Error("must not import")') + await fs.writeFile(files.indexFile, 'throw new Error("must not import")') + const config = path.join(files.dir, 'store.json') + await fs.writeFile(config, JSON.stringify(files)) + const script = fileURLToPath(new URL('../scripts/recover.mjs', import.meta.url)) + const result = await promisify(execFile)(process.execPath, + [script, 'init-integrity', '--store', config]) + assert.equal(JSON.parse(result.stdout).integrityFile, files.integrityFile) + assert.equal((await inspectStore(files)).ready, true) + await assertUnlocked(files) +}) + +test('initialization never replaces an existing baseline, even after tampering', async t => { + const files = await fixture(t) + await initializeIntegrity(files) + const before = await fs.readFile(files.integrityFile) + await fs.appendFile(files.datafile, '\n') + await assert.rejects(initializeIntegrity(files), /already exists/) + assert.deepEqual(await fs.readFile(files.integrityFile), before) + await assert.rejects(loadFileData({dataFile: files.datafile, commands: []}), + /Integrity mismatch/) + await assertUnlocked(files) +}) + +for (const [name, mutate] of [ + ['stale ID index', async files => { + await fs.writeFile(path.join(files.dir, 'index.id.json'), '{}') + }], + ['stale offset index', async files => { + await fs.writeFile(path.join(files.dir, 'index.offset.json'), '{}') + }], + ['invalid unread record body', async files => { + const record = text => `(${Buffer.byteLength(text)})${text}\n` + await fs.writeFile(files.datafile, + record('{"persons":[~1]}') + record('{"x":}')) + await fs.unlink(path.join(files.dir, 'index.offset.json')) + }], + ['duplicate IDs', async files => { + const record = text => `(${Buffer.byteLength(text)})${text}\n` + await fs.writeFile(files.datafile, record('{"persons":[~1,~2]}') + + record('{}') + record('{}')) + await fs.unlink(path.join(files.dir, 'index.offset.json')) + await fs.unlink(path.join(files.dir, 'index.id.json')) + }], + ['pending history', async files => { + await fs.writeFile(files.commandLog, + '{"id":"pending","name":"addPerson"}\n') + await fs.writeFile(files.commandStatus, + '{"command":"pending","status":"accepted","code":202}\n') + }], + ['unexplained changeset', async files => { + await fs.writeFile(path.join(files.dir, 'data.unknown.jsontag'), '') + }] +]) { + test(`initialization rejects ${name} without writing a baseline`, async t => { + const files = await fixture(t) + await mutate(files) + const before = await snapshot(files) + await assert.rejects(initializeIntegrity(files)) + assert.deepEqual(await snapshot(files), before) + await assert.rejects(fs.access(files.integrityFile), /ENOENT/) + await assertUnlocked(files) + }) +} + +test('initialization respects ownership and detects changes during validation', async t => { + const files = await fixture(t) + const owner = await acquireOwnership([files.dir]) + await assert.rejects(initializeIntegrity(files), /locked/) + await owner.release() + const read = fs.readFile + let changed = false + t.mock.method(fs, 'readFile', async (...args) => { + const result = await read(...args) + if (!changed && args[0] === files.commandLog) { + changed = true + await fs.appendFile(files.datafile, '\n') + } + return result + }) + await assert.rejects(initializeIntegrity(files), /changed during inspection/) + await assert.rejects(fs.access(files.integrityFile), /ENOENT/) + await assertUnlocked(files) +}) + +test('uncertain manifest publication retains ownership for inspection', async t => { + const files = await fixture(t) + const rename = fs.rename + t.mock.method(fs, 'rename', async (...args) => { + if (args[1] === files.integrityFile) { + throw new Error('injected publication failure') + } + return rename(...args) + }) + await assert.rejects(initializeIntegrity(files), error => { + return error.storageFailure && /injected/.test(error.message) + }) + await assert.rejects(acquireOwnership([files.dir]), /locked/) +}) + +test('artifact hashes share one durable append and synchronization', async t => { + const files = await fixture(t) + const source = scanDataFile(files.datafile) + const open = fs.open + let appends = 0 + let syncs = 0 + t.mock.method(fs, 'open', async (...args) => { + const handle = await open(...args) + if (args[0] === files.integrityFile && args[1] === 'a') { + appends++ + const datasync = handle.datasync.bind(handle) + handle.datasync = async () => { + syncs++ + return datasync() + } + } + return handle + }) + await appendArtifactIntegrity(files.integrityFile, files.datafile, + source.digest, {data: files.dir}) + assert.equal(appends, 1) + assert.equal(syncs, 1) + assert.equal((await loadIntegrityManifest(files.integrityFile)).size, 3) +}) + +test('a manifest synchronization failure prevents command success', async t => { + const files = await fixture(t) + await initializeIntegrity(files) + const loaded = await loadFileData({dataFile: files.datafile, commands: []}) + t.after(close) + await initialize({...loaded, ...files}) + const open = fs.open + t.mock.method(fs, 'open', async (...args) => { + const handle = await open(...args) + if (args[0] === files.integrityFile && args[1] === 'a') { + handle.datasync = async () => { + throw new Error('injected manifest sync failure') + } + } + return handle + }) + await assert.rejects(runCommand(JSON.stringify({id: 'failed', + name: 'addPerson', value: {name: 'uncommitted'}})), error => { + return error.storageFailure && /manifest sync failure/.test(error.message) + }) + assert.equal(await fs.readFile(files.commandStatus, 'utf8'), '') +}) + +test('initialization supports missing indexes and startup verifies their data', async t => { + const files = await fixture(t) + await fs.unlink(path.join(files.dir, 'index.id.json')) + await fs.unlink(path.join(files.dir, 'index.offset.json')) + const result = await initializeIntegrity(files) + assert.equal(result.files, 1) + const loaded = await loadFileData({dataFile: files.datafile, commands: []}) + assert.equal(loaded.meta.index.id.get('first'), 1) + await fs.appendFile(files.datafile, '\n') + await assert.rejects(loadFileData({dataFile: files.datafile, commands: [], + rebuildIndexes: true, integrityRequired: false}), /Integrity mismatch/) +}) diff --git a/test/store-runtime.mjs b/test/store-runtime.mjs index fa83d57..07743ba 100644 --- a/test/store-runtime.mjs +++ b/test/store-runtime.mjs @@ -69,7 +69,7 @@ async function openRuntime(t, overrides = {}) { async runWorker() { events.push('committed data loaded') return { - data: new Uint8Array([1]), + sources: [{file: fixture.datafile}], meta: { index: { id: new Map() }, parts: 1 @@ -102,7 +102,7 @@ async function openRuntime(t, overrides = {}) { test('runtime opens committed state and closes its owned resources', async t => { const opened = await openRuntime(t) - assert.equal(opened.runtime.data.length, 1) + assert.equal(opened.runtime.sources.length, 1) assert.equal(opened.runtime.meta.parts, 1) assert.equal(opened.pools.length, 2) assert.equal(opened.ownershipReleased(), false) @@ -177,7 +177,7 @@ test('runtime keeps concurrent acceptance and execution in command-log order', const parsed = JSONTag.parse(task.command) starts.push(parsed.id) return { - data: new Uint8Array([starts.length + 1]), + source: { file: `data.${parsed.id}.jsontag` }, meta: { parts: starts.length + 1 } } }