diff --git a/.github/workflows/spiral-integration.yml b/.github/workflows/spiral-integration.yml index c13b7e5..879fa84 100644 --- a/.github/workflows/spiral-integration.yml +++ b/.github/workflows/spiral-integration.yml @@ -27,6 +27,17 @@ jobs: - run: pip install -r .spiral-core/requirements.txt + - name: Test provenance validation + run: python test/spiral-provenance.py + + - name: Validate every introduced artifact version and the candidate snapshot + env: + SPIRAL_BASE: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} + SPIRAL_HEAD: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + python scripts/validate-spiral-provenance.py --range "$SPIRAL_BASE..$SPIRAL_HEAD" + python scripts/validate-spiral-provenance.py --tree "$SPIRAL_HEAD" + - if: github.event_name == 'pull_request' run: >- node .spiral-core/bin/spiral.mjs validate integration diff --git a/.spiral/cycles/CYC-20260819-09ZEF-5.md b/.spiral/cycles/CYC-20260819-09ZEF-5.md new file mode 100644 index 0000000..c6f7ab7 --- /dev/null +++ b/.spiral/cycles/CYC-20260819-09ZEF-5.md @@ -0,0 +1,222 @@ +--- +id: CYC-20260819-09ZEF-5 +--- + +# Cycle: Integrity Detection + +Repository branch: `spiral/CYC-20260819-09ZEF-5-integrity-detection` +Branch verified: verified on 2026-08-19 with `git branch --show-current`, branched from `master` after accepted CYC-018 was merged. + +## Analyze + +Project state / prior evaluation that makes this cycle relevant: + +SimplyStore has completed the Spiral process update cycle and returns to the durability production-readiness roadmap. Recent durability cycles established executable crash/replay behavior, retry/idempotency semantics, OD-JSONTag framing failure behavior, after-change characterization, and command/load timeout behavior. The human has especially called out concern about corrupted or altered OD-JSONTag files. + +Nearest important risk / uncertainty / desired movement: + +The remaining durability claim is still too weak against well-formed but altered durable data. Current evidence covers malformed or truncated OD-JSONTag record framing, not intentional or accidental content alteration that still parses. A developer evaluating SimplyStore needs to know whether altered durable artifacts are detected, ignored, recovered, or silently trusted. + +Relevant human direction / feedback: + +The human agreed that the next cycle should continue the durability roadmap toward integrity/tamper detection while preserving simplicity and avoiding silent API or on-disk format changes. + +Governing higher-level plan / direction: + +- `SRC-001` at `f2c3a597c12c2dbe6d7467582d5a3f4308acf02d` gives the ordered durability/extensibility direction. +- `REQ-001` at `6a9977eb22b0a610a82da5c52864e3bb72b164c9` requests explicit, mechanically testable durability evidence. +- `DES-001` at `85913e982e163c7b05a8e616c00e3f2014e775cc` defines D6 detectable corruption and places integrity-root work after crash/replay, adversarial storage, idempotency, and handler-contract evidence. + +Current position in that plan: + +Continue. The project has already covered crash/fault-injection, reconstruction/idempotency slices, adversarial malformed-framing cases, and handler/timeout characterization. The next coherent risk is integrity checking for altered durable data, with care not to overclaim broader repair or production safety. + +## Plan + +### Cycle goal + +Assess and implement the first simple integrity-detection behavior for durable OD-JSONTag data so altered committed storage fails explicitly instead of being silently used, while preserving SimplyStore's simplicity and avoiding unrelated runtime/API changes. + +### Why now / why this cycle boundary + +This is the next durability-roadmap uncertainty with high downstream leverage: integrity metadata choices affect on-disk compatibility, recovery behavior, developer-facing claims, and future extension/finalizer design. A focused cycle can decide and test the first narrow integrity behavior without turning SimplyStore into a broader storage framework. + +### Plan continuity decision + +Continue the original durability plan. The cycle pulls forward the integrity/tamper-detection concern because malformed framing has already been characterized and the human explicitly identified corrupted or altered OD-JSONTag files as a concern. + +### Current starting evidence + +- Malformed/truncated OD-JSONTag record framing already fails explicitly in covered cases. +- The project context records that full syntax validation and well-formed tampering need future integrity metadata. +- The current public API, JavaScript query API, and durable on-disk format must not silently change. + +### Evaluation basis + +- Current effective behavior for well-formed altered durable data is inspected and stated before runtime behavior changes. +- The cycle records the Understanding/evidenced-gap checkpoint before consequential product modification. +- Tests demonstrate the chosen integrity behavior on at least one committed durable-data alteration path. +- Any on-disk format implication is explicit, bounded, and documented as either compatible, transitional, or intentionally not yet enabled. +- Existing durability tests continue to pass. + +### Likely work + +- Inspect current durable file layout, OD-JSONTag load path, and existing adversarial storage tests. +- Run a small probe or characterization test for well-formed altered committed data. +- Present the Spiral checkpoint: My understanding / Current effective behavior / Evidenced gap / Material assumptions. +- If confirmed, design the smallest integrity mechanism and tests that can detect altered committed storage without broad framework machinery. +- Implement only the committed slice and add verification evidence. + +### Explicit non-goals + +- Automatic repair of corrupted data. +- A Merkle tree, streaming integrity framework, database WAL, event-sourcing framework, message bus, or schema registry. +- Broad REST API or JavaScript query API changes. +- Silent durable format changes. +- Solving VM2/query sandbox security. +- Completing every possible corruption/tamper case in one cycle. + +### Pause / re-plan conditions + +- Current behavior already detects the targeted alteration class. +- A safe first integrity mechanism requires a breaking on-disk format change that has not been explicitly accepted. +- Investigation shows integrity checking depends on unresolved handler/commit-order semantics outside this cycle. +- The smallest viable design conflicts with SimplyStore's simplicity or public compatibility commitments. + +## Act + +Important artifacts / semantic commits produced: + +- `EVD-20260819-09ZEF-6` records the baseline probe showing that same-length payload alteration can preserve OD-JSONTag framing and load as ordinary altered data. +- `DES-20260819-09ZEF-7` defines an optional durable file integrity manifest as the first narrow integrity-detection mechanism. +- `IMP-20260819-09ZEF-8` implements the optional integrity manifest, startup verification, command changeset digest recording, and focused tests. +- `EVD-20260819-09ZEF-9` verifies the implementation with startup, full-server durability, ACID, handler, and touched-file lint checks. + +Material implementation decisions or deviations from the initial likely work: + +- The first step was kept to behavior characterization and evidence until the Spiral Understanding/evidenced-gap checkpoint was presented and confirmed. +- The implemented integrity behavior is opt-in/manifest-driven so existing datasets without sidecar integrity metadata remain loadable by default. +- The first integrity scope is deliberately limited to base data and committed changesets; command log/status and manifest self-protection remain out of scope. + +Out-of-scope discoveries retained for later: + +- Whole-repo ESLint still reports unrelated legacy errors in `src/index.id.mjs`, `src/index.offset.mjs`, `src/produce.mjs`, and `src/triplestore.mjs`. +- The integrity manifest itself is not yet protected against coordinated data+manifest rewrite. +- Command log/status integrity remains outside this first durable-data slice. + +## Evaluate + +Integrated result against cycle goal: + +Implemented and verified the first narrow integrity-detection behavior for durable OD-JSONTag data. Integrity-enabled stores now verify base data and committed changesets against an append-readable SHA-256 sidecar manifest before parsing. If a covered file's bytes change, startup fails explicitly instead of silently loading altered state. Existing stores without integrity metadata remain loadable by default. + +Evidence / acceptance result: + +Verification evidence is recorded in `EVD-20260819-09ZEF-9`. + +Automated checks: + +- `npm test` passed: 19/19 startup durability tests. +- `npm run test:durability` passed: 16/16 full-server durability tests. +- `npm run test:acid` passed: 5/5 ACID baseline tests. +- `npm run test:handlers` passed: 3/3 handler characterization tests. +- Touched-file ESLint passed. +- Whole-repo ESLint still fails on unrelated legacy files. + +The human accepted the cycle on 2026-08-19. + +Metric or risk movement: + +The D6 detectable-corruption claim is stronger for manifest-covered base data and committed changesets. A same-length payload alteration that previously loaded silently is now detected when integrity metadata exists. This is still not a full tamper-resistance claim because coordinated data+manifest rewrite remains possible. + +What changed in our understanding: + +The smallest useful integrity mechanism can be compatible and opt-in: a sidecar manifest can prove altered-byte detection without immediately migrating all existing datasets or changing REST/query APIs. + +Surprises / model mismatches: + +No major design mismatch appeared. A full-server test was needed to verify the digest is written before the existing `done` commit boundary, because unit startup tests alone could not prove the command lifecycle ordering. + +Known compromises: + +- Integrity is not default-required for legacy datasets. +- The manifest is append-readable and latest-wins, but is not chained, signed, or protected from coordinated rewrite. +- Command log/status integrity remains outside this cycle. +- Existing datasets need an explicit manifest entry before integrity-required startup succeeds. + +Unresolved issues within current goal: + +No known unresolved issue blocks evaluation of the agreed first narrow integrity slice. + +Candidate next-cycle inputs: + +- Decide whether to add a simple script to initialize/rebuild integrity manifests for existing datasets. +- Consider extending integrity coverage to command log/status files. +- Consider documenting the durability envelope in a developer-facing `DURABILITY.md` once enough slices have accumulated. +- Retain the earlier reset/clean dataset script idea for dirty examples and local development stores. + +Human evaluation / feedback: + +Accepted by the human on 2026-08-19 after reviewing the implementation summary of the integrity check. + +Cycle accepted, still open, or deliberately re-planned: + +Accepted. Ready for prospective integration validation against the current `master`. + +## Process learning + +What context/constraint/evaluation helped: + +The explicit compatibility constraint was useful: making integrity manifest-driven avoided silently changing the durable format contract for existing datasets while still proving the targeted altered-data case. + +What bookkeeping was useless: + +None identified. + +What should the environment learn from this cycle: + +For SimplyStore durability work, pair direct startup/unit checks with at least one full-server lifecycle test when the claim depends on command commit ordering. + +## Compliance correction — 2026-09-16 + +Reopened on the same branch at the maintainer's explicit request to correct the +compliance audit findings (`DEF-20260916-TTZ7C-1`). The original acceptance above +remains historical evidence for the runtime integrity slice. The cycle was reopened as Active for this bounded process correction; the +correction was subsequently accepted as recorded below. + +Branch checked on reopening: `spiral/CYC-20260819-09ZEF-5-integrity-detection`. +Scope: repair evidence references and governed implementation lineage, replace +partial parsing with standards-based validation, add staged/range prevention, +and reconcile context. No SimplyStore runtime changes are authorized by this +correction. Governing durability direction remains unchanged. + +### Correction evaluation — 2026-09-16 + +The four audit findings have concrete corrections in new commits: + +- `91f5d09` repairs the seven references while preserving historical test claims + and records the defect/reopened scope. +- `f66ea74` defines standards-based reference validation and lineage repair. +- `adccdd1` implements the validator, 16 regression tests, staged hook, CI range + checks, current provenance/lineage projections, and operator guidance. + +`EVD-20260916-TTZ7C-3` records verification. The current graph and introduced +history pass. Checking the original snapshot still reports all seven missing +artifact versions. No runtime source or dependency changed. + +At evaluation, acceptance was pending and the integration validator correctly +blocked the Active cycle. A disposable hypothetical Accepted candidate passed +the other integration constraints. The actual acceptance follows below. + +### Correction acceptance — 2026-09-16 + +After reviewing the correction summary and the explanation of the integration +gate, the maintainer replied "Agreed" to the explicit question asking whether +they accepted the corrections. This accepts the completed compliance correction +at `6f71005388b1e393eb1a6b30059e6e096c751bb2`, with verification in +`EVD-20260916-TTZ7C-3`. The original runtime acceptance remains unchanged. + +Current cycle status: **Accepted and closed**. The cycle is ready for integration +validation against the current target; acceptance does not waive that check. +Historical invalid references remain preserved and reported by history audits. +No runtime code or dependencies changed as part of this acceptance. diff --git a/.spiral/cycles/CYC-20260819-09ZEF-5.ttl b/.spiral/cycles/CYC-20260819-09ZEF-5.ttl new file mode 100644 index 0000000..6fd555b --- /dev/null +++ b/.spiral/cycles/CYC-20260819-09ZEF-5.ttl @@ -0,0 +1,9 @@ +@prefix sd: . +@prefix dcterms: . +@prefix project: . + +project:CYC-20260819-09ZEF-5 + a sd:Cycle ; + dcterms:identifier "CYC-20260819-09ZEF-5" ; + sd:repositoryPath ".spiral/cycles/CYC-20260819-09ZEF-5.md" ; + sd:status sd:Accepted . diff --git a/.spiral/defects/DEF-20260916-TTZ7C-1.md b/.spiral/defects/DEF-20260916-TTZ7C-1.md new file mode 100644 index 0000000..d8488e0 --- /dev/null +++ b/.spiral/defects/DEF-20260916-TTZ7C-1.md @@ -0,0 +1,68 @@ +--- +id: DEF-20260916-TTZ7C-1 +--- + +# Defect: Incomplete Spiral Provenance Validation + +## Source and authorization + +On 2026-09-16 the maintainer requested a compliance audit of the checked-out +SimplyStore integrity branch, then responded "Lets try to correct this" and +"Please continue". This is an explicit instruction to correct the reported +process defects. The connected conversation is the primary source; this record +retains the operative wording rather than claiming to retain a full transcript. + +## Understanding and evidenced gap + +Correct the current causal records and prevent recurrence without changing +SimplyStore runtime behavior or rewriting Git history. At `fa329453bf640fab6083e39d57edfdbde222e282`: + +- Seven evidence references name implementation artifacts before their creation. +- The integrity slice adds a pre-commit prerequisite and loader verification to + governed paths without recording their predecessor lineage. +- The local partial-Turtle validator fails on vocabulary.ttl, and the core + snapshot/integration validator does not check historical target existence or + every introduced artifact version. Both official checks nevertheless passed. +- Context calls the accepted integrity cycle active. + +The earliest actionable cause is insufficient validation, not a runtime defect. + +## Exact reference repairs + +| Evidence | Target | Invalid historical target | First retained artifact version | +|---|---|---|---| +| EVD-006 | IMP-002 | `da55e78ec53465e1b02aaac1bf69c25c409ed89d` | `676fa5c6038f92219e902ef126f284360c1763d6` | +| EVD-007 | IMP-003 | `0fa4c441e8e8cb27562486c225bb02e8038d564e` | `c13d08a9e442e30ca4d66f28597939f55ef0a513` | +| EVD-009 | IMP-004 | `29f016cdb3ec6c3cc03ff9acebac26691a095326` | `36bb7b1013c2de219d4a730556ae9ab98cea7113` | +| EVD-010 | IMP-005 | `e2574dd0df70b1803c61f416c537a7f15376b92d` | `08f53c1d25468e97dbc517b3a6bcd7fb98bf4e5e` | +| EVD-011 | IMP-006 | `7d514568349d6a14ac494155549579533af68bf4` | `2eeecded67ac99c7ab757113fc8f299af1deafc2` | +| EVD-012 | IMP-007 | `570d35a4fa47db5edf4784c998ee441ce4f20821` | `28034073cf15170e367fe1bf9ca9846c299d23ea` | +| EVD-014 | IMP-009 | `bc376bfc353fe2489ba61a99ca0dc47b38fc1214` | `313b5b71016067388c542120712082d6023d87d4` | + +For each row, `git diff --name-only ` shows only `.spiral/` +changes. The recorded product implementation and historical test claims are +therefore retained. Corrected references do not retroactively validate the old +artifact versions. Raw history audits must continue to report those violations. + +## Correction boundary + +Continue the existing integrity branch as a bounded correction of its review +readiness. Reopen its cycle for process correction; preserve the original +2026-08-19 acceptance as acceptance of the runtime slice, not advance approval +of these corrections. No new product cycle or roadmap direction is selected. + +Record lineage prospectively for changed governed paths; do not assign every +shared source file to one owner. New integrity helpers remain a new concern. +Replace the partial parser with rdflib, add staged and introduced-range checks, +and keep core integration validation as a separate check. No historical +allowlist, amended commits, production changes, or new runtime dependencies. + +## Evaluation basis + +- Current graph references resolve to existing artifact versions at admissible commits. +- Repaired evidence remains honest about when artifacts and tests existed. +- Changed governed concerns preserve effective causes and predecessor lineage. +- Regression tests reject malformed, dangling, non-ancestor and transient bad references. +- CI checks both the introduced range and the prospective result. +- Raw history checks still fail on known old violations. +- Human evaluation of the correction remains pending until explicitly received. diff --git a/.spiral/defects/DEF-20260916-TTZ7C-1.ttl b/.spiral/defects/DEF-20260916-TTZ7C-1.ttl new file mode 100644 index 0000000..3d651cb --- /dev/null +++ b/.spiral/defects/DEF-20260916-TTZ7C-1.ttl @@ -0,0 +1,15 @@ +@prefix sd: . +@prefix dcterms: . +@prefix project: . + +project:DEF-20260916-TTZ7C-1 + a sd:Defect ; + dcterms:identifier "DEF-20260916-TTZ7C-1" ; + sd:repositoryPath ".spiral/defects/DEF-20260916-TTZ7C-1.md" ; + sd:status sd:Active ; + sd:provenanceConfidence sd:Evidenced ; + sd:observes [ + a sd:ArtifactReference ; + sd:artifact project:CYC-20260819-09ZEF-5 ; + sd:gitCommit "fa329453bf640fab6083e39d57edfdbde222e282" + ] . diff --git a/.spiral/designs/DES-20260819-09ZEF-7.md b/.spiral/designs/DES-20260819-09ZEF-7.md new file mode 100644 index 0000000..f249e2e --- /dev/null +++ b/.spiral/designs/DES-20260819-09ZEF-7.md @@ -0,0 +1,80 @@ +--- +id: DES-20260819-09ZEF-7 +--- + +# Design: Optional Durable File Integrity Manifest + +## Role + +Define the first narrow integrity-detection behavior for durable OD-JSONTag files. + +## Why this exists + +`DES-001` requires detectable corruption, and `EVD-20260819-09ZEF-6` shows that current OD-JSONTag framing validation does not detect a same-length payload alteration. The project needs a small integrity mechanism that improves developer-evaluable durability without silently changing the public API or forcing existing datasets through an implicit migration. + +## Behavior / responsibilities + +### I1 - integrity manifest is explicit and compatible + +SimplyStore may be configured with an integrity manifest file. If an integrity manifest exists or integrity checking is explicitly enabled, startup verifies the base data file and every committed changeset against the manifest before parsing them as durable state. + +Existing datasets without an integrity manifest remain loadable by default. This avoids silently converting the on-disk format contract for existing users. + +### I2 - committed writes update integrity metadata + +When integrity is enabled, creating a committed changeset also records that changeset file's digest before the command is marked `done`. This preserves the existing commit-before-query-visibility invariant: a `done` status should not be written until the durable data needed for restart, including integrity metadata, exists. + +### I3 - altered committed bytes fail explicitly + +If a manifest entry exists and the durable file's current bytes do not match the recorded digest, startup fails with an explicit recovery/integrity error rather than silently exposing altered state. + +### I4 - missing manifest entries fail only when integrity is required + +If integrity is explicitly enabled, missing digest entries for required durable files fail startup. If integrity is not enabled and no manifest exists, startup keeps legacy behavior. + +## Boundaries + +What belongs here: + +- SHA-256 digest verification for base data and committed changeset files; +- a simple append-readable manifest file; +- tests for well-formed altered committed durable data; +- explicit startup failure when required integrity metadata is missing or mismatched. + +What remains outside: + +- automatic repair; +- Merkle trees or streaming root proofs; +- command log/status integrity; +- schema/index artifact integrity; +- a migration tool for existing datasets; +- changing REST or JavaScript query APIs; +- changing OD-JSONTag syntax validation beyond existing framing checks. + +## Design choices + +| Choice | Reason | +|---|---| +| Use SHA-256 from Node's built-in `crypto` module. | Avoids a dependency and is adequate for a first byte-integrity check. | +| Store integrity metadata in a sidecar file. | Keeps existing OD-JSONTag durable data files readable and makes the new format implication explicit. | +| Verify before parsing. | Prevents altered bytes from becoming normal recovered state. | +| Treat the manifest as append-readable latest-wins records. | Matches the existing simple log style and keeps the first version easy to inspect. | +| Write changeset integrity before `done`. | Preserves the existing committed-state recovery boundary. | + +## Compatibility / migration position + +This cycle does not require all existing datasets to have integrity metadata. The first supported path is opt-in integrity for new or explicitly configured stores. A later cycle may add a reset/clean/rebuild command or migration strategy if developer evaluation shows that is the right next step. + +## Verification plan + +1. Unit-test digest manifest read/write and mismatch behavior. +2. Startup-test an integrity-enabled store whose base data is altered after manifest creation. +3. Startup-test an integrity-enabled committed changeset altered after its manifest entry is recorded. +4. Preserve existing malformed-framing behavior and normal durability tests. + +## Open questions / deferred decisions + +- Whether integrity should become default for newly initialized stores. +- Whether command log/status files should receive the same digest treatment. +- Whether a future manifest should include file size, command ordering, previous digest links, or a root digest. +- Whether recovery tooling should be able to rebuild or deliberately accept a new manifest. diff --git a/.spiral/designs/DES-20260819-09ZEF-7.ttl b/.spiral/designs/DES-20260819-09ZEF-7.ttl new file mode 100644 index 0000000..09e3b1f --- /dev/null +++ b/.spiral/designs/DES-20260819-09ZEF-7.ttl @@ -0,0 +1,19 @@ +@prefix sd: . +@prefix dcterms: . +@prefix project: . + +project:DES-20260819-09ZEF-7 + a sd:Design ; + dcterms:identifier "DES-20260819-09ZEF-7" ; + sd:repositoryPath ".spiral/designs/DES-20260819-09ZEF-7.md" ; + sd:status sd:Active ; + sd:derivedFrom [ + a sd:ArtifactReference ; + sd:artifact project:DES-001 ; + sd:gitCommit "85913e982e163c7b05a8e616c00e3f2014e775cc" + ] ; + sd:observes [ + a sd:ArtifactReference ; + sd:artifact project:EVD-20260819-09ZEF-6 ; + sd:gitCommit "a05b433c35c8fa90a29d9a0cdf3169b027ff9a0c" + ] . diff --git a/.spiral/designs/DES-20260916-TTZ7C-2.md b/.spiral/designs/DES-20260916-TTZ7C-2.md new file mode 100644 index 0000000..3e5ad7e --- /dev/null +++ b/.spiral/designs/DES-20260916-TTZ7C-2.md @@ -0,0 +1,85 @@ +--- +id: DES-20260916-TTZ7C-2 +--- + +# Design: Prospective Provenance Repair And Historical Validation + +## Intent and boundary + +Correct `DEF-20260916-TTZ7C-1` while preserving historical evidence and the +accepted integrity runtime. This is a bounded process correction inside the +reopened integrity cycle. It does not start a new durability feature. + +## Validation responsibilities + +Use Python rdflib (already required by the pinned core) for Turtle parsing. +Keep `node scripts/validate-spiral-provenance.mjs` as a thin entry point; remove +the CYC-016-specific assumptions and handwritten Turtle parser. The validator +reads only the project's `.spiral/**/*.ttl`, including vocabulary documents. +It complements the pinned core's identity/coherence/integration validator. + +- Default: validate the committed `HEAD` snapshot. Determine each Turtle file's + source version from Git, not the current wall clock or tip alone. +- `--staged`: read the Git index, not unstaged files. New/changed staged + references may target current HEAD; persisted unchanged files retain strict + ancestor checks. Use this before semantic causal commits. +- `--tree `: validate that committed snapshot and reference origins. +- `--range ..`: check changed Turtle versions at every introduced + commit, including merge changes and versions later deleted or repaired. + Accept `--range ` to audit all reachable history. Report old violations + as failures; do not introduce a historical allowlist. + +Every versioned reference must have exactly one artifact IRI and one full Git +hash, resolve to a real commit, and name an artifact definition and repository +path present at that commit. Local references must point backward through Git +ancestry (or to HEAD for new staged content). Resolve references semantically +through the ontology's historical-reference hierarchy, including custom +subproperties in the project graph; do not depend on prefixes or formatting. + +The two existing external namespaces (Muze culture and Spiral process artifacts) +are resolved only through the explicitly pinned `.spiral-core` submodule. +Verify the target artifact at its external hash and that hash's inclusion in +the source version's pinned core history. Do not compare hashes from separate +repositories as if they shared local ancestry, or silently skip external links. +An unavailable process checkout causes an actionable validation failure. + +## CI and local prevention + +Run the validator's regression tests and reference-range validation for pull +requests and merge groups, using event hashes passed through environment +variables. Check the candidate tip/prospective merge snapshot too. Preserve the +existing core integration check for acceptance, identity, and convergence. + +Document a repository hook that runs staged validation, installed explicitly +per checkout; keep CI authoritative for introduced commit ranges. A hook is an +optional local convenience, not evidence that historical commits were checked. + +## Prospective lineage correction + +The original integrity artifact is a new concern, but it also changed existing +loader verification and the changeset/pre-done boundary. Update the affected +`IMP-001`, `IMP-004`, and `IMP-006` projections with the integrity design as an +additional effective cause, their exact immediate predecessors, and this +correction as transition cause. Preserve their still-valid durability reasons. +Explain that this metadata revision records the already-existing runtime change; +it does not pretend that the original implementation commit carried the links. + +Update the integrity projection with its own immediate predecessor and references +to the governed units extended by that original change. Mark this metadata-only +revision behavior-preserving. Preserve the original design as effective cause. +Likewise revise `IMP-010` and the CYC-018 process-tooling projection when replacing +their validator/CI implementation, preserving predecessors and surviving causes. + +## Verification and acceptance + +Use disposable Git repositories to test valid prefix/format variations, staged +vs unstaged content, malformed references, missing target versions/paths, +non-ancestor targets, and a bad intermediate commit repaired later. Validate the +real repaired snapshot and the branch's introduced range. Audit the old snapshot +separately and show the seven original failures remain detectable. + +Run the pinned integration check with the real Active correction status: rejection +until human acceptance is expected. A disposable prospective Accepted tree can +check the remaining integration constraints, but must not change or imply actual +acceptance. The user reviews the completed correction; only explicit acceptance +closes the reopened cycle. No runtime test claims are manufactured by this work. diff --git a/.spiral/designs/DES-20260916-TTZ7C-2.ttl b/.spiral/designs/DES-20260916-TTZ7C-2.ttl new file mode 100644 index 0000000..7d409fc --- /dev/null +++ b/.spiral/designs/DES-20260916-TTZ7C-2.ttl @@ -0,0 +1,14 @@ +@prefix sd: . +@prefix dcterms: . +@prefix project: . + +project:DES-20260916-TTZ7C-2 + a sd:Design ; + dcterms:identifier "DES-20260916-TTZ7C-2" ; + sd:repositoryPath ".spiral/designs/DES-20260916-TTZ7C-2.md" ; + sd:status sd:Active ; + sd:derivedFrom [ + a sd:ArtifactReference ; + sd:artifact project:DEF-20260916-TTZ7C-1 ; + sd:gitCommit "91f5d0917959715748444860932aeefe401b3fc5" + ] . diff --git a/.spiral/evidence/EVD-006.md b/.spiral/evidence/EVD-006.md index 5faffcd..a01492a 100644 --- a/.spiral/evidence/EVD-006.md +++ b/.spiral/evidence/EVD-006.md @@ -75,3 +75,12 @@ The new process test covers: ## Evidence quality This evidence is stronger than timing-based crash attempts because the process is killed at a named boundary in the real server code path, guarded by test-only runtime configuration. It is intentionally narrow and should not be described as complete durability proof. + +## Prospective reference correction — 2026-09-16 + +The original test results and source-code commit `da55e78ec53465e1b02aaac1bf69c25c409ed89d` above are unchanged. +The earlier Turtle link named `IMP-002` at that commit, before the artifact existed. +This revision points to its first retained artifact version at `676fa5c6038f92219e902ef126f284360c1763d6`. +Git comparison confirms that only `.spiral/` records changed between those commits. +This is a metadata correction, not a new test run or a claim that the old link was valid. +See `DEF-20260916-TTZ7C-1` for the retained audit and correction scope. diff --git a/.spiral/evidence/EVD-006.ttl b/.spiral/evidence/EVD-006.ttl index 29362a0..539d34a 100644 --- a/.spiral/evidence/EVD-006.ttl +++ b/.spiral/evidence/EVD-006.ttl @@ -10,5 +10,5 @@ project:EVD-006 sd:verifies [ a sd:ArtifactReference ; sd:artifact project:IMP-002 ; - sd:gitCommit "da55e78ec53465e1b02aaac1bf69c25c409ed89d" + sd:gitCommit "676fa5c6038f92219e902ef126f284360c1763d6" ] . diff --git a/.spiral/evidence/EVD-007.md b/.spiral/evidence/EVD-007.md index 7ac24ec..59d9a7c 100644 --- a/.spiral/evidence/EVD-007.md +++ b/.spiral/evidence/EVD-007.md @@ -66,3 +66,12 @@ The new tests cover: ## Evidence quality This evidence covers both direct recovery classification and real process restarts. It specifically addresses automatic restart loops for repeatedly crashing accepted commands, while preserving the intended replay behavior for ordinary accepted commands. + +## Prospective reference correction — 2026-09-16 + +The original test results and source-code commit `0fa4c441e8e8cb27562486c225bb02e8038d564e` above are unchanged. +The earlier Turtle link named `IMP-003` at that commit, before the artifact existed. +This revision points to its first retained artifact version at `c13d08a9e442e30ca4d66f28597939f55ef0a513`. +Git comparison confirms that only `.spiral/` records changed between those commits. +This is a metadata correction, not a new test run or a claim that the old link was valid. +See `DEF-20260916-TTZ7C-1` for the retained audit and correction scope. diff --git a/.spiral/evidence/EVD-007.ttl b/.spiral/evidence/EVD-007.ttl index 3badcec..58fd159 100644 --- a/.spiral/evidence/EVD-007.ttl +++ b/.spiral/evidence/EVD-007.ttl @@ -10,5 +10,5 @@ project:EVD-007 sd:verifies [ a sd:ArtifactReference ; sd:artifact project:IMP-003 ; - sd:gitCommit "0fa4c441e8e8cb27562486c225bb02e8038d564e" + sd:gitCommit "c13d08a9e442e30ca4d66f28597939f55ef0a513" ] . diff --git a/.spiral/evidence/EVD-009.md b/.spiral/evidence/EVD-009.md index 6eb134b..723d888 100644 --- a/.spiral/evidence/EVD-009.md +++ b/.spiral/evidence/EVD-009.md @@ -76,3 +76,12 @@ The process durability tests cover: ## Evidence quality This evidence materially improves durability confidence because each server restart assertion is paired with direct reconstruction from durable command status, command log, base data, and committed changesets. It remains bounded evidence for SimplyStore's current command update lifecycle, not a general production-safety claim. + +## Prospective reference correction — 2026-09-16 + +The original test results and source-code commit `29f016cdb3ec6c3cc03ff9acebac26691a095326` above are unchanged. +The earlier Turtle link named `IMP-004` at that commit, before the artifact existed. +This revision points to its first retained artifact version at `36bb7b1013c2de219d4a730556ae9ab98cea7113`. +Git comparison confirms that only `.spiral/` records changed between those commits. +This is a metadata correction, not a new test run or a claim that the old link was valid. +See `DEF-20260916-TTZ7C-1` for the retained audit and correction scope. diff --git a/.spiral/evidence/EVD-009.ttl b/.spiral/evidence/EVD-009.ttl index 6038a97..01e4192 100644 --- a/.spiral/evidence/EVD-009.ttl +++ b/.spiral/evidence/EVD-009.ttl @@ -10,5 +10,5 @@ project:EVD-009 sd:verifies [ a sd:ArtifactReference ; sd:artifact project:IMP-004 ; - sd:gitCommit "29f016cdb3ec6c3cc03ff9acebac26691a095326" + sd:gitCommit "36bb7b1013c2de219d4a730556ae9ab98cea7113" ] . diff --git a/.spiral/evidence/EVD-010.md b/.spiral/evidence/EVD-010.md index d1f1175..278e6ae 100644 --- a/.spiral/evidence/EVD-010.md +++ b/.spiral/evidence/EVD-010.md @@ -68,3 +68,12 @@ The ACID baseline tests cover: ## Evidence quality The evidence is sufficient to support a developer-facing baseline statement that SimplyStore has executable smoke coverage for non-durability ACID claims. Durability remains the deeper, higher-risk proof area. + +## Prospective reference correction — 2026-09-16 + +The original test results and source-code commit `e2574dd0df70b1803c61f416c537a7f15376b92d` above are unchanged. +The earlier Turtle link named `IMP-005` at that commit, before the artifact existed. +This revision points to its first retained artifact version at `08f53c1d25468e97dbc517b3a6bcd7fb98bf4e5e`. +Git comparison confirms that only `.spiral/` records changed between those commits. +This is a metadata correction, not a new test run or a claim that the old link was valid. +See `DEF-20260916-TTZ7C-1` for the retained audit and correction scope. diff --git a/.spiral/evidence/EVD-010.ttl b/.spiral/evidence/EVD-010.ttl index a485bac..b68aa8c 100644 --- a/.spiral/evidence/EVD-010.ttl +++ b/.spiral/evidence/EVD-010.ttl @@ -10,5 +10,5 @@ project:EVD-010 sd:verifies [ a sd:ArtifactReference ; sd:artifact project:IMP-005 ; - sd:gitCommit "e2574dd0df70b1803c61f416c537a7f15376b92d" + sd:gitCommit "08f53c1d25468e97dbc517b3a6bcd7fb98bf4e5e" ] . diff --git a/.spiral/evidence/EVD-011.md b/.spiral/evidence/EVD-011.md index c927d38..9769512 100644 --- a/.spiral/evidence/EVD-011.md +++ b/.spiral/evidence/EVD-011.md @@ -68,3 +68,12 @@ The new/expanded startup tests cover: ## Evidence quality This evidence is strong for malformed/truncated OD-JSONTag record-framing classification. It deliberately does not overclaim full syntax validation or tamper detection for well-formed altered files; that requires future integrity metadata. + +## Prospective reference correction — 2026-09-16 + +The original test results and source-code commit `7d514568349d6a14ac494155549579533af68bf4` above are unchanged. +The earlier Turtle link named `IMP-006` at that commit, before the artifact existed. +This revision points to its first retained artifact version at `2eeecded67ac99c7ab757113fc8f299af1deafc2`. +Git comparison confirms that only `.spiral/` records changed between those commits. +This is a metadata correction, not a new test run or a claim that the old link was valid. +See `DEF-20260916-TTZ7C-1` for the retained audit and correction scope. diff --git a/.spiral/evidence/EVD-011.ttl b/.spiral/evidence/EVD-011.ttl index 100028e..6178798 100644 --- a/.spiral/evidence/EVD-011.ttl +++ b/.spiral/evidence/EVD-011.ttl @@ -10,5 +10,5 @@ project:EVD-011 sd:verifies [ a sd:ArtifactReference ; sd:artifact project:IMP-006 ; - sd:gitCommit "7d514568349d6a14ac494155549579533af68bf4" + sd:gitCommit "2eeecded67ac99c7ab757113fc8f299af1deafc2" ] . diff --git a/.spiral/evidence/EVD-012.md b/.spiral/evidence/EVD-012.md index cce6981..471001b 100644 --- a/.spiral/evidence/EVD-012.md +++ b/.spiral/evidence/EVD-012.md @@ -66,3 +66,12 @@ The new/expanded tests cover: ## Evidence quality This evidence is strong for command-ID retry behavior within the current server/process harness. It does not prove generalized distributed exactly-once semantics; it proves SimplyStore's local command ID produces at most one committed transition across the tested retry paths. + +## Prospective reference correction — 2026-09-16 + +The original test results and source-code commit `570d35a4fa47db5edf4784c998ee441ce4f20821` above are unchanged. +The earlier Turtle link named `IMP-007` at that commit, before the artifact existed. +This revision points to its first retained artifact version at `28034073cf15170e367fe1bf9ca9846c299d23ea`. +Git comparison confirms that only `.spiral/` records changed between those commits. +This is a metadata correction, not a new test run or a claim that the old link was valid. +See `DEF-20260916-TTZ7C-1` for the retained audit and correction scope. diff --git a/.spiral/evidence/EVD-012.ttl b/.spiral/evidence/EVD-012.ttl index 31d6229..4f093cc 100644 --- a/.spiral/evidence/EVD-012.ttl +++ b/.spiral/evidence/EVD-012.ttl @@ -10,5 +10,5 @@ project:EVD-012 sd:verifies [ a sd:ArtifactReference ; sd:artifact project:IMP-007 ; - sd:gitCommit "570d35a4fa47db5edf4784c998ee441ce4f20821" + sd:gitCommit "28034073cf15170e367fe1bf9ca9846c299d23ea" ] . diff --git a/.spiral/evidence/EVD-014.md b/.spiral/evidence/EVD-014.md index dcae36a..71d6bd5 100644 --- a/.spiral/evidence/EVD-014.md +++ b/.spiral/evidence/EVD-014.md @@ -67,3 +67,12 @@ The new handler tests cover: ## Evidence quality This evidence is strong for current legacy index handler behavior under the server harness. It deliberately preserves uncomfortable findings, especially external derived file escape after failure, so the next contract implementation can address them explicitly. + +## Prospective reference correction — 2026-09-16 + +The original test results and source-code commit `bc376bfc353fe2489ba61a99ca0dc47b38fc1214` above are unchanged. +The earlier Turtle link named `IMP-009` at that commit, before the artifact existed. +This revision points to its first retained artifact version at `313b5b71016067388c542120712082d6023d87d4`. +Git comparison confirms that only `.spiral/` records changed between those commits. +This is a metadata correction, not a new test run or a claim that the old link was valid. +See `DEF-20260916-TTZ7C-1` for the retained audit and correction scope. diff --git a/.spiral/evidence/EVD-014.ttl b/.spiral/evidence/EVD-014.ttl index ef70201..d2148ca 100644 --- a/.spiral/evidence/EVD-014.ttl +++ b/.spiral/evidence/EVD-014.ttl @@ -10,5 +10,5 @@ project:EVD-014 sd:verifies [ a sd:ArtifactReference ; sd:artifact project:IMP-009 ; - sd:gitCommit "bc376bfc353fe2489ba61a99ca0dc47b38fc1214" + sd:gitCommit "313b5b71016067388c542120712082d6023d87d4" ] . diff --git a/.spiral/evidence/EVD-20260819-09ZEF-6.md b/.spiral/evidence/EVD-20260819-09ZEF-6.md new file mode 100644 index 0000000..12f2eaa --- /dev/null +++ b/.spiral/evidence/EVD-20260819-09ZEF-6.md @@ -0,0 +1,69 @@ +--- +id: EVD-20260819-09ZEF-6 +--- + +# Verification Evidence: Well-Formed OD-JSONTag Alteration Is Currently Trusted + +## Claim being checked + +Current SimplyStore startup validation detects malformed OD-JSONTag framing, but does not detect a same-length payload alteration that preserves valid framing. + +## Why this evidence exists + +CYC-20260819-09ZEF-5 is about the first simple integrity-detection behavior for durable OD-JSONTag data. Before changing runtime behavior, Spiral requires current effective behavior and the evidenced gap to be established. + +## Design concern observed + +`DES-001` defines D6 detectable corruption: malformed, missing, truncated, or inconsistent durable artifacts must not silently become valid state. Earlier evidence covered malformed framing. This probe checks a narrower remaining gap: well-formed altered content. + +## Evidence method + +- [x] Automated probe +- [x] Property/invariant check +- [ ] Static analysis +- [ ] Benchmark +- [x] Manual observation +- [ ] Integration exercise +- [ ] Other: + +## Result + +Command run: + +```text +node --input-type=module - <<'NODE' +import assert from 'node:assert/strict' +import JSONTag from '@muze-nl/jsontag' +import Parser from '@muze-nl/od-jsontag/src/parse.mjs' +import serialize from '@muze-nl/od-jsontag/src/serialize.mjs' +import { assertOdJsonTagFraming } from './src/recovery.mjs' + +const original = Buffer.from(serialize(JSONTag.parse('{"persons":[{"name":"Ada"}]}'))) +const altered = Buffer.from(original.toString('utf8').replace('Ada', 'Eve')) +assert.equal(original.length, altered.length) +assertOdJsonTagFraming(altered, 'altered-data.jsontag', 'base OD-JSONTag data') +const data = new Parser().parse(altered) +console.log(JSON.stringify({loadedName: data.persons[0].name, bytesChanged: Buffer.compare(original, altered) !== 0})) +NODE +``` + +Observed output: + +```text +{"loadedName":"Eve","bytesChanged":true} +``` + +Interpretation: + +- The durable bytes changed. +- The OD-JSONTag frame lengths remained valid. +- `assertOdJsonTagFraming()` accepted the altered buffer. +- The parser loaded the altered value as ordinary data. + +## Failure cases / limits + +This is a targeted baseline probe, not a complete corruption matrix. It does not check command status/log tampering, schema tampering, different-length payload alterations, missing integrity metadata, backward compatibility behavior, or automatic recovery. + +## Evidence quality + +The evidence is sufficient to establish a concrete current-behavior gap before product modification: SimplyStore currently trusts well-formed altered OD-JSONTag payload bytes when no stronger integrity metadata exists. diff --git a/.spiral/evidence/EVD-20260819-09ZEF-6.ttl b/.spiral/evidence/EVD-20260819-09ZEF-6.ttl new file mode 100644 index 0000000..cb1f5d7 --- /dev/null +++ b/.spiral/evidence/EVD-20260819-09ZEF-6.ttl @@ -0,0 +1,14 @@ +@prefix sd: . +@prefix dcterms: . +@prefix project: . + +project:EVD-20260819-09ZEF-6 + a sd:VerificationEvidence ; + dcterms:identifier "EVD-20260819-09ZEF-6" ; + sd:repositoryPath ".spiral/evidence/EVD-20260819-09ZEF-6.md" ; + sd:status sd:Active ; + sd:observes [ + a sd:ArtifactReference ; + sd:artifact project:DES-001 ; + sd:gitCommit "85913e982e163c7b05a8e616c00e3f2014e775cc" + ] . diff --git a/.spiral/evidence/EVD-20260819-09ZEF-9.md b/.spiral/evidence/EVD-20260819-09ZEF-9.md new file mode 100644 index 0000000..2992756 --- /dev/null +++ b/.spiral/evidence/EVD-20260819-09ZEF-9.md @@ -0,0 +1,69 @@ +--- +id: EVD-20260819-09ZEF-9 +--- + +# Verification Evidence: Optional Integrity Manifest + +## Claim being verified + +`IMP-20260819-09ZEF-8` realizes `DES-20260819-09ZEF-7`: when durable-file integrity is enabled or a manifest is present, SimplyStore detects altered base data and committed changeset bytes instead of silently loading them as valid state, while preserving existing non-integrity startup behavior. + +## Implementation under test + +| Artifact/path/symbol | Commit | Role | +|---|---|---| +| `IMP-20260819-09ZEF-8` | `7a42ce8c33a3b08c7d6e3af9ccaf7f1f5896a9a4` | Optional integrity manifest implementation. | +| `src/integrity.mjs` | `7a42ce8c33a3b08c7d6e3af9ccaf7f1f5896a9a4` | Digest manifest helpers. | +| `src/load-worker.mjs` | `7a42ce8c33a3b08c7d6e3af9ccaf7f1f5896a9a4` | Startup verification before parsing. | +| `src/command-worker-module.mjs` | `7a42ce8c33a3b08c7d6e3af9ccaf7f1f5896a9a4` | Changeset digest recording before `done`. | + +## Evidence method + +- [x] Automated test +- [x] Property/invariant check +- [x] Static analysis +- [ ] Benchmark +- [ ] Manual observation +- [x] Integration exercise +- [ ] Other: + +## Result + +Commands run after the implementation commit: + +```text +npm test +npm run test:durability +npm run test:acid +npm run test:handlers +npx eslint src/integrity.mjs src/server.mjs src/load-worker.mjs src/command-worker-module.mjs test/durability-startup.mjs test/durability-crash.mjs test/durability-helpers.mjs +npx eslint src test +``` + +Results: + +- `npm test` passed: 19/19 startup durability tests. +- `npm run test:durability` passed: 16/16 full-server durability tests. +- `npm run test:acid` passed: 5/5 ACID baseline tests. +- `npm run test:handlers` passed: 3/3 handler characterization tests. +- Touched-file ESLint passed. +- Whole-repo ESLint still fails with 11 pre-existing errors in unrelated legacy files: `src/index.id.mjs`, `src/index.offset.mjs`, `src/produce.mjs`, and `src/triplestore.mjs`. + +Important assertions now covered: + +- An integrity manifest detects a same-length altered base OD-JSONTag payload. +- An integrity manifest detects a same-length altered committed changeset payload. +- Integrity-enabled loading requires a base data manifest entry. +- A full server command with integrity enabled records the committed changeset digest before `done`, and restart fails explicitly if that committed changeset is later altered. +- Existing startup tests without integrity metadata still pass, preserving default legacy compatibility. + +## Failure cases / limits + +- The manifest itself is not yet chained, signed, or protected from coordinated rewrite. +- Command log/status integrity remains outside this slice. +- Existing datasets need an explicit manifest entry before integrity-required startup succeeds. +- No automatic repair or manifest rebuild tool exists yet. + +## Evidence quality + +This is direct executable evidence for the first narrow integrity behavior and regression evidence for adjacent server/load behavior. It does not prove full tamper resistance; it proves detection of byte changes for manifest-covered base data and committed changesets. diff --git a/.spiral/evidence/EVD-20260819-09ZEF-9.ttl b/.spiral/evidence/EVD-20260819-09ZEF-9.ttl new file mode 100644 index 0000000..dfa39b1 --- /dev/null +++ b/.spiral/evidence/EVD-20260819-09ZEF-9.ttl @@ -0,0 +1,14 @@ +@prefix sd: . +@prefix dcterms: . +@prefix project: . + +project:EVD-20260819-09ZEF-9 + a sd:VerificationEvidence ; + dcterms:identifier "EVD-20260819-09ZEF-9" ; + sd:repositoryPath ".spiral/evidence/EVD-20260819-09ZEF-9.md" ; + sd:status sd:Active ; + sd:verifies [ + a sd:ArtifactReference ; + sd:artifact project:IMP-20260819-09ZEF-8 ; + sd:gitCommit "7a42ce8c33a3b08c7d6e3af9ccaf7f1f5896a9a4" + ] . diff --git a/.spiral/evidence/EVD-20260916-TTZ7C-3.md b/.spiral/evidence/EVD-20260916-TTZ7C-3.md new file mode 100644 index 0000000..e9ec894 --- /dev/null +++ b/.spiral/evidence/EVD-20260916-TTZ7C-3.md @@ -0,0 +1,69 @@ +--- +id: EVD-20260916-TTZ7C-3 +--- + +# Verification: Spiral Compliance Correction + +## Claim and implementation under test + +The process implementation at `adccdd13f4de9c0501a5f6e0c10a2778aa2b0eb8` +realizes `DES-20260916-TTZ7C-2`: exact references and lineage are corrected +prospectively, and historical defects are detected without rewriting history. +`IMP-010` governs the validator; the other verified implementation projections +record metadata corrections and CI adoption. This evidence makes no new claim +about SimplyStore runtime behavior. + +Pinned process: `8d4b2c738a413abd4cccca740ce958f486e5f7af`. +Local integration target: `master` at `9b3bac5541b84420999fbadefeffc3be153e1d17`. +Branch checked: `spiral/CYC-20260819-09ZEF-5-integrity-detection`. + +## Checks and observed results — 2026-09-16 + +Python used the existing `/tmp/simplystore-spiral-venv` environment with rdflib +7.6.0. That environment is local tooling, not a runtime dependency. + +| Check | Result | +|---|---| +| `python test/spiral-provenance.py` | 16 regression tests passed. | +| Node entry point / committed snapshot | 66 Turtle documents, 85 references, no errors. | +| `python scripts/validate-spiral-provenance.py --range master..HEAD` | 10 commits, 22 changed Turtle versions, 41 references, no errors. | +| Staged validation before the implementation commit | 66 documents, 85 references, no errors. | +| `PYTHON= scripts/hooks/pre-commit` | Passed; the optional hook was exercised directly without replacing local hook configuration. | +| Pinned core validation of implementation tree | 66 Turtle files, 744 triples, no errors. | +| Original snapshot `fa329453bf640fab6083e39d57edfdbde222e282` | Failed with exactly the seven known `missing-artifact` errors. | +| `git diff --check` | Passed. | +| `git diff fa329453bf640fab6083e39d57edfdbde222e282 HEAD -- src package.json` | Empty: no runtime source or package changes. | + +The regression histories cover malformed Turtle, custom relation/prefix syntax, +short hashes, missing target versions/paths, unavailable commits, sibling-branch +references, multiple reference targets, staged versus unstaged content, merges, +external references against source-version core pins, and invalid intermediate +versions later repaired or deleted. The raw-history fixture continues to fail +after a later repair; no historical suppression mechanism exists. + +## Integration boundary + +The pinned branch-aware integration validator against the local target rejects +the real candidate with `open-cycle-integration`. This is expected: the correction +reopened the cycle and has not received human acceptance. + +A disposable clone used the same implementation tree with only the cycle status +changed to Accepted, in hypothetical commit +`fbd2f448a9f8e946bc1b048a005f0ec251722f00`. Integration validation passed there +(66 Turtle files, 744 triples). This is a diagnostic fixture only, not acceptance, +a proposed real commit, or a reason to bypass the real integration gate. + +## Limits + +- Human acceptance of the correction is pending. The runtime slice's earlier + acceptance remains historical evidence. +- Remote branch protection and hosted CI execution were not verified. The local + target may need refreshing before actual integration. +- The existing invalid historical versions remain invalid. This is not a claim + that all reachable history is pristine or has been fully audited. +- Runtime test suites were not rerun because runtime code and dependencies did + not change; the earlier runtime test results remain historical claims. +- Lineage ownership still needs human/agent review; the checker cannot infer it + merely from overlapping source paths. +- This evidence was added after testing the implementation commit; its own + references are validated before the evidence commit and again at the final tip. diff --git a/.spiral/evidence/EVD-20260916-TTZ7C-3.ttl b/.spiral/evidence/EVD-20260916-TTZ7C-3.ttl new file mode 100644 index 0000000..09e16c2 --- /dev/null +++ b/.spiral/evidence/EVD-20260916-TTZ7C-3.ttl @@ -0,0 +1,39 @@ +@prefix sd: . +@prefix dcterms: . +@prefix project: . + +project:EVD-20260916-TTZ7C-3 + a sd:VerificationEvidence ; + dcterms:identifier "EVD-20260916-TTZ7C-3" ; + sd:repositoryPath ".spiral/evidence/EVD-20260916-TTZ7C-3.md" ; + sd:status sd:Active ; + sd:verifies [ + a sd:ArtifactReference ; + sd:artifact project:IMP-010 ; + sd:gitCommit "adccdd13f4de9c0501a5f6e0c10a2778aa2b0eb8" + ] ; + sd:verifies [ + a sd:ArtifactReference ; + sd:artifact project:IMP-001 ; + sd:gitCommit "adccdd13f4de9c0501a5f6e0c10a2778aa2b0eb8" + ] ; + sd:verifies [ + a sd:ArtifactReference ; + sd:artifact project:IMP-004 ; + sd:gitCommit "adccdd13f4de9c0501a5f6e0c10a2778aa2b0eb8" + ] ; + sd:verifies [ + a sd:ArtifactReference ; + sd:artifact project:IMP-006 ; + sd:gitCommit "adccdd13f4de9c0501a5f6e0c10a2778aa2b0eb8" + ] ; + sd:verifies [ + a sd:ArtifactReference ; + sd:artifact project:IMP-20260819-09ZEF-8 ; + sd:gitCommit "adccdd13f4de9c0501a5f6e0c10a2778aa2b0eb8" + ] ; + sd:verifies [ + a sd:ArtifactReference ; + sd:artifact project:IMP-20260818-09ZEF-1 ; + sd:gitCommit "adccdd13f4de9c0501a5f6e0c10a2778aa2b0eb8" + ] . diff --git a/.spiral/implementations/IMP-001.md b/.spiral/implementations/IMP-001.md index c3da4de..14dffe6 100644 --- a/.spiral/implementations/IMP-001.md +++ b/.spiral/implementations/IMP-001.md @@ -67,3 +67,13 @@ No new package dependencies. - Cross-file inconsistency classes beyond missing committed changesets and malformed status/log records remain future work. > Record current/effective causal links in the companion Turtle resource. For a material revision of a governed implementation, also record `sd:transforms`, `sd:changeCausedBy`, and `sd:implementationChangeKind` to exact historical versions. + +## Prospective compliance correction — 2026-09-16 + +The current path also implements `DES-20260819-09ZEF-7`'s optional integrity +checks: durable bytes are verified before loading, and a changeset digest is +recorded before the pre-done boundary when integrity is enabled. The prior +durability design remains an effective cause. This metadata-only revision links +the immediate predecessor and `DES-20260916-TTZ7C-2` as its correction cause; +it records the already-existing integrity runtime change prospectively. It does +not imply that the original integrity commit carried this lineage. diff --git a/.spiral/implementations/IMP-001.ttl b/.spiral/implementations/IMP-001.ttl index d3bcc52..6d3e64b 100644 --- a/.spiral/implementations/IMP-001.ttl +++ b/.spiral/implementations/IMP-001.ttl @@ -20,14 +20,19 @@ project:IMP-001 sd:artifact project:DES-001 ; sd:gitCommit "85913e982e163c7b05a8e616c00e3f2014e775cc" ] ; + sd:implements [ + a sd:ArtifactReference ; + sd:artifact project:DES-20260819-09ZEF-7 ; + sd:gitCommit "471e28e350e8095adc2f956d16389790e47efdd0" + ] ; sd:transforms [ a sd:ArtifactReference ; sd:artifact project:IMP-001 ; - sd:gitCommit "dc0fb8a9537233f0ddc9ff75092b9a49b0edf2ba" + sd:gitCommit "68c38971e11a1eb81a34e7f1da1eb6f18019a3c4" ] ; sd:changeCausedBy [ a sd:ArtifactReference ; - sd:artifact project:CYC-016 ; - sd:gitCommit "36fc816a6357b57f99843359aeb4a2329677d6c1" + sd:artifact project:DES-20260916-TTZ7C-2 ; + sd:gitCommit "f66ea74b740ec9ea17598e0212f0daa53fcd81c0" ] ; sd:implementationChangeKind sd:BehaviorPreservingChange . diff --git a/.spiral/implementations/IMP-004.md b/.spiral/implementations/IMP-004.md index 7b6e4e3..eed3adb 100644 --- a/.spiral/implementations/IMP-004.md +++ b/.spiral/implementations/IMP-004.md @@ -54,3 +54,13 @@ The durability test suite still starts local SimplyStore server processes and bi - It does not define public documentation for every durable status. - It does not fully classify duplicate conflicting command-log records. - It does not prove non-deterministic command handler replay safety. + +## Prospective compliance correction — 2026-09-16 + +The current path also implements `DES-20260819-09ZEF-7`'s optional integrity +checks: durable bytes are verified before loading, and a changeset digest is +recorded before the pre-done boundary when integrity is enabled. The prior +durability design remains an effective cause. This metadata-only revision links +the immediate predecessor and `DES-20260916-TTZ7C-2` as its correction cause; +it records the already-existing integrity runtime change prospectively. It does +not imply that the original integrity commit carried this lineage. diff --git a/.spiral/implementations/IMP-004.ttl b/.spiral/implementations/IMP-004.ttl index 7c681f3..f16ed07 100644 --- a/.spiral/implementations/IMP-004.ttl +++ b/.spiral/implementations/IMP-004.ttl @@ -18,14 +18,19 @@ project:IMP-004 sd:artifact project:DES-001 ; sd:gitCommit "85913e982e163c7b05a8e616c00e3f2014e775cc" ] ; + sd:implements [ + a sd:ArtifactReference ; + sd:artifact project:DES-20260819-09ZEF-7 ; + sd:gitCommit "471e28e350e8095adc2f956d16389790e47efdd0" + ] ; sd:transforms [ a sd:ArtifactReference ; sd:artifact project:IMP-004 ; - sd:gitCommit "36bb7b1013c2de219d4a730556ae9ab98cea7113" + sd:gitCommit "68c38971e11a1eb81a34e7f1da1eb6f18019a3c4" ] ; sd:changeCausedBy [ a sd:ArtifactReference ; - sd:artifact project:CYC-016 ; - sd:gitCommit "36fc816a6357b57f99843359aeb4a2329677d6c1" + sd:artifact project:DES-20260916-TTZ7C-2 ; + sd:gitCommit "f66ea74b740ec9ea17598e0212f0daa53fcd81c0" ] ; sd:implementationChangeKind sd:BehaviorPreservingChange . diff --git a/.spiral/implementations/IMP-006.md b/.spiral/implementations/IMP-006.md index 5e6fe13..e55566c 100644 --- a/.spiral/implementations/IMP-006.md +++ b/.spiral/implementations/IMP-006.md @@ -47,3 +47,13 @@ No new package dependencies. - Well-formed malicious changeset alteration is not detectable without future integrity metadata. - The validator does not prove that changeset content corresponds to the command that produced it. - No cleanup/reset/repair tooling is added. + +## Prospective compliance correction — 2026-09-16 + +The current path also implements `DES-20260819-09ZEF-7`'s optional integrity +checks: durable bytes are verified before loading, and a changeset digest is +recorded before the pre-done boundary when integrity is enabled. The prior +durability design remains an effective cause. This metadata-only revision links +the immediate predecessor and `DES-20260916-TTZ7C-2` as its correction cause; +it records the already-existing integrity runtime change prospectively. It does +not imply that the original integrity commit carried this lineage. diff --git a/.spiral/implementations/IMP-006.ttl b/.spiral/implementations/IMP-006.ttl index 8cc1316..b603eb6 100644 --- a/.spiral/implementations/IMP-006.ttl +++ b/.spiral/implementations/IMP-006.ttl @@ -15,14 +15,19 @@ project:IMP-006 sd:artifact project:DES-001 ; sd:gitCommit "85913e982e163c7b05a8e616c00e3f2014e775cc" ] ; + sd:implements [ + a sd:ArtifactReference ; + sd:artifact project:DES-20260819-09ZEF-7 ; + sd:gitCommit "471e28e350e8095adc2f956d16389790e47efdd0" + ] ; sd:transforms [ a sd:ArtifactReference ; sd:artifact project:IMP-006 ; - sd:gitCommit "2eeecded67ac99c7ab757113fc8f299af1deafc2" + sd:gitCommit "68c38971e11a1eb81a34e7f1da1eb6f18019a3c4" ] ; sd:changeCausedBy [ a sd:ArtifactReference ; - sd:artifact project:CYC-016 ; - sd:gitCommit "36fc816a6357b57f99843359aeb4a2329677d6c1" + sd:artifact project:DES-20260916-TTZ7C-2 ; + sd:gitCommit "f66ea74b740ec9ea17598e0212f0daa53fcd81c0" ] ; sd:implementationChangeKind sd:BehaviorPreservingChange . diff --git a/.spiral/implementations/IMP-010.md b/.spiral/implementations/IMP-010.md index 3bacbff..62661a3 100644 --- a/.spiral/implementations/IMP-010.md +++ b/.spiral/implementations/IMP-010.md @@ -2,61 +2,53 @@ id: IMP-010 --- -# Implementation: Causal Provenance Graph Repair And Validator +# Implementation: Causal Provenance Repair And Validation -## Scope +## Scope and effective provenance -Process-only provenance repair for the Spiral graph plus a repository-local validator that mechanically checks representative causal traversals. +Preserve the CYC-016 causal repair and extend its validator under +`DES-20260916-TTZ7C-2`. The implementation is development tooling; SimplyStore +runtime behavior and package dependencies are unchanged. ## Observable behavior -- Current effective provenance for representative durability and handler artifacts can be traversed from implementation to design, request, understanding, and retained human source. -- Historical implementation lineage remains separate through `sd:transforms` and `sd:changeCausedBy`. -- Machine-readable historical references use full 40-character Git commit hashes. -- `.spiral/evidence/EVD-008.ttl` no longer contains a shortened Git hash. -- `node scripts/validate-spiral-provenance.mjs` parses the repository's current Spiral Turtle subset, validates historical Git references, checks strict ancestry, checks branch/cycle consistency, and demonstrates the requested traversal examples. -- After cycle acceptance, the validator accepts CYC-016 as either the current active cycle or the latest accepted cycle in project context. +- `scripts/validate-spiral-provenance.py` parses RDF with rdflib and checks exact + target definitions, paths, full hashes, and admissible ancestry. +- `scripts/validate-spiral-provenance.mjs` preserves the Node entry point and + delegates to that parser; `PYTHON` selects the interpreter when necessary. +- Default / `--tree` validates committed snapshots; `--staged` reads the index; + `--range base..head` audits introduced artifact versions, even if later removed. +- `--range HEAD` audits all reachable history without suppressing known violations. +- External culture/profile references resolve through the source version's + pinned core repository; process Turtle is not part of the project graph. +- `test/spiral-provenance.py` exercises the failure modes with disposable Git histories. +- `scripts/hooks/pre-commit` provides opt-in local prevention. CI runs range and + snapshot validation alongside the pinned core integration validator. ## Repository locations -| Path / symbol | Role | +| Path | Responsibility | |---|---| -| `.spiral/sources/SRC-001.*` | Prospective retained-source version for the repaired effective provenance chain. | -| `.spiral/understandings/UND-001.*` | Prospective understanding version linked to the retained source. | -| `.spiral/requests/REQ-001.*` | Prospective request version linked to the understanding. | -| `.spiral/designs/DES-001.*` | Prospective durability design version linked to the request. | -| `.spiral/designs/DES-002.*` | After-change design version linked to the repaired request/design chain. | -| `.spiral/implementations/IMP-001.ttl` | Current effective provenance projection for startup recovery. | -| `.spiral/implementations/IMP-002.ttl` | Current effective provenance projection for awaited durable appends. | -| `.spiral/implementations/IMP-003.ttl` | Current effective provenance projection for active command replay safety. | -| `.spiral/implementations/IMP-004.ttl` | Current effective provenance projection for commit-before-query visibility. | -| `.spiral/implementations/IMP-006.ttl` | Current effective provenance projection for malformed durable framing failures. | -| `.spiral/implementations/IMP-007.ttl` | Current effective provenance projection for retry/idempotency behavior. | -| `.spiral/implementations/IMP-009.ttl` | Current effective provenance projection for after-change/index characterization. | -| `.spiral/evidence/EVD-008.*` | Full-hash repair for existing Spiral guardrail evidence. | -| `scripts/validate-spiral-provenance.mjs` | Mechanical validation and traversal script for this repair cycle. | - -## Effective provenance - -This implementation realizes `CYC-016` as a current effective causal interpretation of why the existing durability and handler implementation concerns remain present. - -It does not claim that the original code was historically produced from graph links that did not exist at the time. Historical/change provenance is preserved separately with `sd:transforms`, `sd:changeCausedBy`, and existing verification evidence. - -This revision also corrects the validator's branch/cycle consistency check so CYC-016 can validate after human acceptance marks it accepted rather than active. - -## Important implementation decisions - -- No SimplyStore product/runtime code changed. -- The early source, understanding, request, and design artifacts were versioned in separate commits so strict-ancestor references can be expressed honestly. -- Representative current implementation projections were updated rather than rewriting historical artifacts. -- The validator is dependency-free and repository-local so it can run in the current project without adding package dependencies. - -## New dependencies / capabilities / permissions - -No new package dependencies. - -## Known limits - -- The validator is a repository-local parser for the current Spiral Turtle subset, not a general standards-conformant Turtle parser. -- The repair covers representative durability and after-change paths required by `CYC-016`; it does not attempt to make every historical or process artifact answer every possible intent query. -- Some remaining provenance questions still require reading artifact prose, especially where the graph distinguishes current effective justification from historical transition causes but does not encode a single artificial linear cause. +| `.spiral/evidence/EVD-{006,007,009,010,011,012,014}.*` | Prospective repair retaining historical code/test claims. | +| `.spiral/implementations/IMP-*.ttl` | Current effective causes, lineage and transitions. | +| `scripts/validate-spiral-provenance.py` | Standards-based reference validation. | +| `scripts/validate-spiral-provenance.mjs` | Existing Node entry point. | +| `scripts/hooks/pre-commit` | Optional index validation hook. | +| `test/spiral-provenance.py` | Executable regression evidence. | +| `docs/spiral-validation.md` | Usage and limitations. | +| `.github/workflows/spiral-integration.yml` | Required-check implementation (hosting enforcement remains external). | + +## Lineage and correction meaning + +The Turtle projection retains the immediate predecessor and the correction +design as transition cause. The validator's semantics change; the product's +semantics do not. Old partial-parser claims remain historical evidence and are +superseded by this implementation. No old commit is amended or relabelled valid. + +## Dependencies and limits + +rdflib is already required by the pinned Spiral core; no runtime dependency was +added. The validator does not replace the core coherence/integration checks or +human judgment about intent, implementation ownership, or acceptance. There are +no SHACL shapes in the pinned core. Old invalid references remain visible to raw +history audits. Current validity does not imply pristine history. diff --git a/.spiral/implementations/IMP-010.ttl b/.spiral/implementations/IMP-010.ttl index 24a767c..5e4fde4 100644 --- a/.spiral/implementations/IMP-010.ttl +++ b/.spiral/implementations/IMP-010.ttl @@ -7,6 +7,11 @@ project:IMP-010 dcterms:identifier "IMP-010" ; sd:repositoryPath ".spiral/implementations/IMP-010.md" ; sd:status sd:Active ; + sd:implementationLocation ".github/workflows/spiral-integration.yml" ; + sd:implementationLocation "docs/spiral-validation.md" ; + sd:implementationLocation "test/spiral-provenance.py" ; + sd:implementationLocation "scripts/hooks/pre-commit" ; + sd:implementationLocation "scripts/validate-spiral-provenance.py" ; sd:implementationLocation ".spiral/sources/SRC-001.md" ; sd:implementationLocation ".spiral/sources/SRC-001.ttl" ; sd:implementationLocation ".spiral/understandings/UND-001.md" ; @@ -27,14 +32,24 @@ project:IMP-010 sd:implementationLocation ".spiral/evidence/EVD-008.md" ; sd:implementationLocation ".spiral/evidence/EVD-008.ttl" ; sd:implementationLocation "scripts/validate-spiral-provenance.mjs" ; + sd:derivedFrom [ + a sd:ArtifactReference ; + sd:artifact project:CYC-016 ; + sd:gitCommit "fa329453bf640fab6083e39d57edfdbde222e282" + ] ; + sd:implements [ + a sd:ArtifactReference ; + sd:artifact project:DES-20260916-TTZ7C-2 ; + sd:gitCommit "f66ea74b740ec9ea17598e0212f0daa53fcd81c0" + ] ; sd:transforms [ a sd:ArtifactReference ; sd:artifact project:IMP-010 ; - sd:gitCommit "2006532235a6e6fb7366747026d6849827fcad4a" + sd:gitCommit "b26e30f0bc67e96744a9fb18aff0ceaf23e3ec53" ] ; sd:changeCausedBy [ a sd:ArtifactReference ; - sd:artifact project:CYC-016 ; - sd:gitCommit "36fc816a6357b57f99843359aeb4a2329677d6c1" + sd:artifact project:DES-20260916-TTZ7C-2 ; + sd:gitCommit "f66ea74b740ec9ea17598e0212f0daa53fcd81c0" ] ; - sd:implementationChangeKind sd:BehaviorPreservingChange . + sd:implementationChangeKind sd:SemanticChange . diff --git a/.spiral/implementations/IMP-20260818-09ZEF-1.md b/.spiral/implementations/IMP-20260818-09ZEF-1.md index 6bb9d8f..d040bdb 100644 --- a/.spiral/implementations/IMP-20260818-09ZEF-1.md +++ b/.spiral/implementations/IMP-20260818-09ZEF-1.md @@ -63,3 +63,13 @@ This revision also repairs the first implementation version's accidental incompa - The GitHub workflow is only effective as a merge gate after repository hosting settings require it. - Existing repository-local `scripts/validate-spiral-provenance.mjs` remains CYC-016-specific. - `node .spiral-core/bin/spiral.mjs validate` remains unsuitable locally until upstream excludes nested submodule Turtle files or gains a project-root selector. + +## Prospective compliance correction — 2026-09-16 + +The CYC-018 integration workflow now also runs the standards-based local +historical-reference validator and its regression tests. The pinned core check +remains responsible for identity, coherence, cycle acceptance, and convergence. +`DES-20260916-TTZ7C-2` governs this extension; the immediate previous adoption +projection is preserved through `sd:transforms`. This supersedes the old local +validator limitation above: the entry point now delegates to rdflib and no +longer assumes CYC-016 or rejects vocabulary documents. diff --git a/.spiral/implementations/IMP-20260818-09ZEF-1.ttl b/.spiral/implementations/IMP-20260818-09ZEF-1.ttl index caa2bc6..bf0eda6 100644 --- a/.spiral/implementations/IMP-20260818-09ZEF-1.ttl +++ b/.spiral/implementations/IMP-20260818-09ZEF-1.ttl @@ -13,14 +13,24 @@ project:IMP-20260818-09ZEF-1 sd:implementationLocation ".github/workflows/spiral-integration.yml" ; sd:implementationLocation ".spiral/vocabulary.ttl" ; sd:implementationLocation "scripts/validate-spiral-provenance.mjs" ; + sd:derivedFrom [ + a sd:ArtifactReference ; + sd:artifact project:CYC-018 ; + sd:gitCommit "fa329453bf640fab6083e39d57edfdbde222e282" + ] ; + sd:implements [ + a sd:ArtifactReference ; + sd:artifact project:DES-20260916-TTZ7C-2 ; + sd:gitCommit "f66ea74b740ec9ea17598e0212f0daa53fcd81c0" + ] ; sd:transforms [ a sd:ArtifactReference ; sd:artifact project:IMP-20260818-09ZEF-1 ; - sd:gitCommit "89376889923eb10bc8b431cd22e422797c9eb5ca" + sd:gitCommit "725a861ccb577fa5918202d1f486b2e19d02c896" ] ; sd:changeCausedBy [ a sd:ArtifactReference ; - sd:artifact project:CYC-018 ; - sd:gitCommit "4cefd007da682ce8ef46ccfc4852882fb34e2003" + sd:artifact project:DES-20260916-TTZ7C-2 ; + sd:gitCommit "f66ea74b740ec9ea17598e0212f0daa53fcd81c0" ] ; sd:implementationChangeKind sd:SemanticChange . diff --git a/.spiral/implementations/IMP-20260819-09ZEF-8.md b/.spiral/implementations/IMP-20260819-09ZEF-8.md new file mode 100644 index 0000000..859b240 --- /dev/null +++ b/.spiral/implementations/IMP-20260819-09ZEF-8.md @@ -0,0 +1,62 @@ +--- +id: IMP-20260819-09ZEF-8 +--- + +# Implementation: Optional Durable File Integrity Manifest + +## Scope + +Implement the first narrow integrity-detection mechanism for durable OD-JSONTag data. + +## Observable behavior + +- `src/integrity.mjs` provides SHA-256 digest helpers, append-readable integrity manifest parsing, digest record appending, and explicit mismatch/missing-entry failures. +- `src/server.mjs` enables integrity verification when `options.integrity`, `options.integrityFile`, or the default integrity sidecar file is present. +- `src/load-worker.mjs` verifies base data and committed changesets against the manifest before OD-JSONTag framing validation and parsing. +- `src/command-worker-module.mjs` appends a changeset digest after the changeset file is written and before the existing post-changeset fault point / `done` status path. +- Startup tests cover same-length payload alteration for base data and committed changesets, plus required missing manifest entries. +- Full-server durability tests cover an integrity-enabled command writing the changeset digest before `done`, then refusing restart after the committed changeset is altered. + +## Repository locations + +| Path / symbol | Role | +|---|---| +| `src/integrity.mjs` | Integrity manifest read/write/verify helpers. | +| `src/server.mjs` | Integrity option/default sidecar selection and worker task wiring. | +| `src/load-worker.mjs` | Startup verification before durable bytes become parsed state. | +| `src/command-worker-module.mjs` | Changeset digest recording before command commit. | +| `test/durability-startup.mjs` | Unit/startup integrity cases. | +| `test/durability-crash.mjs` | Full-server committed changeset integrity case. | +| `test/durability-helpers.mjs` | Test runner option wiring. | + +## Effective provenance + +This implementation realizes `DES-20260819-09ZEF-7`. + +## Important implementation decisions + +- Existing datasets without an integrity manifest remain loadable by default. +- The default sidecar path is derived from the data file, for example `data.integrity.jsontag` for `data.jsontag`. +- Manifest entries store paths relative to the manifest directory so the file stays inspectable and movable with its dataset directory. +- Manifest records are append-readable and latest-wins by file key, matching SimplyStore's existing simple log style. +- The first integrity scope is base data plus committed changeset files only. Command log/status integrity remains out of scope. + +## New dependencies / capabilities / permissions + +No package dependency was added. SHA-256 uses Node's built-in `crypto` module. + +## Known limits + +- Enabling integrity for an existing dataset requires a manifest entry for the base data before startup succeeds. +- The first mechanism detects byte changes to files that have manifest entries; it does not attempt recovery or prove command log/status authenticity. +- The sidecar manifest itself is not yet chained or signed, so a malicious actor able to rewrite both data and manifest can still hide alteration. + +## Prospective compliance correction — 2026-09-16 + +This metadata-only revision retains the integrity design as the effective cause, +records its own immediate predecessor, and makes explicit the governed recovery, +changeset commit, and framing-validation units extended by the original integrity +slice (`IMP-001`, `IMP-004`, `IMP-006`). Their still-valid concerns remain separately +governed. The correction does not claim that these links existed in the original +implementation commit. Runtime behavior is unchanged; the original semantic +change remains visible in the predecessor. diff --git a/.spiral/implementations/IMP-20260819-09ZEF-8.ttl b/.spiral/implementations/IMP-20260819-09ZEF-8.ttl new file mode 100644 index 0000000..26d365c --- /dev/null +++ b/.spiral/implementations/IMP-20260819-09ZEF-8.ttl @@ -0,0 +1,47 @@ +@prefix sd: . +@prefix dcterms: . +@prefix project: . + +project:IMP-20260819-09ZEF-8 + a sd:Implementation ; + dcterms:identifier "IMP-20260819-09ZEF-8" ; + sd:repositoryPath ".spiral/implementations/IMP-20260819-09ZEF-8.md" ; + sd:status sd:Active ; + sd:implementationLocation "src/integrity.mjs" ; + sd:implementationLocation "src/server.mjs" ; + sd:implementationLocation "src/load-worker.mjs" ; + sd:implementationLocation "src/command-worker-module.mjs" ; + sd:implementationLocation "test/durability-startup.mjs" ; + sd:implementationLocation "test/durability-crash.mjs" ; + sd:implementationLocation "test/durability-helpers.mjs" ; + sd:implements [ + a sd:ArtifactReference ; + sd:artifact project:DES-20260819-09ZEF-7 ; + sd:gitCommit "471e28e350e8095adc2f956d16389790e47efdd0" + ] ; + sd:transforms [ + a sd:ArtifactReference ; + sd:artifact project:IMP-20260819-09ZEF-8 ; + sd:gitCommit "7a42ce8c33a3b08c7d6e3af9ccaf7f1f5896a9a4" + ] ; + sd:transforms [ + a sd:ArtifactReference ; + sd:artifact project:IMP-001 ; + sd:gitCommit "68c38971e11a1eb81a34e7f1da1eb6f18019a3c4" + ] ; + sd:transforms [ + a sd:ArtifactReference ; + sd:artifact project:IMP-004 ; + sd:gitCommit "68c38971e11a1eb81a34e7f1da1eb6f18019a3c4" + ] ; + sd:transforms [ + a sd:ArtifactReference ; + sd:artifact project:IMP-006 ; + sd:gitCommit "68c38971e11a1eb81a34e7f1da1eb6f18019a3c4" + ] ; + sd:changeCausedBy [ + a sd:ArtifactReference ; + sd:artifact project:DES-20260916-TTZ7C-2 ; + sd:gitCommit "f66ea74b740ec9ea17598e0212f0daa53fcd81c0" + ] ; + sd:implementationChangeKind sd:BehaviorPreservingChange . diff --git a/.spiral/project-context.md b/.spiral/project-context.md index fbbc043..63ded97 100644 --- a/.spiral/project-context.md +++ b/.spiral/project-context.md @@ -18,7 +18,7 @@ Project causal-graph namespace: `https://github.com/simplyedit/simplystore/spira Spiral core source: `.spiral-core/`, git submodule for `https://github.com/muze-labs/spiral-developer.git`, currently checked out at `8d4b2c738a413abd4cccca740ce958f486e5f7af`. -Current active Spiral cycle: `.spiral/cycles/CYC-018.md` (`Spiral Core Process Update`). Latest accepted Spiral cycle: `.spiral/cycles/CYC-017.md` (`Command And Load Worker Timeout Semantics`). +Current active Spiral cycle: none. Latest accepted cycle: `.spiral/cycles/CYC-20260819-09ZEF-5.md` (`Integrity Detection`). The runtime slice was accepted on 2026-08-19; the compliance correction in `DEF-20260916-TTZ7C-1`, verified by `EVD-20260916-TTZ7C-3`, was explicitly accepted on 2026-09-16. Integration must still be validated against the current target. ## Intake State @@ -60,6 +60,10 @@ Local validator dependency: the current Spiral CLI requires Python `rdflib` from Local validator limitation: `node .spiral-core/bin/spiral.mjs validate` currently scans the `.spiral-core` submodule's own Turtle files and reports duplicate legacy artifact identifiers. Use prospective tree/integration validation for SimplyStore until upstream validator behavior excludes nested process repositories. +Historical-reference validation: `scripts/validate-spiral-provenance.py` uses rdflib and complements the pinned core. Run `--staged` before semantic commits, and `--range ..` plus `--tree ` for integration. The Node entry point delegates to the same implementation. CI covers both introduced versions and the candidate snapshot. See `docs/spiral-validation.md` for the optional local hook and explicit external process-repository resolution. + +Known historical defects: `DEF-20260916-TTZ7C-1` records seven evidence links that originally named artifacts before their creation. Current references are corrected prospectively; raw history audits must continue reporting the invalid old versions. + Local vocabulary: `.spiral/vocabulary.ttl` carries the minimal Spiral relation hierarchy needed for tree-based validation of this consuming repository. Without it, integration validation sees the `.spiral-core` submodule only as a Git gitlink and cannot derive `sd:causalReference` subproperties from the core ontology. Artifact allocation: new Spiral artifacts after CYC-018 should use distributed-safe IDs allocated by: @@ -68,7 +72,7 @@ Artifact allocation: new Spiral artifacts after CYC-018 should use distributed-s node .spiral-core/bin/spiral.mjs allocate ``` -Legacy sequential artifact IDs remain valid and should not be renamed. CYC-018 itself was opened with the legacy sequential ID before this core update was adopted. This checkout has worktree-local allocator namespace `09ZEF` under `.git/spiral`; that namespace is private local allocation state, not shared project content and not actor identity. +Legacy sequential artifact IDs remain valid and should not be renamed. CYC-018 itself was opened with the legacy sequential ID before this core update was adopted. Allocator state is private to each checkout under `.git/spiral`; inspect it with `spiral status`. The historical `09ZEF` artifacts keep their IDs. This checkout allocated the 2026-09-16 corrections in namespace `TTZ7C`; neither namespace is actor identity. ## Intended Users diff --git a/AGENTS.md b/AGENTS.md index bed8aa4..e8351d7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -55,6 +55,8 @@ Do not create Spiral artifacts merely to populate folders. Add `.spiral/` artifa - An Active cycle branch must not merge into `master` or another Active cycle branch. Record human acceptance by changing the cycle to `sd:Accepted` before proposing integration. - Before proposing or merging accepted cycle work, run `node .spiral-core/bin/spiral.mjs validate integration --base --head --base-branch master --head-branch `, or rely on a hosting check that validates the exact prospective merged result with branch metadata. - When accepted `master` work is merged into an Active cycle branch, prefer `git merge --no-commit master` if governed artifacts may have changed on both sides. If both parent histories materially changed the same governed artifact, reconcile the merge version and record both immediate predecessor versions with `sd:transforms`. +- Before every semantic causal commit, run `python3 scripts/validate-spiral-provenance.py --staged` (or the Node entry point with `PYTHON` set) and `git diff --cached --check`. The optional hook in `scripts/hooks/pre-commit` runs staged validation; see `docs/spiral-validation.md` before installing it. +- Before integration, also run `python3 scripts/validate-spiral-provenance.py --range ..` and `--tree `. This complements the core integration check with target-existence and per-commit historical-reference validation. Old corrected violations must remain visible in raw history audits. - The Spiral validator requires Python `rdflib` from `.spiral-core/requirements.txt`. Local worktree validation currently sees `.spiral-core`'s own Turtle files because the process is installed as a submodule, so prefer tree/prospective-integration validation until upstream validator behavior excludes nested process repositories. - Treat planning, evaluation, and ambiguous human suggestions as discourse until there is an explicit commitment. Do not turn tentative comments into scope or architecture merely because they are implementable. - Treat human confirmation of a sufficiently explicit cycle goal as the commitment boundary for execution within that goal and its non-goals. diff --git a/docs/spiral-validation.md b/docs/spiral-validation.md new file mode 100644 index 0000000..c2e4fdb --- /dev/null +++ b/docs/spiral-validation.md @@ -0,0 +1,63 @@ +# Spiral provenance validation + +The pinned core checks graph coherence and prospective integration. The local +validator adds exact target existence and historical ancestry checks using +Python rdflib from `.spiral-core/requirements.txt`. + +Initialize the pinned process with `git submodule update --init .spiral-core`, +then use a Python environment with those requirements installed. The existing +Node entry point delegates to Python; set `PYTHON` if it is not `python3` on PATH. + +```sh +python3 scripts/validate-spiral-provenance.py # committed HEAD +python3 scripts/validate-spiral-provenance.py --staged # Git index, pre-commit +python3 scripts/validate-spiral-provenance.py --tree HEAD +python3 scripts/validate-spiral-provenance.py --range master..HEAD +python3 test/spiral-provenance.py +node scripts/validate-spiral-provenance.mjs --staged +``` + +Default validation inspects committed HEAD, not unstaged edits. Stage a proposed +change before `--staged`; references in new/changed staged Turtle may target HEAD, +which becomes an ancestor when committed. Committed references must point to a +strict ancestor of their source artifact version. Every target must have an +artifact definition and an existing repository path at its exact hash. + +The project graph is `.spiral/**/*.ttl`; vocabulary files are valid RDF documents +and the process submodule is not merged into this graph. External Muze culture +and Spiral profile references are resolved through `.spiral-core` at the hash +recorded in the source version's gitlink. That checkout must contain the referenced +objects. No network requests or dependency installation happen during validation. + +## Before each semantic causal commit + +Run `--staged` and `git diff --cached --check`. An optional local hook does the +first check automatically. Inspect existing hooks/configuration before enabling +it; compose with existing hooks rather than overwriting another workflow. + +```sh +git config --get core.hooksPath +# When no existing hook setup needs preserving: +git config core.hooksPath scripts/hooks +``` + +Ensure rdflib is available to the Python interpreter used by the hook (`PYTHON` +or `python3`). Hook configuration belongs to the local checkout and is not shared. + +## Integration and history + +CI runs the regression tests, checks all introduced Turtle versions, and validates +the candidate snapshot. The pinned core's branch-aware integration check remains +separate and must still pass immediately before integration. Active cycles must +remain blocked until the human accepts them. Revalidate if the target moves. + +To audit all retained history, run `--range HEAD`. It intentionally reports old +violations even after later correction. In particular the seven invalid evidence +references documented in `DEF-20260916-TTZ7C-1` remain historical failures. A green +current snapshot is not a claim of pristine history; there is no allowlist that +silences old invalid references. + +The local checker does not infer whether a changed implementation needs new +lineage, confirm human intent/acceptance, or prove runtime claims. Those remain +review responsibilities. The pinned core provides identity/coherence/convergence +checks; no SHACL shapes are shipped in this pinned installation. diff --git a/scripts/hooks/pre-commit b/scripts/hooks/pre-commit new file mode 100755 index 0000000..80505e6 --- /dev/null +++ b/scripts/hooks/pre-commit @@ -0,0 +1,5 @@ +#!/bin/sh +# Opt in with git config core.hooksPath scripts/hooks after checking existing hooks. +set -eu +spiral_repo=$(git rev-parse --show-toplevel) +exec "${PYTHON:-python3}" "$spiral_repo/scripts/validate-spiral-provenance.py" --repo "$spiral_repo" --staged diff --git a/scripts/validate-spiral-provenance.mjs b/scripts/validate-spiral-provenance.mjs index e2c8c8e..8e4e12a 100644 --- a/scripts/validate-spiral-provenance.mjs +++ b/scripts/validate-spiral-provenance.mjs @@ -1,343 +1,12 @@ -import fs from 'node:fs' -import path from 'node:path' import { spawnSync } from 'node:child_process' +import { fileURLToPath } from 'node:url' -const root = process.cwd() -const ttlRoot = path.join(root, '.spiral') -const projectPrefix = 'https://github.com/simplyedit/simplystore/spiral#' - -function runGit(args, options = {}) { - const result = spawnSync('git', args, { - cwd: root, - encoding: 'utf8', - ...options - }) - if (result.status !== 0) { - throw new Error(`git ${args.join(' ')} failed: ${result.stderr || result.stdout}`) - } - return result.stdout.trim() -} - -function gitOk(args) { - const result = spawnSync('git', args, { - cwd: root, - stdio: 'ignore' - }) - return result.status === 0 -} - -function listTurtleFiles(dir = ttlRoot) { - const entries = fs.readdirSync(dir, {withFileTypes: true}) - const files = [] - for (const entry of entries) { - const fullPath = path.join(dir, entry.name) - if (entry.isDirectory()) { - files.push(...listTurtleFiles(fullPath)) - } else if (entry.isFile() && entry.name.endsWith('.ttl')) { - files.push(fullPath) - } - } - return files.sort() -} - -function compactTerm(term) { - if (term.startsWith('project:')) { - return term.slice('project:'.length) - } - if (term.startsWith('sd:')) { - return term.slice('sd:'.length) - } - return term -} - -function assertBalanced(text, file) { - let square = 0 - let inString = false - let escaped = false - for (const char of text) { - if (inString) { - if (escaped) { - escaped = false - } else if (char === '\\') { - escaped = true - } else if (char === '"') { - inString = false - } - continue - } - if (char === '"') { - inString = true - } else if (char === '[') { - square++ - } else if (char === ']') { - square-- - if (square < 0) { - throw new Error(`${file}: unmatched ]`) - } - } - } - if (inString) { - throw new Error(`${file}: unterminated string`) - } - if (square !== 0) { - throw new Error(`${file}: unbalanced []`) - } +// Preserve the existing entry point; Python/rdflib also serves the pinned core. +const script = fileURLToPath(new URL('./validate-spiral-provenance.py', import.meta.url)) +const result = spawnSync(process.env.PYTHON || 'python3', [script, ...process.argv.slice(2)], { + stdio: 'inherit' +}) +if (result.error) { + console.error(`Cannot run Spiral validation: ${result.error.message}`) } - -function parseTurtleSubset(text, file) { - assertBalanced(text, file) - if (!text.includes('@prefix sd:') || !text.includes('@prefix project:')) { - throw new Error(`${file}: expected sd and project prefixes`) - } - if (!text.trim().endsWith('.')) { - throw new Error(`${file}: expected final .`) - } - - const subjectMatch = text.match(/project:([A-Za-z][A-Za-z0-9-]*)\s+([\s\S]*)\.\s*$/) - if (!subjectMatch) { - throw new Error(`${file}: expected one project subject block`) - } - const artifact = subjectMatch[1] - const body = subjectMatch[2] - const typeMatch = body.match(/(?:^|\s)a\s+sd:([A-Za-z]+)\s*;/) - if (!typeMatch) { - throw new Error(`${file}: expected rdf type`) - } - const pathMatch = body.match(/sd:repositoryPath\s+"([^"]+)"/) - if (!pathMatch) { - throw new Error(`${file}: expected sd:repositoryPath`) - } - - const references = [] - const refRegex = /sd:([A-Za-z]+)\s+\[\s*a\s+sd:ArtifactReference\s*;\s*sd:artifact\s+project:([A-Za-z][A-Za-z0-9-]*)\s*;\s*sd:gitCommit\s+"([0-9a-f]+)"(?:\s*;\s*sd:fragment\s+"([^"]+)")?\s*\]/g - let match - while ((match = refRegex.exec(body)) !== null) { - references.push({ - predicate: match[1], - artifact: match[2], - gitCommit: match[3], - fragment: match[4] || null - }) - } - - const locations = [...body.matchAll(/sd:implementationLocation\s+"([^"]+)"/g)] - .map(locationMatch => locationMatch[1]) - const statusMatch = body.match(/sd:status\s+sd:([A-Za-z]+)/) - - return { - file, - ttlPath: file, - artifact, - type: typeMatch[1], - repositoryPath: pathMatch[1], - status: statusMatch ? statusMatch[1] : null, - references, - locations, - text - } -} - -function loadCurrentGraph() { - const graph = new Map() - for (const file of listTurtleFiles()) { - const parsed = parseTurtleSubset(fs.readFileSync(file, 'utf8'), path.relative(root, file)) - graph.set(parsed.artifact, parsed) - } - return graph -} - -function loadHistoricalArtifact(artifact, commit) { - const current = currentGraph.get(artifact) - if (!current) { - throw new Error(`Unknown artifact ${artifact}`) - } - const text = runGit(['show', `${commit}:${current.ttlPath}`]) - return parseTurtleSubset(text, `${current.ttlPath}@${commit}`) -} - -function latestCommitForPath(repositoryPath) { - return runGit(['log', '-1', '--format=%H', '--', repositoryPath]) -} - -function validateReferences(graph) { - const errors = [] - const shortHashes = [] - const checked = [] - - for (const artifact of graph.values()) { - const sourceCommit = latestCommitForPath(artifact.ttlPath) - for (const reference of artifact.references) { - const label = `${artifact.artifact}.${reference.predicate}->${reference.artifact}@${reference.gitCommit}` - if (!/^[0-9a-f]{40}$/.test(reference.gitCommit)) { - shortHashes.push(label) - continue - } - if (!gitOk(['cat-file', '-e', `${reference.gitCommit}^{commit}`])) { - errors.push(`${label}: commit does not resolve`) - continue - } - if (reference.gitCommit === sourceCommit) { - errors.push(`${label}: target equals source artifact commit ${sourceCommit}`) - continue - } - if (!gitOk(['merge-base', '--is-ancestor', reference.gitCommit, sourceCommit])) { - errors.push(`${label}: target is not a strict ancestor of source artifact commit ${sourceCommit}`) - continue - } - checked.push(label) - } - } - - if (shortHashes.length) { - errors.push(`short sd:gitCommit references: ${shortHashes.join(', ')}`) - } - if (errors.length) { - throw new Error(errors.join('\n')) - } - return checked -} - -function findImplementationByLocation(location, preferredArtifact) { - const matches = [...currentGraph.values()] - .filter(artifact => artifact.type === 'Implementation' && artifact.locations.includes(location)) - .map(artifact => artifact.artifact) - .sort() - if (preferredArtifact && matches.includes(preferredArtifact)) { - return preferredArtifact - } - if (matches.length !== 1) { - throw new Error(`Expected one implementation for ${location}, found ${matches.join(', ')}`) - } - return matches[0] -} - -function ref(artifact, predicate, target = null) { - const matches = artifact.references.filter(reference => { - return reference.predicate === predicate && (!target || reference.artifact === target) - }) - if (!matches.length) { - throw new Error(`${artifact.artifact}: missing ${predicate}${target ? ` -> ${target}` : ''}`) - } - return matches -} - -function traverseIntent(implementationId) { - const implementation = currentGraph.get(implementationId) - const designRef = ref(implementation, 'implements')[0] - const design = loadHistoricalArtifact(designRef.artifact, designRef.gitCommit) - - let requestRef - if (design.artifact === 'DES-002') { - requestRef = ref(design, 'derivedFrom', 'REQ-001')[0] - } else { - requestRef = ref(design, 'derivedFrom')[0] - } - const request = loadHistoricalArtifact(requestRef.artifact, requestRef.gitCommit) - const understandingRef = ref(request, 'derivedFrom')[0] - const understanding = loadHistoricalArtifact(understandingRef.artifact, understandingRef.gitCommit) - const sourceRef = ref(understanding, 'interprets')[0] - const source = loadHistoricalArtifact(sourceRef.artifact, sourceRef.gitCommit) - - return [ - {artifact: implementation.artifact, type: implementation.type}, - {artifact: design.artifact, type: design.type, commit: designRef.gitCommit}, - {artifact: request.artifact, type: request.type, commit: requestRef.gitCommit}, - {artifact: understanding.artifact, type: understanding.type, commit: understandingRef.gitCommit}, - {artifact: source.artifact, type: source.type, commit: sourceRef.gitCommit} - ] -} - -function historyForImplementation(implementationId) { - const implementation = currentGraph.get(implementationId) - const transforms = implementation.references.filter(reference => reference.predicate === 'transforms') - const changeCauses = implementation.references.filter(reference => reference.predicate === 'changeCausedBy') - const verification = [...currentGraph.values()] - .filter(artifact => artifact.type === 'VerificationEvidence') - .flatMap(artifact => { - return artifact.references - .filter(reference => reference.predicate === 'verifies' && reference.artifact === implementationId) - .map(reference => ({ - evidence: artifact.artifact, - verifies: `${reference.artifact}@${reference.gitCommit}` - })) - }) - .sort((a, b) => a.evidence.localeCompare(b.evidence)) - - const predecessorDetails = transforms.map(transform => { - const predecessor = loadHistoricalArtifact(transform.artifact, transform.gitCommit) - return { - artifact: transform.artifact, - commit: transform.gitCommit, - changeCausedBy: predecessor.references - .filter(reference => reference.predicate === 'changeCausedBy') - .map(reference => `${reference.artifact}@${reference.gitCommit}`), - transforms: predecessor.references - .filter(reference => reference.predicate === 'transforms') - .map(reference => `${reference.artifact}@${reference.gitCommit}`) - } - }) - - return { - currentChangeCausedBy: changeCauses.map(reference => `${reference.artifact}@${reference.gitCommit}`), - transforms: transforms.map(reference => `${reference.artifact}@${reference.gitCommit}`), - predecessorDetails, - verification - } -} - -function validateBranchCycle() { - const branch = runGit(['branch', '--show-current']) - if (!branch.includes('CYC-016')) { - throw new Error(`Expected CYC-016 branch, got ${branch}`) - } - const context = fs.readFileSync(path.join(root, '.spiral/project-context.md'), 'utf8') - const contextNamesCyc016 = context.includes('Current active Spiral cycle: `.spiral/cycles/CYC-016.md`') - || context.includes('Latest accepted Spiral cycle: `.spiral/cycles/CYC-016.md`') - if (!contextNamesCyc016) { - throw new Error('project context does not identify CYC-016 as active or latest accepted') - } - const cycle = currentGraph.get('CYC-016') - if (!cycle || !['Active', 'Accepted'].includes(cycle.status)) { - throw new Error('CYC-016 TTL is neither active nor accepted') - } - return branch -} - -const currentGraph = loadCurrentGraph() -const checkedReferences = validateReferences(currentGraph) -const branch = validateBranchCycle() - -const examples = [ - { - name: 'server done commit-before-query visibility', - code: 'src/server.mjs#runNextCommand', - implementation: findImplementationByLocation('src/server.mjs#runNextCommand', 'IMP-004') - }, - { - name: 'startup recovery with several historical contributions', - code: 'src/recovery.mjs#loadCommandStatus', - implementation: findImplementationByLocation('src/recovery.mjs#loadCommandStatus', 'IMP-001') - }, - { - name: 'recent after-change/index characterization', - code: 'test/after-change-handlers.mjs', - implementation: findImplementationByLocation('test/after-change-handlers.mjs', 'IMP-009') - } -] - -const traversals = examples.map(example => ({ - ...example, - intentPath: traverseIntent(example.implementation), - history: historyForImplementation(example.implementation) -})) - -const output = { - parser: 'repository-local Spiral Turtle subset parser', - projectPrefix, - branch, - turtleFilesParsed: currentGraph.size, - historicalReferencesChecked: checkedReferences.length, - examples: traversals -} - -console.log(JSON.stringify(output, null, 2)) +process.exit(result.status ?? 1) diff --git a/scripts/validate-spiral-provenance.py b/scripts/validate-spiral-provenance.py new file mode 100644 index 0000000..c45814f --- /dev/null +++ b/scripts/validate-spiral-provenance.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 +"""Validate versioned Spiral references without forgiving invalid old history.""" +import argparse +from functools import lru_cache +import json +from pathlib import Path +import re +import subprocess +import sys + +try: + from rdflib import Graph, Literal, Namespace, URIRef + from rdflib.namespace import DCTERMS, RDF, RDFS +except ImportError: + sys.exit('rdflib is required; install .spiral-core/requirements.txt with your Python interpreter') + +SD = Namespace('https://muze.nl/ns/spiral-developer#') +EXTERNAL_PREFIXES = ( + 'https://muze.nl/ns/spiral-developer/culture/', + 'https://muze.nl/projects/spiral-developer/', +) + + +class Repository: + def __init__(self, path): + self.path = Path(path).resolve() + + @lru_cache(maxsize=None) + def git(self, *args): + result = subprocess.run(['git', '-C', str(self.path), *args], + capture_output=True, text=True) + if result.returncode: + raise ValueError(result.stderr.strip() or 'git command failed') + return result.stdout + + def commit(self, ref): + return self.git('rev-parse', '--verify', '--end-of-options', ref + '^{commit}').strip() + + def ancestor(self, older, newer): + result = subprocess.run(['git', '-C', str(self.path), 'merge-base', + '--is-ancestor', older, newer], capture_output=True) + if result.returncode not in (0, 1): + raise ValueError(result.stderr.decode().strip()) + return result.returncode == 0 + + def paths(self, revision, external=False): + if revision == ':': + paths = self.git('ls-files', '-z').split('\0') + else: + paths = self.git('ls-tree', '-r', '--name-only', '-z', revision).split('\0') + return sorted(p for p in paths if p.endswith('.ttl') and + (external or p.startswith('.spiral/')) and + 'node_modules' not in Path(p).parts) + + def read(self, revision, path): + return self.git('show', f'{revision.rstrip(":")}:{path}') + + @lru_cache(maxsize=None) + def graph(self, revision, external=False): + graph = Graph() + for path in self.paths(revision, external): + # Parse documents separately so blank-node IDs cannot collide across files. + graph += Graph().parse(data=self.read(revision, path), format='turtle', + publicID='urn:spiral:file:' + path) + return graph + + def exists(self, revision, path): + # Repository paths must remain repository-relative; fragments are symbols. + path = path.split('#', 1)[0] + if not path or Path(path).is_absolute() or '..' in Path(path).parts: + return False + try: + self.git('cat-file', '-e', f'{revision}:{path}') + return True + except ValueError: + return False + + +class Validator: + def __init__(self, repo, core): + self.repo = repo + self.core = Repository(core) + self.ontology = Graph().parse(self.core.path / 'ontology/spiral-developer.ttl', + format='turtle') + self.errors = [] + self.references = 0 + self.documents = 0 + + def error(self, code, path, source, message, **details): + self.errors.append(dict(code=code, path=path, source=source, + message=message, **details)) + + def properties(self, graph): + properties = {SD.historicalReference} + hierarchy = list(self.ontology.subject_objects(RDFS.subPropertyOf)) + hierarchy += list(graph.subject_objects(RDFS.subPropertyOf)) + while True: + expanded = properties | {child for child, parent in hierarchy if parent in properties} + if expanded == properties: + return properties + properties = expanded + + def core_pin(self, source, staged): + if staged: + entry = self.repo.git('ls-files', '--stage', '--', '.spiral-core').strip() + else: + entry = self.repo.git('ls-tree', source, '--', '.spiral-core').strip() + fields = entry.split() + if not fields or fields[0] != '160000': + raise ValueError('source version has no pinned .spiral-core gitlink') + return fields[1] if staged else fields[2] + + def document(self, path, text, source, properties, staged=False): + self.documents += 1 + try: + graph = Graph().parse(data=text, format='turtle', publicID='urn:spiral:file:' + path) + except Exception as exc: + self.error('invalid-turtle', path, source, str(exc)) + return + for subject, predicate, node in graph: + if predicate not in properties: + continue + self.references += 1 + targets = list(graph.objects(node, SD.artifact)) + hashes = list(graph.objects(node, SD.gitCommit)) + details = dict(artifact=str(subject), relation=str(predicate)) + if ((node, RDF.type, SD.ArtifactReference) not in graph or + len(targets) != 1 or not isinstance(targets[0], URIRef) or + len(hashes) != 1 or not isinstance(hashes[0], Literal)): + self.error('invalid-reference', path, source, + 'Expected an ArtifactReference with exactly one artifact IRI and Git hash', **details) + continue + target, commit = targets[0], str(hashes[0]) + details.update(target=str(target), commit=commit) + if not re.fullmatch(r'(?:[0-9a-f]{40}|[0-9a-f]{64})', commit): + self.error('invalid-hash', path, source, 'Expected a full Git hash', **details) + continue + external = str(target).startswith(EXTERNAL_PREFIXES) + target_repo = self.core if external else self.repo + try: + resolved = target_repo.commit(commit) + if resolved != commit: + raise ValueError('Hash does not identify the commit itself') + bound = self.core_pin(source, staged) if external else source + admissible = target_repo.ancestor(commit, bound) + if not admissible or (not external and not staged and commit == source): + self.error('not-ancestor', path, source, + 'Target must precede the source (external targets must be in its pinned core)', **details) + continue + historical = target_repo.graph(commit, external=external) + if not list(historical.objects(target, DCTERMS.identifier)): + self.error('missing-artifact', path, source, + 'Target artifact does not exist at the referenced commit', **details) + continue + locations = list(historical.objects(target, SD.repositoryPath)) + if not locations or any(not target_repo.exists(commit, str(p)) for p in locations): + self.error('missing-artifact-path', path, source, + 'Target artifact has no existing repositoryPath at its commit', **details) + except Exception as exc: + self.error('unresolvable-reference', path, source, str(exc), **details) + + def snapshot(self, revision, staged=False): + revision = ':' if staged else self.repo.commit(revision) + head = self.repo.commit('HEAD') + # Validate union syntax before deriving custom relation properties. + try: + properties = self.properties(self.repo.graph(revision)) + except Exception as exc: + self.error('invalid-turtle', '.spiral/', revision, str(exc)) + return + for path in self.repo.paths(revision): + text = self.repo.read(revision, path) + changed = False + if staged: + try: + changed = text != self.repo.read(head, path) + except ValueError: + changed = True + if changed: + source = head + else: + source = self.repo.git('log', '-1', '--format=%H', head if staged else revision, + '--', path).strip() + self.document(path, text, source, properties, staged=changed) + + def history(self, revision_range): + if '...' in revision_range: + raise ValueError('Use base..head, not the symmetric-difference base...head range') + if '..' in revision_range: + base, head = revision_range.split('..', 1) + selection = self.repo.commit(base) + '..' + self.repo.commit(head) + else: + selection = self.repo.commit(revision_range) + commits = self.repo.git('rev-list', '--reverse', '--topo-order', selection, '--').splitlines() + for commit in commits: + changed = set(self.repo.git('diff-tree', '--root', '-m', '--no-commit-id', + '--name-only', '-r', '-z', commit, '--', '.spiral').split('\0')) + paths = changed.intersection(self.repo.paths(commit)) + if not paths: + continue + try: + properties = self.properties(self.repo.graph(commit)) + except Exception as exc: + self.error('invalid-turtle', '.spiral/', commit, str(exc)) + continue + for path in sorted(paths): + self.document(path, self.repo.read(commit, path), commit, properties) + return len(commits) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--repo', default='.') + parser.add_argument('--core', help='Pinned core checkout; defaults to /.spiral-core') + mode = parser.add_mutually_exclusive_group() + mode.add_argument('--tree', default=None, help='Committed snapshot (default HEAD)') + mode.add_argument('--staged', action='store_true', help='Validate the Git index before committing') + mode.add_argument('--range', dest='revision_range', help='Audit base..head, or all history reachable from head') + args = parser.parse_args() + try: + repo = Repository(args.repo) + repo = Repository(repo.git('rev-parse', '--show-toplevel').strip()) + validator = Validator(repo, args.core or repo.path / '.spiral-core') + count = None + if args.revision_range: + count = validator.history(args.revision_range) + else: + validator.snapshot(args.tree or 'HEAD', staged=args.staged) + result = dict(ok=not validator.errors, scope='range' if args.revision_range else + 'staged' if args.staged else 'snapshot', documents=validator.documents, + references=validator.references, errors=validator.errors) + if count is not None: + result['commits'] = count + except Exception as exc: + result = dict(ok=False, errors=[dict(code='validation-unavailable', message=str(exc))]) + print(json.dumps(result, indent=2)) + return 0 if result['ok'] else 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/command-worker-module.mjs b/src/command-worker-module.mjs index e53b29c..48e9033 100644 --- a/src/command-worker-module.mjs +++ b/src/command-worker-module.mjs @@ -4,12 +4,13 @@ import Parser from '@muze-nl/od-jsontag/src/parse.mjs' import serialize from '@muze-nl/od-jsontag/src/serialize.mjs' import writeFileAtomic from 'write-file-atomic' import { faultPoint } from './faults.mjs' +import { appendIntegrityRecord } from './integrity.mjs' let commands = {} let index = {} let resultArr = [] let dataspace -let datafile, basefile, extension +let datafile, basefile, extension, integrityFile let meta = {} let metaProxy = { index: { @@ -76,6 +77,7 @@ export async function initialize(task) { metaProxy.schema = meta.schema } datafile = task.datafile + integrityFile = task.integrityFile extension = datafile.split('.').pop() basefile = datafile.substring(0, datafile.length - (extension.length + 1)) //+1 for . character commands = await import(task.commandsFile).then(mod => { @@ -117,6 +119,9 @@ export default async function runCommand(commandStr, request) { let newfilename = basefile + '.' + task.id + '.' + extension await faultPoint('before-command-changeset-write') await writeFileAtomic(newfilename, uint8sab) + if (integrityFile) { + await appendIntegrityRecord(integrityFile, newfilename, uint8sab) + } await faultPoint('after-command-changeset-write') meta.parts++ response.meta.parts = meta.parts diff --git a/src/integrity.mjs b/src/integrity.mjs new file mode 100644 index 0000000..acf7d8c --- /dev/null +++ b/src/integrity.mjs @@ -0,0 +1,106 @@ +import { createHash } from 'node:crypto' +import fs from 'node:fs/promises' +import path from 'node:path' +import JSONTag from '@muze-nl/jsontag' +import { RecoveryIntegrityError } from './recovery.mjs' +import { appendFile } from './util.mjs' + +const integrityAlgorithm = 'sha256' + +export function getDefaultIntegrityFile(dataFile) { + const extension = dataFile.split('.').pop() + const basefile = dataFile.substring(0, dataFile.length - (extension.length + 1)) + return `${basefile}.integrity.${extension}` +} + +export function digestBuffer(buffer) { + return createHash(integrityAlgorithm).update(buffer).digest('hex') +} + +function integrityFileKey(integrityFile, file) { + return path.relative(path.dirname(integrityFile), file) +} + +export async function loadIntegrityManifest(integrityFile) { + const entries = new Map() + let text + try { + text = await fs.readFile(integrityFile, 'utf8') + } catch (error) { + if (error.code === 'ENOENT') { + return entries + } + throw error + } + for (const [index, line] of text.split('\n').entries()) { + if (!line) { + continue + } + let record + try { + record = JSONTag.parse(line) + } catch (cause) { + throw new RecoveryIntegrityError('Invalid integrity manifest record', { + file: integrityFile, + lineNumber: index + 1, + recordKind: 'integrity manifest', + cause + }) + } + if (!record || typeof record !== 'object' || Array.isArray(record)) { + throw new RecoveryIntegrityError('Invalid integrity manifest record: record must be an object', { + file: integrityFile, + lineNumber: index + 1, + recordKind: 'integrity manifest' + }) + } + if (typeof record.file !== 'string' || record.file === '') { + throw new RecoveryIntegrityError('Invalid integrity manifest record: missing string field "file"', { + file: integrityFile, + lineNumber: index + 1, + recordKind: 'integrity manifest' + }) + } + if (record.algorithm !== integrityAlgorithm || typeof record.digest !== 'string' || record.digest === '') { + throw new RecoveryIntegrityError('Invalid integrity manifest record: invalid digest', { + file: integrityFile, + lineNumber: index + 1, + recordKind: 'integrity manifest' + }) + } + entries.set(record.file, record) + } + return entries +} + +export function verifyIntegrity(manifest, integrityFile, file, buffer, options = {}) { + const key = integrityFileKey(integrityFile, file) + const record = manifest.get(key) + if (!record) { + if (options.required) { + throw new RecoveryIntegrityError(`Missing integrity manifest entry for ${key}`, { + file, + recordKind: 'OD-JSONTag data' + }) + } + return false + } + const actual = digestBuffer(buffer) + if (record.digest !== actual) { + throw new RecoveryIntegrityError(`Integrity mismatch for ${key}`, { + file, + recordKind: 'OD-JSONTag data' + }) + } + return true +} + +export async function appendIntegrityRecord(integrityFile, file, buffer) { + const record = { + file: integrityFileKey(integrityFile, file), + algorithm: integrityAlgorithm, + digest: digestBuffer(buffer) + } + await appendFile(integrityFile, JSONTag.stringify(record)) + return record +} diff --git a/src/load-worker.mjs b/src/load-worker.mjs index f099c7c..c90bcf1 100644 --- a/src/load-worker.mjs +++ b/src/load-worker.mjs @@ -5,6 +5,7 @@ import fs from 'fs' import path from 'path' import serialize from '@muze-nl/od-jsontag/src/serialize.mjs' import { assertChangesetExists, assertOdJsonTagFraming } from './recovery.mjs' +import { loadIntegrityManifest, verifyIntegrity } from './integrity.mjs' const parser = new Parser() @@ -19,6 +20,9 @@ parentPort.on('message', async (files) => { const extension = files.dataFile.split('.').pop() const basefile = files.dataFile.substring(0, files.dataFile.length - (extension.length + 1)) //+1 for . character meta.data = path.dirname(basefile) + const integrityManifest = files.integrityFile + ? await loadIntegrityManifest(files.integrityFile) + : null let data let jsontag // TODO @@ -31,12 +35,22 @@ parentPort.on('message', async (files) => { // add version info in proxies with a symbol to get that information if (fs.existsSync(files.dataFile)) { jsontag = fs.readFileSync(files.dataFile) + if (integrityManifest) { + verifyIntegrity(integrityManifest, files.integrityFile, files.dataFile, jsontag, { + required: files.integrityRequired + }) + } assertOdJsonTagFraming(jsontag, files.dataFile, 'base OD-JSONTag data') data = parser.parse(jsontag) } for (let command of files.commands) { const changesetFile = assertChangesetExists(files.dataFile, command) jsontag = fs.readFileSync(changesetFile) + if (integrityManifest) { + verifyIntegrity(integrityManifest, files.integrityFile, changesetFile, jsontag, { + required: files.integrityRequired + }) + } assertOdJsonTagFraming(jsontag, changesetFile, 'changeset OD-JSONTag data') data = parser.parse(jsontag) } diff --git a/src/server.mjs b/src/server.mjs index cf2c08e..5ce4609 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -11,6 +11,7 @@ import process from 'node:process' import { getCommittedCommandIds, loadCommandLog, loadCommandStatus, nextActiveCommandStatus, pendingCommandStatus, recoverActiveCommands, unsafeCommandStatus } from './recovery.mjs' import { assertRuntimeEnvironmentConfiguration } from './runtime-environment.mjs' import { faultPoint } from './faults.mjs' +import { getDefaultIntegrityFile } from './integrity.mjs' const server = express() const __dirname = path.dirname(path.dirname(fileURLToPath(import.meta.url))) @@ -57,6 +58,8 @@ async function main(options) { const indexFile = options.indexFile || __dirname+'/src/index.mjs' const commandLog = options.commandLog || './command-log.jsontag' const commandStatus = options.commandStatus || './command-status.jsontag' + const integrityFile = options.integrityFile || getDefaultIntegrityFile(datafile) + const integrityEnabled = Boolean(options.integrity || options.integrityFile || fs.existsSync(integrityFile)) const maxCommandCrashAttempts = options.maxCommandCrashAttempts ?? 2 const access = options.access || null const timeout = options.timeout || 1000 @@ -91,7 +94,9 @@ async function main(options) { data:jsontagBuffers, commandsFile, indexFile, - datafile + datafile, + integrityFile: integrityEnabled ? integrityFile : null, + integrityRequired: integrityEnabled }) const queryWorkerInitTask = () => { @@ -185,7 +190,14 @@ async function main(options) { void finish(reject, error) }) try { - worker.postMessage({dataFile:datafile,indexFile,schemaFile,commands}) + worker.postMessage({ + dataFile:datafile, + indexFile, + schemaFile, + commands, + integrityFile: integrityEnabled ? integrityFile : null, + integrityRequired: integrityEnabled + }) } catch (error) { void finish(reject, error) } @@ -301,7 +313,9 @@ async function main(options) { data:jsontagBuffers, commandsFile, indexFile, - datafile + datafile, + integrityFile: integrityEnabled ? integrityFile : null, + integrityRequired: integrityEnabled }) await drainCommandQueue() } catch(err) { diff --git a/test/durability-crash.mjs b/test/durability-crash.mjs index efe689f..3d4be48 100644 --- a/test/durability-crash.mjs +++ b/test/durability-crash.mjs @@ -3,6 +3,8 @@ import fs from 'node:fs/promises' import path from 'node:path' import test from 'node:test' import { faultPoint } from '../src/faults.mjs' +import { appendIntegrityRecord, getDefaultIntegrityFile, loadIntegrityManifest } from '../src/integrity.mjs' +import { getChangesetPath } from '../src/recovery.mjs' import { assertRuntimeEnvironmentConfiguration, getRuntimeEnvironment } from '../src/runtime-environment.mjs' import { getCommandStatus, @@ -355,6 +357,45 @@ parentPort.on('message', () => { assert.doesNotMatch(running.getOutput(), /SimplyStore listening/) }) +test('integrity-enabled command writes changeset digest before done status', async t => { + const fixture = await makeServerFixture(t) + const port = await getOpenPort() + const integrityFile = getDefaultIntegrityFile(fixture.datafile) + await appendIntegrityRecord(integrityFile, fixture.datafile, await fs.readFile(fixture.datafile)) + + const running = startServer(t, fixture, { + port, + integrityFile + }) + await waitForServer(running.child, running.getOutput, port) + + const commandId = 'integrity-command' + const response = await postCommand(port, { + id: commandId, + name: 'addPerson', + value: {name: 'Ada'} + }) + assert.equal(response.status, 202) + await waitForCommandStatus(port, commandId, 'done') + + const changesetPath = getChangesetPath(fixture.datafile, commandId) + const manifest = await loadIntegrityManifest(integrityFile) + assert.ok(manifest.has(path.basename(changesetPath))) + + await stopServer(running.child) + + const original = await fs.readFile(changesetPath, 'utf8') + await fs.writeFile(changesetPath, original.replace('Ada', 'Eve')) + + const restarted = startServer(t, fixture, { + port, + integrityFile + }) + const exit = await waitForExit(restarted.child) + assert.equal(exit.code, 1) + assert.match(restarted.getOutput(), /Integrity mismatch/) +}) + test('normal restart preserves committed state according to reconstruction oracle', async t => { const fixture = await makeServerFixture(t) const port = await getOpenPort() diff --git a/test/durability-helpers.mjs b/test/durability-helpers.mjs index 819e84a..fdc92f6 100644 --- a/test/durability-helpers.mjs +++ b/test/durability-helpers.mjs @@ -69,6 +69,8 @@ export function startServer(t, fixture, options = {}) { wwwroot: path.join(rootDir, 'www'), maxWorkers: 1, maxCommandCrashAttempts: options.maxCommandCrashAttempts, + integrity: options.integrity, + integrityFile: options.integrityFile, commandTimeout: options.commandTimeout, loadTimeout: options.loadTimeout, loadWorker: options.loadWorker diff --git a/test/durability-startup.mjs b/test/durability-startup.mjs index 5371ba6..c8d3d92 100644 --- a/test/durability-startup.mjs +++ b/test/durability-startup.mjs @@ -9,6 +9,7 @@ import { fileURLToPath } from 'node:url' import JSONTag from '@muze-nl/jsontag' import Parser from '@muze-nl/od-jsontag/src/parse.mjs' import serialize from '@muze-nl/od-jsontag/src/serialize.mjs' +import { appendIntegrityRecord, getDefaultIntegrityFile } from '../src/integrity.mjs' import { activeCommandStatus, getCommittedCommandIds, @@ -63,7 +64,7 @@ async function writeChangeset(dataFile, commandId, change) { })) } -function loadDataset({dataFile, indexFile, commands}) { +function loadDataset({dataFile, indexFile, commands, integrityFile = null, integrityRequired = false}) { return new Promise((resolve, reject) => { const worker = new Worker(loadWorker) worker.on('message', result => { @@ -78,11 +79,17 @@ function loadDataset({dataFile, indexFile, commands}) { dataFile, indexFile, schemaFile: null, - commands: [...commands] + commands: [...commands], + integrityFile, + integrityRequired }) }) } +async function writeIntegrityEntry(integrityFile, file) { + await appendIntegrityRecord(integrityFile, file, await fs.readFile(file)) +} + test('accepted command changeset is not treated as committed state on startup', async t => { const fixture = await makeFixture(t) await writeChangeset(fixture.dataFile, 'accepted-command', data => { @@ -398,3 +405,64 @@ test('OD-JSONTag framing validation accepts changeset skip records', () => { assertOdJsonTagFraming(buffer, 'data.patch.jsontag', 'changeset OD-JSONTag data') }) }) + +test('integrity manifest detects same-length altered base OD-JSONTag payload', async t => { + const fixture = await makeFixture(t) + const integrityFile = getDefaultIntegrityFile(fixture.dataFile) + await writeIntegrityEntry(integrityFile, fixture.dataFile) + + const original = await fs.readFile(fixture.dataFile, 'utf8') + await fs.writeFile(fixture.dataFile, original.replace('persons', 'qersons')) + + await assert.rejects( + loadDataset({ + ...fixture, + commands: [], + integrityFile, + integrityRequired: true + }), + /Integrity mismatch|base OD-JSONTag data/i + ) +}) + +test('integrity manifest detects same-length altered committed changeset payload', async t => { + const fixture = await makeFixture(t) + const commandId = 'tampered-changeset' + const status = new Map([ + [commandId, {command: commandId, code: 200, status: 'done'}] + ]) + const integrityFile = getDefaultIntegrityFile(fixture.dataFile) + await writeIntegrityEntry(integrityFile, fixture.dataFile) + await writeChangeset(fixture.dataFile, commandId, data => { + data.persons.push({name: 'Ada'}) + }) + const changesetPath = getChangesetPath(fixture.dataFile, commandId) + await writeIntegrityEntry(integrityFile, changesetPath) + + const original = await fs.readFile(changesetPath, 'utf8') + await fs.writeFile(changesetPath, original.replace('Ada', 'Eve')) + + await assert.rejects( + loadDataset({ + ...fixture, + commands: getCommittedCommandIds(status), + integrityFile, + integrityRequired: true + }), + /Integrity mismatch|changeset OD-JSONTag data/i + ) +}) + +test('integrity enabled requires manifest entry for base data', async t => { + const fixture = await makeFixture(t) + + await assert.rejects( + loadDataset({ + ...fixture, + commands: [], + integrityFile: getDefaultIntegrityFile(fixture.dataFile), + integrityRequired: true + }), + /Missing integrity manifest entry/i + ) +}) diff --git a/test/spiral-provenance.py b/test/spiral-provenance.py new file mode 100644 index 0000000..cda42e1 --- /dev/null +++ b/test/spiral-provenance.py @@ -0,0 +1,199 @@ +"""Regression tests for historical provenance checks using disposable Git histories.""" +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +VALIDATOR = ROOT / 'scripts/validate-spiral-provenance.py' +CORE = ROOT / '.spiral-core' +PREFIXES = '''@prefix p: . +@prefix s: . +@prefix d: . +''' + + +def artifact(name, reference=''): + return PREFIXES + f'''p:{name} a s:Design; d:identifier "{name}"; + s:repositoryPath ".spiral/{name}.md" {reference} . +''' + + +def reference(target, commit, predicate='s:derivedFrom'): + return f'''; {predicate} [ a s:ArtifactReference; + s:artifact p:{target}; s:gitCommit "{commit}" ]''' + + +class ProvenanceTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory(prefix='spiral-provenance-test-') + self.addCleanup(self.tmp.cleanup) + self.repo = Path(self.tmp.name) + self.git('init', '-b', 'main') + self.git('config', 'user.name', 'Spiral test') + self.git('config', 'user.email', 'spiral-test@example.invalid') + self.git('config', 'commit.gpgsign', 'false') + self.git('config', 'core.hooksPath', '/dev/null') + self.write('A', artifact('A')) + self.base = self.commit() + + def git(self, *args): + return subprocess.check_output(['git', '-C', str(self.repo), *args], + stderr=subprocess.PIPE, text=True).strip() + + def write(self, name, text): + (self.repo / '.spiral').mkdir(exist_ok=True) + (self.repo / f'.spiral/{name}.ttl').write_text(text) + (self.repo / f'.spiral/{name}.md').write_text(name) + + def commit(self): + self.git('add', '.') + self.git('commit', '-m', 'fixture') + return self.git('rev-parse', 'HEAD') + + def validate(self, *args, ok=True): + result = subprocess.run([sys.executable, str(VALIDATOR), '--repo', str(self.repo), + '--core', str(CORE), *args], capture_output=True, text=True) + data = json.loads(result.stdout) + self.assertEqual(data['ok'], ok, data) + self.assertEqual(result.returncode, 0 if ok else 1, data) + return data + + def assert_code(self, code, *args): + data = self.validate(*args, ok=False) + self.assertIn(code, [e['code'] for e in data['errors']], data) + return data + + def test_valid_vocabulary_and_custom_relation(self): + self.write('vocabulary', PREFIXES + '''@prefix r: . + p:reason r:subPropertyOf s:derivedFrom .''') + self.write('B', artifact('B', reference('A', self.base, 'p:reason'))) + self.commit() + result = self.validate() + self.assertEqual(result['references'], 1) + self.validate('--range', self.base + '..HEAD') + + def test_staged_head_reference_allowed(self): + self.write('B', artifact('B', reference('A', self.base))) + self.git('add', '.') + self.validate('--staged') + self.commit() + self.validate() + + def test_index_is_independent_of_unstaged_files(self): + self.write('B', artifact('B', reference('A', self.base))) + self.git('add', '.') + self.write('B', 'invalid turtle') + self.validate('--staged') + self.git('add', '.') + self.assert_code('invalid-turtle', '--staged') + + def test_valid_unstaged_repair_does_not_hide_bad_index(self): + self.write('B', artifact('B', reference('MISSING', self.base))) + self.git('add', '.') + self.write('B', artifact('B', reference('A', self.base))) + self.assert_code('missing-artifact', '--staged') + + def test_short_hash_fails(self): + self.write('B', artifact('B', reference('A', self.base[:7]))) + self.commit() + self.assert_code('invalid-hash') + + def test_missing_target_at_exact_version(self): + self.write('B', artifact('B', reference('C', self.base))) + self.write('C', artifact('C')) + self.commit() + self.assert_code('missing-artifact') + + def test_missing_artifact_path_fails(self): + (self.repo / '.spiral/A.md').unlink() + missing = self.commit() + self.write('B', artifact('B', reference('A', missing))) + self.commit() + self.assert_code('missing-artifact-path') + + def test_sibling_commit_is_not_an_ancestor(self): + self.git('checkout', '-b', 'other') + self.write('C', artifact('C')) + sibling = self.commit() + self.git('checkout', 'main') + self.write('B', artifact('B', reference('C', sibling))) + self.commit() + self.assert_code('not-ancestor') + + def test_unknown_commit_fails(self): + self.write('B', artifact('B', reference('A', 'f' * 40))) + self.commit() + self.assert_code('unresolvable-reference') + + def test_multiple_reference_targets_fail(self): + ref = reference('A', self.base).replace('p:A;', 'p:A, p:B;') + self.write('B', artifact('B', ref)) + self.commit() + self.assert_code('invalid-reference') + + def test_repaired_intermediate_version_still_fails_range(self): + self.write('B', artifact('B', reference('MISSING', self.base))) + bad = self.commit() + self.write('B', artifact('B', reference('A', self.base))) + self.commit() + self.validate() + result = self.assert_code('missing-artifact', '--range', self.base + '..HEAD') + self.assertEqual(result['errors'][0]['source'], bad) + self.assert_code('missing-artifact', '--range', 'HEAD') + + def test_deleted_intermediate_version_still_fails_range(self): + self.write('B', artifact('B', reference('MISSING', self.base))) + self.commit() + (self.repo / '.spiral/B.ttl').unlink() + (self.repo / '.spiral/B.md').unlink() + self.commit() + self.validate() + self.assert_code('missing-artifact', '--range', self.base + '..HEAD') + + def test_merge_versions_are_checked(self): + self.git('checkout', '-b', 'other') + self.write('B', artifact('B', reference('A', self.base))) + self.commit() + self.git('checkout', 'main') + self.write('C', artifact('C')) + self.commit() + self.git('merge', '--no-ff', 'other', '-m', 'merge fixture') + self.validate('--range', self.base + '..HEAD') + + def test_process_submodule_not_parsed_as_project(self): + (self.repo / '.spiral-core').mkdir() + (self.repo / '.spiral-core/bad.ttl').write_text('invalid turtle') + self.commit() + self.validate() + + def test_external_reference_uses_source_core_pin(self): + core_hash = subprocess.check_output(['git', '-C', str(CORE), 'rev-parse', 'HEAD'], + text=True).strip() + self.git('update-index', '--add', '--cacheinfo', '160000', core_hash, '.spiral-core') + self.git('commit', '-m', 'pin process') + target = 'https://muze.nl/ns/spiral-developer/culture/CUL-MUZE-001' + ref = reference('A', core_hash, 's:adoptsCulture').replace('p:A;', '<' + target + '>;') + self.write('B', artifact('B', ref)) + # Do not git add the empty gitlink path: preserve the explicit pin in the index. + self.git('add', '.spiral') + self.git('commit', '-m', 'adopt culture') + self.validate() + self.write('B', artifact('B', ref.replace(core_hash, 'f' * 40))) + self.git('add', '.spiral') + self.assert_code('unresolvable-reference', '--staged') + + def test_external_reference_requires_a_pin(self): + core_hash = subprocess.check_output(['git', '-C', str(CORE), 'rev-parse', 'HEAD'], + text=True).strip() + ref = reference('A', core_hash, 's:adoptsCulture').replace( + 'p:A;', ';') + self.write('B', artifact('B', ref)) + self.commit() + self.assert_code('unresolvable-reference') + + +if __name__ == '__main__': + unittest.main()