diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4b9d12d8..e67a9d7d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,60 @@ jobs: - name: Tests run: pytest -q + conformance: + name: cross-implementation agreement + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + # The Node implementation is deliberately not installed from npm: it has + # zero dependencies, so a bare runtime is the whole requirement. + - uses: actions/setup-node@v4 + with: + node-version: "20" + - name: Install + run: | + python -m pip install -U pip + pip install -e '.[test]' + - name: Canonicalization census + # Gates on cc.canonical.v2 only. v1 is retained read-only for + # historical receipts and is expected to carry its known defects. + run: python scripts/canonicalization_probe.py + - name: Evidence cards are current + run: python scripts/build_evidence_cards.py --check + - name: Corpus is current + run: python scripts/build_conformance_corpus.py --check + - name: Independent Node implementation agrees + run: node verifiers/node/cc_compose_verify.mjs + - name: Differential fuzz + run: python scripts/differential_compose.py --cases 4000 --seed 1 + - name: Conformance, canonicalization and acceptance suites + run: pytest -q tests/conformance tests/acceptance tests/unit/compose tests/unit/cli tests/unit/canonical tests/unit/evidence_card + + test-extra-is-sufficient: + name: the documented install path works + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + # Installs '.[test]' SPECIFICALLY, not '.[dev]'. The extra strangers are + # told to use must be the extra that is tested; before this job the two + # diverged and the documented path produced a red suite (finding F-01). + - name: Install the documented test extra + run: | + python -m pip install -U pip + pip install -e '.[test]' + - name: Tests + run: pytest -q + package-build: runs-on: ubuntu-latest timeout-minutes: 15 diff --git a/.gitignore b/.gitignore index ae2fa91b..03fc7d1e 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,7 @@ protocol_*.jsonl # Generated bundles and built docs (never track) *.zip + +# Film render intermediates (the .mp4 masters and posters are tracked; these are not) +visual_identity/before_you_see_it/renders/.frames-*/ +visual_identity/before_you_see_it/renders/*.webm diff --git a/Makefile b/Makefile index 97b5f888..52704294 100644 --- a/Makefile +++ b/Makefile @@ -113,12 +113,14 @@ GOV_CAPSULE_TESTS := tests/integration/test_claim_governance_capsule.py dev install setup init lock deps \ fmt lint type security package-build \ test test-unit test-int test-kernel test-release test-reporting test-week3 test-week6 cov bench \ + conformance differential acceptance test-compose canon-probe evidence-cards \ check-artifact-boundary check-repro-clean \ enterprise-smoke \ reproduce-smoke reproduce-mvp reproduce-figures figures reports ccc \ reproduce-paper verify-paper-artifacts paper-smoke \ verify-invariants verify-statistics verify-audit \ docs docs-serve \ + film film-ghost-ark \ docker-build docker-run \ clean distclean \ carto-install carto-smoke carto-mvp carto-verify-audit carto-verify-stats carto-suggest \ @@ -301,6 +303,41 @@ test-release: test-kernel test-reporting: install PYTHONPATH=src $(VENV_DIR)/bin/pytest tests/unit/reporting -q +# --- cross-implementation agreement (upgrade workstream W5) ----------------- +# Agreement here establishes that two implementations of one specification +# compute the same values. It does NOT establish that either is correct: both +# were authored in this project and can share a misreading of the spec. + +conformance: install + PYTHONPATH=src $(VENV_DIR)/bin/python scripts/build_conformance_corpus.py --check + node verifiers/node/cc_compose_verify.mjs + PYTHONPATH=src $(VENV_DIR)/bin/pytest tests/conformance -q + +differential: install + PYTHONPATH=src $(VENV_DIR)/bin/python scripts/differential_compose.py --cases 4000 --seed 1 + +# The acceptance gate for consumability: an external consumer's PUBLISHED +# four-control result, reproduced from cc.compose. Not "the API exists" -- +# their numbers, from this library. +acceptance: install + PYTHONPATH=src $(VENV_DIR)/bin/pytest tests/acceptance -q + +test-compose: install + PYTHONPATH=src $(VENV_DIR)/bin/pytest tests/unit/compose tests/unit/cli -q + $(VENV_DIR)/bin/ruff check src/cc/compose src/cc/cli/guard.py + +# Reproduces the S1 canonicalization findings. Exits non-zero while any +# unintended-kernel class remains, so the findings stay falsifiable. +# Evidence cards for an external Atlas. The committed cards are the not-run +# scaffold: verdicts are host-specific and must not be published from a laptop. +evidence-cards: install + PYTHONPATH=src $(VENV_DIR)/bin/python scripts/build_evidence_cards.py --check + PYTHONPATH=src $(VENV_DIR)/bin/pytest tests/unit/evidence_card -q + +canon-probe: install + PYTHONPATH=src $(VENV_DIR)/bin/python scripts/canonicalization_probe.py + PYTHONPATH=src $(VENV_DIR)/bin/pytest tests/unit/canonical -q + test-week3: install $(ACT); pytest -q --disable-warnings \ tests/unit/test_fh_intervals_alpha_cap.py \ @@ -472,6 +509,19 @@ docs: install docs-serve: install $(ACT); mkdocs serve -a 127.0.0.1:8000 +# ====================================================================== +# Film +# ====================================================================== +# "Before You See It" - deterministic 15s film, rendered frame by frame from +# visual_identity/before_you_see_it/film.html. Needs playwright + a chromium +# build; see that directory's README. + +film: install + $(ACT); python visual_identity/before_you_see_it/render_film.py --cut cc-framework + +film-ghost-ark: install + $(ACT); python visual_identity/before_you_see_it/render_film.py --cut ghost-ark + # ====================================================================== # Verifications # ====================================================================== diff --git a/README.md b/README.md index ac69ee54..f33d87e2 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,36 @@ stacked-system failure, not a safety certification.** +--- + +

+ + A frozen protocol: claim, falsifier, control, non-claim, and one empty result slot + +

+ +

+ A story can start a question. It cannot finish an answer.
+ Before You See It +  ·  a 15-second film  ·  Keep the wonder. Check the claim. +

+ +Those five cards are the entire discipline. Name the **claim**. Name the +**falsifier** that would end it. Name the **control** that could embarrass it. +Name the **non-claim** it will never support. Then lock all four *before* the +result slot is filled, and let the result disagree. + +This repository is the machinery for doing that to composed AI guardrail +evidence: Frechet-Hoeffding bounds that stay honest about unknown dependence, +evidence roles that decide what a measurement is allowed to say, receipts that +prove bytes and refuse to prove safety, and claims that expire. The film has one +`INCONCLUSIVE` frame in it on purpose - and when it has no verifier output to +show, it stamps `ILLUSTRATION` on its own footage rather than imply a result it +was not given. + +--- + CC-Framework is a Python research framework for dependence-aware analysis of composed binary guardrail failures. It computes what available guardrail evidence supports under stated dependence assumptions. It does not certify that diff --git a/conformance/cc-kernel-v1/SPEC.md b/conformance/cc-kernel-v1/SPEC.md new file mode 100644 index 00000000..bbd8d14a --- /dev/null +++ b/conformance/cc-kernel-v1/SPEC.md @@ -0,0 +1,256 @@ +# cc-kernel-v1 — Normative Specification + +A CC composition kernel bounds the probability of a composed binary event from +the marginal probabilities of its parts, under unknown dependence. + +This document is **normative and self-contained**. An implementer works from +this document and [`cases/`](cases/), never from the Python source. That is the +point: an implementation written by reading `cc.compose` would inherit its +mistakes, and agreement between it and the reference would establish nothing. + +> **What passing establishes.** That an implementation computes the same +> intervals as the reference on the cases in this corpus. +> +> **What passing does not establish.** That either implementation is correct — +> two implementations can share a misreading of this spec. That the corpus +> covers the input space; it is a curated census whose size is an authoring +> decision. That any bound means a system is safe, a threshold is right, or a +> marginal is valid. + +--- + +## 1. Objects + +### 1.1 Event + +A named binary indicator. `1` denotes occurrence of the event being bounded — +by CC convention, a **guardrail failure or unsafe pass**, not a success. + +An event is a name and a marginal probability. It has no threshold, no +operating curve, and no false-positive rate. A deterministic refusal rule and a +tuned classifier are the same kind of input here. + +### 1.2 Marginals + +A mapping from event name to probability. + +- Names MUST be non-empty strings. +- Probabilities MUST be finite and in `[0, 1]`. +- The mapping MUST contain at least one entry. +- Order MUST NOT affect any output value. It MAY affect `binding_event` only + through the tie rule in §4.3, which is order-independent by construction. + +### 1.3 Event kind + +| Canonical | Aliases | Meaning | +|---|---|---| +| `all` | `and`, `AND`, `intersection` | `P(every event occurs)` — a conjunction | +| `any` | `or`, `OR`, `union` | `P(at least one occurs)` — a union | + +Any other value MUST be refused (§5, `unknown_event_kind`). + +### 1.4 Dependence assumption + +| Value | Meaning | +|---|---| +| `unconstrained` | Nothing assumed. Returns the sharp interval. The honest default. | +| `independent` | A point value, returned as a degenerate interval. | +| `comonotone` | The upper Fréchet corner, as a degenerate interval. | +| `countermonotone` | **Defined only for exactly two events.** See §6. | + +Any other value MUST be refused (`unknown_dependence`). An implementation MUST +NOT default an unrecognized value to `unconstrained`. + +--- + +## 2. The unconstrained interval + +For marginals `p_1 … p_n`: + +**Conjunction** (`event = "all"`): + +``` +lower = max(0, (Σ p_i) − (n − 1)) +upper = min_i p_i +``` + +**Union** (`event = "any"`): + +``` +lower = max_i p_i +upper = min(1, Σ p_i) +``` + +These are the Fréchet–Hoeffding inequalities. The mathematics is classical — +Fréchet 1935, Hoeffding 1940 — and this specification claims no novelty for it. + +Both endpoints are **pointwise sharp**: for any fixed marginals, some joint +distribution attains each. Neither can be tightened without dependence evidence. + +Both endpoints MUST be clipped to `[0, 1]` after computation, to absorb +floating-point excursions at the boundary. + +### 2.1 The finding this encodes + +Under a conjunction the upper bound is `min_i p_i`, which does **not** depend on +`n`. A conjunction of controls is no stronger than its single strongest member. +Adding a further control cannot raise the upper bound and can only lower the +floor. Case family `cliff-homogeneous-m*` pins this: ten events at `p = 0.1` +still bound to `[0, 0.1]` while independence predicts `1e-10`. + +--- + +## 3. The independence baseline + +Computed and reported on **every** result, whatever the dependence assumption: + +``` +independence_point = Π p_i (conjunction) +independence_point = 1 − Π (1 − p_i) (union) +``` + +This is a **comparison baseline, never an answer.** It exists so the gap it +understates stays visible. An implementation MUST report it even when +`dependence = "independent"`, where it coincides with the interval. + +Derived quantities: + +``` +width = max(0, upper − lower) +independence_regret = upper − independence_point +understatement_factor = upper / independence_point, undefined when the point is 0 +``` + +`understatement_factor` MUST serialize as JSON `null` when +`independence_point` is zero. JSON has no infinity, and a null here means the +ratio is *undefined*, not large. + +--- + +## 4. `binding_event` + +The name of the event whose marginal the upper bound turns on, or `null`. + +This field is the actionable part of the result: it names where effort belongs. + +### 4.1 Conjunction + +The upper bound is `min_i p_i`, so the argmin binds — improving any other event +moves the upper bound not at all. + +### 4.2 Union + +The upper bound is `min(1, Σ p_i)`, a sum rather than a single event. No event +binds, and `binding_event` MUST be `null` — **except** when exactly one event is +present, where it is that event. + +### 4.3 Ties + +If two or more events attain the minimum (within tolerance), `binding_event` +MUST be `null`. Improving either leaves the bound pinned by the other, so naming +one of them would misdirect effort. + +### 4.4 Constrained regimes + +When `dependence` is anything other than `unconstrained`, `binding_event` MUST +be `null`. The interval is then a stipulated point, not a bound turning on an +event. + +--- + +## 5. Refusals + +An implementation MUST refuse, not default, in each case below. Refusal reasons +are the identifiers used by `expect_refusal` in +[`cases/adversarial.json`](cases/adversarial.json). + +| Reason | Condition | +|---|---| +| `marginal_out_of_range` | a marginal outside `[0, 1]` | +| `marginal_not_finite` | a marginal that is NaN or infinite | +| `no_events` | an empty marginals mapping | +| `unknown_event_kind` | an event kind not in §1.3 | +| `unknown_dependence` | a dependence value not in §1.4 | +| `countermonotone_undefined` | `countermonotone` with more than two events (§6) | + +**JSON note.** JSON has no NaN literal. In `cases/adversarial.json` the +non-finite marginal is encoded as the **string** `"NaN"`. An implementation +running the corpus MUST map that string to its language's NaN before calling the +kernel — and MUST NOT accept a string as a probability in normal operation. + +--- + +## 6. Countermonotonicity is bivariate + +`dependence = "countermonotone"` with `n > 2` MUST be refused. + +Countermonotonicity is a strictly bivariate concept. Two events can be perfectly +negatively dependent — one occurs exactly when the other does not — but three +cannot all be pairwise mutually exclusive and exhaustive in that way, and there +is no `n`-dimensional countermonotonic structure for `n > 2`. + +The matching fact about the bound: the Fréchet–Hoeffding lower bound +`max(0, Σ p_i − (n−1))` **is not a copula in dimension ≥ 3**. It remains +*pointwise sharp* — for any fixed marginals some joint distribution attains it — +but no single dependence structure attains it everywhere. So there is no +countermonotone regime to tabulate beside independence and comonotonicity. + +An implementation that returns a number here is wrong. Silently offering a +"countermonotone" option for four events produces a value with the form of a +dependence regime and none of the content. + +For `n = 2`: + +``` +countermonotone value = max(0, p_1 + p_2 − 1) (conjunction) +countermonotone value = min(1, p_1 + p_2) (union) +``` + +--- + +## 7. Tolerance + +Numeric comparisons use an absolute tolerance of **1e-12**, declared in +[`manifest.json`](manifest.json). + +This sits far above the worst disagreement measured between the reference +closed form and the finite-atom LP (3.3e-16 over 600 randomized cases), and far +below any difference that would change a reported bound. An implementation +needing a looser tolerance to pass has a defect, not a rounding difference. + +--- + +## 8. Running the corpus + +``` +cases/composition.json accept cases — reproduce every field in `expect` +cases/adversarial.json reject cases — refuse with the named reason +manifest.json digests, tolerance, case count, non-claims +``` + +For each accept case: call the kernel with `input`, compare every key in +`expect` within tolerance. `binding_event` is compared exactly. + +For each reject case: call the kernel with `input` and require a refusal. An +implementation SHOULD distinguish the reason; a corpus runner MAY accept any +refusal if the implementation does not expose typed reasons, but MUST record +that it did so. + +A runner MUST report `pass/fail` counts as exact integers. **No confidence +interval may be attached to a pass rate over this corpus** — it is the whole +population, not a sample, and its size is an authoring decision. + +--- + +## 9. Versioning + +`cc-kernel-v1` is stable. Adding a case is a **new corpus version**, not an edit +to this one — a passing implementation must not start failing because the corpus +grew underneath it. + +Fixing a case whose expected value was **wrong** is a defect correction, and +MUST be recorded in the corpus changelog with the reason. Regenerating the +corpus to match a changed implementation, without such a record, is the failure +mode this discipline exists to prevent. + +**Never weaken a case to make an implementation pass.** diff --git a/conformance/cc-kernel-v1/cases/adversarial.json b/conformance/cc-kernel-v1/cases/adversarial.json new file mode 100644 index 00000000..7584f739 --- /dev/null +++ b/conformance/cc-kernel-v1/cases/adversarial.json @@ -0,0 +1,107 @@ +{ + "case_count": 8, + "cases": [ + { + "description": "Countermonotonicity is strictly bivariate. The FH lower bound is not a copula in dimension >= 3, though it stays pointwise sharp. An implementation that returns a number here is wrong.", + "expect_refusal": "countermonotone_undefined", + "id": "reject-countermonotone-three-events", + "input": { + "dependence": "countermonotone", + "event": "all", + "marginals": { + "A": 0.3, + "B": 0.4, + "C": 0.5 + } + } + }, + { + "description": "Countermonotonicity is a relation between two events; with fewer than two there is nothing to be countermonotone with. Found by differential fuzz (seed 1): the Python reference raised IndexError and the Node implementation silently returned NaN. Both were wrong, differently.", + "expect_refusal": "countermonotone_undefined", + "id": "reject-countermonotone-one-event", + "input": { + "dependence": "countermonotone", + "event": "all", + "marginals": { + "A": 0.3 + } + } + }, + { + "description": "A marginal outside [0, 1] is not a probability.", + "expect_refusal": "marginal_out_of_range", + "id": "reject-marginal-above-one", + "input": { + "dependence": "unconstrained", + "event": "all", + "marginals": { + "A": 1.5, + "B": 0.2 + } + } + }, + { + "description": "A negative marginal is not a probability.", + "expect_refusal": "marginal_out_of_range", + "id": "reject-marginal-negative", + "input": { + "dependence": "unconstrained", + "event": "all", + "marginals": { + "A": -0.1, + "B": 0.2 + } + } + }, + { + "description": "A composition over zero events has no meaning.", + "expect_refusal": "no_events", + "id": "reject-empty-marginals", + "input": { + "dependence": "unconstrained", + "event": "all", + "marginals": {} + } + }, + { + "description": "Only conjunction and union are defined by this corpus.", + "expect_refusal": "unknown_event_kind", + "id": "reject-unknown-event-kind", + "input": { + "dependence": "unconstrained", + "event": "xor", + "marginals": { + "A": 0.1 + } + } + }, + { + "description": "An unrecognized dependence assumption must fail closed, not default.", + "expect_refusal": "unknown_dependence", + "id": "reject-unknown-dependence", + "input": { + "dependence": "sorta-dependent", + "event": "all", + "marginals": { + "A": 0.1, + "B": 0.2 + } + } + }, + { + "description": "NaN is not a probability. Note this case is expressed as a string in JSON because JSON has no NaN literal; see the spec.", + "expect_refusal": "marginal_not_finite", + "id": "reject-non-finite-marginal", + "input": { + "dependence": "unconstrained", + "event": "all", + "marginals": { + "A": "NaN", + "B": 0.2 + } + } + } + ], + "corpus": "cc-kernel-v1", + "kind": "reject" +} diff --git a/conformance/cc-kernel-v1/cases/composition.json b/conformance/cc-kernel-v1/cases/composition.json new file mode 100644 index 00000000..ba1b7bee --- /dev/null +++ b/conformance/cc-kernel-v1/cases/composition.json @@ -0,0 +1,512 @@ +{ + "case_count": 24, + "cases": [ + { + "description": "Classical two-event conjunction. Lower is max(0, sum-1)=0; upper is min(p)=0.1.", + "expect": { + "binding_event": "A", + "independence_point": 0.020000000000000004, + "lower": 0.0, + "understatement_factor": 4.999999999999999, + "upper": 0.1, + "width": 0.1 + }, + "id": "and-two-classical", + "input": { + "dependence": "unconstrained", + "event": "all", + "marginals": { + "A": 0.1, + "B": 0.2 + } + } + }, + { + "description": "Marginals summing above 1 force a positive lower bound: 0.9+0.8-1 = 0.7.", + "expect": { + "binding_event": "B", + "independence_point": 0.7200000000000001, + "lower": 0.7000000000000002, + "understatement_factor": 1.111111111111111, + "upper": 0.8, + "width": 0.09999999999999987 + }, + "id": "and-two-overlap-forced", + "input": { + "dependence": "unconstrained", + "event": "all", + "marginals": { + "A": 0.9, + "B": 0.8 + } + } + }, + { + "description": "Tied minima: no single event binds, so binding_event is null.", + "expect": { + "binding_event": null, + "independence_point": 0.0625, + "lower": 0.0, + "understatement_factor": 4.0, + "upper": 0.25, + "width": 0.25 + }, + "id": "and-two-equal-tie", + "input": { + "dependence": "unconstrained", + "event": "all", + "marginals": { + "A": 0.25, + "B": 0.25 + } + } + }, + { + "description": "A narrow interval from extreme marginals. Narrowness is not evidence of validity.", + "expect": { + "binding_event": "A", + "independence_point": 0.0099, + "lower": 0.0, + "understatement_factor": 1.01010101010101, + "upper": 0.01, + "width": 0.01 + }, + "id": "and-two-extreme-asymmetry", + "input": { + "dependence": "unconstrained", + "event": "all", + "marginals": { + "A": 0.01, + "B": 0.99 + } + } + }, + { + "description": "Classical two-event union. Lower is max(p)=0.2; upper is min(1, sum)=0.3.", + "expect": { + "binding_event": null, + "independence_point": 0.2799999999999999, + "lower": 0.2, + "understatement_factor": 1.0714285714285718, + "upper": 0.30000000000000004, + "width": 0.10000000000000003 + }, + "id": "or-two-classical", + "input": { + "dependence": "unconstrained", + "event": "any", + "marginals": { + "A": 0.1, + "B": 0.2 + } + } + }, + { + "description": "Union upper bound clipped at 1 because the marginals sum above it.", + "expect": { + "binding_event": null, + "independence_point": 0.94, + "lower": 0.8, + "understatement_factor": 1.0638297872340425, + "upper": 1.0, + "width": 0.19999999999999996 + }, + "id": "or-two-clipped", + "input": { + "dependence": "unconstrained", + "event": "any", + "marginals": { + "A": 0.7, + "B": 0.8 + } + } + }, + { + "description": "Four deterministic controls, assumed evasion rates. Reproduces a published external result: lower 0, upper 0.01, independence 1.2e-5, factor 833.", + "expect": { + "binding_event": null, + "independence_point": 1.2e-05, + "lower": 0.0, + "understatement_factor": 833.3333333333334, + "upper": 0.01, + "width": 0.01 + }, + "id": "and-four-vinctura-small-chapter", + "input": { + "dependence": "unconstrained", + "event": "all", + "marginals": { + "IMMUTABLE": 0.01, + "SELF_REPORTED": 0.3, + "SEPARATION": 0.4, + "STALE": 0.01 + } + } + }, + { + "description": "Same four controls, optimistic assumed rates. Tied minima at 0.01.", + "expect": { + "binding_event": null, + "independence_point": 5.000000000000001e-07, + "lower": 0.0, + "understatement_factor": 19999.999999999996, + "upper": 0.01, + "width": 0.01 + }, + "id": "and-four-vinctura-optimistic", + "input": { + "dependence": "unconstrained", + "event": "all", + "marginals": { + "IMMUTABLE": 0.01, + "SELF_REPORTED": 0.05, + "SEPARATION": 0.1, + "STALE": 0.01 + } + } + }, + { + "description": "Same four controls, pessimistic assumed rates. IMMUTABLE binds at 0.05.", + "expect": { + "binding_event": "IMMUTABLE", + "independence_point": 0.0015000000000000002, + "lower": 0.0, + "understatement_factor": 33.33333333333333, + "upper": 0.05, + "width": 0.05 + }, + "id": "and-four-vinctura-pessimistic", + "input": { + "dependence": "unconstrained", + "event": "all", + "marginals": { + "IMMUTABLE": 0.05, + "SELF_REPORTED": 0.5, + "SEPARATION": 0.6, + "STALE": 0.1 + } + } + }, + { + "description": "Degenerate single-event conjunction. The interval collapses to the marginal.", + "expect": { + "binding_event": "A", + "independence_point": 0.42, + "lower": 0.42, + "understatement_factor": 1.0, + "upper": 0.42, + "width": 0.0 + }, + "id": "and-single-event", + "input": { + "dependence": "unconstrained", + "event": "all", + "marginals": { + "A": 0.42 + } + } + }, + { + "description": "Degenerate single-event union. The interval collapses to the marginal.", + "expect": { + "binding_event": "A", + "independence_point": 0.41999999999999993, + "lower": 0.42, + "understatement_factor": 1.0000000000000002, + "upper": 0.42, + "width": 0.0 + }, + "id": "or-single-event", + "input": { + "dependence": "unconstrained", + "event": "any", + "marginals": { + "A": 0.42 + } + } + }, + { + "description": "A zero marginal pins the conjunction to zero regardless of the other event.", + "expect": { + "binding_event": "A", + "independence_point": 0.0, + "lower": 0.0, + "understatement_factor": null, + "upper": 0.0, + "width": 0.0 + }, + "id": "and-zero-marginal", + "input": { + "dependence": "unconstrained", + "event": "all", + "marginals": { + "A": 0.0, + "B": 0.5 + } + } + }, + { + "description": "Certain events. The conjunction is certain.", + "expect": { + "binding_event": null, + "independence_point": 1.0, + "lower": 1.0, + "understatement_factor": 1.0, + "upper": 1.0, + "width": 0.0 + }, + "id": "and-unit-marginals", + "input": { + "dependence": "unconstrained", + "event": "all", + "marginals": { + "A": 1.0, + "B": 1.0 + } + } + }, + { + "description": "Independence returns a degenerate interval at the product. A baseline, not an answer.", + "expect": { + "binding_event": null, + "independence_point": 0.12, + "lower": 0.12, + "understatement_factor": 1.0, + "upper": 0.12, + "width": 0.0 + }, + "id": "and-two-independent", + "input": { + "dependence": "independent", + "event": "all", + "marginals": { + "A": 0.3, + "B": 0.4 + } + } + }, + { + "description": "Independent union: 1 - (1-0.3)(1-0.4) = 0.58.", + "expect": { + "binding_event": null, + "independence_point": 0.5800000000000001, + "lower": 0.5800000000000001, + "understatement_factor": 1.0, + "upper": 0.5800000000000001, + "width": 0.0 + }, + "id": "or-two-independent", + "input": { + "dependence": "independent", + "event": "any", + "marginals": { + "A": 0.3, + "B": 0.4 + } + } + }, + { + "description": "The upper Frechet corner as a degenerate interval.", + "expect": { + "binding_event": null, + "independence_point": 0.12, + "lower": 0.3, + "understatement_factor": 2.5, + "upper": 0.3, + "width": 0.0 + }, + "id": "and-two-comonotone", + "input": { + "dependence": "comonotone", + "event": "all", + "marginals": { + "A": 0.3, + "B": 0.4 + } + } + }, + { + "description": "Defined only for exactly two events: max(0, 0.3+0.4-1) = 0.", + "expect": { + "binding_event": null, + "independence_point": 0.12, + "lower": 0.0, + "understatement_factor": 0.0, + "upper": 0.0, + "width": 0.0 + }, + "id": "and-two-countermonotone", + "input": { + "dependence": "countermonotone", + "event": "all", + "marginals": { + "A": 0.3, + "B": 0.4 + } + } + }, + { + "description": "Two-event countermonotone with a positive floor: 0.7+0.8-1 = 0.5.", + "expect": { + "binding_event": null, + "independence_point": 0.5599999999999999, + "lower": 0.5, + "understatement_factor": 0.8928571428571429, + "upper": 0.5, + "width": 0.0 + }, + "id": "and-two-countermonotone-positive", + "input": { + "dependence": "countermonotone", + "event": "all", + "marginals": { + "A": 0.7, + "B": 0.8 + } + } + }, + { + "description": "Correlation cliff at m=2: independence predicts 1e-2, the sharp upper bound stays 0.1. Stacking buys nothing under adversarial dependence.", + "expect": { + "binding_event": null, + "independence_point": 0.010000000000000002, + "lower": 0.0, + "understatement_factor": 9.999999999999998, + "upper": 0.1, + "width": 0.1 + }, + "id": "cliff-homogeneous-m2", + "input": { + "dependence": "unconstrained", + "event": "all", + "marginals": { + "G0": 0.1, + "G1": 0.1 + } + } + }, + { + "description": "Correlation cliff at m=3: independence predicts 1e-3, the sharp upper bound stays 0.1. Stacking buys nothing under adversarial dependence.", + "expect": { + "binding_event": null, + "independence_point": 0.0010000000000000002, + "lower": 0.0, + "understatement_factor": 99.99999999999999, + "upper": 0.1, + "width": 0.1 + }, + "id": "cliff-homogeneous-m3", + "input": { + "dependence": "unconstrained", + "event": "all", + "marginals": { + "G0": 0.1, + "G1": 0.1, + "G2": 0.1 + } + } + }, + { + "description": "Correlation cliff at m=4: independence predicts 1e-4, the sharp upper bound stays 0.1. Stacking buys nothing under adversarial dependence.", + "expect": { + "binding_event": null, + "independence_point": 0.00010000000000000003, + "lower": 0.0, + "understatement_factor": 999.9999999999998, + "upper": 0.1, + "width": 0.1 + }, + "id": "cliff-homogeneous-m4", + "input": { + "dependence": "unconstrained", + "event": "all", + "marginals": { + "G0": 0.1, + "G1": 0.1, + "G2": 0.1, + "G3": 0.1 + } + } + }, + { + "description": "Correlation cliff at m=5: independence predicts 1e-5, the sharp upper bound stays 0.1. Stacking buys nothing under adversarial dependence.", + "expect": { + "binding_event": null, + "independence_point": 1.0000000000000004e-05, + "lower": 0.0, + "understatement_factor": 9999.999999999996, + "upper": 0.1, + "width": 0.1 + }, + "id": "cliff-homogeneous-m5", + "input": { + "dependence": "unconstrained", + "event": "all", + "marginals": { + "G0": 0.1, + "G1": 0.1, + "G2": 0.1, + "G3": 0.1, + "G4": 0.1 + } + } + }, + { + "description": "Correlation cliff at m=8: independence predicts 1e-8, the sharp upper bound stays 0.1. Stacking buys nothing under adversarial dependence.", + "expect": { + "binding_event": null, + "independence_point": 1.0000000000000005e-08, + "lower": 0.0, + "understatement_factor": 9999999.999999996, + "upper": 0.1, + "width": 0.1 + }, + "id": "cliff-homogeneous-m8", + "input": { + "dependence": "unconstrained", + "event": "all", + "marginals": { + "G0": 0.1, + "G1": 0.1, + "G2": 0.1, + "G3": 0.1, + "G4": 0.1, + "G5": 0.1, + "G6": 0.1, + "G7": 0.1 + } + } + }, + { + "description": "Correlation cliff at m=10: independence predicts 1e-10, the sharp upper bound stays 0.1. Stacking buys nothing under adversarial dependence.", + "expect": { + "binding_event": null, + "independence_point": 1.0000000000000006e-10, + "lower": 0.0, + "understatement_factor": 999999999.9999995, + "upper": 0.1, + "width": 0.1 + }, + "id": "cliff-homogeneous-m10", + "input": { + "dependence": "unconstrained", + "event": "all", + "marginals": { + "G0": 0.1, + "G1": 0.1, + "G2": 0.1, + "G3": 0.1, + "G4": 0.1, + "G5": 0.1, + "G6": 0.1, + "G7": 0.1, + "G8": 0.1, + "G9": 0.1 + } + } + } + ], + "corpus": "cc-kernel-v1", + "kind": "accept", + "tolerance": 1e-12 +} diff --git a/conformance/cc-kernel-v1/manifest.json b/conformance/cc-kernel-v1/manifest.json new file mode 100644 index 00000000..72dd3b55 --- /dev/null +++ b/conformance/cc-kernel-v1/manifest.json @@ -0,0 +1,14 @@ +{ + "corpus": "cc-kernel-v1", + "files": { + "cases/adversarial.json": "a394f6220cf04586f20ba1b504e01e51dc0810d8f1c0d79667b2a8b697ff71d4", + "cases/composition.json": "71740d53edd0b9e9fb297e6b84a6d1c508c1ee2c5fde27e6bf9b496e623b7492" + }, + "non_claims": [ + "Passing this corpus establishes that an implementation computes the same intervals as the reference on these cases. It does not establish that either implementation is correct: two implementations can share a misreading.", + "The corpus is a curated census, not a sample. Its size is an authoring decision, it carries no coverage claim, and no confidence interval may be attached to a pass rate over it.", + "Passing establishes nothing about safety, calibration, threshold choice, or whether the supplied marginals mean anything." + ], + "tolerance": 1e-12, + "total_cases": 32 +} diff --git a/docs/api.md b/docs/api.md index 6acd1a0d..58fd1ff0 100644 --- a/docs/api.md +++ b/docs/api.md @@ -19,6 +19,52 @@ The supported stable kernel symbols are the names exported by `independence_regret`, `independent_event_probability`, endpoint witness result types, and finite-sample count-to-constraint helpers. +`cc.compose` is the stable composition surface for named binary events. It +takes marginal probabilities keyed by event name and returns a sharp identified +interval, with the independence baseline, the binding event, the marginal +provenance, and the non-claims attached to the result. Its supported symbols are +the names exported by `cc.compose.__all__`, including `compose_bounds`, +`sensitivity`, `CompositionBounds`, `CountermonotoneUndefinedError`, and +`marginal_from_operating_point`. + +The surface deliberately contains no ROC, Youden, threshold, or operating-point +concept: a deterministic refusal rule is as valid an input as a tuned +classifier, and detectors reach the same surface through +`marginal_from_operating_point`. That direction is part of the contract and is +enforced by a test. + +Its behavior is pinned by the language-agnostic corpus in +`conformance/cc-kernel-v1/`, whose `SPEC.md` is the normative statement. +Changing a value the corpus pins is a breaking change to this contract. + +`cc.compose` does not certify safety, validate the supplied marginals, or claim +novelty for the Frechet-Hoeffding inequality, which is classical. + +`cc.cli.guard` (the `cc-guard` console script) is the stable surface for the +inference guards. Its `check` subcommand reads a JSON request on stdin and +writes a verdict on stdout; its `table` subcommand emits the decision rules as +pure data for callers that cannot spawn a Python process. `DECISION_TABLE` is +the normative statement of the rules and is asserted to agree with the +implementation. + +A permitted verdict means the guard found no reason to refuse. It does not mean +an estimate is correct or a measurement was well designed. + +`cc.evidence_card` is the stable surface for emitting `cc.evidence_card.v1` +cards and the `cc.site_evidence_manifest.v1` bundle. Its supported symbols are +the names exported by `cc.evidence_card.__all__`, including `EvidenceCard`, +`ArtifactRef`, `cards_to_site_manifest`, and `render_labels`. + +Its three labels — `evidence_state`, `verdict`, and `publication_state` — are +orthogonal and part of the contract. No consumer may collapse them into a single +status, and the module deliberately provides no composite property, no aggregate +field, and no single-label renderer. Adding one is a breaking change to this +contract, not a convenience. + +`cc.evidence_card` does not establish that any claim is true. A card records a +claim, the command that tests it, what would falsify it, and what it does not +claim. + The evidence/reporting surface is stable only where it is used for release evidence and claim-governance capsules: @@ -50,7 +96,7 @@ guarantees of `cc.kernel.strict`: - `cc.exp`: experiment runners and two-world workflow support. - `cc.guardrails`: local toy guardrail implementations. - `cc.io`, `cc.cli`, and `cc.utils`: storage, manifest, plotting, dashboard, and - utility helpers. + utility helpers, outside the named stable `cc.cli.guard` behavior above. - `cc._legacy`: retained only for backward compatibility and migration context. Experimental/backcompat APIs may change without compatibility guarantees unless diff --git a/docs/architecture/CANONICAL_PROFILE.md b/docs/architecture/CANONICAL_PROFILE.md new file mode 100644 index 00000000..bd3620d0 --- /dev/null +++ b/docs/architecture/CANONICAL_PROFILE.md @@ -0,0 +1,212 @@ +# Canonicalization Profiles + +> **Status: implemented.** `cc.canonical.v2` is the default for artifacts +> written by this repository. `cc.canonical.v1` is retained read-only. +> +> **Non-claim.** A canonicalization profile determines which documents share a +> receipt identity. It does not establish that a document is true, that its +> producer was honest, or that the system it describes is safe. A receipt binds +> bytes, not intentions. + +A receipt identifies a document only up to the **kernel** of its canonicalizer: +the set of distinct documents that receive the same canonical bytes. Every +member of that kernel is a pair of documents a receipt cannot tell apart. The +job of a profile is to keep that kernel equal to JSON equality — no larger, and +no smaller. + +Every receipt names its profile in `receipt.canonicalization_method`, and +verification dispatches on that name rather than assuming the current default. + +--- + +## The two profiles + +| | `cc.canonical.v1` | `cc.canonical.v2` | +|---|---|---| +| Identifier | `json.dumps(sort_keys=True,separators=(',', ':'),ensure_ascii=False,allow_nan=False); receipt.canonical_hash excluded` | `cc.canonical.v2/RFC8785; receipt.canonical_hash excluded` | +| Status | **read-only**, historical | **default** for new artifacts | +| Basis | Python `json.dumps` conventions | RFC 8785 (JCS) | +| Key order | Unicode code point | **UTF-16 code unit** | +| Unicode normalization | NFC applied to keys and values | **none** | +| Number form | Python `repr` | **ECMAScript `Number::toString`** | +| Integer domain | unbounded Python `int` | **IEEE-754 safe range** | +| Unpaired surrogates | emitted | **refused** | +| Cross-language verifiable | no | yes | + +--- + +## Why v1 was replaced + +An adversarial census — `scripts/canonicalization_probe.py`, modelled on +Ghost-Ark's E1 provenance-kernel census — found two `unintended-kernel` classes +in v1: documents a consumer needs distinguished that received identical bytes. + +### The silent key merge + +v1 applied `unicodedata.normalize("NFC", …)` to every mapping key and wrote the +results into a fresh dict. Two byte-distinct keys sharing an NFC form silently +became one: + +```python +>>> canonical_json_bytes({"é": 1, "é": 2}, profile=LEGACY_SORT_KEYS) +b'{"\xc3\xa9":2}' +``` + +Two keys in, one key out, **no exception**. The receipt then attested faithfully +to a document with a field missing. The same collapse occurred nested inside an +object, which a guard inspecting only top-level keys would have missed. + +Recorded as [F-03](../upgrade/FINDINGS_REGISTER.md#f-03). + +### Bytes no other language could agree on + +v1 emitted Python `repr` number forms — `1.0`, `-0.0`, `1e-07` — which diverge +from RFC 8785 on five of six probed forms. A verifier written in any other +language computes different bytes for the same document, so independent +verification was impossible: not because of a bug, but because the two sides +never agreed on what the bytes are. + +v1 also emitted Python integers of unbounded size. A JavaScript `JSON.parse` +collapses values beyond 2⁵³ before any verifier code runs — the kernel is set by +the parser, and no downstream fix reaches it. + +Recorded as [F-04](../upgrade/FINDINGS_REGISTER.md#f-04) and +[F-05](../upgrade/FINDINGS_REGISTER.md#f-05). + +--- + +## What v2 does + +### It does not normalize + +RFC 8785 is explicit that Unicode normalization is the **producer's** +responsibility, not the canonicalizer's. v2 follows it. + +This is the fix for F-03, and it is a fix by construction rather than by patch: +two keys that differ in Unicode form are two keys, which is what JSON says they +are, so there is nothing left to collide. A canonicalizer that mutates content +is not a canonicalizer. + +Producers that would rather refuse such a document can call +`assert_no_confusable_keys` before hashing. It is deliberately **not** on the +hash path — v1's mistake was exactly that a content-altering rule lived inside +canonicalization. + +### Numbers follow ECMAScript + +`Number::toString`, as RFC 8785 requires. Python's `repr` already yields the +shortest round-tripping digits; what v2 supplies is the *formatting* of those +digits. + +| Value | v1 emitted | v2 emits | +|---|---|---| +| `1.0` | `1.0` | `1` | +| `-0.0` | `-0.0` | `0` | +| `1e-7` | `1e-07` | `1e-7` | +| `100.0` | `100.0` | `100` | +| `1e20` | `1e+20` | `100000000000000000000` | +| `1e21` | `1e+21` | `1e+21` | + +The `-0.0` case is not academic: `scipy.optimize.linprog` returns a negative +zero at a lower bound of zero, so under v1 two numerically identical reports +could receive different receipts. + +### Keys sort by UTF-16 code unit + +Not by code point. The orders disagree above the BMP: U+10000 encodes as the +surrogate pair D800 DC00, so it sorts *before* U+FFFD under UTF-16 and *after* +it under code point. Comparing big-endian UTF-16 bytes reproduces the required +order exactly. + +### One declared narrowing: the integer domain + +RFC 8785 is defined over JSON numbers, which are IEEE-754 doubles. Python's +`int` has no such bound, so a profile must supply one. + +v2 refuses any integer with `|n| > 2**53 - 1`. That bound is the standard +`MAX_SAFE_INTEGER` — the largest *n* for which both *n* and *n+1* are exactly +representable — which conservatively also refuses 2⁵³ itself. Floats are +unaffected: they are already doubles, so `1e300` serializes normally. + +**This is a narrowing of JCS, and it is declared here rather than left as an +undocumented difference.** A caller needing a larger integer must carry it as a +string. Refusing is the fail-closed choice: emitting bytes that cannot survive a +round trip through a conforming parser would produce a receipt no one else can +check. + +--- + +## Migration + +The change altered every receipt hash in the repository. Two things made that +safe to do: + +1. **Verification dispatches on the declared profile.** A pre-migration receipt + names v1 and is recomputed under v1. Recomputing it under RFC 8785 would + report a mismatch for a report that is in fact intact — the opposite of what + a receipt is for. Breaking historical receipts to fix the canonicalizer would + trade one integrity failure for another. +2. **The regenerated artifacts were diffed before being accepted.** Every file + in the claim-governance capsule was compared with hashes, hash-derived ids, + and the profile identifier scrubbed. All eleven were byte-identical under + that scrub: no content, no claim text, and no non-claim changed. Only the + hashes moved. + +`cc.canonical.v1` is not deprecated-but-available. It is **read-only**: nothing +in the repository writes it, and a test asserts that its known defects are still +present. If v1 were ever "fixed", every pre-migration receipt would silently +become unverifiable. + +--- + +## Verification + +```bash +make canon-probe # the adversarial census, both profiles +pytest tests/unit/canonical +``` + +The census gates on v2 only. v1 is **expected to fail** — that is why it was +replaced — and a test asserts it still does. + +Current v2 census: **12 sound, 1 fail-closed, 1 sound-by-rejection**; zero +`unintended-kernel`, zero `rejection-asymmetry`, zero `over-discrimination`; 15 +of 15 number forms conformant. + +### One intent was corrected during this work + +The class `int-vs-float-same-value` was originally declared with intent +`distinct`. That declaration described Python's type system, not JSON's: JSON +has exactly one number type, so `1` and `1.0` are the same JSON number, and a +consumer needing them distinguished is asking JSON for something it does not +provide. The intent is now `equivalent`. + +Correcting a declaration because it was wrong about the domain is legitimate. +Correcting one to flatter a measured result is not. This was the former, and it +is recorded in the probe source and here rather than edited away. + +--- + +## What a clean census does not establish + +That the kernel has no other members. The corpus is **curated** — its coverage +is an authoring decision, not a measurement — and it carries no confidence +interval, because it is the whole population rather than a sample. + +Known gaps, stated so absence is not read as a null result: + +- **No fuzzing of the canonicalizer.** Every class was authored by hand. +- **No cross-language differential test on receipts.** `conformance/cc-kernel-v1/` + establishes agreement on the *composition kernel*; nothing yet re-canonicalizes + a CC report in another language and compares digests. That is the natural next + step now that v2 makes it possible, and until it runs, "cross-language + verifiable" describes the profile's design rather than a demonstrated result. +- **No external reviewer** has attacked either profile. + +## Binding rules + +- **Never weaken a corpus class to make a test pass.** A class that starts + failing is a kernel regression. +- **Never change a profile in place.** A new canonicalization is a new profile + identifier, so existing receipts keep verifying. +- **Never write v1.** It exists to read history. diff --git a/docs/architecture/EVIDENCE_CARDS.md b/docs/architecture/EVIDENCE_CARDS.md new file mode 100644 index 00000000..c58a3b67 --- /dev/null +++ b/docs/architecture/EVIDENCE_CARDS.md @@ -0,0 +1,192 @@ +# Evidence Cards + +> **Status: implemented.** `cc.evidence_card` emits `cc.evidence_card.v1` cards +> and a `cc.site_evidence_manifest.v1` bundle from the claim-boundary manifest. +> +> **Non-claim.** A card lists a claim and the command that tests it. It does not +> establish that the claim is true. A card carrying `verdict: pass` establishes +> only that the named command exited successfully on some host — nothing about +> deployment, safety, or the world. + +An evidence card is a claim together with everything a reader needs in order to +**disagree** with it: how it was operationalized, which command reproduces it, +what would falsify it, what it assumes, and what it explicitly does not claim. + +This repository emits cards. It does not host an Atlas, a site, or any surface +that renders them — those live elsewhere. What is here is the instrument. + +--- + +## The object + +``` +claim → maturity → source revision → artifact digests → command + → result → falsifier → counterevidence → assumptions → non-claims +``` + +Two fields are required that most claim registries treat as optional: + +- **`falsifier`** — what observation would show the claim is wrong. A claim with + no falsifier is an assertion, not evidence, and `EvidenceCard.__init__` + refuses it. +- **`non_claims`** — what the card explicitly does not establish. Also required. + +Both are now mandatory *in the claim-boundary manifest itself*, enforced by +`scripts/validate_claim_boundary_manifest.py`. Every claim in this repository +carries a falsifier because the validator will not accept one that does not. + +--- + +## Three labels that never collapse + +``` +evidence_state local-only | aws-synth-only | aws-live | illustrative +verdict pass | fail | unverifiable | not-run +publication_state draft | released | superseded | retracted +``` + +They answer different questions and **none implies another**: + +- A `pass` that is `local-only` says nothing about deployed behaviour. +- `aws-live` evidence can record a `fail` — provenance is not quality. +- A `retracted` card may still carry a `pass`: the run happened, and the claim + was withdrawn anyway. +- `released` says a card was reviewed for release, not that its claim is true. + +This orthogonality is enforced, not merely documented: + +| Enforcement | Where | +|---|---| +| No `status`, `score`, `is_ok`, `badge`, `health`, `overall` on the card | a test asserts each is absent | +| No aggregate field in the site manifest | a test asserts absence there too | +| No single-label renderer | `render_labels` returns all three or raises | +| Schema defines no aggregate | asserted against the committed schema | +| All 64 label combinations constructible | a test builds every one | + +The last row is the important one. If a future change ever makes one label +constrain another — "a retracted card cannot be passing", say — that test +fails. The point of a failure museum is that withdrawn claims keep their +results visible. + +Every label value ships with a **meaning string** rendered beside it, so a bare +token in a UI is never left to interpretation: + +> `not-run` — The command has not been executed for this card. No result is +> claimed. + +--- + +## `not-run` is the default, and that is the design + +A card cannot acquire a passing verdict by being written confidently. The +generator emits `not-run` unless `--run` is passed, and `--run` executes the +commands and records what actually happened. + +The committed cards in `evidence-cards/` are always the **not-run scaffold**. A +test asserts this: `pass` and `fail` counts must both be zero in the committed +manifest. Verdicts are host-specific, and a repository that ships someone's +laptop results as published evidence is doing the thing this object exists to +prevent. + +--- + +## `unverifiable` is not `fail` + +The distinction earned itself on the first run of the generator. + +`make test-kernel` exited non-zero because its dependency-install step could not +reach the network. The kernel tests never executed. The first version of the +harness mapped any non-zero exit to `fail`, and so reported a **failure nobody +observed**. + +The harness now never guesses. A non-zero exit becomes `fail` only when it can +be attributed to a check running and not succeeding — pytest exit code 1, whose +meaning is documented. Codes 2–5 mean usage error, internal error, +interruption, or no tests collected; in each the check did not run. Everything +else is `unverifiable`, with the reason recorded. + +Per-command outcomes are preserved rather than collapsed: + +``` +[pass=1, unverifiable=1] pytest tests/unit/kernel/test_classical_frechet_special_cases.py + -> exit 0: 6 passed in 0.63s | make test-kernel -> exit 2: ... + | exit not attributable to a check running +``` + +The card verdict is the worst outcome present — conservative — but a reader can +still see that the tests passed and only the `make` target was unrunnable. + +--- + +## What the generator does + +```bash +python scripts/build_evidence_cards.py # emit the not-run scaffold +python scripts/build_evidence_cards.py --run # execute commands, record results +python scripts/build_evidence_cards.py --check # verify the committed scaffold is current +``` + +It reads `docs/claims/claim_boundary_manifest.v0.1.json`, binds each claim's +supporting files by SHA-256, records the git revision (with a `-dirty` suffix +when the working tree differs — a card bound to a revision the tree does not +match is bound to the wrong thing), and writes one card per claim plus the site +manifest. + +Because cards bind their supporting files **by digest**, editing any of those +files makes the committed cards stale. Regenerate as the last step before +committing; `--check` runs in CI and will catch it otherwise. That staleness is +the feature working: a card whose artifact digests no longer match the tree is +describing a repository that no longer exists. + +Under `--run`, commands are executed only if they match a small allowlist +(`pytest `, `python `, `make `). A card generator that shells out arbitrary +strings from a data file is a code-execution surface, not an evidence tool. +Anything outside the allowlist is recorded `unverifiable` with that as the +reason. + +--- + +## Current state + +Eight claims, from the claim-boundary manifest: + +| Label | Counts | +|---|---| +| `evidence_state` | `local-only` 7, `aws-synth-only` 1 | +| `verdict` | `not-run` 8 | +| `publication_state` | `draft` 8 | + +The single `aws-synth-only` card is the enterprise reference, whose evidence is +moto emulation. Its assumptions say so on the card: *"Emulated evidence is never +live evidence, and no live-AWS claim rests on this lane."* Nothing in this +repository is `aws-live`, and nothing should claim to be. + +All eight are `draft`. Release is a human decision in a publication workflow, +not a side effect of generation. + +--- + +## What this does not do + +Stated so absence is not read as capability: + +- **No Atlas, site, or renderer.** This repository emits cards. Anything that + displays them is a separate surface with its own design and its own review. +- **No hosting, no AWS, no deployment.** Nothing here provisions or publishes + anything. +- **No cross-repository aggregation.** The cards describe *this* repository's + claims. Combining cards from several projects into one view is a different + problem with different failure modes, and none of them are solved here. +- **No verdict earned by generation.** The committed scaffold claims no results. +- **No commercial material.** Pricing, go-to-market, and venture planning are + out of scope for a research repository, and the sibling institutional + repository enforces the equivalent rule with a test. + +## Binding rules + +- **Never add a composite status.** Not to the card, the manifest, or the + schema. The tests are the tripwire; do not edit them to make one pass. +- **Never commit run verdicts.** The committed scaffold is `not-run`. +- **Never let a claim into the manifest without a falsifier.** The validator + refuses it; do not relax the validator. +- **`unverifiable` is never rounded to `fail` or `pass`.** It is its own fact. diff --git a/docs/claims/claim_boundary_manifest.v0.1.json b/docs/claims/claim_boundary_manifest.v0.1.json index 9d170fd8..57e55d18 100644 --- a/docs/claims/claim_boundary_manifest.v0.1.json +++ b/docs/claims/claim_boundary_manifest.v0.1.json @@ -1,7 +1,7 @@ { "schema_version": "cc.claim_boundary_manifest.v0.1", "generated_for": "cc-framework", - "purpose": "Map public CC-Framework claims to evidence, validation lanes, supporting files, tests or commands, and explicit non-claims so that public communication remains evidence-bound.", + "purpose": "Map public CC-Framework claims to evidence, validation lanes, supporting files, tests or commands, and explicit non-claims so that public communication remains evidence-bound. Every claim additionally carries a falsifier, its assumptions, and an evidence state, so that it can be rendered as an evidence card without the renderer inventing any of them.", "claim_levels": { "C0": "Repository Metadata Claim: basic facts about files, docs, package metadata, or repository organization.", "C1": "Structural Validation Claim: a schema, report, or artifact conforms to declared validation rules.", @@ -24,7 +24,7 @@ "src/cc/kernel/sensitivity.py" ], "supporting_tests_or_commands": [ - "tests/unit/kernel/test_classical_frechet_special_cases.py", + "pytest tests/unit/kernel/test_classical_frechet_special_cases.py", "make test-kernel" ], "non_claims": [ @@ -32,7 +32,15 @@ "Does not prove deployment safety.", "Does not prove marginals are representative." ], - "risk_if_overstated": "Could be misrepresented as a safety certification or as evidence that exact singleton marginals identify the true deployment risk." + "risk_if_overstated": "Could be misrepresented as a safety certification or as evidence that exact singleton marginals identify the true deployment risk.", + "evidence_state": "local-only", + "falsifier": "A marginal configuration in which identified_region returns an interval differing from max(0, sum(p) - (m-1)) and min(p) by more than the declared solver tolerance.", + "assumptions": [ + "Exact singleton marginals are supplied; they are not estimated here.", + "No side constraints beyond the marginals.", + "Finite binary atom space with deterministic atom ordering.", + "IEEE-754 double arithmetic; agreement is asserted only to the declared tolerance." + ] }, { "id": "kernel.identified_interval", @@ -46,8 +54,8 @@ "docs/api.md" ], "supporting_tests_or_commands": [ - "tests/unit/kernel/test_sensitivity.py", - "tests/unit/kernel/test_monotonic_tightening.py", + "pytest tests/unit/kernel/test_sensitivity.py", + "pytest tests/unit/kernel/test_monotonic_tightening.py", "make test-kernel" ], "non_claims": [ @@ -55,7 +63,14 @@ "Does not validate semantic safety.", "Does not establish representativeness." ], - "risk_if_overstated": "Could turn a conditional mathematical identification result into an unsupported empirical or deployment claim." + "risk_if_overstated": "Could turn a conditional mathematical identification result into an unsupported empirical or deployment claim.", + "evidence_state": "local-only", + "falsifier": "A feasible atom distribution whose query value falls outside the reported interval, or a value strictly inside the interval attained by no feasible distribution.", + "assumptions": [ + "The constraint set defines a nonempty feasible polytope.", + "Constraints and the query are linear in the atom probabilities.", + "The LP solver reports optimal status rather than a tolerance-limited stop." + ] }, { "id": "kernel.finite_sample_outer_interval", @@ -70,15 +85,23 @@ "src/cc/kernel/sensitivity.py" ], "supporting_tests_or_commands": [ - "tests/unit/kernel/test_finite_sample_constraints.py", - "tests/unit/kernel/test_sample_complexity.py", + "pytest tests/unit/kernel/test_finite_sample_constraints.py", + "pytest tests/unit/kernel/test_sample_complexity.py", "make test-kernel" ], "non_claims": [ "Not valid after uncorrected adaptive target selection.", "Not a deployment certificate." ], - "risk_if_overstated": "Could imply finite-sample uncertainty handling proves future behavior or deployment safety outside the named assumptions." + "risk_if_overstated": "Could imply finite-sample uncertainty handling proves future behavior or deployment safety outside the named assumptions.", + "evidence_state": "local-only", + "falsifier": "A simulation under the stated iid Bernoulli sampling in which count-derived intervals cover the true query value at a rate materially below the nominal level.", + "assumptions": [ + "Labels and the query are fixed before the data are seen.", + "Counts are iid Bernoulli draws from the target population.", + "Moment coverage holds simultaneously across constraints.", + "The exact assumptions supplied to the LP are true of that population." + ] }, { "id": "reporting.receipt_integrity", @@ -95,8 +118,8 @@ "src/cc/evidence/anchoring.py" ], "supporting_tests_or_commands": [ - "tests/unit/reporting/test_reporting.py", - "tests/unit/evidence/test_transparency_log_adversarial.py", + "pytest tests/unit/reporting/test_reporting.py", + "pytest tests/unit/evidence/test_transparency_log_adversarial.py", "PYTHONPATH=src .venv/bin/python -m pytest -q tests/unit/evidence tests/unit/reporting" ], "non_claims": [ @@ -106,7 +129,14 @@ "Does not prove compliance.", "Does not prove safety." ], - "risk_if_overstated": "Could convert byte integrity or provenance into an unsupported empirical validity, compliance, or safety claim." + "risk_if_overstated": "Could convert byte integrity or provenance into an unsupported empirical validity, compliance, or safety claim.", + "evidence_state": "local-only", + "falsifier": "Two byte-distinct payloads that a consumer needs distinguished receiving the same canonical hash, or a payload whose recomputed hash differs from its receipt while its bytes are unchanged.", + "assumptions": [ + "SHA-256 remains collision resistant.", + "The canonicalization profile named on the receipt is the one used to verify it.", + "Receipt-covered JSON is read with strict duplicate-key rejection." + ] }, { "id": "evidence.non_claim_boundaries", @@ -130,7 +160,13 @@ "non_claims": [ "Does not prevent bad-faith actors from making misleading claims outside the framework." ], - "risk_if_overstated": "Could imply documentation controls alone prevent institutional overclaiming or misuse outside the repository." + "risk_if_overstated": "Could imply documentation controls alone prevent institutional overclaiming or misuse outside the repository.", + "evidence_state": "local-only", + "falsifier": "An evidence-bound claim package that passes verify-claim-governance while omitting a non-claim its declared evidence roles require.", + "assumptions": [ + "The producer declares evidence roles honestly; the verifier checks structure, not sincerity.", + "The v0 non-claims engine detects the intended substance rather than only exact strings." + ] }, { "id": "release.paper_core_v0_3_rc1", @@ -155,7 +191,13 @@ "non_claims": [ "Does not promote enterprise, dashboard, vendor, cloud, or experimental lanes into paper core." ], - "risk_if_overstated": "Could present a release-candidate paper-core boundary as whole-repository production quality." + "risk_if_overstated": "Could present a release-candidate paper-core boundary as whole-repository production quality.", + "evidence_state": "local-only", + "falsifier": "A paper artifact that reproduce_paper.py regenerates with different bytes, or a theorem-ledger row whose named test does not exist or does not exercise the stated implementation.", + "assumptions": [ + "The pinned environment and seeds reproduce the recorded run.", + "Artifact determinism is a property of this host and toolchain, not of the mathematics." + ] }, { "id": "enterprise.reference_v0_1", @@ -180,7 +222,13 @@ "Not compliance certification.", "Not deployment safety." ], - "risk_if_overstated": "Could imply a moto-backed reference path is production infrastructure, live-cloud validation, compliance certification, or deployment safety." + "risk_if_overstated": "Could imply a moto-backed reference path is production infrastructure, live-cloud validation, compliance certification, or deployment safety.", + "evidence_state": "aws-synth-only", + "falsifier": "An enterprise emulation run that reports success while omitting a KMS verification or object-integrity check the reference claims to perform.", + "assumptions": [ + "moto emulates the AWS behaviours under test faithfully enough for the checks performed.", + "Emulated evidence is never live evidence, and no live-AWS claim rests on this lane." + ] }, { "id": "open_core.strategy", @@ -199,7 +247,13 @@ "Does not make the project a product today.", "Does not claim enterprise readiness." ], - "risk_if_overstated": "Could turn a strategic boundary document into an unsupported claim that a commercial product already exists." + "risk_if_overstated": "Could turn a strategic boundary document into an unsupported claim that a commercial product already exists.", + "evidence_state": "local-only", + "falsifier": "A module the strategy document places in the open core that is absent from the published wheel, or an importable wheel surface classified nowhere in docs/api.md.", + "assumptions": [ + "The wheel built in CI is the artifact the strategy describes.", + "Classification in docs/api.md is kept current by the packaging boundary test." + ] } ], "forbidden_upgrades": [ diff --git a/docs/index.md b/docs/index.md index 5644a127..9a9b7acc 100644 --- a/docs/index.md +++ b/docs/index.md @@ -96,6 +96,9 @@ The experiment flow is illustrated in * [Repository Migration Manifest](release/MIGRATION_MANIFEST.md) * [Audit and Evidence Boundaries](architecture/AUDIT_EVIDENCE_BOUNDARIES.md) * [CC Reports and Receipts](research/CC_REPORTS.md) +* [Future Expansion: Epistemic Research Upgrade](research/future-expansion/README.md) +* [The Epistemic Program](research/epistemic-program/README.md) +* [Public Positioning](product/PUBLIC_POSITIONING.md) * [Experiments Guide](experiments-guide.md) * [Reproducibility Notes](reproducibility.md) diff --git a/docs/product/PUBLIC_POSITIONING.md b/docs/product/PUBLIC_POSITIONING.md new file mode 100644 index 00000000..db8994d1 --- /dev/null +++ b/docs/product/PUBLIC_POSITIONING.md @@ -0,0 +1,172 @@ +# Public Positioning + +## Status + +Positioning and communication guidance. This document is a C0 repository/strategy +artifact. It creates no claim, promotes no surface into Paper Core, and is not +evidence for anything. Where it describes offers or a schedule, those are +intentions, not commitments already met. + +## The idea + +```text +Before a green check becomes a claim, make its assumptions, evidence, +and limits inspectable. +``` + +That is not safety branding. It is a public method for turning impressive +technical promises into questions that can genuinely fail. + +## The category + +Not a generic security company. A **public laboratory for doubt**. + +The advantage is not "we have the answer." The advantage is **"we make the +answer show its work."** + +## Lockup and voice + +```text +Pranav Bhave / Cubits11 +Make the claim smaller. Make the evidence stronger. +``` + +Flagship: + +```text +A story can start a question. +Evidence must finish the answer. + +Cubits11 +Keep the wonder. Check the claim. +``` + +Bio: + +```text +I build research software and public experiences that turn vague technical +claims into inspectable evidence. +``` + +## Name boundaries + +| Name | Public role | Boundary | +| --- | --- | --- | +| **Pranav Bhave** | The author, builder, and speaker. | The human voice. | +| **Cubits11** | Independent studio for evidence design, research software, and education. | The commercial home. Sells clarity, never certainty. | +| **CC-Framework** | Research on what stacked guardrail evidence supports when failures may be correlated. | Not a safety score and not a certification. | +| **Ghost-Ark** | A separate institutional research artifact on bounded receipts and verifiable evidence. | Not a Cubits11 product and not a university endorsement. | + +**The Ghost-Ark rule is strict.** It stays out of Cubits11 sales material +entirely. Its own public-interface rules prohibit commercial planning and +unapproved institutional representation. Reference it as a clearly labelled +research case study or not at all. The film's `ghost-ark` cut carries no +Cubits11 mark for exactly this reason. + +## The film is not an ad + +The strongest thing in the 15-second film is not the notebook, the grid, or the +light. It is this: + +> The boldest thing a technical brand can say is `INCONCLUSIVE`. + +The film says it on purpose, and when it has no verifier output to show it +stamps `ILLUSTRATION` on its own footage rather than imply a result it was not +given. That behavior is enforced in the source, not in a style guide: see +`visual_identity/before_you_see_it/`. + +## The release loop + +```text +Film → curiosity → evidence page → interactive check → external scrutiny + ↑ ↓ + └────── next experiment, including failures and limits ──┘ +``` + +The canonical page carries all five required elements: the film and its +transcript, a source ledger, a visible "what this does not establish" strip, one +real interactive check, and the invitation. It lives at +`visual_identity/canonical_page/index.html`. + +**Never fake a verifier result.** If a screen is illustrative, label it on the +screen. If it is real, link the fixture, the command, and the limitation. The +canonical page's check recomputes SHA-256 in the visitor's browser over a real +capsule fixture and compares it against the digest recorded in +`examples/claim_governance_capsule/manifest.expected.json`. Nothing is +hard-coded, and the limitation is printed directly beneath it: integrity only. + +## What the studio sells + +Small, fixed-scope offers. + +1. **Claim-to-Evidence Sprint.** A team brings one important product, research, + or AI-system claim. They get back a claim map, evidence gaps, alternative + explanations, the smallest useful test, and a plain-English limitation + statement. +2. **Evidence Narrative Package.** Dense technical work becomes a film, an + evidence page, a source trail, and a limitation section. +3. **Evidence Literacy Lab.** A workshop built on + `story → claim → test → result → limitation → next decision`. +4. **Research-to-Product Evidence Design.** For teams with real technical work + and no coherent way to show what is implemented, measured, assumed, or merely + planned. + +**Not sold:** validation, compliance, "safe AI", or a seal of approval. What is +sold is better questions, stronger evidence, clearer public communication, and +more useful failure boundaries. + +Likely first customers: applied-AI founders, research groups, technical product +teams, security and evaluation teams, and public-interest organizations. Not +buyers shopping for a compliance badge. + +## The ninety days + +| Window | Work | +| --- | --- | +| **1-10 — establish the flag** | Publish the film and one canonical page. Publish the fixed bio and the method. Put one interactive artifact beside the film. Make authorship, sources, and limitations visible. | +| **11-30 — become useful in public** | Publish "an intuition can start a question; it cannot finish it." Release three short clips: *a signature is not truth*, *a test must be able to fail*, *inconclusive is a result*. Write five genuinely useful technical responses to people already working on evaluation, provenance, security, and reproducibility. No spam. | +| **31-60 — invite scrutiny** | Run a small live Claim Clinic: five to ten people, one claim each. Ask three independent technical people to try to reproduce or break one narrow artifact. Publish what they found, including the uncomfortable parts. | +| **61-90 — turn trust into work** | Offer the first paid Claim-to-Evidence Sprints. Publish one approved or anonymized lesson. Package repeated work into the workshop. Consider hosted software only after repeated demand for the same workflow. | + +## What to measure + +Independent reproductions. External issues, corrections, and contributions. +Invitations to teach or speak. Qualified technical conversations. Paid pilots. +Citations and references. + +Views and followers are not evidence that anyone trusts the work. Do not confuse +the two, and do not report them as traction. + +## Where scrutiny lives + +- Communities where scrutiny is the culture, entered with a narrow artifact or a + practical question rather than a pitch: + [OpenSSF working groups](https://openssf.org/community/openssf-working-groups/), + which include AI/ML security and supply-chain integrity. +- Provenance can eventually become part of the art itself. Use + [C2PA](https://spec.c2pa.org/specifications/specifications/1.1/specs/C2PA_Specification.html) + only to show signed origin and edit history, never to imply that a video is + factually true. That distinction is the same one this whole method rests on. +- Model the research culture on artifact availability and reproducibility rather + than prestige. The + [USENIX Security artifact evaluation program](https://secartifacts.github.io/usenixsec2026/) + is a useful benchmark for that mindset. + +## The sentence that matters + +Not "Pranav Bhave is brilliant." This one: + +> "When Pranav Bhave publishes something, I can see what it claims, what it does +> not claim, and how to challenge it." + +## Non-claims + +- This document does not establish that the method works, that the studio has + customers, or that any offer has been delivered. +- The ninety-day schedule is a plan. Publishing it is not evidence of executing + it. +- Nothing here authorizes representing Ghost-Ark, Penn State, or any other + institution commercially. +- No superlative in this document ("first", "only", "best") is licensed for + public use. Any such wording requires a named evidentiary standard under + `docs/claims/CLAIM_BOUNDARY_MANIFEST.md` before it ships. diff --git a/docs/research/NON_CLAIMS.md b/docs/research/NON_CLAIMS.md index c2b46618..29ecd505 100644 --- a/docs/research/NON_CLAIMS.md +++ b/docs/research/NON_CLAIMS.md @@ -43,6 +43,14 @@ The project does not claim: validated by a separate confirmatory procedure. - A receipt verifies artifact integrity, not statistical validity or deployment safety. +- A capsule that reproduces deterministically from its declared inputs does not + establish that those inputs were measured rather than asserted. Changing an + input and regenerating the chain yields a fully self-consistent capsule, a + valid receipt, and a PASS verdict over a fabricated value. See + `docs/research/epistemic-program/challenges.md` (CH-001). +- A scalar that happens to be expressible over a checked-in row count is not + thereby derived from that matrix. The event, population, and numerator must + be explicitly declared and checked before any derivation claim is made. - A PASS verdict from `cc-report verify-claim-governance` means the evidence-bound claim package is internally consistent under the verifier rules. It does not mean the AI system is safe in deployment. diff --git a/docs/research/epistemic-program/README.md b/docs/research/epistemic-program/README.md new file mode 100644 index 00000000..09b87eaa --- /dev/null +++ b/docs/research/epistemic-program/README.md @@ -0,0 +1,91 @@ +# The Epistemic Program + +> Don't manufacture visibility. Manufacture things worth inspecting, then make +> them impossible to miss. + +## Governing objective + +Turn scattered artifacts into one coherent public research program, then +multiply the number of surfaces through which someone can encounter it. + +The work here is synthesis, selection, positioning, and distribution. It is not +implementation. Implementation happens elsewhere in this repository and is +referenced, not repeated. + +## The three resolutions + +The same idea at three magnifications. + +**One field-sized question** + +```text +How should computational evidence be represented so that claims +remain inspectable and bounded? +``` + +**One human-sized obsession** + +```text +I don't like green checks that hide what they actually establish. +``` + +**One public sentence** + +```text +Make the claim smaller. Make the evidence stronger. +``` + +## What is being discovered + +Not a company. A territory: + +> How computational claims can remain attached to the evidence, assumptions, +> transformations, and boundaries that make them defensible. + +Several projects approach it from different directions. They are not five +businesses. They are **five observations from one laboratory** — and the +[research graph](research-graph.md) says exactly how they relate without +claiming they should merge. + +## Contents + +| Document | What it does | +| --- | --- | +| [Inventory](inventory.md) | Every artifact, and the question it attempted to answer. Not a portfolio — an excavation. | +| [Research graph](research-graph.md) | The invariant beneath the artifacts, and the node structure that connects them. | +| [Evidence scope inflation](evidence-scope-inflation.md) | The core concept: how a narrow verification silently becomes a broad claim. Includes an attack on this repository's own guards. | +| [Challenges](challenges.md) | Open adversarial challenges, and the results of the ones already run. | +| [Visibility manifold](visibility-manifold.md) | The method for turning one inspectable object into many entry points. | +| [Distribution packet 001](distribution-packet.md) | The first packet, and the posts derived from it. | +| [Drafts](drafts/README.md) | The five posts, written out in publishable form. | +| [Claim intake](claim-intake.md) | The participation loop. What actually happens when someone brings a claim. | +| [Release freeze](release-freeze.md) | What ships, what is not claimed, and the one action a visitor should take. | + +## Definition of success + +Not hours worked, files produced, or ideas brainstormed. Four things: + +1. An inventory of what has actually been built. +2. A graph showing the intellectual relationship among those things. +3. One canonical public object everything can point toward. +4. A repeatable method for multiplying every real artifact into multiple public + entry points. + +Then the engine runs: + +```text +artifact → claim → evidence → limitation → story → challenge → distribution + ↑ ↓ + └──────────────────────── response ────────────────────────────┘ +``` + +## Non-claims + +- This program does not establish that the method works. It establishes what the + method is and where it has been applied. +- An artifact appearing in the inventory is not a validated result. The inventory + records what each artifact attempted, not what it proved. +- The distribution material here describes intended public work. Publishing a + plan is not evidence of executing it. +- Nothing in this directory promotes any surface into Paper Core or adds a claim + to `docs/claims/CLAIM_BOUNDARY_MANIFEST.md`. diff --git a/docs/research/epistemic-program/challenges.md b/docs/research/epistemic-program/challenges.md new file mode 100644 index 00000000..70c7ddd1 --- /dev/null +++ b/docs/research/epistemic-program/challenges.md @@ -0,0 +1,278 @@ +# Challenges + +Adversarial participation, run as public research rather than as marketing. + +A challenge is an invitation to falsify something specific in this repository. +Results are published whether or not they are flattering, and the uncomfortable +ones are published first. + +## Completed + +### CH-001 — Break the canonical page's drift guard + +**Challenge.** Can the canonical page be made to display `BYTES MATCH` over a +different underlying fixture, without causing the drift guard to fail? + +**Answer: yes.** Two attempts were run in a detached worktree at `ceeca6b`. + +#### Attempt A — repair the consistency chain + +Forge `expected/calibration.json` (`realized_fpr` 0.041666666667 → 0.011111111111), +recompute its digest, patch `manifest.expected.json`, and patch the page's +embedded copy and digest to match. + +| Guard | Result | +| --- | --- | +| `tests/unit/docs/test_canonical_page_fixture.py` | **4 passed** — the forgery is internally consistent | +| `tests/integration/test_claim_governance_capsule.py` | **failed** — regeneration from declared inputs produced the original file | + +The drift guard alone does not detect this. The capsule integration test does, +because it rebuilds the artifact rather than comparing it to itself. + +*(A first run of this attempt reported a false failure. The attack script used +`re.sub` with a replacement string containing `\n`, which Python interprets as a +newline, corrupting the page instead of forging it. The tooling failed, not the +guard. It is recorded here because a challenge result produced by a broken +attacker is worse than no result.)* + +#### Attempt B — move the forgery upstream + +Change the **declared input** (`inputs/capsule_config.json`, same field, same +value), regenerate the capsule with `build_capsule.py`, refresh all eleven +expected artifacts and the manifest from the regenerated outputs, then update the +page. + +| Guard | Result | +| --- | --- | +| `tests/unit/docs/test_canonical_page_fixture.py` | **4 passed** | +| `tests/integration/test_claim_governance_capsule.py` | **5 passed** | +| Governance verdict in the regenerated manifest | `pass` | +| Report receipt | new, valid, `c1643673596fab68…` | +| Canonical page in a real browser | `BYTES MATCH` over `"realized_fpr": 0.011111111111` | + +Every check green. Every hash correct. The displayed number is fiction. + +#### What CH-001 establishes + +The guard chain establishes **deterministic reproducibility from declared +inputs**. It cannot distinguish a measured input from an asserted one. This is +rung 4 of the ladder in +[evidence scope inflation](evidence-scope-inflation.md). + +#### What CH-001 does not establish + +- Not that the capsule is broken. It does exactly what it says, and Attempt A + shows the chain catching the naive forgery. +- Not a general result about reproducible builds. One repository, one chain. +- Not that an attacker could do this to the published repository — the forgery + requires commit access, and a reviewer reading the diff would see the input + change. The finding is about what the *green checks* prove, not about + compromise. + +#### What changed because of it + +- The limitation is now stated on the canonical page itself, not only here. +- The drift guard's docstring states what it does not establish. +- A new entry in `docs/research/NON_CLAIMS.md`. +- Closing rung 4 requires an anchor outside the repository. There is none today, + and the documents now say so rather than implying otherwise. + +## Open / partially answered + +### CH-002 — Close rung 4 + +**Challenge.** Propose a mechanism that would let an outside reviewer +distinguish a measured input from an asserted one, without requiring them to +trust the author. + +**Answer: no repository-local mechanism can, and the reason is uncomfortable — +deterministic reproducibility is what makes the forgery cheap.** + +#### The argument + +Let `R` be a repository controlled by author `A`, and let `V` be any +verification procedure whose inputs are entirely contained in `R`. `V` cannot +distinguish "input `x` was measured" from "input `x` was asserted by `A`". + +`V` is a function of `R`'s contents, so `A` can compute it. To place a chosen +value `x'` into a state that `V` accepts, `A` does not need to invert anything: +the generator's outputs are a deterministic function of its inputs, and the +checks verify exactly that functional relationship. So `A` edits `x'`, +regenerates, and `V(R') = holds` by construction. CH-001 is this argument +executed rather than asserted. + +The uncomfortable part is that the property being exploited is the one we want. +Reproducibility guarantees that a change to an input propagates consistently +through every downstream artifact, hash, and receipt. That is exactly why the +capsule is valuable — and it applies to honest and dishonest inputs equally: + +> **Reproducibility amplifies consistency, not truth.** + +A less reproducible pipeline would be *harder* to forge coherently, and worse in +every other respect. This is not an argument against reproducible builds. It is +an argument for knowing which of the two things they establish. + +#### One cheap partial defence, and its limit + +`realized_fpr` is `0.041666666667`, which is exactly `1/24`. The capsule ships +`inputs/failure_matrix.csv`, which has exactly 24 rows. The declared statistic +has a denominator structure matching the declared sample size — but nothing +checks the relationship, because the summary is a *declared input* rather than a +*derived* one. + +CH-001's forgery set it to `0.011111111111`, which is `1/90`. That is not +expressible as a count over 24 observations. A **derivability check** — requiring +declared statistics to be expressible over the declared sample size — would have +caught it, costs almost nothing, and needs no external party. + +It does not close rung 4. An attacker who picks `2/24` instead of `1/90` passes. +It raises the cost of forgery from *any number* to *any number consistent with +the declared n*, which is a real improvement and a bounded one. + +**It is deliberately not implemented.** The columns of the failure matrix give +rates of 9/24, 12/24, and 9/24; none is 1/24. So `realized_fpr` is plausibly a +false-positive count over the same 24 prompts, and plausibly a quantity from a +different sample that happens to share a denominator. Implementing a check that +assumes the first reading would encode an unverified assumption about what the +field means — the same error class this whole program is about, committed in the +act of defending against it. The correct next step is to establish what the field +denotes, then check it. + +#### Continuation — make the missing semantics executable + +The fixture now carries three independent facts about `realized_fpr`: +`semantics.status: unresolved`, `origin.kind: asserted`, and +`external_anchor: null`. That makes the absence of a derivation visible in the +generated calibration artifact instead of allowing the value's `1/24` form to +imply one. + +The capsule also has two opt-in mechanisms for a future value whose meaning has +actually been declared: + +- `origin.kind: deterministic_derivation` accepts only a named binary column + of the checked-in matrix and verifies its source digest, numerator, + denominator, decimal rendering, and scalar value. It verifies arithmetic, not + that the named column is semantically an FPR or that the matrix represents an + external population. +- `external_anchor` requires a URI, issuer, timestamp, and a digest over the + exact value, semantics, and origin. It records that reference without + fetching it. It binds a declaration to an external object; it does not + establish that the issuer measured correctly. + +Neither mechanism is applied to the current field. In particular, there is still no +derivability check that silently interprets the current `realized_fpr` as a +matrix-column rate. The remaining next artifact is a real external anchor at +collection time, not a richer local label. + +#### What would actually close it + +Ranked by cost, each closing a different threat: + +| Anchor | Closes | Leaves open | +| --- | --- | --- | +| Timestamp the input at collection (RFC 3161, a transparency log) | Fabricating or revising an input *after seeing results* | Fabricating it at collection time | +| Signature by the measuring instrument or a second party | Binding the value to someone other than its writer | Trust in that party | +| Independent replication | Measurement itself | Nothing — but it is a social process, not a mechanism | + +Only the third establishes measurement, and it is definitionally not +repository-local. The first is the cheapest and maps precisely onto +preregistration: freeze the protocol, and the input, before the observation. + +#### Convergence worth noting + +This is the same shape as the attestation boundary in the +[eight-week plan](../future-expansion/eight-week-plan.md): an enclave attests a +measurement, App Attest validates an app instance, C2PA binds an assertion to an +artifact — and none of them establishes that the assertion is true. Rung 4 is +that boundary, met from the reproducible-build direction instead of the +hardware-attestation direction. Two tracks, one wall. + +#### Status + +Partially answered. The impossibility argument is stated and demonstrated; the +current field's unresolved semantics and asserted origin are now explicit and +machine-checked. No independent external anchor has been recorded or verified. +CH-002 stays open for anyone who can refute the argument or implement the first +anchor. + +### Further open challenges + +### CH-003 — Make the film show something false + +The film renders whatever is in `window.__VERDICT` and stamps `ILLUSTRATION` +when nothing is injected. Find a path that produces an un-stamped frame +displaying a verdict that no verifier issued. + +### CH-004 — Find a reserved-vocabulary bypass + +The report validator rejects reserved overclaim vocabulary in claim statements +and evidence roles. Construct a report that passes validation and still reads, +to an ordinary reader, as a safety claim. Metadata, field names, and alternate +routes all count. + +### CH-005 — Widen an interval by adding evidence + +The kernel's intervals should tighten monotonically as constraints are added. +Find declared evidence whose addition makes the reported interval wider, or an +argument for why the monotonicity test does not cover a real case. + +### CH-006 — A docstring contract, falsified by a property test + +**Not opened by us.** Hypothesis found it while the full suite was running at +the end of this session, and it is recorded here because a challenge programme +that only publishes the challenges it chose is not a challenge programme. + +`ModelBase.migrate()` is documented as best-effort, and its test states the +contract explicitly: + +> migrate() should be best-effort and never throw on arbitrary old dicts. + +It throws. Given `{"updated_at": ""}` it raises a `ValidationError`, because +`migrate()` passes arbitrary values straight into `model_validate` and +`updated_at` carries a validator that rejects a non-numeric timestamp. + +| Fact | Status | +| --- | --- | +| Reproduces on `main` | Yes, identically — verified in a detached worktree | +| Touched by this branch | No. `src/cc/core/models.py` is unmodified here | +| Deterministic in CI | No. The failing example lives in a gitignored `.hypothesis/` database, so CI may or may not draw it | + +#### The interesting part + +The obvious fix — make `migrate()` swallow invalid values for known fields — is +**silent repair**, which this repository rejects everywhere else. The report +validator refuses rather than repairing. The governance verifier fails closed. +A migration that quietly drops a malformed timestamp would hide exactly the kind +of data problem the rest of the codebase is built to surface. + +So the defect is plausibly in the *contract*, not the code: + +| Option | Effect | Cost | +| --- | --- | --- | +| A — make `migrate()` best-effort for real | Honours the docstring | Silent repair; contradicts the repository's fail-closed posture | +| B — narrow the contract | `migrate()` must not throw on **unknown** keys; a known field with an invalid value fails closed, as it should | Changes a documented contract and a test in the core model layer | + +**Recommendation: B**, with the Hypothesis strategy excluding known field names +and an explicit test asserting that an invalid known field *does* raise. + +**Not applied.** This is a semantic decision in the core model layer, made on a +pre-existing bug that is unrelated to this branch's work. Changing it +unilaterally would be its own kind of scope inflation — repairing something +quietly because it was inconvenient to the session's green build. + +## How results are handled + +1. Reproduce it here, in a detached worktree, with the commands recorded. +2. Publish the result — including a result that makes this project look worse. +3. Change the artifact, the limitation text, or both. +4. Record what changed, so the challenge's effect is inspectable too. + +A challenge that produces no change to any artifact is recorded as such. That is +also a result. + +## Non-claims + +- A challenge going unanswered is not evidence that a property holds. +- The list of open challenges is not a claim that these are the only weaknesses. + They are the weaknesses currently visible from the inside, which is exactly the + vantage point that missed rung 4. diff --git a/docs/research/epistemic-program/claim-intake.md b/docs/research/epistemic-program/claim-intake.md new file mode 100644 index 00000000..1d4ca0e0 --- /dev/null +++ b/docs/research/epistemic-program/claim-intake.md @@ -0,0 +1,105 @@ +# Claim Intake + +## The point + +The invitation says *bring one claim that matters*. This document takes that +sentence seriously enough to define what happens next. + +The answer is never `TRUE` or `FALSE`. Returning a verdict would be the same +error the whole program is about: a narrow analysis inflated into a broad +judgment. What comes back is a **decomposition** — the claim taken apart into +the pieces that decide whether it holds, with the unresolved pieces left visibly +unresolved. + +The secondary purpose is that the method itself becomes observable. Broadcasting +expertise persuades nobody; doing the work in public, on someone else's claim, +shows the thinking rather than asserting it. + +## The worked example + +Someone brings: + +> Our agent catches 95% of policy violations. + +### 1. Claim + +What exactly does 95% refer to? Caught *out of what*? Violations that occurred, +violations that were labelled, or violations the agent was shown? Is 95% a +recall, a precision, an accuracy, or an average of several runs? Restate it in a +form where the denominator is explicit and the estimand is named. Most claims +lose half their apparent strength at this step, before any evidence is examined. + +### 2. Population + +What distribution was this measured over, and what does it not represent? +Adversarial or organic traffic? Which policies, which locales, which model +version? A number measured on a curated benchmark and a number measured on +production traffic are different claims that share a digit. + +### 3. Evidence + +Where did the measurement originate? Which artifact holds it, who produced it, +when, under what code version, and can it be re-derived? An unreproducible +number is not disqualified — it is relabelled as a report of a past observation +rather than a property of the system. + +### 4. Alternative explanations + +Could leakage explain it — benchmark contamination, labels visible at inference, +the same examples used to tune the threshold and to score it? Could selection +explain it — failures filtered before counting, runs discarded, an evaluation +window chosen after seeing results? Could the metric explain it — a class +imbalance that makes 95% the score of a system that always says "allow"? + +### 5. Smallest challenge + +The cheapest experiment that would cause the statement to be revised. Not the +best study — the smallest one whose negative result would actually change what +gets said in public. Usually one held-out slice, one re-run with the threshold +frozen in advance, or one adversarial batch nobody has seen. + +### 6. Result + +`supported` / `contradicted` / `inconclusive`, at the scope defined in step 1. +`inconclusive` is a legitimate terminal state and is reported as often as it +occurs. A process that never returns inconclusive is not measuring anything. + +### 7. Boundary + +What larger claim remains unestablished. Explicitly. In the same document, at +the same size as the result. For this example the boundary usually reads: the +measurement says something about detection on a stated distribution; it says +nothing about coverage of violations nobody wrote a policy for, and nothing +about behaviour after the next model update. + +## What comes back + +| Deliverable | Content | +| --- | --- | +| Claim map | The original sentence, decomposed, with the estimand and denominator made explicit. | +| Evidence trail | Which artifact supports which part, and how to inspect each one. | +| Alternative explanations | Ranked by how cheaply they could be ruled out. | +| Smallest useful test | One experiment, with its stop rule and what it cannot settle. | +| Limitation statement | Plain English, publishable as-is, sized to the evidence. | + +## What does not come back + +No validation. No compliance mapping. No safety assertion. No seal, badge, or +score. If a client's actual need is a badge, this is the wrong studio, and +saying so early is part of the service. + +## Why this could become a verb + +Not by forcing the phrase. By making the behaviour repeatable and public until +the decomposition is recognisable on sight — the way a diff or a postmortem is +recognisable. Category creation is a consequence of a distinctive, repeated, +observable behaviour. It is not a naming exercise, and any attempt to name it in +advance would be its own scope inflation. + +## Non-claims + +- This protocol does not establish that a claim is true or false. It establishes + what would have to hold, and which parts remain unresolved. +- No intake has been run yet. This is a defined procedure, not a track record. +- A completed intake is a document, not a certification, and may not be + represented as one. diff --git a/docs/research/epistemic-program/distribution-packet.md b/docs/research/epistemic-program/distribution-packet.md new file mode 100644 index 00000000..0d0816c6 --- /dev/null +++ b/docs/research/epistemic-program/distribution-packet.md @@ -0,0 +1,93 @@ +# Distribution Packet 001 + +The first packet. Everything in it points at one inspectable object. + +## The object + +The canonical page: `visual_identity/canonical_page/index.html`. + +## The hook + +> We made the green check break. + +## The demonstration + +Change one character in the box. The verification fails closed. Restore it and +the verification returns. Nothing is simulated; the browser is computing a real +SHA-256 over the bytes you can see. + +## The surprising point + +The successful hash still does not establish that the value inside the file is +true. + +```text +Bytes verified. Claim unresolved. +``` + +## The philosophy + +Evidence should expose the boundary of what it establishes. + +## The invitation + +> Bring one claim that matters. + +## The copy + +The whole announcement, without inflation: + +```text +We built a browser check that verifies 189 bytes against a manifest. +Change one character and it fails closed. +It still cannot tell you whether the number inside those bytes is true. +That's the point. +``` + +No revolutionary framework. No changing AI safety forever. The restraint is the +differentiator, and it is also honest, which is why it is sustainable. + +## Derived posts + +Each points back to the same object. None is standalone. + +| # | Angle | Core line | Ends at | Copy | +| --- | --- | --- | --- | --- | +| 1 | The demonstration | A green check should have a falsifiable path to red. | The page. | [draft](drafts/01-the-demonstration.md) | +| 2 | The boundary | Integrity is not truth: these bytes are those bytes, and that is all. | The limitation text. | [draft](drafts/02-integrity-is-not-truth.md) | +| 3 | The self-attack | We tried to forge our own verified page. It worked. | [CH-001](challenges.md). | [draft](drafts/03-we-forged-our-own-page.md) | +| 4 | The method | Name the claim, the falsifier, the control, and the non-claim — before the result exists. | The film. | [draft](drafts/04-name-it-before-you-see-it.md) | +| 5 | The invitation | Bring one claim that matters. Here is exactly what you get back. | [Claim intake](claim-intake.md). | [draft](drafts/05-bring-one-claim.md) | + +All five are written and sit in [`drafts/`](drafts/README.md), in publishable +form, for the author to edit into his own voice. + +Post 3 is the one that will travel furthest, because publishing a successful +attack on your own verification is rare enough to be surprising and cheap enough +to be honest. It should not be held back for a better moment. It *is* the moment. + +## Sequencing + +Ship 1 and 2 with the page. Hold 3 until CH-001's write-up and the page's +updated limitation are both live, so the post lands on an artifact that already +reflects the finding — never on a promise to fix it later. Then 4, then 5. + +## Rules for every public artifact in this packet + +Each one carries three things, visibly: + +```text +CLAIM what is being asserted +EVIDENCE what supports it, and where to inspect that +LIMIT what remains unestablished +``` + +If a post cannot carry all three, it is not ready. This is the point at which +the marketing becomes a demonstration of the philosophy rather than a departure +from it. + +## Non-claims + +- This packet has not been published. Writing it is not distributing it. +- No claim is made about how any of it will perform. +- The ordering is a judgment about honesty and readiness, not about reach. diff --git a/docs/research/epistemic-program/drafts/01-the-demonstration.md b/docs/research/epistemic-program/drafts/01-the-demonstration.md new file mode 100644 index 00000000..38dbcbc6 --- /dev/null +++ b/docs/research/epistemic-program/drafts/01-the-demonstration.md @@ -0,0 +1,42 @@ +# Draft 01 — The demonstration + +**Surface:** aphorism + experience. **Points at:** the canonical page. + +--- + +We built a browser check that verifies 189 bytes against a manifest. + +Change one character and it fails closed. + +It still cannot tell you whether the number inside those bytes is true. + +That's the point. + +--- + +## Longer version + +There is a real evidence file in our repository: a calibration record, 189 +bytes, holding a false-positive rate. There is a real digest for it, recorded in +a manifest that was generated when the artifact was built. + +The page puts both in front of you and recomputes the digest in your browser, +over exactly the bytes you can see. No server. No API. `crypto.subtle.digest`, +the same primitive your browser uses for TLS. + +Then it gives you a button that breaks it. + +That button is the whole design. A verification you have personally made fail is +a verification whose scope you understand. Everyone who clicks it stops +believing that the green state means the file is *correct*, because they have +just watched what the check is actually sensitive to: bytes, and nothing else. + +**A green check should have a falsifiable path to red.** Most don't. Not because +anyone is lying, but because the path is buried three systems away from the +person reading the result. + +Claim: these bytes hash to that digest. +Evidence: recomputed in your browser, over bytes you can edit. +Limit: says nothing about whether 0.041666666667 is a real false-positive rate. + +Try to break it: `visual_identity/canonical_page/index.html` diff --git a/docs/research/epistemic-program/drafts/02-integrity-is-not-truth.md b/docs/research/epistemic-program/drafts/02-integrity-is-not-truth.md new file mode 100644 index 00000000..d2434649 --- /dev/null +++ b/docs/research/epistemic-program/drafts/02-integrity-is-not-truth.md @@ -0,0 +1,48 @@ +# Draft 02 — Integrity is not truth + +**Surface:** philosophical. **Points at:** the limitation text. + +--- + +A digest match establishes exactly one thing: + +> These bytes are those bytes. + +Here is what happens next, in an honest reader's head, in under a second and +without any awareness that it happened: + +```text +1. hash(A) == manifest_hash → A is the file that was recorded +2. A is the recorded file → the contents of A are accurate +3. the contents of A are accurate → the claim A represents is true +``` + +Step 1 is licensed. Steps 2 and 3 are not. Nobody computed anything about +whether the number in the file is real. It could have been typed in. + +I call this **evidence scope inflation**, and the thing that makes it worth +building a laboratory around is that it requires no bad actors. It is the +default behaviour of a careful, honest reader. The inflation happens in the gap +between what a mechanism computes and what a person needs to know — and the +mechanism is usually silent about the gap, because it does not know the gap +exists. + +Every mechanism worth trusting has this shape: + +| Establishes | Routinely read as | +| --- | --- | +| A key signed this | The signer is honest and the content is true | +| The declared tests passed on this commit | The change is correct | +| This measurement ran in that enclave | The computation was correct | +| An assertion is bound to an artifact | The assertion is true | +| The package is internally consistent | The system is safe | + +The right column is where products live. The left column is where truth lives. +The distance between them is the interesting part. + +The fix is not better cryptography. It is **placement**: make the boundary +travel attached to the result, at the same size, in the same frame, at the same +moment. Not a footnote. Not a linked document. Directly under the verdict, +naming the specific thing it does not vouch for. + +Bytes verified. Claim unresolved. diff --git a/docs/research/epistemic-program/drafts/03-we-forged-our-own-page.md b/docs/research/epistemic-program/drafts/03-we-forged-our-own-page.md new file mode 100644 index 00000000..bd03812a --- /dev/null +++ b/docs/research/epistemic-program/drafts/03-we-forged-our-own-page.md @@ -0,0 +1,77 @@ +# Draft 03 — We forged our own verified page + +**Surface:** challenge. **Points at:** CH-001. + +--- + +We published a page that verifies an evidence file in your browser and fails +closed if you change one character. + +Then we tried to forge it. It worked. Here is exactly how, and what it means. + +--- + +## Attempt one: repair the consistency chain + +Change the false-positive rate in the evidence file from `0.041666666667` to +`0.011111111111`. Recompute its digest. Patch the manifest to record the new +digest. Patch the page's embedded copy to match. + +Everything is now internally consistent, and the drift guard that exists +specifically to catch this **passes — 4 tests green.** + +But the capsule integration test fails. It does not compare the artifact to +itself; it *rebuilds* the artifact from the declared inputs and finds the +original value. Caught. + +Good. That is what a reproducible build is for. + +## Attempt two: move the forgery upstream + +So don't forge the output. Forge the input. + +Change the same field in `inputs/capsule_config.json`. Regenerate the whole +chain with the project's own build script. Refresh all eleven expected artifacts +and the manifest from the regenerated outputs. Update the page. + +| Check | Result | +| --- | --- | +| Drift guard | 4 passed | +| Capsule integration | 5 passed | +| Governance verdict | `pass` | +| Report receipt | new, valid | +| The page, in a real browser | **BYTES MATCH** over `"realized_fpr": 0.011111111111` | + +Every check green. Every hash correct. The number is fiction. + +## What this actually means + +Not that the capsule is broken — it does exactly what it claims. The finding is +about what the green checks *prove*: + +> The chain establishes deterministic reproducibility from declared inputs. +> It cannot distinguish a measured input from an asserted one. + +That is a fourth unauthorized inference, one level above the three a reader +makes about a digest — and this one is committed by the *builder*. By someone +who had already written the project's non-claims document. By us. + +Knowing about scope inflation does not immunize you against it. Only a mutation +test finds the next rung. + +## What changed + +The page now says this on the page, not in a document nobody opens. The +repository's non-claims file has a new entry. The drift guard's own docstring +states what it does not establish. + +Closing the gap needs an anchor outside the repository: a signature over the +input at collection time, an independent party who performed the measurement, or +a transparency log outside the author's control. **We don't have one.** That is +now written down instead of implied. + +If you can close it, or show that no repository-local mechanism can, that's +CH-002 and it's open. + +Full write-up, commands, and the exact worktree state: +`docs/research/epistemic-program/challenges.md` diff --git a/docs/research/epistemic-program/drafts/04-name-it-before-you-see-it.md b/docs/research/epistemic-program/drafts/04-name-it-before-you-see-it.md new file mode 100644 index 00000000..d00a4b7e --- /dev/null +++ b/docs/research/epistemic-program/drafts/04-name-it-before-you-see-it.md @@ -0,0 +1,43 @@ +# Draft 04 — Name it before you see it + +**Surface:** the method. **Points at:** the film. + +--- + +An intuition can start a question. It cannot finish one. + +Before you see an answer, name what would count: + +**Claim** — what is asserted, and how narrowly. +**Falsifier** — the result that would end it. +**Control** — the comparison that could embarrass it. +**Non-claim** — what it will never support. + +Then freeze all four, *before* the result exists, and let the result disagree. + +--- + +The freezing is the part people skip, and it is the only part that does any +work. A threshold chosen after seeing the data is not a threshold; it is a +description. A control added after a promising result is not a control; it is a +defence. The order matters more than the rigor. + +We made a fifteen-second film about this. It has a shot in it where a protocol +locks over an empty result slot, and holds there for three seconds. That empty +slot is the whole idea: the moment where the standard exists and the answer +does not. + +And when the result finally arrives, it says `INCONCLUSIVE`. + +That was deliberate. The boldest thing a technical brand can say is that it +doesn't know yet. So the film says it — in its own advertising, where it costs +something. + +There's one more detail I'd defend to anyone. The film renders whatever verdict +it is handed. If it is handed nothing, it falls back to an illustrative result +**and stamps `ILLUSTRATION` in the corner of its own footage.** + +A film about not overclaiming cannot display an unearned result. So it labels +itself. That behaviour is in the source, not in a style guide. + +`visual_identity/before_you_see_it/` diff --git a/docs/research/epistemic-program/drafts/05-bring-one-claim.md b/docs/research/epistemic-program/drafts/05-bring-one-claim.md new file mode 100644 index 00000000..81ed2644 --- /dev/null +++ b/docs/research/epistemic-program/drafts/05-bring-one-claim.md @@ -0,0 +1,61 @@ +# Draft 05 — Bring one claim that matters + +**Surface:** the invitation. **Points at:** claim intake. + +--- + +Bring one sentence your team says in public and cannot fully defend in private. + +A benchmark number. A security property. A "proven". A "guaranteed". A 95%. + +You do not get back `TRUE` or `FALSE`. A verdict would be the same error this +whole thing is about — a narrow analysis inflated into a broad judgment. You get +back the claim taken apart into the pieces that decide whether it holds, with +the unresolved pieces left visibly unresolved. + +--- + +Take a real one: + +> Our agent catches 95% of policy violations. + +**Claim.** 95% of *what*? Caught out of violations that occurred, that were +labelled, or that the agent was shown? Is that recall, precision, or accuracy? +Most claims lose half their apparent strength here, before any evidence is +examined. + +**Population.** Measured over which distribution, and what does it not +represent? Adversarial or organic traffic? Which policies, which locales, which +model version? A number from a curated benchmark and a number from production +share a digit and nothing else. + +**Evidence.** Which artifact holds the measurement, who produced it, when, under +what code version, and can it be re-derived? An unreproducible number isn't +disqualified — it's relabelled as a report of a past observation rather than a +property of the system. + +**Alternative explanations.** Could leakage explain it? Could selection explain +it — failures filtered before counting, an evaluation window chosen after seeing +results? Could the metric explain it — a class imbalance that makes 95% the +score of a system that always says "allow"? + +**Smallest challenge.** Not the best study. The cheapest experiment whose +negative result would actually change what you say in public. + +**Result.** Supported, contradicted, or inconclusive — at the scope defined in +step one. A process that never returns inconclusive isn't measuring anything. + +**Boundary.** What larger claim remains unestablished, stated in the same +document, at the same size as the result. + +--- + +What comes back: a claim map, an evidence trail, ranked alternative +explanations, one smallest useful test, and a limitation statement in plain +English that you can publish as written. + +What does not come back: validation, compliance, a safety assertion, a badge, or +a score. If what you need is a badge, this is the wrong studio, and I'd rather +say so in the first email. + +`docs/research/epistemic-program/claim-intake.md` diff --git a/docs/research/epistemic-program/drafts/README.md b/docs/research/epistemic-program/drafts/README.md new file mode 100644 index 00000000..8ca30400 --- /dev/null +++ b/docs/research/epistemic-program/drafts/README.md @@ -0,0 +1,21 @@ +# Drafts + +Publishable copy for [distribution packet 001](../distribution-packet.md). Each +post points back at the same inspectable object and carries a claim, its +evidence, and its limit. + +These are drafts for the author to edit into his own voice, not finished +statements from anyone else. Nothing here has been published, and publishing a +draft is not evidence of anything it describes. + +| Draft | Surface | Points at | +| --- | --- | --- | +| [01 — the demonstration](01-the-demonstration.md) | C, A | The canonical page | +| [02 — integrity is not truth](02-integrity-is-not-truth.md) | G | The limitation text | +| [03 — we forged our own verified page](03-we-forged-our-own-page.md) | F | CH-001 | +| [04 — name it before you see it](04-name-it-before-you-see-it.md) | — | The film | +| [05 — bring one claim that matters](05-bring-one-claim.md) | — | Claim intake | + +Draft 03 is the one to lead with once its artifact is live. Publishing a +successful attack on your own verification is rare enough to be surprising and +cheap enough to be honest. diff --git a/docs/research/epistemic-program/evidence-scope-inflation.md b/docs/research/epistemic-program/evidence-scope-inflation.md new file mode 100644 index 00000000..0f7a04d2 --- /dev/null +++ b/docs/research/epistemic-program/evidence-scope-inflation.md @@ -0,0 +1,137 @@ +# Evidence Scope Inflation + +## The error + +A verification establishes something narrow. A reader converts it into something +broad, without noticing that a conversion happened. + +The canonical page's check makes the whole sequence visible in about four +seconds. The machine observes exactly one thing: + +```text +sha256(bytes_in_the_box) == digest_recorded_in_the_manifest +``` + +The reader's mind, unprompted, performs three more steps: + +```text +1. hash(A) == manifest_hash → A is the file that was recorded +2. A is the recorded file → the contents of A are accurate +3. the contents of A are accurate → the claim A represents is true +``` + +Step 1 is licensed. Steps 2 and 3 are not. Nothing computed anything about +whether `0.041666666667` is a real false-positive rate. The number could have +been typed in. + +**Bytes verified. Claim unresolved.** + +That sentence is the whole discipline compressed into three words and a +concession. A less careful project ships `VERIFIED ✓`. + +## Why it is worth building a laboratory around + +The error is not exotic. It is the default behaviour of an attentive, honest +reader. Nobody has to be fooled. The inflation happens in the gap between what a +mechanism computes and what a person needs to know, and the mechanism is usually +silent about the gap because it does not know the gap exists. + +Every mechanism worth trusting has this shape: + +| Mechanism | Actually establishes | Routinely read as | +| --- | --- | --- | +| A digest match | These bytes are those bytes | This artifact is legitimate | +| A signature | A key signed this | The signer is honest and the content is true | +| A green CI check | The declared tests passed on this commit | The change is correct | +| An enclave attestation | This measurement ran in that enclave | The computation was correct | +| Provenance metadata | An assertion is bound to an artifact | The assertion is true | +| A governance PASS | The package is internally consistent under verifier rules | The system is safe | +| A reproducible build | The output follows from the declared inputs | The inputs describe reality | + +The right-hand column is where products live. The left-hand column is where +truth lives. The distance between them is the territory. + +## The ladder does not stop at three rungs + +The three jumps above are the ones a reader makes. There are more, and they are +made by *builders* — including by this repository. + +[CH-001](challenges.md) attacked this project's own guards to find the next rung. +The result, in full there and in summary here: + +- Editing an evidence artifact and repairing the manifest and the page is caught, + because regeneration from declared inputs disagrees. **Consistency alone is not + enough, and the capsule knows it.** +- Editing the *declared input*, regenerating the whole chain, and refreshing the + goldens passes **every** check. Governance verdict `pass`. A new, valid receipt. + The page displays `BYTES MATCH` over a false-positive rate that was simply + asserted. + +So the next rung is: + +```text +4. the chain reproduces from declared inputs + → the declared inputs describe a real measurement +``` + +Also unlicensed. Every hash in that chain did its job correctly. Not one of them +reaches back to the world. The capsule establishes **deterministic +reproducibility from declared inputs** — which is genuinely valuable, and is not +measurement. + +This is the same error as steps 2 and 3, one level up and committed by someone +who had already written the non-claims document. That is the point worth +sitting with: knowing about scope inflation does not immunize you against it. +Only a mutation test finds the next rung. + +## The general form + +Every verification is a function with a domain and a codomain. + +```text +verify : artifact → {holds, fails} over a specific property P +``` + +Scope inflation is silently widening `P` after the fact. The defence is not +better hashes. It is making `P` travel attached to the result, at the same size, +in the same frame, at the same moment. + +Four things make that happen in practice: + +1. **Co-located limitation.** The boundary is printed where the result is + printed — not in a footnote, not in a linked document, not on a second page. + On the canonical page, the limitation sits directly under the verdict and + names the specific number it does not vouch for. +2. **A mutation affordance.** Give the reader a button that breaks it. A result + the reader has personally made fail is a result they understand the scope of. + Nobody who has clicked *Mutate one byte* believes the check is about truth. +3. **Refusal vocabulary.** Make the overclaim unsayable in the artifact itself: + reserved words rejected by the report validator, a language quarantine over + paper prose, forbidden *upgrades* enumerated in the claim manifest. +4. **An external anchor, or an honest admission that there is none.** Rung 4 + cannot be closed from inside the repository — [CH-002](challenges.md) gives + the argument and the reason: deterministic regeneration lets the author + construct any state a local check accepts, so reproducibility amplifies + consistency rather than truth. Closing it needs a timestamp at collection, a + second party who measured, or independent replication. None is implemented + here, and the correct move until one is, is to say so. + +## What this is not + +It is not a claim that hashing is useless. The check is real, the digest match +is real, and tamper evidence is worth having. + +It is a claim about *placement*: a mechanism that establishes something narrow +should be presented at the size of the thing it establishes. Most of the value +in this program comes from insisting on that, out loud, in public, including +when the resulting screen is less impressive than a green tick. + +## Non-claims + +- This document does not establish that scope inflation is common, only that it + is available — and that this project committed a rung of it while explicitly + trying not to. +- CH-001 is a demonstration on one repository's own guards. It is not a general + result about reproducible-build systems. +- Naming the four countermeasures is not evidence that they work. Three of them + are implemented here; the fourth, an external anchor, is not. diff --git a/docs/research/epistemic-program/inventory.md b/docs/research/epistemic-program/inventory.md new file mode 100644 index 00000000..c67590a1 --- /dev/null +++ b/docs/research/epistemic-program/inventory.md @@ -0,0 +1,75 @@ +# Epistemic Inventory + +## How to read this + +One row per artifact that contains an actual intellectual move. The +qualifying question is **not** "is this impressive enough?" It is: + +> What question did this artifact attempt to answer? + +Small things count. A test is an artifact. A refusal is an artifact. A failed +idea is an artifact, and often a better one than a success. + +The `Limit` column is mandatory. An artifact with no stated limit has not been +inventoried; it has been advertised. + +## Kernel and mathematics + +| Artifact | Question | Claim | Evidence | Limit | Interesting because | Public | +| --- | --- | --- | --- | --- | --- | --- | +| `cc.kernel.sensitivity` finite-atom LP | What does evidence about guardrail failures identify when the joint law is unknown? | Sharp lower and upper values for a declared linear composition query over feasible finite binary atom distributions. | `tests/unit/kernel/test_sensitivity.py`, monotonic tightening tests, theorem ledger T1. | Says nothing about upstream data collection, semantic safety, or representativeness. | The answer is an interval, and its width is itself the finding. | Research | +| `cc.kernel.frechet_classes` | What do classical Fréchet-Hoeffding bounds give for composed binary failure? | Bounds under exact singleton marginals with no side constraints. | `test_classical_frechet_special_cases.py`, theorem ledger T2. | Assumes nothing about dependence, and therefore proves nothing about a deployment. | The independence calculation is exposed as a *baseline*, never as truth. | Research | +| `cc.kernel.sample_complexity` | When counts replace probabilities, what survives? | Under stated iid Bernoulli sampling and simultaneous moment coverage, count-derived intervals form an outer confidence interval. | `test_finite_sample_constraints.py`, `test_sample_complexity.py`, theorem ledger T6. | Invalid after uncorrected adaptive target selection. Not a deployment certificate. | Names the exact condition under which its own guarantee dissolves. | Research | +| Endpoint witnesses | Is a reported bound actually attainable? | The returned joint law attains the reported endpoint under declared constraints. | Witness verification table in the paper artifacts. | Witnesses the mathematics of the bound, not any real system's behavior. | A bound that cannot exhibit a witness is not reported. | Research | +| `cc.kernel.cliff` | Do composed risks jump discontinuously as dependence changes? | Characterizes dependence-driven jumps under declared models. | Correlation-cliff experiments under `experiments/`. | Exploratory. Not a confirmatory certificate. | The failure mode is a cliff, not a slope — averages hide it. | Experimental | + +## Evidence governance + +| Artifact | Question | Claim | Evidence | Limit | Interesting because | Public | +| --- | --- | --- | --- | --- | --- | --- | +| `cc.evidence.role_ontology` | Should all evidence be allowed to say the same things? | Role determines what a piece of evidence may support. | `tests/unit/evidence/test_role_ontology.py`. | A correct role assignment does not make the underlying measurement correct. | Typing evidence is a stronger idea than scoring it. | Research | +| `cc.evidence.confirmatory_protocol` | When does an exploratory finding become a confirmatory one? | Only under a separately registered protocol; never by reinterpretation. | `test_confirmatory_protocol.py`. | Registration does not validate. It only prevents silent revision. | It makes the exploratory/confirmatory boundary machine-checkable. | Research | +| `cc.evidence.decay` | Do claims expire? | A claim carries a policy defining when it must be rechecked, degraded, or expired. | `tests/unit/evidence/` decay tests. | Does not establish that the system is currently safe. It defines when to look again. | Treats claims as mortal by construction. | Research | +| `cc.evidence.merkle_log` / `anchoring` | Can a reviewer detect a silent rewrite of history? | Tamper evidence over recorded bytes under canonical serialization. | `test_transparency_log_adversarial.py`. | Integrity only. Never statistical validity, never safety. | The adversarial test is the artifact; the log is just the subject. | Research | +| `cc.evidence.claim_governance` | Can a claim package be checked for internal consistency? | PASS means internally consistent under verifier rules. | `test_claim_governance.py`, capsule integration. | PASS is not deployment safety. The verdict says so in its own payload. | The success message carries its own limitation. | Research | +| `cc.reporting.report` strict models | Can a report be repaired quietly into validity? | Fail-closed validation: missing boundaries, extra fields, and reserved overclaim vocabulary are rejected. | `tests/unit/reporting/test_reporting.py` round-trip and rejection tests. | Schema validity is not semantic truth. | The word list means a report literally cannot say "proves". | Research | +| Deterministic claim-governance capsule | Can a whole governance chain be reproduced byte for byte? | Regeneration from declared inputs reproduces the manifest exactly. | `tests/integration/test_claim_governance_capsule.py`. | **Reproducibility from declared inputs is not measurement.** See [CH-001](challenges.md). | The strongest guard in the repository, and its boundary is now demonstrated rather than asserted. | Public | + +## Boundary and refusal artifacts + +These are the ones a portfolio would omit. They are the most characteristic. + +| Artifact | Question | Claim | Evidence | Limit | Interesting because | Public | +| --- | --- | --- | --- | --- | --- | --- | +| `docs/research/NON_CLAIMS.md` | What will this project never say? | Ten enumerated non-claims with reasons and substitutes. | The document, plus verifier-mandated boundaries it lists. | Cannot prevent bad-faith actors from overclaiming outside the framework. | A repository that publishes its own refusals is unusual. | Public | +| `CLAIM_BOUNDARY_MANIFEST` + validator | Can public language be checked against evidence? | Every active claim maps to a level, lane, support, test, and non-claim. | `tests/unit/docs/test_claim_boundary_manifest.py`. | Documents boundaries; creates no authority. | Forbidden *upgrades* are enumerated, not just forbidden words. | Public | +| Paper-core language quarantine | Can legacy vocabulary leak into the paper? | Named terms cannot appear in paper-core texts. | `tests/unit/kernel/test_paper_core_language_quarantine.py`. | Only covers the listed files and terms. | A unit test whose subject is prose. | Public | +| Generated-artifact boundary checker | Which outputs may be tracked as evidence? | Runtime, fixture, archive, and release artifacts are separated and enforced. | `scripts/check_artifact_boundary.py --static`, CI. | Static rules; does not judge artifact quality. | Stops generated results from quietly becoming evidence. | Public | +| `cc_max` deprecation warning | What happens to a metric that outlived its justification? | Legacy exploratory metric, not a partial-identification claim, not evidence of safety. | The `FutureWarning` raised at call time. | Preserved for compatibility, which is itself a risk. | The code apologizes for itself at runtime. | Public | + +## Communication and interface + +| Artifact | Question | Claim | Evidence | Limit | Interesting because | Public | +| --- | --- | --- | --- | --- | --- | --- | +| Claim Observatory World V3 | Can epistemic constraints be spatial? | Eleven chambers mapping claim integrity stages to physical metaphors. | Blender scenes, renders, `visual_world_manifest_v3.json`, world bible. | A teaching environment. Not a dashboard and not a certification surface. | Law 10: the world must make overclaiming uncomfortable. | Public | +| *Before You See It* (15s film) | Can epistemic restraint be compelling? | None. It is a statement of method. | The film itself. | Establishes no technical claim; labels its own unverified frame. | It says INCONCLUSIVE on purpose, in its own advertising. | Public | +| Canonical page | Can a visitor encounter the philosophy and test it in the same minute? | The embedded bytes hash to the digest recorded in the capsule manifest. | Live WebCrypto recomputation in the visitor's browser. | Integrity only. Says nothing about whether the number in the file is true. | The claim can actually fail in front of the reader. | Public | +| Drift guard | Can an honest screen go stale and start lying? | The page's embedded fixture and digest match the capsule. | `tests/unit/docs/test_canonical_page_fixture.py`. | Enforces internal consistency, not authenticity. Demonstrated by [CH-001](challenges.md). | The guard's own limit was found by attacking it, not by reasoning about it. | Public | +| Source ledger | What may be inferred from a narrative? | Nothing, until translated, predeclared, and observed. | The ledger itself, applied to a real source. | Does not establish that its source is accurate or inaccurate. | It takes the story seriously enough to test it, and refuses to mock it. | Public | +| Eight-week plan + prompt library | Can a method be handed to someone else? | A schedule with gates, and eleven prompts that produce artifacts rather than predictions. | The documents. | Completing them produces no certainty. | Every week has a stop condition. | Public | + +## Adjacent, institutionally separate + +| Artifact | Question | Relationship | +| --- | --- | --- | +| Ghost-Ark | Can a hostile reviewer independently determine what a system does and does not prove — by replaying receipts, inspecting malicious corpora, and validating evidence windows? | Informs the thinking. Not a Cubits11 product, not a university endorsement, not a commercial credential. Referenced only as a labelled research case study. | + +## What the inventory shows + +Ninety Python modules, one hundred and six test files, ninety-six documents — +and the through-line is not any of them. It is that a surprising fraction of the +artifacts exist **to prevent a conclusion**, not to produce one: a quarantine on +vocabulary, a boundary on generated files, a warning attached to a metric, a +verdict that carries its own caveat, a film that labels its own footage. + +That is the invariant. It is developed in the [research graph](research-graph.md). diff --git a/docs/research/epistemic-program/release-freeze.md b/docs/research/epistemic-program/release-freeze.md new file mode 100644 index 00000000..a787e3b2 --- /dev/null +++ b/docs/research/epistemic-program/release-freeze.md @@ -0,0 +1,91 @@ +# Release Freeze + +Written last, on purpose. No new ideas after this point — only a record of what +exists and a decision about what ships. + +Temperance gets the session. The Emperor gets the last ten minutes. + +## What exists now + +| Layer | Artifact | State | +| --- | --- | --- | +| Research | Finite-atom kernel, Fréchet bounds, endpoint witnesses, sample complexity | Release-candidate, tested | +| Governance | Role ontology, confirmatory protocol, decay, receipts, claim governance | Implemented, tested | +| Capsule | Deterministic claim-governance capsule | Reproducible; boundary demonstrated by CH-001 | +| Method | Future-expansion package: source ledger, eight-week plan, prompt library | Written | +| Film | *Before You See It*, three cuts, 15.000s each | Rendered | +| Page | Canonical page with live WebCrypto check | Built and browser-tested | +| Program | Inventory, research graph, scope inflation, challenges, manifold, packet, intake | Written this session | +| Guards | Drift guard, artifact boundary, language quarantine, claim manifest validator | Passing | + +## The decisions + +**What ships.** +The canonical page and the film. Nothing else in the same release. + +**The central demonstration.** +The browser integrity check. One object, operable by a stranger in under a +minute, breakable on purpose. + +**The sentence that accompanies it.** + +```text +Bytes verified. Claim unresolved. +``` + +**What is explicitly not being claimed.** + +- Not that any system is safe. +- Not that a digest match says anything about the value inside the file. +- Not that reproducibility from declared inputs establishes that those inputs + were measured — CH-001 shows it does not. +- Not that the method has been validated. It has been defined and applied here. +- Not that any offer has been delivered or any intake has been run. + +**Where someone inspects the source.** + +```text +page visual_identity/canonical_page/index.html +fixture examples/claim_governance_capsule/expected/calibration.json +digest examples/claim_governance_capsule/manifest.expected.json +guard tests/unit/docs/test_canonical_page_fixture.py +film visual_identity/before_you_see_it/ +finding docs/research/epistemic-program/challenges.md (CH-001) +``` + +**The one action a visitor should take.** + +Change one character in the box and watch it fail. Everything else — the film, +the ledger, the invitation — is optional. That single interaction is the whole +argument, and it is the only thing the release asks for. + +## What is deliberately not in this release + +- The three short clips. Ready to derive, not ready to ship. +- Posts 3–5 of the packet. Post 3 waits for CH-001's write-up and the page's + updated limitation to be live, so it lands on an artifact that already + reflects the finding. +- Any paid offer. The offers are defined; none has been sold or delivered. +- CH-002. Attempted and partially answered; its current field remains asserted + with unresolved semantics, and no independent anchor exists yet. +- CH-003 through CH-005. Open, unattempted, and published as open. + +## The next artifact + +CH-002 was attempted in the same session and came back with the second outcome: +no repository-local mechanism can distinguish a measured input from an asserted +one, because deterministic regeneration lets the author construct any state the +checks accept. Reproducibility amplifies consistency, not truth. + +The completed local step is an explicit provenance record: `realized_fpr` is +now labelled as semantically unresolved and asserted, rather than silently +looking derived because it resembles `1/24`. A future derivation must name its +event, population, source bytes, counts, and rendering; this fixture does not. + +The next artifact remains the cheapest external anchor from that write-up — +timestamping an input at collection, outside the author's control. It requires +an independent service or measuring party and is not simulated here. + +## Frozen + +No redesigning the philosophy at minute 178. diff --git a/docs/research/epistemic-program/research-graph.md b/docs/research/epistemic-program/research-graph.md new file mode 100644 index 00000000..0b6f66b6 --- /dev/null +++ b/docs/research/epistemic-program/research-graph.md @@ -0,0 +1,142 @@ +# The Research Graph + +## The invariant + +Reading the [inventory](inventory.md) for repeated verbs rather than repeated +nouns, the same loop appears in the kernel, in the evidence layer, in the tests, +in the film, and on the canonical page: + +```text +claim + → decompose + → bind to artifact + → reproduce + → mutate + → observe + → bound the conclusion + → expose the limitation +``` + +The `mutate` step is the one most projects skip. It is the difference between a +system that reports success and a system that can be shown to fail. Every +artifact in the inventory that feels distinctive has a mutation step somewhere: +the adversarial transparency-log test, the one-character edit on the canonical +page, the reserved-vocabulary rejection, [CH-001](challenges.md). + +## Ten ways to say it + +Written to be accurate rather than good. + +1. Across my work, I keep trying to **make technical claims executable enough to challenge**. +2. …to **preserve the path between an assertion and the artifact underneath it**. +3. …to **stop green checks from silently expanding into larger claims**. +4. …to **design interfaces where evidence retains its boundaries**. +5. …to **make uncertainty inspectable without making it boring**. +6. …to **give a result somewhere to fail, in public, before anyone asks**. +7. …to **keep the assumptions attached to the number after it leaves the room**. +8. …to **make the width of an interval as visible as its midpoint**. +9. …to **build systems that can say INCONCLUSIVE without treating it as failure**. +10. …to **make overclaiming physically uncomfortable to do**. + +Numbers 3 and 10 are the annoyingly obvious ones. Those are the keepers. + +## The graph + +At the centre, one edge: + +```text +CLAIM ────────────► EVIDENCE + supported by +``` + +Everything else is a question about how that edge can fail. + +```mermaid +graph TD + C[CLAIM] -->|supported by| E[EVIDENCE] + + V[Verification
can another system reproduce it?] --> E + P[Provenance
where did this artifact come from?] --> E + K[Composition
what happens when evidence interacts?] --> E + R[Correlation
when do independent checks give less than expected?] --> E + I[Interfaces
how do humans see the evidence?] --> C + U[Uncertainty
can the system say INCONCLUSIVE?] --> C + A[Adversarial challenge
can someone try to falsify it?] --> E + M[Communication
how is this explained without killing curiosity?] --> C +``` + +### The eight nodes + +| Node | The question it holds open | +| --- | --- | +| **Verification** | Can another system, run by someone else, reproduce the observation? | +| **Provenance** | Where did this artifact come from, and what has happened to it since? | +| **Composition** | What happens when individual pieces of evidence are combined? | +| **Correlation** | When do apparently independent checks provide less evidence than their count suggests? | +| **Interfaces** | How does a human actually see the evidence, and what do they see instead of it? | +| **Uncertainty** | Can the system say INCONCLUSIVE, and does anything downstream survive it? | +| **Adversarial challenge** | Can someone deliberately attempt to falsify the result, and is that attempt welcome? | +| **Communication** | How is all of this explained without destroying the curiosity that brought someone here? | + +## Where the artifacts attach + +| Artifact | Nodes | +| --- | --- | +| Finite-atom LP, Fréchet bounds, endpoint witnesses | Composition, Correlation | +| Correlation cliff experiments | Correlation, Uncertainty | +| Sample complexity, outer intervals | Verification, Uncertainty | +| Role ontology, confirmatory protocol | Composition, Verification | +| Merkle log, anchoring, receipts | Provenance, Verification | +| Decay policy | Provenance, Uncertainty | +| Claim governance verdict | Composition, Interfaces | +| Deterministic capsule | Verification, Provenance | +| Adversarial transparency-log tests, CH-001 | Adversarial challenge, Verification | +| Non-claims, claim boundary manifest, language quarantine | Interfaces, Communication | +| Claim Observatory | Interfaces, Communication | +| *Before You See It* | Communication, Uncertainty, Interfaces | +| Canonical page + WebCrypto check | Verification, Interfaces, Communication | +| Source ledger, eight-week plan, prompt library | Uncertainty, Communication | +| Ghost-Ark | Provenance, Verification, Adversarial challenge | + +Every node has at least two artifacts attached, and no artifact attaches to only +one. That is what makes this a program rather than a portfolio: the artifacts +were not planned against this graph, and they populate it anyway. + +## The thinnest node + +**Adversarial challenge** was, until this session, held up almost entirely by +one adversarial test file. [CH-001](challenges.md) was run specifically to load +it, and it produced the most useful result in the inventory. The node is still +thin. That is where the next work goes. + +## Organizational separation + +Intellectual connection does not require organizational merger. + +It is accurate to say: + +> These projects inform my thinking. + +It is not accurate, and not permitted, to say: + +> These are all Cubits11 products. + +| Name | In the graph | In the org | +| --- | --- | --- | +| CC-Framework | Composition, Correlation, bounded evidence | Research repository | +| Ghost-Ark | Provenance, Verification, Adversarial challenge | **Separate institutional artifact.** Not a product, not a credential, not an endorsement. | +| Canonical page, film | Interfaces, Communication, Uncertainty | Cubits11 public work | + +Keeping these distinct is not modesty. A shared intellectual frame that is +presented as a shared commercial entity is itself a scope inflation — the same +error described in +[evidence scope inflation](evidence-scope-inflation.md), applied to +institutions instead of hashes. + +## Non-claims + +- The graph is a description of existing work, not a claim that the work is + correct, complete, or novel. +- Attaching an artifact to a node does not mean that node's question has been + answered. +- The ten sentences are attempts at accuracy about intent. They are not findings. diff --git a/docs/research/epistemic-program/visibility-manifold.md b/docs/research/epistemic-program/visibility-manifold.md new file mode 100644 index 00000000..1f7d2798 --- /dev/null +++ b/docs/research/epistemic-program/visibility-manifold.md @@ -0,0 +1,134 @@ +# The Visibility Manifold + +## The reframe + +Do not think: *make more posts.* + +Think: + +> Increase the number of paths through which the same intellectual object can be +> encountered. + +Five independent posts decay independently. Seven surfaces of one inspectable +object compound, because every surface is a different-sized door into the same +room, and everyone who walks through any door arrives at the same place. + +```text + short clip + ↓ + technical note → CANONICAL ARTIFACT ← philosophical post + ↑ + source + tests + ↑ + challenge +``` + +The rule: **every surface must point back at something that can fail.** A +surface that points only at another surface is marketing. A surface that points +at an object a stranger can break is research. + +## The worked example + +One object. The whole thing is: + +```text +189 raw bytes → manifest SHA-256 → browser WebCrypto → equality + → mutation → FAIL CLOSED +``` + +That is small enough to hold in one hand and real enough to break. Seven +surfaces come off it. + +### Surface A — the experience + +The canonical page. Someone reads two paragraphs, edits a character, and watches +a verification fail. Total time: under a minute. No install, no signup, no +server. + +### Surface B — the visual + +A ten-second silent recording, no narration: + +```text +BYTES MATCH → one character changes → FAIL CLOSED → restore → BYTES MATCH +``` + +Rendered deterministically from the page itself, the same way the film is, so +the clip cannot drift from the artifact it depicts. Built by +`visual_identity/before_you_see_it/render_check_clip.py`. + +### Surface C — the aphorism + +> A green check should have a falsifiable path to red. + +Carries on its own, out of context, and survives being quoted by someone who +never saw the page. + +### Surface D — the technical note + +Why hashing the *raw bytes* matters; why the manifest digest has to mean the same +thing the browser computes; why the drift guard exists; and precisely what the +match does not prove. Ends at the limitation, not at the success. + +### Surface E — the source + +Direct links, no navigation required: the fixture, the manifest entry, the page, +the drift guard test. Anyone can run `sha256sum` and get the same number. + +### Surface F — the challenge + +> Can you make this page display `BYTES MATCH` while changing the underlying +> fixture, without causing the drift guard to fail? + +This one has already been answered — by us, against ourselves. See +[CH-001](challenges.md). The challenge surface is the strongest of the seven, +because it converts an audience into participants and because its most valuable +possible outcome is being proven wrong in public. + +### Surface G — the philosophical post + +> Integrity is not truth. + +The digest proves something narrow: *these bytes correspond to those bytes.* It +does not prove *the number inside the file is correct.* And CH-001 showed the +next rung: reproducing the whole chain from declared inputs does not prove the +inputs were measured. Developed in +[evidence scope inflation](evidence-scope-inflation.md). + +## Why this object works + +Not because SHA-256 is novel. It is the opposite of novel. It works because the +**epistemic boundary is visible**, and because the demonstration is small enough +that the boundary is the only interesting thing in the frame. + +That is the selection rule for future objects: + +| Test | Why | +| --- | --- | +| Can a stranger make it fail in under a minute? | Failure they cause themselves is understood, not believed. | +| Is the mechanism boring? | A novel mechanism draws attention to itself and away from the boundary. | +| Is the limitation more interesting than the success? | If not, the object is a demo, not a demonstration. | +| Does the source fit on one screen? | Inspectability is a size property before it is a licensing one. | +| Would publishing a negative result about it improve it? | If not, it cannot participate in the loop. | + +## The multiplication routine + +For each new real artifact: + +1. Name the one-line mechanism. If it takes a paragraph, it is not the object yet. +2. Build Surface A — the thing a stranger can operate. +3. Record Surface B from A directly, never re-staged. +4. Write the limitation before the announcement. +5. Extract the aphorism from the limitation, not from the success. +6. Publish the source path in the same breath as the result. +7. Open a challenge against it, and answer that challenge yourself first. + +Step 7 is what stops the manifold from becoming a funnel. + +## Non-claims + +- This is a method for distribution, not evidence that the distribution works. + No reach, engagement, or reproduction has been measured. +- The seven surfaces are a decomposition of one object, not a content calendar. +- Surface F's value depends entirely on people actually attempting it. Until + someone outside this repository does, the adversarial node stays thin. diff --git a/docs/research/future-expansion/README.md b/docs/research/future-expansion/README.md new file mode 100644 index 00000000..b717b7cf --- /dev/null +++ b/docs/research/future-expansion/README.md @@ -0,0 +1,134 @@ +# Future Expansion: Epistemic Research Upgrade + +## Status + +Method documentation. This package defines how a narrative, an intuition, or a +motivational source is converted into research objects that can be tested, +replicated, or discarded. + +It adds no claim to the +[Claim Boundary Manifest](../../claims/CLAIM_BOUNDARY_MANIFEST.md), changes no +kernel behavior, produces no measurement, and is not evidence for anything. It +is a procedure for producing evidence later, under stated rules. + +## Purpose + +Turn subjective intuition into disciplined inquiry without pretending that +intuition is evidence. + +A source narrative may remain meaningful as reflection or motivation. It must +not be used to predict revenue, demand, personal outcomes, technical +feasibility, or market timing. The operating rule is: + +```text +A story may generate a question. +Only a registered observation may support an answer. +``` + +## The operating loop + +```text +Narrative / intuition + -> candidate hypothesis + -> alternative explanation + -> predeclared indicator + -> reversible test + -> observed result + -> replication or control + -> bounded decision +``` + +The loop is one-directional. A result may retire a hypothesis; it may not +retroactively edit the indicator that was predeclared to test it. + +## Non-negotiable rules + +1. No narrative-derived statement becomes a business, product, or technical + claim. +2. Every experiment names a decision it can change and a result that would stop + the work. +3. Results, interpretation, and non-claims remain separate fields. +4. No human research, customer outreach, external data collection, paid API + use, or live cloud work occurs without explicit authorization and + appropriate consent. +5. Tracks stay separate. Two projects may share epistemic methods without + sharing customers, claims, data, or branding. + +## Research foundations + +- Personalized but broadly applicable descriptions can feel unusually accurate. + When studying "resonance," use blinded or shuffled comparison material rather + than assuming felt accuracy reflects predictive accuracy. + [Mason & Budge, 2011](https://pubmed.ncbi.nlm.nih.gov/21315874/) +- Freeze protocols before observation. OSF registrations are designed as a + fixed project state that allows a visible withdrawal record rather than + silent revision. + [OSF registration guidance](https://help.osf.io/article/330-welcome-to-registrations) +- Treat small early studies as feasibility work with explicit progression + criteria, not as demonstrations of effectiveness. + [CONSORT pilot and feasibility guidance](https://www.bmj.com/content/355/bmj.i5239) +- Attestation mechanisms have separate proof boundaries. AWS Nitro attests + enclave measurements; Apple App Attest validates an app instance connecting + to a server; C2PA validates association and integrity of signed assertions, + not whether those assertions are true. + [AWS Nitro](https://docs.aws.amazon.com/enclaves/latest/user/set-up-attestation.html), + [Apple App Attest](https://developer.apple.com/documentation/DeviceCheck?changes=_3), + [C2PA](https://spec.c2pa.org/specifications/specifications/1.1/specs/C2PA_Specification.html) + +## Contents + +| Document | What it is | When to use it | +| --- | --- | --- | +| [Source Ledger](source-ledger.md) | A worked Week 1 artifact: one narrative source classified statement by statement into reflection, forecast, causal assertion, action heuristic, empirical claim, and marketing funnel, with testable translations and disallowed inferences. | Whenever a narrative, pitch, reading, or founder intuition is proposed as a reason to act. | +| [Eight-Week Plan](eight-week-plan.md) | The schedule: convert, destroy boundaries, pre-register, build instruments, run reversible work, replicate, review adversarially, decide. | Starting a bounded research push where the deliverable is a dated evidence packet. | +| [Prompt Library](prompt-library.md) | Eleven prompts that produce research artifacts rather than predictions. | Drafting a ledger entry, a boundary audit, a prior-art sweep, a preregistration, a replay handoff, a decision memo, or a retraction. | + +## Relationship to existing CC-Framework surfaces + +This package does not create a parallel governance process. Where CC-Framework +already has a mechanism, the plan uses it: + +| Plan artifact | Existing surface | +| --- | --- | +| Predeclared claim with owner, falsifier, decision, window, non-claim | [Claim Boundary Manifest](../../claims/CLAIM_BOUNDARY_MANIFEST.md) and `docs/claims/claim_boundary_manifest.v0.1.json` | +| Explicit non-claim per result | [Non-Claims](../NON_CLAIMS.md) | +| Evidence typed by what it can support | [Evidence Role Ontology](../../design-specs/evidence_role_ontology.md) | +| Exploratory result kept separate from confirmatory result | `src/cc/evidence/confirmatory_protocol.py` | +| Result that expires and must be rechecked | `src/cc/evidence/decay.py` | +| Replayable record of what was run | [CC Reports and Receipts](../CC_REPORTS.md) | +| Scoped human sign-off that does not upgrade evidence | `human_review_note` handling in `src/cc/evidence/claim_governance.py`; the dedicated artifact is still a planned [prompt-roadmap item](../prompt-roadmap/09_human_review_artifact.md) | +| Prompt-shaped execution specs | [Prompt Roadmap](../prompt-roadmap/02_claim_envelope_boundary_envelope.md) | + +The one boundary worth restating: a governance PASS means an evidence-bound +package is internally consistent under the verifier rules. It does not mean the +underlying result is correct, and it does not mean a system is safe. + +## Porting to sibling projects + +The eight-week plan carries two project-specific tracks, written to be lifted +into their own repositories rather than run from here. When those repositories +are available as writable workspaces, copy this directory to: + +| Track | Destination | +| --- | --- | +| B2B / Assay-Limina | `limina/docs/future-expansion/` | +| Vinctura | `docs/pivot/research/future-expansion/` | + +Two conditions apply to that copy. First, reconcile the plan with whatever +experiment registry and claim register already exist in the destination +repository; do not create a second, competing set of metrics. Second, check the +destination working tree before writing, since uncommitted work in an unrelated +area is easy to bury. + +## Non-claims + +- This package does not establish that any narrative source is accurate, + predictive, or causal. +- Classifying a statement as testable does not mean it has been tested. +- Completing the eight weeks does not produce certainty; the completion + standard is fewer unsupported claims, better instruments, clearer stop + conditions, preserved negative evidence, at least one independently + replayable result, and a more honest next decision. +- Nothing here authorizes human-subject research, customer outreach, external + data collection, or paid or live cloud work. Those require separate, + explicit authorization. diff --git a/docs/research/future-expansion/eight-week-plan.md b/docs/research/future-expansion/eight-week-plan.md new file mode 100644 index 00000000..0e96a7a1 --- /dev/null +++ b/docs/research/future-expansion/eight-week-plan.md @@ -0,0 +1,285 @@ +# Eight-Week Epistemic Research Upgrade + +## Purpose + +Turn subjective intuition into disciplined inquiry without pretending that +intuition is evidence. + +A source narrative may remain meaningful as reflection or motivation. It must +not be used to predict revenue, demand, personal outcomes, technical +feasibility, or market timing. + +```text +A story may generate a question. +Only a registered observation may support an answer. +``` + +The operating loop, the non-negotiable rules, and the research foundations are +stated once in the [package README](README.md) and are not repeated here. + +## Tracks + +The plan runs two independent project tracks that share methods and nothing +else - no customers, claims, data, or branding. + +- **B2B / Assay-Limina.** A technical attestation and provenance wedge. Its + central risk is claiming more than a cryptographic mechanism establishes. +- **Vinctura.** A service operation. Its central risk is promising an operating + standard that has not been performed and measured. + +CC-Framework hosts this package but is not a third track. Its role is the +[mapping appendix](#appendix-mapping-onto-cc-framework): the plan reuses the +governance surfaces this repository already has instead of inventing parallel +ones. + +## Week 1 - Convert story into research objects + +**Shared work** + +Create a four-column source ledger. A worked example against a real source is in +[Source Ledger](source-ledger.md). + +| Source statement | Classification | Testable translation | Disallowed inference | +| --- | --- | --- | --- | +| "A breakthrough is near" | Forecast | Does a predeclared intervention improve a measured outcome? | Revenue or success is inevitable. | +| "Follow signs" | Ambiguous heuristic | Which traction indicators are defined before results appear? | Retrospective coincidence proves causation. | +| "Experiment, then formalize what works" | Method heuristic | Does the result reproduce with a control or second run? | One outcome proves a scalable model. | +| "You are inventive" | Reflective affirmation | Does reflective framing alter effort or experiment completion? | Personality claims are facts. | + +**B2B / Assay-Limina** + +- Inventory every public technical claim. +- Map: claim, mechanism, trust root, measured artifact, missing evidence, and + explicit non-claim. +- Flag any claim that moves from "enclave measurement" to "physical capture," + "fraud elimination," or legal admissibility without a demonstrated binding. + +**Vinctura** + +- Freeze the actual near-term question: whether the operating promise - queue + avoidance, dependable delivery, and a well-run session - can be performed and + measured. +- Reconcile the plan with the existing `business/experiments/registry.yml`. Do + not create parallel competing metrics. +- Keep narrative-derived language out of customer-facing copy. + +**Gate** + +Every active proposition has an owner, evidence state, falsifier, decision, time +window, and non-claim. + +## Week 2 - Prior-art and boundary destruction + +**B2B / Assay-Limina** + +- Build an attestation-boundary brief covering what Nitro PCRs establish, what + App Attest establishes, what C2PA establishes, and what none of them + establishes alone. +- Run a prior-art and public-incident sweep for canonicalization, parse + disagreement, provenance failures, and attestation misuse. +- Create a claim kill list: language that must not ship without specific + evidence. + +**Vinctura** + +- Audit every current customer-facing statement against the existing + `business/evidence/claims.yml`. +- Separate externally sourced market facts, founder assertions, local + observations, and assumptions. +- Define a customer-problem evidence map: queue, turnaround, booking friction, + group coordination, route reliability, weather, and portfolio quality. + +**Gate** + +No "novel," "eliminates," "proves," "secure," "guarantees," or "best" wording +survives without a named evidentiary standard. + +## Week 3 - Pre-register the decisive experiments + +**B2B / Assay-Limina** + +Prioritize existing high-information work. + +1. **X2 - Model-consumer kernel.** Test whether model consumers unify + distinctions that named consumers require, with repeated runs, multiple model + families, a constrained-output control, a frozen corpus, and an explicit null + condition. +2. **X5 - Consequence search.** Systematically identify public incidents where + canonicalization, parser differential, or identity collapse caused a + consequential failure. A sparse result is still a result. + +**Vinctura** + +Prioritize the existing decision-changing experiments. + +1. `EXP-FULFILLMENT` - timed end-to-end gallery completion. +2. `EXP-PRICE` - qualified inquiry-to-booking and package mix. +3. `EXP-GROUP` - coordination cost and realized contribution per founder hour. +4. `EXP-ROUTE` - whether published session routes actually fit their time + budget. + +For each, preserve the current success and failure branches rather than +rewriting them after a result appears. + +**Gate** + +A protocol specifies population, provenance, denominator, controls or +alternative explanations, stop and scale rules, missing-data handling, and the +non-claim. + +## Week 4 - Build instruments before collecting outcomes + +**B2B / Assay-Limina** + +- Create a research-only proof-boundary notebook or browser lesson recording the + claim, the exact verification input, the trusted root, the mutation that must + fail, and what remains unknown. +- Add a positive-control requirement: deliberately break one verifier assumption + and show that the instrument reports the break. +- Write the independent replay instructions before running the experiment. + +**Vinctura** + +- Prepare a consent-aware operational log for real sessions: inquiry source, + quote, booking outcome, package, route timestamps, queue minutes, delivery + timestamps, founder minutes by phase, deviations, and customer-initiated + comments. +- Do not introduce third-party tracking merely to measure a funnel. Preserve the + site's privacy commitments unless a documented decision changes them. + +**Gate** + +An instrument that cannot reveal a deliberately introduced defect is not ready +to measure success. + +## Week 5 - Run reversible, low-risk evidence work + +**B2B / Assay-Limina** + +- Run local, reproducible corpus and verifier work only. +- Record source revision, host, tool versions, inputs, output hashes, + unavailable arms, and failures. +- Do not run live cloud validation or describe attestation as deployed without + explicit approval. + +**Vinctura** + +- Run only real operational observations authorized by the founder: founding + cohort portfolio work, stopwatch-based fulfillment timing, route timing, + inquiry logging, and group coordination observations. +- Do not state a delivery promise until it is repeatedly met under recorded + conditions. + +**Gate** + +Raw observations are stored before interpretation. "It felt like momentum" is +never a result field. + +## Week 6 - Replication, controls, and negative evidence + +**B2B / Assay-Limina** + +- Re-run selected cells on a second environment or an independent + implementation. +- Test the constrained-output model arm against the same corpus. +- Publish negative findings, unavailable arms, and disagreements. + +**Vinctura** + +- Compare first-session and later-session timing rather than treating the first + session as representative. +- For delivery experiments, retain the planned alternating or control pattern. +- Analyze whether the actual bottleneck is editing, planning, travel, + coordination, or customer response. + +**Gate** + +A one-off success produces a feasibility note, never a mature product claim. + +## Week 7 - Adversarial review + +**B2B / Assay-Limina** + +Ask an independent reviewer to attack trust-root substitutions, replay windows, +parser and canonicalization mismatch, device-to-byte binding gaps, hidden +assumptions in "physical capture" language, missing controls, and unreported +unavailable arms. + +**Vinctura** + +Ask a skeptical reviewer to attack package-price comparability, portfolio claim +boundaries, delivery and route promises, customer-data privacy, operational +feasibility during peak commencement periods, and whether a claim gate hides a +claim in metadata or alternate routes. + +**Gate** + +Every substantive objection ends as corrected, rejected with evidence, +explicitly unresolved, or retracted. + +## Week 8 - Decision and publication boundary + +Produce two dated evidence packets. + +**B2B / Assay-Limina packet** + +- Claim-to-evidence matrix. +- Experiment reports and replay commands. +- Technical boundary chart. +- Prior-art and incident-search results. +- Retracted or narrowed statements. +- Decision: continue research, narrow to a verifier-side learning module, pursue + independent replication, or halt a weak wedge. + +**Vinctura packet** + +- Experiment registry status. +- Observed versus assumed operating model. +- Actual fulfillment and route data. +- Claim-register updates. +- Launch-readiness status. +- Decision: proceed, hold, reprice, simplify the offer, or remove an unevidenced + promise. + +## Completion standard + +At the end of eight weeks, neither project should claim more certainty. Each +should have: + +- fewer unsupported claims, +- better instruments, +- clearer stop conditions, +- preserved negative evidence, +- at least one independently replayable result, +- a more honest next decision. + +## Appendix: mapping onto CC-Framework + +CC-Framework already implements most of what this plan asks for. When the plan +is run from this repository, use these surfaces rather than building new ones. + +| Week | Plan artifact | Existing surface | +| --- | --- | --- | +| 1 | Proposition with owner, falsifier, decision, window, non-claim | [Claim Boundary Manifest](../../claims/CLAIM_BOUNDARY_MANIFEST.md) and its JSON companion | +| 2 | Claim kill list and boundary brief | [Non-Claims](../NON_CLAIMS.md), specifically the forbidden-upgrade table | +| 3 | Preregistered protocol with stop and scale rules | [Confirmatory Evidence Protocol](../../design-specs/confirmatory_protocol.md) | +| 4 | Instrument that must detect a deliberate defect | Adversarial tests under `tests/unit/evidence/`, e.g. `test_transparency_log_adversarial.py` | +| 5 | Recorded run with inputs, versions, and output hashes | [CC Reports and Receipts](../CC_REPORTS.md) and the deterministic capsule under `examples/claim_governance_capsule/` | +| 6 | Replication and evidence typed by what it supports | [Evidence Role Ontology](../../design-specs/evidence_role_ontology.md); exploratory results stay separate from confirmatory ones | +| 7 | Review that does not upgrade evidence | `human_review_note` handling in `src/cc/evidence/claim_governance.py` | +| 8 | Dated packet with a bounded decision | [Claim Envelope](../../design-specs/claim_envelope.md) plus decay semantics in `src/cc/evidence/decay.py` | + +Two boundaries carry over unchanged. A governance PASS means an evidence-bound +package is internally consistent under the verifier rules; it is not a statement +that a system is safe. A receipt establishes byte integrity; it is not a +statement that a measurement was valid. + +## Non-claims + +- Completing this plan does not validate any project, claim, or narrative. +- Neither evidence packet is a certification, an audit opinion, or a compliance + artifact. +- The Week 5 and Week 6 activities described here are not authorization to + perform them. Human research, customer outreach, external data collection, + paid API use, and live cloud work each require separate explicit + authorization, and human observation requires appropriate consent. diff --git a/docs/research/future-expansion/prompt-library.md b/docs/research/future-expansion/prompt-library.md new file mode 100644 index 00000000..64565e37 --- /dev/null +++ b/docs/research/future-expansion/prompt-library.md @@ -0,0 +1,277 @@ +# Epistemic Prompt Library + +## How to use these + +These prompts produce research artifacts, not predictions. Replace bracketed +text before use. Require citations, source dates, uncertainty, and an explicit +non-claim in every answer. + +Two standing constraints apply to every prompt below. + +- A model's output is a draft artifact, not evidence. It enters the record the + same way any other draft does: predeclared, reviewed, and bounded. +- If a prompt's answer would change a financial, legal, medical, safety, or + otherwise consequential decision, the answer is insufficient on its own. + +The prompts map onto the [eight-week plan](eight-week-plan.md) roughly in order: +1 for Week 1, 2 and 3 for Week 2, 4 and 5 for Week 3, 6 and 7 for Weeks 3 to 5, +8 for Week 4, 9 for Week 6, 10 for Week 8, and 11 whenever a prior statement +stops being supportable. + +## 1. Narrative-to-study translator + +```text +You are an epistemic editor. Analyze the following narrative without validating +its predictions or metaphysical claims: + +[PASTE TEXT] + +For every substantive statement, label it as: +REFLECTION, IDENTITY AFFIRMATION, FORECAST, CAUSAL ASSERTION, +ACTION HEURISTIC, EMPIRICAL CLAIM, or MARKETING FUNNEL. + +Then produce: +1. a charitable interpretation; +2. a falsifiable translation, if one exists; +3. the minimum observation needed to test it; +4. one alternative explanation; +5. a non-claim; +6. whether using it for a financial, legal, medical, or consequential decision + would be inappropriate. + +Do not infer personal facts, future outcomes, or hidden causes. +``` + +## 2. Claim-to-proof boundary auditor + +```text +You are a hostile technical reviewer. For each claim below, identify exactly +what the mechanism proves, what it assumes, and what it cannot establish. + +[CLAIMS] +[ARCHITECTURE OR CODE LINKS] + +Return a table: +claim | cryptographic/operational object | trust root | verifier input | +attack or failure mode | evidence currently present | evidence missing | +allowed wording | prohibited wording | explicit non-claim + +Reject any leap from: +- app integrity to physical-world truth; +- enclave measurement to correct execution; +- signed assertion to assertion truth; +- provenance to authenticity; +- one successful run to universal reliability. +``` + +## 3. Prior-art destroyer + +```text +Act as a prior-art researcher trying to falsify this proposed novelty claim: + +[CLAIM] + +Search primary sources, standards, vendor documentation, peer-reviewed papers, +public incident reports, and active products. Separate: +- already solved; +- partially solved; +- adjacent but materially different; +- commercially occupied; +- genuinely unresolved. + +For every source, provide URL, publication date, source type, exact relation to +the claim, and a confidence rating. End with: +"what remains narrow enough to investigate honestly." + +Do not use search-result snippets as evidence. +``` + +## 4. Experiment preregistration architect + +```text +Turn this question into a preregistered experiment: + +[QUESTION] +[DECISION IT MUST CHANGE] +[AVAILABLE DATA / ACCESS LIMITS] + +Specify: +- hypothesis and falsifier; +- population and what it does not represent; +- sampling frame or census rationale; +- primary measure and denominator; +- control arm or alternative explanation; +- required positive-control/discriminator test; +- exclusions and unavailable-arm treatment; +- repeat count and stability plan; +- stop, scale, hold, and kill criteria; +- ethics/privacy requirements; +- exact non-claim. + +Do not choose thresholds after viewing results. +``` + +## 5. Signal-versus-traction designer + +```text +Convert this intuitive "signal" into a predeclared traction measure: + +[OBSERVATION OR INTUITION] + +Return: +1. the observable event; +2. the baseline or comparison; +3. the collection method; +4. the time window; +5. what would count as noise; +6. what result changes the decision; +7. the smallest reversible next action; +8. what must not be inferred. + +Explicitly distinguish attention, inquiries, conversion, repeat behavior, +revenue, margin, and retention. Never substitute one for another. +``` + +## 6. Customer-evidence interviewer + +```text +Design a neutral customer-discovery protocol for this decision: + +[QUEUE / DELIVERY / GROUPS / ROUTES / PRICING / OTHER] + +Create: +- recruitment criteria; +- non-leading interview questions; +- exact operational observations to collect; +- consent and privacy boundary; +- disconfirming answers to look for; +- how to avoid treating compliments as demand; +- a decision rule tied to the existing experiment registry; +- a summary template that separates direct quotes, observations, inference, + and non-claim. + +Do not promise services, ask for sensitive data, or create a marketing claim +from a single interview. +``` + +## 7. B2B consequence-search protocol + +```text +Research whether this technical failure class has caused a real consequence: + +[FAILURE CLASS] + +Search CVEs, advisories, standards errata, incident postmortems, court filings, +and vendor security reports. For each candidate, classify: +- mechanism; +- affected representation or parser; +- attacker capability; +- observed consequence; +- evidence quality; +- remediation; +- relation to our corpus; +- whether the incident supports prevalence, possibility only, or neither. + +Report null findings and search limitations. Do not turn a mechanism into a +market claim unless consequence evidence supports that step. +``` + +## 8. Instrument-sensitivity challenge + +```text +You are validating a research instrument, not its preferred answer. + +[INSTRUMENT] +[INTENDED DETECTION OR MEASUREMENT] + +Propose: +- one deliberate defect the instrument must detect; +- one negative control it must not flag; +- one boundary case; +- one unavailable-arm condition; +- one mutation that should change the result; +- one mutation that should not; +- a pass/fail criterion for each; +- the consequence if the instrument fails any test. + +Explain why a green result without these controls would be uninformative. +``` + +## 9. Independent replay handoff + +```text +Create a handoff for an independent reviewer to reproduce this result: + +[CLAIM] +[ARTIFACTS] +[COMMANDS] +[ENVIRONMENT] + +Include: +- exact inputs and hashes; +- expected output and failing output; +- tool versions; +- trust roots and keys that may be public; +- no-secret setup; +- likely failure modes; +- how to report disagreement; +- what agreement would and would not establish. + +The handoff must allow the reviewer to disagree safely and visibly. +``` + +## 10. Evidence-to-decision board + +```text +Given this evidence packet: + +[PACKET] + +Make a four-way decision memo: +CONTINUE, HOLD, NARROW, or STOP. + +For each option, state: +- facts supporting it; +- facts against it; +- assumptions still carrying the conclusion; +- the next cheapest high-information test; +- irreversible actions that must wait; +- claims that must be removed or downgraded now; +- the condition that would reverse the recommendation. + +Do not reward activity, novelty, or optimism. Reward only decision-relevant +evidence. +``` + +## 11. Retraction and revision writer + +```text +An earlier statement is no longer supportable: + +[OLD STATEMENT] +[NEW EVIDENCE] + +Write: +1. a precise retraction or downgrade; +2. what remains true; +3. what was not measured; +4. the artifact or source revision that changed the conclusion; +5. the new allowed wording; +6. the prohibited wording; +7. the next experiment, if any. + +Never silently delete the old claim or rewrite history as if it was never made. +``` + +## Non-claims + +- These prompts do not validate their own output. A completed table is a draft, + not a finding. +- A model asked to search may return plausible sources that do not exist or do + not say what the summary says. Every citation is checked against the primary + source before it enters an artifact. +- Prompt 1 classifies statements; it does not determine whether a narrative + source is meaningful to the person who brought it. +- Prompts 6 and 7 describe protocols. Running them against real people or + external systems requires separate explicit authorization and, for human + contact, appropriate consent. diff --git a/docs/research/future-expansion/source-ledger.md b/docs/research/future-expansion/source-ledger.md new file mode 100644 index 00000000..16355080 --- /dev/null +++ b/docs/research/future-expansion/source-ledger.md @@ -0,0 +1,122 @@ +# Source Ledger: A Worked Week 1 Artifact + +## Why this document exists + +Week 1 of the [eight-week plan](eight-week-plan.md) converts a narrative source +into research objects. This is a worked example of that conversion, using an +actual source: a recorded tarot reading addressed to a general sign-based +audience, brought into a working session as motivational material alongside two +active projects. + +The point is not to evaluate the source. The point is to demonstrate the gate +every narrative passes through before it is allowed near a decision. A source +that survives the gate contributes a *question*. It never contributes an +*answer*. + +The reading may remain meaningful as reflection. Nothing below asks anyone to +stop finding it meaningful. + +## Classification labels + +| Label | Definition | Evidential weight | +| --- | --- | --- | +| `REFLECTION` | A description of the subject's present state or recent behavior. | None. May be accurate and still carry no predictive content. | +| `IDENTITY AFFIRMATION` | A statement about who the subject is. | None. | +| `FORECAST` | A statement about what will happen. | None unless independently registered and observed. | +| `CAUSAL ASSERTION` | A statement that one thing brings about another. | None. Causal claims require causal assumptions and a design. | +| `ACTION HEURISTIC` | A rule for what to do next. | None as evidence, but may be a useful, separately justified practice. | +| `EMPIRICAL CLAIM` | A statement checkable against the world. | Only after predeclaration, observation, and control. | +| `MARKETING FUNNEL` | Content whose function is engagement or conversion for the source. | None, and a reason for caution about felt accuracy. | + +A statement can carry more than one label. Where it does, the strictest handling +applies. + +## The ledger + +| Source statement | Classification | Testable translation | Disallowed inference | +| --- | --- | --- | --- | +| "Things are just about to turn for you"; the wheel is moving in your favor. | `FORECAST` | None directly. The nearest registerable question is whether a predeclared intervention changes a predeclared measure inside a fixed window. | That improvement is inevitable, scheduled, or owed. Not usable for runway, hiring, spend, or pricing. | +| "You're going to experience more signs, symbols, and synchronicities" indicating things are turning. | `CAUSAL ASSERTION` / `ACTION HEURISTIC` | Name the traction indicators and the observation window *before* any result is seen, then count them. | That noticing a coincidence afterwards is evidence that a turn occurred. Retrospective matching is the failure mode this whole package exists to block. | +| "There's been a lot of experimentation going on... you've been testing different approaches." | `REFLECTION` | Count registered experiments started, completed, and abandoned in a fixed prior period. This is directly measurable from an experiment registry. | That the description being accurate makes the forecasts accurate. Broadly applicable descriptions feel personal ([Mason & Budge, 2011](https://pubmed.ncbi.nlm.nih.gov/21315874/)). | +| "Nobody knows what they're doing. We're all just experimenting." | `ACTION HEURISTIC` | Not a claim. Usable as morale. | That the absence of certainty excuses the absence of protocol. | +| "This is exactly why this is going to work for you" - because the approach is unconventional. | `FORECAST` + `CAUSAL ASSERTION` | None. The implied mechanism (unconventional therefore successful) is not identifiable from this source. | That unconventionality predicts success. Selection effects make surviving unconventional bets highly visible and failed ones invisible. | +| "You're inventing something here... solving a problem in a way that nobody ever thought of solving it before." | `IDENTITY AFFIRMATION` + implicit novelty claim | Run a prior-art sweep ([Prompt 3](prompt-library.md#3-prior-art-destroyer)). Novelty is established by failing to find prior art, not by feeling original. | That "novel," "first," or "nobody has done this" may appear in any public copy. Those words require a named evidentiary standard. | +| "You're noticing the thing that other people haven't noticed." | `IDENTITY AFFIRMATION` | Consequence search ([Prompt 7](prompt-library.md#7-b2b-consequence-search-protocol)): has this failure class produced a documented, consequential outcome anywhere? | That an unnoticed thing is therefore an important thing. A sparse search result is a result, and it may mean the thing is unimportant. | +| "Divine timing. Right place, right time, right idea." | `CAUSAL ASSERTION` | None. Unfalsifiable as stated. | That timing is externally arranged, and therefore that acting now is lower risk than acting later. | +| "I'm seeing a blue notebook"; someone working on a laptop in a car; "your phone screen is cracked"; a person with curly hair connected to the innovation; a conversation near a vending machine. | `EMPIRICAL CLAIM` | These are the only literally checkable statements in the source. See [The falsification exercise](#the-falsification-exercise) below - they can be tested, but only under predeclaration, base-rate comparison, and a blinded control. | That a later match confirms anything. Each item is high-prevalence, and "connected to the innovation" is elastic enough to fit almost any encounter unless bounded in advance. | +| "You get overwhelmed by data sometimes... tied up in a big tangle." | `REFLECTION` | Measure where founder hours actually go by phase. The stated bottleneck and the measured bottleneck are frequently different. | That the felt bottleneck is the real bottleneck. | +| "The right piece of information comes in at the right time and breaks the loop." | `ACTION HEURISTIC` | Identify the single cheapest measurement that would change the current decision, and run that one first. | That the information will arrive on its own, or that waiting is a strategy. | +| "There is money attached to this... revenue arriving... a sale, a commission, a royalty, or a referral." | `FORECAST` (financial) | None from this source. Conversion questions belong to an already-registered pricing or funnel experiment that exists independently of the reading. | Hard block: no revenue projection, no pricing decision, no spend commitment, no runway assumption, and no investor- or customer-facing statement may cite or rest on this. | +| "We're breaking through a ceiling... your income or role or reach is infinitely bigger than you thought." | `FORECAST` | None. | That current constraints are illusory. Constraints are measured, not dispelled. | +| "An experiment might work once... figure out why it worked, how it worked, and can I make it work again." | `ACTION HEURISTIC` (method) | Does the result reproduce on a second run, a second environment, or an independent implementation? This is the Week 6 replication gate. | That a single success is a scalable model. One-off success supports a feasibility note only ([CONSORT pilot guidance](https://www.bmj.com/content/355/bmj.i5239)). | +| "Now you've discovered something that works, you don't go messing with it... I need the discipline to not change anything anymore." | `ACTION HEURISTIC` (method) | Freeze the protocol before observation and record deviations rather than absorbing them ([OSF registration guidance](https://help.osf.io/article/330-welcome-to-registrations)). | That freezing a protocol makes the result correct. A frozen protocol prevents silent revision; it does not validate. | +| "Redirect effort away from things that haven't worked and toward the area that's demonstrating momentum." | `ACTION HEURISTIC` | Predeclare the traction indicators and the stop/scale thresholds, then reallocate when a threshold is crossed ([Prompt 5](prompt-library.md#5-signal-versus-traction-designer)). | That momentum can be judged after the fact from whatever moved. Post-hoc threshold selection converts noise into a mandate. | +| "More inquiries, more questions, more emails coming in, better numbers, repeat customers, and increasing demand." | `FORECAST` naming real indicators | This is the one place the source names measurable quantities. Adopt the indicator *names*, discard the prediction that they will rise. | That attention equals inquiries, inquiries equal conversion, conversion equals revenue, or revenue equals retention. These are separate measures and are never substituted for one another. | +| "Claim authority over it, structure it, give it order." | `ACTION HEURISTIC` | Optional and low priority: does writing the protocol down reduce run-to-run variance? | That structure substitutes for evidence. Formalizing an unvalidated result makes it durable, not true. | +| "Don't let your emotions dictate what you do." | `ACTION HEURISTIC` | Consistent with predeclaring thresholds before results are visible. | That discipline about feelings implies discipline about inference. | +| "Follow the signs... something you've been experimenting with is starting to answer you back." | `ACTION HEURISTIC` + `FORECAST` | Only in the predeclared-indicator form above. | That ambiguous events count as replies. | +| "Hit the like button... subscribe... tell me in the comments what you're experimenting with... click the link for the extended reading." | `MARKETING FUNNEL` | Not a claim about the subject. | That felt resonance is independent of the source's incentive to produce resonance. The comment prompt also collects the subject's own specifics, which raises the apparent hit rate of future readings. | +| "The most important part of any tarot reading is you." | `REFLECTION` | Not a claim. | Nothing follows from it either way - though it is, structurally, an accurate description of where the content comes from. | + +## What survives + +Three method heuristics survive translation: + +1. **Experiment, then replicate, then formalize.** Try things; when something + works, find out why and whether it recurs before building on it. +2. **Allocate on predeclared indicators.** Move resources toward what moves the + indicator you named in advance, not toward what looks like momentum in + hindsight. +3. **Freeze the protocol once it works.** Stop tinkering during measurement; + record deviations instead of absorbing them. + +Each is independently supported by ordinary research practice. None of them is +supported *by the reading*. The reading is a prompt that surfaced them, which is +exactly the permitted use: a story generated a question. + +Everything else in the ledger - the turn, the timing, the ceiling, the revenue - +carries no weight and enters no decision. + +## The falsification exercise + +The five concrete specifics (blue notebook, laptop in a car, cracked phone +screen, curly-haired person, vending-machine conversation) are the only part of +the source that could be tested at all. Testing them is optional and cheap. If +it is done, it is done properly or not at all. + +**Decision it can change.** Not a business decision. It calibrates the observer: +if retrospective matching produces a high hit rate on control material, the rule +that indicators must be predeclared gets tighter, and narrative-derived signals +stay out of indicator selection entirely. + +**Protocol requirements.** + +- Predeclare, in writing and before any observation, what counts as a match for + each of the five items - including the boundary of "connected to the + innovation," which is otherwise elastic enough to absorb any encounter. +- Fix the observation window and the scorer before starting. +- Score against base rates, not against zero. Cracked screens, blue notebooks, + curly hair, and vending machines are common; the comparison quantity is how + often these appear in an arbitrary equivalent window, not whether they appear. +- Include a control: score the same five items against a reading drawn for a + different sign, or a shuffled transcript, with the scorer blind to which + transcript is which ([Mason & Budge, 2011](https://pubmed.ncbi.nlm.nih.gov/21315874/)). +- Record the result, including a null result, before interpreting it. + +**Expected outcome.** Matches on several items, at a rate indistinguishable from +the control. That is the informative outcome, and it is worth having in writing +the next time a specific detail feels uncanny. + +## Non-claims + +- This ledger does not establish that the source is accurate, inaccurate, + predictive, or causal. It establishes only what may and may not be inferred + from it. +- Classifying a statement as testable does not mean it has been tested. No + observation in this document has been made. +- The three surviving heuristics are not endorsed *because* the source stated + them. They are endorsed because they are separately defensible, and they would + survive the source being discarded entirely. +- No statement in the ledger's "testable translation" column authorizes human + research, customer outreach, or external data collection. Those require + separate explicit authorization and appropriate consent. diff --git a/docs/upgrade/BASELINE_MEASUREMENTS.md b/docs/upgrade/BASELINE_MEASUREMENTS.md new file mode 100644 index 00000000..6e8e4541 --- /dev/null +++ b/docs/upgrade/BASELINE_MEASUREMENTS.md @@ -0,0 +1,441 @@ +# Baseline Measurements — cc-framework + +> **Status: measured.** Every number in this document was produced by a command +> run against commit `3e22c39` on 2026-08-19, on the host described in +> [Host](#host). Numbers that are unflattering are reported anyway. Where a +> number was *not* measured, the row says so rather than estimating. +> +> **Non-claim.** These measurements describe this repository on one host at one +> commit. They are not a statement about correctness, safety, or the behaviour +> of any deployed system. A passing test suite is evidence of regression +> resistance, not of correctness. + +Regenerate with `scripts/upgrade_baseline.py` (see [Reproduction](#reproduction)). + +--- + +## Host + +| Field | Value | +|---|---| +| Platform | Linux 6.18.5, x86-64 | +| Python | 3.11.15 (CPython) | +| NumPy / SciPy | 2.x / 1.17.1 | +| Environment | ephemeral container, `uv venv`, `uv pip install -e '.[test]'` | +| Commit | `3e22c39` | +| Date | 2026-08-19 | + +A latency figure without a machine is not reproducible. Every timing below is +host-relative and must be re-measured before it is cited. + +--- + +## 1. Size + +| Metric | Value | Command | +|---|---:|---| +| Python files in `src/cc` | 90 | `find src -name '*.py' \| wc -l` | +| Lines in `src/cc` | 41,645 | `find src -name '*.py' -exec cat {} + \| wc -l` | +| Test files | 108 | `find tests -name '*.py' \| wc -l` | +| Markdown files (repo) | 136 | `find . -name '*.md' -not -path './.git/*' \| wc -l` | +| Markdown files (`docs/`) | 90 | `find docs -name '*.md' \| wc -l` | +| JSON schemas | 2 | `ls schemas/*.json schemas/evidence/*.json` | +| Working tree (excl. `.git`) | 54 MB | `du -sh --exclude=.git .` | +| `.git` | 43 MB | `du -sh .git` | + +### Per-package line counts + +| Package | Files | Lines | +|---|---:|---:| +| `src/cc/core` | 13 | 10,462 | +| `src/cc/evidence` | 11 | 9,946 | +| `src/cc/kernel` | 12 | 6,389 | +| `src/cc/analysis` | 9 | 4,384 | +| `src/cc/cartographer` | 9 | 2,896 | +| `src/cc/redteam` | 2 | 1,413 | +| `src/cc/reporting` | 4 | 1,343 | +| `src/cc/adapters` | 6 | 1,191 | +| `src/cc/evals` | 3 | 993 | +| `src/cc/guardrails` | 6 | 726 | +| `src/cc/exp` | 1 | 710 | +| `src/cc/enterprise` | 2 | 524 | +| `src/cc/io` | 2 | 204 | +| `src/cc/utils` | 6 | 236 | +| `src/cc/cli` | 2 | 117 | +| `src/cc/_legacy` | 1 | 89 | + +`cc.core` and `cc.evidence` together are 20,408 lines — 49% of the source tree — +against a `cc.kernel` of 6,389 lines. The mathematical core the project exists +to provide is 15% of its own source. + +--- + +## 2. Test suite + +### Out of the box, from the declared test extra + +```bash +uv venv .venv && uv pip install -e '.[test]' && pytest +``` + +| Result | Count | +|---|---:| +| Passed | 685 | +| **Failed** | **1** | +| **Errors** | **3** | +| Skipped | 9 | +| Wall time | 87 s | + +Failures: + +- `tests/unit/packaging/test_wheel_boundary.py` — 3 errors, `No module named build` +- `tests/regression/week2/test_week2_deliverables.py::test_unit_tests_pass` — cascaded + +After installing `build`, a second, distinct failure appears: +`BackendUnavailable: Cannot import 'setuptools.build_meta'`. + +**The `[test]` extra cannot run the test suite.** It omits `build`, `setuptools`, +and `wheel`, all of which `tests/unit/packaging/` requires. See finding +[F-01](FINDINGS_REGISTER.md#f-01). + +### After adding the three undeclared dependencies + +| Result | Count | +|---|---:| +| Passed | 689 | +| Failed | 0 | +| Skipped | 9 | +| Wall time | 90 s | + +This is the green baseline the rest of the plan builds on. + +### Skipped tests (9) + +Skips are gated by environment variables or optional dependencies: +`CC_RUN_EXPERIMENTS`, `CC_RUN_PERF`, `guardrails`, `fastavro`, `protobuf`, +`SQLAlchemy`. No CI job sets any of them, so **no CI run has ever executed the +experiment or performance lanes.** + +--- + +## 3. Coverage + +`pytest --cov=src/cc --cov-report=term-missing` + +| Metric | Value | +|---|---:| +| Statements | 15,149 | +| Missed | 3,904 | +| Branches | 4,762 | +| Partial branches | 1,070 | +| **Total coverage** | **69.91%** | + +There is **no coverage gate in CI**. This number has never been enforced. + +### Modules below 60% + +| Module | Stmts | Cover | Note | +|---|---:|---:|---| +| `reporting/cli.py` | 204 | **0.00%** | the `cc-report` entry point — the product surface | +| `cli/manifest.py` | 68 | **0.00%** | | +| `core/audit_runner.py` | 199 | 17.45% | | +| `guardrails/semantic_filter.py` | 38 | 18.75% | | +| `guardrails/toy_threshold.py` | 42 | 21.43% | | +| `enterprise/aws_reference.py` | 143 | 23.87% | | +| `guardrails/regex_filters.py` | 138 | 24.18% | | +| `analysis/reporting.py` | 153 | 26.42% | | +| **`core/stats.py`** | **756** | **38.16%** | **the statistics engine; 434 statements unexercised** | +| `evals/run_bench.py` | 189 | 49.39% | | +| `cartographer/bounds.py` | 272 | 52.99% | | +| `utils/artifacts.py` | 50 | 53.70% | | +| `analysis/cc_estimation.py` | 118 | 56.08% | | +| `core/guardrail_api.py` | 95 | 57.48% | | +| `core/registry.py` | 79 | 57.89% | | +| `core/metrics.py` | 244 | 59.15% | | + +The two largest uncovered surfaces are the **CLI that emits reports** (0%) and +the **statistics module** (38% of 756 statements). Both are on the path from +evidence to a published number. + +### Modules above 90% + +`kernel/frechet_classes.py` 98.87%, `evidence/permission_compiler.py` 95.28%, +`reporting/canonical.py` 91.89%, `core/manifest.py` 90.74%, +`evidence/role_ontology.py` 90.53%. + +The kernel is well covered. The plumbing around it is not. + +--- + +## 4. Type checking + +`pyproject.toml` declares: + +```toml +[tool.mypy] +packages = ["cc"] +strict = true +``` + +Running mypy **at its own declared scope**: + +``` +Found 279 errors in 47 files (checked 101 source files) +``` + +CI runs mypy on an explicit list of **7 files**: + +``` +src/cc/adapters/base.py src/cc/cartographer/audit.py +src/cc/cartographer/bounds.py src/cc/cartographer/intervals.py +src/cc/io/storage.py src/cc/utils/artifacts.py src/cc/utils/timing.py +``` + +`.pre-commit-config.yaml` uses the same 7-file list. + +The repository declares strict typing across the `cc` package and enforces it on +7.8% of the files. See finding [F-02](FINDINGS_REGISTER.md#f-02). + +--- + +## 5. Kernel numerics + +### Fréchet–Hoeffding recovery + +`identified_region` versus the closed form `[max(0, pA+pB-1), min(pA, pB)]`: + +| `(pA, pB)` | LP interval | Closed form | Abs. error | +|---|---|---|---:| +| (0.1, 0.2) | [0, 0.1] | [0, 0.1] | 0 | +| (0.5, 0.5) | [0, 0.5] | [0, 0.5] | 0 | +| (0.9, 0.8) | [0.7, 0.8] | [0.7, 0.8] | 1.11e-16 | +| (0.01, 0.99) | [0, 0.01] | [0, 0.01] | 0 | +| (1e-6, 1e-6) | [0, 1e-6] | [0, 1e-6] | 0 | +| (0.999999, 0.999999) | [0.999998, 0.999999] | same | 0 | + +Worst absolute error across the six configurations: **1.11e-16** — one unit in +the last place of a float64. **The LP kernel is numerically exact on this +family.** This is the strongest single result in the repository and it is +currently unadvertised. + +Coverage caveat: six marginal configurations is a hand-chosen census, not a +sample. It carries no confidence interval and does not establish exactness +outside these points. Property-based testing over the marginal simplex is +workstream [W4](EPISTEMIC_UPGRADE_PLAN.md#w4). + +### Scaling: atoms = 2^m + +`m` guardrails, all marginals fixed at 0.1, query = intersection of all: + +| m | atoms | wall time | interval | +|---:|---:|---:|---| +| 2 | 4 | 0.003 s | [0, 0.1] | +| 6 | 64 | 0.005 s | [0, 0.1] | +| 10 | 1,024 | 0.039 s | [0, 0.1] | +| 12 | 4,096 | 0.146 s | [0, 0.1] | +| 14 | 16,384 | 0.766 s | [0, 0.1] | +| 15 | 32,768 | 1.738 s | [0, 0.1] | +| 16 | 65,536 | 3.642 s | [0, 0.1] | +| 17 | 131,072 | 8.962 s | [0, 0.1] | +| 18 | 262,144 | 19.978 s | [0, 0.1] | + +Single measurement per point, no repetition, so **no dispersion is reported and +none should be inferred**. Repeated-measurement timing with p50 and IQR is +workstream [W7](EPISTEMIC_UPGRADE_PLAN.md#w7). + +Practical ceiling for the exact LP on this host: **m ≈ 18**, roughly doubling +per added guardrail. This is the measured basis for the relaxation hierarchy in +VISION Pillar VI — not an assumption. + +### The correlation cliff, reproduced + +Eighteen guardrails, each failing at p = 0.1, composed as a conjunction: + +- independence would predict `0.1^18 = 1e-18` +- the sharp identified interval is **[0, 0.1]** + +Under adversarial dependence, **eighteen stacked filters bound no better than +one**. The upper bound is `min_i p_i`, independent of `m`. This is the +project's thesis and it reproduces in 20 seconds on a laptop-class host. + +--- + +## 6. Canonicalization kernel + +Probing `cc.reporting.canonical.canonical_json_bytes` in the style of +Ghost-Ark's E1 provenance-kernel census. Intent is declared per class; the +verdict compares observation to intent. + +```bash +PYTHONPATH=src python scripts/canonicalization_probe.py +``` + +| Class | Intent | Observed | Verdict | +|---|---|---|---| +| unicode-key-collision (NFC vs NFD) | distinct | collapsed | **unintended-kernel** | +| nested-unicode-key-collision | distinct | collapsed | **unintended-kernel** | +| negative-zero | equivalent | distinct | over-discrimination | +| integer-above-2^53 | distinct | distinct | sound (CPython only — see below) | +| int-vs-float-same-value | distinct | distinct | sound | +| bool-vs-int | distinct | distinct | sound | +| float-exponent-form | distinct | distinct | sound | +| safe-integer-neighbours | distinct | distinct | sound (positive control) | +| object-key-order | equivalent | collapsed | sound (positive control) | +| array-element-order | distinct | distinct | sound (positive control) | +| large-document-single-byte | distinct | distinct | sound (positive control) | + +**Verdict counts: 8 sound, 2 unintended-kernel, 1 over-discrimination.** +Provenance is `census` — exact counts, no confidence intervals. + +The four positive controls pass, which is what makes the two failures credible +rather than an artifact of an over-strict probe. + +Findings [F-03](FINDINGS_REGISTER.md#f-03) through +[F-07](FINDINGS_REGISTER.md#f-07) record these in full. The headline: + +```python +>>> canonical_json_bytes({"é": 1, "é": 2}) # NFC key, NFD key +b'{"\xc3\xa9":2}' +``` + +Two distinct input keys, one output key, **no error raised**. The receipt hash +covers a document that is not the document supplied. + +The probe exits non-zero while any class carries `unintended-kernel` or +`rejection-asymmetry`, so this measurement is falsifiable rather than asserted: +fix the canonicalizer and the probe goes green. + +### RFC 8785 (JCS) conformance + +| Value | cc-framework emits | JCS emits | | +|---|---|---|---| +| `1e30` | `1e+30` | `1e+30` | ok | +| `10**30` | `1000000000000000000000000000000` | `1e+30` | **diverges** | +| `1.0` | `1.0` | `1` | **diverges** | +| `-0.0` | `-0.0` | `0` | **diverges** | +| `1e-7` | `1e-07` | `1e-7` | **diverges** | +| `100.0` | `100.0` | `100` | **diverges** | + +Five of six number forms diverge from RFC 8785. cc-framework's canonical form is +`json.dumps(sort_keys=True)`, which is a *convention*, not a *standard*, and it +is not the convention its sibling repositories use. + +--- + +## 7. Claim-discipline surface + +### What exists + +| Artifact | Lines | Enforced by | +|---|---:|---| +| `docs/research/NON_CLAIMS.md` | — | prose + verifier substring engine | +| `docs/claims/CLAIM_BOUNDARY_MANIFEST.md` (C0–C5 levels) | — | not cross-checked against the JSON | +| `docs/claims/claim_boundary_manifest.v0.1.json` | — | `tests/unit/docs/test_claim_boundary_manifest.py` | +| `scripts/validate_claim_boundary_manifest.py` | — | called by that test, which CI runs | +| `src/cc/evidence/permission_compiler.py` forbidden-phrase list | 202 stmts | unit tests | +| `docs/theory/theorem_ledger.md` | 10 KB | test witnesses named per theorem | +| `docs/research/THEOREM_LEDGER.md` | 3 KB | — (second, divergent ledger) | + +`grep` across `.github/workflows/` and `.pre-commit-config.yaml` for +`claim`/`scan` returns only Bandit and detect-secrets, so **no forbidden-phrase +scanner runs in CI**. + +The claim-boundary manifest, however, *is* enforced — via pytest rather than a +named workflow step. `tests/unit/docs/test_claim_boundary_manifest.py` calls +`validate_claim_boundary_manifest.validate_manifest`, which checks required +keys, unique ids, level resolution, non-empty non-claims, and **the existence of +every `supporting_files` path on disk**. It passes with zero errors over all +eight declared claims. See the correction note in +[F-12](FINDINGS_REGISTER.md#f-12); an earlier draft of this document said +otherwise and was wrong. + +### Proto-scan: Ghost-Ark's forbidden patterns applied here + +| Pattern | Hits | +|---|---:| +| `guarantee` | 70 | +| `production[- ]ready` | 7 | +| `enterprise[- ]ready` | 2 | +| `tamper[- ]proof` | 1 | +| `enterprise[- ]grade`, `formally verified`, `zero risk`, `secure by default`, `unbreakable`, `audit complete`, `one-click compliance` | 0 | + +Inspection of the hits matters more than the count: + +- The `guarantee` hits are **overwhelmingly negated** — `does not guarantee`, + `no guarantee of`. A naive scanner ported from Ghost-Ark would produce 70 + false positives on this repository and be switched off within a week. +- The `production[- ]ready` and `tamper[- ]proof` hits are mostly **inside the + claim-discipline machinery itself** (`permission_compiler.py`'s forbidden list, + `NON_CLAIMS.md`, the release checklist saying the project is *not* + production-ready). + +So the honest reading is: **cc-framework's prose discipline is already good; its +enforcement is absent.** The scanner this repository needs is not Ghost-Ark's — +it needs negation and allowlist handling from day one. See +[W2](EPISTEMIC_UPGRADE_PLAN.md#w2). + +--- + +## 8. CI gates + +| Gate | Present | Scope | +|---|---|---| +| ruff lint | yes | whole repo | +| ruff format | yes (pre-commit) | whole repo | +| mypy | yes | **7 files of 90** | +| pytest | yes | full suite, Python 3.10–3.13 | +| coverage threshold | **no** | — | +| claim scanner (forbidden phrases) | **no** | — | +| claim-boundary manifest validation | yes | via pytest, not a named CI step | +| artifact boundary | yes | `scripts/check_artifact_boundary.py --static` | +| package build + twine check | yes | — | +| enterprise smoke (moto) | yes | — | +| minimal example | yes | `examples/minimal/run_bounds.py` | +| Bandit / detect-secrets / pip-audit | yes | `security.yml` | +| mutation testing | **no** | — | +| property-based test gate | **no** | hypothesis is a declared dep; no gate | +| cross-language verifier agreement | **no** | no second implementation exists | +| adversarial receipt corpus | **no** | — | +| reproducibility replay manifest | partial | `scripts/reproduce_paper.py`, not a manifest | + +--- + +## 9. Repository hygiene + +| Item | Measurement | +|---|---| +| Largest tracked files | 7 Blender renders, 2.2–2.8 MB each, **17.8 MB total** | +| Archived JSONL checkpoints | 4 × 636 KB in `docs/archive/generated-checkpoints/` | +| `.git` size | 43 MB against a 54 MB working tree | +| Duplicate theorem ledgers | `docs/theory/theorem_ledger.md` (10 KB) and `docs/research/THEOREM_LEDGER.md` (3 KB) | +| Legacy shim | `src/cc/_legacy/` — 89 lines, one file | +| `build/` directory on disk | present, untracked (0 files in git) | +| Schema `$id` host | `cc-framework.local` — not resolvable | + +--- + +## Reproduction + +Every table above is regenerated by: + +```bash +python scripts/upgrade_baseline.py --out docs/upgrade/baseline.json +``` + +The script is a deliverable of workstream [W0](EPISTEMIC_UPGRADE_PLAN.md#w0). Until +it exists, this document is a **hand-recorded measurement**, and it says so +here rather than implying automation that is not present. + +## What was not measured + +Reported so that absence is not mistaken for a null result: + +- **Mutation score.** No mutation testing has ever been run on this repository. +- **Cross-language agreement.** No second implementation of the kernel exists + inside this repository to disagree with. +- **External reviewer reproduction.** No stranger has ever reproduced these + artifacts. Every adversary in this tree was written by the author of the code + it attacks. +- **Timing dispersion.** Every timing above is a single measurement. +- **Real guardrail marginals.** No `p_i` in this repository was measured against + a production guardrail. All are supplied, synthetic, or assumed. diff --git a/docs/upgrade/COMMITTEE_SCORECARD.md b/docs/upgrade/COMMITTEE_SCORECARD.md new file mode 100644 index 00000000..07dd6aa0 --- /dev/null +++ b/docs/upgrade/COMMITTEE_SCORECARD.md @@ -0,0 +1,438 @@ +# Committee Scorecard — cc-framework, August 2026 + +> Modelled on Ghost-Ark's committee scorecard. Scores are **judgements, not +> measurements** — they are defensible only because each one names the evidence +> that would raise it and the evidence that would lower it. A score whose +> movement conditions are unstated is meaningless. +> +> Passing tests are evidence. Passing tests are not correctness. Passing tests +> are not safety. Passing tests are not production readiness. + +Baseline facts: [BASELINE_MEASUREMENTS.md](BASELINE_MEASUREMENTS.md). +Defects: [FINDINGS_REGISTER.md](FINDINGS_REGISTER.md). + +--- + +## Summary + +| # | Dimension | Now | Target (v0.4) | Workstream | +|---|---|---:|---:|---| +| 1 | Mathematical correctness | **8.9** | 9.4 | W4 | +| 2 | Canonicalization & provenance integrity | **8.2** | 8.5 | W3 | +| 3 | Claim discipline — prose | **8.8** | 9.2 | W2 | +| 4 | Claim discipline — enforcement | **4.0** | 8.5 | W2 | +| 5 | Test depth | **6.4** | 8.5 | W4 | +| 6 | External reproducibility | **4.2** | 8.0 | W0, W7 | +| 7 | Consumability as a library | **6.8** | 8.5 | W5 | +| 8 | Cross-implementation agreement | **6.5** | 8.0 | W5 | +| 9 | Empirical grounding | **2.4** | 6.5 | W7 | +| 10 | Statistical honesty machinery | **7.9** | 9.0 | W4, W7 | +| 11 | Evidence-governance architecture | **7.6** | 8.5 | W6 | +| 12 | Formal readiness | **3.1** | 4.5 | W4 | + +**Committee verdict.** cc-framework is approximately: + +- **8.4 / 10** as a *mathematical* research artifact — the kernel is exact, the + theorem ledger names witnesses, the non-claims are unusually disciplined. +- **4.1 / 10** as a *verifiable evidence* artifact — the canonicalization kernel + has an unexamined collapse, nothing enforces the claim manifest, and no + independent implementation exists to disagree with it. +- **6.8 / 10** as a *consumable library* — was 2.6. W5 shipped an ROC-free + composition surface, a published conformance corpus, an independent Node + implementation, and a cross-language guard. A real consumer's published + numbers now reproduce from the library. + +The gap between the first and third numbers is the whole plan. W5 has closed +most of it; W6 and the empirical lanes remain. + +--- + +## 1. Mathematical correctness — 8.9 {#mathematical-correctness} + +*Measures: whether the computed bounds are the bounds claimed.* + +**Evidence supporting.** FH recovery exact to 1.11e-16 worst case across six +marginal configurations. `kernel/frechet_classes.py` 98.87% covered. Theorem +ledger T1–T6 with proof status, implementation witness, test witness, and a +per-theorem non-claim. Infeasibility raises rather than returning a plausible +number. Monotonic tightening tested. LP is sharp by finite-dimensional convexity, +computationally witnessed. + +**What would raise it.** Property-based tests over the marginal simplex rather +than six chosen points (the current evidence is a census, not a sample, and +carries no interval). Dual certificates emitted alongside primal witnesses so a +skeptic can verify optimality without re-solving. Exact rational arithmetic for +small instances as a cross-check on the float path. + +**What would lower it.** Any marginal configuration where the LP and the closed +form disagree beyond solver tolerance. A degenerate constraint set that returns +`optimal` on an empty feasible region. + +--- + +## 2. Canonicalization & provenance integrity — 8.2 {#canonicalization-provenance-integrity} + +*Measures: whether a receipt identifies the document it claims to.* + +> **Revised upward from 3.8 on 2026-08-19**, on delivered evidence. Every S1 +> finding in this dimension is fixed. See +> [CANONICAL_PROFILE.md](../architecture/CANONICAL_PROFILE.md). + +**Evidence supporting.** `cc.canonical.v2` implements RFC 8785: ECMAScript +`Number::toString` number forms, UTF-16 code-unit key ordering, no +normalization, and one declared narrowing (the IEEE-754 safe integer range) +documented rather than left implicit. 15 of 15 probed number forms conform. + +The census at `scripts/canonicalization_probe.py` runs 14 declared-intent +classes across both profiles and reports, under v2: **12 sound, 1 fail-closed, +1 sound-by-rejection** — zero `unintended-kernel`, zero `rejection-asymmetry`, +zero `over-discrimination`. Four positive controls pass, which is what makes the +result a fix rather than a trade. `tests/unit/canonical/` holds all of it with 57 +tests, and a further test asserts v1 *still* carries its defects, so the legacy +profile cannot be silently "fixed" out from under historical receipts. + +Duplicate keys are refused on read at any nesting depth, wired into the report +CLI, the claim-governance readers, and the Merkle log. + +Migration was done without breaking history: verification dispatches on the +profile each receipt declares, and every regenerated capsule artifact was diffed +with hashes, hash-derived ids, and the profile identifier scrubbed. All eleven +were byte-identical under that scrub — only hashes moved. + +**What still lowers it.** No fuzzing of the canonicalizer: every class was +authored by hand, so the census establishes that the declared classes behave as +declared and nothing about the rest of the input space. **No cross-language +differential test on receipts** — the corpus establishes agreement on the +composition kernel, but nothing yet re-canonicalizes a CC report in another +language and compares digests, so "cross-language verifiable" currently +describes v2's design rather than a demonstrated result. No external reviewer +has attacked either profile. + +**What would raise it further.** A receipt-level differential against a +non-Python JCS implementation. A fuzzer over the canonicalizer. + +**What would lower it.** A collision found by anyone outside the project. + +--- + +## 3. Claim discipline — prose — 8.8 {#claim-discipline-prose} + +*Measures: whether the words stay inside the evidence.* + +**Evidence supporting.** `NON_CLAIMS.md` is a genuine artifact — ten non-claims, +each with why-not and what-instead. The C0–C5 claim-boundary manifest maps every +public claim to level, lane, support, tests, files, and non-claim. `VISION.md` +labels itself aspirational in its first line and refuses to relax a single +non-claim. `cliff.py` refuses confidence claims on post-selection intervals *in +code*, with a comment explaining the statistics. Legacy metrics emit +`FutureWarning` naming what they are not. + +This is, in places, stronger than the flagship's. The C0–C5 lattice has no +Ghost-Ark equivalent. + +**What would raise it.** Adopting Vinctura's stricter rule — *no orphan claims: +every public sentence carries the command that falsifies it, or an explicit +limitation* — and applying it to the README. Conceding the 1935 point +([F-18](FINDINGS_REGISTER.md#f-18)) above the fold instead of leaving it for a +critic to make. + +**What would lower it.** Any README sentence a reader cannot map to a command. + +--- + +## 4. Claim discipline — enforcement — 4.0 {#claim-discipline-enforcement} + +*Measures: whether the discipline survives an author in a hurry.* + +> **Revised upward from 2.1 on 2026-08-19.** The original score rested partly on +> the claim that `validate_claim_boundary_manifest.py` was "wired into nothing — +> not CI, not the Makefile, not a test." The first two were right and the third +> was wrong: it is called by +> `tests/unit/docs/test_claim_boundary_manifest.py`, which runs in the pytest +> suite CI executes on four Python versions. See the correction note in +> [F-12](FINDINGS_REGISTER.md#f-12). + +**Evidence supporting.** The claim-boundary manifest **is** enforced, and more +thoroughly than most such artifacts: the validator checks required keys, unique +ids, claim-level resolution, non-empty non-claims per claim, and the existence +on disk of every `supporting_files` path. It passes with zero errors across all +eight declared claims. `permission_compiler.py` carries a forbidden-phrase list +and is 95.28% covered. The claim-governance verifier checks non-claim substance +rather than exact strings. `check_artifact_boundary.py` runs in CI. + +**What lowers it.** No forbidden-phrase scanner in CI at all — the gate +Ghost-Ark runs on every file has no counterpart here. Two divergent theorem +ledgers with no governing rule ([F-15](FINDINGS_REGISTER.md#f-15)). Strict typing +declared for the `cc` package and enforced on 7 of 90 files +([F-02](FINDINGS_REGISTER.md#f-02)) — claim inflation expressed in build +configuration. Two narrow manifest gaps remain unenforced +([F-12](FINDINGS_REGISTER.md#f-12)): test paths are not checked for existence, +and the Markdown and JSON manifests are not cross-checked. + +The prose is 8.8 and the enforcement is 4.0. The gap is real but narrower than +first reported: what is missing is a phrase-level scanner and honest typing, +not the claim ledger itself. + +**What would raise it.** A claim scanner with negation and allowlist handling +from day one — a naive port of Ghost-Ark's would fire on 70 negated uses of +"guarantee" and be disabled within a week. The typing ratchet. Closing the two +manifest gaps. + +**What would lower it.** A published claim that no artifact supports, merged +green. + +--- + +## 5. Test depth — 6.4 {#test-depth} + +*Measures: whether the suite would notice a defect.* + +**Evidence supporting.** 689 tests, 108 files, unit/integration/e2e/regression/ +performance/experiment separation, adversarial tests for the transparency log, +fail-closed semantics tested, selective-inference guard tested, 69.91% coverage +with branch coverage on. + +**What lowers it.** The report CLI at 0% ([F-13](FINDINGS_REGISTER.md#f-13)). +`core/stats.py` at 38% over 756 statements +([F-14](FINDINGS_REGISTER.md#f-14)) — the statistics engine, in a repository +about statistical honesty. No coverage gate. No mutation testing, so the score is +*unknown*, not low. Hypothesis is a declared dependency with no gate. A +regression test that shells out to the whole unit suite and masks real failures +([F-20](FINDINGS_REGISTER.md#f-20)). Nine tests that have never run in CI +([F-17](FINDINGS_REGISTER.md#f-17)). + +**What would raise it.** Coverage floor at the measured 69.91%, ratcheting. +Mutation testing on `kernel/` and `reporting/` with a published score. Property- +based tests on the statistics module. CLI golden tests. + +**What would lower it.** A defect found downstream that the suite could have +caught. + +--- + +## 6. External reproducibility — 4.2 {#external-reproducibility} + +*Measures: whether a stranger can reproduce the artifacts.* + +**Evidence supporting.** `scripts/reproduce_paper.py`, +`scripts/verify_paper_artifacts.py`, a claim-governance capsule with expected +outputs and a manifest, `examples/minimal/`, a devcontainer, a Dockerfile, +`make reproduce-*` targets, pinned artifact manifests. + +**What lowers it.** The documented install path produces a red suite +([F-01](FINDINGS_REGISTER.md#f-01)) — the first thing a stranger encounters is a +failure that is not a real failure. No single-command reproduction. No replay +manifest in Ghost-Ark's sense. **No stranger has ever done it**; every adversary +in this tree was written by the author of the code it attacks. + +**What would raise it.** `make verify` that goes green from a clean clone on a +clean machine, tested in CI from `.[test]` specifically. A reviewer Dockerfile. +One external person reproducing the paper artifacts and saying so in writing. + +**What would lower it.** An external reviewer failing to reproduce. + +--- + +## 7. Consumability as a library — 6.8 {#consumability-as-a-library} + +*Measures: whether a downstream project can depend on this instead of rewriting it.* + +> **Revised upward from 2.6 on 2026-08-19**, on delivered evidence rather than +> intent. What moved it is listed below; each item is a command. + +**Evidence supporting.** `cc.compose` takes named marginals and returns a sharp +interval, with no ROC, Youden, threshold, or operating-point concept anywhere in +its signature — pinned by a test that greps the signatures for that vocabulary, +so the surface cannot drift back to the shape a consumer already walked away +from. Deterministic predicates are first-class; detectors reach the same surface +through `marginal_from_operating_point`, which points *inward*. + +The result object carries what qualifies the number: the independence baseline, +the understatement factor, the binding event, the marginal provenance +(`measured` / `assumed` / `supplied`), and the non-claims. A bound over assumed +rates cannot be serialized without the word `assumed` attached. + +`cc-guard` exposes the inference guards as a stdin/stdout JSON subcommand *and* +as a pure-data decision table, so a non-Python caller needs no Python process. +A test asserts the table and the implementation agree rule by rule, and another +asserts `cc-guard` reaches the same verdict as `cc.kernel.cliff.cliff_certificate` +in both directions — the CLI can neither permit what the kernel refuses nor +refuse what it permits. + +**The acceptance gate passed.** `tests/acceptance/test_external_consumer_reproduction.py` +reproduces an external consumer's *published* four-control result from +`cc.compose`: interval `[0, 0.01]`, independence baseline `1.2e-5`, +understatement factor `833×`, all three published scenarios, and their +sensitivity finding that improving a weak control moves the upper bound by +`0.00pp`. Ten tests. Their 214 lines of JavaScript could be deleted. + +**What still lowers it.** No ingest path for the contract Ghost-Ark declares +binding ([F-11](FINDINGS_REGISTER.md#f-11)) — that is W6. The constrained LP path +is not exposed through `cc.compose`; only the closed form is. And no downstream +project has actually adopted any of this yet: the obstacle is removed, the +adoption is theirs to make. + +**What would raise it further.** W6's `cc.ingest.discretization`. Side +constraints on the `cc.compose` surface. A downstream repository importing it in +anger. + +**What would lower it.** A fourth reimplementation appearing anyway — which +would mean the surface is still the wrong shape. + +--- + +## 8. Cross-implementation agreement — 6.5 {#cross-implementation-agreement} + +*Measures: whether independent implementations produce the same answer.* + +> **Revised upward from 0.0 on 2026-08-19.** There was nothing to score; now +> there is. + +**Evidence supporting.** + +- `conformance/cc-kernel-v1/` is published: 24 accept cases with pinned exact + values, 8 reject cases with typed refusal reasons, a manifest with digests and + a declared 1e-12 tolerance, and a 256-line normative `SPEC.md`. Every accept + case is cross-checked against the finite-atom LP at generation time, and the + build **refuses to write a case** the closed form and the LP disagree on. +- `verifiers/node/cc_compose_verify.mjs` is a zero-dependency Node + implementation written from `SPEC.md` and the JSON, not from the Python. It + passes 24/24 accept and 8/8 reject. +- `scripts/differential_compose.py` fuzzes both sides on randomized inputs, + including malformed ones, requiring agreement on the answer *or* the refusal. + **23,000 cases across six seeds, zero disagreements** — and it earned its + keep immediately by finding a real bug (see below). +- Three of these run in CI as pytest cases, so agreement is enforced rather than + demonstrated once. + +**The fuzzer found a bug on its first run.** `dependence="countermonotone"` with +exactly *one* event: the Python raised `IndexError`, the Node silently returned +`NaN`. Both were wrong, differently, and the curated corpus had not thought to +ask. Both are fixed, and the case is now pinned as +`reject-countermonotone-one-event` with its provenance recorded in the corpus. +That is the argument for randomized differential testing over a corpus alone. + +**Why this is 6.5 and not 8.0.** The honest limit: **the Node implementation and +the Python reference were authored in the same project.** A specification that +is wrong yields two implementations that are wrong together. This is a +differential-testing instrument, not an independent replication, and both the +verifier's own output and the corpus manifest say so in their non-claims. + +The one check here that is *not* same-author is the external oracle: an +outside project's **published** numbers, produced independently for its own +purposes before this corpus existed, reproduced exactly. That is one oracle, on +one scenario family. + +**What would raise it.** Ghost-Ark's TypeScript `ccCorrelation.ts` running this +corpus. A third implementation by someone who has not read either of these. More +external oracles. Extending the corpus to the constrained LP path, which neither +the corpus nor the fuzzer currently covers. + +**What would lower it.** A disagreement found by anyone outside the project — or +a case quietly weakened to make an implementation pass. + +--- + +## 9. Empirical grounding — 2.4 {#empirical-grounding} + +*Measures: whether any number here came from a real system.* + +**Evidence supporting.** A dependence benchmark, adapter implementations for +Llama Guard / NeMo / Guardrails-AI, a rails demo, correlation-cliff copula +simulations, week-by-week experiment memos. + +**What lowers it.** **No `p_i` in this repository was measured against a +production guardrail.** Every marginal is supplied, synthetic, or assumed. The +adapters exist; no measurement campaign has run through them. The experiment lane +has never executed in CI ([F-17](FINDINGS_REGISTER.md#f-17)). VISION names the +Correlation Atlas as the empirical keystone and the most citable artifact +available; it does not exist. + +**What would raise it.** The Atlas: two or three real guardrails on one public +jailbreak corpus, reporting per-pair marginals, observed joint rates, phi, and +the independence-versus-worst-case gap, each row emitting a certificate. Even a +small one changes this score more than any amount of engineering. + +**What would lower it.** Publishing a measured-looking number that came from a +simulation. + +--- + +## 10. Statistical honesty machinery — 7.9 {#statistical-honesty-machinery} + +*Measures: whether the code refuses claims the data cannot support.* + +**Evidence supporting.** This is the repository's most distinctive asset. The +post-selection refusal at `cliff.py:321` returns `discovery-only` with no +confidence claim, and the comment explains why: *"A discovery is a hypothesis; +only held-out data can certify it."* Confirmatory/exploratory provenance is a +first-class type. Wilson intervals rather than normal approximations. Legacy +metrics warn about what they are not. Non-finite bootstrap samples are dropped +with a warning rather than silently. + +A downstream repository independently identified this guard as *"the single most +elegant thing available in this program."* External parties do not say that about +statistical software often. + +**What would raise it.** Adopting Ghost-Ark's reporting rules as executable +assertions: no proportion without a denominator, no interval over a curated +census, no interval below n = 30, no point estimate without dispersion. Ghost-Ark +enforces these in `reportProportion` / `assertCensusReporting`; cc-framework +enforces the post-selection rule and not the others. Confidence bands around +`[L, U]` under estimated marginals — VISION Pillar I — would raise it furthest. + +**What would lower it.** Any path that emits a confidence interval over a +hand-authored corpus. + +--- + +## 11. Evidence-governance architecture — 7.6 {#evidence-governance-architecture} + +*Measures: whether evidence carries its own boundaries.* + +**Evidence supporting.** Claim envelopes, role ontology, decay/hazard policy, +extremal scenarios, confirmatory protocol, Merkle log with adversarial tests, +permission compiler, a governance capsule with expected artifacts and a manifest. +9,946 lines, mostly 80–95% covered. Non-claims are carried *inside* signed +payloads, not merely alongside them. + +**What lowers it.** No ingest contract for the upstream that feeds it +([F-11](FINDINGS_REGISTER.md#f-11)). Two schemas with a non-resolvable `$id` and +version in the title ([F-19](FINDINGS_REGISTER.md#f-19)). The governance verdict +is only as good as the canonicalization beneath it — dimension 2 at 3.8 caps this +one. + +**What would raise it.** The discretization ingest module. Versioned resolvable +schema identifiers. An evidence-window contract in Ghost-Ark's sense. + +--- + +## 12. Formal readiness — 3.1 {#formal-readiness} + +*Measures: distance to machine-checked proof.* + +**Evidence supporting.** Theorem ledger with explicit proof status per theorem +and honest labelling of what is "standard theorem, computationally witnessed" +versus proved here. Explicit invariants in the kernel. LP sharpness follows from +finite-dimensional convexity, which is a proof, just not a mechanised one. + +**What lowers it.** No mechanised proof of any kind. No TLA+, Alloy, Lean, or +Coq. No exact-arithmetic cross-check of the float LP path. Dual certificates are +not emitted, so LP optimality is asserted by the solver rather than independently +checkable. + +**What would raise it.** Emitting dual certificates — the cheapest real move, +turning "the solver said optimal" into "here is the certificate, check it +yourself." An exact rational LP for small `m` as an oracle. A Lean statement of +the FH recovery theorem is a stretch goal, not a v0.4 target. + +**What would lower it.** Claiming formal verification. The word does not appear +in this repository today; keep it that way. + +--- + +## The committee's closing question + +For every score above: *what evidence would raise it, and what evidence would +lower it?* Both are stated for all twelve. Where a future revision cannot answer +both, that score should be deleted rather than defended. diff --git a/docs/upgrade/DOWNSTREAM_CONTRACTS.md b/docs/upgrade/DOWNSTREAM_CONTRACTS.md new file mode 100644 index 00000000..0afa6aa0 --- /dev/null +++ b/docs/upgrade/DOWNSTREAM_CONTRACTS.md @@ -0,0 +1,355 @@ +# Downstream Contracts + +> **Scope note.** Every change described here is made **in cc-framework**. This +> document describes what the other repositories need and what this repository +> must offer; it does not propose edits to them. Where a downstream change would +> be required to complete a loop, it is named as *their* decision, not scheduled +> here. +> +> **Non-claim.** Nothing in this document establishes that any downstream system +> is safe, correct, or compliant. A contract that carries lineage is a contract +> that can be checked — not a contract that is true. + +Three consumers, three different failures of the current library, three +different fixes. + +| Consumer | Relationship | Current state | What cc-framework owes it | +|---|---|---|---| +| [ghost-ark](#ghost-ark) | upstream producer *and* parallel implementer | reimplemented the calculus in TypeScript; declared a binding ingest rule this repo cannot honour | an ingest module and a conformance corpus | +| [vinctura](#vinctura) | rejected adopter | read the source, rejected the API, wrote 214 lines of JS; has a dated request this repo cannot serve | an ROC-free composition surface and a cross-language guard | +| [b2b-spatial](#b2b-spatial-intelligence-engine) | skeptic | demoted the core contribution to "engineering" | a defensible answer, and instrumentation it can actually use | + +--- + +## The shared finding + +All three relationships fail the same way. cc-framework's mathematics is sound +and its packaging makes the mathematics unreachable. Every consumer independently +concluded that reimplementing was cheaper than depending — and **each was right**, +given the API they were offered. + +The upgrade is therefore not "add features." It is: make the thing that already +works reachable from outside Python, in a shape that matches how the calculus is +actually used. + +--- + +## ghost-ark + +`PSUCyberSecurityLab/ghost-ark` — the AWS-native evidence and control plane. +Public. 501 TypeScript files, 169 test files, 156 markdown documents. + +### What Ghost-Ark already decided + +`docs/research/CLAIM_EVIDENCE_MATRIX.md` carries a rule stated as binding: + +> CC-Framework must not consume naked binary labels from Ghost-Ark. Binary +> variables must be tied to a discretization rule, threshold, comparator, +> calibration digest, scoring digest, validity window, and parent evidence +> lineage. + +and a layer table that assigns the claim boundary at each hop: + +| Layer | Boundary | +|---|---| +| Ghost discretization rule receipt | defines how a score becomes a binary variable; does not prove the score is valid | +| Ghost binary observation | records applying the rule; does not prove the threshold is optimal | +| CC evidence bundle | provides CC-compatible binaries and provenance; does not prove safety | +| **CC-Framework bounds report** | **computes what follows under stated assumptions; does not validate upstream data collection** | +| Ghost signed claim envelope | binds evidence, assumptions, result digest; does not widen the claim | + +The object is specified: `ghost.discretization_rule_receipt.v1`, with a monotonic +risk invariant (`higher_is_riskier` admits only `>=` and `>`), seventeen required +fields, and eleven verification preconditions. + +**cc-framework implements none of it** ([F-11](FINDINGS_REGISTER.md#f-11)). The +rule binds a repository that cannot honour it. + +### The parallel implementation + +`packages/research-frontier/src/ccCorrelation.ts` computes, in TypeScript: +per-variable failure counts and rates, 95% Wilson intervals, `n00/n01/n10/n11` +co-failure tables, observed joint rates with Wilson intervals, empirical phi, and +**pairwise Fréchet lower and upper bounds**. It requires a complete rectangular +grid and rejects mixed cohorts, non-binary values, and absent stationarity +declarations. + +It is a careful implementation of cc-framework's calculus by someone who could +not call cc-framework. It has never been compared against the Python. + +### What cc-framework ships + +**C1 — `cc.ingest.discretization`** (owner [W6](EPISTEMIC_UPGRADE_PLAN.md#w6)) + +A fail-closed reader for `ghost.discretization_rule_receipt.v1` implementing all +eleven preconditions: + +| Precondition | Refusal on violation | +|---|---| +| binary domain | value is exactly 0 or 1 | +| failure semantics | 1 means guardrail failure / unsafe pass | +| bounded score domain | finite lower and upper bounds declared | +| threshold legality | threshold inside the declared domain | +| signed comparator | comparator included in the rule digest | +| monotonic risk invariant | comparator direction matches score polarity | +| calibration digest | present | +| scoring digest | scoring function, model, or policy digest present | +| temporal validity | observation timestamp inside the rule validity window | +| parent lineage | observation references a parent receipt | +| stationarity declaration | cohort declares whether joint dependence is assumed stable | + +A marginal arriving without lineage is **refused**, not defaulted. The negative +corpus carries one fixture per precondition, and each must be rejected with a +distinguishable reason — a reader that rejects everything for the same reason is +not enforcing eleven rules, it is enforcing one. + +`declared_reference` means the observation carries receipt identifiers. It does +**not** mean cc-framework verified that receipt or its signature. That non-claim +travels with the ingested object. + +**C2 — the conformance corpus** (owner [W5](EPISTEMIC_UPGRADE_PLAN.md#w5)) + +`conformance/cc-kernel-v1/` — language-agnostic cases with pinned expected +values, covering pairwise FH bounds, Wilson intervals, phi, and the n-ary +conjunction and union bounds. Ghost-Ark's TypeScript either passes it or it is +not a CC kernel. Running it there is Ghost-Ark's decision; publishing it is this +repository's obligation. + +**C3 — the canonicalization treaty** (owner [W3](EPISTEMIC_UPGRADE_PLAN.md#w3)) + +[F-04](FINDINGS_REGISTER.md#f-04) and [F-05](FINDINGS_REGISTER.md#f-05) mean a CC +report canonicalized here and re-canonicalized in Ghost-Ark's TypeScript can +disagree — five of six number forms diverge from RFC 8785, and integers above +2^53 do not survive `JSON.parse`. The treaty is a written, versioned +`cc.canonical.v1` profile stating its relationship to RFC 8785, plus a schema +constraint bounding receipt-covered integers to the IEEE-754 safe range. + +This must be decided **before** an independent verifier is written. Ghost-Ark's +E1 census found that the kernel is set by the parser, not the canonicalizer; the +same corollary applies here and cannot be fixed downstream. + +### What this does not create + +No claim that Ghost-Ark's evidence is valid, that its scores are calibrated, or +that a CC report over Ghost-Ark observations says anything about deployed +safety. The ingest module checks that evidence carries its lineage. Whether the +lineage describes a well-designed measurement is outside every layer in the +table above. + +--- + +## vinctura + +`Cubits11/vinctura` — graduation photography, and underneath it a provenance +kernel and a verifiable-credential register. Private. + +Vinctura's `CLAIMS.md` opens with the strictest rule in the program: + +> Every public claim this repository makes, with the command that falsifies it or +> an explicit limitation. **No orphan claims.** A sentence that no command can +> demonstrate is deleted or given a command. +> +> Limitations come first, in this file as in every other. + +cc-framework should adopt this rule ([W2](EPISTEMIC_UPGRADE_PLAN.md#w2)). It is +stronger than anything currently enforced here. + +### Failure 1 — the API rejected + +`scripts/compose-bounds.js` records, in its header, why cc-framework was not used +([F-08](FINDINGS_REGISTER.md#f-08)): `cc.core.composition_theory` operates on ROC +point sets and bounds the Youden J statistic, which presumes each guardrail is a +classifier with a threshold and an operating curve. Vinctura's four controls are +deterministic refusal rules — `SELF_REPORTED` refuses iff `loggedBy === memberId`. +No threshold, no operating point, no false-positive rate. + +> Forcing a deterministic refusal rule into an ROC shape would produce numbers +> with the form of a measurement and none of the content. + +The consumer was right, and identified exactly what transfers: "the FH inequality +is applied directly to the events. **That is the part that transfers**; the ROC +machinery is not." + +**C4 — `cc.compose`** (owner [W5](EPISTEMIC_UPGRADE_PLAN.md#w5)) + +A surface whose signature contains no ROC concept at all: + +```python +compose_bounds( + marginals: Mapping[str, float], # P(E_i) — event probabilities, not TPR/FPR + event: Literal["all", "any"] | LinearQuery, + constraints: Sequence[Constraint] = (), +) -> Interval # sharp [L, U], plus witnesses +``` + +Deterministic predicates are first-class inputs. Detectors reach the same surface +by supplying their operating point as a marginal — the ROC path becomes an +optional adapter *on top of* the event calculus, never underneath it. + +**Acceptance gate.** Not "the API exists." The gate is: Vinctura's four-control +result — including the sensitivity analysis over assumed detection rates — is +reproduced from `cc.compose` in a cc-framework test, and their 214 lines could be +deleted. If it cannot be reproduced, the surface is still wrong. + +The test must also reproduce Vinctura's **correct refusal**: there is no +countermonotone regime for n > 2, because the FH lower bound is not a copula in +dimension ≥ 3 though it remains pointwise sharp. A composition API that silently +offers a "countermonotone" option for four events is wrong, and Vinctura noticed +before this repository did. + +### Failure 2 — the guard that cannot be reached + +`docs/research/program/ultracode/UC-10-KERNEL-BRIDGE.md` §4.3 +([F-10](FINDINGS_REGISTER.md#f-10)): + +> Optional stopping invalidates the density estimate entirely — and the sibling +> project `cc-framework` already refuses confidence claims on post-selection +> intervals at `src/cc/kernel/cliff.py:321`. **Use that.** [...] Making one +> repository's guardrail catch another repository's error is the strongest +> possible demonstration that the guardrail is real. +> +> That last point is the single most elegant thing available in this program. Do it. + +The guard exists and is correct. It is reachable only from Python, in-process. + +**C5 — `cc-guard`** (owner [W5](EPISTEMIC_UPGRADE_PLAN.md#w5)) + +Two deliverables, because one is not enough: + +1. A `cc-guard` subcommand reading JSON on stdin and writing a verdict on + stdout — `{"regime": "discovery-only", "confidence_claim": null, "reason": ...}` — + exposing the provenance-tagged guards without a Python API. +2. The same logic as a **pure-data decision table** in the conformance corpus, so + a JavaScript caller can enforce the rule with no Python process at all. A + subprocess dependency is a weaker bridge than a table. + +**Acceptance gate.** Vinctura's G3 gate — `npm test -- probe.post-selection-refused` — +passes against cc-framework's guard, by either route. Whether they wire it is +their call; that it *can* be wired is this repository's obligation. + +### Failure 3 — the kernel probe + +UC-10 specifies a `kernel-probe` measuring `|ker C|` for a canonicalization +function `C : Events → Records`, with `cheapestCollisionCostRatio` as the +security-relevant quantity — pre-registered sampling design, committed stopping +rule, and results reported whether or not they are interesting. + +cc-framework's canonicalizer is a **valid target**, and +[F-03](FINDINGS_REGISTER.md#f-03) is a member of its kernel found by hand in +under an hour. The adversarial corpus in [W3](EPISTEMIC_UPGRADE_PLAN.md#w3) +should be built in a shape a probe can consume: declared intent per class, +machine-readable verdicts, no confidence interval over a curated census. + +That last constraint is the pleasing part. Vinctura's probe wants to route its +density estimate through cc-framework's post-selection refusal; cc-framework's +own corpus is a census and must therefore carry no interval. **The guard applies +to the repository that wrote it.** + +--- + +## b2b-spatial-intelligence-engine + +`Cubits11/b2b-spatial-intelligence-engine` ("Assay") — hardware-attested capture +provenance via Nitro Enclaves, App Attest, StrongBox, NIST beacons, C2PA/JUMBF. +Private. + +### The skeptic's verdict + +`docs/latent-research-program.md` runs a kill-ledger over the program's research +concepts. The relevant row ([F-18](FINDINGS_REGISTER.md#f-18)): + +> **Fréchet ceiling + reachability sharpening** — "Adversarial robustness ≠ +> average-case robustness" — standard in ML security since 2014. **DEMOTED to +> engineering.** The mathematics is 1935; the sharpening is a known distinction. +> Survives as reporting practice, not as science. + +This is correct and cc-framework should say so first. See +[W9](EPISTEMIC_UPGRADE_PLAN.md#w9). + +What survives the demotion is worth more than what it removes: reporting practice +that a hostile reviewer trusts is exactly what an evidence platform needs, and +"survives as reporting practice" is a *use case*, not a dismissal. + +### Where the calculus actually applies + +Assay composes several independent checks before issuing a verdict: device +certificate chain validation, NIST beacon temporal freshness, offline Merkle CRL +revocation, and PCR0 enclave measurement. Its own README frames the value as +replacing probabilistic detection with deterministic attestation — and the +composed question is exactly CC's object: + +> Given per-check evasion marginals, what is the sharp bound on an artifact +> passing **all** checks while being fabricated, under unknown dependence between +> the checks? + +These checks are deterministic predicates, not classifiers — the same shape as +Vinctura's controls, and the same reason `cc.compose` (C4) is the surface that +fits. An attacker who compromises a signing key correlates several checks at +once; independence across them is precisely the assumption an adversary attacks. + +**C6 — the deterministic-predicate composition path** is C4. No separate +deliverable; the same surface serves both consumers, which is the argument for +building it once and building it right. + +### The three-valued verdict + +`schemas/vinctura_spatial_verdict.v1.schema.json` carries a lesson cc-framework +should learn rather than teach: + +``` +"class": ["WITHIN_DECLARED_BOUND", "EXCEEDS_DECLARED_BOUND", "INDETERMINATE"] +"INDETERMINATE is not a pass. Relying parties MUST NOT collapse it to either other value." +``` + +and, on the convenience boolean: + +> Deliberately NOT named 'compliant' [...] A boolean asserting legal compliance +> would be a claim Vinctura cannot support. Consumers MUST branch on 'class'; +> this field exists so that a null-unaware consumer fails loudly rather than +> silently reading INDETERMINATE as a pass. + +**C7 — an explicit indeterminate state in the CC report** +(owner [W6](EPISTEMIC_UPGRADE_PLAN.md#w6)) + +Today, a CC report either carries an interval or an error is raised. There is no +first-class "the evidence does not identify this" verdict that survives +serialization. Insufficient evidence, an infeasible constraint set, and a +refused post-selection claim are all *indeterminate*, and each should serialize +as such with a machine-readable reason — never as `[0, 1]`, which a careless +consumer reads as a computed bound. + +The design constraint is Assay's, and it is right: a null-unaware consumer must +fail loudly rather than silently read indeterminate as a pass. + +### The convergence worth naming + +Assay ships a research module titled `01-canonicalization-collapse.html`. +Ghost-Ark's E1 measured five unintended kernel members in its own pipeline. +cc-framework's canonicalizer silently merges Unicode-distinct keys +([F-03](FINDINGS_REGISTER.md#f-03)). + +**Three repositories in one program are working on canonicalization collapse. +Two are studying it. One has it and had not looked.** W3 closes that, and the +corpus it produces is the artifact the other two can measure against. + +--- + +## Contract summary + +| ID | Deliverable | Serves | Workstream | Acceptance | +|---|---|---|---|---| +| C1 | `cc.ingest.discretization` | ghost-ark | W6 | eleven preconditions, one negative fixture each, distinguishable refusals | +| C2 | `conformance/cc-kernel-v1/` | ghost-ark, vinctura | W5 | pinned expected values; adversarial section; runs in CI | +| C3 | `cc.canonical.v1` profile + integer bound | ghost-ark | W3 | written RFC 8785 relationship; safe-range constraint in schema | +| C4 | `cc.compose` (ROC-free) | vinctura, b2b-spatial | W5 | Vinctura's four-control numbers reproduce; no countermonotone option for n>2 | +| C5 | `cc-guard` CLI + decision table | vinctura | W5 | Vinctura's G3 gate passes by either route | +| C6 | deterministic-predicate path | b2b-spatial | W5 | same surface as C4 | +| C7 | indeterminate verdict state | b2b-spatial | W6 | serializes with reason; never `[0, 1]` | + +## What none of this establishes + +The contracts make evidence checkable. They do not make it true. No consumer +should read a passing contract check as establishing that a guardrail works, a +threshold is right, a cohort generalizes, an artifact is authentic, or a system +is safe. Each contract carries its own non-claim above, and those non-claims +travel with the artifacts, not merely with this document. diff --git a/docs/upgrade/EPISTEMIC_UPGRADE_PLAN.md b/docs/upgrade/EPISTEMIC_UPGRADE_PLAN.md new file mode 100644 index 00000000..65b8069c --- /dev/null +++ b/docs/upgrade/EPISTEMIC_UPGRADE_PLAN.md @@ -0,0 +1,769 @@ +# Epistemic Upgrade Plan — cc-framework + +> **Status: plan.** Nothing in this document is implemented. It describes work +> proposed for v0.4, grounded in measurements taken at commit `3e22c39` on +> 2026-08-19. +> +> **Non-claim.** This plan does not assert that cc-framework is safe, correct, +> production-ready, or fit for any deployment. It does not claim the work +> described here will succeed, and it does not relax a single boundary in +> [`NON_CLAIMS.md`](../research/NON_CLAIMS.md) or the +> [claim-boundary manifest](../claims/CLAIM_BOUNDARY_MANIFEST.md). A plan is a +> statement of intent. Only the acceptance gates below can turn any of it into +> evidence. + +| | | +|---|---| +| **Scope** | `Cubits11/cc-framework` only. No change is proposed to any other repository. | +| **Measured baseline** | [BASELINE_MEASUREMENTS.md](BASELINE_MEASUREMENTS.md) | +| **Defects found** | [FINDINGS_REGISTER.md](FINDINGS_REGISTER.md) — 20 findings, 5 at S1 | +| **Scored dimensions** | [COMMITTEE_SCORECARD.md](COMMITTEE_SCORECARD.md) — 12 dimensions | +| **Target tiers** | [VERIFICATION_LADDER.md](VERIFICATION_LADDER.md) | +| **Consumer obligations** | [DOWNSTREAM_CONTRACTS.md](DOWNSTREAM_CONTRACTS.md) — C1–C7 | + +--- + +## 1. The finding that organizes everything + +cc-framework's mathematics is **exact**. The LP recovers Fréchet–Hoeffding to +1.11e-16 worst case. It reproduces the correlation cliff in twenty seconds: +eighteen guardrails at p = 0.1 each, composed as a conjunction, bound to +`[0, 0.1]` — where independence would predict 1e-18. Eighteen filters buy +nothing under adversarial dependence. That result is the thesis, and it works. + +**And three separate projects reimplemented it rather than depend on this +repository.** Ghost-Ark in TypeScript, Vinctura in JavaScript, and one of them +wrote down exactly why: + +> The obvious move was to call `cc.core.composition_theory` [...] it operates on +> ROC POINT SETS and bounds the Youden J statistic. That is a DETECTOR framing — +> it assumes each guardrail is a classifier with a threshold and an operating +> curve. Vinctura's controls are not classifiers. +> +> — `vinctura/scripts/compose-bounds.js` + +A consumer read the source, found the mathematics correct and the ontology wrong, +and wrote 214 lines of JavaScript instead. That is not a documentation gap. It is +an API that models the wrong thing. + +So the scorecard reads: mathematical correctness **8.9**, consumability **2.6**, +cross-implementation agreement **0.0**. The plan closes the second and third +without touching the first. + +### The three failures, stated once + +1. **Reachability.** The calculus is correct and unreachable — wrong ontology in + the API, Python-only, no conformance corpus. Nobody can use it, so everybody + rewrites it. → W5, W6 +2. **Enforcement.** The prose discipline scores 8.8; the enforcement scores 2.1. + Strict typing is declared for 90 files and enforced on 7. The claim-boundary + manifest is validated by nothing. Everything holding this repository honest is + a person remembering to be honest. → W1, W2 +3. **Unexamined foundations.** The canonicalization kernel — which every signed + artifact routes through — silently merges Unicode-distinct keys, diverges from + RFC 8785 on five of six number forms, and has never been attacked. One hour of + probing found four defects. → W3 + +--- + +## 2. What this plan refuses to claim + +Stated before the workstreams, because a plan that leads with ambition and buries +its boundaries has already failed the discipline it proposes. + +- **The mathematics is not new.** Fréchet–Hoeffding is 1935. A sibling repository + already demoted "Fréchet ceiling + reachability sharpening" to engineering, and + it was right ([F-18](FINDINGS_REGISTER.md#f-18)). This plan does not claim + novelty for the bounds. The contribution is elsewhere — see W9. +- **No downstream integration is promised.** Whether Ghost-Ark runs the + conformance corpus, or Vinctura wires the guard, is their decision. This plan + commits only to making both *possible*. +- **No empirical claim is created by planning one.** The Correlation Atlas (W7) + is the highest-value item here and it does not exist. Until it runs, empirical + grounding stays at 2.4 and should be quoted as 2.4. +- **Completing every workstream would not establish safety.** It would establish + that this repository does what it says. Whether what it says is worth anything + is answered by someone outside the project, and no amount of internal work + substitutes for that. +- **This plan is one excavation pass.** Absence of a finding is not evidence of + absence. `redteam/dependence_search.py` — 709 statements at 65.6% coverage, + performing the search whose post-selection bias `cliff.py` refuses to certify — + was **not examined** and is the most likely home of a subtle statistical defect. + +--- + +## 3. Workstreams + +Ten workstreams. Each states its findings, deliverables, acceptance gate, and +what completing it does *not* establish. A workstream without a falsifiable gate +is a wish; every gate below is a command. + +--- + +### W0 — Make the documented path work {#w0} + +**Findings:** [F-01](FINDINGS_REGISTER.md#f-01), [F-20](FINDINGS_REGISTER.md#f-20) +· **Effort:** hours · **Prerequisite for everything else** + +A stranger following the install instructions gets a red suite. That is the first +thing an external reviewer sees, and it is not a real failure — it is a missing +dependency in the `[test]` extra. Nothing else on this plan matters if the front +door is broken. + +**Deliverables** + +1. Add `build`, `setuptools`, `wheel` to `[project.optional-dependencies].test`. +2. CI job installing `.[test]` **specifically** — not `.[dev]` — and running the + suite. The extra strangers are told to use becomes the extra that is tested. +3. Delete `tests/regression/week2/test_week2_deliverables.py::test_unit_tests_pass`. + It shells out to run the whole unit suite and masks real failures — it is how + F-01 presented, as an unrelated `assert 1 == 0`. If the intent was "week-2 + deliverables exist," assert that directly. +4. `scripts/upgrade_baseline.py` — regenerates every table in + BASELINE_MEASUREMENTS.md into `docs/upgrade/baseline.json`. The plan's own + numbers become reproducible, or the plan fails its own standard. + +**Acceptance gate** + +```bash +git clean -xdf && uv venv .venv && uv pip install -e '.[test]' && pytest +# expect: 0 failed, 0 errors +python scripts/upgrade_baseline.py --check # regenerated numbers match committed +``` + +**Does not establish.** Anything about correctness. This is a packaging fix. + +--- + +### W1 — Make the type discipline true {#w1} + +**Findings:** [F-02](FINDINGS_REGISTER.md#f-02) · **Effort:** 1 week setup, then continuous + +`pyproject.toml` declares `strict = true` over `packages = ["cc"]`. At that scope +mypy reports **279 errors in 47 files**. CI enforces 7 files. The configuration +asserts a discipline the repository does not have — claim inflation in TOML. + +Do not delete `strict = true`. Do not silence 279 errors in one pass; a +thousand-line `type: ignore` commit is how strictness dies quietly. + +**Deliverables** + +1. `docs/upgrade/typing-ratchet.json` — per-module current error count, committed. +2. CI job running mypy at the declared scope, failing only if a module's count + **increases**. New code is strict from birth; existing debt is visible and + frozen. +3. Burn-down order: `kernel/` → `reporting/` → `evidence/` → the rest. The + modules that emit signed artifacts get types first. +4. Remove the 7-file list from CI and pre-commit once the ratchet is live. + +**Acceptance gate** + +```bash +mypy # declared scope +python scripts/check_typing_ratchet.py # no module regressed +``` + +with `kernel/` and `reporting/` at zero errors by the end of the workstream. + +**Does not establish.** That the code is correct. Types constrain shape, not +meaning. A well-typed wrong bound is still wrong. + +--- + +### W2 — Make the claim discipline executable {#w2} + +**Findings:** [F-12](FINDINGS_REGISTER.md#f-12), [F-15](FINDINGS_REGISTER.md#f-15) +· **Effort:** 1–2 weeks + +The prose scores 8.8 and the enforcement scores 2.1. This workstream closes the +gap without weakening the prose. + +**The design constraint that matters.** A naive port of Ghost-Ark's scanner would +fire on 70 uses of "guarantee" in this repository, **almost all of them negated** — +`does not guarantee`, `no guarantee of`. It would be switched off within a week, +and switching off a claim gate is worse than never having one. So negation +handling and an allowlist are day-one requirements, not refinements. + +**Deliverables** + +1. `scripts/check_claims.py` — forbidden-phrase scanner with: + - negation-aware matching (a phrase inside "does not X" is not a violation); + - an allowlist with a **written reason per entry**, following Ghost-Ark's + pattern — policy documents and the scanner itself legitimately quote + forbidden wording; + - scanning of `.md`, `.py`, `.tex`, `.yml`, `.json`, `.sh`, and `Makefile`, + because reviewer-facing claim text lives in build files too; + - exclusion of generated output, so the verdict describes the committed tree + rather than local run state. +2. Close the two residual manifest gaps ([F-12](FINDINGS_REGISTER.md#f-12), corrected): + resolve path-like tokens in `supporting_tests_or_commands`, and assert the + Markdown and JSON claim-id sets are equal. The validator already runs (via + pytest) and already checks `supporting_files` existence — an earlier draft of + this plan said otherwise and was wrong. +3. Resolve the two theorem ledgers. `docs/theory/theorem_ledger.md` governs; the + other becomes a pointer or is deleted. Add a test asserting exactly one. +4. **Adopt Vinctura's rule:** *no orphan claims — every public sentence carries + the command that falsifies it or an explicit limitation.* Apply to README + first, then `docs/index.md`. This is the strongest claim rule in the program + and it comes from a consumer, not from here. + +**Acceptance gate** + +```bash +make claims # scanner + manifest validation, both clean +pytest tests/unit/docs/ # every manifest row's files and tests exist +``` + +plus: **every README claim maps to a command in the manifest.** + +**Does not establish.** That the claims are true. A scanner catches known +overclaim phrasings. It cannot detect a number that was fabricated, a citation +that does not say what it is claimed to say, or a subtler overclaim in unfamiliar +words. + +--- + +### W3 — Attack the canonicalization kernel {#w3} + +> **DELIVERED 2026-08-19.** Canonicalization & provenance integrity +> **3.8 → 8.2**. All five findings fixed (F-03, F-04, F-05 at S1; F-06, F-07 at +> S2). `cc.canonical.v2` (RFC 8785) is the default; `cc.canonical.v1` is +> read-only so historical receipts stay verifiable. Delivery record in +> [§3.3 below](#w3-delivery-record); profile documented in +> [CANONICAL_PROFILE.md](../architecture/CANONICAL_PROFILE.md). Contract +> [C3](DOWNSTREAM_CONTRACTS.md#contract-summary) is shipped. + +**Findings:** [F-03](FINDINGS_REGISTER.md#f-03) **S1**, [F-04](FINDINGS_REGISTER.md#f-04) **S1**, [F-05](FINDINGS_REGISTER.md#f-05) **S1**, [F-06](FINDINGS_REGISTER.md#f-06), [F-07](FINDINGS_REGISTER.md#f-07) +· **Effort:** 2–3 weeks · **Highest severity** + +Every signed artifact this repository emits routes through +`canonical_json_bytes`. One hour of probing found four defects, including a +silent two-keys-to-one collapse with no error raised. Ghost-Ark found five +unintended kernel members in its own pipeline by building a 31-class census; this +repository has never run the equivalent. + +**Deliverables** + +1. **Fail closed on key collision.** After NFC normalization, compare key-set + cardinality. If it shrank, raise `CanonicalJSONError` naming both keys. Detect, + never resolve. +2. **Strict duplicate-key rejection on read.** An `object_pairs_hook` that raises + on repeated keys, applied to the report reader, evidence-bundle reader, and + claim-envelope reader. +3. **Normalize `-0.0` to `0.0`** — with a positive control that genuinely distinct + near-zero values stay distinct. A rule that rejects honest documents is a + trade, not a fix. +4. **`docs/architecture/CANONICAL_PROFILE.md`** — `cc.canonical.v1`, versioned, + stating its relationship to RFC 8785 explicitly. Either adopt JCS number + serialization or declare a non-JCS profile with a written rationale. Both are + defensible; silence is not. **Decide before an independent verifier is + written.** +5. **Bound receipt-covered integers** to the IEEE-754 safe range in the schema, or + carry them as strings. A JS `JSON.parse` collapses `2^53+1` before any + verifier code runs — the kernel is set by the parser, and no downstream fix + reaches it. +6. **`tests/canonical_corpus/`** — an adversarial census in Ghost-Ark's E1 shape: + declared intent per class (`distinct` / `equivalent`), machine-readable + verdicts (`sound`, `unintended-kernel`, `over-discrimination`, `fail-closed`, + `sound-by-rejection`, `rejection-asymmetry`), and **positive controls** so + strictness is a fix rather than a trade. Seed classes: the six probed for this + plan, plus duplicate-key-nested-in-array, empty-key-repeated, lone surrogate, + deep nesting, large-document-single-byte, safe-integer-neighbours. + **Provenance is `census` — no confidence intervals, exact counts only.** + +**Acceptance gate** + +```bash +make canon-corpus # every class matches its declared verdict +make receipt-replay # capsule rebuilds byte-identically +``` + +with **zero classes at `unintended-kernel`** and **zero at +`rejection-asymmetry`** — the second is what distinguishes a fix from a trade. + +**Binding rule:** never weaken the corpus to make a test pass. A class that starts +failing is a kernel regression. + +**Does not establish.** That the canonicalizer has no collisions — only that it +has none in the declared classes. The corpus is a census and its size is an +authoring decision, so it carries no interval and no coverage claim. + +#### Delivery record {#w3-delivery-record} + +| Deliverable | Shipped as | Evidence | +|---|---|---| +| RFC 8785 profile | `cc.canonical.v2` in `src/cc/reporting/canonical.py` | 15/15 probed number forms conform; 16 pinned individually | +| Legacy profile retained | `cc.canonical.v1`, read-only | verification dispatches on the declared profile; a test asserts v1 still carries its defects | +| Collision fixed | no normalization, per RFC 8785 | two byte-distinct keys go in, two come out, at top level and nested | +| Producer lint | `assert_no_confusable_keys` | opt-in, deliberately off the hash path | +| Strict reads | `strict_json_loads` | wired into the report CLI, claim-governance readers, Merkle log | +| Integer bound | `MAX_SAFE_INTEGER` | declared narrowing of JCS, documented not implicit | +| Census | `scripts/canonicalization_probe.py` | 14 classes, both profiles, 4 positive controls | +| Corpus | `tests/unit/canonical/` | 57 tests | + +**The remedy shipped is not the one first proposed here.** The plan said +normalization must *detect* collision and fail closed. The better fix, taken +instead, was to stop normalizing: RFC 8785 makes normalization the producer's +job, and a canonicalizer that mutates content is not a canonicalizer. With no +normalization there is nothing left to collide. + +**Numbers.** v2 census: 12 sound, 1 fail-closed, 1 sound-by-rejection; zero +`unintended-kernel`, zero `rejection-asymmetry`, zero `over-discrimination`. +Suite 816 → **873 passing**, 0 failing. + +**Migration was verified, not assumed.** Every regenerated capsule artifact was +diffed with hashes, hash-derived ids, and the profile identifier scrubbed; all +eleven were byte-identical under that scrub. No content, no claim text, and no +non-claim changed — only hashes moved. + +**One intent was corrected.** `int-vs-float-same-value` moved from `distinct` to +`equivalent`: the original declaration described Python's type system, not +JSON's, which has one number type. Correcting a declaration that was wrong about +the domain is legitimate; correcting one to flatter a result is not. Recorded in +the probe source and the profile doc rather than edited away. + +**Honest limits.** No fuzzing of the canonicalizer — every class was authored by +hand. **No cross-language differential on receipts yet**: the corpus establishes +agreement on the composition kernel, but nothing re-canonicalizes a CC report in +another language and compares digests, so "cross-language verifiable" describes +v2's design rather than a demonstrated result. No external reviewer has attacked +either profile. + +--- + +### W4 — Deepen the test floor {#w4} + +**Findings:** [F-13](FINDINGS_REGISTER.md#f-13), [F-14](FINDINGS_REGISTER.md#f-14) +· **Effort:** 3–4 weeks + +Coverage is 69.91% with no gate. The `cc-report` CLI — the product surface, the +thing a reviewer actually runs — is at **0%**. `core/stats.py` is at **38% across +756 statements**: the statistics engine, in a repository about statistical +honesty, is its least-tested large component. Untested statistical code does not +fail loudly; it returns a plausible number. + +**Deliverables** + +1. **CLI golden tests** — build a report, verify it, verify a tampered copy is + rejected, verify claim-governance PASS and FAIL paths. Target `reporting/cli.py` + from 0% to >80%. +2. **Property-based tests on `core/stats.py`** using Hypothesis (already a + declared dependency, currently ungated): interval coverage, monotonicity, + boundary behaviour. Properties, not examples — the failure mode here is a + plausible wrong number, which example tests are poor at catching. +3. **Property-based kernel tests** over the marginal simplex, replacing the + six-point census with sampled invariants: FH recovery, monotone tightening + under added constraints, `L <= U` always, infeasibility raises. +4. **Coverage floor at the measured 69.91%**, ratcheting upward. A floor set below + the current value is theatre. +5. **Mutation testing** on `kernel/` and `reporting/`, score published in the + scorecard. The current score is *unknown*, not low, and that distinction is + worth removing. +6. **Dual certificates** from the LP, emitted with the primal witness, so a + skeptic verifies optimality without re-solving. Cheapest available move on + formal readiness. + +**Acceptance gate** + +```bash +pytest --cov=src/cc --cov-fail-under=69.91 +make kernel-properties && make kernel-duals && make mutation +``` + +with `reporting/cli.py` > 80%, `core/stats.py` > 70%, and a published mutation +score. + +**Does not establish.** Correctness. Higher coverage means more code was executed, +not that it was executed correctly. Mutation score bounds the suite's sensitivity +on the mutated subset, nothing wider. + +--- + +### W5 — Make the kernel consumable {#w5} + +> **DELIVERED 2026-08-19.** Consumability **2.6 → 6.8**, cross-implementation +> agreement **0.0 → 6.5**. Evidence in +> [§3.5 Delivery record](#w5-delivery-record) below; scores and their limits in +> [COMMITTEE_SCORECARD.md](COMMITTEE_SCORECARD.md#consumability-as-a-library). +> C4, C5, C6 and the C2 corpus are shipped. C1, C3, C7 remain with W3 and W6. + +**Findings:** [F-08](FINDINGS_REGISTER.md#f-08) **S3**, [F-09](FINDINGS_REGISTER.md#f-09) **S3**, [F-10](FINDINGS_REGISTER.md#f-10) **S3** +· **Contracts:** [C2, C4, C5, C6](DOWNSTREAM_CONTRACTS.md#contract-summary) +· **Effort:** 4–6 weeks · **Highest leverage** + +Three reimplementations, one written rejection, one dated unfillable request. +This workstream is why the repository scores 2.6 on consumability against 8.9 on +mathematics. + +**Deliverables** + +1. **`cc.compose`** — marginals in, sharp bounds out, **no ROC concept anywhere in + the signature**: + + ```python + compose_bounds( + marginals: Mapping[str, float], + event: Literal["all", "any"] | LinearQuery, + constraints: Sequence[Constraint] = (), + ) -> Interval + ``` + + Deterministic predicates are first-class. Detectors reach the same surface by + supplying an operating point as a marginal — the ROC path becomes an optional + adapter *above* the event calculus, never underneath it. This inverts the + relationship that caused the rejection. + +2. **`conformance/cc-kernel-v1/`** — language-agnostic corpus: + `(marginals, constraints, query) → [L, U]` with exact expected values, plus an + adversarial section (infeasible sets, degenerate marginals, boundary values, + `p=0`, `p=1`, near-machine-epsilon marginals). Any implementation in any + language either passes or is not a CC kernel. Run in CI here; published for + the others. + +3. **`cc-guard`** — a stdin/stdout JSON subcommand exposing the provenance-tagged + guards (post-selection refusal first), **plus** the same logic as a pure-data + decision table in the corpus, so a JS caller can enforce it with no Python + process. A subprocess dependency is a weaker bridge than a table. + +4. **A second implementation, in this repository, in another language.** Small, + dependency-free, deriving from the published corpus and not from the Python + source. Without it, Tier 3 of the ladder cannot exist and + cross-implementation agreement stays at 0.0. + +5. **Differential harness** — randomized `(marginals, query)` cases through both + implementations, comparing to a declared tolerance. Divergence is recorded as + a finding, never reconciled by fiat. + +**Acceptance gate** + +```bash +make conformance-publish && make differential-agreement && make differential-fuzz +pytest tests/acceptance/test_vinctura_four_control.py # their numbers, from cc.compose +``` + +The Vinctura acceptance test is the real gate: **their four-control result, +including the sensitivity analysis over assumed detection rates, reproduced from +`cc.compose`, such that their 214 lines could be deleted.** It must also +reproduce their correct refusal — no countermonotone regime for n > 2, since the +FH lower bound is not a copula in dimension ≥ 3 though it stays pointwise sharp. +If the API silently offers a countermonotone option for four events, it is wrong, +and the consumer noticed before this repository did. + +**Does not establish.** That downstream projects will adopt it. That is their +decision. The gate is that adoption becomes *possible* and that a real consumer's +published numbers reproduce. + +#### Delivery record {#w5-delivery-record} + +Measured on 2026-08-19. Every row is a command. + +| Deliverable | Shipped as | Evidence | +|---|---|---| +| ROC-free surface | `src/cc/compose/` | 34 unit tests; a test greps the public signatures for `roc`, `youden`, `tpr`, `fpr`, `threshold`, `operating_point` and fails if any reappears | +| Conformance corpus | `conformance/cc-kernel-v1/` | 24 accept + 8 reject cases, `SPEC.md` (256 lines), manifest with digests, 1e-12 tolerance | +| Second implementation | `verifiers/node/cc_compose_verify.mjs` | zero-dependency Node, written from `SPEC.md`; 24/24 accept, 8/8 reject | +| Differential harness | `scripts/differential_compose.py` | 23,000 randomized cases across 6 seeds, **0 disagreements** | +| Cross-language guard | `src/cc/cli/guard.py`, `cc-guard` | stdin/stdout JSON **and** a pure-data decision table; table-vs-code agreement asserted | +| Acceptance gate | `tests/acceptance/` | an external consumer's **published** four-control result reproduced: `[0, 0.01]`, independence `1.2e-5`, **833×**, all three scenarios, their sensitivity finding | + +Enforced by `make conformance`, `make differential`, `make acceptance`, +`make test-compose`, and two new CI jobs. + +**Numbers.** Suite 689 → **808 passing**, 0 failing. Coverage 69.91% → **70.13%**. +`cc/compose/_bounds.py` **97.09%**, `cc/cli/guard.py` **97.56%**. The closed form +agrees with the finite-atom LP to **3.3e-16** over 600 randomized cases. + +**The fuzzer earned its keep on its first run.** `dependence="countermonotone"` +with exactly one event: the Python raised `IndexError`, the Node silently +returned `NaN`. Both wrong, differently; the curated corpus had not thought to +ask. Both fixed, and pinned as `reject-countermonotone-one-event`. That is the +argument for randomized differential testing over a corpus alone. + +**Honest limits.** The Node implementation and the Python reference were +authored in the same project — a wrong specification yields two implementations +wrong together. This is a differential-testing instrument, not an independent +replication, and both the verifier output and the corpus manifest say so in +their non-claims. The one genuinely non-same-author check is the external +oracle, and it is **one** oracle on **one** scenario family. Neither the corpus +nor the fuzzer covers the constrained LP path. No downstream project has adopted +any of this yet: the obstacle is removed, the adoption is theirs. + +**Also fixed in passing.** [F-01](FINDINGS_REGISTER.md#f-01): `build`, +`setuptools`, and `wheel` added to the `[test]` extra, plus a CI job that +installs `.[test]` specifically, so the extra strangers are told to use is the +extra that is tested. + +--- + +### W6 — Contract the boundaries {#w6} + +**Findings:** [F-11](FINDINGS_REGISTER.md#f-11), [F-19](FINDINGS_REGISTER.md#f-19) +· **Contracts:** [C1, C3, C7](DOWNSTREAM_CONTRACTS.md#contract-summary) +· **Effort:** 3–4 weeks + +Ghost-Ark declares a binding rule that cc-framework must not consume naked binary +labels. cc-framework has no ingest module, no schema, and no test for it. The rule +binds a repository that cannot honour it. + +**Deliverables** + +1. **`cc.ingest.discretization`** — fail-closed reader for + `ghost.discretization_rule_receipt.v1` implementing all eleven preconditions + (binary domain, failure semantics, bounded score domain, threshold legality, + signed comparator, monotonic risk invariant, calibration digest, scoring + digest, temporal validity, parent lineage, stationarity declaration). A + marginal without lineage is refused, not defaulted. +2. **Negative corpus** — one fixture per precondition, each rejected with a + **distinguishable reason**. A reader that rejects everything for the same + reason enforces one rule, not eleven. +3. **Versioned resolvable schemas** — `cc..v` identifiers, version in + `$id` rather than `title`. A schema whose version lives in its title can change + without its identifier changing. +4. **Indeterminate verdict state** (C7) — insufficient evidence, infeasible + constraints, and refused post-selection claims all serialize as + `INDETERMINATE` with a machine-readable reason, **never as `[0, 1]`**, which a + careless consumer reads as a computed bound. The design constraint is Assay's + and it is right: a null-unaware consumer must fail loudly rather than silently + read indeterminate as a pass. +5. **`declared_reference` non-claim travels with the object** — carrying receipt + identifiers is not verifying them, and the ingested artifact must say so. + +**Acceptance gate** + +```bash +make ingest-corpus # every precondition fixture refused, reasons distinct +pytest tests/unit/evidence/test_indeterminate_serialization.py +``` + +**Does not establish.** That upstream discretization was appropriate, the +threshold well chosen, or the score calibrated. The contract checks that evidence +carries its lineage, not that the lineage is wise. + +--- + +### W7 — Ground it empirically {#w7} + +**Findings:** [F-17](FINDINGS_REGISTER.md#f-17) · **Effort:** 4–8 weeks +· **Highest value, lowest certainty** + +Empirical grounding scores **2.4**. No `p_i` in this repository was measured +against a production guardrail. Every marginal is supplied, synthetic, or +assumed. VISION names the Correlation Atlas as the empirical keystone and the +most citable single artifact the program can produce; it does not exist. The +adapters that would collect it do exist and have never been run in a campaign. + +**Deliverables** + +1. **Turn on the dark lanes.** Scheduled workflow setting `CC_RUN_EXPERIMENTS=1` + and `CC_RUN_PERF=1` with optional extras installed, publishing artifacts. Nine + tests have never executed in CI. Allowed to be red without blocking merges — a + lane that must stay green gets weakened until it does. +2. **Executable reporting rules**, adopted from Ghost-Ark and enforced in code, + not prose: + - no point estimate without dispersion (p50 with IQR, never a bare p50); + - no proportion without a denominator, no rate without its control arm; + - **no confidence interval over a curated census** — a hand-authored corpus is + the whole population and its size is an authoring decision; + - no interval below n = 30; + - intent declared before results, pinned by a test; + - state the host; + - report what was not measured. + + These become assertions in `cc.core.stats`, in the shape of Ghost-Ark's + `reportProportion` / `assertCensusReporting`. cc-framework enforces the + post-selection rule already; these are the rest of the family. +3. **Minimal Correlation Atlas** — two or three real guardrails on one public + jailbreak corpus. Per-pair marginals, observed joint rates, phi, and the + independence-versus-worst-case gap. Each row emits a certificate. Pre-register + the cohort and the stopping rule before the first measurement. +4. **Repeated-measurement timing** replacing the single-shot scaling table in the + baseline, with p50 and IQR and a named host. + +**Acceptance gate** + +```bash +make atlas # produces a signed, replayable atlas artifact +``` + +with: pre-registration digest published **before** results; every rate carrying +its denominator; no interval over a census; cohort, stationarity declaration, and +non-claims present on every row; **and results reported whether or not they are +interesting.** A finding of "independence was approximately right on this cohort" +is a real result and gets published. + +**Does not establish.** That a measured regret on one cohort predicts another. The +Atlas is **descriptive, never predictive**. A number measured on one corpus of +jailbreaks says nothing about the next one, and the artifact must say so on its +face. + +--- + +### W8 — Hygiene {#w8} + +**Findings:** [F-16](FINDINGS_REGISTER.md#f-16) · **Effort:** days · **Do during a release** + +17.8 MB of Blender renders tracked in git; `.git` is 43 MB against a 54 MB +working tree. Clone cost for external reviewers, no verification benefit. + +**Deliverables.** Renders to release assets or LFS, one small preview retained. +Archived JSONL checkpoints reviewed. `src/cc/_legacy/` either documented with a +removal version or deleted. + +**Acceptance gate.** `.git` under 15 MB; `git clone --depth 1` under 20 MB. + +**Does not establish.** Anything. This is housekeeping and is scheduled last on +purpose. + +--- + +### W9 — Reframe the contribution {#w9} + +**Findings:** [F-18](FINDINGS_REGISTER.md#f-18) **S3** · **Effort:** 1 week +· **Do first, it is nearly free** + +A sibling repository's research ledger demoted the core contribution: + +> **Fréchet ceiling + reachability sharpening** [...] **DEMOTED to engineering.** +> The mathematics is 1935; the sharpening is a known distinction. Survives as +> reporting practice, not as science. + +This is correct. Any framing that positions cc-framework's value as "we compute +Fréchet–Hoeffding bounds" is answering a criticism that has already been made and +sustained *from inside the program*. + +**The move is to concede it first, above the fold.** A criticism you state +yourself cannot be used against you. + +**What is genuinely not 1935:** + +| Contribution | Status | +|---|---| +| Statistical inference for partial-identification bounds under **estimated** marginals — a confidence band around `[L, U]`, not a point interval | thin in the literature; VISION Pillar I; **open** | +| **Measured** co-failure dependence on real guardrail stacks — how badly independence lies in practice | nobody has published it; VISION Pillar V; **open** | +| A conformance corpus making three independent implementations agree | engineering, but engineering nobody has done for this calculus | +| **Refusing to certify** — the post-selection guard at `cliff.py:321` | a design commitment most statistical software does not make; a consumer called it "the single most elegant thing available in this program" | + +**Deliverables** + +1. README rewritten: concede the theorem, claim the measurement. Lead with + limitations, following Vinctura's convention. +2. `docs/research/CONTRIBUTION_BOUNDARY.md` — what is classical, what is + engineering, what is open. Cite the demotion by name rather than working + around it. +3. VISION.md pillars re-sequenced against this framing. Pillars I and V are the + contribution; II and III are the infrastructure that makes them checkable. + +**Acceptance gate.** The README states, in its first screen, that the underlying +inequality is classical and names what is not. Reviewed by someone who has read +the demotion. + +**Does not establish.** That the open problems will be solved. Naming a gap +honestly is not filling it. + +--- + +## 4. Sequencing + +Ordered by leverage × prerequisite, not by size. + +### Phase 0 — Credibility (weeks 1–2) + +**W0** (documented path works) + **W9** (reframe). Both are nearly free and both +are prerequisites for anyone taking the rest seriously. A reviewer who hits a red +suite, or a framing already publicly demoted, stops reading. + +### Phase 1 — Foundations (weeks 2–6) + +**W3** (canonicalization) + **W2** (claim enforcement) in parallel. W3 carries +every S1 finding and everything signed depends on it. W2 stops the discipline +from depending on memory. Start **W1** (typing ratchet) here — it is cheap to +start and pays continuously. + +### Phase 2 — Reachability (weeks 5–12) + +**W5** (consumable kernel) + **W6** (contracts). The highest-leverage work, and it +needs W3 finished first: a conformance corpus over a canonicalization that +silently collapses keys would pin the defect into the contract. + +### Phase 3 — Depth (weeks 8–16, overlapping) + +**W4** (test floor). Overlaps Phase 2; the CLI golden tests and property tests can +start as soon as W0 lands. + +### Phase 4 — Evidence (weeks 12–24) + +**W7** (empirical grounding). Highest value and lowest certainty, so it goes last +and is allowed to fail. It depends on W5 (certificates per Atlas row) and W2 +(reporting rules enforced). + +### Anytime + +**W8** (hygiene), during a release. + +``` +wk: 1 2 3 4 5 6 7 8 9 10 11 12 ... 16 ... 24 +W0 ██ +W9 ████ +W3 ████████████ +W2 ████████ +W1 ████████████████████████████████████████████ (continuous ratchet) +W5 ████████████████████████ +W6 ████████████████ +W4 ████████████████████████ +W7 ████████████████████████████ +W8 ██ +``` + +--- + +## 5. What "done" looks like + +At the end, the scorecard should read: + +| Dimension | Now | Target | +|---|---:|---:| +| Mathematical correctness | 8.9 | 9.4 | +| Canonicalization integrity | 3.8 | 8.5 | +| Claim discipline — enforcement | 2.1 | 8.5 | +| Consumability | 2.6 | 8.5 | +| Cross-implementation agreement | 0.0 | 8.0 | +| Empirical grounding | 2.4 | 6.5 | + +and these statements should be true, each checkable by a command: + +- A stranger clones, runs one command, and gets a green suite. +- A stranger recomputes an interval from its witness without trusting the + producer, and checks the dual certificate without re-solving the LP. +- An implementation in another language passes the published corpus, and a + differential harness proves it agrees. +- The canonicalizer fails closed on every declared collision class, with zero + `unintended-kernel` and zero `rejection-asymmetry` verdicts. +- No claim reaches `main` without a manifest row naming a file and test that + exist. +- A consumer computes composition bounds from marginals with no ROC concept in + sight, and a real consumer's published numbers reproduce from the library. +- At least one number in the repository was measured on a real guardrail, and is + reported with its denominator, its cohort, and what it does not predict. + +None of that establishes safety. It establishes that this repository does what it +says — which is the only thing a repository can establish about itself. + +--- + +## 6. The one question, per feature + +Adapted from Ghost-Ark's frontier questions. Every proposed change to +cc-framework must answer all ten before it merges: + +1. What claim is being introduced, and at which claim-boundary level? +2. What evidence supports it, and where does that evidence live? +3. What command replays it? +4. What conformance case or malicious fixture disproves it? +5. Which boundary fails closed when it is violated? +6. Which non-claim accompanies it? +7. Which reviewer, outside this project, could reproduce it? +8. Which downstream contract does it satisfy or break? +9. Is the number a census or a sample — and does its interval match? +10. What assumptions remain invisible? + +If these cannot be answered, the design is incomplete. diff --git a/docs/upgrade/FINDINGS_REGISTER.md b/docs/upgrade/FINDINGS_REGISTER.md new file mode 100644 index 00000000..ac857b4f --- /dev/null +++ b/docs/upgrade/FINDINGS_REGISTER.md @@ -0,0 +1,690 @@ +# Findings Register — cc-framework Epistemic Excavation + +> **Status: measured findings.** Every finding below was reproduced by a command +> against commit `3e22c39`. Each carries the reproduction, the impact, the +> proposed remedy, and the workstream that owns it. +> +> **Non-claim.** This register is not a security audit and not a completeness +> claim. It records what one excavation pass found. The absence of a finding is +> not evidence of absence — see [Not searched](#what-was-not-searched). + +Severity scale: + +| | Meaning | +|---|---| +| **S1** | An artifact this repository signs or publishes can be wrong or misleading, silently. | +| **S2** | A gate the repository claims to enforce is not enforced. | +| **S3** | A consumer cannot use the repository for its stated purpose. | +| **S4** | Hygiene, drift, or duplication that will become S1–S3 if left. | + +--- + +## Index + +| ID | Severity | Finding | Owner | +|---|---|---|---| +| [F-01](#f-01) | S2 | The declared `[test]` extra cannot run the test suite | W0 | +| [F-02](#f-02) | S2 | `strict = true` mypy is declared for `cc`, enforced on 7 of 90 files | W1 | +| [F-03](#f-03) | ~~S1~~ **FIXED** | Canonicalization silently merges Unicode-distinct keys | W3 | +| [F-04](#f-04) | ~~S1~~ **FIXED** | Canonical form is not RFC 8785; five of six number forms diverge | W3 | +| [F-05](#f-05) | ~~S1~~ **FIXED** | Cross-language receipt divergence above 2^53 is undetectable | W3, W5 | +| [F-06](#f-06) | ~~S2~~ **FIXED** | `-0.0` and `0.0` produce different receipts for identical numbers | W3 | +| [F-07](#f-07) | ~~S2~~ **FIXED** | Duplicate JSON keys are accepted last-wins on the parse side | W3 | +| [F-08](#f-08) | **S3** | A real consumer read the composition API and rejected it | W5 | +| [F-09](#f-09) | **S3** | The kernel calculus is independently reimplemented three times | W5 | +| [F-10](#f-10) | **S3** | Vinctura's requested cross-language guard surface does not exist | W5 | +| [F-11](#f-11) | S3 | Ghost-Ark's binding ingest rule has no enforcing code here | W6 | +| [F-12](#f-12) | S4 | The manifest validator has two narrow unenforced gaps | W2 | +| [F-13](#f-13) | S2 | No coverage gate; the report CLI is at 0% | W4 | +| [F-14](#f-14) | S2 | `core/stats.py` is 38% covered across 756 statements | W4 | +| [F-15](#f-15) | S4 | Two divergent theorem ledgers | W2 | +| [F-16](#f-16) | S4 | 17.8 MB of render binaries tracked in git | W8 | +| [F-17](#f-17) | S2 | Experiment and performance lanes have never run in CI | W7 | +| [F-18](#f-18) | S3 | A sibling repository demoted the core contribution to "engineering" | W9 | +| [F-19](#f-19) | S4 | Schema surface is two files with a non-resolvable `$id` | W6 | +| [F-20](#f-20) | S2 | A regression test shells out to run the whole unit suite | W0 | + +--- + +## F-01 + +**The declared `[test]` extra cannot run the test suite.** — S2, owner W0 + +`pyproject.toml`'s `[project.optional-dependencies].test` omits `build`, +`setuptools`, and `wheel`. `tests/unit/packaging/test_wheel_boundary.py` +subprocesses `python -m build --no-isolation`, which needs all three. + +```bash +uv venv .venv && uv pip install -e '.[test]' && pytest +# 1 failed, 685 passed, 9 skipped, 3 errors +# E No module named build +# then, after installing build: +# E BackendUnavailable: Cannot import 'setuptools.build_meta' +``` + +**Impact.** A stranger following the documented install path gets a red suite and +has no way to tell a real regression from a packaging gap. CI passes only +because `[dev]` happens to pull `build`, so the defect is invisible to the +people who would fix it. This is the single cheapest fix in the register and it +is the first thing an external reviewer hits. + +**Remedy.** Add `build`, `setuptools`, `wheel` to `[test]`. Add a CI job that +installs `.[test]` *specifically* — not `.[dev]` — and runs the suite, so the +extra that strangers are told to use is the extra that is tested. + +--- + +## F-02 + +**Strict typing is declared for the package and enforced on 7 files.** — S2, owner W1 + +`pyproject.toml` declares `packages = ["cc"]` with `strict = true`. CI and +pre-commit both override this with an explicit 7-file list. + +```bash +mypy # at the declared scope +# Found 279 errors in 47 files (checked 101 source files) +``` + +**Impact.** The configuration asserts a discipline the repository does not have. +Any reader who checks `pyproject.toml` to see how strict the project is will be +misled by 279 errors' worth. This is claim inflation *in the build +configuration* — the same defect class the project's own documents exist to +prevent, expressed in TOML instead of prose. + +**Remedy.** Do not delete `strict = true`, and do not silence 279 errors in one +pass. Ratchet: introduce `docs/upgrade/typing-ratchet.json` recording the +current per-module error count, add a CI job that fails if any module's count +*increases*, and burn the list down module by module, kernel first. The declared +scope becomes true incrementally and cannot regress. + +--- + +## F-03 + +> **FIXED 2026-08-19 by W3.** `cc.canonical.v2` (RFC 8785) is now the default +> profile; `cc.canonical.v1` is retained read-only so pre-migration receipts stay +> verifiable, and verification dispatches on the profile each receipt declares. +> See [CANONICAL_PROFILE.md](../architecture/CANONICAL_PROFILE.md). The census +> at `scripts/canonicalization_probe.py` now reports zero `unintended-kernel`, +> zero `rejection-asymmetry`, and zero `over-discrimination` under v2, and +> `tests/unit/canonical/` holds it there. + +**Canonicalization silently merges Unicode-distinct keys.** — **S1**, owner W3 + +`_normalize_json_value` applies `unicodedata.normalize("NFC", key)` to every +mapping key and writes results into a fresh `dict`. When two distinct source +keys share an NFC form, the second overwrites the first. + +```python +>>> from cc.reporting.canonical import canonical_json_bytes +>>> canonical_json_bytes({"é": 1, "é": 2}) # U+00E9 key, then U+0065 U+0301 key +b'{"\xc3\xa9":2}' +``` + +Two keys in. One key out. **No exception.** + +The committed census (`scripts/canonicalization_probe.py`) finds **two** +`unintended-kernel` classes, not one. Widening the corpus from the hand-probed +six classes to eleven surfaced `nested-unicode-key-collision` — the same +collision one level down, inside a nested object: + +```python +>>> canonical_json_bytes({"outer": {"é": 1}}) == canonical_json_bytes({"outer": {"é": 1}}) +True +``` + +This matters on its own: a guard that inspected only top-level keys would pass +the flat class while leaving the defect fully exploitable. It also reproduces +Ghost-Ark's E1 lesson exactly — *widening the alphabet found more defects* — and +it is the reason the remedy below is a corpus rather than a patch. + +**Impact.** This is the most serious finding in the register. The receipt hash is +computed over the *normalized* document, so: + +1. A report can lose a field between construction and hashing, and the receipt + will faithfully attest to the truncated document. +2. Two semantically different payloads can be constructed with the same receipt + hash — a collision in the provenance kernel, reachable with no cryptography + and no privileged access. +3. Ghost-Ark's E1 census classifies exactly this pattern as an *unintended kernel + member*. cc-framework has one, in the module every signed artifact routes + through, and has never looked. + +The irony is load-bearing: `b2b-spatial-intelligence-engine` ships a research +module titled `01-canonicalization-collapse.html`. The sibling repositories study +this defect class. This repository has it. + +**Remedy as shipped** — and it is *not* the remedy first proposed here. The +original plan was to detect the collision and fail closed. The better fix, taken +instead, was to stop normalizing at all: RFC 8785 is explicit that normalization +is the producer's responsibility, and a canonicalizer that mutates content is not +a canonicalizer. Two keys that differ in Unicode form are two keys, which is what +JSON says they are, so there is nothing left to collide. + +The detection is still available as `assert_no_confusable_keys`, an **opt-in +producer lint** that is deliberately off the hash path — v1's mistake was +precisely that a content-altering rule lived inside canonicalization. + +--- + +## F-04 + +> **FIXED 2026-08-19 by W3.** `cc.canonical.v2` (RFC 8785) is now the default +> profile; `cc.canonical.v1` is retained read-only so pre-migration receipts stay +> verifiable, and verification dispatches on the profile each receipt declares. +> See [CANONICAL_PROFILE.md](../architecture/CANONICAL_PROFILE.md). The census +> at `scripts/canonicalization_probe.py` now reports zero `unintended-kernel`, +> zero `rejection-asymmetry`, and zero `over-discrimination` under v2, and +> `tests/unit/canonical/` holds it there. + +**The canonical form is not RFC 8785, and diverges on five of six number +forms.** — **S1**, owner W3 + +| Value | cc-framework | RFC 8785 (JCS) | +|---|---|---| +| `10**30` | `1000000000000000000000000000000` | `1e+30` | +| `1.0` | `1.0` | `1` | +| `-0.0` | `-0.0` | `0` | +| `1e-7` | `1e-07` | `1e-7` | +| `100.0` | `100.0` | `100` | +| `1e30` | `1e+30` | `1e+30` | + +The implementation is `json.dumps(sort_keys=True, separators=(",",":"))`, which +is a Python convention, not an interoperable standard. + +**Impact.** Any non-Python verifier that implements JCS — the obvious choice for +an independent verifier, and the family Ghost-Ark surveyed in +`CANONICALIZATION_LAYER_SURVEY.md` — computes a different digest for the same +report. Independent verification is impossible not because of a bug but because +the two sides never agreed on what the bytes are. + +**Remedy as shipped.** RFC 8785 adopted. `_es_number_to_string` implements the +ECMAScript `Number::toString` algorithm; 15 of 15 probed forms now conform, and +`tests/unit/canonical/` pins 16 of them individually. + +--- + +## F-05 + +> **FIXED 2026-08-19 by W3.** `cc.canonical.v2` (RFC 8785) is now the default +> profile; `cc.canonical.v1` is retained read-only so pre-migration receipts stay +> verifiable, and verification dispatches on the profile each receipt declares. +> See [CANONICAL_PROFILE.md](../architecture/CANONICAL_PROFILE.md). The census +> at `scripts/canonicalization_probe.py` now reports zero `unintended-kernel`, +> zero `rejection-asymmetry`, and zero `over-discrimination` under v2, and +> `tests/unit/canonical/` holds it there. + +**Cross-language receipt divergence above 2^53 is undetectable here.** — **S1**, owner W3, W5 + +```python +>>> canonical_json_bytes({"n": 2**53 + 1}) +b'{"n":9007199254740993}' +``` + +Python's arbitrary-precision integers preserve this. A JavaScript or TypeScript +verifier's `JSON.parse` collapses both `2**53+1` and `2**53+2` to +`9007199254740992` before any verifier code runs — the collapse happens inside +the parser, which is Ghost-Ark's E1 corollary C1 exactly: *the kernel is set by +the parser, and auditing the canonicalizer alone cannot find it.* + +**Impact.** Ghost-Ark holds a TypeScript reimplementation of this calculus +(`packages/research-frontier/src/ccCorrelation.ts`). If a CC report carrying a +large integer count is verified there, the two sides disagree — and nothing in +either repository would notice, because there is no differential test spanning +them. + +**Remedy as shipped.** v2 refuses any integer with `|n| > 2**53 - 1` rather +than emitting bytes that cannot survive a round trip through a conforming parser. +This is a declared *narrowing* of RFC 8785 — JCS is defined over doubles, and +Python's `int` has no such bound — documented in +[CANONICAL_PROFILE.md](../architecture/CANONICAL_PROFILE.md) rather than left as +an undocumented difference. Floats are unaffected. + +--- + +## F-06 + +> **FIXED 2026-08-19 by W3.** `cc.canonical.v2` (RFC 8785) is now the default +> profile; `cc.canonical.v1` is retained read-only so pre-migration receipts stay +> verifiable, and verification dispatches on the profile each receipt declares. +> See [CANONICAL_PROFILE.md](../architecture/CANONICAL_PROFILE.md). The census +> at `scripts/canonicalization_probe.py` now reports zero `unintended-kernel`, +> zero `rejection-asymmetry`, and zero `over-discrimination` under v2, and +> `tests/unit/canonical/` holds it there. + +**`-0.0` and `0.0` produce different receipts for the same number.** — S2, owner W3 + +```python +>>> canonical_json_bytes({"n": -0.0}) +b'{"n":-0.0}' +``` + +`-0.0 == 0.0` is true in IEEE-754 and both denote zero. Two numerically identical +reports receive different receipt hashes. + +**Impact.** Over-discrimination, in Ghost-Ark's E1 vocabulary. Less dangerous than +F-03 — it produces false *differences* rather than false *identities* — but it +breaks replay determinism whenever an LP solver returns a negative zero, which +`scipy.optimize.linprog` does routinely at a lower bound of zero. A replay that +should be byte-identical will not be. + +**Remedy as shipped.** Fixed as a consequence of adopting RFC 8785, which +requires `-0` to serialize as `0`. The positive controls hold: `5e-324` and other +genuinely distinct near-zero values remain distinct. + +--- + +## F-07 + +> **FIXED 2026-08-19 by W3.** `cc.canonical.v2` (RFC 8785) is now the default +> profile; `cc.canonical.v1` is retained read-only so pre-migration receipts stay +> verifiable, and verification dispatches on the profile each receipt declares. +> See [CANONICAL_PROFILE.md](../architecture/CANONICAL_PROFILE.md). The census +> at `scripts/canonicalization_probe.py` now reports zero `unintended-kernel`, +> zero `rejection-asymmetry`, and zero `over-discrimination` under v2, and +> `tests/unit/canonical/` holds it there. + +**Duplicate JSON keys are accepted last-wins on the parse side.** — S2, owner W3 + +```python +>>> json.loads('{"amount":1,"amount":2}') +{'amount': 2} +``` + +Reports read from disk pass through `json.loads`, which silently keeps the last +value. `canonical_json_bytes` then attests to the survivor. + +**Impact.** The same class as F-03 and reachable from any file the repository +reads. Ghost-Ark's E1 records `duplicate-key-last-wins` as an unintended kernel +member in four of five arms; the one sound arm was the one with a different +parser. + +**Remedy as shipped.** `strict_json_loads` raises `DuplicateJSONKeyError` on +any repeated key, at any nesting depth, and is wired into the report CLI reader, +the claim-governance report and artifact readers, and the Merkle transparency-log +line reader. + +--- + +## F-08 + +**A real consumer read the composition API and rejected it.** — **S3**, owner W5 + +`vinctura/scripts/compose-bounds.js`, in its file header, records the decision +not to use this repository: + +> The obvious move was to call `cc.core.composition_theory`, which implements FH +> bounds for guardrail composition. Reading it rather than its symbol list: it +> operates on **ROC POINT SETS** and bounds the **Youden J statistic** +> (J = TPR − FPR). That is a DETECTOR framing — it assumes each guardrail is a +> classifier with a threshold and an operating curve. +> +> Vinctura's controls are not classifiers. `SELF_REPORTED` refuses if and only if +> `loggedBy === memberId`. There is no threshold, no operating point, and no +> false-positive rate to trade against. Forcing a deterministic refusal rule into +> an ROC shape would produce numbers with the form of a measurement and none of +> the content. + +The consumer then wrote 214 lines of JavaScript to do it themselves. + +**Impact.** This is the most valuable single artifact the excavation found, because +it is a *rejected-adoption report written by someone who read the source*. It +names the defect precisely: the most discoverable composition entry point, +`cc.core.composition_theory`, imposes a detector ontology on events that have +none. Deterministic predicates — a refusal rule, a schema check, a signature +verification — are binary failure events with marginals and no ROC curve. They +are squarely inside the mathematics and outside the API. + +Note also what the consumer *did* keep: "the FH inequality is applied directly to +the events. **That is the part that transfers**; the ROC machinery is not." The +consumer correctly identified the kernel and correctly identified the packaging +as the obstacle. + +**Remedy.** W5 ships `cc.compose` — a marginals-in, bounds-out surface with no ROC +concept anywhere in its signature — and re-derives Vinctura's four-control result +from it as an acceptance test. The gate is not "the API exists"; the gate is +"Vinctura's own numbers reproduce from the library, and their 214 lines can be +deleted." + +--- + +## F-09 + +**The kernel calculus is independently reimplemented three times.** — **S3**, owner W5 + +| Repository | File | Language | What it reimplements | +|---|---|---|---| +| cc-framework | `src/cc/kernel/` | Python | normative source | +| ghost-ark | `packages/research-frontier/src/ccCorrelation.ts` | TypeScript | FH pairwise bounds, Wilson intervals, phi | +| vinctura | `scripts/compose-bounds.js` (214 LOC) | JavaScript | FH n-ary conjunction bounds | + +No cross-checking. No shared corpus. No agreement test. Three implementations of +one theorem, each trusted because it looks right. + +**Impact.** Ghost-Ark treats *its own* cross-language verifier agreement as a +critical architectural invariant — "breaking independent verifier agreement is a +critical architectural event." Across the CC calculus, the same project has +three implementations and zero agreement tests. Any one of them can drift and +nothing detects it. + +There is also an opportunity here that is larger than the defect. Ghost-Ark's E5 +(cross-language verifier agreement) and E7 (differential fuzz) exist and work. +Pointing that machinery at the CC kernel costs far less than inventing it. + +**Remedy.** W5 publishes `conformance/cc-kernel-v1/` — a language-agnostic corpus +of `(marginals, constraints, query) → [L, U]` cases with exact expected values, +plus an adversarial section (infeasible sets, degenerate marginals, boundary +values). Any implementation in any language either passes or is not a CC kernel. +cc-framework runs it in CI; the corpus is published for the others. + +--- + +## F-10 + +**Vinctura's requested cross-language guard surface does not exist.** — **S3**, owner W5 + +`vinctura/docs/research/program/ultracode/UC-10-KERNEL-BRIDGE.md` §4.3: + +> Optional stopping invalidates the density estimate entirely — and the sibling +> project `cc-framework` already refuses confidence claims on post-selection +> intervals at `src/cc/kernel/cliff.py:321`. **Use that.** Route the density +> estimate through the same provenance-tagged machinery so that a post-selection +> interval is refused rather than reported. Making one repository's guardrail +> catch another repository's error is the strongest possible demonstration that +> the guardrail is real. +> +> That last point is the single most elegant thing available in this program. Do it. + +The refusal exists and is correct — `cliff_certificate(..., provenance="post-selection")` +returns `regime="discovery-only"` with no confidence claim. It is reachable only +from Python, in-process. + +**Impact.** A named downstream consumer has a written, dated request to route +through a specific guard in this repository, and cannot, because there is no +callable surface across the language boundary. The most compelling demonstration +available to the whole program is blocked on packaging, not on science. + +**Remedy.** W5 ships `cc-guard` — a stdin/stdout JSON subcommand exposing the +provenance-tagged guards, and the same logic as a pure-data decision table in the +conformance corpus so a JS caller can enforce it without a Python process at all. +Acceptance: Vinctura's G3 gate (`probe.post-selection-refused`) passes against +cc-framework's guard. + +--- + +## F-11 + +**Ghost-Ark's binding ingest rule has no enforcing code here.** — S3, owner W6 + +`ghost-ark/docs/research/CLAIM_EVIDENCE_MATRIX.md` states as a binding rule: + +> CC-Framework must not consume naked binary labels from Ghost-Ark. Binary +> variables must be tied to a discretization rule, threshold, comparator, +> calibration digest, scoring digest, validity window, and parent evidence +> lineage. + +Ghost-Ark specifies the object (`ghost.discretization_rule_receipt.v1`), the +monotonic risk invariant, and eleven verification preconditions. cc-framework +has **no ingest module, no schema, and no test** for any of it. The rule binds a +repository that cannot honour it. + +**Impact.** The bridge between the two repositories is currently prose in one of +them. Every guarantee the contract offers — that a `Z_i = 1` means what it says, +that the comparator matches score polarity, that the observation is inside the +rule's validity window — is unenforced at the point of consumption. + +**Remedy.** W6 ships `cc.ingest.discretization`: a fail-closed reader for +`ghost.discretization_rule_receipt.v1` implementing all eleven preconditions, +with a negative corpus for each. A marginal that arrives without lineage is +refused, not defaulted. + +--- + +## F-12 + +**The manifest validator has two narrow unenforced gaps.** — S4, owner W2 + +> **Corrected 2026-08-19.** This finding was first written as "the +> claim-boundary manifest is validated by nothing," at severity S2. That was +> wrong. The original grep covered `.github/workflows/` and `Makefile`, found +> nothing, and I concluded the validator was unwired — without checking +> `tests/`, where it is in fact called. The corrected finding is below, and the +> enforcement score in +> [COMMITTEE_SCORECARD.md](COMMITTEE_SCORECARD.md#claim-discipline-enforcement) +> was raised from 2.1 to 4.0 as a result. + +`docs/claims/CLAIM_BOUNDARY_MANIFEST.md` and its JSON companion define C0–C5 +claim levels and map every public claim to evidence, tests, files, and +non-claims. It is one of the best-designed artifacts in the repository, and it +**is** enforced: `tests/unit/docs/test_claim_boundary_manifest.py` calls +`scripts/validate_claim_boundary_manifest.py`, and that test runs in the normal +pytest suite, which CI runs on four Python versions. + +Measured against the current tree, the validator checks — and passes — all of: + +``` +validate_manifest errors: none +claims in JSON: 8 +``` + +- required top-level and per-claim keys; +- unique claim ids; +- every `level` resolving to a declared claim level; +- **every `supporting_files` path existing on disk**; +- non-empty `non_claims` on every claim; +- non-empty `supporting_tests_or_commands` on every claim; +- `forbidden_upgrades` entries carrying `from`, `to`, and `reason`. + +**What is actually missing.** Two narrow gaps, both currently satisfied by +discipline rather than by a gate: + +1. **Test paths are not checked for existence.** `supporting_files` entries are + verified against the filesystem; `supporting_tests_or_commands` entries are + only checked for being a non-empty list. A renamed test file would pass. All + named paths do currently exist — verified separately for this register — but + nothing keeps that true. +2. **The Markdown and JSON manifests are not cross-checked.** They carry the + same eight claim ids today, with no id in one and not the other, but no test + asserts it. The prose table and the machine-checkable file can diverge + silently. + +**Impact.** Much smaller than first stated. The failure mode is a renamed test +or a claim added to one manifest and not the other, not a wholly unenforced +document. + +**Remedy.** Extend the validator to resolve path-like tokens in +`supporting_tests_or_commands`, and add a test asserting the Markdown and JSON +claim-id sets are equal. Roughly fifteen lines; it belongs in W2 with the claim +scanner, not ahead of it. + +--- + +## F-13 + +**No coverage gate; the report CLI is at 0%.** — S2, owner W4 + +Total coverage 69.91%. `src/cc/reporting/cli.py` — the `cc-report` console entry +point, 204 statements — is at **0.00%**. `src/cc/cli/manifest.py`, 68 statements, +also 0.00%. No CI job measures or enforces coverage. + +**Impact.** `cc-report build-report` and `cc-report verify-claim-governance` are +the commands an external reviewer would actually run. They are the repository's +product surface and they are entirely unexercised by the suite. The verifier that +issues PASS verdicts on claim-governance packages has no test covering its own +CLI path. + +**Remedy.** W4 adds CLI golden-output tests (build a report, verify it, verify a +tampered copy fails), then sets a coverage floor at the measured value and +ratchets it upward. A floor set below the current value is theatre; set it at +69.91% and raise it only with evidence. + +--- + +## F-14 + +**`core/stats.py` is 38.16% covered across 756 statements.** — S2, owner W4 + +434 statements in the statistics module are never executed by the suite. + +**Impact.** In a repository whose entire purpose is to be honest about +statistical claims, the statistics module is its least-tested large component. +Untested statistical code does not fail loudly — it returns a plausible number. +That is the specific failure mode this project exists to prevent, located inside +the project. + +**Remedy.** W4 treats `core/stats.py` as the highest-priority coverage target, +with property-based tests (Hypothesis is already a declared dependency and +currently ungated) over interval coverage, monotonicity, and boundary behaviour +rather than example-based tests alone. + +--- + +## F-15 + +**Two divergent theorem ledgers.** — S4, owner W2 + +`docs/theory/theorem_ledger.md` (10 KB, T1–T6 with proof status, implementation +witness, test witness, and non-claims per theorem) and +`docs/research/THEOREM_LEDGER.md` (3 KB). Different content, same name, no +cross-reference declaring which governs. + +**Impact.** Ghost-Ark's matrix names its source of truth explicitly and states the +tie-break: "If the ladder and this matrix disagree, downgrade the claim." Two +ledgers with no such rule means a reader cannot tell which one binds, and an +author can satisfy whichever is convenient. + +**Remedy.** `docs/theory/theorem_ledger.md` is the ledger. The other becomes a +pointer, or is deleted. Add a test asserting exactly one file matches +`*theorem*ledger*`. + +--- + +## F-16 + +**17.8 MB of render binaries tracked in git.** — S4, owner W8 + +Seven Blender renders in `visual_identity/claim_observatory/renders/`, 2.2–2.8 MB +each. Four 636 KB JSONL checkpoints under `docs/archive/generated-checkpoints/`. +`.git` is 43 MB against a 54 MB working tree. + +**Impact.** Clone cost for external reviewers, for no verification benefit. The +renders are brand assets, not evidence. + +**Remedy.** Move to a release asset or LFS. Keep one small preview in-tree. Not +urgent; do it during a release, not mid-workstream. + +--- + +## F-17 + +**Experiment and performance lanes have never run in CI.** — S2, owner W7 + +Nine tests skip behind `CC_RUN_EXPERIMENTS`, `CC_RUN_PERF`, or optional imports +(`guardrails`, `fastavro`, `protobuf`, `SQLAlchemy`). No workflow sets any of them. + +**Impact.** The lanes that produce empirical numbers are the lanes that never run. +`tests/performance/test_adapter_perf.py` and +`tests/experiments/test_experiment_leak_metrics.py` are, operationally, dead code +that looks like coverage. + +**Remedy.** W7 adds a scheduled workflow that sets both variables and installs the +optional extras, publishing results as artifacts. A lane that runs weekly and +reports honestly beats a lane that is skipped daily and looks green. + +--- + +## F-18 + +**A sibling repository demoted the core contribution to "engineering."** — **S3**, owner W9 + +`b2b-spatial-intelligence-engine/docs/latent-research-program.md` runs a +kill-ledger over the program's research concepts. The row that matters: + +> **Fréchet ceiling + reachability sharpening** — "Adversarial robustness ≠ +> average-case robustness" — standard in ML security since 2014. **DEMOTED to +> engineering.** The mathematics is 1935; the sharpening is a known distinction. +> Survives as reporting practice, not as science. + +**Impact.** This is the hardest single piece of feedback in the excavation, it +comes from inside the program, and it is **substantially correct**. +Fréchet–Hoeffding is 1935. Restating it for guardrails is not a contribution. +Any plan that positions cc-framework's value as "we compute FH bounds" is +answering a criticism that has already been made and sustained. + +It is also *not* fatal, and the reason matters. What is not 1935: + +- **Statistical inference for partial-identification bounds under estimated + marginals** — a confidence band around `[L, U]` rather than a point interval, + which is thin in the literature and is VISION Pillar I. +- **Measured co-failure dependence on real guardrail stacks** — nobody has + published how badly independence lies in practice, which is Pillar V. +- **A conformance corpus that makes three independent implementations agree** — + engineering, yes, but engineering nobody has done for this calculus. +- **Refusing to certify** — the post-selection guard at `cliff.py:321` is a + design commitment most statistical software does not make. + +**Remedy.** W9 rewrites the framing to concede the theorem and claim the +measurement. The repository's public position becomes: *the mathematics is +classical and we say so; the contribution is the estimation loop, the measured +atlas, the certificate, and the refusal.* Concede early, in the README, above the +fold. A criticism you state yourself cannot be used against you. + +--- + +## F-19 + +**Schema surface is two files with a non-resolvable `$id`.** — S4, owner W6 + +`schemas/cc_report.schema.json` and `schemas/evidence/evidence-item.schema.json`. +Both `$id` values point at `https://cc-framework.local/...`. Version lives in +`title` ("CC Report v0.3.1"), not in the `$id`. + +For comparison, Ghost-Ark carries 182 JSON files with versioned identifiers of the +form `ghost.discretization_rule_receipt.v1.json`. + +**Impact.** A consumer cannot resolve a schema, cannot pin a version, and cannot +tell from an artifact which schema version produced it. Version-in-title means a +schema can change without its identifier changing. + +**Remedy.** W6 adopts `cc..v` identifiers, moves version into `$id`, +and adds a test asserting every emitted artifact names a resolvable, versioned +schema. + +--- + +## F-20 + +**A regression test shells out to run the whole unit suite.** — S2, owner W0 + +`tests/regression/week2/test_week2_deliverables.py::test_unit_tests_pass` +subprocesses `pytest tests/unit -q` and asserts the return code is zero. + +**Impact.** Every unit-test failure is reported twice — once truthfully, once as a +meaningless `assert 1 == 0` in an unrelated file. With `--maxfail=1` in +`addopts`, whichever fires first can mask the real one. This is how F-01 +presented: the visible failure was a week-2 deliverables test, and the actual +cause was a missing `build` module three directories away. + +**Remedy.** Delete the test. If the intent was "week 2 deliverables still exist," +assert that directly — check the files and the entry points, not the exit code of +a nested test runner. + +--- + +## What was not searched + +Stated so that absence is not read as a null result: + +- **No fuzzing.** No fuzz target exists for the report reader, the LP input path, + or the canonicalizer. +- **No mutation testing.** Mutation score is unknown, not low. +- **No adversarial corpus.** There is no committed set of malformed reports that + must be rejected. The negative tests that exist are hand-written per-feature. +- **No dependency audit in this pass.** `pip-audit` runs in `security.yml`; its + current output was not collected here. +- **No review of `_legacy`, `exp/`, `theory/`, or the notebooks** beyond size. +- **No numerical audit of `redteam/dependence_search.py`** (709 statements, + 65.6% covered), which performs the search whose post-selection bias + `cliff.py` refuses to certify. It is the most likely location of a subtle + statistical defect and it was not examined in this pass. diff --git a/docs/upgrade/README.md b/docs/upgrade/README.md new file mode 100644 index 00000000..af86b713 --- /dev/null +++ b/docs/upgrade/README.md @@ -0,0 +1,137 @@ +# Epistemic Upgrade — cc-framework + +A deep excavation of `Cubits11/cc-framework` at commit `3e22c39`, measured +against the standard set by the flagship sibling repository +[`PSUCyberSecurityLab/ghost-ark`](https://github.com/PSUCyberSecurityLab/ghost-ark), +and a plan to close the gap. + +> **Status: plan and measurements.** The measurements are real and reproducible. +> The plan is not implemented. Nothing here claims cc-framework is safe, correct, +> or production-ready, and nothing here relaxes a boundary in +> [`NON_CLAIMS.md`](../research/NON_CLAIMS.md). + +--- + +## Read in this order + +| # | Document | What it is | +|---|---|---| +| 1 | [BASELINE_MEASUREMENTS.md](BASELINE_MEASUREMENTS.md) | Every number, with the command that produced it. Read first — everything else rests on it. | +| 2 | [FINDINGS_REGISTER.md](FINDINGS_REGISTER.md) | 20 findings, 5 at severity S1, each with reproduction and remedy. | +| 3 | [COMMITTEE_SCORECARD.md](COMMITTEE_SCORECARD.md) | 12 dimensions scored, each naming what would raise it and what would lower it. | +| 4 | [VERIFICATION_LADDER.md](VERIFICATION_LADDER.md) | The tiers this repository should have, and what each does *not* establish. | +| 5 | [DOWNSTREAM_CONTRACTS.md](DOWNSTREAM_CONTRACTS.md) | What ghost-ark, vinctura, and b2b-spatial need. Contracts C1–C7. | +| 6 | [EPISTEMIC_UPGRADE_PLAN.md](EPISTEMIC_UPGRADE_PLAN.md) | Ten workstreams, acceptance gates, sequencing. | + +--- + +## The short version + +**The mathematics is exact and nobody can use it.** + +The LP recovers Fréchet–Hoeffding to 1.11e-16 worst case. It reproduces the +correlation cliff in twenty seconds: eighteen guardrails at p = 0.1 each, +composed as a conjunction, bound to `[0, 0.1]` where independence predicts +1e-18. That result is the project's thesis and it works. + +Three separate projects reimplemented the calculus rather than depend on this +repository, and one wrote down why: the most discoverable composition entry point +imposes an ROC/detector ontology on events that are deterministic predicates. A +consumer read the source, found the mathematics right and the API wrong, and +wrote 214 lines of JavaScript instead. + +So the scorecard reads: + +| | At excavation | Now | +|---|---:|---:| +| Mathematical correctness | 8.9 | 8.9 | +| Claim discipline — prose | 8.8 | 8.8 | +| Consumability as a library | 2.6 | **6.8** | +| Claim discipline — enforcement | 2.1 † | **4.0** † | +| Cross-implementation agreement | 0.0 | **6.5** | + +† Corrected, not improved — see [F-12](FINDINGS_REGISTER.md#f-12). + +Three failures, stated once: + +1. **Reachability.** Correct and unreachable — wrong ontology, Python-only, no + conformance corpus. Everybody rewrites it. **Largely closed by + [W5](EPISTEMIC_UPGRADE_PLAN.md#w5-delivery-record).** +2. **Enforcement.** Prose 8.8, enforcement 4.0. Strict typing declared over 90 + files, enforced on 7. No forbidden-phrase scanner anywhere. The claim ledger + itself *is* enforced — an earlier draft said otherwise and was wrong. +3. **Unexamined foundations.** The canonicalization kernel every signed artifact + routes through silently merges Unicode-distinct keys, diverges from RFC 8785 + on five of six number forms, and had never been attacked. One hour of probing + found four defects. **Open — this is W3, the highest-severity work remaining.** + +--- + +## What has been built + +[W5](EPISTEMIC_UPGRADE_PLAN.md#w5-delivery-record) is delivered. Reproduce it: + +```bash +make test-compose # the ROC-free surface, 34 tests +make conformance # corpus current + an independent Node implementation agrees +make differential # 4,000 randomized cases through both implementations +make acceptance # an external consumer's PUBLISHED numbers, from this library +``` + +The acceptance gate is the one that matters. It is not "the API exists" — it is +a real consumer's published four-control result reproduced from `cc.compose`: +interval `[0, 0.01]`, independence baseline `1.2e-5`, understatement factor +`833×`, all three of their scenarios, and their sensitivity finding. Their 214 +lines of JavaScript could be deleted. + +The differential fuzzer found a real bug on its first run — `countermonotone` +with one event, where the Python raised `IndexError` and the Node silently +returned `NaN`. Both wrong, differently; the curated corpus had not thought to +ask. + +**The honest limit:** both implementations were authored in the same project, so +this is a differential-testing instrument, not an independent replication. The +one genuinely non-same-author check is a single external oracle on a single +scenario family. + +--- + +## Reproduce the S1 findings + +The canonicalization defects — [F-03](FINDINGS_REGISTER.md#f-03) through +[F-07](FINDINGS_REGISTER.md#f-07) — reproduce in one command: + +```bash +PYTHONPATH=src python scripts/canonicalization_probe.py +``` + +The probe exits non-zero while any class carries an `unintended-kernel` verdict, +so the finding is falsifiable: fix the canonicalizer and the probe goes green. + +Headline: + +```python +>>> canonical_json_bytes({"é": 1, "é": 2}) # U+00E9 key, then U+0065 U+0301 key +b'{"\xc3\xa9":2}' +``` + +Two distinct input keys. One output key. No exception. The receipt hash covers a +document that is not the document supplied. + +--- + +## What this excavation did not do + +Absence of a finding is not evidence of absence. + +- No fuzzing, no mutation testing, no dependency audit in this pass. +- `redteam/dependence_search.py` — 709 statements at 65.6% coverage, performing + the search whose post-selection bias `cliff.py` refuses to certify — was **not + examined**. It is the most likely home of a subtle statistical defect. +- No external reviewer was involved. Every finding here was produced by the same + kind of process that produced the code. +- One finding was **wrong on first publication**. F-12 claimed the + claim-boundary manifest was "validated by nothing"; the original grep covered + `.github/` and `Makefile` but not `tests/`, where the validator is in fact + called. It is corrected in place, with the correction visible rather than + quietly edited away, and the enforcement score was revised from 2.1 to 4.0. diff --git a/docs/upgrade/VERIFICATION_LADDER.md b/docs/upgrade/VERIFICATION_LADDER.md new file mode 100644 index 00000000..3a5957d9 --- /dev/null +++ b/docs/upgrade/VERIFICATION_LADDER.md @@ -0,0 +1,239 @@ +# Verification Ladder — cc-framework + +> Modelled on Ghost-Ark's repository verification ladder. Each tier states what +> passing establishes **and what it does not**. A tier that cannot name its +> non-claims is decoration. +> +> Tiers marked **[planned]** do not exist yet and are workstream deliverables. +> They are listed here so the target shape is legible, not to imply they run. + +--- + +## Why a ladder rather than "run the tests" + +`pytest` returning 0 currently means one thing: no assertion in this repository +disagreed with this repository. That is regression resistance. It is not +correctness, it is not agreement with any other implementation, and it is +certainly not safety. + +A ladder separates those. A reviewer who runs Tier 1 knows exactly which of them +they have obtained. + +--- + +## Tier 0 — Fast validation + +```bash +make fmt-check # ruff format --check +make lint # ruff check +make types # [planned] mypy at declared scope, against the ratchet +make claims # [planned] claim scanner + boundary-manifest validation +make docs-check # [planned] required docs exist and cross-references resolve +``` + +**Establishes.** The tree is formatted, lints clean, introduces no new type +errors beyond the recorded ratchet, contains no forbidden claim phrasing outside +the allowlist, and every claim-manifest row names a file and test that exist. + +**Does not establish.** That any computation is correct. Tier 0 is a text and +configuration gate. + +Runtime target: under 60 seconds. + +--- + +## Tier 1 — Kernel correctness + +```bash +make test-kernel # existing +make kernel-conformance # [planned] the published corpus, in-repo +make kernel-properties # [planned] Hypothesis over the marginal simplex +make kernel-duals # [planned] verify emitted dual certificates +``` + +**Establishes.** The LP recovers Fréchet–Hoeffding on the tested family; +infeasible constraint sets raise rather than returning a number; monotone +tightening holds; every conformance case in `conformance/cc-kernel-v1/` matches +its pinned expected interval; sampled marginals satisfy the declared invariants; +emitted dual certificates verify against their primal solutions. + +**Does not establish.** That the marginals fed to the kernel are valid, +representative, or measured. That an implementation in another language agrees +(that is Tier 3). That the result means anything about a deployed system. + +Current status: the first target exists and passes; the rest are W4/W5 +deliverables. + +--- + +## Tier 2 — Receipt and canonicalization integrity + +```bash +make canon-corpus # [planned] the adversarial canonicalization census +make receipt-verify-sample # [planned] verify a committed golden report +make receipt-verify-corpus # [planned] every malicious fixture must be REJECTED +make receipt-replay # [planned] rebuild artifacts, assert byte-identity +make test-reporting # existing +``` + +**Establishes.** Canonical bytes are deterministic and stable; every class in the +adversarial corpus produces the declared verdict; documents that must collapse do +and documents that must stay distinct do; malicious fixtures fail closed; +a committed capsule replays byte-identically. + +**Does not establish.** Semantic truth of the report. That the numbers inside are +correct. That the producer was honest — a receipt binds bytes, not intentions. + +This tier is where [F-03](FINDINGS_REGISTER.md#f-03) through +[F-07](FINDINGS_REGISTER.md#f-07) get caught, permanently. + +**The rule that makes this tier real:** *never weaken the corpus to make a test +pass.* A corpus class that starts failing is a regression in the kernel, not a +problem with the corpus. + +--- + +## Tier 3 — Cross-implementation agreement + +```bash +make conformance-publish # [planned] emit conformance/cc-kernel-v1/ +make differential-agreement # [planned] Python vs. second implementation +make differential-fuzz # [planned] randomized cases, both sides, compare +``` + +**Establishes.** Two independently written implementations, on the same inputs, +produce the same intervals to a declared tolerance — and where they diverge, the +divergence is recorded as a finding rather than reconciled by fiat. + +**Does not establish.** That either is correct. Two implementations can share a +misreading. Agreement is necessary, not sufficient. + +**Why this tier matters most.** Three implementations of this calculus exist +across the program ([F-09](FINDINGS_REGISTER.md#f-09)) and none has ever been +compared to another. Ghost-Ark treats breaking verifier agreement as a critical +architectural event. Here, agreement has never been established, so it cannot yet +be broken. + +Breaking agreement, once established, is a critical architectural event here too. + +--- + +## Tier 4 — Downstream contract conformance + +```bash +make ingest-corpus # [planned] discretization receipts: accept/reject +make guard-contract # [planned] cc-guard decision table vs. implementation +make compose-acceptance # [planned] reproduce Vinctura's published numbers +``` + +**Establishes.** A `ghost.discretization_rule_receipt.v1` missing lineage, or +carrying a comparator inconsistent with score polarity, or outside its validity +window, is **refused**; the guard decision table matches the Python +implementation case-for-case; the composition surface reproduces a real +consumer's published bounds. + +**Does not establish.** That the upstream discretization was appropriate, that the +threshold was well chosen, or that the score was calibrated. The contract checks +that evidence carries its lineage, not that the lineage is wise. + +--- + +## Tier 5 — Full baseline + +```bash +make test # the whole suite +make cov # coverage against the floor +make mutation # [planned] mutation score on kernel/ and reporting/ +``` + +**Establishes.** Repository consistency, regression resistance, coverage at or +above the recorded floor, and a published mutation score. + +**Does not establish.** Correctness. This is the tier most likely to be +misquoted; state its boundary whenever it is cited. + +--- + +## Tier 6 — Empirical lanes (scheduled, not per-commit) + +```bash +CC_RUN_EXPERIMENTS=1 CC_RUN_PERF=1 make test-empirical # [planned] +make atlas # [planned] Correlation Atlas run +``` + +**Establishes.** The experiment and performance lanes execute and report; measured +co-failure dependence on the declared cohort is published with its provenance and +its non-claims. + +**Does not establish.** That a measured regret on one cohort predicts another. +The Atlas is **descriptive**, never predictive. A number measured on one corpus +of jailbreaks says nothing about the next one. + +Runs on a schedule, publishes artifacts, and is allowed to be red without +blocking merges — a lane that must stay green will be quietly weakened until it +does. + +--- + +## Independent verification boundary + +Once Tier 3 exists, the following hold, borrowed directly from Ghost-Ark and +adapted: + +- A standalone verifier must not import `cc` internals. It re-derives from the + published corpus and the schema. +- Independent verification must recompute the interval, not accept a reported one. +- Conformance failures must be deterministic — the same input yields the same + verdict on every run. +- Malicious fixtures must fail closed, never "best effort." +- Unicode-sensitive canonicalization must remain stable across releases. +- **Never weaken verifier strictness to make tests pass.** + +--- + +## What no tier establishes + +No tier on this ladder, at any level, establishes that: + +- an AI system is safe, +- a guardrail is effective, +- a threshold is appropriate, +- a marginal is representative, +- a dependence structure is stable, +- a cohort generalizes, +- a deployment decision is correct, +- a compliance obligation is met. + +The ladder measures whether this repository does what it says. Whether what it +says is worth anything is a separate question, answered by +[COMMITTEE_SCORECARD.md](COMMITTEE_SCORECARD.md) and, ultimately, by someone +outside the project. + +--- + +## North star + +cc-framework succeeds when a hostile reviewer can say: + +> I do not trust the author. +> +> I do not trust the README. +> +> I do not trust the theorem ledger. +> +> But I can recompute the interval from the witness. +> +> I can check the dual certificate without re-solving the LP. +> +> I can run the conformance corpus against my own implementation. +> +> I can find the canonicalization collisions they already found, and no others. +> +> I can see which marginals were measured and which were assumed. +> +> I can read what they refused to claim, and check that the code refuses it too. +> +> And I can determine exactly what cc-framework does and does not establish +> without asking anyone. + +Everything else is secondary. diff --git a/evidence-cards/cards/enterprise.reference_v0_1.json b/evidence-cards/cards/enterprise.reference_v0_1.json new file mode 100644 index 00000000..f29d0f90 --- /dev/null +++ b/evidence-cards/cards/enterprise.reference_v0_1.json @@ -0,0 +1,68 @@ +{ + "artifacts": [ + { + "bytes": 7871, + "missing": false, + "path": "docs/validation_matrix.md", + "sha256": "0c4aeb300be94508644c7ca2f38b1cb6552ce4fb31c10cdea0769b809904c5d5" + }, + { + "bytes": 18487, + "missing": false, + "path": "docs/release/V0_3_RC1_CHECKLIST.md", + "sha256": "87cbba56c855c978f37d31c63f6354b025c5cb8a2e37bebc944e4a2f0794dc3f" + }, + { + "bytes": 16907, + "missing": false, + "path": "src/cc/enterprise/aws_reference.py", + "sha256": "f9138cd1d0cd1eaa2b5db0b1be5b95f0ff507a3081cec4a4aa7cad9bcdaaa9d4" + }, + { + "bytes": 5144, + "missing": false, + "path": "infra/lambda/verify_handler.py", + "sha256": "d1bbcefddda9c91578ac9d4d6fda9113da6fccbd965cd42c49b7592f602e8ee2" + }, + { + "bytes": 634, + "missing": false, + "path": "apps/dashboard/scripts/require-enterprise-bundle.mjs", + "sha256": "013caae404ca3b0ed3557dc61bb1887d4708f1f0d6803499702444f1f12414c5" + } + ], + "assumptions": [ + "moto emulates the AWS behaviours under test faithfully enough for the checks performed.", + "Emulated evidence is never live evidence, and no live-AWS claim rests on this lane." + ], + "card_id": "enterprise.reference_v0_1", + "claim": "Enterprise Reference v0.1 is an experimental evidence-integrity reference architecture with moto-backed smoke tests.", + "command": [ + "make enterprise-smoke", + "PYTHONPATH=src .venv/bin/pytest tests/integration/test_enterprise_aws_emulation.py tests/e2e/test_enterprise_smoke.py -q" + ], + "counterevidence": [], + "falsifier": "An enterprise emulation run that reports success while omitting a KMS verification or object-integrity check the reference claims to perform.", + "label_meanings": { + "evidence_state": "Produced against synthesized or emulated AWS, not a live account. Synthetic evidence is never live evidence.", + "publication_state": "Not yet reviewed for publication.", + "verdict": "The command has not been executed for this card. No result is claimed." + }, + "labels": { + "evidence_state": "aws-synth-only", + "publication_state": "draft", + "verdict": "not-run" + }, + "maturity": "C0/C2", + "non_claims": [ + "Not enterprise-ready.", + "Not live-AWS proof.", + "Not compliance certification.", + "Not deployment safety." + ], + "result_detail": null, + "retraction_reason": null, + "schema_version": "cc.evidence_card.v1", + "source_revision": "faf81103b9eceaa1a47daac8d6cd76d7cb382f53", + "supersedes": null +} diff --git a/evidence-cards/cards/evidence.non_claim_boundaries.json b/evidence-cards/cards/evidence.non_claim_boundaries.json new file mode 100644 index 00000000..acc8b736 --- /dev/null +++ b/evidence-cards/cards/evidence.non_claim_boundaries.json @@ -0,0 +1,72 @@ +{ + "artifacts": [ + { + "bytes": 19959, + "missing": false, + "path": "README.md", + "sha256": "f314d19df1890888a9e69131004e1ef6bfc7ab67f721e7211ad468c824159c36" + }, + { + "bytes": 7871, + "missing": false, + "path": "docs/validation_matrix.md", + "sha256": "0c4aeb300be94508644c7ca2f38b1cb6552ce4fb31c10cdea0769b809904c5d5" + }, + { + "bytes": 10127, + "missing": false, + "path": "docs/theory/theorem_ledger.md", + "sha256": "a8ed8bb949c97ad2a157654e7121f0d8150ce65f876a94ef6577b8b185a3ae20" + }, + { + "bytes": 4687, + "missing": false, + "path": "docs/briefs/rick_mergenthaler_ai_safety_disclosure_controls.md", + "sha256": "65240508d532ad7467aa4c2c14b67ae32a1acd68a3b21c5b6f37dd8984536b2f" + }, + { + "bytes": 2437, + "missing": false, + "path": "docs/briefs/three_minute_demo.md", + "sha256": "ef4ae474b3fe93aa1aa0b5d02a3ea58fda616da30b4a47c7a640a84fa115bd40" + }, + { + "bytes": 7225, + "missing": false, + "path": "docs/research/NON_CLAIMS.md", + "sha256": "8ba6e6a27c445a282b07d1ba704544382570b2c7285f4f73238dec7658428627" + } + ], + "assumptions": [ + "The producer declares evidence roles honestly; the verifier checks structure, not sincerity.", + "The v0 non-claims engine detects the intended substance rather than only exact strings." + ], + "card_id": "evidence.non_claim_boundaries", + "claim": "Documentation and reporting surfaces explicitly separate what each validation result supports from what it does not support.", + "command": [ + ".venv/bin/mkdocs build --strict", + "PYTHONPATH=src .venv/bin/python -m pytest -q tests/unit/evidence", + "PYTHONPATH=src .venv/bin/python -m pytest -q tests/integration/test_claim_governance_capsule.py" + ], + "counterevidence": [], + "falsifier": "An evidence-bound claim package that passes verify-claim-governance while omitting a non-claim its declared evidence roles require.", + "label_meanings": { + "evidence_state": "Produced on a local machine. Says nothing about cloud or deployed behaviour.", + "publication_state": "Not yet reviewed for publication.", + "verdict": "The command has not been executed for this card. No result is claimed." + }, + "labels": { + "evidence_state": "local-only", + "publication_state": "draft", + "verdict": "not-run" + }, + "maturity": "C4", + "non_claims": [ + "Does not prevent bad-faith actors from making misleading claims outside the framework." + ], + "result_detail": null, + "retraction_reason": null, + "schema_version": "cc.evidence_card.v1", + "source_revision": "faf81103b9eceaa1a47daac8d6cd76d7cb382f53", + "supersedes": null +} diff --git a/evidence-cards/cards/kernel.finite_sample_outer_interval.json b/evidence-cards/cards/kernel.finite_sample_outer_interval.json new file mode 100644 index 00000000..f15d78dc --- /dev/null +++ b/evidence-cards/cards/kernel.finite_sample_outer_interval.json @@ -0,0 +1,63 @@ +{ + "artifacts": [ + { + "bytes": 4317, + "missing": false, + "path": "docs/theory/finite_sample_identification.md", + "sha256": "a7228c069ed5cc20677f9b06fadf81112fc9f604e310cbe6fd080b522c9709c0" + }, + { + "bytes": 10127, + "missing": false, + "path": "docs/theory/theorem_ledger.md", + "sha256": "a8ed8bb949c97ad2a157654e7121f0d8150ce65f876a94ef6577b8b185a3ae20" + }, + { + "bytes": 20804, + "missing": false, + "path": "src/cc/kernel/sample_complexity.py", + "sha256": "aaef21b3d037a4e23cd067bdf2585227899feefe05634d9b7cd07a03223295f6" + }, + { + "bytes": 33874, + "missing": false, + "path": "src/cc/kernel/sensitivity.py", + "sha256": "6ff24bf54969a046c774c8b3ab251de31fbb4c0d505db88a9b4b283aea562422" + } + ], + "assumptions": [ + "Labels and the query are fixed before the data are seen.", + "Counts are iid Bernoulli draws from the target population.", + "Moment coverage holds simultaneously across constraints.", + "The exact assumptions supplied to the LP are true of that population." + ], + "card_id": "kernel.finite_sample_outer_interval", + "claim": "Under stated iid Bernoulli sampling assumptions and simultaneous moment coverage, count-derived LP intervals form an outer confidence interval for the target query.", + "command": [ + "pytest tests/unit/kernel/test_finite_sample_constraints.py", + "pytest tests/unit/kernel/test_sample_complexity.py", + "make test-kernel" + ], + "counterevidence": [], + "falsifier": "A simulation under the stated iid Bernoulli sampling in which count-derived intervals cover the true query value at a rate materially below the nominal level.", + "label_meanings": { + "evidence_state": "Produced on a local machine. Says nothing about cloud or deployed behaviour.", + "publication_state": "Not yet reviewed for publication.", + "verdict": "The command has not been executed for this card. No result is claimed." + }, + "labels": { + "evidence_state": "local-only", + "publication_state": "draft", + "verdict": "not-run" + }, + "maturity": "C3", + "non_claims": [ + "Not valid after uncorrected adaptive target selection.", + "Not a deployment certificate." + ], + "result_detail": null, + "retraction_reason": null, + "schema_version": "cc.evidence_card.v1", + "source_revision": "faf81103b9eceaa1a47daac8d6cd76d7cb382f53", + "supersedes": null +} diff --git a/evidence-cards/cards/kernel.frechet_bounds.json b/evidence-cards/cards/kernel.frechet_bounds.json new file mode 100644 index 00000000..080eb4c8 --- /dev/null +++ b/evidence-cards/cards/kernel.frechet_bounds.json @@ -0,0 +1,63 @@ +{ + "artifacts": [ + { + "bytes": 19959, + "missing": false, + "path": "README.md", + "sha256": "f314d19df1890888a9e69131004e1ef6bfc7ab67f721e7211ad468c824159c36" + }, + { + "bytes": 10127, + "missing": false, + "path": "docs/theory/theorem_ledger.md", + "sha256": "a8ed8bb949c97ad2a157654e7121f0d8150ce65f876a94ef6577b8b185a3ae20" + }, + { + "bytes": 21621, + "missing": false, + "path": "src/cc/kernel/frechet_classes.py", + "sha256": "60109fb97bc22bc732b1aa7fe660b5e4c4d03c5d1f2e13e4b2e54b1e032d1c58" + }, + { + "bytes": 33874, + "missing": false, + "path": "src/cc/kernel/sensitivity.py", + "sha256": "6ff24bf54969a046c774c8b3ab251de31fbb4c0d505db88a9b4b283aea562422" + } + ], + "assumptions": [ + "Exact singleton marginals are supplied; they are not estimated here.", + "No side constraints beyond the marginals.", + "Finite binary atom space with deterministic atom ordering.", + "IEEE-754 double arithmetic; agreement is asserted only to the declared tolerance." + ], + "card_id": "kernel.frechet_bounds", + "claim": "The strict kernel computes classical Fréchet-Hoeffding bounds for finite binary guardrail failure composition under exact singleton marginals and no additional side constraints.", + "command": [ + "pytest tests/unit/kernel/test_classical_frechet_special_cases.py", + "make test-kernel" + ], + "counterevidence": [], + "falsifier": "A marginal configuration in which identified_region returns an interval differing from max(0, sum(p) - (m-1)) and min(p) by more than the declared solver tolerance.", + "label_meanings": { + "evidence_state": "Produced on a local machine. Says nothing about cloud or deployed behaviour.", + "publication_state": "Not yet reviewed for publication.", + "verdict": "The command has not been executed for this card. No result is claimed." + }, + "labels": { + "evidence_state": "local-only", + "publication_state": "draft", + "verdict": "not-run" + }, + "maturity": "C3", + "non_claims": [ + "Does not assume independence.", + "Does not prove deployment safety.", + "Does not prove marginals are representative." + ], + "result_detail": null, + "retraction_reason": null, + "schema_version": "cc.evidence_card.v1", + "source_revision": "faf81103b9eceaa1a47daac8d6cd76d7cb382f53", + "supersedes": null +} diff --git a/evidence-cards/cards/kernel.identified_interval.json b/evidence-cards/cards/kernel.identified_interval.json new file mode 100644 index 00000000..89874f16 --- /dev/null +++ b/evidence-cards/cards/kernel.identified_interval.json @@ -0,0 +1,57 @@ +{ + "artifacts": [ + { + "bytes": 10127, + "missing": false, + "path": "docs/theory/theorem_ledger.md", + "sha256": "a8ed8bb949c97ad2a157654e7121f0d8150ce65f876a94ef6577b8b185a3ae20" + }, + { + "bytes": 33874, + "missing": false, + "path": "src/cc/kernel/sensitivity.py", + "sha256": "6ff24bf54969a046c774c8b3ab251de31fbb4c0d505db88a9b4b283aea562422" + }, + { + "bytes": 7346, + "missing": false, + "path": "docs/api.md", + "sha256": "34db452fa048483afc06caaf7806703b53ea984e6625d681a31b35db11d71c33" + } + ], + "assumptions": [ + "The constraint set defines a nonempty feasible polytope.", + "Constraints and the query are linear in the atom probabilities.", + "The LP solver reports optimal status rather than a tolerance-limited stop." + ], + "card_id": "kernel.identified_interval", + "claim": "The finite atom LP computes sharp lower and upper values for declared linear composition queries over feasible finite binary atom distributions.", + "command": [ + "pytest tests/unit/kernel/test_sensitivity.py", + "pytest tests/unit/kernel/test_monotonic_tightening.py", + "make test-kernel" + ], + "counterevidence": [], + "falsifier": "A feasible atom distribution whose query value falls outside the reported interval, or a value strictly inside the interval attained by no feasible distribution.", + "label_meanings": { + "evidence_state": "Produced on a local machine. Says nothing about cloud or deployed behaviour.", + "publication_state": "Not yet reviewed for publication.", + "verdict": "The command has not been executed for this card. No result is claimed." + }, + "labels": { + "evidence_state": "local-only", + "publication_state": "draft", + "verdict": "not-run" + }, + "maturity": "C3", + "non_claims": [ + "Does not validate upstream data collection.", + "Does not validate semantic safety.", + "Does not establish representativeness." + ], + "result_detail": null, + "retraction_reason": null, + "schema_version": "cc.evidence_card.v1", + "source_revision": "faf81103b9eceaa1a47daac8d6cd76d7cb382f53", + "supersedes": null +} diff --git a/evidence-cards/cards/open_core.strategy.json b/evidence-cards/cards/open_core.strategy.json new file mode 100644 index 00000000..5f3743f1 --- /dev/null +++ b/evidence-cards/cards/open_core.strategy.json @@ -0,0 +1,47 @@ +{ + "artifacts": [ + { + "bytes": 2303, + "missing": false, + "path": "docs/product/OPEN_CORE_STRATEGY.md", + "sha256": "5aafa0ce83ebd3eba11dad99326efaa8516b6fcbd51f62304f3bceee25622baa" + }, + { + "bytes": 19959, + "missing": false, + "path": "README.md", + "sha256": "f314d19df1890888a9e69131004e1ef6bfc7ab67f721e7211ad468c824159c36" + } + ], + "assumptions": [ + "The wheel built in CI is the artifact the strategy describes.", + "Classification in docs/api.md is kept current by the packaging boundary test." + ], + "card_id": "open_core.strategy", + "claim": "The public research core is intended to remain inspectable for credibility and reproducibility, while possible private/commercial layers may operationalize the primitives.", + "command": [ + ".venv/bin/mkdocs build --strict" + ], + "counterevidence": [], + "falsifier": "A module the strategy document places in the open core that is absent from the published wheel, or an importable wheel surface classified nowhere in docs/api.md.", + "label_meanings": { + "evidence_state": "Produced on a local machine. Says nothing about cloud or deployed behaviour.", + "publication_state": "Not yet reviewed for publication.", + "verdict": "The command has not been executed for this card. No result is claimed." + }, + "labels": { + "evidence_state": "local-only", + "publication_state": "draft", + "verdict": "not-run" + }, + "maturity": "C0", + "non_claims": [ + "Does not make the project a product today.", + "Does not claim enterprise readiness." + ], + "result_detail": null, + "retraction_reason": null, + "schema_version": "cc.evidence_card.v1", + "source_revision": "faf81103b9eceaa1a47daac8d6cd76d7cb382f53", + "supersedes": null +} diff --git a/evidence-cards/cards/release.paper_core_v0_3_rc1.json b/evidence-cards/cards/release.paper_core_v0_3_rc1.json new file mode 100644 index 00000000..fcee24a2 --- /dev/null +++ b/evidence-cards/cards/release.paper_core_v0_3_rc1.json @@ -0,0 +1,68 @@ +{ + "artifacts": [ + { + "bytes": 18487, + "missing": false, + "path": "docs/release/V0_3_RC1_CHECKLIST.md", + "sha256": "87cbba56c855c978f37d31c63f6354b025c5cb8a2e37bebc944e4a2f0794dc3f" + }, + { + "bytes": 7871, + "missing": false, + "path": "docs/validation_matrix.md", + "sha256": "0c4aeb300be94508644c7ca2f38b1cb6552ce4fb31c10cdea0769b809904c5d5" + }, + { + "bytes": 7346, + "missing": false, + "path": "docs/api.md", + "sha256": "34db452fa048483afc06caaf7806703b53ea984e6625d681a31b35db11d71c33" + }, + { + "bytes": 8198, + "missing": false, + "path": "docs/research/PAPER_CORE.md", + "sha256": "ca05c26005224d62a288d2c9f396fe08fc3fca3f48e5f8c10f45421898a61ff8" + }, + { + "bytes": 7225, + "missing": false, + "path": "docs/research/NON_CLAIMS.md", + "sha256": "8ba6e6a27c445a282b07d1ba704544382570b2c7285f4f73238dec7658428627" + } + ], + "assumptions": [ + "The pinned environment and seeds reproduce the recorded run.", + "Artifact determinism is a property of this host and toolchain, not of the mathematics." + ], + "card_id": "release.paper_core_v0_3_rc1", + "claim": "Paper Core v0.3 is release-candidate quality for the finite atom kernel, canonical metrics, endpoint witnesses, deterministic paper artifacts, and documentation spine.", + "command": [ + "make test-kernel", + "make test-release", + "make docs", + "make paper-smoke", + "make verify-paper-artifacts" + ], + "counterevidence": [], + "falsifier": "A paper artifact that reproduce_paper.py regenerates with different bytes, or a theorem-ledger row whose named test does not exist or does not exercise the stated implementation.", + "label_meanings": { + "evidence_state": "Produced on a local machine. Says nothing about cloud or deployed behaviour.", + "publication_state": "Not yet reviewed for publication.", + "verdict": "The command has not been executed for this card. No result is claimed." + }, + "labels": { + "evidence_state": "local-only", + "publication_state": "draft", + "verdict": "not-run" + }, + "maturity": "C0/C3", + "non_claims": [ + "Does not promote enterprise, dashboard, vendor, cloud, or experimental lanes into paper core." + ], + "result_detail": null, + "retraction_reason": null, + "schema_version": "cc.evidence_card.v1", + "source_revision": "faf81103b9eceaa1a47daac8d6cd76d7cb382f53", + "supersedes": null +} diff --git a/evidence-cards/cards/reporting.receipt_integrity.json b/evidence-cards/cards/reporting.receipt_integrity.json new file mode 100644 index 00000000..0c52ee77 --- /dev/null +++ b/evidence-cards/cards/reporting.receipt_integrity.json @@ -0,0 +1,77 @@ +{ + "artifacts": [ + { + "bytes": 10127, + "missing": false, + "path": "docs/theory/theorem_ledger.md", + "sha256": "a8ed8bb949c97ad2a157654e7121f0d8150ce65f876a94ef6577b8b185a3ae20" + }, + { + "bytes": 7871, + "missing": false, + "path": "docs/validation_matrix.md", + "sha256": "0c4aeb300be94508644c7ca2f38b1cb6552ce4fb31c10cdea0769b809904c5d5" + }, + { + "bytes": 17592, + "missing": false, + "path": "src/cc/reporting/canonical.py", + "sha256": "64e9c1f1efaa1b53fdfeec8c26a070089dc36e46c82f2e13ce1404d9088288c4" + }, + { + "bytes": 29932, + "missing": false, + "path": "src/cc/reporting/report.py", + "sha256": "31c3e6f192b671a40e272e95c5177dc6c36f69bfb428016dca1ccfa958ab7f7d" + }, + { + "bytes": 19054, + "missing": false, + "path": "src/cc/evidence/merkle_log.py", + "sha256": "66ee5c8b9931f64b3a1e05e71375ca73f315252de280780fd7b4ad45aeb8e58a" + }, + { + "bytes": 10654, + "missing": false, + "path": "src/cc/evidence/anchoring.py", + "sha256": "c38d55e7baafd860bc46bd05e52b08f2de8cdd6cec1f64e2722be24d92e48d8c" + } + ], + "assumptions": [ + "SHA-256 remains collision resistant.", + "The canonicalization profile named on the receipt is the one used to verify it.", + "Receipt-covered JSON is read with strict duplicate-key rejection." + ], + "card_id": "reporting.receipt_integrity", + "claim": "Receipt/hash verification can provide tamper evidence for recorded bytes under canonical serialization and verification rules.", + "command": [ + "pytest tests/unit/reporting/test_reporting.py", + "pytest tests/unit/evidence/test_transparency_log_adversarial.py", + "PYTHONPATH=src .venv/bin/python -m pytest -q tests/unit/evidence tests/unit/reporting" + ], + "counterevidence": [], + "falsifier": "Two byte-distinct payloads that a consumer needs distinguished receiving the same canonical hash, or a payload whose recomputed hash differs from its receipt while its bytes are unchanged.", + "label_meanings": { + "evidence_state": "Produced on a local machine. Says nothing about cloud or deployed behaviour.", + "publication_state": "Not yet reviewed for publication.", + "verdict": "The command has not been executed for this card. No result is claimed." + }, + "labels": { + "evidence_state": "local-only", + "publication_state": "draft", + "verdict": "not-run" + }, + "maturity": "C2", + "non_claims": [ + "Does not prove statistical validity.", + "Does not prove source data truth.", + "Does not prove representativeness.", + "Does not prove compliance.", + "Does not prove safety." + ], + "result_detail": null, + "retraction_reason": null, + "schema_version": "cc.evidence_card.v1", + "source_revision": "faf81103b9eceaa1a47daac8d6cd76d7cb382f53", + "supersedes": null +} diff --git a/evidence-cards/site-evidence-manifest.v1.json b/evidence-cards/site-evidence-manifest.v1.json new file mode 100644 index 00000000..93b36bf2 --- /dev/null +++ b/evidence-cards/site-evidence-manifest.v1.json @@ -0,0 +1,569 @@ +{ + "card_count": 8, + "cards": [ + { + "artifacts": [ + { + "bytes": 19959, + "missing": false, + "path": "README.md", + "sha256": "f314d19df1890888a9e69131004e1ef6bfc7ab67f721e7211ad468c824159c36" + }, + { + "bytes": 10127, + "missing": false, + "path": "docs/theory/theorem_ledger.md", + "sha256": "a8ed8bb949c97ad2a157654e7121f0d8150ce65f876a94ef6577b8b185a3ae20" + }, + { + "bytes": 21621, + "missing": false, + "path": "src/cc/kernel/frechet_classes.py", + "sha256": "60109fb97bc22bc732b1aa7fe660b5e4c4d03c5d1f2e13e4b2e54b1e032d1c58" + }, + { + "bytes": 33874, + "missing": false, + "path": "src/cc/kernel/sensitivity.py", + "sha256": "6ff24bf54969a046c774c8b3ab251de31fbb4c0d505db88a9b4b283aea562422" + } + ], + "assumptions": [ + "Exact singleton marginals are supplied; they are not estimated here.", + "No side constraints beyond the marginals.", + "Finite binary atom space with deterministic atom ordering.", + "IEEE-754 double arithmetic; agreement is asserted only to the declared tolerance." + ], + "card_id": "kernel.frechet_bounds", + "claim": "The strict kernel computes classical Fréchet-Hoeffding bounds for finite binary guardrail failure composition under exact singleton marginals and no additional side constraints.", + "command": [ + "pytest tests/unit/kernel/test_classical_frechet_special_cases.py", + "make test-kernel" + ], + "counterevidence": [], + "falsifier": "A marginal configuration in which identified_region returns an interval differing from max(0, sum(p) - (m-1)) and min(p) by more than the declared solver tolerance.", + "label_meanings": { + "evidence_state": "Produced on a local machine. Says nothing about cloud or deployed behaviour.", + "publication_state": "Not yet reviewed for publication.", + "verdict": "The command has not been executed for this card. No result is claimed." + }, + "labels": { + "evidence_state": "local-only", + "publication_state": "draft", + "verdict": "not-run" + }, + "maturity": "C3", + "non_claims": [ + "Does not assume independence.", + "Does not prove deployment safety.", + "Does not prove marginals are representative." + ], + "result_detail": null, + "retraction_reason": null, + "schema_version": "cc.evidence_card.v1", + "source_revision": "faf81103b9eceaa1a47daac8d6cd76d7cb382f53", + "supersedes": null + }, + { + "artifacts": [ + { + "bytes": 10127, + "missing": false, + "path": "docs/theory/theorem_ledger.md", + "sha256": "a8ed8bb949c97ad2a157654e7121f0d8150ce65f876a94ef6577b8b185a3ae20" + }, + { + "bytes": 33874, + "missing": false, + "path": "src/cc/kernel/sensitivity.py", + "sha256": "6ff24bf54969a046c774c8b3ab251de31fbb4c0d505db88a9b4b283aea562422" + }, + { + "bytes": 7346, + "missing": false, + "path": "docs/api.md", + "sha256": "34db452fa048483afc06caaf7806703b53ea984e6625d681a31b35db11d71c33" + } + ], + "assumptions": [ + "The constraint set defines a nonempty feasible polytope.", + "Constraints and the query are linear in the atom probabilities.", + "The LP solver reports optimal status rather than a tolerance-limited stop." + ], + "card_id": "kernel.identified_interval", + "claim": "The finite atom LP computes sharp lower and upper values for declared linear composition queries over feasible finite binary atom distributions.", + "command": [ + "pytest tests/unit/kernel/test_sensitivity.py", + "pytest tests/unit/kernel/test_monotonic_tightening.py", + "make test-kernel" + ], + "counterevidence": [], + "falsifier": "A feasible atom distribution whose query value falls outside the reported interval, or a value strictly inside the interval attained by no feasible distribution.", + "label_meanings": { + "evidence_state": "Produced on a local machine. Says nothing about cloud or deployed behaviour.", + "publication_state": "Not yet reviewed for publication.", + "verdict": "The command has not been executed for this card. No result is claimed." + }, + "labels": { + "evidence_state": "local-only", + "publication_state": "draft", + "verdict": "not-run" + }, + "maturity": "C3", + "non_claims": [ + "Does not validate upstream data collection.", + "Does not validate semantic safety.", + "Does not establish representativeness." + ], + "result_detail": null, + "retraction_reason": null, + "schema_version": "cc.evidence_card.v1", + "source_revision": "faf81103b9eceaa1a47daac8d6cd76d7cb382f53", + "supersedes": null + }, + { + "artifacts": [ + { + "bytes": 4317, + "missing": false, + "path": "docs/theory/finite_sample_identification.md", + "sha256": "a7228c069ed5cc20677f9b06fadf81112fc9f604e310cbe6fd080b522c9709c0" + }, + { + "bytes": 10127, + "missing": false, + "path": "docs/theory/theorem_ledger.md", + "sha256": "a8ed8bb949c97ad2a157654e7121f0d8150ce65f876a94ef6577b8b185a3ae20" + }, + { + "bytes": 20804, + "missing": false, + "path": "src/cc/kernel/sample_complexity.py", + "sha256": "aaef21b3d037a4e23cd067bdf2585227899feefe05634d9b7cd07a03223295f6" + }, + { + "bytes": 33874, + "missing": false, + "path": "src/cc/kernel/sensitivity.py", + "sha256": "6ff24bf54969a046c774c8b3ab251de31fbb4c0d505db88a9b4b283aea562422" + } + ], + "assumptions": [ + "Labels and the query are fixed before the data are seen.", + "Counts are iid Bernoulli draws from the target population.", + "Moment coverage holds simultaneously across constraints.", + "The exact assumptions supplied to the LP are true of that population." + ], + "card_id": "kernel.finite_sample_outer_interval", + "claim": "Under stated iid Bernoulli sampling assumptions and simultaneous moment coverage, count-derived LP intervals form an outer confidence interval for the target query.", + "command": [ + "pytest tests/unit/kernel/test_finite_sample_constraints.py", + "pytest tests/unit/kernel/test_sample_complexity.py", + "make test-kernel" + ], + "counterevidence": [], + "falsifier": "A simulation under the stated iid Bernoulli sampling in which count-derived intervals cover the true query value at a rate materially below the nominal level.", + "label_meanings": { + "evidence_state": "Produced on a local machine. Says nothing about cloud or deployed behaviour.", + "publication_state": "Not yet reviewed for publication.", + "verdict": "The command has not been executed for this card. No result is claimed." + }, + "labels": { + "evidence_state": "local-only", + "publication_state": "draft", + "verdict": "not-run" + }, + "maturity": "C3", + "non_claims": [ + "Not valid after uncorrected adaptive target selection.", + "Not a deployment certificate." + ], + "result_detail": null, + "retraction_reason": null, + "schema_version": "cc.evidence_card.v1", + "source_revision": "faf81103b9eceaa1a47daac8d6cd76d7cb382f53", + "supersedes": null + }, + { + "artifacts": [ + { + "bytes": 10127, + "missing": false, + "path": "docs/theory/theorem_ledger.md", + "sha256": "a8ed8bb949c97ad2a157654e7121f0d8150ce65f876a94ef6577b8b185a3ae20" + }, + { + "bytes": 7871, + "missing": false, + "path": "docs/validation_matrix.md", + "sha256": "0c4aeb300be94508644c7ca2f38b1cb6552ce4fb31c10cdea0769b809904c5d5" + }, + { + "bytes": 17592, + "missing": false, + "path": "src/cc/reporting/canonical.py", + "sha256": "64e9c1f1efaa1b53fdfeec8c26a070089dc36e46c82f2e13ce1404d9088288c4" + }, + { + "bytes": 29932, + "missing": false, + "path": "src/cc/reporting/report.py", + "sha256": "31c3e6f192b671a40e272e95c5177dc6c36f69bfb428016dca1ccfa958ab7f7d" + }, + { + "bytes": 19054, + "missing": false, + "path": "src/cc/evidence/merkle_log.py", + "sha256": "66ee5c8b9931f64b3a1e05e71375ca73f315252de280780fd7b4ad45aeb8e58a" + }, + { + "bytes": 10654, + "missing": false, + "path": "src/cc/evidence/anchoring.py", + "sha256": "c38d55e7baafd860bc46bd05e52b08f2de8cdd6cec1f64e2722be24d92e48d8c" + } + ], + "assumptions": [ + "SHA-256 remains collision resistant.", + "The canonicalization profile named on the receipt is the one used to verify it.", + "Receipt-covered JSON is read with strict duplicate-key rejection." + ], + "card_id": "reporting.receipt_integrity", + "claim": "Receipt/hash verification can provide tamper evidence for recorded bytes under canonical serialization and verification rules.", + "command": [ + "pytest tests/unit/reporting/test_reporting.py", + "pytest tests/unit/evidence/test_transparency_log_adversarial.py", + "PYTHONPATH=src .venv/bin/python -m pytest -q tests/unit/evidence tests/unit/reporting" + ], + "counterevidence": [], + "falsifier": "Two byte-distinct payloads that a consumer needs distinguished receiving the same canonical hash, or a payload whose recomputed hash differs from its receipt while its bytes are unchanged.", + "label_meanings": { + "evidence_state": "Produced on a local machine. Says nothing about cloud or deployed behaviour.", + "publication_state": "Not yet reviewed for publication.", + "verdict": "The command has not been executed for this card. No result is claimed." + }, + "labels": { + "evidence_state": "local-only", + "publication_state": "draft", + "verdict": "not-run" + }, + "maturity": "C2", + "non_claims": [ + "Does not prove statistical validity.", + "Does not prove source data truth.", + "Does not prove representativeness.", + "Does not prove compliance.", + "Does not prove safety." + ], + "result_detail": null, + "retraction_reason": null, + "schema_version": "cc.evidence_card.v1", + "source_revision": "faf81103b9eceaa1a47daac8d6cd76d7cb382f53", + "supersedes": null + }, + { + "artifacts": [ + { + "bytes": 19959, + "missing": false, + "path": "README.md", + "sha256": "f314d19df1890888a9e69131004e1ef6bfc7ab67f721e7211ad468c824159c36" + }, + { + "bytes": 7871, + "missing": false, + "path": "docs/validation_matrix.md", + "sha256": "0c4aeb300be94508644c7ca2f38b1cb6552ce4fb31c10cdea0769b809904c5d5" + }, + { + "bytes": 10127, + "missing": false, + "path": "docs/theory/theorem_ledger.md", + "sha256": "a8ed8bb949c97ad2a157654e7121f0d8150ce65f876a94ef6577b8b185a3ae20" + }, + { + "bytes": 4687, + "missing": false, + "path": "docs/briefs/rick_mergenthaler_ai_safety_disclosure_controls.md", + "sha256": "65240508d532ad7467aa4c2c14b67ae32a1acd68a3b21c5b6f37dd8984536b2f" + }, + { + "bytes": 2437, + "missing": false, + "path": "docs/briefs/three_minute_demo.md", + "sha256": "ef4ae474b3fe93aa1aa0b5d02a3ea58fda616da30b4a47c7a640a84fa115bd40" + }, + { + "bytes": 7225, + "missing": false, + "path": "docs/research/NON_CLAIMS.md", + "sha256": "8ba6e6a27c445a282b07d1ba704544382570b2c7285f4f73238dec7658428627" + } + ], + "assumptions": [ + "The producer declares evidence roles honestly; the verifier checks structure, not sincerity.", + "The v0 non-claims engine detects the intended substance rather than only exact strings." + ], + "card_id": "evidence.non_claim_boundaries", + "claim": "Documentation and reporting surfaces explicitly separate what each validation result supports from what it does not support.", + "command": [ + ".venv/bin/mkdocs build --strict", + "PYTHONPATH=src .venv/bin/python -m pytest -q tests/unit/evidence", + "PYTHONPATH=src .venv/bin/python -m pytest -q tests/integration/test_claim_governance_capsule.py" + ], + "counterevidence": [], + "falsifier": "An evidence-bound claim package that passes verify-claim-governance while omitting a non-claim its declared evidence roles require.", + "label_meanings": { + "evidence_state": "Produced on a local machine. Says nothing about cloud or deployed behaviour.", + "publication_state": "Not yet reviewed for publication.", + "verdict": "The command has not been executed for this card. No result is claimed." + }, + "labels": { + "evidence_state": "local-only", + "publication_state": "draft", + "verdict": "not-run" + }, + "maturity": "C4", + "non_claims": [ + "Does not prevent bad-faith actors from making misleading claims outside the framework." + ], + "result_detail": null, + "retraction_reason": null, + "schema_version": "cc.evidence_card.v1", + "source_revision": "faf81103b9eceaa1a47daac8d6cd76d7cb382f53", + "supersedes": null + }, + { + "artifacts": [ + { + "bytes": 18487, + "missing": false, + "path": "docs/release/V0_3_RC1_CHECKLIST.md", + "sha256": "87cbba56c855c978f37d31c63f6354b025c5cb8a2e37bebc944e4a2f0794dc3f" + }, + { + "bytes": 7871, + "missing": false, + "path": "docs/validation_matrix.md", + "sha256": "0c4aeb300be94508644c7ca2f38b1cb6552ce4fb31c10cdea0769b809904c5d5" + }, + { + "bytes": 7346, + "missing": false, + "path": "docs/api.md", + "sha256": "34db452fa048483afc06caaf7806703b53ea984e6625d681a31b35db11d71c33" + }, + { + "bytes": 8198, + "missing": false, + "path": "docs/research/PAPER_CORE.md", + "sha256": "ca05c26005224d62a288d2c9f396fe08fc3fca3f48e5f8c10f45421898a61ff8" + }, + { + "bytes": 7225, + "missing": false, + "path": "docs/research/NON_CLAIMS.md", + "sha256": "8ba6e6a27c445a282b07d1ba704544382570b2c7285f4f73238dec7658428627" + } + ], + "assumptions": [ + "The pinned environment and seeds reproduce the recorded run.", + "Artifact determinism is a property of this host and toolchain, not of the mathematics." + ], + "card_id": "release.paper_core_v0_3_rc1", + "claim": "Paper Core v0.3 is release-candidate quality for the finite atom kernel, canonical metrics, endpoint witnesses, deterministic paper artifacts, and documentation spine.", + "command": [ + "make test-kernel", + "make test-release", + "make docs", + "make paper-smoke", + "make verify-paper-artifacts" + ], + "counterevidence": [], + "falsifier": "A paper artifact that reproduce_paper.py regenerates with different bytes, or a theorem-ledger row whose named test does not exist or does not exercise the stated implementation.", + "label_meanings": { + "evidence_state": "Produced on a local machine. Says nothing about cloud or deployed behaviour.", + "publication_state": "Not yet reviewed for publication.", + "verdict": "The command has not been executed for this card. No result is claimed." + }, + "labels": { + "evidence_state": "local-only", + "publication_state": "draft", + "verdict": "not-run" + }, + "maturity": "C0/C3", + "non_claims": [ + "Does not promote enterprise, dashboard, vendor, cloud, or experimental lanes into paper core." + ], + "result_detail": null, + "retraction_reason": null, + "schema_version": "cc.evidence_card.v1", + "source_revision": "faf81103b9eceaa1a47daac8d6cd76d7cb382f53", + "supersedes": null + }, + { + "artifacts": [ + { + "bytes": 7871, + "missing": false, + "path": "docs/validation_matrix.md", + "sha256": "0c4aeb300be94508644c7ca2f38b1cb6552ce4fb31c10cdea0769b809904c5d5" + }, + { + "bytes": 18487, + "missing": false, + "path": "docs/release/V0_3_RC1_CHECKLIST.md", + "sha256": "87cbba56c855c978f37d31c63f6354b025c5cb8a2e37bebc944e4a2f0794dc3f" + }, + { + "bytes": 16907, + "missing": false, + "path": "src/cc/enterprise/aws_reference.py", + "sha256": "f9138cd1d0cd1eaa2b5db0b1be5b95f0ff507a3081cec4a4aa7cad9bcdaaa9d4" + }, + { + "bytes": 5144, + "missing": false, + "path": "infra/lambda/verify_handler.py", + "sha256": "d1bbcefddda9c91578ac9d4d6fda9113da6fccbd965cd42c49b7592f602e8ee2" + }, + { + "bytes": 634, + "missing": false, + "path": "apps/dashboard/scripts/require-enterprise-bundle.mjs", + "sha256": "013caae404ca3b0ed3557dc61bb1887d4708f1f0d6803499702444f1f12414c5" + } + ], + "assumptions": [ + "moto emulates the AWS behaviours under test faithfully enough for the checks performed.", + "Emulated evidence is never live evidence, and no live-AWS claim rests on this lane." + ], + "card_id": "enterprise.reference_v0_1", + "claim": "Enterprise Reference v0.1 is an experimental evidence-integrity reference architecture with moto-backed smoke tests.", + "command": [ + "make enterprise-smoke", + "PYTHONPATH=src .venv/bin/pytest tests/integration/test_enterprise_aws_emulation.py tests/e2e/test_enterprise_smoke.py -q" + ], + "counterevidence": [], + "falsifier": "An enterprise emulation run that reports success while omitting a KMS verification or object-integrity check the reference claims to perform.", + "label_meanings": { + "evidence_state": "Produced against synthesized or emulated AWS, not a live account. Synthetic evidence is never live evidence.", + "publication_state": "Not yet reviewed for publication.", + "verdict": "The command has not been executed for this card. No result is claimed." + }, + "labels": { + "evidence_state": "aws-synth-only", + "publication_state": "draft", + "verdict": "not-run" + }, + "maturity": "C0/C2", + "non_claims": [ + "Not enterprise-ready.", + "Not live-AWS proof.", + "Not compliance certification.", + "Not deployment safety." + ], + "result_detail": null, + "retraction_reason": null, + "schema_version": "cc.evidence_card.v1", + "source_revision": "faf81103b9eceaa1a47daac8d6cd76d7cb382f53", + "supersedes": null + }, + { + "artifacts": [ + { + "bytes": 2303, + "missing": false, + "path": "docs/product/OPEN_CORE_STRATEGY.md", + "sha256": "5aafa0ce83ebd3eba11dad99326efaa8516b6fcbd51f62304f3bceee25622baa" + }, + { + "bytes": 19959, + "missing": false, + "path": "README.md", + "sha256": "f314d19df1890888a9e69131004e1ef6bfc7ab67f721e7211ad468c824159c36" + } + ], + "assumptions": [ + "The wheel built in CI is the artifact the strategy describes.", + "Classification in docs/api.md is kept current by the packaging boundary test." + ], + "card_id": "open_core.strategy", + "claim": "The public research core is intended to remain inspectable for credibility and reproducibility, while possible private/commercial layers may operationalize the primitives.", + "command": [ + ".venv/bin/mkdocs build --strict" + ], + "counterevidence": [], + "falsifier": "A module the strategy document places in the open core that is absent from the published wheel, or an importable wheel surface classified nowhere in docs/api.md.", + "label_meanings": { + "evidence_state": "Produced on a local machine. Says nothing about cloud or deployed behaviour.", + "publication_state": "Not yet reviewed for publication.", + "verdict": "The command has not been executed for this card. No result is claimed." + }, + "labels": { + "evidence_state": "local-only", + "publication_state": "draft", + "verdict": "not-run" + }, + "maturity": "C0", + "non_claims": [ + "Does not make the project a product today.", + "Does not claim enterprise readiness." + ], + "result_detail": null, + "retraction_reason": null, + "schema_version": "cc.evidence_card.v1", + "source_revision": "faf81103b9eceaa1a47daac8d6cd76d7cb382f53", + "supersedes": null + } + ], + "counts_by_label": { + "evidence_state": { + "aws-live": 0, + "aws-synth-only": 1, + "illustrative": 0, + "local-only": 7 + }, + "publication_state": { + "draft": 8, + "released": 0, + "retracted": 0, + "superseded": 0 + }, + "verdict": { + "fail": 0, + "not-run": 8, + "pass": 0, + "unverifiable": 0 + } + }, + "generated_note": "Generated from docs/claims/claim_boundary_manifest.v0.1.json. Verdicts are 'not-run': no command was executed during generation.", + "label_meanings": { + "evidence_state": { + "aws-live": "Produced against a live AWS account, for the named run only.", + "aws-synth-only": "Produced against synthesized or emulated AWS, not a live account. Synthetic evidence is never live evidence.", + "illustrative": "A worked example or figure. Not measured evidence, and must never be cited as a result.", + "local-only": "Produced on a local machine. Says nothing about cloud or deployed behaviour." + }, + "publication_state": { + "draft": "Not yet reviewed for publication.", + "released": "Published. Says nothing about whether the claim is true, only that it was reviewed for release.", + "retracted": "Withdrawn. The reason is on the card; the card is not deleted.", + "superseded": "A later card replaces this one. Retained so the record is not rewritten." + }, + "verdict": { + "fail": "The command ran and its checks did not succeed.", + "not-run": "The command has not been executed for this card. No result is claimed.", + "pass": "The command ran and its checks succeeded. Scope is whatever the command covers -- no wider.", + "unverifiable": "The check could not be performed. This is not a pass and not a fail." + } + }, + "non_claims": [ + "This manifest lists claims and the commands that test them. It does not establish that any claim is true.", + "The three labels are orthogonal. A passing verdict on local-only evidence says nothing about deployed behaviour, and a released publication state says only that a card was reviewed for release.", + "Counts are reported per label. There is no aggregate score, and any surface that computes one from this file is misusing it.", + "A card with verdict 'not-run' has no result. It must not be displayed as passing, pending, or healthy." + ], + "schema_version": "cc.site_evidence_manifest.v1", + "source_revision": "faf81103b9eceaa1a47daac8d6cd76d7cb382f53" +} diff --git a/examples/claim_governance_capsule/README.md b/examples/claim_governance_capsule/README.md index e72612ea..348f90b4 100644 --- a/examples/claim_governance_capsule/README.md +++ b/examples/claim_governance_capsule/README.md @@ -34,6 +34,28 @@ confirmatory-failure-matrix artifact because those verifier layers are already implemented in this repository. It does not emit a fake package/signature layer beyond the manifest. +## Calibration provenance + +`calibration.realized_fpr` has an explicit provenance record in the checked-in +input and generated `calibration.json`. It separates three questions that must +not collapse: what the field means, how the local value was obtained, and +whether an external record binds that declaration. The current fixture has +**unresolved** semantics, an **asserted** origin, and no external anchor. Its +numerical resemblance to `1/24` is not treated as a derivation from the 24-row +failure matrix. + +| Axis | Current value | What the builder checks | What it still does not establish | +| --- | --- | --- | --- | +| Semantics | `unresolved` | Requires a reason saying what has not been established. | That the field denotes any particular event or population. | +| Origin | `asserted` | Records the value without inferring a derivation. | That the number was measured, or that it relates to the matrix. | +| Origin (future) | `deterministic_derivation` | A named binary matrix column, source digest, exact numerator/denominator, decimal rendering, and scalar value must agree. | That the declared column is semantically a false-positive rate or that the matrix is representative. | +| External reference | `null` | A future `reference_only/v1` record must bind the exact value, semantics, and origin with an HTTPS URI, issuer, timestamp, and subject hash. The builder does not fetch it. | That the external party measured the value correctly. | + +The future origin and anchor paths are independent: an asserted value can be +referenced externally, and a derivation can be referenced externally. Neither +path upgrades this fixture by itself. This is the CH-002 boundary in +[`docs/research/epistemic-program/challenges.md`](../../docs/research/epistemic-program/challenges.md). + ## Run From the repository root: @@ -103,3 +125,5 @@ examples/claim_governance_capsule/reproduce.sh --update-expected operational safety. - The confirmatory protocol checks fixed-plan separation; it does not certify external validity or release readiness. +- A calibrated value marked `asserted` is a declared input, not a measurement + derived from the checked-in failure matrix. diff --git a/examples/claim_governance_capsule/build_capsule.py b/examples/claim_governance_capsule/build_capsule.py index 0029f0ff..72a3a7cb 100644 --- a/examples/claim_governance_capsule/build_capsule.py +++ b/examples/claim_governance_capsule/build_capsule.py @@ -20,6 +20,7 @@ from collections.abc import Mapping, Sequence from contextlib import suppress from datetime import datetime, timezone +from decimal import ROUND_HALF_EVEN, Decimal, InvalidOperation from pathlib import Path from typing import Any @@ -71,6 +72,13 @@ "extremal_upper.json": "extremal_scenario", } +CALIBRATION_PROVENANCE_SCHEMA = "cc.calibration.realized-fpr-provenance.v1" +CALIBRATION_ORIGIN_KINDS = frozenset({"asserted", "deterministic_derivation"}) +CALIBRATION_PROVENANCE_NON_CLAIM = ( + "Calibration provenance records how a value is described; it does not establish that the " + "value was measured, that labels are correct, or that the population is representative." +) + class CapsuleError(RuntimeError): """Raised when the capsule cannot be regenerated or verified.""" @@ -145,7 +153,12 @@ def build_capsule(capsule_dir: Path) -> None: ) write_json(outputs_dir / "bounds.json", bounds_payload) - calibration_payload = dict(config["calibration"]) + calibration_payload = build_calibration_payload( + calibration=config["calibration"], + rows=rows, + guardrails=guardrails, + matrix_sha256=input_hash, + ) write_json(outputs_dir / "calibration.json", calibration_payload) write_json( @@ -319,6 +332,359 @@ def build_confirmatory_failure_matrix_payload( } +def build_calibration_payload( + *, + calibration: Any, + rows: Sequence[Mapping[str, int | str]], + guardrails: tuple[str, ...], + matrix_sha256: str, +) -> dict[str, Any]: + """Emit calibration with independent semantic, origin, and anchor axes. + + The current fixture deliberately records unresolved semantics and an + asserted origin. A future fixture may opt into an allowlisted deterministic + derivation only after it declares a concrete event, denominator, and + population. An external anchor, if supplied, is a separate reference that + can accompany either origin; this local builder never calls it proof of a + measurement. + """ + + if not isinstance(calibration, Mapping): + raise CapsuleError("calibration must be an object") + + payload = dict(calibration) + realized_fpr = _decimal_probability(payload.get("realized_fpr"), "calibration.realized_fpr") + raw_provenance = payload.get("realized_fpr_provenance") + if not isinstance(raw_provenance, Mapping): + raise CapsuleError("calibration.realized_fpr_provenance must be an object") + + provenance = dict(raw_provenance) + _require_exact_keys( + provenance, + {"schema", "field", "semantics", "origin", "external_anchor"}, + "calibration.realized_fpr_provenance", + ) + if provenance["schema"] != CALIBRATION_PROVENANCE_SCHEMA: + raise CapsuleError( + f"calibration.realized_fpr_provenance.schema must be {CALIBRATION_PROVENANCE_SCHEMA}" + ) + if provenance["field"] != "realized_fpr": + raise CapsuleError("calibration provenance field must be realized_fpr") + + semantics = _validate_calibration_semantics(provenance["semantics"]) + origin, origin_verification = _validate_calibration_origin( + origin=provenance["origin"], + semantics=semantics, + realized_fpr=realized_fpr, + rows=rows, + guardrails=guardrails, + matrix_sha256=matrix_sha256, + ) + anchor_subject = _anchor_subject( + realized_fpr=realized_fpr, + semantics=semantics, + origin=origin, + ) + external_anchor, anchor_verification = _validate_external_anchor( + provenance["external_anchor"], + subject=anchor_subject, + ) + + payload["realized_fpr_provenance"] = { + "schema": CALIBRATION_PROVENANCE_SCHEMA, + "field": "realized_fpr", + "semantics": semantics, + "origin": origin, + "external_anchor": external_anchor, + "verification": { + "origin": origin_verification, + "external_anchor": anchor_verification, + }, + "non_claims": _provenance_non_claims( + semantics=semantics, + origin=origin, + external_anchor=external_anchor, + ), + } + return payload + + +def _validate_calibration_semantics(value: Any) -> dict[str, Any]: + if not isinstance(value, Mapping): + raise CapsuleError("calibration provenance.semantics must be an object") + semantics = dict(value) + status = semantics.get("status") + if status == "unresolved": + _require_exact_keys( + semantics, + {"status", "reason"}, + "unresolved calibration semantics", + ) + _require_nonblank(semantics["reason"], "calibration provenance.semantics.reason") + return semantics + if status == "defined": + _require_exact_keys( + semantics, + {"status", "numerator", "denominator", "population"}, + "defined calibration semantics", + ) + for field_name in ("numerator", "denominator", "population"): + _require_nonblank( + semantics[field_name], f"calibration provenance.semantics.{field_name}" + ) + return semantics + raise CapsuleError("calibration provenance.semantics.status must be unresolved or defined") + + +def _validate_calibration_origin( + *, + origin: Any, + semantics: Mapping[str, Any], + realized_fpr: Decimal, + rows: Sequence[Mapping[str, int | str]], + guardrails: tuple[str, ...], + matrix_sha256: str, +) -> tuple[dict[str, Any], dict[str, Any]]: + if not isinstance(origin, Mapping): + raise CapsuleError("calibration provenance.origin must be an object") + origin_payload = dict(origin) + kind = origin_payload.get("kind") + if kind not in CALIBRATION_ORIGIN_KINDS: + allowed = ", ".join(sorted(CALIBRATION_ORIGIN_KINDS)) + raise CapsuleError(f"calibration provenance origin kind must be one of: {allowed}") + if kind == "asserted": + _require_exact_keys(origin_payload, {"kind"}, "asserted calibration origin") + return ( + origin_payload, + { + "status": "asserted_not_derivable", + "reason": ( + "The capsule records this declared value without inferring an event, " + "denominator, or source population." + ), + }, + ) + + if semantics["status"] != "defined": + raise CapsuleError("deterministic calibration derivation requires defined semantics") + _require_exact_keys( + origin_payload, + {"kind", "algorithm", "source", "column", "inclusion_rule", "exact_result", "rendering"}, + "deterministic calibration origin", + ) + if origin_payload["algorithm"] != "binary_column_rate/v1": + raise CapsuleError( + "deterministic calibration origin algorithm must be binary_column_rate/v1" + ) + if origin_payload["inclusion_rule"] != "all_rows": + raise CapsuleError("deterministic calibration origin inclusion_rule must be all_rows") + + source = origin_payload["source"] + if not isinstance(source, Mapping): + raise CapsuleError("deterministic calibration origin.source must be an object") + source_payload = dict(source) + _require_exact_keys( + source_payload, + {"path", "sha256"}, + "deterministic calibration origin.source", + ) + if source_payload["path"] != "inputs/failure_matrix.csv": + raise CapsuleError("deterministic calibration derivation must use failure_matrix.csv") + if source_payload["sha256"] != matrix_sha256: + raise CapsuleError( + "deterministic calibration derivation source sha256 does not match matrix" + ) + + column = origin_payload["column"] + if not isinstance(column, str) or column not in guardrails: + raise CapsuleError("deterministic calibration origin column must name a guardrail") + + exact_result = origin_payload["exact_result"] + if not isinstance(exact_result, Mapping): + raise CapsuleError("deterministic calibration origin.exact_result must be an object") + exact_result_payload = dict(exact_result) + _require_exact_keys( + exact_result_payload, + {"numerator", "denominator"}, + "deterministic calibration origin.exact_result", + ) + if any( + isinstance(value, bool) or not isinstance(value, int) or value < 0 + for value in exact_result_payload.values() + ): + raise CapsuleError( + "deterministic calibration exact_result counts must be non-negative integers" + ) + + rendering = origin_payload["rendering"] + if not isinstance(rendering, Mapping): + raise CapsuleError("deterministic calibration origin.rendering must be an object") + rendering_payload = dict(rendering) + _require_exact_keys( + rendering_payload, + {"decimal_places", "rounding"}, + "deterministic calibration origin.rendering", + ) + if rendering_payload != {"decimal_places": 12, "rounding": "half_even"}: + raise CapsuleError( + "deterministic calibration rendering must use 12 decimal places and half_even rounding" + ) + + denominator_count = len(rows) + numerator_count = sum(int(row[column]) for row in rows) + if exact_result_payload != { + "numerator": numerator_count, + "denominator": denominator_count, + }: + raise CapsuleError( + "deterministic calibration exact_result does not match the checked-in matrix count" + ) + computed_value = (Decimal(numerator_count) / Decimal(denominator_count)).quantize( + Decimal("0.000000000001"), + rounding=ROUND_HALF_EVEN, + ) + if realized_fpr != computed_value: + raise CapsuleError( + "calibration.realized_fpr does not match the deterministic " + f"{column} rate: declared {realized_fpr}, computed {computed_value}" + ) + return ( + origin_payload, + { + "status": "derivation_verified_from_declared_bytes", + "source_sha256": matrix_sha256, + "numerator_count": numerator_count, + "denominator_count": denominator_count, + "computed_value": format(computed_value, ".12f"), + }, + ) + + +def _anchor_subject( + *, + realized_fpr: Decimal, + semantics: Mapping[str, Any], + origin: Mapping[str, Any], +) -> dict[str, Any]: + return { + "schema": CALIBRATION_PROVENANCE_SCHEMA, + "field": "realized_fpr", + "value": format(realized_fpr, "f"), + "semantics": dict(semantics), + "origin": dict(origin), + } + + +def _validate_external_anchor( + anchor: Any, + *, + subject: Mapping[str, Any], +) -> tuple[dict[str, Any] | None, dict[str, Any]]: + if anchor is None: + return None, {"status": "absent"} + if not isinstance(anchor, Mapping): + raise CapsuleError("calibration provenance.external_anchor must be an object or null") + anchor_payload = dict(anchor) + _require_exact_keys( + anchor_payload, + {"mechanism", "uri", "subject_sha256", "issued_at", "issuer"}, + "calibration provenance.external_anchor", + ) + if anchor_payload["mechanism"] != "reference_only/v1": + raise CapsuleError( + "calibration provenance external_anchor mechanism must be reference_only/v1" + ) + uri = anchor_payload["uri"] + if not isinstance(uri, str) or not uri.startswith("https://"): + raise CapsuleError("calibration provenance external_anchor.uri must be an https URL") + digest = anchor_payload["subject_sha256"] + if ( + not isinstance(digest, str) + or len(digest) != 64 + or any(character not in "0123456789abcdef" for character in digest) + ): + raise CapsuleError( + "calibration provenance external_anchor.subject_sha256 must be lowercase hex" + ) + expected_subject_hash = hash_json(subject) + if digest != expected_subject_hash: + raise CapsuleError( + "calibration provenance external_anchor does not bind this exact subject" + ) + _require_nonblank(anchor_payload["issuer"], "calibration provenance external_anchor.issuer") + if not isinstance(anchor_payload["issued_at"], str): + raise CapsuleError("calibration provenance external_anchor.issued_at must be a timestamp") + parse_utc(anchor_payload["issued_at"]) + return ( + anchor_payload, + { + "status": "reference_recorded_not_externally_verified", + "reason": ( + "The capsule checked that the reference binds this declared subject, but did " + "not contact the issuer or verify that it measured the value." + ), + }, + ) + + +def _provenance_non_claims( + *, + semantics: Mapping[str, Any], + origin: Mapping[str, Any], + external_anchor: Mapping[str, Any] | None, +) -> list[str]: + non_claims = [CALIBRATION_PROVENANCE_NON_CLAIM] + if origin["kind"] == "asserted": + non_claims.append( + "This value is asserted; the capsule does not infer a numerator, denominator, event, " + "or relation to failure_matrix.csv." + ) + else: + non_claims.append( + "The deterministic derivation verifies only the declared calculation over named bytes; " + "it does not establish that the named binary column is a false-positive rate." + ) + if semantics["status"] == "unresolved": + non_claims.append( + "The event, numerator, denominator, and source population remain unresolved in this fixture." + ) + if external_anchor is not None: + non_claims.append( + "The external reference is recorded but not externally verified by this capsule." + ) + return non_claims + + +def _decimal_probability(value: Any, field_name: str) -> Decimal: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise CapsuleError(f"{field_name} must be a finite probability") + try: + numeric = Decimal(str(value)) + except (InvalidOperation, ValueError) as exc: + raise CapsuleError(f"{field_name} must be a finite probability") from exc + if not numeric.is_finite() or not Decimal("0") <= numeric <= Decimal("1"): + raise CapsuleError(f"{field_name} must be a finite probability") + return numeric + + +def _require_nonblank(value: Any, field_name: str) -> None: + if not isinstance(value, str) or not value.strip(): + raise CapsuleError(f"{field_name} must be a non-empty string") + + +def _require_exact_keys(payload: Mapping[str, Any], expected: set[str], context: str) -> None: + actual = set(payload) + if actual != expected: + missing = sorted(expected - actual) + extra = sorted(actual - expected) + details = [] + if missing: + details.append(f"missing {missing}") + if extra: + details.append(f"unexpected {extra}") + raise CapsuleError(f"{context} has invalid fields: {', '.join(details)}") + + def build_decay_payload( *, config: Mapping[str, Any], evidence_refs: tuple[str, ...] ) -> dict[str, Any]: diff --git a/examples/claim_governance_capsule/expected/calibration.json b/examples/claim_governance_capsule/expected/calibration.json index 21742e3d..f6d40f15 100644 --- a/examples/claim_governance_capsule/expected/calibration.json +++ b/examples/claim_governance_capsule/expected/calibration.json @@ -5,6 +5,32 @@ "upper": 0.07 }, "realized_fpr": 0.041666666667, + "realized_fpr_provenance": { + "external_anchor": null, + "field": "realized_fpr", + "non_claims": [ + "Calibration provenance records how a value is described; it does not establish that the value was measured, that labels are correct, or that the population is representative.", + "This value is asserted; the capsule does not infer a numerator, denominator, event, or relation to failure_matrix.csv.", + "The event, numerator, denominator, and source population remain unresolved in this fixture." + ], + "origin": { + "kind": "asserted" + }, + "schema": "cc.calibration.realized-fpr-provenance.v1", + "semantics": { + "reason": "The repository does not establish realized_fpr's numerator, denominator, event, or source population.", + "status": "unresolved" + }, + "verification": { + "external_anchor": { + "status": "absent" + }, + "origin": { + "reason": "The capsule records this declared value without inferring an event, denominator, or source population.", + "status": "asserted_not_derivable" + } + } + }, "status": "pass", "target_fpr": 0.05, "threshold": 0.42 diff --git a/examples/claim_governance_capsule/expected/cc_report.json b/examples/claim_governance_capsule/expected/cc_report.json index 274ddb7c..991101de 100644 --- a/examples/claim_governance_capsule/expected/cc_report.json +++ b/examples/claim_governance_capsule/expected/cc_report.json @@ -46,10 +46,10 @@ "sha256": "ce2cbba19dbfee5516143ce8f94cd8c8acc7b1ab2517809695cd52f8bd62fe74" }, { - "bytes": 189, + "bytes": 1289, "path": "calibration.json", "role": "calibration_evidence", - "sha256": "fb6632f6958567326a2e31a41e62a839dd31ea751ff0af69a562adeccaded014" + "sha256": "a7d8be80173008354fd1936a45136f88ba3c62a524e4f42dc763ad0c019eb3d4" }, { "bytes": 3712, @@ -73,13 +73,13 @@ "bytes": 4559, "path": "extremal_lower.json", "role": "extremal_scenario", - "sha256": "ab5bf859ed132c8f9f68262a00470d6312c035953b14696e7dda368a569d437f" + "sha256": "bb9e9cf4e1aec1aef6e670c06e10bad89cb96ccdb7222d1ec3919e79b3c7a120" }, { "bytes": 4557, "path": "extremal_upper.json", "role": "extremal_scenario", - "sha256": "ce08e2c34974d8187824413deec8b7314e3a95559c1a2b5c0d53beaf15dceba3" + "sha256": "f2aaf4d8b700069449f3364068fc024f28f718173aa9f9312137b29788541925" } ], "audit_log": { @@ -111,15 +111,15 @@ } }, "receipt": { - "canonical_hash": "209fa93b57dda3ceccf4b7680211a19f2e07105fc6b1561c89b64ab8093fd811", - "canonicalization_method": "json.dumps(sort_keys=True,separators=(',', ':'),ensure_ascii=False,allow_nan=False); receipt.canonical_hash excluded", + "canonical_hash": "9742e11a31f35697d6170b4ea228a0371b60fd708b57606a01ac91b5385cdad2", + "canonicalization_method": "cc.canonical.v2/RFC8785; receipt.canonical_hash excluded", "hash_algorithm": "sha256", "previous_hash": null }, "report_id": "cc-claim-governance-capsule", "run": { "command": "examples/claim_governance_capsule/reproduce.sh", - "config_hash": "7342579696302b3673c868baf7affb4ffe794ec00d0e51b70e8e5f61d74639d2", + "config_hash": "1451bd242ca3069cac394eb2f84f4833db303938e90bf451dea41d4bcd1784e0", "config_path": "examples/claim_governance_capsule/inputs/capsule_config.json", "run_id": "claim-governance-capsule-run", "seed": 314159 diff --git a/examples/claim_governance_capsule/expected/claim_envelope.json b/examples/claim_governance_capsule/expected/claim_envelope.json index a553612b..b5974461 100644 --- a/examples/claim_governance_capsule/expected/claim_envelope.json +++ b/examples/claim_governance_capsule/expected/claim_envelope.json @@ -118,7 +118,7 @@ "claim_id": "cc-claim-governance-capsule", "created_at": "2026-01-01T00:00:00Z", "evaluated_at": "2026-01-02T00:00:00Z", - "source_report_hash": "209fa93b57dda3ceccf4b7680211a19f2e07105fc6b1561c89b64ab8093fd811", + "source_report_hash": "9742e11a31f35697d6170b4ea228a0371b60fd708b57606a01ac91b5385cdad2", "source_report_id": "cc-claim-governance-capsule", "source_report_schema": "cc.report.v0.3.1", "subject_ref": "cc.report.v0.3.1:cc-claim-governance-capsule" @@ -220,7 +220,7 @@ "reason": null, "role": "measurement_evidence", "schema": "cc.report.measurement.v0.3.1", - "sha256": "569331caa16c17cdaf1531a4cbc318f1b6ccf44204dc0d0018dfa1af4a0c261b", + "sha256": "dac150e456c0fbd1294e866cc24b76f11e6d553ef51a7a87a72b74be01fad717", "status": null, "subject_ref": "cc.report.v0.3.1:cc-claim-governance-capsule" }, @@ -256,7 +256,7 @@ }, { "artifact_id": "evidence:calibration_evidence:0001:calibration.json", - "bytes": 189, + "bytes": 1289, "created_at": null, "evaluated_at": null, "metadata": {}, @@ -264,7 +264,7 @@ "reason": "Evidence artifact hash and byte count verified.", "role": "calibration_evidence", "schema": null, - "sha256": "fb6632f6958567326a2e31a41e62a839dd31ea751ff0af69a562adeccaded014", + "sha256": "a7d8be80173008354fd1936a45136f88ba3c62a524e4f42dc763ad0c019eb3d4", "status": "present", "subject_ref": "cc.report.v0.3.1:cc-claim-governance-capsule" }, @@ -322,7 +322,7 @@ "reason": "Canonical report receipt binds the report bytes and named evidence hashes.", "role": "receipt_integrity", "schema": "cc.report.receipt.v0.3.1", - "sha256": "209fa93b57dda3ceccf4b7680211a19f2e07105fc6b1561c89b64ab8093fd811", + "sha256": "9742e11a31f35697d6170b4ea228a0371b60fd708b57606a01ac91b5385cdad2", "status": null, "subject_ref": "cc.report.v0.3.1:cc-claim-governance-capsule" } @@ -339,7 +339,7 @@ "reason": "Evidence artifact hash and byte count verified.", "role": "extremal_scenario", "schema": null, - "sha256": "ab5bf859ed132c8f9f68262a00470d6312c035953b14696e7dda368a569d437f", + "sha256": "bb9e9cf4e1aec1aef6e670c06e10bad89cb96ccdb7222d1ec3919e79b3c7a120", "status": "present", "subject_ref": "cc.report.v0.3.1:cc-claim-governance-capsule" }, @@ -353,7 +353,7 @@ "reason": "Evidence artifact hash and byte count verified.", "role": "extremal_scenario", "schema": null, - "sha256": "ce08e2c34974d8187824413deec8b7314e3a95559c1a2b5c0d53beaf15dceba3", + "sha256": "f2aaf4d8b700069449f3364068fc024f28f718173aa9f9312137b29788541925", "status": "present", "subject_ref": "cc.report.v0.3.1:cc-claim-governance-capsule" } diff --git a/examples/claim_governance_capsule/expected/claim_governance_audit.json b/examples/claim_governance_capsule/expected/claim_governance_audit.json index 87e799be..75e12728 100644 --- a/examples/claim_governance_capsule/expected/claim_governance_audit.json +++ b/examples/claim_governance_capsule/expected/claim_governance_audit.json @@ -147,13 +147,13 @@ "status": "present" }, { - "bytes_actual": 189, - "bytes_expected": 189, + "bytes_actual": 1289, + "bytes_expected": 1289, "path": "calibration.json", "reason": "Evidence artifact hash and byte count verified.", "role": "calibration_evidence", - "sha256_actual": "fb6632f6958567326a2e31a41e62a839dd31ea751ff0af69a562adeccaded014", - "sha256_expected": "fb6632f6958567326a2e31a41e62a839dd31ea751ff0af69a562adeccaded014", + "sha256_actual": "a7d8be80173008354fd1936a45136f88ba3c62a524e4f42dc763ad0c019eb3d4", + "sha256_expected": "a7d8be80173008354fd1936a45136f88ba3c62a524e4f42dc763ad0c019eb3d4", "status": "present" }, { @@ -192,8 +192,8 @@ "path": "extremal_lower.json", "reason": "Evidence artifact hash and byte count verified.", "role": "extremal_scenario", - "sha256_actual": "ab5bf859ed132c8f9f68262a00470d6312c035953b14696e7dda368a569d437f", - "sha256_expected": "ab5bf859ed132c8f9f68262a00470d6312c035953b14696e7dda368a569d437f", + "sha256_actual": "bb9e9cf4e1aec1aef6e670c06e10bad89cb96ccdb7222d1ec3919e79b3c7a120", + "sha256_expected": "bb9e9cf4e1aec1aef6e670c06e10bad89cb96ccdb7222d1ec3919e79b3c7a120", "status": "present" }, { @@ -202,8 +202,8 @@ "path": "extremal_upper.json", "reason": "Evidence artifact hash and byte count verified.", "role": "extremal_scenario", - "sha256_actual": "ce08e2c34974d8187824413deec8b7314e3a95559c1a2b5c0d53beaf15dceba3", - "sha256_expected": "ce08e2c34974d8187824413deec8b7314e3a95559c1a2b5c0d53beaf15dceba3", + "sha256_actual": "f2aaf4d8b700069449f3364068fc024f28f718173aa9f9312137b29788541925", + "sha256_expected": "f2aaf4d8b700069449f3364068fc024f28f718173aa9f9312137b29788541925", "status": "present" }, { @@ -247,7 +247,7 @@ "reasons": [], "receipt": { "artifact_hashes_verified": true, - "canonical_hash": "209fa93b57dda3ceccf4b7680211a19f2e07105fc6b1561c89b64ab8093fd811", + "canonical_hash": "9742e11a31f35697d6170b4ea228a0371b60fd708b57606a01ac91b5385cdad2", "reason": "Canonical report hash verified. Evidence artifact hashes verified. Receipt verification checks artifact integrity only; it does not prove statistical validity, deployment safety, production readiness, or compliance.", "report_hash_verified": true }, @@ -270,8 +270,8 @@ "present": true, "scenario_count": 2, "scenario_ids": [ - "scenario-frechet-lower-7a00219842c1", - "scenario-frechet-upper-bff6e6e0b031" + "scenario-frechet-lower-83135b67bb38", + "scenario-frechet-upper-2292c147ab06" ] }, "schema": "cc/claim-governance-audit.v1", diff --git a/examples/claim_governance_capsule/expected/extremal_lower.json b/examples/claim_governance_capsule/expected/extremal_lower.json index 47e7f551..a4713c94 100644 --- a/examples/claim_governance_capsule/expected/extremal_lower.json +++ b/examples/claim_governance_capsule/expected/extremal_lower.json @@ -139,7 +139,7 @@ "Scenario kind and feasibility status are evidence-artifact properties, not claim lifecycle states." ], "objective": null, - "scenario_id": "scenario-frechet-lower-7a00219842c1", + "scenario_id": "scenario-frechet-lower-83135b67bb38", "schema": "cc.extremal_scenario.v1", "schema_version": "cc.extremal_scenario.v1", "source": "frechet", diff --git a/examples/claim_governance_capsule/expected/extremal_upper.json b/examples/claim_governance_capsule/expected/extremal_upper.json index 61e70d2c..bac642cd 100644 --- a/examples/claim_governance_capsule/expected/extremal_upper.json +++ b/examples/claim_governance_capsule/expected/extremal_upper.json @@ -139,7 +139,7 @@ "Scenario kind and feasibility status are evidence-artifact properties, not claim lifecycle states." ], "objective": null, - "scenario_id": "scenario-frechet-upper-bff6e6e0b031", + "scenario_id": "scenario-frechet-upper-2292c147ab06", "schema": "cc.extremal_scenario.v1", "schema_version": "cc.extremal_scenario.v1", "source": "frechet", diff --git a/examples/claim_governance_capsule/inputs/capsule_config.json b/examples/claim_governance_capsule/inputs/capsule_config.json index 3328825a..6e066390 100644 --- a/examples/claim_governance_capsule/inputs/capsule_config.json +++ b/examples/claim_governance_capsule/inputs/capsule_config.json @@ -13,6 +13,18 @@ "upper": 0.07 }, "realized_fpr": 0.041666666667, + "realized_fpr_provenance": { + "external_anchor": null, + "field": "realized_fpr", + "origin": { + "kind": "asserted" + }, + "schema": "cc.calibration.realized-fpr-provenance.v1", + "semantics": { + "reason": "The repository does not establish realized_fpr's numerator, denominator, event, or source population.", + "status": "unresolved" + } + }, "status": "pass", "target_fpr": 0.05, "threshold": 0.42 diff --git a/examples/claim_governance_capsule/manifest.expected.json b/examples/claim_governance_capsule/manifest.expected.json index adb5f3d0..8f690fcc 100644 --- a/examples/claim_governance_capsule/manifest.expected.json +++ b/examples/claim_governance_capsule/manifest.expected.json @@ -14,28 +14,28 @@ "sha256": "ce2cbba19dbfee5516143ce8f94cd8c8acc7b1ab2517809695cd52f8bd62fe74" }, { - "bytes": 189, + "bytes": 1289, "filename": "calibration.json", "role": "calibration_evidence", - "sha256": "fb6632f6958567326a2e31a41e62a839dd31ea751ff0af69a562adeccaded014" + "sha256": "a7d8be80173008354fd1936a45136f88ba3c62a524e4f42dc763ad0c019eb3d4" }, { - "bytes": 4702, + "bytes": 4643, "filename": "cc_report.json", "role": "cc_report", - "sha256": "faad40a4f2018b2031593452c32c8f1e33fe8d139fe7b96c4b158f41961ff012" + "sha256": "8f48cf752ead0fe3d46d3eeb2e6b9ac7e7868ec1c1925973b45725fdeae97573" }, { - "bytes": 21103, + "bytes": 21104, "filename": "claim_envelope.json", "role": "claim_envelope", - "sha256": "2f686a9b0e095f19f4af6ca7d36f4c7ed19e386bc8f83d1412f83b3eeaa75658" + "sha256": "ad47582c938cb31595218b191f33ca946d8403c152bc93bc13bcb13bb1a597d1" }, { - "bytes": 13150, + "bytes": 13152, "filename": "claim_governance_audit.json", "role": "claim_governance_audit", - "sha256": "9af0d97908983987051b78703fc7c7b13caea859202fff307daaeb9061cf0498" + "sha256": "8f32aa8e1dae91bb98c6c8ed3e89a483ffe84d6778c0ae83f4a092aace2d2450" }, { "bytes": 3712, @@ -59,13 +59,13 @@ "bytes": 4559, "filename": "extremal_lower.json", "role": "extremal_scenario", - "sha256": "ab5bf859ed132c8f9f68262a00470d6312c035953b14696e7dda368a569d437f" + "sha256": "bb9e9cf4e1aec1aef6e670c06e10bad89cb96ccdb7222d1ec3919e79b3c7a120" }, { "bytes": 4557, "filename": "extremal_upper.json", "role": "extremal_scenario", - "sha256": "ce08e2c34974d8187824413deec8b7314e3a95559c1a2b5c0d53beaf15dceba3" + "sha256": "f2aaf4d8b700069449f3364068fc024f28f718173aa9f9312137b29788541925" } ], "fixed_now": "2026-01-02T00:00:00Z", @@ -73,9 +73,9 @@ "governance_verdict": "pass", "inputs": [ { - "bytes": 2398, + "bytes": 2798, "filename": "inputs/capsule_config.json", - "sha256": "7342579696302b3673c868baf7affb4ffe794ec00d0e51b70e8e5f61d74639d2" + "sha256": "1451bd242ca3069cac394eb2f84f4833db303938e90bf451dea41d4bcd1784e0" }, { "bytes": 352, @@ -85,7 +85,7 @@ ], "pass_caveat": "PASS means internal consistency under verifier rules; it does not mean the AI system is safe in deployment.", "report_id": "cc-claim-governance-capsule", - "report_receipt_sha256": "209fa93b57dda3ceccf4b7680211a19f2e07105fc6b1561c89b64ab8093fd811", + "report_receipt_sha256": "9742e11a31f35697d6170b4ea228a0371b60fd708b57606a01ac91b5385cdad2", "schema_version": "cc.claim_governance_capsule_manifest.v1", "seed": 314159 } diff --git a/pyproject.toml b/pyproject.toml index 540d6899..63a92e37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,6 +116,13 @@ test = [ "pandas>=2.2", "matplotlib>=3.8", "scikit-learn>=1.3", + # tests/unit/packaging/ subprocesses `python -m build --no-isolation`, which + # needs the build frontend and the setuptools backend present in the same + # environment. Without these three the declared test extra cannot run the + # test suite (upgrade finding F-01). + "build>=1.2", + "setuptools>=69", + "wheel", ] security = [ @@ -133,6 +140,7 @@ all = [ cc-bundle = "cc.core.evidence_bundle:main" cc-cartographer = "cc.cartographer.cli:main" cc-dependence-bench = "cc.evals.dependence_benchmark:main" +cc-guard = "cc.cli.guard:main" cc-report = "cc.reporting.cli:main" [project.urls] diff --git a/schemas/cc.evidence_card.v1.json b/schemas/cc.evidence_card.v1.json new file mode 100644 index 00000000..12fd86a5 --- /dev/null +++ b/schemas/cc.evidence_card.v1.json @@ -0,0 +1,296 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://cc-framework.local/schemas/cc.evidence_card.v1.json", + "title": "cc.evidence_card.v1", + "description": "One claim with everything needed to disagree with it. The three labels are ORTHOGONAL and a consumer MUST NOT collapse them into a single status: evidence_state says where evidence came from, verdict says what happened when the command ran, publication_state says where the card is in its own lifecycle. None implies another. This schema deliberately defines no aggregate score, badge, or health field.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "card_id", + "claim", + "maturity", + "source_revision", + "command", + "artifacts", + "labels", + "label_meanings", + "falsifier", + "assumptions", + "non_claims" + ], + "properties": { + "schema_version": { + "const": "cc.evidence_card.v1" + }, + "card_id": { + "type": "string", + "minLength": 1 + }, + "claim": { + "type": "string", + "minLength": 1 + }, + "maturity": { + "type": "string", + "minLength": 1, + "description": "Claim-boundary level, e.g. C3. Not a quality rating." + }, + "source_revision": { + "type": "string", + "minLength": 1, + "description": "Git revision the card was generated from. A '-dirty' suffix means the working tree differed." + }, + "command": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + }, + "description": "How a reader reproduces this. A claim no command can demonstrate is an assertion." + }, + "artifacts": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "sha256", + "bytes", + "missing" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "sha256": { + "type": [ + "string", + "null" + ], + "pattern": "^[0-9a-f]{64}$" + }, + "bytes": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "missing": { + "type": "boolean", + "description": "True when a claim names a path that does not exist. Recorded, not skipped." + } + } + } + }, + "labels": { + "type": "object", + "additionalProperties": false, + "required": [ + "evidence_state", + "verdict", + "publication_state" + ], + "description": "All three are required. A surface showing fewer will show the flattering one.", + "properties": { + "evidence_state": { + "enum": [ + "local-only", + "aws-synth-only", + "aws-live", + "illustrative" + ] + }, + "verdict": { + "enum": [ + "pass", + "fail", + "unverifiable", + "not-run" + ], + "description": "'unverifiable' is not a fail: the check could not be performed at all. 'not-run' claims no result." + }, + "publication_state": { + "enum": [ + "draft", + "released", + "superseded", + "retracted" + ], + "description": "'released' says a card was reviewed for release, not that its claim is true." + } + } + }, + "label_meanings": { + "type": "object", + "additionalProperties": false, + "required": [ + "evidence_state", + "verdict", + "publication_state" + ], + "description": "Gloss shown beside each value so a reader never infers what a bare token means.", + "properties": { + "evidence_state": { + "type": "string", + "minLength": 1 + }, + "verdict": { + "type": "string", + "minLength": 1 + }, + "publication_state": { + "type": "string", + "minLength": 1 + } + } + }, + "result_detail": { + "type": [ + "string", + "null" + ], + "description": "What actually ran. Required whenever verdict is pass or fail." + }, + "falsifier": { + "type": "string", + "minLength": 1, + "description": "What observation would show this claim is wrong. Required: a claim with no falsifier is not evidence." + }, + "counterevidence": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "assumptions": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + }, + "non_claims": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + }, + "description": "What this card explicitly does not establish." + }, + "supersedes": { + "type": [ + "string", + "null" + ] + }, + "retraction_reason": { + "type": [ + "string", + "null" + ] + } + }, + "allOf": [ + { + "if": { + "properties": { + "labels": { + "properties": { + "verdict": { + "enum": [ + "pass", + "fail" + ] + } + }, + "required": [ + "verdict" + ] + } + }, + "required": [ + "labels" + ] + }, + "then": { + "properties": { + "result_detail": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "result_detail" + ] + } + }, + { + "if": { + "properties": { + "labels": { + "properties": { + "publication_state": { + "const": "retracted" + } + }, + "required": [ + "publication_state" + ] + } + }, + "required": [ + "labels" + ] + }, + "then": { + "properties": { + "retraction_reason": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "retraction_reason" + ] + } + }, + { + "if": { + "properties": { + "labels": { + "properties": { + "publication_state": { + "const": "superseded" + } + }, + "required": [ + "publication_state" + ] + } + }, + "required": [ + "labels" + ] + }, + "then": { + "properties": { + "supersedes": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "supersedes" + ] + } + } + ] +} diff --git a/schemas/cc_report.schema.json b/schemas/cc_report.schema.json index 9e553d08..68f26574 100644 --- a/schemas/cc_report.schema.json +++ b/schemas/cc_report.schema.json @@ -40,23 +40,36 @@ "git": { "type": "object", "additionalProperties": false, - "required": ["commit", "dirty", "branch"], + "required": [ + "commit", + "dirty", + "branch" + ], "properties": { "commit": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ] }, "dirty": { "type": "boolean" }, "branch": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ] } } }, "environment": { "type": "object", "additionalProperties": false, - "required": ["python_version", "platform"], + "required": [ + "python_version", + "platform" + ], "properties": { "python_version": { "type": "string", @@ -81,24 +94,42 @@ "run": { "type": "object", "additionalProperties": false, - "required": ["run_id", "config_path", "config_hash", "seed", "command"], + "required": [ + "run_id", + "config_path", + "config_hash", + "seed", + "command" + ], "properties": { "run_id": { "type": "string", "minLength": 1 }, "config_path": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ] }, "config_hash": { - "type": ["string", "null"], + "type": [ + "string", + "null" + ], "pattern": "^[0-9a-f]{64}$" }, "seed": { - "type": ["integer", "null"] + "type": [ + "integer", + "null" + ] }, "command": { - "type": ["string", "null"] + "type": [ + "string", + "null" + ] } } }, @@ -121,17 +152,26 @@ ], "properties": { "target_fpr": { - "type": ["number", "null"], + "type": [ + "number", + "null" + ], "minimum": 0, "maximum": 1 }, "alpha_cap": { - "type": ["number", "null"], + "type": [ + "number", + "null" + ], "minimum": 0, "maximum": 1 }, "realized_fpr": { - "type": ["number", "null"], + "type": [ + "number", + "null" + ], "minimum": 0, "maximum": 1 }, @@ -139,10 +179,16 @@ "type": "object" }, "threshold": { - "type": ["number", "null"] + "type": [ + "number", + "null" + ] }, "status": { - "enum": ["pass", "fail"] + "enum": [ + "pass", + "fail" + ] } }, "anyOf": [ @@ -185,7 +231,10 @@ "interval": { "type": "object", "additionalProperties": false, - "required": ["lower", "upper"], + "required": [ + "lower", + "upper" + ], "properties": { "lower": { "type": "number" @@ -196,12 +245,18 @@ } }, "confidence_level": { - "type": ["number", "null"], + "type": [ + "number", + "null" + ], "minimum": 0, "maximum": 1 }, "delta": { - "type": ["number", "null"], + "type": [ + "number", + "null" + ], "minimum": 0, "maximum": 1 }, @@ -239,7 +294,11 @@ "type": "object", "description": "Evidence references bound by SHA-256 hashes. These hashes identify bytes for audit; they do not prove that the evidence is sufficient for release or compliance.", "additionalProperties": false, - "required": ["artifacts", "audit_log", "figure_manifest"], + "required": [ + "artifacts", + "audit_log", + "figure_manifest" + ], "properties": { "artifacts": { "type": "array", @@ -273,7 +332,11 @@ "type": "object", "description": "The single claim this report is allowed to support, plus explicit boundaries on what the report does not claim.", "additionalProperties": false, - "required": ["statement", "allowed_claim_level", "non_claims"], + "required": [ + "statement", + "allowed_claim_level", + "non_claims" + ], "properties": { "statement": { "type": "string", @@ -295,10 +358,16 @@ "allOf": [ { "if": { - "required": ["allowed_claim_level"], + "required": [ + "allowed_claim_level" + ], "properties": { "allowed_claim_level": { - "enum": ["bounded_empirical", "reproducible_run", "release_claim"] + "enum": [ + "bounded_empirical", + "reproducible_run", + "release_claim" + ] } } }, @@ -332,11 +401,17 @@ "const": "sha256" }, "canonicalization_method": { - "description": "Exact deterministic JSON method used before hashing. All report fields except receipt.canonical_hash are included.", - "const": "json.dumps(sort_keys=True,separators=(',', ':'),ensure_ascii=False,allow_nan=False); receipt.canonical_hash excluded" + "description": "Canonicalization profile used before hashing. All report fields except receipt.canonical_hash are included. Verification MUST dispatch on this value rather than assuming the current default: pre-migration receipts carry the legacy profile and must stay verifiable. See docs/architecture/CANONICAL_PROFILE.md.", + "enum": [ + "cc.canonical.v2/RFC8785; receipt.canonical_hash excluded", + "json.dumps(sort_keys=True,separators=(',', ':'),ensure_ascii=False,allow_nan=False); receipt.canonical_hash excluded" + ] }, "previous_hash": { - "type": ["string", "null"], + "type": [ + "string", + "null" + ], "pattern": "^[0-9a-f]{64}$" } } @@ -368,7 +443,12 @@ "type": "object", "description": "Named evidence file with its byte count and SHA-256 digest.", "additionalProperties": false, - "required": ["path", "sha256", "bytes", "role"], + "required": [ + "path", + "sha256", + "bytes", + "role" + ], "properties": { "path": { "type": "string", diff --git a/scripts/build_conformance_corpus.py b/scripts/build_conformance_corpus.py new file mode 100755 index 00000000..8b81c95e --- /dev/null +++ b/scripts/build_conformance_corpus.py @@ -0,0 +1,438 @@ +#!/usr/bin/env python3 +"""Generate the ``cc-kernel-v1`` conformance corpus. + +The corpus is the normative statement of what a CC composition kernel must +compute. It is language-agnostic JSON: an implementation in any language either +reproduces every expected value within tolerance, or it is not a CC kernel. + +Expected values are produced here by :mod:`cc.compose`, and every ``accept`` +case is independently cross-checked against the finite-atom LP in +:mod:`cc.kernel.sensitivity` before it is written. A case that the closed form +and the LP disagree on is a defect in this repository and the build fails +rather than pinning the disagreement into the corpus. + +Usage:: + + python scripts/build_conformance_corpus.py # write the corpus + python scripts/build_conformance_corpus.py --check # verify it is current +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sys +from pathlib import Path +from typing import Any + +from cc.compose import CountermonotoneUndefinedError, compose_bounds +from cc.kernel.sensitivity import AssumptionSet, LinearQuery, identified_region +from cc.reporting.canonical import canonical_json_bytes + +CORPUS_VERSION = "cc-kernel-v1" +ROOT = Path(__file__).resolve().parents[1] +CORPUS_DIR = ROOT / "conformance" / CORPUS_VERSION + +#: Absolute tolerance an implementation is allowed on every numeric field. +#: Chosen at 1e-12, far above the 3.3e-16 worst disagreement measured between +#: the closed form and the LP, and far below any difference that would change a +#: reported bound. +TOLERANCE = 1.0e-12 + +# --- accept cases: (id, marginals, event, dependence, description) ----------- + +_ACCEPT: list[tuple[str, dict[str, float], str, str, str]] = [ + ( + "and-two-classical", + {"A": 0.1, "B": 0.2}, + "all", + "unconstrained", + "Classical two-event conjunction. Lower is max(0, sum-1)=0; upper is min(p)=0.1.", + ), + ( + "and-two-overlap-forced", + {"A": 0.9, "B": 0.8}, + "all", + "unconstrained", + "Marginals summing above 1 force a positive lower bound: 0.9+0.8-1 = 0.7.", + ), + ( + "and-two-equal-tie", + {"A": 0.25, "B": 0.25}, + "all", + "unconstrained", + "Tied minima: no single event binds, so binding_event is null.", + ), + ( + "and-two-extreme-asymmetry", + {"A": 0.01, "B": 0.99}, + "all", + "unconstrained", + "A narrow interval from extreme marginals. Narrowness is not evidence of validity.", + ), + ( + "or-two-classical", + {"A": 0.1, "B": 0.2}, + "any", + "unconstrained", + "Classical two-event union. Lower is max(p)=0.2; upper is min(1, sum)=0.3.", + ), + ( + "or-two-clipped", + {"A": 0.7, "B": 0.8}, + "any", + "unconstrained", + "Union upper bound clipped at 1 because the marginals sum above it.", + ), + ( + "and-four-vinctura-small-chapter", + {"SELF_REPORTED": 0.30, "STALE": 0.01, "IMMUTABLE": 0.01, "SEPARATION": 0.40}, + "all", + "unconstrained", + "Four deterministic controls, assumed evasion rates. Reproduces a published " + "external result: lower 0, upper 0.01, independence 1.2e-5, factor 833.", + ), + ( + "and-four-vinctura-optimistic", + {"SELF_REPORTED": 0.05, "STALE": 0.01, "IMMUTABLE": 0.01, "SEPARATION": 0.10}, + "all", + "unconstrained", + "Same four controls, optimistic assumed rates. Tied minima at 0.01.", + ), + ( + "and-four-vinctura-pessimistic", + {"SELF_REPORTED": 0.50, "STALE": 0.10, "IMMUTABLE": 0.05, "SEPARATION": 0.60}, + "all", + "unconstrained", + "Same four controls, pessimistic assumed rates. IMMUTABLE binds at 0.05.", + ), + ( + "and-single-event", + {"A": 0.42}, + "all", + "unconstrained", + "Degenerate single-event conjunction. The interval collapses to the marginal.", + ), + ( + "or-single-event", + {"A": 0.42}, + "any", + "unconstrained", + "Degenerate single-event union. The interval collapses to the marginal.", + ), + ( + "and-zero-marginal", + {"A": 0.0, "B": 0.5}, + "all", + "unconstrained", + "A zero marginal pins the conjunction to zero regardless of the other event.", + ), + ( + "and-unit-marginals", + {"A": 1.0, "B": 1.0}, + "all", + "unconstrained", + "Certain events. The conjunction is certain.", + ), + ( + "and-two-independent", + {"A": 0.3, "B": 0.4}, + "all", + "independent", + "Independence returns a degenerate interval at the product. A baseline, not an answer.", + ), + ( + "or-two-independent", + {"A": 0.3, "B": 0.4}, + "any", + "independent", + "Independent union: 1 - (1-0.3)(1-0.4) = 0.58.", + ), + ( + "and-two-comonotone", + {"A": 0.3, "B": 0.4}, + "all", + "comonotone", + "The upper Frechet corner as a degenerate interval.", + ), + ( + "and-two-countermonotone", + {"A": 0.3, "B": 0.4}, + "all", + "countermonotone", + "Defined only for exactly two events: max(0, 0.3+0.4-1) = 0.", + ), + ( + "and-two-countermonotone-positive", + {"A": 0.7, "B": 0.8}, + "all", + "countermonotone", + "Two-event countermonotone with a positive floor: 0.7+0.8-1 = 0.5.", + ), +] + +# The correlation cliff, as a family: m events all at p=0.1. Independence +# predicts 0.1**m; the sharp upper bound stays at 0.1 for every m. This is the +# repository's headline result, pinned so no implementation can lose it. +for _m in (2, 3, 4, 5, 8, 10): + _ACCEPT.append( + ( + f"cliff-homogeneous-m{_m}", + {f"G{i}": 0.1 for i in range(_m)}, + "all", + "unconstrained", + f"Correlation cliff at m={_m}: independence predicts 1e-{_m}, the sharp " + f"upper bound stays 0.1. Stacking buys nothing under adversarial dependence.", + ) + ) + +# --- reject cases ------------------------------------------------------------ + +_REJECT: list[tuple[str, dict[str, Any], str, str]] = [ + ( + "reject-countermonotone-three-events", + { + "marginals": {"A": 0.3, "B": 0.4, "C": 0.5}, + "event": "all", + "dependence": "countermonotone", + }, + "countermonotone_undefined", + "Countermonotonicity is strictly bivariate. The FH lower bound is not a " + "copula in dimension >= 3, though it stays pointwise sharp. An " + "implementation that returns a number here is wrong.", + ), + ( + "reject-countermonotone-one-event", + { + "marginals": {"A": 0.3}, + "event": "all", + "dependence": "countermonotone", + }, + "countermonotone_undefined", + "Countermonotonicity is a relation between two events; with fewer than two " + "there is nothing to be countermonotone with. Found by differential fuzz " + "(seed 1): the Python reference raised IndexError and the Node " + "implementation silently returned NaN. Both were wrong, differently.", + ), + ( + "reject-marginal-above-one", + {"marginals": {"A": 1.5, "B": 0.2}, "event": "all", "dependence": "unconstrained"}, + "marginal_out_of_range", + "A marginal outside [0, 1] is not a probability.", + ), + ( + "reject-marginal-negative", + {"marginals": {"A": -0.1, "B": 0.2}, "event": "all", "dependence": "unconstrained"}, + "marginal_out_of_range", + "A negative marginal is not a probability.", + ), + ( + "reject-empty-marginals", + {"marginals": {}, "event": "all", "dependence": "unconstrained"}, + "no_events", + "A composition over zero events has no meaning.", + ), + ( + "reject-unknown-event-kind", + {"marginals": {"A": 0.1}, "event": "xor", "dependence": "unconstrained"}, + "unknown_event_kind", + "Only conjunction and union are defined by this corpus.", + ), + ( + "reject-unknown-dependence", + {"marginals": {"A": 0.1, "B": 0.2}, "event": "all", "dependence": "sorta-dependent"}, + "unknown_dependence", + "An unrecognized dependence assumption must fail closed, not default.", + ), + ( + "reject-non-finite-marginal", + {"marginals": {"A": float("nan"), "B": 0.2}, "event": "all", "dependence": "unconstrained"}, + "marginal_not_finite", + "NaN is not a probability. Note this case is expressed as a string in JSON " + "because JSON has no NaN literal; see the spec.", + ), +] + + +def _lp_crosscheck(marginals: dict[str, float], event: str) -> tuple[float, float]: + """Recompute the interval with the finite-atom LP.""" + names = tuple(marginals) + assumptions = AssumptionSet.empty(names) + for name in names: + assumptions = assumptions.with_marginal_interval(name, marginals[name], marginals[name]) + query = ( + LinearQuery.intersection(names, names) + if event == "all" + else LinearQuery.union(names, names) + ) + result = identified_region(query, assumptions) + return result.lower_bound, result.upper_bound + + +def build_accept() -> list[dict[str, Any]]: + cases: list[dict[str, Any]] = [] + for case_id, marginals, event, dependence, description in _ACCEPT: + bounds = compose_bounds(marginals, event=event, dependence=dependence) + payload = bounds.to_json() + + # Every unconstrained case must agree with the LP. A disagreement is a + # defect here, not a corpus decision. + if dependence == "unconstrained": + lp_lower, lp_upper = _lp_crosscheck(marginals, event) + drift = max(abs(lp_lower - bounds.lower), abs(lp_upper - bounds.upper)) + if drift > TOLERANCE: + raise SystemExit( + f"REFUSING to pin {case_id}: closed form and LP disagree by " + f"{drift:.3e}, above the {TOLERANCE:.0e} tolerance. Fix the " + f"kernel before regenerating the corpus." + ) + + cases.append( + { + "id": case_id, + "description": description, + "input": { + "marginals": dict(marginals), + "event": event, + "dependence": dependence, + }, + "expect": { + "lower": payload["lower"], + "upper": payload["upper"], + "width": payload["width"], + "independence_point": payload["independence_point"], + "understatement_factor": payload["understatement_factor"], + "binding_event": payload["binding_event"], + }, + } + ) + return cases + + +def build_reject() -> list[dict[str, Any]]: + cases: list[dict[str, Any]] = [] + for case_id, raw_input, reason, description in _REJECT: + marginals = raw_input["marginals"] + # Confirm the implementation actually refuses, so the corpus cannot + # claim a refusal that does not happen. + try: + compose_bounds( + marginals, + event=raw_input["event"], + dependence=raw_input["dependence"], + ) + except (CountermonotoneUndefinedError, ValueError, TypeError): + pass + else: + raise SystemExit(f"REFUSING to pin {case_id}: cc.compose accepted it.") + + serializable = dict(raw_input) + serializable["marginals"] = { + k: ("NaN" if isinstance(v, float) and v != v else v) for k, v in marginals.items() + } + cases.append( + { + "id": case_id, + "description": description, + "input": serializable, + "expect_refusal": reason, + } + ) + return cases + + +def build() -> dict[str, dict[str, Any]]: + accept = build_accept() + reject = build_reject() + composition = { + "corpus": CORPUS_VERSION, + "kind": "accept", + "tolerance": TOLERANCE, + "case_count": len(accept), + "cases": accept, + } + adversarial = { + "corpus": CORPUS_VERSION, + "kind": "reject", + "case_count": len(reject), + "cases": reject, + } + return {"composition": composition, "adversarial": adversarial} + + +def _digest(payload: dict[str, Any]) -> str: + return hashlib.sha256(canonical_json_bytes(payload)).hexdigest() + + +def write(files: dict[str, dict[str, Any]]) -> dict[str, Any]: + CORPUS_DIR.mkdir(parents=True, exist_ok=True) + (CORPUS_DIR / "cases").mkdir(exist_ok=True) + digests: dict[str, str] = {} + for name, payload in files.items(): + path = CORPUS_DIR / "cases" / f"{name}.json" + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + digests[f"cases/{name}.json"] = _digest(payload) + + manifest = { + "corpus": CORPUS_VERSION, + "tolerance": TOLERANCE, + "files": digests, + "total_cases": sum(p["case_count"] for p in files.values()), + "non_claims": [ + "Passing this corpus establishes that an implementation computes " + "the same intervals as the reference on these cases. It does not " + "establish that either implementation is correct: two " + "implementations can share a misreading.", + "The corpus is a curated census, not a sample. Its size is an " + "authoring decision, it carries no coverage claim, and no " + "confidence interval may be attached to a pass rate over it.", + "Passing establishes nothing about safety, calibration, threshold " + "choice, or whether the supplied marginals mean anything.", + ], + } + (CORPUS_DIR / "manifest.json").write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return manifest + + +def check(files: dict[str, dict[str, Any]]) -> int: + manifest_path = CORPUS_DIR / "manifest.json" + if not manifest_path.exists(): + print("corpus manifest missing; run without --check to generate", file=sys.stderr) + return 1 + committed = json.loads(manifest_path.read_text(encoding="utf-8")) + expected = {f"cases/{name}.json": _digest(payload) for name, payload in files.items()} + if committed.get("files") != expected: + print("corpus is stale; regenerate with:", file=sys.stderr) + print(" python scripts/build_conformance_corpus.py", file=sys.stderr) + for key, digest in expected.items(): + was = committed.get("files", {}).get(key) + if was != digest: + print(f" {key}: {was} -> {digest}", file=sys.stderr) + return 1 + print(f"corpus current: {committed['total_cases']} cases, {len(expected)} files") + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--check", action="store_true", help="verify the committed corpus is current" + ) + args = parser.parse_args(argv) + + files = build() + if args.check: + return check(files) + + manifest = write(files) + print(f"wrote {manifest['total_cases']} cases to {CORPUS_DIR.relative_to(ROOT)}") + for name, digest in sorted(manifest["files"].items()): + print(f" {name} {digest[:16]}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/build_evidence_cards.py b/scripts/build_evidence_cards.py new file mode 100755 index 00000000..5dbf3999 --- /dev/null +++ b/scripts/build_evidence_cards.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python3 +"""Emit evidence cards and the site manifest an Evidence Atlas consumes. + +Reads the claim-boundary manifest, binds each claim to its artifacts by digest +and to the current source revision, and writes one evidence card per claim plus +a ``cc.site_evidence_manifest.v1`` bundle. + +The default verdict is ``not-run``. A ``pass`` is reachable only with ``--run``, +which executes each claim's commands and records what actually happened. A card +cannot acquire a passing verdict by being written confidently. + +Nothing here computes an aggregate. Counts are reported per label, because a +site reporting "7/8 passing" would be describing something nobody measured. + +Usage:: + + python scripts/build_evidence_cards.py # verdicts: not-run + python scripts/build_evidence_cards.py --run # execute commands + python scripts/build_evidence_cards.py --check # verify committed output +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shlex +import subprocess +import sys +from pathlib import Path +from typing import Any + +from cc.evidence_card import ArtifactRef, EvidenceCard, cards_to_site_manifest + +ROOT = Path(__file__).resolve().parents[1] +MANIFEST = ROOT / "docs" / "claims" / "claim_boundary_manifest.v0.1.json" +OUT_DIR = ROOT / "evidence-cards" +SITE_MANIFEST = OUT_DIR / "site-evidence-manifest.v1.json" + +#: Commands that are safe to execute under --run. Anything not matching is +#: recorded as unverifiable rather than being shelled out blindly: a card +#: generator that runs arbitrary strings from a data file is a code-execution +#: surface, not an evidence tool. +RUNNABLE_PREFIXES = ("pytest ", "python ", "make ") + + +def _git_revision() -> str: + try: + out = subprocess.run( + ["git", "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=False, + cwd=ROOT, + ) + revision = out.stdout.strip() + except OSError: + revision = "" + if not revision: + return "unknown-revision" + dirty = subprocess.run( + ["git", "status", "--porcelain"], + capture_output=True, + text=True, + check=False, + cwd=ROOT, + ).stdout.strip() + # A dirty tree is recorded, not hidden: a card bound to "abc123" when the + # working tree differs from abc123 is bound to the wrong thing. + return f"{revision}-dirty" if dirty else revision + + +def _artifact(path_str: str) -> ArtifactRef: + path = ROOT / path_str + if not path.is_file(): + return ArtifactRef(path=path_str, missing=True) + data = path.read_bytes() + return ArtifactRef( + path=path_str, + sha256=hashlib.sha256(data).hexdigest(), + bytes_len=len(data), + ) + + +#: pytest exit code 1 means "tests ran and some failed" -- the one non-zero exit +#: this harness can confidently attribute to a check performing and not +#: succeeding. Codes 2-5 mean usage error, internal error, interruption, or no +#: tests collected: in each the check did not run, which is `unverifiable`, not +#: `fail`. Everything else is likewise unattributable. +PYTEST_TESTS_FAILED = 1 + + +def _run_one(command: str, env: dict[str, str]) -> tuple[str, str]: + """Run a single command. Returns (outcome, one-line record).""" + if not command.startswith(RUNNABLE_PREFIXES): + return ( + "unverifiable", + f"{command} -> not executed: outside the runnable allowlist {RUNNABLE_PREFIXES}", + ) + argv = shlex.split(command) + is_pytest = argv[0] == "pytest" + if is_pytest: + argv = [sys.executable, "-m", *argv] + elif argv[0] == "python": + argv = [sys.executable, *argv[1:]] + proc = subprocess.run(argv, capture_output=True, text=True, check=False, cwd=ROOT, env=env) + tail = (proc.stdout or proc.stderr).strip().splitlines() + summary = (tail[-1] if tail else "")[:160] + record = f"{command} -> exit {proc.returncode}: {summary}" + if proc.returncode == 0: + return "pass", record + if is_pytest and proc.returncode == PYTEST_TESTS_FAILED: + return "fail", record + return "unverifiable", record + " | exit not attributable to a check running" + + +def _run_commands(commands: list[str]) -> tuple[str, str]: + """Run every command and derive the card verdict conservatively. + + Each command is recorded with its own outcome, so a claim whose tests pass + but whose `make` target cannot run here reports both facts rather than + collapsing to one. The card verdict is the worst outcome present: + + * any `fail` -> the card fails + * else any `unverifiable` -> the card is unverifiable + * else -> the card passes + + The harness never guesses a `fail`. A non-zero exit becomes `fail` only + when it can be attributed to a check running and not succeeding -- pytest + exit code 1. Everything else is `unverifiable`, which is a different fact + about the world. + + This is not pedantry. On the first run of this script `make test-kernel` + exited non-zero because its dependency-install step could not reach the + network, and the kernel tests never executed. Reporting that as `fail` + would have published a failure nobody observed. + """ + env = {**os.environ, "PYTHONPATH": str(ROOT / "src")} + outcomes: list[str] = [] + records: list[str] = [] + for command in commands: + outcome, record = _run_one(command, env) + outcomes.append(outcome) + records.append(record) + + if "fail" in outcomes: + verdict = "fail" + elif "unverifiable" in outcomes: + verdict = "unverifiable" + else: + verdict = "pass" + + tally = ", ".join( + f"{o}={outcomes.count(o)}" for o in ("pass", "fail", "unverifiable") if outcomes.count(o) + ) + return verdict, f"[{tally}] " + " | ".join(records) + + +def build(*, run: bool) -> tuple[list[EvidenceCard], dict[str, Any]]: + manifest = json.loads(MANIFEST.read_text(encoding="utf-8")) + revision = _git_revision() + cards: list[EvidenceCard] = [] + + for claim in manifest["claims"]: + commands = tuple(claim["supporting_tests_or_commands"]) + if run: + verdict, detail = _run_commands(list(commands)) + else: + verdict, detail = "not-run", None + + cards.append( + EvidenceCard( + card_id=claim["id"], + claim=claim["claim_text"], + maturity=claim["level"], + source_revision=revision, + command=commands, + artifacts=tuple(_artifact(p) for p in claim["supporting_files"]), + falsifier=claim["falsifier"], + assumptions=tuple(claim["assumptions"]), + non_claims=tuple(claim["non_claims"]), + evidence_state=claim["evidence_state"], + verdict=verdict, + # Every card starts as a draft. Release is a human decision made + # in the publication workflow, not a side effect of generation. + publication_state="draft", + result_detail=detail, + ) + ) + + note = "Generated from docs/claims/claim_boundary_manifest.v0.1.json. " + ( + "Verdicts reflect commands executed during generation." + if run + else "Verdicts are 'not-run': no command was executed during generation." + ) + return cards, cards_to_site_manifest(cards, source_revision=revision, generated_note=note) + + +def _write(site: dict[str, Any], cards: list[EvidenceCard]) -> None: + OUT_DIR.mkdir(exist_ok=True) + (OUT_DIR / "cards").mkdir(exist_ok=True) + for card in cards: + path = OUT_DIR / "cards" / f"{card.card_id}.json" + path.write_text( + json.dumps(card.to_json(), indent=2, sort_keys=True, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + SITE_MANIFEST.write_text( + json.dumps(site, indent=2, sort_keys=True, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + + +def _stable(payload: dict[str, Any]) -> dict[str, Any]: + """Drop fields that legitimately change between runs, for --check.""" + out = json.loads(json.dumps(payload)) + out.pop("source_revision", None) + out.pop("generated_note", None) + for card in out.get("cards", []): + card.pop("source_revision", None) + return out + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--run", + action="store_true", + help="execute each claim's commands and record real verdicts. Results are " + "host-specific, so the COMMITTED cards are always the not-run scaffold; " + "regenerate without --run before committing.", + ) + parser.add_argument( + "--check", + action="store_true", + help="verify the committed cards match the manifest (ignoring revision)", + ) + args = parser.parse_args(argv) + + cards, site = build(run=args.run) + + if args.check: + if not SITE_MANIFEST.exists(): + print("site manifest missing; run without --check", file=sys.stderr) + return 1 + committed = json.loads(SITE_MANIFEST.read_text(encoding="utf-8")) + if _stable(committed) != _stable(site): + print( + "evidence cards are stale; regenerate with:\n" + " python scripts/build_evidence_cards.py", + file=sys.stderr, + ) + return 1 + print(f"evidence cards current: {site['card_count']} cards") + return 0 + + _write(site, cards) + print(f"wrote {site['card_count']} evidence cards to {OUT_DIR.relative_to(ROOT)}") + if args.run: + print( + " NOTE: verdicts are specific to this host and toolchain. The " + "committed\n cards are the not-run scaffold -- regenerate " + "without --run before committing." + ) + for label, counts in site["counts_by_label"].items(): + shown = ", ".join(f"{k}={v}" for k, v in counts.items() if v) + print(f" {label:20s} {shown}") + print("\n Counts are per label and orthogonal. There is no aggregate score,") + print(" and any surface computing one from this file is misusing it.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/canonicalization_probe.py b/scripts/canonicalization_probe.py new file mode 100755 index 00000000..130a2668 --- /dev/null +++ b/scripts/canonicalization_probe.py @@ -0,0 +1,462 @@ +#!/usr/bin/env python3 +"""Adversarial census over the CC report canonicalization kernel. + +Modelled on Ghost-Ark's E1 provenance-kernel census. For each pathology class we +declare, *before* observing anything, whether a consumer needs the two documents +distinguished (``distinct``) or unified (``equivalent``). The verdict compares +the observation against that declared intent: + +``sound`` + The kernel did what the declared consumer intent requires. +``unintended-kernel`` + Two documents a consumer needs distinguished received the same canonical + identity. This is the dangerous verdict: it issues a false shared identity. +``over-discrimination`` + Two documents every consumer treats as identical received different + identities. Breaks replay determinism; does not forge identity. +``fail-closed`` + Both documents were refused. No false identity is issued. +``sound-by-rejection`` + One side was admitted and the other refused where a consumer needs them + distinguished. No false identity is issued and the honest document still + receives one. +``rejection-asymmetry`` + One of two documents every consumer treats as identical was refused. This is + the failure mode of an over-strict rule, and it is what distinguishes a fix + from a trade. + +Provenance of this corpus is ``census``: the classes are curated and adversarial, +the population is exactly what is written here, and its size is an authoring +decision. Exact counts only. **No confidence intervals** -- attaching one to a +hand-authored corpus would be precisely the error `cc.kernel.cliff` refuses to +make elsewhere in this repository. + +Exit status is non-zero while any class carries ``unintended-kernel`` or +``rejection-asymmetry``, so the findings in +``docs/upgrade/FINDINGS_REGISTER.md`` are falsifiable rather than asserted: fix +the canonicalizer and this probe goes green. + +Usage:: + + PYTHONPATH=src python scripts/canonicalization_probe.py + PYTHONPATH=src python scripts/canonicalization_probe.py --json +""" + +from __future__ import annotations + +import argparse +import json +import sys +from dataclasses import dataclass +from typing import Any, Literal + +from cc.reporting.canonical import ( + LEGACY_SORT_KEYS, + RFC8785, + DuplicateJSONKeyError, + canonical_json_bytes, + strict_json_loads, +) + +Intent = Literal["distinct", "equivalent"] + +# Two representations of the same grapheme: precomposed U+00E9, and the +# decomposed pair U+0065 U+0301. They are different byte sequences that NFC +# normalization maps to the same string. +KEY_NFC = "é" +KEY_NFD = "é" + + +@dataclass(frozen=True) +class PathologyClass: + """One declared-intent pair of documents.""" + + name: str + intent: Intent + left: dict[str, Any] + right: dict[str, Any] + note: str + + +# Declared BEFORE observation. Editing an intent to match a measured result +# requires editing this list, which surfaces it in review. +CLASSES: tuple[PathologyClass, ...] = ( + PathologyClass( + name="unicode-key-collision", + intent="distinct", + left={KEY_NFC: 1}, + right={KEY_NFD: 1}, + note="Distinct source keys whose NFC forms coincide.", + ), + PathologyClass( + name="integer-above-2^53", + intent="distinct", + left={"n": 2**53 + 1}, + right={"n": 2**53 + 2}, + note="v1 emitted both, but a JS JSON.parse collapses them before any " + "verifier code runs -- the kernel is set by the parser, and no " + "downstream fix reaches it. v2 refuses both rather than emitting bytes " + "that cannot survive a round trip (F-05).", + ), + PathologyClass( + name="int-vs-float-same-value", + intent="equivalent", + left={"n": 1}, + right={"n": 1.0}, + note="INTENT CORRECTED 2026-08-19, from 'distinct' to 'equivalent'. The " + "original declaration described Python's type system, not JSON's: JSON " + "has exactly one number type, so 1 and 1.0 are the same JSON number, and " + "a consumer needing them distinguished is asking JSON for something it " + "does not provide. Correcting a declaration because it was wrong about " + "the domain is legitimate; correcting one to flatter a measured result " + "is not. This is the former, recorded here rather than edited away.", + ), + PathologyClass( + name="negative-zero", + intent="equivalent", + left={"n": 0.0}, + right={"n": -0.0}, + note="IEEE-754 -0.0 == 0.0. scipy.optimize.linprog returns -0.0 at a " + "zero lower bound, so this reaches real reports.", + ), + PathologyClass( + name="bool-vs-int", + intent="distinct", + left={"n": True}, + right={"n": 1}, + note="Python bool is a subclass of int; JSON true is not 1.", + ), + PathologyClass( + name="float-exponent-form", + intent="distinct", + left={"n": 1e30}, + right={"n": 10**30}, + note="A float and a Python int of the same magnitude. Under v2 the int " + "is refused for exceeding the IEEE-754 safe range, so no false shared " + "identity is issued and the float still receives one.", + ), + PathologyClass( + name="safe-integer-neighbours", + intent="distinct", + left={"n": 2**53 - 2}, + right={"n": 2**53 - 1}, + note="Positive control: adjacent integers inside the safe range must stay distinct.", + ), + PathologyClass( + name="object-key-order", + intent="equivalent", + left={"a": 1, "b": 2}, + right={"b": 2, "a": 1}, + note="Positive control: key order carries no meaning in JSON.", + ), + PathologyClass( + name="array-element-order", + intent="distinct", + left={"xs": [1, 2]}, + right={"xs": [2, 1]}, + note="Positive control: arrays are ordered; no arm may sort them.", + ), + PathologyClass( + name="nested-unicode-key-collision", + intent="distinct", + left={"outer": {KEY_NFC: 1}}, + right={"outer": {KEY_NFD: 1}}, + note="A guard inspecting only top-level keys would pass the flat class " + "while leaving this fully exploitable.", + ), + PathologyClass( + name="utf16-vs-codepoint-key-order", + intent="distinct", + left={"\ufffd": 1, "\U00010000": 2}, + right={"\ufffd": 2, "\U00010000": 1}, + note="Positive control for key ordering. RFC 8785 sorts by UTF-16 code " + "unit, not code point, and the orders disagree above the BMP: U+10000 " + "encodes as D800 DC00, so it sorts before U+FFFD under UTF-16 and after " + "it under code point. Different values under the same keys must stay " + "distinct whichever order is used.", + ), + PathologyClass( + name="control-character-escape", + intent="distinct", + left={"k": "a\u0001b"}, + right={"k": "a\u0002b"}, + note="Positive control: C0 control characters are escaped as \\u00XX " + "and must not collapse to one another.", + ), + PathologyClass( + name="tab-vs-escaped-tab-text", + intent="distinct", + left={"k": "a\tb"}, + right={"k": "a\\tb"}, + note="Positive control: a real tab and the two-character text " + "backslash-t are different strings and must stay distinct after escaping.", + ), + PathologyClass( + name="large-document-single-byte", + intent="distinct", + left={"pad": "x" * 65536, "tail": "a"}, + right={"pad": "x" * 65536, "tail": "b"}, + note="Positive control: catches a digest computed over a prefix.", + ), +) + + +def _canonical(doc: dict[str, Any], profile: str) -> bytes | None: + """Canonical bytes, or None when the document is refused.""" + try: + return canonical_json_bytes(doc, profile=profile) # type: ignore[arg-type] + except Exception: # any refusal is a refusal, for this census + return None + + +def _verdict(intent: Intent, left: bytes | None, right: bytes | None) -> str: + if left is None and right is None: + return "fail-closed" + if left is None or right is None: + return "sound-by-rejection" if intent == "distinct" else "rejection-asymmetry" + if left == right: + return "sound" if intent == "equivalent" else "unintended-kernel" + return "sound" if intent == "distinct" else "over-discrimination" + + +def _observation(left: bytes | None, right: bytes | None) -> str: + if left is None and right is None: + return "rejected-both" + if left is None or right is None: + return "rejected-one" + return "collapsed" if left == right else "distinct" + + +def run(profile: str) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for cls in CLASSES: + left = _canonical(cls.left, profile) + right = _canonical(cls.right, profile) + rows.append( + { + "class": cls.name, + "intent": cls.intent, + "observed": _observation(left, right), + "verdict": _verdict(cls.intent, left, right), + "note": cls.note, + } + ) + return rows + + +def silent_key_loss(profile: str) -> dict[str, Any]: + """F-03: under v1 a single object carrying both key forms loses one, silently. + + Under v2 nothing is normalized, so the two keys stay two keys and the + canonical bytes carry both -- which is what JSON says the document is. + """ + both = {KEY_NFC: 1, KEY_NFD: 2} + try: + out = canonical_json_bytes(both, profile=profile) # type: ignore[arg-type] + except Exception as exc: + return {"keys_in": len(both), "refused": True, "error": type(exc).__name__} + return { + "keys_in": len(both), + "keys_out": len(strict_json_loads(out)), + "refused": False, + "output": out.decode("utf-8"), + } + + +def duplicate_key_parse() -> dict[str, Any]: + """F-07: the permissive parser keeps the last key; the strict one refuses.""" + raw = '{"amount":1,"amount":2}' + permissive = json.loads(raw) + try: + strict_json_loads(raw) + except DuplicateJSONKeyError: + strict_refused = True + else: + strict_refused = False + return { + "raw": raw, + "permissive_parse": permissive, + "permissive_collides_with_survivor": canonical_json_bytes(permissive) + == canonical_json_bytes({"amount": 2}), + "strict_json_loads_refuses": strict_refused, + } + + +# Expected serializations under **cc.canonical.v2**, which is RFC 8785 plus one +# declared narrowing: a Python ``int`` outside the IEEE-754 safe integer range +# is refused rather than emitted. JCS is defined over JSON numbers, which are +# doubles; Python ints have no such bound, so the profile must supply one. The +# bound is the standard ``2**53 - 1`` -- the largest n for which both n and n+1 +# are exactly representable -- which conservatively also refuses 2**53 itself. +# +# ``None`` means "must be refused". A refusal that JCS would have serialized is +# a narrowing, and it is declared in docs/architecture/CANONICAL_PROFILE.md +# rather than left as an undocumented difference. +JCS_EXPECTED: tuple[tuple[Any, str | None], ...] = ( + (1e30, "1e+30"), + (10**30, None), # int beyond the safe range + (1.0, "1"), + (-0.0, "0"), + (1e-7, "1e-7"), + (100.0, "100"), + (0.1, "0.1"), + (1e21, "1e+21"), + (1e20, "100000000000000000000"), # float: no int bound applies + (5e-324, "5e-324"), + (1.7976931348623157e308, "1.7976931348623157e+308"), + (float(2**53), "9007199254740992"), # float is unaffected by the int rule + (2**53, None), # int at the boundary: conservatively refused + (2**53 - 1, "9007199254740991"), # the largest int the profile accepts + (2**53 + 1, None), +) + + +def jcs_conformance(profile: str) -> list[dict[str, Any]]: + """F-04: how the emitted number form compares with RFC 8785.""" + rows: list[dict[str, Any]] = [] + for value, expected in JCS_EXPECTED: + try: + emitted: str | None = canonical_json_bytes({"n": value}, profile=profile).decode( # type: ignore[arg-type] + "utf-8" + )[5:-1] + except Exception: + emitted = None + rows.append( + { + "value": repr(value), + "emitted": emitted, + "jcs_expects": expected, + "conformant": emitted == expected, + } + ) + return rows + + +FAILING_VERDICTS = {"unintended-kernel", "rejection-asymmetry"} + + +PROFILES: dict[str, str] = {"v1": LEGACY_SORT_KEYS, "v2": RFC8785} + + +def census(profile_name: str) -> dict[str, Any]: + profile = PROFILES[profile_name] + rows = run(profile) + counts: dict[str, int] = {} + for row in rows: + counts[row["verdict"]] = counts.get(row["verdict"], 0) + 1 + return { + "profile": profile_name, + "profile_id": profile, + "classes": rows, + "verdict_counts": counts, + "failing": sum(counts.get(v, 0) for v in FAILING_VERDICTS), + "silent_key_loss": silent_key_loss(profile), + "jcs_conformance": jcs_conformance(profile), + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--json", action="store_true", help="emit machine-readable JSON") + parser.add_argument( + "--profile", + choices=("v2", "v1", "both"), + default="both", + help="which canonicalization profile to census (default: both, so the " + "migration is visible)", + ) + args = parser.parse_args(argv) + + names = ["v1", "v2"] if args.profile == "both" else [args.profile] + results = {name: census(name) for name in names} + + payload = { + "provenance": "census", + "confidence_intervals": None, + "confidence_interval_non_claim": ( + "This corpus is the whole population and its size is an authoring " + "decision. A confidence interval would describe sampling " + "variability that does not exist here." + ), + "profiles": results, + "duplicate_key_parse": duplicate_key_parse(), + "gated_profile": "v2", + "gate_non_claim": ( + "A clean census establishes that the declared classes behave as " + "declared. It does not establish that the kernel has no other " + "members: the corpus is curated, and its coverage is an authoring " + "decision, not a measurement." + ), + } + + # Only the default profile gates. v1 is retained read-only for historical + # receipts and is EXPECTED to fail -- that is why it was replaced. + failed = results["v2"]["failing"] if "v2" in results else 0 + + if args.json: + json.dump(payload, sys.stdout, indent=2, ensure_ascii=False) + sys.stdout.write("\n") + return 1 if failed else 0 + + print("cc-framework canonicalization kernel census") + print("provenance: census - exact counts only, no confidence intervals\n") + + for name in names: + result = results[name] + gated = " (GATED)" if name == "v2" else " (read-only, historical)" + print(f"profile {name}{gated} {result['profile_id']}") + width = max(len(r["class"]) for r in result["classes"]) + for row in result["classes"]: + flag = " <<<" if row["verdict"] in FAILING_VERDICTS else "" + print( + f" {row['class']:<{width}} intent={row['intent']:<10}" + f" observed={row['observed']:<13} {row['verdict']}{flag}" + ) + counts = result["verdict_counts"] + print(" counts: " + ", ".join(f"{k}={counts[k]}" for k in sorted(counts))) + + loss = result["silent_key_loss"] + if loss["refused"]: + print(f" both-key-forms document: refused with {loss['error']}") + elif loss["keys_out"] < loss["keys_in"]: + print( + f" both-key-forms document: {loss['keys_in']} keys in -> " + f"{loss['keys_out']} out, NO EXCEPTION <<< silent key loss" + ) + else: + print( + f" both-key-forms document: {loss['keys_in']} keys in -> " + f"{loss['keys_out']} out, no loss" + ) + + nonconf = [r for r in result["jcs_conformance"] if not r["conformant"]] + print( + f" RFC 8785 number forms: " + f"{len(result['jcs_conformance']) - len(nonconf)}" + f"/{len(result['jcs_conformance'])} conformant" + ) + for row in nonconf: + print( + f" diverges {row['value']:<12} emitted={row['emitted']!s:<26}" + f" jcs={row['jcs_expects']!s}" + ) + print("") + + dup = payload["duplicate_key_parse"] + print("duplicate keys on the parse side:") + print(f" {dup['raw']} -> json.loads gives {dup['permissive_parse']}") + print(f" strict_json_loads refuses it: {dup['strict_json_loads_refuses']}") + print("") + + if failed: + print( + f"FAIL: profile v2 has {failed} class(es) at {' or '.join(sorted(FAILING_VERDICTS))}." + ) + return 1 + print("PASS: profile v2 has no unintended-kernel or rejection-asymmetry class.") + print("\n " + payload["gate_non_claim"].replace("(.{70}) ", "")) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/differential_compose.py b/scripts/differential_compose.py new file mode 100755 index 00000000..c063cb0f --- /dev/null +++ b/scripts/differential_compose.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python3 +"""Differential fuzz between the Python and Node composition kernels. + +The conformance corpus is a curated census: it checks the cases someone thought +to write down. This harness checks the cases nobody thought of, by generating +randomized inputs -- including malformed ones -- and requiring both +implementations to agree on the answer *or* on the refusal. + +The Node side never sees the Python answers. It receives inputs on stdin and +returns its own results, so it cannot converge on the reference. + +Seeds are reported. A disagreement is reproducible with ``--seed``. + +**What agreement establishes.** That two implementations of one specification +compute the same values over the sampled region. **What it does not +establish.** That either is correct -- both were authored in the same project +and a wrong specification yields two implementations wrong together. Nor does it +establish coverage: the generator's distribution is stated below and it has +blind spots. + +Usage:: + + python scripts/differential_compose.py --cases 2000 + python scripts/differential_compose.py --seed 7 --cases 500 --json +""" + +from __future__ import annotations + +import argparse +import json +import math +import random +import shutil +import subprocess +import sys +from pathlib import Path +from typing import Any + +from cc.compose import CountermonotoneUndefinedError, compose_bounds + +ROOT = Path(__file__).resolve().parents[1] +VERIFIER = ROOT / "verifiers" / "node" / "cc_compose_verify.mjs" + +TOLERANCE = 1.0e-12 + +EVENTS = ("all", "any", "and", "or", "AND", "OR", "intersection", "union") +DEPENDENCE = ("unconstrained", "independent", "comonotone", "countermonotone") + +#: What the generator can produce. Stated before the numbers, because a +#: generator's coverage bounds what its agreement means. +GENERATOR_COVERAGE = ( + "Marginals drawn from a mixture: uniform on [0,1], values snapped to the " + "boundaries 0 and 1, values within 1e-9 of a boundary, and near-tied " + "values that exercise the binding_event tie rule. Event counts 1-8. All " + "event-kind aliases and all four dependence assumptions. Roughly one case " + "in six is deliberately malformed: out-of-range, non-finite, empty, or an " + "unknown event/dependence value." +) + +#: What it cannot produce. A generator's blind spots are part of its result. +GENERATOR_BLIND_SPOTS = ( + "Event counts above 8.", + "Marginals that are not IEEE-754 doubles (no decimal or rational inputs).", + "Non-string event names, and names differing only by Unicode normalization " + "-- the canonicalization kernel's collapse is probed separately by " + "scripts/canonicalization_probe.py.", + "Side constraints beyond marginals: the corpus and this harness both cover " + "the closed-form path only, not the constrained LP path.", + "Adversarially chosen floating-point values selected to maximize " + "divergence; the draw is random, not searched.", +) + + +def _draw_marginal(rng: random.Random) -> float: + roll = rng.random() + if roll < 0.08: + return 0.0 + if roll < 0.16: + return 1.0 + if roll < 0.24: + return rng.choice([1e-9, 1.0 - 1e-9, 1e-15, 5e-324]) + return rng.random() + + +def _generate(rng: random.Random, index: int) -> dict[str, Any]: + n = rng.randint(1, 8) + names = [f"E{i}" for i in range(n)] + values = [_draw_marginal(rng) for _ in range(n)] + + # Near-ties exercise the binding_event tie rule from SPEC section 4.3. + if n >= 2 and rng.random() < 0.25: + values[1] = values[0] + + case: dict[str, Any] = { + "id": f"fuzz-{index}", + "marginals": dict(zip(names, values, strict=True)), + "event": rng.choice(EVENTS), + "dependence": rng.choice(DEPENDENCE), + } + + # Roughly one in six is malformed, so refusal agreement is exercised too. + roll = rng.random() + if roll < 0.04: + case["marginals"][names[0]] = rng.choice([1.5, -0.1, 2.0, -1e-6]) + elif roll < 0.07: + case["marginals"][names[0]] = "NaN" + elif roll < 0.10: + case["marginals"] = {} + elif roll < 0.13: + case["event"] = rng.choice(["xor", "nand", "", "ALL"]) + elif roll < 0.16: + case["dependence"] = rng.choice(["mixed", "", "Independent"]) + return case + + +def _python_side(case: dict[str, Any]) -> dict[str, Any]: + marginals = {k: (math.nan if v == "NaN" else v) for k, v in case["marginals"].items()} + try: + bounds = compose_bounds(marginals, event=case["event"], dependence=case["dependence"]) + except CountermonotoneUndefinedError: + return {"ok": False, "refusal": "countermonotone_undefined"} + except (ValueError, TypeError) as exc: + return {"ok": False, "refusal": _classify(str(exc))} + payload = bounds.to_json() + return {"ok": True, **payload} + + +def _classify(message: str) -> str: + """Map a Python message onto a SPEC section 5 refusal identifier.""" + lowered = message.lower() + if "at least one event" in lowered or "mapping" in lowered: + return "no_events" + if "must be finite" in lowered: + return "marginal_not_finite" + if "must lie in [0, 1]" in lowered: + return "marginal_out_of_range" + if "unknown event kind" in lowered: + return "unknown_event_kind" + if "unknown dependence" in lowered: + return "unknown_dependence" + if "real number" in lowered: + return "marginal_not_finite" + return f"unclassified:{message[:60]}" + + +def _node_side(cases: list[dict[str, Any]]) -> list[dict[str, Any]]: + node = shutil.which("node") + if node is None: + raise SystemExit("node is not on PATH; the differential harness needs it") + proc = subprocess.run( + [node, str(VERIFIER), "--batch"], + input=json.dumps({"cases": cases}), + capture_output=True, + text=True, + check=False, + ) + if proc.returncode != 0: + raise SystemExit(f"node verifier failed:\n{proc.stdout}\n{proc.stderr}") + return json.loads(proc.stdout)["results"] + + +_COMPARED = ( + "lower", + "upper", + "width", + "independence_point", + "independence_regret", + "understatement_factor", + "binding_event", +) + + +def _compare(case: dict[str, Any], py: dict[str, Any], js: dict[str, Any]) -> list[dict[str, Any]]: + if py["ok"] != js["ok"]: + return [ + { + "id": case["id"], + "field": "", + "python": "accepted" if py["ok"] else py["refusal"], + "node": "accepted" if js["ok"] else js["refusal"], + "input": case, + } + ] + if not py["ok"]: + if py["refusal"] != js["refusal"]: + return [ + { + "id": case["id"], + "field": "", + "python": py["refusal"], + "node": js["refusal"], + "input": case, + } + ] + return [] + + out: list[dict[str, Any]] = [] + for field in _COMPARED: + a, b = py[field], js[field] + if field == "binding_event" or a is None or b is None: + if a != b: + out.append( + {"id": case["id"], "field": field, "python": a, "node": b, "input": case} + ) + continue + if abs(a - b) > TOLERANCE: + out.append( + { + "id": case["id"], + "field": field, + "python": a, + "node": b, + "delta": abs(a - b), + "input": case, + } + ) + return out + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--cases", type=int, default=2000, help="number of randomized cases") + parser.add_argument("--seed", type=int, default=None, help="seed; random if omitted") + parser.add_argument("--json", action="store_true", help="machine-readable output") + args = parser.parse_args(argv) + + seed = args.seed if args.seed is not None else random.randrange(2**31) + rng = random.Random(seed) + cases = [_generate(rng, i) for i in range(args.cases)] + + py_results = [_python_side(c) for c in cases] + js_results = _node_side(cases) + + disagreements: list[dict[str, Any]] = [] + for case, py, js in zip(cases, py_results, js_results, strict=True): + disagreements.extend(_compare(case, py, js)) + + accepted = sum(1 for r in py_results if r["ok"]) + report = { + "seed": seed, + "cases": len(cases), + "accepted": accepted, + "refused": len(cases) - accepted, + "disagreements": len(disagreements), + "tolerance": TOLERANCE, + "generator_coverage": GENERATOR_COVERAGE, + "generator_blind_spots": list(GENERATOR_BLIND_SPOTS), + "provenance": "sampled", + "non_claim": ( + "Agreement over a sampled region does not establish correctness. " + "Both implementations were authored in the same project and a wrong " + "specification produces two implementations that are wrong together. " + "Coverage is bounded by the generator described above." + ), + "detail": disagreements[:20], + } + + if args.json: + json.dump(report, sys.stdout, indent=2) + sys.stdout.write("\n") + return 1 if disagreements else 0 + + print("differential fuzz — Python vs Node composition kernel") + print(f" seed {seed} (reproduce with --seed {seed})") + print(f" cases {len(cases)} ({accepted} accepted, {len(cases) - accepted} refused)") + print(f" tolerance {TOLERANCE:g}") + print(" provenance sampled") + print("") + if disagreements: + for row in disagreements[:20]: + print(f" DISAGREE {json.dumps(row, default=str)[:220]}") + if len(disagreements) > 20: + print(f" ... and {len(disagreements) - 20} more") + print(f"\nFAIL: {len(disagreements)} disagreement(s).") + return 1 + print("PASS: no disagreement.") + print("\ngenerator coverage:") + for line in GENERATOR_COVERAGE.split(". "): + if line.strip(): + print(f" {line.strip().rstrip('.')}.") + print("\ngenerator blind spots:") + for spot in GENERATOR_BLIND_SPOTS: + print(f" - {spot}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validate_claim_boundary_manifest.py b/scripts/validate_claim_boundary_manifest.py index 107800b0..c8c94466 100644 --- a/scripts/validate_claim_boundary_manifest.py +++ b/scripts/validate_claim_boundary_manifest.py @@ -1,11 +1,11 @@ from __future__ import annotations import json +import re import sys from pathlib import Path from typing import Any - ROOT = Path(__file__).resolve().parents[1] MANIFEST_PATH = ROOT / "docs" / "claims" / "claim_boundary_manifest.v0.1.json" @@ -29,8 +29,17 @@ "supporting_tests_or_commands", "non_claims", "risk_if_overstated", + # Required so every claim can be rendered as an evidence card without the + # renderer inventing anything. A claim with no falsifier is an assertion. + "evidence_state", + "falsifier", + "assumptions", } +#: Where a claim's evidence came from. Mirrors cc.evidence_card.EVIDENCE_STATES; +#: a test asserts the two lists agree. +EVIDENCE_STATES = ("local-only", "aws-synth-only", "aws-live", "illustrative") + def load_manifest(path: Path = MANIFEST_PATH) -> dict[str, Any]: return json.loads(path.read_text(encoding="utf-8")) @@ -94,6 +103,34 @@ def validate_manifest(manifest: dict[str, Any], root: Path = ROOT) -> list[str]: tests = claim.get("supporting_tests_or_commands") if not isinstance(tests, list) or not tests: errors.append(f"{claim_id}: supporting_tests_or_commands must be a non-empty list") + else: + # Path-like tokens in a command must resolve. A renamed test would + # otherwise leave the claim pointing at nothing. + for entry in tests: + if not isinstance(entry, str) or not entry.strip(): + errors.append(f"{claim_id}: command entries must be non-empty strings") + continue + for token in re.findall(r"(?:tests|src|scripts|examples)/[\w/.\-]+", entry): + if not (root / token.split("::")[0]).exists(): + errors.append(f"{claim_id}: command names a missing path: {token}") + + evidence_state = claim.get("evidence_state") + if evidence_state not in EVIDENCE_STATES: + errors.append( + f"{claim_id}: evidence_state must be one of {EVIDENCE_STATES}, " + f"got {evidence_state!r}" + ) + + falsifier = claim.get("falsifier") + if not isinstance(falsifier, str) or not falsifier.strip(): + errors.append( + f"{claim_id}: falsifier must be a non-empty string. A claim no " + "observation could refute is an assertion, not evidence." + ) + + assumptions = claim.get("assumptions") + if not isinstance(assumptions, list) or not assumptions: + errors.append(f"{claim_id}: assumptions must be a non-empty list") forbidden_upgrades = manifest.get("forbidden_upgrades") if not isinstance(forbidden_upgrades, list) or not forbidden_upgrades: diff --git a/src/cc/cli/guard.py b/src/cc/cli/guard.py new file mode 100644 index 00000000..28027801 --- /dev/null +++ b/src/cc/cli/guard.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python3 +"""``cc-guard`` -- the inference guards, reachable without a Python API. + +A downstream project asked, in writing, to route its density estimate through +this repository's post-selection refusal so that an optional-stopping error +would be *refused* rather than reported. It could not: the guard existed only +as an in-process Python call, and the caller was JavaScript. + +This module is the answer. It exposes the guards two ways, because one is not +enough: + +1. ``cc-guard check`` reads a JSON request on stdin and writes a verdict on + stdout. Any language that can spawn a process can use it. +2. ``cc-guard table`` emits the decision rules as **pure data**, so a caller + can enforce them with no Python process at all. A subprocess dependency is a + weaker bridge than a table -- it fails in CI sandboxes, in browsers, and + anywhere Python is not installed. + +The guards refuse claims the supplied evidence cannot support. Refusal is the +product. A guard that can be talked into a confidence claim is not a guard. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from typing import Any + +__all__ = ["DECISION_TABLE", "check", "main"] + +#: The decision rules, as data. This is the normative statement of the guard; +#: :func:`check` implements it and ``tests/unit/cli/test_guard.py`` asserts the +#: two agree case for case, so the table cannot drift from the code. +#: +#: Each rule names the condition, the verdict, whether a confidence claim +#: survives, and the reason. A caller reimplementing this in another language +#: needs nothing beyond this structure. +DECISION_TABLE: dict[str, Any] = { + "version": "cc-guard-v1", + "guards": [ + { + "id": "post_selection_interval", + "applies_to": "An interval around an estimate that was selected by " + "a search -- a maximum, a best-fit, a scan over candidates, or any " + "procedure with optional stopping.", + "input_fields": ["provenance", "estimate", "ci_low", "ci_high"], + "rules": [ + { + "when": {"provenance": "post-selection"}, + "verdict": "discovery-only", + "confidence_claim_permitted": False, + "reason": "The estimate is the maximum over a search, so it " + "is biased upward and an interval around it does not attain " + "its nominal level for the true parameter. A discovery is a " + "hypothesis; only held-out data can certify it.", + "remedy": "Re-estimate on data not used for selection, then " + "resubmit with provenance='confirmatory'.", + }, + { + "when": {"provenance": "confirmatory"}, + "verdict": "confirmatory", + "confidence_claim_permitted": True, + "reason": "The interval was computed on data not used to " + "select the target, so its nominal level applies.", + "remedy": None, + }, + ], + "default": { + "verdict": "refused", + "confidence_claim_permitted": False, + "reason": "Unknown provenance. A guard that defaults to " + "permitting a claim is not a guard.", + "remedy": "Declare provenance as 'confirmatory' or 'post-selection'.", + }, + }, + { + "id": "census_interval", + "applies_to": "A proportion computed over a curated corpus rather " + "than a random sample.", + "input_fields": ["provenance", "n"], + "rules": [ + { + "when": {"provenance": "census"}, + "verdict": "no-interval", + "confidence_claim_permitted": False, + "reason": "A confidence interval describes sampling " + "variability under repeated random draws. A hand-authored " + "corpus has none: it is the whole population and its size is " + "an authoring decision.", + "remedy": "Report exact counts and the denominator.", + }, + { + "when": {"provenance": "sampled", "n_below": 30}, + "verdict": "no-interval", + "confidence_claim_permitted": False, + "reason": "Below n=30 a normal-approximation interval is not " + "trustworthy, and a Wilson interval is wide enough that " + "quoting it invites misreading.", + "remedy": "Report exact counts, or collect more samples.", + }, + { + "when": {"provenance": "sampled"}, + "verdict": "interval-permitted", + "confidence_claim_permitted": True, + "reason": "A genuine sample of adequate size supports a proportion interval.", + "remedy": None, + }, + ], + "default": { + "verdict": "refused", + "confidence_claim_permitted": False, + "reason": "Unknown provenance. Declare whether the denominator " + "is a census or a sample.", + "remedy": "Declare provenance as 'census' or 'sampled'.", + }, + }, + ], + "non_claims": [ + "A permitted verdict means this guard found no reason to refuse. It " + "does not mean the estimate is correct, the model is right, or the " + "measurement was well designed.", + "These guards check the provenance a caller declares. They cannot " + "detect a caller who declares 'confirmatory' for a post-selection " + "interval.", + ], +} + +#: The minimum sample size below which no proportion interval is permitted. +#: Matches the ``n_below`` threshold in the census guard above. +MIN_N_FOR_PROPORTION_INTERVAL = 30 + + +def _post_selection(request: dict[str, Any]) -> dict[str, Any]: + provenance = request.get("provenance") + if provenance == "post-selection": + return { + "guard": "post_selection_interval", + "verdict": "discovery-only", + "confidence_claim": None, + "confidence_claim_permitted": False, + "reason": "The estimate is the maximum over a search, so it is " + "biased upward and an interval around it does not attain its " + "nominal level for the true parameter. A discovery is a hypothesis; " + "only held-out data can certify it.", + "remedy": "Re-estimate on data not used for selection, then " + "resubmit with provenance='confirmatory'.", + } + if provenance == "confirmatory": + return { + "guard": "post_selection_interval", + "verdict": "confirmatory", + "confidence_claim": request.get("ci_low") is not None + and request.get("ci_high") is not None, + "confidence_claim_permitted": True, + "reason": "The interval was computed on data not used to select the " + "target, so its nominal level applies.", + "remedy": None, + } + return { + "guard": "post_selection_interval", + "verdict": "refused", + "confidence_claim": None, + "confidence_claim_permitted": False, + "reason": f"Unknown provenance {provenance!r}. A guard that defaults to " + "permitting a claim is not a guard.", + "remedy": "Declare provenance as 'confirmatory' or 'post-selection'.", + } + + +def _census(request: dict[str, Any]) -> dict[str, Any]: + provenance = request.get("provenance") + if provenance == "census": + return { + "guard": "census_interval", + "verdict": "no-interval", + "confidence_claim_permitted": False, + "reason": "A confidence interval describes sampling variability " + "under repeated random draws. A hand-authored corpus has none: it " + "is the whole population and its size is an authoring decision.", + "remedy": "Report exact counts and the denominator.", + } + if provenance == "sampled": + n = request.get("n") + if not isinstance(n, int) or isinstance(n, bool) or n < 0: + return { + "guard": "census_interval", + "verdict": "refused", + "confidence_claim_permitted": False, + "reason": f"A sampled proportion needs a non-negative integer " + f"denominator; got {n!r}. No proportion without a denominator.", + "remedy": "Supply 'n' as the number of draws.", + } + if n < MIN_N_FOR_PROPORTION_INTERVAL: + return { + "guard": "census_interval", + "verdict": "no-interval", + "confidence_claim_permitted": False, + "reason": f"n={n} is below the minimum of " + f"{MIN_N_FOR_PROPORTION_INTERVAL}. Below that a " + "normal-approximation interval is not trustworthy, and a Wilson " + "interval is wide enough that quoting it invites misreading.", + "remedy": "Report exact counts, or collect more samples.", + } + return { + "guard": "census_interval", + "verdict": "interval-permitted", + "confidence_claim_permitted": True, + "reason": "A genuine sample of adequate size supports a proportion interval.", + "remedy": None, + } + return { + "guard": "census_interval", + "verdict": "refused", + "confidence_claim_permitted": False, + "reason": f"Unknown provenance {provenance!r}. Declare whether the " + "denominator is a census or a sample.", + "remedy": "Declare provenance as 'census' or 'sampled'.", + } + + +_GUARDS = { + "post_selection_interval": _post_selection, + "census_interval": _census, +} + + +def check(request: dict[str, Any]) -> dict[str, Any]: + """Apply the named guard to a request and return its verdict. + + Args: + request: Must carry ``guard`` naming one of + ``post_selection_interval`` or ``census_interval``, plus that + guard's declared ``input_fields``. + + Returns: + A JSON-native verdict. ``confidence_claim_permitted`` is the field a + caller acts on; a ``False`` there means the caller must not attach a + confidence statement to the number, whatever else it does with it. + """ + guard = request.get("guard") + handler = _GUARDS.get(guard) + if handler is None: + return { + "guard": guard, + "verdict": "refused", + "confidence_claim_permitted": False, + "reason": f"Unknown guard {guard!r}. Known guards: {', '.join(sorted(_GUARDS))}.", + "remedy": "Name a guard from 'cc-guard table'.", + } + return handler(request) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="cc-guard", + description="Inference guards that refuse claims the evidence cannot support.", + ) + sub = parser.add_subparsers(dest="command", required=True) + + check_parser = sub.add_parser("check", help="Apply a guard to a JSON request read from stdin.") + check_parser.add_argument( + "--strict-exit", + action="store_true", + help="Exit 1 when the verdict forbids a confidence claim, so a shell " + "caller can gate on it without parsing JSON.", + ) + + sub.add_parser("table", help="Emit the decision rules as pure data.") + + args = parser.parse_args(argv) + + if args.command == "table": + json.dump(DECISION_TABLE, sys.stdout, indent=2) + sys.stdout.write("\n") + return 0 + + raw = sys.stdin.read() + try: + request = json.loads(raw) if raw.strip() else {} + except json.JSONDecodeError as exc: + json.dump( + { + "verdict": "refused", + "confidence_claim_permitted": False, + "reason": f"Request is not valid JSON: {exc}", + "remedy": "Send a JSON object on stdin.", + }, + sys.stdout, + indent=2, + ) + sys.stdout.write("\n") + return 2 + if not isinstance(request, dict): + json.dump( + { + "verdict": "refused", + "confidence_claim_permitted": False, + "reason": "Request must be a JSON object.", + "remedy": "Send a JSON object on stdin.", + }, + sys.stdout, + indent=2, + ) + sys.stdout.write("\n") + return 2 + + verdict = check(request) + json.dump(verdict, sys.stdout, indent=2) + sys.stdout.write("\n") + if args.strict_exit and not verdict["confidence_claim_permitted"]: + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/cc/compose/__init__.py b/src/cc/compose/__init__.py new file mode 100644 index 00000000..d95841e5 --- /dev/null +++ b/src/cc/compose/__init__.py @@ -0,0 +1,31 @@ +"""ROC-free composition bounds over named binary events. + +The entry point is :func:`compose_bounds`: marginals in, a sharp interval out. +See :mod:`cc.compose._bounds` for the design note on why this surface exists. +""" + +from __future__ import annotations + +from cc.compose._bounds import ( + CompositionBounds, + CountermonotoneUndefinedError, + DependenceAssumption, + EventKind, + MarginalProvenance, + SensitivityRow, + compose_bounds, + marginal_from_operating_point, + sensitivity, +) + +__all__ = [ + "CompositionBounds", + "CountermonotoneUndefinedError", + "DependenceAssumption", + "EventKind", + "MarginalProvenance", + "SensitivityRow", + "compose_bounds", + "marginal_from_operating_point", + "sensitivity", +] diff --git a/src/cc/compose/_bounds.py b/src/cc/compose/_bounds.py new file mode 100644 index 00000000..922e4368 --- /dev/null +++ b/src/cc/compose/_bounds.py @@ -0,0 +1,490 @@ +"""Composition bounds over named binary events. + +This module is the answer to a rejected-adoption report. A downstream consumer +read ``cc.core.composition_theory``, found that it operates on ROC point sets +and bounds the Youden J statistic, and concluded -- correctly -- that a +deterministic refusal rule has no threshold, no operating point, and no +false-positive rate to trade against. They reimplemented the inequality by hand +rather than force their controls into a detector shape. + +Nothing in this module's public surface mentions ROC curves, thresholds, +operating points, TPR, or FPR. A binary event is a name and a marginal +probability. Detectors reach this surface through +:func:`cc.compose.marginal_from_operating_point`, which converts an operating +point into a marginal -- so the detector framing sits *above* the event +calculus rather than underneath it. + +The mathematics is classical. Frechet-Hoeffding is 1935 and this module claims +no novelty for it. What it provides is a surface shaped like the way composed +binary events are actually reasoned about, and a result object that carries the +independence baseline, the binding event, and the non-claims alongside the +interval so none of them can be quoted apart from the others. +""" + +from __future__ import annotations + +import math +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any, Literal, TypeAlias + +__all__ = [ + "CompositionBounds", + "CountermonotoneUndefinedError", + "DependenceAssumption", + "EventKind", + "MarginalProvenance", + "SensitivityRow", + "compose_bounds", + "marginal_from_operating_point", + "sensitivity", +] + +#: ``"all"`` bounds P(every event occurs) -- a conjunction. ``"any"`` bounds +#: P(at least one occurs) -- a union. The aliases ``"and"``/``"or"`` are +#: accepted because downstream documents use that vocabulary. +EventKind: TypeAlias = Literal["all", "any"] + +#: What is assumed about the joint dependence structure. +#: +#: ``"unconstrained"`` +#: Nothing is assumed. Returns the sharp Frechet-Hoeffding interval. This +#: is the honest default when no dependence evidence exists. +#: ``"independent"`` +#: A point value, returned as a degenerate interval. Reported as a +#: *baseline for comparison*, never as an answer -- see the module note on +#: ``independence_point``. +#: ``"comonotone"`` +#: The upper Frechet corner, returned as a degenerate interval. +#: ``"countermonotone"`` +#: Defined only for exactly two events. Refused for more; see +#: :class:`CountermonotoneUndefinedError`. +DependenceAssumption: TypeAlias = Literal[ + "unconstrained", "independent", "comonotone", "countermonotone" +] + +#: Where each marginal came from. This is a producer declaration, not a +#: statistical test, and it is carried on the result so a number cannot be +#: quoted without it. +MarginalProvenance: TypeAlias = Literal["measured", "assumed", "supplied"] + +_ALIASES: dict[str, EventKind] = { + "all": "all", + "and": "all", + "AND": "all", + "intersection": "all", + "any": "any", + "or": "any", + "OR": "any", + "union": "any", +} + +_TOL = 1.0e-12 + +_NON_CLAIMS: tuple[str, ...] = ( + "A bound is not a measurement. It states what the supplied marginals " + "permit under unknown dependence, and nothing else.", + "These bounds do not identify a copula, establish independence, infer " + "causation, or validate stationarity.", + "A narrow interval can result from extreme marginals and is not evidence " + "that the upstream measurements are valid.", + "The independence point value is reported as a comparison baseline. It is " + "not an estimate of the joint probability and must not be quoted as one.", + "This is not a safety certification, a compliance statement, or a claim " + "about any deployed system.", +) + + +class CountermonotoneUndefinedError(ValueError): + """Raised when countermonotonicity is requested for other than two events. + + Countermonotonicity is a strictly bivariate concept. Two events can be + perfectly negatively dependent -- one occurs exactly when the other does + not -- but three cannot all be pairwise mutually exclusive and exhaustive + in that way, and there is no n-dimensional countermonotonic structure for + ``n > 2``. + + The matching fact about the bound: the Frechet-Hoeffding lower bound + ``max(0, sum(p) - (n-1))`` **is not a copula in dimension >= 3**. It + remains *pointwise sharp* -- for any fixed marginals some joint + distribution attains it -- but no single dependence structure attains it + everywhere, so there is no countermonotone regime to tabulate beside + independence and comonotonicity. + + This error exists because silently offering a "countermonotone" option for + four events would produce a number with the form of a dependence regime and + none of the content. + """ + + +@dataclass(frozen=True) +class SensitivityRow: + """How the interval moves when one event's marginal is perturbed.""" + + event: str + marginal: float + perturbed_marginal: float + lower_delta: float + upper_delta: float + + @property + def moves_upper(self) -> bool: + """Whether this event moves the upper bound at all.""" + return abs(self.upper_delta) > _TOL + + def to_json(self) -> dict[str, Any]: + return { + "event": self.event, + "marginal": self.marginal, + "perturbed_marginal": self.perturbed_marginal, + "lower_delta": self.lower_delta, + "upper_delta": self.upper_delta, + "moves_upper": self.moves_upper, + } + + +@dataclass(frozen=True) +class CompositionBounds: + """A sharp interval for a composed binary event, with its context attached. + + The interval is the result. Everything else on this object exists so the + interval cannot be quoted without the things that qualify it: what was + assumed, which event the bound turns on, and what the number does not mean. + """ + + lower: float + upper: float + event: EventKind + marginals: Mapping[str, float] + dependence: DependenceAssumption + #: The product of the marginals. A comparison baseline, never an answer. + independence_point: float + #: Name of the event whose marginal the upper bound turns on, when the + #: bound is determined by a single event. ``None`` when the upper bound is + #: a sum or a clip rather than a single marginal. + binding_event: str | None + marginal_provenance: MarginalProvenance + non_claims: tuple[str, ...] = field(default=_NON_CLAIMS) + + @property + def width(self) -> float: + """``upper - lower``, with numerical negatives clipped to zero.""" + return max(0.0, self.upper - self.lower) + + @property + def independence_regret(self) -> float: + """How much the independence baseline understates the admissible worst case. + + ``upper - independence_point``. This is the quantity that makes the + product assumption's error visible in absolute terms. + """ + return self.upper - self.independence_point + + @property + def understatement_factor(self) -> float: + """``upper / independence_point``, or infinity when the product is zero. + + The relative form of :attr:`independence_regret`. For a homogeneous + stack of ``m`` events at rate ``p`` this grows like ``p**(1-m)``, which + is the correlation cliff in one number. + """ + if self.independence_point <= 0.0: + return math.inf + return self.upper / self.independence_point + + def to_json(self) -> dict[str, Any]: + """Return a JSON-native mapping. Used by the conformance corpus.""" + factor = self.understatement_factor + return { + "lower": self.lower, + "upper": self.upper, + "width": self.width, + "event": self.event, + "marginals": dict(self.marginals), + "dependence": self.dependence, + "independence_point": self.independence_point, + "independence_regret": self.independence_regret, + # JSON has no infinity. A null here means the product was zero, so + # the ratio is undefined rather than large. + "understatement_factor": None if math.isinf(factor) else factor, + "binding_event": self.binding_event, + "marginal_provenance": self.marginal_provenance, + "non_claims": list(self.non_claims), + } + + +def _canonical_event(event: str) -> EventKind: + try: + return _ALIASES[event] + except KeyError: + allowed = ", ".join(sorted(set(_ALIASES))) + raise ValueError(f"unknown event kind {event!r}; expected one of: {allowed}") from None + + +def _validated_marginals(marginals: Mapping[str, float]) -> dict[str, float]: + if not isinstance(marginals, Mapping): + raise TypeError( + "marginals must be a mapping of event name to probability; " + f"got {type(marginals).__name__}. A bare sequence is refused " + "because an unnamed marginal cannot be reported against a " + "binding event." + ) + if not marginals: + raise ValueError("marginals must name at least one event.") + + out: dict[str, float] = {} + for name, value in marginals.items(): + if not isinstance(name, str) or not name: + raise ValueError(f"event name must be a non-empty string; got {name!r}") + try: + p = float(value) + except (TypeError, ValueError): + raise TypeError(f"marginal for {name!r} must be a real number; got {value!r}") from None + if not math.isfinite(p): + raise ValueError(f"marginal for {name!r} must be finite; got {p!r}") + if not (-_TOL <= p <= 1.0 + _TOL): + raise ValueError(f"marginal for {name!r} must lie in [0, 1]; got {p!r}") + out[name] = min(1.0, max(0.0, p)) + return out + + +def _clip01(value: float) -> float: + return min(1.0, max(0.0, value)) + + +def _classical(values: Sequence[float], event: EventKind) -> tuple[float, float]: + """Closed-form n-way Frechet-Hoeffding bounds.""" + n = len(values) + total = math.fsum(values) + if event == "all": + return _clip01(max(0.0, total - (n - 1))), _clip01(min(values)) + return _clip01(max(values)), _clip01(min(1.0, total)) + + +def _binding_event(names: Sequence[str], values: Sequence[float], event: EventKind) -> str | None: + """Name the event the upper bound turns on, when a single event determines it. + + For a conjunction the upper bound is ``min(p)``, so the argmin binds and + improving any other event moves the upper bound not at all -- which is the + actionable finding. For a union the upper bound is ``min(1, sum(p))``, + which is a sum rather than a single event, so no event binds unless the + clip at 1 is inactive and only one event is present. + """ + if event == "all": + best = min(range(len(values)), key=lambda i: values[i]) + ties = [i for i, v in enumerate(values) if abs(v - values[best]) <= _TOL] + # A tie means no single event binds: improving either leaves the bound + # pinned by the other. Reporting one of them would be misleading. + return names[best] if len(ties) == 1 else None + if len(values) == 1: + return names[0] + return None + + +def compose_bounds( + marginals: Mapping[str, float], + *, + event: str = "all", + dependence: DependenceAssumption = "unconstrained", + marginal_provenance: MarginalProvenance = "supplied", +) -> CompositionBounds: + """Bound the probability of a composed binary event from its marginals. + + Each event is a named binary failure indicator with a marginal probability. + No ROC curve, threshold, or operating point is involved: a deterministic + refusal rule is as valid an input as a tuned classifier. + + Args: + marginals: Mapping of event name to marginal probability in ``[0, 1]``. + Names are required -- an unnamed marginal cannot be reported + against a binding event. + event: ``"all"`` for a conjunction, ``"any"`` for a union. ``"and"`` + and ``"or"`` are accepted aliases. + dependence: What is assumed about the joint structure. The default, + ``"unconstrained"``, assumes nothing and returns the sharp + interval. + marginal_provenance: Whether the marginals were ``"measured"``, + ``"assumed"``, or merely ``"supplied"``. Carried on the result so a + bound over assumed rates cannot be quoted as a measurement. + + Returns: + A :class:`CompositionBounds` carrying the interval, the independence + baseline, the binding event, and the non-claims. + + Raises: + CountermonotoneUndefinedError: ``dependence="countermonotone"`` with + more than two events. + ValueError: A marginal outside ``[0, 1]``, an empty mapping, or an + unknown event kind. + + Example: + Three deterministic controls, each with an assumed evasion + probability:: + + >>> b = compose_bounds( + ... {"SELF_REPORTED": 0.30, "STALE": 0.01, "SEPARATION": 0.40}, + ... event="all", + ... marginal_provenance="assumed", + ... ) + >>> b.lower, b.upper + (0.0, 0.01) + >>> b.binding_event + 'STALE' + + The upper bound is ``min(p)``: under arbitrary dependence a conjunction + of controls is no stronger than its single strongest member. Improving + ``SEPARATION`` moves the upper bound not at all. + + When two events tie for the minimum, no single event binds and + ``binding_event`` is ``None`` -- improving either one leaves the bound + pinned by the other, so naming one of them would misdirect effort. + """ + kind = _canonical_event(event) + checked = _validated_marginals(marginals) + names = tuple(checked) + values = tuple(checked[n] for n in names) + + if dependence == "countermonotone" and len(values) != 2: + if len(values) < 2: + raise CountermonotoneUndefinedError( + f"countermonotonicity is undefined for {len(values)} event(s). It " + "is a relation between two events; with fewer than two there is " + "nothing for an event to be countermonotone with. Use " + "dependence='unconstrained'." + ) + raise CountermonotoneUndefinedError( + f"countermonotonicity is undefined for {len(values)} events. It is a " + "strictly bivariate concept: there is no n-dimensional " + "countermonotonic structure for n > 2, and the Frechet-Hoeffding " + "lower bound is not a copula in dimension >= 3 (though it remains " + "pointwise sharp). Use dependence='unconstrained' for the sharp " + "interval, whose lower endpoint is that pointwise-sharp floor." + ) + + # The independence baseline is computed once and reported on every result, + # whatever the dependence assumption, so the gap it understates stays + # visible even when the caller asked for a different regime. + independence_point = ( + math.prod(values) if kind == "all" else _clip01(1.0 - math.prod(1.0 - v for v in values)) + ) + lower, upper = _classical(values, kind) + + if dependence == "independent": + lower = upper = independence_point + elif dependence == "comonotone": + lower = upper = _clip01(min(values)) if kind == "all" else _clip01(max(values)) + elif dependence == "countermonotone": + # Exactly two events; every other count was refused above. + lower = upper = ( + _clip01(max(0.0, values[0] + values[1] - 1.0)) + if kind == "all" + else _clip01(min(1.0, values[0] + values[1])) + ) + elif dependence != "unconstrained": + raise ValueError( + f"unknown dependence assumption {dependence!r}; expected one of: " + "unconstrained, independent, comonotone, countermonotone" + ) + + binding = _binding_event(names, values, kind) if dependence == "unconstrained" else None + + return CompositionBounds( + lower=lower, + upper=upper, + event=kind, + marginals=checked, + dependence=dependence, + independence_point=independence_point, + binding_event=binding, + marginal_provenance=marginal_provenance, + ) + + +def sensitivity( + marginals: Mapping[str, float], + *, + event: str = "all", + delta: float = 0.05, + direction: Literal["increase", "decrease"] = "decrease", + marginal_provenance: MarginalProvenance = "supplied", +) -> tuple[SensitivityRow, ...]: + """Perturb each marginal in turn and report how the interval moves. + + Under a conjunction the upper bound is ``min(p)``, so only the event + holding the minimum can move it. Every other row will report an + ``upper_delta`` of zero, and that zero is the finding: effort spent on a + weak event buys nothing against the worst admissible dependence. + + Args: + marginals: As :func:`compose_bounds`. + event: As :func:`compose_bounds`. + delta: Magnitude of the perturbation, in probability units. + direction: ``"decrease"`` lowers each marginal (a control improving, if + the marginal is an evasion probability); ``"increase"`` raises it. + marginal_provenance: As :func:`compose_bounds`. + + Returns: + One :class:`SensitivityRow` per event, in the order given. + """ + if not (0.0 < delta <= 1.0): + raise ValueError(f"delta must lie in (0, 1]; got {delta!r}") + + base = compose_bounds(marginals, event=event, marginal_provenance=marginal_provenance) + checked = dict(base.marginals) + sign = -1.0 if direction == "decrease" else 1.0 + + rows: list[SensitivityRow] = [] + for name, value in checked.items(): + perturbed = _clip01(value + sign * delta) + trial = dict(checked) + trial[name] = perturbed + after = compose_bounds(trial, event=event, marginal_provenance=marginal_provenance) + rows.append( + SensitivityRow( + event=name, + marginal=value, + perturbed_marginal=perturbed, + lower_delta=after.lower - base.lower, + upper_delta=after.upper - base.upper, + ) + ) + return tuple(rows) + + +def marginal_from_operating_point( + *, + false_negative_rate: float, + prevalence: float = 1.0, +) -> float: + """Convert a detector's operating point into an event marginal. + + This is the *only* bridge from detector vocabulary into this module, and it + points inward: a classifier supplies its miss rate at a chosen threshold + and receives a marginal. The ROC framing sits above the event calculus and + is optional; nothing downstream of this function knows a threshold existed. + + Args: + false_negative_rate: Probability the detector fails to flag a case that + should be flagged, at the operating point actually deployed. For a + guardrail this is the unsafe-pass rate. + prevalence: Probability the case arises at all. Defaults to 1.0, which + makes the marginal equal to the miss rate -- appropriate when the + composition is conditioned on an attack already occurring. + + Returns: + ``false_negative_rate * prevalence``, suitable as a + :func:`compose_bounds` marginal. + + Note: + Choosing the threshold is outside this function and outside this + module. A marginal derived here inherits every assumption of the + threshold choice, and is no more valid than that choice was. + """ + for label, value in ( + ("false_negative_rate", false_negative_rate), + ("prevalence", prevalence), + ): + p = float(value) + if not math.isfinite(p) or not (-_TOL <= p <= 1.0 + _TOL): + raise ValueError(f"{label} must be a finite probability in [0, 1]; got {value!r}") + return _clip01(float(false_negative_rate) * float(prevalence)) diff --git a/src/cc/evidence/claim_governance.py b/src/cc/evidence/claim_governance.py index 57fb6279..1b673a6b 100644 --- a/src/cc/evidence/claim_governance.py +++ b/src/cc/evidence/claim_governance.py @@ -73,7 +73,7 @@ roles_requiring_semantic_payload_validation, validate_role_payload, ) -from cc.reporting.canonical import sha256_canonical +from cc.reporting.canonical import sha256_canonical, strict_json_loads from cc.reporting.report import ( ALLOWED_CLAIM_LEVELS, CLAIM_LEVELS, @@ -325,7 +325,8 @@ def verify_claim_governance( evaluated_at = _utc_iso(evaluation_time) try: - report = json.loads(report_file.read_text(encoding="utf-8")) + # A repeated key here would change which report is verified (F-07). + report = strict_json_loads(report_file.read_text(encoding="utf-8")) except Exception as exc: return _failure_audit( report_id=report_file.stem or "", @@ -824,7 +825,7 @@ def _read_semantic_payload( root_dir: Path, ) -> Mapping[str, Any] | None: try: - payload = json.loads(_resolve_path(artifact_audit.path, root_dir).read_text()) + payload = strict_json_loads(_resolve_path(artifact_audit.path, root_dir).read_text()) except Exception as exc: artifact_audit.status = EvidenceRoleStatus.UNREADABLE artifact_audit.reason = f"Artifact could not be parsed as JSON: {exc}" diff --git a/src/cc/evidence/merkle_log.py b/src/cc/evidence/merkle_log.py index 88402286..f779712b 100644 --- a/src/cc/evidence/merkle_log.py +++ b/src/cc/evidence/merkle_log.py @@ -20,7 +20,7 @@ from pathlib import Path from typing import Any -from cc.reporting.canonical import canonical_json_bytes +from cc.reporting.canonical import CanonicalJSONError, canonical_json_bytes, strict_json_loads EMPTY_ROOT_HASH = hashlib.sha256(b"").hexdigest() LOG_RECORD_SCHEMA = "cc/merkle-log-record.v1" @@ -422,9 +422,14 @@ def _load(self) -> None: if not raw.strip(): continue try: - entry = json.loads(raw) + # A log line that repeats a key must not be silently + # resolved: the leaf hash would cover the survivor, not the + # line as written (finding F-07). + entry = strict_json_loads(raw) except json.JSONDecodeError as exc: raise MerkleLogError(f"line {line_number}: invalid JSON") from exc + except CanonicalJSONError as exc: + raise MerkleLogError(f"line {line_number}: {exc}") from exc if not isinstance(entry, Mapping): raise MerkleLogError(f"line {line_number}: log entry must be an object") if entry.get("schema") != LOG_RECORD_SCHEMA: diff --git a/src/cc/evidence_card/__init__.py b/src/cc/evidence_card/__init__.py new file mode 100644 index 00000000..19a8ea03 --- /dev/null +++ b/src/cc/evidence_card/__init__.py @@ -0,0 +1,40 @@ +"""Evidence cards -- the unit an Evidence Atlas displays. + +A card is a claim plus everything needed to disagree with it. See +:mod:`cc.evidence_card._card` for the design note, in particular why there is no +composite status property. +""" + +from __future__ import annotations + +from cc.evidence_card._card import ( + EVIDENCE_STATES, + LABEL_MEANINGS, + PUBLICATION_STATES, + SCHEMA_VERSION, + VERDICTS, + ArtifactRef, + EvidenceCard, + EvidenceCardError, + EvidenceState, + PublicationState, + Verdict, + cards_to_site_manifest, + render_labels, +) + +__all__ = [ + "EVIDENCE_STATES", + "LABEL_MEANINGS", + "PUBLICATION_STATES", + "SCHEMA_VERSION", + "VERDICTS", + "ArtifactRef", + "EvidenceCard", + "EvidenceCardError", + "EvidenceState", + "PublicationState", + "Verdict", + "cards_to_site_manifest", + "render_labels", +] diff --git a/src/cc/evidence_card/_card.py b/src/cc/evidence_card/_card.py new file mode 100644 index 00000000..875c7f50 --- /dev/null +++ b/src/cc/evidence_card/_card.py @@ -0,0 +1,318 @@ +"""Evidence cards: the unit an Atlas displays, and the thing it refuses to be. + +An evidence card is a claim together with everything a reader needs in order to +disagree with it: how it was operationalized, which command produces it, what +would falsify it, what it assumes, and what it explicitly does not claim. + +The design constraint that shapes this module is negative. A card is **not** a +trust score, and no function here computes one. Three labels travel with every +card and they are deliberately *orthogonal*: + +``evidence_state`` + Where the evidence came from -- ``local-only``, ``aws-synth-only``, + ``aws-live``, ``illustrative``. +``verdict`` + What happened when the command ran -- ``pass``, ``fail``, ``unverifiable``, + ``not-run``. +``publication_state`` + Where the card is in its own lifecycle -- ``draft``, ``released``, + ``superseded``, ``retracted``. + +They answer different questions and none implies another. A ``pass`` that is +``local-only`` says nothing about deployed behaviour. A ``retracted`` card may +still carry a ``pass``: the run happened, and the claim was withdrawn anyway. +Collapsing them into one green checkmark is exactly the failure this object +exists to prevent, so :class:`EvidenceCard` provides no composite status +property, and :func:`render_labels` returns all three or raises. + +Two fields are required that most claim registries treat as optional: + +* ``falsifier`` -- what observation would show the claim is wrong. A claim with + no falsifier is not evidence; it is an assertion, and the emitter refuses it. +* ``non_claims`` -- what the card explicitly does not establish. + +The default verdict is ``not-run``. A ``pass`` is only reachable by executing +the command and recording the result, so a card cannot acquire a passing verdict +by being written confidently. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any, Final, Literal, TypeAlias + +__all__ = [ + "EVIDENCE_STATES", + "PUBLICATION_STATES", + "SCHEMA_VERSION", + "VERDICTS", + "ArtifactRef", + "EvidenceCard", + "EvidenceCardError", + "EvidenceState", + "PublicationState", + "Verdict", + "render_labels", +] + +SCHEMA_VERSION: Final = "cc.evidence_card.v1" + +#: Where the evidence came from. This is a statement about provenance, not +#: about quality: ``aws-live`` evidence can record a failure, and +#: ``local-only`` evidence can be entirely sound for a local claim. +EvidenceState: TypeAlias = Literal[ + "local-only", + "aws-synth-only", + "aws-live", + "illustrative", +] + +#: What happened when the command ran. ``not-run`` is the default and the only +#: honest value for a card whose command has not been executed. ``unverifiable`` +#: is distinct from ``fail``: the check could not be performed at all, which is +#: a different fact about the world than the check performing and failing. +Verdict: TypeAlias = Literal["pass", "fail", "unverifiable", "not-run"] + +#: Where the card is in its own lifecycle. Retraction is a first-class state, +#: not a deletion: a withdrawn claim stays visible with its reason. +PublicationState: TypeAlias = Literal["draft", "released", "superseded", "retracted"] + +EVIDENCE_STATES: Final[tuple[EvidenceState, ...]] = ( + "local-only", + "aws-synth-only", + "aws-live", + "illustrative", +) +VERDICTS: Final[tuple[Verdict, ...]] = ("pass", "fail", "unverifiable", "not-run") +PUBLICATION_STATES: Final[tuple[PublicationState, ...]] = ( + "draft", + "released", + "superseded", + "retracted", +) + +#: Human-readable gloss for each label value. The Atlas shows these next to the +#: value so a reader never has to infer what a bare token means. +LABEL_MEANINGS: Final[Mapping[str, Mapping[str, str]]] = { + "evidence_state": { + "local-only": "Produced on a local machine. Says nothing about cloud or " + "deployed behaviour.", + "aws-synth-only": "Produced against synthesized or emulated AWS, not a " + "live account. Synthetic evidence is never live evidence.", + "aws-live": "Produced against a live AWS account, for the named run only.", + "illustrative": "A worked example or figure. Not measured evidence, and " + "must never be cited as a result.", + }, + "verdict": { + "pass": "The command ran and its checks succeeded. Scope is whatever the " + "command covers -- no wider.", + "fail": "The command ran and its checks did not succeed.", + "unverifiable": "The check could not be performed. This is not a pass and not a fail.", + "not-run": "The command has not been executed for this card. No result is claimed.", + }, + "publication_state": { + "draft": "Not yet reviewed for publication.", + "released": "Published. Says nothing about whether the claim is true, only " + "that it was reviewed for release.", + "superseded": "A later card replaces this one. Retained so the record is not rewritten.", + "retracted": "Withdrawn. The reason is on the card; the card is not deleted.", + }, +} + + +class EvidenceCardError(ValueError): + """Raised when a card is missing something that makes it evidence.""" + + +@dataclass(frozen=True) +class ArtifactRef: + """A file the claim rests on, bound by digest. + + ``sha256`` may be ``None`` only when ``missing`` is true -- a path named by a + claim that does not exist on disk. That is recorded rather than skipped: a + claim pointing at a file that is gone is a finding, not an absence. + """ + + path: str + sha256: str | None = None + bytes_len: int | None = None + missing: bool = False + + def to_json(self) -> dict[str, Any]: + return { + "path": self.path, + "sha256": self.sha256, + "bytes": self.bytes_len, + "missing": self.missing, + } + + +@dataclass(frozen=True) +class EvidenceCard: + """One claim, with everything needed to disagree with it. + + Raises: + EvidenceCardError: A required field is empty, or a label carries a value + outside its declared set. Construction is the enforcement point -- + an invalid card cannot exist to be rendered. + """ + + card_id: str + claim: str + maturity: str + source_revision: str + command: tuple[str, ...] + falsifier: str + assumptions: tuple[str, ...] + non_claims: tuple[str, ...] + evidence_state: EvidenceState + verdict: Verdict + publication_state: PublicationState + artifacts: tuple[ArtifactRef, ...] = () + result_detail: str | None = None + counterevidence: tuple[str, ...] = () + supersedes: str | None = None + retraction_reason: str | None = None + schema_version: str = field(default=SCHEMA_VERSION) + + def __post_init__(self) -> None: + for name in ("card_id", "claim", "maturity", "source_revision", "falsifier"): + value = getattr(self, name) + if not isinstance(value, str) or not value.strip(): + raise EvidenceCardError( + f"{self.card_id or ''}: {name} must be a non-empty string" + ) + + if not self.command: + raise EvidenceCardError( + f"{self.card_id}: a card needs at least one command. A claim no " + "command can demonstrate is an assertion, not evidence." + ) + if not self.non_claims: + raise EvidenceCardError( + f"{self.card_id}: a card needs at least one explicit non-claim. " + "Stating what a result does not establish is the point." + ) + + for label, allowed in ( + ("evidence_state", EVIDENCE_STATES), + ("verdict", VERDICTS), + ("publication_state", PUBLICATION_STATES), + ): + value = getattr(self, label) + if value not in allowed: + raise EvidenceCardError( + f"{self.card_id}: {label}={value!r} is not one of {allowed}" + ) + + if self.publication_state == "retracted" and not (self.retraction_reason or "").strip(): + raise EvidenceCardError( + f"{self.card_id}: a retracted card must carry its retraction_reason. " + "Withdrawing a claim without saying why rewrites the record." + ) + if self.publication_state == "superseded" and not (self.supersedes or "").strip(): + raise EvidenceCardError( + f"{self.card_id}: a superseded card must set 'supersedes' to name " + "the card that replaces it." + ) + if self.verdict in ("pass", "fail") and not (self.result_detail or "").strip(): + raise EvidenceCardError( + f"{self.card_id}: verdict={self.verdict!r} requires result_detail " + "naming what actually ran. A verdict without a record of the run " + "is an assertion." + ) + + # NOTE: there is deliberately no `status`, `score`, `is_ok`, or `badge` + # property on this class. The three labels answer different questions and a + # single composite would let a reader stop before asking any of them. If a + # caller wants one glyph, they must choose which question they are asking. + + def to_json(self) -> dict[str, Any]: + """Return the JSON-native card. Every label is present and separate.""" + return { + "schema_version": self.schema_version, + "card_id": self.card_id, + "claim": self.claim, + "maturity": self.maturity, + "source_revision": self.source_revision, + "command": list(self.command), + "artifacts": [artifact.to_json() for artifact in self.artifacts], + "labels": { + "evidence_state": self.evidence_state, + "verdict": self.verdict, + "publication_state": self.publication_state, + }, + "label_meanings": { + "evidence_state": LABEL_MEANINGS["evidence_state"][self.evidence_state], + "verdict": LABEL_MEANINGS["verdict"][self.verdict], + "publication_state": LABEL_MEANINGS["publication_state"][self.publication_state], + }, + "result_detail": self.result_detail, + "falsifier": self.falsifier, + "counterevidence": list(self.counterevidence), + "assumptions": list(self.assumptions), + "non_claims": list(self.non_claims), + "supersedes": self.supersedes, + "retraction_reason": self.retraction_reason, + } + + +def render_labels(card: EvidenceCard) -> dict[str, dict[str, str]]: + """Return all three labels with their meanings, for display. + + This is the only rendering helper provided, and it returns **three** + entries. There is no single-label variant, because a surface that can show + one label will show the flattering one. + """ + return { + label: { + "value": getattr(card, label), + "meaning": LABEL_MEANINGS[label][getattr(card, label)], + } + for label in ("evidence_state", "verdict", "publication_state") + } + + +def cards_to_site_manifest( + cards: Sequence[EvidenceCard], + *, + source_revision: str, + generated_note: str, +) -> dict[str, Any]: + """Bundle cards into the manifest a static Atlas consumes. + + The manifest carries counts *per label*, never an aggregate. A reader + learns how many cards are ``not-run`` and how many are ``local-only`` + separately, because a site that reported "7/8 passing" would be describing + something nobody measured. + """ + by_label: dict[str, dict[str, int]] = { + "evidence_state": dict.fromkeys(EVIDENCE_STATES, 0), + "verdict": dict.fromkeys(VERDICTS, 0), + "publication_state": dict.fromkeys(PUBLICATION_STATES, 0), + } + for card in cards: + for label in by_label: + by_label[label][getattr(card, label)] += 1 + + return { + "schema_version": "cc.site_evidence_manifest.v1", + "source_revision": source_revision, + "generated_note": generated_note, + "card_count": len(cards), + "counts_by_label": by_label, + "label_meanings": {k: dict(v) for k, v in LABEL_MEANINGS.items()}, + "cards": [card.to_json() for card in cards], + "non_claims": [ + "This manifest lists claims and the commands that test them. It does " + "not establish that any claim is true.", + "The three labels are orthogonal. A passing verdict on local-only " + "evidence says nothing about deployed behaviour, and a released " + "publication state says only that a card was reviewed for release.", + "Counts are reported per label. There is no aggregate score, and any " + "surface that computes one from this file is misusing it.", + "A card with verdict 'not-run' has no result. It must not be " + "displayed as passing, pending, or healthy.", + ], + } diff --git a/src/cc/reporting/canonical.py b/src/cc/reporting/canonical.py index 2681896e..8eada83e 100644 --- a/src/cc/reporting/canonical.py +++ b/src/cc/reporting/canonical.py @@ -1,7 +1,55 @@ -"""Canonical JSON helpers for CC report receipts. +"""Canonical JSON for CC receipts, as versioned profiles. -The receipt hash is SHA-256 over a strict JSON representation: -sorted object keys, compact separators, UTF-8 bytes, and no NaN/Infinity. +A receipt identifies a document only up to the kernel of its canonicalizer: +the set of distinct documents that receive the same canonical bytes. Every +member of that kernel is a pair of documents a receipt cannot tell apart. The +job here is to keep that kernel equal to JSON equality -- no larger, and no +smaller. + +Two profiles exist. + +``cc.canonical.v2`` (:data:`RFC8785`) is the default for new artifacts and +implements RFC 8785, the JSON Canonicalization Scheme. It is what makes an +independent verifier in another language possible: JCS pins number formatting +to the ECMAScript ``Number::toString`` algorithm and key ordering to UTF-16 +code-unit order, so a conforming implementation in any language produces the +same bytes. + +``cc.canonical.v1`` (:data:`LEGACY_SORT_KEYS`) is the original +``json.dumps(sort_keys=True, ...)`` method. It is retained **read-only** so +receipts written before the migration stay verifiable. Breaking historical +receipts to fix the canonicalizer would trade one integrity failure for +another. + +Every receipt names its profile in ``receipt.canonicalization_method``, and +verification dispatches on that name rather than assuming the current default. + +Why v1 is not merely "the old one" +---------------------------------- + +An adversarial census (``scripts/canonicalization_probe.py``) found two +``unintended-kernel`` classes in v1 -- documents a consumer needs +distinguished that received identical bytes: + +* v1 applied ``unicodedata.normalize("NFC", ...)`` to every mapping key and + wrote the results into a fresh dict. Two byte-distinct keys sharing an NFC + form silently became one, with no error, at top level and nested. The receipt + then attested to a document with a field missing. +* v1 emitted Python ``repr`` number forms (``1.0``, ``-0.0``, ``1e-07``), + which diverge from RFC 8785 on five of six probed forms, so no non-Python + verifier could agree on the bytes. + +v2 fixes both by construction rather than by patch. It does **not** normalize: +RFC 8785 is explicit that normalization is the producer's responsibility, and a +canonicalizer that mutates content is not a canonicalizer. Two keys that differ +in Unicode form are two keys, which is what JSON says they are, so there is +nothing left to collide. Producers that want the stricter reading can call +:func:`assert_no_confusable_keys` as a separate lint -- it is deliberately not +part of the hash path. + +Both profiles reject non-finite floats, non-string keys, and non-JSON-native +values. v2 additionally rejects integers outside the IEEE-754 safe range (see +:data:`MAX_SAFE_INTEGER`) and strings containing unpaired surrogates. """ from __future__ import annotations @@ -11,60 +59,232 @@ import math import unicodedata from collections.abc import Mapping, Sequence -from typing import Any +from decimal import Decimal +from typing import Any, Final, Literal, TypeAlias JsonValue = str | int | float | bool | None | list["JsonValue"] | dict[str, "JsonValue"] +#: Identifier written into ``receipt.canonicalization_method`` by the original +#: method. Retained verbatim so pre-migration receipts still validate. +LEGACY_SORT_KEYS: Final = ( + "json.dumps(sort_keys=True,separators=(',', ':'),ensure_ascii=False,allow_nan=False); " + "receipt.canonical_hash excluded" +) + +#: Identifier for the RFC 8785 profile. Default for new artifacts. +RFC8785: Final = "cc.canonical.v2/RFC8785; receipt.canonical_hash excluded" + +CanonicalProfile: TypeAlias = Literal[ + "json.dumps(sort_keys=True,separators=(',', ':'),ensure_ascii=False,allow_nan=False); " + "receipt.canonical_hash excluded", + "cc.canonical.v2/RFC8785; receipt.canonical_hash excluded", +] + +DEFAULT_PROFILE: Final[CanonicalProfile] = RFC8785 + +#: ``2**53 - 1``. Integers beyond this are not exactly representable as IEEE-754 +#: doubles, so a JavaScript ``JSON.parse`` collapses neighbouring values before +#: any verifier code runs -- the kernel is set by the parser, and no downstream +#: fix reaches it. v2 refuses them rather than emitting bytes that cannot +#: survive a round trip. +MAX_SAFE_INTEGER: Final = 9007199254740991 + +__all__ = [ + "DEFAULT_PROFILE", + "LEGACY_SORT_KEYS", + "MAX_SAFE_INTEGER", + "RFC8785", + "CanonicalJSONError", + "CanonicalProfile", + "ConfusableKeyError", + "assert_no_confusable_keys", + "canonical_json_bytes", + "sha256_canonical", + "strict_json_loads", +] + class CanonicalJSONError(TypeError): """Raised when a value cannot be represented in canonical report JSON.""" -def canonical_json_bytes(obj: Mapping[str, Any]) -> bytes: - """Return deterministic UTF-8 JSON bytes for a JSON-native mapping. +class ConfusableKeyError(CanonicalJSONError): + """Raised when two byte-distinct keys share a Unicode normal form. - Non-string mapping keys, non-finite floats, bytes, sets, and arbitrary - objects are rejected instead of being stringified implicitly. + Under RFC 8785 these are two separate keys and canonicalization proceeds. + This error is raised only by the opt-in :func:`assert_no_confusable_keys` + lint, because a document containing both is nearly always a producer bug -- + two fields where one was meant. """ - normalized = _normalize_json_value(obj, path="$") - if not isinstance(normalized, dict): - raise CanonicalJSONError("canonical_json_bytes requires a JSON object at the top level") - return json.dumps( - normalized, - sort_keys=True, - separators=(",", ":"), - ensure_ascii=False, - allow_nan=False, - ).encode("utf-8") +class DuplicateJSONKeyError(CanonicalJSONError): + """Raised when a parsed JSON object repeats a key. + + ``json.loads`` keeps the last occurrence silently, so a receipt computed + over the parsed result attests to a document the sender did not send. + """ + + +# -------------------------------------------------------------------------- +# RFC 8785 number serialization +# -------------------------------------------------------------------------- -def sha256_canonical(obj: Mapping[str, Any], *, exclude_receipt_hash: bool = True) -> str: - """Hash canonical JSON bytes with SHA-256. - When ``exclude_receipt_hash`` is true, ``receipt.canonical_hash`` is removed - before hashing so reports can contain their own receipt without a circular - dependency. +def _es_number_to_string(value: float) -> str: + """Serialize a double per the ECMAScript ``Number::toString`` algorithm. + + RFC 8785 section 3.2.2.3 defers to ECMA-262. Python's ``repr`` already + yields the shortest round-tripping digits; what differs is the *formatting* + of those digits, which this function supplies. """ + if not math.isfinite(value): + raise CanonicalJSONError(f"non-finite numbers are not JSON: {value!r}") + # Covers -0.0, which RFC 8785 requires be emitted as "0". + if value == 0.0: + return "0" + if value < 0: + return "-" + _es_number_to_string(-value) - payload: Mapping[str, Any] = _without_receipt_hash(obj) if exclude_receipt_hash else obj - return hashlib.sha256(canonical_json_bytes(payload)).hexdigest() + _, raw_digits, exponent = Decimal(repr(value)).as_tuple() + digits = "".join(str(d) for d in raw_digits) + exponent = int(exponent) + # Trailing zeros are an artifact of repr ("1.0" -> digits "10"); the ES + # algorithm requires the shortest digit string k. + while len(digits) > 1 and digits.endswith("0"): + digits = digits[:-1] + exponent += 1 + k = len(digits) + n = k + exponent # value == 0. * 10**n + + if k <= n <= 21: + return digits + "0" * (n - k) + if 0 < n <= 21: + return digits[:n] + "." + digits[n:] + if -6 < n <= 0: + return "0." + "0" * (-n) + digits + sign = "+" if n - 1 >= 0 else "-" + exp = f"e{sign}{abs(n - 1)}" + if k == 1: + return digits + exp + return digits[0] + "." + digits[1:] + exp + + +_SHORT_ESCAPES: Final[dict[str, str]] = { + '"': '\\"', + "\\": "\\\\", + "\b": "\\b", + "\f": "\\f", + "\n": "\\n", + "\r": "\\r", + "\t": "\\t", +} + + +def _es_quote(text: str) -> str: + """Quote a string per RFC 8785 section 3.2.2.2. + + Control characters below 0x20 are escaped, using the short forms where they + exist; everything else is emitted literally as UTF-8. Non-ASCII is **not** + escaped, and no normalization is applied. + """ + out = ['"'] + for char in text: + escape = _SHORT_ESCAPES.get(char) + if escape is not None: + out.append(escape) + elif char < "\x20": + out.append(f"\\u{ord(char):04x}") + elif "\ud800" <= char <= "\udfff": + # An unpaired surrogate cannot be encoded as UTF-8. Emitting one + # would produce bytes no conforming parser can read, so refuse. + raise CanonicalJSONError( + f"string contains an unpaired surrogate U+{ord(char):04X}; " + "it has no UTF-8 encoding and cannot be canonicalized" + ) + else: + out.append(char) + out.append('"') + return "".join(out) + + +def _utf16_sort_key(key: str) -> bytes: + """Return a sort key ordering strings by UTF-16 code unit. + + RFC 8785 sorts object keys by UTF-16 code units, **not** by code point. + The two orders differ above the BMP: U+10000 encodes as the surrogate pair + D800 DC00, so it sorts *before* U+FFFD under UTF-16 and *after* it under + code point. Comparing big-endian UTF-16 bytes reproduces the required + order exactly. + """ + try: + return key.encode("utf-16-be") + except UnicodeEncodeError as exc: + raise CanonicalJSONError( + f"object key {key!r} contains an unpaired surrogate and cannot be ordered" + ) from exc + + +def _serialize_rfc8785(value: Any, *, path: str) -> str: + if value is None: + return "null" + # bool must precede int: Python's bool is an int subclass. + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, str): + return _es_quote(value) + if isinstance(value, int): + if abs(value) > MAX_SAFE_INTEGER: + raise CanonicalJSONError( + f"{path} integer {value} exceeds the IEEE-754 safe range " + f"(|n| <= {MAX_SAFE_INTEGER}). A JSON parser backed by doubles " + "collapses values beyond it, so no cross-language verifier " + "could agree on this document. Carry the value as a string." + ) + return str(value) + if isinstance(value, float): + if not math.isfinite(value): + raise CanonicalJSONError(f"{path} must be finite, got {value!r}") + return _es_number_to_string(value) + if isinstance(value, Mapping): + items = [] + seen: set[str] = set() + for key in value: + if not isinstance(key, str): + raise CanonicalJSONError(f"{path} contains non-string key {key!r}") + if key in seen: # pragma: no cover - a dict cannot repeat a key + raise CanonicalJSONError(f"{path} repeats key {key!r}") + seen.add(key) + items.append(key) + items.sort(key=_utf16_sort_key) + parts = [ + f"{_es_quote(key)}:{_serialize_rfc8785(value[key], path=f'{path}.{key}')}" + for key in items + ] + return "{" + ",".join(parts) + "}" + if isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray, str)): + parts = [ + _serialize_rfc8785(item, path=f"{path}[{index}]") for index, item in enumerate(value) + ] + return "[" + ",".join(parts) + "]" + raise CanonicalJSONError( + f"{path} contains non-JSON-native value of type {type(value).__name__}" + ) -def _without_receipt_hash(obj: Mapping[str, Any]) -> dict[str, Any]: - normalized = _normalize_json_value(obj, path="$") - if not isinstance(normalized, dict): - raise CanonicalJSONError("report payload must be a JSON object") - report = dict(normalized) - receipt = report.get("receipt") - if isinstance(receipt, dict): - receipt_copy = dict(receipt) - receipt_copy.pop("canonical_hash", None) - report["receipt"] = receipt_copy - return report +# -------------------------------------------------------------------------- +# Legacy profile (read-only) +# -------------------------------------------------------------------------- -def _normalize_json_value(value: Any, *, path: str) -> JsonValue: + +def _normalize_legacy(value: Any, *, path: str) -> JsonValue: + """Reproduce the v1 normalization exactly, including its NFC key merge. + + This is deliberately bug-compatible. Its purpose is to recompute the hash a + pre-migration receipt actually carries; "fixing" it here would make old + receipts unverifiable, which is the failure this profile exists to prevent. + """ if value is None or isinstance(value, bool): return value if isinstance(value, str): @@ -81,12 +301,144 @@ def _normalize_json_value(value: Any, *, path: str) -> JsonValue: if not isinstance(key, str): raise CanonicalJSONError(f"{path} contains non-string key {key!r}") norm_key = unicodedata.normalize("NFC", key) - out[norm_key] = _normalize_json_value(item, path=f"{path}.{norm_key}") + out[norm_key] = _normalize_legacy(item, path=f"{path}.{norm_key}") return out if isinstance(value, Sequence) and not isinstance(value, (bytes, bytearray, str)): - return [ - _normalize_json_value(item, path=f"{path}[{idx}]") for idx, item in enumerate(value) - ] + return [_normalize_legacy(item, path=f"{path}[{i}]") for i, item in enumerate(value)] raise CanonicalJSONError( f"{path} contains non-JSON-native value of type {type(value).__name__}" ) + + +def _serialize_legacy(obj: Mapping[str, Any]) -> bytes: + normalized = _normalize_legacy(obj, path="$") + if not isinstance(normalized, dict): + raise CanonicalJSONError("canonical_json_bytes requires a JSON object at the top level") + return json.dumps( + normalized, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + + +# -------------------------------------------------------------------------- +# Public surface +# -------------------------------------------------------------------------- + + +def canonical_json_bytes( + obj: Mapping[str, Any], + *, + profile: CanonicalProfile = DEFAULT_PROFILE, +) -> bytes: + """Return deterministic UTF-8 JSON bytes for a JSON-native mapping. + + Args: + obj: A JSON object. Non-string keys, non-finite floats, bytes, sets, + and arbitrary objects are rejected rather than stringified. + profile: Which canonicalization profile to apply. Defaults to + :data:`RFC8785`. Pass :data:`LEGACY_SORT_KEYS` only to recompute + the hash of a pre-migration receipt. + + Raises: + CanonicalJSONError: The value cannot be canonicalized, or (under + :data:`RFC8785`) carries an integer outside the safe range or an + unpaired surrogate. + """ + if profile == LEGACY_SORT_KEYS: + return _serialize_legacy(obj) + if profile != RFC8785: + raise CanonicalJSONError( + f"unknown canonicalization profile {profile!r}; expected " + f"{RFC8785!r} or {LEGACY_SORT_KEYS!r}" + ) + if not isinstance(obj, Mapping): + raise CanonicalJSONError("canonical_json_bytes requires a JSON object at the top level") + return _serialize_rfc8785(obj, path="$").encode("utf-8") + + +def sha256_canonical( + obj: Mapping[str, Any], + *, + exclude_receipt_hash: bool = True, + profile: CanonicalProfile = DEFAULT_PROFILE, +) -> str: + """Hash canonical JSON bytes with SHA-256. + + When ``exclude_receipt_hash`` is true, ``receipt.canonical_hash`` is + removed before hashing so a report can contain its own receipt without a + circular dependency. + """ + payload: Mapping[str, Any] = _without_receipt_hash(obj) if exclude_receipt_hash else obj + return hashlib.sha256(canonical_json_bytes(payload, profile=profile)).hexdigest() + + +def strict_json_loads(text: str | bytes) -> Any: + """Parse JSON, refusing any object that repeats a key. + + ``json.loads`` keeps the last occurrence of a repeated key and reports + nothing. A receipt computed over that result attests to a document the + sender did not send, so every receipt-covered read goes through here. + """ + + def _hook(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + seen: set[str] = set() + for key, _ in pairs: + if key in seen: + raise DuplicateJSONKeyError( + f"JSON object repeats key {key!r}; the value that would " + "survive is not the document that was sent" + ) + seen.add(key) + return dict(pairs) + + return json.loads(text, object_pairs_hook=_hook) + + +def assert_no_confusable_keys(obj: Any, *, path: str = "$") -> None: + """Raise if any object contains two keys sharing a Unicode normal form. + + This is an **opt-in producer lint**, not part of canonicalization. Under + RFC 8785 such keys are simply two keys and hashing proceeds correctly; but + a document carrying both is nearly always a bug -- two fields where one was + intended -- and a producer that would rather fail than ship it can call + this first. + + Keeping it out of the hash path is deliberate. v1 merged such keys *inside* + canonicalization and silently dropped a field; the lesson is that a + canonicalizer must not mutate, and a mutation-free canonicalizer has no + business deciding a document is wrong. + """ + if isinstance(obj, Mapping): + by_normal_form: dict[str, str] = {} + for key in obj: + if not isinstance(key, str): + continue + normal = unicodedata.normalize("NFC", key) + previous = by_normal_form.get(normal) + if previous is not None and previous != key: + raise ConfusableKeyError( + f"{path} contains keys {previous!r} and {key!r}, which are " + "byte-distinct but share the Unicode normal form " + f"{normal!r}. Under RFC 8785 these are two separate fields." + ) + by_normal_form[normal] = key + for key, value in obj.items(): + assert_no_confusable_keys(value, path=f"{path}.{key}") + elif isinstance(obj, Sequence) and not isinstance(obj, (bytes, bytearray, str)): + for index, item in enumerate(obj): + assert_no_confusable_keys(item, path=f"{path}[{index}]") + + +def _without_receipt_hash(obj: Mapping[str, Any]) -> dict[str, Any]: + if not isinstance(obj, Mapping): + raise CanonicalJSONError("report payload must be a JSON object") + report = dict(obj) + receipt = report.get("receipt") + if isinstance(receipt, Mapping): + receipt_copy = dict(receipt) + receipt_copy.pop("canonical_hash", None) + report["receipt"] = receipt_copy + return report diff --git a/src/cc/reporting/cli.py b/src/cc/reporting/cli.py index a86e0825..1bbdb9a2 100644 --- a/src/cc/reporting/cli.py +++ b/src/cc/reporting/cli.py @@ -12,6 +12,7 @@ from typing import Any from cc.evidence.claim_governance import GovernanceVerdict, verify_claim_governance +from cc.reporting.canonical import strict_json_loads from cc.reporting.report import ( CLAIM_LEVEL_DESCRIPTIONS, CLAIM_LEVELS, @@ -237,7 +238,10 @@ def _cmd_verify_claim_governance(args: argparse.Namespace) -> int: def _read_json(path: Path) -> dict[str, Any]: if not path.exists(): raise FileNotFoundError(f"JSON input not found: {path}") - payload = json.loads(path.read_text(encoding="utf-8")) + # strict_json_loads, not json.loads: a repeated key would otherwise be + # silently resolved last-wins, and the receipt computed over the survivor + # would attest to a document the sender did not send (finding F-07). + payload = strict_json_loads(path.read_text(encoding="utf-8")) if not isinstance(payload, dict): raise ValueError(f"{path} must contain a JSON object") return payload diff --git a/src/cc/reporting/report.py b/src/cc/reporting/report.py index 2d206161..49df8b7f 100644 --- a/src/cc/reporting/report.py +++ b/src/cc/reporting/report.py @@ -45,14 +45,15 @@ ) from cc import __version__ as framework_version_default -from cc.reporting.canonical import sha256_canonical +from cc.reporting.canonical import DEFAULT_PROFILE, sha256_canonical SCHEMA_VERSION = "cc.report.v0.3.1" HASH_ALGORITHM = "sha256" -CANONICALIZATION_METHOD = ( - "json.dumps(sort_keys=True,separators=(',', ':'),ensure_ascii=False,allow_nan=False); " - "receipt.canonical_hash excluded" -) +#: Profile written into new receipts. Reports produced before the RFC 8785 +#: migration carry the legacy profile instead, and verification dispatches on +#: whichever profile the receipt itself declares -- see +#: ``docs/architecture/CANONICAL_PROFILE.md``. +CANONICALIZATION_METHOD = DEFAULT_PROFILE # Report maturity/support labels. # @@ -323,9 +324,13 @@ def _non_diagnostic_claims_need_boundaries(self) -> ClaimSummaryModel: class ReportReceiptModel(_StrictReportModel): canonical_hash: str = Field(pattern=r"^[0-9a-f]{64}$") hash_algorithm: Literal["sha256"] + #: Both profiles are accepted on read. New reports are written with + #: the RFC 8785 profile; the legacy one appears only on pre-migration receipts, + #: which must stay verifiable. canonicalization_method: Literal[ + "cc.canonical.v2/RFC8785; receipt.canonical_hash excluded", "json.dumps(sort_keys=True,separators=(',', ':'),ensure_ascii=False,allow_nan=False); " - "receipt.canonical_hash excluded" + "receipt.canonical_hash excluded", ] previous_hash: str | None = Field(default=None, pattern=r"^[0-9a-f]{64}$") @@ -384,11 +389,17 @@ def _receipt_and_boundary_invariants(self) -> CCReport: + ", ".join(overclaims) ) - computed_hash = sha256_canonical(payload) + # Dispatch on the profile the receipt declares, not on the current + # default. Recomputing a pre-migration receipt under RFC 8785 would + # report a mismatch for a report that is in fact intact, which is the + # opposite of what a receipt is for. + computed_hash = sha256_canonical(payload, profile=self.receipt.canonicalization_method) if computed_hash != self.receipt.canonical_hash: raise ValueError( "receipt.canonical_hash does not match canonical report payload: " - f"expected {self.receipt.canonical_hash}, computed {computed_hash}" + f"expected {self.receipt.canonical_hash}, computed {computed_hash} " + f"under canonicalization profile " + f"{self.receipt.canonicalization_method!r}" ) return self diff --git a/tests/acceptance/test_external_consumer_reproduction.py b/tests/acceptance/test_external_consumer_reproduction.py new file mode 100644 index 00000000..7adeef08 --- /dev/null +++ b/tests/acceptance/test_external_consumer_reproduction.py @@ -0,0 +1,180 @@ +"""Acceptance gate: reproduce an external consumer's published result. + +A downstream project needed Frechet-Hoeffding bounds over four deterministic +controls. It read this repository's most discoverable composition entry point, +``cc.core.composition_theory``, found that it operates on ROC point sets and +bounds the Youden J statistic, and concluded that its controls -- which refuse +if and only if a predicate holds, with no threshold and no operating curve -- +did not fit. It implemented the inequality itself in 214 lines of JavaScript. + +This module is the gate on that failure being fixed. It is not "the API +exists": it is *their published numbers reproduce from this library*, so those +214 lines could be deleted. + +The numbers below are quoted from that project's published composition +document, not recomputed from its source. They are therefore an **external +oracle**: an implementation written independently, for another purpose, without +reference to this corpus or this code. Agreement with it is the only check in +this repository that is not same-author. + +Their non-claim travels with their numbers and is asserted here too: every +detection rate is ASSUMED. Zero records had been issued when they were +published, so no rate was ever observed. These are bounds over assumed inputs, +never measurements. +""" + +from __future__ import annotations + +import pytest + +from cc.compose import compose_bounds, sensitivity + +#: Published detection rates, small-chapter scenario. ASSUMED, never observed. +PUBLISHED_DETECTION_RATES = { + "SELF_REPORTED": 0.70, + "STALE": 0.99, + "IMMUTABLE": 0.99, + "SEPARATION": 0.60, +} + +#: Published output, quoted from the consumer's document: +#: +#: if independent (the fiction) 0.0012% +#: Frechet-Hoeffding lower 0% +#: Frechet-Hoeffding upper 1.00% +#: width of the bound 1.00% +#: +#: Assuming independence understates the admissible worst case by 833x. +PUBLISHED_LOWER = 0.00 +PUBLISHED_UPPER = 0.01 +PUBLISHED_WIDTH = 0.01 +PUBLISHED_INDEPENDENCE = 0.000012 +PUBLISHED_UNDERSTATEMENT_FACTOR = 833 + + +def _evasion_marginals(detection: dict[str, float]) -> dict[str, float]: + """A control's evasion probability is one minus its detection rate. + + The composed failure event is a conjunction: an inflated entry must evade + *every* control. Getting this backwards -- treating it as a union -- would + invert every number, which is why the consumer argued it explicitly rather + than assuming it. + """ + return {name: 1.0 - rate for name, rate in detection.items()} + + +def test_published_interval_reproduces(): + bounds = compose_bounds( + _evasion_marginals(PUBLISHED_DETECTION_RATES), + event="all", + marginal_provenance="assumed", + ) + assert bounds.lower == pytest.approx(PUBLISHED_LOWER, abs=1e-12) + assert bounds.upper == pytest.approx(PUBLISHED_UPPER, abs=1e-12) + assert bounds.width == pytest.approx(PUBLISHED_WIDTH, abs=1e-12) + + +def test_published_independence_baseline_reproduces(): + bounds = compose_bounds( + _evasion_marginals(PUBLISHED_DETECTION_RATES), + event="all", + marginal_provenance="assumed", + ) + assert bounds.independence_point == pytest.approx(PUBLISHED_INDEPENDENCE, abs=1e-12) + + +def test_published_understatement_factor_reproduces(): + """The headline: independence understates the admissible worst case by 833x.""" + bounds = compose_bounds( + _evasion_marginals(PUBLISHED_DETECTION_RATES), + event="all", + marginal_provenance="assumed", + ) + assert round(bounds.understatement_factor) == PUBLISHED_UNDERSTATEMENT_FACTOR + + +def test_published_sensitivity_finding_reproduces(): + """Their finding: improving a weak control moves the upper bound by 0.00pp. + + Quoted: "In the small-chapter scenario, a 5-point improvement to + SELF_REPORTED or SEPARATION moves it by 0.00pp. Only the strongest controls + move it." + """ + rows = sensitivity( + _evasion_marginals(PUBLISHED_DETECTION_RATES), + event="all", + delta=0.05, + direction="decrease", + marginal_provenance="assumed", + ) + by_event = {row.event: row for row in rows} + + assert by_event["SELF_REPORTED"].upper_delta == pytest.approx(0.0, abs=1e-12) + assert by_event["SEPARATION"].upper_delta == pytest.approx(0.0, abs=1e-12) + # The two strong controls are tied at the minimum, so each moves the bound. + assert by_event["STALE"].upper_delta == pytest.approx(-0.01, abs=1e-12) + assert by_event["IMMUTABLE"].upper_delta == pytest.approx(-0.01, abs=1e-12) + + +def test_the_tie_is_reported_as_a_tie(): + """Two controls tie at the minimum, so no single event binds. + + Naming one of them would misdirect effort: improving either leaves the + bound pinned by the other. The consumer's sensitivity table shows exactly + this -- both STALE and IMMUTABLE move the bound. + """ + bounds = compose_bounds( + _evasion_marginals(PUBLISHED_DETECTION_RATES), + event="all", + marginal_provenance="assumed", + ) + assert bounds.binding_event is None + + +@pytest.mark.parametrize( + ("scenario", "detection", "expected_upper"), + [ + ("optimistic", {"a": 0.95, "b": 0.99, "c": 0.99, "d": 0.90}, 0.01), + ("small-chapter", {"a": 0.70, "b": 0.99, "c": 0.99, "d": 0.60}, 0.01), + ("pessimistic", {"a": 0.50, "b": 0.90, "c": 0.95, "d": 0.40}, 0.05), + ], +) +def test_all_three_published_scenarios_reproduce(scenario, detection, expected_upper): + bounds = compose_bounds( + _evasion_marginals(detection), event="all", marginal_provenance="assumed" + ) + assert bounds.upper == pytest.approx(expected_upper, abs=1e-12), scenario + assert bounds.lower == pytest.approx(0.0, abs=1e-12), scenario + + +def test_the_finding_holds_a_fifth_control_cannot_help(): + """Quoted: "Adding a fifth control cannot improve the bound." + + Under a conjunction the upper bound is min(p), so a further control can + only lower the floor. This is the correlation cliff stated as an + operational fact. + """ + base = compose_bounds( + _evasion_marginals(PUBLISHED_DETECTION_RATES), + event="all", + marginal_provenance="assumed", + ) + with_fifth = compose_bounds( + {**_evasion_marginals(PUBLISHED_DETECTION_RATES), "A_FIFTH_CONTROL": 0.20}, + event="all", + marginal_provenance="assumed", + ) + assert with_fifth.upper <= base.upper + 1e-12 + assert with_fifth.lower <= base.lower + 1e-12 + + +def test_the_assumed_provenance_travels_with_the_result(): + """A bound over assumed rates must not be quotable as a measurement.""" + bounds = compose_bounds( + _evasion_marginals(PUBLISHED_DETECTION_RATES), + event="all", + marginal_provenance="assumed", + ) + assert bounds.marginal_provenance == "assumed" + assert bounds.to_json()["marginal_provenance"] == "assumed" + assert any("not a measurement" in claim for claim in bounds.non_claims) diff --git a/tests/conformance/test_cc_kernel_v1.py b/tests/conformance/test_cc_kernel_v1.py new file mode 100644 index 00000000..5ca8a65e --- /dev/null +++ b/tests/conformance/test_cc_kernel_v1.py @@ -0,0 +1,198 @@ +"""The ``cc-kernel-v1`` conformance corpus, enforced. + +Three checks, in increasing strength: + +1. The committed corpus is current -- it has not drifted from the kernel. +2. The Python reference passes every case in it. +3. An independent Node implementation, written from the specification rather + than from this source, passes it too, and reproduces an external consumer's + published numbers. + +Check 3 is skipped when ``node`` is absent, and the skip is loud: a green run +without it establishes strictly less. + +**What passing establishes.** That the implementations compute the same values +on these cases. **What it does not establish.** That any of them is correct: +the Python and Node sides were authored in the same project and can share a +misreading of the specification. +""" + +from __future__ import annotations + +import json +import math +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from cc.compose import CountermonotoneUndefinedError, compose_bounds + +ROOT = Path(__file__).resolve().parents[2] +CORPUS = ROOT / "conformance" / "cc-kernel-v1" +VERIFIER = ROOT / "verifiers" / "node" / "cc_compose_verify.mjs" + +MANIFEST = json.loads((CORPUS / "manifest.json").read_text(encoding="utf-8")) +TOLERANCE = MANIFEST["tolerance"] + +ACCEPT = json.loads((CORPUS / "cases" / "composition.json").read_text(encoding="utf-8")) +REJECT = json.loads((CORPUS / "cases" / "adversarial.json").read_text(encoding="utf-8")) + +_requires_node = pytest.mark.skipif( + shutil.which("node") is None, + reason="node is not installed; cross-implementation agreement is NOT established", +) + + +# --- 1. the corpus is current ------------------------------------------------- + + +def test_committed_corpus_is_current(): + """A stale corpus pins yesterday's kernel and silently stops testing today's.""" + proc = subprocess.run( + [sys.executable, str(ROOT / "scripts" / "build_conformance_corpus.py"), "--check"], + capture_output=True, + text=True, + check=False, + cwd=ROOT, + ) + assert proc.returncode == 0, proc.stdout + proc.stderr + + +def test_manifest_counts_match_the_case_files(): + assert ACCEPT["case_count"] == len(ACCEPT["cases"]) + assert REJECT["case_count"] == len(REJECT["cases"]) + assert MANIFEST["total_cases"] == len(ACCEPT["cases"]) + len(REJECT["cases"]) + + +def test_manifest_carries_non_claims(): + assert MANIFEST["non_claims"] + assert any("does not establish" in claim for claim in MANIFEST["non_claims"]) + + +def test_case_ids_are_unique(): + ids = [c["id"] for c in ACCEPT["cases"]] + [c["id"] for c in REJECT["cases"]] + assert len(ids) == len(set(ids)) + + +# --- 2. the Python reference passes ------------------------------------------- + + +@pytest.mark.parametrize("case", ACCEPT["cases"], ids=lambda c: c["id"]) +def test_python_reference_reproduces_accept_case(case): + bounds = compose_bounds( + case["input"]["marginals"], + event=case["input"]["event"], + dependence=case["input"]["dependence"], + ) + got = bounds.to_json() + for field, want in case["expect"].items(): + have = got[field] + if field == "binding_event" or want is None or have is None: + assert have == want, f"{case['id']}.{field}" + else: + assert have == pytest.approx(want, abs=TOLERANCE), f"{case['id']}.{field}" + + +@pytest.mark.parametrize("case", REJECT["cases"], ids=lambda c: c["id"]) +def test_python_reference_refuses_reject_case(case): + marginals = {k: (math.nan if v == "NaN" else v) for k, v in case["input"]["marginals"].items()} + with pytest.raises((CountermonotoneUndefinedError, ValueError, TypeError)): + compose_bounds( + marginals, + event=case["input"]["event"], + dependence=case["input"]["dependence"], + ) + + +# --- 3. cross-implementation agreement ---------------------------------------- + + +@_requires_node +def test_independent_node_implementation_agrees(): + """The check that moves cross-implementation agreement off zero.""" + proc = subprocess.run( + ["node", str(VERIFIER), "--json"], + capture_output=True, + text=True, + check=False, + cwd=ROOT, + ) + report = json.loads(proc.stdout) + assert report["accept"]["failed"] == 0, report["accept"]["failures"] + assert report["reject"]["failed"] == 0, report["reject"]["failures"] + assert report["accept"]["total"] == len(ACCEPT["cases"]) + assert report["reject"]["total"] == len(REJECT["cases"]) + assert proc.returncode == 0 + + +@_requires_node +def test_node_implementation_reproduces_the_external_oracle(): + """The one check here that is not same-author. + + The oracle numbers were published by a separate project that implemented + these bounds for its own purposes, without reference to this corpus. + """ + proc = subprocess.run( + ["node", str(VERIFIER), "--json"], + capture_output=True, + text=True, + check=False, + cwd=ROOT, + ) + report = json.loads(proc.stdout) + assert report["external_oracle"]["total"] >= 1 + assert report["external_oracle"]["failed"] == 0, report["external_oracle"]["failures"] + + +@_requires_node +def test_agreement_report_states_what_it_does_not_establish(): + """Agreement that does not name its limits invites being over-read.""" + proc = subprocess.run( + ["node", str(VERIFIER), "--json"], + capture_output=True, + text=True, + check=False, + cwd=ROOT, + ) + report = json.loads(proc.stdout) + assert "does not establish that either is correct" in report["agreement_non_claim"] + assert report["provenance"] == "census" + + +@_requires_node +@pytest.mark.parametrize("seed", [1, 2, 3]) +def test_differential_fuzz_finds_no_disagreement(seed): + """Randomized cases explore what the curated corpus does not. + + Seeds are fixed here so CI is deterministic. The harness accepts any seed; + a wider unseeded sweep belongs in the scheduled empirical lane, not in the + per-commit gate. + """ + proc = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "differential_compose.py"), + "--cases", + "400", + "--seed", + str(seed), + "--json", + ], + capture_output=True, + text=True, + check=False, + cwd=ROOT, + ) + report = json.loads(proc.stdout) + assert report["disagreements"] == 0, report["detail"] + assert report["cases"] == 400 + # A sweep that refused everything would agree trivially. + assert report["accepted"] > 0 + assert report["refused"] > 0 + # Coverage and blind spots must be stated, not implied. + assert report["generator_coverage"] + assert report["generator_blind_spots"] + assert report["provenance"] == "sampled" diff --git a/tests/integration/test_claim_governance_capsule.py b/tests/integration/test_claim_governance_capsule.py index 37e07c75..f135420e 100644 --- a/tests/integration/test_claim_governance_capsule.py +++ b/tests/integration/test_claim_governance_capsule.py @@ -152,6 +152,160 @@ def test_claim_governance_capsule_temp_copy_regeneration_is_byte_identical( ).read_bytes() +def test_capsule_marks_current_realized_fpr_as_asserted_not_matrix_derived() -> None: + result = run_capsule() + assert result.returncode == 0, result.stderr + result.stdout + + calibration = load_json(OUTPUTS / "calibration.json") + provenance = calibration["realized_fpr_provenance"] + + assert provenance["schema"] == "cc.calibration.realized-fpr-provenance.v1" + assert provenance["field"] == "realized_fpr" + assert provenance["semantics"]["status"] == "unresolved" + assert provenance["origin"] == {"kind": "asserted"} + assert provenance["external_anchor"] is None + assert provenance["verification"]["origin"]["status"] == "asserted_not_derivable" + assert any("does not infer a numerator" in item.lower() for item in provenance["non_claims"]) + + +def test_forged_asserted_calibration_value_remains_regenerable_and_visibly_asserted( + tmp_path: Path, +) -> None: + """CH-001 still holds: an asserted upstream value can be regenerated coherently.""" + + capsule_root = tmp_path / "asserted-forgery" + shutil.copytree(CAPSULE, capsule_root) + config_path = capsule_root / "inputs" / "capsule_config.json" + config = load_json(config_path) + config["calibration"]["realized_fpr"] = 0.011111111111 + write_json(config_path, config) + + result = run_capsule_at(capsule_root, "--update-expected") + assert result.returncode == 0, result.stderr + result.stdout + + calibration = load_json(capsule_root / "outputs" / "calibration.json") + provenance = calibration["realized_fpr_provenance"] + assert calibration["realized_fpr"] == 0.011111111111 + assert provenance["origin"] == {"kind": "asserted"} + assert provenance["verification"]["origin"]["status"] == "asserted_not_derivable" + + +def test_capsule_checks_opt_in_deterministically_derived_calibration_rate(tmp_path: Path) -> None: + capsule_root = tmp_path / "derived-capsule" + shutil.copytree(CAPSULE, capsule_root) + config_path = capsule_root / "inputs" / "capsule_config.json" + config = load_json(config_path) + config["calibration"]["realized_fpr"] = 0.375 + config["calibration"]["realized_fpr_provenance"] = { + "schema": "cc.calibration.realized-fpr-provenance.v1", + "field": "realized_fpr", + "semantics": { + "status": "defined", + "numerator": "Rows whose guardrail_keyword value equals one.", + "denominator": "All checked-in failure-matrix rows.", + "population": "The checked-in capsule failure-matrix rows.", + }, + "origin": { + "kind": "deterministic_derivation", + "algorithm": "binary_column_rate/v1", + "source": { + "path": "inputs/failure_matrix.csv", + "sha256": sha256(capsule_root / "inputs" / "failure_matrix.csv"), + }, + "column": "guardrail_keyword", + "inclusion_rule": "all_rows", + "exact_result": {"numerator": 9, "denominator": 24}, + "rendering": {"decimal_places": 12, "rounding": "half_even"}, + }, + "external_anchor": None, + } + write_json(config_path, config) + + result = run_capsule_at(capsule_root, "--update-expected") + assert result.returncode == 0, result.stderr + result.stdout + + calibration = load_json(capsule_root / "outputs" / "calibration.json") + verification = calibration["realized_fpr_provenance"]["verification"]["origin"] + assert verification == { + "computed_value": "0.375000000000", + "denominator_count": 24, + "numerator_count": 9, + "source_sha256": sha256(capsule_root / "inputs" / "failure_matrix.csv"), + "status": "derivation_verified_from_declared_bytes", + } + + config["calibration"]["realized_fpr_provenance"]["origin"]["source"]["sha256"] = "0" * 64 + write_json(config_path, config) + stale_source = run_capsule_at(capsule_root) + assert stale_source.returncode == 1 + assert "source sha256 does not match matrix" in stale_source.stdout + + +def test_capsule_rejects_a_wrong_opt_in_derived_calibration_rate(tmp_path: Path) -> None: + capsule_root = tmp_path / "mismatched-derived-capsule" + shutil.copytree(CAPSULE, capsule_root) + config_path = capsule_root / "inputs" / "capsule_config.json" + config = load_json(config_path) + config["calibration"]["realized_fpr"] = 0.011111111111 + config["calibration"]["realized_fpr_provenance"] = { + "schema": "cc.calibration.realized-fpr-provenance.v1", + "field": "realized_fpr", + "semantics": { + "status": "defined", + "numerator": "Rows whose guardrail_keyword value equals one.", + "denominator": "All checked-in failure-matrix rows.", + "population": "The checked-in capsule failure-matrix rows.", + }, + "origin": { + "kind": "deterministic_derivation", + "algorithm": "binary_column_rate/v1", + "source": { + "path": "inputs/failure_matrix.csv", + "sha256": sha256(capsule_root / "inputs" / "failure_matrix.csv"), + }, + "column": "guardrail_keyword", + "inclusion_rule": "all_rows", + "exact_result": {"numerator": 9, "denominator": 24}, + "rendering": {"decimal_places": 12, "rounding": "half_even"}, + }, + "external_anchor": None, + } + write_json(config_path, config) + + result = run_capsule_at(capsule_root) + assert result.returncode == 1 + assert "does not match the deterministic guardrail_keyword rate" in result.stdout + + +def test_capsule_records_but_does_not_fetch_an_external_anchor(tmp_path: Path) -> None: + capsule_root = tmp_path / "anchored-capsule" + shutil.copytree(CAPSULE, capsule_root) + config_path = capsule_root / "inputs" / "capsule_config.json" + config = load_json(config_path) + config["calibration"]["realized_fpr_provenance"]["external_anchor"] = { + "mechanism": "reference_only/v1", + "uri": "https://example.test/calibration/receipt", + "subject_sha256": calibration_anchor_subject_hash(config["calibration"]), + "issued_at": "2026-01-01T00:00:00Z", + "issuer": "fixture instrument", + } + write_json(config_path, config) + + result = run_capsule_at(capsule_root, "--update-expected") + assert result.returncode == 0, result.stderr + result.stdout + + calibration = load_json(capsule_root / "outputs" / "calibration.json") + assert calibration["realized_fpr_provenance"]["verification"]["external_anchor"]["status"] == ( + "reference_recorded_not_externally_verified" + ) + + config["calibration"]["realized_fpr"] = 0.125 + write_json(config_path, config) + wrong_subject = run_capsule_at(capsule_root) + assert wrong_subject.returncode == 1 + assert "does not bind this exact subject" in wrong_subject.stdout + + def test_claim_governance_capsule_tamper_changes_manifest_and_governance_verdict( tmp_path: Path, ) -> None: @@ -255,13 +409,14 @@ def run_capsule() -> subprocess.CompletedProcess[str]: ) -def run_capsule_at(capsule_dir: Path) -> subprocess.CompletedProcess[str]: +def run_capsule_at(capsule_dir: Path, *args: str) -> subprocess.CompletedProcess[str]: return subprocess.run( [ sys.executable, str(CAPSULE / "build_capsule.py"), "--capsule-dir", str(capsule_dir), + *args, ], cwd=ROOT, env=env(), @@ -301,6 +456,29 @@ def load_json(path: Path) -> dict[str, Any]: return payload +def write_json(path: Path, payload: dict[str, Any]) -> None: + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def calibration_anchor_subject_hash(calibration: dict[str, Any]) -> str: + provenance = calibration["realized_fpr_provenance"] + subject = { + "schema": provenance["schema"], + "field": provenance["field"], + "value": str(calibration["realized_fpr"]), + "semantics": provenance["semantics"], + "origin": provenance["origin"], + } + encoded = json.dumps( + subject, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() + + def assert_json_equal( actual: dict[str, Any], expected: dict[str, Any], diff --git a/tests/test_selective_inference_guard.py b/tests/test_selective_inference_guard.py index 90223088..da992bbb 100644 --- a/tests/test_selective_inference_guard.py +++ b/tests/test_selective_inference_guard.py @@ -30,7 +30,6 @@ from cc.kernel.cliff import cliff_certificate - CRITICAL = 0.20 diff --git a/tests/unit/canonical/test_canonical_profiles.py b/tests/unit/canonical/test_canonical_profiles.py new file mode 100644 index 00000000..613119e2 --- /dev/null +++ b/tests/unit/canonical/test_canonical_profiles.py @@ -0,0 +1,303 @@ +"""The canonicalization corpus, enforced. + +Every class in ``scripts/canonicalization_probe.py`` is asserted here, so a +regression in the kernel fails the suite rather than only the probe. The +binding rule for this file: **never weaken a case to make a test pass.** A +class that starts failing is a kernel regression, not a corpus problem. + +Provenance is ``census``. The classes are curated and adversarial, their number +is an authoring decision, and no coverage claim or confidence interval attaches +to a pass count over them. +""" + +from __future__ import annotations + +import json +import math +import subprocess +import sys +from pathlib import Path + +import pytest + +from cc.reporting.canonical import ( + DEFAULT_PROFILE, + LEGACY_SORT_KEYS, + MAX_SAFE_INTEGER, + RFC8785, + CanonicalJSONError, + ConfusableKeyError, + DuplicateJSONKeyError, + assert_no_confusable_keys, + canonical_json_bytes, + sha256_canonical, + strict_json_loads, +) + +ROOT = Path(__file__).resolve().parents[3] + +KEY_NFC = "é" # e-acute, precomposed +KEY_NFD = "é" # e + combining acute + + +def _probe_json() -> dict: + proc = subprocess.run( + [sys.executable, str(ROOT / "scripts" / "canonicalization_probe.py"), "--json"], + capture_output=True, + text=True, + check=False, + cwd=ROOT, + ) + return json.loads(proc.stdout) + + +# --- the corpus gate ---------------------------------------------------------- + + +def test_probe_reports_no_unintended_kernel_under_the_default_profile(): + """The acceptance gate for W3: zero unintended-kernel, zero rejection-asymmetry.""" + report = _probe_json() + v2 = report["profiles"]["v2"] + counts = v2["verdict_counts"] + + assert counts.get("unintended-kernel", 0) == 0, v2["classes"] + assert counts.get("rejection-asymmetry", 0) == 0, v2["classes"] + assert counts.get("over-discrimination", 0) == 0, v2["classes"] + assert v2["failing"] == 0 + # The census must carry its own boundary, not just its counts. + assert report["provenance"] == "census" + assert report["confidence_intervals"] is None + + +def test_probe_still_records_the_legacy_defects(): + """v1 must keep failing. It is retained as history, not as an option. + + If this ever passes trivially it means the legacy profile was "fixed", + which would silently change the hash of every pre-migration receipt and + defeat the reason v1 exists at all. + """ + v1 = _probe_json()["profiles"]["v1"] + assert v1["verdict_counts"].get("unintended-kernel", 0) == 2 + assert v1["silent_key_loss"]["keys_out"] < v1["silent_key_loss"]["keys_in"] + + +# --- F-03: the silent key merge ----------------------------------------------- + + +def test_default_profile_keeps_unicode_distinct_keys_distinct(): + assert canonical_json_bytes({KEY_NFC: 1}) != canonical_json_bytes({KEY_NFD: 1}) + + +def test_default_profile_does_not_lose_a_key(): + """Two byte-distinct keys go in; two come out, and neither is dropped.""" + record = {KEY_NFC: 1, KEY_NFD: 2, "plain": 3} + decoded = strict_json_loads(canonical_json_bytes(record).decode("utf-8")) + assert len(decoded) == 3 + assert decoded[KEY_NFC] == 1 + assert decoded[KEY_NFD] == 2 + + +def test_nested_unicode_keys_also_survive(): + """A guard inspecting only top-level keys would miss this.""" + assert canonical_json_bytes({"outer": {KEY_NFC: 1}}) != canonical_json_bytes( + {"outer": {KEY_NFD: 1}} + ) + + +def test_legacy_profile_still_merges_them(): + """Bug-compatible on purpose: old receipts must stay verifiable.""" + merged = canonical_json_bytes({KEY_NFC: 1, KEY_NFD: 2}, profile=LEGACY_SORT_KEYS) + assert len(json.loads(merged)) == 1 + + +# --- the opt-in producer lint ------------------------------------------------- + + +def test_confusable_key_lint_flags_what_canonicalization_permits(): + record = {KEY_NFC: 1, KEY_NFD: 2} + canonical_json_bytes(record) # canonicalization itself is content-neutral + with pytest.raises(ConfusableKeyError, match="normal form"): + assert_no_confusable_keys(record) + + +def test_confusable_key_lint_descends_into_nested_structures(): + with pytest.raises(ConfusableKeyError): + assert_no_confusable_keys({"a": [{"b": {KEY_NFC: 1, KEY_NFD: 2}}]}) + + +def test_confusable_key_lint_accepts_ordinary_documents(): + assert_no_confusable_keys({"a": 1, "b": [{"c": 2}], "d": None}) + + +# --- F-04: RFC 8785 number forms ---------------------------------------------- + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (1.0, "1"), + (-0.0, "0"), + (0.0, "0"), + (100.0, "100"), + (0.1, "0.1"), + (1.5, "1.5"), + (1e-7, "1e-7"), + (1e-6, "0.000001"), + (1e20, "100000000000000000000"), + (1e21, "1e+21"), + (1e30, "1e+30"), + (5e-324, "5e-324"), + (1.7976931348623157e308, "1.7976931348623157e+308"), + (-1.5, "-1.5"), + (1.2345e-10, "1.2345e-10"), + (1234567890.0, "1234567890"), + ], +) +def test_number_forms_follow_ecmascript_tostring(value, expected): + assert canonical_json_bytes({"n": value}).decode("utf-8") == f'{{"n":{expected}}}' + + +def test_negative_zero_and_zero_share_an_identity(): + """scipy returns -0.0 at a zero lower bound, so this reaches real reports.""" + assert sha256_canonical({"n": -0.0}) == sha256_canonical({"n": 0.0}) + + +def test_int_and_float_of_equal_value_share_an_identity(): + """JSON has one number type; 1 and 1.0 are the same JSON number.""" + assert canonical_json_bytes({"n": 1}) == canonical_json_bytes({"n": 1.0}) + + +# --- F-05: the integer safe range --------------------------------------------- + + +@pytest.mark.parametrize("value", [2**53, 2**53 + 1, 10**30, -(2**53), -(10**30)]) +def test_integers_outside_the_safe_range_are_refused(value): + """A JS JSON.parse collapses these before any verifier code runs.""" + with pytest.raises(CanonicalJSONError, match="safe range"): + canonical_json_bytes({"n": value}) + + +@pytest.mark.parametrize("value", [0, 1, -1, MAX_SAFE_INTEGER, -MAX_SAFE_INTEGER]) +def test_integers_inside_the_safe_range_are_accepted(value): + assert canonical_json_bytes({"n": value}) == f'{{"n":{value}}}'.encode() + + +def test_floats_are_not_subject_to_the_integer_bound(): + """The bound exists because Python ints are unbounded. Floats already are not.""" + assert canonical_json_bytes({"n": 1e300}).decode("utf-8") == '{"n":1e+300}' + + +# --- F-07: duplicate keys on read --------------------------------------------- + + +def test_strict_json_loads_refuses_a_repeated_key(): + with pytest.raises(DuplicateJSONKeyError, match="repeats key"): + strict_json_loads('{"amount":1,"amount":2}') + + +def test_strict_json_loads_refuses_a_repeated_key_when_nested(): + with pytest.raises(DuplicateJSONKeyError): + strict_json_loads('{"outer":{"a":1,"a":2}}') + + +def test_strict_json_loads_accepts_ordinary_json(): + assert strict_json_loads('{"a":1,"b":[1,2],"c":{"d":null}}') == { + "a": 1, + "b": [1, 2], + "c": {"d": None}, + } + + +# --- string and key handling -------------------------------------------------- + + +def test_control_characters_are_escaped_with_short_forms_where_defined(): + # \u0001 has no short escape, so it must come out as \\u0001; tab and + # newline do have one and must not be emitted as \\u0009 / \\u000a. + got = canonical_json_bytes({"k": "a\tb\nc\u0001d"}).decode("utf-8") + assert got == '{"k":"a\\tb\\nc\\u0001d"}' + + +def test_non_ascii_is_emitted_literally_not_escaped(): + text = "café" + assert canonical_json_bytes({"k": text}).decode("utf-8") == f'{{"k":"{text}"}}' + + +def test_keys_sort_by_utf16_code_unit_not_code_point(): + """The orders disagree above the BMP. + + U+10000 encodes as the surrogate pair D800 DC00, so it sorts *before* + U+FFFD under UTF-16 and *after* it under code point. + """ + bmp = "�" + astral = "\U00010000" + got = canonical_json_bytes({bmp: 1, astral: 2}).decode("utf-8") + assert got.index(astral) < got.index(bmp) + + +def test_unpaired_surrogates_are_refused(): + lone = "\ud800" + with pytest.raises(CanonicalJSONError, match="surrogate"): + canonical_json_bytes({"k": lone}) + with pytest.raises(CanonicalJSONError, match="surrogate"): + canonical_json_bytes({lone: "v"}) + + +# --- profile plumbing --------------------------------------------------------- + + +def test_default_profile_is_the_rfc8785_one(): + assert DEFAULT_PROFILE == RFC8785 + + +def test_unknown_profile_fails_closed(): + with pytest.raises(CanonicalJSONError, match="unknown canonicalization profile"): + canonical_json_bytes({"a": 1}, profile="cc.canonical.v99") # type: ignore[arg-type] + + +def test_profiles_disagree_which_is_the_point_of_versioning(): + """If these ever agreed, dispatching on the declared profile would be moot.""" + payload = {"n": 1.0, "m": -0.0} + assert canonical_json_bytes(payload, profile=RFC8785) != canonical_json_bytes( + payload, profile=LEGACY_SORT_KEYS + ) + + +# --- shared refusals ---------------------------------------------------------- + + +@pytest.mark.parametrize("profile", [RFC8785, LEGACY_SORT_KEYS]) +def test_both_profiles_refuse_non_finite_floats(profile): + for bad in (math.nan, math.inf, -math.inf): + with pytest.raises(CanonicalJSONError): + canonical_json_bytes({"n": bad}, profile=profile) + + +@pytest.mark.parametrize("profile", [RFC8785, LEGACY_SORT_KEYS]) +def test_both_profiles_refuse_non_string_keys(profile): + with pytest.raises(CanonicalJSONError, match="non-string key"): + canonical_json_bytes({1: "v"}, profile=profile) # type: ignore[dict-item] + + +@pytest.mark.parametrize("profile", [RFC8785, LEGACY_SORT_KEYS]) +def test_both_profiles_refuse_non_json_native_values(profile): + for bad in ({"s"}, b"bytes", object()): + with pytest.raises(CanonicalJSONError, match="non-JSON-native"): + canonical_json_bytes({"k": bad}, profile=profile) + + +# --- determinism -------------------------------------------------------------- + + +def test_key_order_does_not_affect_identity(): + assert canonical_json_bytes({"b": 1, "a": 2}) == canonical_json_bytes({"a": 2, "b": 1}) + + +def test_array_order_does_affect_identity(): + assert canonical_json_bytes({"xs": [1, 2]}) != canonical_json_bytes({"xs": [2, 1]}) + + +def test_canonical_bytes_are_stable_across_repeated_calls(): + payload = {"z": [1, {"b": 2, "a": 1}], "a": "x", "n": 0.1} + first = canonical_json_bytes(payload) + assert all(canonical_json_bytes(payload) == first for _ in range(50)) diff --git a/tests/unit/cli/test_guard.py b/tests/unit/cli/test_guard.py new file mode 100644 index 00000000..1bb41e0f --- /dev/null +++ b/tests/unit/cli/test_guard.py @@ -0,0 +1,311 @@ +"""Tests for ``cc-guard``: the inference guards, and their decision table. + +The table is data a caller in another language reimplements from. If it drifts +from the code, every such caller silently enforces the wrong rule -- so the +agreement between the two is asserted here, case by case. +""" + +from __future__ import annotations + +import json +import subprocess +import sys + +import pytest + +from cc.cli.guard import DECISION_TABLE, MIN_N_FOR_PROPORTION_INTERVAL, check, main +from cc.kernel.cliff import CliffCertificate, cliff_certificate + + +def _run(payload: dict, *args: str) -> tuple[int, dict]: + proc = subprocess.run( + [sys.executable, "-m", "cc.cli.guard", *args], + input=json.dumps(payload), + capture_output=True, + text=True, + check=False, + ) + return proc.returncode, json.loads(proc.stdout) + + +# --- the guard refuses what it must ------------------------------------------- + + +def test_post_selection_interval_is_refused(): + verdict = check( + { + "guard": "post_selection_interval", + "provenance": "post-selection", + "estimate": 0.42, + "ci_low": 0.21, + "ci_high": 0.63, + } + ) + assert verdict["verdict"] == "discovery-only" + assert verdict["confidence_claim_permitted"] is False + assert verdict["remedy"] + + +def test_confirmatory_interval_is_permitted(): + verdict = check( + { + "guard": "post_selection_interval", + "provenance": "confirmatory", + "estimate": 0.42, + "ci_low": 0.21, + "ci_high": 0.63, + } + ) + assert verdict["verdict"] == "confirmatory" + assert verdict["confidence_claim_permitted"] is True + + +@pytest.mark.parametrize("provenance", [None, "", "maybe", "CONFIRMATORY", 7]) +def test_unknown_provenance_fails_closed(provenance): + """A guard that defaults to permitting a claim is not a guard.""" + verdict = check({"guard": "post_selection_interval", "provenance": provenance}) + assert verdict["verdict"] == "refused" + assert verdict["confidence_claim_permitted"] is False + + +def test_census_interval_is_refused(): + verdict = check({"guard": "census_interval", "provenance": "census"}) + assert verdict["verdict"] == "no-interval" + assert verdict["confidence_claim_permitted"] is False + + +@pytest.mark.parametrize("n", [0, 1, 2, 29]) +def test_small_sample_interval_is_refused(n): + verdict = check({"guard": "census_interval", "provenance": "sampled", "n": n}) + assert verdict["verdict"] == "no-interval" + assert verdict["confidence_claim_permitted"] is False + + +@pytest.mark.parametrize("n", [30, 31, 1000]) +def test_adequate_sample_interval_is_permitted(n): + verdict = check({"guard": "census_interval", "provenance": "sampled", "n": n}) + assert verdict["verdict"] == "interval-permitted" + assert verdict["confidence_claim_permitted"] is True + + +@pytest.mark.parametrize("n", [None, -1, 3.5, "30", True]) +def test_sampled_proportion_without_a_valid_denominator_is_refused(n): + """No proportion without a denominator.""" + verdict = check({"guard": "census_interval", "provenance": "sampled", "n": n}) + assert verdict["verdict"] == "refused" + + +def test_unknown_guard_is_refused(): + verdict = check({"guard": "vibes"}) + assert verdict["verdict"] == "refused" + assert verdict["confidence_claim_permitted"] is False + + +# --- the table and the code must agree ---------------------------------------- + + +def test_decision_table_matches_the_implementation(): + """The table is what a non-Python caller enforces. It must not drift.""" + for guard in DECISION_TABLE["guards"]: + guard_id = guard["id"] + for rule in guard["rules"]: + request = {"guard": guard_id, **rule["when"]} + # The census guard's threshold rule is expressed as `n_below`. + n_below = request.pop("n_below", None) + if n_below is not None: + request["n"] = n_below - 1 + elif guard_id == "census_interval" and request.get("provenance") == "sampled": + request["n"] = MIN_N_FOR_PROPORTION_INTERVAL + verdict = check(request) + assert verdict["verdict"] == rule["verdict"], (guard_id, rule["when"]) + assert verdict["confidence_claim_permitted"] == rule["confidence_claim_permitted"], ( + guard_id, + rule["when"], + ) + + +def test_decision_table_threshold_matches_the_constant(): + census = next(g for g in DECISION_TABLE["guards"] if g["id"] == "census_interval") + thresholds = [r["when"]["n_below"] for r in census["rules"] if "n_below" in r["when"]] + assert thresholds == [MIN_N_FOR_PROPORTION_INTERVAL] + + +def test_decision_table_is_json_native(): + assert json.loads(json.dumps(DECISION_TABLE)) == DECISION_TABLE + + +def test_decision_table_carries_non_claims(): + assert DECISION_TABLE["non_claims"] + assert any("does not mean" in c for c in DECISION_TABLE["non_claims"]) + + +# --- agreement with the original in-kernel guard ------------------------------ + + +def test_guard_agrees_with_cliff_certificate_on_post_selection(): + """`cc-guard` must reach the same verdict as the guard it exposes. + + `cc.kernel.cliff.cliff_certificate` is the original refusal, reachable only + from Python. If the CLI ever permitted what the kernel refuses, every + cross-language caller would be worse off than a Python one. + """ + certificate = cliff_certificate(0.42, (0.21, 0.63), provenance="post-selection") + assert isinstance(certificate, CliffCertificate) + assert certificate.regime == "discovery-only" + + verdict = check( + { + "guard": "post_selection_interval", + "provenance": "post-selection", + "estimate": 0.42, + "ci_low": 0.21, + "ci_high": 0.63, + } + ) + assert verdict["verdict"] == certificate.regime + assert verdict["confidence_claim_permitted"] is False + + +def test_guard_and_cliff_certificate_agree_that_confirmatory_is_not_refused(): + """The other direction: the CLI must not refuse what the kernel allows.""" + certificate = cliff_certificate(0.42, (0.21, 0.63), provenance="confirmatory") + assert certificate.regime != "discovery-only" + + verdict = check( + { + "guard": "post_selection_interval", + "provenance": "confirmatory", + "estimate": 0.42, + "ci_low": 0.21, + "ci_high": 0.63, + } + ) + assert verdict["confidence_claim_permitted"] is True + + +# --- the process boundary ----------------------------------------------------- + + +def test_cli_check_refuses_and_can_gate_a_shell_caller(): + code, verdict = _run( + {"guard": "post_selection_interval", "provenance": "post-selection"}, + "check", + "--strict-exit", + ) + assert code == 1 + assert verdict["confidence_claim_permitted"] is False + + +def test_cli_check_permits_with_exit_zero(): + code, verdict = _run( + {"guard": "post_selection_interval", "provenance": "confirmatory"}, + "check", + "--strict-exit", + ) + assert code == 0 + assert verdict["confidence_claim_permitted"] is True + + +def test_cli_rejects_malformed_json(): + proc = subprocess.run( + [sys.executable, "-m", "cc.cli.guard", "check"], + input="not json", + capture_output=True, + text=True, + check=False, + ) + assert proc.returncode == 2 + assert json.loads(proc.stdout)["confidence_claim_permitted"] is False + + +def test_cli_rejects_a_non_object_request(): + proc = subprocess.run( + [sys.executable, "-m", "cc.cli.guard", "check"], + input="[1, 2, 3]", + capture_output=True, + text=True, + check=False, + ) + assert proc.returncode == 2 + assert json.loads(proc.stdout)["verdict"] == "refused" + + +def test_cli_table_emits_the_decision_data(): + proc = subprocess.run( + [sys.executable, "-m", "cc.cli.guard", "table"], + capture_output=True, + text=True, + check=False, + ) + assert proc.returncode == 0 + assert json.loads(proc.stdout) == DECISION_TABLE + + +def test_main_requires_a_subcommand(): + with pytest.raises(SystemExit): + main([]) + + +# --- in-process CLI coverage -------------------------------------------------- +# +# The subprocess tests above exercise the real process boundary, which is what a +# non-Python caller actually uses. Coverage cannot see into a subprocess, so +# these drive main() in-process as well: without them cc/cli/guard.py reports +# ~64% while its CLI body is in fact fully exercised. Both kinds are kept -- +# dropping the subprocess tests would stop testing the boundary that matters. + + +def test_main_check_refuses_in_process(monkeypatch, capsys): + monkeypatch.setattr( + "sys.stdin", + __import__("io").StringIO( + json.dumps({"guard": "post_selection_interval", "provenance": "post-selection"}) + ), + ) + assert main(["check", "--strict-exit"]) == 1 + verdict = json.loads(capsys.readouterr().out) + assert verdict["verdict"] == "discovery-only" + + +def test_main_check_permits_in_process(monkeypatch, capsys): + monkeypatch.setattr( + "sys.stdin", + __import__("io").StringIO( + json.dumps({"guard": "post_selection_interval", "provenance": "confirmatory"}) + ), + ) + assert main(["check", "--strict-exit"]) == 0 + assert json.loads(capsys.readouterr().out)["confidence_claim_permitted"] is True + + +def test_main_check_without_strict_exit_returns_zero_even_when_refusing(monkeypatch, capsys): + """The exit code is opt-in; the verdict is always in the payload.""" + monkeypatch.setattr( + "sys.stdin", + __import__("io").StringIO(json.dumps({"guard": "census_interval", "provenance": "census"})), + ) + assert main(["check"]) == 0 + assert json.loads(capsys.readouterr().out)["confidence_claim_permitted"] is False + + +def test_main_check_on_empty_stdin_refuses(monkeypatch, capsys): + monkeypatch.setattr("sys.stdin", __import__("io").StringIO("")) + assert main(["check", "--strict-exit"]) == 1 + assert json.loads(capsys.readouterr().out)["verdict"] == "refused" + + +def test_main_check_on_malformed_json_in_process(monkeypatch, capsys): + monkeypatch.setattr("sys.stdin", __import__("io").StringIO("{not json")) + assert main(["check"]) == 2 + assert json.loads(capsys.readouterr().out)["confidence_claim_permitted"] is False + + +def test_main_check_on_non_object_in_process(monkeypatch, capsys): + monkeypatch.setattr("sys.stdin", __import__("io").StringIO("[1,2,3]")) + assert main(["check"]) == 2 + assert json.loads(capsys.readouterr().out)["verdict"] == "refused" + + +def test_main_table_in_process(capsys): + assert main(["table"]) == 0 + assert json.loads(capsys.readouterr().out) == DECISION_TABLE diff --git a/tests/unit/compose/test_compose_bounds.py b/tests/unit/compose/test_compose_bounds.py new file mode 100644 index 00000000..10657208 --- /dev/null +++ b/tests/unit/compose/test_compose_bounds.py @@ -0,0 +1,263 @@ +"""Unit tests for the ROC-free composition surface.""" + +from __future__ import annotations + +import json +import math + +import pytest +from hypothesis import given, settings +from hypothesis import strategies as st + +from cc.compose import ( + CountermonotoneUndefinedError, + compose_bounds, + marginal_from_operating_point, + sensitivity, +) +from cc.kernel.sensitivity import AssumptionSet, LinearQuery, identified_region + +PROBS = st.floats(min_value=0.0, max_value=1.0, allow_nan=False, allow_infinity=False) + + +def _lp_interval(marginals: dict[str, float], event: str) -> tuple[float, float]: + names = tuple(marginals) + assumptions = AssumptionSet.empty(names) + for name in names: + assumptions = assumptions.with_marginal_interval(name, marginals[name], marginals[name]) + query = ( + LinearQuery.intersection(names, names) + if event == "all" + else LinearQuery.union(names, names) + ) + result = identified_region(query, assumptions) + return result.lower_bound, result.upper_bound + + +# --- the surface must not speak ROC ------------------------------------------ + + +def test_public_surface_mentions_no_detector_vocabulary(): + """The rejected-adoption report named ROC framing as the obstacle. + + A deterministic refusal rule has no threshold and no operating curve. If + that vocabulary reappears in the public signature, the surface has drifted + back to the shape a consumer already walked away from. + """ + import inspect + + from cc.compose import _bounds + + forbidden = ("roc", "youden", "tpr", "fpr", "threshold", "operating_point") + for name in ("compose_bounds", "sensitivity"): + signature = str(inspect.signature(getattr(_bounds, name))).lower() + for word in forbidden: + assert word not in signature, f"{name} signature mentions {word!r}" + + +def test_operating_point_adapter_points_inward(): + """Detectors reach the event calculus, not the other way round.""" + assert marginal_from_operating_point(false_negative_rate=0.2) == pytest.approx(0.2) + assert marginal_from_operating_point(false_negative_rate=0.2, prevalence=0.5) == pytest.approx( + 0.1 + ) + with pytest.raises(ValueError): + marginal_from_operating_point(false_negative_rate=1.5) + + +# --- agreement with the LP kernel -------------------------------------------- + + +@settings(max_examples=250, deadline=None) +@given( + values=st.lists(PROBS, min_size=1, max_size=6), + event=st.sampled_from(["all", "any"]), +) +def test_closed_form_agrees_with_finite_atom_lp(values, event): + """The fast closed form and the general LP must not diverge. + + Two code paths compute the same object. If they disagree, one is wrong and + every downstream artifact inherits it. + """ + marginals = {f"E{i}": v for i, v in enumerate(values)} + bounds = compose_bounds(marginals, event=event) + lp_lower, lp_upper = _lp_interval(marginals, event) + assert bounds.lower == pytest.approx(lp_lower, abs=1e-9) + assert bounds.upper == pytest.approx(lp_upper, abs=1e-9) + + +# --- invariants --------------------------------------------------------------- + + +@settings(max_examples=400, deadline=None) +@given( + values=st.lists(PROBS, min_size=1, max_size=8), + event=st.sampled_from(["all", "any"]), +) +def test_interval_invariants(values, event): + marginals = {f"E{i}": v for i, v in enumerate(values)} + bounds = compose_bounds(marginals, event=event) + assert 0.0 <= bounds.lower <= 1.0 + assert 0.0 <= bounds.upper <= 1.0 + assert bounds.lower <= bounds.upper + 1e-12 + assert bounds.width == pytest.approx(bounds.upper - bounds.lower, abs=1e-12) + # The independence point must lie inside the sharp interval: it is one + # admissible dependence structure among many. + assert bounds.lower - 1e-12 <= bounds.independence_point <= bounds.upper + 1e-12 + + +@settings(max_examples=200, deadline=None) +@given(values=st.lists(PROBS, min_size=2, max_size=6)) +def test_conjunction_upper_bound_does_not_depend_on_count(values): + """The correlation cliff, as an invariant. + + The conjunction upper bound is min(p). Adding an event can only lower it, + never raise it -- so stacking controls cannot improve the worst-case bound. + """ + marginals = {f"E{i}": v for i, v in enumerate(values)} + before = compose_bounds(marginals, event="all") + marginals["EXTRA"] = 1.0 # the weakest possible additional control + after = compose_bounds(marginals, event="all") + assert after.upper <= before.upper + 1e-12 + + +def test_cliff_is_reproduced_at_scale(): + """Ten filters at p=0.1 bound to [0, 0.1]; independence predicts 1e-10.""" + bounds = compose_bounds({f"G{i}": 0.1 for i in range(10)}, event="all") + assert bounds.lower == pytest.approx(0.0) + assert bounds.upper == pytest.approx(0.1) + assert bounds.independence_point == pytest.approx(1e-10) + assert bounds.understatement_factor > 1e8 + + +# --- binding event ------------------------------------------------------------ + + +def test_binding_event_names_the_argmin(): + bounds = compose_bounds({"A": 0.3, "B": 0.01, "C": 0.4}, event="all") + assert bounds.binding_event == "B" + + +def test_binding_event_is_none_on_a_tie(): + """A tie means no single event binds; naming one would misdirect effort.""" + bounds = compose_bounds({"A": 0.01, "B": 0.01, "C": 0.4}, event="all") + assert bounds.binding_event is None + + +def test_binding_event_is_none_for_a_multi_event_union(): + """The union upper bound is a sum, not a single event.""" + assert compose_bounds({"A": 0.1, "B": 0.2}, event="any").binding_event is None + assert compose_bounds({"A": 0.1}, event="any").binding_event == "A" + + +def test_binding_event_is_none_in_a_stipulated_regime(): + bounds = compose_bounds({"A": 0.3, "B": 0.01}, event="all", dependence="comonotone") + assert bounds.binding_event is None + + +# --- countermonotonicity ------------------------------------------------------ + + +@pytest.mark.parametrize("n", [1, 3, 4, 8]) +def test_countermonotone_is_refused_for_other_than_two_events(n): + """Countermonotonicity is strictly bivariate. + + There is no n-dimensional countermonotonic structure for n > 2, and the FH + lower bound is not a copula in dimension >= 3 though it stays pointwise + sharp. The n=1 case was found by differential fuzz: the reference raised + IndexError and the Node implementation returned NaN. + """ + marginals = {f"E{i}": 0.3 for i in range(n)} + with pytest.raises(CountermonotoneUndefinedError): + compose_bounds(marginals, event="all", dependence="countermonotone") + + +def test_countermonotone_is_permitted_for_exactly_two(): + bounds = compose_bounds({"A": 0.7, "B": 0.8}, event="all", dependence="countermonotone") + assert bounds.lower == bounds.upper == pytest.approx(0.5) + + +def test_unknown_dependence_fails_closed(): + with pytest.raises(ValueError, match="unknown dependence"): + compose_bounds({"A": 0.3}, dependence="sorta-dependent") # type: ignore[arg-type] + + +# --- input validation --------------------------------------------------------- + + +@pytest.mark.parametrize("bad", [1.5, -0.1, float("nan"), float("inf")]) +def test_marginals_outside_the_unit_interval_are_refused(bad): + with pytest.raises(ValueError): + compose_bounds({"A": bad, "B": 0.2}) + + +def test_empty_marginals_are_refused(): + with pytest.raises(ValueError, match="at least one event"): + compose_bounds({}) + + +def test_a_bare_sequence_is_refused(): + """An unnamed marginal cannot be reported against a binding event.""" + with pytest.raises(TypeError, match="mapping"): + compose_bounds([0.1, 0.2]) # type: ignore[arg-type] + + +def test_unknown_event_kind_is_refused(): + with pytest.raises(ValueError, match="unknown event kind"): + compose_bounds({"A": 0.1}, event="xor") + + +@pytest.mark.parametrize( + ("alias", "canonical"), + [ + ("and", "all"), + ("AND", "all"), + ("intersection", "all"), + ("or", "any"), + ("OR", "any"), + ("union", "any"), + ], +) +def test_event_aliases_canonicalize(alias, canonical): + assert compose_bounds({"A": 0.1, "B": 0.2}, event=alias).event == canonical + + +# --- the result object carries its context ------------------------------------ + + +def test_result_carries_provenance_and_non_claims(): + bounds = compose_bounds({"A": 0.3}, marginal_provenance="assumed") + assert bounds.marginal_provenance == "assumed" + assert bounds.non_claims + assert any("not a measurement" in c for c in bounds.non_claims) + + +def test_to_json_is_json_native_and_round_trips(): + bounds = compose_bounds({"A": 0.3, "B": 0.4}, event="all") + payload = bounds.to_json() + assert json.loads(json.dumps(payload)) == payload + + +def test_understatement_factor_serializes_as_null_when_undefined(): + """JSON has no infinity. Null means undefined, not large.""" + bounds = compose_bounds({"A": 0.0, "B": 0.4}, event="all") + assert math.isinf(bounds.understatement_factor) + assert bounds.to_json()["understatement_factor"] is None + + +# --- sensitivity -------------------------------------------------------------- + + +def test_sensitivity_shows_only_the_binding_event_moves_the_upper_bound(): + """The actionable finding: effort on a weak control buys nothing.""" + rows = sensitivity({"WEAK": 0.40, "STRONG": 0.01, "MIDDLE": 0.30}, event="all", delta=0.05) + by_event = {r.event: r for r in rows} + assert by_event["STRONG"].moves_upper + assert not by_event["WEAK"].moves_upper + assert not by_event["MIDDLE"].moves_upper + assert by_event["WEAK"].upper_delta == pytest.approx(0.0) + + +def test_sensitivity_rejects_an_out_of_range_delta(): + with pytest.raises(ValueError, match="delta"): + sensitivity({"A": 0.3}, delta=0.0) diff --git a/tests/unit/docs/test_canonical_page_fixture.py b/tests/unit/docs/test_canonical_page_fixture.py new file mode 100644 index 00000000..4085aaa8 --- /dev/null +++ b/tests/unit/docs/test_canonical_page_fixture.py @@ -0,0 +1,58 @@ +"""The canonical page embeds a real fixture, so it must not drift from the capsule. + +`visual_identity/canonical_page/index.html` recomputes SHA-256 in the visitor's +browser and compares it against a digest recorded in the deterministic claim +governance capsule. If the capsule is regenerated and the page is not updated, +the page would show a legitimate artifact failing closed - an honest-looking +screen making a false statement. These tests fail first instead. + +What these tests do NOT establish: they check internal consistency between the +page, the fixture, and the manifest. They cannot tell a measured input from an +asserted one. Changing the capsule's declared input and regenerating the chain +passes every guard here and in the capsule integration test - see CH-001 in +docs/research/epistemic-program/challenges.md. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] +PAGE = ROOT / "visual_identity" / "canonical_page" / "index.html" +CAPSULE = ROOT / "examples" / "claim_governance_capsule" +FIXTURE = CAPSULE / "expected" / "calibration.json" +MANIFEST = CAPSULE / "manifest.expected.json" + + +def _embedded(name: str) -> str: + match = re.search(rf"^const {name} = (.+);$", PAGE.read_text(), re.MULTILINE) + assert match, f"{PAGE.name} no longer defines {name}" + return json.loads(match.group(1)) + + +def _recorded_digest() -> str: + manifest = json.loads(MANIFEST.read_text()) + entry = next(f for f in manifest["files"] if f["filename"] == FIXTURE.name) + return entry["sha256"] + + +def test_embedded_document_matches_the_capsule_fixture() -> None: + assert _embedded("DOC") == FIXTURE.read_text() + + +def test_embedded_digest_matches_the_capsule_manifest() -> None: + assert _embedded("EXPECTED") == _recorded_digest() + + +def test_page_check_passes_on_unmodified_bytes() -> None: + digest = hashlib.sha256(_embedded("DOC").encode()).hexdigest() + + assert digest == _embedded("EXPECTED") + + +def test_capsule_manifest_digest_is_a_plain_file_digest() -> None: + """The browser hashes raw bytes; the manifest must mean the same thing.""" + assert hashlib.sha256(FIXTURE.read_bytes()).hexdigest() == _recorded_digest() diff --git a/tests/unit/docs/test_claim_boundary_manifest.py b/tests/unit/docs/test_claim_boundary_manifest.py index cae52281..5b572d9c 100644 --- a/tests/unit/docs/test_claim_boundary_manifest.py +++ b/tests/unit/docs/test_claim_boundary_manifest.py @@ -1,7 +1,10 @@ from __future__ import annotations -from scripts.validate_claim_boundary_manifest import CLAIM_REQUIRED, load_manifest, validate_manifest - +from scripts.validate_claim_boundary_manifest import ( + CLAIM_REQUIRED, + load_manifest, + validate_manifest, +) MAJOR_FORBIDDEN_UPGRADES = { ("Governance PASS", "deployment safety"), @@ -23,7 +26,7 @@ def test_claim_boundary_manifest_required_fields_and_unique_ids() -> None: seen: set[str] = set() for claim in manifest["claims"]: - assert CLAIM_REQUIRED <= claim.keys() + assert claim.keys() >= CLAIM_REQUIRED assert claim["id"] not in seen seen.add(claim["id"]) assert claim["non_claims"] @@ -33,4 +36,4 @@ def test_claim_boundary_manifest_contains_major_forbidden_upgrades() -> None: manifest = load_manifest() upgrades = {(item["from"], item["to"]) for item in manifest["forbidden_upgrades"]} - assert MAJOR_FORBIDDEN_UPGRADES <= upgrades + assert upgrades >= MAJOR_FORBIDDEN_UPGRADES diff --git a/tests/unit/evidence/test_transparency_log_adversarial.py b/tests/unit/evidence/test_transparency_log_adversarial.py index 8f8ed408..c268d324 100644 --- a/tests/unit/evidence/test_transparency_log_adversarial.py +++ b/tests/unit/evidence/test_transparency_log_adversarial.py @@ -148,12 +148,47 @@ def test_merkle_canonicalization_matches_reporting_canonical_json() -> None: assert MerkleLog(records=[record]).leaf_hashes == [expected] -def test_merkle_log_normalizes_unicode_nfc_like_reporting() -> None: +def test_merkle_log_does_not_normalize_unicode() -> None: + """Byte-distinct strings stay distinct. Their normal forms are not the log's business. + + This test previously asserted the opposite -- that a composed and a + decomposed spelling produce the *same* leaf hash -- and was named + ``test_merkle_log_normalizes_unicode_nfc_like_reporting``. That behavior was + finding F-03: canonicalization applied NFC to every key and wrote the + results into a fresh dict, so two byte-distinct keys sharing a normal form + silently became one, with no error, and the receipt attested to a document + with a field missing. + + RFC 8785 is explicit that normalization is the producer's responsibility. A + canonicalizer that mutates content is not a canonicalizer, and a + transparency log must record what was written rather than a normalized + rendering of it. Under ``cc.canonical.v2`` the two spellings are two + strings, which is what JSON says they are. + + Producers that would rather refuse such a document can call + ``assert_no_confusable_keys`` before hashing; it is deliberately not on the + hash path. + """ composed = {"é": "café", "event": "audit"} decomposed = {"e\u0301": "cafe\u0301", "event": "audit"} - assert canonical_json_bytes(composed) == canonical_json_bytes(decomposed) - assert leaf_hash(composed) == leaf_hash(decomposed) + assert canonical_json_bytes(composed) != canonical_json_bytes(decomposed) + assert leaf_hash(composed) != leaf_hash(decomposed) + + +def test_merkle_log_keeps_both_key_spellings_in_one_record() -> None: + """The silent key loss of F-03, asserted not to recur. + + Under the legacy profile this record went in with two keys and came out + with one, and nothing raised. + """ + record = {"é": 1, "e\u0301": 2, "event": "audit"} + + decoded = json.loads(canonical_json_bytes(record).decode("utf-8")) + + assert len(decoded) == 3 + assert decoded["é"] == 1 + assert decoded["e\u0301"] == 2 def test_merkle_log_rejects_non_string_mapping_keys(tmp_path: Path) -> None: diff --git a/tests/unit/evidence_card/test_evidence_card.py b/tests/unit/evidence_card/test_evidence_card.py new file mode 100644 index 00000000..111def83 --- /dev/null +++ b/tests/unit/evidence_card/test_evidence_card.py @@ -0,0 +1,288 @@ +"""Evidence cards: the object, and the things it refuses to be. + +Most of these tests assert an *absence*. That is deliberate. The failure mode +this object exists to prevent is a surface that collapses three orthogonal +labels into one green checkmark, so the tests that matter most are the ones +asserting no such collapse is reachable. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path + +import pytest +from jsonschema import Draft202012Validator + +from cc.evidence_card import ( + EVIDENCE_STATES, + LABEL_MEANINGS, + PUBLICATION_STATES, + VERDICTS, + ArtifactRef, + EvidenceCard, + EvidenceCardError, + cards_to_site_manifest, + render_labels, +) + +ROOT = Path(__file__).resolve().parents[3] +SCHEMA = json.loads((ROOT / "schemas" / "cc.evidence_card.v1.json").read_text()) +CARDS_DIR = ROOT / "evidence-cards" / "cards" +SITE_MANIFEST = ROOT / "evidence-cards" / "site-evidence-manifest.v1.json" + + +def _card(**overrides) -> EvidenceCard: + base = { + "card_id": "demo.claim", + "claim": "The kernel computes classical Frechet-Hoeffding bounds.", + "maturity": "C3", + "source_revision": "abc123", + "command": ("pytest tests/unit/kernel",), + "falsifier": "A marginal configuration where the LP and the closed form disagree.", + "assumptions": ("Exact marginals are supplied.",), + "non_claims": ("Does not prove deployment safety.",), + "evidence_state": "local-only", + "verdict": "not-run", + "publication_state": "draft", + } + base.update(overrides) + return EvidenceCard(**base) + + +# --- the refusals that make a card evidence ---------------------------------- + + +def test_a_card_without_a_falsifier_is_refused(): + """A claim no observation could refute is an assertion, not evidence.""" + with pytest.raises(EvidenceCardError, match="falsifier"): + _card(falsifier=" ") + + +def test_a_card_without_a_command_is_refused(): + with pytest.raises(EvidenceCardError, match="at least one command"): + _card(command=()) + + +def test_a_card_without_a_non_claim_is_refused(): + with pytest.raises(EvidenceCardError, match="explicit non-claim"): + _card(non_claims=()) + + +@pytest.mark.parametrize("verdict", ["pass", "fail"]) +def test_a_verdict_without_a_record_of_the_run_is_refused(verdict): + """You cannot assert a result without saying what produced it.""" + with pytest.raises(EvidenceCardError, match="result_detail"): + _card(verdict=verdict, result_detail=None) + + +def test_a_retraction_without_a_reason_is_refused(): + """Withdrawing a claim without saying why rewrites the record.""" + with pytest.raises(EvidenceCardError, match="retraction_reason"): + _card(publication_state="retracted") + + +def test_a_superseded_card_must_name_its_successor(): + with pytest.raises(EvidenceCardError, match="supersedes"): + _card(publication_state="superseded") + + +@pytest.mark.parametrize( + ("label", "bad"), + [ + ("evidence_state", "production"), + ("verdict", "green"), + ("verdict", "ok"), + ("publication_state", "live"), + ], +) +def test_labels_outside_their_declared_sets_are_refused(label, bad): + with pytest.raises(EvidenceCardError, match=label): + _card(**{label: bad}) + + +# --- the collapse this object exists to prevent ------------------------------- + + +def test_the_card_exposes_no_composite_status(): + """No `status`, `score`, `is_ok`, or `badge`. + + A single glyph lets a reader stop before asking any of the three questions + the labels answer. If one is ever added, this test is the tripwire. + """ + card = _card() + for forbidden in ( + "status", + "score", + "is_ok", + "ok", + "badge", + "health", + "passing", + "overall", + "summary", + "grade", + ): + assert not hasattr(card, forbidden), f"EvidenceCard grew a composite: {forbidden}" + + +def test_render_labels_returns_all_three_or_nothing(): + """There is no single-label renderer: a surface that can show one will show + the flattering one.""" + rendered = render_labels(_card()) + assert set(rendered) == {"evidence_state", "verdict", "publication_state"} + for entry in rendered.values(): + assert entry["value"] + assert entry["meaning"] + + +def test_labels_are_independent_every_combination_is_constructible(): + """No label constrains another. + + A retracted card may still carry a passing verdict: the run happened, and + the claim was withdrawn anyway. A local-only pass says nothing about + deployment. If a future change couples them, this fails. + """ + built = 0 + for evidence_state in EVIDENCE_STATES: + for verdict in VERDICTS: + for publication_state in PUBLICATION_STATES: + card = _card( + evidence_state=evidence_state, + verdict=verdict, + publication_state=publication_state, + result_detail="ran: 6 passed" if verdict in ("pass", "fail") else None, + retraction_reason=( + "superseded by a sharper bound" + if publication_state == "retracted" + else None + ), + supersedes=("demo.claim.v2" if publication_state == "superseded" else None), + ) + assert card.evidence_state == evidence_state + assert card.verdict == verdict + assert card.publication_state == publication_state + built += 1 + assert built == len(EVIDENCE_STATES) * len(VERDICTS) * len(PUBLICATION_STATES) + + +def test_every_label_value_carries_a_meaning(): + """A bare token in a UI is an invitation to guess.""" + for label, values in ( + ("evidence_state", EVIDENCE_STATES), + ("verdict", VERDICTS), + ("publication_state", PUBLICATION_STATES), + ): + for value in values: + assert LABEL_MEANINGS[label][value].strip() + + +def test_not_run_is_the_default_verdict_of_the_generator(): + """A pass must be earned by running something.""" + assert _card().verdict == "not-run" + + +# --- serialization ------------------------------------------------------------ + + +def test_to_json_keeps_the_three_labels_separate(): + payload = _card().to_json() + assert set(payload["labels"]) == {"evidence_state", "verdict", "publication_state"} + # No top-level flattened alias may appear beside them. + for forbidden in ("status", "score", "badge", "ok", "passing"): + assert forbidden not in payload + + +def test_to_json_round_trips(): + payload = _card().to_json() + assert json.loads(json.dumps(payload)) == payload + + +def test_missing_artifacts_are_recorded_not_skipped(): + """A claim pointing at a file that is gone is a finding, not an absence.""" + card = _card(artifacts=(ArtifactRef(path="docs/gone.md", missing=True),)) + artifact = card.to_json()["artifacts"][0] + assert artifact["missing"] is True + assert artifact["sha256"] is None + + +# --- the site manifest -------------------------------------------------------- + + +def test_site_manifest_counts_per_label_and_never_aggregates(): + cards = [ + _card(card_id="a", verdict="not-run"), + _card(card_id="b", verdict="pass", result_detail="6 passed"), + _card(card_id="c", evidence_state="aws-synth-only", verdict="unverifiable"), + ] + site = cards_to_site_manifest(cards, source_revision="abc", generated_note="test") + + assert site["card_count"] == 3 + assert site["counts_by_label"]["verdict"]["pass"] == 1 + assert site["counts_by_label"]["verdict"]["not-run"] == 1 + assert site["counts_by_label"]["evidence_state"]["aws-synth-only"] == 1 + # No aggregate anywhere. + for forbidden in ("score", "passing_rate", "health", "overall", "summary_status"): + assert forbidden not in site + + +def test_site_manifest_states_that_its_counts_are_not_a_score(): + site = cards_to_site_manifest([_card()], source_revision="abc", generated_note="n") + joined = " ".join(site["non_claims"]).lower() + assert "orthogonal" in joined + assert "no aggregate score" in joined + assert "not-run" in joined + + +# --- the committed artifacts -------------------------------------------------- + + +def test_schema_is_valid_and_defines_no_aggregate(): + Draft202012Validator.check_schema(SCHEMA) + properties = set(SCHEMA["properties"]) + for forbidden in ("status", "score", "badge", "health", "overall"): + assert forbidden not in properties + + +@pytest.mark.parametrize("path", sorted(CARDS_DIR.glob("*.json")), ids=lambda p: p.stem) +def test_committed_cards_validate_against_the_schema(path): + errors = list(Draft202012Validator(SCHEMA).iter_errors(json.loads(path.read_text()))) + assert not errors, [e.message for e in errors[:3]] + + +def test_committed_cards_are_current(): + proc = subprocess.run( + [sys.executable, str(ROOT / "scripts" / "build_evidence_cards.py"), "--check"], + capture_output=True, + text=True, + check=False, + cwd=ROOT, + env={"PYTHONPATH": str(ROOT / "src"), "PATH": "/usr/bin:/bin"}, + ) + assert proc.returncode == 0, proc.stdout + proc.stderr + + +def test_committed_cards_claim_no_results(): + """The committed scaffold must never carry a verdict from someone's laptop.""" + site = json.loads(SITE_MANIFEST.read_text()) + verdicts = site["counts_by_label"]["verdict"] + assert verdicts["not-run"] == site["card_count"] + assert verdicts["pass"] == 0 + assert verdicts["fail"] == 0 + + +def test_every_committed_card_has_a_falsifier_and_a_non_claim(): + for path in sorted(CARDS_DIR.glob("*.json")): + card = json.loads(path.read_text()) + assert card["falsifier"].strip(), path.name + assert card["non_claims"], path.name + assert card["assumptions"], path.name + + +def test_the_synthetic_aws_lane_is_labelled_as_such(): + """Synthetic evidence is never live evidence, and the label must say so.""" + card = json.loads((CARDS_DIR / "enterprise.reference_v0_1.json").read_text()) + assert card["labels"]["evidence_state"] == "aws-synth-only" + assert "never live evidence" in " ".join(card["assumptions"]).lower() diff --git a/tests/unit/reporting/test_reporting.py b/tests/unit/reporting/test_reporting.py index 8489dc2d..84501ed6 100644 --- a/tests/unit/reporting/test_reporting.py +++ b/tests/unit/reporting/test_reporting.py @@ -296,10 +296,14 @@ def test_schema_contract_matches_report_builder_constants() -> None: ) assert schema_claim_levels == CLAIM_LEVELS - assert ( - schema["properties"]["receipt"]["properties"]["canonicalization_method"]["const"] - == CANONICALIZATION_METHOD - ) + # The schema accepts both canonicalization profiles so pre-migration + # receipts stay verifiable; the builder writes the first of them. + schema_profiles = schema["properties"]["receipt"]["properties"]["canonicalization_method"][ + "enum" + ] + assert CANONICALIZATION_METHOD in schema_profiles + assert schema_profiles[0] == CANONICALIZATION_METHOD + assert len(schema_profiles) == 2 report = _report() report["claim"]["non_claims"] = [] diff --git a/verifiers/node/cc_compose_verify.mjs b/verifiers/node/cc_compose_verify.mjs new file mode 100644 index 00000000..4b46eab4 --- /dev/null +++ b/verifiers/node/cc_compose_verify.mjs @@ -0,0 +1,387 @@ +#!/usr/bin/env node +/** + * ═══════════════════════════════════════════════════════════════════════════ + * INDEPENDENT CC COMPOSITION KERNEL — Node, zero dependencies + * + * Written from `conformance/cc-kernel-v1/SPEC.md` and the JSON corpus. It does + * not import, transcribe, or consult `cc.compose`. That constraint is the + * whole value: an implementation written by reading the Python would inherit + * its mistakes, and agreement between the two would establish nothing. + * + * ── What agreement here does and does not establish ──────────────────────── + * + * Agreement establishes that two implementations, working from a written + * specification, compute the same intervals on the corpus. It does NOT + * establish that either is correct. Both were authored in the same project, + * and a specification that is wrong produces two implementations that are + * wrong together. This is a DIFFERENTIAL TESTING INSTRUMENT, not an + * independent replication. + * + * The genuinely independent check is `--external-oracle`, which reproduces + * numbers published by a separate project that implemented these bounds for + * its own purposes, without reference to this corpus. See §"external oracle" + * below. + * + * node verifiers/node/cc_compose_verify.mjs run the corpus + * node verifiers/node/cc_compose_verify.mjs --json machine-readable + * node verifiers/node/cc_compose_verify.mjs --external-oracle + * node verifiers/node/cc_compose_verify.mjs --batch < cases.json + * ═══════════════════════════════════════════════════════════════════════════ + */ + +import { readFileSync, readSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const CORPUS = join(HERE, '..', '..', 'conformance', 'cc-kernel-v1'); + +/* ── SPEC §1.3 · event kinds ─────────────────────────────────────────────── */ + +const EVENT_ALIASES = new Map([ + ['all', 'all'], ['and', 'all'], ['AND', 'all'], ['intersection', 'all'], + ['any', 'any'], ['or', 'any'], ['OR', 'any'], ['union', 'any'] +]); + +/* ── SPEC §1.4 · dependence assumptions ──────────────────────────────────── */ + +const DEPENDENCE = new Set([ + 'unconstrained', 'independent', 'comonotone', 'countermonotone' +]); + +/** A refusal carrying the SPEC §5 reason identifier. */ +class Refusal extends Error { + constructor(reason, detail) { + super(`${reason}: ${detail}`); + this.reason = reason; + } +} + +const clip01 = (x) => Math.min(1, Math.max(0, x)); + +/* ── SPEC §1.2 · marginal validation ─────────────────────────────────────── */ + +function validateMarginals(marginals) { + if (marginals === null || typeof marginals !== 'object' || Array.isArray(marginals)) { + throw new Refusal('no_events', 'marginals must be a mapping of name to probability'); + } + const names = Object.keys(marginals); + if (names.length === 0) throw new Refusal('no_events', 'at least one event is required'); + + const out = []; + for (const name of names) { + if (typeof name !== 'string' || name.length === 0) { + throw new Refusal('no_events', `event name must be a non-empty string: ${name}`); + } + const p = marginals[name]; + if (typeof p !== 'number' || !Number.isFinite(p)) { + throw new Refusal('marginal_not_finite', `${name} = ${p}`); + } + if (p < 0 || p > 1) { + throw new Refusal('marginal_out_of_range', `${name} = ${p}`); + } + out.push([name, p]); + } + return out; +} + +/* ── SPEC §2 · the unconstrained interval ────────────────────────────────── */ + +function classical(values, kind) { + const n = values.length; + const sum = values.reduce((a, b) => a + b, 0); + if (kind === 'all') { + return [clip01(Math.max(0, sum - (n - 1))), clip01(Math.min(...values))]; + } + return [clip01(Math.max(...values)), clip01(Math.min(1, sum))]; +} + +/* ── SPEC §3 · the independence baseline ─────────────────────────────────── */ + +function independencePoint(values, kind) { + if (kind === 'all') return values.reduce((a, b) => a * b, 1); + return clip01(1 - values.reduce((a, b) => a * (1 - b), 1)); +} + +/* ── SPEC §4 · binding_event ─────────────────────────────────────────────── */ + +const TIE_TOL = 1e-12; + +function bindingEvent(pairs, kind) { + if (kind === 'all') { + const min = Math.min(...pairs.map(([, p]) => p)); + const atMin = pairs.filter(([, p]) => Math.abs(p - min) <= TIE_TOL); + // §4.3 — a tie means no single event binds. + return atMin.length === 1 ? atMin[0][0] : null; + } + // §4.2 — the union upper bound is a sum, not a single event. + return pairs.length === 1 ? pairs[0][0] : null; +} + +/* ── the kernel ──────────────────────────────────────────────────────────── */ + +export function composeBounds(marginals, { event = 'all', dependence = 'unconstrained' } = {}) { + const kind = EVENT_ALIASES.get(event); + if (kind === undefined) throw new Refusal('unknown_event_kind', String(event)); + if (!DEPENDENCE.has(dependence)) throw new Refusal('unknown_dependence', String(dependence)); + + const pairs = validateMarginals(marginals); + const values = pairs.map(([, p]) => p); + + // §6 — countermonotonicity is strictly bivariate: exactly two, never more, + // never fewer. Without this the n=1 branch reads values[1] as undefined and + // returns NaN rather than refusing. Found by differential fuzz, seed 1. + if (dependence === 'countermonotone' && values.length !== 2) { + throw new Refusal( + 'countermonotone_undefined', + values.length < 2 + ? `${values.length} event(s); countermonotonicity is a relation between ` + + 'two events, so there is nothing to be countermonotone with' + : `${values.length} events; there is no n-dimensional countermonotonic ` + + 'structure for n > 2, and the FH lower bound is not a copula in ' + + 'dimension >= 3 (though it stays pointwise sharp)' + ); + } + + const indep = independencePoint(values, kind); + let [lower, upper] = classical(values, kind); + + if (dependence === 'independent') { + lower = upper = indep; + } else if (dependence === 'comonotone') { + lower = upper = kind === 'all' ? clip01(Math.min(...values)) : clip01(Math.max(...values)); + } else if (dependence === 'countermonotone') { + lower = upper = kind === 'all' + ? clip01(Math.max(0, values[0] + values[1] - 1)) + : clip01(Math.min(1, values[0] + values[1])); + } + + // §4.4 — no event binds in a stipulated regime. + const binding = dependence === 'unconstrained' ? bindingEvent(pairs, kind) : null; + + return { + lower, + upper, + width: Math.max(0, upper - lower), + event: kind, + dependence, + independence_point: indep, + independence_regret: upper - indep, + // §3 — JSON has no infinity; null means undefined, not large. + understatement_factor: indep > 0 ? upper / indep : null, + binding_event: binding + }; +} + +/* ── corpus runner ───────────────────────────────────────────────────────── */ + +const readJson = (p) => JSON.parse(readFileSync(p, 'utf8')); + +function runAccept(tolerance) { + const doc = readJson(join(CORPUS, 'cases', 'composition.json')); + const failures = []; + for (const c of doc.cases) { + let got; + try { + got = composeBounds(c.input.marginals, { + event: c.input.event, + dependence: c.input.dependence + }); + } catch (err) { + failures.push({ id: c.id, field: '', detail: err.message }); + continue; + } + for (const [field, want] of Object.entries(c.expect)) { + const have = got[field]; + if (field === 'binding_event' || want === null || have === null) { + // §4 — compared exactly, including the null cases. + if (have !== want) failures.push({ id: c.id, field, want, have }); + continue; + } + if (Math.abs(have - want) > tolerance) { + failures.push({ id: c.id, field, want, have, delta: Math.abs(have - want) }); + } + } + } + return { total: doc.cases.length, failures }; +} + +function runReject() { + const doc = readJson(join(CORPUS, 'cases', 'adversarial.json')); + const failures = []; + for (const c of doc.cases) { + // §5 JSON note — the corpus encodes NaN as the string "NaN". + const marginals = {}; + for (const [k, v] of Object.entries(c.input.marginals)) { + marginals[k] = v === 'NaN' ? Number.NaN : v; + } + try { + composeBounds(marginals, { event: c.input.event, dependence: c.input.dependence }); + failures.push({ id: c.id, detail: 'accepted, but the corpus requires a refusal' }); + } catch (err) { + if (!(err instanceof Refusal)) { + failures.push({ id: c.id, detail: `threw ${err.constructor.name}, not a typed refusal` }); + } else if (err.reason !== c.expect_refusal) { + failures.push({ id: c.id, want: c.expect_refusal, have: err.reason }); + } + } + } + return { total: doc.cases.length, failures }; +} + +/* ── external oracle ───────────────────────────────────────────────────────── + * + * Numbers PUBLISHED by a separate project that implemented these bounds + * independently, for its own purposes, before this corpus existed. Reproducing + * them is the one check here that is not same-author: neither this file nor + * the Python reference was consulted when those numbers were produced. + * + * Source: a four-control ledger composition study. Detection rates are ASSUMED + * by that project — zero records had been issued — so these are bounds over + * assumed inputs, never measurements. + */ +const EXTERNAL_ORACLE = [ + { + name: 'four-control ledger, small-chapter scenario', + // Published as detection rates; evasion is 1 - detection. + detection: { SELF_REPORTED: 0.70, STALE: 0.99, IMMUTABLE: 0.99, SEPARATION: 0.60 }, + published: { lower: 0.0, upper: 0.01, width: 0.01, independence_point: 0.000012 }, + // Published rounded, as "understates the admissible worst case by 833x". + publishedFactorRounded: 833 + } +]; + +function runExternalOracle(tolerance) { + const failures = []; + for (const o of EXTERNAL_ORACLE) { + const marginals = {}; + for (const [k, d] of Object.entries(o.detection)) marginals[k] = 1 - d; + const got = composeBounds(marginals, { event: 'all' }); + for (const [field, want] of Object.entries(o.published)) { + if (Math.abs(got[field] - want) > tolerance) { + failures.push({ oracle: o.name, field, want, have: got[field] }); + } + } + const factor = Math.round(got.understatement_factor); + if (factor !== o.publishedFactorRounded) { + failures.push({ + oracle: o.name, field: 'understatement_factor(rounded)', + want: o.publishedFactorRounded, have: factor + }); + } + } + return { total: EXTERNAL_ORACLE.length, failures }; +} + +/* ── batch mode, for the differential harness ──────────────────────────────── + * + * Reads {"cases":[{"id","marginals","event","dependence"}]} on stdin and emits + * one result or one typed refusal per case. The harness on the other side + * compares; this side never sees the reference answers, so it cannot converge + * on them. + */ +/** + * Read all of stdin. `readFileSync(0)` returns only the first chunk when stdin + * is a pipe, which silently truncated large batches at 64 KiB. + */ +function readStdin() { + const chunks = []; + const buf = Buffer.alloc(1 << 16); + for (;;) { + let n; + try { + n = readSync(0, buf, 0, buf.length, null); + } catch (err) { + if (err.code === 'EAGAIN') continue; + if (err.code === 'EOF') break; + throw err; + } + if (n === 0) break; + chunks.push(Buffer.from(buf.subarray(0, n))); + } + return Buffer.concat(chunks).toString('utf8'); +} + +function runBatch() { + const doc = JSON.parse(readStdin()); + const results = doc.cases.map((c) => { + try { + const got = composeBounds(c.marginals, { + event: c.event ?? 'all', + dependence: c.dependence ?? 'unconstrained' + }); + return { id: c.id, ok: true, ...got }; + } catch (err) { + return { + id: c.id, + ok: false, + refusal: err instanceof Refusal ? err.reason : `untyped:${err.constructor.name}` + }; + } + }); + process.stdout.write(JSON.stringify({ results })); + return 0; +} + +/* ── main ────────────────────────────────────────────────────────────────── */ + +function main(argv) { + if (argv.includes('--batch')) return runBatch(); + const wantJson = argv.includes('--json'); + const oracleOnly = argv.includes('--external-oracle'); + const manifest = readJson(join(CORPUS, 'manifest.json')); + const tol = manifest.tolerance; + + // The oracle numbers are published to 1e-6, not to the corpus tolerance. + const oracle = runExternalOracle(1e-9); + const accept = oracleOnly ? { total: 0, failures: [] } : runAccept(tol); + const reject = oracleOnly ? { total: 0, failures: [] } : runReject(); + + const failed = accept.failures.length + reject.failures.length + oracle.failures.length; + const report = { + corpus: manifest.corpus, + tolerance: tol, + implementation: 'verifiers/node/cc_compose_verify.mjs', + derived_from: 'conformance/cc-kernel-v1/SPEC.md', + accept: { total: accept.total, failed: accept.failures.length, failures: accept.failures }, + reject: { total: reject.total, failed: reject.failures.length, failures: reject.failures }, + external_oracle: { total: oracle.total, failed: oracle.failures.length, failures: oracle.failures }, + // Provenance: exact counts, no interval. The corpus is the whole + // population, not a sample. + provenance: 'census', + agreement_non_claim: + 'Agreement establishes that two implementations of one specification ' + + 'compute the same values. It does not establish that either is correct: ' + + 'both were authored in the same project and can share a misreading. ' + + 'Only external_oracle is not same-author.' + }; + + if (wantJson) { + console.log(JSON.stringify(report, null, 2)); + return failed ? 1 : 0; + } + + console.log('independent CC composition kernel — corpus agreement'); + console.log(` corpus ${manifest.corpus} (tolerance ${tol})`); + console.log(` derived ${report.derived_from}`); + console.log(''); + if (!oracleOnly) { + console.log(` accept ${accept.total - accept.failures.length}/${accept.total} agree`); + console.log(` reject ${reject.total - reject.failures.length}/${reject.total} refuse correctly`); + } + console.log(` oracle ${oracle.total - oracle.failures.length}/${oracle.total} external published results reproduced`); + console.log(''); + for (const f of [...accept.failures, ...reject.failures, ...oracle.failures]) { + console.log(` FAIL ${JSON.stringify(f)}`); + } + console.log(failed ? `FAIL: ${failed} disagreement(s).` : 'PASS: full agreement.'); + console.log('\n ' + report.agreement_non_claim.replace(/(.{72}) /g, '$1\n ')); + return failed ? 1 : 0; +} + +if (import.meta.url === `file://${process.argv[1]}`) { + // NOT process.exit(): it discards pending stdout writes to a pipe, which + // truncated batch output at the 64 KiB pipe buffer. Setting exitCode lets + // Node flush and exit on its own. + process.exitCode = main(process.argv.slice(2)); +} diff --git a/visual_identity/before_you_see_it/DIRECTORS_CUT.md b/visual_identity/before_you_see_it/DIRECTORS_CUT.md new file mode 100644 index 00000000..242f3a70 --- /dev/null +++ b/visual_identity/before_you_see_it/DIRECTORS_CUT.md @@ -0,0 +1,170 @@ +# Before You See It - Director's Cut + +## Logline + +The moment a seductive story is forced to become a test. + +## Thesis + +```text +A story can start a question. +Evidence must finish the answer. +``` + +Campaign line: **Keep the wonder. Check the claim.** + +The film honors curiosity without letting it impersonate evidence. It does not +mock the story. It makes the standard of proof impossible to evade. That +distinction is the whole brief: tenderness first, then ruthlessness. + +## Placement in the visual world + +This is **Chamber 00** of the Claim Observatory - the room before the Arrival +Hall. Every later chamber assumes a claim already exists as a bounded artifact. +Chamber 00 is where the claim is born: the instant an intuition stops being a +feeling and is written down in a form that can lose. + +It inherits the [World Bible V3](../claim_observatory/WORLD_BIBLE_V3.md) laws +without exception, including Law 10 - *the world must make overclaiming +uncomfortable*. + +## Beat sheet + +Runtime 15.000s. 1920x1080. 60fps. Silent master. + +| Time | Beat | Visual | On-screen copy | +| --- | --- | --- | --- | +| 0.00-2.00 | **The lure** | An electric-blue notebook in darkness, lit warm and almost oracular from above left. Gold embers drift. A slow 4.5% push-in. | `THIS FEELS TRUE.` | +| 2.00-3.40 | **The freeze** | Hard cut: the light goes cold, the push-in stops dead, and the embers quantize into a 12x7 grid of candidate matches - `BLUE NOTEBOOK`, `CRACKED SCREEN`, `CURLY HAIR`, `VENDING MACHINE`, `FELT LIKE MOMENTUM`. The notebook drains of color and becomes one cell among eighty-four. | `BUT WHAT WOULD COUNT?` | +| 3.40-6.50 | **The ledger** | The grid dissolves. Four tablets snap in on a 0.6s beat, the row re-centering as each one lands. | `CLAIM` `FALSIFIER` `CONTROL` `NON-CLAIM` | +| 6.50-9.50 | **The lock** | A fifth slot appears, dashed and empty, labeled `RESULT`. A protocol bar locks a timestamp and a hash beneath the row while that slot is still empty. Three seconds of near stillness. | `NAME IT BEFORE YOU SEE IT.` | +| 9.50-11.80 | **The disagreement** | The slot fills with `INCONCLUSIVE`, not a green tick. The falsifier's rose accent flares once - it did its job. At 10.80 the result becomes `NEXT EXPERIMENT`. | `LET THE RESULT DISAGREE.` | +| 11.80-15.00 | **The lockup** | The five accents compress into a ledger rule under the wordmark. | `CC-FRAMEWORK` / `Keep the wonder. Check the claim.` | + +The emotional center is the empty dashed slot at 6.50-9.50. Everything before it +is seduction; everything after it is consequence. The protocol is frozen while +the answer is still unknown, on screen, in front of the viewer. + +## Voiceover + +Quiet, calm, unsentimental. Never wry. Never triumphant. + +> "An intuition can start a question. Before you see an answer, name what would +> count. Then let the result disagree." + +Timing: line 1 lands over the freeze (2.2-3.3), line 2 over the lock +(6.8-9.2), line 3 over the disagreement (9.8-11.4). The lockup plays silent. + +## Sound design + +The master ships silent; `render_film.py` will mux an audio track if one is +supplied. The spec, if it is scored: + +- 0.00-2.00: a low warm drone, close and intimate, a little too pleasant. +- 2.00: **total silence for 400ms.** The cut is the loudest moment in the film + and it is made of nothing. +- 2.40-3.40: a fine granular texture as the embers quantize. +- 3.55, 4.15, 4.75, 5.35: four dry mechanical snaps, no reverb tail. +- 7.85: one clack as the protocol locks. +- 9.70: no sting. The result arrives without a reward sound. This is deliberate. +- 11.80-15.00: the drone returns, resolved a fifth lower. + +## Color law + +| Token | Hex | Means | +| --- | --- | --- | +| Ground | `#0E0E10` | The room | +| Lure amber | `#E9A23B` | Reflective and proposed material, unresolved pressure | +| Protocol white | `#EDF2F7` | Structure, procedure, what was fixed in advance | +| Evidence blue | `#7FA8D9` | Evidence and replayable structure | +| Confirmatory cyan | `#5FD3D0` | Confirmatory evidence under a pre-registered protocol | +| Invalidation rose | `#D9534F` | Invalidation, rejection, the falsifier | +| Pale | `#C9CEDA` | Integrity only; a result that has not separated from its control | + +**Deliberate deviation from the original brief.** The brief reserved emerald for +a real verifier result. This film uses no green at any point. World Bible V3 +lists "green checkmark as final truth" under forbidden imagery and states plainly +that there is no green safety color. Confirmatory cyan carries that role instead, +and it appears exactly once - on `NEXT EXPERIMENT`, which is a commitment, not a +verdict. Continuity with the existing world outranks the brief. + +## Typography + +- Thesis lines: Bitstream Charter, falling back to Liberation Serif and Georgia. + Wide tracking, generous size, never bold. +- Protocol and labels: DejaVu Sans Mono, uppercase, 0.24em tracking. +- Wordmark: Liberation Sans, 0.44em tracking. + +Only fonts present in a standard Linux render environment are used, so the film +renders identically on CI without shipping font binaries. + +## Production rules + +1. Start sensual and human. End exact and calm. Never the reverse. +2. The oracle imagery is original and abstract. No tarot card art, no recording + or likeness from any source creator, no borrowed footage. +3. The blue notebook is an easter egg from + [the source ledger](../../docs/research/future-expansion/source-ledger.md). + It is not proof of anything, and the film demotes it to one cell in a grid on + purpose. +4. One honest `INCONCLUSIVE` moment is mandatory. It is the anti-marketing flex: + the repository is willing to preserve uncertainty in its own advertising. +5. No terminal footage. The hero is the conceptual transformation, not code. +6. No verdict the film was not given. See below. + +## The verdict rule + +The result card renders whatever is in `window.__VERDICT` and nothing else. With +no verdict injected, the card falls back to an illustrative `INCONCLUSIVE` and +**stamps `ILLUSTRATION` in the corner of the frame** for the whole beat. + +That tag is not decoration. A film about not overclaiming cannot itself display +an unearned result, so the film labels its own unverified moment on screen. + +To show a real one, pass a verifier's own output: + +```bash +# a real result, produced by a real run, displayed verbatim +python3 visual_identity/before_you_see_it/render_film.py \ + --cut ghost-ark --verdict verdict.json +``` + +```json +{ "state": "FAIL CLOSED", "detail": "mutated receipt rejected\nverifier: independent", "color": "#D9534F" } +``` + +For the Ghost-Ark cut this is the intended use: run the malicious-corpus +verifier, take its actual output, and let the film show a mutation failing +closed. An unmutated decorative checkmark would violate both repositories' +north star. + +## Poster frame + +Pulled from t=7.60s - the frozen protocol with the empty result slot. + +```text +WHEN DOES A SIGN +BECOME EVIDENCE? +BEFORE YOU SEE IT. +``` + +## Cuts + +| Cut | Wordmark | Tagline | Footer | +| --- | --- | --- | --- | +| `cc-framework` | CC-FRAMEWORK | Keep the wonder. Check the claim. | Not a safety proof. A method for making claims checkable. | +| `ghost-ark` | GHOST-ARK | Evidence you can inspect. | Replay the receipt. Inspect the non-claims. | + +Same film, same timing, same frames - only the final lockup changes. + +## Non-claims + +- The film is a statement of method. It is not evidence, and it reports no + measurement. +- The result card is an illustration of a protocol outcome unless a verdict was + injected, in which case it displays that verdict verbatim and drops the + `ILLUSTRATION` tag. +- The timestamp and hash in the protocol bar are set dressing at fixed values. + They are not a real receipt over a real artifact. +- Neither lockup asserts that the named project has proven anything. Both name a + standard the project holds itself to. diff --git a/visual_identity/before_you_see_it/README.md b/visual_identity/before_you_see_it/README.md new file mode 100644 index 00000000..7a49ff9f --- /dev/null +++ b/visual_identity/before_you_see_it/README.md @@ -0,0 +1,105 @@ +# Before You See It + +A 15-second film about the moment a seductive story is forced to become a test. + +```text +A story can start a question. +Evidence must finish the answer. +``` + +**Keep the wonder. Check the claim.** + +| | | +| --- | --- | +| Treatment | [DIRECTORS_CUT.md](DIRECTORS_CUT.md) - beats, color law, voiceover, sound, production rules | +| Source | [film.html](film.html) - the film itself, one self-contained file | +| Render | [render_film.py](render_film.py) - deterministic frame capture and encode | +| Clip | [render_check_clip.py](render_check_clip.py) - Surface B, the ten-second `BYTES MATCH → FAIL CLOSED` clip, driven against the real page | +| Chamber | Chamber 00 of the [Claim Observatory](../claim_observatory/WORLD_BIBLE_V3.md) | +| Method | [Future Expansion](../../docs/research/future-expansion/README.md) - the research package the film dramatizes | +| Page | [Canonical page](../canonical_page/index.html) - the film, transcript, ledger, limits strip, and one real check | + +## Watch it + +Open `film.html` in any browser. It plays on load; click to replay. + +## Render it + +```bash +python3 -m pip install playwright # Chromium is already present in CI images +python3 visual_identity/before_you_see_it/render_film.py --cut cc-framework +python3 visual_identity/before_you_see_it/render_film.py --cut ghost-ark +``` + +Outputs land in `renders/`: an H.264 `.mp4`, a VP9 `.webm`, and a poster frame. + +The film is a **pure function of time**. The page exposes `window.__seek(t)` and +the renderer walks the timeline frame by frame rather than recording playback, so +the same commit produces the same frames on any machine. Nothing depends on +wall-clock speed, and there is no font download - only faces already present in a +standard Linux render environment are used. + +## The two cuts + +Same film, same timing, same frames. Only the final lockup changes. + +| Cut | Wordmark | Tagline | +| --- | --- | --- | +| `cc-framework` | CC-FRAMEWORK | Keep the wonder. Check the claim. | +| `ghost-ark` | GHOST-ARK | Evidence you can inspect. | +| `cubits11` | CUBITS11 | Keep the wonder. Check the claim. | + +## The verdict rule + +The result card shows `window.__VERDICT` and nothing else. With no verdict +injected it falls back to an illustrative `INCONCLUSIVE` **and stamps +`ILLUSTRATION` in the corner of the frame** for the whole beat. + +A film about not overclaiming cannot display an unearned result, so it labels its +own unverified moment on screen. To show a real one, hand it a real verifier's +output: + +```bash +python3 visual_identity/before_you_see_it/render_film.py \ + --cut ghost-ark --verdict verdict.json +``` + +```json +{ "state": "FAIL CLOSED", "detail": "mutated receipt rejected\nverifier: independent", "color": "#D9534F" } +``` + +For the Ghost-Ark cut that is the intended path: run the malicious-corpus +verifier, take its actual output, and let the film show a real mutation failing +closed rather than a decorative checkmark. + +## Placing the Ghost-Ark cut + +The Ghost-Ark cut is built to live in `PSUCyberSecurityLab/ghost-ark`. It was not +pushed there from this repository - that session could not attach a repository +outside the `Cubits11` owner - so it ships here, ready to move: + +```bash +# from a session or checkout that has ghost-ark +mkdir -p docs/media +cp renders/before_you_see_it__ghost-ark.mp4 docs/media/ +cp renders/poster__ghost-ark.png docs/media/ +``` + +Then in the Ghost-Ark README: + +```markdown +https://github.com/PSUCyberSecurityLab/ghost-ark/raw/main/docs/media/before_you_see_it__ghost-ark.mp4 +``` + +Regenerating it there needs only `film.html` and `render_film.py`; both are +self-contained and have no repository-specific dependencies. + +## Non-claims + +- The film is a statement of method. It is not evidence and reports no + measurement. +- The result card is an illustration unless a verdict was injected. +- The timestamp and hash in the protocol bar are set dressing at fixed values. + They are not a receipt over a real artifact. +- Neither lockup asserts that the named project has proven anything. Both name a + standard the project holds itself to. diff --git a/visual_identity/before_you_see_it/film.html b/visual_identity/before_you_see_it/film.html new file mode 100644 index 00000000..8c6157a5 --- /dev/null +++ b/visual_identity/before_you_see_it/film.html @@ -0,0 +1,433 @@ + + +Before You See It + + + +
+ +
+
THIS FEELS TRUE.
+
BUT WHAT WOULD COUNT?
+
NAME IT BEFORE YOU SEE IT.
+
LET THE RESULT DISAGREE.
+ +
+
Claim
+
What is asserted, and how narrowly.
01
+
Falsifier
+
The result that would end this.
02
+
Control
+
The comparison that could embarrass it.
03
+
Non-claim
+
What this will never support.
04
+
Result
+
ILLUSTRATION
+
+
05
+
+ +
+ protocol frozen  2026-08-20T14:36:00Z + sha256  3f9c…a7e1 + result  not yet observed +
+ +
+
CC-FRAMEWORK
+
+
Keep the wonder. Check the claim.
+ +
+
+
+ + diff --git a/visual_identity/before_you_see_it/render_check_clip.py b/visual_identity/before_you_see_it/render_check_clip.py new file mode 100644 index 00000000..23300384 --- /dev/null +++ b/visual_identity/before_you_see_it/render_check_clip.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +"""Render Surface B: a ten-second silent clip of the canonical page's check. + + BYTES MATCH -> one character changes -> FAIL CLOSED -> restore -> BYTES MATCH + +The clip is driven against the real page in a real browser and captured frame by +frame, so it cannot drift from the artifact it depicts. Nothing is re-staged, +mocked up, or animated separately: every digest on screen was computed by +WebCrypto during the capture. + +Usage: + python3 visual_identity/before_you_see_it/render_check_clip.py +""" + +from __future__ import annotations + +import argparse +import shutil +import subprocess +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) + +from render_film import find_ffmpeg # noqa: E402 (path set above; shared helper) + +PAGE = HERE.parents[1] / "visual_identity" / "canonical_page" / "index.html" +DURATION = 10.0 + +# (at second, action). "select" highlights the digit; "type" replaces it. +SCRIPT: tuple[tuple[float, str, str], ...] = ( + (2.00, "select", ""), + (2.40, "type", "9"), + (5.40, "select", ""), + (5.80, "type", "1"), +) + +SELECT_DIGIT = """ +() => { + const t = document.getElementById('doc'); + const i = t.value.indexOf('0.041') + 4; + t.focus(); + t.setSelectionRange(i, i + 1); +} +""" + +TYPE_DIGIT = """ +(ch) => { + const t = document.getElementById('doc'); + const s = t.selectionStart, e = t.selectionEnd; + t.value = t.value.slice(0, s) + ch + t.value.slice(e); + t.setSelectionRange(s + 1, s + 1); + t.dispatchEvent(new Event('input', { bubbles: true })); +} +""" + + +def capture(frames_dir: Path, fps: int) -> int: + from playwright.sync_api import sync_playwright + + total = round(DURATION * fps) + with sync_playwright() as pw: + launch: dict = {} + root = Path("/opt/pw-browsers") + for exe in sorted(root.glob("chromium-*/chrome-linux/chrome")): + launch["executable_path"] = str(exe) + break + browser = pw.chromium.launch(**launch) + page = browser.new_page(viewport={"width": 1440, "height": 1080}, device_scale_factor=1) + page.goto(PAGE.as_uri(), wait_until="load") + page.wait_for_function("document.getElementById('actual').textContent.length === 64") + page.locator("#doc").scroll_into_view_if_needed() + page.wait_for_timeout(200) + + head = page.locator("section:has(#doc) h3").bounding_box() + box = page.locator("section:has(#doc) .check").bounding_box() + pad = 34 + top = head["y"] - pad + clip = { + "x": int(box["x"] - pad) & ~1, + "y": int(top) & ~1, + "width": int(box["width"] + 2 * pad) & ~1, + "height": int(box["y"] + box["height"] + 12 - top) & ~1, + } + print(f" framing {clip['width']}x{clip['height']} at ({clip['x']},{clip['y']})") + + pending = list(SCRIPT) + for i in range(total): + now = i / fps + while pending and pending[0][0] <= now: + _, action, arg = pending.pop(0) + if action == "select": + page.evaluate(SELECT_DIGIT) + else: + page.evaluate(TYPE_DIGIT, arg) + page.wait_for_timeout(30) + page.screenshot( + path=str(frames_dir / f"f{i:05d}.png"), clip=clip, animations="disabled" + ) + if i % 60 == 0: + print(f" frame {i}/{total}", flush=True) + browser.close() + return total + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--fps", type=int, default=30) + ap.add_argument("--out-dir", type=Path, default=HERE / "renders") + ap.add_argument("--keep-frames", action="store_true") + args = ap.parse_args(argv) + + args.out_dir.mkdir(parents=True, exist_ok=True) + frames_dir = args.out_dir / ".frames-check-clip" + if frames_dir.exists(): + shutil.rmtree(frames_dir) + frames_dir.mkdir(parents=True) + + ffmpeg = find_ffmpeg() + print(f"capturing the check at {args.fps}fps -> {frames_dir}") + capture(frames_dir, args.fps) + + mp4 = args.out_dir / "check__bytes_match_fail_closed.mp4" + subprocess.run( + [ + ffmpeg, + "-y", + "-framerate", + str(args.fps), + "-i", + str(frames_dir / "f%05d.png"), + "-c:v", + "libx264", + "-preset", + "slow", + "-crf", + "20", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + str(mp4), + ], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + if not args.keep_frames: + shutil.rmtree(frames_dir) + print(f" {mp4.relative_to(HERE.parents[1])} {mp4.stat().st_size / 1e6:.2f} MB") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/visual_identity/before_you_see_it/render_film.py b/visual_identity/before_you_see_it/render_film.py new file mode 100755 index 00000000..c4ce53bc --- /dev/null +++ b/visual_identity/before_you_see_it/render_film.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""Render `film.html` to a deterministic 15-second film. + +The film is a pure function of time: the page exposes ``window.__seek(t)`` and +this script walks the timeline frame by frame, so two renders of the same commit +produce byte-identical frames. Nothing is recorded in real time. + +Usage: + python3 visual_identity/before_you_see_it/render_film.py --cut cc-framework + python3 visual_identity/before_you_see_it/render_film.py --cut ghost-ark \ + --verdict path/to/verdict.json + +`--verdict` takes a JSON object ``{"state": ..., "detail": ..., "color": ...}`` +produced by a real verifier run. Without it the result card is rendered with an +on-screen ILLUSTRATION tag: the film does not display a verdict it was not given. +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +FILM = HERE / "film.html" +WIDTH, HEIGHT = 1920, 1080 + + +def _has_x264(binary: str) -> bool: + try: + out = subprocess.run( + [binary, "-hide_banner", "-encoders"], capture_output=True, text=True, timeout=30 + ) + except (OSError, subprocess.SubprocessError): + return False + return "libx264" in out.stdout + + +def find_ffmpeg() -> str: + """First ffmpeg on the box that can actually encode H.264. + + Playwright ships an ffmpeg built only for VP8 screen recording, so it is + tried last and usually rejected here. + """ + candidates: list[str] = [] + if os.environ.get("FFMPEG"): + candidates.append(os.environ["FFMPEG"]) + on_path = shutil.which("ffmpeg") + if on_path: + candidates.append(on_path) + try: + import imageio_ffmpeg + + candidates.append(imageio_ffmpeg.get_ffmpeg_exe()) + except Exception: + pass + root = Path(os.environ.get("PLAYWRIGHT_BROWSERS_PATH", "/opt/pw-browsers")) + candidates += [str(p) for p in sorted(root.glob("ffmpeg-*/ffmpeg-linux"))] + + for candidate in candidates: + if _has_x264(candidate): + return candidate + raise SystemExit( + "no ffmpeg with libx264 found. `pip install imageio-ffmpeg`, or set " + "FFMPEG=/path/to/ffmpeg (the Playwright bundle is VP8-only)." + ) + + +def find_chromium() -> str | None: + root = Path(os.environ.get("PLAYWRIGHT_BROWSERS_PATH", "/opt/pw-browsers")) + for candidate in sorted(root.glob("chromium-*/chrome-linux/chrome")): + return str(candidate) + return None + + +def capture(frames_dir: Path, cut: str, fps: int, verdict: dict | None) -> int: + from playwright.sync_api import sync_playwright + + total = round(15.0 * fps) + url = f"{FILM.as_uri()}?capture=1&cut={cut}" + launch: dict = {"args": ["--force-color-profile=srgb", "--font-render-hinting=none"]} + exe = find_chromium() + if exe: + launch["executable_path"] = exe + + with sync_playwright() as pw: + browser = pw.chromium.launch(**launch) + page = browser.new_page(viewport={"width": WIDTH, "height": HEIGHT}, device_scale_factor=1) + page.goto(url, wait_until="load") + if verdict is not None: + page.evaluate("v => { window.__VERDICT = v; }", verdict) + page.wait_for_function("typeof window.__seek === 'function'") + for i in range(total): + page.evaluate("t => window.__seek(t)", i / fps) + page.screenshot(path=str(frames_dir / f"f{i:05d}.png"), animations="disabled") + if i % 120 == 0: + print(f" frame {i}/{total}", flush=True) + browser.close() + return total + + +def encode(ffmpeg: str, frames_dir: Path, out_dir: Path, cut: str, fps: int) -> list[Path]: + stem = f"before_you_see_it__{cut}" + mp4 = out_dir / f"{stem}.mp4" + webm = out_dir / f"{stem}.webm" + common = [ffmpeg, "-y", "-framerate", str(fps), "-i", str(frames_dir / "f%05d.png")] + subprocess.run( + [ + *common, + "-c:v", + "libx264", + "-preset", + "slow", + "-crf", + "19", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + str(mp4), + ], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + subprocess.run( + [ + *common, + "-c:v", + "libvpx-vp9", + "-b:v", + "0", + "-crf", + "32", + "-pix_fmt", + "yuv420p", + "-row-mt", + "1", + str(webm), + ], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + return [mp4, webm] + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument( + "--cut", choices=["cc-framework", "ghost-ark", "cubits11"], default="cc-framework" + ) + ap.add_argument("--fps", type=int, default=60) + ap.add_argument("--out-dir", type=Path, default=HERE / "renders") + ap.add_argument( + "--poster-at", + type=float, + default=7.6, + help="seconds; the poster frame is pulled from this moment", + ) + ap.add_argument( + "--verdict", + type=Path, + default=None, + help="JSON file holding a real verifier result to display", + ) + ap.add_argument("--keep-frames", action="store_true") + ap.add_argument( + "--encode-only", + action="store_true", + help="reuse frames already on disk instead of re-capturing", + ) + args = ap.parse_args(argv) + + verdict = json.loads(args.verdict.read_text()) if args.verdict else None + args.out_dir.mkdir(parents=True, exist_ok=True) + frames_dir = args.out_dir / f".frames-{args.cut}" + ffmpeg = find_ffmpeg() + + if args.encode_only: + total = len(list(frames_dir.glob("f*.png"))) + if not total: + raise SystemExit(f"--encode-only: no frames in {frames_dir}") + print(f"reusing {total} frames in {frames_dir}") + else: + if frames_dir.exists(): + shutil.rmtree(frames_dir) + frames_dir.mkdir(parents=True) + print(f"capturing {args.cut} at {args.fps}fps -> {frames_dir}") + total = capture(frames_dir, args.cut, args.fps, verdict) + + outputs = encode(ffmpeg, frames_dir, args.out_dir, args.cut, args.fps) + poster_idx = min(total - 1, round(args.poster_at * args.fps)) + poster = args.out_dir / f"poster__{args.cut}.png" + shutil.copyfile(frames_dir / f"f{poster_idx:05d}.png", poster) + outputs.append(poster) + + if not args.keep_frames: + shutil.rmtree(frames_dir) + + for path in outputs: + print(f" {path.relative_to(HERE.parents[1])} {path.stat().st_size / 1e6:.2f} MB") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/visual_identity/before_you_see_it/renders/before_you_see_it__cc-framework.mp4 b/visual_identity/before_you_see_it/renders/before_you_see_it__cc-framework.mp4 new file mode 100644 index 00000000..a7ba31a3 Binary files /dev/null and b/visual_identity/before_you_see_it/renders/before_you_see_it__cc-framework.mp4 differ diff --git a/visual_identity/before_you_see_it/renders/before_you_see_it__cubits11.mp4 b/visual_identity/before_you_see_it/renders/before_you_see_it__cubits11.mp4 new file mode 100644 index 00000000..9f4ef948 Binary files /dev/null and b/visual_identity/before_you_see_it/renders/before_you_see_it__cubits11.mp4 differ diff --git a/visual_identity/before_you_see_it/renders/before_you_see_it__ghost-ark.mp4 b/visual_identity/before_you_see_it/renders/before_you_see_it__ghost-ark.mp4 new file mode 100644 index 00000000..c55b86a7 Binary files /dev/null and b/visual_identity/before_you_see_it/renders/before_you_see_it__ghost-ark.mp4 differ diff --git a/visual_identity/before_you_see_it/renders/check__bytes_match_fail_closed.mp4 b/visual_identity/before_you_see_it/renders/check__bytes_match_fail_closed.mp4 new file mode 100644 index 00000000..40e7795d Binary files /dev/null and b/visual_identity/before_you_see_it/renders/check__bytes_match_fail_closed.mp4 differ diff --git a/visual_identity/before_you_see_it/renders/poster__cc-framework.png b/visual_identity/before_you_see_it/renders/poster__cc-framework.png new file mode 100644 index 00000000..9ed9f091 Binary files /dev/null and b/visual_identity/before_you_see_it/renders/poster__cc-framework.png differ diff --git a/visual_identity/before_you_see_it/renders/poster__cubits11.png b/visual_identity/before_you_see_it/renders/poster__cubits11.png new file mode 100644 index 00000000..9ed9f091 Binary files /dev/null and b/visual_identity/before_you_see_it/renders/poster__cubits11.png differ diff --git a/visual_identity/before_you_see_it/renders/poster__ghost-ark.png b/visual_identity/before_you_see_it/renders/poster__ghost-ark.png new file mode 100644 index 00000000..9ed9f091 Binary files /dev/null and b/visual_identity/before_you_see_it/renders/poster__ghost-ark.png differ diff --git a/visual_identity/canonical_page/index.html b/visual_identity/canonical_page/index.html new file mode 100644 index 00000000..c495107a --- /dev/null +++ b/visual_identity/canonical_page/index.html @@ -0,0 +1,303 @@ + + + + +Bring one claim that matters — Cubits11 + + + +
+ +
+
Pranav Bhave  /  Cubits11
+
A story can start a question.
Evidence must finish the answer.
+
Keep the wonder. Check the claim.
+ +
Before You See It  ·  15.000s  ·  silent master  ·  transcript below
+
+ +
+

The method

+
Before a green check becomes a claim, make its assumptions, evidence, and limits inspectable.
+
+

I build research software and public experiences that turn vague technical + claims into inspectable evidence.

+

This is not a safety score, a certification, or a seal. It is a loop: name + the claim, name the falsifier that would end it, name the + control that could embarrass it, name the non-claim it will + never support — then lock all four before the result exists, and let the + result disagree.

+

The boldest thing a technical brand can say is INCONCLUSIVE. The + film says it on purpose, and when it has no verifier output to show it stamps + ILLUSTRATION on its own footage rather than imply a result it was + not given.

+
+ +
+

Transcript

+ + + + + + + +
0.00–2.00THIS FEELS TRUE.
A notebook under warm, almost oracular light. Gold embers drift. A slow push-in.
2.00–3.40BUT WHAT WOULD COUNT?
Hard cut to cold. The embers quantize into 84 candidate matches — blue notebook, cracked screen, curly hair, vending machine, felt like momentum. The notebook drains of colour and becomes one cell among them.
3.40–6.50CLAIM   FALSIFIER   CONTROL   NON-CLAIM
Four tablets snap into a ledger that re-centres as each one lands.
6.50–9.50NAME IT BEFORE YOU SEE IT.
A fifth slot appears, dashed and empty. A protocol bar locks a timestamp and a hash while that slot is still empty. Three seconds of near stillness.
9.50–11.80LET THE RESULT DISAGREE.
The slot fills with INCONCLUSIVE, not a green tick. The falsifier's accent flares once. At 10.80 the result becomes NEXT EXPERIMENT.
11.80–15.00CUBITS11 — KEEP THE WONDER. CHECK THE CLAIM.
The five accents compress into a ledger rule under the wordmark.
+

“An intuition can start a question. Before you see an answer, + name what would count. Then let the result disagree.”

+
+ +
+

Source ledger

+

The film began with a real narrative source — a recorded tarot reading + brought into a working session as motivation. Before any of it was allowed + near a decision, every substantive statement was classified. This is the + short version; the + full ledger + carries every line, its testable translation, and the inference it forbids.

+ + + + + + + + +
Source statementClassTestable translationDisallowed inference
“Things are about to turn in your favour.”forecastNone directly. The nearest registerable question is whether a predeclared intervention moves a predeclared measure inside a fixed window.That improvement is scheduled. Not usable for runway, hiring, spend, or pricing.
“You’ll see more signs and synchronicities.”causalName the traction indicators and the window before any result is seen, then count them.That noticing a coincidence afterwards is evidence. This is the failure mode the whole method exists to block.
“There’s been a lot of experimentation.”reflectionCount experiments started, completed, and abandoned in a fixed prior period.That an accurate description makes the forecasts accurate. Broad descriptions feel personal.
“A blue notebook. A cracked screen. Curly hair. A vending machine.”empiricalThe only literally checkable lines. Testable only under predeclaration, base-rate comparison, and a blinded control.That a later match confirms anything. Each item is high-prevalence and elastic.
“There is money attached to this.”forecastNone from this source. Conversion belongs to an experiment that exists independently of it.Hard block: no revenue projection, pricing decision, spend commitment, or investor-facing statement rests on this.
“Hit like, subscribe, tell me in the comments.”funnelNot a claim about the subject.That felt resonance is independent of the source’s incentive to produce resonance.
+

Three method heuristics survived translation: + experiment then replicate then formalize; allocate on predeclared indicators; + freeze the protocol once it works. Each is independently defensible, and each + would survive the source being discarded entirely. That is the permitted use — + a story generated a question.

+
+ +
+

What this does not establish

+
    +
  • Nothing on this page proves any AI system, product, or claim is safe.
  • +
  • The check below establishes byte integrity only. It says nothing about whether the numbers inside the file are correct.
  • +
  • A digest match is not a statement about study design, sampling, labelling, or interpretation.
  • +
  • The film reports no measurement. Its result card is an illustration unless a real verdict was injected, and it labels itself when it is.
  • +
  • The source ledger does not establish that its source is accurate, predictive, or causal — only what may and may not be inferred from it.
  • +
  • Reproducing this capsule from its declared inputs does not establish that those inputs were measured. We forged our own verified page to check, and it worked — CH-001.
  • +
  • No claim here has been externally reproduced yet. When it is, the discrepancies get published too.
  • +
+
+ +
+

One real check

+

Change one character. Watch it fail closed.

+

This is a real evidence file from this repository, and a real digest + recorded in its manifest. Your browser recomputes SHA-256 over exactly the + bytes in the box. Edit anything — a digit, a space — and the recomputation + stops matching. Nothing is simulated and no result is hard-coded.

+ +
+
+
fixture · calibration_evidence
+ + + +
+
+
recorded in manifest
+
recomputed in your browser
+
+
+
+
+ +
+ fixture   examples/claim_governance_capsule/expected/calibration.json
+ digest   examples/claim_governance_capsule/manifest.expected.json
+ command   sha256sum examples/claim_governance_capsule/expected/calibration.json
+ limitation   Integrity only. This does not establish that 0.041666666667 is + the true false-positive rate, that the calibration was well designed, that the + population was representative, or that any system is safe.
+ and one level up   The capsule this fixture comes from reproduces deterministically + from its declared inputs. That is not the same as those inputs having been measured. We + changed an input, regenerated the whole chain, and every guard stayed green over a number + we invented — CH-001. +
+
+ +
+

The invitation

+

Bring one claim that matters.

+

One sentence your team says in public and cannot + fully defend in private. A benchmark number, a security property, a “proven” + or a “guaranteed”. You get back a claim map, the evidence that supports it, the + alternative explanations, the smallest useful test, and a limitation statement + in plain English.

+

Not validation. Not compliance. Not a seal of approval. + Better questions, stronger evidence, clearer public language, and a more useful + failure boundary.

+
+ +
+

Boundaries

+ + + + + + +
NamePublic roleBoundary
Pranav BhaveAuthor, builder, speaker.The human voice.
Cubits11Independent studio for evidence design, research software, and education.The commercial home. It sells clarity, never certainty.
CC-FrameworkResearch on what stacked guardrail evidence supports when failures may be correlated.Not a safety score and not a certification.
Ghost-ArkA separate institutional research artifact on bounded receipts and verifiable evidence.Not a Cubits11 product and not a university endorsement. Referenced only as a labelled research case study, never as a commercial credential.
+
+ +
+ Pranav Bhave / Cubits11  ·  Make the claim smaller. Make the evidence stronger.
+ This page states a method. It is not evidence that the method works. +
+
+ + +